// The host-retry log: where `carlos instances retry ` grants ride // (zero-downtime spec §8.4, §11; readiness round 2, Task 3). // // Bucket layout addition: // // apps///host-retry.json CAS-guarded, schema-1 request log // // It has the restart-scope log's JSON shape and append (a log of requests, // oldest first, each with a random id, at most restartScopeKeep kept), and // every entry is a RestartScopeRetry whose Signed is the console's // root-signed HostRetryGrant. Grants used to ride the restart-scope log, // and that is why they no longer do: a console built before // RestartScope.Signed re-marshals that whole log on every restart it // appends, which strips the signature from every pending retry in it, and // the member's retry silently clears nothing. No released binary reads or // writes this key, so no older console can rewrite it, and no older box // ever sees a grant. // // The box reads it on the serial move, as it reads the restart-scope log // (internal/host hostRetryPass), and applies every grant still inside its // window that it has not handled in this process, an agent restart // included: a grant spent or made stale since is a no-op under the failure // book's generation rule (rollout.FailureBook.RetryHostAt). package release import ( "context" "errors" "fmt" "os" ) // HostRetryKey is the bucket key of app's host-retry log, derived from // AppPrefix like every other per-app key. func HostRetryKey(acct, app string) string { return AppPrefix(acct, app) + "host-retry.json" } // LoadHostRetries reads app's host-retry log. found=false means the object // is absent, the ordinary state of almost every app. A newer schema is an // error, as in LoadRestartScopes: a box that cannot be sure what an entry // means must do nothing with it. func LoadHostRetries(ctx context.Context, s Store, acct, app string) (*ScopedRestartGeneration, bool, error) { key := HostRetryKey(acct, app) raw, _, err := s.Get(ctx, key) if errors.Is(err, os.ErrNotExist) { return &ScopedRestartGeneration{Schema: 1}, false, nil } if err != nil { return nil, false, err } g, err := parseRestartScopes(key, raw) if err != nil { return nil, false, err } return g, true, nil } // claimRawAppendHostRetry records one signed retry of host in app's // host-retry log and returns it. signed is the console's root-signed grant // (SignHostRetry); an entry without one is refused here, since every box // would drop it. The append, its CAS retries and its bound are the // restart-scope log's (claimRawAppendRequestLog). func claimRawAppendHostRetry(ctx context.Context, s Store, acct, app, host, signed string) (RestartRequest, error) { if !ValidAccount(acct) { return RestartRequest{}, fmt.Errorf("retry: invalid account %q", acct) } if !ValidApp(app) { return RestartRequest{}, fmt.Errorf("retry: invalid app %q", app) } if err := validScopeValue(RestartScopeRetry, host); err != nil { return RestartRequest{}, err } switch { case signed == "": return RestartRequest{}, errors.New("retry: no signed grant; every box would drop it") case len(signed) > MaxRootEnvelopeBytes: return RestartRequest{}, errors.New("retry: the signed grant is too large") } return claimRawAppendRequestLog(ctx, s, HostRetryKey(acct, app), RestartScope{Kind: RestartScopeRetry, Value: host, Signed: signed}, "retry") } // AppendHostRetry is claimRawAppendHostRetry under the app's mutation // claim. func AppendHostRetry(ctx context.Context, s Store, acct, app, host, signed string) (claimResult0 RestartRequest, claimResult1 error) { claimResult1 = WithAppMutation(ctx, s, acct, app, "release.AppendHostRetry", func(claimContext context.Context) error { claimResult0, claimResult1 = claimRawAppendHostRetry(claimContext, s, acct, app, host, signed) return claimResult1 }) return }