// The host-retry door (zero-downtime spec §8.4, §9.1): `carlos instances // retry ` clears one instance's failure hold, and its next attempt // runs as a canary. The app-wide retry is `carlos promote --retry`, which // re-signs the channel pointer. // // The request rides the app's host-retry log (release.AppendHostRetry, // readiness round 2, Task 3), which no older console rewrites and no older // box reads. Every box can write that log (Astra impl-6 finding 5), so the // entry carries a grant the root key signs here (release.HostRetryGrant, // spec §11; Paul's ruling on plan 6 Q4, amadan #272), naming the held build // and its restriction's epoch and book as the box reported them, and a box // clears nothing on an entry without one. The door writes only for a box // whose report says it can take the grant (retryHostBuild): an older box, // or one whose failure book is not saved yet, is refused with a message // and nothing is written. A console with no key (read-only) answers 503 // and writes nothing. ROLE: operate, as `carlos restart` is. package console import ( "context" "errors" "fmt" "log" "net/http" "os" "time" "github.com/carlosframework/platform/internal/provision" "github.com/carlosframework/platform/internal/release" ) const ( apiRetryStore = "could not write the retry request. Try again" apiRetryHostAudit = "requested a retry of %s on %s (%s)" apiRetryNotHeld = "%s is not held back by a failed build, as far as its box last reported, so there is nothing to retry" // apiRetryNoKey is a read-only console's answer: a retry is a grant the // root key signs (spec §11), and there is no key here to sign it. apiRetryNoKey = "no signing key on this host. this console cannot request a retry" // apiRetryUnsupported answers a box whose report has no retry_v or no // held_book: it reads no grant from the host-retry log, so a retry // written for it would clear nothing. apiRetryUnsupported = "this instance's box is too old to take a retry. update the box, then try again" // apiRetryUnsaved answers a held host reported at epoch 0: its box's // failure book owes a save, and no grant clears anything until it lands. apiRetryUnsaved = "this instance's box has not saved its failure yet. try again in a minute" ) var ( errRetryNoRecord = errors.New("retry: no such instance") errRetryNotHeld = errors.New("retry: not held") errRetryNoKey = errors.New("retry: no signing key on this console") // errRetryUnsupported is a held host whose box cannot take a retry: // its report has no retry_v (a box built before the host-retry log) or // no held_book (one built before the book's identity). errRetryUnsupported = errors.New("retry: the box cannot take a retry") // errRetryUnsaved is a held host whose box reports retry_v but epoch 0: // its failure book is not saved (readiness round 2, Task 1), and a grant // signed now would name no epoch the box accepts. errRetryUnsaved = errors.New("retry: the box has not saved its failure book") ) // retryHostBuild is the build holding host back, and the epoch of that // restriction and the failure book it counts in (plan review finding 13; // final review round 1, finding 3), as its box reports them. The record // must exist in this account and app (the instance family's 404), and the // report must be this app's own (boxJoin.rollout), read through the same // loader as the list and the instance page (loadRolloutJoin). A box that // could not arm its scheduler is not held, except one that declined the // flag (a decline code, rolloutwords.UnavailableDeclines) and reports the // host held by the failure book it still honours: that box verifies and // applies a signed retry too (Task 10, plan review #5). A refusing box, or // one whose code this console does not know, is never retried here. // // Then the box's report is asked whether it can take the grant (readiness // round 2, Task 3), and a retry it could not take is refused with a // message rather than written to clear nothing: a report with no retry_v // is a box built before the host-retry log, which never reads it, and one // with no held_book is older than the book's identity // (errRetryUnsupported); a held host at epoch 0 is a box whose failure // book owes a save, and no grant clears anything until it lands // (errRetryUnsaved). func (s *Server) retryHostBuild(ctx context.Context, acct *Account, app, host string) (heldRestriction, error) { if _, err := provision.GetRecord(ctx, s.cfg.Releases, acct.Sqid, app, host); err != nil { if errors.Is(err, os.ErrNotExist) { return heldRestriction{}, errRetryNoRecord } return heldRestriction{}, err } join, _ := s.loadRolloutJoin(ctx, acct.Sqid, time.Now(), nil) ro := join.rollout(host, acct.Sqid, app) if !ro.retryable() { return heldRestriction{}, errRetryNotHeld } if err := ro.retryRefusal(); err != nil { return heldRestriction{}, err } return heldRestriction{build: ro.HeldBy, book: ro.heldBook, epoch: ro.heldEpoch, declined: ro.Unavailable != ""}, nil } // heldRestriction is what a retry grant names: the build holding the host // back, and that restriction's epoch and book as its box reported them. // declined marks a box that declined the flag: it runs no canary, so the // browser's confirmation says so. type heldRestriction struct { build, book string epoch uint64 declined bool } // requestHostRetry signs the retry grant for held (its build, book and // epoch) with the root key, appends it to the app's host-retry log // (release.AppendHostRetry), and announces it on the serial. With no key // it writes nothing (errRetryNoKey): an unsigned entry is one every box // drops. func (s *Server) requestHostRetry(ctx context.Context, acct, app, host string, held heldRestriction) (release.RestartRequest, error) { if s.cfg.Key == nil { return release.RestartRequest{}, errRetryNoKey } signed, err := release.SignHostRetry(release.RootSigner(s.cfg.Key), acct, app, host, held.build, held.book, held.epoch, time.Now()) if err != nil { return release.RestartRequest{}, err } req, err := release.AppendHostRetry(ctx, s.cfg.Releases, acct, app, host, signed) if err != nil { return release.RestartRequest{}, err } s.bumpSerial(ctx) return req, nil } func (s *Server) handleAPIInstanceRetry(w http.ResponseWriter, r *http.Request) { addr, acct, app, ok := s.apiAccountApp(w, r) if !ok { return } host := r.PathValue("host") if !provision.ValidHostShape(host) { http.NotFound(w, r) return } p, _ := principalFrom(r.Context()) if !s.allowWrite(p, time.Now()) { jsonError(w, http.StatusTooManyRequests, "too many requests. wait a minute and try again") return } if s.cfg.Releases == nil { jsonError(w, http.StatusServiceUnavailable, "no deployment bucket on this host. this console cannot request a retry") return } if s.cfg.Key == nil { jsonError(w, http.StatusServiceUnavailable, apiRetryNoKey) return } var req struct { Message string `json:"message"` } if !readJSON(w, r, &req) { return } message, ok := checkedActivityMessage(w, req.Message) if !ok { return } ctx, cancel := context.WithTimeout(r.Context(), 15*time.Second) defer cancel() held, err := s.retryHostBuild(ctx, acct, app, host) switch { case errors.Is(err, errRetryNoRecord): http.NotFound(w, r) return case errors.Is(err, errRetryNotHeld): jsonError(w, http.StatusConflict, fmt.Sprintf(apiRetryNotHeld, host)) return case errors.Is(err, errRetryUnsupported): jsonError(w, http.StatusConflict, apiRetryUnsupported) return case errors.Is(err, errRetryUnsaved): jsonError(w, http.StatusConflict, apiRetryUnsaved) return case err != nil: log.Printf("console: api instance retry %s/%s/%s: %v", acct.Sqid, app, host, err) jsonError(w, http.StatusBadGateway, "could not read the instance") return } build := held.build rr, err := s.requestHostRetry(ctx, acct.Sqid, app, host, held) if err != nil { log.Printf("console: api instance retry %s/%s/%s: %v", acct.Sqid, app, host, err) jsonError(w, http.StatusBadGateway, apiRetryStore) return } s.auditActivity(acct, app, addr, Activity{Action: "retry", TargetType: "instance", Target: host, Message: message, Detail: fmt.Sprintf(apiRetryHostAudit, build, host, "cli")}) writeJSON(w, http.StatusOK, map[string]any{"result": "requested", "id": rr.ID, "version": build, "host": host}) }