package console // claim_restore.go is the flag-on half of restoreApp (restore.go). Both doors // still call restoreApp, which hands over here when HostClaim is on; its // errors are the same errNotInTrash, *temporaryCapError and *restoreFailure // values, so each door's existing responses (and copy) are unchanged. // // The order, under the same account-then-app locks: // // 0. finish any interrupted restore of this app (ControlWriter.RecoverRestore) // 1. project canonical state into this row (the flag-on settle) // 2. not in the trash / already restored / cap check, as restoreApp // 3. read the exact marker version // 4. BeginExclusive (app-mutation, action restore) // 5. PrepareRestore: the receipt, with the marker ETag, BEFORE any effect is // issued and before any SQL // 6. Issue the operation's one effect (the live permit) // 7. under the effect permit: restoring = 1 (this console's slot) // 8. DeleteIfMatch of exactly that version, through the data boundary // 9. CompleteRestore, settle the effect, FinishMutation // 10. under the receipt grant: this console's row (FinishClaimRestoreRow) // // Steps 5, 8 and 9 are the global effect. Once step 5 has saved the receipt // the restore is committed: any console's RecoverMutation finishes the // receipt's exact-ETag delete (which cannot remove a successor marker), and // AbandonRestore refuses it. Before step 5 nothing was issued, so a failure // just reopens. The expiry bump is projection, folded into step 10, which // every console (this one included) replays from the completed receipt in // materializeClaimApp. Nothing here writes SQL except through // Store.WithAppMutation, and nothing writes canonical admission except the // typed ControlWriter. import ( "context" "crypto/rand" "database/sql" "encoding/hex" "errors" "fmt" "log" "strconv" "time" "github.com/carlosframework/platform/internal/hostops" "github.com/carlosframework/platform/internal/release" ) // claimRestoreAbandonAfter is how long a restore operation that never saved // its receipt is left to its issuer before another console abandons it (see // hostops/restore_recover.go for why abandoning is a fence). const claimRestoreAbandonAfter = 2 * time.Minute // errClaimRestoreUnconfirmed is a trashed row with no marker and no completed // receipt: canonical state does not say this app is in the trash, so there // is nothing a restore may delete. var errClaimRestoreUnconfirmed = errors.New("console: the trash marker is missing and no restore receipt confirms one") func newClaimRestoreID() (string, error) { var b [16]byte if _, err := rand.Read(b[:]); err != nil { return "", err } return "restore-" + hex.EncodeToString(b[:]), nil } // claimRestoreRuntime is the typed control writer and this console's // controller identity, as ConfigureClaimRuntime installed them. Anything // missing, or an ungated database, refuses. func (s *Server) claimRestoreRuntime() (*hostops.ControlWriter, hostops.RegistrationOwner, error) { a := s.cfg.ClaimAuthority if a == nil || a.Control == nil || a.ConsoleID == "" || a.Enrollment == "" || s.cfg.Releases == nil || !s.store.claimGated() { return nil, hostops.RegistrationOwner{}, hostops.ErrContract } return a.Control, hostops.RegistrationOwner{Console: a.ConsoleID, Enrollment: a.Enrollment}, nil } // restoreClaimApp is restoreApp with HostClaim on. The caller has checked // s.cfg.Releases != nil. func (s *Server) restoreClaimApp(ctx context.Context, acct *Account, app string, now time.Time) error { w, self, err := s.claimRestoreRuntime() if err != nil { return &restoreFailure{Marker: true, Err: err} } defer s.lockAccount(acct.Sqid)() defer s.lockApp(acct.Sqid, app)() appID, err := s.store.ClaimAppID(acct.ID, app) if errors.Is(err, sql.ErrNoRows) { return errNotInTrash } if err != nil { return &restoreFailure{Marker: true, Err: err} } prior, err := s.store.AppLife(acct.ID, app) if err != nil { return &restoreFailure{Marker: true, Err: err} } // 0. ErrPending is a young operation another console still owns; step 4 // refuses it below. if _, err := w.RecoverRestore(ctx, acct.Sqid, app, appID, now, claimRestoreAbandonAfter); err != nil && !errors.Is(err, hostops.ErrPending) { return &restoreFailure{Marker: true, Err: err} } // 1. ErrPending is a projection that must wait (an operation owns the // app, or a receipt is not this row's); the checks below decide. if _, err := materializeClaimApp(ctx, s.cfg.Releases, s.store, acct.ID, acct.Sqid, app, now); err != nil && !errors.Is(err, hostops.ErrPending) { return &restoreFailure{Marker: true, Err: err} } life, err := s.store.AppLife(acct.ID, app) if err != nil { return &restoreFailure{Marker: true, Err: err} } // 2. As restoreApp: a restore finished by step 0 or 1 is this restore, // answered as one. if prior != nil && prior.Trashed() && life != nil && !life.Trashed() { return nil } if life == nil || !life.Trashed() { return errNotInTrash } // Plans 3a: restoring brings a running thing back, so it meets the cap. if err := s.checkRestore(ctx, acct, app); err != nil { if isPlanCap(err) { return err } return &restoreFailure{Marker: true, Err: err} } if life.Temporary() { if err := s.temporaryCapCheck(acct.ID); err != nil { var capErr *temporaryCapError if errors.As(err, &capErr) { return capErr } return &restoreFailure{Marker: true, Err: err} } } // 3. tomb, etag, err := readTombstoneVersion(ctx, s.cfg.Releases, acct.Sqid, app) if err != nil { return &restoreFailure{Marker: true, Err: err} } if tomb == nil { return &restoreFailure{Marker: true, Err: errClaimRestoreUnconfirmed} } // 4. op, err := newClaimRestoreID() if err != nil { return &restoreFailure{Marker: true, Err: err} } owner := self owner.Controller = op h, err := w.BeginExclusive(ctx, acct.Sqid, app, appID, hostops.AppOperation{ID: op, Kind: hostops.OpAppMutation, AppID: appID, Phase: "issued", Intent: map[string]string{"action": "restore", hostops.RestoreActionStarted: strconv.FormatInt(now.Unix(), 10)}}, owner) if err != nil { return &restoreFailure{Marker: true, Err: err} } // 5. The receipt, before any effect or SQL. A lost response may still // have saved it; only an operation with no receipt (and so nothing // issued) is reopened here. if err := h.PrepareRestore(ctx, etag, tomb.DeletedAt.Unix()); err != nil { current, rerr := hostops.ReadAdmission(ctx, s.cfg.Releases, acct.Sqid, app, appID) if rerr != nil || current.Value.LastRestore == nil || current.Value.LastRestore.ID != op { return &restoreFailure{Marker: true, Err: errors.Join(err, rerr, h.FinishMutation(context.WithoutCancel(ctx)))} } } // 6. From here the receipt commits the restore: a failure is finished by // the receipt's recovery, never abandoned. pctx, settle, err := h.Runner(time.Now).Issue(ctx, hostops.Effect{ID: op, Resource: hostops.OpAppMutation, Action: "restore", Inputs: map[string]string{"tombstone_etag": etag}}) if err != nil { if ferr := s.recoverClaimRestoreReceipt(ctx, w, acct, app, appID, op, err); ferr != nil { return ferr } } else { // 7. This console's restoring slot, under the live permit. A failure // here is local projection only; the delete still runs, and step 10 // (or any later materialization) clears the flag. scope := claimSQLScope{AccountID: acct.ID, App: app, AppID: appID} if err := s.store.WithAppMutation(ctx, []sqlGrant{permitGrant(pctx, s.cfg.Releases, acct.Sqid, scope)}, func(v *Store) error { _, err := v.SetClaimRestoring(acct.ID, app, appID, true) return err }); err != nil { log.Printf("console: claim restore %s/%s: restoring slot: %v", acct.Sqid, app, err) } // 8 and 9. derr := release.DeleteIfMatch(pctx, s.cfg.Releases, release.AppTombstoneKey(acct.Sqid, app), etag) var cerr error if derr == nil { cerr = h.CompleteRestore(ctx) } if stepErr := errors.Join(derr, cerr); stepErr != nil { // Revoke the permit and record uncertainty, then let the bucket // decide: RecoverMutation finishes the receipt's exact delete. settleErr := settle("", stepErr) if ferr := s.recoverClaimRestoreReceipt(ctx, w, acct, app, appID, op, errors.Join(stepErr, settleErr)); ferr != nil { return ferr } } else { if err := settle("restored", nil); err != nil { // The marker is gone and the receipt is complete; the sweep's // RecoverMutation settles the effect and reopens. return &restoreFailure{Marker: false, Err: err} } if err := h.FinishMutation(ctx); err != nil { return &restoreFailure{Marker: false, Err: err} } } } // 10. This console's projection, from the completed receipt. current, err := hostops.ReadAdmission(ctx, s.cfg.Releases, acct.Sqid, app, appID) if err != nil { return &restoreFailure{Marker: false, Err: err} } receipt := current.Value.LastRestore if receipt == nil || receipt.ID != op || !restoreReceiptCurrent(current.Value, *receipt) { return &restoreFailure{Marker: false, Err: fmt.Errorf("console: restore %s/%s: receipt %s is not settled: %w", acct.Sqid, app, op, hostops.ErrPending)} } _, err = materializeClaimApp(ctx, s.cfg.Releases, s.store, acct.ID, acct.Sqid, app, now) if err != nil { return &restoreFailure{Marker: false, Err: err} } return nil } // recoverClaimRestoreReceipt finishes a restore whose receipt is saved but // whose issuer path failed (stepErr): RecoverMutation replays the receipt's // exact-ETag delete through the control writer and reopens. nil means the // restore finished and step 10 may run; otherwise the door's failure, with // Marker set while any marker still hides the app. func (s *Server) recoverClaimRestoreReceipt(ctx context.Context, w *hostops.ControlWriter, acct *Account, app, appID, op string, stepErr error) error { rerr := w.RecoverMutation(context.WithoutCancel(ctx), acct.Sqid, app, appID, op) if rerr == nil { return nil } log.Printf("console: claim restore %s/%s: operation %s keeps its receipt for recovery: %v", acct.Sqid, app, op, errors.Join(stepErr, rerr)) _, seen, terr := readTombstoneVersion(ctx, s.cfg.Releases, acct.Sqid, app) if terr != nil || seen != "" { // any marker still hides the app everywhere return &restoreFailure{Marker: true, Err: errors.Join(stepErr, rerr, terr)} } return &restoreFailure{Marker: false, Err: errors.Join(stepErr, rerr)} } // reconcileClaimRestore is the admission sweep's restore case: finish a // saved receipt, or abandon an aged receipt-less restore. It never projects // SQL; each console's materialization replays the completed receipt. func (s *Server) reconcileClaimRestore(ctx context.Context, account, app, appID string) error { w, _, err := s.claimRestoreRuntime() if err != nil { return err } out, err := w.RecoverRestore(ctx, account, app, appID, time.Now(), claimRestoreAbandonAfter) if err != nil { return err } if out != hostops.RestoreNone { log.Printf("console: claim restore %s/%s: recovered (%s)", account, app, out) } return nil }