package console import ( "context" "errors" "fmt" "log" "os" "time" "github.com/carlosframework/platform/internal/hostactors" "github.com/carlosframework/platform/internal/hostclaim" "github.com/carlosframework/platform/internal/hostfence" "github.com/carlosframework/platform/internal/hostops" "github.com/carlosframework/platform/internal/provision" "github.com/carlosframework/platform/internal/release" ) // ConfigureClaimRuntime builds the enabled runtime exactly once, after // NewServer and before any Start* goroutine or handler can run. It replaces // every raw handle in s.cfg with its boundary; the raw bucket survives only // inside the ControlWriter and the boundary's private fields. func (s *Server) ConfigureClaimRuntime(ctx context.Context) error { if !s.cfg.HostClaim { return nil } if s.cfg.Releases == nil || s.cfg.DBVersions == nil || s.cfg.Bucket == "" || s.cfg.HostConsoleID == "" || len(s.cfg.Key) == 0 || len(s.cfg.ReleasePub) == 0 { return hostops.ErrContract } if s.claimRuntimeConfigured { return hostops.ErrContract } raw := s.cfg.Releases control, err := hostops.NewControlWriter(raw, s.cfg.ReleasePub, time.Now, s.cfg.dbClaimHomeRegion()) if err != nil { return err } check, err := hostops.NewProductionMutationCheck(raw, control) if err != nil { return err } records, err := hostops.NewConditionalRecordStore(raw, s.cfg.ReleasePub, s.cfg.dbClaimHomeRegion()) if err != nil { return err } data, err := hostops.NewObservedMutationBoundary(true, records, check, s.claimMutationObserver) if err != nil { return err } var dataBuckets []string if s.cfg.HibernateBucket != "" && s.cfg.HibernateBucket != s.cfg.Bucket { dataBuckets = append(dataBuckets, s.cfg.HibernateBucket) } versions, err := hostops.NewVersionedBoundary(true, s.cfg.DBVersions, raw, control, s.cfg.Bucket, dataBuckets, s.claimMutationObserver) if err != nil { return err } guard := backendGuard{data: data, control: control, observe: s.claimBackendObserver, deployment: s.cfg.StoreDeployment} d := hostactors.Directory{Store: data, PublicKey: s.cfg.ReleasePub, Bucket: s.cfg.Bucket} roster, err := d.Read(ctx) if err != nil { return err } if !roster.Roster.MatchesHome(s.cfg.dbClaimHomeRegion()) { return hostops.ErrContract } self, ok := roster.Roster.Consoles[s.cfg.HostConsoleID] if !ok || len(self.Namespaces) != 1 { return hostops.ErrContract } if err = s.store.migrateClaimAppIdentity(); err != nil { return err } a := &hostops.ProductionAuthority{HomeRegion: s.cfg.dbClaimHomeRegion(), Store: data, Control: control, Directory: d, Now: time.Now, ConsoleID: s.cfg.HostConsoleID, Enrollment: self.Enrollment, Namespace: self.Namespaces[0], Signer: release.RootSigner(s.cfg.Key)} a.Backfill = &hostops.Backfill{Store: data, Directory: d} // Region HA ยง5.2 (L8): create's executor from the chooser, only with // Config.RegionHA on; nil keeps today's create path. a.Executors = s.executorChooser(d) a.Evidence = hostactors.ExactEvidence{PublicKey: s.cfg.ReleasePub, Store: data, Boxes: d.PrivateBoxReport, Self: s.localHostReport} c := hostops.Controller{Store: data, Control: control, Authority: a, Now: time.Now, NewID: claimOperationID} owner := hostops.RegistrationOwner{Console: s.cfg.HostConsoleID, Enrollment: self.Enrollment, Controller: claimOperationID()} if owner.Controller == "" { return hostops.ErrContract } if err = hostops.BindMutationRunner(data, control, owner, s.mutationAppID); err != nil { return err } if err = s.store.bindClaimSQLMutations(data); err != nil { return err } // From here on no field of s.cfg refers to a raw handle. s.cfg.Releases = data if s.cfg.WakeRetryReset != nil { door := *s.cfg.WakeRetryReset door.Claims = data if door.production { door = *productionWakeRetryReset(a) } s.cfg.WakeRetryReset = &door } s.rebindGeoipStore() s.cfg.DBVersions = versions s.cfg.StoreProvisioner = newClaimStoreProvisioner(s.cfg.StoreProvisioner, guard) s.cfg.StoreScanner = newClaimStoreScanner(s.cfg.StoreScanner, guard) s.cfg.MailProvisioner = newClaimMailProvisioner(s.cfg.MailProvisioner, guard) s.cfg.DNSWriter = newClaimDNSWriter(s.cfg.DNSWriter, guard) purge := &ClaimPurgeCoordinator{Server: s, Controller: c, Authority: a, Versions: versions, ControlBucket: s.cfg.Bucket, Owner: owner} placement := &ClaimPlacementCoordinator{Server: s, Controller: c, Authority: a, Owner: owner} s.cfg.ClaimAuthority = a s.cfg.HostClaimOps = &c s.cfg.ClaimPurge = purge s.cfg.ClaimPlacement = placement s.claimOwner = owner s.claimRuntimeConfigured = true if s.cfg.MovesEnabled && s.cfg.ClaimMoveRecovery == nil { // The console's move pass (moves Plan 5 Task 7): the moves body of // claim Plan 10's Recovery.Moves hook, bound only with the moves // switch on, and only now that every box close is record first // (ruling M4, P3-6; TestClaimMoveRecoveryWiringWaitsForRecordFirstOwners). s.cfg.ClaimMoveRecovery = s.movesRecovery } var moveLogs func(context.Context, *hostops.MoveLogWriter) error if s.cfg.MovesEnabled { // The delivery of the move pass's log entries (Plan 6 Task 3), once // a pass and independent of any move's state or grant. moveLogs = s.drainMoveLog } s.cfg.HostClaimRecovery = &hostops.Recovery{ ScheduleLossPages: s.scheduleClaimLossPages, ReportAutomaticLosses: func(ctx context.Context, host string) error { return a.Control.ReportAutomaticLosses(ctx, a.Signer, host) }, Moves: s.cfg.ClaimMoveRecovery, MoveLogs: moveLogs, MoveFenceAuthority: hostfence.ClaimFenceAuthority{Directory: a.Directory, Now: c.Now}, Controller: c, SweepRecords: func(ctx context.Context) error { return c.SweepAllRecords(ctx) }, SweepPurges: func(ctx context.Context) error { scanErr := hostops.SweepAllPurges(ctx, c.Store, func(ctx context.Context, app hostops.App, p hostops.Purge) error { adm, err := hostops.ReadCanonicalAdmission(ctx, c.Store, app.Account, app.Name) if err != nil { return err } j, _, err := hostops.ReadPurgeJournal(ctx, c.Store, app.Account, app.Name, p.ID, adm.Value.AppID) if err != nil { return err } return purge.Resume(ctx, app.Account, app.Name, j.ID, j.AppID) }) return errors.Join(scanErr, s.SweepClaimPurgeRequests(ctx)) }, Purge: func(ctx context.Context, obs hostclaim.Observed) error { if obs.Claim == nil { return hostops.ErrIdentity } if obs.Claim.Holder == nil { return hostops.ErrPending } h := obs.Claim.Holder adm, err := hostops.ReadCanonicalAdmission(ctx, c.Store, h.Account, h.App) if err != nil { return err } j, _, err := hostops.ReadPurgeJournal(ctx, c.Store, h.Account, h.App, obs.Claim.Op.ID, adm.Value.AppID) if err != nil { return err } return purge.Resume(ctx, h.Account, h.App, j.ID, j.AppID) }, Placement: func(ctx context.Context, obs hostclaim.Observed) error { if obs.Claim == nil || obs.Claim.Holder == nil { return hostops.ErrIdentity } h := obs.Claim.Holder return placement.Resume(ctx, h.Account, h.App, obs.Claim.Op.Intent["app_id"], obs.Claim.Op.ID) }, SweepAdmissions: s.SweepClaimAdmissions, } return nil } func (s *Server) SweepClaimAdmissions(ctx context.Context) error { if !s.cfg.HostClaim { return nil } c := s.cfg.HostClaimOps if c == nil || c.Control == nil || s.cfg.ClaimPlacement == nil || s.cfg.ClaimPurge == nil { return hostops.ErrContract } // A projection error on one console row must not stop independent // canonical operations on every other app. identityErr := s.SweepClaimAppIdentities(ctx) apps, err := hostops.Apps(ctx, c.Store) if err != nil { return errors.Join(identityErr, err) } all := []error{identityErr} for _, app := range apps { if err := s.sweepOneClaimAdmission(ctx, app); err != nil && !errors.Is(err, hostops.ErrPending) { all = append(all, err) } } return errors.Join(all...) } func (s *Server) sweepOneClaimAdmission(ctx context.Context, app hostops.App) (result error) { c := s.cfg.HostClaimOps a, err := hostops.ReadCanonicalAdmission(ctx, c.Store, app.Account, app.Name) if errors.Is(err, os.ErrNotExist) { return nil } if err != nil { return err } appID := a.Value.AppID // SQL errors remain visible but do not prevent unrelated registration or // operation recovery. Parent operations already refuse unsettled children. var sqlErrors []error defer func() { result = errors.Join(result, errors.Join(sqlErrors...)) }() var database string var databaseErr error databaseChecked := false for id, e := range a.Value.Effects { if e.Action != "sql-ingest" || e.State == "complete" { continue } after, parseErr := time.Parse(time.RFC3339Nano, e.Inputs["sql_recover_after"]) reg, registered := a.Value.Registrations[e.OperationID] if parseErr != nil { if !registered || hostops.RegistrationLive(reg, c.Now()) { continue } } else if c.Now().Before(after) { continue } // Nothing above this point needs the local receipt database. In // particular, apps with no outstanding SQL never try to open it. if !databaseChecked { databaseChecked = true database, databaseErr = s.store.sqlDatabaseID() if databaseErr != nil { sqlErrors = append(sqlErrors, databaseErr) } } // An identity error here means the file was replaced under this // running console (review m-4): no effect can settle against it, so // each is alerted exactly as one bound to a restored-away database. replacedUnderUs := errors.Is(databaseErr, hostops.ErrIdentity) if databaseErr != nil && !replacedUnderUs { continue } if replacedUnderUs || e.Inputs["sql_database"] != database { // Bound to a receipt database this console no longer has (a // restore or replacement changed its identity). Never inferred: // it stays unsettled, visible and alerted until an operator // takes the restore runbook's break-glass path. sqlErrors = append(sqlErrors, fmt.Errorf("%w: %s/%s effect %s", ErrSQLDatabaseReplaced, app.Account, app.Name, id)) s.alertSQLDatabaseReplaced(ctx, app.Account, app.Name, id) continue } if err := c.Control.RecoverSQLTransaction(ctx, app.Account, app.Name, appID, id, s.store); err != nil { sqlErrors = append(sqlErrors, err) } } for id, reg := range a.Value.Registrations { if reg.Kind != hostops.RegistrationIngest { continue } if err := c.Control.SettleRegistration(ctx, app.Account, app.Name, appID, id, c.Now()); err != nil && !errors.Is(err, hostops.ErrPending) { return err } } if op := a.Value.Op; op != nil { switch op.Kind { case hostops.OpAppMutation: if hostops.IsRestoreOp(op) { // Review P2-4: a receipt-bound restore is finished from the // receipt (prepared receipt with no effect included); a // receipt-less one is AbandonRestore's (sql-work), which // never settles an issued or uncertain effect. return s.reconcileClaimRestore(ctx, app.Account, app.Name, appID) } // Mail members reopen only after their gate is released or // reconciled (RecoverMutation's mail branch). return c.Control.RecoverMutation(ctx, app.Account, app.Name, appID, op.ID) case hostops.OpPlacing: return s.cfg.ClaimPlacement.Resume(ctx, app.Account, app.Name, appID, op.ID) case hostops.OpPurging: return s.cfg.ClaimPurge.Resume(ctx, app.Account, app.Name, op.ID, appID) case hostops.OpDeleting: // Resume auxiliary cleanup even after the host claim became free. return s.retireClaimHostRecord(ctx, provision.RetireRef{Account: app.Account, App: app.Name, Host: op.Intent["host"], Family: op.Intent["family"], Name: op.Intent["name"], Incarnation: op.Intent["incarnation"], By: "console-recovery"}) default: return hostops.ErrContract } } var all []error for id := range a.Value.Registrations { if err := c.Control.SettleRegistration(ctx, app.Account, app.Name, appID, id, c.Now()); err != nil && !errors.Is(err, hostops.ErrPending) { all = append(all, err) } } return errors.Join(all...) } // ErrSQLDatabaseReplaced is an SQL effect past sql_recover_after whose // receipt database is not this console's current one. var ErrSQLDatabaseReplaced = errors.New("console: SQL effect bound to a replaced receipt database") // sqlReceiptAlertBox is the alert_seen box for receipt database alerts. const sqlReceiptAlertBox = "claim-sql" func sqlReplacedRule(account, app, effect string) string { return "replaced-database " + account + "/" + app + "/" + effect } // sqlAlertSendTimeout bounds all keymail sends for one effect's alert, so a // stalled mail service cannot hold the serial sweep for long. const sqlAlertSendTimeout = 15 * time.Second // alertSQLDatabaseReplaced tells operators once per effect, through the same // alert memory as box alerts: keymail where a grant exists, otherwise a log // line. Sends run under the sweep's context. func (s *Server) alertSQLDatabaseReplaced(ctx context.Context, account, app, effect string) { rule := sqlReplacedRule(account, app, effect) subject := fmt.Sprintf("[carlos] SQL effect %s on %s/%s needs the restore break-glass path", effect, account, app) body := fmt.Sprintf("SQL effect %s on %s/%s is bound to a console database that has been replaced. It cannot be settled automatically.\nFollow the break-glass section of the console restore runbook.\n", effect, account, app) var run *alertRun if s.sendApp != nil { r, err := s.alertRecipients() if err != nil { // Possibly transient: record nothing, so the next sweep tries // again rather than filing the alert as undeliverable for good. log.Printf("console: claim SQL alert: recipients: %v", err) return } run = r } if run == nil || len(run.to) == 0 { if s.alertDue(sqlReceiptAlertBox, rule, noMailboxAddress, true) { log.Printf("console: ALERT %s", subject) s.recordAlertSeen(sqlReceiptAlertBox, rule, noMailboxAddress, true) } return } ctx, cancel := context.WithTimeout(ctx, sqlAlertSendTimeout) defer cancel() for _, op := range run.to { if !s.alertDue(sqlReceiptAlertBox, rule, op.address, true) { continue } // Only when there is something to deliver, as in alertPass. run.noteMissing() _, err := s.sendApp(ctx, op.token, op.address, subject, body) if err != nil { log.Printf("console: claim SQL alert: keymail to %s: %v", op.address, err) continue } s.recordAlertSeen(sqlReceiptAlertBox, rule, op.address, true) } }