package console import ( "bytes" "context" "encoding/json" "errors" "fmt" "os" "time" "github.com/carlosframework/platform/internal/dnsrec" "github.com/carlosframework/platform/internal/domains" "github.com/carlosframework/platform/internal/hostops" "github.com/carlosframework/platform/internal/lease" "github.com/carlosframework/platform/internal/mail" "github.com/carlosframework/platform/internal/objstore" "github.com/carlosframework/platform/internal/release" ) type ClaimPurgeCoordinator struct { Server *Server Controller hostops.Controller Authority *hostops.ProductionAuthority Versions lease.VersionedStore ControlBucket string Owner hostops.RegistrationOwner } func (p *ClaimPurgeCoordinator) control() (*hostops.ControlWriter, error) { if p.Controller.Control == nil || p.Authority == nil || p.Versions == nil || p.ControlBucket == "" { return nil, hostops.ErrContract } return p.Controller.Control, nil } // stage runs one purge stage as a durable effect under the purge operation, // so every bucket write it makes passes the data boundary on that permit. // A stage that returns ErrPending joined cleanly and is retried later. func (p *ClaimPurgeCoordinator) stage(ctx context.Context, h *hostops.OperationHandle, name string, fn func(context.Context) error) error { id, err := claimEffectID() if err != nil { return err } var callErr error _, err = h.Runner(p.Controller.Now).Run(ctx, hostops.Effect{ID: id, Resource: "purge-stage:" + name, Action: name}, func(ctx context.Context) (string, error) { observed, issued := hostops.ObserveAppMutationEffects(ctx) callErr = fn(observed) if callErr == nil { return "joined", nil } if !issued() { return "", errors.Join(hostops.ErrNotApplied, callErr) } // Only the plain sentinel says the joined stage needs another pass. // A joined transport error or a nested not-applied result cannot clear // ownership of an effect this outer stage already issued. if callErr == hostops.ErrPending { return "joined", nil } return "", fmt.Errorf("purge stage %s returned after an issued effect: %v", name, callErr) }) return errors.Join(callErr, err) } // localSQLStage is restricted to the two joined SQL projections. Their // failures remain retryable without inventing an uncertain remote effect. func (p *ClaimPurgeCoordinator) localSQLStage(ctx context.Context, h *hostops.OperationHandle, name string, fn func(context.Context) error) error { if name != "bind-operation" && name != "app-row" { return hostops.ErrContract } var localErr error err := p.stage(ctx, h, name, func(ctx context.Context) error { localErr = fn(ctx); return nil }) return errors.Join(localErr, err) } func (p *ClaimPurgeCoordinator) Request(ctx context.Context, account, app, id, reason, by string) (string, error) { s := p.Server w, err := p.control() if err != nil { return "", err } acct, err := s.store.AccountBySqid(account) if err != nil { return "", err } if acct == nil { return "", hostops.ErrIdentity } journalKey, err := hostops.PurgeKey(account, app) if err != nil { return "", err } existingRaw, _, err := s.cfg.Releases.Get(ctx, journalKey) if err == nil { var existing hostops.PurgeJournal if err = json.Unmarshal(existingRaw, &existing); err != nil { return "", err } return existing.ID, p.Resume(ctx, account, app, existing.ID, existing.AppID) } if !errors.Is(err, os.ErrNotExist) { return "", err } appID, err := s.store.ClaimAppID(acct.ID, app) if err != nil { return "", err } admission, err := hostops.ReadAdmission(ctx, s.cfg.Releases, account, app, appID) if err != nil { return "", err } if admission.Value.LastPurge != nil && admission.Value.LastPurge.ID == id { return id, nil } if id == "" || (reason != "owner" && reason != "automatic") { return "", hostops.ErrIdentity } control, _, err := s.store.AppFlags(acct.ID, app) if err != nil { return "", err } if control { return "", hostops.ErrPending } if err = release.PurgeReady(ctx, s.cfg.Releases, s.cfg.ReleasePub, account, app); err != nil { return "", err } tomb, err := release.ReadAppTombstone(ctx, s.cfg.Releases, account, app) if err != nil { return "", err } if tomb == nil { return "", release.ErrNotTrashed } if reason == "automatic" && p.Controller.Now().Before(tomb.DeletedAt.Add(release.TrashCooloff)) { return "", hostops.ErrPending } key := release.AppTombstoneKey(account, app) body, tag, err := s.cfg.Releases.Get(ctx, key) if err != nil { return "", err } // Backend inventory/readiness precedes every destructive action. effects := map[string]json.RawMessage{} envs, err := objstore.StrictStoreEnvironments(ctx, s.cfg.Releases, account, app) if err != nil { return "", err } for _, env := range append(envs, "") { rec, _, err := objstore.StrictGetIn(ctx, s.cfg.Releases, account, app, env) if err != nil { return "", err } if rec != nil && (rec.Provisioned != nil || rec.Grant != nil) && s.cfg.StoreProvisioner == nil { return "", hostops.ErrContract } raw, err := json.Marshal(rec) if err != nil { return "", err } effects["store."+env] = raw } mrec, _, err := mail.StrictGet(ctx, s.cfg.Releases, account, app) if err != nil { return "", err } mailDomains, err := s.mailAppDomains(ctx, acct, app) if err != nil { return "", err } if s.cfg.MailProvisioner == nil && ((mrec != nil && (mrec.Provisioned != nil || len(mrec.Standalone) > 0)) || len(mailDomains) > 0) { return "", hostops.ErrContract } if err = p.mailReady(ctx, account, app); err != nil { return "", err } j := hostops.PurgeJournal{V: 1, Purge: hostops.Purge{ID: id, Phase: "reserved"}, AppID: appID, Account: account, App: app, RequestedBy: by, RequestedAt: p.Controller.Now().UTC().Format(time.RFC3339Nano), Reason: reason, AdmissionEpoch: admission.Value.Epoch, Tombstone: hostops.PurgeObject{Key: key, ETag: tag, Body: json.RawMessage(body)}, Hosts: map[string]hostops.PurgeHost{}, Effects: effects} // Registered reservation, then create-only journal: both typed. if _, err = w.ReservePurge(ctx, j, p.Owner, p.Controller.Now()); err != nil { return "", err } return id, p.Resume(ctx, account, app, id, appID) } func (p *ClaimPurgeCoordinator) Resume(ctx context.Context, account, app, id, appID string) error { s := p.Server w, err := p.control() if err != nil { return err } j, _, err := hostops.ReadPurgeJournal(ctx, s.cfg.Releases, account, app, id, appID) if errors.Is(err, os.ErrNotExist) { // Journal gone: only a completed purge's tail (request discard). return w.RemovePurgeJournal(ctx, account, app, id) } if err != nil { return err } acct, err := s.store.AccountBySqid(account) if err != nil { return err } if acct == nil { return hostops.ErrIdentity } a, err := hostops.ReadAdmission(ctx, s.cfg.Releases, account, app, appID) if err != nil { return err } if j.Phase == "done" && a.Value.Op == nil { return w.RemovePurgeJournal(ctx, account, app, id) } // BeginPurge transfers a reserved journal or replays the existing op. h, err := w.BeginPurge(ctx, account, app, appID, id, p.Owner) if err != nil { return err } // Every claim write in this pass (host claims, fences) carries the purge // operation's lifecycle grant. ctx = h.Lifecycle(ctx) if j.Phase == "done" { return p.Controller.CompletePurgeFences(ctx, h) } scope := claimSQLScope{AccountID: acct.ID, App: app, AppID: appID} if j.Phase != "retired" { if a, err = hostops.ReadAdmission(ctx, s.cfg.Releases, account, app, appID); err != nil { return err } revision := a.Value.Revision if err = p.localSQLStage(ctx, h, "bind-operation", func(ctx context.Context) error { return s.store.WithAppMutation(ctx, []sqlGrant{permitGrant(ctx, s.cfg.Releases, account, scope)}, func(st *Store) error { return st.BindClaimAppOperation(acct.ID, app, appID, id, revision) }) }); err != nil { return err } } switch j.Phase { case "reserved": body, tag, err := s.cfg.Releases.Get(ctx, j.Tombstone.Key) if err != nil { return err } if tag != j.Tombstone.ETag || !bytes.Equal(body, j.Tombstone.Body) { return hostops.ErrPending } if err = release.PurgeReady(ctx, s.cfg.Releases, s.cfg.ReleasePub, account, app); err != nil { return err } protected, _, err := s.store.AppFlags(acct.ID, app) if err != nil { return err } if protected { return hostops.ErrPending } if err = w.DrainPurgeRegistrations(ctx, account, app, appID, id, p.Controller.Now(), p.Controller.RecoverCreate); err != nil { return err } // The census is keyed by this purge's reservation: until every // roster box has published its report for it, the owned purge is // waiting (journaled, admission closed), not failed. census, err := p.Authority.Census(ctx, account, app, id) if errors.Is(err, os.ErrNotExist) { return errors.Join(hostops.ErrPending, err) } if err != nil { return err } records := map[string]bool{} for _, r := range census.Records { records[r.Holder.Host] = true } for _, report := range census.Reports { for _, installation := range report.Installations { if installation.Account == account && installation.App == app && !records[installation.Host] { return hostops.ErrPending } } } if err = p.captureDomains(ctx, account, app, id, appID); err != nil { return err } if err = p.Controller.CensusPurge(ctx, account, app, id, appID); err != nil { return err } return hostops.ErrPending case "adopted": return p.Controller.ClaimPurgeHosts(ctx, account, app, id, appID) case "claimed": if err = p.Controller.PurgeBarrier(ctx, account, app, id, appID); err != nil { return err } scope := mail.ClaimMailScope{Account: account, App: app, AppID: appID, Operation: id} if err = p.stage(ctx, h, "teardown", func(ctx context.Context) error { return mail.WithClaimMailResource(mail.WithClaimMailScope(ctx, scope), s.claimMailStore(), func(held context.Context) error { return p.teardown(held, h, account, app, id, appID) }) }); err != nil { return err } fresh, tag, err := hostops.ReadPurgeJournal(ctx, s.cfg.Releases, account, app, id, appID) if err != nil { return err } if fresh.Phase != "claimed" { return hostops.ErrPending } fresh.Phase = "torn-down" _, err = w.SavePurgeJournal(ctx, fresh, tag) return err case "torn-down": if err = p.stage(ctx, h, "retire-hosts", func(ctx context.Context) error { return p.Controller.RetirePurgeHosts(ctx, account, app, id, appID) }); err != nil { return err } if err = p.stage(ctx, h, "retire-domains", func(ctx context.Context) error { return p.retireDomains(ctx, account, app, id, appID) }); err != nil { return err } fresh, tag, err := hostops.ReadPurgeJournal(ctx, s.cfg.Releases, account, app, id, appID) if err != nil { return err } fresh.Phase = "retired" _, err = w.SavePurgeJournal(ctx, fresh, tag) return err case "retired": default: return hostops.ErrPending } if err = p.stage(ctx, h, "sweep", func(ctx context.Context) error { if err := hostops.PreparePurgeSweep(ctx, s.cfg.Releases, w, p.Versions, p.ControlBucket, account, app, id, appID); err != nil { return err } if err := hostops.DeletePurgeSweep(ctx, s.cfg.Releases, w, p.Versions, p.ControlBucket, account, app, id, appID); err != nil { return err } return hostops.VerifyPurgeSweep(ctx, s.cfg.Releases, w, p.Versions, p.ControlBucket, account, app, id, appID) }); err != nil { return err } j, tag, err := hostops.ReadPurgeJournal(ctx, s.cfg.Releases, account, app, id, appID) if err != nil { return err } if !j.Cleanup.Tombstone { if err = p.stage(ctx, h, "tombstone", func(ctx context.Context) error { err := release.DeleteIfMatch(ctx, s.cfg.Releases, j.Tombstone.Key, j.Tombstone.ETag) if err != nil { if _, _, rerr := s.cfg.Releases.Get(ctx, j.Tombstone.Key); !errors.Is(rerr, os.ErrNotExist) { return errors.Join(err, rerr) } } return nil }); err != nil { return err } j.Cleanup.Tombstone = true if tag, err = w.SavePurgeJournal(ctx, j, tag); err != nil { return err } } if !j.Cleanup.AppRow { // The SQL boundary admits this delete on the stage's permit. if err = p.localSQLStage(ctx, h, "app-row", func(ctx context.Context) error { return s.store.WithAppMutation(ctx, []sqlGrant{permitGrant(ctx, s.cfg.Releases, account, scope)}, func(st *Store) error { if err := st.MarkClaimAppRetired(acct.ID, app, appID, p.Controller.Now()); err != nil { return err } return st.PurgeClaimAppRow(ctx, acct.ID, app, appID, id) }) }); err != nil { return err } j.Cleanup.AppRow = true if tag, err = w.SavePurgeJournal(ctx, j, tag); err != nil { return err } } // The purge retired its instances' incarnations (retire-hosts); their // move log outbox rows go with them, before the purge is done. if err = s.purgeMoveLogOfPurgedHosts(j); err != nil { return err } j.Phase = "done" if _, err = w.SavePurgeJournal(ctx, j, tag); err != nil { return err } return p.Controller.CompletePurgeFences(ctx, h) } // purgeMoveLogOfPurgedHosts drops the move log outbox rows of every instance // incarnation the purge journal records (Plan 6 Task 3). It is local and // idempotent, so a replayed stage repeats it harmlessly. func (s *Server) purgeMoveLogOfPurgedHosts(j hostops.PurgeJournal) error { for host, ph := range j.Hosts { if ph.Holder == nil || ph.Holder.Incarnation == "" { continue } if err := s.store.PurgeMoveLogIncarnation(host, ph.Holder.Incarnation); err != nil { return err } } return nil } func (p *ClaimPurgeCoordinator) SweepRequests(ctx context.Context) error { w, err := p.control() if err != nil { return err } apps, err := hostops.Apps(ctx, p.Controller.Store) if err != nil { return err } for _, app := range apps { key, err := hostops.PurgeRequestKey(app.Account, app.Name) if err != nil { return err } raw, _, err := p.Controller.Store.Get(ctx, key) if errors.Is(err, os.ErrNotExist) { continue } if err != nil { return err } removed, err := w.DiscardPurgeRequest(ctx, app.Account, app.Name) if err != nil { return err } if removed { continue } var request release.ClaimPurgeRequest if err = json.Unmarshal(raw, &request); err != nil { return err } if request.V != 1 || request.ID == "" || request.AppID == "" || request.Account != app.Account || request.App != app.Name || request.TombstoneKey != release.AppTombstoneKey(app.Account, app.Name) { return hostops.ErrIdentity } // An existing reservation resumes with its captured marker. existing, _, perr := hostops.ReadPurge(ctx, p.Controller.Store, app.Account, app.Name) if perr == nil { if err = p.Resume(ctx, app.Account, app.Name, existing.ID, request.AppID); err != nil && !errors.Is(err, hostops.ErrPending) { return err } continue } if !errors.Is(perr, os.ErrNotExist) { return perr } body, tag, err := p.Controller.Store.Get(ctx, request.TombstoneKey) if err != nil { return err } if tag != request.TombstoneETag || !bytes.Equal(body, request.Tombstone) { return hostops.ErrPending } acct, err := p.Server.store.AccountBySqid(app.Account) if err != nil { return err } if acct == nil { return hostops.ErrIdentity } appID, err := p.Server.store.ClaimAppID(acct.ID, app.Name) if err != nil { return err } if appID != request.AppID { return hostops.ErrIdentity } if _, err = p.Request(ctx, app.Account, app.Name, request.ID, "automatic", "auto-purge"); err != nil && !errors.Is(err, hostops.ErrPending) { return err } } return nil } func (p *ClaimPurgeCoordinator) mailReady(ctx context.Context, account, app string) error { estate := false record, _, err := mail.StrictGet(ctx, p.Controller.Store, account, app) if err != nil { return err } if record != nil && (record.Provisioned != nil || len(record.Standalone) != 0) { estate = true } // Complete inventories (C6): an unreadable record or claim stops the purge // instead of being skipped and outliving it. domainRecords, err := mail.StrictListDomains(ctx, p.Controller.Store, account) if err != nil { return err } _, dnsWriter := p.Server.mailDeploymentFor() for _, record := range domainRecords { for _, member := range record.Apps { if member == app { estate = true if record.Kind == mail.KindPlatform && len(record.Apps) == 1 && dnsWriter == nil { return hostops.ErrContract } } } } claims, err := mail.StrictDomainClaims(ctx, p.Controller.Store) if err != nil { return err } for _, claim := range claims { if claim.Account == account && claim.App == app { estate = true } } if estate && p.Server.cfg.MailProvisioner == nil { return hostops.ErrContract } return nil } func (p *ClaimPurgeCoordinator) captureDomains(ctx context.Context, account, app, id, appID string) error { w, err := p.control() if err != nil { return err } keys, err := p.Controller.Store.List(ctx, domains.ClaimsPrefix) if err != nil { return err } j, tag, err := hostops.ReadPurgeJournal(ctx, p.Controller.Store, account, app, id, appID) if err != nil { return err } if j.Phase != "reserved" { return hostops.ErrPending } if j.Effects == nil { j.Effects = map[string]json.RawMessage{} } for _, key := range keys { raw, etag, err := p.Controller.Store.Get(ctx, key) if err != nil { return err } var claim domains.Claim if err = json.Unmarshal(raw, &claim); err != nil { return err } if claim.Account != account || claim.App != app { continue } if claim.Acquisition == "" || claim.OwnershipState == "quarantined" || etag == "" || key != domains.ClaimKey(claim.Host) { return hostops.ErrPending } captured, err := json.Marshal(hostops.PurgeObject{Key: key, ETag: etag, Body: raw}) if err != nil { return err } j.Effects["domain."+claim.Host] = captured } _, err = w.SavePurgeJournal(ctx, j, tag) return err } // retireDomains runs inside the "retire-domains" stage: its exact-ETag // deletes pass the data boundary on that stage's permit. func (p *ClaimPurgeCoordinator) retireDomains(ctx context.Context, account, app, id, appID string) error { j, _, err := hostops.ReadPurgeJournal(ctx, p.Controller.Store, account, app, id, appID) if err != nil { return err } if j.Phase != "torn-down" { return hostops.ErrPending } for _, host := range j.Hosts { if host.Holder != nil && !host.Retired { return hostops.ErrPending } } for name, raw := range j.Effects { if len(name) < 7 || name[:7] != "domain." { continue } var captured hostops.PurgeObject if err = json.Unmarshal(raw, &captured); err != nil { return err } var old domains.Claim if err = json.Unmarshal(captured.Body, &old); err != nil { return err } if old.Account != account || old.App != app || old.Acquisition == "" || captured.Key != domains.ClaimKey(old.Host) { return hostops.ErrIdentity } body, etag, err := p.Controller.Store.Get(ctx, captured.Key) if errors.Is(err, os.ErrNotExist) { continue } if err != nil { return err } var current domains.Claim if err = json.Unmarshal(body, ¤t); err != nil { return err } if current.Acquisition != old.Acquisition { continue } if etag != captured.ETag || !bytes.Equal(body, captured.Body) { return hostops.ErrPending } if err = release.DeleteIfMatch(ctx, p.Controller.Store, captured.Key, captured.ETag); err != nil { return err } } return nil } // teardown runs inside the "teardown" stage and the held mail gate. Each // backend call is its own effect under the purge operation. func (p *ClaimPurgeCoordinator) teardown(ctx context.Context, h *hostops.OperationHandle, account, app, id, appID string) error { s := p.Server w, err := p.control() if err != nil { return err } if err := p.mailReady(ctx, account, app); err != nil { return err } j, tag, err := hostops.ReadPurgeJournal(ctx, p.Controller.Store, account, app, id, appID) if err != nil { return err } if j.Phase != "claimed" { return hostops.ErrPending } if j.Effects == nil { j.Effects = map[string]json.RawMessage{} } // Capture every backend reconstruction input under both gates, before // the first destructive call. envs, err := objstore.StrictStoreEnvironments(ctx, p.Controller.Store, account, app) if err != nil { return err } for _, env := range append(envs, "") { record, _, err := objstore.StrictGetIn(ctx, p.Controller.Store, account, app, env) if err != nil { return err } if record != nil && (record.Provisioned != nil || record.Grant != nil) && s.cfg.StoreProvisioner == nil { return hostops.ErrContract } raw, err := json.Marshal(record) if err != nil { return err } j.Effects["store."+env] = raw } record, _, err := mail.StrictGet(ctx, p.Controller.Store, account, app) if err != nil { return err } raw, err := json.Marshal(record) if err != nil { return err } j.Effects["email.record"] = raw domainRecords, err := mail.StrictListDomains(ctx, p.Controller.Store, account) if err != nil { return err } for _, record := range domainRecords { raw, err := json.Marshal(record) if err != nil { return err } j.Effects["email.domain."+record.Domain] = raw } domainClaims, err := mail.StrictDomainClaims(ctx, p.Controller.Store) if err != nil { return err } for _, claim := range domainClaims { if claim.Account == account && claim.App == app { raw, err := json.Marshal(claim) if err != nil { return err } j.Effects["email.claim."+claim.Domain] = raw } } if _, err = w.SavePurgeJournal(ctx, j, tag); err != nil { return err } effects := claimPurgeEffects{h: h} for _, env := range append(envs, "") { rec, _, err := objstore.StrictGetIn(ctx, p.Controller.Store, account, app, env) if err != nil { return err } if rec == nil || (rec.Provisioned == nil && rec.Grant == nil) { continue } var scanner objstore.Scanner if s.cfg.StoreScanner != nil { scanner = claimPurgeScanner{raw: s.cfg.StoreScanner, e: effects} } if err = objstore.TeardownIn(ctx, p.Controller.Store, claimPurgeStore{raw: s.cfg.StoreProvisioner, e: effects}, scanner, account, app, env); err != nil { return err } } if s.cfg.MailProvisioner != nil { deployment, writer := s.mailDeploymentFor() var guardedDNS dnsrec.Writer if writer != nil { guardedDNS = claimPurgeDNS{raw: writer, e: effects} } if err = mail.Teardown(ctx, s.claimMailStore(), claimPurgeMail{raw: s.cfg.MailProvisioner, e: effects}, guardedDNS, deployment, account, app); err != nil { return err } } return nil } func (s *Server) SweepClaimPurgeRequests(ctx context.Context) error { if s.cfg.ClaimPurge == nil { return hostops.ErrContract } return s.cfg.ClaimPurge.SweepRequests(ctx) }