package hostops import ( "context" "crypto/sha256" "encoding/hex" "encoding/json" "errors" "fmt" "os" "slices" "strings" "time" ) // Fenced reconciliation of a retained mail-resource gate (review P1-2). // // A gated mail call that fails after an effect leaves every member effect // "uncertain" and the gate held. Nothing here asks a caller whether the call // worked. The settle that recorded "uncertain" also recorded, in the effect's // result, exactly which surfaces the call attempted (MailAttempt). The // reconciler: // // 1. fences the gate: one CAS bumps its revision and clears its token, so no // lease of the old holder passes CheckMailGate (every mail backend and // mail-gated bucket write checks it) and no new holder can acquire; // 2. reads back every attempted surface it has read authority for, from the // writer's own raw bucket and a typed SES identity reader; // 3. refuses (ErrMailUnproven, gate stays held and fenced) if any attempted // surface has no readback (DNS, IAM, tenants, configuration sets), if any // member effect is still "issued" (the issuer may be mid-call), or if a // result carries no attempt record; // 4. settles each uncertain effect complete with the evidence digest, each // merely prepared one "superseded", then frees the gate with the full // evidence stored on the gate object. // // Member operations end afterwards through RecoverMutation, which requires // the gate to no longer hold their gate ID: ops still end only after the // gate's release CAS. // ErrMailUnproven is a reconciliation the writer cannot prove from readback. // It wraps ErrPending: the gate and every member stay held. var ErrMailUnproven = fmt.Errorf("%w: mail gate reconciliation lacks readback evidence", ErrPending) // MailAttempt is one mutating surface a gated mail call reached (it was not // refused before effect). Surfaces: "store" (Key), "ses-identity" (Region, // Domain), and the surfaces with no readback: "ses-tenant", "ses-config-set", // "iam" (Name), "dns" (Name). type MailAttempt struct { Surface string `json:"surface"` Method string `json:"method"` Key string `json:"key,omitempty"` Region string `json:"region,omitempty"` Domain string `json:"domain,omitempty"` Name string `json:"name,omitempty"` } const mailAttemptsPrefix = "mail-attempts:" // MailAttemptsResult encodes the attempts into an uncertain effect's result. func MailAttemptsResult(attempts []MailAttempt) string { body, err := json.Marshal(normalizeAttempts(attempts)) if err != nil { return "" } return mailAttemptsPrefix + string(body) } func parseMailAttempts(result string) ([]MailAttempt, bool) { rest, ok := strings.CutPrefix(result, mailAttemptsPrefix) if !ok { return nil, false } var out []MailAttempt dec := json.NewDecoder(strings.NewReader(rest)) dec.DisallowUnknownFields() if err := dec.Decode(&out); err != nil { return nil, false } return out, true } func normalizeAttempts(in []MailAttempt) []MailAttempt { out := slices.Clone(in) slices.SortFunc(out, func(a, b MailAttempt) int { return strings.Compare(a.Surface+"\x00"+a.Method+"\x00"+a.Key+"\x00"+a.Region+"\x00"+a.Domain+"\x00"+a.Name, b.Surface+"\x00"+b.Method+"\x00"+b.Key+"\x00"+b.Region+"\x00"+b.Domain+"\x00"+b.Name) }) return slices.Compact(out) } // MailIdentityReader is the typed SES readback. The console adapts its RAW // mail provisioner's IdentityStatus (a read) and returns its canonical JSON. type MailIdentityReader interface { MailIdentityState(ctx context.Context, region, domain string) (json.RawMessage, error) } // MailReadback is one surface's observed state after the fence. type MailReadback struct { Attempt MailAttempt `json:"attempt"` Absent bool `json:"absent,omitempty"` ETag string `json:"etag,omitempty"` SHA256 string `json:"sha256,omitempty"` State json.RawMessage `json:"state,omitempty"` } // MailGateReconciliation is the evidence a reconciliation recorded. type MailGateReconciliation struct { Gate string `json:"gate"` Revision uint64 `json:"revision"` // the fenced revision Members []MailGateMember `json:"members"` Readback []MailReadback `json:"readback"` At time.Time `json:"at"` Digest string `json:"digest"` // sha256 of the record with Digest empty } // ReconcileMailGate fences, reads back and settles the retained gate gateID. // minHeld is liveness only (do not interrupt a holder that is probably still // running); it is never the safety argument and never settles an issued // effect. Idempotent: a gate already reconciled for gateID returns its // record. func (w *ControlWriter) ReconcileMailGate(ctx context.Context, gateID string, minHeld time.Duration, ses MailIdentityReader) (MailGateReconciliation, error) { if err := w.ready(); err != nil { return MailGateReconciliation{}, err } if gateID == "" { return MailGateReconciliation{}, ErrIdentity } g, tag, err := w.readMailGate(ctx) if err != nil { return MailGateReconciliation{}, err } if g.State != "held" || g.ID != gateID { if g.Reconciled != nil && g.Reconciled.Gate == gateID { return *g.Reconciled, nil } return MailGateReconciliation{}, ErrPending } // 1. Fence. if !g.Reconciling { if w.now().Before(g.HeldAt.Add(minHeld)) { return MailGateReconciliation{}, ErrPending } g.Revision++ g.TokenSHA256, g.Reconciling = "", true body, err := json.Marshal(g) if err != nil { return MailGateReconciliation{}, err } if tag, err = w.raw.Put(ctx, MailGateKey, body, tag); err != nil { return MailGateReconciliation{}, err } } fenced := g.Revision // 2/3. Classify members and collect attempts. var attempts []MailAttempt for _, m := range g.Members { a, err := w.read(ctx, m.Account, m.App, m.AppID) if err != nil { return MailGateReconciliation{}, err } e, ok := a.Value.Effects[m.Effect] if !ok { continue // admitted, never prepared: nothing to settle } if e.OperationID != m.Operation || e.AppID != m.AppID { return MailGateReconciliation{}, ErrIdentity } switch e.State { case "complete", "prepared": case "uncertain": // A readback cannot exclude a request arriving after the client's // timeout. Keep durable ownership until quiescence can be proved. return MailGateReconciliation{}, fmt.Errorf("%w: %s/%s has no remote quiescence proof", ErrMailUnproven, m.Account, m.App) default: // issued: the issuer may still be inside its call. No issuer // death proof exists; it settles or stays (no age settlement). return MailGateReconciliation{}, fmt.Errorf("%w: %s/%s effect %s is still issued", ErrMailUnproven, m.Account, m.App, m.Effect) } } attempts = normalizeAttempts(attempts) var unproven []string readback := make([]MailReadback, 0, len(attempts)) for _, at := range attempts { rb := MailReadback{Attempt: at} switch at.Surface { case "store": body, etag, err := w.raw.Get(ctx, at.Key) switch { case errors.Is(err, os.ErrNotExist): rb.Absent = true case err != nil: return MailGateReconciliation{}, err default: sum := sha256.Sum256(body) rb.ETag, rb.SHA256 = etag, hex.EncodeToString(sum[:]) } case "ses-identity": if ses == nil || at.Region == "" || at.Domain == "" { unproven = append(unproven, at.Surface+":"+at.Method+":"+at.Domain) continue } state, err := ses.MailIdentityState(ctx, at.Region, at.Domain) if err != nil { return MailGateReconciliation{}, err } rb.State = state default: unproven = append(unproven, at.Surface+":"+at.Method+":"+at.Name) continue } readback = append(readback, rb) } if len(unproven) != 0 { return MailGateReconciliation{}, fmt.Errorf("%w: no readback authority for %s", ErrMailUnproven, strings.Join(unproven, ", ")) } rec := MailGateReconciliation{Gate: gateID, Revision: fenced, Members: slices.Clone(g.Members), Readback: readback, At: w.now().UTC()} body, err := json.Marshal(rec) if err != nil { return MailGateReconciliation{}, err } sum := sha256.Sum256(body) rec.Digest = hex.EncodeToString(sum[:]) // 4. Settle members, then free the gate. for _, m := range g.Members { if _, err := w.retryAdmission(ctx, m.Account, m.App, m.AppID, AdmissionObserved{}, func(v *AppAdmission) error { e, ok := v.Effects[m.Effect] if !ok { return nil } switch e.State { case "complete": return nil case "prepared": e.State, e.ResultReference = "complete", "superseded" case "uncertain": e.State, e.ResultReference = "complete", "mail-reconciled:sha256:"+rec.Digest default: return ErrPending } v.Effects[m.Effect] = e return nil }); err != nil { return MailGateReconciliation{}, err } } g, tag, err = w.readMailGate(ctx) if err != nil { return MailGateReconciliation{}, err } if g.State != "held" || g.ID != gateID || !g.Reconciling || g.Revision != fenced { return MailGateReconciliation{}, ErrPending } g.Revision++ g.State, g.TokenSHA256, g.HeldAt, g.Members, g.Reconciling, g.Reconciled = "free", "", time.Time{}, nil, false, &rec if body, err = json.Marshal(g); err != nil { return MailGateReconciliation{}, err } if _, err = w.raw.Put(ctx, MailGateKey, body, tag); err != nil { return MailGateReconciliation{}, err } return rec, nil }