package hostops import ( "context" "crypto/ed25519" "errors" "strings" "sync" "testing" "time" "github.com/carlosframework/platform/internal/hostclaim" "github.com/carlosframework/platform/internal/release" ) // racingStore loses the next armed admission CAS: just before it, it runs // race (a concurrent renewal, settle or phase write through the same raw // store), so the armed Put's ETag is stale and fails with ErrCASConflict. type racingStore struct { *release.DirStore mu sync.Mutex race func() fired int } func (s *racingStore) arm(race func()) { s.mu.Lock() s.race = race s.mu.Unlock() } func (s *racingStore) Put(ctx context.Context, key string, body []byte, tag string) (string, error) { s.mu.Lock() race := s.race if strings.HasSuffix(key, "/admission.json") && race != nil { s.race = nil s.fired++ } else { race = nil } s.mu.Unlock() if race != nil { race() } return s.DirStore.Put(ctx, key, body, tag) } func racingWriter(t *testing.T) (*ControlWriter, *racingStore, func() time.Time) { t.Helper() raw := &racingStore{DirStore: release.NewDirStore(t.TempDir())} pub, _, err := ed25519.GenerateKey(nil) if err != nil { t.Fatal(err) } now := time.Date(2026, 10, 7, 12, 0, 0, 0, time.UTC) clock := func() time.Time { return now } w, err := NewControlWriter(raw, pub, clock) if err != nil { t.Fatal(err) } return w, raw, clock } func startRegistration(t *testing.T, w *ControlWriter, clock func() time.Time, id string) *StartHandle { t.Helper() r := Registration{Kind: "create", Host: "x.example", Owner: testOwner, AppID: "app", Epoch: 1, Claim: SavedClaim{Absent: true}} h, err := w.RegisterStart(t.Context(), "bab", "demo", "app", id, r, clock()) if err != nil { t.Fatal(err) } return h } // Review P1-3: a routine renewal racing the settle CAS must not leave a known // outcome "issued" forever. func TestSettleSurvivesRenewalCAS(t *testing.T) { ctx := t.Context() w, raw, clock := racingWriter(t) seedAdmission(t, raw.DirStore, "bab", "demo", openAdmission("app")) h := startRegistration(t, w, clock, "reg") _, settle, err := h.Runner(clock).Issue(ctx, Effect{ID: "eff", Resource: "store:x", Action: "grant"}) if err != nil { t.Fatal(err) } raw.arm(func() { if err := h.Renew(ctx, clock()); err != nil { t.Errorf("renew: %v", err) } }) if err = settle("ok", nil); err != nil { t.Fatalf("settle lost to a renewal: %v", err) } if raw.fired != 1 { t.Fatalf("race not exercised: %d", raw.fired) } a, err := ReadAdmission(ctx, raw.DirStore, "bab", "demo", "app") if err != nil { t.Fatal(err) } if e := a.Value.Effects["eff"]; e.State != "complete" || e.ResultReference != "ok" { t.Fatalf("effect after settle: %+v", e) } } // Review P1-3: Renew and Bind themselves retry a lost CAS. func TestRenewAndBindRetryLostCAS(t *testing.T) { ctx := t.Context() w, raw, clock := racingWriter(t) seedAdmission(t, raw.DirStore, "bab", "demo", openAdmission("app")) one := startRegistration(t, w, clock, "one") two := startRegistration(t, w, clock, "two") raw.arm(func() { _ = two.Renew(ctx, clock()) }) if err := one.Renew(ctx, clock()); err != nil { t.Fatalf("renew lost to a sibling renewal: %v", err) } raw.arm(func() { _ = two.Renew(ctx, clock()) }) if err := one.Bind(ctx, clock(), "inc-1", clock()); err != nil { t.Fatalf("bind lost to a sibling renewal: %v", err) } } // Review P1-3: the prepare and issue CASes run before the call, so they // replay after a lost race; the effect ends issued exactly once. func TestIssueReplaysPreparedAcrossCAS(t *testing.T) { ctx := t.Context() w, raw, clock := racingWriter(t) seedAdmission(t, raw.DirStore, "bab", "demo", openAdmission("app")) other := startRegistration(t, w, clock, "other") h := startRegistration(t, w, clock, "reg") raw.arm(func() { _ = other.Renew(ctx, clock()) }) _, settle, err := h.Runner(clock).Issue(ctx, Effect{ID: "eff", Resource: "store:x", Action: "grant"}) if err != nil { t.Fatalf("issue lost to a renewal: %v", err) } a, _ := ReadAdmission(ctx, raw.DirStore, "bab", "demo", "app") if a.Value.Effects["eff"].State != "issued" { t.Fatalf("effect: %+v", a.Value.Effects["eff"]) } if err = settle("ok", nil); err != nil { t.Fatal(err) } } // Review P1-3: SetPhase and the reopen retry; a retried reopen bumps the // epoch exactly once. func TestSetPhaseAndFinishRetryLostCAS(t *testing.T) { ctx := t.Context() w, raw, clock := racingWriter(t) seedAdmission(t, raw.DirStore, "bab", "demo", openAdmission("app")) h, err := w.BeginExclusive(ctx, "bab", "demo", "app", AppOperation{ID: "op", Kind: OpAppMutation, AppID: "app"}, testOwner) if err != nil { t.Fatal(err) } _, settle, err := h.Runner(clock).Issue(ctx, Effect{ID: "op", Resource: "app-mutation", Action: "test"}) if err != nil { t.Fatal(err) } if err = settle("joined", nil); err != nil { t.Fatal(err) } bump := func() { before, err := ReadAdmission(ctx, raw.DirStore, "bab", "demo", "app") if err != nil { t.Error(err) return } // An unrelated same-op intent write (another worker's phase journal). if _, err = w.updateAdmission(ctx, before, func(v *AppAdmission) error { v.Op.Intent["note"] = "x"; return nil }); err != nil { t.Error(err) } } raw.arm(bump) if err = h.SetPhase(ctx, "", "done", map[string]string{"k": "v"}); err != nil { t.Fatalf("SetPhase lost: %v", err) } raw.arm(bump) if err = h.FinishMutation(ctx); err != nil { t.Fatalf("finish lost: %v", err) } a, _ := ReadAdmission(ctx, raw.DirStore, "bab", "demo", "app") if a.Value.State != "open" || a.Value.Epoch != 2 { t.Fatalf("reopen: state %s epoch %d", a.Value.State, a.Value.Epoch) } } func restoreFixture(t *testing.T) (*ControlWriter, *release.DirStore, func() time.Time, *OperationHandle, string) { t.Helper() ctx := t.Context() w, raw, clock := testWriter(t) seedAdmission(t, raw, "bab", "demo", openAdmission("app")) if err := release.TrashApp(ctx, raw, "bab", "demo", "owner", clock()); err != nil { t.Fatal(err) } _, tag, err := raw.Get(ctx, release.AppTombstoneKey("bab", "demo")) if err != nil { t.Fatal(err) } h, err := w.BeginExclusive(ctx, "bab", "demo", "app", AppOperation{ID: "op", Kind: OpAppMutation, AppID: "app", Intent: map[string]string{"action": "restore"}}, testOwner) if err != nil { t.Fatal(err) } if err = h.PrepareRestore(ctx, tag, clock().Unix()); err != nil { t.Fatal(err) } return w, raw, clock, h, tag } // SQL-fix order BeginExclusive -> PrepareRestore -> Issue: a crash after // the receipt and before Issue is recovered from the receipt alone, the // epoch advances once, and a late Issue gets the recorded result, never a // permit. func TestRecoverPreparedReceiptWithNoEffect(t *testing.T) { ctx := t.Context() w, raw, clock, h, _ := restoreFixture(t) if err := w.RecoverMutation(ctx, "bab", "demo", "app", "op"); err != nil { t.Fatal(err) } a, err := ReadAdmission(ctx, raw, "bab", "demo", "app") if err != nil { t.Fatal(err) } e := a.Value.Effects["op"] if a.Value.State != "open" || a.Value.Op != nil || a.Value.Epoch != 2 || a.Value.LastRestore.State != "complete" || e.State != "complete" || e.ResultReference != "restore-reconciled" { t.Fatalf("not recovered: %+v", a.Value) } if _, _, err = raw.Get(ctx, release.AppTombstoneKey("bab", "demo")); err == nil { t.Fatal("tombstone survived") } _, _, err = h.Runner(clock).Issue(ctx, Effect{ID: "op", Resource: OpAppMutation, Action: "restore", Inputs: map[string]string{"tombstone_etag": "x"}}) if result, ok := EffectResult(err); !ok || result != "restore-reconciled" { t.Fatalf("late issue got a permit or the wrong refusal: %v", err) } // Idempotent: the issuer's own recovery call now succeeds. if err = w.RecoverMutation(ctx, "bab", "demo", "app", "op"); err != nil { t.Fatalf("second recovery: %v", err) } a, _ = ReadAdmission(ctx, raw, "bab", "demo", "app") if a.Value.Epoch != 2 { t.Fatalf("epoch bumped twice: %d", a.Value.Epoch) } } // Recovery's decision on an issued restore effect stands: the issuer's late // settle neither overwrites it nor fails. func TestSettleNeverOverwritesRecoveredOutcome(t *testing.T) { ctx := t.Context() w, raw, clock, h, _ := restoreFixture(t) _, settle, err := h.Runner(clock).Issue(ctx, Effect{ID: "op", Resource: OpAppMutation, Action: "restore"}) if err != nil { t.Fatal(err) } if err = w.RecoverMutation(ctx, "bab", "demo", "app", "op"); err != nil { t.Fatal(err) } callErr := errors.New("lost response") if err = settle("", callErr); !errors.Is(err, callErr) { t.Fatalf("late settle: %v", err) } a, _ := ReadAdmission(ctx, raw, "bab", "demo", "app") if e := a.Value.Effects["op"]; e.State != "complete" || e.ResultReference != "restore-reconciled" { t.Fatalf("recovered outcome overwritten: %+v", e) } } func mailMember(t *testing.T, w *ControlWriter, clock func() time.Time) (*MailGateLease, *OperationHandle, func(string, error) error) { t.Helper() ctx := t.Context() l, err := w.AcquireMailGate(ctx, MailGateOwner{Console: "c", Enrollment: "e", Controller: "k"}, MailGateScope{}) if err != nil { t.Fatal(err) } if err = w.RecordMailGateMember(ctx, l, MailGateMember{Account: "bab", App: "demo", AppID: "app", Operation: "op", Effect: "op"}); err != nil { t.Fatal(err) } h, err := w.BeginExclusive(ctx, "bab", "demo", "app", AppOperation{ID: "op", Kind: OpAppMutation, AppID: "app", Intent: map[string]string{"action": "mail", "gate": l.ID}}, testOwner) if err != nil { t.Fatal(err) } _, settle, err := h.Runner(clock).Issue(ctx, Effect{ID: "op", Resource: "mail-gate:" + l.ID, Action: "mail", Inputs: map[string]string{"gate": l.ID}}) if err != nil { t.Fatal(err) } return l, h, settle } // Review P1-2: a mail member whose effect completed is recovered after a // crash, but only once the gate no longer holds its gate ID. func TestMailMutationRecoversOnlyAfterGateRelease(t *testing.T) { ctx := t.Context() w, raw, clock := testWriter(t) seedAdmission(t, raw, "bab", "demo", openAdmission("app")) l, _, settle := mailMember(t, w, clock) if err := settle("joined", nil); err != nil { t.Fatal(err) } // Crash before Release: the gate still holds this gate ID. if err := w.RecoverMutation(ctx, "bab", "demo", "app", "op"); !errors.Is(err, ErrPending) { t.Fatalf("reopened while the gate is held: %v", err) } if err := w.ReleaseMailGate(ctx, l); err != nil { t.Fatal(err) } if err := w.RecoverMutation(ctx, "bab", "demo", "app", "op"); err != nil { t.Fatalf("complete effect + free gate not recovered: %v", err) } a, _ := ReadAdmission(ctx, raw, "bab", "demo", "app") if a.Value.State != "open" || a.Value.Epoch != 2 { t.Fatalf("not reopened: %+v", a.Value) } } // Review P1-2: an uncertain mail effect keeps gate and op until the fenced // reconciler settles it from readback; the old lease is fenced; the op then // recovers. func TestReconcileMailGateReadbackDoesNotProveQuiescence(t *testing.T) { ctx := t.Context() w, raw, clock := testWriter(t) seedAdmission(t, raw, "bab", "demo", openAdmission("app")) l, _, settle := mailMember(t, w, clock) callErr := errors.New("timeout") _ = settle(MailAttemptsResult([]MailAttempt{{Surface: "store", Method: "Put", Key: "accounts/bab/email/domains/example.com.json"}}), callErr) if _, err := w.ReconcileMailGate(ctx, l.ID, 0, nil); !errors.Is(err, ErrMailUnproven) { t.Fatalf("uncertain effect reopened: %v", err) } if err := w.RecoverMutation(ctx, "bab", "demo", "app", "op"); !errors.Is(err, ErrPending) { t.Fatalf("op reopened: %v", err) } a, _ := ReadAdmission(ctx, raw, "bab", "demo", "app") if a.Value.Effects["op"].State != "uncertain" || a.Value.Op == nil { t.Fatal("durable ownership was lost") } } // Review P1-2: no readback authority (DNS) or a still-issued effect refuses; // gate, op and effect all stay held. func TestReconcileMailGateRefusesUnproven(t *testing.T) { for _, tc := range []string{"dns", "issued"} { t.Run(tc, func(t *testing.T) { ctx := t.Context() w, raw, clock := testWriter(t) seedAdmission(t, raw, "bab", "demo", openAdmission("app")) l, _, settle := mailMember(t, w, clock) want := "issued" if tc == "dns" { _ = settle(MailAttemptsResult([]MailAttempt{{Surface: "dns", Method: "EnsureRecord", Name: "x.example.com|TXT"}}), errors.New("timeout")) want = "uncertain" } if _, err := w.ReconcileMailGate(ctx, l.ID, 0, nil); !errors.Is(err, ErrMailUnproven) { t.Fatalf("unproven reconciled: %v", err) } g, _, _ := w.readMailGate(ctx) a, _ := ReadAdmission(ctx, raw, "bab", "demo", "app") if g.State != "held" || g.ID != l.ID || a.Value.Effects["op"].State != want || a.Value.Op == nil { t.Fatalf("released something: gate %+v effect %+v", g, a.Value.Effects["op"]) } if err := w.RecoverMutation(ctx, "bab", "demo", "app", "op"); !errors.Is(err, ErrPending) { t.Fatalf("reopened: %v", err) } }) } } // Review P1-1 (v2 residue): an app grant can neither replace a foreign // holder by naming its own op in the new claim, nor overwrite a foreign op // on a free claim. func TestAppGrantCannotTakeForeignClaim(t *testing.T) { g := lifecycleGrant{kind: "app", account: "bab", app: "demo", appID: "app", operation: "mine"} foreign := &hostclaim.Claim{Host: "x.example", Gen: 4, Holder: &hostclaim.Holder{Account: "zed", App: "web", Incarnation: "z1"}, Op: hostclaim.Op{Kind: hostclaim.KindServing}} steal := &hostclaim.Claim{Host: "x.example", Gen: 5, Holder: &hostclaim.Holder{Account: "bab", App: "demo", Incarnation: "b1"}, Op: hostclaim.Op{ID: "mine"}} if appGrantAllows(g, foreign, steal) { t.Fatal("foreign holder replaced by naming the grant's op") } evict := &hostclaim.Claim{Host: "x.example", Gen: 5, Op: hostclaim.Op{ID: "mine", Kind: hostclaim.KindPurging}} if appGrantAllows(g, foreign, evict) { t.Fatal("foreign holder removed") } fenced := &hostclaim.Claim{Host: "x.example", Gen: 4, Op: hostclaim.Op{ID: "theirs", Kind: hostclaim.KindPurging}} if appGrantAllows(g, fenced, evict) { t.Fatal("foreign purge fence overwritten") } free := &hostclaim.Claim{Host: "x.example", Gen: 4} if !appGrantAllows(g, free, evict) { t.Fatal("own fence on a free claim refused") } if claimTransitionShape("x.example", free, &hostclaim.Claim{Host: "x.example", Gen: 9}) || claimTransitionShape("y.example", nil, &hostclaim.Claim{Host: "x.example", Gen: 1}) { t.Fatal("gen skip or host mismatch admitted") } } // Review P1-1 (v2 residue): a lifecycle grant dies with its registration. func TestLifecycleGrantDiesWithRegistration(t *testing.T) { ctx := t.Context() w, raw, clock := testWriter(t) seedAdmission(t, raw, "bab", "demo", openAdmission("app")) h := startRegistration(t, w, clock, "reg") grants, _ := h.Lifecycle(ctx).Value(lifecycleGrantKey{}).([]lifecycleGrant) if len(grants) != 1 { t.Fatal("no grant") } if live, err := grantLive(ctx, raw, grants[0]); err != nil || !live { t.Fatalf("live registration's grant refused: %v", err) } before, _ := ReadAdmission(ctx, raw, "bab", "demo", "app") if _, err := w.updateAdmission(ctx, before, func(v *AppAdmission) error { delete(v.Registrations, "reg"); return nil }); err != nil { t.Fatal(err) } if live, err := grantLive(ctx, raw, grants[0]); err != nil || live { t.Fatalf("drained registration's grant still live: %v", err) } }