package hostops import ( "context" "crypto/ed25519" "encoding/json" "errors" "strconv" "strings" "sync" "testing" "time" "github.com/carlosframework/platform/internal/release" ) // hookStore runs hook once, on the first admission Put whose next value // has an effect in state want. A nil hook error lets the Put through; on a // hook error, land decides whether the Put still reaches the store (a lost // response) or not. type hookStore struct { *release.DirStore mu sync.Mutex want string land bool hook func(ctx context.Context) error } func (s *hookStore) Put(ctx context.Context, key string, body []byte, tag string) (string, error) { if hook := s.take(key, body); hook != nil { herr := hook(ctx) if herr == nil { return s.DirStore.Put(ctx, key, body, tag) } if !s.land { return "", herr } if _, err := s.DirStore.Put(context.WithoutCancel(ctx), key, body, tag); err != nil { return "", err } return "", herr } return s.DirStore.Put(ctx, key, body, tag) } func (s *hookStore) take(key string, body []byte) func(context.Context) error { s.mu.Lock() defer s.mu.Unlock() if s.hook == nil || !strings.HasSuffix(key, "/admission.json") { return nil } var a AppAdmission if json.Unmarshal(body, &a) != nil { return nil } for _, e := range a.Effects { if e.State == s.want { hook := s.hook s.hook = nil return hook } } return nil } type mutationFixture struct { raw *hookStore w *ControlWriter data release.Store now time.Time owner RegistrationOwner } func newMutationFixture(t *testing.T) *mutationFixture { t.Helper() f := &mutationFixture{raw: &hookStore{DirStore: release.NewDirStore(t.TempDir())}, now: time.Date(2026, 10, 7, 12, 0, 0, 0, time.UTC)} pub, _, err := ed25519.GenerateKey(nil) if err != nil { t.Fatal(err) } if f.w, err = NewControlWriter(f.raw, pub, func() time.Time { return f.now }); err != nil { t.Fatal(err) } check, err := NewProductionMutationCheck(f.raw, f.w) if err != nil { t.Fatal(err) } if f.data, err = NewMutationBoundary(true, f.raw, check); err != nil { t.Fatal(err) } f.owner = RegistrationOwner{Console: "console", Enrollment: "enrollment", Controller: "controller"} if err = BindMutationRunner(f.data, f.w, f.owner, func(context.Context, string, string) (string, error) { return "app", nil }); err != nil { t.Fatal(err) } seedAdmission(t, f.raw, "bab", "demo", openAdmission("app")) return f } func (f *mutationFixture) admission(t *testing.T) AppAdmission { t.Helper() a, err := ReadAdmission(t.Context(), f.raw, "bab", "demo", "app") if err != nil { t.Fatal(err) } return a.Value } func onlyEffect(t *testing.T, a AppAdmission) Effect { t.Helper() if len(a.Effects) != 1 { t.Fatalf("want one effect, got %+v", a.Effects) } for _, e := range a.Effects { return e } return Effect{} } // Review P1-1: Issue failing after BeginExclusive (cancelled request, // create freeze, lost issued-CAS response) must not leave the app closed // when nothing was issued; the caller sees the original error, unchanged in // kind. An issued effect is never settled by the issuer's recovery. func TestRunAppMutationIssueFailureRecovers(t *testing.T) { for _, tc := range []struct { name string want string // effect state that triggers the hook land bool freeze bool wantErr error reopened bool effect string // "" none, else the effect's final state/result }{ {name: "cancelled before prepare", want: "prepared", wantErr: context.Canceled, reopened: true}, {name: "cancelled after prepare", want: "issued", wantErr: context.Canceled, reopened: true, effect: "complete/superseded"}, {name: "create freeze", freeze: true, wantErr: ErrIdentity, reopened: true}, {name: "issued response lost", want: "issued", land: true, wantErr: context.Canceled, effect: "issued/"}, } { t.Run(tc.name, func(t *testing.T) { f := newMutationFixture(t) ctx, cancel := context.WithCancel(t.Context()) defer cancel() if tc.freeze { // An unverifiable backfill plan: CheckCreateFreeze refuses. if _, err := f.raw.DirStore.Put(ctx, BackfillKey, []byte(`{}`), ""); err != nil { t.Fatal(err) } } else { f.raw.want, f.raw.land = tc.want, tc.land f.raw.hook = func(context.Context) error { cancel(); return context.Canceled } } called := false err := release.WithAppMutation(ctx, f.data, "bab", "demo", "test", func(context.Context) error { called = true; return nil }) if called { t.Fatal("callback ran without an issued effect") } if !errors.Is(err, tc.wantErr) { t.Fatalf("original error lost: %v", err) } if !errors.Is(tc.wantErr, ErrPending) && errors.Is(err, ErrPending) { t.Fatalf("recovery outcome reclassified the error as pending: %v", err) } a := f.admission(t) if tc.reopened { if a.State != "open" || a.Op != nil || a.Epoch != 2 { t.Fatalf("unissued operation wedged the app: %+v", a) } } else if a.State != "closed" || a.Op == nil || a.Epoch != 1 { t.Fatalf("issued effect lost its operation: %+v", a) } switch { case tc.effect == "" && len(a.Effects) != 0: t.Fatalf("unexpected effects: %+v", a.Effects) case tc.effect != "": if e := onlyEffect(t, a); e.State+"/"+e.ResultReference != tc.effect { t.Fatalf("effect %+v, want %s", e, tc.effect) } } if !tc.reopened { // Neither the sweep, at any age, may settle the issued effect. f.now = f.now.Add(24 * time.Hour) if err := f.w.RecoverMutation(t.Context(), "bab", "demo", "app", a.Op.ID); !errors.Is(err, ErrPending) { t.Fatalf("issued effect recovered: %v", err) } } }) } } // Review P1-1: recovery between a live issuer's prepare and issued CASes // supersedes the prepared effect and reopens with a new epoch; the late // issued CAS is refused, the callback never runs, and a fresh mutation // proceeds. func TestLateIssueIsFencedByUnissuedRecovery(t *testing.T) { f := newMutationFixture(t) ctx := t.Context() var opID string f.raw.want = "issued" f.raw.hook = func(context.Context) error { a := f.admission(t) opID = a.Op.ID // Too young: the sweep leaves a live issuer alone. if err := f.w.RecoverMutation(ctx, "bab", "demo", "app", opID); !errors.Is(err, ErrPending) { t.Fatalf("recovered a young operation: %v", err) } f.now = f.now.Add(UnissuedMutationRecoverAfter + time.Second) if err := f.w.RecoverMutation(ctx, "bab", "demo", "app", opID); err != nil { t.Fatalf("aged unissued operation not recovered: %v", err) } return nil // the issuer's issued Put proceeds, against a stale etag } called := false err := release.WithAppMutation(ctx, f.data, "bab", "demo", "test", func(context.Context) error { called = true; return nil }) if called || !errors.Is(err, ErrPending) { t.Fatalf("late issuer was not fenced: called=%v err=%v", called, err) } a := f.admission(t) if a.State != "open" || a.Op != nil || a.Epoch != 2 { t.Fatalf("not reopened once: %+v", a) } if e := a.Effects[opID]; e.State != "complete" || e.ResultReference != "superseded" { t.Fatalf("prepared effect not superseded: %+v", e) } if err = release.WithAppMutation(ctx, f.data, "bab", "demo", "test", func(context.Context) error { return nil }); err != nil { t.Fatalf("app stayed wedged: %v", err) } } // Review P1-1: a lost-response begin (or a crash right after it) leaves an // operation with no effect; the sweep reopens it once it is old enough, and // the stale handle can neither issue nor finish. func TestRecoverMutationAbandonsEffectlessGenericOperation(t *testing.T) { f := newMutationFixture(t) ctx := t.Context() started := strconv.FormatInt(f.now.Unix(), 10) h, err := f.w.BeginExclusive(ctx, "bab", "demo", "app", AppOperation{ID: "op", Kind: OpAppMutation, AppID: "app", Intent: map[string]string{"action": "test", MutationActionStarted: started}}, f.owner) if err != nil { t.Fatal(err) } if err = f.w.RecoverMutation(ctx, "bab", "demo", "app", "op"); !errors.Is(err, ErrPending) { t.Fatalf("young operation recovered: %v", err) } f.now = f.now.Add(UnissuedMutationRecoverAfter) if err = f.w.RecoverMutation(ctx, "bab", "demo", "app", "op"); err != nil { t.Fatal(err) } if a := f.admission(t); a.State != "open" || a.Op != nil || a.Epoch != 2 || len(a.Effects) != 0 { t.Fatalf("not reopened: %+v", a) } if _, _, err = h.Runner(func() time.Time { return f.now }).Issue(ctx, Effect{ID: "op", Resource: OpAppMutation, Action: "test"}); !errors.Is(err, ErrPending) { t.Fatalf("stale handle issued: %v", err) } if err = h.FinishMutation(ctx); !errors.Is(err, ErrPending) { t.Fatalf("stale handle finished: %v", err) } if a := f.admission(t); a.State != "open" || a.Epoch != 2 || len(a.Effects) != 0 { t.Fatalf("stale handle changed the admission: %+v", a) } } // Generic recovery never settles issued or uncertain work by age, never // touches a registration-holding app, and leaves restore and mail to their // specialized branches. func TestRecoverMutationGenericRefusals(t *testing.T) { old := strconv.FormatInt(time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC).Unix(), 10) for _, tc := range []struct { name string action string mutate func(*AppAdmission) }{ {"issued", "test", func(a *AppAdmission) { a.Effects["op"] = Effect{ID: "op", OperationID: "op", AppID: "app", Resource: OpAppMutation, Action: "test", State: "issued"} }}, {"uncertain", "test", func(a *AppAdmission) { a.Effects["op"] = Effect{ID: "op", OperationID: "op", AppID: "app", Resource: OpAppMutation, Action: "test", State: "uncertain"} }}, {"sibling uncertain", "test", func(a *AppAdmission) { a.Effects["op"] = Effect{ID: "op", OperationID: "op", AppID: "app", Resource: OpAppMutation, Action: "test", State: "prepared"} a.Effects["x"] = Effect{ID: "x", OperationID: "op", AppID: "app", Resource: "other", Action: "test", State: "uncertain"} }}, {"foreign prepared", "test", func(a *AppAdmission) { a.Effects["x"] = Effect{ID: "x", OperationID: "other", AppID: "app", Resource: "other", Action: "test", State: "prepared"} }}, {"registration", "test", func(a *AppAdmission) { a.Registrations["r"] = Registration{} }}, {"restore", "restore", func(*AppAdmission) {}}, {"mail", "mail", func(*AppAdmission) {}}, } { t.Run(tc.name, func(t *testing.T) { f := newMutationFixture(t) a := openAdmission("app") a.State = "closed" a.Op = &AppOperation{ID: "op", Kind: OpAppMutation, AppID: "app", Intent: map[string]string{"action": tc.action, MutationActionStarted: old, "gate": "g"}} tc.mutate(&a) seedAdmission(t, f.raw, "bab", "demo", a) before, _ := ReadAdmission(t.Context(), f.raw, "bab", "demo", "app") // Restore and mail keep their specialized RecoverMutation // branches; the generic transition itself refuses them. tryRecover := func() error { return f.w.RecoverMutation(t.Context(), "bab", "demo", "app", "op") } if tc.action != "test" { tryRecover = func() error { return f.w.abandonUnissuedMutation(t.Context(), "bab", "demo", "app", "op", 0) } } if err := tryRecover(); !errors.Is(err, ErrPending) { t.Fatalf("recovered: %v", err) } after, _ := ReadAdmission(t.Context(), f.raw, "bab", "demo", "app") if after.ETag != before.ETag { t.Fatalf("refused recovery changed the admission: %+v", after.Value) } }) } } // Review P1-2: a definitive refusal before any write (stale etag at the // boundary, validation) releases the operation; a refusal after a write // keeps it. func TestRunAppMutationRefusalDoesNotWedge(t *testing.T) { f := newMutationFixture(t) ctx := t.Context() key := "apps/bab/demo/test.json" err := release.WithAppMutation(ctx, f.data, "bab", "demo", "test", func(pctx context.Context) error { _, err := f.data.Put(pctx, key, []byte(`{}`), "stale") return err }) if !errors.Is(err, release.ErrCASConflict) { t.Fatalf("stale etag: %v", err) } if a := f.admission(t); a.State != "open" || a.Op != nil || !effectsSettled(&a, "") { t.Fatalf("stale-etag refusal wedged the app: %+v", a) } failure := errors.New("second write refused") err = release.WithAppMutation(ctx, f.data, "bab", "demo", "test", func(pctx context.Context) error { if _, err := f.data.Put(pctx, key, []byte(`{}`), ""); err != nil { return err } return failure }) if !errors.Is(err, failure) { t.Fatal(err) } if a := f.admission(t); a.State != "closed" || a.Op == nil { t.Fatalf("an issued write released its operation: %+v", a) } }