package maintenanceguard_test import ( "bytes" "context" "crypto/sha256" "encoding/base64" "encoding/hex" "errors" "fmt" "os" "sync" "testing" "time" guard "github.com/carlosframework/platform/internal/maintenanceguard" "github.com/carlosframework/platform/internal/release" ) var ctx = context.Background() func id(b byte) string { return base64.RawURLEncoding.EncodeToString(bytes.Repeat([]byte{b}, 32)) } func hash(s string) string { x := sha256.Sum256([]byte(s)); return hex.EncodeToString(x[:]) } type fixture struct { store *release.DirStore repo *guard.Repository identity guard.Identity } func setup(t *testing.T) fixture { t.Helper() s := release.NewDirStore(t.TempDir()) identity, e := guard.RecordIdentity("aaa", "home", "one.home.example", time.Date(2026, 9, 8, 0, 0, 0, 1, time.UTC)) if e != nil { t.Fatal(e) } return fixture{s, guard.New(s), guard.Identity{Account: "aaa", App: "home", Host: "one.home.example", Record: identity}} } func (f fixture) ref(t *testing.T, kind guard.ReferenceKind, n byte) guard.Reference { t.Helper() r, e := f.repo.PutReference(ctx, f.identity.Host, kind, id(n), []byte(fmt.Sprintf(`{"metadata":"%s-%d"}`, kind, n))) if e != nil { t.Fatal(e) } return r } func (f fixture) claim(t *testing.T, n byte) guard.ClaimSnapshot { t.Helper() s, e := f.repo.AcquireClaim(ctx, "aaa", "home", id(n), guard.MaintenanceOperation) if e != nil { t.Fatal(e) } return s } func (f fixture) initial(t *testing.T, c guard.ClaimSnapshot) guard.Guard { t.Helper() return guard.Guard{Version: 1, Identity: f.identity, Generation: 1, Phase: guard.Requested, OperationID: c.Value().OperationID, OwnerBox: "box-one", RecordRevision: `"original-record-etag"`, Request: f.ref(t, guard.RequestReference, 2), Attempt: f.ref(t, guard.AttemptReference, 3), ArtifactVersion: "v1", ArtifactDigest: hash("artifact"), ConfigDigest: hash("randomized-config-envelope"), RequiredContracts: []string{}} } func (f fixture) create(t *testing.T, c guard.ClaimSnapshot) guard.GuardSnapshot { t.Helper() g, e := f.repo.CreateGuard(ctx, c, f.initial(t, c)) if e != nil { t.Fatal(e) } return g } func (f fixture) step(t *testing.T, c guard.ClaimSnapshot, s guard.GuardSnapshot, phase guard.Phase) guard.GuardSnapshot { t.Helper() g := s.Value() g.Generation++ g.Phase = phase if phase == guard.ResumeRequested { g.Request = f.ref(t, guard.RequestReference, 4) g.Attempt = f.ref(t, guard.AttemptReference, 5) g.RequiredContracts = []string{"home.operator.v1"} g.Evidence = nil } if phase == guard.Succeeded || phase == guard.Resumed { ref := f.ref(t, guard.ReceiptReference, byte(g.Generation+10)) g.Evidence = &ref } next, e := f.repo.AdvanceGuard(ctx, c, s, g) if e != nil { t.Fatal(phase, e) } return next } func (f fixture) resumed(t *testing.T, c guard.ClaimSnapshot) guard.GuardSnapshot { s := f.create(t, c) for _, p := range []guard.Phase{guard.Quiescing, guard.Fenced, guard.Executing, guard.Succeeded, guard.ResumeRequested, guard.Resumed} { s = f.step(t, c, s, p) } return s } func denied(t *testing.T, f fixture, p *guard.Projection, contracts []string) { t.Helper() d, e := f.repo.CheckServing(ctx, f.identity, p, contracts) if d != guard.Denied || !errors.Is(e, guard.ErrDenied) { t.Fatal(d, e) } } func TestGuardUnmanagedAndAbsenceRecheck(t *testing.T) { f := setup(t) if d, e := f.repo.CheckServing(ctx, f.identity, nil, nil); e != nil || d != guard.Unmanaged { t.Fatal(d, e) } p := guard.Projection{Version: 1, Generation: 1, GuardDigest: hash("guard")} denied(t, f, &p, nil) c := f.claim(t, 1) // A claim alone is not a host fence. This models the crash before guard creation. if d, e := f.repo.CheckServing(ctx, f.identity, nil, nil); e != nil || d != guard.Unmanaged { t.Fatal(d, e) } s := f.create(t, c) p, e := s.Projection() if e != nil { t.Fatal(e) } // A worker that observed absence before acquiring its lease must recheck. denied(t, f, nil, nil) denied(t, f, &p, nil) } func TestGuardResumeCrashProjectionAndContracts(t *testing.T) { f := setup(t) c := f.claim(t, 1) s := f.create(t, c) for _, phase := range []guard.Phase{guard.Quiescing, guard.Fenced, guard.Executing, guard.Succeeded, guard.ResumeRequested} { s = f.step(t, c, s, phase) p, _ := s.Projection() denied(t, f, &p, []string{"home.operator.v1"}) } oldProjection, _ := s.Projection() s = f.step(t, c, s, guard.Resumed) // Durable guard publication without the record CAS must not open admission. denied(t, f, &oldProjection, []string{"home.operator.v1"}) denied(t, f, nil, []string{"home.operator.v1"}) reopened := guard.New(f.store) loaded, e := reopened.ReadGuard(ctx, f.identity.Host) if e != nil { t.Fatal(e) } p, _ := loaded.Projection() if d, e := reopened.CheckServing(ctx, f.identity, &p, []string{"home.operator.v1"}); e != nil || d != guard.Managed { t.Fatal(d, e) } denied(t, f, &p, nil) denied(t, f, &p, []string{"home.operator.v0"}) foreign := f.identity foreign.Account = "bbb" if d, e := f.repo.CheckServing(ctx, foreign, &p, []string{"home.operator.v1"}); d != guard.Denied || e == nil { t.Fatal("hostname reassigned") } foreign = f.identity foreign.Record = hash("replacement") if d, e := f.repo.CheckServing(ctx, foreign, &p, []string{"home.operator.v1"}); d != guard.Denied || e == nil { t.Fatal("record replaced") } if _, e = f.repo.ReleaseClaim(ctx, c); e != nil { t.Fatal(e) } held, e := f.repo.ReadLifecycle(ctx, "aaa", "home") if e != nil || held.Value().State != "idle" || held.Value().Generation != 2 { t.Fatal(held.Value(), e) } } func TestGuardStaleWorkersAndExecutingRecovery(t *testing.T) { f := setup(t) c := f.claim(t, 1) first := f.create(t, c) second := f.step(t, c, first, guard.Quiescing) stale := first.Value() stale.Generation++ stale.Phase = guard.Quiescing if _, e := f.repo.AdvanceGuard(ctx, c, first, stale); !errors.Is(e, release.ErrCASConflict) { t.Fatal("stale write did not preserve CAS conflict", e) } s := f.step(t, c, second, guard.Fenced) s = f.step(t, c, s, guard.Executing) afterCrash, e := f.repo.ReadGuard(ctx, f.identity.Host) if e != nil { t.Fatal(e) } repeat := afterCrash.Value() repeat.Generation++ if _, e = f.repo.AdvanceGuard(ctx, c, afterCrash, repeat); !errors.Is(e, guard.ErrConflict) { t.Fatal("executing automatically repeated", e) } s = f.step(t, c, afterCrash, guard.Ambiguous) retry := s.Value() retry.Generation++ retry.Phase = guard.Requested retry.Request = f.ref(t, guard.RequestReference, 8) retry.Attempt = f.ref(t, guard.AttemptReference, 9) retry.Evidence = nil if _, e = f.repo.AdvanceGuard(ctx, c, s, retry); e != nil { t.Fatal("explicit new attempt refused", e) } } func TestGuardContinuityAndContractFloorCannotShrink(t *testing.T) { f := setup(t) c := f.claim(t, 1) s := f.create(t, c) for _, change := range []func(*guard.Guard){func(g *guard.Guard) { g.Generation-- }, func(g *guard.Guard) { g.Generation++ }, func(g *guard.Guard) { g.Identity.Record = hash("other") }, func(g *guard.Guard) { g.OwnerBox = "box-two" }, func(g *guard.Guard) { g.Request.Digest = hash("other") }, func(g *guard.Guard) { g.ArtifactDigest = hash("other") }} { g := s.Value() g.Generation++ g.Phase = guard.Quiescing change(&g) if _, e := f.repo.AdvanceGuard(ctx, c, s, g); !errors.Is(e, guard.ErrConflict) { t.Fatal("continuity changed", e) } } for _, p := range []guard.Phase{guard.Quiescing, guard.Fenced, guard.ResumeRequested, guard.Resumed} { s = f.step(t, c, s, p) } if _, e := f.repo.ReleaseClaim(ctx, c); e != nil { t.Fatal(e) } c = f.claim(t, 20) g := s.Value() g.Generation++ g.Phase = guard.Requested g.OperationID = c.Value().OperationID g.Request = f.ref(t, guard.RequestReference, 21) g.Attempt = f.ref(t, guard.AttemptReference, 22) g.Evidence = nil g.RequiredContracts = nil if _, e := f.repo.AdvanceGuard(ctx, c, s, g); !errors.Is(e, guard.ErrConflict) { t.Fatal("floor removed", e) } } func TestImmutableReferenceAndMissingEvidence(t *testing.T) { f := setup(t) r := f.ref(t, guard.RequestReference, 2) if _, e := f.repo.PutReference(ctx, f.identity.Host, guard.RequestReference, r.ID, []byte("changed")); !errors.Is(e, guard.ErrConflict) { t.Fatal(e) } c := f.claim(t, 1) s := f.resumed(t, c) p, _ := s.Projection() // Private payload expiry is separate and cannot remove permanent metadata. private := "private/expired-payload" if _, e := f.store.Put(ctx, private, []byte("synthetic-private"), ""); e != nil { t.Fatal(e) } if e := f.store.Delete(ctx, private); e != nil { t.Fatal(e) } if d, e := f.repo.CheckServing(ctx, f.identity, &p, []string{"home.operator.v1"}); e != nil || d != guard.Managed { t.Fatal(d, e) } key, _ := guard.ReceiptKey(f.identity.Host, s.Value().Evidence.ID) if e := f.store.Delete(ctx, key); e != nil { t.Fatal(e) } denied(t, f, &p, []string{"home.operator.v1"}) } // absenceBarrier forces both contenders to observe absence before either CAS. type absenceBarrier struct { release.Store key string mu sync.Mutex n int ready chan struct{} } func (s *absenceBarrier) Get(ctx context.Context, key string) ([]byte, string, error) { b, etag, e := s.Store.Get(ctx, key) if key == s.key && errors.Is(e, os.ErrNotExist) { s.mu.Lock() s.n++ if s.n == 2 { close(s.ready) } s.mu.Unlock() <-s.ready } return b, etag, e } func TestLifecycleSimultaneousAbsenceMaintenanceVersusPurge(t *testing.T) { for i := 0; i < 20; i++ { f := setup(t) key, _ := guard.LifecycleKey("aaa", "home") barrier := &absenceBarrier{Store: f.store, key: key, ready: make(chan struct{})} r := guard.New(barrier) record := "apps/aaa/home/instances/one.home.example.json" if _, e := f.store.Put(ctx, record, []byte("existing-record"), ""); e != nil { t.Fatal(e) } type result struct { kind guard.OperationKind claim guard.ClaimSnapshot err error } out := make(chan result, 2) for n, kind := range []guard.OperationKind{guard.MaintenanceOperation, guard.PurgeOperation} { go func(n int, kind guard.OperationKind) { c, e := r.AcquireClaim(ctx, "aaa", "home", id(byte(n+40)), kind) out <- result{kind, c, e} }(n, kind) } a, b := <-out, <-out if (a.err == nil) == (b.err == nil) { t.Fatal("two absence readers both won or lost", a.err, b.err) } winner, loser := a, b if winner.err != nil { winner, loser = b, a } if !errors.Is(loser.err, release.ErrCASConflict) { t.Fatal(loser.err) } if winner.kind == guard.PurgeOperation { if e := f.store.Delete(ctx, record); e != nil { t.Fatal(e) } } else { g := f.initial(t, winner.claim) if _, e := r.CreateGuard(ctx, winner.claim, g); e != nil { t.Fatal(e) } } s, e := r.ReadGuard(ctx, f.identity.Host) if e != nil { t.Fatal(e) } _, _, recordErr := f.store.Get(ctx, record) if winner.kind == guard.MaintenanceOperation && (!s.Exists() || recordErr != nil) { t.Fatal("purge crossed maintenance claim") } if winner.kind == guard.PurgeOperation && (s.Exists() || !errors.Is(recordErr, os.ErrNotExist)) { t.Fatal("maintenance crossed purge claim") } } }