package rollout import ( "encoding/json" "errors" "math" "os" "path/filepath" "testing" "time" "github.com/carlosframework/platform/internal/activator" ) // The crash-epoch hard limit (zero-downtime flag readiness round 2, Task 1; // Paul's ruling 2026-10-12): a stale signed retry grant never clears a later // failure. A restriction's generation is per (account, app), moves on every // change of the app's restrictions, is published only once it is on disk, // never repeats across a restart, and counts in a book identity that comes // from crypto/rand and is rotated when the rule is adopted. // diskBook is a book saved under a data dir of its own, its owed save // landed, and the dir. func diskBook(t *testing.T) (*FailureBook, string) { t.Helper() dd := t.TempDir() b, err := OpenFailureBook(dd) if err != nil { t.Fatal(err) } if err := b.Flush(); err != nil { t.Fatal(err) } return b, dd } // published is the generation status publishes for build's restriction of // host in bab/blog. func published(b *FailureBook, build, host string) uint64 { return b.PublishedEpoch("bab", "blog", build, host) } // grantAt applies a signed host retry of build on host naming b's identity // at epoch. func grantAt(b *FailureBook, build, host string, epoch uint64) (rec, mark bool) { rec, mark, _, _ = b.RetryHostAt("bab", "blog", build, host, b.ID(), epoch) return rec, mark } func mustRecord(t *testing.T, b *FailureBook, r FailureRecord) RecordResult { t.Helper() res, err := b.Record(r) if err != nil { t.Fatal(err) } return res } func mustMark(t *testing.T, b *FailureBook, host string, h HoldRan) { t.Helper() if err := b.MarkHoldRan(host, h); err != nil { t.Fatal(err) } } // Plan 8: a book whose save fails publishes nothing, and a grant for the // generation published before the change clears nothing. Once the owed save // lands the new generation is published and a grant at it clears. func TestAFailingDiskPublishesNoEpoch(t *testing.T) { b, dd := diskBook(t) mustRecord(t, b, FailureRecord{Account: "bab", App: "blog", Build: "v2", Host: "h1", Mode: string(activator.ModeHold), HeldDown: []string{"h1"}}) e1 := published(b, "v2", "h1") if e1 == 0 { t.Fatal("a saved restriction published no generation") } readOnly(t, dd) if _, err := b.Record(FailureRecord{Account: "bab", App: "blog", Build: "v3", Host: "h2"}); err == nil { t.Fatal("the save of v3's record did not fail: the case needs a failing disk") } if e := published(b, "v2", "h1"); e != 0 { t.Fatalf("published %d while the book's save is owed, want 0", e) } if rec, mark, _, err := b.RetryHostAt("bab", "blog", "v2", "h1", b.ID(), e1); rec || mark { t.Fatalf("a grant from before the unsaved change cleared record %v, marker %v (%v)", rec, mark, err) } if !b.HeldDown("bab", "blog", "v2", "h1") { t.Fatal("h1 is no longer held down") } writable(t, dd) if err := b.SaveIfDirty(); err != nil { t.Fatalf("the owed save did not land: %v", err) } e2 := published(b, "v2", "h1") if e2 <= e1 { t.Fatalf("published %d after the save landed, want past %d", e2, e1) } if rec, _ := grantAt(b, "v2", "h1", e2); !rec { t.Fatal("a grant at the published generation cleared nothing") } again, err := OpenFailureBook(dd) if err != nil { t.Fatal(err) } if again.HeldDown("bab", "blog", "v2", "h1") { t.Fatal("the retry was not saved") } } // Plan 1: every removal of a restriction of the app moves the app's // generation, so a grant signed before it clears nothing. func TestEveryRemovalMovesTheGeneration(t *testing.T) { const host = "h1" // v2's record (from a sibling's failure) holds h1, and v2's marker // stands on h1. setup := func(t *testing.T) *FailureBook { b, _ := diskBook(t) mustRecord(t, b, FailureRecord{Account: "bab", App: "blog", Build: "v2", Host: "sib", PromotedAt: promotedAt, At: t0}) mustMark(t, b, host, HoldRan{Account: "bab", App: "blog", Build: "v2", Exec: "a1", At: t0}) return b } for name, change := range map[string]func(t *testing.T, b *FailureBook){ "the marker replaced by another build's": func(t *testing.T, b *FailureBook) { mustMark(t, b, host, HoldRan{Account: "bab", App: "blog", Build: "v3", Exec: "a2", At: t0}) }, "the record cleared by a pointer": func(t *testing.T, b *FailureBook) { if ok, err := b.ClearOnPointer("bab", "blog", "v2", promotedAt.Add(time.Minute)); !ok || err != nil { t.Fatalf("ClearOnPointer = %v, %v", ok, err) } }, "the record pruned": func(t *testing.T, b *FailureBook) { if err := b.Prune(func(string, string, string) bool { return false }); err != nil { t.Fatal(err) } if _, ok := b.Lookup("bab", "blog", "v2"); ok { t.Fatal("the record was not pruned") } }, "the marker cleared by a good write": func(t *testing.T, b *FailureBook) { if ok, err := b.ClearHoldRanBefore(host, b.Snapshot()); !ok || err != nil { t.Fatalf("ClearHoldRanBefore = %v, %v", ok, err) } }, "the marker cleared by a rollback": func(t *testing.T, b *FailureBook) { if ok, err := b.ClearHoldRanOnPointer(host, "bab", "blog", t0.Add(time.Minute)); !ok || err != nil { t.Fatalf("ClearHoldRanOnPointer = %v, %v", ok, err) } }, "the marker cleared": func(t *testing.T, b *FailureBook) { if ok, err := b.ClearHoldRan(host); !ok || err != nil { t.Fatalf("ClearHoldRan = %v, %v", ok, err) } }, "the record merged with a newer pointer": func(t *testing.T, b *FailureBook) { if res := mustRecord(t, b, FailureRecord{Account: "bab", App: "blog", Build: "v2", Host: "sib2", PromotedAt: promotedAt.Add(time.Minute)}); res != Merged { t.Fatalf("Record = %v, want merged", res) } }, } { t.Run(name, func(t *testing.T) { b := setup(t) movesAcross(t, b, "blog", "v2", host, func() { change(t, b) }) }) } // Fix round 1, finding 1a: a grant that clears only a marker (no record // of its build holds the host) moves the app's generation, so a grant at // the value published before it for another build of the app clears // nothing. t.Run("a marker-only grant for another build", func(t *testing.T) { b, _ := diskBook(t) mustRecord(t, b, FailureRecord{Account: "bab", App: "blog", Build: "v2", Host: "sib", PromotedAt: promotedAt, At: t0}) mustMark(t, b, host, HoldRan{Account: "bab", App: "blog", Build: "v3", Exec: "a1", At: t0}) movesAcross(t, b, "blog", "v2", "h2", func() { if rec, mark := grantAt(b, "v3", host, published(b, "v3", host)); rec || !mark { t.Fatalf("the grant for v3 on h1 cleared record %v, marker %v; want the marker only", rec, mark) } }) }) // Fix round 1, finding 1b: a host's marker of one app replaced by another // app's (the host now runs another app) moves the first app's generation. t.Run("the marker replaced by another app's", func(t *testing.T) { b := setup(t) movesAcross(t, b, "blog", "v2", host, func() { mustMark(t, b, host, HoldRan{Account: "bab", App: "shop", Build: "w1", Exec: "a2", At: t0}) }) }) } // movesAcross checks that change moves bab/app's generation as build's // restriction of host publishes it, that build still restricts host after // it, and that a grant at the value published before it clears nothing. func movesAcross(t *testing.T, b *FailureBook, app, build, host string, change func()) { t.Helper() e := b.PublishedEpoch("bab", app, build, host) if e == 0 { t.Fatal("no generation published before the change") } change() now := b.PublishedEpoch("bab", app, build, host) if now == e { t.Fatalf("the generation stayed at %d across the change", e) } if now == 0 { t.Fatalf("%s still restricts %s through its record or marker, but no generation is published", build, host) } if rec, mark, _, _ := b.RetryHostAt("bab", app, build, host, b.ID(), e); rec || mark { t.Fatalf("a grant from before the change cleared record %v, marker %v", rec, mark) } } // A call that changes nothing (a stale failure, an unchanged one, a repeated // marker of the same execution) moves nothing: a grant at the generation // published before it still clears. func TestANoOpOutcomeDoesNotMoveTheGeneration(t *testing.T) { b, _ := diskBook(t) mustRecord(t, b, FailureRecord{Account: "bab", App: "blog", Build: "v2", Host: "sib", Mode: string(activator.ModeHold), HeldDown: []string{"sib"}, MarkHoldRan: true, HoldRanExec: "a1", HoldRanAt: t0}) snap := b.Snapshot() // an execution of v2 on h1 starts if rec, _ := grantAt(b, "v2", "h1", published(b, "v2", "h1")); !rec { t.Fatal("the retry of h1 cleared nothing") } e := published(b, "v2", "h2") if e == 0 { t.Fatal("no generation published for h2") } steady := func(what string) { t.Helper() if now := published(b, "v2", "h2"); now != e { t.Fatalf("%s moved the generation from %d to %d", what, e, now) } } if res := mustRecord(t, b, FailureRecord{Account: "bab", App: "blog", Build: "v2", Host: "h1", Seq: snap}); res != PreRetry { t.Fatalf("the failure that began before the retry = %v, want pre-retry", res) } steady("a pre-retry failure") if res := mustRecord(t, b, FailureRecord{Account: "bab", App: "blog", Build: "v2", Host: "sib"}); res != Unchanged { t.Fatalf("a repeated failure = %v, want unchanged", res) } steady("an unchanged failure") mustMark(t, b, "sib", HoldRan{Account: "bab", App: "blog", Build: "v2", Exec: "a1", At: t0}) steady("a repeated marker of the same execution") if res := mustRecord(t, b, FailureRecord{Account: "bab", App: "blog", Build: "v2", Host: "sib", Mode: string(activator.ModeHold), HeldDown: []string{"sib"}, MarkHoldRan: true, HoldRanExec: "a1", HoldRanAt: t0}); res != Unchanged { t.Fatalf("a repeated hold failure = %v, want unchanged", res) } steady("a repeated hold failure of the same execution") if rec, _ := grantAt(b, "v2", "h2", e); !rec { t.Fatal("a grant at the unchanged generation cleared nothing") } } // Plan 2: a file an earlier binary wrote (no durable_epochs) may hold // epochs that binary published and never saved. Opening it under the rule // rotates the book's identity, so no grant issued before clears anything, // and nothing is published until the rotation is on disk. func TestABookFromBeforeTheRuleGetsANewIdentity(t *testing.T) { dd := t.TempDir() raw, err := json.Marshal(map[string]any{ "schema": 2, "book": "old", "seq": 10, "records": []map[string]any{{"account": "bab", "app": "blog", "build": "v2", "host": "h1", "held_down": []string{"h1"}, "epochs": map[string]uint64{"": 11, "h1": 11}}}, }) if err != nil { t.Fatal(err) } if err := os.WriteFile(filepath.Join(dd, FailureFile), raw, 0o644); err != nil { t.Fatal(err) } b, err := OpenFailureBook(dd) if err != nil { t.Fatal(err) } id := b.ID() if id == "old" || id == "" { t.Fatalf("identity %q after adopting the rule, want a new one", id) } if e := published(b, "v2", "h1"); e != 0 { t.Fatalf("published %d before the rotation was saved, want 0", e) } for _, epoch := range []uint64{11, 12} { if rec, mark, _, _ := b.RetryHostAt("bab", "blog", "v2", "h1", "old", epoch); rec || mark { t.Fatalf("a grant naming the old identity at %d cleared record %v, marker %v", epoch, rec, mark) } } if err := b.Flush(); err != nil { t.Fatal(err) } e := published(b, "v2", "h1") if e <= 11 { t.Fatalf("published %d after the save, want a generation past the file's 11", e) } again, err := OpenFailureBook(dd) if err != nil { t.Fatal(err) } if again.ID() != id || published(again, "v2", "h1") != e { t.Fatalf("reopened as %q at %d, want %q at %d", again.ID(), published(again, "v2", "h1"), id, e) } if f := fileOf(t, dd); !f.Durable || f.Book != id { t.Fatalf("file durable_epochs %v, book %q: want true and %q", f.Durable, f.Book, id) } if rec, _ := grantAt(again, "v2", "h1", e); !rec { t.Fatal("a grant naming the new identity at the published generation cleared nothing") } } // Fix round 1, finding 2: the book is written as schema 3, which an agent // before the rule (it accepts schema 1 and 2) refuses, so it can never keep // the identity while publishing unsaved epochs. A schema-3 file round-trips // with its identity and owes no save. A file of schema 1 or 2 still opens, // with a new identity, even one that claims durable_epochs: only an agent of // this rule writes schema 3. func TestTheBookIsSchema3AndOlderSchemasOpenRotated(t *testing.T) { t.Run("schema 3 round-trips", func(t *testing.T) { b, dd := diskBook(t) mustRecord(t, b, FailureRecord{Account: "bab", App: "blog", Build: "v2", Host: "h1", HeldDown: []string{"h1"}}) e := published(b, "v2", "h1") if e == 0 { t.Fatal("a saved restriction published no generation") } if f := fileOf(t, dd); f.Schema != 3 || !f.Durable || f.Book != b.ID() { t.Fatalf("file schema %d, durable_epochs %v, book %q: want 3, true, %q", f.Schema, f.Durable, f.Book, b.ID()) } again, err := OpenFailureBook(dd) if err != nil { t.Fatalf("a schema-3 file did not open: %v", err) } if again.ID() != b.ID() || again.SaveOwed() || published(again, "v2", "h1") != e { t.Fatalf("reopened as %q, save owed %v, at %d: want %q, false, %d", again.ID(), again.SaveOwed(), published(again, "v2", "h1"), b.ID(), e) } if rec, _ := grantAt(again, "v2", "h1", e); !rec { t.Fatal("a grant at the generation published before the reopen cleared nothing") } }) for name, file := range map[string]map[string]any{ "schema 1": {"schema": 1}, "schema 2": {"schema": 2}, "schema 2 with durable_epochs": {"schema": 2, "durable_epochs": true, "gens": map[string]uint64{"bab/blog": 10}}, } { t.Run(name, func(t *testing.T) { dd := t.TempDir() file["book"], file["seq"] = "old", 10 file["records"] = []map[string]any{{"account": "bab", "app": "blog", "build": "v2", "host": "h1", "held_down": []string{"h1"}}} raw, err := json.Marshal(file) if err != nil { t.Fatal(err) } if err := os.WriteFile(filepath.Join(dd, FailureFile), raw, 0o644); err != nil { t.Fatal(err) } b, err := OpenFailureBook(dd) if err != nil { t.Fatalf("the file did not open: %v", err) } if !b.HeldDown("bab", "blog", "v2", "h1") { t.Fatal("the file's record was lost") } if id := b.ID(); id == "old" || id == "" { t.Fatalf("identity %q, want a new one", id) } if !b.SaveOwed() || published(b, "v2", "h1") != 0 { t.Fatalf("save owed %v, published %d: want the rotation owed and nothing published", b.SaveOwed(), published(b, "v2", "h1")) } if rec, mark, _, _ := b.RetryHostAt("bab", "blog", "v2", "h1", "old", 10); rec || mark { t.Fatalf("a grant naming the old identity cleared record %v, marker %v", rec, mark) } if err := b.Flush(); err != nil { t.Fatal(err) } if f := fileOf(t, dd); f.Schema != 3 || !f.Durable || f.Book != b.ID() { t.Fatalf("written as schema %d, durable_epochs %v, book %q: want 3, true, %q", f.Schema, f.Durable, f.Book, b.ID()) } e := published(b, "v2", "h1") if e == 0 { t.Fatal("published nothing after the rotation was saved") } if rec, _ := grantAt(b, "v2", "h1", e); !rec { t.Fatal("a grant naming the new identity at the published generation cleared nothing") } }) } } // Rule 3: a reopen starts from the disk's sequence, so every generation // handed out after it is larger than every one published before it. func TestAReopenNeverReissuesAPublishedGeneration(t *testing.T) { b, dd := diskBook(t) var before uint64 read := func(b *FailureBook) uint64 { t.Helper() e := max(published(b, "v2", "h1"), published(b, "v3", "h1"), published(b, "v4", "h1"), published(b, "v5", "h2")) if e == 0 { t.Fatal("nothing published") } return e } mustRecord(t, b, FailureRecord{Account: "bab", App: "blog", Build: "v2", Host: "sib", PromotedAt: promotedAt}) before = max(before, read(b)) mustRecord(t, b, FailureRecord{Account: "bab", App: "blog", Build: "v2", Host: "sib", HeldDown: []string{"h1"}}) before = max(before, read(b)) mustMark(t, b, "h1", HoldRan{Account: "bab", App: "blog", Build: "v3", Exec: "a1", At: t0}) before = max(before, read(b)) mustRecord(t, b, FailureRecord{Account: "bab", App: "blog", Build: "v4", Host: "sib"}) before = max(before, read(b)) again, err := OpenFailureBook(dd) if err != nil { t.Fatal(err) } if err := again.Flush(); err != nil { t.Fatal(err) } for i, change := range []func(){ func() { mustRecord(t, again, FailureRecord{Account: "bab", App: "blog", Build: "v5", Host: "sib"}) }, func() { mustMark(t, again, "h2", HoldRan{Account: "bab", App: "blog", Build: "v5", Exec: "a2", At: t0}) }, func() { if ok, err := again.ClearOnPointer("bab", "blog", "v4", t0); !ok || err != nil { t.Fatalf("ClearOnPointer = %v, %v", ok, err) } }, } { change() for _, c := range [][2]string{{"v2", "h1"}, {"v3", "h1"}, {"v5", "h2"}} { if e := published(again, c[0], c[1]); e <= before { t.Fatalf("after change %d of the reopened book, %s on %s published %d, not past %d", i+1, c[0], c[1], e, before) } } } } // Rule 5 and finding 3: two books opened with equal numeric state publish // the same generation; a grant for one clears nothing in the other. func TestAGrantForAnotherBookClearsNothing(t *testing.T) { one, _ := diskBook(t) other, _ := diskBook(t) for _, b := range []*FailureBook{one, other} { mustRecord(t, b, FailureRecord{Account: "bab", App: "blog", Build: "v2", Host: "h", Mode: string(activator.ModeHold), HeldDown: []string{"h"}, MarkHoldRan: true, HoldRanExec: "a1", HoldRanAt: t0}) } e := published(one, "v2", "h") if e == 0 || e != published(other, "v2", "h") { t.Fatalf("published %d and %d: the case needs equal generations", e, published(other, "v2", "h")) } if one.ID() == other.ID() { t.Fatalf("two books share the identity %q", one.ID()) } if rec, mark, _, _ := other.RetryHostAt("bab", "blog", "v2", "h", one.ID(), e); rec || mark { t.Fatalf("a grant for one book cleared record %v, marker %v in another", rec, mark) } if !other.HeldDown("bab", "blog", "v2", "h") { t.Fatal("the other book's hold-down was lifted") } if rec, mark := grantAt(one, "v2", "h", e); !rec || !mark { t.Fatalf("the grant cleared record %v, marker %v in its own book; want both", rec, mark) } } // Rule 5: a book's identity comes from crypto/rand only. A book that needs // one and cannot get it does not open; one already on disk under the rule // needs none. func TestABookThatCannotGetAnIdentityDoesNotOpen(t *testing.T) { durable, ddDurable := diskBook(t) mustRecord(t, durable, FailureRecord{Account: "bab", App: "blog", Build: "v2", Host: "h"}) ddOld := t.TempDir() if err := os.WriteFile(filepath.Join(ddOld, FailureFile), []byte(`{"schema": 2, "book": "old", "records": []}`), 0o644); err != nil { t.Fatal(err) } broken := errors.New("no entropy") old := randRead randRead = func([]byte) (int, error) { return 0, broken } t.Cleanup(func() { randRead = old }) for name, dd := range map[string]string{"a new book": t.TempDir(), "a book from before the rule": ddOld} { if b, err := OpenFailureBook(dd); !errors.Is(err, broken) || b != nil { t.Fatalf("%s opened (%v, %v) without an identity", name, b, err) } } b, err := OpenFailureBook(ddDurable) if err != nil { t.Fatalf("a book with its identity on disk did not open: %v", err) } if b.ID() != durable.ID() { t.Fatalf("identity %q, want %q", b.ID(), durable.ID()) } } // Fix wave 1, item 5 (Astra final review 1, finding 5): the sequence never // wraps under one identity. The largest value it hands out is // MaxUint64-2. A change that needs another refuses to save, so nothing is // published and no grant clears; a snapshot at the boundary reads "not // known" (0); and the file keeps the last value that landed, so a reopen // refuses the same way. func TestAnExhaustedSequenceFailsClosed(t *testing.T) { b, dd := diskBook(t) b.mu.Lock() b.seq = math.MaxUint64 - 3 b.mu.Unlock() mustRecord(t, b, FailureRecord{Account: "bab", App: "blog", Build: "v2", Host: "h1"}) e := published(b, "v2", "h1") if e != math.MaxUint64-2 { t.Fatalf("published %d, want the last usable value %d", e, uint64(math.MaxUint64-2)) } if s := b.Snapshot(); s != 0 { t.Fatalf("Snapshot = %d at the boundary, want 0 (not known)", s) } if err := b.MarkHoldRan("h1", HoldRan{Account: "bab", App: "blog", Build: "v2", Exec: "e1"}); err == nil { t.Fatal("a change that needs a value past the boundary was saved") } if p := published(b, "v2", "h1"); p != 0 { t.Fatalf("published %d after the sequence ran out, want 0", p) } if rec, mark := grantAt(b, "v2", "h1", e); rec || mark { t.Fatalf("a grant cleared record %v, marker %v on an exhausted book", rec, mark) } again, err := OpenFailureBook(dd) if err != nil { t.Fatal(err) } again.mu.Lock() seq, gen := again.seq, again.gens[genKey("bab", "blog")] again.mu.Unlock() if seq != math.MaxUint64-2 || gen != e { t.Fatalf("the file holds seq %d and generation %d, want the last landed %d", seq, gen, e) } if _, err := again.Record(FailureRecord{Account: "bab", App: "blog", Build: "v3", Host: "h2"}); err == nil { t.Fatal("the reopened book saved a change past the boundary") } if p := published(again, "v2", "h1"); p != 0 { t.Fatalf("the reopened book published %d after it ran out, want 0", p) } }