package rollout import ( "context" "encoding/json" "errors" "os" "path/filepath" "strings" "testing" "time" "github.com/carlosframework/platform/internal/activator" "github.com/carlosframework/platform/internal/status" ) // Task 5 (Astra F5; R1 #5, R3 ยง5 item 2): failure state survives a failed // book write followed by an agent restart. Every execution that may leave a // failure writes an intent first; a restart turns an intent left on disk // into the failure it may have been. // // A save fails by making the data dir read-only (WriteFileAtomic writes its // temp file beside the target). Running as root defeats it: the tests skip. func readOnly(t *testing.T, dir string) { t.Helper() if os.Geteuid() == 0 { t.Skip("running as root: a read-only directory does not fail a save") } if err := os.Chmod(dir, 0o500); err != nil { t.Fatal(err) } t.Cleanup(func() { os.Chmod(dir, 0o700) }) } func writable(t *testing.T, dir string) { t.Helper() if err := os.Chmod(dir, 0o700); err != nil { t.Fatal(err) } } // onDisk gives the harness a book saved under a data dir of its own, before // anything has run, and returns the dir. func (h *harness) onDisk() string { h.t.Helper() dir := h.t.TempDir() b, err := OpenFailureBook(dir) if err != nil { h.t.Fatal(err) } b.now = h.clock h.book, h.m.d.Book = b, b return dir } // fileOf reads the book file in dir as it stands on disk. func fileOf(t *testing.T, dir string) failureFile { t.Helper() raw, err := os.ReadFile(filepath.Join(dir, FailureFile)) if errors.Is(err, os.ErrNotExist) { return failureFile{} } if err != nil { t.Fatal(err) } var f failureFile if err := json.Unmarshal(raw, &f); err != nil { t.Fatal(err) } return f } // Review Focus 1: B's failure is held only in memory because the save // failed; the agent restarts. The intent written before B ran is still on // disk and comes back as B's failure. func TestAFailedSaveThenRestartKeepsTheFailure(t *testing.T) { dd := t.TempDir() b, _ := OpenFailureBook(dd) start := t0.Add(-time.Minute) if err := b.Begin("h", Intent{Account: "bab", App: "blog", Build: "v2", Mode: "hold", ID: "a7", Start: start}); err != nil { t.Fatal(err) } readOnly(t, dd) if _, err := b.Record(FailureRecord{Account: "bab", App: "blog", Build: "v2", Host: "h", Mode: "hold", HeldDown: []string{"h"}}); err == nil { t.Skip("the save did not fail") } _ = b.End("h", "a7") // fails too: the intent stays on disk writable(t, dd) b2, err := OpenFailureBook(dd) if err != nil { t.Fatal(err) } if b2.Reconciled() != 1 || !b2.HeldDown("bab", "blog", "v2", "h") { t.Fatalf("after restart: reconciled %d, held down %v", b2.Reconciled(), b2.HeldDown("bab", "blog", "v2", "h")) } m, ok := b2.HoldRanOn("h") if !ok || m.Build != "v2" { t.Fatal("a hold intent must leave its marker") } if m.Exec != "a7" || !m.At.Equal(start) { t.Fatalf("marker %+v: want the intent's execution (a7) and its start %v", m, start) } } func TestABeginThatCannotSaveRefusesAndLeavesNothing(t *testing.T) { dd := t.TempDir() b, _ := OpenFailureBook(dd) readOnly(t, dd) if err := b.Begin("h", Intent{Account: "bab", App: "blog", Build: "v2", Mode: "overlap", ID: "a1"}); err == nil { t.Skip("the save did not fail") } if _, ok := b.intents["h"]; ok { t.Fatal("a refused begin kept its intent in memory") } writable(t, dd) b2, _ := OpenFailureBook(dd) if b2.Reconciled() != 0 { t.Fatal("a refused begin left an intent") } // The memory a later save writes has no intent either. if err := b.Flush(); err != nil { t.Fatal(err) } if b3, _ := OpenFailureBook(dd); b3.Reconciled() != 0 { t.Fatal("a refused begin's intent reached the file at the next save") } } func TestASchema1BookReadsAndIsWrittenAsSchema3(t *testing.T) { dd := t.TempDir() raw := `{"schema":1,"records":[{"account":"bab","app":"blog","build":"v2","host":"h","cause":"release","mode":"overlap","reason":"x","promoted_at":"2026-10-01T00:00:00Z","at":"2026-10-01T00:00:00Z"}]}` if err := os.WriteFile(filepath.Join(dd, FailureFile), []byte(raw), 0o644); err != nil { t.Fatal(err) } b, err := OpenFailureBook(dd) if err != nil || !b.Blocks("bab", "blog", "v2", "x", false) { t.Fatalf("schema 1: %v", err) } if err := b.Flush(); err != nil { t.Fatal(err) } out, _ := os.ReadFile(filepath.Join(dd, FailureFile)) if !strings.Contains(string(out), `"schema": 3`) { t.Fatalf("written as %s", out) } if _, err := OpenFailureBook(dd); err != nil { t.Fatalf("schema 3 written, then refused: %v", err) } } // Review Focus 2: what an older agent sees. Its reader refused anything // but schema 1 (OpenFailureBook before Task 5), or but schema 1 and 2 // (before the crash-epoch hard limit, readiness round 2 Task 1, fix round 1). func TestTheBookIsUnreadableToAnOlderReader(t *testing.T) { var f struct { Schema int `json:"schema"` } dd := t.TempDir() b, _ := OpenFailureBook(dd) if err := b.Flush(); err != nil { t.Fatal(err) } raw, _ := os.ReadFile(filepath.Join(dd, FailureFile)) if err := json.Unmarshal(raw, &f); err != nil { t.Fatal(err) } if f.Schema == 1 { t.Fatal("an old reader would accept the file and drop markers and intents") } if f.Schema == 2 { t.Fatal("a reader before the crash-epoch rule would accept the file, keep its identity and publish unsaved epochs") } } func TestABookOfAnUnknownSchemaIsRefused(t *testing.T) { for _, schema := range []string{"0", "4"} { dd := t.TempDir() if err := os.WriteFile(filepath.Join(dd, FailureFile), []byte(`{"schema":`+schema+`,"records":[]}`), 0o644); err != nil { t.Fatal(err) } if _, err := OpenFailureBook(dd); err == nil { t.Fatalf("schema %s was read", schema) } } } // Rule 3: each intent left in the file becomes a failure record for its // build on that host: cause release, the intent's mode, held down with a // marker in hold mode, stamped with the open time, and a reason the member // surfaces word as a failed health check. The intents go, and the book is // owed a save (Flush), which leaves none on disk. func TestAnIntentLeftOnDiskBecomesItsFailure(t *testing.T) { dd := t.TempDir() b, _ := OpenFailureBook(dd) start := t0.Add(-time.Hour) for host, in := range map[string]Intent{ "a": {Account: "bab", App: "blog", Build: "v2", Mode: "overlap", ID: "a3", Start: start}, "b": {Account: "bab", App: "blog", Build: "v2", Mode: "hold", ID: "w9", Start: start}, "c": {Account: "bab", App: "blog", Build: "v3", Mode: "hold", ID: "a4"}, } { if err := b.Begin(host, in); err != nil { t.Fatal(err) } } before := time.Now() b2, err := OpenFailureBook(dd) if err != nil { t.Fatal(err) } after := time.Now() if b2.Reconciled() != 3 { t.Fatalf("reconciled %d, want 3", b2.Reconciled()) } r, ok := b2.Lookup("bab", "blog", "v2") if !ok || r.Cause != "release" || len(r.HeldDown) != 1 || r.HeldDown[0] != "b" { t.Fatalf("v2's record %+v: want a release failure holding down b (the hold intent) alone", r) } if r.At.Before(before) || r.At.After(after) || !r.PromotedAt.Equal(r.At) { t.Fatalf("v2's record stamped at %v, promoted at %v: want the open time for both", r.At, r.PromotedAt) } if reasonWord(r.Reason) != status.ReasonHealth { t.Fatalf("reason %q words as %q", r.Reason, reasonWord(r.Reason)) } if !b2.Blocks("bab", "blog", "v2", "a", false) || b2.HeldDown("bab", "blog", "v2", "a") { t.Fatal("the overlap intent's host must be blocked, not held down") } if _, ok := b2.HoldRanOn("a"); ok { t.Fatal("an overlap intent left a hold-ran marker") } if m, ok := b2.HoldRanOn("b"); !ok || m.Build != "v2" || m.Exec != "w9" || !m.At.Equal(start) { t.Fatalf("b's marker %+v, %v: want v2's, from w9, at its start", m, ok) } rc, ok := b2.Lookup("bab", "blog", "v3") if !ok || rc.Mode != "hold" || !b2.HeldDown("bab", "blog", "v3", "c") { t.Fatalf("v3's record %+v: want c held down", rc) } if m, ok := b2.HoldRanOn("c"); !ok || m.Exec != "a4" || m.At.Before(before) || m.At.After(after) { t.Fatalf("c's marker %+v: an intent with no start is stamped with the open time", m) } if len(b2.intents) != 0 { t.Fatalf("intents %v stayed in memory", b2.intents) } // Nothing is written at open: the caller flushes. if f := fileOf(t, dd); len(f.Intents) != 3 { t.Fatalf("the open rewrote the file: %d intents on disk", len(f.Intents)) } if err := b2.Flush(); err != nil { t.Fatal(err) } if f := fileOf(t, dd); len(f.Intents) != 0 || len(f.Records) != 2 || f.Schema != failureSchema { t.Fatalf("after the flush: %+v", f) } if b3, _ := OpenFailureBook(dd); b3.Reconciled() != 0 || !b3.HeldDown("bab", "blog", "v2", "b") { t.Fatal("the flushed book did not keep the reconciled records") } } // Rule 1 and 2: a newer wake's intent replaces an older wake's for the host; // an attempt's intent is replaced by nothing, and an attempt never replaces // a wake's; End removes only its own intent. func TestBeginAndEndKeepOneIntentPerHost(t *testing.T) { dd := t.TempDir() b, _ := OpenFailureBook(dd) in := func(id string) Intent { return Intent{Account: "bab", App: "blog", Build: "v2", Mode: "overlap", ID: id} } if err := b.Begin("h", in("w1")); err != nil { t.Fatal(err) } if err := b.Begin("h", in("w2")); err != nil { t.Fatalf("a newer wake's intent was refused: %v", err) } if err := b.Begin("h", in("a5")); !errors.Is(err, ErrIntentStands) { t.Fatalf("an attempt replaced a wake's intent: %v", err) } if f := fileOf(t, dd); f.Intents["h"].ID != "w2" { t.Fatalf("on disk %+v, want w2's intent", f.Intents) } if err := b.End("h", "w1"); err != nil { t.Fatal(err) } if f := fileOf(t, dd); f.Intents["h"].ID != "w2" { t.Fatal("an older wake's End removed the newer wake's intent") } if err := b.End("h", "w2"); err != nil { t.Fatal(err) } if f := fileOf(t, dd); len(f.Intents) != 0 { t.Fatalf("End left %+v on disk", f.Intents) } if err := b.Begin("h", in("a6")); err != nil { t.Fatal(err) } for _, id := range []string{"w3", "a7"} { if err := b.Begin("h", in(id)); !errors.Is(err, ErrIntentStands) { t.Fatalf("%s replaced an attempt's intent: %v", id, err) } } if err := b.Begin("other", in("w4")); err != nil { t.Fatalf("another host's intent was refused: %v", err) } } // Manager (a): an adoption whose intent cannot be written never reaches // Swap. It is abandoned unexecuted, waits out the abandon deferral, and runs // once the book can be written. func TestAnAdoptionWhoseIntentCannotBeWrittenNeverRuns(t *testing.T) { h := newHarness(t, "a") dir := h.onDisk() h.stage("v2") readOnly(t, dir) h.install("v2", "a") h.pass() waitFor(t, "the attempt to be abandoned and deferred", func() bool { s := h.m.schedulerOf("a") if s == nil { return false } s.mu.Lock() defer s.mu.Unlock() hs := s.q.hosts["a"] return hs != nil && hs.running == nil && hs.notBefore.After(h.clock()) }) if calls, started, _ := h.act.snapshot(); len(calls) != 0 || len(started) != 0 { t.Fatalf("Swap ran (%+v, %v) with no intent written", calls, started) } if len(h.book.List()) != 0 { t.Fatal("a refused attempt wrote a failure") } writable(t, dir) h.advance(abandonDeferral + time.Second) h.pass() waitFor(t, "the attempt after the wait", func() bool { return h.servingAll("v2", "a") }) waitFor(t, "the intent to end", func() bool { return len(fileOf(t, dir).Intents) == 0 }) } // An adoption's intent is on disk while its Swap runs, with its execution's // identity, mode and start, and is gone once its outcome is written. func TestAnAdoptionsIntentStandsWhileItRuns(t *testing.T) { for _, fail := range []bool{false, true} { t.Run(map[bool]string{false: "done", true: "failed"}[fail], func(t *testing.T) { h := newHarness(t, "a") dir := h.onDisk() h.stage("v2") if fail { h.act.outcome = func(activator.Attempt) activator.SwapOutcome { return activator.SwapFailed } } gate := h.act.hold("a") h.install("v2", "a") h.pass() waitFor(t, "the swap to start", func() bool { _, started, _ := h.act.snapshot(); return len(started) == 1 }) in, ok := fileOf(t, dir).Intents["a"] if !ok || !strings.HasPrefix(in.ID, "a") || in.Build != "v2" || in.Mode != "overlap" || !in.Start.Equal(t0) { t.Fatalf("intent on disk during the swap: %+v, %v", in, ok) } close(gate) waitFor(t, "the intent to end", func() bool { return len(fileOf(t, dir).Intents) == 0 }) recorded := false for _, r := range fileOf(t, dir).Records { recorded = recorded || r.Build == "v2" } if recorded != fail { t.Fatalf("failed %v, but v2 recorded %v", fail, recorded) } }) } } // Plan review #2: Swap runs an overlap attempt on a foreign-layout row as a // hold. The intent carries the mode the activator will run (EffectiveMode), // so a crash mid-swap reopens the book with the instance held down and // marked, as a hold that went down would have left it. func TestAForeignLayoutHoldCrashReopensHeldDown(t *testing.T) { h := newHarness(t, "a") dir := h.onDisk() h.stage("v2") h.act.mode = activator.ModeHold // the foreign row's override gate := h.act.hold("a") // The parked attempt's done() writes the failure book and ends its intent // in dir once the gate opens, on a goroutine the test does not own. Cleanups // run last in, first out and onDisk's TempDir is already registered, so // this one opens the gate and joins the manager before dir is removed. t.Cleanup(func() { close(gate) ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() h.m.Close(ctx) }) h.install("v2", "a") h.pass() waitFor(t, "the swap to start", func() bool { _, started, _ := h.act.snapshot(); return len(started) == 1 }) if in := fileOf(t, dir).Intents["a"]; in.Mode != "hold" { t.Fatalf("intent %+v: want the effective mode, hold", in) } // The agent dies here; the next one opens the book. b, err := OpenFailureBook(dir) if err != nil { t.Fatal(err) } if b.Reconciled() != 1 || !b.HeldDown("bab", "blog", "v2", "a") { t.Fatalf("reconciled %d: want a held down on v2", b.Reconciled()) } m, ok := b.HoldRanOn("a") if !ok || m.Build != "v2" || !strings.HasPrefix(m.Exec, "a") || !m.At.Equal(t0) { t.Fatalf("marker %+v, %v: want v2's, from the attempt, at its dispatch", m, ok) } } // Manager (b): a first wake whose intent cannot be written is refused with // the approved text, and nothing of it is registered. func TestAFirstWakeWhoseIntentCannotBeWrittenIsRefused(t *testing.T) { h := newHarness(t, "a") dir := h.onDisk() h.stage("v2") h.registerThenInstall("v2", "a") h.wakeHost("a") readOnly(t, dir) p := h.m.Plan(context.Background(), h.rows.get("a")) if p.Refuse != BookUnreadableRefusal || p.Boot.StagedPath != "" { t.Fatalf("plan %+v: want the wake refused with %q", p, BookUnreadableRefusal) } h.m.mu.Lock() _, planned := h.m.planned["a"] h.m.mu.Unlock() if planned { t.Fatal("a refused wake stayed registered as planned") } } // A planned wake does not take over an attempt's intent: its plan is // refused as the attempt's own planning refusal is, and nothing is lost. func TestAFirstWakeNeverReplacesAnAttemptsIntent(t *testing.T) { h := newHarness(t, "a") h.onDisk() h.stage("v2") h.registerThenInstall("v2", "a") h.wakeHost("a") if err := h.book.Begin("a", Intent{Account: "bab", App: "blog", Build: "v3", Mode: "hold", ID: "a1"}); err != nil { t.Fatal(err) } p := h.m.Plan(context.Background(), h.rows.get("a")) if p.Refuse != "this instance is still moving to another build" { t.Fatalf("plan %+v: want the moving refusal", p) } if in := h.book.intents["a"]; in.ID != "a1" { t.Fatalf("intent %+v: the attempt's was replaced", in) } } // Manager (c), with plan review #1: a planned first wake's intent is written // at its plan, with the wake's identity and start, and ends with the first // event that settles its build's fate, never with the activator letting the // host go (an entry deleted before its failure is delivered). func TestAPlannedWakesIntentEndsWithItsOutcome(t *testing.T) { ctx := context.Background() setup := func(t *testing.T) (*harness, string, activator.WakePlan) { h := newHarness(t, "a") dir := h.onDisk() h.stage("v2") h.registerThenInstall("v2", "a") h.wakeHost("a") p := h.m.Plan(ctx, h.rows.get("a")) if !p.FirstWake || p.Boot.Version != "v2" { t.Fatalf("plan %+v: want a first wake of v2", p) } in, ok := fileOf(t, dir).Intents["a"] if !ok || in.ID != h.m.wakeExecID(p.Token) || in.Build != "v2" || in.Mode != string(p.Mode) || !in.Start.Equal(t0) { t.Fatalf("intent on disk %+v, %v: want the plan's", in, ok) } return h, dir, p } ev := func(p activator.WakePlan, h *harness, kind activator.GenEventKind, first bool) activator.GenEvent { return activator.GenEvent{Kind: kind, Route: h.rows.get("a"), Gen: activator.Gen{N: 3, Build: "v2", Digest: "sha-v2"}, Mode: p.Mode, FirstWake: first, Token: p.Token, ConfigRev: p.ConfigRev} } for _, tc := range []struct { name string kinds []activator.GenEventKind first bool ends bool }{ {"healthy first wake, under observation", []activator.GenEventKind{activator.GenHealthy}, true, false}, {"settled", []activator.GenEventKind{activator.GenHealthy, activator.GenSettled}, true, true}, {"failed", []activator.GenEventKind{activator.GenFailed}, true, true}, {"cancelled with nothing run", []activator.GenEventKind{activator.GenCancelled}, true, true}, {"the good build booted untried", []activator.GenEventKind{activator.GenHealthy}, false, true}, } { t.Run(tc.name, func(t *testing.T) { h, dir, p := setup(t) for _, k := range tc.kinds { e := ev(p, h, k, tc.first) if k == activator.GenFailed { e.Failed = p.Boot } h.m.OnGen(e) } if _, stands := fileOf(t, dir).Intents["a"]; stands == tc.ends { t.Fatalf("after %v: intent stands %v", tc.kinds, stands) } }) } t.Run("an older wake's event leaves a newer wake's intent", func(t *testing.T) { h, dir, old := setup(t) cur := h.m.Plan(ctx, h.rows.get("a")) h.m.OnGen(ev(old, h, activator.GenCancelled, true)) if in := fileOf(t, dir).Intents["a"]; in.ID != h.m.wakeExecID(cur.Token) { t.Fatalf("intent %+v, want the newer wake's", in) } }) t.Run("not by the activator letting the host go", func(t *testing.T) { h, dir, _ := setup(t) h.sleepHost("a") // Owns false: the failed wake's entry is gone h.pass() if _, stands := fileOf(t, dir).Intents["a"]; !stands { t.Fatal("dropUnownedWakes ended the intent of a wake whose outcome is not delivered") } }) }