package rollout import ( "encoding/json" "os" "path/filepath" "testing" "time" "github.com/carlosframework/platform/internal/activator" "github.com/carlosframework/platform/internal/status" ) // retry is the console's signed host retry as the box applies it: a grant // naming build, carrying the generation the box's status publishes for it. func (h *harness) retry(host, build string) bool { return h.m.RetryHost(host, build, h.book.ID(), h.book.PublishedEpoch("bab", "blog", build, host)) } // Plan review finding 13, under the crash-epoch hard limit (readiness round // 2, Task 1): a restriction's epoch is its app's generation, a persisted // book sequence that moves whenever any restriction of the app changes. func TestRestrictionEpochFollowsTheRecordAndTheMarker(t *testing.T) { dd := t.TempDir() b, err := OpenFailureBook(dd) if err != nil { t.Fatal(err) } if e := b.RestrictionEpoch("bab", "blog", "v2", "a"); e != 0 { t.Fatalf("epoch with nothing recorded = %d, want 0", e) } b.Record(FailureRecord{Account: "bab", App: "blog", Build: "v2", Host: "a"}) e1 := b.RestrictionEpoch("bab", "blog", "v2", "a") if e1 == 0 || b.RestrictionEpoch("bab", "blog", "v2", "b") != e1 { t.Fatalf("after the first failure: a %d, sibling b %d; want one non-zero epoch for both", e1, b.RestrictionEpoch("bab", "blog", "v2", "b")) } if res, _ := b.Record(FailureRecord{Account: "bab", App: "blog", Build: "v2", Host: "a"}); res != Unchanged || b.RestrictionEpoch("bab", "blog", "v2", "a") != e1 { t.Fatalf("an unchanged failure moved the epoch to %d", b.RestrictionEpoch("bab", "blog", "v2", "a")) } b.Record(FailureRecord{Account: "bab", App: "blog", Build: "v2", Host: "a", HeldDown: []string{"a"}}) e2 := b.RestrictionEpoch("bab", "blog", "v2", "a") if e2 <= e1 || b.RestrictionEpoch("bab", "blog", "v2", "b") <= e1 { t.Fatalf("a merged hold-down: a %d, b %d; both must move past %d", e2, b.RestrictionEpoch("bab", "blog", "v2", "b"), e1) } b.MarkHoldRan("c", HoldRan{Account: "bab", App: "blog", Build: "v3"}) e3 := b.RestrictionEpoch("bab", "blog", "v3", "c") if e3 <= e2 { t.Fatalf("a marker-only restriction's epoch = %d, want the app's generation moved past %d", e3, e2) } if e := b.RestrictionEpoch("bab", "blog", "v2", "b"); e != e3 { t.Fatalf("v2's epoch for b = %d after c's marker: one generation per app, want %d", e, e3) } if e := b.RestrictionEpoch("bab", "blog", "v3", "b"); e != 0 { t.Fatalf("b's epoch for v3 = %d: c's marker of v3 restricts only c, want 0", e) } if e := b.RestrictionEpoch("bab", "shop", "v3", "c"); e != 0 { t.Fatalf("another app's epoch = %d, want 0", e) } b2, err := OpenFailureBook(dd) if err != nil { t.Fatal(err) } for _, c := range [][2]string{{"v2", "a"}, {"v2", "b"}, {"v3", "c"}} { if got, want := b2.RestrictionEpoch("bab", "blog", c[0], c[1]), b.RestrictionEpoch("bab", "blog", c[0], c[1]); got != want { t.Fatalf("after a reopen %s on %s has epoch %d, want %d", c[0], c[1], got, want) } } } // A book an earlier agent wrote has no generations: the open gives its app // one, and its marker a sequence, newer than anything in the file, so its // holds can still be retried (a zero epoch is never accepted). One app has // one generation (the crash-epoch hard limit, rule 1), so the record's and // the marker's restrictions share it. func TestOpenGivesAnOlderFilesRestrictionsAnEpoch(t *testing.T) { dd := t.TempDir() raw, _ := json.Marshal(map[string]any{ "schema": 1, "records": []map[string]any{{"account": "bab", "app": "blog", "build": "v2", "host": "a"}}, "hold_ran": map[string]any{"c": map[string]any{"account": "bab", "app": "blog", "build": "v3"}}, "seq": 5, }) if err := os.WriteFile(filepath.Join(dd, FailureFile), raw, 0o644); err != nil { t.Fatal(err) } b, err := OpenFailureBook(dd) if err != nil { t.Fatal(err) } rec := b.RestrictionEpoch("bab", "blog", "v2", "a") mark := b.RestrictionEpoch("bab", "blog", "v3", "c") if rec <= 5 || rec != mark { t.Fatalf("epochs record %d, marker %d: want the app's one generation, newer than the file's 5", rec, mark) } if m, _ := b.HoldRanOn("c"); m.Seq <= 5 { t.Fatalf("the marker's sequence = %d, want one newer than the file's 5", m.Seq) } if b.Snapshot() <= max(rec, mark) { t.Fatal("a snapshot after the open must be newer than the epochs it handed out") } } // Plan review finding 5: a grant names the build whose restriction it lifts. // One naming another build than the record's clears nothing, the marker // included. func TestARetryGrantForAnotherBuildClearsNothing(t *testing.T) { h := newHarness(t, "a", "b") h.stage("v2") h.book.Record(FailureRecord{Account: "bab", App: "blog", Build: "v2", Host: "c", PromotedAt: promotedAt, At: t0}) h.book.MarkHoldRan("b", markV2) h.install("v2", "a", "b") h.pass() epoch := h.book.RestrictionEpoch("bab", "blog", "v2", "b") if h.m.RetryHost("b", "v9", h.book.ID(), epoch) { t.Fatal("a grant for v9 cleared something") } if !h.book.Blocks("bab", "blog", "v2", "b", false) { t.Fatal("a grant for v9 cleared v2's record for b") } if _, ok := h.book.HoldRanOn("b"); !ok { t.Fatal("a grant for v9 cleared b's v2 marker") } if h.m.RetryHost("b", "v2", h.book.ID(), 0) { t.Fatal("a zero epoch cleared something") } if !h.retry("b", "v2") { t.Fatal("the grant for v2 at its epoch cleared nothing") } if _, ok := h.book.HoldRanOn("b"); ok || h.book.Blocks("bab", "blog", "v2", "b", false) { t.Fatal("the grant for v2 left b restricted") } waitFor(t, "b on v2", func() bool { return h.servingAll("v2", "b") }) } // The marker names B while the slot wants A: a grant for B lifts the marker // and nothing of A's. func TestARetryGrantOfTheMarkersBuildClearsOnlyTheMarker(t *testing.T) { h := newHarness(t, "a") h.stage("v2") h.install("v2", "a") h.sleepHost("a") h.pass() h.book.Record(FailureRecord{Account: "bab", App: "blog", Build: "v2", Host: "sib", At: t0}) h.book.MarkHoldRan("a", HoldRan{Account: "bab", App: "blog", Build: "v3"}) if !h.retry("a", "v3") { t.Fatal("the grant for the marker's build cleared nothing") } if _, ok := h.book.HoldRanOn("a"); ok { t.Fatal("the marker of v3 survived its grant") } if !h.book.Blocks("bab", "blog", "v2", "a", false) { t.Fatal("a grant for v3 cleared v2's record for a") } } // Plan review finding 13: a grant is for the restriction the member saw. // One whose epoch predates a second failure of the same build on the same // host clears nothing, and a fresh grant does. func TestARetryGrantPredatingASecondFailureClearsNothing(t *testing.T) { h := newHarness(t, "a", "b") h.stage("v2") h.failing.Store(true) h.act.outcome = func(a activator.Attempt) activator.SwapOutcome { if a.Build == "v2" && h.failing.Load() { return activator.SwapFailed } return activator.SwapDone } h.install("v2", "a", "b") h.pass() waitFor(t, "a failure record", func() bool { _, ok := h.book.Lookup("bab", "blog", "v2"); return ok }) first := h.book.RestrictionEpoch("bab", "blog", "v2", "b") if !h.m.RetryHost("b", "v2", h.book.ID(), first) { t.Fatal("the first grant cleared nothing") } // b's canary fails: the same build fails again on the same host. waitFor(t, "b's canary to fail", func() bool { return !h.m.attemptRunning("b") && h.book.Blocks("bab", "blog", "v2", "b", false) }) if h.m.RetryHost("b", "v2", h.book.ID(), first) { t.Fatal("a grant from before b's second failure cleared it") } if !h.book.Blocks("bab", "blog", "v2", "b", false) { t.Fatal("b is not held after the stale grant") } h.failing.Store(false) if !h.retry("b", "v2") { t.Fatal("a fresh grant cleared nothing") } waitFor(t, "b on v2", func() bool { return h.servingAll("v2", "b") }) } // The same at the book: a merge that holds the host down after the grant // was signed moves the epoch, so the grant clears nothing. func TestARetryGrantPredatingAHoldDownClearsNothing(t *testing.T) { b := &FailureBook{} b.Record(FailureRecord{Account: "bab", App: "blog", Build: "v2", Host: "h"}) signed := b.RestrictionEpoch("bab", "blog", "v2", "h") b.Record(FailureRecord{Account: "bab", App: "blog", Build: "v2", Host: "h", HeldDown: []string{"h"}}) if rec, mark, _, _ := b.RetryHostAt("bab", "blog", "v2", "h", b.ID(), signed); rec || mark { t.Fatalf("a grant predating the hold-down cleared record %v, marker %v", rec, mark) } if !b.HeldDown("bab", "blog", "v2", "h") { t.Fatal("h is no longer held down") } } // A used grant replayed after the agent reopens its book clears nothing // new: the clear moved the host's epoch, and the epoch is on disk. func TestARetryGrantReplayedAfterAReopenClearsNothing(t *testing.T) { dd := t.TempDir() b, err := OpenFailureBook(dd) if err != nil { t.Fatal(err) } b.Record(FailureRecord{Account: "bab", App: "blog", Build: "v2", Host: "h", Mode: string(activator.ModeHold), HeldDown: []string{"h"}, MarkHoldRan: true, HoldRanExec: "a1", HoldRanAt: t0}) grant := b.RestrictionEpoch("bab", "blog", "v2", "h") if rec, mark, _, err := b.RetryHostAt("bab", "blog", "v2", "h", b.ID(), grant); !rec || !mark || err != nil { t.Fatalf("the grant cleared record %v, marker %v (%v); want both", rec, mark, err) } b2, err := OpenFailureBook(dd) if err != nil { t.Fatal(err) } // The retried canary fails again, after the reopen. b2.Record(FailureRecord{Account: "bab", App: "blog", Build: "v2", Host: "h", Mode: string(activator.ModeHold), HeldDown: []string{"h"}, MarkHoldRan: true, HoldRanExec: "a2", HoldRanAt: t0.Add(time.Minute), Seq: b2.Snapshot()}) if rec, mark, _, _ := b2.RetryHostAt("bab", "blog", "v2", "h", b.ID(), grant); rec || mark { t.Fatalf("the replayed grant cleared record %v, marker %v", rec, mark) } if !b2.HeldDown("bab", "blog", "v2", "h") { t.Fatal("the replay lifted the second hold-down") } if _, ok := b2.HoldRanOn("h"); !ok { t.Fatal("the replay lifted the second marker") } } // Status publishes the epoch the console signs, beside HeldBy, on both the // record arm and the marker arm. func TestStatusPublishesTheHeldEpoch(t *testing.T) { h := newHarness(t, "a", "b") h.stage("v2", "v3") h.install("v2", "a") h.install("v3", "b") h.sleepHost("a") h.sleepHost("b") h.pass() h.book.Record(FailureRecord{Account: "bab", App: "blog", Build: "v2", Host: "sib", Reason: "exited early", At: t0}) h.book.MarkHoldRan("b", HoldRan{Account: "bab", App: "blog", Build: "v4"}) r := h.m.Status() if got := r.Hosts["a"]; got.HeldBy != "v2" || got.HeldEpoch == 0 || got.HeldEpoch != h.book.RestrictionEpoch("bab", "blog", "v2", "a") { t.Fatalf("a = %+v: want held by v2 at its epoch %d", got, h.book.RestrictionEpoch("bab", "blog", "v2", "a")) } if got := r.Hosts["b"]; got.HeldBy != "v4" || got.HeldEpoch == 0 || got.HeldEpoch != h.book.RestrictionEpoch("bab", "blog", "v4", "b") { t.Fatalf("b = %+v: want held by the marker's v4 at its epoch", got) } if got := r.Hosts["b"]; got.HeldBook == "" || got.HeldBook != h.book.ID() || r.Hosts["a"].HeldBook != h.book.ID() { t.Fatalf("held books %q and %q: want the book's identity %q on both arms", r.Hosts["a"].HeldBook, got.HeldBook, h.book.ID()) } // Beside HeldBook, the retry protocol this box takes (readiness round 2, // Task 3): its grants ride the host-retry log, which the console checks // before it writes one. if r.Hosts["a"].RetryV != status.HostRetryV || r.Hosts["b"].RetryV != status.HostRetryV || status.HostRetryV != 1 { t.Fatalf("retry_v %d and %d: want %d on both arms", r.Hosts["a"].RetryV, r.Hosts["b"].RetryV, status.HostRetryV) } if !h.m.RetryHost("b", r.Hosts["b"].HeldBy, r.Hosts["b"].HeldBook, r.Hosts["b"].HeldEpoch) { t.Fatal("a grant signed from the status report cleared nothing") } }