package rollout import ( "maps" "slices" "testing" "time" "github.com/carlosframework/platform/internal/activator" ) // retryHarness is a harness whose s fails v3 and whose t fails v4: each // failure writes a record that holds h too, and halts that build's cohort. // h serves and wants v1 throughout. func retryHarness(t *testing.T) (*harness, func(build string) bool) { t.Helper() k := appKey{"bab", "blog"} h := newHarness(t, "h", "s", "t") h.stage("v2", "v3", "v4") h.act.mu.Lock() h.act.outcome = func(a activator.Attempt) activator.SwapOutcome { if (a.Host == "s" && a.Build == "v3") || (a.Host == "t" && a.Build == "v4") { return activator.SwapFailed } return activator.SwapDone } h.act.mu.Unlock() h.pass() s := h.m.scheduler(k) halted := func(build string) bool { s.mu.Lock() defer s.mu.Unlock() return s.q.cohortFor(cohortKey(build, classAdopt)).halted } h.install("v3", "s") h.pass() waitFor(t, "v3's cohort to halt", func() bool { return halted("v3") }) h.install("v4", "t") h.pass() waitFor(t, "v4's cohort to halt", func() bool { return halted("v4") }) return h, halted } // running is h's running attempt, nil when none. func (h *harness) running(host string) *dispatch { s := h.m.scheduler(appKey{"bab", "blog"}) s.mu.Lock() defer s.mu.Unlock() if d := s.q.hosts[host].running; d != nil { c := *d return &c } return nil } // Final review round 2, finding 5 (Astra): retry entitlements are kept per // host and build. B's (v3's) cohort is halted; h is retried for v3 while it // wants v1, then for another restricted build C (v4) before it adopts v3. // The second retry must not overwrite the first: when h comes to want v3, // the book already lists h as retried for v3 (so a second retry of v3 // clears nothing), and only the entitlement the first retry bound can take // h through v3's halt. func TestARetryForASecondBuildKeepsTheFirstBuildsCanary(t *testing.T) { h, halted := retryHarness(t) if !h.retry("h", "v3") { t.Fatal("the retry for v3 cleared nothing") } if !h.retry("h", "v4") { t.Fatal("the retry for v4 cleared nothing") } if h.retry("h", "v3") { t.Fatal("a second retry of v3 cleared something: the case needs the book to refuse it") } if !halted("v3") || !halted("v4") { t.Fatal("a cohort is no longer halted: the case needs both halts") } gate := h.act.hold("h") h.install("v3", "h") h.pass() waitFor(t, "h's adoption onto v3 through the halt", func() bool { _, started, _ := h.act.snapshot() return slices.Contains(started, "h:v3") }) if d := h.running("h"); d == nil || d.ID.Build != "v3" || !d.Canary || !d.Forced { t.Fatalf("h's running attempt %+v: want the forced canary onto v3", d) } close(gate) waitFor(t, "h's adoption onto v3", func() bool { return h.servingAll("v3", "h") && !h.m.attemptRunning("h") }) // v4's entitlement was neither overwritten nor spent by v3's adoption. gate = h.act.hold("h") defer close(gate) h.install("v4", "h") h.pass() waitFor(t, "h's adoption onto v4 through the halt", func() bool { _, started, _ := h.act.snapshot() return slices.Contains(started, "h:v4") }) if d := h.running("h"); d == nil || d.ID.Build != "v4" || !d.Canary || !d.Forced { t.Fatalf("h's running attempt %+v: want the forced canary onto v4", d) } } // An entitlement lasts only while the book lists the host as retried for its // build. Once the record is gone (cleared by a signed pointer, or pruned when // no route installs the build), the next pass drops it: the set cannot grow // without bound, and a record the build may get later starts with no // entitlement from a retry of the old one. func TestARetryEntitlementIsDroppedWhenTheBookNoLongerListsTheHost(t *testing.T) { for _, tc := range []struct { name string clear func(h *harness) }{ {"cleared by a pointer", func(h *harness) { // s still wants v3 and may adopt it before h: it succeeds now, // so no fresh record of v3 races h's adoption. h.act.mu.Lock() h.act.outcome = func(a activator.Attempt) activator.SwapOutcome { if a.Host == "t" && a.Build == "v4" { return activator.SwapFailed } return activator.SwapDone } h.act.mu.Unlock() if ok, _ := h.book.ClearOnPointer("bab", "blog", "v3", t0.Add(time.Hour)); !ok { h.t.Fatal("the pointer cleared nothing") } }}, {"pruned", func(h *harness) { h.install("v1", "s") // no route installs v3 any more }}, } { t.Run(tc.name, func(t *testing.T) { h, halted := retryHarness(t) if !h.retry("h", "v3") { t.Fatal("the retry for v3 cleared nothing") } if !h.retry("h", "v4") { t.Fatal("the retry for v4 cleared nothing") } tc.clear(h) h.pass() if halted("v3") { t.Fatal("v3's halt outlived its record") } s := h.m.scheduler(appKey{"bab", "blog"}) s.mu.Lock() got := slices.Sorted(maps.Keys(s.q.hosts["h"].forceBuilds)) s.mu.Unlock() if !slices.Equal(got, []string{"v4"}) { t.Fatalf("h's entitlements %v, want only v4's: v3's record is gone", got) } gate := h.act.hold("h") defer close(gate) h.install("v3", "h") h.pass() waitFor(t, "h's adoption onto v3", func() bool { _, started, _ := h.act.snapshot() return slices.Contains(started, "h:v3") }) // Whether it is the cohort's canary depends on whether s went // first; either way it is not a retried host's forced canary. if d := h.running("h"); d == nil || d.ID.Build != "v3" || d.Forced { t.Fatalf("h's running attempt %+v: want an ordinary adoption onto v3, not a retried one", d) } }) } }