package rollout import ( "sync/atomic" "testing" "time" "github.com/carlosframework/platform/internal/activator" "github.com/carlosframework/platform/internal/release" ) // Final review F8: `carlos instances retry ` promises that host's next // adoption runs as a canary through its cohort's halt. Admission spends the // allowance (forceBuilds) when the attempt starts. An attempt that then ends // before its build ran (abandoned with SwapResult.Executed false, or refused // as a staging error) spent nothing of it: the allowance comes back, so the // canary still runs once the host's deferral ends, with no second retry // (which the book would refuse anyway: the host is already in Retried). An // attempt whose build ran spends it, abandoned or not. func TestARetryAllowanceComesBackOnlyWhenNothingRan(t *testing.T) { for _, tc := range []struct { name string staging bool // b's first retry attempt is refused as a staging error executed bool // b's first retry attempt is abandoned after its build ran deferral time.Duration }{ {"abandoned before its build ran", false, false, abandonDeferral}, {"refused as a staging error", true, false, stagingDeferral}, {"abandoned after its build ran", false, true, abandonDeferral}, } { t.Run(tc.name, func(t *testing.T) { h := newHarness(t, "a", "b") h.stage("v2") var bTries atomic.Int32 h.act.outcome = func(a activator.Attempt) activator.SwapOutcome { switch { case a.Build != "v2": return activator.SwapDone case a.Host == "a": return activator.SwapFailed // the canary that halts the cohort case bTries.Add(1) == 1 && !tc.staging: return activator.SwapAbandoned } return activator.SwapDone } h.act.executed = func(_ activator.Attempt, out activator.SwapOutcome) bool { return out != activator.SwapAbandoned || tc.executed } h.install("v2", "a", "b") h.pass() waitFor(t, "the halt", func() bool { _, ok := h.book.Lookup("bab", "blog", "v2"); return ok }) if tc.staging { h.mu.Lock() h.mismatch["v2"] = true h.mu.Unlock() } if !h.retry("b", "v2") { t.Fatal("RetryHost did not clear b") } if tc.staging { waitFor(t, "the staging refusal", func() bool { return h.touched.Load() > 0 }) h.mu.Lock() h.mismatch["v2"] = false h.mu.Unlock() } else { waitFor(t, "b's first retry attempt", func() bool { return bTries.Load() >= 1 }) } // Its slot is free and its deferral set (done) before the next pass. waitFor(t, "b's slot to free", func() bool { return !h.m.attemptRunning("b") }) before, _, _ := h.act.snapshot() h.advance(tc.deferral + time.Second) h.pass() if tc.executed { time.Sleep(100 * time.Millisecond) if calls, _, _ := h.act.snapshot(); len(calls) != len(before) { t.Fatalf("calls %+v: an attempt whose build ran spent the retry, yet b ran again through the halt", calls[len(before):]) } return } waitFor(t, "b's retry canary", func() bool { return h.servingAll("v2", "b") }) calls, _, _ := h.act.snapshot() if c := calls[len(calls)-1].a; c.Host != "b" || !c.Canary || c.Cause != activator.CauseRetry { t.Fatalf("attempt after the re-armed retry = %+v, want b's retry canary", c) } if h.servingAll("v2", "a") { t.Fatal("the re-armed retry lifted the halt for another host") } }) } } // Final review wave 2, N4: only proof that nothing ran gives the allowance // back. A worker attempt abandoned when its caller left can still have had // its respawn exec the build (the activator joins the respawn, and past its // bound reports Executed true: unknown). Executed true spends the // allowance, whether the build surely ran or only may have; Executed false, // the activator's proof that nothing ran, gives it back. func TestARetryAllowanceIsSpentWhenAWorkerAttemptMayHaveRun(t *testing.T) { for _, tc := range []struct { name string executed bool }{ {"execution unknown", true}, {"proven unexecuted", false}, } { t.Run(tc.name, func(t *testing.T) { h := workerHarness(t, "a", "b") h.stage("v2") var bTries atomic.Int32 h.act.mu.Lock() h.act.worker = func(a activator.Attempt, _ *release.StagedIdentity) activator.SwapResult { switch { case a.Build != "v2": return activator.SwapResult{Outcome: activator.SwapDone} case a.Host == "a": return activator.SwapResult{Outcome: activator.SwapFailed, Reason: "exited within 10s of starting", Executed: true} case bTries.Add(1) == 1: return activator.SwapResult{Outcome: activator.SwapAbandoned, Reason: "swap: an intentional stop cancelled the attempt", Executed: tc.executed} } return activator.SwapResult{Outcome: activator.SwapDone, Executed: true} } h.act.mu.Unlock() h.install("v2", "a", "b") h.pass() waitFor(t, "the halt", func() bool { _, ok := h.book.Lookup("bab", "blog", "v2"); return ok }) waitFor(t, "a's slot to free", func() bool { return !h.m.attemptRunning("a") }) if !h.retry("b", "v2") { t.Fatal("RetryHost did not clear b") } waitFor(t, "b's first retry attempt", func() bool { return bTries.Load() >= 1 }) waitFor(t, "b's slot to free", func() bool { return !h.m.attemptRunning("b") }) h.advance(abandonDeferral + time.Second) h.pass() if tc.executed { time.Sleep(100 * time.Millisecond) if n := bTries.Load(); n != 1 { t.Fatalf("b ran %d attempts: an attempt that may have run its build spent the retry, yet b ran again through the halt", n) } return } waitFor(t, "b's re-armed retry canary", func() bool { return bTries.Load() == 2 }) }) } }