package activator import ( "context" "errors" "fmt" "strings" "testing" "time" "github.com/carlosframework/platform/internal/cgroup" "github.com/carlosframework/platform/internal/hostguard" "github.com/carlosframework/platform/internal/hostwrite" "github.com/carlosframework/platform/internal/instancectl" "github.com/carlosframework/platform/internal/registry" ) // The happy path, in spec order: raise, exec the staged path, health // through the handle, switch, then SIGTERM the retiring generation, then // restore the parent. The live link names v3 (root moved it) and makes no // difference: the successor execs the attempt's staged v2 (review focus 2). // The retiring generation is signalled only after AppSocket has moved // (review focus 1). func TestSwapOverlapSwitchesAfterHealthAndRetiresTheIncumbent(t *testing.T) { f := newSwapFixture(t) f.b.execVerFn = func() string { return "v3" } incumbent := f.b.lastProc() res := f.a.Swap(context.Background(), f.attempt(ModeOverlap), f.newest) if res.Outcome != SwapDone || res.Gen.Build != "v2" || res.Gen.StagedPath != f.staged { t.Fatalf("Swap = %+v", res) } if got := f.appSocket(t); got != res.Gen.Socket { t.Fatalf("AppSocket = %q, want the successor's %q", got, res.Gen.Socket) } ev := f.b.events() order := []int{ indexPrefix(ev, "raise"), indexPrefix(ev, "startGen:"+f.staged), indexPrefix(ev, fmt.Sprintf("switch:%d", res.Gen.N)), indexPrefix(ev, "stop:instance"), // The parent restore. A bare "restore" is the wake's data restore // (fakeBackend.Restore), logged before any of this. indexPrefix(ev, "restore:"), } for i := range order { if order[i] < 0 || (i > 0 && order[i] < order[i-1]) { t.Fatalf("event order %v (indexes %v): want raise, startGen, switch, stop:instance, restore", ev, order) } } if !incumbent.isStopped() { t.Fatal("the retiring generation was never stopped") } cur, members := f.a.genStateForTest(f.host) if cur.N != res.Gen.N || len(members) != 0 { t.Fatalf("after the swap: current %d, members %v", cur.N, members) } if _, raises, restores := f.b.calls(); raises != 1 || restores != 1 { t.Fatalf("raises %d restores %d, want 1 and 1", raises, restores) } if got := f.b.startCount(); got != 1 { t.Fatalf("Start (live link) calls = %d, want only the original wake", got) } } // During the overlap the parent holds the incumbent's tier plus the // successor's, each captured from its own artifact (spec §7.3), and the // restore is to whichever generation is current afterwards. Here the // incumbent (v1) declares children (1 GiB) and the successor (v2) does // not (512 MiB). Summing one tier twice gives a 1 GiB parent (or 2 GiB), // and restoring to the live tier leaves the wrong limit after a failure. func TestSwapOverlapUsesEachArtifactsTier(t *testing.T) { t.Parallel() const gib, half = int64(1 << 30), int64(1 << 29) t.Run("switched: restored to the successor's tier", func(t *testing.T) { f := newSwapFixture(t) f.b.tiers = map[string]cgroup.Limit{"v1": cgroup.TierChildren, "v2": cgroup.TierDefault} res := f.a.Swap(context.Background(), f.attempt(ModeOverlap), f.newest) if res.Outcome != SwapDone || res.Gen.Tier != cgroup.TierDefault { t.Fatalf("Swap = %+v, want done with the successor's own tier", res) } ev := f.b.events() if indexPrefix(ev, fmt.Sprintf("raise:%d+%d", gib, half)) < 0 || indexPrefix(ev, fmt.Sprintf("restore:%d", half)) < 0 { t.Fatalf("events %v: want raise:%d+%d and restore:%d", ev, gib, half, half) } if g, _ := f.b.lastGen(); g.Tier != cgroup.TierDefault { t.Fatalf("the successor spawned at %v, want its own 512 MiB tier", g.Tier) } }) t.Run("failed: restored to the incumbent's tier", func(t *testing.T) { f := newSwapFixture(t) f.b.tiers = map[string]cgroup.Limit{"v1": cgroup.TierChildren, "v2": cgroup.TierDefault} f.a.genHealth = func(context.Context, Gen) error { return errors.New("unhealthy: HTTP 503") } if res := f.a.Swap(context.Background(), f.attempt(ModeOverlap), f.newest); res.Outcome != SwapFailed { t.Fatalf("Swap = %+v", res) } if ev := f.b.events(); indexPrefix(ev, fmt.Sprintf("restore:%d", gib)) < 0 { t.Fatalf("events %v: want restore:%d (the incumbent is still current)", ev, gib) } }) } // The "swapped" event carries the retiring generation's drain time (spec // §9.4), so the result measures it: from SIGTERM to the proven exit. func TestSwapOverlapReportsTheDrainTime(t *testing.T) { t.Parallel() f := newSwapFixture(t) gate := make(chan struct{}) f.b.lastProc().setStopGate(gate) time.AfterFunc(80*time.Millisecond, func() { close(gate) }) res := f.a.Swap(context.Background(), f.attempt(ModeOverlap), f.newest) if res.Outcome != SwapDone { t.Fatalf("Swap = %+v", res) } if res.Drain < 50*time.Millisecond || res.Drain > 5*time.Second { t.Fatalf("Drain = %v, want the retiring generation's stop (about 80ms)", res.Drain) } } // A successor that fails its health check, fails to spawn, or exits before // the switch leaves the incumbent current and untouched (spec §8.1). func TestSwapOverlapFailureKeepsTheIncumbent(t *testing.T) { t.Parallel() for _, tc := range []struct { name string arrange func(f *swapFixture) reason string }{ {"unhealthy", func(f *swapFixture) { f.a.genHealth = func(context.Context, Gen) error { return errors.New("unhealthy: HTTP 503") } }, "health"}, {"spawn fails", func(f *swapFixture) { f.b.genErr = errors.New("exec format error") }, "spawn"}, {"exits before the switch", func(f *swapFixture) { f.a.genHealth = func(context.Context, Gen) error { _, p := f.b.lastGen() p.exit() return nil } }, errSuccessorExited.Error()}, } { t.Run(tc.name, func(t *testing.T) { f := newSwapFixture(t) tc.arrange(f) incumbent := f.b.lastProc() res := f.a.Swap(context.Background(), f.attempt(ModeOverlap), f.newest) if res.Outcome != SwapFailed || !strings.Contains(res.Reason, tc.reason) { t.Fatalf("Swap = %+v, want failed with %q", res, tc.reason) } if got := f.appSocket(t); got != f.incumbent.Socket { t.Fatalf("AppSocket moved to %q after a failure", got) } if incumbent.isStopped() { t.Fatal("the incumbent was stopped by a failed swap") } if _, p := f.b.lastGen(); p != nil && !p.isStopped() { t.Fatal("the failed successor was left running") } if _, raises, restores := f.b.calls(); raises != restores { t.Fatalf("raises %d restores %d: the parent limit was not restored", raises, restores) } }) } } // Every refusal spawns nothing. Swaps off is the flag-off test. func TestSwapRefusals(t *testing.T) { t.Parallel() for _, tc := range []struct { name string arrange func(t *testing.T, f *swapFixture, at *Attempt) want error }{ {"swap off", func(_ *testing.T, f *swapFixture, _ *Attempt) { f.a.gens = false }, errSwapOff}, {"asleep", func(_ *testing.T, _ *swapFixture, at *Attempt) { at.Host = "asleep.example.com" }, errSwapNotAwake}, {"worker", func(_ *testing.T, f *swapFixture, _ *Attempt) { f.a.mu.Lock() f.a.entries[f.host].worker = true f.a.mu.Unlock() }, errSwapKind}, {"busy", func(t *testing.T, f *swapFixture, _ *Attempt) { plantMember(t, f.a, f.host, roleSuccessor, &fakeProc{}, 77) }, errSwapBusy}, {"unknown mode", func(_ *testing.T, _ *swapFixture, at *Attempt) { at.Mode = "" }, errSwapMode}, {"superseded", func(_ *testing.T, f *swapFixture, _ *Attempt) { f.rev.Add(1) }, errSwapSuperseded}, {"fenced", func(_ *testing.T, f *swapFixture, _ *Attempt) { f.a.mu.Lock() f.a.fences = map[string]uint64{f.host: 3} f.a.mu.Unlock() }, errSwapFenced}, {"lease lost", func(_ *testing.T, f *swapFixture, _ *Attempt) { f.b.lastRep().setErr(errors.New("renewal failed")) }, errSwapLeaseLost}, } { t.Run(tc.name, func(t *testing.T) { f := newSwapFixture(t) at := f.attempt(ModeOverlap) tc.arrange(t, f, &at) res := f.a.Swap(context.Background(), at, f.newest) if res.Outcome != SwapAbandoned || !strings.Contains(res.Reason, tc.want.Error()) { t.Fatalf("Swap = %+v, want abandoned: %v", res, tc.want) } if gens, raises, _ := f.b.calls(); gens != 0 || raises != 0 { t.Fatalf("a refused attempt spawned %d generation(s), raised %d time(s)", gens, raises) } }) } t.Run("digest mismatch", func(t *testing.T) { f := newSwapFixture(t) at := f.attempt(ModeOverlap) at.Digest = strings.Repeat("0", 64) res := f.a.Swap(context.Background(), at, f.newest) if res.Outcome != SwapAbandoned || !strings.Contains(res.Reason, "staging") { t.Fatalf("Swap = %+v, want abandoned as a staging error", res) } if gens, _, _ := f.b.calls(); gens != 0 { t.Fatal("a mismatched artifact reached StartGen") } }) t.Run("backend without generations", func(t *testing.T) { a, _, _, _ := testActivator(t, time.Hour) a.gens = true if res := a.Swap(context.Background(), Attempt{Host: "x.example.com", Mode: ModeOverlap}, func(string) uint64 { return 0 }); res.Outcome != SwapAbandoned || !strings.Contains(res.Reason, errSwapNoGens.Error()) { t.Fatalf("Swap = %+v", res) } }) } // A stopping entry admits no successor (spec §7.4: "the entry is still // awake"). This is the witness for mutation M5. func TestSwapWitnessNoAdmissionOntoAStoppingEntry(t *testing.T) { f := newSwapFixture(t) gate := make(chan struct{}) f.b.lastProc().setStopGate(gate) f.a.Bounce(f.host) waitFor(t, "the entry to stop", func() bool { f.a.mu.Lock() defer f.a.mu.Unlock() e := f.a.entries[f.host] return e != nil && e.state == actStopping }) // The refusal is immediate. A Swap that admits a successor instead // switches and then drains the incumbent, whose Stop the gate holds, so // it cannot return until the gate opens: bound the wait, so the // mutation fails here by name rather than hanging the run. results := make(chan SwapResult, 1) go func() { results <- f.a.Swap(context.Background(), f.attempt(ModeOverlap), f.newest) }() var res SwapResult select { case res = <-results: case <-time.After(5 * time.Second): close(gate) select { case res = <-results: t.Fatalf("Swap waited on the stopping incumbent's drain, then returned %+v: it admitted a successor onto a stopping entry", res) case <-time.After(5 * time.Second): t.Fatal("Swap onto a stopping entry did not return") } } close(gate) if res.Outcome != SwapAbandoned || !strings.Contains(res.Reason, errSwapNotAwake.Error()) { t.Fatalf("Swap = %+v, want abandoned: not awake", res) } if gens, _, _ := f.b.calls(); gens != 0 { t.Fatalf("StartGen ran %d time(s) on a stopping entry", gens) } } // The one outcome classifier (spec §6.4, §7.4): only a failure of the build // is stepFailed. A cancelled lifecycle, an intentional stop and a refusal // by the attempt gate record nothing; a health deadline that expires is the // build's. func TestClassifyStep(t *testing.T) { live, ended := context.Background(), func() context.Context { ctx, cancel := context.WithCancel(context.Background()) cancel() return ctx }() for _, tc := range []struct { name string ctx context.Context err error want stepOutcome }{ {"ok", live, nil, stepOK}, {"the lifecycle ended", ended, errors.New("dial: connection refused"), stepCancelled}, {"cancelled", live, fmt.Errorf("health: %w", context.Canceled), stepCancelled}, {"fenced", live, fmt.Errorf("wake: %w", ErrFenced), stepCancelled}, {"the barrier", live, errSwapStopped, stepCancelled}, {"superseded", live, fmt.Errorf("pin: %w", errSwapSuperseded), stepStale}, {"lease lost", live, errSwapLeaseLost, stepStale}, // Claims validation at the exec (spawnClaimedGen's Work.With): // nothing ran, so no failure of the build and no halted cohort. {"a changed stamp at the exec", live, fmt.Errorf("with: %w", hostguard.ErrStampMismatch), stepStale}, {"a durable fence at the exec", live, fmt.Errorf("with: %w", hostguard.ErrFenced), stepStale}, // The strict-host gate refused a placed spawn (plan 2), and every // claims-on launch is closed on this base: nothing of the build ran. {"the strict gate", live, fmt.Errorf("x: %w: %w", ErrStrictRequired, ErrStrictGroupInUse), stepStale}, {"a closed claims launch", live, fmt.Errorf("launch: %w", ErrClaimIsolationUnavailable), stepStale}, // A row that cannot run a staged generation at all. {"a unit row", live, errUnitNoGens, stepStale}, // Amadan #272: the generation start refused before any exec because // the host's parent holds a process directly. {"a populated legacy parent", live, fmt.Errorf("h: %w: holds 1 process(es)", cgroup.ErrLegacyParentPopulated), stepStale}, // Task 19: the start's killed cgroup could not be made fresh, so the // manager refused before any exec. {"a killed cgroup that would not go", live, fmt.Errorf("%w: carlos-exec@h/g2: device or resource busy", cgroup.ErrKilledCgroupKept), stepStale}, // Task 7 (readiness round 2): a generation whose limits could not be // applied is refused before any exec: the box's condition. {"a generation's limits not applied", live, fmt.Errorf("h: %w: g2: memory.max: is a directory", cgroup.ErrLimitNotApplied), stepStale}, // Root refused the nested staged executable before it mounted or // started anything (Task 9): an abandonment, not a failed build. {"root refused the staged exe", live, fmt.Errorf("start unit: %w: not blog's staged v2", instancectl.ErrStagedExe), stepStale}, {"a backend without generations", live, errSwapNoGens, stepStale}, {"a health deadline", live, fmt.Errorf("health: %w", context.DeadlineExceeded), stepFailed}, {"HTTP 500", live, errors.New("GET / answered 500"), stepFailed}, } { if got := classifyStep(tc.ctx, tc.err); got != tc.want { t.Errorf("%s: %d, want %d", tc.name, got, tc.want) } } } // The gate's verdict is typed (spec §6.3, §7.4): a supersession may restore // behind a fresh ownership check; a lost lease, a fence, shutdown or a // claims refusal stops every spawn. func TestVerdictOf(t *testing.T) { for _, tc := range []struct { err error want gateVerdict }{ {nil, gateProceed}, {errors.New("GET / answered 500"), gateProceed}, {fmt.Errorf("gate: %w", errSwapSuperseded), gateSuperseded}, {errSwapLeaseLost, gateNoOwnership}, {errSwapFenced, gateNoOwnership}, {errSwapShutdown, gateNoOwnership}, {errSwapNotAwake, gateNoOwnership}, // The barrier's cancel (memberCurrentLocked): the entry is stopping, // so nothing may spawn for it, a restore included. {errSwapStopped, gateNoOwnership}, {fmt.Errorf("with: %w", hostguard.ErrStampMismatch), gateNoOwnership}, {fmt.Errorf("with: %w", hostguard.ErrFenced), gateNoOwnership}, // P5 review partial new #3: an absent claim, and a claim read that // failed before the exec, are no failure of the build either. {fmt.Errorf("with: %w", hostguard.ErrNoClaim), gateNoOwnership}, {claimAdmission(errors.New("s3: GetObject: request timeout"), false), gateNoOwnership}, {claimAdmission(errors.New("GET / answered 500"), true), gateProceed}, // Plan 2's strict-host gate on placed spawns (index P2-8), as the // exec backend wraps them. {fmt.Errorf("h: %w", ErrStrictRequired), gateNoOwnership}, {fmt.Errorf("h: %w: %w", ErrStrictRequired, ErrStrictGroupInUse), gateNoOwnership}, {fmt.Errorf("h: %w", ErrStrictGroupInUse), gateNoOwnership}, {fmt.Errorf("h: %w", ErrStrictCleanupPending), gateNoOwnership}, // Every claims-on launch is closed on this base (index P2-9). It // wraps hostbox.ErrUnsupported, and is still NoOwnership, raw or // as a claims admission. {ErrClaimIsolationUnavailable, gateNoOwnership}, {fmt.Errorf("start: %w", ErrClaimIsolationUnavailable), gateNoOwnership}, {claimAdmission(ErrClaimIsolationUnavailable, false), gateNoOwnership}, // Amadan #272: a populated legacy parent refuses the start before any // exec, and no build is at fault. {fmt.Errorf("h: %w: holds 1 process(es)", cgroup.ErrLegacyParentPopulated), gateNoOwnership}, // Task 19: a start whose killed cgroup could not be made fresh is // refused before the command is built; no build is at fault. {cgroup.ErrKilledCgroupKept, gateNoOwnership}, {fmt.Errorf("%w: carlos-exec@h/role-sidecar: device or resource busy", cgroup.ErrKilledCgroupKept), gateNoOwnership}, // Task 7: a generation start refused because its limits could not // all be applied (or it could only be started then moved) is refused // before the exec; no build is at fault. {cgroup.ErrLimitNotApplied, gateNoOwnership}, {fmt.Errorf("h: %w: the placement shim is not available", cgroup.ErrLimitNotApplied), gateNoOwnership}, // A row that cannot run a staged generation: nothing staged spawns. {errUnitNoGens, gateNoOwnership}, {instancectl.ErrStagedExe, gateNoOwnership}, {fmt.Errorf("start unit: %w: x", instancectl.ErrStagedExe), gateNoOwnership}, {errSwapNoGens, gateNoOwnership}, } { if got := verdictOf(tc.err); got != tc.want { t.Errorf("verdictOf(%v) = %d, want %d", tc.err, got, tc.want) } } } // Spec §6.3, §7.4: the checks run again at the exec, not only at the // admission. A newer attempt that lands while the parent is being raised // (or, with plan 5, while the pins are being published) means nothing // execs, and the raise is undone. Checked only at the admission, the // superseded build execs and is abandoned only at the switch. func TestSwapOverlapReChecksAtTheExec(t *testing.T) { f := newSwapFixture(t) f.b.onRaise = func() { f.rev.Add(1) } res := f.a.Swap(context.Background(), f.attempt(ModeOverlap), f.newest) if res.Outcome != SwapAbandoned || !strings.Contains(res.Reason, errSwapSuperseded.Error()) { t.Fatalf("Swap = %+v, want abandoned: superseded", res) } if gens, raises, restores := f.b.calls(); gens != 0 || raises != restores { t.Fatalf("spawned %d, raises %d, restores %d: want nothing exec'd and the raise undone", gens, raises, restores) } } // A newer attempt that lands during the health check is seen at the switch. // The successor is abandoned and stopped, and the incumbent keeps serving. func TestSwapOverlapSupersededAtTheSwitch(t *testing.T) { f := newSwapFixture(t) f.a.genHealth = func(context.Context, Gen) error { f.rev.Add(1); return nil } res := f.a.Swap(context.Background(), f.attempt(ModeOverlap), f.newest) if res.Outcome != SwapAbandoned || !strings.Contains(res.Reason, errSwapSuperseded.Error()) { t.Fatalf("Swap = %+v", res) } if got := f.appSocket(t); got != f.incumbent.Socket { t.Fatalf("AppSocket = %q, want the incumbent", got) } if _, p := f.b.lastGen(); !p.isStopped() { t.Fatal("the abandoned successor was left running") } } // useClaimSpawn stands the claimSpawn seam in for the kernel: a claims-on // spawn needs a delegated cgroup tree and an open isolated launch, and // every claims-on launch is closed on this base. func useClaimSpawn(f *swapFixture) { f.a.claimSpawn = func(ctx context.Context, _ *actEntry, rt registry.Route, _ hostwrite.Work, g Gen) (Gen, Proc, error) { return f.b.StartStaged(ctx, rt, g) } } // Claims on: a refused selection or a refused check, before the spawn or // at the switch, abandons the attempt and is never retried as claims off. func TestSwapClaimsOn(t *testing.T) { t.Parallel() for _, tc := range []struct { name string sel func() hostwrite.Selector wantGens int }{ {"selector refuses", func() hostwrite.Selector { return &swapSelector{err: hostwrite.ErrUnavailable} }, 0}, {"no selector", func() hostwrite.Selector { return nil }, 0}, {"refused before the spawn", func() hostwrite.Selector { return &swapSelector{work: &swapWork{denyAt: 1}} }, 0}, {"refused at the exec", func() hostwrite.Selector { return &swapSelector{work: &swapWork{denyAt: 2}} }, 0}, {"refused at the switch", func() hostwrite.Selector { return &swapSelector{work: &swapWork{denyAt: 3}} }, 1}, } { t.Run(tc.name, func(t *testing.T) { f := newSwapFixture(t) f.a.cfg.HostWrites = hostwrite.Config{Enabled: true, Selector: tc.sel()} // A runtime that can start a generation of its own, so each row // is refused where it names, not by errClaimsNoGenerations. f.a.cfg.Claims = &ClaimRuntime{Pins: true, Generations: true} // The seam stands in for an open launch, so each row reaches the // refusal it names; without it, claimsSwapRefusal refuses every // row first (the "closed launch, no seam" case below). useClaimSpawn(f) res := f.a.Swap(context.Background(), f.attempt(ModeOverlap), f.newest) if res.Outcome != SwapAbandoned { t.Fatalf("Swap = %+v, want abandoned", res) } gens, raises, restores := f.b.calls() if gens != tc.wantGens || raises != restores { t.Fatalf("gens %d (want %d), raises %d, restores %d", gens, tc.wantGens, raises, restores) } if got := f.appSocket(t); got != f.incumbent.Socket { t.Fatalf("AppSocket = %q, want the incumbent", got) } if _, p := f.b.lastGen(); p != nil && !p.isStopped() { t.Fatal("a refused successor was left running") } }) } t.Run("allowed", func(t *testing.T) { f := newSwapFixture(t) w := &swapWork{} f.a.cfg.HostWrites = hostwrite.Config{Enabled: true, Selector: &swapSelector{work: w}} f.a.cfg.Claims = &ClaimRuntime{Pins: true, Generations: true} useClaimSpawn(f) if res := f.a.Swap(context.Background(), f.attempt(ModeOverlap), f.newest); res.Outcome != SwapDone { t.Fatalf("Swap = %+v", res) } if w.withCalls() != 3 { t.Fatalf("With calls = %d, want 3 (the admission, the exec and the switch)", w.withCalls()) } }) // Every claims-on launch is closed on this base (index P2-9). Without // the seam a claims-on swap is refused before its admission and its // raise (claimsSwapRefusal, final review F9): abandoned (never failed, // which would halt the cohort for a build that never ran), nothing // raised or spawned, and never sent down the claims-off StartStaged. t.Run("closed launch, no seam", func(t *testing.T) { f := newSwapFixture(t) w := &swapWork{} f.a.cfg.HostWrites = hostwrite.Config{Enabled: true, Selector: &swapSelector{work: w}} f.a.cfg.Claims = &ClaimRuntime{Pins: true, Generations: true} res := f.a.Swap(context.Background(), f.attempt(ModeOverlap), f.newest) if res.Outcome != SwapAbandoned || !strings.Contains(res.Reason, ErrClaimIsolationUnavailable.Error()) { t.Fatalf("Swap = %+v, want abandoned: %v", res, ErrClaimIsolationUnavailable) } gens, raises, restores := f.b.calls() if gens != 0 || raises != 0 || restores != 0 { t.Fatalf("StartStaged %d, raises %d, restores %d: want nothing raised or spawned", gens, raises, restores) } if ev := f.b.events(); indexPrefix(ev, "startGen:") >= 0 { t.Fatalf("events %v: a claims-on swap went down the claims-off spawn", ev) } if w.withCalls() != 0 { t.Fatalf("With calls = %d, want 0 (refused before the admission)", w.withCalls()) } if got := f.appSocket(t); got != f.incumbent.Socket { t.Fatalf("AppSocket = %q, want the incumbent", got) } if _, members := f.a.genStateForTest(f.host); len(members) != 0 { t.Fatalf("members %v left in the operation set", members) } }) }