package activator import ( "context" "errors" "fmt" "go/ast" "go/parser" "go/token" "io/fs" "os/exec" "strings" "testing" "time" "github.com/carlosframework/platform/internal/hostclaim" "github.com/carlosframework/platform/internal/hostguard" "github.com/carlosframework/platform/internal/hostwrite" "github.com/carlosframework/platform/internal/registry" ) // claimedStartFor is a ClaimStart for obs with every hook a no-op, version // v. It is the one ClaimStart these tests build: main's claims-on launch // needs a RoleLauncher and CurrentAttempt for valid() (claimwindow_test.go), // and on this base RoleLaunch.Start refuses every shape before its before // hook runs (index P2-9). func claimedStartFor(obs hostclaim.Observed, work hostwrite.Work, v string) ClaimStart { return ClaimStart{RoleLauncher: &RoleLaunch{}, CurrentAttempt: func() hostclaim.WakeRetryAttempt { return hostclaim.WakeRetryAttempt{} }, Host: obs.Host, Work: work, Identity: hostwrite.Identity{Config: "c", Release: "r"}, Version: v, Restore: func(context.Context) error { return nil }, Materialize: func(context.Context) error { return nil }, Build: func() *exec.Cmd { return nil }, Ready: func(context.Context, *exec.Cmd) error { return nil }, Reserve: func(context.Context, int) error { return nil }, Failed: func(context.Context, int) error { return nil }, Healthy: func(context.Context, int, *exec.Cmd) error { return nil }, Replicate: func(context.Context, string) (Rep, error) { return nil, nil }, Publish: func(context.Context, hostclaim.Observed) (hostclaim.Observed, error) { return obs, nil }, Retain: func(Proc, Rep, error) {}} } func claimedObs(host string) hostclaim.Observed { return hostclaim.Observed{Host: host, ETag: "one", Claim: &hostclaim.Claim{ Holder: &hostclaim.Holder{Location: "instances/" + host + "/inc", Session: 1, Generation: "one"}, Op: hostclaim.Op{Kind: hostclaim.KindServing, Exec: "box"}, }} } // Without a runtime that builds per-generation starts, a claims-on swap is // refused before anything is raised, stopped or spawned. func TestClaimsOnSwapNeedsGenerations(t *testing.T) { t.Parallel() for _, mode := range []SwapMode{ModeOverlap, ModeHold} { f := newSwapFixture(t) f.a.cfg.HostWrites = hostwrite.Config{Enabled: true, Selector: &swapSelector{work: &swapWork{}}} f.a.cfg.Claims = &ClaimRuntime{Pins: true} // Generations false res := f.a.Swap(context.Background(), f.attempt(mode), f.newest) if res.Outcome != SwapAbandoned || !strings.Contains(res.Reason, errClaimsNoGenerations.Error()) { t.Fatalf("%s: Swap = %+v", mode, res) } if gens, raises, _ := f.b.calls(); gens != 0 || raises != 0 { t.Fatalf("%s: spawned %d, raised %d", mode, gens, raises) } if f.b.lastProc().isStopped() { t.Fatalf("%s: the incumbent was stopped", mode) } } } // A provider that binds anything but the generation the activator chose is // refused before Restore touches the database. With generations off (a // zero Gen) nothing is checked, as today. func TestStartAdmittedRefusesAGenerationMismatch(t *testing.T) { obs := claimedObs("x.example") work, _ := (hostwrite.Config{}).Select(context.Background(), obs.Host, "spawn") stop := errors.New("stop at restore") want := Gen{N: 9, Socket: "/run/carlos/g/abc.9.sock", Control: "/run/carlos/g/abc.9.sock.control"} for _, tc := range []struct { name string policy Gen bound Gen restored bool }{ {"matching", want, want, true}, {"another socket", want, Gen{N: 9, Socket: "/run/carlos/x.example.sock", Control: want.Control}, false}, {"another control socket", want, Gen{N: 9, Socket: want.Socket, Control: "/run/carlos/x.example.sock.control"}, false}, {"another number", want, Gen{N: 10, Socket: want.Socket, Control: want.Control}, false}, {"generations off", Gen{}, Gen{}, true}, } { restored := false a := &Service{cfg: Config{HostWrites: hostwrite.Config{Enabled: true}, Claims: &ClaimRuntime{Box: "box", Read: func(context.Context, string) (hostclaim.Observed, error) { return obs, nil }, Prepare: func(context.Context, registry.Route, hostclaim.Observed, startPolicy) (ClaimStart, error) { s := claimedStartFor(obs, work, "v1") s.Gen = tc.bound s.Restore = func(context.Context) error { restored = true; return stop } return s, nil }}}} err := a.startAdmitted(context.Background(), &actEntry{host: obs.Host}, obs, startPolicy{MaxStarts: 1, Window: time.Second, Gen: tc.policy}) if restored != tc.restored { t.Fatalf("%s: restored = %v, want %v (err %v)", tc.name, restored, tc.restored, err) } if !tc.restored && !errors.Is(err, hostwrite.ErrUnavailable) { t.Fatalf("%s: %v, want hostwrite.ErrUnavailable", tc.name, err) } if tc.restored && !errors.Is(err, stop) { t.Fatalf("%s: %v, want the restore's own error", tc.name, err) } } } // The activator chooses claims-on generation sockets the way plan 1 lays // them out, and a canonical handle with a real number where there is no // generation path. func TestClaimGenAt(t *testing.T) { b := testExecBackend(t) b.RunDirOverride = t.TempDir() a := New(Config{}, b, nil) flat := registry.Route{Host: "c.example.com", Kind: "instance", DBPath: b.DataDir + "/c.example.com.db", Socket: b.RunDirOverride + "/c.example.com.sock"} if g := a.claimGenAt(flat, 4); g.N != 4 || g.Socket != b.flatGenSocket(flat.Host, 4) || g.Control != g.Socket+".control" { t.Fatalf("flat: %+v", g) } foreign := registry.Route{Host: "f.example.com", Kind: "instance", Socket: t.TempDir() + "/odd.sock"} if g := a.claimGenAt(foreign, 5); g.N != 5 || g.Socket != foreign.Socket || g.Control != foreign.Socket+".control" { t.Fatalf("foreign: %+v", g) } // The split backend asks its exec half; a unit-backed row has no // generation path of its own. split := New(Config{}, &SplitBackend{Exec: b}, nil) if g := split.claimGenAt(flat, 6); g.N != 6 || g.Socket != b.flatGenSocket(flat.Host, 6) { t.Fatalf("split, flat: %+v", g) } unit := flat unit.Backing = registry.BackingUnit if g := split.claimGenAt(unit, 7); g.N != 7 || g.Socket != unit.Socket { t.Fatalf("split, unit: %+v", g) } } // Spec §7.4 (index P2-9): every claims-on launch is closed on this base, and // the work's own validation refuses before the exec. Nothing ran, so the // attempt is abandoned, never SwapFailed: a failure record would halt the // cohort for a build that never executed. func TestAClaimsRefusalAtTheExecIsNoFailure(t *testing.T) { t.Parallel() for _, refusal := range []error{hostguard.ErrStampMismatch, hostguard.ErrFenced, hostguard.ErrNoClaim} { f := newSwapFixture(t) f.a.cfg.HostWrites = hostwrite.Config{Enabled: true, Selector: &swapSelector{work: &swapWork{}}} f.a.cfg.Claims = &ClaimRuntime{Pins: true, Generations: true} f.a.claimSpawn = func(context.Context, *actEntry, registry.Route, hostwrite.Work, Gen) (Gen, Proc, error) { return Gen{}, nil, fmt.Errorf("work: %w", refusal) } if res := f.a.Swap(context.Background(), f.attempt(ModeOverlap), f.newest); res.Outcome != SwapAbandoned { t.Fatalf("%v at the exec: Swap = %+v, want abandoned", refusal, res) } } // No seam: the closed launch refuses the swap before anything runs // (claimsSwapRefusal), as the admission's refusal (claimAdmission), and // the attempt is abandoned. The runtime is never read or prepared: // there is no launch to prepare for. f := newSwapFixture(t) f.a.cfg.HostWrites = hostwrite.Config{Enabled: true, Selector: &swapSelector{work: &swapWork{}}} f.a.cfg.Claims = &ClaimRuntime{Pins: true, Generations: true, Read: func(context.Context, string) (hostclaim.Observed, error) { return hostclaim.Observed{}, errors.New("s3: GetObject: request timeout") }, Prepare: func(context.Context, registry.Route, hostclaim.Observed, startPolicy) (ClaimStart, error) { t.Fatal("prepared a successor on a closed launch") return ClaimStart{}, nil }} if res := f.a.Swap(context.Background(), f.attempt(ModeOverlap), f.newest); res.Outcome != SwapAbandoned { t.Fatalf("a closed launch: Swap = %+v, want abandoned", res) } } // Index P2-9, final review F9: a claims-on overlap whose runtime has // Generations and Pins, with no test seam standing in for the launch, is // refused by the closed launch before its admission and its raise. It is // abandoned with nothing raised or spawned, the incumbent still serving and // no failure recorded (SwapFailed would halt the cohort). func TestAClaimsOnOverlapOnAClosedLaunchIsAbandoned(t *testing.T) { f := newSwapFixture(t) w := &swapWork{} f.a.cfg.HostWrites = hostwrite.Config{Enabled: true, Selector: &swapSelector{work: w}} f.a.cfg.Claims = &ClaimRuntime{Pins: true, Generations: true} res := f.a.Swap(context.Background(), f.attempt(ModeOverlap), f.newest) if res.Outcome != SwapAbandoned || !strings.Contains(res.Reason, ErrClaimIsolationUnavailable.Error()) { t.Fatalf("Swap = %+v, want abandoned on the closed launch", res) } if !strings.Contains(res.Reason, errClaimsAdmission.Error()) { t.Fatalf("Reason %q: the closed launch is the admission's refusal", res.Reason) } gens, raises, restores := f.b.calls() if gens != 0 || raises != 0 || restores != 0 { t.Fatalf("StartStaged %d, raises %d, restores %d: want nothing raised or spawned", gens, raises, restores) } if n := w.withCalls(); n != 0 { t.Fatalf("With calls = %d: the refusal comes before the admission", n) } if f.b.lastProc().isStopped() { t.Fatal("the incumbent was stopped") } if got := f.appSocket(t); got != f.incumbent.Socket { t.Fatalf("AppSocket = %q, want the incumbent", got) } f.a.mu.Lock() e := f.a.entries[f.host] state, pinFailed := e.state, e.pinFailed f.a.mu.Unlock() if state != actAwake || pinFailed != nil { t.Fatalf("entry state %v, pinFailed %+v: the incumbent should serve on, with nothing recorded", state, pinFailed) } } // Index P2-9: a claims-on hold whose runtime cannot start a generation of // its own is abandoned before the incumbent stops. A hold is stop-then-start // (spec §7.8): refused after the stop, it would only cost the instance its // running process, since the closed launch could never start the pin. func TestAClaimsOnHoldWithoutGenerationsIsAbandonedBeforeTheStop(t *testing.T) { f := newSwapFixture(t) f.a.cfg.HostWrites = hostwrite.Config{Enabled: true, Selector: &swapSelector{work: &swapWork{}}} touched := false f.a.cfg.Claims = &ClaimRuntime{Box: "box", Pins: true, // Generations false Read: func(context.Context, string) (hostclaim.Observed, error) { touched = true return claimedObs(f.host), nil }, Prepare: func(context.Context, registry.Route, hostclaim.Observed, startPolicy) (ClaimStart, error) { touched = true return ClaimStart{}, nil }} res := f.a.Swap(context.Background(), f.attempt(ModeHold), f.newest) if res.Outcome != SwapAbandoned || !strings.Contains(res.Reason, errClaimsNoGenerations.Error()) { t.Fatalf("Swap = %+v, want abandoned: %v", res, errClaimsNoGenerations) } if f.b.lastProc().isStopped() { t.Fatal("the incumbent was stopped by a hold that could never start its pin") } if ev := f.b.events(); indexPrefix(ev, "stop:") >= 0 { t.Fatalf("events %v: the hold began a stop", ev) } if touched { t.Fatal("the claim runtime was read or prepared for a refused hold") } f.a.mu.Lock() e := f.a.entries[f.host] state, held := e.state, e.holdNext f.a.mu.Unlock() if state != actAwake || held != nil { t.Fatalf("entry state %v, holdNext %v: want awake, no hold", state, held) } } // Spec §6.4, §7.4 (index P2-9): a claimed planned wake on this base meets // the closed launch, which refuses before its exec hook. That refusal is the // admission's (claimAdmission), so it is stale, never a failed build: no // pinFailed, no fallback. Prepare and Restore ran once. func TestAClaimedPlannedWakeOnAClosedLaunchRecordsNoFailure(t *testing.T) { obs := claimedObs("x.example") work, _ := (hostwrite.Config{}).Select(context.Background(), obs.Host, "spawn") prepares, restores, failed := 0, 0, 0 a := &Service{cfg: Config{HostWrites: hostwrite.Config{Enabled: true}, Claims: &ClaimRuntime{Box: "box", Read: func(context.Context, string) (hostclaim.Observed, error) { return obs, nil }, Prepare: func(_ context.Context, _ registry.Route, _ hostclaim.Observed, p startPolicy) (ClaimStart, error) { prepares++ s := claimedStartFor(obs, work, p.Pin.Build) s.Restore = func(context.Context) error { restores++; return nil } s.Failed = func(context.Context, int) error { failed++; return nil } return s, nil }}}} e := &actEntry{host: obs.Host} gateCalls := 0 policy := startPolicy{MaxStarts: 1, Window: time.Second, Pin: &Gen{Build: "v2"}, Fallback: &Gen{Build: "v1"}, HealthBy: time.Now().Add(time.Second), Gate: func() error { gateCalls++; return nil }, PinCheck: func(Gen) error { gateCalls++; return nil }} err := a.startAdmitted(context.Background(), e, obs, policy) if !errors.Is(err, ErrClaimIsolationUnavailable) || !errors.Is(err, errClaimsAdmission) { t.Fatalf("startAdmitted = %v, want the closed launch as the admission's refusal", err) } if got := classifyStep(nil, err); got != stepStale { t.Fatalf("classifyStep = %v, want stepStale", got) } if prepares != 1 || restores != 1 { t.Fatalf("Prepare ran %d times, Restore %d: want once each, and no fallback", prepares, restores) } if failed != 1 { t.Fatalf("Failed ran %d times: main's retry accounting is unchanged", failed) } if e.pinFailed != nil || e.pinFailReason != "" { t.Fatalf("pinFailed = %+v (%q): a refused launch was recorded as a failed build", e.pinFailed, e.pinFailReason) } if gateCalls != 0 { t.Fatalf("Gate/PinCheck ran %d times: the closed launch never reaches its exec hook", gateCalls) } if e.pinExec { t.Fatal("pinExec set: nothing reached the exec") } } // Spec §7.4 (Task 2): a failed wake whose generation is not proven stopped // still holds the host's lease. A claims-on wake is refused at its head, as // doWake refuses one, before the claim is even read. func TestAClaimedWakeRefusesWhileTheLastFailedWakeIsUnproven(t *testing.T) { read := false a := &Service{cfg: Config{HostWrites: hostwrite.Config{Enabled: true}, Claims: &ClaimRuntime{Box: "box", Read: func(context.Context, string) (hostclaim.Observed, error) { read = true return claimedObs("x.example"), nil }, StopForeign: func(context.Context, string) error { return nil }}}} a.unprovenWakes = map[string]*unprovenWake{"x.example": {}} e := &actEntry{host: "x.example", route: registry.Route{Host: "x.example", Kind: "instance"}} if _, err := a.doClaimWake(context.Background(), e); !errors.Is(err, errWakeUnproven) { t.Fatalf("doClaimWake = %v, want errWakeUnproven", err) } if read { t.Fatal("the claim was read for a host whose last failed wake is unproven") } } // orderContainment records whether the leader had already exited when // KillAndEmpty ran, then kills and waits like the real one. type orderContainment struct { p *claimProcess exitedAtKill bool } func (c *orderContainment) KillAndEmpty(context.Context) error { c.exitedAtKill = c.p.Exited() if c.p.cmd.Process != nil { _ = c.p.cmd.Process.Kill() } <-c.p.done return nil } func (c *orderContainment) Close() error { return nil } // Spec §7.5 step 6 (index P6 #4): a retiring claims-on generation gets // SIGTERM and its drain before KillAndEmpty, so its in-flight requests // finish. An unready candidate is still killed at once. Stop alone // (cgroup.kill first) killed the retiring generation's requests. func TestARetiringClaimedGenerationDrainsBeforeItsKill(t *testing.T) { if _, err := exec.LookPath("sleep"); err != nil { t.Skip("no sleep") } for _, tc := range []struct { role genRole drained bool }{{roleRetiring, true}, {roleSuccessor, false}} { cmd := exec.Command("sleep", "30") if err := cmd.Start(); err != nil { t.Fatal(err) } p := newClaimProcess(cmd, nil) c := &orderContainment{p: p} p.prepared = c a := &Service{} e := &actEntry{host: "claims.example.com"} if !a.stopMember(e, &genMember{gen: Gen{N: 2}, role: tc.role, proc: p}) { t.Fatalf("%s: stop not proven", tc.role) } if c.exitedAtKill != tc.drained { t.Fatalf("%s: leader exited before KillAndEmpty = %v, want %v", tc.role, c.exitedAtKill, tc.drained) } } } // Final review F9: on this base every claims-backed swap (work that is not // hostwrite.Legacy) is refused before its side effects, whatever the // runtime's Generations and Pins say: no admission inside the work, no // parent raise, no stop of the incumbent, no spawn. The guard opens only // with a working isolated launch, which this base does not have. func TestEveryClaimsBackedSwapIsRefusedBeforeItsSideEffects(t *testing.T) { t.Parallel() runtimes := []struct { name string r *ClaimRuntime }{ {"no runtime", nil}, {"neither", &ClaimRuntime{}}, {"generations only", &ClaimRuntime{Generations: true}}, {"pins only", &ClaimRuntime{Pins: true}}, {"generations and pins", &ClaimRuntime{Generations: true, Pins: true}}, } for _, mode := range []SwapMode{ModeOverlap, ModeHold} { for _, rc := range runtimes { t.Run(string(mode)+"/"+rc.name, func(t *testing.T) { f := newSwapFixture(t) w := &swapWork{} f.a.cfg.HostWrites = hostwrite.Config{Enabled: true, Selector: &swapSelector{work: w}} f.a.cfg.Claims = rc.r res := f.a.Swap(context.Background(), f.attempt(mode), f.newest) if res.Outcome != SwapAbandoned { t.Fatalf("Swap = %+v, want abandoned", res) } if !strings.Contains(res.Reason, errClaimsNoGenerations.Error()) && !strings.Contains(res.Reason, ErrClaimIsolationUnavailable.Error()) { t.Fatalf("Reason %q: want the claims guard's refusal", res.Reason) } gens, raises, restores := f.b.calls() if gens != 0 || raises != 0 || restores != 0 { t.Fatalf("StartStaged %d, raises %d, restores %d: want no side effect", gens, raises, restores) } if f.b.lastProc().isStopped() { t.Fatal("the incumbent was stopped") } if ev := f.b.events(); indexPrefix(ev, "stop:") >= 0 || indexPrefix(ev, "raise:") >= 0 { t.Fatalf("events %v: a refused claims-backed swap acted", ev) } if n := w.withCalls(); n != 0 { t.Fatalf("With calls = %d: a refused swap entered the selected work", n) } f.a.mu.Lock() e := f.a.entries[f.host] state, held, members := e.state, e.holdNext, len(e.ops.members) f.a.mu.Unlock() if state != actAwake || held != nil || members != 0 { t.Fatalf("entry state %v, holdNext %v, members %d: want awake, untouched", state, held, members) } }) } } } // claimSpawn stands in for an open isolated launch, and only tests may set // it: production must always meet claimsSwapRefusal's refusal. No // non-test file of the package assigns the field. func TestClaimSpawnIsATestSeam(t *testing.T) { t.Parallel() fset := token.NewFileSet() pkgs, err := parser.ParseDir(fset, ".", func(fi fs.FileInfo) bool { return !strings.HasSuffix(fi.Name(), "_test.go") }, 0) if err != nil { t.Fatal(err) } for _, pkg := range pkgs { for name, file := range pkg.Files { ast.Inspect(file, func(n ast.Node) bool { switch x := n.(type) { case *ast.AssignStmt: for _, l := range x.Lhs { if sel, ok := l.(*ast.SelectorExpr); ok && sel.Sel.Name == "claimSpawn" { t.Errorf("%s: %s assigns claimSpawn", name, fset.Position(x.Pos())) } } case *ast.KeyValueExpr: if id, ok := x.Key.(*ast.Ident); ok && id.Name == "claimSpawn" { t.Errorf("%s: %s sets claimSpawn in a literal", name, fset.Position(x.Pos())) } } return true }) } } }