package activator import ( "context" "errors" "fmt" "log" "github.com/carlosframework/platform/internal/cgroup" "github.com/carlosframework/platform/internal/instancectl" "github.com/carlosframework/platform/internal/registry" "github.com/carlosframework/platform/internal/schedule" ) var ( errSwapNoGens = errors.New("swap: backend cannot run staged generations") errUnitNoGens = errors.New("swap: unit-backed routes keep stop-then-start") errSwapLayout = errors.New("swap: this row's layout has no generation sockets") ) // SwapBackend is what a swap needs from a backend beyond Backend and plan // 1's GenBackend (spec §7.3–§7.5). ExecBackend implements it on top of plan // 1's generation spawn and plan 2's containment. SplitBackend forwards it // for exec rows and refuses unit rows, which keep stop-then-start (spec §2 // decision 9). type SwapBackend interface { // StartStaged spawns generation g.N of rt from g.StagedPath, never // through the live link, in its own child cgroup at g.Tier, on the // generation's own socket (a foreign row: its canonical socket). It // returns as soon as the process runs, with g's Socket and Control // filled in, and never waits for health: the caller checks health // through that handle under its own deadline, and stops (or kills) a // process that fails. A nil Proc means nothing ran, unless the error // says otherwise (startMayHaveRun; final review wave 2, N4): a start // whose kill is unproven (errGenUnproven), or one handed to root whose // outcome is uncertain (startUncertain). // // ctx may carry the caller's exec gate (withExecGate; final review F1). // An implementation asks it (runExecGate) at its final launch point, // after all of its preparation and immediately before the process or // unit is started, and when it refuses starts nothing, cleans up what // its preparation made, and returns the gate's error as it is with a // nil Proc. StartStaged(ctx context.Context, rt registry.Route, g Gen) (Gen, Proc, error) // GenTier is build's own tier, from its staged manifest (spec §7.3). // The swap captures it into Gen.Tier at admission. GenTier(rt registry.Route, build string) cgroup.Limit // GenReady runs once a pinned wake's generation is healthy: it widens // the socket's mode (issue #103), as the live-link spawn's health tail // does. A swap's successor gets the same from GenSwitched. GenReady(rt registry.Route, g Gen, p Proc) // RaiseGenParent sets the host's cgroup parent to the incumbent's tier // plus the successor's, before the successor spawns (spec §7.3). RaiseGenParent(ctx context.Context, rt registry.Route, incumbent, successor cgroup.Limit) error // RestoreGenParent sets the parent back to tier, the current // generation's captured tier, once every other generation is empty. RestoreGenParent(ctx context.Context, rt registry.Route, tier cgroup.Limit) error // GenSwitched runs under the entry lock at the switch (spec §7.5 step // 5), before the in-memory switch. It points the canonical path at the // new generation's socket (rev 15), purges the host's cached responses // and widens the new socket's mode. An error refuses the switch. GenSwitched(rt registry.Route, g Gen, p Proc) error } // execGateKey carries a staged spawn's exec gate on its context // (withExecGate). type execGateKey struct{} // withExecGate returns ctx carrying gate, the caller's composed ownership // and attempt checks for a staged spawn (final review F1): a planned wake's // B or its fallback, a hold's pinned build, an overlap successor, and every // worker start that has a gate (startWorker; Astra R3 §5 item 4). The // backend asks it at the very end of its preparation, right before the exec // (flat, and a worker) or the root start call (nested), so a lease lost, a // fence set or an attempt superseded while the backend read config, bound // sockets, wrote launch files or prepared the cgroup execs nothing. It rides // on the context, as withBeforeExec does, so StartStaged's and // StartWorkerStaged's signatures and every forwarding backend // (SplitBackend) stay as they are. func withExecGate(ctx context.Context, gate func() error) context.Context { if gate == nil { return ctx } return context.WithValue(ctx, execGateKey{}, gate) } // runExecGate asks ctx's exec gate, if it carries one, then writes ctx's // pending wake record, if it carries one (runPendingRecord, amadan #272), // and, once that write has run, asks the gate again (Astra review of // zdd-owed-items, P1): the lease, a fence or the attempt can change during // the bucket write, and the exec follows the second answer. A spawn with a // gate also honours its context there: one that has ended starts nothing. // With neither (every live-link spawn, the flag off) it is a no-op, so those // spawns are exactly as they were. Every refusal is marked launchRefused: // it is asked immediately before the exec, so nothing of the launch ran. func runExecGate(ctx context.Context) error { if err := askExecGate(ctx); err != nil { return launchRefused{err} } wrote, err := runPendingRecord(ctx) if err != nil { return launchRefused{err} } if wrote { if err := askExecGate(ctx); err != nil { return launchRefused{err} } } return nil } // RunExecGate is runExecGate for a Backend implemented outside this // package: such a backend asks it immediately before its exec, as // ExecBackend does, so the caller's gate and the wake's pending record // (amadan #272) are honoured there. func RunExecGate(ctx context.Context) error { return runExecGate(ctx) } // askExecGate is runExecGate's gate alone. func askExecGate(ctx context.Context) error { gate, ok := ctx.Value(execGateKey{}).(func() error) if !ok || gate == nil { return nil } if err := ctx.Err(); err != nil { return err } return gate() } // launchGate is a flat staged spawn's (an instance generation's or a // worker's) before-start hook for the cgroup // manager (final review F1): launchHook (the test seam), then ctx's exec // gate, asked immediately before each start attempt, after the cgroup's // preparation, then ctx's pending wake record (runPendingRecord), and the // gate again once that write has run. Nil // when ctx carries no gate and no pending record and no test seam is set, so // a live-link spawn hands the manager no hook and starts exactly as it did. func (b *ExecBackend) launchGate(ctx context.Context, host string) func() error { return b.launchGateWith(ctx, host, nil) } // launchGateWith is launchGate with live, a live-link spawn's LiveExecGate // check for the build it resolved (Task 9), asked after ctx's exec gate at // the same point, before the pending record and again after its write. A // nil live is launchGate // exactly, so a spawn on a box with no LiveExecGate hands the manager the // hook it always did. func (b *ExecBackend) launchGateWith(ctx context.Context, host string, live func() error) func() error { if _, ok := ctx.Value(execGateKey{}).(func() error); !ok && b.launchHook == nil && live == nil && pendingRecordFrom(ctx) == nil { return nil } return func() error { if b.launchHook != nil { b.launchHook(host) } if err := askExecGate(ctx); err != nil { return err } if live != nil { if err := live(); err != nil { return err } } wrote, err := runPendingRecord(ctx) if err != nil || !wrote { return err } // The write took time: the gate and the live check are asked again // immediately before the exec (Astra review of zdd-owed-items, P1). if err := askExecGate(ctx); err != nil { return err } if live != nil { return live() } return nil } } // liveExecGateSetter is a backend that honours a live-link exec gate // (ExecBackend.LiveExecGate), or forwards it to one that does // (SplitBackend). type liveExecGateSetter interface { SetLiveExecGate(gate func(rt registry.Route, build string) error) } var ( _ liveExecGateSetter = (*ExecBackend)(nil) _ liveExecGateSetter = (*SplitBackend)(nil) ) // SetLiveExecGate sets LiveExecGate. Call it before the backend spawns // anything, as Service.SetLiveExecGate does. func (b *ExecBackend) SetLiveExecGate(gate func(rt registry.Route, build string) error) { b.LiveExecGate = gate } // SetLiveExecGate forwards to the exec half: unit-backed routes belong to // systemd, and the rollout never holds them (rollout.Managed). func (b *SplitBackend) SetLiveExecGate(gate func(rt registry.Route, build string) error) { if s, ok := b.Exec.(liveExecGateSetter); ok { s.SetLiveExecGate(gate) } } // SetLiveExecGate installs gate as the backend's LiveExecGate and as the // activator's own check at a worker's converge and recycle, which stop the // old child before the backend's spawn could refuse the new one. Only a box // that declined CARLOS_SWAP=overlap and honours failure records or hold-ran // markers calls it (internal/host, honourDeclinedFailures), and before // Start, like Planner. gate is asked under a.mu, so it must answer from // memory, the row and the live link, and never call back into Service. func (a *Service) SetLiveExecGate(gate func(rt registry.Route, build string) error) { a.mu.Lock() a.liveExecGate = gate a.mu.Unlock() if s, ok := a.backend.(liveExecGateSetter); ok { s.SetLiveExecGate(gate) } } // StartStaged is plan 1's StartGen with perGen true, except for the // executable, the tier and the health wait, which is the caller's. A flat // child execs g.StagedPath directly. A nested unit's executable mount names // it, which is Task 9's leg; until then a nested row is refused. A foreign // row keeps its canonical socket (plan 1): it gets one generation at a time // (a cold, pinned or hold start), and Swap never overlaps it. A flat row // whose generation path would not fit a unix socket path is handled the // same way (genLayout, rev 19 rule 4). func (b *ExecBackend) StartStaged(ctx context.Context, rt registry.Route, g Gen) (Gen, Proc, error) { if g.Tier == (cgroup.Limit{}) { g.Tier = b.GenTier(rt, g.Build) } p := b.pathsFor(rt) switch genLayout(p.layout, rt.Host) { case LayoutFlat: sock := b.flatGenSocket(rt.Host, g.N) if err := clearOwnGenSocket(sock); err != nil { return g, nil, fmt.Errorf("%s: generation socket: %w", rt.Host, err) } proc, _, err := b.spawnChildFrom(ctx, rt, sock, g.N, g.StagedPath, g.Tier) if err != nil { return g, proc, err } g.Socket, g.Control = sock, sock+".control" return g, proc, nil case LayoutForeign: // As StartGen does for a canonical-socket spawn: a link left there // by an agent that ran with generations on would fail the app's bind. if err := clearLeftoverCanonicalLink(rt.Socket); err != nil { return g, nil, fmt.Errorf("%s: leftover canonical link: %w", rt.Host, err) } proc, _, err := b.spawnChildFrom(ctx, rt, rt.Socket, g.N, g.StagedPath, g.Tier) if err != nil { return g, proc, err } g.Socket, g.Control = rt.Socket, rt.Socket+".control" return g, proc, nil case LayoutNested: sock := nestedGenSocketIn(p.runDir, g.N) // A start with no staged path (a cold wake) is on the live link, as // the flat arm is. var exe *instancectl.StagedExe if g.StagedPath != "" { exe = &instancectl.StagedExe{Path: g.StagedPath, Version: g.Build, Digest: g.Digest} } proc, _, err := b.spawnUnitFrom(ctx, rt, sock, g.N, exe, g.Tier) if err != nil { return g, proc, err } g.Socket, g.Control = sock, sock+".control" return g, proc, nil case LayoutFenced, LayoutConflict: return g, nil, fmt.Errorf("%s: refusing to spawn: %s", rt.Host, p.reason) } return g, nil, errSwapLayout } // GenTier is build's own tier: TierOf when it is wired (the agent reads // the build's staged manifest), the live tier otherwise. func (b *ExecBackend) GenTier(rt registry.Route, build string) cgroup.Limit { if b.TierOf != nil && build != "" { return b.TierOf(rt, build) } return b.tier(rt) } // GenReady widens a healthy pinned generation's socket mode (issue #103). func (b *ExecBackend) GenReady(rt registry.Route, g Gen, p Proc) { b.normalizeSocketMode(rt, g.Socket, p) } // RaiseGenParent sets the parent to the incumbent's tier plus the // successor's (spec §7.3, plan 2 D3), each captured from its own artifact // at the attempt's admission, through plan 2's ParentLimit. That keeps the // parent's expected-limit record in step. func (b *ExecBackend) RaiseGenParent(ctx context.Context, rt registry.Route, incumbent, successor cgroup.Limit) error { return b.ParentLimit(ctx, rt, cgroup.OverlapLimit(incumbent, successor)) } // RestoreGenParent puts the parent back at the current generation's tier. func (b *ExecBackend) RestoreGenParent(ctx context.Context, rt registry.Route, tier cgroup.Limit) error { return b.ParentLimit(ctx, rt, tier) } // GenSwitched points the canonical socket path at g's socket first, with // plan 1's atomic helper. A failure refuses the switch, because outside // dialers (#103) would otherwise be left on a process that is about to // drain. Then it drops the host's cached responses (plan 1's generation tag // already makes them miss; this reclaims the space) and widens the new // socket to 0660 (issue #103). // // Lock contract (plan 1, Task 4). The caller holds a.mu, the lock under // which plan 1 creates and removes the link, and sets e.gen to g in the // same critical section. So finishStop always removes the link that names // the generation it retires, and one entry's stop can never delete the // next entry's link. func (b *ExecBackend) GenSwitched(rt registry.Route, g Gen, p Proc) error { if !linksCanonical(rt, g) { return errSwapLayout } if err := swapCanonicalLink(rt.Socket, g.Socket); err != nil { return fmt.Errorf("%s: canonical socket link: %w", rt.Host, err) } if b.Cache != nil { if n := b.Cache.PurgeHost(rt.Host); n > 0 { log.Printf("activator: purged %d cached response(s) for %s at the switch to generation %d", n, rt.Host, g.N) } } b.normalizeSocketMode(rt, g.Socket, p) return nil } // genExec picks the exec half for a swap operation. Unit rows refuse, and // so does an exec half that cannot run staged generations. func (b *SplitBackend) genExec(rt registry.Route) (SwapBackend, error) { if rt.Backing == registry.BackingUnit { return nil, errUnitNoGens } g, ok := b.Exec.(SwapBackend) if !ok { return nil, errSwapNoGens } return g, nil } func (b *SplitBackend) StartStaged(ctx context.Context, rt registry.Route, g Gen) (Gen, Proc, error) { gb, err := b.genExec(rt) if err != nil { return g, nil, err } return gb.StartStaged(ctx, rt, g) } func (b *SplitBackend) GenTier(rt registry.Route, build string) cgroup.Limit { gb, err := b.genExec(rt) if err != nil { return cgroup.TierDefault } return gb.GenTier(rt, build) } func (b *SplitBackend) GenReady(rt registry.Route, g Gen, p Proc) { if gb, err := b.genExec(rt); err == nil { gb.GenReady(rt, g, p) } } func (b *SplitBackend) RaiseGenParent(ctx context.Context, rt registry.Route, incumbent, successor cgroup.Limit) error { gb, err := b.genExec(rt) if err != nil { return err } return gb.RaiseGenParent(ctx, rt, incumbent, successor) } func (b *SplitBackend) RestoreGenParent(ctx context.Context, rt registry.Route, tier cgroup.Limit) error { gb, err := b.genExec(rt) if err != nil { return err } return gb.RestoreGenParent(ctx, rt, tier) } func (b *SplitBackend) GenSwitched(rt registry.Route, g Gen, p Proc) error { gb, err := b.genExec(rt) if err != nil { return err } return gb.GenSwitched(rt, g, p) } var errNestedSidecarGen = errors.New("nested sidecar: the generation has no staged executable") // SidecarGenStarter starts a legacy co-spawned sidecar from a generation's // captured staged path (spec §7.5: "Every legacy co-spawned sidecar runs // its instance's serving artifact"). type SidecarGenStarter interface { StartSidecarGen(ctx context.Context, rt registry.Route, g Gen) (Proc, error) } // StartSidecarGen execs g.StagedPath for a flat (or foreign) host. A // nested host's sidecar unit binds what root lays out for it, so the agent // names the generation's staged executable and root validates it. A // generation with none (it was woken on the live link) refuses. It never // reads the live link to decide: a repoint can land between that read and // the start, and after a fallback the live link names a build the instance // is not serving. The supervisor retries with its backoff. func (b *ExecBackend) StartSidecarGen(ctx context.Context, rt registry.Route, g Gen) (Proc, error) { p := b.pathsFor(rt) switch p.layout { case LayoutNested: // The unit binds what root lays out, so root is told the // generation's staged build and validates it (Task 9). Nothing here // reads the live link: a fallback or a repoint cannot change what // this sidecar runs. if g.StagedPath == "" { return nil, errNestedSidecarGen } exe := instancectl.StagedExe{Path: g.StagedPath, Version: g.Build, Digest: g.Digest} return b.startSidecarUnitFrom(ctx, rt, &exe, g.Tier) case LayoutFenced, LayoutConflict: return nil, fmt.Errorf("%s: refusing to start sidecar: %s", rt.Host, p.reason) } return b.startSidecarChildFrom(ctx, rt, g.StagedPath) } // StartSidecarGen forwards to the exec half for exec rows. A unit row keeps // its own StartSidecar. func (b *SplitBackend) StartSidecarGen(ctx context.Context, rt registry.Route, g Gen) (Proc, error) { if rt.Backing != registry.BackingUnit { if s, ok := b.Exec.(SidecarGenStarter); ok { return s.StartSidecarGen(ctx, rt, g) } } return b.pick(rt).StartSidecar(ctx, rt) } // KillSidecar kills whatever is left of the host's legacy sidecar and // returns once its cgroup is empty (zero-downtime spec §7.4, §7.5): the // leader's exit says nothing of a process it forked. Flat with containment: // the role-sidecar child (plan 2 D4). Nested: the sidecar unit's stop, which // returns once systemd has emptied its cgroup (KillMode=mixed sends SIGKILL // to whatever SIGTERM left). Without containment the sidecar shares the // host cgroup with the instance, and the terminal stop's evidence on that // cgroup is the proof (errNoGenContainment, as KillGen answers). // // The role child is kept after the kill, and the next sidecar start // (StartChildGated) removes it and makes it fresh first, or refuses: a // cgroup that took cgroup.kill is never spawned into again (amadan #272, // internal/cgroup/recreate_linux.go). func (b *ExecBackend) KillSidecar(ctx context.Context, rt registry.Route) error { switch p := b.pathsFor(rt); p.layout { case LayoutNested: if b.Control == nil { return fmt.Errorf("%s: no instance control channel to prove the sidecar unit stopped", rt.Host) } return b.Control.Stop(ctx, rt.Host, instancectl.RoleSidecar) case LayoutFenced, LayoutConflict: return fmt.Errorf("%s: refusing to kill a sidecar: %s", rt.Host, p.reason) } if !b.GenContainment { return errNoGenContainment } if b.Cgroups == nil { return cgroup.ErrNoDelegation } c, err := b.Cgroups.OpenCleanupChild(rt.Host, cgroup.RoleChild("sidecar")) if errors.Is(err, cgroup.ErrNoCgroup) { return nil // removed after the leader's exit, so empty } if err != nil { return err } if err := c.KillAndEmpty(ctx); err != nil { _ = c.Close() return err } return c.Close() } // KillSidecar forwards to the exec half for exec rows; systemd owns a unit // row's tree. func (b *SplitBackend) KillSidecar(ctx context.Context, rt registry.Route) error { if rt.Backing == registry.BackingUnit { return nil } if k, ok := b.Exec.(sidecarKiller); ok { return k.KillSidecar(ctx, rt) } return nil } // writeFence is MountGated's fence for the control socket of host's // generation gen. It is nil when the backend has no ScheduleFence seam. func (b *ExecBackend) writeFence(host string, gen uint64) schedule.WriteFence { if b.ScheduleFence == nil { return nil } return func(write func()) bool { return b.ScheduleFence(host, gen, write) } } // genLayouter is a backend whose rows can read differently to generations // than to Classify (genLayout, rev 19 rule 4). The swap's layout decisions // ask it before Classifier (Service.swapLayout). type genLayouter interface { GenLayout(rt registry.Route) Layout } var ( _ genLayouter = (*ExecBackend)(nil) _ genLayouter = (*SplitBackend)(nil) ) // GenLayout is Layout as generations see it: a flat row whose generation // path would not fit a unix socket path reads foreign (genLayout). func (b *ExecBackend) GenLayout(rt registry.Route) Layout { l, _ := b.Layout(rt) return genLayout(l, rt.Host) } // GenLayout forwards to the exec half for exec rows. A unit row reads as // its Layout does. func (b *SplitBackend) GenLayout(rt registry.Route) Layout { if rt.Backing != registry.BackingUnit { if g, ok := b.Exec.(genLayouter); ok { return g.GenLayout(rt) } } l, _ := b.Layout(rt) return l } // genPather is a backend that can name a generation's sockets. type genPather interface { GenPaths(rt registry.Route, n uint64) (Gen, bool) } // GenPaths is the generation socket pair a spawn of generation n of rt // binds, as rev 19 lays them out. It reports false for a row with no // generation path (foreign, fenced, conflict, and a flat row whose // generation path would not fit a unix socket path, genLayout). func (b *ExecBackend) GenPaths(rt registry.Route, n uint64) (Gen, bool) { p := b.pathsFor(rt) var sock string switch genLayout(p.layout, rt.Host) { case LayoutFlat: sock = b.flatGenSocket(rt.Host, n) case LayoutNested: sock = nestedGenSocketIn(p.runDir, n) default: return Gen{}, false } return Gen{N: n, Socket: sock, Control: sock + ".control"}, true } // GenPaths forwards to the exec half for exec rows. A unit row has no // generation path of its own. func (b *SplitBackend) GenPaths(rt registry.Route, n uint64) (Gen, bool) { if rt.Backing == registry.BackingUnit { return Gen{}, false } if gp, ok := b.Exec.(genPather); ok { return gp.GenPaths(rt, n) } return Gen{}, false } // KillHost kills whatever is left in rt's own host cgroup and returns once // it is empty: a worker's termination proof (zero-downtime spec §7.4, §7.9). // It does not look at the row's layout: StartWorker spawns every worker // through spawnInCgroup into carlos-exec@, whatever the layout says, // so a nested-looking worker is proven there too. An absent cgroup is // empty. No cgroup delegation at all is an unproven kill. // // The host cgroup is kept after the kill, and the worker's next start // (StartGated) removes it and makes it fresh first, or refuses: a cgroup // that took cgroup.kill is never spawned into again (amadan #272, // internal/cgroup/recreate_linux.go). That // holds with the flag off too, where only Task 15's F6 path reaches here. func (b *ExecBackend) KillHost(ctx context.Context, rt registry.Route) error { if b.Cgroups == nil { return cgroup.ErrNoDelegation } c, err := b.Cgroups.OpenCleanup(rt.Host) if errors.Is(err, cgroup.ErrNoCgroup) { return nil } if err != nil { return err } if err := c.KillAndEmpty(ctx); err != nil { _ = c.Close() return err } return c.Close() } // KillHost dispatches like Start: a unit-backed row's one process tree is // systemd's, so there is no host cgroup of ours to kill. func (b *SplitBackend) KillHost(ctx context.Context, rt registry.Route) error { if rt.Backing == registry.BackingUnit { return nil // systemd owns the unit tenant's tree } if k, ok := b.Exec.(hostKiller); ok { return k.KillHost(ctx, rt) } return nil } // EmptyHost proves rt's own host cgroup empty for a flag-off terminal stop // (Task 8, F6): it reads cgroup.events and kills only when something is // left, so a clean stop leaves no kill mark and nothing for the next start // to recreate. A populated cgroup goes through KillAndEmpty, which the // next start answers by making it fresh (Task 19). An absent cgroup is // empty; no delegated cgroup at all is cgroup.ErrNoDelegation, which the // caller treats as the leader-only proof. So is a cgroup with no // cgroup.kill (cgroup.ErrNoKill, a kernel before 5.14): nothing could kill // a descendant there, and reading it as an unproven kill would hold every // flag-off stop's lease on such a box. func (b *ExecBackend) EmptyHost(ctx context.Context, rt registry.Route) error { open := b.openHostCleanup if open == nil { if b.Cgroups == nil { return cgroup.ErrNoDelegation } open = b.Cgroups.OpenCleanup } c, err := open(rt.Host) if errors.Is(err, cgroup.ErrNoCgroup) { return nil } if errors.Is(err, cgroup.ErrNoKill) { log.Printf("activator: %s: this kernel has no cgroup.kill, so the host cgroup cannot be killed; its leader's exit is the only proof there is", rt.Host) return fmt.Errorf("%w: %w", cgroup.ErrNoDelegation, err) } if err != nil { return err } populated, err := c.Populated() if err == nil && populated { err = c.KillAndEmpty(ctx) } if cerr := c.Close(); err == nil { err = cerr } return err } // EmptyHost dispatches like KillHost: a unit-backed row has no host cgroup // of ours. An exec side without the read-first proof gets KillHost. func (b *SplitBackend) EmptyHost(ctx context.Context, rt registry.Route) error { if rt.Backing == registry.BackingUnit { return nil // systemd owns the unit tenant's tree } if e, ok := b.Exec.(hostEmptier); ok { return e.EmptyHost(ctx, rt) } if k, ok := b.Exec.(hostKiller); ok { return k.KillHost(ctx, rt) } return nil }