package activator import ( "context" "errors" "fmt" "os/exec" "github.com/carlosframework/platform/internal/cgroup" "github.com/carlosframework/platform/internal/instancectl" "github.com/carlosframework/platform/internal/registry" ) var ( errNoGenContainment = errors.New("activator: generation containment is off") errNoGenControl = errors.New("activator: the instance control channel cannot run generation units") // errGenUnproven is a generation start that failed and whose kill did // not prove the generation gone. It may still be running: its listeners // and launch file are kept, and a caller must keep whatever it owns for // the host (a later plan's swap keeps the instance's ownership). errGenUnproven = errors.New("activator: generation start failed and its termination is unproven") ) // startUncertain marks a start's error when the start was handed to // something that may have run it before failing (final review wave 2, N4): // root's instance-control start call answered an error other than // instancectl.ErrStagedExe, or no invocation. Root may have started the // unit, and the build may have run, whatever became of it after. The // wrapper changes neither the text nor errors.Is, so every existing // classification and member text is the same; only startMayHaveRun reads // it. type startUncertain struct{ err error } func (e startUncertain) Error() string { return e.err.Error() } func (e startUncertain) Unwrap() error { return e.err } // markStartUncertain wraps err as startUncertain. func markStartUncertain(err error) error { return startUncertain{err: err} } // legacyStartUncertain marks a failed start of a nested host's original // run unit (generation 0, carlos-instance@) that root was asked for and may // still carry out (#292): cancelling the client's /start does not cancel a // start job systemd has queued, and that job can launch a writer later. // The failed wake's cleanup must stop the unit and prove it gone // (legacyRunProver) before the lease goes; unproven, the lease is kept. // Like startUncertain it changes neither the text nor errors.Is, so the // member's wording is unchanged. type legacyStartUncertain struct{ err error } func (e legacyStartUncertain) Error() string { return e.err.Error() } func (e legacyStartUncertain) Unwrap() error { return e.err } // markLegacyStartUncertain wraps err as legacyStartUncertain. func markLegacyStartUncertain(err error) error { return legacyStartUncertain{err: err} } // launchRefused marks a start that its pre-exec gate refused // (runExecGate): the caller's exec gate, the wake's pending record or the // recheck after it answered no, immediately before the exec or before root // was asked. Nothing of that launch ran, so a failed wake whose start comes // back marked is known not started, not uncertain (seeded pre-land review, // P2). Like startUncertain it changes neither the text nor errors.Is. type launchRefused struct{ err error } func (e launchRefused) Error() string { return e.err.Error() } func (e launchRefused) Unwrap() error { return e.err } // launchNotStarted reports whether err is a start its pre-exec gate // refused (launchRefused), and nothing marks it as one that may have run. func launchNotStarted(err error) bool { var r launchRefused return errors.As(err, &r) && !legacyStartMayRun(err) && !startMayHaveRun(nil, err) } // legacyStartMayRun reports whether err is a legacy run unit's start whose // queued job may still run (legacyStartUncertain). func legacyStartMayRun(err error) bool { var l legacyStartUncertain return errors.As(err, &l) } // legacyRunProver is the backend half of that proof: stop the host's // original run unit through instance-control and prove it gone (stop // answered, unit inactive, its cgroup empty). nil is the proof. // ExecBackend has it; SplitBackend passes it to its exec side. type legacyRunProver interface { LegacyRunGone(ctx context.Context, rt registry.Route) error } // errNoLegacyProof: the backend cannot stop a legacy run unit, so nothing // proves an aborted start gone and its lease is kept. var errNoLegacyProof = errors.New("activator: this backend cannot prove a run unit stopped") // legacyStopBudget bounds one legacy proof: the stop call and the wait for // the unit to read stopped, on a context of its own (a cancelled wake's // context is often why the start was aborted). const legacyStopBudget = instancectl.DeadlineControl // legacyRunGone runs the backend's proof for rt under legacyStopBudget and // never past Shutdown's own budget (Astra, rounds 2 and 3): a proof started // once Shutdown has begun runs under its context, and every proof is // registered so that one already running when Shutdown began is cancelled // the moment that budget ends (cancelProofs). A proof cut short is // unproven, so the lease is kept. func (a *Service) legacyRunGone(rt registry.Route) error { p, ok := a.backend.(legacyRunProver) if !ok { return errNoLegacyProof } parent := context.Background() token := new(byte) a.mu.Lock() if a.shutdownCtx != nil { parent = a.shutdownCtx } ctx, cancel := context.WithTimeout(parent, legacyStopBudget) if a.proofCancels == nil { a.proofCancels = make(map[*byte]context.CancelFunc) } a.proofCancels[token] = cancel a.mu.Unlock() defer func() { a.mu.Lock() delete(a.proofCancels, token) a.mu.Unlock() cancel() }() return p.LegacyRunGone(ctx, rt) } // cancelProofs cancels every legacy proof still running: Shutdown arms it // on its own context, so none outlives the shutdown budget. func (a *Service) cancelProofs() { a.mu.Lock() defer a.mu.Unlock() for _, cancel := range a.proofCancels { cancel() } } // errStartedThenExited is the one uncertain start whose outcome is known to // be a crash at the start: root started the unit and it had exited before // root read its invocation (Astra R3 ruling 206, zdd-readiness Task 6). The // build ran and died on its own, so a planned wake fails the build in // overlap mode too, where any other uncertain start is only "may have run". // It is wrapped by markStartUncertain, so startMayHaveRun still reads it. var errStartedThenExited = errors.New("generation exited before it was observed") // exitedUnobserved is errStartedThenExited in the unit's own words. Its // text is exactly the one spawnUnitFrom always wrote, because a flag-off // nested wake fails with it and memberSafeWakeReason passes it to the // member; only errors.Is learns the sentinel. type exitedUnobserved struct{ unit string } func (e exitedUnobserved) Error() string { return e.unit + " exited before it was observed" } func (e exitedUnobserved) Is(target error) bool { return target == errStartedThenExited } // startMayHaveRun is a spawn's execution evidence, tri-state collapsed to // what an attempt may rely on (final review wave 2, N4): false only when it // is PROVEN that nothing of the build ran (no process came back, and the // error neither says the start's kill is unproven nor that the start was // handed off with an uncertain outcome); true when it ran or may have. A // flat start that failed is proven: the exec never happened. Every refusal // before the start (a gate, a preparation step, root's ErrStagedExe) is // proven too. func startMayHaveRun(proc Proc, err error) bool { if proc != nil || genStartUnproven(err) { return true } var u startUncertain return errors.As(err, &u) } // spawnKind is which child shape a spawn is, for placeFor. type spawnKind int const ( spawnInstance spawnKind = iota spawnSidecar // the legacy co-spawned sidecar (-sidecar.env) spawnSeed spawnWorker // workers and native sidecar members (StartWorker) ) // placeFor names the child cgroup a spawn lands in under the generation // layout (zero-downtime spec §7.3, plan 2 D4): g with the tier's full // limits for an instance generation; role-sidecar and role-seed, with no // limit of their own, for the legacy co-spawned sidecar and the seed, which // share the instance's host cgroup and so cannot sit in a parent that holds // no processes. Workers keep one cgroup per host because they never overlap // (§7.9). "" (today's single-level cgroup) for every kind while // GenContainment is off. func (b *ExecBackend) placeFor(kind spawnKind, n uint64) (child string, ownTier bool, err error) { if !b.GenContainment { return "", false, nil } switch kind { case spawnInstance: if n == 0 { return "", false, errors.New("activator: generation 0 is not a generation") } return cgroup.GenChild(n), true, nil case spawnSidecar: return cgroup.RoleChild("sidecar"), false, nil case spawnSeed: return cgroup.RoleChild("seed"), false, nil } return "", false, nil } // spawnInCgroupAt is spawnInCgroupAtTier with the live tier: the spawn of a // placed child whose artifact is the one the live link names. func (b *ExecBackend) spawnInCgroupAt(rt registry.Route, child string, ownTier bool, build func() *exec.Cmd) (*exec.Cmd, func(), error) { return b.spawnInCgroupAtTier(rt, child, ownTier, cgroup.Limit{}, build, nil) } // spawnInCgroupAtTier is spawnInCgroup for a placed child, with the // artifact's tier named: the generation child's own limit, and the parent's // floor for StartChild (which never lowers a parent a swap has raised). Zero // tier is b.tier(rt). With no child it IS spawnInCgroup at that tier, so a // flag-off spawn with a zero tier is byte for byte today's, and a recorded // build's (ruling R-TIER) caps the host cgroup at its own tier. // // A placed spawn is never strict: strict containment of a generation layout // is the claim path's (PrepareStrictParent, plan 5). It still honours the // host gate, though, so a placed spawn can never start beside a strict // owner's group, and a host that is strict is refused rather than started // outside its strict group. The returned settle is spawnInCgroup's: a no-op // here. A spawn that fails removes the child it may have created. // // before, when set, is the cgroup manager's before-start hook (final review // F1): asked immediately before each start attempt, after the parent and // child are prepared, and on the PID-move retry too. A refusal starts // nothing, is cleaned up as a failed start is, and comes back as before's // own error. Nil is the spawn exactly as it was. func (b *ExecBackend) spawnInCgroupAtTier(rt registry.Route, child string, ownTier bool, tier cgroup.Limit, build func() *exec.Cmd, before func() error) (*exec.Cmd, func(), error) { if b.Cfg.HostWrites.Enabled { return nil, nil, ErrClaimIsolationUnavailable } if child == "" { return b.spawnInCgroupGated(rt, tier, build, before) } start := func() (*exec.Cmd, func(), error) { t := tier if t == (cgroup.Limit{}) { t = b.tier(rt) } own := cgroup.Limit{} if ownTier { own = t } cmd, err := b.Cgroups.StartChildGated(rt.Host, child, t, own, build, before) if err != nil { b.Cgroups.RemoveChild(rt.Host, child) return nil, nil, err } return cmd, func() {}, nil } if b.Strict == nil { return start() } g := b.gate(rt.Host) g.mu.Lock() defer g.mu.Unlock() if b.Strict(rt.Host) { return nil, nil, fmt.Errorf("%s: %w: a generation layout is not started strict here", rt.Host, ErrStrictRequired) } if o := g.owner; o != nil { cause := ErrStrictGroupInUse if o.pending { cause = ErrStrictCleanupPending } return nil, nil, fmt.Errorf("%s: %w", rt.Host, cause) } return start() } // genControl is Control's generation half, or errNoGenControl. func (b *ExecBackend) genControl() (GenControl, error) { if b.Control == nil { return nil, errNoGenControl } gc, ok := b.Control.(GenControl) if !ok { return nil, errNoGenControl } return gc, nil } // errNoStagedExeControl refuses a start or seed of an original template // that names a staged build when the control channel cannot name one. // Binding the live link instead would run a build nobody recorded. var errNoStagedExeControl = errors.New("activator: the instance control channel cannot name an executable for this unit") // stagedExeControl is Control's original-template executable half, or // errNoStagedExeControl. func (b *ExecBackend) stagedExeControl() (StagedExeControl, error) { if b.Control == nil { return nil, errNoStagedExeControl } xc, ok := b.Control.(StagedExeControl) if !ok { return nil, errNoStagedExeControl } return xc, nil } // unitGen is the generation a nested run unit is started as: 0 (the // original template) while GenContainment is off, n otherwise. n of 0 // with containment on is refused. func (b *ExecBackend) unitGen(n uint64) (uint64, error) { if !b.GenContainment { return 0, nil } if n == 0 { return 0, errors.New("activator: generation 0 is not a generation") } return n, nil } // startRole starts one role's unit and returns its invocation: the // original template through Control, a generation through GenControl. exe // names a staged build (nil: the live link, plan 2's StartGen), and tier // is that build's own (zero: b.tier(rt)). // // At generation 0 a named exe is the build a cold wake recorded (settings // spec §4.2), started through StagedExeControl. A channel without it // refuses the start: dropping exe would bind the live link, a build // nobody recorded. func (b *ExecBackend) startRole(ctx context.Context, rt registry.Route, role instancectl.Role, gen uint64, exe *instancectl.StagedExe, tier cgroup.Limit) (string, error) { if b.Cfg.HostWrites.Enabled { return "", ErrClaimIsolationUnavailable } if gen == 0 { if exe == nil { return b.Control.Start(ctx, rt.Host, role, b.tier(rt).MemoryMax) } xc, err := b.stagedExeControl() if err != nil { return "", fmt.Errorf("%s: not started on %s: %w", rt.Host, exe.Version, err) } if tier == (cgroup.Limit{}) { tier = b.tier(rt) } return xc.StartExe(ctx, rt.Host, role, tier.MemoryMax, *exe) } gc, err := b.genControl() if err != nil { return "", err } if tier == (cgroup.Limit{}) { tier = b.tier(rt) } if exe != nil { // The staged artifact's own tier, not the live link's. return gc.StartGenExe(ctx, rt.Host, gen, tier.MemoryMax, *exe) } return gc.StartGen(ctx, rt.Host, gen, tier.MemoryMax) } // ParentLimit sets rt's generation parent to l: carlos-exec@ for a // flat or foreign host, the host slice for a nested one (spec §7.3). The // swap raises it to cgroup.OverlapLimit before the successor spawns and // restores the tier once the retiring generation's child is empty. With // GenContainment off there is no parent, and it refuses, so a caller can // never believe it raised a limit that does not exist. func (b *ExecBackend) ParentLimit(ctx context.Context, rt registry.Route, l cgroup.Limit) error { if !b.GenContainment { return errNoGenContainment } switch p := b.pathsFor(rt); p.layout { case LayoutNested: gc, err := b.genControl() if err != nil { return err } return gc.Parent(ctx, rt.Host, l.MemoryMax) case LayoutFenced, LayoutConflict: return fmt.Errorf("%s: refusing to set a parent limit: %s", rt.Host, p.reason) } // The manager's SetParentLimit answers nil with no manager; here that // would admit an overlap with no parent raised and no generation child // to kill, so it refuses. if b.Cgroups == nil { return cgroup.ErrNoDelegation } return b.Cgroups.SetParentLimit(rt.Host, l) } // KillGen kills generation n outright and returns once its child is empty // (spec §7.3, §8.2 step 1). It acts on that generation's child only. func (b *ExecBackend) KillGen(ctx context.Context, rt registry.Route, n uint64) error { if !b.GenContainment || n == 0 { return errNoGenContainment } switch p := b.pathsFor(rt); p.layout { case LayoutNested: gc, err := b.genControl() if err != nil { return err } return gc.StopGen(ctx, rt.Host, n, true) case LayoutFenced, LayoutConflict: return fmt.Errorf("%s: refusing to kill generation %d: %s", rt.Host, n, p.reason) } if b.Cgroups == nil { return cgroup.ErrNoDelegation } child := cgroup.GenChild(n) if err := b.killChild(ctx, rt.Host, child); err != nil { return err } b.Cgroups.RemoveChild(rt.Host, child) return nil } // killChild kills whatever is in one child of host's generation parent and // returns once that child is empty. It touches no sibling. An absent child // is empty; no delegated cgroup at all is cgroup.ErrNoDelegation. The child // is kept: the caller removes it (RemoveChild) once it is done with it. func (b *ExecBackend) killChild(ctx context.Context, host, child string) error { c, err := b.Cgroups.OpenCleanupChild(host, child) if errors.Is(err, cgroup.ErrNoCgroup) { return nil // already removed, so already empty } if err != nil { return err } if err := c.KillAndEmpty(ctx); err != nil { _ = c.Close() return err } return c.Close() }