// Package cgroup gives each exec-backed child its own cgroup under the // subtree systemd delegates to carlos-edge.service (Delegate= in the unit // file), so the metrics sampler can report per-instance figures for routes // that run as exec children of the agent — today those live inside the // agent's own cgroup and can never show readings. Design: // docs/superpowers/specs/2026-08-07-exec-child-accounting-design.md. Since // the memory-pressure spec (2026-08-28) the same per-host cgroup also // carries the instance's memory cap (Limit, the tiers). // // The agent is NOT root: everything here works through ownership of the // delegated subtree, never privilege. When delegation is absent (old unit // file, a test, darwin) Setup returns a nil *Manager and every method on it // degrades to today's behavior — fail toward absent readings, never toward // a broken spawn path. The generation layout (StartChildGated, armed only // under CARLOS_SWAP) is the exception once a manager exists: a generation // whose limits cannot be applied is refused (ErrLimitNotApplied), never run // uncapped. package cgroup import ( "context" "errors" "fmt" "os" "os/exec" "path/filepath" "strconv" "strings" "sync" "sync/atomic" "testing" ) // execPrefix is the ONE definition of the per-host exec cgroup naming — // activator.ExecName and the sampler's directory match both come through // this package. const execPrefix = "carlos-exec@" // ExecName is the cgroup directory name — and therefore the metrics sample // key — for an exec-backed host's children. Deliberately NOT dressed up as // a fake ".service": an operator who takes a dashboard row to `systemctl // status` must not be told a unit exists that doesn't. func ExecName(host string) string { return execPrefix + host } // IsExecName reports whether a cgroup directory name is one of our per-host // exec cgroups: the carlos-exec@ prefix with a non-empty host after it. func IsExecName(name string) bool { return strings.HasPrefix(name, execPrefix) && len(name) > len(execPrefix) } // The claim containment groups' host names: claim-.local, where id is // the hex digest internal/host derives from the claim's fence target. This // is the ONE definition of the format (final review round 1, merge item 8): // internal/host builds names with ClaimContainmentName, and the retaining // setup (SetupRetaining), which runs flag off too, recognises them with // IsClaimContainment, so the two cannot drift apart. const ( claimContainmentPrefix = "claim-" claimContainmentSuffix = ".local" ) // ClaimContainmentName is the containment group host name for a claim whose // identity digest is id (hex). func ClaimContainmentName(id string) string { return claimContainmentPrefix + id + claimContainmentSuffix } // IsClaimContainment reports whether name is a claim containment group's: // its host name (ClaimContainmentName) or its exec cgroup's directory name // (ExecName of one). func IsClaimContainment(name string) bool { host := strings.TrimPrefix(name, execPrefix) return strings.HasPrefix(host, claimContainmentPrefix) && strings.HasSuffix(host, claimContainmentSuffix) && len(host) > len(claimContainmentPrefix)+len(claimContainmentSuffix) } // Limit is what Start writes into a host's cgroup before the child lands // in it. MemoryMax is bytes; zero means "write nothing" — the pre-spec // behaviour for a caller with no tier. memory.high is derived (90% of // max) rather than a second field: a tenant that is throttled before it is // killed is the whole point, and there is no honest reason for a caller to // pick a different band. type Limit struct { MemoryMax int64 } // The tiers are PLATFORM-chosen (memory-pressure spec §2.3), mirroring // what caps/render.go already writes for unit-backed apps: the carlos-app@ // template's 512M, and the 1G a forking family gets when its manifest // declares `children`. No number comes from a manifest. var ( TierDefault = Limit{MemoryMax: 512 << 20} TierChildren = Limit{MemoryMax: 1 << 30} ) // Shares is the CPU and IO half of a tier (#214 item 1). Memory alone let // one tenant take every core, or the disk's whole queue, from every other // tenant on the box. Like the memory caps these are PLATFORM-chosen: the // tier a route gets is decided by its manifest's one bit (children), and // its shares follow from the tier, so nothing new travels on the wire and // no number comes from a manifest. // // The weights are the kernel's default, written explicitly: a weight is // what makes the per-host slices take turns under contention, and setting // one is also what turns the controller on for the subtree. The quota is // the hard ceiling, in percent of one CPU. IOWeight is honoured only where // the disk has a weight-aware controller (io.cost or BFQ); the probe // reports whether this box has one. type Shares struct { CPUWeight int CPUQuotaPercent int IOWeight int } // SharesOf is the tier's shares. A forking family gets two CPUs' worth of // quota, as it gets twice the memory; anything else, including a limit // that is not a tier, gets one: the smaller, and so the safe direction. func SharesOf(l Limit) Shares { if l == TierChildren { return Shares{CPUWeight: 100, CPUQuotaPercent: 200, IOWeight: 100} } return Shares{CPUWeight: 100, CPUQuotaPercent: 100, IOWeight: 100} } // HighOf is the memory.high for a memory.max: 90 percent, computed as the // cgroup manager always has, so a unit tier and an exec-child tier are the // same number. func HighOf(max int64) int64 { return max / 10 * 9 } // ExecOOMScoreAdj is written to every exec child's /proc//oom_score_adj // right after it starts. The kernel's OOM killer — the memcg one that fires // when carlos-edge.service's own MemoryMax is hit included — scores per // PROCESS by RSS, and the agent (~110 MB) outweighs any single tenant // process on the flagship today, so without this the outer wall's victim // would be the agent, and Restart=always would then cycle every awake // instance: precisely what OOMPolicy=continue exists to prevent. +500 adds // half the box's RAM to a child's score; the agent stays at 0 and is never // the first pick. Raising one's own-uid process's score needs no privilege // (lowering does), which is why this is done here rather than by a unit // directive. Grandchildren inherit the value at fork; a family that forks // in the microseconds between exec and this write keeps 0 for those early // forks — the safe direction is still "a tenant process before the agent" // for everything forked after. const ExecOOMScoreAdj = "500" // oomScoreAdjPath is a seam so a test can point the write at a temp dir. var oomScoreAdjPath = func(pid int) string { return filepath.Join("/proc", strconv.Itoa(pid), "oom_score_adj") } // ParseSelfCgroup extracts the cgroup v2 path from /proc/self/cgroup // content — the "0::" unified-hierarchy line — relative (no leading slash), // ready to join under /sys/fs/cgroup. Not-ok when there is no v2 line (a // v1-only box) or the process sits in the root cgroup: neither is a subtree // systemd could have delegated to us. func ParseSelfCgroup(data string) (string, bool) { for _, line := range strings.Split(data, "\n") { rest, found := strings.CutPrefix(line, "0::") if !found { continue } p := strings.TrimPrefix(rest, "/") if p == "" { return "", false } return p, true } return "", false } // Manager owns the delegated subtree. A nil *Manager is the documented // "no delegation" state: every method is nil-safe and degrades to today's // behavior. type Manager struct { root string logf func(string, ...any) // locks holds each host's start lock (hostLock), made on first use. locksMu sync.Mutex locks map[string]*sync.Mutex // placing is the in-flight registry (inflight.go): placements by // cleaned target directory. placeWake is closed and dropped whenever a // count drops or a hold ends, so waitPlacementsUnder and a registration // waiting on a hold wake and recheck. windows are the readQuiet reads in // progress; holds, the directories no placement may register under. placeMu sync.Mutex placing map[string]int placeWake chan struct{} windows map[*placeWindow]struct{} holds map[string]int } // hostLock is host's start lock. A start holds it while it checks for, and // removes, a cgroup of the host that took cgroup.kill (freshen, amadan // #272), so two starts of one host never both remove one. It is the // innermost lock: nothing is taken while it is held, and it is never held // across a before-start hook or a start. func (m *Manager) hostLock(host string) *sync.Mutex { m.locksMu.Lock() defer m.locksMu.Unlock() if m.locks == nil { m.locks = make(map[string]*sync.Mutex) } mu, ok := m.locks[host] if !ok { mu = &sync.Mutex{} m.locks[host] = mu } return mu } // Root is the delegated subtree's path — the sampler's third scan root. // "" for a nil manager (no delegation): the sampler skips an empty root. func (m *Manager) Root() string { if m == nil { return "" } return m.root } // Setup arms exec-child accounting for the running agent: resolve our own // cgroup from /proc/self/cgroup and run the delegation dance there. Any // failure — v1-only box, old unit file without Delegate=, a test, darwin — // returns nil with one logged line, and every spawn behaves exactly as // today. Fail toward absent readings, never toward a broken spawn path. func Setup(logf func(string, ...any)) *Manager { return setup(logf, false) } func SetupRetaining(logf func(string, ...any)) *Manager { return setup(logf, true) } func setup(logf func(string, ...any), retain bool) *Manager { b, err := os.ReadFile("/proc/self/cgroup") if err != nil { logf("agent: exec-child accounting disabled: %v", err) return nil } rel, ok := ParseSelfCgroup(string(b)) if !ok { logf("agent: exec-child accounting disabled: no cgroup v2 path in /proc/self/cgroup") return nil } m, err := setupAt(filepath.Join("/sys/fs/cgroup", rel), logf, retain) if err != nil { logf("agent: exec-child accounting disabled (no delegated subtree — unit file without Delegate=?): %v", err) return nil } logf("agent: exec-child accounting armed (delegated subtree %s)", m.Root()) return m } // SetupAt is the delegation dance against an explicit subtree root — the // testable core Setup wraps. Order matters: probe, move self to a leaf, THEN // enable controllers — cgroup v2 refuses subtree_control writes while the // cgroup itself still has member processes ("no internal processes"). func SetupAt(root string, logf func(string, ...any)) (*Manager, error) { return setupAt(root, logf, false) } func SetupAtRetaining(root string, logf func(string, ...any)) (*Manager, error) { return setupAt(root, logf, true) } func setupAt(root string, logf func(string, ...any), retain bool) (*Manager, error) { // 1. Probe live delegation: both control files must be writable by us. // systemd chowns exactly these to User= when Delegate= is set; without // it this open is the cheap, side-effect-free "no" that keeps an // undelegated box on today's path. for _, name := range []string{"cgroup.procs", "cgroup.subtree_control"} { f, err := os.OpenFile(filepath.Join(root, name), os.O_WRONLY, 0) if err != nil { return nil, fmt.Errorf("probe delegation: %w", err) } f.Close() } // 2. Move ourselves into a supervise/ leaf, making step 3 legal. sup := filepath.Join(root, "supervise") if err := os.MkdirAll(sup, 0o755); err != nil { return nil, fmt.Errorf("create supervise leaf: %w", err) } if err := os.WriteFile(filepath.Join(sup, "cgroup.procs"), []byte(strconv.Itoa(os.Getpid())), 0o644); err != nil { return nil, fmt.Errorf("move self into supervise leaf: %w", err) } // 3. Enable the controllers per-child cgroups need for memory.current, // io.stat and pids accounting (cpu.stat exists regardless, but +cpu // keeps utilization attributable). One write, all four: the kernel // applies subtree_control writes atomically. if err := os.WriteFile(filepath.Join(root, "cgroup.subtree_control"), []byte("+cpu +memory +io +pids"), 0o644); err != nil { return nil, fmt.Errorf("enable controllers: %w", err) } m := &Manager{root: root, logf: logf} // 4. Sweep per-host dirs a previous agent left behind — empty after a // restart killed their members. rmdir refuses a non-empty dir, which is // exactly the tolerance we want. // A retaining setup keeps them, and marks the empty ones instead, so a // cgroup an earlier run killed and kept is made fresh before anything // spawns into it (amadan #272, seedKilled). if !retain { m.sweepStale() } else { m.seedKilled() } return m, nil } // sweepStale removes leftover carlos-exec@* dirs under the root. Failures // are logged and tolerated: a dir that won't rmdir has something alive in // it, and killing is the service manager's job, not ours. func (m *Manager) sweepStale() { entries, err := os.ReadDir(m.root) if err != nil { return } for _, e := range entries { if !e.IsDir() || !IsExecName(e.Name()) { continue } dir := filepath.Join(m.root, e.Name()) // A generation parent's children go first: rmdir refuses a cgroup // that has child cgroups, empty or not (zero-downtime spec §7.3). if kids, err := os.ReadDir(dir); err == nil { for _, k := range kids { if !k.IsDir() || !isChildName(k.Name()) { continue } if err := os.Remove(filepath.Join(dir, k.Name())); err != nil { m.logf("cgroup: stale %s/%s not removed: %v", e.Name(), k.Name(), err) } } } if err := os.Remove(dir); err != nil { m.logf("cgroup: stale %s not removed: %v", e.Name(), err) } } } // ErrKilledCgroupKept is a start refused because the cgroup it would spawn // into took cgroup.kill and could not be removed and made fresh first // (amadan #272). Spawning into it anyway would have the kernel SIGKILL the // child at once, so nothing is started. var ErrKilledCgroupKept = errors.New("cgroup: a cgroup that took cgroup.kill could not be made fresh before a spawn") // startCmd is the package's one spawn seam: every exec.Cmd.Start, the // shim's own included, goes through it, so a test can fake a start without // a process. var startCmd = func(cmd *exec.Cmd) error { return cmd.Start() } // placeStart is startIn's placement: the exec-shim (startViaShim) puts the // process in dir before any of its code runs. FakePlacementForTest swaps // it for a test whose cgroupfs is a temp directory. var placeStart = func(m *Manager, dir string, cmd *exec.Cmd, before func() error) (bool, error) { return m.startViaShim(dir, cmd, before) } // hostDir is the per-host cgroup: instance, sidecar and seed all share it — // a seed's first-boot cost and a sidecar's load ARE that instance's work. func (m *Manager) hostDir(host string) string { return filepath.Join(m.root, ExecName(host)) } // Start runs build() and starts the result inside host's cgroup, with // limit applied to that cgroup first (memory-pressure spec §2.3). Nil-safe: // with no manager (or any accounting failure) the child starts exactly as // today — a spawn must never fail because accounting failed. build must // return a fully configured, unstarted Cmd; it may be called twice, because // a Cmd cannot be reused after a failed Start. On a non-nil error the // returned Cmd may be nil. // // Primary mechanism: the placement exec-shim (startViaShim, amadan #272). // The child starts as this binary, writes its own PID into the cgroup, // and only then execs the command, so none of the command's code runs in // the wrong cgroup. clone3 CLONE_INTO_CGROUP did this until the shipped // unit's RestrictNamespaces=yes made clone3 answer ENOSYS. If the shim // proves it could not place the child before exec (a placement error), // this one spawn falls back to starting the command normally and moving it // by PID; the next Start still uses the shim. The window between start and // move charges the child's first moments to the agent: mis-accounting in // the safe direction. Note the PID write migrates only the child process // itself — any grandchildren it has already forked by then stay charged to // the agent's cgroup, not the child's. A failure that is the command's own // (a missing binary, a bad Dir) is returned as is and never retried. func (m *Manager) Start(host string, limit Limit, build func() *exec.Cmd) (*exec.Cmd, error) { return m.StartGated(host, limit, build, nil) } // StartGated is Start with a before-start hook. before, when set, is asked // immediately before EACH exec of the command, after the cgroup is made and // its limits are written (zero-downtime deploys final review F1: the // caller's ownership can change during that preparation). Through the shim // that is after the child is placed and before it execs (Ruling 13); on // the PID-move fallback it is before the start. A refusal runs nothing of // the command (a placed shim is killed and reaped), undoes that attempt's // preparation as a failed start does (the cgroup dir this spawn would have // used is removed if empty), and comes back as before's own error, never // joined with an earlier attempt's. A nil before is Start exactly. // // A host cgroup that took cgroup.kill (KillHost keeps it) is removed and // made again before anything is spawned into it, and a start whose killed // cgroup will not go is refused with ErrKilledCgroupKept, never run // unaccounted (amadan #272, freshen). func (m *Manager) StartGated(host string, limit Limit, build func() *exec.Cmd, before func() error) (*exec.Cmd, error) { if m == nil { return startPlain(build, before) } if err := m.freshen(host, m.hostDir(host)); err != nil { return nil, err } return m.startIn(host, m.hostDir(host), limit, false, build, before) } // FakePlacementForTest is for another package's tests over a fake cgroupfs: // a temp directory is no cgroup, so the placement shim cannot place a // process there, and a generation start (StartChildGated) is never retried // by start-and-move. Until t's cleanup, every start that would go through // the shim instead starts the command directly behind its before hook, as // if it had been placed, and the shim counts as available even in a test // binary whose TestMain has no RunShimIfRequested. That is safe because the // fake never runs a shim, so it cannot make a test binary re-run itself. // Both are restored at t's cleanup. t is a *testing.T. It panics outside a // test binary. It swaps package globals, so a test that calls it, and every // other test in that binary, must not use t.Parallel. func FakePlacementForTest(t interface { Helper() Cleanup(func()) }) { if !testing.Testing() { panic("cgroup: FakePlacementForTest called outside a test") } t.Helper() orig := placeStart placeStart = func(_ *Manager, _ string, cmd *exec.Cmd, before func() error) (bool, error) { refused, _, err := startGated(cmd, before, nil) return refused, err } forced := shimForcedForTest.Swap(true) t.Cleanup(func() { placeStart = orig shimForcedForTest.Store(forced) }) } // startGated starts cmd once before, when set, has agreed. refused reports // that before said no: nothing was started, and err is before's own error. // // register, when set, is called after before agrees and immediately before // the start, and the done it returns comes back for the caller to call once // the process is where it belongs. A start that fails ends it here, and // with no register done is a no-op; either way done may be called. func startGated(cmd *exec.Cmd, before func() error, register func() func()) (refused bool, done func(), err error) { done = func() {} if before != nil { if err := before(); err != nil { return true, done, err } } if register != nil { done = register() } if err := startCmd(cmd); err != nil { done() return false, done, err } return false, done, nil } // startPlain is the unaccounted start (no manager, or no cgroup dir): one // start behind before. With before nil it is today's `cmd, startCmd(cmd)`. func startPlain(build func() *exec.Cmd, before func() error) (*exec.Cmd, error) { cmd := build() refused, _, err := startGated(cmd, before, nil) if refused { return nil, err } return cmd, err } // startIn is Start against an explicit cgroup directory: the host's own // (Start) or one of its children (StartChild). Every start places its // process through the shim (placeStart); before is StartGated's. // // required is the generation layout's (StartChildGated, Task 7): every // limit must be in place before the process exists. With it, a process // that no shim can place here (a binary without RunShimIfRequested) is // refused before the child dir is made or build() runs, a child dir that // cannot be made or a limit write that fails refuses the start, and a // placement that fails refuses it too: all with ErrLimitNotApplied, the // box's condition. A required start is never started outside its cgroup // and moved by PID: it never reaches startAndMoveIn. Without required // (StartGated, the flat start) every one of those stays best-effort, and a // placement failure falls back to startAndMoveIn for that spawn (Ruling 2). // // Either way a failure that is the command's own (a missing binary, a bad // Dir) is returned as is, unwrapped, and never retried, and an empty dir // this spawn made is removed. A start whose exec.CommandContext context // ends during the placement is the caller's own condition too, never the // box's: the shim reports it as a placement failure (it could not start, // or was killed before placement), so startIn tells it apart (cancelWatch) // and, required or not, answers what exec.Cmd.Start answers for a command // whose context has ended: the context's own error (context.Canceled or // context.DeadlineExceeded), bare. Nothing ran and nothing is retried. The // placement's detail goes to the operator log only, so a member reading // the wake's failure sees what a direct start's cancellation always showed // (whole-branch review, finding 3). func (m *Manager) startIn(host, dir string, limit Limit, required bool, build func() *exec.Cmd, before func() error) (*exec.Cmd, error) { if required && !shimAvailable() { m.logf("cgroup: %s: the placement shim is not available in this process, and a generation is never started then moved — generation start refused", host) return nil, fmt.Errorf("%w: %s: the placement shim is not available", ErrLimitNotApplied, host) } if err := os.MkdirAll(dir, 0o755); err != nil { if required { m.logf("cgroup: %s: create cgroup: %v — generation start refused", host, err) return nil, fmt.Errorf("%w: %s: create %s: %w", ErrLimitNotApplied, host, filepath.Base(dir), err) } m.logf("cgroup: %s: create cgroup: %v — spawning unaccounted", host, err) return startPlain(build, before) } if err := m.applyLimit(host, dir, limit, required); err != nil && required { _ = os.Remove(dir) m.logf("cgroup: %s: %s: %v — generation start refused", host, filepath.Base(dir), err) return nil, fmt.Errorf("%w: %s: %w", ErrLimitNotApplied, filepath.Base(dir), err) } if !required && !shimAvailable() { cmd, _, err := m.startAndMoveIn(host, dir, build, before) return cmd, err } cmd := build() watch := watchCancel(cmd) refused, err := placeStart(m, dir, cmd, before) switch { case refused: // Nothing of the command ran, and no retry: the dir goes as it does // after a spawn that never happened (startAndMoveIn). _ = os.Remove(dir) return nil, err case err == nil: m.biasOOM(host, cmd) return cmd, nil case !isPlacementError(err): // The command's own failure: nothing ran, and a retry would fail the // same way. _ = os.Remove(dir) return nil, err } if ctxErr, ended := watch.endedErr(err); ended { // The caller's context ended: nothing ran, and it says nothing // about the box. No fallback either: its Start would answer the // same error at once. _ = os.Remove(dir) m.logf("cgroup: %s: the start's context ended during placement (%v); nothing ran", host, err) return nil, ctxErr } if required { _ = os.Remove(dir) m.logf("cgroup: %s: placement before exec failed (%v) — generation start refused", host, err) return nil, fmt.Errorf("%w: %s: %w", ErrLimitNotApplied, host, err) } // The flat start alone, after a placement failure the shim proved // happened before exec. m.logf("cgroup: %s: placement before exec failed (%v) — using the PID-move fallback for this spawn", host, err) cmd, _, err = m.startAndMoveIn(host, dir, build, before) return cmd, err } // cancelWatch records whether cmd's exec.CommandContext context ended // while it was being started: exec.Cmd calls Cancel (by default, a kill) // when the context ends after Start, which during a placement kills the // shim. A cmd without a context has no Cancel, and nothing is recorded. type cancelWatch struct { ended atomic.Bool // probe is a copy of cmd taken before its start, for the context // exec.Cmd keeps unexported (contextErr). Nil without a Cancel. probe *exec.Cmd } func watchCancel(cmd *exec.Cmd) *cancelWatch { w := &cancelWatch{} if cmd.Cancel != nil { probe := *cmd w.probe = &probe cancel := cmd.Cancel cmd.Cancel = func() error { w.ended.Store(true) return cancel() } } return w } // endedErr reports whether a failed placement was the start's context // ending, and that context's error: before the shim could start (Start // answered the context's error, which the placement error wraps), or // during it (Cancel ran, and the shim's answer names no context). func (w *cancelWatch) endedErr(err error) (error, bool) { switch { case errors.Is(err, context.Canceled): return context.Canceled, true case errors.Is(err, context.DeadlineExceeded): return context.DeadlineExceeded, true case w.ended.Load(): return w.contextErr(), true } return nil, false } // contextErr is the error of the command's context, once Cancel has run, // which exec.Cmd does only after that context is done. exec.Cmd keeps the // context unexported, but its Start answers ctx.Err() for a context that // has ended, before it opens a descriptor or forks. So the copy taken // before the start is asked to start, stripped of everything it could // otherwise act on: no files, no attributes, no Cancel, and /dev/null as // its path, which no exec would run. Should the answer ever be something // else, the context still ended, and Canceled is the caller's own end. // The unit tests pin both Canceled and DeadlineExceeded through it. func (w *cancelWatch) contextErr() error { if w.probe != nil { p := w.probe p.Path, p.Args, p.Env, p.Dir = "/dev/null", nil, []string{}, "" p.Stdin, p.Stdout, p.Stderr, p.ExtraFiles = nil, nil, nil, nil p.SysProcAttr, p.Cancel, p.WaitDelay = nil, nil, 0 if err := p.Start(); errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { return err } } return context.Canceled } // applyLimit writes limit into dir. Best-effort in the package's usual // sense: a refused write (a kernel without the memory controller here, a // control file that isn't ours) is one logged line and an uncapped child. // It returns the first write that failed, for a caller that must not start // without the whole limit (startIn's required); the flat start ignores it. // With required, the first failure returns at once and logs nothing here: // no child will run, so "runs uncapped" would be untrue, and startIn logs // the one refusal line (Task 7 fix round 1). Without it, every line is as // before. Written BEFORE the child is in the cgroup, so the cap holds from // its first page. memory.max first: if that one fails, high is pointless. // // The cap bounds the cgroup's TOTAL charge — anonymous memory plus the page // cache its reads pull in — not RSS. A tenant whose cache alone exceeds // memory.high lives under constant reclaim: slow, and nothing in the // journal says why. Hence the log line: the tier a host landed on is the // first thing to read when an app "got slow after the roll". // // memory.oom.group=1 makes an OOM kill inside this cgroup take the whole // family: a hub whose children outlive it would otherwise leave orphans // holding the socket while the activator respawns a fresh hub beside them. func (m *Manager) applyLimit(host, dir string, limit Limit, required bool) error { if limit.MemoryMax <= 0 { return nil } if err := os.WriteFile(filepath.Join(dir, "memory.max"), []byte(strconv.FormatInt(limit.MemoryMax, 10)), 0o644); err != nil { if !required { m.logf("cgroup: %s: memory.max: %v — child runs uncapped", host, err) } return fmt.Errorf("memory.max: %w", err) } var first error if err := os.WriteFile(filepath.Join(dir, "memory.high"), []byte(strconv.FormatInt(HighOf(limit.MemoryMax), 10)), 0o644); err != nil { if required { return fmt.Errorf("memory.high: %w", err) } m.logf("cgroup: %s: memory.high: %v — child is capped but not throttled first", host, err) first = fmt.Errorf("memory.high: %w", err) } // Swap is bounded too, or the cap isn't one: with zram on the box a // cgroup at memory.max simply pushes its anonymous pages into // compressed swap and carries on — a 200 MB allocator under a 64M cap // "survived" on the flagship, live, until this line. Same number as // max: a tenant may spill up to its cap into swap (the slow-then-dead // gradient), then the kill is real. if err := os.WriteFile(filepath.Join(dir, "memory.swap.max"), []byte(strconv.FormatInt(limit.MemoryMax, 10)), 0o644); err != nil { if required { return fmt.Errorf("memory.swap.max: %w", err) } m.logf("cgroup: %s: memory.swap.max: %v — the cap can be exceeded into swap", host, err) if first == nil { first = fmt.Errorf("memory.swap.max: %w", err) } } if err := os.WriteFile(filepath.Join(dir, "memory.oom.group"), []byte("1"), 0o644); err != nil { if required { return fmt.Errorf("memory.oom.group: %w", err) } m.logf("cgroup: %s: memory.oom.group: %v — an OOM kill here takes one process, not the family", host, err) if first == nil { first = fmt.Errorf("memory.oom.group: %w", err) } } m.logf("cgroup: %s: memory.max=%dM memory.high=%dM", host, limit.MemoryMax>>20, HighOf(limit.MemoryMax)>>20) return first } func (m *Manager) startAndMove(host string, build func() *exec.Cmd) (*exec.Cmd, error) { cmd, _, err := m.startAndMoveIn(host, m.hostDir(host), build, nil) return cmd, err } // startAndMoveIn is startAndMove into an explicit cgroup directory, behind // StartGated's before. refused reports that before said no (err is its // own error). It is the flat start's fallback only: a generation start // never comes here. // // From the moment before agrees until the PID write has returned, the // child runs in the agent's cgroup, not dir, so dir's placement is in // flight for that window (the registry, inflight.go): a cleanup's proof // that dir is empty must not miss it, as it must not miss a shim's. func (m *Manager) startAndMoveIn(host, dir string, build func() *exec.Cmd, before func() error) (*exec.Cmd, bool, error) { cmd := build() refused, done, err := startGated(cmd, before, func() func() { return m.beginPlacement(dir) }) if err != nil { // The spawn never happened: drop the now-ownerless dir so a failed // wake doesn't leave the sampler reporting permanent zeros for a // host with nothing running. rmdir-safe: os.Remove refuses a // non-empty dir, so this can never take a live sibling's dir. _ = os.Remove(dir) return nil, refused, err } if cmd.Process == nil { // A broken startCmd seam reporting success with no process: this // package must never panic the agent over it. Log and skip the // move; the child, if any, simply stays unaccounted. done() m.logf("cgroup: %s: start reported success with no process — skipping pid move", host) return cmd, false, nil } procs := filepath.Join(dir, "cgroup.procs") err = os.WriteFile(procs, []byte(strconv.Itoa(cmd.Process.Pid)), 0o644) done() if err != nil { m.logf("cgroup: %s: move pid %d: %v — child stays unaccounted", host, cmd.Process.Pid, err) } m.biasOOM(host, cmd) return cmd, false, nil } // biasOOM writes ExecOOMScoreAdj for a just-started child. Best-effort: a // refused write is one line, and the child runs with the agent's score. func (m *Manager) biasOOM(host string, cmd *exec.Cmd) { if cmd == nil || cmd.Process == nil { return } if err := os.WriteFile(oomScoreAdjPath(cmd.Process.Pid), []byte(ExecOOMScoreAdj), 0o644); err != nil { // A child that crashed in its first milliseconds is a zombie by the // time this write lands, and a zombie's oom_score_adj refuses the // write (seen live: birthday-alarm's crash loop logged one of these // per wake). A dead child needs no bias — say nothing. if procGone(cmd.Process.Pid) { return } m.logf("cgroup: %s: oom_score_adj for pid %d: %v — the agent and this child score alike at an edge-tree OOM", host, cmd.Process.Pid, err) } } // procGone reports whether pid is dead — absent from /proc, or a zombie // (state Z in /proc//stat, third field after the parenthesised comm). func procGone(pid int) bool { b, err := os.ReadFile(filepath.Join("/proc", strconv.Itoa(pid), "stat")) if err != nil { return true } // comm may contain spaces and parens; the state letter follows the LAST ')'. s := string(b) i := strings.LastIndexByte(s, ')') if i < 0 || i+2 >= len(s) { return false } return s[i+2] == 'Z' } // CleanupHost removes host's per-host cgroup dir after its children exit. // Nil-safe and rmdir-tolerant: while a sibling (instance vs sidecar) is // still alive the dir is non-empty and the remove fails — the LAST exiter's // call succeeds, and the next agent start's sweep catches anything missed. // (On real cgroupfs the control files applyLimit wrote don't count: rmdir // ignores interface files. In the temp-dir test fixture they are ordinary // files, so a test that caps a host and then expects CleanupHost to remove // the dir must delete them first — that failure has no production analogue.) func (m *Manager) CleanupHost(host string) { if m == nil { return } _ = removeEmptyDir(m.hostDir(host)) }