package cgroup import ( "context" "errors" "fmt" "io" "os" "os/exec" "path/filepath" "slices" "strconv" "strings" "sync" "syscall" "testing" "time" ) // quietOOM points the oom_score_adj write at a temp dir, so a fake start // that reports our own PID never re-scores the test process. func quietOOM(t *testing.T) { t.Helper() dir := t.TempDir() orig := oomScoreAdjPath oomScoreAdjPath = func(pid int) string { return filepath.Join(dir, strconv.Itoa(pid)) } t.Cleanup(func() { oomScoreAdjPath = orig }) } func writeFixture(t *testing.T, path, body string) { t.Helper() if err := os.WriteFile(path, []byte(body), 0o644); err != nil { t.Fatal(err) } } func readFixture(t *testing.T, path string) string { t.Helper() b, err := os.ReadFile(path) if err != nil { t.Fatalf("read %s: %v", path, err) } return string(b) } // genManager is a delegated fixture root whose host dir already lists the // four controllers, as a real child of the delegated root would. A temp // directory is no cgroupfs, so no shim can place a process there: the // placement is faked (FakePlacementForTest), as if the kernel had placed // it. genFixture is the same root with the real placement. func genManager(t *testing.T, host string) (*Manager, string) { t.Helper() m, parent := genFixture(t, host) FakePlacementForTest(t) return m, parent } func genFixture(t *testing.T, host string) (*Manager, string) { t.Helper() quietOOM(t) m, err := SetupAt(newDelegatedRoot(t), t.Logf) if err != nil { t.Fatal(err) } parent := m.hostDir(host) if err := os.MkdirAll(parent, 0o755); err != nil { t.Fatal(err) } writeFixture(t, filepath.Join(parent, "cgroup.controllers"), "cpu io memory pids\n") return m, parent } func TestGenAndRoleChildNames(t *testing.T) { if GenChild(7) != "g7" || RoleChild("sidecar") != "role-sidecar" { t.Fatalf("GenChild(7) = %q, RoleChild(sidecar) = %q", GenChild(7), RoleChild("sidecar")) } for name, want := range map[string]bool{ "g7": true, "g18446744073709551615": true, "role-seed": true, "role-sidecar": true, "g0": false, "g07": false, "g": false, "gx": false, "g-1": false, "role-": false, "role-a/b": false, "role-..": false, "child": false, "carlos-exec@h.example.com": false, "../g7": false, } { if got := isChildName(name); got != want { t.Errorf("isChildName(%q) = %v, want %v", name, got, want) } } } func TestOverlapAndValidParentLimits(t *testing.T) { if got := OverlapLimit(TierDefault, TierChildren); got.MemoryMax != 1536<<20 { t.Errorf("OverlapLimit(512M, 1G) = %d", got.MemoryMax) } for max, want := range map[int64]bool{ 512 << 20: true, 1 << 30: true, 1536 << 20: true, 2 << 30: true, 0: false, 64 << 20: false, 3 << 30: false, } { if got := ValidParentLimit(Limit{MemoryMax: max}); got != want { t.Errorf("ValidParentLimit(%d) = %v, want %v", max, got, want) } } } func TestStartChildPutsTheTierOnTheGenerationAndTheParent(t *testing.T) { const host = "h.example.com" m, parent := genManager(t, host) fakeStart(t, func(*exec.Cmd) error { return nil }) if _, err := m.StartChild(host, GenChild(1), TierDefault, TierDefault, buildTrue); err != nil { t.Fatal(err) } high := strconv.FormatInt(HighOf(TierDefault.MemoryMax), 10) child := filepath.Join(parent, "g1") for file, want := range map[string]string{ filepath.Join(child, "memory.max"): "536870912", filepath.Join(child, "memory.high"): high, filepath.Join(child, "memory.swap.max"): "536870912", filepath.Join(child, "memory.oom.group"): "1", filepath.Join(parent, "memory.max"): "536870912", filepath.Join(parent, "memory.high"): high, filepath.Join(parent, "memory.swap.max"): "536870912", filepath.Join(parent, "memory.oom.group"): "0", filepath.Join(parent, "cgroup.subtree_control"): "+cpu +memory +io +pids", } { if got := readFixture(t, file); got != want { t.Errorf("%s = %q, want %q", file, got, want) } } if _, err := os.Stat(filepath.Join(parent, "cgroup.procs")); !os.IsNotExist(err) { t.Errorf("the parent must hold no process, but its cgroup.procs was written (%v)", err) } } func TestStartChildRoleChildCarriesNoLimitOfItsOwn(t *testing.T) { const host = "h.example.com" m, parent := genManager(t, host) fakeStart(t, func(*exec.Cmd) error { return nil }) if _, err := m.StartChild(host, RoleChild("sidecar"), TierDefault, Limit{}, buildTrue); err != nil { t.Fatal(err) } if _, err := os.Stat(filepath.Join(parent, "role-sidecar", "memory.max")); !os.IsNotExist(err) { t.Errorf("a role child got a limit of its own: %v", err) } if got := readFixture(t, filepath.Join(parent, "memory.max")); got != "536870912" { t.Errorf("parent memory.max = %q, want the tier", got) } } // A successor spawns while the incumbent lives: the swap has raised the // parent on purpose, and the spawn must not lower it. func TestStartChildLeavesALiveParentAlone(t *testing.T) { const host = "h.example.com" m, parent := genManager(t, host) fakeStart(t, func(*exec.Cmd) error { return nil }) writeFixture(t, filepath.Join(parent, "cgroup.events"), "populated 1\nfrozen 0\n") writeFixture(t, filepath.Join(parent, "cgroup.subtree_control"), "cpu memory io pids\n") writeFixture(t, filepath.Join(parent, "memory.max"), "1073741824") if _, err := m.StartChild(host, GenChild(2), TierDefault, TierDefault, buildTrue); err != nil { t.Fatal(err) } if got := readFixture(t, filepath.Join(parent, "memory.max")); got != "1073741824" { t.Errorf("a successor's spawn changed the raised parent to %q", got) } } // Review focus 1: an agent killed between the raise and the restore leaves // the parent at two tiers. The next cold wake puts it back. func TestStartChildResetsAStaleRaisedParentOnAColdWake(t *testing.T) { const host = "h.example.com" m, parent := genManager(t, host) fakeStart(t, func(*exec.Cmd) error { return nil }) writeFixture(t, filepath.Join(parent, "cgroup.events"), "populated 0\nfrozen 0\n") writeFixture(t, filepath.Join(parent, "memory.max"), "1073741824") if _, err := m.StartChild(host, GenChild(3), TierDefault, TierDefault, buildTrue); err != nil { t.Fatal(err) } if got := readFixture(t, filepath.Join(parent, "memory.max")); got != "536870912" { t.Errorf("cold wake left the parent at %q, want the tier", got) } } func TestStartChildRefusesAForeignChildName(t *testing.T) { m, _ := genManager(t, "h.example.com") calls := fakeStart(t, func(*exec.Cmd) error { return nil }) if _, err := m.StartChild("h.example.com", "../carlos-exec@other.example.com", TierDefault, TierDefault, buildTrue); err == nil { t.Fatal("a child name outside the layout was accepted") } if len(*calls) != 0 { t.Fatalf("spawned %d times for a refused child name", len(*calls)) } } func TestSetParentLimitWritesTheBackstopAndReportsFailure(t *testing.T) { const host = "h.example.com" m, parent := genManager(t, host) if err := m.SetParentLimit(host, OverlapLimit(TierDefault, TierDefault)); err != nil { t.Fatal(err) } for file, want := range map[string]string{ "memory.max": "1073741824", "memory.high": strconv.FormatInt(HighOf(1<<30), 10), "memory.swap.max": "1073741824", "memory.oom.group": "0", } { if got := readFixture(t, filepath.Join(parent, file)); got != want { t.Errorf("%s = %q, want %q", file, got, want) } } // A raise that did not land must say so: the swap stops on it. if err := os.Remove(filepath.Join(parent, "memory.max")); err != nil { t.Fatal(err) } if err := os.Mkdir(filepath.Join(parent, "memory.max"), 0o755); err != nil { t.Fatal(err) } if err := m.SetParentLimit(host, TierDefault); err == nil { t.Fatal("a limit write that failed reported success") } var nilM *Manager if err := nilM.SetParentLimit(host, TierDefault); err != nil { t.Fatalf("nil manager: %v, want nil (no delegation, nothing to set)", err) } } // Review focus 2: rmdir refuses a cgroup with child cgroups, empty or not. func TestSweepRemovesEmptyGenerationChildrenThenTheParent(t *testing.T) { root := newDelegatedRoot(t) stale := filepath.Join(root, ExecName("old.example.com")) for _, d := range []string{"g3", "role-seed"} { if err := os.MkdirAll(filepath.Join(stale, d), 0o755); err != nil { t.Fatal(err) } } live := filepath.Join(root, ExecName("live.example.com")) if err := os.MkdirAll(filepath.Join(live, "g4"), 0o755); err != nil { t.Fatal(err) } // A file stands in for a member: rmdir refuses the directory. writeFixture(t, filepath.Join(live, "g4", "cgroup.procs"), "123\n") if _, err := SetupAt(root, t.Logf); err != nil { t.Fatal(err) } if _, err := os.Stat(stale); !os.IsNotExist(err) { t.Errorf("stale generation parent survived the sweep: %v", err) } if _, err := os.Stat(filepath.Join(live, "g4")); err != nil { t.Errorf("a populated generation child must be left alone: %v", err) } } func TestRemoveChildRemovesOnlyItsOwnNames(t *testing.T) { const host = "h.example.com" m, parent := genManager(t, host) for _, d := range []string{"g5", "keep"} { if err := os.Mkdir(filepath.Join(parent, d), 0o755); err != nil { t.Fatal(err) } } m.RemoveChild(host, "g5") m.RemoveChild(host, "keep") m.RemoveChild(host, "../"+ExecName("other.example.com")) if _, err := os.Stat(filepath.Join(parent, "g5")); !os.IsNotExist(err) { t.Errorf("empty g5 survived: %v", err) } if _, err := os.Stat(filepath.Join(parent, "keep")); err != nil { t.Errorf("a foreign child was removed: %v", err) } var nilM *Manager nilM.RemoveChild(host, "g5") // must not panic } // Flag off: Start keeps today's single-level layout. func TestStartKeepsTheSingleLevelLayout(t *testing.T) { quietOOM(t) m, err := SetupAt(newDelegatedRoot(t), t.Logf) if err != nil { t.Fatal(err) } fakeStart(t, func(*exec.Cmd) error { return nil }) if _, err := m.Start("h.example.com", TierDefault, buildTrue); err != nil { t.Fatal(err) } dir := m.hostDir("h.example.com") if got := readFixture(t, filepath.Join(dir, "memory.oom.group")); got != "1" { t.Errorf("host cgroup memory.oom.group = %q, want 1", got) } if _, err := os.Stat(filepath.Join(dir, "cgroup.subtree_control")); !os.IsNotExist(err) { t.Errorf("flag-off Start delegated controllers: %v", err) } entries, err := os.ReadDir(dir) if err != nil { t.Fatal(err) } for _, e := range entries { if e.IsDir() { t.Errorf("flag-off Start made a child cgroup %s", e.Name()) } } } // Zero-downtime final review wave 2 (F1 completion): StartChildGated's // before-start hook is asked after the whole preparation, the parent's tier // and the generation child's limits both written, and a refusal starts // nothing and comes back as the hook's own error. A hook that agrees // starts exactly once, as StartChild does. func TestStartChildGatedAsksAfterThePreparation(t *testing.T) { const host = "h.example.com" m, parent := genManager(t, host) calls := fakeStart(t, func(*exec.Cmd) error { return nil }) refused := errors.New("the attempt was superseded") var prepared bool cmd, err := m.StartChildGated(host, GenChild(1), TierDefault, TierDefault, buildTrue, func() error { _, perr := os.Stat(filepath.Join(parent, "memory.max")) _, cerr := os.Stat(filepath.Join(parent, "g1", "memory.max")) prepared = perr == nil && cerr == nil return refused }) if err != refused || cmd != nil { t.Fatalf("StartChildGated = %v, %v: want the hook's own error and no command", cmd, err) } if !prepared { t.Fatal("the hook was asked before the parent's tier and the child's limits were written") } if len(*calls) != 0 { t.Fatalf("started %d times after the hook refused, want none", len(*calls)) } if _, err := m.StartChildGated(host, GenChild(2), TierDefault, TierDefault, buildTrue, func() error { return nil }); err != nil { t.Fatal(err) } if len(*calls) != 1 { t.Fatalf("started %d times with an agreeing hook, want 1", len(*calls)) } } // With no manager (no delegation) the hook still guards the one plain // start. func TestStartGatedWithNoManagerAsksBeforeThePlainStart(t *testing.T) { calls := fakeStart(t, func(*exec.Cmd) error { return nil }) refused := errors.New("lease lost") var m *Manager if cmd, err := m.StartGated("h.example.com", TierDefault, buildTrue, func() error { return refused }); err != refused || cmd != nil { t.Fatalf("StartGated = %v, %v: want the hook's own error and no command", cmd, err) } if cmd, err := m.StartChildGated("h.example.com", GenChild(1), TierDefault, TierDefault, buildTrue, func() error { return refused }); err != refused || cmd != nil { t.Fatalf("StartChildGated = %v, %v: want the hook's own error and no command", cmd, err) } if len(*calls) != 0 { t.Fatalf("started %d times after the hook refused, want none", len(*calls)) } } // legacyParent is genManager over a manager whose log lines are kept, with // the parent's own cgroup.procs listing pids: the shape the dev box showed // (amadan #272, check (d)), a carlos-exec@ an older agent started a // process in directly. func legacyParent(t *testing.T, host, procs string) (*Manager, string, *[]string) { t.Helper() m, parent := genManager(t, host) var lines []string m.logf = func(format string, args ...any) { lines = append(lines, fmt.Sprintf(format, args...)) } writeFixture(t, filepath.Join(parent, "cgroup.procs"), procs) return m, parent, &lines } func noGenerationChild(t *testing.T, parent string) { t.Helper() ents, err := os.ReadDir(parent) if err != nil { t.Fatal(err) } for _, e := range ents { if e.IsDir() { t.Errorf("the refused start created %s under the parent", e.Name()) } } } // Flag blocker (amadan #272): a generation started under a parent that // still holds a process directly runs with no memory limit, because a // cgroup with processes of its own cannot delegate the memory controller. // The start is refused, names the host and the count, and creates nothing. func TestAPopulatedLegacyParentIsRefusedBeforeAnyChildExists(t *testing.T) { const host = "h.example.com" m, parent, lines := legacyParent(t, host, "4242\n4243\n") calls := fakeStart(t, func(*exec.Cmd) error { return nil }) cmd, err := m.StartChild(host, GenChild(1), TierDefault, TierDefault, buildTrue) if !errors.Is(err, ErrLegacyParentPopulated) || cmd != nil { t.Fatalf("StartChild = %v, %v: want ErrLegacyParentPopulated and no command", cmd, err) } if len(*calls) != 0 { t.Fatalf("started %d times under a populated parent", len(*calls)) } noGenerationChild(t, parent) if _, serr := os.Stat(filepath.Join(parent, "cgroup.subtree_control")); !os.IsNotExist(serr) { t.Errorf("the refused start delegated controllers anyway (%v)", serr) } if len(*lines) != 1 || !strings.Contains((*lines)[0], host) || !strings.Contains((*lines)[0], "2 process") { t.Errorf("log lines = %q, want one naming %s and 2 processes", *lines, host) } } // A parent that holds a process directly is refused whatever children it // has (plan review #12): an empty g1 left from an earlier generation does // not make the parent a generation parent. func TestAPopulatedParentWithAStaleEmptyGenerationIsRefused(t *testing.T) { const host = "h.example.com" m, parent, _ := legacyParent(t, host, "4242\n") if err := os.MkdirAll(filepath.Join(parent, "g1"), 0o755); err != nil { t.Fatal(err) } writeFixture(t, filepath.Join(parent, "g1", "cgroup.procs"), "") writeFixture(t, filepath.Join(parent, "cgroup.events"), "populated 1\nfrozen 0\n") calls := fakeStart(t, func(*exec.Cmd) error { return nil }) if _, err := m.StartChild(host, GenChild(2), TierDefault, TierDefault, buildTrue); !errors.Is(err, ErrLegacyParentPopulated) { t.Fatalf("StartChild = %v, want ErrLegacyParentPopulated", err) } if len(*calls) != 0 { t.Fatalf("started %d times", len(*calls)) } if _, serr := os.Stat(filepath.Join(parent, "g2")); !os.IsNotExist(serr) { t.Errorf("the refused start created g2 (%v)", serr) } } // The refusal is about direct members only. A parent whose processes all // live in generation children is the layout working: cgroup.procs is empty // and cgroup.events reports the descendants. func TestAnEmptyParentWithLiveGenerationsStillStartsASuccessor(t *testing.T) { const host = "h.example.com" m, parent, _ := legacyParent(t, host, "") if err := os.MkdirAll(filepath.Join(parent, "g1"), 0o755); err != nil { t.Fatal(err) } writeFixture(t, filepath.Join(parent, "g1", "cgroup.procs"), "5151\n") writeFixture(t, filepath.Join(parent, "cgroup.events"), "populated 1\nfrozen 0\n") writeFixture(t, filepath.Join(parent, "cgroup.subtree_control"), "cpu memory io pids\n") writeFixture(t, filepath.Join(parent, "memory.max"), "1073741824") calls := fakeStart(t, func(*exec.Cmd) error { return nil }) if _, err := m.StartChild(host, GenChild(2), TierDefault, TierDefault, buildTrue); err != nil { t.Fatalf("StartChild beside a live generation: %v", err) } if len(*calls) != 1 { t.Fatalf("started %d times, want 1", len(*calls)) } if got := readFixture(t, filepath.Join(parent, "memory.max")); got != "1073741824" { t.Errorf("the successor's start lowered the raised parent to %q", got) } } // A parent cgroup.procs that cannot be read is not known to be empty, so it // is refused as populated is (parentPopulated's rule: unreadable counts as // populated). A parent that has no cgroup.procs yet is a fresh directory. func TestAnUnreadableParentProcsFileIsRefusedAndAnAbsentOneIsNot(t *testing.T) { const host = "h.example.com" m, parent, _ := legacyParent(t, host, "") if err := os.Remove(filepath.Join(parent, "cgroup.procs")); err != nil { t.Fatal(err) } fakeStart(t, func(*exec.Cmd) error { return nil }) if _, err := m.StartChild(host, GenChild(1), TierDefault, TierDefault, buildTrue); err != nil { t.Fatalf("a parent with no cgroup.procs file: %v", err) } if err := os.MkdirAll(filepath.Join(parent, "cgroup.procs"), 0o755); err != nil { // reading a directory fails t.Fatal(err) } if _, err := m.StartChild(host, GenChild(2), TierDefault, TierDefault, buildTrue); !errors.Is(err, ErrLegacyParentPopulated) { t.Fatalf("StartChild over an unreadable cgroup.procs = %v, want ErrLegacyParentPopulated", err) } } // Task 7 (zdd flag readiness round 2): a generation whose limits cannot be // applied does not start. Each case below is one of the branches that used // to log a line and run the generation uncapped (recon C, A to G). Each // refuses with ErrLimitNotApplied before anything is exec'd (the build // function is never even called) and leaves no child cgroup behind. // refusedChild runs StartChild for g1 at the default tier and checks the // refusal: ErrLimitNotApplied (or want), no command, nothing started, and // the command never built. func refusedChild(t *testing.T, m *Manager, host, child string, own Limit, want error) error { t.Helper() calls := fakeStart(t, func(*exec.Cmd) error { return nil }) built := false cmd, err := m.StartChild(host, child, TierDefault, own, func() *exec.Cmd { built = true; return buildTrue() }) if !errors.Is(err, want) || cmd != nil { t.Fatalf("StartChild(%s) = %v, %v: want %v and no command", child, cmd, err, want) } if len(*calls) != 0 || built { t.Fatalf("StartChild(%s): built %v, started %d times; want neither", child, built, len(*calls)) } return err } // noChildBut is noGenerationChild for a fixture that plants a directory // under the parent (a control file made a directory so its write fails): // the planted ones are the only directories allowed. func noChildBut(t *testing.T, parent string, planted ...string) { t.Helper() ents, err := os.ReadDir(parent) if err != nil { t.Fatal(err) } for _, e := range ents { if e.IsDir() && !slices.Contains(planted, e.Name()) { t.Errorf("the refused start created %s under the parent", e.Name()) } } } // A: the parent cgroup cannot be made. It used to start the generation in // the agent's own cgroup, unaccounted and uncapped. The flat start (flag // off) keeps its best-effort plain start. func TestAGenerationParentThatCannotBeMadeIsRefused(t *testing.T) { const host = "h.example.com" m, parent := genManager(t, host) if err := os.RemoveAll(parent); err != nil { t.Fatal(err) } writeFixture(t, parent, "a file where the parent goes") // MkdirAll: ENOTDIR refusedChild(t, m, host, GenChild(1), TierDefault, ErrLimitNotApplied) if fi, err := os.Stat(parent); err != nil || fi.IsDir() { t.Fatalf("the refused start replaced the parent: %v", err) } calls := fakeStart(t, func(*exec.Cmd) error { return nil }) if cmd, err := m.Start(host, TierDefault, buildTrue); err != nil || cmd == nil || len(*calls) != 1 { t.Fatalf("flag-off Start = %v, %v after %d starts: want today's plain start", cmd, err, len(*calls)) } } // B: a parent with live generations is left as it is, but only when it // delegates memory to its children. Without memory in its subtree_control // (absent, unreadable, or listing other controllers) a new g under it // would get no memory.max at all. func TestALiveParentThatDoesNotDelegateMemoryIsRefused(t *testing.T) { const host = "h.example.com" for name, subtree := range map[string]*string{ "absent": nil, "without memory": ptr("cpu pids\n"), "empty": ptr(""), } { t.Run(name, func(t *testing.T) { m, parent := genManager(t, host) writeFixture(t, filepath.Join(parent, "cgroup.events"), "populated 1\nfrozen 0\n") if subtree != nil { writeFixture(t, filepath.Join(parent, "cgroup.subtree_control"), *subtree) } refusedChild(t, m, host, GenChild(2), TierDefault, ErrLimitNotApplied) noGenerationChild(t, parent) }) } t.Run("unreadable", func(t *testing.T) { m, parent := genManager(t, host) writeFixture(t, filepath.Join(parent, "cgroup.events"), "populated 1\nfrozen 0\n") if err := os.Mkdir(filepath.Join(parent, "cgroup.subtree_control"), 0o755); err != nil { t.Fatal(err) } refusedChild(t, m, host, GenChild(2), TierDefault, ErrLimitNotApplied) noChildBut(t, parent, "cgroup.subtree_control") }) // Both spellings delegate: the kernel reads back "memory", and this // package's own write (and its fixtures) store "+memory". for _, subtree := range []string{"cpu memory io pids\n", "+cpu +memory +io +pids"} { m, parent := genManager(t, host) writeFixture(t, filepath.Join(parent, "cgroup.events"), "populated 1\nfrozen 0\n") writeFixture(t, filepath.Join(parent, "cgroup.subtree_control"), subtree) calls := fakeStart(t, func(*exec.Cmd) error { return nil }) if _, err := m.StartChild(host, GenChild(2), TierDefault, TierDefault, buildTrue); err != nil || len(*calls) != 1 { t.Fatalf("subtree_control %q: StartChild = %v after %d starts, want one start", subtree, err, len(*calls)) } } } func ptr(s string) *string { return &s } // C: the controllers cannot be delegated on a cold start. A parent with no // memory controller to hand on, or a subtree_control write that fails, is // ErrLimitNotApplied; a write the kernel answers EBUSY is the populated // parent race (a process landed in the parent after the cgroup.procs // check), so it is ErrLegacyParentPopulated. Nothing else is written: no // parent limit, no child. func TestAParentWhoseControllersCannotBeDelegatedIsRefused(t *testing.T) { const host = "h.example.com" t.Run("no memory controller", func(t *testing.T) { m, parent := genManager(t, host) writeFixture(t, filepath.Join(parent, "cgroup.controllers"), "cpu pids\n") refusedChild(t, m, host, GenChild(1), TierDefault, ErrLimitNotApplied) noGenerationChild(t, parent) if _, err := os.Stat(filepath.Join(parent, "memory.max")); !os.IsNotExist(err) { t.Errorf("the refused start wrote the parent's limit: %v", err) } }) for name, tc := range map[string]struct { write error want error }{ "the write is refused": {syscall.EACCES, ErrLimitNotApplied}, "a process landed in the parent": {syscall.EBUSY, ErrLegacyParentPopulated}, } { t.Run(name, func(t *testing.T) { m, parent := genManager(t, host) orig := writeSubtreeControl writeSubtreeControl = func(path string, _ []byte) error { return &os.PathError{Op: "write", Path: path, Err: tc.write} } t.Cleanup(func() { writeSubtreeControl = orig }) err := refusedChild(t, m, host, GenChild(1), TierDefault, tc.want) noGenerationChild(t, parent) if tc.want == ErrLegacyParentPopulated && errors.Is(err, ErrLimitNotApplied) { t.Error("the EBUSY race reads as a limit failure") } if _, err := os.Stat(filepath.Join(parent, "memory.max")); !os.IsNotExist(err) { t.Errorf("the refused start wrote the parent's limit: %v", err) } }) } } // D: the parent's backstop cannot be written. A role child (the legacy // sidecar, the seed) has no limit of its own, so without the parent's it // would have none at all. func TestAParentWhoseLimitCannotBeWrittenIsRefused(t *testing.T) { const host = "h.example.com" for _, file := range []string{"memory.max", "memory.oom.group"} { for _, child := range []string{GenChild(1), RoleChild("sidecar")} { m, parent := genManager(t, host) if err := os.Mkdir(filepath.Join(parent, file), 0o755); err != nil { // WriteFile: EISDIR t.Fatal(err) } own := Limit{} if child == GenChild(1) { own = TierDefault } refusedChild(t, m, host, child, own, ErrLimitNotApplied) noChildBut(t, parent, file) } } } // E: the generation's own child cgroup cannot be made. It used to start in // the agent's cgroup, unaccounted and uncapped. func TestAGenerationChildThatCannotBeMadeIsRefused(t *testing.T) { const host = "h.example.com" m, parent := genManager(t, host) writeFixture(t, filepath.Join(parent, "g1"), "a file where g1 goes") // MkdirAll: ENOTDIR refusedChild(t, m, host, GenChild(1), TierDefault, ErrLimitNotApplied) noGenerationChild(t, parent) } // F: one of the generation's four limit writes fails. All four are the // child's limits (spec ยง7.3), so any failure refuses the start, and the // process never lands in the child (no cgroup.procs write). The flat start // keeps logging and spawning (TestStartLimitWriteFailureStillSpawns). func TestAGenerationWhoseLimitsCannotBeWrittenIsRefused(t *testing.T) { const host = "h.example.com" for _, file := range []string{"memory.max", "memory.high", "memory.swap.max", "memory.oom.group"} { t.Run(file, func(t *testing.T) { m, parent := genManager(t, host) g1 := filepath.Join(parent, "g1") if err := os.MkdirAll(filepath.Join(g1, file), 0o755); err != nil { // WriteFile: EISDIR t.Fatal(err) } refusedChild(t, m, host, GenChild(1), TierDefault, ErrLimitNotApplied) if _, err := os.Stat(filepath.Join(g1, "cgroup.procs")); !os.IsNotExist(err) { t.Errorf("a process was placed in the refused child: %v", err) } noChildBut(t, parent, "g1") }) } // applyLimit reports the first failure; with every write landing it // answers nil. m, parent := genManager(t, host) dir := filepath.Join(parent, "g9") if err := os.Mkdir(dir, 0o755); err != nil { t.Fatal(err) } for _, required := range []bool{false, true} { if err := m.applyLimit(host, dir, TierDefault, required); err != nil { t.Fatalf("applyLimit(required %v) with every write landing = %v", required, err) } if err := m.applyLimit(host, dir, Limit{}, required); err != nil { t.Fatalf("applyLimit(required %v) with no limit = %v", required, err) } } } // Fix round 1, finding 7: a generation whose memory.max cannot be written // is refused, so the journal must not say it "runs uncapped". The required // path logs one refusal line naming the file; the flat start's lines are // unchanged. func TestARefusedGenerationsLimitFailureLogsTheRefusalOnly(t *testing.T) { const host = "h.example.com" m, parent, lines := legacyParent(t, host, "") if err := os.MkdirAll(filepath.Join(parent, "g1", "memory.max"), 0o755); err != nil { // WriteFile: EISDIR t.Fatal(err) } refusedChild(t, m, host, GenChild(1), TierDefault, ErrLimitNotApplied) var refusals int for _, l := range *lines { if strings.Contains(l, "uncapped") || strings.Contains(l, ": memory.max=") { // the parent's own "parent memory.max=" line is true t.Errorf("a refused generation logged %q", l) } if strings.Contains(l, "g1: memory.max:") && strings.Contains(l, "generation start refused") { refusals++ } } if refusals != 1 { t.Errorf("log lines = %q, want one refusal naming g1's memory.max", *lines) } // The flat start keeps its line. *lines = nil flat := m.hostDir("flat.example.com") if err := os.MkdirAll(filepath.Join(flat, "memory.max"), 0o755); err != nil { t.Fatal(err) } fakeStart(t, func(*exec.Cmd) error { return nil }) if _, err := m.Start("flat.example.com", TierDefault, buildTrue); err != nil { t.Fatal(err) } if !slices.ContainsFunc(*lines, func(l string) bool { return strings.Contains(l, "memory.max:") && strings.Contains(l, "child runs uncapped") }) { t.Errorf("flat log lines = %q, want the unchanged \"child runs uncapped\" line", *lines) } } // G: with no shim in this process (a binary whose main lacks // RunShimIfRequested) a generation could only be started and then moved by // PID: running, for that window, in the agent's cgroup with no limit, and // for good if the move fails. It is refused before the child is created // or build() runs (ensureParent has already run). The flat start keeps the // PID-move start. func TestGenerationRefusedWithoutTheShim(t *testing.T) { const host = "h.example.com" m, parent := genFixture(t, host) noShim(t) for _, child := range []string{GenChild(1), RoleChild("seed")} { own := Limit{} if child == GenChild(1) { own = TierDefault } refusedChild(t, m, host, child, own, ErrLimitNotApplied) noGenerationChild(t, parent) } calls := fakeStart(t, func(*exec.Cmd) error { return nil }) if cmd, err := m.Start("flat.example.com", TierDefault, buildTrue); err != nil || cmd == nil || len(*calls) != 1 { t.Fatalf("flag-off Start without the shim = %v, %v after %d starts: want the PID-move start", cmd, err, len(*calls)) } if _, err := os.Stat(filepath.Join(m.hostDir("flat.example.com"), "cgroup.procs")); err != nil { t.Fatalf("flag-off Start did not move the PID: %v", err) } } // noShim makes this test binary one without RunShimIfRequested until t's // cleanup. func noShim(t *testing.T) { t.Helper() hooked, forced := shimHooked.Swap(false), shimForcedForTest.Swap(false) t.Cleanup(func() { shimHooked.Store(hooked) shimForcedForTest.Store(forced) }) } // stubPlacement swaps the placement seam until t's cleanup. func stubPlacement(t *testing.T, f func(m *Manager, dir string, cmd *exec.Cmd, before func() error) (bool, error)) { t.Helper() orig := placeStart placeStart = f t.Cleanup(func() { placeStart = orig }) } // The fixture's limit files are ordinary files, which keep a refused // child's rmdir from succeeding (CleanupHost's comment). The removal tests // below start their generation with no limit of its own, so the child is // empty when it should go. // Review focus 5: a missing binary is the command's own failure. The shim // answers fork/exec's own error; the start returns it as is, and it is not // the box's condition, so it is not ErrLimitNotApplied. The empty child goes. func TestGenerationMissingBinaryIsNotABoxCondition(t *testing.T) { const host = "h.example.com" m, parent := genManager(t, host) missing := &os.PathError{Op: "fork/exec", Path: "/x", Err: syscall.ENOENT} stubPlacement(t, func(*Manager, string, *exec.Cmd, func() error) (bool, error) { return false, missing }) builds := 0 cmd, err := m.StartChild(host, GenChild(1), TierDefault, Limit{}, func() *exec.Cmd { builds++; return buildTrue() }) if err != missing || cmd != nil { t.Fatalf("StartChild = %v, %v: want the command's own error, unwrapped, and no command", cmd, err) } if errors.Is(err, ErrLimitNotApplied) { t.Fatal("a missing binary read as the box's condition") } if builds != 1 { t.Fatalf("built %d times, want once: the command's own failure is never retried", builds) } if _, err := os.Stat(filepath.Join(parent, "g1")); !os.IsNotExist(err) { t.Fatalf("the empty child survived a failed start: %v", err) } } // A placement failure is the box's condition: ErrLimitNotApplied, wrapping // the shim's answer, with no retry (a generation is never started then // moved) and the empty child removed. func TestGenerationPlacementFailureIsLimitNotApplied(t *testing.T) { const host = "h.example.com" m, parent, lines := legacyParent(t, host, "") failed := &placementError{errors.New("the shim exited before placement")} stubPlacement(t, func(*Manager, string, *exec.Cmd, func() error) (bool, error) { return false, failed }) calls := fakeStart(t, func(*exec.Cmd) error { return nil }) builds := 0 cmd, err := m.StartChild(host, GenChild(1), TierDefault, Limit{}, func() *exec.Cmd { builds++; return buildTrue() }) if !errors.Is(err, ErrLimitNotApplied) || !errors.Is(err, failed) || cmd != nil { t.Fatalf("StartChild = %v, %v: want ErrLimitNotApplied wrapping the placement error, and no command", cmd, err) } if builds != 1 || len(*calls) != 0 { t.Fatalf("built %d, started %d: want one build and no start-and-move", builds, len(*calls)) } if _, err := os.Stat(filepath.Join(parent, "g1")); !os.IsNotExist(err) { t.Fatalf("the empty child survived a failed placement: %v", err) } if !slices.ContainsFunc(*lines, func(l string) bool { return strings.Contains(l, "placement before exec failed") && strings.Contains(l, "generation start refused") }) { t.Errorf("log lines = %q, want the refusal line", *lines) } } // Ruling 6: a sibling generation being placed is something alive under the // parent, so a cold-parent reset (enable the controllers, write the tier) // must not run: a swap may have raised that parent on purpose. func TestEnsureParentCountsAPlacementBelowTheParent(t *testing.T) { const host = "h.example.com" m, parent := genManager(t, host) fakeStart(t, func(*exec.Cmd) error { return nil }) writeFixture(t, filepath.Join(parent, "cgroup.events"), "populated 0\nfrozen 0\n") writeFixture(t, filepath.Join(parent, "cgroup.subtree_control"), "cpu memory io pids\n") writeFixture(t, filepath.Join(parent, "memory.max"), "1073741824") done := m.beginPlacement(filepath.Join(parent, GenChild(1))) defer done() if _, err := m.StartChildGated(host, GenChild(2), TierDefault, TierDefault, buildTrue, nil); err != nil { t.Fatalf("StartChildGated beside a generation being placed: %v", err) } if got := readFixture(t, filepath.Join(parent, "memory.max")); got != "1073741824" { t.Errorf("the parent's memory.max was rewritten to %q: a placement in flight was taken for a cold parent", got) } if got := readFixture(t, filepath.Join(parent, "cgroup.subtree_control")); got != "cpu memory io pids\n" { t.Errorf("the parent's subtree_control was rewritten to %q", got) } } // Ruling 6: a flat start being placed into the host directory itself is a // direct process of the parent, so a generation start is refused as for a // legacy parent, before anything is built. func TestEnsureParentCountsAPlacementAtTheParent(t *testing.T) { const host = "h.example.com" m, parent := genManager(t, host) done := m.beginPlacement(parent) defer done() err := refusedChild(t, m, host, GenChild(1), TierDefault, ErrLegacyParentPopulated) if !strings.Contains(err.Error(), "1 process") { t.Errorf("refusal %q, want it to count the placement as one process", err) } noGenerationChild(t, parent) } // A generation start whose exec.CommandContext context ends during the // placement (a seed's wake timing out, the agent stopping) is the caller's // condition, not the box's: the shim reports it as a placement failure, // but the start must not come back as ErrLimitNotApplied. Two ways in: // the context had ended before the shim could start (Start answers its // error, which the placement error wraps), and it ends while the shim is // being placed (exec.Cmd's Cancel kills the shim, whose answer names no // context). Either way the start answers the context's own error, bare, // as exec.Cmd.Start does for an ended context: context.Canceled or // context.DeadlineExceeded as the context ended, with no placement text, // which goes to the operator log (whole-branch review, finding 3). func TestGenerationCancelledStartIsNotABoxCondition(t *testing.T) { const host = "h.example.com" ended := map[string]func(t *testing.T) context.Context{ "cancelled": func(t *testing.T) context.Context { ctx, cancel := context.WithCancel(t.Context()) cancel() return ctx }, "past its deadline": func(t *testing.T) context.Context { ctx, cancel := context.WithDeadline(t.Context(), time.Unix(1, 0)) t.Cleanup(cancel) return ctx }, } for name, place := range map[string]func(*Manager, string, *exec.Cmd, func() error) (bool, error){ "ended before the shim started": func(_ *Manager, _ string, cmd *exec.Cmd, _ func() error) (bool, error) { return false, &placementError{fmt.Errorf("start the shim: %w", cmd.Start())} }, "ended during the placement": func(_ *Manager, _ string, cmd *exec.Cmd, _ func() error) (bool, error) { _ = cmd.Cancel() // what exec.Cmd's context watcher does return false, &placementError{errors.New("the shim exited before placement")} }, } { for ctxName, mkCtx := range ended { t.Run(name+", "+ctxName, func(t *testing.T) { m, parent, lines := legacyParent(t, host, "") stubPlacement(t, place) ctx := mkCtx(t) cmd, err := m.StartChild(host, GenChild(1), TierDefault, Limit{}, func() *exec.Cmd { c := exec.CommandContext(ctx, "/bin/true") c.Cancel = func() error { return nil } // no process to kill here return c }) if cmd != nil || err != ctx.Err() { t.Fatalf("StartChild = %v, %v: want the context's own error, %v, bare", cmd, err, ctx.Err()) } if errors.Is(err, ErrLimitNotApplied) || isPlacementError(err) { t.Fatalf("StartChild = %v: a cancelled start read as the box's condition", err) } if _, serr := os.Stat(filepath.Join(parent, "g1")); !os.IsNotExist(serr) { t.Fatalf("the empty child survived: %v", serr) } logged := false for _, l := range *lines { if strings.Contains(l, "generation start refused") { t.Errorf("a cancelled start logged a box condition: %q", l) } if strings.Contains(l, "the start's context ended during placement") && strings.Contains(l, "placement before exec") { logged = true } } if !logged { t.Errorf("log lines = %q, want the placement's detail in the operator line", *lines) } }) } } } // contextErr asks a copy of the command, taken before its start, for the // context's error. The copy must never run anything, even with the // original's files and attributes set, and must leave the original as it // was. func TestCancelWatchContextErrRunsNothing(t *testing.T) { for _, tc := range []struct { name string ctx func(t *testing.T) context.Context want error }{ {"cancelled", func(t *testing.T) context.Context { ctx, cancel := context.WithCancel(t.Context()) cancel() return ctx }, context.Canceled}, {"past its deadline", func(t *testing.T) context.Context { ctx, cancel := context.WithDeadline(t.Context(), time.Unix(1, 0)) t.Cleanup(cancel) return ctx }, context.DeadlineExceeded}, } { t.Run(tc.name, func(t *testing.T) { marker := filepath.Join(t.TempDir(), "ran") r, w, err := os.Pipe() if err != nil { t.Fatal(err) } defer r.Close() defer w.Close() ctx := tc.ctx(t) cmd := exec.CommandContext(ctx, "/bin/sh", "-c", `touch "$0"`, marker) cmd.ExtraFiles = []*os.File{w} cmd.Stdout = io.Discard cmd.Cancel = func() error { return nil } // no process to kill here watch := watchCancel(cmd) _ = cmd.Cancel() // the context watcher's call got, ended := watch.endedErr(&placementError{errors.New("the shim exited before placement")}) if !ended || got != tc.want { t.Fatalf("endedErr = %v, %v: want %v", got, ended, tc.want) } if _, err := os.Stat(marker); err == nil { t.Fatal("the probe ran the command") } if cmd.Path != "/bin/sh" || len(cmd.ExtraFiles) != 1 || cmd.Stdout != io.Discard || cmd.Process != nil { t.Fatalf("the probe changed the original command: %+v", cmd) } }) } // Without a context nothing is watched and nothing ended. if _, ended := watchCancel(exec.Command("/bin/true")).endedErr(&placementError{errors.New("x")}); ended { t.Fatal("a command without a context read as ended") } } // Whole-branch review, finding 1, at ensureParent: a sibling generation in // flight when the parent's population is read lands, and ends its // registration, before the registry is asked. Read in that order, the // parent would look cold (the kernel read predates the process, the // registry no longer lists it) and its limits, which a swap may have raised, // would be reset. The hook ends the placement right after the kernel read. func TestEnsureParentCountsASiblingThatLandsBetweenTheReads(t *testing.T) { const host = "h.example.com" m, parent := genManager(t, host) fakeStart(t, func(*exec.Cmd) error { return nil }) writeFixture(t, filepath.Join(parent, "cgroup.events"), "populated 0\nfrozen 0\n") writeFixture(t, filepath.Join(parent, "cgroup.subtree_control"), "cpu memory io pids\n") writeFixture(t, filepath.Join(parent, "memory.max"), "1073741824") done := m.beginPlacement(filepath.Join(parent, GenChild(1))) defer done() hooked := false setPlacementReadHook(t, func(string) { if !hooked { hooked = true done() // g1's shim answered placed: the registry drops it } }) if _, err := m.StartChildGated(host, GenChild(2), TierDefault, TierDefault, buildTrue, nil); err != nil { t.Fatalf("StartChildGated beside a sibling that landed: %v", err) } if !hooked { t.Fatal("the read hook never ran: the test proves nothing") } if got := readFixture(t, filepath.Join(parent, "memory.max")); got != "1073741824" { t.Errorf("the parent's memory.max was rewritten to %q: a sibling that landed between the reads was taken for a cold parent", got) } if got := readFixture(t, filepath.Join(parent, "cgroup.subtree_control")); got != "cpu memory io pids\n" { t.Errorf("the parent's subtree_control was rewritten to %q", got) } } // Finding 1, the other schedule at ensureParent: a sibling placement that // tries to register after the parent was found cold must wait until the // cold parent's controllers and limits are written. Otherwise the reset // could land on a parent that already has a live generation under it. The // hook starts the sibling's registration right after the population read // and waits until the registration reports that a hold stopped it. func TestEnsureParentHoldsPlacementsUntilTheColdParentIsWritten(t *testing.T) { const host = "h.example.com" m, parent := genManager(t, host) fakeStart(t, func(*exec.Cmd) error { return nil }) writeFixture(t, filepath.Join(parent, "cgroup.events"), "populated 0\nfrozen 0\n") blocked := make(chan struct{}) var once sync.Once origHeld := placementHeldHook placementHeldHook = func(string) { once.Do(func() { close(blocked) }) } t.Cleanup(func() { placementHeldHook = origHeld }) type seen struct { max, subtree string done func() } registered := make(chan seen, 1) hooked := false setPlacementReadHook(t, func(string) { if hooked { return } hooked = true go func() { done := m.beginPlacement(filepath.Join(parent, GenChild(1))) // What the sibling's start would find once it is registered. max, _ := os.ReadFile(filepath.Join(parent, "memory.max")) sub, _ := os.ReadFile(filepath.Join(parent, "cgroup.subtree_control")) registered <- seen{string(max), string(sub), done} }() select { case <-blocked: case r := <-registered: r.done() t.Errorf("a sibling registered while the cold parent was being written (memory.max %q)", r.max) case <-time.After(5 * time.Second): t.Error("the sibling's registration neither registered nor waited on a hold") } }) if _, err := m.StartChildGated(host, GenChild(2), TierDefault, TierDefault, buildTrue, nil); err != nil { t.Fatalf("StartChildGated on a cold parent: %v", err) } if !hooked { t.Fatal("the read hook never ran: the test proves nothing") } select { case r := <-registered: defer r.done() if want := strconv.FormatInt(TierDefault.MemoryMax, 10); r.max != want { t.Errorf("the sibling registered with the parent's memory.max %q, want the cold parent's tier %q written first", r.max, want) } if !strings.Contains(r.subtree, "+memory") { t.Errorf("the sibling registered with the parent's subtree_control %q, want the controllers delegated first", r.subtree) } case <-time.After(5 * time.Second): t.Fatal("the sibling's registration was never released after the parent was written") } if n := m.placementsUnder(parent); n != 1 { t.Fatalf("%d placements registered, want the sibling's one", n) } } // setPlacementReadHook sets placementReadHook for one test. func setPlacementReadHook(t *testing.T, f func(dir string)) { t.Helper() orig := placementReadHook placementReadHook = f t.Cleanup(func() { placementReadHook = orig }) }