//go:build linux package cgroup import ( "bytes" "context" "errors" "fmt" "os" "os/exec" "path/filepath" "strconv" "strings" "syscall" "time" "golang.org/x/sys/unix" ) // placementCheckName is the scratch child of the delegated root that // VerifyPlacement places its probe in. It is no carlos-exec@ name, so no // host's cleanup or the stale sweep ever takes it for a host's cgroup. const placementCheckName = "carlos-placement-check" // placementProbeArgs is the probe tenant's argv after shimExe. Only tests // change it, to stand in a probe that misreports or never exits. var placementProbeArgs = []string{ProbeVerb} // placementCheckBudget bounds VerifyPlacement's waits, the shim's two and // the probe's run, so a decline is decided in time for the restarted edge // to answer `carlos system update`'s 20 s gate (systemGateWait) with the // rest of its boot work. What runs after the budget is the reap of a // SIGKILLed shim or probe and an rmdir. That is milliseconds on the // assumption that the kernel ends a killed process promptly, which the // second of headroom under 5 s covers. It is an assumption, not a bound: // SIGKILL does not end a process in uninterruptible sleep (stuck kernel // I/O) until that sleep ends, and the reap waits for it (reap, in // shim_linux.go; awaitProbe). A box stalled that way can take longer than // 4 s to decline (whole-branch review, finding 4). const placementCheckBudget = 4 * time.Second // VerifyPlacement does one real placement, through the shim, into a scratch // child cgroup with the default tier's limits, and answers nil only if the // placed process saw itself in that cgroup and the limit reads back. It is // the arm-time question for CARLOS_SWAP=overlap (#272): the second dev-box // check found a box that armed and then failed every generation start, // because no placement was possible under carlos-edge.service's hardening. // // It calls the shim itself (startViaShimBy), never the placeStart seam, so a // test's FakePlacementForTest cannot make it pass. Its waits end at ctx's // deadline or after placementCheckBudget, whichever is sooner; the shim's // waits are capped at the same instant. The reap of a process it killed // then follows, outside that budget (placementCheckBudget says why). The // scratch cgroup is removed once the probe is reaped; a removal failure is // logged, not returned. func (m *Manager) VerifyPlacement(ctx context.Context) error { if m == nil { return ErrNoDelegation } if !shimAvailable() { return errors.New("cgroup: placement check: the placement shim is not available in this process") } ctx, cancel := context.WithTimeout(ctx, placementCheckBudget) defer cancel() by, _ := ctx.Deadline() dir := filepath.Join(m.root, placementCheckName) // A leftover from an earlier run is empty: the unit's restart killed // its members. One that will not go fails the Mkdir below. _ = os.Remove(dir) // Nothing is created under a root that is no cgroup: the limit files // would be plain files there, and the directory could not be removed. var st unix.Statfs_t if err := unix.Statfs(m.root, &st); err != nil { return fmt.Errorf("cgroup: placement check: statfs %s: %w", m.root, err) } if st.Type != unix.CGROUP2_SUPER_MAGIC { return fmt.Errorf("cgroup: placement check: %s is not a cgroup2 directory", m.root) } if err := os.Mkdir(dir, 0o755); err != nil { return fmt.Errorf("cgroup: placement check: %w", err) } // Every path below reaps any probe it started before it returns, so // this runs on an empty cgroup. defer func() { if err := os.Remove(dir); err != nil { m.logf("cgroup: placement check: %s not removed: %v", dir, err) } }() if err := m.applyLimit("placement check", dir, TierDefault, true); err != nil { return fmt.Errorf("cgroup: placement check: %w", err) } var out bytes.Buffer cmd := exec.Command(shimExe, placementProbeArgs...) cmd.Env = []string{} cmd.Stdout = &out if _, err := m.startViaShimBy(dir, cmd, nil, by); err != nil { // A placement error or not, the check failed; the shim call has // reaped anything it started. return fmt.Errorf("cgroup: placement check: %w", err) } if err := awaitProbe(ctx, cmd); err != nil { return fmt.Errorf("cgroup: placement check: %w", err) } // What the probe saw in /proc/self/cgroup: dir as startViaShim names it // to the shim, below /sys/fs/cgroup. expected := strings.TrimPrefix(dir, "/sys/fs/cgroup") if got := out.String(); got != expected+"\n" { return fmt.Errorf("cgroup: placement check: the probe reported cgroup %q, want %q", got, expected+"\n") } raw, err := os.ReadFile(filepath.Join(dir, "memory.max")) if err != nil { return fmt.Errorf("cgroup: placement check: %w", err) } if got, want := strings.TrimSpace(string(raw)), strconv.FormatInt(TierDefault.MemoryMax, 10); got != want { return fmt.Errorf("cgroup: placement check: memory.max reads %q, want %q", got, want) } return nil } // awaitProbe reaps the started probe: nil when it exited 0, its exit as the // error otherwise. When ctx ends first the probe is SIGKILLed and reaped // before this returns, and the check fails on ctx's error. The judgement is // the reaped status, never which channel was ready first: a probe that had // already exited when ctx ended is a zombie the kill cannot touch, so it is // judged by its own exit, not declined as one that hung. func awaitProbe(ctx context.Context, cmd *exec.Cmd) error { waited := make(chan error, 1) go func() { waited <- cmd.Wait() }() var werr error select { case werr = <-waited: case <-ctx.Done(): _ = cmd.Process.Kill() // Bounded once the kernel has ended the killed probe, which holds no // other process; not by ctx (placementCheckBudget). werr = <-waited var ee *exec.ExitError if errors.As(werr, &ee) { if ws, ok := ee.Sys().(syscall.WaitStatus); ok && ws.Signaled() && ws.Signal() == syscall.SIGKILL { return fmt.Errorf("the probe did not exit: %w", ctx.Err()) } } } if werr != nil { return fmt.Errorf("the probe failed: %w", werr) } return nil }