//go:build linux package stopevidence import ( "context" "errors" "fmt" "os" "path" "path/filepath" "slices" "sort" "strconv" "strings" "time" "github.com/carlosframework/platform/internal/cgroup" "github.com/carlosframework/platform/internal/controlname" "github.com/carlosframework/platform/internal/moves" ) // treeDrainLimit bounds step 1's kill-and-drain when the caller's context // has no earlier deadline (KillAndEmpty refuses a context without one). It // matches the activator's strictDrainTimeout. const treeDrainLimit = 30 * time.Second // treeGroup is the host cgroup as step 1 uses it: *cgroup.Cleanup, or a // fake in tests. type treeGroup interface { KillAndEmpty(context.Context) error Close() error } // Reconstruct is §5.3's evidence from durable, observable state: it reads // nothing from the agent's memory, so a restarted agent gives the same // answer. It runs steps 1 to 4 and records each in the report's Steps; the // caller does step 5 (the final checkpoint and the digests, then // StepReplica), since only it knows the replica it closes. Role, Box, // Incarnation, ClaimGen and OpID are the caller's to fill in too. // // A step that observed a failure is a Check with OK false and a reason; // the pass retries until its stop evidence budget runs out. A step that // could not observe (an unreadable /proc or cgroup file, a lease read // error, the caller's context ending) is a returned error, and the report // beside it is partial and must not be written. // // Reconstruct takes neither per-host gate. schedule.HostGate is the timers // drain's gate (the runner, the converger and moveCfg.TimersDrained); the // activator's strictGates (ExecBackend.gate) serialise a strict spawn with // its own exit cleanup. By the time the cut calls this, the registry hold // and the activator's stop fence have closed admission, so nothing spawns // for the host; the kill works on the group whoever last owned it, and a // strict owner's concurrent cleanup only kills and drains the same group. func Reconstruct(ctx context.Context, in ReconstructInputs) (moves.StoppedReport, error) { if in.Cgroups == nil { return moves.StoppedReport{V: 1, Host: in.Host}, ErrInvalidInput } m := in.Cgroups return reconstruct(ctx, in, func(host string) (treeGroup, error) { c, err := m.OpenCleanup(host) if err != nil { return nil, err } return c, nil }) } func reconstruct(ctx context.Context, in ReconstructInputs, open func(string) (treeGroup, error)) (moves.StoppedReport, error) { rep := moves.StoppedReport{V: 1, Host: in.Host, Steps: map[string]moves.Check{}} if ctx == nil || open == nil || in.LeaseFree == nil || !controlname.ValidHostShape(in.Host) { return rep, ErrInvalidInput } procRoot := in.ProcRoot if procRoot == "" { procRoot = "/proc" } if !filepath.IsAbs(procRoot) { return rep, ErrInvalidInput } dbFiles, err := dbPathsWithSidecars(in.DBPaths) if err != nil { return rep, err } if err := ctx.Err(); err != nil { return rep, err } // 1. Kill and drain the tree: cgroup.kill on carlos-exec@, then // wait for it to empty. Phase 1 moves flat exec instances only (S1 // refuses a nested one), so this is the whole of step 1. tree, err := stopTree(ctx, in.Host, open) if err != nil { return rep, err } rep.Steps[moves.StepTree] = tree // 2. No stray process in the agent's own leaf. A spawn that escaped its // host cgroup (best-effort placement before the move went strict) lands // here, which is why an absent host cgroup counts as an empty tree only // alongside this step. So does any short-lived child the agent forks // for itself: it fails StepStray for this pass, and the cut's retry // until its deadline is the intended behaviour. Only a repeat is worth // the alert (M-C20). The leaf is read again after steps 3 and 4 (below). self := os.Getpid() first, err := agentLeaf(procRoot, self) if err != nil { return rep, err } rep.Steps[moves.StepStray] = strayCheck(first, self) // 3. No open database. scanFDs is FDHolders with the pass's context: it // matches by path text and by device and inode, so a hard link, a // symlinked path or another mount namespace's bind path is the same // file. The scan covers every process /proc shows. On a box the // agent runs under ProtectProc=invisible (carlos-edge.service), so /proc // shows only the carlos uid's processes: §5.3's "restricted to the // carlos uid". A process it can see but not read makes the scan // incomplete, which is an error, never a pass. holders, err := scanFDs(ctx, procRoot, dbFiles) if err != nil { return rep, err } if len(holders) == 0 { rep.Steps[moves.StepFDs] = moves.Check{OK: true} } else { rep.Steps[moves.StepFDs] = moves.Check{Reason: fmt.Sprintf("%d process(es) hold the database open", len(holders))} } // 4. Lease released. free, err := in.LeaseFree(ctx) if err != nil { return rep, err } if free { rep.Steps[moves.StepLease] = moves.Check{OK: true} } else { rep.Steps[moves.StepLease] = moves.Check{Reason: "lock.json still held"} } // 2, again. The first read is a snapshot taken before the descriptor // scan and the lease read, which take time; a process that joined the // agent's leaf meanwhile and holds no database descriptor would pass // both. Any pid in the second read but the agent fails StepStray for // this pass, and the cut's retry covers it. Rep.Stray lists the strays // of both reads. second, err := agentLeaf(procRoot, self) if err != nil { return rep, err } if rep.Steps[moves.StepStray].OK { rep.Steps[moves.StepStray] = strayCheck(second, self) } rep.Stray = strays(self, first, second) // The evidence time is when the last step was observed, not when the // pass began: §5.3's late-completion horizon runs from it, and a later // time is the safe direction. rep.CheckedAt = time.Now().UTC() return rep, nil } // stopTree is step 1. A group that never existed, or that its last strict // owner removed, is an empty tree. A drain that runs out of time is an // observed failure; any other failure to open, kill or read the group is // an error. func stopTree(ctx context.Context, host string, open func(string) (treeGroup, error)) (moves.Check, error) { c, err := open(host) if errors.Is(err, cgroup.ErrNoCgroup) { return moves.Check{OK: true}, nil } if err != nil { return moves.Check{}, err } defer c.Close() drain, cancel := context.WithTimeout(ctx, treeDrainLimit) defer cancel() err = c.KillAndEmpty(drain) switch { case err == nil: return moves.Check{OK: true}, nil case ctx.Err() != nil: return moves.Check{}, ctx.Err() case errors.Is(err, context.DeadlineExceeded): return moves.Check{Reason: "the instance's processes could not be stopped"}, nil } return moves.Check{}, err } // strayCheck is StepStray's verdict on one read of the agent's leaf. A pid // other than the agent's fails it with that pid. A 0 entry (the kernel's // name for a member outside this process's pid namespace) fails it too: it // is a process the box cannot identify, so it is never taken for the agent // and never for nothing. func strayCheck(pids []int, self int) moves.Check { unknown := 0 for _, p := range pids { switch { case p == 0: unknown++ case p != self: return moves.Check{Reason: fmt.Sprintf("pid %d in the agent's cgroup is not the agent", p)} } } if unknown > 0 { return moves.Check{Reason: fmt.Sprintf("%d process(es) in the agent's cgroup cannot be identified", unknown)} } return moves.Check{OK: true} } // agentLeaf reads the agent's own leaf and requires the agent in it: a // listing without the agent is the wrong directory, an error. func agentLeaf(procRoot string, self int) ([]int, error) { pids, err := ownLeafProcs(procRoot) if err != nil { return nil, err } if !slices.Contains(pids, self) { return nil, fmt.Errorf("stopevidence: the agent (pid %d) is not in the cgroup its /proc entry names", self) } return pids, nil } // strays is every pid but self across the reads, sorted, each once. A 0 // is the marker for members the box could not identify (once, however many // there were; the step's reason carries the count). func strays(self int, reads ...[]int) []int { var out []int for _, pids := range reads { for _, p := range pids { if p != self && !slices.Contains(out, p) { out = append(out, p) } } } sort.Ints(out) return out } // dbPathsWithSidecars is every database file with its -wal and -shm, clean, // sorted and without repeats. Each path must be absolute, and there must be // at least one: a cut always has a database path, so an empty list is a // caller's mistake, not "nothing to hold open". func dbPathsWithSidecars(paths []string) ([]string, error) { if len(paths) == 0 { return nil, ErrInvalidInput } seen := map[string]bool{} for _, p := range paths { if !filepath.IsAbs(p) { return nil, ErrInvalidInput } p = filepath.Clean(p) seen[p], seen[p+"-wal"], seen[p+"-shm"] = true, true, true } out := make([]string, 0, len(seen)) for p := range seen { out = append(out, p) } sort.Strings(out) return out, nil } // ownLeafProcs reads the pids in the agent's own cgroup leaf: the v2 path // /self/cgroup names, under the cgroup2 mount // /self/mountinfo names (ruling G32: never a hard-coded // /sys/fs/cgroup). func ownLeafProcs(procRoot string) ([]int, error) { self, err := os.ReadFile(filepath.Join(procRoot, "self", "cgroup")) if err != nil { return nil, err } leaf, ok := cgroup.ParseSelfCgroup(string(self)) if !ok { return nil, errors.New("stopevidence: own cgroup unreadable") } mounts, err := os.ReadFile(filepath.Join(procRoot, "self", "mountinfo")) if err != nil { return nil, err } dir, err := cgroupLeafDir(string(mounts), leaf) if err != nil { return nil, err } return readProcs(filepath.Join(dir, "cgroup.procs")) } // cgroupLeafDir is the directory of the v2 cgroup leaf (as // ParseSelfCgroup returns it, relative to the hierarchy's root) under the // cgroup2 mount that contains it. A mount's root field is the part of the // hierarchy it shows; the mount whose root holds the leaf most closely // wins, and of equal roots the first listed. A leaf that climbs out of // the hierarchy (a cgroup outside the agent's cgroup namespace shows as // "/../..") or that no cgroup2 mount shows is an error. func cgroupLeafDir(mountinfo, leaf string) (string, error) { abs := "/" + leaf if path.Clean(abs) != abs { return "", fmt.Errorf("stopevidence: own cgroup %q is outside the hierarchy", leaf) } best, bestRoot := "", -1 for _, line := range strings.Split(mountinfo, "\n") { fields := strings.Fields(line) if len(fields) < 7 { continue } sep := slices.Index(fields[6:], "-") if sep < 0 || 6+sep+1 >= len(fields) || fields[6+sep+1] != "cgroup2" { continue } root, mnt := unescapeMount(fields[3]), unescapeMount(fields[4]) if !filepath.IsAbs(mnt) { continue } var rel string switch { case root == "/": rel = abs case abs == root: rel = "/" case strings.HasPrefix(abs, root+"/"): rel = strings.TrimPrefix(abs, root) default: continue } if len(root) > bestRoot { best, bestRoot = filepath.Join(mnt, rel), len(root) } } if bestRoot < 0 { return "", fmt.Errorf("stopevidence: no cgroup2 mount shows own cgroup %q", leaf) } return best, nil } // unescapeMount undoes mountinfo's octal escapes (\040 for a space, \011, // \012, \134). func unescapeMount(s string) string { if !strings.Contains(s, `\`) { return s } var b strings.Builder for i := 0; i < len(s); i++ { if s[i] == '\\' && i+4 <= len(s) { if v, err := strconv.ParseUint(s[i+1:i+4], 8, 8); err == nil { b.WriteByte(byte(v)) i += 3 continue } } b.WriteByte(s[i]) } return b.String() } // readProcs parses cgroup.procs: one pid per line. The kernel lists pid 0 // for a member that is not visible in the reader's pid namespace; it is // returned as 0, for the caller to treat as an unidentifiable member. Any // other entry that is not a plain non-negative number is an error. func readProcs(file string) ([]int, error) { raw, err := os.ReadFile(file) if err != nil { return nil, err } var pids []int for _, line := range strings.Split(string(raw), "\n") { line = strings.TrimSpace(line) if line == "" { continue } pid, err := strconv.Atoi(line) if err != nil || pid < 0 || line[0] == '+' || line[0] == '-' { return nil, fmt.Errorf("stopevidence: %s: invalid pid %q", file, line) } pids = append(pids, pid) } return pids, nil }