// The box-side half of the metrics sampler: every function here reads local // kernel/OS state (cgroup v2 and /proc files) and MUST fail toward nil, never // a guessed zero. This file is thin OS glue — os.ReadFile and os.ReadDir // wrapping the pure parsers in internal/metrics — and the readers are // deliberately untested by the same convention as probes.go readers; // internal/metrics tests exercise the parsers with fixtures instead. The // exceptions are listCarlosUnits, listStampedUnits and listExecCgroups: they are directory // WALKs, not single reads, and which directories they walk is a decision that // has already been wrong once (see listCarlosUnits' doc comment), so they take // their roots as arguments and are pinned by metrics_probes_test.go against // temp-dir fixtures. package host import ( "os" "path/filepath" "strings" "github.com/carlosframework/platform/internal/cgroup" "github.com/carlosframework/platform/internal/metrics" ) // readProcStat reads /proc/stat's aggregate cpu line and parses its counters. // Read failure or parse failure -> nil. func readProcStat() *metrics.CPUCounters { b, err := os.ReadFile("/proc/stat") if err != nil { return nil } return metrics.ParseProcStat(string(b)) } // readLoadAvg reads /proc/loadavg and parses the 1/5/15-minute load averages. // Any read or parse failure (too few fields, non-numeric values) -> all three // results nil, since a partial load average is not a value worth publishing // on its own. func readLoadAvg() (l1, l5, l15 *float64) { b, err := os.ReadFile("/proc/loadavg") if err != nil { return nil, nil, nil } return metrics.ParseLoadAvg(string(b)) } // cgroupSliceDir is the cgroup v2 directory where CARLOS units report their // resource usage. It is the default root for listCarlosUnits, which takes the // root as an argument only so the discovery walk below can be tested against // a temp-dir fixture instead of the live kernel. const cgroupSliceDir = "/sys/fs/cgroup/system.slice" // cgroupCarlosSliceDir is the second root listCarlosUnits and // listStampedUnits must scan: carlos-edge.service now lives in carlos.slice, // not system.slice, because the isolated-exec design (§4.3) moved the edge // there to share its memory wall with the exec-child instances it supervises // (see internal/host/units/carlos-edge.service's Slice=carlos.slice). On a // flat fleet — no tenant instances yet delegated their own cgroups — the // edge's own leaf is still the hierarchical total of everything under it, // delegated exec children included, so publishing that same // memory.current/cpu.stat/io.stat reading under the unchanged unit-name key // "carlos-edge.service" preserves the number operators already look at; only // the directory that number is read from has moved. const cgroupCarlosSliceDir = "/sys/fs/cgroup/carlos.slice" // unitCgroup is one discovered CARLOS unit: its systemd unit name (the key // everything downstream — sample maps, the console's activator.UnitName // match — uses) and the cgroup directory that unit's files actually live in. // The two are NOT interchangeable: only flat units sit at // /, and rebuilding the path from the name would miss every // tenant instance (see listCarlosUnits). type unitCgroup struct { name string dir string } // listCarlosUnits lists all active CARLOS systemd units under root by // filtering directory names for the "carlos-" prefix and ".service" suffix. // Read failure -> nil. // // Two levels are scanned, because systemd puts CARLOS's two kinds of unit in // two different places: // // - Flat, directly under root: the platform's fixed units // (carlos-edge.service, carlos-litestream, carlos-adopt, // carlos-restore-verify) — plain non-templated services land in // system.slice itself. // - Nested one level down, inside a "system-*.slice" child: INSTANTIATED // units. carlos-app@.service sets no Slice= (see // internal/host/units/carlos-app@.service), so systemd places every // instance of the template in the implicit per-template slice // system.slice/system-carlos\x2dapp.slice/. Scanning only the top level // therefore finds every platform unit and not a single tenant — the exact // set the per-unit table and the owner app-metrics page exist to show. // // Only these two levels are walked: a unit's own cgroup directory has // children of its own on some systems, and nothing of ours ever nests deeper // than one slice. func listCarlosUnits(root string) []unitCgroup { entries, err := os.ReadDir(root) if err != nil { return nil } var units []unitCgroup for _, entry := range entries { if !entry.IsDir() { continue } name := entry.Name() if isCarlosUnitName(name) { units = append(units, unitCgroup{name: name, dir: filepath.Join(root, name)}) continue } if !strings.HasPrefix(name, "system-") || !strings.HasSuffix(name, ".slice") { continue } sliceDir := filepath.Join(root, name) nested, err := os.ReadDir(sliceDir) if err != nil { continue // an unreadable slice loses its units, never the whole scan } for _, child := range nested { if !child.IsDir() || !isCarlosUnitName(child.Name()) { continue } units = append(units, unitCgroup{name: child.Name(), dir: filepath.Join(sliceDir, child.Name())}) } } return units } // listStampedUnits finds the cgroup directories of operator-stamped bespoke // units — registry Route.Unit (#110), the routes the carlos- naming // convention above cannot see (an operator's console.service, tito's // titogo-canary@.service). Read failure -> nil, same as // listCarlosUnits. // // Same two-level walk, for the same reason: a flat unit sits under root, a // template instance inside its implicit system-*.slice. The stamped NAME says // nothing about which of the two systemd chose, so both levels are matched // against the same wanted set. // // A stamped unit with no cgroup dir under root is simply not returned — no // reading is the honest report for a unit that isn't running, and the console // renders that absence rather than a guessed zero. Names arrive from the // registry already shaped by registry.ValidUnitName (the door validates; see // SetUnit), so nothing here re-validates them: an unmatchable name matches // nothing, which is the same outcome as a stopped unit. func listStampedUnits(root string, stamped []string) []unitCgroup { if len(stamped) == 0 { return nil } want := make(map[string]bool, len(stamped)) for _, u := range stamped { want[u] = true } entries, err := os.ReadDir(root) if err != nil { return nil } var units []unitCgroup for _, entry := range entries { if !entry.IsDir() { continue } name := entry.Name() if want[name] { units = append(units, unitCgroup{name: name, dir: filepath.Join(root, name)}) continue } if !strings.HasPrefix(name, "system-") || !strings.HasSuffix(name, ".slice") { continue } sliceDir := filepath.Join(root, name) nested, err := os.ReadDir(sliceDir) if err != nil { continue // an unreadable slice loses its units, never the whole scan } for _, child := range nested { if !child.IsDir() || !want[child.Name()] { continue } units = append(units, unitCgroup{name: child.Name(), dir: filepath.Join(sliceDir, child.Name())}) } } return units } // isCarlosUnitName reports whether a cgroup directory name is one of ours: // the "carlos-" prefix and the ".service" suffix, with at least one character // between them. func isCarlosUnitName(name string) bool { return len(name) > len("carlos-.service") && strings.HasPrefix(name, "carlos-") && strings.HasSuffix(name, ".service") } // unitRaw is a single reading of a systemd unit's resource usage from its // cgroup v2 files. Each field represents a metric that may have failed to // read independently. type unitRaw struct { cpuUsec *int64 memBytes *int64 memPeakBytes *int64 readBytes *int64 writeBytes *int64 } // readUnitRaw reads a unit's resource usage from the cgroup v2 files in // unitPath — the directory listCarlosUnits reported for that unit, which for // an instantiated unit is NOT /. Each file read failure // sets only that field to nil; the others still read independently. Parse // failures also yield nil for that field only. func readUnitRaw(unitPath string) unitRaw { // Read cpu.stat for usage_usec raw := unitRaw{} if b, err := os.ReadFile(filepath.Join(unitPath, "cpu.stat")); err == nil { raw.cpuUsec = metrics.ParseCgroupCPUUsec(string(b)) } // Read memory.current if b, err := os.ReadFile(filepath.Join(unitPath, "memory.current")); err == nil { raw.memBytes = metrics.ParseCgroupInt(string(b)) } // Read memory.peak if b, err := os.ReadFile(filepath.Join(unitPath, "memory.peak")); err == nil { raw.memPeakBytes = metrics.ParseCgroupInt(string(b)) } // Read io.stat for rbytes and wbytes if b, err := os.ReadFile(filepath.Join(unitPath, "io.stat")); err == nil { rbytes, wbytes := metrics.ParseCgroupIO(string(b)) raw.readBytes = rbytes raw.writeBytes = wbytes } return raw } // listExecCgroups lists the per-host exec-child cgroups under the agent's // delegated subtree — the third scan root, alongside listCarlosUnits' two // levels. An empty root (no delegation armed) or a read failure -> nil. The // agent's own supervise leaf fails IsExecName and is skipped: that process // is already reported as carlos-edge.service. A dir whose cgroup.procs is // empty or unreadable is skipped too: a failed seed or an aborted wake // leaves the dir behind (until a later exit's rmdir or the next agent // start's sweep), and reporting it would render a host whose wake FAILED // as a live instance at zero — absence is the data, never a guessed zero. // readUnitRaw reads the included dirs unchanged — same cgroup v2 files, // same fail-toward-nil fields. func listExecCgroups(root string) []unitCgroup { if root == "" { return nil } entries, err := os.ReadDir(root) if err != nil { return nil } var units []unitCgroup for _, entry := range entries { if !entry.IsDir() || !cgroup.IsExecName(entry.Name()) { continue } dir := filepath.Join(root, entry.Name()) procs, err := os.ReadFile(filepath.Join(dir, "cgroup.procs")) if err != nil { continue } // A generation parent (zero-downtime spec §7.3) holds no process of // its own; its generations do. cgroup.events "populated 1" is the // kernel's word that something under it is alive. if len(strings.TrimSpace(string(procs))) == 0 && !subtreePopulated(dir) { continue } units = append(units, unitCgroup{name: entry.Name(), dir: dir}) } return units } // subtreePopulated reports whether dir's cgroup.events says a process is // alive anywhere under it. Unreadable is no: absence is the data. func subtreePopulated(dir string) bool { b, err := os.ReadFile(filepath.Join(dir, "cgroup.events")) return err == nil && strings.Contains(string(b), "populated 1") }