// The box-side half of the fleet status page's Probes contract // (internal/status.Probes): every function here reads local kernel/OS state // and MUST fail toward nil, never a guessed zero — status.CollectBox already // treats a nil probe as "unknown, publish null" everywhere, so a probe that // invents a number on failure would silently turn an honest unknown into a // false reading on the public page. Deliberately small, single-purpose // functions (rather than one combined "gather everything" call) so // host_test.go's statusTick tests never have to touch /proc or a live // syscall — they inject a status.Probes fixture directly and never call // anything in this file. package host import ( "bufio" "io" "os" "os/exec" "strconv" "strings" "sync" "github.com/carlosframework/platform/internal/status" "github.com/carlosframework/platform/internal/registry" ) // adoptTimerUnit is the unit whose liveness IS "this box adopts releases on // its own" — the same name bootstrap installs and arms (see bootstrap.go's // enabledUnits). const adoptTimerUnit = "carlos-adopt.timer" // adoptTimerActive asks systemd whether the adoption timer is ACTIVE, not // merely enabled. The distinction is the whole point: `systemctl enable` // writes a .wants symlink and nothing more, so a timer can sit enabled and // never armed until the box next reboots — which is exactly how the flagship // ran for weeks without a single automatic adoption. `is-active` is the only // question that distinguishes the two. // // The exit status is deliberately ignored in favour of the printed word: // is-active exits non-zero for every not-active state while still printing // which state it is, so the error would tell us less than the output already // does. A systemctl that couldn't run at all prints nothing, and nothing maps // to nil. func adoptTimerActive() *bool { out, _ := exec.Command("systemctl", "is-active", adoptTimerUnit).Output() return parseIsActive(string(out)) } // configPathUnit is the unit whose liveness IS "config reaches this box's // unit-backed routes in seconds" (issue #44): the path unit watching // /data/config-sync-requested, which the agent touches and which starts // carlos-config-sync.service. Installed and armed by bootstrap alongside the // timer (see enabledUnits). const configPathUnit = "carlos-adopt.path" // configPathActive asks the same question adoptTimerActive asks, about the // unit with the same failure mode: `systemctl enable` on a path unit writes a // .wants symlink and watches nothing until the box next reboots. An unarmed // path unit is invisible from inside the agent — it writes its marker file // happily, and nothing ever reads it — so this probe is the only thing that // can tell an operator the box quietly fell back to the hourly config floor. func configPathActive() *bool { out, _ := exec.Command("systemctl", "is-active", configPathUnit).Output() return parseIsActive(string(out)) } // restartPathUnit is the unit whose liveness IS "a member's `carlos restart` // reaches this box's units in seconds" (issue #113): the path unit watching // /data/restart-requested, which the agent touches and which starts // carlos-restart.service. Installed and armed by bootstrap alongside the other // two (see enabledUnits). const restartPathUnit = "carlos-restart.path" // restartPathActive asks the same question configPathActive asks, about a unit // with the same silent failure mode — and one worse consequence. An unarmed // config trigger costs an env var an hour; an unarmed restart trigger means // someone typed `carlos restart` at a wedged instance, was told "requested", // and will stand there watching a process that will not cycle until the hourly // pass comes round. Nothing inside the agent can see the difference: it writes // its marker either way. func restartPathActive() *bool { out, _ := exec.Command("systemctl", "is-active", restartPathUnit).Output() return parseIsActive(string(out)) } // systemUpdatePathUnit is the unit whose liveness IS "an agent's fetched // platform candidate reaches a root apply" (spec §4): the path unit watching // /data/system-update-requested. Installed and armed by bootstrap alongside // the other path units (see enabledUnits). const systemUpdatePathUnit = "carlos-system-update.path" // systemUpdatePathActive asks the same question restartPathActive asks, about // the self-update relay: an unarmed path here means an agent that fetched and // verified a new build touches its marker and nothing ever picks it up. func systemUpdatePathActive() *bool { out, _ := exec.Command("systemctl", "is-active", systemUpdatePathUnit).Output() return parseIsActive(string(out)) } // parseIsActive maps systemd's is-active vocabulary onto the tri-state the // status page publishes. "unknown" — systemd's answer when it has no such unit // loaded — is a genuine false: a box with no adoption timer adopts nothing. // Anything this function does not recognise (empty output because systemctl // isn't there, a word a future systemd invents) is nil, because the one answer // this probe must never invent is a reassuring one. func parseIsActive(out string) *bool { yes, no := true, false switch strings.TrimSpace(out) { case "active": return &yes case "inactive", "failed", "deactivating", "unknown": return &no } return nil } // cachedInstanceID reads this box's cloud machine id once and remembers it // — the id cannot change under a running process, and the status loop asks // every five minutes. // // Only a SUCCESSFUL read is cached. An empty answer is retried next tick, // because the two reasons for one are not alike: off-cloud it stays empty // forever and costs a couple of unroutable milliseconds a tick, while on a // box whose metadata service was briefly unreachable at boot, caching the // empty would cost the console's box-to-machine join for the whole life of // the agent — cured only by a restart nobody would know to perform. // // This is the one probe in this file that reaches off the box, and it obeys // the same rule as the rest: it fails toward "", which CollectBox publishes // as absence, never a guess. var ( instanceIDMu sync.Mutex instanceIDSeen string ) func cachedInstanceID() string { instanceIDMu.Lock() defer instanceIDMu.Unlock() if instanceIDSeen != "" { return instanceIDSeen } instanceIDSeen = status.InstanceID() return instanceIDSeen } // readMachineID reads /etc/machine-id, trimmed. Unreadable (missing file, // permission denied — plausible inside some containers) -> "", NOT a // hash-of-hostname fallback. A hostname-derived id would look exactly like a // real one to status.Label, silently manufacturing a stable-looking label // for a box that never actually reported a machine id; "" is honest instead // — it still hashes to a deterministic Label(""), it just always means "this // probe never worked," which is the true state of affairs. func readMachineID() string { b, err := os.ReadFile("/etc/machine-id") if err != nil { return "" } return strings.TrimSpace(string(b)) } // readMemFrac reads /proc/meminfo's MemAvailable/MemTotal (both in kB) and // returns the used fraction (1 - available/total). MemAvailable (not // MemFree) is the kernel's own "how much could a new process actually get // without swapping" estimate — it already accounts for reclaimable // page/slab cache, which MemFree does not, so MemFree would read as // permanently "critical" on any box with a healthy page cache. Either field // missing or unparsable -> nil. func readMemFrac() *float64 { f, err := os.Open("/proc/meminfo") if err != nil { return nil } defer f.Close() var totalKB, availKB int64 haveTotal, haveAvail := false, false scanner := bufio.NewScanner(f) for scanner.Scan() { fields := strings.Fields(scanner.Text()) if len(fields) < 2 { continue } v, err := strconv.ParseInt(fields[1], 10, 64) if err != nil { continue } switch fields[0] { case "MemTotal:": totalKB, haveTotal = v, true case "MemAvailable:": availKB, haveAvail = v, true } } if err := scanner.Err(); err != nil || !haveTotal || !haveAvail || totalKB <= 0 { return nil } frac := float64(totalKB-availKB) / float64(totalKB) return &frac } // readDiskFrac reports how full the data volume is (1 - available/total), // via diskBytes on dataDir() — /data by default, CARLOS_DATA_DIR when // set, matching every other on-box path in this package. There is no // existing "floor" config to mirror: ops/litestream (grepped before writing // this) carries no reserved-space setting, so this is the plain fraction, // not a fraction-of-headroom-above-a-floor. Statfs failure (unmounted, // missing dir) -> nil; a zero-block filesystem (shouldn't happen, but would // divide by zero) also -> nil rather than a crash. func readDiskFrac() *float64 { total, avail, err := diskBytes(dataDir()) if err != nil || total == 0 { return nil } frac := 1 - float64(avail)/float64(total) return &frac } // listeningProbe reads /proc/net/unix — the kernel's own table of bound // unix-domain sockets — rather than dialing sock. Dialing would itself be a // connection attempt on a hibernating tenant's socket, which is exactly the // event that wakes it: the probe would falsify the very thing it's trying to // observe. Presence of sock's path in the table is the awake signal, absence // is asleep; only a failure to read the table at all -> nil ("unknown"). // // This reading is correct ONLY for exec-backed routes, where the socket only // exists in the kernel while the spawned child holds it open — asleep, the // child is gone and the entry is gone with it. It is CONFIDENTLY WRONG for // systemd-socket-activated (registry.BackingUnit) routes: the carlos-app@.socket // unit holds the listener open in the kernel table at all times, awake or // asleep, specifically so systemd can catch the first connection and spawn // the service — so this function alone would report "awake" forever for a // sleeping unit-backed tenant. Callers MUST route unit-backed sockets through // wrapListeningProbe below rather than calling this directly; statusLoop is // the only caller and does exactly that. func listeningProbe(sock string) *bool { f, err := os.Open("/proc/net/unix") if err != nil { return nil } defer f.Close() found := false scanner := bufio.NewScanner(f) scanner.Scan() // header line ("Num RefCount Protocol Flags Type St Inode Path") — not a data row for scanner.Scan() { fields := strings.Fields(scanner.Text()) if len(fields) == 0 { continue } // Path, when present, is always the last column; sockets with no // Path (anonymous socketpairs) simply never match sock. if fields[len(fields)-1] == sock { found = true break } } if err := scanner.Err(); err != nil { return nil } return &found } // unitBackedSockets returns the set of socket paths belonging to routes // whose Backing is registry.BackingUnit — the systemd-socket-activated // whole-app tenants that make listeningProbe's kernel-table reading // unreliable (see its doc comment). Pure and routes-only so it and // wrapListeningProbe below are each independently testable at the seam // host_test.go exercises, with no /proc read anywhere near the test. func unitBackedSockets(routes []registry.Route) map[string]bool { set := map[string]bool{} for _, r := range routes { if r.Backing == registry.BackingUnit { set[r.Socket] = true } } return set } // wrapListeningProbe wraps probe so a unit-backed socket (per unitBacked) // always reports nil — unknown — instead of whatever probe would say. // systemd's .socket unit holds a unit-backed tenant's listener open in the // kernel table at ALL times, awake or asleep; a raw kernel-table read can // therefore never tell the two states apart for these routes, and reporting // "awake" from that alone would be a confidently wrong answer where the // status page's commitment demands an honest null. This stays true until the // agent learns to ask systemd itself (e.g. `systemctl is-active // carlos-app@.service`) — a real fix, not implemented here, tracked as // future work rather than guessed at. Exec-backed sockets (not in the set) // pass straight through to probe unchanged, since listeningProbe's // kernel-table reading IS reliable for them. func wrapListeningProbe(unitBacked map[string]bool, probe func(string) *bool) func(string) *bool { return func(sock string) *bool { if unitBacked[sock] { return nil } return probe(sock) } } // currentSocketProbe points probe at each activator-managed route's CURRENT // generation socket (zero-downtime spec §7.1): with generations on, nothing // binds a managed route's canonical path, so the kernel table would call // every awake instance asleep. A route the activator reports no current // generation for (asleep) is probed at its canonical path, where nothing // listens: asleep, which is the truth. Any other socket passes straight // through. appSocket is the activator's AppSocket. func currentSocketProbe(routes []registry.Route, appSocket func(host string) (string, bool), probe func(string) *bool) func(string) *bool { hostOf := map[string]string{} for _, r := range routes { if r.Kind == "instance" && r.Hibernate && r.Socket != "" && r.Backing != registry.BackingUnit { hostOf[r.Socket] = r.Host } } return func(sock string) *bool { if host, ok := hostOf[sock]; ok { if cur, ok := appSocket(host); ok { return probe(cur) } } return probe(sock) } } // parseMeminfoBytes reads /proc/meminfo's MemTotal and MemAvailable (both in // kB) and returns them converted to bytes. Either field missing or // unparsable -> nil for both. func parseMeminfoBytes(r io.Reader) (*int64, *int64) { var totalKB, availKB int64 haveTotal, haveAvail := false, false scanner := bufio.NewScanner(r) for scanner.Scan() { fields := strings.Fields(scanner.Text()) if len(fields) < 2 { continue } v, err := strconv.ParseInt(fields[1], 10, 64) if err != nil { continue } switch fields[0] { case "MemTotal:": totalKB, haveTotal = v, true case "MemAvailable:": availKB, haveAvail = v, true } } if err := scanner.Err(); err != nil || !haveTotal || !haveAvail || totalKB <= 0 { return nil, nil } totalBytes := totalKB * 1024 availBytes := availKB * 1024 return &totalBytes, &availBytes } // readMemBytes reads /proc/meminfo and returns MemTotal and MemAvailable as // bytes. Unreadable or unparseable -> nil, nil. func readMemBytes() (*int64, *int64) { f, err := os.Open("/proc/meminfo") if err != nil { return nil, nil } defer f.Close() return parseMeminfoBytes(f) } // parseLoad1 parses /proc/loadavg's first field (1-minute load average) from // a string. The file format is: "load1 load5 load15 running/total lastpid". // Empty or unparseable -> nil. func parseLoad1(s string) *float64 { fields := strings.Fields(s) if len(fields) == 0 { return nil } l, err := strconv.ParseFloat(fields[0], 64) if err != nil { return nil } return &l } // readLoad1 reads /proc/loadavg and returns the 1-minute load average. // Unreadable or unparseable -> nil. func readLoad1() *float64 { b, err := os.ReadFile("/proc/loadavg") if err != nil { return nil } return parseLoad1(string(b)) } // readDiskFreeBytes reports how much free space is available on the data // volume, via diskBytes on dataDir() — /data by default, CARLOS_DATA_DIR // when set. Statfs failure or a zero-block filesystem -> nil rather than a // crash. func readDiskFreeBytes() *int64 { total, avail, err := diskBytes(dataDir()) if err != nil || total == 0 { return nil } freeBytes := int64(avail) return &freeBytes } // dbFileBytes is status.Probes.DBBytes: the on-disk size of a route's SQLite // database, the main file plus its -wal. The WAL counts because in WAL mode // recent writes live there until a checkpoint, so the main file alone // understates a busy database. A missing or unreadable main file is nil // (unknown), never 0: a route that has not created its database yet has no // size. A missing -wal is normal after a checkpoint and adds nothing. A // hibernating route's file stays on disk while it sleeps (the activator keeps // it as a warm cache), so asleep routes are sized too. func dbFileBytes(dbPath string) *int64 { fi, err := os.Stat(dbPath) if err != nil || !fi.Mode().IsRegular() { return nil } n := fi.Size() if wal, err := os.Stat(dbPath + "-wal"); err == nil && wal.Mode().IsRegular() { n += wal.Size() } return &n }