//go:build linux package host import ( "context" "errors" "fmt" "io/fs" "os" "os/exec" "path/filepath" "reflect" "sort" "strings" "syscall" "time" "github.com/carlosframework/platform/internal/cgroup" "github.com/carlosframework/platform/internal/hostbox" "github.com/carlosframework/platform/internal/hostguard" "github.com/carlosframework/platform/internal/instancectl" "github.com/carlosframework/platform/internal/provision" "github.com/carlosframework/platform/internal/stopevidence" ) // stopKernelRoots are the trees root observes. Production uses the real // /proc and /sys/fs/cgroup; the fixture tests use stand-ins. type stopKernelRoots struct { Proc, Cgroup string // Managed maps the inventory's managed-root names to directories. Managed map[string]string // AgentCgroup is the agent unit's delegated cgroup directory, under // which the host's own containment would be. AgentCgroup string } // stopLaunchHosts is the launch journal's host index. type stopLaunchHosts interface { HostLaunches(context.Context, string) ([]hostbox.LaunchInventory, error) } func (k LinuxStopKernel) realRoots() (stopKernelRoots, error) { if k.Journal == nil || k.CgroupRoot != "/sys/fs/cgroup" || k.ProcRoot != "/proc" || os.Geteuid() != 0 { return stopKernelRoots{}, ErrClaimStopUnavailable } if err := verifyStopKernelMounts(k.CgroupRoot, k.ProcRoot); err != nil { return stopKernelRoots{}, err } return stopKernelRoots{Proc: k.ProcRoot, Cgroup: k.CgroupRoot, Managed: k.ManagedRoots}, nil } // VerifyWritersStopped is root's writers-stopped observation (R3). func (k LinuxStopKernel) VerifyWritersStopped(ctx context.Context, t hostguard.FenceTarget, inv StopInventory, peer StopPeer, hints []stopevidence.DescriptorHint) (WritersKernelEvidence, error) { roots, err := k.realRoots() if err != nil { return WritersKernelEvidence{}, err } if _, err := k.readContainedJournal(ctx, t); err != nil { return WritersKernelEvidence{}, err } return verifyWritersAt(ctx, roots, k.Journal, t, inv, peer, hints) } // VerifyNoProcess is root's session-zero witness (R4). func (k LinuxStopKernel) VerifyNoProcess(ctx context.Context, t hostguard.FenceTarget, inv StopInventory) (StopKernelEvidence, error) { roots, err := k.realRoots() if err != nil { return StopKernelEvidence{}, err } launches, ok := k.Journal.(stopLaunchHosts) if !ok || k.AgentCgroup == nil { return StopKernelEvidence{}, ErrClaimStopUnavailable } if roots.AgentCgroup, err = k.AgentCgroup(ctx); err != nil { return StopKernelEvidence{}, err } boot, err := os.ReadFile(filepath.Join(k.ProcRoot, "sys/kernel/random/boot_id")) if err != nil { return StopKernelEvidence{}, err } if observed := strings.TrimSpace(string(boot)); observed == "" || observed != t.BootID { return StopKernelEvidence{}, ErrClaimStopIdentity } evidence, err := verifyNoProcessAt(ctx, roots, launches, t, inv) if err != nil { return StopKernelEvidence{}, err } evidence.ObservedBootID = t.BootID return evidence, nil } // verifyWritersAt: the session's launch journal is settled with no pending // role and every journaled containment is empty (scanStopCgroups, by device // and inode); every descriptor on the installation's database files, found // by root's complete scan, is a hinted replicator descriptor of the // authenticated agent; and the journal did not change across the scan. func verifyWritersAt(ctx context.Context, roots stopKernelRoots, journal hostbox.StopKernelJournalReader, t hostguard.FenceTarget, inv StopInventory, peer StopPeer, hints []stopevidence.DescriptorHint) (WritersKernelEvidence, error) { if journal == nil || t.Session == 0 || peer.PID <= 0 || peer.ProcessStart == "" { return WritersKernelEvidence{}, ErrClaimStopUnavailable } j, err := journal.ReadStopKernelJournal(ctx, t) if err != nil { return WritersKernelEvidence{}, err } if !j.WritersClosed || j.V != 1 || !sameStopSession(j.Target, t) || len(j.Cgroups) == 0 || !j.Settled || j.PendingRoles { return WritersKernelEvidence{}, ErrClaimStopInventory } groups, err := scanStopCgroups(ctx, roots.Cgroup, j.Cgroups) if err != nil { return WritersKernelEvidence{}, err } paths, prefixes, err := rootInstallationCoverage(roots.Managed, t.Host, inv, true) if err != nil { return WritersKernelEvidence{}, err } files := make([]stopevidence.FileIdentity, 0, len(j.Files)) for _, f := range j.Files { files = append(files, stopevidence.FileIdentity{Device: f.Device, Inode: f.Inode}) } retained := []stopevidence.Descriptor{} if len(paths)+len(prefixes)+len(files) > 0 { held, err := stopevidence.HoldersWithPrefixes(ctx, roots.Proc, paths, prefixes, files) if err != nil { return WritersKernelEvidence{}, err } retained, err = stopevidence.ReplicatorOnly(ctx, roots.Proc, held, stopevidence.Agent{PID: peer.PID, Start: peer.ProcessStart}, hints) if err != nil { return WritersKernelEvidence{}, err } } again, err := journal.ReadStopKernelJournal(ctx, t) if err != nil || !reflect.DeepEqual(j, again) { return WritersKernelEvidence{}, ErrClaimStopInventory } if _, err := scanStopCgroups(ctx, roots.Cgroup, j.Cgroups); err != nil { return WritersKernelEvidence{}, err } return WritersKernelEvidence{CheckedPaths: paths, CgroupIdentities: groups, Retained: retained}, nil } // verifyNoProcessAt: the launch journal has no entry for this installation, the // host's containment under the agent's cgroup is absent or empty // throughout, and no process holds any of the installation's files; the // journal is read again after the scan. func verifyNoProcessAt(ctx context.Context, roots stopKernelRoots, launches stopLaunchHosts, t hostguard.FenceTarget, inv StopInventory) (StopKernelEvidence, error) { if launches == nil || t.Session != 0 || !filepath.IsAbs(roots.AgentCgroup) { return StopKernelEvidence{}, ErrClaimStopUnavailable } return verifyNoLaunchAt(ctx, roots, launches, t, inv) } func (k LinuxStopKernel) VerifyUnjournaled(ctx context.Context, t hostguard.FenceTarget, inv StopInventory) (StopKernelEvidence, error) { roots, err := k.realRoots() if err != nil { return StopKernelEvidence{}, err } launches, ok := k.Journal.(stopLaunchHosts) if !ok || k.AgentCgroup == nil || t.Session == 0 || t.FenceID == "" { return StopKernelEvidence{}, ErrClaimStopUnavailable } if roots.AgentCgroup, err = k.AgentCgroup(ctx); err != nil { return StopKernelEvidence{}, err } boot, err := os.ReadFile(filepath.Join(roots.Proc, "sys/kernel/random/boot_id")) if err != nil || strings.TrimSpace(string(boot)) == "" { return StopKernelEvidence{}, errors.Join(ErrClaimStopIdentity, err) } evidence, err := verifyNoLaunchAt(ctx, roots, launches, t, inv) evidence.ObservedBootID = strings.TrimSpace(string(boot)) if err == nil { evidence.CgroupIdentities = append(evidence.CgroupIdentities, "unlaunched:"+digestStop(t)) } return evidence, err } func verifyNoLaunchAt(ctx context.Context, roots stopKernelRoots, launches stopLaunchHosts, t hostguard.FenceTarget, inv StopInventory) (StopKernelEvidence, error) { records, err := launches.HostLaunches(ctx, t.Host) if err != nil { return StopKernelEvidence{}, err } for _, in := range records { if in.Target.Host != t.Host { return StopKernelEvidence{}, ErrClaimStopInventory } if in.Target.Incarnation == t.Incarnation && in.Target.Stamp == t.Stamp { return StopKernelEvidence{}, ErrClaimStopInventory } } containments := records if t.Session != 0 { containments = append(append([]hostbox.LaunchInventory(nil), records...), hostbox.LaunchInventory{Target: t}) } groups, err := noProcessGroups(ctx, roots, t.Host, containments...) if err != nil { return StopKernelEvidence{}, err } paths, prefixes, err := rootInstallationCoverage(roots.Managed, t.Host, inv, false) if err != nil { return StopKernelEvidence{}, err } if len(paths)+len(prefixes) == 0 { return StopKernelEvidence{}, ErrClaimStopInventory } held, err := stopevidence.HoldersWithPrefixes(ctx, roots.Proc, paths, prefixes, nil) if err != nil { return StopKernelEvidence{}, err } if len(held) != 0 { return StopKernelEvidence{}, fmt.Errorf("%w: pid %d holds an installation file", ErrClaimStopIdentity, held[0].PID) } again, err := launches.HostLaunches(ctx, t.Host) if err != nil || !reflect.DeepEqual(records, again) { return StopKernelEvidence{}, errors.Join(ErrClaimStopInventory, err) } if again, err := noProcessGroups(ctx, roots, t.Host, containments...); err != nil || !reflect.DeepEqual(again, groups) { return StopKernelEvidence{}, errors.Join(ErrClaimStopInventory, err) } journal := digestStop(records) return StopKernelEvidence{InventorySHA256: digestStop(inv), LocalJournalSHA256: journal, CgroupIdentities: groups, UnitInvocationIDs: []string{}}, nil } // rootInstallationCoverage derives mandatory coverage from the root-owned layout. // Agent entries can add paths but cannot remove the host's database or container. // Name and prefix matches cover root's mount namespace only. Units.Gone must // be proved first: isolated namespaces can hide newly created or deleted names. func rootInstallationCoverage(managed map[string]string, host string, inv StopInventory, dataOnly bool) ([]string, []string, error) { if !provision.ValidHostShape(host) || !filepath.IsAbs(managed["data"]) { return nil, nil, ErrClaimStopInventory } dir := provision.InstanceDir(managed["data"], host) rel, err := filepath.Rel(managed["data"], dir) if err != nil { return nil, nil, err } entry := StopInventoryEntry{Kind: "path", ManagedRoot: "data", RelativePath: rel, Type: syscall.S_IFDIR, Materialization: "intended"} info, err := os.Lstat(dir) if err == nil { st, ok := info.Sys().(*syscall.Stat_t) if !ok || !info.IsDir() { return nil, nil, ErrClaimStopInventory } entry.Device, entry.Inode, entry.Materialization = uint64(st.Dev), st.Ino, "materialized" entry.Identity, err = hostbox.CaptureInventoryIdentity(managed["data"], rel) if err != nil { return nil, nil, err } } else if !errors.Is(err, os.ErrNotExist) { return nil, nil, err } inv.Entries = append(append([]StopInventoryEntry(nil), inv.Entries...), entry) paths, prefixes, err := installationCoverage(managed, inv, dataOnly) if err != nil { return nil, nil, err } db := provision.NestedDBPath(managed["data"], host) paths = append(paths, db, db+"-wal", db+"-shm", db+"-journal") sort.Strings(paths) return paths, prefixes, nil } // installationCoverage returns both the currently named files and the // inventory directory prefixes. The latter keep both root witnesses // effective for nested and open-but-unlinked files, including an installation // directory whose current listing is empty. func installationCoverage(managed map[string]string, inv StopInventory, dataOnly bool) ([]string, []string, error) { if !inv.Complete || len(managed) == 0 { return nil, nil, ErrClaimStopInventory } seen := map[string]bool{} prefixSet := map[string]bool{} dirs := 0 for _, e := range inv.Entries { if e.Kind != "path" && e.Kind != "live-link" { continue } if dataOnly && e.ManagedRoot != "data" { continue } root := managed[e.ManagedRoot] if !filepath.IsAbs(root) || !filepath.IsLocal(e.RelativePath) { return nil, nil, ErrClaimStopInventory } path := filepath.Join(root, e.RelativePath) if e.Type&syscall.S_IFMT != syscall.S_IFDIR { seen[path], seen[path+"-wal"], seen[path+"-shm"] = true, true, true continue } dirs++ prefixSet[filepath.Clean(path)] = true info, err := os.Lstat(path) if errors.Is(err, os.ErrNotExist) && e.Materialization != "materialized" { continue } if err != nil { return nil, nil, err } if !info.IsDir() || hostbox.VerifyInventoryIdentity(root, e) != nil { return nil, nil, fmt.Errorf("%w: %s is not the inventory's directory", ErrClaimStopInventory, path) } entries, err := os.ReadDir(path) if err != nil { return nil, nil, err } for _, entry := range entries { if entry.Type().IsRegular() { seen[filepath.Join(path, entry.Name())] = true } } } if dataOnly && dirs == 0 { return nil, nil, fmt.Errorf("%w: the inventory names no state directory", ErrClaimStopInventory) } out := make([]string, 0, len(seen)) for path := range seen { out = append(out, path) } sort.Strings(out) prefixes := make([]string, 0, len(prefixSet)) for prefix := range prefixSet { prefixes = append(prefixes, prefix) } sort.Strings(prefixes) return out, prefixes, nil } // hostContainmentEmpty checks the host's containment under the agent's // delegated cgroup: absent, or every cgroup in it unpopulated. It returns // the identity it observed. func hostContainmentEmpty(ctx context.Context, agent, host string) (string, error) { name := cgroup.ExecName(host) return emptyContainmentAt(ctx, filepath.Join(agent, name), name) } func emptyContainmentAt(ctx context.Context, dir, name string) (string, error) { info, err := os.Lstat(dir) if errors.Is(err, os.ErrNotExist) { return "absent:" + name, nil } if err != nil { return "", err } st, ok := info.Sys().(*syscall.Stat_t) if !ok || !info.IsDir() { return "", ErrClaimStopInventory } err = filepath.WalkDir(dir, func(path string, d fs.DirEntry, walkErr error) error { if walkErr != nil { return walkErr } if err := ctx.Err(); err != nil { return err } if d.Type()&os.ModeSymlink != 0 { return ErrClaimStopInventory } if !d.IsDir() { return nil } events, err := os.ReadFile(filepath.Join(path, "cgroup.events")) if err != nil { return err } populated := "" for _, line := range strings.Split(string(events), "\n") { if f := strings.Fields(line); len(f) == 2 && f[0] == "populated" { if populated != "" { return ErrClaimStopInventory } populated = f[1] } } if populated != "0" { return fmt.Errorf("%w: %s is populated", ErrClaimStopInventory, path) } return nil }) if err != nil { return "", err } return fmt.Sprintf("empty:%s:%d:%d", name, st.Dev, st.Ino), nil } // agentUnitCgroup is the production AgentCgroup: the edge unit's cgroup as // systemd reports it. The agent's cgroup manager delegates its hosts' // containments below it. func agentUnitCgroup(ctx context.Context) (string, error) { ctx, cancel := context.WithTimeout(ctx, 3*time.Second) defer cancel() out, err := exec.CommandContext(ctx, "systemctl", "show", "-p", "ControlGroup", "--value", "carlos-edge.service").Output() if err != nil { return "", err } rel := strings.TrimPrefix(strings.TrimSpace(string(out)), "/") if rel == "" || !filepath.IsLocal(rel) { return "", ErrClaimStopUnavailable } return filepath.Join("/sys/fs/cgroup", rel), nil } // noProcessGroups includes every isolated role beneath the host slice. func noProcessGroups(ctx context.Context, roots stopKernelRoots, host string, records ...hostbox.LaunchInventory) ([]string, error) { legacy, err := hostContainmentEmpty(ctx, roots.AgentCgroup, host) if err != nil { return nil, err } if !filepath.IsAbs(roots.Cgroup) { return nil, ErrClaimStopUnavailable } rel := filepath.Dir(instancectl.ClaimUnitControlGroup(host)) isolated, err := emptyContainmentAt(ctx, filepath.Join(roots.Cgroup, rel), rel) if err != nil { return nil, err } out := []string{legacy, isolated} seen := map[string]bool{} for _, in := range records { name := cgroup.ExecName(claimContainmentName(in.Target)) if seen[name] { continue } seen[name] = true observed, err := emptyContainmentAt(ctx, filepath.Join(roots.AgentCgroup, name), name) if err != nil { return nil, err } out = append(out, observed) } return out, nil } func rootStopAgentCgroup(ctx context.Context) (string, error) { ctx, cancel := context.WithTimeout(ctx, 3*time.Second) defer cancel() raw, err := exec.CommandContext(ctx, "systemctl", "show", "-p", "ControlGroup,Id,Slice,ActiveState", "carlos-edge.service").Output() if err != nil { return "", err } return rootStopAgentGroup(raw) } func rootStopAgentGroup(raw []byte) (string, error) { fields := map[string]string{} for _, line := range strings.Split(strings.TrimSpace(string(raw)), "\n") { key, value, ok := strings.Cut(line, "=") if !ok { return "", ErrClaimStopUnavailable } if _, exists := fields[key]; exists { return "", ErrClaimStopUnavailable } fields[key] = value } if fields["Id"] != "carlos-edge.service" { return "", ErrClaimStopUnavailable } rel := strings.TrimPrefix(fields["ControlGroup"], "/") if rel == "" { if fields["ActiveState"] != "inactive" && fields["ActiveState"] != "failed" { return "", ErrClaimStopUnavailable } slice := fields["Slice"] if !strings.HasSuffix(slice, ".slice") || !filepath.IsLocal(slice) || filepath.Base(slice) != slice { return "", ErrClaimStopUnavailable } if slice != "-.slice" { parts := strings.Split(strings.TrimSuffix(slice, ".slice"), "-") for i, part := range parts { if part == "" { return "", ErrClaimStopUnavailable } rel = filepath.Join(rel, strings.Join(parts[:i+1], "-")+".slice") } } rel = filepath.Join(rel, fields["Id"]) } if !filepath.IsLocal(rel) || filepath.Clean(rel) != rel { return "", ErrClaimStopUnavailable } return filepath.Join("/sys/fs/cgroup", rel), nil } func stopNeverIntended(records []hostbox.LaunchInventory, t hostguard.FenceTarget) bool { found := false for _, in := range records { if in.Target.Host != t.Host { return false } if in.Target.Incarnation != t.Incarnation || in.Target.Stamp != t.Stamp { continue } if !in.WritersClosed || len(in.Groups) != 1 { return false } found = true for _, r := range in.Reservations { if !hostbox.IsUnitReservation(r) || r.State != hostbox.UnitSettled || !r.NeverIntended || r.PID != 0 || r.ProcessStart != "" || r.Group.Device != 0 || r.Group.Inode != 0 || r.Group.InvocationID != "" { return false } } } return found }