//go:build linux package host import ( "context" "errors" "fmt" "io/fs" "os" "path/filepath" "reflect" "sort" "strconv" "strings" "syscall" "time" "github.com/carlosframework/platform/internal/hostbox" "github.com/carlosframework/platform/internal/hostguard" ) type LinuxStopKernel struct { rootAuthority bool Journal hostbox.StopKernelJournalReader ManagedRoots map[string]string CgroupRoot, ProcRoot string // AgentCgroup returns the agent unit's delegated cgroup directory, for // the session-zero witness (VerifyNoProcess). Nil refuses the witness. AgentCgroup func(context.Context) (string, error) } func sameStopSession(a, b hostguard.FenceTarget) bool { return a.Host == b.Host && a.Incarnation == b.Incarnation && a.Stamp == b.Stamp && a.Session == b.Session && a.Box == b.Box && a.Enrollment == b.Enrollment && a.BootID == b.BootID && a.ControllerID == b.ControllerID } func (k LinuxStopKernel) VerifyStopped(ctx context.Context, t hostguard.FenceTarget, inv StopInventory, ack DrainAcknowledgement) (StopKernelEvidence, error) { if k.Journal == nil || k.CgroupRoot != "/sys/fs/cgroup" || k.ProcRoot != "/proc" || os.Geteuid() != 0 { return StopKernelEvidence{}, ErrClaimStopUnavailable } if err := verifyStopKernelMounts(k.CgroupRoot, k.ProcRoot); err != nil { return StopKernelEvidence{}, err } j, err := k.readContainedJournal(ctx, t) if err != nil { return StopKernelEvidence{}, err } if !inv.Complete || j.V != 1 || !sameStopSession(j.Target, t) || len(j.Cgroups) == 0 || ack.FenceID != t.FenceID || ack.ControllerID != t.ControllerID { return StopKernelEvidence{}, ErrClaimStopInventory } if err := validateClaimDrainProducers(j.ProducerIDs, ack, j.Completed); err != nil { return StopKernelEvidence{}, err } if j.PendingRoles { if err := k.resolveRoleIntentions(ctx, t, false); err != nil { return StopKernelEvidence{}, err } j, err = k.readContainedJournal(ctx, t) if err != nil { return StopKernelEvidence{}, err } } if !j.Settled || j.PendingRoles { return StopKernelEvidence{}, ErrClaimStopInventory } groups, err := scanStopCgroups(ctx, k.CgroupRoot, j.Cgroups) if err != nil { return StopKernelEvidence{}, err } paths, err := k.stopDescriptorPaths(inv) if err != nil { return StopKernelEvidence{}, err } if err := scanStopDescriptors(ctx, k.ProcRoot, j.Files, paths); err != nil { return StopKernelEvidence{}, err } again, err := k.Journal.ReadStopKernelJournal(ctx, t) if err != nil || !reflect.DeepEqual(j, again) { return StopKernelEvidence{}, ErrClaimStopInventory } if _, err := scanStopCgroups(ctx, k.CgroupRoot, j.Cgroups); err != nil { return StopKernelEvidence{}, err } if raw, ok := k.Journal.(stopLaunchHosts); ok && k.rootAuthority { records, err := raw.HostLaunches(ctx, t.Host) if err != nil { return StopKernelEvidence{}, err } if stopNeverIntended(records, t) { groups = append(groups, "unlaunched:"+digestStop(t)) } } units := []string{} return StopKernelEvidence{ObservedBootID: t.BootID, InventorySHA256: digestStop(inv), LocalJournalSHA256: digestStop(j), CgroupIdentities: groups, UnitInvocationIDs: units}, nil } func stopIdentity(path string) (hostbox.StopFileIdentity, error) { f, err := os.Stat(path) if err != nil { return hostbox.StopFileIdentity{}, err } s, ok := f.Sys().(*syscall.Stat_t) if !ok { return hostbox.StopFileIdentity{}, ErrClaimStopUnavailable } return hostbox.StopFileIdentity{Device: uint64(s.Dev), Inode: s.Ino}, nil } func scanStopCgroups(ctx context.Context, root string, groups []hostbox.StopCgroupIdentity) ([]string, error) { return scanStopCgroupsMode(ctx, root, groups, true) } func scanStopCgroupsMode(ctx context.Context, root string, groups []hostbox.StopCgroupIdentity, empty bool) ([]string, error) { if len(groups) == 0 { return nil, ErrClaimStopInventory } byPath := map[string]hostbox.StopCgroupIdentity{} for _, g := range groups { if !filepath.IsLocal(g.RelativePath) || g.RelativePath == "." || filepath.Clean(g.RelativePath) != g.RelativePath || g.Inode == 0 || g.Device == 0 { return nil, ErrClaimStopInventory } if _, ok := byPath[g.RelativePath]; ok { return nil, ErrClaimStopInventory } byPath[g.RelativePath] = g } parent := "" for path := range byPath { if _, ok := byPath[filepath.Dir(path)]; !ok { if parent != "" { return nil, ErrClaimStopInventory } parent = path } } if parent == "" { return nil, ErrClaimStopInventory } for path := range byPath { if path != parent && !strings.HasPrefix(path, parent+string(filepath.Separator)) { return nil, ErrClaimStopInventory } } seen := map[string]bool{} out := []string{} err := filepath.WalkDir(filepath.Join(root, parent), func(path string, d fs.DirEntry, walkErr error) error { if walkErr != nil { return walkErr } if err := ctx.Err(); err != nil { return err } if d.Type()&os.ModeSymlink != 0 { return ErrClaimStopInventory } if !d.IsDir() { return nil } rel, err := filepath.Rel(root, path) if err != nil { return err } g, ok := byPath[rel] if !ok { return ErrClaimStopInventory } id, err := stopIdentity(path) if err != nil { return err } if id.Device != g.Device || id.Inode != g.Inode { return ErrClaimStopInventory } events, err := os.ReadFile(filepath.Join(path, "cgroup.events")) if err != nil { return err } populated := "" for _, line := range strings.Split(string(events), "\n") { f := strings.Fields(line) if len(f) == 2 && f[0] == "populated" { if populated != "" { return ErrClaimStopInventory } populated = f[1] } } if populated != "0" && (empty || populated != "1") { return ErrClaimStopInventory } seen[rel] = true out = append(out, fmt.Sprintf("%s:%d:%d", rel, id.Device, id.Inode)) return nil }) if err != nil { return nil, err } if len(seen) != len(byPath) { return nil, ErrClaimStopInventory } sort.Strings(out) return out, nil } func scanStopDescriptors(ctx context.Context, proc string, files []hostbox.StopFileIdentity, paths ...[]string) error { wanted := map[hostbox.StopFileIdentity]bool{} for _, id := range files { if id.Device == 0 || id.Inode == 0 { return ErrClaimStopInventory } wanted[id] = true } entries, err := os.ReadDir(proc) if err != nil { return err } for _, entry := range entries { if _, err := strconv.ParseUint(entry.Name(), 10, 32); err != nil { continue } if err := ctx.Err(); err != nil { return err } pidDir := filepath.Join(proc, entry.Name()) before, err := os.ReadFile(filepath.Join(pidDir, "stat")) if err != nil { if errors.Is(err, os.ErrNotExist) { continue } return err } descriptors, err := os.ReadDir(filepath.Join(pidDir, "fd")) if err != nil { if _, gone := os.Stat(pidDir); errors.Is(gone, os.ErrNotExist) { continue } return err } for _, fd := range descriptors { descriptor := filepath.Join(pidDir, "fd", fd.Name()) name, linkErr := os.Readlink(descriptor) if linkErr != nil && !errors.Is(linkErr, os.ErrNotExist) { return linkErr } name = strings.TrimSuffix(name, " (deleted)") for _, list := range paths { for _, owned := range list { if name == owned || (strings.HasSuffix(owned, "/") && strings.HasPrefix(name, owned)) { return fmt.Errorf("claim stop: managed state descriptor remains at %s", descriptor) } } } id, err := stopIdentity(descriptor) if err != nil { if errors.Is(err, os.ErrNotExist) { continue } return err } if wanted[id] { return fmt.Errorf("claim stop: database descriptor remains at %s/fd/%s", entry.Name(), fd.Name()) } } after, err := os.ReadFile(filepath.Join(pidDir, "stat")) if err != nil { if errors.Is(err, os.ErrNotExist) { continue } return err } start := func(b []byte) string { i := strings.LastIndexByte(string(b), ')') if i < 0 { return "" } f := strings.Fields(string(b[i+1:])) if len(f) <= 19 { return "" } return f[19] } if start(before) == "" || start(before) != start(after) { return ErrClaimStopIdentity } } return nil } func newProductionStopKernel(j hostbox.StopKernelJournalReader, roots map[string]string, rootAuthority ...bool) StopKernelVerifier { agent := agentUnitCgroup if len(rootAuthority) == 1 && rootAuthority[0] { agent = rootStopAgentCgroup } return LinuxStopKernel{rootAuthority: len(rootAuthority) == 1 && rootAuthority[0], Journal: j, ManagedRoots: roots, CgroupRoot: "/sys/fs/cgroup", ProcRoot: "/proc", AgentCgroup: agent} } func (k LinuxStopKernel) RecoverStopped(ctx context.Context, t hostguard.FenceTarget, in StopInventory) (StopKernelEvidence, DrainAcknowledgement, error) { refuse := func(err error) (StopKernelEvidence, DrainAcknowledgement, error) { return StopKernelEvidence{}, DrainAcknowledgement{}, err } if os.Geteuid() != 0 || k.CgroupRoot != "/sys/fs/cgroup" || k.ProcRoot != "/proc" || k.Journal == nil { return refuse(ErrClaimStopUnavailable) } if err := verifyStopKernelMounts(k.CgroupRoot, k.ProcRoot); err != nil { return refuse(err) } journal, ok := k.Journal.(interface { SettleRecovered(context.Context, hostguard.FenceTarget) error }) if !ok { return refuse(ErrClaimStopUnavailable) } j, err := k.readContainedJournal(ctx, t) if err != nil { return refuse(err) } if j.ControllerPID <= 0 || j.ProcessStart == "" || !sameStopSession(j.Target, t) { return refuse(ErrClaimStopIdentity) } observedBoot, err := recoveryControllerAt(k.ProcRoot, t, j) if err != nil { return refuse(err) } if observedBoot != t.BootID { return k.recoverAfterReboot(ctx, t, in, j, observedBoot, journal) } if j.PendingRoles { if err := k.resolveRoleIntentions(ctx, t, false); err != nil { return refuse(err) } j, err = k.readContainedJournal(ctx, t) if err != nil { return refuse(err) } } if _, err := scanStopCgroupsMode(ctx, k.CgroupRoot, j.Cgroups, false); err != nil { if errors.Is(err, os.ErrNotExist) { return k.recoverUnlaunchedAbsent(ctx, t, in, j, journal) } return refuse(err) } groups := append([]hostbox.StopCgroupIdentity(nil), j.Cgroups...) sort.Slice(groups, func(i, j int) bool { return len(groups[i].RelativePath) < len(groups[j].RelativePath) }) if len(groups) == 0 { return refuse(ErrClaimStopInventory) } if err := os.WriteFile(filepath.Join(k.CgroupRoot, groups[0].RelativePath, "cgroup.kill"), []byte("1"), 0600); err != nil { return refuse(err) } for { if _, err := scanStopCgroups(ctx, k.CgroupRoot, j.Cgroups); err == nil { break } select { case <-ctx.Done(): return refuse(ctx.Err()) case <-time.After(20 * time.Millisecond): } } paths, err := k.stopDescriptorPaths(in) if err != nil { return refuse(err) } if err := scanStopDescriptors(ctx, k.ProcRoot, j.Files, paths); err != nil { return refuse(err) } if err := journal.SettleRecovered(ctx, t); err != nil { return refuse(err) } ack := DrainClaimAcknowledgement(t.FenceID, t.ControllerID, j.ProcessStart, j.ProducerIDs, j.Completed) evidence, err := k.VerifyStopped(ctx, t, in, ack) return evidence, ack, err } func (k LinuxStopKernel) stopDescriptorPaths(in StopInventory) ([]string, error) { if !in.Complete || len(k.ManagedRoots) == 0 { return nil, ErrClaimStopInventory } out := []string{} for _, a := range in.Entries { if a.Kind != "path" && a.Kind != "live-link" { continue } root := k.ManagedRoots[a.ManagedRoot] if !filepath.IsAbs(root) || !filepath.IsLocal(a.RelativePath) { return nil, ErrClaimStopInventory } path := filepath.Join(root, a.RelativePath) out = append(out, path, path+"-wal", path+"-shm") if a.Type&syscall.S_IFMT == syscall.S_IFDIR { out = append(out, path+"/") } } return out, nil } func (k LinuxStopKernel) recoverAfterReboot(ctx context.Context, t hostguard.FenceTarget, in StopInventory, j hostbox.StopKernelJournal, observedBoot string, journal interface { SettleRecovered(context.Context, hostguard.FenceTarget) error }) (StopKernelEvidence, DrainAcknowledgement, error) { refuse := func(err error) (StopKernelEvidence, DrainAcknowledgement, error) { return StopKernelEvidence{}, DrainAcknowledgement{}, err } if observedBoot == "" || t.BootID == "" || observedBoot == t.BootID || !sameStopSession(t, j.Target) || j.ControllerPID <= 0 || j.ProcessStart == "" || j.V != 1 { return refuse(ErrClaimStopIdentity) } if err := k.validateContainment(ctx, t, j); err != nil { return refuse(err) } bootPath := filepath.Join(k.ProcRoot, "sys/kernel/random/boot_id") checkBoot := func() error { b, err := os.ReadFile(bootPath) if err != nil { return err } if strings.TrimSpace(string(b)) != observedBoot { return ErrClaimStopIdentity } return nil } if err := checkBoot(); err != nil { return refuse(err) } if j.PendingRoles { if err := k.resolveRoleIntentions(ctx, t, true); err != nil { return refuse(err) } var err error j, err = k.readContainedJournal(ctx, t) if err != nil { return refuse(err) } } groups, err := scanRebootCgroups(ctx, k.CgroupRoot, j.Cgroups, t.BootID, observedBoot) if err != nil { return refuse(err) } paths, err := k.stopDescriptorPaths(in) if err != nil { return refuse(err) } files, err := k.rebootStopFiles(ctx, in, j.Files) if err != nil { return refuse(err) } if err := scanStopDescriptors(ctx, k.ProcRoot, files, paths); err != nil { return refuse(err) } if err := journal.SettleRecovered(ctx, t); err != nil { return refuse(err) } fresh, err := k.readContainedJournal(ctx, t) if err != nil { return refuse(err) } settled := j settled.Settled = true if !reflect.DeepEqual(settled, fresh) { return refuse(ErrClaimStopInventory) } if _, err := scanRebootCgroups(ctx, k.CgroupRoot, j.Cgroups, t.BootID, observedBoot); err != nil { return refuse(err) } if err := checkBoot(); err != nil { return refuse(err) } ack := DrainClaimAcknowledgement(t.FenceID, t.ControllerID, j.ProcessStart, j.ProducerIDs, j.Completed) return StopKernelEvidence{ObservedBootID: observedBoot, InventorySHA256: digestStop(in), LocalJournalSHA256: digestStop(fresh), CgroupIdentities: groups, UnitInvocationIDs: []string{}}, ack, nil } func scanRebootCgroups(ctx context.Context, root string, groups []hostbox.StopCgroupIdentity, previousBoot, observedBoot string) ([]string, error) { if previousBoot == "" || observedBoot == "" || previousBoot == observedBoot || len(groups) == 0 { return nil, ErrClaimStopIdentity } info, err := os.Stat(root) if err != nil { return nil, err } if !info.IsDir() { return nil, ErrClaimStopInventory } seen := map[string]bool{} out := []string{} for _, g := range groups { if err := ctx.Err(); err != nil { return nil, err } if !filepath.IsLocal(g.RelativePath) || g.RelativePath == "." || filepath.Clean(g.RelativePath) != g.RelativePath || g.Device == 0 || g.Inode == 0 || seen[g.RelativePath] { return nil, ErrClaimStopInventory } seen[g.RelativePath] = true if _, err := os.Lstat(filepath.Join(root, g.RelativePath)); !errors.Is(err, os.ErrNotExist) { if err != nil { return nil, err } return nil, ErrClaimStopInventory } out = append(out, fmt.Sprintf("boot=%s;previous_boot=%s;absent=%s;previous_identity=%d:%d", observedBoot, previousBoot, g.RelativePath, g.Device, g.Inode)) } parents := 0 parent := "" for path := range seen { if !seen[filepath.Dir(path)] { parents++ parent = path } } if parents != 1 { return nil, ErrClaimStopInventory } for path := range seen { if path != parent && !strings.HasPrefix(path, parent+"/") { return nil, ErrClaimStopInventory } } sort.Strings(out) return out, nil } func verifyStopKernelMounts(cgroupRoot, procRoot string) error { var control, process syscall.Statfs_t if err := syscall.Statfs(cgroupRoot, &control); err != nil { return err } if err := syscall.Statfs(procRoot, &process); err != nil { return err } if uint64(control.Type) != 0x63677270 || uint64(process.Type) != 0x9fa0 { return ErrClaimStopUnavailable } return nil } func (k LinuxStopKernel) resolveRoleIntentions(ctx context.Context, t hostguard.FenceTarget, reboot bool) error { if _, err := k.readContainedJournal(ctx, t); err != nil { return err } resolver, ok := k.Journal.(interface { ResolveRoleIntentions(context.Context, hostguard.FenceTarget, string, bool) error }) if !ok { return ErrClaimStopUnavailable } return resolver.ResolveRoleIntentions(ctx, t, k.CgroupRoot, reboot) } func (k LinuxStopKernel) validateContainment(ctx context.Context, t hostguard.FenceTarget, j hostbox.StopKernelJournal) error { if !sameStopSession(j.Target, t) || k.AgentCgroup == nil { return ErrClaimStopInventory } agent, err := k.AgentCgroup(ctx) if err != nil { return err } return validateStopClaimGroups(t, j, agent) } func (k LinuxStopKernel) readContainedJournal(ctx context.Context, t hostguard.FenceTarget) (hostbox.StopKernelJournal, error) { j, err := k.Journal.ReadStopKernelJournal(ctx, t) if err != nil { return j, err } if err = k.validateContainment(ctx, t, j); err != nil { return hostbox.StopKernelJournal{}, err } // Intentions are persisted separately from the concrete groups and must be // checked before their resolver opens any privileged recovery target. if raw, ok := k.Journal.(interface { Read(context.Context, hostguard.FenceTarget) (hostbox.LaunchInventory, error) }); ok { in, e := raw.Read(ctx, t) if e != nil { return hostbox.StopKernelJournal{}, e } agent, e := k.AgentCgroup(ctx) if e != nil { return hostbox.StopKernelJournal{}, e } parent, e := claimContainmentParent(agent, t) if e != nil { return hostbox.StopKernelJournal{}, e } if e = validateClaimContainment(in, parent); e != nil { return hostbox.StopKernelJournal{}, e } } else if j.PendingRoles { return hostbox.StopKernelJournal{}, ErrClaimStopInventory } return j, nil } // CheckRecoveryController lets root restart while the live edge finishes its fence. func (k LinuxStopKernel) CheckRecoveryController(ctx context.Context, t hostguard.FenceTarget) error { if _, err := k.realRoots(); err != nil { return err } j, err := k.readContainedJournal(ctx, t) if err != nil { return err } _, err = recoveryControllerAt(k.ProcRoot, t, j) return err } func recoveryControllerAt(proc string, t hostguard.FenceTarget, j hostbox.StopKernelJournal) (string, error) { if j.ControllerPID <= 0 || j.ProcessStart == "" || !sameStopSession(j.Target, t) { return "", ErrClaimStopIdentity } boot, err := os.ReadFile(filepath.Join(proc, "sys/kernel/random/boot_id")) if err != nil { return "", err } observed := strings.TrimSpace(string(boot)) if observed == "" { return "", ErrClaimStopIdentity } if observed != t.BootID { return observed, nil } raw, err := os.ReadFile(filepath.Join(proc, fmt.Sprint(j.ControllerPID), "stat")) if errors.Is(err, os.ErrNotExist) { return observed, nil } if err != nil { return "", err } i := strings.LastIndexByte(string(raw), ')') if i < 0 { return "", ErrClaimStopIdentity } fields := strings.Fields(string(raw)[i+1:]) if len(fields) <= 19 { return "", ErrClaimStopIdentity } if fields[19] == j.ProcessStart { return "", ErrClaimStopControllerAlive } return observed, nil }