package instanceexec import ( "crypto/rand" "encoding/hex" "encoding/json" "errors" "flag" "fmt" "net" "os" "path/filepath" "sort" "strconv" "syscall" "time" "github.com/carlosframework/platform/internal/cgroup" "github.com/carlosframework/platform/internal/cliflag" "github.com/carlosframework/platform/internal/controlsock" "github.com/carlosframework/platform/internal/instancectl" "github.com/carlosframework/platform/internal/instancefs" ) // probeEnv is everything the checks touch, so the verdict logic is testable // against a fixture tree and fake syscalls while production runs against // the real namespace. type probeEnv struct { dataRoot string // /data as the probe sees it runRoot string // /run/carlos as the probe sees it varCache string // /var/cache varLib string // /var/lib etcCarlos string // /etc/carlos devShm string // /dev/shm, which must be this unit's own procRoot string // /proc controlDir string // /run/carlos-control, which must be unusable from in here controlSock string // the control socket inside it, which must not dial cgroupProcs string // the edge's delegated cgroup.procs, to test writability on pid func() int writable func(path string) error sendUDP func(network, addr string) error connectTCP func(addr string, timeout time.Duration) error mountNoSym func(mountpoint string) (bool, error) // mountOption asks whether a mountpoint carries a given per-mount // option; wired to instancefs.MountOptionOf in production. It backs // the cgroup-ro-mount check, which asserts /sys/fs/cgroup itself // carries "ro" rather than trusting cgroup-ro's access(W_OK) alone. mountOption func(mountpoint, opt string) (bool, error) // symlinkCheck plants a symlink under dir and reports whether it was // refused. Production wires this to realSymlinkCheck; the test fixture // stubs it because the fixture's tree is a plain temp dir with no // nosymfollow mount, so the real check would fail there for a reason // unrelated to the property under test in the other cases. symlinkCheck func(dir string) error // The hardening of #214 and #178. optRoot string // /opt/carlos, which must hold only this unit's own build resolvConf string // /etc/resolv.conf, whose resolvers must be reachable devMqueue string // /dev/mqueue, which must not be the host's // ipcInode is this process's IPC namespace inode; the box's own is // initIPCInode. ipcInode func() (uint64, error) // abstractScoped proves the trampoline's Landlock scope on this // kernel: nil when a scoped connect to an outside abstract name was // refused with EPERM. abstractScoped func() error // selfCgroup is this unit's own cgroup directory, where the probe // unit's CPUQuota=, CPUWeight= and IOWeight= must have landed. selfCgroup func() (string, error) // ioEnforced says whether io.weight has a controller behind it. ioEnforced func(selfCgroup string) bool } // cgroupMountpoint is the fixed system path the cgroup-ro-mount check // asks instancefs.MountOptionOf about. It is not a probeEnv field: every // test that cares stubs the mountOption function itself, so no fixture // needs a real mount at this path. const cgroupMountpoint = "/sys/fs/cgroup" var errNotELOOP = errors.New("symlink was followed") // RunProbe is `carlos instance-probe`: it runs inside the probe unit's // namespace (the same text as the tenant templates) and prints a // ProbeReport on stdout, which the unit's StandardOutput=truncate: lands in // /run/carlos-control/probe.out for the control service to read (design // §8.5). It exits 0 whatever the verdict: the verdict is the output, and a // failed unit would hide it. func RunProbe(args []string) { fs := flag.NewFlagSet("instance-probe", flag.ExitOnError) fs.Usage = cliflag.Usage(fs, "usage: carlos instance-probe") _ = fs.Parse(args) r := runChecks(productionProbeEnv()) enc := json.NewEncoder(os.Stdout) enc.SetIndent("", " ") _ = enc.Encode(r) } func productionProbeEnv() probeEnv { return probeEnv{ dataRoot: "/data", runRoot: "/run/carlos", varCache: "/var/cache", varLib: "/var/lib", etcCarlos: "/etc/carlos", devShm: "/dev/shm", procRoot: "/proc", controlDir: controlsock.Dir, controlSock: controlsock.Path, // The edge's own delegated cgroup.procs, not the probe's: Delegate= // on carlos-edge.service chowns this file to carlos, so DAC passes // and only the mount's ro flag can refuse the write, making EROFS // evidence rather than an artifact of ownership (final review, // plan 2, item 1). This depends on the edge being loaded, which // every instance unit's BindsTo=carlos-edge.service already // assumes. cgroupProcs: "/sys/fs/cgroup/carlos.slice/carlos-edge.service/cgroup.procs", pid: os.Getpid, writable: accessWritable, sendUDP: sendOneDatagram, connectTCP: connectWithTimeout, mountNoSym: instancefs.MountNoSymfollow, mountOption: instancefs.MountOptionOf, symlinkCheck: realSymlinkCheck, optRoot: "/opt/carlos", resolvConf: "/etc/resolv.conf", devMqueue: "/dev/mqueue", ipcInode: ipcNamespaceInode, abstractScoped: realAbstractCheck, selfCgroup: ownCgroupDir, ioEnforced: ioWeightEnforced, } } // ownCgroupDir is /sys/fs/cgroup plus this process's unified-hierarchy // path from /proc/self/cgroup. func ownCgroupDir() (string, error) { b, err := os.ReadFile("/proc/self/cgroup") if err != nil { return "", err } rel, ok := cgroup.ParseSelfCgroup(string(b)) if !ok { return "", errors.New("no cgroup v2 path in /proc/self/cgroup") } return filepath.Join(cgroupMountpoint, rel), nil } func listNames(dir string) ([]string, error) { entries, err := os.ReadDir(dir) if err != nil { return nil, err } names := make([]string, 0, len(entries)) for _, e := range entries { names = append(names, e.Name()) } sort.Strings(names) return names, nil } func exactly(dir string, want ...string) (string, bool) { got, err := listNames(dir) if err != nil { return err.Error(), false } sort.Strings(want) if fmt.Sprint(got) != fmt.Sprint(want) { return fmt.Sprintf("%s lists %v, want %v", dir, got, want), false } return "", true } // runChecks is the verdict. Each line names the property it checks; "fail" // blocks readiness, "unverified" does not, "ok" is what a ready box shows // on every line. func runChecks(e probeEnv) instancectl.ProbeReport { var r instancectl.ProbeReport add := func(check, verdict, detail string) { r.Lines = append(r.Lines, instancectl.VerdictLine{Check: check, Verdict: verdict, Detail: detail}) } pass := func(check string, detail string, ok bool) { if ok { add(check, "ok", "") } else { add(check, "fail", detail) } } // The two trees hold exactly the probe's own directory. d1, ok1 := exactly(e.dataRoot, "instances") d2, ok2 := exactly(filepath.Join(e.dataRoot, "instances"), instancectl.ProbeDirName) pass("data-tree", d1+d2, ok1 && ok2) r1, ok1 := exactly(e.runRoot, "instances") r2, ok2 := exactly(filepath.Join(e.runRoot, "instances"), instancectl.ProbeDirName) pass("run-tree", r1+r2, ok1 && ok2) // The two trees other carlos-uid units write are empty. v1, ok1 := exactly(e.varCache) v2, ok2 := exactly(e.varLib) pass("var-empty", v1+v2, ok1 && ok2) // A private pid namespace: this process is pid 1 and /proc lists no // other pid. pids, err := listNames(e.procRoot) others := "" for _, n := range pids { if v, err := strconv.Atoi(n); err == nil && v != 1 { others += n + " " } } pid := e.pid() pass("pid1", fmt.Sprintf("pid=%d others=[%s] err=%v", pid, others, err), err == nil && pid == 1 && others == "") // cgroupfs is read-only, checked two ways. access(W_OK) on the edge's // delegated cgroup.procs must answer EROFS: DAC is checked before the // mount's read-only flag, so EACCES here means the file is not // delegated to carlos (not a pass, whatever the mount says) and ENOENT // means the edge unit is not loaded at all, neither of which the mount // flag could produce on its own. And /sys/fs/cgroup itself must carry // the "ro" mount option, so the EROFS above is the mount flag's doing // and not some other cause (final review, plan 2, item 1). werr := e.writable(e.cgroupProcs) detail, ok := cgroupROVerdict(werr) pass("cgroup-ro", detail, ok) mro, mroErr := e.mountOption(cgroupMountpoint, "ro") pass("cgroup-ro-mount", fmt.Sprintf("MountOptionOf(%s, ro) = %v, %v", cgroupMountpoint, mro, mroErr), mro && mroErr == nil) // The control socket is unreachable from in here: InaccessiblePaths= // in the shared namespace block (design §5.2) makes a connect from // inside any instance impossible outright, not merely refused by the // peer check on the other end. The path itself still answers Lstat, // because systemd hides it under a mode 000 root-owned node rather // than removing it, so the check is what a tenant could actually do: // list the directory, or dial the socket (#126, first roll). // Both must be refused for permission or absence: a refused // connection or a timeout means the path was reachable. _, cerr := os.ReadDir(e.controlDir) kerr := dialUnix(e.controlSock) pass("control-unreachable", fmt.Sprintf("readdir=%v dial=%v", cerr, kerr), unreachable(cerr) && unreachable(kerr)) // /etc/carlos cannot be opened and its fixture cannot be read. _, derr := os.ReadDir(e.etcCarlos) _, ferr := os.ReadFile(filepath.Join(e.etcCarlos, "probe-secret")) pass("etc-carlos", fmt.Sprintf("readdir=%v readfile=%v", derr, ferr), derr != nil && ferr != nil) // /dev/shm is this unit's own tmpfs, not the host's, which PrivateDevices= // would bind in: the host's holds the fixture /dev/shm/carlos-probe, so // an empty listing here means a sibling's shared memory is out of reach // (#126, R15). s1, sok := exactly(e.devShm) pass("shm-private", s1, sok) // A symlink planted in the probe's own directory is refused with ELOOP. serr := e.symlinkCheck(filepath.Join(e.dataRoot, "instances", instancectl.ProbeDirName)) pass("symlink-eloop", fmt.Sprint(serr), serr == nil) // Both bound directories are on nosymfollow mounts. n1, err1 := e.mountNoSym(filepath.Join(e.dataRoot, "instances", instancectl.ProbeDirName)) n2, err2 := e.mountNoSym(filepath.Join(e.runRoot, "instances", instancectl.ProbeDirName)) pass("nosymfollow", fmt.Sprintf("data=%v/%v run=%v/%v", n1, err1, n2, err2), n1 && n2 && err1 == nil && err2 == nil) // IMDS, for a packet filter: a datagram's verdict comes straight back // from sendmsg. v4 must be EPERM. v6 is EPERM (enforced) or ENETUNREACH // (no route to that address: unverified, not blocking). The loopback // control proves the stack is live. A TCP connect must not succeed. v4 := e.sendUDP("udp4", "169.254.169.254:80") pass("imds-v4", fmt.Sprintf("sendto = %v", v4), errors.Is(v4, syscall.EPERM)) v6 := e.sendUDP("udp6", "[fd00:ec2::254]:80") switch { case errors.Is(v6, syscall.EPERM): add("imds-v6", "ok", "") case errors.Is(v6, syscall.ENETUNREACH): add("imds-v6", "unverified", "no route to the address; deny not exercised") default: add("imds-v6", "fail", fmt.Sprintf("sendto = %v", v6)) } tcp := e.connectTCP("169.254.169.254:80", 2*time.Second) pass("imds-tcp", fmt.Sprintf("connect = %v", tcp), tcp != nil) hardeningChecks(e, add, pass) r.Ready = true for _, l := range r.Lines { if l.Verdict == "fail" { r.Ready = false } } return r } // cgroupROVerdict turns access(W_OK) on the edge's delegated cgroup.procs // into the cgroup-ro check's detail and pass/fail: EROFS is the only // answer the read-only mount flag alone can produce, since the kernel // checks DAC first and Delegate= has already given carlos write access on // the file itself (final review, plan 2, item 1). func cgroupROVerdict(werr error) (detail string, ok bool) { switch { case errors.Is(werr, syscall.EROFS): return "", true case errors.Is(werr, syscall.EACCES): return "not delegated", false case errors.Is(werr, syscall.ENOENT): return "edge not loaded", false case werr == nil: return "writable", false default: return werr.Error(), false } } // randomName returns a 16-hex-digit name. A test overrides it to force a // name collision; production draws 8 random bytes so two probe runs, or a // probe run and a flat-tenant's own writes, never collide by accident. var randomName = func() string { var b [8]byte if _, err := rand.Read(b[:]); err != nil { // crypto/rand.Read only fails if the OS's random source is // unavailable, which is a box-level emergency worth stopping the // probe over rather than silently falling back to a fixed name // that reintroduces the property this check exists to catch. panic(fmt.Sprintf("instance-probe: crypto/rand: %v", err)) } return hex.EncodeToString(b[:]) } // realSymlinkCheck plants a symlink under a random name inside dir, // pointing at a real file under another random name, and tries to open // the link through its name; on a nosymfollow mount the open fails with // ELOOP. Random names mean a same-uid tenant that shares this directory // cannot pre-plant either one (final review, plan 2, item 5); an EEXIST // planting either name is reported as a failure and nothing at that name // is removed, since it was not ours to remove. It returns nil for ELOOP, // errNotELOOP if the link was followed, and any other error. func realSymlinkCheck(dir string) error { target := randomName() f, err := instancefs.OpenBeneath(dir, target, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600) if err != nil { if errors.Is(err, os.ErrExist) { return fmt.Errorf("a planted entry already occupies %s", filepath.Join(dir, target)) } return err } _, werr := f.Write([]byte("x")) cerr := f.Close() targetPath := filepath.Join(dir, target) defer os.Remove(targetPath) if werr != nil { return werr } if cerr != nil { return cerr } link := randomName() linkPath := filepath.Join(dir, link) if err := os.Symlink(targetPath, linkPath); err != nil { if errors.Is(err, os.ErrExist) { return fmt.Errorf("a planted entry already occupies %s", linkPath) } return err } defer os.Remove(linkPath) // A plain open, deliberately: the whole point of this check is that // the MOUNT's nosymfollow option refuses the follow. A resolve flag of // our own (RESOLVE_NO_SYMLINKS) would answer ELOOP whether or not the // mount carries the option, making the line vacuous either way (ledger // ruling, final review, plan 2, item 5). lf, err := os.Open(linkPath) if err == nil { lf.Close() return errNotELOOP } if errors.Is(err, syscall.ELOOP) { return nil } return err } // dialUnix connects to a unix socket and closes the connection at once, // returning nil only if the connect succeeded. func dialUnix(path string) error { c, err := net.DialTimeout("unix", path, 2*time.Second) if err != nil { return err } return c.Close() } // unreachable reports whether err is a refusal for permission or absence: // the answers a hidden path gives. Any other error, a refused connection // among them, means the path itself was reached. func unreachable(err error) bool { return errors.Is(err, syscall.EACCES) || errors.Is(err, syscall.EPERM) || errors.Is(err, syscall.ENOENT) || errors.Is(err, syscall.ENOTDIR) }