//go:build linux package instanceexec import ( "errors" "net" "os" "path/filepath" "strings" "syscall" "testing" "time" "github.com/carlosframework/platform/internal/instancectl" ) // goodEnv is a fixture that passes every check: the trees an isolated // instance sees, pid 1, a read-only cgroupfs, an unopenable /etc/carlos, a // nosymfollow mount, EPERM on both metadata addresses, a live loopback and // a TCP connect that never succeeds. func goodEnv(t *testing.T) probeEnv { t.Helper() root := shortProbeSocketDir(t) mk := func(p string) string { if err := os.MkdirAll(filepath.Join(root, p), 0o700); err != nil { t.Fatal(err) } return filepath.Join(root, p) } data := mk("data") mk("data/instances/_probe") run := mk("run/carlos") mk("run/carlos/instances/_probe") varCache := mk("var/cache") varLib := mk("var/lib") etc := filepath.Join(root, "etc", "carlos") // deliberately absent: opening it fails shm := mk("dev/shm") proc := mk("proc") mk("proc/1") mk("proc/self") opt := mk("opt/carlos") mk("opt/carlos/live") for _, f := range []string{"opt/carlos/app", "opt/carlos/carlos"} { if err := os.WriteFile(filepath.Join(root, f), nil, 0o755); err != nil { t.Fatal(err) } } if err := os.Symlink("../app", filepath.Join(opt, "live", "_probe")); err != nil { t.Fatal(err) } resolv := filepath.Join(root, "resolv.conf") if err := os.WriteFile(resolv, []byte("# fixture\nnameserver 10.0.0.2\nsearch eu-west-1.compute.internal\n"), 0o644); err != nil { t.Fatal(err) } mqueue := mk("dev/mqueue") cg := mk("cgroup/self") for name, v := range map[string]string{"cpu.max": "50000 100000\n", "cpu.weight": "50\n", "io.weight": "default 50\n"} { if err := os.WriteFile(filepath.Join(cg, name), []byte(v), 0o644); err != nil { t.Fatal(err) } } return probeEnv{ dataRoot: data, runRoot: run, varCache: varCache, varLib: varLib, etcCarlos: etc, devShm: shm, procRoot: proc, controlDir: filepath.Join(root, "run", "carlos-control"), // deliberately absent: ReadDir and dial fail controlSock: filepath.Join(root, "run", "carlos-control", "control.sock"), cgroupProcs: filepath.Join(root, "sys", "cgroup.procs"), pid: func() int { return 1 }, writable: func(string) error { return syscall.EROFS }, // The fence: everything is refused but the box's resolver. sendUDP: func(network, addr string) error { if addr == "10.0.0.2:53" { return nil } return syscall.EPERM }, connectTCP: func(string, time.Duration) error { return syscall.ETIMEDOUT }, mountNoSym: func(string) (bool, error) { return true, nil }, mountOption: func(string, string) (bool, error) { return true, nil }, symlinkCheck: func(string) error { return nil }, optRoot: opt, resolvConf: resolv, devMqueue: mqueue, ipcInode: func() (uint64, error) { return 4026532412, nil }, abstractScoped: func() error { return nil }, selfCgroup: func() (string, error) { return cg, nil }, ioEnforced: func(string) bool { return true }, } } func verdict(r instancectl.ProbeReport, check string) string { for _, l := range r.Lines { if l.Check == check { return l.Verdict } } return "" } func TestProbePassesOnAnIsolatedNamespace(t *testing.T) { r := runChecks(goodEnv(t)) if !r.Ready { t.Fatalf("not ready: %+v", r.Lines) } for _, c := range []string{"data-tree", "run-tree", "var-empty", "pid1", "cgroup-ro", "cgroup-ro-mount", "control-unreachable", "etc-carlos", "shm-private", "symlink-eloop", "nosymfollow", "imds-v4", "imds-v6", "imds-tcp", "opt-own-build", "dns", "resolver-ports", "loopback-v4", "loopback-v6", "private-v4", "private-v6", "ipc-private", "abstract-unix", "cpu-limits", "io-weight"} { if v := verdict(r, c); v != "ok" { t.Errorf("%s = %s", c, v) } } } func TestProbeFailsWhenASiblingIsVisible(t *testing.T) { e := goodEnv(t) if err := os.MkdirAll(filepath.Join(e.dataRoot, "instances", "other.example.com"), 0o700); err != nil { t.Fatal(err) } r := runChecks(e) if r.Ready || verdict(r, "data-tree") != "fail" { t.Errorf("sibling visible yet ready=%v data-tree=%s", r.Ready, verdict(r, "data-tree")) } } func TestProbeFailsWhenCgroupfsIsWritable(t *testing.T) { e := goodEnv(t) e.writable = func(string) error { return nil } r := runChecks(e) if r.Ready || verdict(r, "cgroup-ro") != "fail" { t.Errorf("writable cgroupfs yet ready=%v", r.Ready) } } // TestCgroupROVerdictDistinguishesTheThreeAnswers pins that DAC (checked // first by the kernel) and the mount flag are told apart, not merged into // one pass/fail: EACCES means the file is not delegated to carlos, ENOENT // means the edge unit is not loaded, and neither is the mount flag's own // answer (final review, plan 2, item 1). func TestCgroupROVerdictDistinguishesTheThreeAnswers(t *testing.T) { cases := []struct { err error detail string ok bool }{ {syscall.EROFS, "", true}, {syscall.EACCES, "not delegated", false}, {syscall.ENOENT, "edge not loaded", false}, {nil, "writable", false}, } for _, c := range cases { detail, ok := cgroupROVerdict(c.err) if detail != c.detail || ok != c.ok { t.Errorf("cgroupROVerdict(%v) = %q, %v, want %q, %v", c.err, detail, ok, c.detail, c.ok) } } } func TestProbeFailsWhenTheCgroupMountLacksRO(t *testing.T) { e := goodEnv(t) e.mountOption = func(string, string) (bool, error) { return false, nil } r := runChecks(e) if r.Ready || verdict(r, "cgroup-ro-mount") != "fail" { t.Errorf("cgroup mount without ro yet ready=%v", r.Ready) } } func TestProbeFailsWhenTheControlSocketDirectoryIsVisible(t *testing.T) { e := goodEnv(t) if err := os.MkdirAll(e.controlDir, 0o750); err != nil { t.Fatal(err) } r := runChecks(e) if r.Ready || verdict(r, "control-unreachable") != "fail" { t.Errorf("visible control socket directory yet ready=%v", r.Ready) } } // TestProbePassesOnAnInaccessibleControlNode pins the shape // InaccessiblePaths= really leaves: a mode 000 directory that still // answers Lstat. The first probe on a box failed here because it checked // Lstat; what a tenant cannot do is list it or dial through it (#126). func TestProbePassesOnAnInaccessibleControlNode(t *testing.T) { if os.Geteuid() == 0 { t.Skip("root reads a mode 000 directory") } e := goodEnv(t) if err := os.MkdirAll(e.controlDir, 0o700); err != nil { t.Fatal(err) } if err := os.Chmod(e.controlDir, 0); err != nil { t.Fatal(err) } t.Cleanup(func() { os.Chmod(e.controlDir, 0o700) }) if _, err := os.Lstat(e.controlDir); err != nil { t.Fatalf("fixture: Lstat = %v, want the node to exist", err) } r := runChecks(e) if v := verdict(r, "control-unreachable"); v != "ok" { t.Errorf("control-unreachable = %s on a mode 000 node: %+v", v, r.Lines) } } // TestProbeFailsWhenTheControlSocketDials: a directory that refuses a // listing but still lets a connect through is not unreachable. func TestProbeFailsWhenTheControlSocketDials(t *testing.T) { if os.Geteuid() == 0 { t.Skip("root reads a mode 0300 directory") } e := goodEnv(t) if err := os.MkdirAll(e.controlDir, 0o700); err != nil { t.Fatal(err) } l, err := net.Listen("unix", e.controlSock) if err != nil { t.Fatalf("listen at %s (a socket test wants a short TMPDIR): %v", e.controlSock, err) } defer l.Close() go func() { for { c, err := l.Accept() if err != nil { return } c.Close() } }() if err := os.Chmod(e.controlDir, 0o300); err != nil { t.Fatal(err) } t.Cleanup(func() { os.Chmod(e.controlDir, 0o700) }) r := runChecks(e) if r.Ready || verdict(r, "control-unreachable") != "fail" { t.Errorf("dialable control socket yet ready=%v control-unreachable=%s", r.Ready, verdict(r, "control-unreachable")) } } // TestProbeFailsOnAStaleControlSocket: a socket file with nothing // listening refuses the connection, which proves the path was reached. func TestProbeFailsOnAStaleControlSocket(t *testing.T) { if os.Geteuid() == 0 { t.Skip("root reads a mode 0300 directory") } e := goodEnv(t) if err := os.MkdirAll(e.controlDir, 0o700); err != nil { t.Fatal(err) } l, err := net.Listen("unix", e.controlSock) if err != nil { t.Fatalf("listen at %s (a socket test wants a short TMPDIR): %v", e.controlSock, err) } l.(*net.UnixListener).SetUnlinkOnClose(false) l.Close() if err := os.Chmod(e.controlDir, 0o300); err != nil { t.Fatal(err) } t.Cleanup(func() { os.Chmod(e.controlDir, 0o700) }) r := runChecks(e) if r.Ready || verdict(r, "control-unreachable") != "fail" { t.Errorf("stale control socket yet ready=%v control-unreachable=%s", r.Ready, verdict(r, "control-unreachable")) } } // TestProbeFailsWhenTheHostShmIsVisible: PrivateDevices= alone binds the // host's /dev/shm, where tmpfiles plants the fixture (#126, R15). func TestProbeFailsWhenTheHostShmIsVisible(t *testing.T) { e := goodEnv(t) if err := os.WriteFile(filepath.Join(e.devShm, "carlos-probe"), []byte("probe"), 0o644); err != nil { t.Fatal(err) } r := runChecks(e) if r.Ready || verdict(r, "shm-private") != "fail" { t.Errorf("host /dev/shm visible yet ready=%v shm-private=%s", r.Ready, verdict(r, "shm-private")) } } func TestProbeFailsWhenEtcCarlosIsReadable(t *testing.T) { e := goodEnv(t) if err := os.MkdirAll(e.etcCarlos, 0o755); err != nil { t.Fatal(err) } if err := os.WriteFile(filepath.Join(e.etcCarlos, "probe-secret"), []byte("probe"), 0o644); err != nil { t.Fatal(err) } r := runChecks(e) if r.Ready || verdict(r, "etc-carlos") != "fail" { t.Errorf("readable /etc/carlos yet ready=%v", r.Ready) } } func TestProbeIMDSVerdicts(t *testing.T) { // v4 must be EPERM; anything else is a failure, including a timeout. e := goodEnv(t) e.sendUDP = func(_, addr string) error { if addr == "169.254.169.254:80" { return syscall.ETIMEDOUT } return syscall.EPERM } if r := runChecks(e); r.Ready || verdict(r, "imds-v4") != "fail" { t.Errorf("v4 timeout: ready=%v v4=%s", r.Ready, verdict(r, "imds-v4")) } // v6 ENETUNREACH is unverified and does not block readiness. e = goodEnv(t) e.sendUDP = func(_, addr string) error { if addr == "[fd00:ec2::254]:80" { return syscall.ENETUNREACH } if addr == "10.0.0.2:53" { return nil } return syscall.EPERM } if r := runChecks(e); !r.Ready || verdict(r, "imds-v6") != "unverified" { t.Errorf("v6 no route: ready=%v v6=%s", r.Ready, verdict(r, "imds-v6")) } // A successful send to either address is a failure. e = goodEnv(t) e.sendUDP = func(string, string) error { return nil } if r := runChecks(e); r.Ready || verdict(r, "imds-v4") != "fail" { t.Errorf("v4 success: ready=%v", r.Ready) } // A TCP connect that succeeds is a failure. e = goodEnv(t) e.connectTCP = func(string, time.Duration) error { return nil } if r := runChecks(e); r.Ready || verdict(r, "imds-tcp") != "fail" { t.Errorf("tcp success: ready=%v", r.Ready) } } func TestProbeSymlinkCheckUsesARealSymlink(t *testing.T) { // The fixture's data tree is a plain temp dir with no nosymfollow, so // the real symlink check FAILS here: that proves the check is real. e := goodEnv(t) if !errors.Is(realSymlinkCheck(filepath.Join(e.dataRoot, "instances", "_probe")), errNotELOOP) { t.Error("a followable symlink was not reported") } } // TestRealSymlinkCheckReportsAPlantedName pins the review fix: a name the // check would have used is refused (EEXIST), reported as a failure naming // the planted entry, and left in place, rather than removed. A same-uid // tenant sharing this directory could otherwise have its own file deleted // by a probe run (final review, plan 2, item 5). func TestRealSymlinkCheckReportsAPlantedName(t *testing.T) { dir := t.TempDir() orig := randomName t.Cleanup(func() { randomName = orig }) randomName = func() string { return "planted" } if err := os.MkdirAll(filepath.Join(dir, "planted"), 0o700); err != nil { t.Fatal(err) } err := realSymlinkCheck(dir) if err == nil { t.Fatal("a colliding name was not reported") } if !strings.Contains(err.Error(), filepath.Join(dir, "planted")) { t.Errorf("error does not name the planted entry: %v", err) } if fi, statErr := os.Stat(filepath.Join(dir, "planted")); statErr != nil || !fi.IsDir() { t.Errorf("the planted entry was removed: %v, %v", fi, statErr) } } func shortProbeSocketDir(t *testing.T) string { t.Helper() dir, err := os.MkdirTemp(os.TempDir(), "probe-") if err != nil { t.Fatal(err) } t.Cleanup(func() { if err := os.RemoveAll(dir); err != nil { t.Error(err) } }) return dir }