package host import ( "os" "path/filepath" "strings" "testing" ) func readUnit(t *testing.T, name string) string { t.Helper() b, err := unitsFS.ReadFile("units/" + name) if err != nil { t.Fatalf("%s: %v", name, err) } return string(b) } // The outer memory wall moves from the edge unit to carlos.slice // (isolated-exec design §4.3): it bounds the agent plus every awake // instance as before, but instances are now sibling units, not children. func TestCarlosSliceCarriesTheOuterWall(t *testing.T) { s := readUnit(t, "carlos.slice") for _, want := range []string{"[Slice]", "MemoryHigh=70%", "MemoryMax=80%"} { if !strings.Contains(s, want) { t.Errorf("carlos.slice lacks %q", want) } } e := readUnit(t, "carlos-edge.service") if !strings.Contains(e, "Slice=carlos.slice") { t.Error("carlos-edge.service is not in carlos.slice") } for _, never := range []string{"MemoryHigh=", "MemoryMax="} { if strings.Contains(e, never) { t.Errorf("carlos-edge.service still carries %q; the wall is the slice's now", never) } } // What must NOT change on the edge unit. The bounding set and // NoNewPrivileges= are here because dropping RestrictSUIDSGID= (below) // rests on them: with no CAP_CHOWN, CAP_FSETID or CAP_SETUID a set-id // file the agent makes is owned by carlos, and under no_new_privs its // own children ignore the bit. for _, keep := range []string{"OOMPolicy=continue", "Delegate=cpu memory io pids", "User=carlos", "AmbientCapabilities=CAP_NET_BIND_SERVICE", "CapabilityBoundingSet=CAP_NET_BIND_SERVICE", "NoNewPrivileges=yes", "ReadWritePaths=/data /run/carlos", "RestrictNamespaces=yes"} { if !strings.Contains(e, keep) { t.Errorf("carlos-edge.service lost %q", keep) } } // RestrictSUIDSGID= makes openat2 answer ENOSYS, and the agent runs in // this unit: its nested opens and every rename of the layout pass are // openat2 calls (#126, the plan 4 blocker). The probe cannot see this // unit's text, so the test pins it. for _, line := range strings.Split(e, "\n") { if strings.HasPrefix(line, "RestrictSUIDSGID=") { t.Errorf("carlos-edge.service carries %q, which blocks openat2 for the agent", line) } } // The same argument needs every other unit that runs as carlos, or // could execute something under /data, to ignore set-id bits too: // /data itself carries no nosuid, so NoNewPrivileges= on each of them // is what makes a set-id file the agent could make worthless to them. for _, name := range []string{"carlos-app@.service", "carlos-relay.service", "carlos-litestream.service", "carlos-hibernate-verify.service", "carlos-restore-verify.service"} { if !strings.Contains(readUnit(t, name), "NoNewPrivileges=yes") { t.Errorf("%s lacks NoNewPrivileges=yes; the edge unit's dropped RestrictSUIDSGID= rests on it", name) } } i := readUnit(t, "carlos-instances.slice") if !strings.Contains(i, "[Slice]") { t.Error("carlos-instances.slice has no [Slice] section") } for _, never := range []string{"MemoryHigh=", "MemoryMax="} { if strings.Contains(i, never) { t.Errorf("carlos-instances.slice carries %q; the walls are carlos.slice and the per-host slices", never) } } } // The three mounts: a bind of a raw directory onto its protected view, // nosymfollow, with DefaultDependencies=no and explicit ordering, because // the default ordering (before local-fs.target) and tmpfiles (after it) // would form a cycle systemd breaks by dropping the mount (codex R2-2). func TestInstanceMountsAreShapedForNosymfollow(t *testing.T) { cases := map[string][2]string{ "data-instances.mount": {"/data/.instances", "/data/instances"}, "run-carlos-instances.mount": {"/run/carlos/.instances", "/run/carlos/instances"}, "run-carlos-logs.mount": {"/run/carlos/.logs", "/run/carlos/logs"}, } for name, ww := range cases { m := readUnit(t, name) for _, want := range []string{ "What=" + ww[0], "Where=" + ww[1], "Type=none", "Options=bind,nosymfollow,nosuid,nodev,noexec", "DefaultDependencies=no", "After=", "systemd-tmpfiles-setup.service", "Before=", "carlos-edge.service", "Conflicts=umount.target", "Before=umount.target", "WantedBy=multi-user.target", } { if !strings.Contains(m, want) { t.Errorf("%s lacks %q", name, want) } } if ww[0] == ww[1] { t.Errorf("%s binds a directory onto itself; the migration rename would fail with EXDEV (codex R4-2)", name) } } d := readUnit(t, "data-instances.mount") for _, want := range []string{"RequiresMountsFor=/data", "local-fs.target", "carlos-litestream.service"} { if !strings.Contains(d, want) { t.Errorf("data-instances.mount lacks %q", want) } } e := readUnit(t, "carlos-edge.service") for _, want := range []string{"Requires=data-instances.mount run-carlos-instances.mount run-carlos-logs.mount", "After=network-online.target data-instances.mount run-carlos-instances.mount run-carlos-logs.mount"} { if !strings.Contains(e, want) { t.Errorf("carlos-edge.service lacks %q: an edge that starts without the mounts must not start", want) } } l := readUnit(t, "carlos-litestream.service") if !strings.Contains(l, "After=network-online.target data-instances.mount") { t.Error("carlos-litestream.service is not ordered after data-instances.mount") } } // Every unit file, whatever its suffix, is embedded: a .slice or .mount // left out of the glob would be silently absent from every box. func TestEmbeddedUnitsIncludeSlicesAndMounts(t *testing.T) { names, err := unitNames() if err != nil { t.Fatal(err) } have := map[string]bool{} for _, n := range names { have[n] = true } for _, want := range []string{"carlos.slice", "carlos-instances.slice", "data-instances.mount", "run-carlos-instances.mount", "run-carlos-logs.mount"} { if !have[want] { t.Errorf("%s is not embedded (bootstrap.go's go:embed glob)", want) } } } // tmpfiles creates the raw directories and the mountpoints every boot // (/run is tmpfs), with the modes the design gives them (§3.1, §3.2): // the socket trees keep the carlos-sockets group and setgid (issue #103), // the log tree and data trees are the agent's alone, and the control // directory is root's so no carlos process can create anything in it. func TestTmpfilesCreatesTheInstanceTrees(t *testing.T) { for _, want := range []string{ "d /data/.instances 0700 carlos carlos -", "d /data/instances 0700 carlos carlos -", "d /data/layout 0700 carlos carlos -", "d /run/carlos/.instances 2750 carlos carlos-sockets -", "d /run/carlos/instances 2750 carlos carlos-sockets -", "d /run/carlos/.logs 0700 carlos carlos -", "d /run/carlos/logs 0700 carlos carlos -", "d /run/carlos-control 0750 root carlos -", "d /run/carlos/hostguard 2770 carlos carlos -", "d /data/hostguard 2770 carlos carlos -", "f /etc/carlos/probe-secret 0640 root carlos - probe", "f /dev/shm/carlos-probe 0644 root root - probe", } { if !strings.Contains(tmpfilesConf, want+"\n") { t.Errorf("tmpfilesConf lacks %q", want) } } // The existing lines stay: the flat socket dir and the apps subtree. for _, keep := range []string{"d /run/carlos 2750 carlos carlos-sockets -", "d /run/carlos/apps 0750 carlos carlos -"} { if !strings.Contains(tmpfilesConf, keep+"\n") { t.Errorf("tmpfilesConf lost %q", keep) } } } // bootstrap -root creates the same directories in the staging tree // (a live run's chown -R covers them), so a fresh box has them before the // mounts try to bind them. func TestBootstrapCreatesTheInstanceDirectories(t *testing.T) { root := t.TempDir() if err := bootstrap(root, nil); err != nil { t.Fatalf("bootstrap: %v", err) } for _, d := range []string{"data/.instances", "data/instances", "data/layout", "run/carlos/.instances", "run/carlos/instances", "run/carlos/.logs", "run/carlos/logs", "run/carlos-control"} { if fi, err := os.Stat(filepath.Join(root, d)); err != nil || !fi.IsDir() { t.Errorf("%s: %v", d, err) } } } // The mounts are enabled AND started by the plan: `systemctl enable` on a // mount unit writes a symlink for the next boot and nothing else, the same // trap timers and path units fell into (codex R1-14). A socket unit the // same. The relay is still the only service the plan starts. func TestSystemctlPlanStartsMountsAndSockets(t *testing.T) { plan := systemctlPlan([]string{"carlos-edge.service", "data-instances.mount", "run-carlos-instances.mount", "run-carlos-logs.mount", "carlos-adopt.timer", "carlos-instance-control.socket"}) var started []string for _, args := range plan { if args[0] == "start" { started = append(started, args[1:]...) } } got := strings.Join(started, " ") for _, want := range []string{"data-instances.mount", "run-carlos-instances.mount", "run-carlos-logs.mount", "carlos-adopt.timer", "carlos-instance-control.socket"} { if !strings.Contains(got, want) { t.Errorf("plan does not start %s: %v", want, plan) } } if strings.Contains(got, "carlos-edge.service") { t.Errorf("plan starts the edge: %v", plan) } } func TestEnabledUnitsIncludeTheMounts(t *testing.T) { have := map[string]bool{} for _, u := range enabledUnits { have[u] = true } for _, want := range []string{"data-instances.mount", "run-carlos-instances.mount", "run-carlos-logs.mount"} { if !have[want] { t.Errorf("enabledUnits lacks %s", want) } } } // The three tenant templates and the probe carry one namespace: byte- // identical hardening blocks, so a property proven by the probe is the // property a tenant gets (design §4.2, §8.5). func TestInstanceTemplatesShareOneNamespace(t *testing.T) { block := func(name string) string { u := readUnit(t, name) start := strings.Index(u, "# --- the namespace") end := strings.Index(u, "# --- per-service resources") if start < 0 || end < 0 || end < start { t.Fatalf("%s: namespace block markers missing", name) } return u[start:end] } ref := block("carlos-instance@.service") for _, name := range []string{"carlos-instance-gen@.service", "carlos-instance-seed@.service", "carlos-instance-sidecar@.service", "carlos-instance-cmd@.service", "carlos-instance-probe.service"} { if block(name) != ref { t.Errorf("%s namespace block differs from carlos-instance@.service", name) } } for _, want := range []string{ "ProtectSystem=strict", "TemporaryFileSystem=/data:ro", "TemporaryFileSystem=/run/carlos:ro", "TemporaryFileSystem=/var/cache:ro", "TemporaryFileSystem=/var/lib:ro", "InaccessiblePaths=/etc/carlos", "TemporaryFileSystem=/dev/shm:mode=1777,nosuid,nodev", "ProtectControlGroups=yes", "ProtectProc=invisible", "ProcSubset=pid", "PrivatePIDs=yes", "RestrictNamespaces=yes", "NoNewPrivileges=yes", "CapabilityBoundingSet=", "IPAddressDeny=localhost ", " 169.254.169.254/32 ", " fd00:ec2::254/128 ", "PrivateIPC=yes", "MemoryDenyWriteExecute=yes", } { if !strings.Contains(ref, want) { t.Errorf("namespace block lacks %q", want) } } // RestrictSUIDSGID= makes openat2 answer ENOSYS, and every instancefs // open inside the namespace is an openat2 (#126, first roll). for _, line := range strings.Split(ref, "\n") { if strings.HasPrefix(line, "RestrictSUIDSGID=") { t.Errorf("namespace block carries %q, which breaks openat2", line) } } for name, wants := range map[string][]string{ "carlos-instance@.service": {"ExecStart=/opt/carlos/carlos instance-exec run %I", "ExecStopPost=/opt/carlos/carlos instance-exit run %I", "BindPaths=/data/instances/%I /run/carlos/instances/%I", "StandardOutput=file:/run/carlos/logs/%I.run.sock", // The instance's own data directory, where a flat child's // relative paths now resolve (#256 D); the IE box's emergency // drop-in 50-workdir-incident-256.conf carries this exact line, // so removing it changes nothing. "WorkingDirectory=/data/instances/%I", "Slice=carlos-instances-%i.slice", "Restart=no", "KillMode=mixed", "TimeoutStopSec=20", "BindsTo=carlos-edge.service", "Requires=data-instances.mount run-carlos-instances.mount run-carlos-logs.mount", "OOMScoreAdjust=500", "OOMPolicy=kill", "TasksMax=256", "Type=simple", // The trampoline's init exits 128+n for a death by signal n (#126). "SuccessExitStatus=129 130 141 143"}, "carlos-instance-gen@.service": {"ExecStart=/opt/carlos/carlos instance-exec run %I", "ExecStopPost=/opt/carlos/carlos instance-exit run %I", "Restart=no", "KillMode=mixed", "TimeoutStopSec=20", "BindsTo=carlos-edge.service", "Requires=data-instances.mount run-carlos-instances.mount run-carlos-logs.mount", "OOMScoreAdjust=500", "OOMPolicy=kill", "TasksMax=256", "Type=simple", "SuccessExitStatus=129 130 141 143"}, "carlos-instance-seed@.service": {"Type=oneshot", "RemainAfterExit=yes", "TimeoutStartSec=120", "ExecStart=/opt/carlos/carlos instance-exec seed %I", "ExecStopPost=/opt/carlos/carlos instance-exit seed %I", "StandardOutput=file:/run/carlos/logs/%I.seed.sock", "KillMode=mixed", "TimeoutStopSec=20", "WorkingDirectory=/data/instances/%I"}, "carlos-instance-cmd@.service": {"Type=oneshot", "RemainAfterExit=yes", "TimeoutStartSec=6h5m", "ExecStart=/opt/carlos/carlos instance-exec cmd %I", "ExecStopPost=/opt/carlos/carlos instance-exit cmd %I", "StandardOutput=file:/run/carlos/logs/%I.cmd.sock", "StandardError=file:/run/carlos/logs/%I.cmd-err.sock", "KillMode=mixed", "TimeoutStopSec=20", "WorkingDirectory=/data/instances/%I"}, "carlos-instance-sidecar@.service": {"Type=simple", "ExecStart=/opt/carlos/carlos instance-exec sidecar %I", "ExecStopPost=/opt/carlos/carlos instance-exit sidecar %I", "StandardOutput=file:/run/carlos/logs/%I.sidecar.sock", "SuccessExitStatus=129 130 141 143", "WorkingDirectory=/data/instances/%I"}, "carlos-instance-probe.service": {"Type=oneshot", "ExecStart=/opt/carlos/carlos instance-probe", "BindPaths=/data/instances/_probe /run/carlos/instances/_probe", "StandardOutput=truncate:/run/carlos-control/probe.out"}, } { u := readUnit(t, name) for _, w := range wants { if !strings.Contains(u, w) { t.Errorf("%s lacks %q", name, w) } } // Exactly one WorkingDirectory= per tenant template, and none in // the probe, whose directory is _probe and whose cwd does not // matter: a second line would silently win (#256 D). if n := strings.Count(u, "\nWorkingDirectory="); (name != "carlos-instance-probe.service" && n != 1) || (name == "carlos-instance-probe.service" && n != 0) { t.Errorf("%s has %d WorkingDirectory= lines", name, n) } if strings.Contains(u, "ExecStopPost=+") { t.Errorf("%s uses ExecStopPost=+: the plus prefix keeps the unit's tmpfs (codex R3-2)", name) } // A seed a signal ended has not seeded (#126). if (name == "carlos-instance-seed@.service" || name == "carlos-instance-cmd@.service") && strings.Contains(u, "\nSuccessExitStatus=") { t.Errorf("%s counts a signal's exit status as success", name) } if name != "carlos-instance-probe.service" && !strings.Contains(u, "User=carlos") { t.Errorf("%s does not run as carlos", name) } } } func TestControlUnitsShape(t *testing.T) { s := readUnit(t, "carlos-instance-control.socket") for _, want := range []string{"ListenStream=/run/carlos-control/control.sock", "SocketUser=carlos", "SocketGroup=carlos", "SocketMode=0600", "Accept=no", "WantedBy=sockets.target"} { if !strings.Contains(s, want) { t.Errorf("socket unit lacks %q", want) } } u := readUnit(t, "carlos-instance-control.service") for _, want := range []string{"ExecStart=/opt/carlos/carlos instance-control", "Requires=carlos-instance-control.socket", "data-instances.mount", "ProtectSystem=strict", "ReadWritePaths=/run/carlos-control /run/systemd/system", "RestrictAddressFamilies=AF_UNIX", "MemoryMax=64M"} { if !strings.Contains(u, want) { t.Errorf("control service lacks %q", want) } } for _, never := range []string{"User=", "NoNewPrivileges"} { if strings.Contains(u, never) { t.Errorf("control service must run as root for systemctl; found %q", never) } } have := map[string]bool{} for _, e := range enabledUnits { have[e] = true } if !have["carlos-instance-control.socket"] { t.Error("enabledUnits lacks the control socket") } if have["carlos-instance-control.service"] || have["carlos-instance-probe.service"] || have["carlos-instance@.service"] || have["carlos-instance-gen@.service"] { t.Error("a template, the probe or the socket-activated service must not be enabled") } for _, want := range []string{"d /data/.instances/_probe 0700 carlos carlos -", "d /run/carlos/.instances/_probe 2750 carlos carlos-sockets -"} { if !strings.Contains(tmpfilesConf, want+"\n") { t.Errorf("tmpfilesConf lacks %q", want) } } } // The same-uid audit (design §9, codex R9-1, R9-2, R10-1): every tree a // carlos-uid unit writes, every carlos-owned cgroup, and every credential a // carlos-group process reads must be hidden from the tenant templates. func TestSameUIDAuditIsCoveredByTheTemplates(t *testing.T) { ns := readUnit(t, "carlos-instance@.service") hidden := func(path string) bool { for _, line := range strings.Split(ns, "\n") { for _, key := range []string{"TemporaryFileSystem=", "InaccessiblePaths="} { if strings.HasPrefix(line, key) { for _, p := range strings.Fields(strings.TrimPrefix(line, key)) { p = strings.TrimSuffix(p, ":ro") if path == p || strings.HasPrefix(path, p+"/") { return true } } } } } return false } names, err := unitNames() if err != nil { t.Fatal(err) } for _, name := range names { u := readUnit(t, name) if !strings.Contains(u, "User=carlos") { continue } for _, line := range strings.Split(u, "\n") { for _, key := range []string{"ReadWritePaths=", "CacheDirectory=", "StateDirectory=", "RuntimeDirectory="} { if !strings.HasPrefix(line, key) { continue } for _, p := range strings.Fields(strings.TrimPrefix(line, key)) { switch key { case "CacheDirectory=": p = "/var/cache/" + p case "StateDirectory=": p = "/var/lib/" + p case "RuntimeDirectory=": p = "/run/" + p } if !hidden(p) && !strings.HasPrefix(p, "/run/carlos/instances/") && !strings.HasPrefix(p, "/data/instances/") { t.Errorf("%s writes %s, which the tenant template does not hide", name, p) } } } } if strings.Contains(u, "Delegate=") && !strings.Contains(ns, "ProtectControlGroups=yes") { t.Errorf("%s delegates a cgroup subtree and the template leaves cgroupfs writable", name) } } // Credential inputs a carlos-group process reads (the second table). for _, cred := range []string{"/etc/carlos/relay-masters"} { if !hidden(cred) { t.Errorf("credential input %s is not hidden from tenants", cred) } } } // The cmd template is the seed template outside seven named lines (run // spec §5.4, codex 4; the executable mount's own sources, merge ruling // M4): every sandbox line is the seed's, so a command has exactly the // reach a seed has. Comments and blank lines are free; every // directive is compared, in order. func TestCmdTemplateEqualsSeedOutsideTheNamedLines(t *testing.T) { directives := func(name string) []string { var out []string for _, line := range strings.Split(readUnit(t, name), "\n") { line = strings.TrimSpace(line) if line == "" || strings.HasPrefix(line, "#") { continue } out = append(out, line) } return out } swap := map[string]string{ "Description=carlos exec-backed instance %I (seed)": "Description=carlos exec-backed instance %I (cmd)", "TimeoutStartSec=120": "TimeoutStartSec=6h5m", "ExecStart=/opt/carlos/carlos instance-exec seed %I": "ExecStart=/opt/carlos/carlos instance-exec cmd %I", "ExecStopPost=/opt/carlos/carlos instance-exit seed %I": "ExecStopPost=/opt/carlos/carlos instance-exit cmd %I", "StandardOutput=file:/run/carlos/logs/%I.seed.sock": "StandardOutput=file:/run/carlos/logs/%I.cmd.sock", "StandardError=inherit": "StandardError=file:/run/carlos/logs/%I.cmd-err.sock", "BindReadOnlyPaths=/run/carlos-control/exe/%I/app:/opt/carlos/app /run/carlos-control/exe/%I/live:/opt/carlos/live": "BindReadOnlyPaths=/run/carlos-control/exe/%I/cmd-app:/opt/carlos/app /run/carlos-control/exe/%I/cmd-live:/opt/carlos/live", } seed, cmd := directives("carlos-instance-seed@.service"), directives("carlos-instance-cmd@.service") want := make([]string, len(seed)) used := map[string]bool{} for i, l := range seed { if c, ok := swap[l]; ok { want[i], used[l] = c, true continue } want[i] = l } for l := range swap { if !used[l] { t.Errorf("the seed template no longer has %q; the run spec names it as a cmd difference", l) } } for i := 0; i < len(want) || i < len(cmd); i++ { var w, g string if i < len(want) { w = want[i] } if i < len(cmd) { g = cmd[i] } if w != g { t.Fatalf("directive %d: cmd template has %q, want %q (the seed's line with the named swaps)", i, g, w) } } for _, keep := range []string{"Type=oneshot", "RemainAfterExit=yes", "KillMode=mixed", "TimeoutStopSec=20", "WorkingDirectory=/data/instances/%I"} { if !hasDirective(readUnit(t, "carlos-instance-cmd@.service"), keep) { t.Errorf("cmd template lacks %q", keep) } } } // The generation template's instance string is ":" (zero-downtime // spec §7.6, plan 2 D1). No specifier can take the instance ID out of it, // so the template keys NOTHING by %i or %I but the trampoline's argument. // The slice, the data and executable mounts and the log socket come from // the control service's per-generation drop-in (instancectl.GenDropIn). func TestGenerationTemplateKeysNothingByItsInstanceString(t *testing.T) { u := readUnit(t, "carlos-instance-gen@.service") for _, d := range directives(u) { for _, key := range []string{"Slice=", "BindPaths=", "ReadWritePaths=", "StandardOutput="} { if strings.HasPrefix(d, key) { t.Errorf("generation template carries %q; it belongs in the drop-in", d) } } if strings.HasPrefix(d, "BindReadOnlyPaths=") && strings.Contains(d, "/run/carlos-control/exe") { t.Errorf("generation template binds an executable itself: %q", d) } if strings.Contains(d, "/opt/carlos/versions") { t.Errorf("generation template names the version store: %q", d) } if strings.Contains(d, "%i") || (strings.Contains(d, "%I") && !strings.HasPrefix(d, "ExecStart=") && !strings.HasPrefix(d, "ExecStopPost=") && !strings.HasPrefix(d, "Description=")) { t.Errorf("generation template keys %q by its instance string", d) } } }