package host
import (
"os"
"path/filepath"
"strings"
"testing"
)
func readUnit(t *testing.T, name string) string {
t.Helper()
b, err := unitsFS.ReadFile("units/" + name)
if err != nil {
t.Fatalf("%s: %v", name, err)
}
return string(b)
}
// The outer memory wall moves from the edge unit to carlos.slice
// (isolated-exec design §4.3): it bounds the agent plus every awake
// instance as before, but instances are now sibling units, not children.
func TestCarlosSliceCarriesTheOuterWall(t *testing.T) {
s := readUnit(t, "carlos.slice")
for _, want := range []string{"[Slice]", "MemoryHigh=70%", "MemoryMax=80%"} {
if !strings.Contains(s, want) {
t.Errorf("carlos.slice lacks %q", want)
}
}
e := readUnit(t, "carlos-edge.service")
if !strings.Contains(e, "Slice=carlos.slice") {
t.Error("carlos-edge.service is not in carlos.slice")
}
for _, never := range []string{"MemoryHigh=", "MemoryMax="} {
if strings.Contains(e, never) {
t.Errorf("carlos-edge.service still carries %q; the wall is the slice's now", never)
}
}
// What must NOT change on the edge unit. The bounding set and
// NoNewPrivileges= are here because dropping RestrictSUIDSGID= (below)
// rests on them: with no CAP_CHOWN, CAP_FSETID or CAP_SETUID a set-id
// file the agent makes is owned by carlos, and under no_new_privs its
// own children ignore the bit.
for _, keep := range []string{"OOMPolicy=continue", "Delegate=cpu memory io pids", "User=carlos",
"AmbientCapabilities=CAP_NET_BIND_SERVICE", "CapabilityBoundingSet=CAP_NET_BIND_SERVICE", "NoNewPrivileges=yes",
"ReadWritePaths=/data /run/carlos", "RestrictNamespaces=yes"} {
if !strings.Contains(e, keep) {
t.Errorf("carlos-edge.service lost %q", keep)
}
}
// RestrictSUIDSGID= makes openat2 answer ENOSYS, and the agent runs in
// this unit: its nested opens and every rename of the layout pass are
// openat2 calls (#126, the plan 4 blocker). The probe cannot see this
// unit's text, so the test pins it.
for _, line := range strings.Split(e, "\n") {
if strings.HasPrefix(line, "RestrictSUIDSGID=") {
t.Errorf("carlos-edge.service carries %q, which blocks openat2 for the agent", line)
}
}
// The same argument needs every other unit that runs as carlos, or
// could execute something under /data, to ignore set-id bits too:
// /data itself carries no nosuid, so NoNewPrivileges= on each of them
// is what makes a set-id file the agent could make worthless to them.
for _, name := range []string{"carlos-app@.service", "carlos-relay.service", "carlos-litestream.service",
"carlos-hibernate-verify.service", "carlos-restore-verify.service"} {
if !strings.Contains(readUnit(t, name), "NoNewPrivileges=yes") {
t.Errorf("%s lacks NoNewPrivileges=yes; the edge unit's dropped RestrictSUIDSGID= rests on it", name)
}
}
i := readUnit(t, "carlos-instances.slice")
if !strings.Contains(i, "[Slice]") {
t.Error("carlos-instances.slice has no [Slice] section")
}
for _, never := range []string{"MemoryHigh=", "MemoryMax="} {
if strings.Contains(i, never) {
t.Errorf("carlos-instances.slice carries %q; the walls are carlos.slice and the per-host slices", never)
}
}
}
// The three mounts: a bind of a raw directory onto its protected view,
// nosymfollow, with DefaultDependencies=no and explicit ordering, because
// the default ordering (before local-fs.target) and tmpfiles (after it)
// would form a cycle systemd breaks by dropping the mount (codex R2-2).
func TestInstanceMountsAreShapedForNosymfollow(t *testing.T) {
cases := map[string][2]string{
"data-instances.mount": {"/data/.instances", "/data/instances"},
"run-carlos-instances.mount": {"/run/carlos/.instances", "/run/carlos/instances"},
"run-carlos-logs.mount": {"/run/carlos/.logs", "/run/carlos/logs"},
}
for name, ww := range cases {
m := readUnit(t, name)
for _, want := range []string{
"What=" + ww[0], "Where=" + ww[1], "Type=none",
"Options=bind,nosymfollow,nosuid,nodev,noexec",
"DefaultDependencies=no",
"After=", "systemd-tmpfiles-setup.service",
"Before=", "carlos-edge.service",
"Conflicts=umount.target", "Before=umount.target",
"WantedBy=multi-user.target",
} {
if !strings.Contains(m, want) {
t.Errorf("%s lacks %q", name, want)
}
}
if ww[0] == ww[1] {
t.Errorf("%s binds a directory onto itself; the migration rename would fail with EXDEV (codex R4-2)", name)
}
}
d := readUnit(t, "data-instances.mount")
for _, want := range []string{"RequiresMountsFor=/data", "local-fs.target", "carlos-litestream.service"} {
if !strings.Contains(d, want) {
t.Errorf("data-instances.mount lacks %q", want)
}
}
e := readUnit(t, "carlos-edge.service")
for _, want := range []string{"Requires=data-instances.mount run-carlos-instances.mount run-carlos-logs.mount",
"After=network-online.target data-instances.mount run-carlos-instances.mount run-carlos-logs.mount"} {
if !strings.Contains(e, want) {
t.Errorf("carlos-edge.service lacks %q: an edge that starts without the mounts must not start", want)
}
}
l := readUnit(t, "carlos-litestream.service")
if !strings.Contains(l, "After=network-online.target data-instances.mount") {
t.Error("carlos-litestream.service is not ordered after data-instances.mount")
}
}
// Every unit file, whatever its suffix, is embedded: a .slice or .mount
// left out of the glob would be silently absent from every box.
func TestEmbeddedUnitsIncludeSlicesAndMounts(t *testing.T) {
names, err := unitNames()
if err != nil {
t.Fatal(err)
}
have := map[string]bool{}
for _, n := range names {
have[n] = true
}
for _, want := range []string{"carlos.slice", "carlos-instances.slice", "data-instances.mount", "run-carlos-instances.mount", "run-carlos-logs.mount"} {
if !have[want] {
t.Errorf("%s is not embedded (bootstrap.go's go:embed glob)", want)
}
}
}
// tmpfiles creates the raw directories and the mountpoints every boot
// (/run is tmpfs), with the modes the design gives them (§3.1, §3.2):
// the socket trees keep the carlos-sockets group and setgid (issue #103),
// the log tree and data trees are the agent's alone, and the control
// directory is root's so no carlos process can create anything in it.
func TestTmpfilesCreatesTheInstanceTrees(t *testing.T) {
for _, want := range []string{
"d /data/.instances 0700 carlos carlos -",
"d /data/instances 0700 carlos carlos -",
"d /data/layout 0700 carlos carlos -",
"d /run/carlos/.instances 2750 carlos carlos-sockets -",
"d /run/carlos/instances 2750 carlos carlos-sockets -",
"d /run/carlos/.logs 0700 carlos carlos -",
"d /run/carlos/logs 0700 carlos carlos -",
"d /run/carlos-control 0750 root carlos -",
"d /run/carlos/hostguard 2770 carlos carlos -",
"d /data/hostguard 2770 carlos carlos -",
"f /etc/carlos/probe-secret 0640 root carlos - probe",
"f /dev/shm/carlos-probe 0644 root root - probe",
} {
if !strings.Contains(tmpfilesConf, want+"\n") {
t.Errorf("tmpfilesConf lacks %q", want)
}
}
// The existing lines stay: the flat socket dir and the apps subtree.
for _, keep := range []string{"d /run/carlos 2750 carlos carlos-sockets -", "d /run/carlos/apps 0750 carlos carlos -"} {
if !strings.Contains(tmpfilesConf, keep+"\n") {
t.Errorf("tmpfilesConf lost %q", keep)
}
}
}
// bootstrap -root
creates the same directories in the staging tree
// (a live run's chown -R covers them), so a fresh box has them before the
// mounts try to bind them.
func TestBootstrapCreatesTheInstanceDirectories(t *testing.T) {
root := t.TempDir()
if err := bootstrap(root, nil); err != nil {
t.Fatalf("bootstrap: %v", err)
}
for _, d := range []string{"data/.instances", "data/instances", "data/layout", "run/carlos/.instances", "run/carlos/instances", "run/carlos/.logs", "run/carlos/logs", "run/carlos-control"} {
if fi, err := os.Stat(filepath.Join(root, d)); err != nil || !fi.IsDir() {
t.Errorf("%s: %v", d, err)
}
}
}
// The mounts are enabled AND started by the plan: `systemctl enable` on a
// mount unit writes a symlink for the next boot and nothing else, the same
// trap timers and path units fell into (codex R1-14). A socket unit the
// same. The relay is still the only service the plan starts.
func TestSystemctlPlanStartsMountsAndSockets(t *testing.T) {
plan := systemctlPlan([]string{"carlos-edge.service", "data-instances.mount", "run-carlos-instances.mount", "run-carlos-logs.mount", "carlos-adopt.timer", "carlos-instance-control.socket"})
var started []string
for _, args := range plan {
if args[0] == "start" {
started = append(started, args[1:]...)
}
}
got := strings.Join(started, " ")
for _, want := range []string{"data-instances.mount", "run-carlos-instances.mount", "run-carlos-logs.mount", "carlos-adopt.timer", "carlos-instance-control.socket"} {
if !strings.Contains(got, want) {
t.Errorf("plan does not start %s: %v", want, plan)
}
}
if strings.Contains(got, "carlos-edge.service") {
t.Errorf("plan starts the edge: %v", plan)
}
}
func TestEnabledUnitsIncludeTheMounts(t *testing.T) {
have := map[string]bool{}
for _, u := range enabledUnits {
have[u] = true
}
for _, want := range []string{"data-instances.mount", "run-carlos-instances.mount", "run-carlos-logs.mount"} {
if !have[want] {
t.Errorf("enabledUnits lacks %s", want)
}
}
}
// The three tenant templates and the probe carry one namespace: byte-
// identical hardening blocks, so a property proven by the probe is the
// property a tenant gets (design §4.2, §8.5).
func TestInstanceTemplatesShareOneNamespace(t *testing.T) {
block := func(name string) string {
u := readUnit(t, name)
start := strings.Index(u, "# --- the namespace")
end := strings.Index(u, "# --- per-service resources")
if start < 0 || end < 0 || end < start {
t.Fatalf("%s: namespace block markers missing", name)
}
return u[start:end]
}
ref := block("carlos-instance@.service")
for _, name := range []string{"carlos-instance-gen@.service", "carlos-instance-seed@.service", "carlos-instance-sidecar@.service", "carlos-instance-cmd@.service", "carlos-instance-probe.service"} {
if block(name) != ref {
t.Errorf("%s namespace block differs from carlos-instance@.service", name)
}
}
for _, want := range []string{
"ProtectSystem=strict", "TemporaryFileSystem=/data:ro", "TemporaryFileSystem=/run/carlos:ro",
"TemporaryFileSystem=/var/cache:ro", "TemporaryFileSystem=/var/lib:ro", "InaccessiblePaths=/etc/carlos",
"TemporaryFileSystem=/dev/shm:mode=1777,nosuid,nodev",
"ProtectControlGroups=yes", "ProtectProc=invisible", "ProcSubset=pid", "PrivatePIDs=yes",
"RestrictNamespaces=yes", "NoNewPrivileges=yes", "CapabilityBoundingSet=",
"IPAddressDeny=localhost ", " 169.254.169.254/32 ", " fd00:ec2::254/128 ", "PrivateIPC=yes", "MemoryDenyWriteExecute=yes",
} {
if !strings.Contains(ref, want) {
t.Errorf("namespace block lacks %q", want)
}
}
// RestrictSUIDSGID= makes openat2 answer ENOSYS, and every instancefs
// open inside the namespace is an openat2 (#126, first roll).
for _, line := range strings.Split(ref, "\n") {
if strings.HasPrefix(line, "RestrictSUIDSGID=") {
t.Errorf("namespace block carries %q, which breaks openat2", line)
}
}
for name, wants := range map[string][]string{
"carlos-instance@.service": {"ExecStart=/opt/carlos/carlos instance-exec run %I", "ExecStopPost=/opt/carlos/carlos instance-exit run %I",
"BindPaths=/data/instances/%I /run/carlos/instances/%I", "StandardOutput=file:/run/carlos/logs/%I.run.sock",
// The instance's own data directory, where a flat child's
// relative paths now resolve (#256 D); the IE box's emergency
// drop-in 50-workdir-incident-256.conf carries this exact line,
// so removing it changes nothing.
"WorkingDirectory=/data/instances/%I",
"Slice=carlos-instances-%i.slice", "Restart=no", "KillMode=mixed", "TimeoutStopSec=20", "BindsTo=carlos-edge.service",
"Requires=data-instances.mount run-carlos-instances.mount run-carlos-logs.mount", "OOMScoreAdjust=500", "OOMPolicy=kill", "TasksMax=256", "Type=simple",
// The trampoline's init exits 128+n for a death by signal n (#126).
"SuccessExitStatus=129 130 141 143"},
"carlos-instance-gen@.service": {"ExecStart=/opt/carlos/carlos instance-exec run %I", "ExecStopPost=/opt/carlos/carlos instance-exit run %I",
"Restart=no", "KillMode=mixed", "TimeoutStopSec=20", "BindsTo=carlos-edge.service",
"Requires=data-instances.mount run-carlos-instances.mount run-carlos-logs.mount", "OOMScoreAdjust=500", "OOMPolicy=kill", "TasksMax=256", "Type=simple",
"SuccessExitStatus=129 130 141 143"},
"carlos-instance-seed@.service": {"Type=oneshot", "RemainAfterExit=yes", "TimeoutStartSec=120", "ExecStart=/opt/carlos/carlos instance-exec seed %I",
"ExecStopPost=/opt/carlos/carlos instance-exit seed %I", "StandardOutput=file:/run/carlos/logs/%I.seed.sock",
"KillMode=mixed", "TimeoutStopSec=20", "WorkingDirectory=/data/instances/%I"},
"carlos-instance-cmd@.service": {"Type=oneshot", "RemainAfterExit=yes", "TimeoutStartSec=6h5m", "ExecStart=/opt/carlos/carlos instance-exec cmd %I",
"ExecStopPost=/opt/carlos/carlos instance-exit cmd %I", "StandardOutput=file:/run/carlos/logs/%I.cmd.sock",
"StandardError=file:/run/carlos/logs/%I.cmd-err.sock", "KillMode=mixed", "TimeoutStopSec=20", "WorkingDirectory=/data/instances/%I"},
"carlos-instance-sidecar@.service": {"Type=simple", "ExecStart=/opt/carlos/carlos instance-exec sidecar %I",
"ExecStopPost=/opt/carlos/carlos instance-exit sidecar %I", "StandardOutput=file:/run/carlos/logs/%I.sidecar.sock",
"SuccessExitStatus=129 130 141 143", "WorkingDirectory=/data/instances/%I"},
"carlos-instance-probe.service": {"Type=oneshot", "ExecStart=/opt/carlos/carlos instance-probe",
"BindPaths=/data/instances/_probe /run/carlos/instances/_probe", "StandardOutput=truncate:/run/carlos-control/probe.out"},
} {
u := readUnit(t, name)
for _, w := range wants {
if !strings.Contains(u, w) {
t.Errorf("%s lacks %q", name, w)
}
}
// Exactly one WorkingDirectory= per tenant template, and none in
// the probe, whose directory is _probe and whose cwd does not
// matter: a second line would silently win (#256 D).
if n := strings.Count(u, "\nWorkingDirectory="); (name != "carlos-instance-probe.service" && n != 1) || (name == "carlos-instance-probe.service" && n != 0) {
t.Errorf("%s has %d WorkingDirectory= lines", name, n)
}
if strings.Contains(u, "ExecStopPost=+") {
t.Errorf("%s uses ExecStopPost=+: the plus prefix keeps the unit's tmpfs (codex R3-2)", name)
}
// A seed a signal ended has not seeded (#126).
if (name == "carlos-instance-seed@.service" || name == "carlos-instance-cmd@.service") && strings.Contains(u, "\nSuccessExitStatus=") {
t.Errorf("%s counts a signal's exit status as success", name)
}
if name != "carlos-instance-probe.service" && !strings.Contains(u, "User=carlos") {
t.Errorf("%s does not run as carlos", name)
}
}
}
func TestControlUnitsShape(t *testing.T) {
s := readUnit(t, "carlos-instance-control.socket")
for _, want := range []string{"ListenStream=/run/carlos-control/control.sock", "SocketUser=carlos", "SocketGroup=carlos", "SocketMode=0600", "Accept=no", "WantedBy=sockets.target"} {
if !strings.Contains(s, want) {
t.Errorf("socket unit lacks %q", want)
}
}
u := readUnit(t, "carlos-instance-control.service")
for _, want := range []string{"ExecStart=/opt/carlos/carlos instance-control", "Requires=carlos-instance-control.socket",
"data-instances.mount", "ProtectSystem=strict", "ReadWritePaths=/run/carlos-control /run/systemd/system", "RestrictAddressFamilies=AF_UNIX", "MemoryMax=64M"} {
if !strings.Contains(u, want) {
t.Errorf("control service lacks %q", want)
}
}
for _, never := range []string{"User=", "NoNewPrivileges"} {
if strings.Contains(u, never) {
t.Errorf("control service must run as root for systemctl; found %q", never)
}
}
have := map[string]bool{}
for _, e := range enabledUnits {
have[e] = true
}
if !have["carlos-instance-control.socket"] {
t.Error("enabledUnits lacks the control socket")
}
if have["carlos-instance-control.service"] || have["carlos-instance-probe.service"] || have["carlos-instance@.service"] || have["carlos-instance-gen@.service"] {
t.Error("a template, the probe or the socket-activated service must not be enabled")
}
for _, want := range []string{"d /data/.instances/_probe 0700 carlos carlos -", "d /run/carlos/.instances/_probe 2750 carlos carlos-sockets -"} {
if !strings.Contains(tmpfilesConf, want+"\n") {
t.Errorf("tmpfilesConf lacks %q", want)
}
}
}
// The same-uid audit (design §9, codex R9-1, R9-2, R10-1): every tree a
// carlos-uid unit writes, every carlos-owned cgroup, and every credential a
// carlos-group process reads must be hidden from the tenant templates.
func TestSameUIDAuditIsCoveredByTheTemplates(t *testing.T) {
ns := readUnit(t, "carlos-instance@.service")
hidden := func(path string) bool {
for _, line := range strings.Split(ns, "\n") {
for _, key := range []string{"TemporaryFileSystem=", "InaccessiblePaths="} {
if strings.HasPrefix(line, key) {
for _, p := range strings.Fields(strings.TrimPrefix(line, key)) {
p = strings.TrimSuffix(p, ":ro")
if path == p || strings.HasPrefix(path, p+"/") {
return true
}
}
}
}
}
return false
}
names, err := unitNames()
if err != nil {
t.Fatal(err)
}
for _, name := range names {
u := readUnit(t, name)
if !strings.Contains(u, "User=carlos") {
continue
}
for _, line := range strings.Split(u, "\n") {
for _, key := range []string{"ReadWritePaths=", "CacheDirectory=", "StateDirectory=", "RuntimeDirectory="} {
if !strings.HasPrefix(line, key) {
continue
}
for _, p := range strings.Fields(strings.TrimPrefix(line, key)) {
switch key {
case "CacheDirectory=":
p = "/var/cache/" + p
case "StateDirectory=":
p = "/var/lib/" + p
case "RuntimeDirectory=":
p = "/run/" + p
}
if !hidden(p) && !strings.HasPrefix(p, "/run/carlos/instances/") && !strings.HasPrefix(p, "/data/instances/") {
t.Errorf("%s writes %s, which the tenant template does not hide", name, p)
}
}
}
}
if strings.Contains(u, "Delegate=") && !strings.Contains(ns, "ProtectControlGroups=yes") {
t.Errorf("%s delegates a cgroup subtree and the template leaves cgroupfs writable", name)
}
}
// Credential inputs a carlos-group process reads (the second table).
for _, cred := range []string{"/etc/carlos/relay-masters"} {
if !hidden(cred) {
t.Errorf("credential input %s is not hidden from tenants", cred)
}
}
}
// The cmd template is the seed template outside seven named lines (run
// spec §5.4, codex 4; the executable mount's own sources, merge ruling
// M4): every sandbox line is the seed's, so a command has exactly the
// reach a seed has. Comments and blank lines are free; every
// directive is compared, in order.
func TestCmdTemplateEqualsSeedOutsideTheNamedLines(t *testing.T) {
directives := func(name string) []string {
var out []string
for _, line := range strings.Split(readUnit(t, name), "\n") {
line = strings.TrimSpace(line)
if line == "" || strings.HasPrefix(line, "#") {
continue
}
out = append(out, line)
}
return out
}
swap := map[string]string{
"Description=carlos exec-backed instance %I (seed)": "Description=carlos exec-backed instance %I (cmd)",
"TimeoutStartSec=120": "TimeoutStartSec=6h5m",
"ExecStart=/opt/carlos/carlos instance-exec seed %I": "ExecStart=/opt/carlos/carlos instance-exec cmd %I",
"ExecStopPost=/opt/carlos/carlos instance-exit seed %I": "ExecStopPost=/opt/carlos/carlos instance-exit cmd %I",
"StandardOutput=file:/run/carlos/logs/%I.seed.sock": "StandardOutput=file:/run/carlos/logs/%I.cmd.sock",
"StandardError=inherit": "StandardError=file:/run/carlos/logs/%I.cmd-err.sock",
"BindReadOnlyPaths=/run/carlos-control/exe/%I/app:/opt/carlos/app /run/carlos-control/exe/%I/live:/opt/carlos/live": "BindReadOnlyPaths=/run/carlos-control/exe/%I/cmd-app:/opt/carlos/app /run/carlos-control/exe/%I/cmd-live:/opt/carlos/live",
}
seed, cmd := directives("carlos-instance-seed@.service"), directives("carlos-instance-cmd@.service")
want := make([]string, len(seed))
used := map[string]bool{}
for i, l := range seed {
if c, ok := swap[l]; ok {
want[i], used[l] = c, true
continue
}
want[i] = l
}
for l := range swap {
if !used[l] {
t.Errorf("the seed template no longer has %q; the run spec names it as a cmd difference", l)
}
}
for i := 0; i < len(want) || i < len(cmd); i++ {
var w, g string
if i < len(want) {
w = want[i]
}
if i < len(cmd) {
g = cmd[i]
}
if w != g {
t.Fatalf("directive %d: cmd template has %q, want %q (the seed's line with the named swaps)", i, g, w)
}
}
for _, keep := range []string{"Type=oneshot", "RemainAfterExit=yes", "KillMode=mixed", "TimeoutStopSec=20", "WorkingDirectory=/data/instances/%I"} {
if !hasDirective(readUnit(t, "carlos-instance-cmd@.service"), keep) {
t.Errorf("cmd template lacks %q", keep)
}
}
}
// The generation template's instance string is ":" (zero-downtime
// spec §7.6, plan 2 D1). No specifier can take the instance ID out of it,
// so the template keys NOTHING by %i or %I but the trampoline's argument.
// The slice, the data and executable mounts and the log socket come from
// the control service's per-generation drop-in (instancectl.GenDropIn).
func TestGenerationTemplateKeysNothingByItsInstanceString(t *testing.T) {
u := readUnit(t, "carlos-instance-gen@.service")
for _, d := range directives(u) {
for _, key := range []string{"Slice=", "BindPaths=", "ReadWritePaths=", "StandardOutput="} {
if strings.HasPrefix(d, key) {
t.Errorf("generation template carries %q; it belongs in the drop-in", d)
}
}
if strings.HasPrefix(d, "BindReadOnlyPaths=") && strings.Contains(d, "/run/carlos-control/exe") {
t.Errorf("generation template binds an executable itself: %q", d)
}
if strings.Contains(d, "/opt/carlos/versions") {
t.Errorf("generation template names the version store: %q", d)
}
if strings.Contains(d, "%i") || (strings.Contains(d, "%I") && !strings.HasPrefix(d, "ExecStart=") && !strings.HasPrefix(d, "ExecStopPost=") && !strings.HasPrefix(d, "Description=")) {
t.Errorf("generation template keys %q by its instance string", d)
}
}
}