//go:build linux && rootrunner package host import ( "context" "encoding/json" "fmt" "io" "net" "net/http" "os" "os/exec" "os/user" "path/filepath" "strconv" "strings" "testing" "time" "github.com/carlosframework/platform/internal/activator" "github.com/carlosframework/platform/internal/hostbox" "github.com/carlosframework/platform/internal/hostclaim" execlease "github.com/carlosframework/platform/internal/hostexec" "github.com/carlosframework/platform/internal/hostwrite" "github.com/carlosframework/platform/internal/instancectl" "github.com/carlosframework/platform/internal/instanceexec" "github.com/carlosframework/platform/internal/provision" "github.com/carlosframework/platform/internal/registry" "github.com/carlosframework/platform/internal/rootparticipation" "github.com/carlosframework/platform/internal/rootstate" "golang.org/x/sys/unix" ) func init() { if len(os.Args) > 1 { switch os.Args[1] { case "instance-exec": instanceexec.RunExec(os.Args[2:]) os.Exit(0) case "instance-exit": instanceexec.RunExit(os.Args[2:]) os.Exit(0) } } } func TestRootFilesSeedIsolatedTenant(t *testing.T) { runServingParkPeerNamespace(t, false, false, "seed-isolated") } type seedIsolatedClient struct { root *ClaimLaunchRoot peer StopPeer } func (c seedIsolatedClient) Capability(context.Context) (string, error) { return hostbox.RoleLaunchProtocol, nil } func (c seedIsolatedClient) Launch(ctx context.Context, q hostbox.RoleLaunchRequest) (hostbox.RoleLaunchGrant, error) { out, err := c.root.Serve(ctx, c.peer, q) if err != nil { return hostbox.RoleLaunchGrant{}, err } return *out.Grant, nil } func (c seedIsolatedClient) Stop(ctx context.Context, q hostbox.RoleLaunchRequest) error { _, err := c.root.Serve(ctx, c.peer, q) return err } func rootFilesSeedIsolated(t *testing.T, binary string) { witness := rootFilesSeedIsolatedWitness(t, binary) runFilesSeedPreparationWithLive(t, "real-seed", "same-controller", []bool{true}, []filesSeedLiveWitness{witness}) } func rootFilesSeedIsolatedWitness(t *testing.T, binary string, twoHosts ...bool) filesSeedLiveWitness { hosts := []string{"seed-isolated.bab.oncarlos.com"} if len(twoHosts) > 0 { hosts = seedTwoHostNames() } for _, host := range hosts { path := "/run/systemd/system.control/" + instancectl.SliceName(host) + ".d" if _, err := os.Lstat(path); !os.IsNotExist(err) { t.Fatal("refuse existing slice properties", path, err) } } for _, dir := range []string{"/data", "/var/lib/carlos-claims", "/opt/carlos", "/run/carlos", "/run/carlos-control", "/etc/carlos"} { if len(twoHosts) > 1 && (dir == "/data" || dir == "/var/lib/carlos-claims" || dir == "/opt/carlos" || dir == "/etc/carlos") { if _, err := os.Stat(dir); err != nil { t.Fatal("missing reboot fixture", dir, err) } } else if err := os.Mkdir(dir, 0755); err != nil { t.Fatal("refuse existing fixture path", dir, err) } t.Cleanup(func() { os.RemoveAll(dir) }) if dir == "/data" && strings.HasPrefix(t.Name(), "TestRootRebootFilesSeedTwoHost") { seedRunnerRebootDisk(t) } } account, err := user.Lookup("carlos") if err != nil { t.Fatal(err) } uid, _ := strconv.Atoi(account.Uid) gid, _ := strconv.Atoi(account.Gid) for _, dir := range []string{"/opt/carlos/versions/fixture", "/run/carlos/instances", "/run/carlos/logs"} { if err := os.MkdirAll(dir, 0755); err != nil { t.Fatal(err) } } image, err := os.ReadFile(binary) if err != nil { t.Fatal(err) } app := "/opt/carlos/versions/fixture/app" for _, path := range []string{"/opt/carlos/carlos", app} { if err := os.WriteFile(path, image, 0755); err != nil { t.Fatal(err) } } raw, err := unitsFS.ReadFile("units/carlos-instance@.service") if err != nil { t.Fatal(err) } lines := []string{} for _, line := range strings.Split(string(raw), "\n") { if strings.HasPrefix(line, "BindsTo=") || strings.HasPrefix(line, "Requires=") || strings.HasPrefix(line, "After=") { continue } lines = append(lines, line) } if os.Getenv("FILES_SEED_ISOLATION_MUTATION") == "1" { lines = append(lines, "[Service]", "BindReadOnlyPaths=/etc/carlos/fixture-secret:/tmp/exposed-root") } unit := "/run/systemd/system/carlos-instance@.service" file, err := os.OpenFile(unit, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0644) if err != nil { t.Fatal(err) } if _, err := file.WriteString(strings.Join(lines, "\n") + "\nIPAddressDeny=any\n"); err != nil { t.Fatal(err) } file.Close() t.Cleanup(func() { ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() for _, host := range hosts { if out, err := exec.CommandContext(ctx, "systemctl", "stop", instancectl.UnitName(instancectl.RoleRun, host)).CombinedOutput(); err != nil { t.Errorf("stop isolated tenant: %v %s", err, out) } os.RemoveAll("/run/systemd/system/" + instancectl.UnitName(instancectl.RoleRun, host) + ".d") slice := instancectl.SliceName(host) if out, err := exec.CommandContext(ctx, "systemctl", "stop", slice).CombinedOutput(); err != nil { t.Errorf("stop isolated slice: %v %s", err, out) } state, err := exec.CommandContext(ctx, "systemctl", "show", slice, "--property=ActiveState", "--value").Output() if err != nil || strings.TrimSpace(string(state)) != "inactive" { t.Errorf("isolated slice survived cleanup: %s %v", state, err) } exec.CommandContext(ctx, "systemctl", "reset-failed", instancectl.UnitName(instancectl.RoleRun, host)).Run() path := "/run/systemd/system.control/" + slice + ".d" if err := os.RemoveAll(path); err != nil { t.Error("remove fixture slice properties", err) } if _, err := os.Lstat(path); !os.IsNotExist(err) { t.Error("fixture slice properties survived", err) } } for _, slice := range []string{"carlos-instances.slice", "carlos.slice"} { if out, err := exec.CommandContext(ctx, "systemctl", "stop", slice).CombinedOutput(); err != nil { t.Errorf("stop isolated parent slice: %v %s", err, out) } state, err := exec.CommandContext(ctx, "systemctl", "show", slice, "--property=ActiveState", "--value").Output() if err != nil || strings.TrimSpace(string(state)) != "inactive" { t.Errorf("isolated parent slice survived cleanup: %s %v", state, err) } } os.Remove(unit) if out, err := exec.CommandContext(ctx, "systemctl", "daemon-reload").CombinedOutput(); err != nil { t.Errorf("reload isolated unit: %v %s", err, out) } }) servingRunnerSystemctl(t, "daemon-reload") denied := []string{"/etc/carlos/fixture-secret", "/var/lib/carlos-claims/fixture-secret", "/data/instances/foreign.example/fixture-secret"} for _, path := range denied { if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil { t.Fatal(err) } if err := os.WriteFile(path, []byte("foreign sentinel"), 0644); err != nil { t.Fatal(err) } } agent := "/sys/fs/cgroup/files-seed-controller" if err := os.Mkdir(agent, 0755); err != nil { t.Fatal("refuse existing controller cgroup", err) } t.Cleanup(func() { if err := os.Remove(agent); err != nil { t.Error(err) } }) witness := rootFilesSeedLiveWitness(t, app) witness.host = "seed-isolated.bab.oncarlos.com" configure := witness.configure witness.configure = func(w *ClaimWakeAssembly) { configure(w) owners := w.Inputs.(*claimServingOwners) selected := owners.Sources.Launch owners.Sources.Launch = func(ctx context.Context, rt registry.Route) (ClaimServingLaunch, error) { in, err := selected(ctx, rt) paths := append([]string(nil), denied...) if os.Getenv("FILES_SEED_ISOLATION_MUTATION") == "1" { paths = append(paths, "/tmp/exposed-root") } if len(twoHosts) > 0 { pair := seedTwoHostNames() other := pair[1] if rt.Host == other { other = pair[0] } paths = append(paths, provision.NestedDBPath("/data", other), filepath.Join("/data/instances", other, other, "payload")) in.Env = append(in.Env, "FILES_SEED_EXPECTED=real seeded bytes:"+rt.Host) } in.Env = append(in.Env, "FILES_SEED_DENIED="+strings.Join(paths, ":")) return in, err } } witness.healthy = true witness.prepare = func(p *servingPreparation) { boot, err := execlease.BootID() if err != nil { t.Fatal(err) } start, err := claimLaunchProcessStart("/proc", os.Getpid()) if err != nil { t.Fatal(err) } p.local.BootID, p.local.ControllerID, p.local.PID, p.local.ProcessStart = boot, "isolated-controller", os.Getpid(), start p.f.f.peer = StopPeer{UID: uint32(uid), PID: os.Getpid(), ProcessStart: start, BootID: boot, ControllerID: p.local.ControllerID} p.wake.Retries.BootID, p.wake.Retries.ControllerID = boot, p.local.ControllerID p.f.p.root.Stop.Coordinator.CarlosUID = uint32(uid) prior := claimContainmentAgentRoot claimContainmentAgentRoot = func(context.Context) (string, error) { return agent, nil } t.Cleanup(func() { claimContainmentAgentRoot = prior }) p.wake.Prelaunch = func(ctx context.Context, o hostclaim.Observed, work hostwrite.Work, in ClaimLaunchInputs) (*activator.RoleLaunch, error) { target := recoveryTuple(o, p.local) group, err := claimContainmentParent(agent, target) if err != nil { return nil, err } path := filepath.Join("/sys/fs/cgroup", group) if err := os.Mkdir(path, 0755); err != nil { return nil, err } t.Cleanup(func() { os.Remove(path) }) var st unix.Stat_t if err := unix.Stat(path, &st); err != nil { return nil, err } err = work.With(ctx, in.Identity, func() error { return p.f.f.journal.Prelaunch(ctx, hostbox.LaunchInventory{Target: target, ControllerPID: p.local.PID, ProcessStart: start, NoState: true, Groups: []hostbox.LaunchGroup{{RelativePath: group, Device: uint64(st.Dev), Inode: st.Ino}}}) }) return &activator.RoleLaunch{Target: target, Journal: p.f.f.journal}, err } state, err := rootstate.Open(filepath.Join(t.TempDir(), "execution"), true, false) if err != nil { t.Fatal(err) } t.Cleanup(func() { state.Close() }) census, _ := json.Marshal(rootparticipation.Census{Schema: 1, Box: p.f.actor.Box, Enrollment: p.f.actor.Enrollment, Migration: 1, Revision: 1, LaunchRevision: 1, Complete: true}) if err := state.Write("census.json", census); err != nil { t.Fatal(err) } units := instancectl.ClaimUnits{Systemctl: func(ctx context.Context, args ...string) (string, error) { out, err := exec.CommandContext(ctx, "systemctl", args...).CombinedOutput() t.Logf("isolated systemctl %q: %s (%v)", args, out, err) return string(out), err }, CgroupRoot: "/sys/fs/cgroup", ExeRoot: "/run/carlos-control/exe", UnitRoot: "/run/systemd/system", DataRoot: "/data"} root := &ClaimLaunchRoot{Guard: p.f.f.guard, Journal: p.f.f.journal, DataRoot: "/data", ExecLockDir: t.TempDir(), Local: &rev2Local{}, Clock: execlease.SystemClock{}, Actor: func(context.Context) (hostbox.Actor, error) { return p.f.actor, nil }, Units: units, Kernel: linuxClaimLaunchKernel{OptRoot: "/opt/carlos", CgroupRoot: "/sys/fs/cgroup", ProcRoot: "/proc"}, ValidateExecutable: func(string, string, string) error { return nil }, executionState: state} client := seedIsolatedClient{root: root, peer: p.f.f.peer} p.wake.Launch = func(ctx context.Context, o hostclaim.Observed, work hostwrite.Work, in ClaimLaunchInputs, pre *activator.RoleLaunch) (*activator.RoleLaunch, error) { paths, err := claimIsolatedPaths(map[string]string{"data": "/data"}, o.Host, in) if err != nil { return nil, err } l := *pre l.Work, l.Identity, l.Root, l.CgroupRoot = work, in.Identity, client, "/sys/fs/cgroup" l.Spec = &hostbox.LaunchSpec{Identity: in.Identity, Executable: in.Executable, Digest: in.ExecutableSHA256, Args: in.Args, Env: in.Env} l.LaunchDir, l.RunDir, l.LogSocket, l.MemoryMax = paths.LaunchDir, paths.RunDir, paths.LogSocket, paths.MemoryMax l.Executable = func(context.Context, *exec.Cmd) (*os.File, error) { return os.Open(app) } l.Initialize = func(context.Context) error { if err := filepath.Walk(l.LaunchDir, func(path string, info os.FileInfo, err error) error { if err != nil { return err } return os.Chown(path, uid, gid) }); err != nil { return err } if err := os.MkdirAll(l.RunDir, 0750); err != nil { return err } return os.Chown(l.RunDir, uid, gid) } return &l, nil } if len(p.wake.pending) != 0 { t.Fatal("fixture rebuilt active wake assembly") } p.wake.pending = nil p.runtime, err = p.wake.Build(t.Context(), p.source, p.writer) if err != nil { t.Fatal(err) } } witness.start = func(t *testing.T, p *servingPreparation, o hostclaim.Observed, start activator.ClaimStart) func() { launch, err := start.PrepareLaunch(t.Context()) if err != nil { t.Fatal("prepare isolated launch", err) } cmd, _, settle, _, err := launch.Start(t.Context(), "main", start.CurrentAttempt(), start.Build, func() error { return nil }) if err != nil { if body, readErr := os.ReadFile(filepath.Join(filepath.Dir(p.f.db), o.Host, "payload.isolation-violation")); readErr == nil { t.Fatalf("isolation mutation detected: %s", body) } out, _ := exec.Command("journalctl", "-u", instancectl.UnitName(instancectl.RoleRun, o.Host), "--no-pager", "-n", "30").CombinedOutput() t.Fatalf("isolated launch: %v\n%s", err, out) } stopped := false stop := func() { if !stopped { stopped = true if err := settle(context.Background()); err != nil { t.Error(err) } } } t.Cleanup(stop) ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) defer cancel() if err := start.Ready(ctx, cmd); err != nil { if body, readErr := os.ReadFile(filepath.Join(filepath.Dir(p.f.db), o.Host, "payload.isolation-violation")); readErr == nil { t.Fatalf("isolation mutation detected: %s", body) } t.Fatal("isolated readiness", err) } wrong, _ := os.FindProcess(os.Getpid()) if err := start.Healthy(ctx, 0, &exec.Cmd{Process: wrong}); err == nil { t.Fatal("controller accepted as healthy tenant") } if err := start.Healthy(ctx, 0, cmd); err != nil { t.Fatal("isolated exact health", err) } transport := &http.Transport{DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) { return (&net.Dialer{}).DialContext(ctx, "unix", provision.NestedSocket(o.Host)) }} defer transport.CloseIdleConnections() response, err := (&http.Client{Transport: transport, Timeout: time.Second}).Get("http://tenant/") if err != nil { t.Fatal(err) } body, err := io.ReadAll(response.Body) response.Body.Close() expected := "real seeded bytes" if len(twoHosts) > 0 { expected += ":" + o.Host } if err != nil || response.StatusCode != 200 || string(body) != expected { t.Fatal("isolated payload", string(body), err) } for _, path := range denied { if body, err := os.ReadFile(path); err != nil || string(body) != "foreign sentinel" { t.Fatal("foreign sentinel changed", path, err) } } t.Log(fmt.Sprintf("healthy isolated wake: host=%s session=%d pid=%d", o.Host, o.Claim.Holder.Session, cmd.Process.Pid)) return stop } witness.serve = func(t *testing.T, p *servingPreparation, o hostclaim.Observed, start activator.ClaimStart) { witness.start(t, p, o, start)() } return witness }