//go:build linux && rootrunner package host import ( "context" "crypto/sha256" "encoding/hex" "fmt" "os" "os/exec" "path/filepath" "runtime" "strconv" "syscall" "testing" "time" "github.com/carlosframework/platform/internal/instancectl" "github.com/carlosframework/platform/internal/instanceexec" "github.com/carlosframework/platform/internal/registry" "github.com/carlosframework/platform/internal/release" "github.com/carlosframework/platform/internal/rootstate" "golang.org/x/sys/unix" ) func TestPlan14GenerationContainedRunningRootRunner(t *testing.T) { const key = "PLAN14_E45_CONTAINED" dir, mode := os.Getenv("PLAN14_E45_DIR"), os.Getenv(key) if mode == "mount" { runtime.LockOSThread() if err := unix.Mount("", "/", "", unix.MS_REC|unix.MS_PRIVATE, ""); err != nil { t.Fatal(err) } if err := unix.Mount("proc", "/proc", "proc", 0, ""); err != nil { t.Fatal(err) } executable := release.VersionPath(filepath.Join(dir, "opt"), "bab", "v1", "shop") if err := unix.Mount(executable, filepath.Join(dir, "bound-app"), "", unix.MS_BIND, ""); err != nil { t.Fatal(err) } if err := unix.Mount("tmpfs", filepath.Join(dir, "opt"), "tmpfs", unix.MS_RDONLY, "mode=0755"); err != nil { t.Fatal(err) } group := instancectl.ClaimUnitControlGroup(filepath.Base(dir) + ".example") if err := os.WriteFile(filepath.Join("/sys/fs/cgroup", group, "cgroup.procs"), []byte("0"), 0600); err != nil { t.Fatal(err) } if err := os.Setenv(key, "init"); err != nil { t.Fatal(err) } env := os.Environ() if err := syscall.Exec("/usr/bin/setpriv", []string{"setpriv", "--reuid=65534", "--regid=65534", "--clear-groups", "--bounding-set=-all", "--no-new-privs", filepath.Join(dir, "helper.test"), "-test.run=^TestPlan14GenerationContainedRunningRootRunner$", "-test.timeout=90s"}, env); err != nil { t.Fatal(err) } return } if mode == "init" { runtime.LockOSThread() if err := instanceexec.ScopeAbstractUnix(); err != nil { t.Fatal(err) } if err := instanceexec.ScopeClaimGeneration(); err != nil { t.Fatal(err) } for _, call := range []uintptr{unix.SYS_RECVMSG, unix.SYS_RECVMMSG, unix.SYS_PIDFD_GETFD, unix.SYS_IO_URING_SETUP} { _, _, e := unix.RawSyscall6(call, ^uintptr(0), 0, 0, 0, 0, 0) if e != unix.EPERM { t.Fatal("descriptor boundary missing", call, e) } } tenant := exec.Command(filepath.Join(dir, "bound-app"), "90") if err := tenant.Start(); err != nil { t.Fatal(err) } if err := tenant.Wait(); err != nil { t.Fatal(err) } return } if mode == "" { dir = generationRootRunnerDir(t) binary, err := os.ReadFile(os.Args[0]) if err != nil { t.Fatal(err) } if err := os.WriteFile(filepath.Join(dir, "helper.test"), binary, 0755); err != nil { t.Fatal(err) } cmd := exec.CommandContext(t.Context(), "unshare", "--mount", "--pid", "--fork", "--mount-proc", filepath.Join(dir, "helper.test"), "-test.run=^TestPlan14GenerationContainedRunningRootRunner$", "-test.v", "-test.timeout=2m") cmd.Dir = dir cmd.Env = append(os.Environ(), key+"=census", "PLAN14_E45_DIR="+dir) if out, err := cmd.CombinedOutput(); err != nil { t.Fatalf("running containment: %v\n%s", err, out) } else { t.Logf("%s", out) } return } if mode != "census" || os.Geteuid() != 0 { t.Fatal("invalid runner entry") } t.Setenv("CARLOS_OPT_DIR", filepath.Join(dir, "opt")) body, err := os.ReadFile("/bin/sleep") if err != nil { t.Fatal(err) } for _, version := range []string{"v1", "v2", "v3", "v4"} { if err := release.StageArtifact(filepath.Join(dir, "opt"), "bab", version, "shop", body); err != nil { t.Fatal(err) } old := time.Now().Add(-48 * time.Hour) if err := os.Chtimes(release.VersionPath(filepath.Join(dir, "opt"), "bab", version, "shop"), old, old); err != nil { t.Fatal(err) } } if err := os.WriteFile(filepath.Join(dir, "bound-app"), nil, 0755); err != nil { t.Fatal(err) } host := filepath.Base(dir) + ".example" group := instancectl.ClaimUnitControlGroup(host) var made []string for p := filepath.Join("/sys/fs/cgroup", group); p != "/sys/fs/cgroup"; p = filepath.Dir(p) { made = append(made, p) } var owned []string for i := len(made) - 1; i >= 0; i-- { if err := os.Mkdir(made[i], 0755); err == nil { owned = append(owned, made[i]) } else if !os.IsExist(err) { t.Fatal(err) } } defer func() { for i := len(owned) - 1; i >= 0; i-- { if err := os.Remove(owned[i]); err != nil { t.Error(err) } } }() cmd := exec.Command(filepath.Join(dir, "helper.test"), "-test.run=^TestPlan14GenerationContainedRunningRootRunner$", "-test.timeout=90s") cmd.Dir = dir cmd.Env = append(os.Environ(), key+"=mount") cmd.SysProcAttr = &syscall.SysProcAttr{Cloneflags: unix.CLONE_NEWNS | unix.CLONE_NEWPID} cmd.Stdout, cmd.Stderr = os.Stdout, os.Stderr if err := cmd.Start(); err != nil { t.Fatal(err) } defer func() { cmd.Process.Kill(); cmd.Wait() }() pid := cmd.Process.Pid k := linuxClaimLaunchKernel{ProcRoot: "/proc", CgroupRoot: "/sys/fs/cgroup", BoundApp: filepath.Join(dir, "bound-app")} var observed ClaimLaunchObserved deadline := time.Now().Add(10 * time.Second) for { observed, err = k.Observe(instancectl.ClaimUnitStarted{MainPID: pid, ControlGroup: group}) if err == nil { break } if time.Now().After(deadline) { t.Fatalf("tenant observation: %+v %v", observed, err) } time.Sleep(10 * time.Millisecond) } boot, err := (generationKernel{proc: "/proc"}).boot() if err != nil { t.Fatal(err) } controller, err := claimLaunchProcessStart("/proc", os.Getpid()) if err != nil { t.Fatal(err) } state, err := rootstate.Open(filepath.Join(dir, "generations"), true, false) if err != nil { t.Fatal(err) } defer state.Close() ledger := &release.RootGenerations{Store: state} x := &release.ReleaseExclusion{Open: func() (rootstate.Store, error) { return rootstate.Open(filepath.Join(dir, "exclusion"), true, false) }} identity := func(path string) rootstate.FileIdentity { id, err := generationIdentity(path) if err != nil { t.Fatal(err) } return id } digest := sha256.Sum256(body) reserve := func(version string) release.GenerationReservation { path := release.VersionPath(filepath.Join(dir, "opt"), "bab", version, "shop") b := release.GenerationBinding{Binding: release.HistoryBinding{Host: host, Account: "bab", App: "shop"}, Target: release.GenerationOwner{Host: host, Incarnation: "fixture", Stamp: 1, Session: 1, Box: "runner", Enrollment: "fixture", BootID: boot, ControllerID: "fixture"}, Installation: "fixture", Attempt: version, Inputs: "fixture", Invocation: version, Record: "fixture", RecordETag: "fixture", ControllerPID: os.Getpid(), ControllerStart: controller, Version: version, Digest: hex.EncodeToString(digest[:]), Path: path, Tree: filepath.Dir(path), Executable: identity(path), TreeIdentity: identity(filepath.Dir(path))} r, err := ledger.Reserve(b) if err != nil { t.Fatal(err) } return r } var v1, v3 release.GenerationReservation if err := x.With(t.Context(), func(context.Context) error { v1, v3 = reserve("v1"), reserve("v3") return ledger.Started(v1, release.GenerationProcess{PID: pid, Start: observed.ProcessStart, TenantPID: observed.TenantPID, TenantStart: observed.TenantStart, Group: group, Invocation: "fixture", GroupIdentity: identity(filepath.Join("/sys/fs/cgroup", group)), LauncherIdentity: identity(filepath.Join("/proc", strconv.Itoa(pid), "exe")), Containment: 1}) }); err != nil { t.Fatal(err) } v1, err = ledger.Read(v1.Binding) if err != nil { t.Fatal(err) } source := generationPinSource{open: func() (rootstate.Store, error) { return rootstate.Open(filepath.Join(dir, "generations"), false, false) }, verify: func(ctx context.Context, rs []release.GenerationReservation) error { // Enrollment/launch-journal agreement is fixture authority in this witness. kernel := generationKernel{proc: "/proc", cgroups: "/sys/fs/cgroup", boundApp: filepath.Join(dir, "bound-app"), contained: rs} _, _, err := kernel.census(ctx, filepath.Join(dir, "opt", "versions")) if err != nil { tasks, _ := filepath.Glob("/proc/[0-9]*/task/[0-9]*") for _, task := range tasks { status, _ := generationRead(task + "/status") ok, e := kernel.containedTask(ctx, task, status) if e != nil { t.Logf("containment %s ok=%v err=%v status=%s", task, ok, e, status) } } } return err }} for _, attack := range []string{"none", "erased legacy", "renamed legacy"} { legacy := filepath.Join(dir, "zdd") if attack == "none" { if err := os.Mkdir(legacy, 0755); err != nil { t.Fatal(err) } if err := release.WriteGenerations(filepath.Join(legacy, "generations.json"), release.Generations{}); err != nil { t.Fatal(err) } } for _, name := range []string{"generations.json", "generations.seq"} { p := filepath.Join(legacy, name) if attack == "erased legacy" { if err := os.Remove(p); err != nil { t.Fatal(err) } } } if attack == "renamed legacy" { if err := os.Rename(legacy, legacy+".old"); err != nil { t.Fatal(err) } } if err := x.With(t.Context(), func(ctx context.Context) error { pins, err := source.StagedPins(ctx, filepath.Join(dir, "opt")) if err != nil { return fmt.Errorf("%s: %w", attack, err) } for _, r := range []release.GenerationReservation{v1, v3} { if ok, err := pins.Refers(r.Binding.Path); err != nil || !ok { t.Fatal("missing pin", err) } } sweepStagedVersions(fakeLister{routes: []registry.Route{{Host: host, Account: "bab", App: "shop", Version: "v2"}}}, filepath.Join(dir, "opt"), filepath.Join(legacy, "generations.json"), time.Hour, time.Now(), source) return nil }); err != nil { t.Fatal(err) } if !staged(t, filepath.Join(dir, "opt"), "v1") || !staged(t, filepath.Join(dir, "opt"), "v2") || !staged(t, filepath.Join(dir, "opt"), "v3") || staged(t, filepath.Join(dir, "opt"), "v4") { t.Fatal("running/pending pins or permitted sweep incorrect") } } if got, err := ledger.Read(v3.Binding); err != nil || got != v3 { t.Fatal("pending reservation changed", err) } }