//go:build linux package activator import ( "context" "errors" "fmt" "os" "os/exec" "path/filepath" "slices" "strconv" "strings" "syscall" "testing" "time" "github.com/carlosframework/platform/internal/cgroup" "github.com/carlosframework/platform/internal/provision" "github.com/carlosframework/platform/internal/registry" ) func TestClaimSpawnKernelChild(t *testing.T) { if os.Getenv("CARLOS_CLAIM_SPAWN_CHILD") != "1" { t.Skip("synthetic contained child only") } time.Sleep(time.Minute) } // TestClaimContainmentCleanupRespawn kills a cgroup and spawns into the // same path again, on a real kernel. On the dev box (amadan #272: arm64, // kernel 6.12.105, systemd 257) a clone3(CLONE_INTO_CGROUP) into a cgroup // that took cgroup.kill is SIGKILLed at once, so each respawn below must // land in a fresh cgroup and stay alive. The strict primitive still places // with clone3; the Manager's starts (the worker's and the sidecar's) now // place through the exec-shim, whose cgroup.procs write is not clone3's, // and must still land in a fresh cgroup, limits written again, and stay // up. Three sequences: the strict // primitive (PrepareStrict, then a cleanup handle's kill), a worker's // KillHost then its respawn into carlos-exec@, and the legacy // sidecar's KillSidecar then its respawn into role-sidecar. func TestClaimContainmentCleanupRespawn(t *testing.T) { if os.Getenv("CARLOS_CLAIM_SPAWN_KERNEL") != "1" { t.Skip("requires a disposable delegated user service") } manager := cgroup.Setup(t.Logf) if manager == nil { t.Fatal("missing delegated cgroup") } dir := t.TempDir() build := func() *exec.Cmd { cmd := exec.Command(os.Args[0], "-test.run=^TestClaimSpawnKernelChild$", "-test.count=1") cmd.Env = []string{"CARLOS_CLAIM_SPAWN_CHILD=1"} cmd.Dir = dir return cmd } t.Run("strict primitive", func(t *testing.T) { claimStrictRespawn(t, manager, build) }) t.Run("KillHost then respawn", func(t *testing.T) { killHostRespawn(t, manager, build) }) t.Run("KillSidecar then respawn", func(t *testing.T) { killSidecarRespawn(t, manager, build) }) } func claimStrictRespawn(t *testing.T, manager *cgroup.Manager, build func() *exec.Cmd) { const host = "claimspawn.example" for attempt := range 2 { // This tests the cgroup primitive with a synthetic test child. It // grants no claim launch authority and is not a tenant-isolation test. prepared, err := manager.PrepareStrict(host, cgroup.TierDefault) if err != nil { t.Fatal(err) } cmd := build() err = prepared.Start(cmd) if err != nil || cmd == nil || prepared == nil { t.Fatalf("spawn %d: cmd=%v prepared=%v err=%v", attempt, cmd, prepared, err) } // The unit's KillMode covers fatal test exits. Normal cleanup below // must release the original reservation without removing the group. if err := prepared.Close(); !errors.Is(err, cgroup.ErrStrictRefused) { t.Fatalf("populated close: %v", err) } cleanup, err := manager.OpenCleanup(host) if err != nil { t.Fatal(err) } ctx, cancel := context.WithTimeout(t.Context(), 3*time.Second) err = cleanup.KillAndEmpty(ctx) cancel() if err != nil { t.Fatal(err) } var exitErr *exec.ExitError if err := cmd.Wait(); err != nil && !errors.As(err, &exitErr) { t.Fatal(err) } if cmd.ProcessState == nil { t.Fatal("child was not reaped") } if err := cleanup.Close(); err != nil { t.Fatal(err) } if other, err := manager.PrepareStrict(host, cgroup.TierDefault); other != nil || !errors.Is(err, cgroup.ErrStrictState) { t.Fatalf("cleanup close released original reservation: %v", err) } if populated, err := prepared.Populated(); err != nil || populated { t.Fatalf("pinned group not proven empty: %v %v", populated, err) } if err := prepared.Close(); err != nil { t.Fatal(err) } } } // spawnStaysUp fails the test if cmd exits within a moment of its start: // the #272 symptom is a child SIGKILLed as it lands. It returns the // channel cmd.Wait's result arrives on. func spawnStaysUp(t *testing.T, label string, cmd *exec.Cmd) <-chan error { t.Helper() done := make(chan error, 1) go func() { done <- cmd.Wait() }() select { case err := <-done: t.Fatalf("%s: the child died as it landed: %v", label, err) case <-time.After(500 * time.Millisecond): } return done } // reaped waits for a killed child's exit. func reaped(t *testing.T, label string, done <-chan error) { t.Helper() select { case <-done: case <-time.After(5 * time.Second): t.Fatalf("%s: the killed child was not reaped", label) } } func cgroupInode(t *testing.T, path string) uint64 { t.Helper() var st syscall.Stat_t if err := syscall.Lstat(path, &st); err != nil { t.Fatalf("stat %s: %v", path, err) } return st.Ino } // killHostRespawn is a worker's sequence: spawned through spawnInCgroup into // carlos-exec@, proven terminated by the real ExecBackend.KillHost, // then respawned into the same path. Each respawn gets a fresh cgroup with // the tier written again, and stays up. func killHostRespawn(t *testing.T, manager *cgroup.Manager, build func() *exec.Cmd) { const host = "killhost.claimspawn.example" b := &ExecBackend{Cgroups: manager} rt := registry.Route{Host: host} dir := filepath.Join(manager.Root(), cgroup.ExecName(host)) var last uint64 for attempt := range 3 { cmd, settle, err := b.spawnInCgroup(rt, build) if err != nil { t.Fatalf("spawn %d: %v", attempt, err) } done := spawnStaysUp(t, fmt.Sprintf("spawn %d", attempt), cmd) ino := cgroupInode(t, dir) if attempt > 0 && ino == last { t.Fatalf("spawn %d reused the killed host cgroup", attempt) } last = ino if got, err := os.ReadFile(filepath.Join(dir, "memory.max")); err != nil || strings.TrimSpace(string(got)) != strconv.FormatInt(cgroup.TierDefault.MemoryMax, 10) { t.Fatalf("spawn %d: memory.max = %q, %v; want the tier", attempt, got, err) } ctx, cancel := context.WithTimeout(t.Context(), 3*time.Second) err = b.KillHost(ctx, rt) cancel() if err != nil { t.Fatalf("KillHost %d: %v", attempt, err) } reaped(t, fmt.Sprintf("spawn %d", attempt), done) settle() } b.cleanupHost(host) } // killSidecarRespawn is the legacy sidecar's sequence under containment: a // flat row's sidecar spawned into carlos-exec@/role-sidecar, proven // gone by the real ExecBackend.KillSidecar, then respawned. Each respawn gets // a fresh role-sidecar under the same parent, and stays up. A live g1 runs // under the parent throughout (zdd readiness round 2, Task 7): a parent // populated only through its children is no legacy parent, so no respawn // is refused, the parent never holds a process directly, it keeps memory // delegated, and g1 is untouched by every kill. func killSidecarRespawn(t *testing.T, manager *cgroup.Manager, build func() *exec.Cmd) { const host = "killsidecar.claimspawn.example" b := &ExecBackend{Cgroups: manager, GenContainment: true, DataDir: t.TempDir(), OptDir: t.TempDir()} rt := registry.Route{Host: host, Kind: "instance", DBPath: provision.DefaultDBPath(b.DataDir, host), Socket: filepath.Join(provision.SocketDir, host+".sock")} parent := filepath.Join(manager.Root(), cgroup.ExecName(host)) child := cgroup.RoleChild("sidecar") g1, _, err := b.spawnInCgroupAt(rt, cgroup.GenChild(1), true, build) if err != nil || g1 == nil { t.Fatalf("live g1: %v, %v", g1, err) } t.Cleanup(func() { _ = g1.Process.Kill() }) g1Done := spawnStaysUp(t, "live g1", g1) var last, parentIno uint64 for attempt := range 3 { cmd, settle, err := b.spawnInCgroupAt(rt, child, false, build) if err != nil { t.Fatalf("spawn %d beside a live g1: %v", attempt, err) } if got, err := os.ReadFile(filepath.Join(parent, "cgroup.procs")); err != nil || strings.TrimSpace(string(got)) != "" { t.Fatalf("spawn %d: the parent holds processes directly: %q, %v", attempt, got, err) } if got, err := os.ReadFile(filepath.Join(parent, "cgroup.subtree_control")); err != nil || !slices.Contains(strings.Fields(string(got)), "memory") { t.Fatalf("spawn %d: the parent's subtree_control is %q, %v; want memory delegated", attempt, got, err) } done := spawnStaysUp(t, fmt.Sprintf("sidecar spawn %d", attempt), cmd) ino := cgroupInode(t, filepath.Join(parent, child)) if attempt > 0 && ino == last { t.Fatalf("sidecar spawn %d reused the killed role-sidecar", attempt) } if p := cgroupInode(t, parent); attempt > 0 && p != parentIno { t.Fatalf("sidecar spawn %d replaced the parent", attempt) } else { parentIno = p } last = ino ctx, cancel := context.WithTimeout(t.Context(), 3*time.Second) err = b.KillSidecar(ctx, rt) cancel() if err != nil { t.Fatalf("KillSidecar %d: %v", attempt, err) } reaped(t, fmt.Sprintf("sidecar spawn %d", attempt), done) settle() } select { case err := <-g1Done: t.Fatalf("the live g1 died during the sidecar respawns: %v", err) default: } ctx, cancel := context.WithTimeout(t.Context(), 3*time.Second) err = b.KillGen(ctx, rt, 1) cancel() if err != nil { t.Fatalf("KillGen 1: %v", err) } reaped(t, "live g1", g1Done) manager.RemoveChild(host, child) manager.CleanupHost(host) }