//go:build linux

package cgroup

import (
	"bufio"
	"context"
	"errors"
	"fmt"
	"io"
	"os"
	"os/exec"
	"path/filepath"
	"strconv"
	"strings"
	"sync"
	"syscall"
	"testing"
	"time"
)

// The generation layout against a real kernel (zero-downtime plan 2, Task
// 4). Opt-in: CARLOS_GEN_KERNEL=1 inside a disposable unit with
// Delegate=yes (the plan's recipe). Without the variable, or without a
// delegated cgroup v2 subtree to work in, every test here skips.

func genKernelManager(t *testing.T) *Manager {
	t.Helper()
	if os.Getenv("CARLOS_GEN_KERNEL") != "1" {
		t.Skip("set CARLOS_GEN_KERNEL=1 inside a unit with Delegate=yes (zero-downtime plan 2, Task 4)")
	}
	raw, err := os.ReadFile("/proc/self/cgroup")
	if err != nil {
		t.Skipf("cgroup v2 delegation unavailable: %v", err)
	}
	rel, ok := ParseSelfCgroup(string(raw))
	if !ok {
		t.Skip("cgroup v2 delegation unavailable: no unified cgroup path")
	}
	root := filepath.Join("/sys/fs/cgroup", rel)
	if filepath.Base(root) == "supervise" {
		root = filepath.Dir(root) // an earlier test in this process moved us
	}
	sup := filepath.Join(root, "supervise")
	if err := os.MkdirAll(sup, 0o755); err != nil {
		t.Skipf("cgroup v2 delegation unavailable: %v", err)
	}
	if err := os.WriteFile(filepath.Join(sup, "cgroup.procs"), []byte(strconv.Itoa(os.Getpid())), 0o644); err != nil {
		t.Skipf("cgroup v2 delegation unavailable: %v", err)
	}
	if err := enableChildren(root); err != nil {
		t.Skipf("cgroup v2 delegation unavailable: %v", err)
	}
	return &Manager{root: root, logf: t.Logf}
}

func genKernelCmd(mode string) *exec.Cmd {
	cmd := exec.Command(os.Args[0], "-test.run=^TestGenKernelChild$", "-test.count=1")
	cmd.Env = []string{"CARLOS_GEN_CHILD=" + mode}
	cmd.Stderr = os.Stderr // a child that fails says why
	return cmd
}

// TestGenKernelChild is the synthetic process: "echo" answers each line it
// reads, standing in for an incumbent serving requests; "hog" touches
// memory until something kills it.
func TestGenKernelChild(t *testing.T) {
	switch os.Getenv("CARLOS_GEN_CHILD") {
	case "echo":
		fmt.Println("ready")
		sc := bufio.NewScanner(os.Stdin)
		for sc.Scan() {
			fmt.Println(sc.Text())
		}
		os.Exit(0)
	case "hog":
		// Wait for the parent's go-ahead, which it gives only after it has
		// checked that this process is inside its capped cgroup. EOF means
		// the parent is gone: exit rather than allocate unwatched.
		if _, err := bufio.NewReader(os.Stdin).ReadString('\n'); err != nil {
			fmt.Fprintln(os.Stderr, "hog: no go-ahead:", err)
			os.Exit(4)
		}
		// One populating mmap: the kernel faults every page in without
		// returning to user space, so memory.high's throttle (a sleep on
		// each return) cannot slow the climb to memory.max. A hog that
		// touches pages one at a time crawls at the throttle, swaps, and
		// can sit under the limit for minutes instead of dying.
		mem, err := syscall.Mmap(-1, 0, 256<<20, syscall.PROT_READ|syscall.PROT_WRITE, syscall.MAP_ANON|syscall.MAP_PRIVATE|syscall.MAP_POPULATE)
		if err != nil {
			fmt.Fprintln(os.Stderr, "hog: mmap:", err)
			os.Exit(2)
		}
		_ = mem
		// Bounded: a hog that survives 1 GiB in all was not confined.
		hold := make([][]byte, 0, 192)
		for len(hold) < cap(hold) {
			b := make([]byte, 4<<20)
			for i := 0; i < len(b); i += 4096 {
				b[i] = 1
			}
			hold = append(hold, b)
		}
		fmt.Fprintln(os.Stderr, "hog: survived 1 GiB: not confined")
		os.Exit(3)
	default:
		// Synthetic child only. It returns rather than skips, so a kernel
		// run's -test.run 'GenKernel' shows no SKIP line (as TestShimKernelChild).
		return
	}
}

type echoChild struct {
	in  io.WriteCloser
	out *bufio.Reader
}

func (e *echoChild) answer(t *testing.T, send, want string) {
	t.Helper()
	got := make(chan string, 1)
	go func() {
		if send != "" {
			fmt.Fprintln(e.in, send)
		}
		line, _ := e.out.ReadString('\n')
		got <- line
	}()
	select {
	case line := <-got:
		if line != want+"\n" {
			t.Fatalf("incumbent answered %q, want %q", line, want)
		}
	case <-time.After(5 * time.Second):
		t.Fatal("incumbent stopped answering")
	}
}

func (e *echoChild) ping(t *testing.T) { t.Helper(); e.answer(t, "ping", "ping") }

func echoPipes(t *testing.T, cmd *exec.Cmd) *echoChild {
	t.Helper()
	in, err := cmd.StdinPipe()
	if err != nil {
		t.Fatal(err)
	}
	out, err := cmd.StdoutPipe()
	if err != nil {
		t.Fatal(err)
	}
	return &echoChild{in: in, out: bufio.NewReader(out)}
}

func expectFile(t *testing.T, path, want string) {
	t.Helper()
	b, err := os.ReadFile(path)
	if err != nil {
		t.Fatal(err)
	}
	if got := strings.TrimSpace(string(b)); got != want {
		t.Fatalf("%s = %q, want %q", path, got, want)
	}
}

func oomKills(t *testing.T, dir string) int {
	t.Helper()
	raw, err := os.ReadFile(filepath.Join(dir, "memory.events"))
	if err != nil {
		t.Fatal(err)
	}
	for _, line := range strings.Split(string(raw), "\n") {
		if f := strings.Fields(line); len(f) == 2 && f[0] == "oom_kill" {
			n, _ := strconv.Atoi(f[1])
			return n
		}
	}
	t.Fatalf("%s/memory.events has no oom_kill line", dir)
	return 0
}

// cgroupOf returns pid's unified cgroup. It reports rather than fails, so
// a caller holding a process that must not run on can kill it first.
func cgroupOf(pid int) (string, error) {
	raw, err := os.ReadFile(fmt.Sprintf("/proc/%d/cgroup", pid))
	if err != nil {
		return "", err
	}
	rel, ok := ParseSelfCgroup(string(raw))
	if !ok {
		return "", fmt.Errorf("pid %d has no unified cgroup", pid)
	}
	return rel, nil
}

// kernelKid is a synthetic child the test started. Its one waiter runs at
// most once, so the test body and the cleanup can both wait on it.
type kernelKid struct {
	cmd  *exec.Cmd
	once sync.Once
	done chan struct{}
	err  error
}

func (k *kernelKid) reaped() <-chan struct{} {
	k.once.Do(func() {
		go func() { k.err = k.cmd.Wait(); close(k.done) }()
	})
	return k.done
}

func (k *kernelKid) wait() error { <-k.reaped(); return k.err }

// reapOnCleanup is registered right after a spawn, before anything that
// can fail: whatever the test's outcome, the child ends. It tries the
// cgroup's own kill first, on a fresh budget, then kills the process
// directly, closes the pipes the test holds, and reaps on a bounded wait.
// A child the spawn never started is left alone.
func reapOnCleanup(t *testing.T, m *Manager, host, child string, cmd *exec.Cmd, pipes ...io.Closer) *kernelKid {
	t.Helper()
	k := &kernelKid{cmd: cmd, done: make(chan struct{})}
	t.Cleanup(func() {
		closePipes := func() {
			for _, p := range pipes {
				if p != nil {
					_ = p.Close()
				}
			}
		}
		if cmd == nil || cmd.Process == nil {
			closePipes()
			return
		}
		if c, err := m.OpenCleanupChild(host, child); err == nil {
			ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
			if err := c.KillAndEmpty(ctx); err != nil {
				t.Logf("cleanup: %s/%s: cgroup kill: %v (killing pid %d directly)", host, child, err, cmd.Process.Pid)
			}
			cancel()
			_ = c.Close()
		}
		_ = cmd.Process.Kill()
		closePipes()
		select {
		case <-k.reaped():
		case <-time.After(5 * time.Second):
			t.Errorf("cleanup: pid %d was not reaped within 5s", cmd.Process.Pid)
		}
	})
	return k
}

// Spec §12, Containment: "a memory-hog successor that overruns its child
// limit and is OOM-killed while the incumbent keeps serving." A small
// limit, not a tier: the accounting path takes any limit, and 64M makes
// the hog die in seconds rather than after a gigabyte of swap.
func TestGenOOMKernel(t *testing.T) {
	m := genKernelManager(t)
	const host = "oom.example"
	small := Limit{MemoryMax: 64 << 20}
	parent := m.hostDir(host)
	// Registered first, so it runs last: after every child is reaped.
	t.Cleanup(func() {
		m.RemoveChild(host, GenChild(1))
		m.RemoveChild(host, GenChild(2))
		m.CleanupHost(host)
	})

	var echo *echoChild
	incumbentCmd, err := m.StartChild(host, GenChild(1), small, small, func() *exec.Cmd {
		cmd := genKernelCmd("echo")
		echo = echoPipes(t, cmd)
		return cmd
	})
	var echoIn io.Closer
	if echo != nil {
		echoIn = echo.in
	}
	incumbent := reapOnCleanup(t, m, host, GenChild(1), incumbentCmd, echoIn)
	if err != nil {
		t.Fatal(err)
	}
	echo.answer(t, "", "ready")
	expectFile(t, filepath.Join(parent, "memory.max"), "67108864")
	expectFile(t, filepath.Join(parent, "memory.oom.group"), "0")
	expectFile(t, filepath.Join(parent, GenChild(1), "memory.oom.group"), "1")
	got1, err := cgroupOf(incumbent.cmd.Process.Pid)
	if want := filepath.Join(strings.TrimPrefix(m.Root(), "/sys/fs/cgroup/"), ExecName(host), GenChild(1)); err != nil || got1 != want {
		t.Fatalf("incumbent runs in %s (%v), want %s", got1, err, want)
	}

	// The raise happens before the successor spawns.
	if err := m.SetParentLimit(host, OverlapLimit(small, small)); err != nil {
		t.Fatal(err)
	}
	expectFile(t, filepath.Join(parent, "memory.max"), "134217728")

	var hogIn io.WriteCloser
	var hogPipeErr error
	hogCmd, err := m.StartChild(host, GenChild(2), small, small, func() *exec.Cmd {
		cmd := genKernelCmd("hog")
		hogIn, hogPipeErr = cmd.StdinPipe()
		return cmd
	})
	hog := reapOnCleanup(t, m, host, GenChild(2), hogCmd, hogIn)
	if err != nil {
		t.Fatal(err)
	}
	if hogPipeErr != nil {
		t.Fatalf("hog stdin: %v", hogPipeErr)
	}
	// StartChild refuses a generation whose limits were not written (Task 7),
	// but prove the confinement here anyway, on the real kernel, before the
	// hog runs: it waits on stdin. Any failure from here kills it: the
	// cleanup registered above runs.
	g2 := filepath.Join(parent, GenChild(2))
	wantG2 := filepath.Join(strings.TrimPrefix(m.Root(), "/sys/fs/cgroup/"), ExecName(host), GenChild(2))
	if got, err := os.ReadFile(filepath.Join(g2, "memory.max")); err != nil || strings.TrimSpace(string(got)) != "67108864" {
		t.Fatalf("g2 memory.max = %q, %v; want 67108864 before the hog runs", got, err)
	}
	if gotCg, err := cgroupOf(hog.cmd.Process.Pid); err != nil || gotCg != wantG2 {
		t.Fatalf("hog runs in %s (%v), want %s", gotCg, err, wantG2)
	}
	if _, err := fmt.Fprintln(hogIn, "go"); err != nil {
		t.Fatalf("hog go-ahead: %v", err)
	}
	select {
	case <-hog.reaped():
		err = hog.err
	case <-time.After(60 * time.Second):
		t.Fatal("the hog was not OOM-killed within 60s")
	}
	var exitErr *exec.ExitError
	if !errors.As(err, &exitErr) {
		t.Fatalf("hog exit: %v", err)
	}
	if ws, ok := exitErr.Sys().(syscall.WaitStatus); !ok || !ws.Signaled() || ws.Signal() != syscall.SIGKILL {
		t.Fatalf("hog ended with %v, want SIGKILL from the OOM killer", err)
	}
	if n := oomKills(t, filepath.Join(parent, GenChild(2))); n < 1 {
		t.Fatalf("g2 memory.events oom_kill = %d, want at least 1", n)
	}
	if n := oomKills(t, filepath.Join(parent, GenChild(1))); n != 0 {
		t.Fatalf("the incumbent's child saw %d OOM kills", n)
	}
	echo.ping(t) // the incumbent keeps serving

	// The restore happens after the retiring child is empty.
	m.RemoveChild(host, GenChild(2))
	if _, err := os.Stat(filepath.Join(parent, GenChild(2))); !os.IsNotExist(err) {
		t.Fatalf("empty g2 not removed: %v", err)
	}
	if err := m.SetParentLimit(host, small); err != nil {
		t.Fatal(err)
	}
	expectFile(t, filepath.Join(parent, "memory.max"), "67108864")
	echo.ping(t)
}

// Strict containment against the kernel: per-generation reservation, the
// parent limit through one API, retirement of one generation, and the
// parent's kill taking every generation.
func TestGenStrictKernel(t *testing.T) {
	m := genKernelManager(t)
	const host = "strictgen.example"
	dir := m.hostDir(host)
	parent, err := m.PrepareStrictParent(host, TierDefault)
	if err != nil {
		t.Fatal(err)
	}
	// Registered first, so it runs last: after every child is reaped. The
	// handles close before the parent's, then the directories go.
	var handles []*Prepared
	t.Cleanup(func() {
		for i := len(handles) - 1; i >= 0; i-- {
			_ = handles[i].Close()
		}
		_ = parent.Close()
		for n := uint64(1); n <= 3; n++ {
			m.RemoveChild(host, GenChild(n))
		}
		m.CleanupHost(host)
	})
	expectFile(t, filepath.Join(dir, "memory.oom.group"), "0")
	expectFile(t, filepath.Join(dir, "memory.max"), "536870912")
	if b, err := os.ReadFile(filepath.Join(dir, "cgroup.subtree_control")); err != nil || !strings.Contains(string(b), "memory") {
		t.Fatalf("parent subtree_control = %q, %v", b, err)
	}
	refused := genKernelCmd("echo")
	err = parent.Start(refused)
	reapOnCleanup(t, m, host, "", refused)
	if !errors.Is(err, ErrStrictState) {
		t.Fatalf("a generation parent accepted a process: %v", err)
	}

	prepareGen := func(n uint64) *Prepared {
		t.Helper()
		g, err := parent.PrepareGen(n, TierDefault)
		if err != nil {
			t.Fatal(err)
		}
		handles = append(handles, g)
		return g
	}
	start := func(g *Prepared, n uint64) (*kernelKid, *echoChild) {
		t.Helper()
		cmd := genKernelCmd("echo")
		e := echoPipes(t, cmd)
		err := g.Start(cmd)
		k := reapOnCleanup(t, m, host, GenChild(n), cmd, e.in)
		if err != nil {
			t.Fatal(err)
		}
		e.answer(t, "", "ready")
		return k, e
	}
	g1 := prepareGen(1)
	kid1, echo1 := start(g1, 1)
	expectFile(t, filepath.Join(dir, GenChild(1), "memory.oom.group"), "1")
	if err := parent.SetParentLimit(OverlapLimit(TierDefault, TierDefault)); err != nil {
		t.Fatal(err)
	}
	expectFile(t, filepath.Join(dir, "memory.max"), "1073741824")
	g2 := prepareGen(2)
	kid2, echo2 := start(g2, 2)
	base := filepath.Join(strings.TrimPrefix(m.Root(), "/sys/fs/cgroup/"), ExecName(host))
	if got, err := cgroupOf(kid1.cmd.Process.Pid); err != nil || got != filepath.Join(base, GenChild(1)) {
		t.Fatalf("g1 process in %s (%v)", got, err)
	}
	if got, err := cgroupOf(kid2.cmd.Process.Pid); err != nil || got != filepath.Join(base, GenChild(2)) {
		t.Fatalf("g2 process in %s (%v)", got, err)
	}
	echo1.ping(t)

	// Retire g1 alone.
	ctx1, cancel1 := context.WithTimeout(t.Context(), 3*time.Second)
	defer cancel1()
	if err := g1.KillAndEmpty(ctx1); err != nil {
		t.Fatal(err)
	}
	_ = kid1.wait()
	echo2.ping(t)
	if err := g1.CloseRemove(); err != nil {
		t.Fatal(err)
	}
	if _, err := os.Stat(filepath.Join(dir, GenChild(1))); !os.IsNotExist(err) {
		t.Fatalf("g1 directory survived CloseRemove: %v", err)
	}
	if err := parent.SetParentLimit(TierDefault); err != nil {
		t.Fatal(err)
	}
	expectFile(t, filepath.Join(dir, "memory.max"), "536870912")

	// A limit this API did not set stops the next spawn.
	if err := os.WriteFile(filepath.Join(dir, "memory.max"), []byte("268435456"), 0o644); err != nil {
		t.Fatal(err)
	}
	g3 := prepareGen(3)
	refused3 := genKernelCmd("echo")
	err = g3.Start(refused3)
	reapOnCleanup(t, m, host, GenChild(3), refused3)
	if !errors.Is(err, ErrStrictRefused) {
		t.Fatalf("spawn under a foreign parent limit: %v", err)
	}
	if err := g3.CloseRemove(); err != nil {
		t.Fatal(err)
	}

	// A terminal stop: the parent's kill takes every generation. Its own
	// budget, not the one g1's retirement spent.
	ctx2, cancel2 := context.WithTimeout(t.Context(), 3*time.Second)
	defer cancel2()
	if err := parent.KillAndEmpty(ctx2); err != nil {
		t.Fatal(err)
	}
	_ = kid2.wait()
	if err := g2.CloseRemove(); err != nil {
		t.Fatal(err)
	}
	if err := parent.Close(); err != nil {
		t.Fatal(err)
	}
	m.CleanupHost(host)
}

// sleepKid starts nothing itself: it is the build for a tenant that sleeps
// until killed.
func sleepKid() *exec.Cmd { return exec.Command("/bin/sleep", "60") }

// staysUp fails the test if k's process exits within a moment of its start
// (amadan #272's symptom: a child SIGKILLed as it lands), and checks that
// the process is the tenant, exec'd, in want (a path below /sys/fs/cgroup).
func staysUp(t *testing.T, label string, k *kernelKid, want string) {
	t.Helper()
	select {
	case <-k.reaped():
		t.Fatalf("%s: the child died as it landed: %v", label, k.err)
	case <-time.After(500 * time.Millisecond):
	}
	pid := k.cmd.Process.Pid
	rel, err := cgroupOf(pid)
	if err != nil || "/sys/fs/cgroup/"+rel != want {
		t.Fatalf("%s: pid %d runs in %q (%v), want %s", label, pid, rel, err, want)
	}
	if comm, err := os.ReadFile(fmt.Sprintf("/proc/%d/comm", pid)); err != nil || strings.TrimSpace(string(comm)) != "sleep" {
		t.Fatalf("%s: pid %d is %q (%v), want the exec'd tenant, sleep", label, pid, comm, err)
	}
}

func inodeOf(t *testing.T, path string) uint64 {
	t.Helper()
	var st syscall.Stat_t
	if err := syscall.Lstat(path, &st); err != nil {
		t.Fatalf("stat %s: %v", path, err)
	}
	return st.Ino
}

// The respawn scenario amadan #272 found with clone3 (a clone3 into a
// cgroup that took cgroup.kill is SIGKILLed at once), now through the
// shim: a flat start (a worker's) lands in carlos-exec@<host> placed before
// exec, a cleanup kills and proves it empty, and each respawn gets a fresh
// cgroup (new inode, the tier written again) and stays up as the tenant.
// The activator's TestClaimContainmentCleanupRespawn runs the same sequence
// through ExecBackend.KillHost, but needs a delegation with io to arm
// cgroup.Setup; this one runs under a user manager's (cpu memory pids).
func TestGenKernelShimRespawnAfterKill(t *testing.T) {
	m := genKernelManager(t)
	const host = "respawn.example"
	dir := m.hostDir(host)
	t.Cleanup(func() { m.CleanupHost(host) })
	tier := strconv.FormatInt(TierDefault.MemoryMax, 10)
	var last uint64
	for attempt := range 3 {
		label := fmt.Sprintf("spawn %d", attempt)
		cmd, err := m.Start(host, TierDefault, sleepKid)
		if err != nil {
			t.Fatalf("%s: %v", label, err)
		}
		k := &kernelKid{cmd: cmd, done: make(chan struct{})}
		t.Cleanup(func() { _ = cmd.Process.Kill(); <-k.reaped() })
		staysUp(t, label, k, dir)
		ino := inodeOf(t, dir)
		if attempt > 0 && ino == last {
			t.Fatalf("%s reused the killed host cgroup", label)
		}
		last = ino
		if got := strings.TrimSpace(readFixture(t, filepath.Join(dir, "memory.max"))); got != tier {
			t.Fatalf("%s: memory.max = %q, want the tier", label, got)
		}
		c, err := m.OpenCleanup(host)
		if err != nil {
			t.Fatal(err)
		}
		ctx, cancel := context.WithTimeout(t.Context(), 3*time.Second)
		err = c.KillAndEmpty(ctx)
		cancel()
		_ = c.Close()
		if err != nil {
			t.Fatalf("%s: KillAndEmpty: %v", label, err)
		}
		select {
		case <-k.reaped():
		case <-time.After(5 * time.Second):
			t.Fatalf("%s: the killed child was not reaped", label)
		}
	}
}

// amadan #272, real-unit check (d), through the shim: a flat start puts its
// process directly in carlos-exec@<host>, and a generation start beside it
// is refused (ErrLegacyParentPopulated) before it builds anything or makes
// a g* child. The recovery control: once the direct member has exited, the
// same host starts a capped g1 under a parent that holds no process.
func TestGenKernelGenerationRefusedBesideAFlatMember(t *testing.T) {
	m := genKernelManager(t)
	const host = "legacy.example"
	parent := m.hostDir(host)
	t.Cleanup(func() {
		m.RemoveChild(host, GenChild(1))
		m.CleanupHost(host)
	})
	legacy, err := m.Start(host, TierDefault, sleepKid)
	if err != nil {
		t.Fatal(err)
	}
	lk := &kernelKid{cmd: legacy, done: make(chan struct{})}
	t.Cleanup(func() { _ = legacy.Process.Kill(); <-lk.reaped() })
	staysUp(t, "flat", lk, parent)

	built := false
	cmd, err := m.StartChild(host, GenChild(1), TierDefault, TierDefault, func() *exec.Cmd { built = true; return sleepKid() })
	if !errors.Is(err, ErrLegacyParentPopulated) || cmd != nil || built {
		t.Fatalf("StartChild beside a flat member = %v, %v (built %v): want ErrLegacyParentPopulated, nothing built", cmd, err, built)
	}
	if _, err := os.Stat(filepath.Join(parent, GenChild(1))); !os.IsNotExist(err) {
		t.Fatalf("the refused start made g1: %v", err)
	}

	_ = legacy.Process.Kill()
	<-lk.reaped()
	for deadline := time.Now().Add(5 * time.Second); strings.TrimSpace(readFixture(t, filepath.Join(parent, "cgroup.procs"))) != ""; time.Sleep(20 * time.Millisecond) {
		if time.Now().After(deadline) {
			t.Fatal("the parent still holds the killed flat process")
		}
	}
	g1, err := m.StartChild(host, GenChild(1), TierDefault, TierDefault, sleepKid)
	if err != nil {
		t.Fatalf("g1 once the flat member exited: %v", err)
	}
	gk := reapOnCleanup(t, m, host, GenChild(1), g1)
	staysUp(t, "g1", gk, filepath.Join(parent, GenChild(1)))
	if got := strings.TrimSpace(readFixture(t, filepath.Join(parent, GenChild(1), "memory.max"))); got != strconv.FormatInt(TierDefault.MemoryMax, 10) {
		t.Errorf("g1 memory.max = %q, want the tier", got)
	}
	if got := strings.TrimSpace(readFixture(t, filepath.Join(parent, "cgroup.procs"))); got != "" {
		t.Errorf("the parent holds processes directly: %q", got)
	}
}

// Review focus 4 on a real kernel: a generation's shim held before its
// placement is registered in flight and still in the agent's own cgroup, so
// the kernel says the host is empty. The host's cleanup must not: Populated
// is true, Members is ErrPlacementInFlight, and KillAndEmpty waits, killing
// nothing, until the placement resolves (here: the shim is SIGKILLed, so
// the start is refused as a placement failure) and only then proves the
// host empty.
func TestGenKernelCleanupWaitsForAPlacementInFlight(t *testing.T) {
	m := genKernelManager(t)
	const host = "inflight.example"
	parent := m.hostDir(host)
	t.Cleanup(func() {
		m.RemoveChild(host, GenChild(1))
		m.CleanupHost(host)
	})
	setShimVar(t, &shimTestHold, 1)
	held := make(chan int, 1)
	orig := startCmd
	t.Cleanup(func() { startCmd = orig })
	startCmd = func(cmd *exec.Cmd) error {
		if err := orig(cmd); err != nil {
			return err
		}
		pid, specFD := cmd.Process.Pid, len(cmd.ExtraFiles) // the shim's spec fd: see TestShimKernelSignalBeforePlacement
		go func() {
			if waitShimHeld(pid, specFD) {
				held <- pid
			} else {
				held <- 0
			}
		}()
		return nil
	}
	started := make(chan error, 1)
	go func() {
		_, err := m.StartChild(host, GenChild(1), TierDefault, TierDefault, sleepKid)
		started <- err
	}()
	pid := <-held
	if pid == 0 {
		t.Fatal("the shim never reached its hold before placement")
	}
	t.Cleanup(func() { _ = syscall.Kill(pid, syscall.SIGKILL) })

	c, err := m.OpenCleanup(host)
	if err != nil {
		t.Fatal(err)
	}
	t.Cleanup(func() { _ = c.Close() })
	events := readFixture(t, filepath.Join(parent, "cgroup.events"))
	if !strings.Contains(events, "populated 0") {
		t.Fatalf("the parent's cgroup.events = %q: want the kernel to see it empty, or this proves nothing", events)
	}
	if populated, err := c.Populated(); err != nil || !populated {
		t.Fatalf("Populated = %v, %v with a shim held before placement: want true", populated, err)
	}
	if _, err := c.Members(); !errors.Is(err, ErrPlacementInFlight) {
		t.Fatalf("Members = %v, want ErrPlacementInFlight", err)
	}
	ctx, cancel := context.WithTimeout(t.Context(), 200*time.Millisecond)
	err = c.KillAndEmpty(ctx)
	cancel()
	if !errors.Is(err, ErrPlacementInFlight) || !errors.Is(err, context.DeadlineExceeded) {
		t.Fatalf("KillAndEmpty with a placement in flight = %v: want ErrPlacementInFlight, unproven", err)
	}
	if procGone(pid) {
		t.Fatal("the held shim died: the unproven proof killed it, or the hold did not hold")
	}

	proof := make(chan error, 1)
	go func() {
		ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second)
		defer cancel()
		proof <- c.KillAndEmpty(ctx)
	}()
	select {
	case err := <-proof:
		t.Fatalf("KillAndEmpty returned %v while the placement was still in flight", err)
	case <-time.After(200 * time.Millisecond):
	}
	if err := syscall.Kill(pid, syscall.SIGKILL); err != nil {
		t.Fatal(err)
	}
	if err := <-started; !errors.Is(err, ErrLimitNotApplied) || !isPlacementError(err) {
		t.Fatalf("the generation start = %v, want ErrLimitNotApplied from the placement failure", err)
	}
	if err := <-proof; err != nil {
		t.Fatalf("KillAndEmpty once the placement resolved = %v, want the proof", err)
	}
	assertNothingInFlight(t, m)
}

// A generation start whose exec.CommandContext context ends while its shim
// is held before placement: exec.Cmd kills the shim, nothing runs, the
// start is not the box's condition (not ErrLimitNotApplied), the empty
// child is removed, and nothing is left in flight.
func TestGenKernelCancelledGenerationStartIsNotABoxCondition(t *testing.T) {
	m := genKernelManager(t)
	const host = "cancel.example"
	parent := m.hostDir(host)
	t.Cleanup(func() {
		m.RemoveChild(host, GenChild(1))
		m.CleanupHost(host)
	})
	setShimVar(t, &shimTestHold, 1)
	ctx, cancel := context.WithCancel(t.Context())
	defer cancel()
	held := make(chan bool, 1)
	orig := startCmd
	t.Cleanup(func() { startCmd = orig })
	startCmd = func(cmd *exec.Cmd) error {
		if err := orig(cmd); err != nil {
			return err
		}
		pid, specFD := cmd.Process.Pid, len(cmd.ExtraFiles)
		go func() {
			ok := waitShimHeld(pid, specFD)
			held <- ok
			cancel()
		}()
		return nil
	}
	marker := filepath.Join(t.TempDir(), "ran")
	var built *exec.Cmd
	cmd, err := m.StartChild(host, GenChild(1), TierDefault, TierDefault, func() *exec.Cmd {
		built = exec.CommandContext(ctx, "/bin/sh", "-c", `touch "$0"; sleep 60`, marker)
		return built
	})
	if !<-held {
		t.Fatal("the shim never reached its hold before placement")
	}
	if err == nil || cmd != nil || errors.Is(err, ErrLimitNotApplied) {
		t.Fatalf("StartChild = %v, %v: want a failure that is not ErrLimitNotApplied", cmd, err)
	}
	// The context's own error, bare, as exec.Cmd.Start answers an ended
	// context (whole-branch review, finding 3).
	if err != context.Canceled {
		t.Fatalf("StartChild = %v: want context.Canceled itself", err)
	}
	t.Logf("cancelled generation start: %v", err)
	assertReaped(t, built)
	assertNoMarker(t, marker)
	assertNothingInFlight(t, m)
	if _, err := os.Stat(filepath.Join(parent, GenChild(1))); !os.IsNotExist(err) {
		t.Fatalf("the empty g1 survived: %v", err)
	}
}