package activator

import (
	"context"
	"errors"
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"strconv"
	"strings"
	"syscall"
	"testing"

	"github.com/carlosframework/platform/internal/cgroup"
	"github.com/carlosframework/platform/internal/instancectl"
	"github.com/carlosframework/platform/internal/registry"
)

// fakeGenControl adds the generation verbs to fakeControl (unitproc_test.go).
type fakeGenControl struct{ *fakeControl }

func (f fakeGenControl) StartGen(_ context.Context, host string, n uint64, max int64) (string, error) {
	f.rec(fmt.Sprintf("startgen %s %d %d", host, n, max))
	if f.onStart != nil {
		f.onStart()
	}
	return f.inv, f.startErr
}
func (f fakeGenControl) StartGenExe(ctx context.Context, host string, n uint64, max int64, _ instancectl.StagedExe) (string, error) {
	return f.StartGen(ctx, host, n, max)
}
func (f fakeGenControl) StartSidecarExe(_ context.Context, host string, max int64, _ instancectl.StagedExe) (string, error) {
	f.rec(fmt.Sprintf("startsidecarexe %s %d", host, max))
	return f.inv, f.startErr
}
func (f fakeGenControl) StopGen(_ context.Context, host string, n uint64, kill bool) error {
	f.rec(fmt.Sprintf("stopgen %s %d kill=%v", host, n, kill))
	if f.stopHook != nil {
		f.stopHook(host, instancectl.RoleRun)
	}
	return f.stopGenErr
}
func (f fakeGenControl) ResetGen(_ context.Context, host string, n uint64) error {
	f.rec(fmt.Sprintf("resetgen %s %d", host, n))
	return nil
}
func (f fakeGenControl) Parent(_ context.Context, host string, max int64) error {
	f.rec(fmt.Sprintf("parent %s %d", host, max))
	return nil
}

func TestPlaceForIsSingleLevelWithContainmentOff(t *testing.T) {
	b := testExecBackend(t)
	for _, kind := range []spawnKind{spawnInstance, spawnSidecar, spawnSeed, spawnWorker} {
		child, own, err := b.placeFor(kind, 7)
		if child != "" || own || err != nil {
			t.Errorf("kind %d with containment off: %q %v %v", kind, child, own, err)
		}
	}
}

func TestPlaceForUnderContainment(t *testing.T) {
	b := testExecBackend(t)
	b.GenContainment = true
	for _, tc := range []struct {
		kind  spawnKind
		child string
		own   bool
	}{
		{spawnInstance, "g7", true},
		{spawnSidecar, "role-sidecar", false},
		{spawnSeed, "role-seed", false},
		{spawnWorker, "", false},
	} {
		child, own, err := b.placeFor(tc.kind, 7)
		if child != tc.child || own != tc.own || err != nil {
			t.Errorf("kind %d: %q %v %v, want %q %v", tc.kind, child, own, err, tc.child, tc.own)
		}
	}
	if _, _, err := b.placeFor(spawnInstance, 0); err == nil {
		t.Error("an instance spawn with generation 0 was placed")
	}
}

func delegatedBackend(t *testing.T, host string) (*ExecBackend, string) {
	t.Helper()
	root := t.TempDir()
	for _, f := range []string{"cgroup.procs", "cgroup.subtree_control"} {
		if err := os.WriteFile(filepath.Join(root, f), nil, 0o644); err != nil {
			t.Fatal(err)
		}
	}
	m, err := cgroup.SetupAt(root, t.Logf)
	if err != nil {
		t.Fatal(err)
	}
	parent := filepath.Join(root, cgroup.ExecName(host))
	if err := os.MkdirAll(parent, 0o755); err != nil {
		t.Fatal(err)
	}
	if err := os.WriteFile(filepath.Join(parent, "cgroup.controllers"), []byte("cpu io memory pids\n"), 0o644); err != nil {
		t.Fatal(err)
	}
	b := testExecBackend(t)
	b.Cgroups = m
	return b, parent
}

func TestSpawnInCgroupAtPutsAGenerationUnderItsParent(t *testing.T) {
	const host = "h.example.com"
	b, parent := delegatedBackend(t, host)
	cgroup.FakePlacementForTest(t) // a temp dir is no cgroupfs, and a generation is never moved by PID
	b.GenContainment = true
	cmd, _, err := b.spawnInCgroupAt(registry.Route{Host: host}, "g7", true, func() *exec.Cmd { return exec.Command("/bin/true") })
	if err != nil {
		t.Fatal(err)
	}
	_ = cmd.Wait()
	for file, want := range map[string]string{
		filepath.Join(parent, "g7", "memory.max"):       "536870912",
		filepath.Join(parent, "g7", "memory.oom.group"): "1",
		filepath.Join(parent, "memory.oom.group"):       "0",
	} {
		if got, err := os.ReadFile(file); err != nil || string(got) != want {
			t.Errorf("%s = %q, %v; want %q", file, got, err, want)
		}
	}
}

// Flag off: the spawn lands in the host cgroup itself, as today.
func TestSpawnInCgroupAtWithNoChildKeepsTheHostCgroup(t *testing.T) {
	const host = "h.example.com"
	b, parent := delegatedBackend(t, host)
	cmd, _, err := b.spawnInCgroupAt(registry.Route{Host: host}, "", false, func() *exec.Cmd { return exec.Command("/bin/true") })
	if err != nil {
		t.Fatal(err)
	}
	_ = cmd.Wait()
	if got, err := os.ReadFile(filepath.Join(parent, "memory.oom.group")); err != nil || string(got) != "1" {
		t.Errorf("host cgroup memory.oom.group = %q, %v; want 1", got, err)
	}
	entries, _ := os.ReadDir(parent)
	for _, e := range entries {
		if e.IsDir() {
			t.Errorf("a flag-off spawn made child cgroup %s", e.Name())
		}
	}
}

func TestParentLimitRefusesWithContainmentOff(t *testing.T) {
	b, _ := delegatedBackend(t, "h.example.com")
	if err := b.ParentLimit(context.Background(), registry.Route{Host: "h.example.com"}, cgroup.TierDefault); !errors.Is(err, errNoGenContainment) {
		t.Fatalf("ParentLimit with containment off: %v", err)
	}
	if err := b.KillGen(context.Background(), registry.Route{Host: "h.example.com"}, 7); !errors.Is(err, errNoGenContainment) {
		t.Fatalf("KillGen with containment off: %v", err)
	}
}

func TestParentLimitFlatWritesTheParent(t *testing.T) {
	const host = "h.example.com"
	b, parent := delegatedBackend(t, host)
	b.GenContainment = true
	if err := b.ParentLimit(context.Background(), registry.Route{Host: host}, cgroup.OverlapLimit(cgroup.TierDefault, cgroup.TierDefault)); err != nil {
		t.Fatal(err)
	}
	if got, err := os.ReadFile(filepath.Join(parent, "memory.max")); err != nil || string(got) != "1073741824" {
		t.Errorf("parent memory.max = %q, %v", got, err)
	}
}

// No cgroup manager (no delegation): there is no parent to raise, so a
// flat ParentLimit refuses rather than let a swap admit an overlap with
// nothing raised.
func TestParentLimitFlatRefusesWithoutACgroupManager(t *testing.T) {
	b := testExecBackend(t)
	b.GenContainment = true
	b.Cgroups = nil
	if err := b.ParentLimit(context.Background(), registry.Route{Host: "h.example.com"}, cgroup.OverlapLimit(cgroup.TierDefault, cgroup.TierDefault)); !errors.Is(err, cgroup.ErrNoDelegation) {
		t.Fatalf("ParentLimit with no cgroup manager: %v, want cgroup.ErrNoDelegation", err)
	}
}

// A fixture is not cgroup v2: KillGen must refuse rather than claim an
// empty child it never proved.
func TestKillGenFlatRefusesWithoutCgroupV2(t *testing.T) {
	const host = "h.example.com"
	b, parent := delegatedBackend(t, host)
	b.GenContainment = true
	if err := os.Mkdir(filepath.Join(parent, "g7"), 0o755); err != nil {
		t.Fatal(err)
	}
	if err := b.KillGen(context.Background(), registry.Route{Host: host}, 7); err == nil {
		t.Fatal("KillGen reported an unproven kill as done")
	}
}

func TestParentLimitAndKillGenNestedAskTheControlService(t *testing.T) {
	ctl := &fakeControl{}
	b, rt := nestedBackend(t, ctl, &fakeUnit{})
	b.GenContainment = true
	b.Control = fakeGenControl{ctl}
	if err := b.ParentLimit(context.Background(), rt, cgroup.OverlapLimit(cgroup.TierDefault, cgroup.TierDefault)); err != nil {
		t.Fatal(err)
	}
	if err := b.KillGen(context.Background(), rt, 7); err != nil {
		t.Fatal(err)
	}
	if want := "parent h.example.com 1073741824\nstopgen h.example.com 7 kill=true"; ctl.joined() != want {
		t.Errorf("calls =\n%s\nwant\n%s", ctl.joined(), want)
	}
	b.Control = ctl // a Control without the generation verbs
	if err := b.ParentLimit(context.Background(), rt, cgroup.TierDefault); !errors.Is(err, errNoGenControl) {
		t.Errorf("ParentLimit through a Control without generations: %v", err)
	}
}

// SplitBackend forwards KillGen to the exec half for an exec row. A
// unit-backed row, and an exec half that cannot kill a generation, refuse:
// a caller must never believe a kill it could not make.
func TestSplitBackendKillGen(t *testing.T) {
	ctl := &fakeControl{}
	b, rt := nestedBackend(t, ctl, &fakeUnit{})
	b.GenContainment = true
	b.Control = fakeGenControl{ctl}
	sb := &SplitBackend{Exec: b, Unit: &UnitBackend{}}
	if err := sb.KillGen(context.Background(), rt, 7); err != nil {
		t.Fatal(err)
	}
	if want := "stopgen h.example.com 7 kill=true"; ctl.joined() != want {
		t.Errorf("calls =\n%s\nwant\n%s", ctl.joined(), want)
	}
	unitRt := rt
	unitRt.Backing = registry.BackingUnit
	if err := sb.KillGen(context.Background(), unitRt, 7); !errors.Is(err, errNoGenContainment) {
		t.Errorf("KillGen of a unit-backed row: %v", err)
	}
	if want := "stopgen h.example.com 7 kill=true"; ctl.joined() != want {
		t.Errorf("a unit-backed KillGen reached the control service:\n%s", ctl.joined())
	}
	plain := &SplitBackend{Exec: &fakeBackend{}, Unit: &UnitBackend{}}
	if err := plain.KillGen(context.Background(), rt, 7); !errors.Is(err, errNoGenContainment) {
		t.Errorf("KillGen through an exec half without KillGen: %v", err)
	}
}

// A placed spawn that fails to start leaves no child cgroup behind.
func TestSpawnInCgroupAtRemovesTheChildOfAFailedSpawn(t *testing.T) {
	const host = "h.example.com"
	b, parent := delegatedBackend(t, host)
	b.GenContainment = true
	// A zero tier writes no limit files, so the fixture's child directory is
	// as empty as a real cgroup's after its control files vanish.
	b.Tier = func(registry.Route) cgroup.Limit { return cgroup.Limit{} }
	_, _, err := b.spawnInCgroupAt(registry.Route{Host: host}, "g7", true, func() *exec.Cmd {
		return exec.Command(filepath.Join(t.TempDir(), "no-such-binary"))
	})
	if err == nil {
		t.Fatal("a spawn of a missing binary succeeded")
	}
	if _, err := os.Stat(filepath.Join(parent, "g7")); !os.IsNotExist(err) {
		t.Errorf("the failed spawn's child cgroup is still there: %v", err)
	}
}

// A placed spawn never starts outside a strict host's group: a host that is
// strict, or one whose group a strict owner holds, is refused.
func TestSpawnInCgroupAtRefusesAStrictHost(t *testing.T) {
	const host = "h.example.com"
	b, parent := delegatedBackend(t, host)
	b.GenContainment = true
	b.Strict = func(string) bool { return true }
	build := func() *exec.Cmd { return exec.Command("/bin/true") }
	if _, _, err := b.spawnInCgroupAt(registry.Route{Host: host}, "g7", true, build); !errors.Is(err, ErrStrictRequired) {
		t.Fatalf("a strict host's placed spawn: %v", err)
	}
	b.Strict = func(string) bool { return false }
	b.gate(host).owner = &strictOwner{}
	if _, _, err := b.spawnInCgroupAt(registry.Route{Host: host}, "role-seed", false, build); !errors.Is(err, ErrStrictGroupInUse) {
		t.Fatalf("a placed spawn beside a strict owner: %v", err)
	}
	if _, err := os.Stat(filepath.Join(parent, "g7")); !os.IsNotExist(err) {
		t.Errorf("a refused spawn made a child cgroup: %v", err)
	}
}

// Amadan #272: a host whose parent cgroup still holds a process directly
// (a legacy flat start) gets no generation, in any child shape. The refusal
// comes back before the exec, leaves no child, and reads as nothing of the
// build having run, so no swap or wake counts it as "may have run" or as a
// failure of the build.
func TestSpawnInCgroupAtRefusesAPopulatedLegacyParent(t *testing.T) {
	const host = "h.example.com"
	b, parent := delegatedBackend(t, host)
	b.GenContainment = true
	if err := os.WriteFile(filepath.Join(parent, "cgroup.procs"), []byte("4242\n"), 0o644); err != nil {
		t.Fatal(err)
	}
	ran := false
	build := func() *exec.Cmd { ran = true; return exec.Command("/bin/true") }
	for _, child := range []string{"g7", "role-seed", "role-sidecar"} {
		cmd, _, err := b.spawnInCgroupAt(registry.Route{Host: host}, child, child == "g7", build)
		if !errors.Is(err, cgroup.ErrLegacyParentPopulated) || cmd != nil {
			t.Fatalf("%s under a populated parent: %v, %v", child, cmd, err)
		}
		if startMayHaveRun(nil, err) {
			t.Errorf("%s: the refusal reads as a start that may have run", child)
		}
		if verdictOf(err) != gateNoOwnership {
			t.Errorf("%s: the refusal is not an abandonment", child)
		}
		if _, serr := os.Stat(filepath.Join(parent, child)); !os.IsNotExist(serr) {
			t.Errorf("%s: a refused spawn made a child cgroup: %v", child, serr)
		}
	}
	if ran {
		t.Error("the build was prepared for exec under a populated parent")
	}
}

// zdd-readiness Task 6: errStartedThenExited is found through every wrap
// spawnUnitFrom gives it (the uncertain mark, and beside errGenUnproven when
// the kill is unproven), and the text is the one it always was: a flag-off
// nested wake fails with it, and the member reads it.
func TestStartedThenExitedIsUncertainAndKeepsItsText(t *testing.T) {
	unit := exitedUnobserved{unit: "carlos-instance-gen@h.example.com:7.service"}
	proven := markStartUncertain(fmt.Errorf("%s: start unit: %w", "h.example.com", unit))
	unproven := markStartUncertain(fmt.Errorf("%s: start unit: %w; %w: %v", "h.example.com", unit, errGenUnproven, errors.New("stop timed out")))
	for _, err := range []error{proven, unproven} {
		if !errors.Is(err, errStartedThenExited) || !startMayHaveRun(nil, err) {
			t.Fatalf("%v: errStartedThenExited %v, may have run %v; want both", err, errors.Is(err, errStartedThenExited), startMayHaveRun(nil, err))
		}
	}
	if want := "h.example.com: start unit: carlos-instance-gen@h.example.com:7.service exited before it was observed"; proven.Error() != want {
		t.Fatalf("text %q, want %q", proven.Error(), want)
	}
	if !genStartUnproven(unproven) || genStartUnproven(proven) {
		t.Fatal("only the unproven kill carries errGenUnproven")
	}
	if errors.Is(markStartUncertain(errors.New("the unit failed to start")), errStartedThenExited) {
		t.Fatal("another uncertain start reads as a crash at the start")
	}
}

// Task 8: a failed flat seed's processes are killed and proven gone before
// its database is removed. Here they cannot be (a temp dir is no cgroupfs,
// so the host cgroup's kill is unproven): the seed answers ErrSeedUnproven,
// its database stays, and a durable obligation is left beside it.
func TestAFailedSeedWhoseCleanupIsUnprovenKeepsTheDBAndLeavesAnObligation(t *testing.T) {
	const host = "unproven-seed.example.com"
	b, _ := delegatedBackend(t, host)
	stageSeedScript(t, b, host, "v1", "printf half > \"$3\"\nexit 1\n")
	rt := registry.Route{Host: host, Kind: "instance", Seed: `["seed"]`}

	err := b.Seed(context.Background(), rt)
	if !errors.Is(err, ErrSeedUnproven) {
		t.Fatalf("Seed = %v, want ErrSeedUnproven: the seed's leftovers were never proven gone", err)
	}
	db := filepath.Join(b.DataDir, host+".db")
	if got, err := os.ReadFile(db); err != nil || string(got) != "half" {
		t.Fatalf("database = %q, %v; want it kept: a process may still be writing it", got, err)
	}
	if _, err := os.Stat(db + seedUnprovenSuffix); err != nil {
		t.Fatalf("no obligation beside the database: %v", err)
	}
}

// Task 8, plan 3: the obligation survives to the next wake. While the proof
// still fails, Restore refuses and the database is never reported warm.
// Once it passes, the database, its WAL and SHM and the obligation are
// removed, the wake is fresh, and the seed runs again.
func TestTheNextWakeAfterAnUnprovenSeedProvesItBeforeTheDBIsWarm(t *testing.T) {
	const host = "obligation.example.com"
	b, _ := delegatedBackend(t, host)
	runs := filepath.Join(t.TempDir(), "runs")
	stageSeedScript(t, b, host, "v1", fmt.Sprintf(
		"echo run >> %q\nprintf half > \"$3\"\n[ \"$(wc -l < %q)\" -ge 2 ] && exit 0\nexit 1\n", runs, runs))
	swapRestoreReplica(t, func(context.Context, string, string, string, string) (bool, error) {
		return false, nil
	})
	rt := registry.Route{Host: host, Kind: "instance", Seed: `["seed"]`}
	db := filepath.Join(b.DataDir, host+".db")

	if err := b.Seed(context.Background(), rt); !errors.Is(err, ErrSeedUnproven) {
		t.Fatalf("first Seed = %v, want ErrSeedUnproven", err)
	}
	for _, p := range []string{db + "-wal", db + "-shm"} {
		if err := os.WriteFile(p, []byte("x"), 0o600); err != nil {
			t.Fatal(err)
		}
	}

	// The proof still fails: refused, and the database is not warm.
	fresh, err := b.Restore(context.Background(), rt)
	if !errors.Is(err, ErrSeedUnproven) || fresh {
		t.Fatalf("Restore = %v, %v; want a refusal with ErrSeedUnproven", fresh, err)
	}
	if got, _ := os.ReadFile(db); string(got) != "half" {
		t.Fatalf("database = %q after a refused wake, want it untouched", got)
	}
	if _, err := os.Stat(db + seedUnprovenSuffix); err != nil {
		t.Fatalf("the obligation went with a refused wake: %v", err)
	}

	// The proof passes: everything of the failed seed goes, and it is fresh.
	var asked []string
	b.cgroupKill = func(_ context.Context, _ registry.Route, child string) error {
		asked = append(asked, child)
		return nil
	}
	fresh, err = b.Restore(context.Background(), rt)
	if err != nil || !fresh {
		t.Fatalf("Restore = %v, %v; want fresh once the proof holds", fresh, err)
	}
	if len(asked) != 1 || asked[0] != "" {
		t.Fatalf("proof asked of %q, want the host cgroup once", asked)
	}
	for _, p := range []string{db, db + "-wal", db + "-shm", db + seedUnprovenSuffix} {
		if _, err := os.Stat(p); !os.IsNotExist(err) {
			t.Fatalf("%s still there (%v) after the proof held", filepath.Base(p), err)
		}
	}
	if err := b.Seed(context.Background(), rt); err != nil {
		t.Fatalf("second Seed: %v", err)
	}
	got, err := os.ReadFile(runs)
	if err != nil || strings.Count(string(got), "run\n") != 2 {
		t.Fatalf("seed runs = %q, %v; want the seed to have run twice", got, err)
	}
}

// Task 8, plan 6: flag on, a successful seed's role child is killed and
// proven empty before the seed returns, so a helper that redirected its
// output (Wait returns a plain nil) does not outlive it.
func TestAFlagOnSeedKillsAHelperThatOutlivesIt(t *testing.T) {
	const host = "helper-seed.example.com"
	b, _ := delegatedBackend(t, host)
	cgroup.FakePlacementForTest(t) // a temp dir is no cgroupfs, and a role child is never moved by PID
	b.GenContainment = true
	pidFile := filepath.Join(t.TempDir(), "helper.pid")
	stageSeedScript(t, b, host, "v1", fmt.Sprintf(
		"sleep 30 >/dev/null 2>&1 &\necho $! > %q\nprintf seeded > \"$3\"\nexit 0\n", pidFile))
	helper := func() int {
		raw, err := os.ReadFile(pidFile)
		if err != nil {
			return 0
		}
		pid, _ := strconv.Atoi(strings.TrimSpace(string(raw)))
		return pid
	}
	t.Cleanup(func() {
		if pid := helper(); pid > 0 {
			_ = syscall.Kill(pid, syscall.SIGKILL)
		}
	})
	var asked []string
	b.cgroupKill = func(_ context.Context, _ registry.Route, child string) error {
		asked = append(asked, child)
		if pid := helper(); pid > 0 {
			_ = syscall.Kill(pid, syscall.SIGKILL) // what cgroup.kill does to the role child
		}
		return nil
	}

	if err := b.Seed(context.Background(), registry.Route{Host: host, Kind: "instance", Seed: `["seed"]`}); err != nil {
		t.Fatalf("Seed: %v", err)
	}
	if len(asked) != 1 || asked[0] != cgroup.RoleChild("seed") {
		t.Fatalf("killed %q before the seed returned, want the seed's role child once", asked)
	}
	if got, err := os.ReadFile(filepath.Join(b.DataDir, host+".db")); err != nil || string(got) != "seeded" {
		t.Fatalf("database = %q, %v; want the seed's work kept", got, err)
	}
}

// Flag on, a successful seed whose role child cannot be proven empty is
// unproven: the helper may still write the database, so the wake must not
// go on to serve it, and the next wake must prove it first.
func TestAFlagOnSeedWhoseHelperIsNotProvenGoneIsUnproven(t *testing.T) {
	const host = "helper-unproven.example.com"
	b, _ := delegatedBackend(t, host)
	cgroup.FakePlacementForTest(t)
	b.GenContainment = true
	stageSeedScript(t, b, host, "v1", "printf seeded > \"$3\"\nexit 0\n")
	b.cgroupKill = func(context.Context, registry.Route, string) error {
		return errors.New("the role child still holds a process")
	}

	err := b.Seed(context.Background(), registry.Route{Host: host, Kind: "instance", Seed: `["seed"]`})
	if !errors.Is(err, ErrSeedUnproven) {
		t.Fatalf("Seed = %v, want ErrSeedUnproven", err)
	}
	if _, err := os.Stat(filepath.Join(b.DataDir, host+".db"+seedUnprovenSuffix)); err != nil {
		t.Fatalf("no obligation beside the database: %v", err)
	}
}

// Task 8 fix round 1, finding 1: the obligation is written ahead. It is on
// disk, durably, before the seed's first instruction runs, so an agent that
// dies mid-seed leaves it beside the partial database, and the next wake
// proves the seed gone before the database can count as warm. A seed that
// succeeds (flag off: no proof of its own) removes it at once.
func TestASeedRecordsItsObligationBeforeItRuns(t *testing.T) {
	const host = "write-ahead.example.com"
	b, _ := delegatedBackend(t, host)
	snap := filepath.Join(t.TempDir(), "obligation-at-run")
	stageSeedScript(t, b, host, "v1", fmt.Sprintf(
		"cp \"$3%s\" %q || exit 7\nprintf seeded > \"$3\"\nexit 0\n", seedUnprovenSuffix, snap))
	swapRestoreReplica(t, func(context.Context, string, string, string, string) (bool, error) {
		return false, nil
	})
	rt := registry.Route{Host: host, Kind: "instance", Seed: `["seed"]`}
	db := filepath.Join(b.DataDir, host+".db")
	ob := db + seedUnprovenSuffix

	if err := b.Seed(context.Background(), rt); err != nil {
		t.Fatalf("Seed = %v; want success, with the obligation there while it ran", err)
	}
	if _, err := os.Stat(ob); !os.IsNotExist(err) {
		t.Fatalf("obligation still there (%v) after a successful flag-off seed", err)
	}

	// The agent died mid-seed: what is on disk is what the seed saw while
	// it ran, the obligation and a partial database.
	raw, err := os.ReadFile(snap)
	if err != nil {
		t.Fatal(err)
	}
	if err := os.WriteFile(ob, raw, 0o600); err != nil {
		t.Fatal(err)
	}
	if err := os.WriteFile(db, []byte("half"), 0o600); err != nil {
		t.Fatal(err)
	}
	b.cgroupKill = func(context.Context, registry.Route, string) error {
		return errors.New("the host cgroup still holds a process")
	}
	if fresh, err := b.Restore(context.Background(), rt); !errors.Is(err, ErrSeedUnproven) || fresh {
		t.Fatalf("Restore = %v, %v; want a refusal while the dead agent's seed is unproven", fresh, err)
	}
	if got, _ := os.ReadFile(db); string(got) != "half" {
		t.Fatalf("database = %q after a refused wake, want it untouched", got)
	}
	b.cgroupKill = func(context.Context, registry.Route, string) error { return nil }
	if fresh, err := b.Restore(context.Background(), rt); err != nil || !fresh {
		t.Fatalf("Restore = %v, %v; want fresh once the proof holds", fresh, err)
	}
	for _, p := range []string{db, ob} {
		if _, err := os.Stat(p); !os.IsNotExist(err) {
			t.Fatalf("%s still there (%v) after the proof held", filepath.Base(p), err)
		}
	}
}

// Finding 1: an obligation that cannot be written (a full disk) refuses the
// seed before anything runs. Nothing ran, so this is an ordinary failure
// (not ErrSeedUnproven), and the wake may release its lease.
func TestASeedWhoseObligationCannotBeWrittenRunsNothing(t *testing.T) {
	const host = "no-room.example.com"
	b, _ := delegatedBackend(t, host)
	runs := filepath.Join(t.TempDir(), "runs")
	stageSeedScript(t, b, host, "v1", fmt.Sprintf("echo run >> %q\nprintf half > \"$3\"\nexit 0\n", runs))
	orig := writeSeedObligationFile
	t.Cleanup(func() { writeSeedObligationFile = orig })
	writeSeedObligationFile = func(string, []byte, os.FileMode) error { return syscall.ENOSPC }

	err := b.Seed(context.Background(), registry.Route{Host: host, Kind: "instance", Seed: `["seed"]`})
	if err == nil || errors.Is(err, ErrSeedUnproven) || !errors.Is(err, syscall.ENOSPC) {
		t.Fatalf("Seed = %v, want a refusal naming the failed write, not ErrSeedUnproven", err)
	}
	if _, err := os.Stat(runs); !os.IsNotExist(err) {
		t.Fatalf("the seed ran (%v) with no obligation recorded", err)
	}
	if _, err := os.Stat(filepath.Join(b.DataDir, host+".db")); !os.IsNotExist(err) {
		t.Fatalf("a database is there (%v) though no seed ran", err)
	}
}

// Findings 1 and 7: a failed seed proven gone has its database removed with
// every error checked, and the obligation goes only after. A database file
// that cannot be removed keeps the obligation (the next wake proves and
// removes again) and fails the wake as an ordinary failure: the seed's
// processes are proven gone, so the lease may go.
func TestAProvenFailedSeedWhoseDBCannotBeRemovedKeepsItsObligation(t *testing.T) {
	const host = "stuck-wal.example.com"
	b, _ := delegatedBackend(t, host)
	stageSeedScript(t, b, host, "v1", "printf half > \"$3\"\nmkdir -p \"$3-wal/x\"\nexit 1\n")
	b.cgroupKill = func(context.Context, registry.Route, string) error { return nil }
	db := filepath.Join(b.DataDir, host+".db")

	err := b.Seed(context.Background(), registry.Route{Host: host, Kind: "instance", Seed: `["seed"]`})
	if err == nil || errors.Is(err, ErrSeedUnproven) {
		t.Fatalf("Seed = %v, want the seed's own failure", err)
	}
	if !strings.Contains(err.Error(), "-wal") {
		t.Fatalf("Seed = %v, want it to name the file that could not be removed", err)
	}
	if _, err := os.Stat(db + seedUnprovenSuffix); err != nil {
		t.Fatalf("obligation gone (%v) though the database was not removed", err)
	}
}

// Finding 6: a database that was there before the seed is never the
// obligation's to remove, but it is still refused as warm while the seed
// beside it is unproven. Once the proof holds, the obligation goes and the
// database stays.
func TestAPreexistingDBBesideAnUnprovenSeedIsRefusedThenKept(t *testing.T) {
	const host = "preexisting.example.com"
	b, _ := delegatedBackend(t, host)
	stageSeedScript(t, b, host, "v1", "exit 1\n")
	db := filepath.Join(b.DataDir, host+".db")
	if err := os.MkdirAll(filepath.Dir(db), 0o755); err != nil {
		t.Fatal(err)
	}
	if err := os.WriteFile(db, []byte("data"), 0o600); err != nil {
		t.Fatal(err)
	}
	rt := registry.Route{Host: host, Kind: "instance", Seed: `["seed"]`}

	if err := b.Seed(context.Background(), rt); !errors.Is(err, ErrSeedUnproven) {
		t.Fatalf("Seed = %v, want ErrSeedUnproven", err)
	}
	if _, err := os.Stat(db + seedUnprovenSuffix); err != nil {
		t.Fatalf("no obligation beside a pre-existing database (%v): the next wake would serve it warm", err)
	}
	if fresh, err := b.Restore(context.Background(), rt); !errors.Is(err, ErrSeedUnproven) || fresh {
		t.Fatalf("Restore = %v, %v; want a refusal while the seed is unproven", fresh, err)
	}
	b.cgroupKill = func(context.Context, registry.Route, string) error { return nil }
	if fresh, err := b.Restore(context.Background(), rt); err != nil || fresh {
		t.Fatalf("Restore = %v, %v; want the warm database once the proof holds", fresh, err)
	}
	if got, err := os.ReadFile(db); err != nil || string(got) != "data" {
		t.Fatalf("database = %q, %v; want the pre-existing one kept", got, err)
	}
	if _, err := os.Stat(db + seedUnprovenSuffix); !os.IsNotExist(err) {
		t.Fatalf("obligation still there (%v) after the proof held", err)
	}
}

// Finding 5, on the seed's own proof: a kernel with no cgroup.kill can kill
// nothing, so the leader's exit is the only proof there is, as with no
// delegation. Read as unproven instead, an obligation could never be
// discharged on such a box and the row would stay down for good.
func TestASeedProofWithNoCgroupKillIsTheLeaderOnlyProof(t *testing.T) {
	const host = "old-kernel-seed.example.com"
	b, _ := delegatedBackend(t, host)
	b.cgroupKill = func(context.Context, registry.Route, string) error {
		return fmt.Errorf("%w: open cgroup.kill: no such file or directory", cgroup.ErrNoKill)
	}
	if err := b.seedDescendantsGone(context.Background(), registry.Route{Host: host}, ""); err != nil {
		t.Fatalf("seedDescendantsGone = %v, want the leader-only proof (nil)", err)
	}
}

// Flag on, a failed seed's role child is proven empty before the database
// it built is removed.
func TestAFlagOnFailedSeedProvesItsRoleChildBeforeRemovingTheDB(t *testing.T) {
	const host = "failed-role-seed.example.com"
	b, _ := delegatedBackend(t, host)
	cgroup.FakePlacementForTest(t)
	b.GenContainment = true
	stageSeedScript(t, b, host, "v1", "printf half > \"$3\"\nexit 1\n")
	db := filepath.Join(b.DataDir, host+".db")
	var asked []string
	b.cgroupKill = func(_ context.Context, _ registry.Route, child string) error {
		if _, err := os.Stat(db); err != nil {
			t.Errorf("the database was gone (%v) before the seed's processes were proven gone", err)
		}
		asked = append(asked, child)
		return nil
	}

	err := b.Seed(context.Background(), registry.Route{Host: host, Kind: "instance", Seed: `["seed"]`})
	if err == nil || errors.Is(err, ErrSeedUnproven) {
		t.Fatalf("Seed = %v, want the seed's own failure", err)
	}
	if len(asked) != 1 || asked[0] != cgroup.RoleChild("seed") {
		t.Fatalf("proved %q, want the seed's role child once", asked)
	}
	for _, p := range []string{db, db + seedUnprovenSuffix} {
		if _, err := os.Stat(p); !os.IsNotExist(err) {
			t.Fatalf("%s still there (%v) after a proven failed seed", filepath.Base(p), err)
		}
	}
}

// Fix wave 1, item 2(a) (Astra final review 1, finding 2): a successful
// flag-off seed has no proof of its own (a helper it left stays in the host
// cgroup), so when its obligation cannot be discharged the wake fails with
// that helper's termination unknown. The proof is tried then: here the
// host cgroup still holds a process, so the seed answers ErrSeedUnproven and
// the lease is left to its TTL. The seed below swaps its obligation for a
// non-empty directory, which the discharge cannot remove.
func TestAFlagOffSeedWhoseDischargeFailsIsUnprovenWhileItsHelperRemains(t *testing.T) {
	const host = "discharge-fails.example.com"
	b, _ := delegatedBackend(t, host)
	stageSeedScript(t, b, host, "v1", fmt.Sprintf(
		"rm -f \"$3%[1]s\" && mkdir -p \"$3%[1]s/x\"\nprintf seeded > \"$3\"\nexit 0\n", seedUnprovenSuffix))
	var asked []string
	b.cgroupKill = func(_ context.Context, _ registry.Route, child string) error {
		asked = append(asked, child)
		return errors.New("the host cgroup still holds a process")
	}

	err := b.Seed(context.Background(), registry.Route{Host: host, Kind: "instance", Seed: `["seed"]`})
	if !errors.Is(err, ErrSeedUnproven) {
		t.Fatalf("Seed = %v, want ErrSeedUnproven: the discharge failed and the seed's helper is not proven gone", err)
	}
	if len(asked) != 1 || asked[0] != "" {
		t.Fatalf("proof asked of %q, want the host cgroup once", asked)
	}
}

// Fix wave 1, item 2(a), the other side: the discharge fails but the proof,
// tried then, holds. Nothing of the seed is left, so the failure is an
// ordinary one and the lease may go; the obligation stays for the next
// wake.
func TestAFlagOffSeedWhoseDischargeFailsAfterAProofIsAnOrdinaryFailure(t *testing.T) {
	const host = "discharge-fails-proven.example.com"
	b, _ := delegatedBackend(t, host)
	stageSeedScript(t, b, host, "v1", fmt.Sprintf(
		"rm -f \"$3%[1]s\" && mkdir -p \"$3%[1]s/x\"\nprintf seeded > \"$3\"\nexit 0\n", seedUnprovenSuffix))
	b.cgroupKill = func(context.Context, registry.Route, string) error { return nil }

	err := b.Seed(context.Background(), registry.Route{Host: host, Kind: "instance", Seed: `["seed"]`})
	if err == nil || errors.Is(err, ErrSeedUnproven) {
		t.Fatalf("Seed = %v, want an ordinary failure: the proof held", err)
	}
	if _, err := os.Stat(filepath.Join(b.DataDir, host+".db"+seedUnprovenSuffix)); err != nil {
		t.Fatalf("the obligation is gone (%v); it must stay for the next wake", err)
	}
}

// Fix wave 1, item 2(b): an obligation that is there but cannot be read
// says nothing of whether its seed's processes are gone, so Restore refuses
// with ErrSeedUnproven (the lease is left to its TTL), never an ordinary
// error that releases it. The host cgroup here still holds a process.
func TestAnUnreadableObligationIsUnproven(t *testing.T) {
	if os.Geteuid() == 0 {
		t.Skip("root reads a mode-000 file")
	}
	const host = "unreadable-obligation.example.com"
	b, _ := delegatedBackend(t, host)
	b.cgroupKill = func(context.Context, registry.Route, string) error {
		return errors.New("the host cgroup still holds a process")
	}
	db := filepath.Join(b.DataDir, host+".db")
	if err := os.MkdirAll(filepath.Dir(db), 0o755); err != nil {
		t.Fatal(err)
	}
	if err := os.WriteFile(db, []byte("half"), 0o600); err != nil {
		t.Fatal(err)
	}
	ob := db + seedUnprovenSuffix
	if err := os.WriteFile(ob, []byte(seedObligationHeader+"\n"+seedObligationOwned+"\n"+seedObligationFlat+"\n"), 0o000); err != nil {
		t.Fatal(err)
	}
	t.Cleanup(func() { _ = os.Chmod(ob, 0o600) })

	fresh, err := b.Restore(context.Background(), registry.Route{Host: host, Kind: "instance", Seed: `["seed"]`})
	if !errors.Is(err, ErrSeedUnproven) || fresh {
		t.Fatalf("Restore = %v, %v; want a refusal with ErrSeedUnproven", fresh, err)
	}
	if got, _ := os.ReadFile(db); string(got) != "half" {
		t.Fatalf("database = %q after a refused wake, want it untouched", got)
	}
}