package activator

import (
	"context"
	"errors"
	"os"
	"path/filepath"
	"slices"
	"strings"
	"sync"
	"testing"
	"time"

	"github.com/carlosframework/platform/internal/cgroup"
	"github.com/carlosframework/platform/internal/provision"
	"github.com/carlosframework/platform/internal/registry"
)

// sidecarStartDoneOf is the host's supervised-start join channel, nil when
// none is in flight.
func (a *Service) sidecarStartDoneOf(host string) chan struct{} {
	a.mu.Lock()
	defer a.mu.Unlock()
	if e := a.entries[host]; e != nil {
		return e.sidecarStartDone
	}
	return nil
}

// sidecarProc is the i'th sidecar the fake backend started.
func (g *fakeSwapBackend) sidecarProc(i int) *fakeProc {
	g.mu.Lock()
	defer g.mu.Unlock()
	if i >= len(g.sidecars) {
		return nil
	}
	return g.sidecars[i]
}

// sidecarStoppingIn reports whether the i'th sidecar's Stop has been
// entered: with a stop gate on it, that Stop is in flight and blocked.
func (g *fakeSwapBackend) sidecarStoppingIn(i int) bool {
	p := g.sidecarProc(i)
	return p != nil && p.stopEntered() > 0
}

// generationSwitched is true once the entry's current generation is not the
// fixture's incumbent.
func (f *swapFixture) generationSwitched() bool {
	g, _ := f.a.genStateForTest(f.host)
	return g.N != f.incumbent.N
}

// Flag off, on a backend that cannot pin an executable (the fixture never
// opts into the identity capability, as a unit row's backend has none): a
// generation with no staged file starts its legacy sidecar (the wake tail
// and the supervisor) through StartSidecar and the live link, as before.
// A generation that names its file runs it whatever the flag (ruling
// R-EXEC-SIDECAR; TestLegacySidecarOwnershipCheckRunsWithSwapOff).
func TestLegacySidecarUsesTheLiveLinkWithSwapOff(t *testing.T) {
	t.Parallel()
	f := newSwapFixture(t)
	f.a.gens = false
	f.b.sidecarStaged = true
	f.a.Bounce(f.host)
	waitFor(t, "the stop", func() bool { return f.a.entryCount() == 0 })
	arrangeAwake(t, f.a, testRoute(f.host))
	if got := f.b.sidecarStartCount(); got != 1 {
		t.Fatalf("StartSidecar calls = %d after the wake, want 1", got)
	}
	f.b.lastSidecar().kill()
	f.clk.advance(3 * time.Second)
	f.a.sweepOnce()
	waitFor(t, "the supervised restart", func() bool { return f.a.sidecarSettled(f.host) && f.b.sidecarStartCount() == 2 })
	if paths := f.b.sidecarGenPaths(); len(paths) != 0 {
		t.Fatalf("StartSidecarGen ran with swaps off: %v", paths)
	}
}

// Flag off, ownership: startSidecar asks ownedLocked whatever the flag (an
// accepted dormancy exception, recorded in task-7-adapt.md). Exactly what
// changes against today, called directly so no earlier sweep check masks it:
// a host that does not own itself (fenced, shutting down, lease lost, no
// longer the host's entry) gets NO co-spawned sidecar, where today's
// StartSidecar would have spawned one. A host that owns itself gets its
// sidecar on the generation's own staged file, with the flag off too
// (ruling R-EXEC-SIDECAR, settings Task 5c), never the live link.
func TestLegacySidecarOwnershipCheckRunsWithSwapOff(t *testing.T) {
	f := newSwapFixture(t)
	f.a.gens = false
	f.b.sidecarStaged = true
	f.a.mu.Lock()
	e := f.a.entries[f.host]
	rt := e.route
	f.a.mu.Unlock()
	rep := f.b.lastRep()

	// Owned: the generation's recorded file, by path.
	sc, err := f.a.startSidecar(context.Background(), e, rt, f.incumbent, false)
	if err != nil || sc == nil {
		t.Fatalf("an owned host's sidecar = %v, %v; want the generation's spawn", sc, err)
	}
	if paths := f.b.sidecarGenPaths(); len(paths) != 1 || paths[0] != f.incumbent.StagedPath {
		t.Fatalf("StartSidecarGen paths = %v, want the generation's %q", paths, f.incumbent.StagedPath)
	}
	if got := f.b.sidecarStartCount(); got != 0 {
		t.Fatalf("StartSidecar (live link) calls = %d, want none", got)
	}

	refused := func(name string, arm, disarm func(), want error) {
		t.Helper()
		arm()
		sc, err := f.a.startSidecar(context.Background(), e, rt, f.incumbent, false)
		disarm()
		if !errors.Is(err, want) || sc != nil {
			t.Fatalf("%s: startSidecar = %v, %v; want %v and no process", name, sc, err, want)
		}
		if got := len(f.b.sidecarGenPaths()); got != 1 || f.b.sidecarStartCount() != 0 {
			t.Fatalf("%s: sidecar spawns = %d (live link %d), want 1 (none beside a host that does not own itself)", name, got, f.b.sidecarStartCount())
		}
	}
	refused("fenced",
		func() { f.a.mu.Lock(); f.a.fences = map[string]uint64{f.host: 1}; f.a.mu.Unlock() },
		func() { f.a.mu.Lock(); f.a.fences = nil; f.a.mu.Unlock() }, errSwapFenced)
	refused("shutting down",
		func() { f.a.mu.Lock(); f.a.shuttingDown = true; f.a.mu.Unlock() },
		func() { f.a.mu.Lock(); f.a.shuttingDown = false; f.a.mu.Unlock() }, errSwapShutdown)
	refused("lease lost",
		func() { rep.setErr(errors.New("lease lost")) },
		func() { rep.setErr(nil) }, errSwapLeaseLost)
	refused("not the host's entry",
		func() { f.a.mu.Lock(); delete(f.a.entries, f.host); f.a.mu.Unlock() },
		func() { f.a.mu.Lock(); f.a.entries[f.host] = e; f.a.mu.Unlock() }, errSwapNotAwake)

	// And once owned again, the same call spawns as before.
	if _, err := f.a.startSidecar(context.Background(), e, rt, f.incumbent, false); err != nil {
		t.Fatalf("owned again: %v", err)
	}
	if got := len(f.b.sidecarGenPaths()); got != 2 {
		t.Fatalf("StartSidecarGen calls = %d, want 2", got)
	}
}

// Flag on: late attachment, the supervisor, a swap's replacement and the
// wake tail all exec the CURRENT generation's captured staged path (spec
// §7.5). The live link names v3 throughout and is never used.
func TestLegacySidecarRunsTheServingGeneration(t *testing.T) {
	t.Parallel()
	t.Run("late attachment and the supervisor", func(t *testing.T) {
		f := newSwapFixture(t)
		f.b.execVerFn = func() string { return "v3" }
		f.b.sidecarStaged = true // staged after the wake: the sweep attaches it
		f.a.sweepOnce()
		waitFor(t, "late attachment", func() bool { return f.a.sidecarSettled(f.host) && len(f.b.sidecarGenPaths()) == 1 })
		f.b.lastSidecar().kill()
		f.clk.advance(3 * time.Second)
		f.a.sweepOnce()
		waitFor(t, "the supervised restart", func() bool { return f.a.sidecarSettled(f.host) && len(f.b.sidecarGenPaths()) == 2 })
		for _, p := range f.b.sidecarGenPaths() {
			if p != f.incumbent.StagedPath {
				t.Fatalf("a sidecar ran %q, want the serving generation's %q", p, f.incumbent.StagedPath)
			}
		}
		if got := f.b.sidecarStartCount(); got != 0 {
			t.Fatalf("StartSidecar (live link) ran %d time(s) with swaps on", got)
		}
	})
	t.Run("a swap replaces the sidecar stop-then-start", func(t *testing.T) {
		f := newSwapFixture(t)
		f.b.sidecarStaged = true
		f.a.sweepOnce()
		waitFor(t, "late attachment", func() bool { return f.a.sidecarSettled(f.host) && len(f.b.sidecarGenPaths()) == 1 })
		old := f.b.lastSidecar()
		if res := f.a.Swap(context.Background(), f.attempt(ModeOverlap), f.newest); res.Outcome != SwapDone {
			t.Fatalf("Swap = %+v", res)
		}
		paths := f.b.sidecarGenPaths()
		if len(paths) != 2 || paths[1] != f.staged {
			t.Fatalf("sidecar spawns %v: the replacement must run the new generation's %q", paths, f.staged)
		}
		if !old.isStopped() {
			t.Fatal("the old sidecar outlived the replacement")
		}
		ev := f.b.events()
		if i, j := slices.Index(ev, "stop:sidecar"), slices.Index(ev, "startSidecarGen:"+f.staged); i < 0 || j < 0 || i > j {
			t.Fatalf("order %v: stop the old sidecar, then start the new one", ev)
		}
	})
	t.Run("the wake tail of a hold", func(t *testing.T) {
		f := newSwapFixture(t)
		f.b.sidecarStaged = true
		if res := f.a.Swap(context.Background(), f.attempt(ModeHold), f.newest); res.Outcome != SwapDone {
			t.Fatalf("Swap = %+v", res)
		}
		if paths := f.b.sidecarGenPaths(); len(paths) == 0 || paths[len(paths)-1] != f.staged {
			t.Fatalf("the hold's wake tail started sidecars %v, want %q last", paths, f.staged)
		}
	})
}

// The swap's replacement holds the supervisor's interlock, so a supervised
// restart that falls due meanwhile cannot start a second sidecar (review
// focus 4).
func TestSwapSidecarReplacementHoldsTheSupervisorOff(t *testing.T) {
	t.Parallel()
	f := newSwapFixture(t)
	f.b.sidecarStaged = true
	f.a.sweepOnce()
	waitFor(t, "late attachment", func() bool { return f.a.sidecarSettled(f.host) && len(f.b.sidecarGenPaths()) == 1 })
	gate := make(chan struct{})
	f.b.gmu.Lock()
	f.b.sidecarGenGate = gate
	f.b.gmu.Unlock()
	results := make(chan SwapResult, 1)
	go func() { results <- f.a.Swap(context.Background(), f.attempt(ModeOverlap), f.newest) }()
	waitFor(t, "the replacement's start", func() bool { return len(f.b.sidecarGenPaths()) == 2 })
	held := f.a.sidecarStartDoneOf(f.host)
	if held == nil {
		t.Fatal("the replacement holds no interlock")
	}
	f.clk.advance(time.Minute)
	f.a.sweepOnce() // the old sidecar is stopped (exited): the supervisor would restart it
	// The supervisor claims sidecarStarting under the sweep's own lock, so a
	// claim would have replaced the join channel before sweepOnce returned.
	if now := f.a.sidecarStartDoneOf(f.host); now != held {
		t.Fatal("a supervised restart claimed the interlock during the replacement")
	}
	if n := len(f.b.sidecarGenPaths()); n != 2 {
		t.Fatalf("sidecar spawns = %d during the replacement, want 2", n)
	}
	close(gate)
	if res := <-results; res.Outcome != SwapDone {
		t.Fatalf("Swap = %+v", res)
	}
}

// A supervised restart already in flight when the swap switches captured the
// OLD generation. The swap's replacement joins it and then replaces what it
// installed, so the new server never runs beside the old build's sidecar
// (spec §7.5). Returning early because sidecarStarting is set (the old
// replaceSidecar) leaves the supervisor's old-build sidecar installed with
// no replacement pending: the third spawn never comes.
func TestSwapSidecarReplacementJoinsAnInFlightSupervisor(t *testing.T) {
	t.Parallel()
	f := newSwapFixture(t)
	f.b.sidecarStaged = true
	f.a.sweepOnce()
	waitFor(t, "late attachment", func() bool { return f.a.sidecarSettled(f.host) && len(f.b.sidecarGenPaths()) == 1 })
	gate := make(chan struct{})
	f.b.gmu.Lock()
	f.b.sidecarGenGate = gate
	f.b.gmu.Unlock()
	f.b.lastSidecar().kill()
	f.clk.advance(3 * time.Second)
	f.a.sweepOnce() // the supervisor starts a sidecar for the incumbent and blocks
	waitFor(t, "the supervisor's start", func() bool { return len(f.b.sidecarGenPaths()) == 2 })
	results := make(chan SwapResult, 1)
	go func() { results <- f.a.Swap(context.Background(), f.attempt(ModeOverlap), f.newest) }()
	waitFor(t, "the switch", f.generationSwitched)
	close(gate)
	if res := <-results; res.Outcome != SwapDone {
		t.Fatalf("Swap = %+v", res)
	}
	waitFor(t, "the replacement onto the new generation", func() bool {
		p := f.b.sidecarGenPaths()
		return len(p) == 3 && f.a.sidecarSettled(f.host)
	})
	paths := f.b.sidecarGenPaths()
	if paths[1] != f.incumbent.StagedPath || paths[2] != f.staged {
		t.Fatalf("sidecar spawns %v: want the supervisor's on %q, then the replacement on %q", paths, f.incumbent.StagedPath, f.staged)
	}
	// The supervisor's sidecar is stopped by the replacement's
	// stop-then-start.
	f.b.mu.Lock()
	supervised, current := f.b.sidecars[1], f.b.sidecars[len(f.b.sidecars)-1]
	f.b.mu.Unlock()
	if !supervised.isStopped() {
		t.Fatal("the supervisor's old-build sidecar outlived the swap")
	}
	f.a.mu.Lock()
	installed := f.a.entries[f.host].sidecar
	f.a.mu.Unlock()
	if installed != Proc(current) {
		t.Fatal("the installed sidecar is not the replacement")
	}
}

// The supervisor's start for the retired generation is joined until that
// process is proven gone (spec §7.4, §7.5): the replacement never starts
// while it still runs. Discarding it asynchronously let the supervisor
// close its join at once, so the replacement started beside a sidecar
// still draining on the old build.
func TestSwapSidecarNeverRunsBesideTheRetiredGenerationsSidecar(t *testing.T) {
	t.Parallel()
	f := newSwapFixture(t)
	f.b.sidecarStaged = true
	f.a.sweepOnce()
	waitFor(t, "late attachment", func() bool { return f.a.sidecarSettled(f.host) && len(f.b.sidecarGenPaths()) == 1 })
	startGate, stopGate := make(chan struct{}), make(chan struct{})
	f.b.gmu.Lock()
	f.b.sidecarGenGate, f.b.sidecarStopGate = startGate, stopGate
	f.b.gmu.Unlock()
	f.b.lastSidecar().kill()
	f.clk.advance(3 * time.Second)
	f.a.sweepOnce() // the supervisor starts the incumbent's sidecar and blocks
	waitFor(t, "the supervisor's start", func() bool { return len(f.b.sidecarGenPaths()) == 2 })
	results := make(chan SwapResult, 1)
	go func() { results <- f.a.Swap(context.Background(), f.attempt(ModeOverlap), f.newest) }()
	waitFor(t, "the switch", f.generationSwitched)
	close(startGate) // the supervisor's sidecar starts; its Stop blocks on stopGate
	// The retired generation's sidecar is being stopped, and the replacement
	// is sequenced behind that Stop: nothing can have started beside it.
	waitFor(t, "the retired sidecar's stop to be in flight", func() bool { return f.b.sidecarStoppingIn(1) })
	if n := len(f.b.sidecarGenPaths()); n != 2 {
		t.Fatalf("%d sidecar starts: the replacement started while the retired generation's sidecar still ran", n)
	}
	select {
	case res := <-results:
		t.Fatalf("the swap returned (%+v) before the retired sidecar was proven gone", res)
	default:
	}
	close(stopGate)
	if res := <-results; res.Outcome != SwapDone {
		t.Fatalf("Swap = %+v", res)
	}
	waitFor(t, "the replacement", func() bool { return len(f.b.sidecarGenPaths()) == 3 })
	if p := f.b.sidecarGenPaths(); p[2] != f.staged {
		t.Fatalf("sidecar starts %v: want the new generation's last", p)
	}
}

// A terminal stop that begins while a supervised sidecar start is in flight
// joins that start, and the sidecar it produced, before the lease is
// released (spec §7.4). Discarding it asynchronously let the supervisor
// close its join at once, so the stop released the lease while the new
// sidecar was still running.
func TestSwapStopJoinsTheSidecarAStartLeftBehind(t *testing.T) {
	t.Parallel()
	f := newSwapFixture(t)
	f.b.sidecarStaged = true
	f.a.sweepOnce()
	waitFor(t, "late attachment", func() bool { return f.a.sidecarSettled(f.host) && len(f.b.sidecarGenPaths()) == 1 })
	startGate, stopGate := make(chan struct{}), make(chan struct{})
	f.b.gmu.Lock()
	f.b.sidecarGenGate, f.b.sidecarStopGate = startGate, stopGate
	f.b.gmu.Unlock()
	f.b.lastSidecar().kill()
	f.clk.advance(3 * time.Second)
	f.a.sweepOnce() // the supervisor's start blocks
	waitFor(t, "the supervisor's start", func() bool { return len(f.b.sidecarGenPaths()) == 2 })
	rep := f.b.lastRep()
	joining := make(chan struct{})
	var joinOnce sync.Once
	f.a.mu.Lock()
	f.a.joinWaiting = func(_, what string) {
		if what == "sidecar start" {
			joinOnce.Do(func() { close(joining) })
		}
	}
	f.a.mu.Unlock()
	f.a.Bounce(f.host)
	// The stop is joined on the supervisor's start: it goes no further (no
	// instance stop, so no lease release) while the start is in flight. Wait
	// for the stop to reach that join (the join seam's positive signal), or,
	// if something skipped it, for the instance's stop.
	waitFor(t, "the stop at its sidecar-start join", func() bool {
		select {
		case <-joining:
			return true
		default:
		}
		return slices.Contains(f.b.events(), "stop:instance")
	})
	if slices.Contains(f.b.events(), "stop:instance") || rep.isClosed() {
		t.Fatal("the stop went past its join on the in-flight sidecar start")
	}
	close(startGate) // the start lands on a stopping entry; its sidecar's Stop blocks
	waitFor(t, "the left-behind sidecar's stop to be in flight", func() bool { return f.b.sidecarStoppingIn(1) })
	if rep.isClosed() {
		t.Fatal("the lease was released while the sidecar a start left behind still ran")
	}
	close(stopGate)
	waitFor(t, "the lease release", rep.isClosed)
	if started := f.b.sidecarProc(1); !started.isStopped() {
		t.Fatal("the sidecar the start left behind was never stopped")
	}
}

// Spec §7.4: a discarded sidecar whose stop is unproven is never forgotten.
// It stays tracked, the host takes no new wake beside it, and the sweep
// stops it again until it exits. Logging it and moving on let the next wake
// run a second sidecar beside a live one.
func TestAStraySidecarIsNeverForgotten(t *testing.T) {
	t.Parallel()
	f := newSwapFixture(t)
	f.a.Bounce(f.host)
	waitFor(t, "the instance asleep", func() bool { return f.a.entryCount() == 0 })
	stray := &fakeProc{label: "sidecar", rec: f.b.fakeBackend, stuck: true}
	f.a.discardSidecar(testRoute(f.host), stray)
	if got := f.a.strayCount(f.host); got != 1 {
		t.Fatalf("tracked strays = %d, want 1", got)
	}
	if _, err := f.a.beginRequest(context.Background(), testRoute(f.host)); err == nil || !strings.Contains(err.Error(), errWakeUnproven.Error()) {
		t.Fatalf("a wake beside an unproven sidecar = %v, want refused", err)
	}
	stray.exit()
	f.a.sweepOnce() // the reaper re-proves it off the lock
	waitFor(t, "the stray proven gone", func() bool { return f.a.strayCount(f.host) == 0 })
	release, err := f.a.beginRequest(context.Background(), testRoute(f.host))
	if err != nil {
		t.Fatalf("the wake once the stray exited: %v", err)
	}
	release()
}

func (a *Service) strayCount(host string) int {
	a.mu.Lock()
	defer a.mu.Unlock()
	return len(a.straySidecars[host])
}

// strayReaping is true while any stray of host has a re-proof in flight.
func (a *Service) strayReaping(host string) bool {
	a.mu.Lock()
	defer a.mu.Unlock()
	for _, s := range a.straySidecars[host] {
		if s.reaping {
			return true
		}
	}
	return false
}

func (a *Service) stopPending(host string) bool {
	a.mu.Lock()
	defer a.mu.Unlock()
	e := a.entries[host]
	return e != nil && e.pendingStop
}

// Spec §7.4, §7.5: a legacy sidecar is gone only when its cgroup is empty.
// childProc.Stop signals only the leader, so a process the sidecar forked
// outlives its exit in role-sidecar. Here the leader exits but a descendant
// stays (KillSidecar cannot empty the cgroup): the replacement does not
// start beside it, and a terminal stop keeps the lease until it is proven
// gone. Accepting Exited() started the new sidecar beside the descendant
// and released the lease under it.
func TestASidecarsDescendantBlocksItsReplacementAndTheLease(t *testing.T) {
	t.Parallel()
	f := newSwapFixture(t)
	f.b.sidecarStaged = true
	f.a.sweepOnce()
	waitFor(t, "late attachment", func() bool { return f.a.sidecarSettled(f.host) && len(f.b.sidecarGenPaths()) == 1 })
	f.b.setSidecarDescendant(true)
	if res := f.a.Swap(context.Background(), f.attempt(ModeOverlap), f.newest); res.Outcome != SwapDone {
		t.Fatalf("Swap = %+v", res)
	}
	if got := f.a.strayCount(f.host); got != 1 {
		t.Fatalf("tracked strays = %d after the replacement, want 1 (the old sidecar's descendant)", got)
	}
	// A supervisor pass that falls due must not start one either. It runs
	// startSidecar, which refuses while the stray is tracked.
	f.clk.advance(time.Minute)
	f.a.sweepOnce()
	waitFor(t, "the supervisor's refused start", func() bool { return f.a.sidecarSettled(f.host) })
	if n := len(f.b.sidecarGenPaths()); n != 1 {
		t.Fatalf("%d sidecar starts: a new sidecar started beside the old one's descendant", n)
	}
	rep := f.b.lastRep()
	f.a.Bounce(f.host)
	waitFor(t, "the stop to wait on the sidecar's cgroup", func() bool { return f.a.stopPending(f.host) })
	// Each sweep re-proves the stray; the descendant is still there.
	for range 2 {
		before := f.b.sidecarKillCount()
		f.a.sweepOnce()
		waitFor(t, "a proof attempt", func() bool { return f.b.sidecarKillCount() > before })
		waitFor(t, "the re-proof to finish", func() bool { return !f.a.strayReaping(f.host) })
		if rep.isClosed() {
			t.Fatal("the lease was released while the sidecar's descendant still ran")
		}
	}
	if got := f.a.strayCount(f.host); got != 1 {
		t.Fatalf("tracked strays = %d with the descendant alive, want 1", got)
	}
	f.b.setSidecarDescendant(false)
	waitFor(t, "the lease release once the cgroup is empty", func() bool {
		f.a.sweepOnce()
		return rep.isClosed()
	})
}

// A terminal stop's co-stop is held to the same proof: the sidecar's leader
// exits but a process it forked is left in the cgroup, so the stop keeps the
// lease (releasable) until the sweep proves the cgroup empty.
func TestATerminalStopKeepsTheLeaseOverASidecarsDescendant(t *testing.T) {
	t.Parallel()
	f := newSwapFixture(t)
	f.b.sidecarStaged = true
	f.a.sweepOnce()
	waitFor(t, "late attachment", func() bool { return f.a.sidecarSettled(f.host) && len(f.b.sidecarGenPaths()) == 1 })
	f.b.setSidecarDescendant(true)
	rep := f.b.lastRep()
	f.a.Bounce(f.host)
	waitFor(t, "the stop to wait on the sidecar's cgroup", func() bool { return f.a.stopPending(f.host) })
	if got := f.a.strayCount(f.host); got != 1 {
		t.Fatalf("tracked strays = %d after the co-stop, want 1", got)
	}
	before := f.b.sidecarKillCount()
	f.a.sweepOnce()
	waitFor(t, "a proof attempt", func() bool { return f.b.sidecarKillCount() > before })
	waitFor(t, "the re-proof to finish", func() bool { return !f.a.strayReaping(f.host) })
	if rep.isClosed() {
		t.Fatal("the lease was released while the sidecar's descendant still ran")
	}
	f.b.setSidecarDescendant(false)
	waitFor(t, "the lease release once the cgroup is empty", func() bool {
		f.a.sweepOnce()
		return rep.isClosed()
	})
}

// A supervised restart of a sidecar that exited on its own is a replacement
// too: if something it forked is left in the cgroup, the old one becomes a
// tracked stray and no new sidecar starts beside it.
func TestASupervisedRestartWaitsForTheCrashedSidecarsDescendant(t *testing.T) {
	t.Parallel()
	f := newSwapFixture(t)
	f.b.sidecarStaged = true
	f.a.sweepOnce()
	waitFor(t, "late attachment", func() bool { return f.a.sidecarSettled(f.host) && len(f.b.sidecarGenPaths()) == 1 })
	f.b.setSidecarDescendant(true)
	f.b.lastSidecar().kill()
	f.clk.advance(3 * time.Second)
	f.a.sweepOnce()
	waitFor(t, "the refused restart", func() bool { return f.a.sidecarSettled(f.host) && f.a.strayCount(f.host) == 1 })
	if n := len(f.b.sidecarGenPaths()); n != 1 {
		t.Fatalf("%d sidecar starts: a restart ran beside the crashed sidecar's descendant", n)
	}
	f.b.setSidecarDescendant(false)
	waitFor(t, "the restart once the cgroup is empty", func() bool {
		f.clk.advance(time.Minute)
		f.a.sweepOnce()
		return f.a.sidecarSettled(f.host) && len(f.b.sidecarGenPaths()) == 2
	})
	if got := f.a.strayCount(f.host); got != 0 {
		t.Fatalf("tracked strays = %d after the proof, want 0", got)
	}
}

// A legacy sidecar left on a retired generation's build (a replacement whose
// stop of it was unproven) is replaced by the sweep's retry.
func TestSweepReplacesASidecarLeftOnARetiredGeneration(t *testing.T) {
	t.Parallel()
	f := newSwapFixture(t)
	f.b.sidecarStaged = true
	f.a.sweepOnce()
	waitFor(t, "late attachment", func() bool { return f.a.sidecarSettled(f.host) && len(f.b.sidecarGenPaths()) == 1 })
	old := f.b.lastSidecar()
	f.a.mu.Lock()
	f.a.entries[f.host].sidecarGen = f.incumbent.N - 1 // a build the entry no longer serves
	f.a.mu.Unlock()
	f.a.sweepOnce()
	waitFor(t, "the replacement", func() bool { return f.a.sidecarSettled(f.host) && len(f.b.sidecarGenPaths()) == 2 })
	if !old.isStopped() {
		t.Fatal("the retired generation's sidecar was not stopped")
	}
	f.a.mu.Lock()
	got := f.a.entries[f.host].sidecarGen
	f.a.mu.Unlock()
	if got != f.incumbent.N {
		t.Fatalf("sidecarGen = %d, want the current generation %d", got, f.incumbent.N)
	}
}

// Under plan 2's containment the sidecar keeps its role-sidecar child: no
// limit of its own, and the generation parent keeps memory.oom.group=0 and
// holds no process (spec §7.3, plan 2 D4). Spawning through spawnInCgroup
// instead writes the tier and memory.oom.group=1 onto the parent and makes
// no role-sidecar child.
func TestStartSidecarGenKeepsTheRoleSidecarChild(t *testing.T) {
	const host = "sc.example.com"
	b, parent := delegatedBackend(t, host)
	cgroup.FakePlacementForTest(t) // a temp dir is no cgroupfs, and a generation is never moved by PID
	b.GenContainment = true
	v2 := filepath.Join(b.OptDir, "versions", "v2", "app")
	if err := os.MkdirAll(filepath.Dir(v2), 0o755); err != nil {
		t.Fatal(err)
	}
	// It stays up until the stop: its exit removes its own child.
	if err := os.WriteFile(v2, []byte("#!/bin/sh\nexec sleep 30\n"), 0o755); err != nil {
		t.Fatal(err)
	}
	if err := os.WriteFile(filepath.Join(b.DataDir, host+"-sidecar.env"), []byte("X=1\n"), 0o600); err != nil {
		t.Fatal(err)
	}
	rt := registry.Route{Host: host, Kind: "instance", DBPath: provision.DefaultDBPath(b.DataDir, host),
		Socket: filepath.Join(provision.SocketDir, host+".sock")}
	p, err := b.StartSidecarGen(context.Background(), rt, Gen{N: 3, Build: "v2", StagedPath: v2})
	if err != nil {
		t.Fatal(err)
	}
	// Checked before the stop: the exit's cleanup removes the empty child
	// (with placement before exec nothing is written into it to keep it).
	if fi, err := os.Stat(filepath.Join(parent, "role-sidecar")); err != nil || !fi.IsDir() {
		_ = p.Stop(context.Background())
		t.Fatalf("no role-sidecar child: %v", err)
	}
	if _, err := os.Stat(filepath.Join(parent, "role-sidecar", "memory.max")); !os.IsNotExist(err) {
		t.Errorf("the sidecar's child got a limit of its own: %v", err)
	}
	_ = p.Stop(context.Background())
	if got, err := os.ReadFile(filepath.Join(parent, "memory.oom.group")); err != nil || string(got) != "0" {
		t.Errorf("parent memory.oom.group = %q, %v; want 0", got, err)
	}
}

// ExecBackend: a flat sidecar execs the generation's staged path. A nested
// sidecar is refused until Task 9 hands root the staged executable: it never
// reads the live link to decide, because a repoint can land between that
// read and the start.
func TestExecBackendStartSidecarGen(t *testing.T) {
	t.Parallel()
	stage := func(t *testing.T, b *ExecBackend, host string) (v2, marker string) {
		t.Helper()
		marker = filepath.Join(t.TempDir(), "argv0")
		v2 = filepath.Join(b.OptDir, "versions", "v2", "app")
		if err := os.MkdirAll(filepath.Dir(v2), 0o755); err != nil {
			t.Fatal(err)
		}
		if err := os.WriteFile(v2, []byte("#!/bin/sh\nprintf '%s' \"$0\" > '"+marker+"'\n"), 0o755); err != nil {
			t.Fatal(err)
		}
		if err := os.WriteFile(filepath.Join(b.DataDir, host+"-sidecar.env"), []byte("X=1\n"), 0o600); err != nil {
			t.Fatal(err)
		}
		return v2, marker
	}
	t.Run("flat", func(t *testing.T) {
		b := testExecBackend(t)
		host := "sc.example.com"
		stageLive(t, b, host, "v1")
		v2, marker := stage(t, b, host)
		// The default db and socket: Classify puts this route in the flat layout.
		rt := registry.Route{Host: host, Kind: "instance", DBPath: provision.DefaultDBPath(b.DataDir, host),
			Socket: filepath.Join(provision.SocketDir, host+".sock")}
		if l, why := b.Layout(rt); l != LayoutFlat {
			t.Fatalf("route classified %v (%s), want the flat layout", l, why)
		}
		p, err := b.StartSidecarGen(context.Background(), rt, Gen{N: 3, Build: "v2", StagedPath: v2})
		if err != nil {
			t.Fatal(err)
		}
		defer p.Stop(context.Background()) //nolint:errcheck
		if got := waitForFile(t, marker, 2*time.Second); got != v2 {
			t.Fatalf("the sidecar execd %q, want the generation's %q", got, v2)
		}
	})
	t.Run("foreign", func(t *testing.T) {
		b := testExecBackend(t)
		host := "fsc.example.com"
		stageLive(t, b, host, "v1")
		v2, marker := stage(t, b, host)
		rt := registry.Route{Host: host, Kind: "instance", Socket: filepath.Join(t.TempDir(), "s.sock")}
		if l, _ := b.Layout(rt); l != LayoutForeign {
			t.Fatalf("route classified %v, want the foreign layout", l)
		}
		p, err := b.StartSidecarGen(context.Background(), rt, Gen{N: 3, Build: "v2", StagedPath: v2})
		if err != nil {
			t.Fatal(err)
		}
		defer p.Stop(context.Background()) //nolint:errcheck
		if got := waitForFile(t, marker, 2*time.Second); got != v2 {
			t.Fatalf("the sidecar execd %q, want the generation's %q", got, v2)
		}
	})
	t.Run("nested", func(t *testing.T) {
		b := testExecBackend(t)
		rd := t.TempDir()
		b.RunDirOverride = rd
		host := "nsc.example.com"
		stageLive(t, b, host, "v1")
		rt := registry.Route{Host: host, Kind: "instance", DBPath: provision.NestedDBPath(b.DataDir, host),
			Socket: filepath.Join(rd, "instances", provision.InstanceID(host), "app.sock")}
		for _, build := range []string{"v1", "v2"} { // the live build and another
			// A generation with no staged executable (woken on the live
			// link) has nothing to name to root, so it still refuses.
			if _, err := b.StartSidecarGen(context.Background(), rt, Gen{N: 3, Build: build}); !errors.Is(err, errNestedSidecarGen) {
				t.Fatalf("serving %s with no staged path: %v, want errNestedSidecarGen", build, err)
			}
		}
	})
}

// KillSidecar: the nested arm asks the sidecar unit's stop, and with no
// Control wired it answers an error, never a nil dereference. Flat without
// containment has no cgroup of its own to prove: errNoGenContainment.
func TestExecBackendKillSidecar(t *testing.T) {
	t.Run("nested without a control channel", func(t *testing.T) {
		b := testExecBackend(t)
		rd := t.TempDir()
		b.RunDirOverride = rd
		host := "nsc.example.com"
		rt := registry.Route{Host: host, Kind: "instance", DBPath: provision.NestedDBPath(b.DataDir, host),
			Socket: filepath.Join(rd, "instances", provision.InstanceID(host), "app.sock")}
		if err := b.KillSidecar(context.Background(), rt); err == nil {
			t.Fatal("KillSidecar with no Control proved a nested sidecar gone")
		}
	})
	t.Run("flat without containment", func(t *testing.T) {
		b := testExecBackend(t)
		host := "sc.example.com"
		rt := registry.Route{Host: host, Kind: "instance", DBPath: provision.DefaultDBPath(b.DataDir, host),
			Socket: filepath.Join(provision.SocketDir, host+".sock")}
		if err := b.KillSidecar(context.Background(), rt); !errors.Is(err, errNoGenContainment) {
			t.Fatalf("KillSidecar = %v, want errNoGenContainment", err)
		}
	})
}

var _ SidecarGenStarter = (*SplitBackend)(nil)