package main

// Unit tests for the activator state machine (activator.go) with a fake
// backend — wake coalescing, idle→sleep, lease-lost stand-down, contention
// 503s. No AWS, no processes: the live end-to-end test (the wake/restore/
// sleep money path against real S3 + a real spawned instance) is
// TestActivatorLiveHibernation in activator_live_test.go.

import (
	"context"
	"errors"
	"fmt"
	"io"
	"net"
	"net/http"
	"net/http/httptest"
	"os"
	"path/filepath"
	"sync"
	"sync/atomic"
	"testing"
	"time"

	"lchat/internal/bot"
)

// --- fakes -------------------------------------------------------------------

type fakeRep struct {
	mu     sync.Mutex
	err    error
	closed bool
}

func (r *fakeRep) Err() error {
	r.mu.Lock()
	defer r.mu.Unlock()
	return r.err
}
func (r *fakeRep) setErr(err error) {
	r.mu.Lock()
	defer r.mu.Unlock()
	r.err = err
}
func (r *fakeRep) Close(context.Context) error {
	r.mu.Lock()
	defer r.mu.Unlock()
	r.closed = true
	return nil
}
func (r *fakeRep) isClosed() bool {
	r.mu.Lock()
	defer r.mu.Unlock()
	return r.closed
}

type fakeProc struct {
	mu      sync.Mutex
	stopped bool
	dead    bool
	// stopGate, when non-nil, blocks stop() until closed — holds an entry in
	// the stopping state so a test can poke at it.
	stopGate chan struct{}
	// label + rec feed the backend's order log ("instance" / "sidecar"), so
	// the co-spawn/co-stop tests can assert interleavings.
	label string
	rec   *fakeBackend
}

func (p *fakeProc) stop(context.Context) error {
	p.mu.Lock()
	gate := p.stopGate
	p.mu.Unlock()
	if gate != nil {
		<-gate
	}
	p.mu.Lock()
	p.stopped = true
	p.mu.Unlock()
	if p.rec != nil {
		p.rec.record("stop:" + p.label)
	}
	return nil
}
func (p *fakeProc) exited() bool {
	p.mu.Lock()
	defer p.mu.Unlock()
	return p.dead
}
func (p *fakeProc) isStopped() bool {
	p.mu.Lock()
	defer p.mu.Unlock()
	return p.stopped
}

type fakeLease struct {
	rep *fakeRep
	b   *fakeBackend // records the sync interval the wake threads through
}

func (l *fakeLease) replicate(_ context.Context, _ string, syncInterval time.Duration) (actRep, error) {
	if l.b != nil {
		l.b.mu.Lock()
		l.b.syncIntervals = append(l.b.syncIntervals, syncInterval)
		l.b.mu.Unlock()
	}
	return l.rep, nil
}
func (l *fakeLease) release(context.Context) error { return nil }

type fakeBackend struct {
	mu         sync.Mutex
	acquires   int
	restores   int
	starts     int
	acquireErr error
	startGate  chan struct{} // when non-nil, start() blocks until closed
	reps       []*fakeRep
	procs      []*fakeProc
	// syncIntervals records, per wake, the replication-class tick doWake
	// threads into replicate() — the ladder's seam assertion.
	syncIntervals []time.Duration
	dueVal     int64 // what the instance reports at the sleep-time next-due probe
	dueErr     error // when non-nil, the probe fails (dead socket, missing token, …)

	// execVerFn models the live/<host> target the child execs at spawn — start()
	// returns it as the spawnVersion, exactly as s3Backend reads the symlink just
	// before Start(). Nil → "" (unknown), which disables convergence, so tests
	// that don't care are unaffected. Point it at the same source as the sweep's
	// liveVer seam to model a symlink faithfully (both read the current target).
	execVerFn func() string

	// Bot sidecar co-spawn. sidecarStaged is hasSidecar's answer (false by
	// default, so every pre-co-spawn test doubles as the "host without a
	// sidecar spawns none" case); sidecarErr injects spawn failures.
	sidecarStaged bool
	sidecarErr    error
	sidecarStarts int
	sidecars      []*fakeProc
	// order interleaves backend calls and proc stops (start:instance,
	// start:sidecar, nextDue, stop:sidecar, stop:instance) for the ordering
	// assertions.
	order []string
}

func (b *fakeBackend) record(ev string) {
	b.mu.Lock()
	b.order = append(b.order, ev)
	b.mu.Unlock()
}

func (b *fakeBackend) events() []string {
	b.mu.Lock()
	defer b.mu.Unlock()
	return append([]string(nil), b.order...)
}

func (b *fakeBackend) acquire(context.Context, string) (actLease, error) {
	b.mu.Lock()
	b.acquires++
	err := b.acquireErr
	rep := &fakeRep{}
	b.reps = append(b.reps, rep)
	b.mu.Unlock()
	if err != nil {
		return nil, err
	}
	return &fakeLease{rep: rep, b: b}, nil
}

func (b *fakeBackend) restore(context.Context, string, string) error {
	b.mu.Lock()
	defer b.mu.Unlock()
	b.restores++
	return nil
}

func (b *fakeBackend) start(context.Context, Route) (actProc, string, error) {
	b.mu.Lock()
	b.starts++
	b.order = append(b.order, "start:instance")
	gate := b.startGate
	fn := b.execVerFn
	proc := &fakeProc{label: "instance", rec: b}
	b.procs = append(b.procs, proc)
	b.mu.Unlock()
	if gate != nil {
		<-gate
	}
	ver := ""
	if fn != nil {
		ver = fn()
	}
	return proc, ver, nil
}

func (b *fakeBackend) nextDue(context.Context, Route) (int64, error) {
	b.mu.Lock()
	defer b.mu.Unlock()
	b.order = append(b.order, "nextDue")
	if b.dueErr != nil {
		return 0, b.dueErr
	}
	return b.dueVal, nil
}

func (b *fakeBackend) hasSidecar(string) bool {
	b.mu.Lock()
	defer b.mu.Unlock()
	return b.sidecarStaged
}

func (b *fakeBackend) startSidecar(context.Context, Route) (actProc, error) {
	b.mu.Lock()
	defer b.mu.Unlock()
	b.sidecarStarts++
	b.order = append(b.order, "start:sidecar")
	if b.sidecarErr != nil {
		return nil, b.sidecarErr
	}
	p := &fakeProc{label: "sidecar", rec: b}
	b.sidecars = append(b.sidecars, p)
	return p, nil
}

func (b *fakeBackend) sidecarStartCount() int {
	b.mu.Lock()
	defer b.mu.Unlock()
	return b.sidecarStarts
}

func (b *fakeBackend) lastSidecar() *fakeProc {
	b.mu.Lock()
	defer b.mu.Unlock()
	return b.sidecars[len(b.sidecars)-1]
}

func (b *fakeBackend) setSidecarErr(err error) {
	b.mu.Lock()
	defer b.mu.Unlock()
	b.sidecarErr = err
}

func (b *fakeBackend) setDue(v int64) {
	b.mu.Lock()
	defer b.mu.Unlock()
	b.dueVal = v
}

func (b *fakeBackend) counts() (acquires, restores, starts int) {
	b.mu.Lock()
	defer b.mu.Unlock()
	return b.acquires, b.restores, b.starts
}

func (b *fakeBackend) lastProc() *fakeProc {
	b.mu.Lock()
	defer b.mu.Unlock()
	return b.procs[len(b.procs)-1]
}

func (b *fakeBackend) lastRep() *fakeRep {
	b.mu.Lock()
	defer b.mu.Unlock()
	return b.reps[len(b.reps)-1]
}

// fakeClock is the activator's `now` seam.
type fakeClock struct {
	mu sync.Mutex
	t  time.Time
}

func (c *fakeClock) now() time.Time {
	c.mu.Lock()
	defer c.mu.Unlock()
	return c.t
}
func (c *fakeClock) advance(d time.Duration) {
	c.mu.Lock()
	defer c.mu.Unlock()
	c.t = c.t.Add(d)
}

// testActivator wires the fakes to a REAL (temp-dir) registry as the due store,
// so the due-time tests exercise the actual routes.next_due SQL, not a fake of it.
func testActivator(t *testing.T, idle time.Duration) (*activatorSvc, *fakeBackend, *fakeClock, *Registry) {
	t.Helper()
	b := &fakeBackend{}
	clk := &fakeClock{t: time.Unix(1_700_000_000, 0)}
	reg, err := openRegistry(filepath.Join(t.TempDir(), "registry.db"))
	if err != nil {
		t.Fatalf("open registry: %v", err)
	}
	a := newActivator(hibernateConfig{Bucket: "test-bucket", Idle: idle}, b, reg)
	a.now = clk.now
	return a, b, clk, reg
}

func testRoute(host string) Route {
	// Hibernate is set because these routes model activator-managed hosts —
	// the per-host gate (dueInstances, the router's upstream flag) only acts
	// on flagged instances.
	return Route{Host: host, Kind: "instance", Socket: "/tmp/nope.sock",
		DBPath: filepath.Join("/tmp", host+".db"), Hibernate: true}
}

// (waitFor — poll until a condition holds — is shared from bot_run_test.go;
// the stop path runs on its own goroutine, so tests observe its completion
// rather than assuming it.)

func (a *activatorSvc) entryCount() int {
	a.mu.Lock()
	defer a.mu.Unlock()
	return len(a.entries)
}

// --- tests ---------------------------------------------------------------------

// Many concurrent requests during a wake coalesce into exactly one wake: one
// lease claim, one restore, one spawn.
func TestActivatorWakeCoalesces(t *testing.T) {
	a, b, _, _ := testActivator(t, time.Minute)
	b.startGate = make(chan struct{})

	const n = 12
	var ok atomic.Int32
	var wg sync.WaitGroup
	for i := 0; i < n; i++ {
		wg.Add(1)
		go func() {
			defer wg.Done()
			release, err := a.beginRequest(context.Background(), testRoute("one.example.com"))
			if err != nil {
				t.Errorf("beginRequest: %v", err)
				return
			}
			ok.Add(1)
			release()
		}()
	}
	// Let the requests pile up on the gated wake, then open it.
	waitFor(t, "the wake to begin", func() bool { _, _, s := b.counts(); return s == 1 })
	time.Sleep(50 * time.Millisecond)
	close(b.startGate)
	wg.Wait()

	if got := ok.Load(); got != n {
		t.Fatalf("%d/%d requests succeeded", got, n)
	}
	acquires, restores, starts := b.counts()
	if acquires != 1 || restores != 1 || starts != 1 {
		t.Fatalf("wake did not coalesce: acquires=%d restores=%d starts=%d (want 1 each)", acquires, restores, starts)
	}
}

// The wake threads the route's replication class into replicate() as the
// litestream sync interval — the per-plan durability ladder's seam. Unset
// (every pre-ladder route) must thread 0, i.e. litestream's default: a deploy
// of the ladder changes no host's durability by itself.
func TestActivatorWakeThreadsReplicationClass(t *testing.T) {
	a, b, _, _ := testActivator(t, time.Minute)

	for i, tc := range []struct {
		class string
		want  time.Duration
	}{
		{"", 0},
		{replicationStream, 0},
		{replicationBatched, 5 * time.Minute},
		{replicationDaily, 24 * time.Hour},
	} {
		rt := testRoute(fmt.Sprintf("ladder%d.example.com", i))
		rt.Replication = tc.class
		release, err := a.beginRequest(context.Background(), rt)
		if err != nil {
			t.Fatalf("beginRequest(%q): %v", tc.class, err)
		}
		release()
		b.mu.Lock()
		got := b.syncIntervals[len(b.syncIntervals)-1]
		b.mu.Unlock()
		if got != tc.want {
			t.Errorf("class %q threaded sync interval %v, want %v", tc.class, got, tc.want)
		}
	}
}

// After the idle window with nothing in flight, the sweeper hibernates the
// instance: process stopped, replication closed (final checkpoint + lease
// release), entry forgotten; the next request wakes it afresh.
func TestActivatorIdleSleep(t *testing.T) {
	a, b, clk, _ := testActivator(t, 10*time.Second)

	release, err := a.beginRequest(context.Background(), testRoute("one.example.com"))
	if err != nil {
		t.Fatalf("beginRequest: %v", err)
	}
	release()

	// Not yet idle: nothing happens.
	clk.advance(9 * time.Second)
	a.sweepOnce()
	if a.entryCount() != 1 {
		t.Fatal("instance hibernated before its idle window elapsed")
	}

	clk.advance(2 * time.Second)
	a.sweepOnce()
	waitFor(t, "hibernation", func() bool { return a.entryCount() == 0 })
	if !b.lastProc().isStopped() {
		t.Fatal("idle instance's process was not stopped")
	}
	if !b.lastRep().isClosed() {
		t.Fatal("idle instance's replication was not closed (no final checkpoint/lease release)")
	}

	// A new request wakes it again — a second full wake.
	release, err = a.beginRequest(context.Background(), testRoute("one.example.com"))
	if err != nil {
		t.Fatalf("beginRequest after sleep: %v", err)
	}
	release()
	if _, _, starts := b.counts(); starts != 2 {
		t.Fatalf("starts=%d after sleep+rewake, want 2", starts)
	}
}

// An in-flight request (an open WebSocket, in production) holds the instance
// awake past the idle window; only after it finishes does the idle clock run.
func TestActivatorInflightKeepsAwake(t *testing.T) {
	a, b, clk, _ := testActivator(t, 10*time.Second)

	release, err := a.beginRequest(context.Background(), testRoute("one.example.com"))
	if err != nil {
		t.Fatalf("beginRequest: %v", err)
	}

	clk.advance(time.Hour)
	a.sweepOnce()
	if a.entryCount() != 1 || b.lastProc().isStopped() {
		t.Fatal("instance with an in-flight request was hibernated")
	}

	release() // stamps lastActive = now
	a.sweepOnce()
	if a.entryCount() != 1 {
		t.Fatal("instance hibernated immediately after its last request finished")
	}
	clk.advance(11 * time.Second)
	a.sweepOnce()
	waitFor(t, "hibernation", func() bool { return a.entryCount() == 0 })
}

// A ship repoints live/<host> under an awake child. A running process never
// re-reads its symlink, so idle-hibernate is the only thing that converges an
// instance — and an always-awake space (every bot-sidecar host) never idles.
// The sweep closes that gap: it notices the drift and, after a grace window and
// rate-limited to one per tick, gracefully cycles the child so its next wake
// execs the new binary. See docs/awake-instance-convergence.md.
func TestActivatorConvergesOnSymlinkRepoint(t *testing.T) {
	a, b, clk, _ := testActivator(t, time.Hour) // idle far away — isolate the convergence path
	var ver atomic.Value                        // the live/<host> symlink target; swapped to model a ship
	ver.Store("v1")
	// Model one symlink: the child execs whatever it points at now (execVerFn,
	// frozen into spawnVersion at wake), and the sweep reads the current target
	// (liveVer). A "ship" swaps ver; the running child keeps its frozen version.
	a.liveVer = func(string) (string, error) { return ver.Load().(string), nil }
	b.execVerFn = func() string { return ver.Load().(string) }

	release, err := a.beginRequest(context.Background(), testRoute("one.example.com"))
	if err != nil {
		t.Fatalf("beginRequest: %v", err)
	}
	release()

	// Symlink unchanged: never cycles, however long it stays awake.
	clk.advance(3 * convergeGrace)
	a.sweepOnce()
	if a.entryCount() != 1 {
		t.Fatal("cycled an instance whose symlink never moved")
	}

	// Ship: repoint the symlink. The first sweep after only starts the grace
	// clock — it must not cycle yet, so a move-then-rollback never drops a live
	// connection.
	ver.Store("v2")
	a.sweepOnce()
	if a.entryCount() != 1 {
		t.Fatal("cycled inside the grace window — a transient repoint would drop a live connection")
	}

	// Past the grace: gracefully cycle (stop → final checkpoint → lease release →
	// asleep). The reconnecting client/sidecar would wake it onto v2.
	clk.advance(convergeGrace + time.Second)
	a.sweepOnce()
	waitFor(t, "convergence cycle", func() bool { return a.entryCount() == 0 })
	if !b.lastProc().isStopped() {
		t.Fatal("stale instance's process was not stopped")
	}
	if !b.lastRep().isClosed() {
		t.Fatal("convergence cycle skipped the final checkpoint / lease release")
	}

	// Re-wake against the now-current symlink: spawnVersion tracks v2, so it
	// never cycles again — no thrash.
	release, err = a.beginRequest(context.Background(), testRoute("one.example.com"))
	if err != nil {
		t.Fatalf("re-wake after convergence: %v", err)
	}
	release()
	clk.advance(3 * convergeGrace)
	a.sweepOnce()
	if a.entryCount() != 1 {
		t.Fatal("cycled again after already converging onto the current build")
	}
}

// Two awake children go stale at once (a fleet-wide ship). One sweep cycles at
// most one — a staggered convergence, never a simultaneous blip of every space.
func TestActivatorConvergenceRateLimitsPerSweep(t *testing.T) {
	a, b, clk, _ := testActivator(t, time.Hour)
	var ver atomic.Value
	ver.Store("v1")
	a.liveVer = func(string) (string, error) { return ver.Load().(string), nil }
	b.execVerFn = func() string { return ver.Load().(string) }

	for _, h := range []string{"one.example.com", "two.example.com"} {
		release, err := a.beginRequest(context.Background(), testRoute(h))
		if err != nil {
			t.Fatalf("beginRequest %s: %v", h, err)
		}
		release()
	}
	if a.entryCount() != 2 {
		t.Fatalf("want 2 awake instances, got %d", a.entryCount())
	}

	ver.Store("v2") // ship under both
	a.sweepOnce()   // starts the grace clock for both
	clk.advance(convergeGrace + time.Second)

	a.sweepOnce() // both are past grace, but only one may cycle this tick
	waitFor(t, "first convergence cycle", func() bool { return a.entryCount() == 1 })
	// A moment later, the sweep takes the second.
	a.sweepOnce()
	waitFor(t, "second convergence cycle", func() bool { return a.entryCount() == 0 })
}

// The feature's whole premise: converge a space that NEVER reaches inflight==0
// (an open WebSocket, or a bot sidecar's persistent connection), which
// idle-hibernate can never touch. Unlike idle-hibernate, convergence fires with
// a request in flight — the graceful stop drains it exactly as a restart would.
func TestActivatorConvergesWithRequestInFlight(t *testing.T) {
	a, b, clk, _ := testActivator(t, time.Hour)
	var ver atomic.Value
	ver.Store("v1")
	a.liveVer = func(string) (string, error) { return ver.Load().(string), nil }
	b.execVerFn = func() string { return ver.Load().(string) }

	// Hold the request open for the whole test — inflight stays 1, so the idle
	// path (which needs inflight==0) can never converge this host.
	release, err := a.beginRequest(context.Background(), testRoute("one.example.com"))
	if err != nil {
		t.Fatalf("beginRequest: %v", err)
	}
	defer release()

	ver.Store("v2") // ship under the busy space
	a.sweepOnce()   // grace starts; must not cycle a busy space mid-grace
	if a.entryCount() != 1 {
		t.Fatal("cycled inside the grace window")
	}
	clk.advance(convergeGrace + time.Second)
	a.sweepOnce()
	// Cycles despite the in-flight request — idle-hibernate never would.
	waitFor(t, "busy-space convergence", func() bool { return a.entryCount() == 0 })
	if !b.lastProc().isStopped() {
		t.Fatal("busy stale instance's process was not stopped")
	}
}

// convergeDecision's state machine, exhaustively: the grace clock starts on
// first drift, survives a rollback correctly, and a read error never clears
// accrued grace (it means "unknown", not "converged").
func TestConvergeDecision(t *testing.T) {
	base := time.Unix(1_700_000_000, 0)
	set := base.Add(-40 * time.Second) // a grace clock that started 40s ago
	errBoom := errors.New("readlink boom")
	cases := []struct {
		name       string
		cur        string
		readErr    error
		spawnVer   string
		staleSince time.Time
		wantCycle  bool
		wantStale  time.Time
	}{
		{"matches — clears grace", "v1", nil, "v1", set, false, time.Time{}},
		{"empty cur — no data, clears", "", nil, "v1", set, false, time.Time{}},
		{"empty spawnVer — unknown, clears", "v2", nil, "", set, false, time.Time{}},
		{"first drift — starts grace, no cycle", "v2", nil, "v1", time.Time{}, false, base},
		{"drift within grace — holds, no cycle", "v2", nil, "v1", base.Add(-30 * time.Second), false, base.Add(-30 * time.Second)},
		{"drift past grace — cycles", "v2", nil, "v1", base.Add(-convergeGrace - time.Second), true, base.Add(-convergeGrace - time.Second)},
		{"read error — preserves accrued grace", "", errBoom, "v1", set, false, set},
		{"read error before any drift — stays zero", "", errBoom, "v1", time.Time{}, false, time.Time{}},
		{"rollback to spawn — clears grace", "v1", nil, "v1", base.Add(-convergeGrace - time.Second), false, time.Time{}},
	}
	for _, c := range cases {
		t.Run(c.name, func(t *testing.T) {
			cycle, next := convergeDecision(c.cur, c.readErr, c.spawnVer, c.staleSince, base, convergeGrace)
			if cycle != c.wantCycle {
				t.Errorf("cycle = %v, want %v", cycle, c.wantCycle)
			}
			if !next.Equal(c.wantStale) {
				t.Errorf("nextStale = %v, want %v", next, c.wantStale)
			}
		})
	}
}

// A lost lease (failed renewal) stands the instance down immediately — no
// waiting for idle: someone else may own the S3 prefix now.
func TestActivatorLeaseLostStandsDown(t *testing.T) {
	a, b, _, _ := testActivator(t, time.Hour)

	release, err := a.beginRequest(context.Background(), testRoute("one.example.com"))
	if err != nil {
		t.Fatalf("beginRequest: %v", err)
	}
	release()

	b.lastRep().setErr(errors.New("lease renewal failed"))
	a.sweepOnce() // idle window is an hour away; the lost lease alone must trigger the stop
	waitFor(t, "stand-down", func() bool { return a.entryCount() == 0 })
	if !b.lastProc().isStopped() {
		t.Fatal("lease-lost instance's process was not stopped")
	}
	if !b.lastRep().isClosed() {
		t.Fatal("lease-lost instance's replication was not closed")
	}
}

// A dead child (crashed instance) is recycled by the next request: the stale
// entry is stopped and a fresh wake happens in the same beginRequest call.
func TestActivatorDeadChildRewakesOnRequest(t *testing.T) {
	a, b, _, _ := testActivator(t, time.Hour)

	release, err := a.beginRequest(context.Background(), testRoute("one.example.com"))
	if err != nil {
		t.Fatalf("beginRequest: %v", err)
	}
	release()

	p := b.lastProc()
	p.mu.Lock()
	p.dead = true
	p.mu.Unlock()

	release, err = a.beginRequest(context.Background(), testRoute("one.example.com"))
	if err != nil {
		t.Fatalf("beginRequest after crash: %v", err)
	}
	release()
	if _, _, starts := b.counts(); starts != 2 {
		t.Fatalf("starts=%d after crash+request, want 2 (recycle + rewake)", starts)
	}
}

// A request that arrives mid-stop waits for the stop, then wakes fresh.
func TestActivatorRequestDuringStopRewakes(t *testing.T) {
	a, b, clk, _ := testActivator(t, 10*time.Second)

	release, err := a.beginRequest(context.Background(), testRoute("one.example.com"))
	if err != nil {
		t.Fatalf("beginRequest: %v", err)
	}
	release()

	// Hold the stop open so the entry sits in stopping.
	gate := make(chan struct{})
	p := b.lastProc()
	p.mu.Lock()
	p.stopGate = gate
	p.mu.Unlock()

	clk.advance(time.Minute)
	a.sweepOnce()

	got := make(chan error, 1)
	go func() {
		release, err := a.beginRequest(context.Background(), testRoute("one.example.com"))
		if err == nil {
			release()
		}
		got <- err
	}()
	time.Sleep(50 * time.Millisecond) // the request should now be parked on e.stopped
	close(gate)
	if err := <-got; err != nil {
		t.Fatalf("request during stop: %v", err)
	}
	if _, _, starts := b.counts(); starts != 2 {
		t.Fatalf("starts=%d, want 2 (sleep then rewake)", starts)
	}
}

// A wake that exceeds the bound turns queued requests away with errWakeTimeout
// (503 at the router) rather than piling connections up behind it.
func TestActivatorWakeTimeout(t *testing.T) {
	a, b, _, _ := testActivator(t, time.Minute)
	old := wakeWaitBound
	wakeWaitBound = 100 * time.Millisecond
	t.Cleanup(func() { wakeWaitBound = old })

	gate := make(chan struct{})
	b.startGate = gate
	t.Cleanup(func() { close(gate) }) // let the stuck wake finish at test end

	_, err := a.beginRequest(context.Background(), testRoute("one.example.com"))
	if !errors.Is(err, errWakeTimeout) {
		t.Fatalf("got %v, want errWakeTimeout", err)
	}
}

// Lease contention surfaces through the router as 503 + Retry-After, and a
// failed wake leaves no entry behind (the next request retries the claim).
func TestActivatorLeaseContention503(t *testing.T) {
	a, b, _, _ := testActivator(t, time.Minute)
	b.acquireErr = &errLeaseHeld{holder: "old-incarnation", retryAfter: 7 * time.Second}

	reg, err := openRegistry(filepath.Join(t.TempDir(), "registry.db"))
	if err != nil {
		t.Fatalf("open registry: %v", err)
	}
	if err := reg.upsert(Route{Host: "held.example.com", Kind: "instance", Socket: "/tmp/held.sock", Hibernate: true}); err != nil {
		t.Fatalf("upsert: %v", err)
	}
	rt, err := newRouter(reg, a)
	if err != nil {
		t.Fatalf("new router: %v", err)
	}
	front := httptest.NewServer(rt)
	t.Cleanup(front.Close)

	req, _ := http.NewRequest("GET", front.URL+"/", nil)
	req.Host = "held.example.com"
	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		t.Fatalf("request: %v", err)
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusServiceUnavailable {
		t.Fatalf("status %d, want 503", resp.StatusCode)
	}
	if ra := resp.Header.Get("Retry-After"); ra != "8" { // 7s truncated + 1
		t.Fatalf("Retry-After %q, want 8", ra)
	}
	if a.entryCount() != 0 {
		t.Fatal("failed wake left an entry behind")
	}
}

// Per-host gating (#550 Part 4 v2 opt-in): with the activator CONFIGURED, only
// a hibernate-flagged instance goes through the wake path; an unflagged one is
// proxied straight, exactly as if there were no activator at all — the
// deploy-safety property that lets the bucket env ship fleet-wide before any
// host is flagged.
func TestRouterPerHostHibernateGating(t *testing.T) {
	a, b, _, reg := testActivator(t, time.Minute)

	// Two live backends playing the instances. The flagged host's listener
	// stands in for the child a wake would have spawned (the fake backend's
	// start() spawns nothing).
	bind := func(name string) string {
		sock := tmpSock(t)
		ln, err := net.Listen("unix", sock)
		if err != nil {
			t.Fatalf("listen unix: %v", err)
		}
		srv := &http.Server{Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
			io.WriteString(w, name)
		})}
		go srv.Serve(ln)
		t.Cleanup(func() { srv.Close() })
		return sock
	}
	if err := reg.upsert(Route{Host: "hib.example.com", Kind: "instance", Socket: bind("hib"), Hibernate: true}); err != nil {
		t.Fatal(err)
	}
	if err := reg.upsert(Route{Host: "plain.example.com", Kind: "instance", Socket: bind("plain")}); err != nil {
		t.Fatal(err)
	}

	rt, err := newRouter(reg, a)
	if err != nil {
		t.Fatalf("new router: %v", err)
	}
	front := httptest.NewServer(rt)
	t.Cleanup(front.Close)

	// The unflagged host: proxied straight, the activator never touched.
	if body, code := proxyGet(t, front.URL, "plain.example.com"); code != 200 || body != "plain" {
		t.Fatalf("unflagged host: got %d %q, want 200 plain", code, body)
	}
	if acquires, _, starts := b.counts(); acquires != 0 || starts != 0 {
		t.Fatalf("unflagged instance touched the activator (acquires=%d starts=%d)", acquires, starts)
	}

	// The flagged host: served through a full wake — lease, restore, spawn.
	if body, code := proxyGet(t, front.URL, "hib.example.com"); code != 200 || body != "hib" {
		t.Fatalf("flagged host: got %d %q, want 200 hib", code, body)
	}
	if acquires, restores, starts := b.counts(); acquires != 1 || restores != 1 || starts != 1 {
		t.Fatalf("flagged host skipped the wake path: acquires=%d restores=%d starts=%d (want 1 each)", acquires, restores, starts)
	}
}

// A reload that finds an awake host no longer hibernate-flagged retires its
// child — stopped, final-checkpointed (replication closed), entry forgotten —
// the router-process half of `lchat hibernate --off`. And once the flag is
// off, a recorded next_due no longer wakes it: the due index only acts on
// hibernating hosts, so a systemd-managed instance can never be spawned
// against its own unit.
func TestActivatorReloadRetiresUnflaggedHost(t *testing.T) {
	a, b, clk, reg := testActivator(t, time.Hour)
	route := testRoute("flip.example.com")
	if err := reg.upsert(route); err != nil {
		t.Fatal(err)
	}
	rt, err := newRouter(reg, a)
	if err != nil {
		t.Fatalf("new router: %v", err)
	}

	release, err := a.beginRequest(context.Background(), route)
	if err != nil {
		t.Fatalf("beginRequest: %v", err)
	}
	release()
	// The retire's sleep re-records next_due AFTER the flag is cleared; an
	// already-past value must stay inert under the flag filter.
	b.setDue(clk.now().Add(-5 * time.Second).Unix())

	if err := reg.setHibernate("flip.example.com", false); err != nil {
		t.Fatal(err)
	}
	if err := rt.reload(); err != nil {
		t.Fatalf("reload: %v", err)
	}
	waitFor(t, "retire", func() bool { return a.entryCount() == 0 })
	if !b.lastProc().isStopped() {
		t.Fatal("retired child was not stopped")
	}
	if !b.lastRep().isClosed() {
		t.Fatal("retired child's replication was not closed (no final checkpoint/lease release)")
	}

	// The due sweeper must not resurrect a host that's back under systemd.
	a.sweepOnce()
	if _, _, starts := b.counts(); starts != 1 {
		t.Fatalf("starts=%d after retire, want 1 (no due wake for a systemd-managed host)", starts)
	}
}

// A retire that lands mid-wake can't stop the child yet — it finishes the job
// once the wake resolves, so a flip-off during the small wake window still
// releases cleanly.
func TestActivatorRetireDuringWake(t *testing.T) {
	a, b, clk, reg := testActivator(t, time.Hour)
	route := testRoute("mid.example.com")
	if err := reg.upsert(route); err != nil {
		t.Fatal(err)
	}
	rt, err := newRouter(reg, a)
	if err != nil {
		t.Fatalf("new router: %v", err)
	}

	// Start a request-less (due-time) wake and hold it in the waking state.
	b.startGate = make(chan struct{})
	if err := reg.setNextDue("mid.example.com", clk.now().Add(-time.Second).Unix()); err != nil {
		t.Fatal(err)
	}
	a.sweepOnce()
	waitFor(t, "the wake to begin", func() bool { _, _, s := b.counts(); return s == 1 })

	// Flip off mid-wake: the reload's retire must defer, not deadlock.
	if err := reg.setHibernate("mid.example.com", false); err != nil {
		t.Fatal(err)
	}
	if err := rt.reload(); err != nil {
		t.Fatalf("reload: %v", err)
	}
	close(b.startGate)

	waitFor(t, "post-wake retire", func() bool { return a.entryCount() == 0 })
	if !b.lastProc().isStopped() {
		t.Fatal("child woken mid-retire was not stopped")
	}
	if !b.lastRep().isClosed() {
		t.Fatal("child woken mid-retire left replication open (lease never released)")
	}
}

// With hibernation on, sibling services (kind "service") are untouched: the
// router proxies them directly, no wake, no activity tracking.
func TestActivatorServiceRoutesBypass(t *testing.T) {
	sock := tmpSock(t)
	ln, err := net.Listen("unix", sock)
	if err != nil {
		t.Fatalf("listen unix: %v", err)
	}
	backend := &http.Server{Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		io.WriteString(w, "service ok")
	})}
	go backend.Serve(ln)
	t.Cleanup(func() { backend.Close() })

	a, b, _, _ := testActivator(t, time.Minute)
	reg, err := openRegistry(filepath.Join(t.TempDir(), "registry.db"))
	if err != nil {
		t.Fatalf("open registry: %v", err)
	}
	if err := reg.upsert(Route{Host: "svc.example.com", Kind: "service", Socket: sock}); err != nil {
		t.Fatalf("upsert: %v", err)
	}
	rt, err := newRouter(reg, a)
	if err != nil {
		t.Fatalf("new router: %v", err)
	}
	front := httptest.NewServer(rt)
	t.Cleanup(front.Close)

	req, _ := http.NewRequest("GET", front.URL+"/", nil)
	req.Host = "svc.example.com"
	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		t.Fatalf("request: %v", err)
	}
	body, _ := io.ReadAll(resp.Body)
	resp.Body.Close()
	if resp.StatusCode != 200 || string(body) != "service ok" {
		t.Fatalf("got %d %q, want 200 service ok", resp.StatusCode, body)
	}
	if acquires, _, _ := b.counts(); acquires != 0 {
		t.Fatal("a service route touched the activator")
	}
}

// shutdown checkpoints and stops every awake child before the router exits.
func TestActivatorShutdownStopsChildren(t *testing.T) {
	a, b, _, _ := testActivator(t, time.Hour)
	for _, h := range []string{"one.example.com", "two.example.com"} {
		release, err := a.beginRequest(context.Background(), testRoute(h))
		if err != nil {
			t.Fatalf("beginRequest %s: %v", h, err)
		}
		release()
	}

	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
	defer cancel()
	a.shutdown(ctx)

	if a.entryCount() != 0 {
		t.Fatal("shutdown left entries behind")
	}
	b.mu.Lock()
	defer b.mu.Unlock()
	for i, p := range b.procs {
		if !p.stopped {
			t.Fatalf("proc %d not stopped on shutdown", i)
		}
	}
	for i, r := range b.reps {
		if !r.closed {
			t.Fatalf("replication %d not closed on shutdown", i)
		}
	}
}

// --- due-time index --------------------------------------------------------------

// entryAwake reports whether host has a live entry in the awake state — how the
// due tests observe an async due-time wake resolving.
func (a *activatorSvc) entryAwake(host string) bool {
	a.mu.Lock()
	defer a.mu.Unlock()
	e := a.entries[host]
	return e != nil && e.state == actAwake
}

// The full self-maintaining cycle: sleep records the instance's next due time
// in the registry; the sweeper wakes the host (no request) when it arrives; the
// following sleep re-records — and once nothing is scheduled, no more due wakes.
func TestActivatorDueTimeWakeCycle(t *testing.T) {
	a, b, clk, reg := testActivator(t, 10*time.Second)
	route := testRoute("due.example.com")
	if err := reg.upsert(route); err != nil {
		t.Fatalf("upsert: %v", err)
	}

	// Wake by request; the instance has a delivery scheduled 30s out.
	dueAt := clk.now().Add(30 * time.Second).Unix()
	b.setDue(dueAt)
	release, err := a.beginRequest(context.Background(), route)
	if err != nil {
		t.Fatalf("beginRequest: %v", err)
	}
	release()

	// Idle out: the sleep probes the instance and records its due time.
	clk.advance(11 * time.Second)
	a.sweepOnce()
	waitFor(t, "hibernation", func() bool { return a.entryCount() == 0 })
	if rt, err := reg.get(route.Host); err != nil || rt == nil || rt.NextDue != dueAt {
		t.Fatalf("registry next_due = %+v, %v; want %d", rt, err, dueAt)
	}

	// Before the due time: sweeps leave it asleep.
	a.sweepOnce()
	if _, _, starts := b.counts(); starts != 1 {
		t.Fatalf("woke before its due time (starts=%d)", starts)
	}

	// At the due time: the sweeper wakes it with no client request.
	clk.advance(20 * time.Second) // now = wake+31s ≥ due
	a.sweepOnce()
	waitFor(t, "due-time wake", func() bool { return a.entryAwake(route.Host) })
	if _, _, starts := b.counts(); starts != 2 {
		t.Fatalf("starts=%d after due wake, want 2", starts)
	}

	// The woken instance stays up for the normal idle window, then sleeps and
	// re-records — nothing scheduled now, so the index clears and stays quiet.
	b.setDue(0)
	clk.advance(11 * time.Second)
	a.sweepOnce()
	waitFor(t, "re-hibernation", func() bool { return a.entryCount() == 0 })
	if rt, _ := reg.get(route.Host); rt.NextDue != 0 {
		t.Fatalf("next_due = %d after empty-schedule sleep, want 0", rt.NextDue)
	}
	clk.advance(time.Hour)
	a.sweepOnce()
	if _, _, starts := b.counts(); starts != 2 {
		t.Fatalf("starts=%d with nothing scheduled, want 2 (no phantom due wake)", starts)
	}
}

// A due time already in the past at sleep time (a delivery that raced the stop)
// is recorded as-is: the sweeper wakes the host on its next tick — once — and
// there is no ping-pong, because a woken instance can't sleep again until a
// full idle window passes.
func TestActivatorDuePastWakesOnceNoPingPong(t *testing.T) {
	a, b, clk, reg := testActivator(t, 10*time.Second)
	route := testRoute("late.example.com")
	if err := reg.upsert(route); err != nil {
		t.Fatalf("upsert: %v", err)
	}
	b.setDue(clk.now().Add(-5 * time.Second).Unix()) // overdue before it even sleeps

	release, err := a.beginRequest(context.Background(), route)
	if err != nil {
		t.Fatalf("beginRequest: %v", err)
	}
	release()
	clk.advance(11 * time.Second)
	a.sweepOnce()
	waitFor(t, "hibernation", func() bool { return a.entryCount() == 0 })

	// Next tick: woken for the overdue delivery.
	a.sweepOnce()
	waitFor(t, "overdue wake", func() bool { return a.entryAwake(route.Host) })

	// No thrash: with the wake fresh (clock unmoved), repeated sweeps neither
	// re-sleep it nor double-wake it, even though next_due is still in the past.
	for i := 0; i < 5; i++ {
		a.sweepOnce()
	}
	if a.entryCount() != 1 || !a.entryAwake(route.Host) {
		t.Fatal("due-woken instance did not stay awake through its idle window")
	}
	if _, _, starts := b.counts(); starts != 2 {
		t.Fatalf("starts=%d, want 2 (request wake + one due wake)", starts)
	}
}

// A failed next-due probe (dead socket, missing admin token, …) records 0 and
// the sleep proceeds — the instance degrades to "sleeps until a request",
// never a crash and never a stuck stop.
func TestActivatorDueProbeFailureRecordsZero(t *testing.T) {
	a, b, clk, reg := testActivator(t, 10*time.Second)
	route := testRoute("deaf.example.com")
	if err := reg.upsert(route); err != nil {
		t.Fatalf("upsert: %v", err)
	}
	b.dueErr = errors.New("dial unix: no such file")

	release, err := a.beginRequest(context.Background(), route)
	if err != nil {
		t.Fatalf("beginRequest: %v", err)
	}
	release()
	clk.advance(11 * time.Second)
	a.sweepOnce()
	waitFor(t, "hibernation despite probe failure", func() bool { return a.entryCount() == 0 })

	if rt, _ := reg.get(route.Host); rt.NextDue != 0 {
		t.Fatalf("next_due = %d after failed probe, want 0", rt.NextDue)
	}
	clk.advance(time.Hour)
	a.sweepOnce()
	if _, _, starts := b.counts(); starts != 1 {
		t.Fatalf("starts=%d, want 1 (a failed probe must not invent a wake)", starts)
	}
}

// The index lives in the registry, so a due time recorded before a router
// restart still fires: a brand-new activator over the same registry — no
// in-memory state carried over — wakes the host when its time arrives.
func TestActivatorDueSurvivesRestart(t *testing.T) {
	a1, b1, clk, reg := testActivator(t, 10*time.Second)
	route := testRoute("reborn.example.com")
	if err := reg.upsert(route); err != nil {
		t.Fatalf("upsert: %v", err)
	}
	b1.setDue(clk.now().Add(30 * time.Second).Unix())

	release, err := a1.beginRequest(context.Background(), route)
	if err != nil {
		t.Fatalf("beginRequest: %v", err)
	}
	release()
	clk.advance(11 * time.Second)
	a1.sweepOnce()
	waitFor(t, "hibernation", func() bool { return a1.entryCount() == 0 })

	// "Restart": a fresh activator, same registry.
	b2 := &fakeBackend{}
	a2 := newActivator(hibernateConfig{Bucket: "test-bucket", Idle: 10 * time.Second}, b2, reg)
	a2.now = clk.now

	clk.advance(25 * time.Second) // past the due time
	a2.sweepOnce()
	waitFor(t, "post-restart due wake", func() bool { return a2.entryAwake(route.Host) })
	if _, _, starts := b2.counts(); starts != 1 {
		t.Fatalf("starts=%d on the new activator, want 1", starts)
	}
}

// --- bot sidecar co-spawn ----------------------------------------------------

// indexOf: position of ev in the backend's order log (-1 = absent).
func indexOf(events []string, ev string) int {
	for i, e := range events {
		if e == ev {
			return i
		}
	}
	return -1
}

// A host with a sidecar staged gets it co-spawned on wake, strictly AFTER the
// instance process is up — the sidecar dials the instance, so spawning it
// first would just burn its reconnect backoff. (Every other test in this file
// runs with sidecarStaged=false and asserts nothing sidecar-ish happens, which
// is the "host without a sidecar behaves exactly as today" half.)
func TestActivatorCoSpawnsSidecarAfterInstance(t *testing.T) {
	a, b, _, _ := testActivator(t, time.Minute)
	b.sidecarStaged = true

	release, err := a.beginRequest(context.Background(), testRoute("bots.example.com"))
	if err != nil {
		t.Fatalf("beginRequest: %v", err)
	}
	release()

	if got := b.sidecarStartCount(); got != 1 {
		t.Fatalf("sidecar starts = %d, want 1", got)
	}
	ev := b.events()
	if i, s := indexOf(ev, "start:instance"), indexOf(ev, "start:sidecar"); i == -1 || s == -1 || s < i {
		t.Fatalf("sidecar did not start strictly after the instance: %v", ev)
	}
}

// A sidecar that fails to spawn must not fail the wake — members > bots: the
// space serves without its bots, and the sweeper keeps retrying with backoff
// until a spawn sticks.
func TestActivatorSidecarFailureDoesNotFailWake(t *testing.T) {
	a, b, clk, _ := testActivator(t, time.Hour)
	b.sidecarStaged = true
	b.setSidecarErr(errors.New("exec: live symlink gone"))
	route := testRoute("botless.example.com")

	release, err := a.beginRequest(context.Background(), route)
	if err != nil {
		t.Fatalf("beginRequest with a failing sidecar: %v", err)
	}
	release()
	if !a.entryAwake(route.Host) {
		t.Fatal("instance not awake after a sidecar-only failure")
	}
	if got := b.sidecarStartCount(); got != 1 {
		t.Fatalf("sidecar attempts = %d, want 1 (the wake's)", got)
	}

	// The sweeper retries: attempt 2 immediately (first retry), then gated by
	// the doubling backoff; once the fault clears, a retry lands the sidecar.
	a.sweepOnce()
	if got := b.sidecarStartCount(); got != 2 {
		t.Fatalf("sidecar attempts after first sweep = %d, want 2", got)
	}
	a.sweepOnce() // still inside the 2s gate — no attempt
	if got := b.sidecarStartCount(); got != 2 {
		t.Fatalf("backoff gate ignored: attempts = %d, want 2", got)
	}
	b.setSidecarErr(nil)
	clk.advance(3 * time.Second)
	a.sweepOnce()
	if got := b.sidecarStartCount(); got != 3 {
		t.Fatalf("sidecar attempts after fault cleared = %d, want 3", got)
	}
	if b.lastSidecar().isStopped() || b.lastSidecar().exited() {
		t.Fatal("recovered sidecar isn't running")
	}
}

// Co-stop ordering at sleep: the sidecar is stopped FIRST — before the
// next-due probe (so bots can't shift the schedule being recorded) and before
// the instance's own stop (so the sidecar never outlives its instance).
func TestActivatorCoStopOrdering(t *testing.T) {
	a, b, clk, _ := testActivator(t, 10*time.Second)
	b.sidecarStaged = true

	release, err := a.beginRequest(context.Background(), testRoute("bots.example.com"))
	if err != nil {
		t.Fatalf("beginRequest: %v", err)
	}
	release()

	clk.advance(11 * time.Second)
	a.sweepOnce()
	waitFor(t, "hibernation", func() bool { return a.entryCount() == 0 })

	if !b.lastSidecar().isStopped() {
		t.Fatal("sidecar not stopped at sleep")
	}
	ev := b.events()
	sc, due, inst := indexOf(ev, "stop:sidecar"), indexOf(ev, "nextDue"), indexOf(ev, "stop:instance")
	if sc == -1 || due == -1 || inst == -1 || !(sc < due && due < inst) {
		t.Fatalf("co-stop order wrong: %v (want stop:sidecar < nextDue < stop:instance)", ev)
	}
}

// Orphan prevention: whatever takes the instance down — here a crashed child,
// but a lost lease and a retire ride the identical stop path — takes the
// sidecar with it. An orphaned sidecar would reconnect-loop against the router
// forever (503s at best; a wake source if the marker rule ever broke).
func TestActivatorInstanceDeathStopsSidecar(t *testing.T) {
	a, b, _, _ := testActivator(t, time.Hour)
	b.sidecarStaged = true

	release, err := a.beginRequest(context.Background(), testRoute("bots.example.com"))
	if err != nil {
		t.Fatalf("beginRequest: %v", err)
	}
	release()

	p := b.lastProc()
	p.mu.Lock()
	p.dead = true
	p.mu.Unlock()
	a.sweepOnce()
	waitFor(t, "stand-down", func() bool { return a.entryCount() == 0 })
	if !b.lastSidecar().isStopped() {
		t.Fatal("instance died but its sidecar was left running (orphan)")
	}
}

// Sidecar death while the instance stays awake: restarted with doubling,
// capped backoff — Restart=always semantics, sweep-driven — and a run that
// held for sidecarHealthyRun earns a fresh backoff.
func TestActivatorSidecarRestartBackoff(t *testing.T) {
	a, b, clk, _ := testActivator(t, time.Hour)
	b.sidecarStaged = true

	release, err := a.beginRequest(context.Background(), testRoute("bots.example.com"))
	if err != nil {
		t.Fatalf("beginRequest: %v", err)
	}
	release()
	kill := func() {
		sc := b.lastSidecar()
		sc.mu.Lock()
		sc.dead = true
		sc.mu.Unlock()
	}

	// T0: first crash — restarted on the next sweep (attempt 2), gate 2s.
	kill()
	a.sweepOnce()
	if got := b.sidecarStartCount(); got != 2 {
		t.Fatalf("first restart: attempts = %d, want 2", got)
	}
	// Crash-loop: still inside the 2s gate — no restart.
	kill()
	a.sweepOnce()
	if got := b.sidecarStartCount(); got != 2 {
		t.Fatalf("gate ignored: attempts = %d, want 2", got)
	}
	// T3: past the 2s gate — attempt 3, gate doubles to 4s.
	clk.advance(3 * time.Second)
	a.sweepOnce()
	if got := b.sidecarStartCount(); got != 3 {
		t.Fatalf("second restart: attempts = %d, want 3", got)
	}
	// T6: 3s later is inside the 4s gate — the doubling is real.
	kill()
	clk.advance(3 * time.Second)
	a.sweepOnce()
	if got := b.sidecarStartCount(); got != 3 {
		t.Fatalf("doubled gate ignored: attempts = %d, want 3", got)
	}
	// T8: past it — attempt 4 (gate now 8s).
	clk.advance(2 * time.Second)
	a.sweepOnce()
	if got := b.sidecarStartCount(); got != 4 {
		t.Fatalf("third restart: attempts = %d, want 4", got)
	}
	// A healthy run (≥ sidecarHealthyRun) resets the ladder: the next crash
	// restarts immediately and its gate is back to the 2s base…
	clk.advance(2 * time.Minute)
	kill()
	a.sweepOnce()
	if got := b.sidecarStartCount(); got != 5 {
		t.Fatalf("post-healthy restart: attempts = %d, want 5", got)
	}
	// …which a 3s advance clears (16s — the unreset ladder — would not).
	kill()
	clk.advance(3 * time.Second)
	a.sweepOnce()
	if got := b.sidecarStartCount(); got != 6 {
		t.Fatalf("backoff not reset after a healthy run: attempts = %d, want 6", got)
	}
}

// The activity question (#550 Part 4 v2): the co-spawned sidecar dials its
// instance's public URL through this very router (the box pins its hosted
// names to loopback), so without the marker its keepalives would be member
// activity and the pair could never sleep. Full front-door check: marked
// requests are served while awake — with the marker stripped before the
// backend — but stamp nothing (the host hibernates on schedule beneath them),
// and once asleep they get 503 and wake nothing. An outsider spoofing the
// marker gets exactly that same treatment: it can't wake the host and can't
// suppress anyone else's stamps (stamps are per-request), so the marker is
// not a DoS lever in either direction.
func TestActivatorSidecarRequestsInvisibleToActivity(t *testing.T) {
	a, b, clk, reg := testActivator(t, 10*time.Second)

	// A live backend listener stands in for the child a wake would have
	// spawned (the fake backend's start() spawns nothing), recording whether
	// the sidecar marker ever leaks through the proxy.
	sock := tmpSock(t)
	ln, err := net.Listen("unix", sock)
	if err != nil {
		t.Fatalf("listen unix: %v", err)
	}
	var leaked atomic.Bool
	srv := &http.Server{Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if r.Header.Get(bot.SidecarHeader) != "" {
			leaked.Store(true)
		}
		io.WriteString(w, "ok")
	})}
	go srv.Serve(ln)
	t.Cleanup(func() { srv.Close() })

	const host = "bots.example.com"
	if err := reg.upsert(Route{Host: host, Kind: "instance", Socket: sock, Hibernate: true}); err != nil {
		t.Fatal(err)
	}
	rt, err := newRouter(reg, a)
	if err != nil {
		t.Fatalf("new router: %v", err)
	}
	front := httptest.NewServer(rt)
	t.Cleanup(front.Close)

	markedGet := func() int {
		req, _ := http.NewRequest("GET", front.URL+"/", nil)
		req.Host = host
		req.Header.Set(bot.SidecarHeader, "1")
		resp, err := http.DefaultClient.Do(req)
		if err != nil {
			t.Fatalf("marked request: %v", err)
		}
		defer resp.Body.Close()
		io.Copy(io.Discard, resp.Body)
		return resp.StatusCode
	}

	// A member request wakes the host.
	if body, code := proxyGet(t, front.URL, host); code != 200 || body != "ok" {
		t.Fatalf("member wake: got %d %q", code, body)
	}
	// Marked requests are served while awake…
	if code := markedGet(); code != 200 {
		t.Fatalf("marked request while awake: %d, want 200", code)
	}
	// …but keep stamping nothing: sidecar traffic riding through the whole
	// idle window must not delay the sleep.
	clk.advance(6 * time.Second)
	if code := markedGet(); code != 200 {
		t.Fatalf("marked request mid-window: %d, want 200", code)
	}
	clk.advance(5 * time.Second) // 11s since the member request
	a.sweepOnce()
	waitFor(t, "hibernation beneath sidecar traffic", func() bool { return a.entryCount() == 0 })

	// Asleep: marked requests are refused and wake NOTHING — this is also the
	// outsider-spoof case, and it's harmless.
	if code := markedGet(); code != 503 {
		t.Fatalf("marked request while asleep: %d, want 503", code)
	}
	if a.entryCount() != 0 {
		t.Fatal("a marked request created an entry (a wake) for a sleeping host")
	}
	if _, _, starts := b.counts(); starts != 1 {
		t.Fatalf("starts = %d after marked requests, want 1 (no sidecar-triggered wake)", starts)
	}
	// An unmarked request still wakes it, and the marker never reached the
	// backend on any request.
	if _, code := proxyGet(t, front.URL, host); code != 200 {
		t.Fatalf("member re-wake: %d, want 200", code)
	}
	if _, _, starts := b.counts(); starts != 2 {
		t.Fatalf("starts = %d after member re-wake, want 2", starts)
	}
	if leaked.Load() {
		t.Fatal("the sidecar marker leaked through the proxy to the backend")
	}
}

func TestHibernateConfigFromEnv(t *testing.T) {
	t.Setenv("LCHAT_HIBERNATE_BUCKET", "")
	if cfg, err := hibernateConfigFromEnv(); err != nil || cfg != nil {
		t.Fatalf("unset bucket: got %+v, %v; want nil, nil (feature off)", cfg, err)
	}

	t.Setenv("LCHAT_HIBERNATE_BUCKET", "b")
	t.Setenv("LCHAT_HIBERNATE_PREFIX", "/instances/")
	t.Setenv("LCHAT_HIBERNATE_IDLE", "90s")
	cfg, err := hibernateConfigFromEnv()
	if err != nil {
		t.Fatalf("config: %v", err)
	}
	if cfg.Bucket != "b" || cfg.Prefix != "instances" || cfg.Idle != 90*time.Second {
		t.Fatalf("got %+v", cfg)
	}
	if got := cfg.hibernatePrefix("x.example.com"); got != "instances/x.example.com" {
		t.Fatalf("prefix: %q", got)
	}

	t.Setenv("LCHAT_HIBERNATE_IDLE", "")
	cfg, err = hibernateConfigFromEnv()
	if err != nil || cfg.Idle != 15*time.Minute {
		t.Fatalf("default idle: %+v, %v", cfg, err)
	}

	t.Setenv("LCHAT_HIBERNATE_IDLE", "banana")
	if _, err := hibernateConfigFromEnv(); err == nil {
		t.Fatal("bad idle duration should error, not default silently")
	}
}

// --- ensureLiveLink (the controld-mediated fix for mesh-replicated routes) ---

// ensureLiveLinkTestEnv redirects optDir/selfPath into temp dirs, same seam
// controld_test.go's controlTestEnv uses, so ensureSelfStaged/pointLive run
// for real against throwaway paths.
func ensureLiveLinkTestEnv(t *testing.T) {
	t.Helper()
	oldOpt, oldSelf := optDir, selfPath
	t.Cleanup(func() { optDir, selfPath = oldOpt, oldSelf })
	optDir = t.TempDir()
	stub := filepath.Join(t.TempDir(), "stub")
	if err := os.WriteFile(stub, []byte("#!/bin/sh\n"), 0o755); err != nil {
		t.Fatal(err)
	}
	selfPath = func() (string, error) { return stub, nil }
}

// Already-materialized: the common case for every existing box A instance —
// neither the direct path nor materialize is ever invoked.
func TestEnsureLiveLinkAlreadyMaterialized(t *testing.T) {
	ensureLiveLinkTestEnv(t)
	ver, err := ensureSelfStaged()
	if err != nil {
		t.Fatal(err)
	}
	if err := pointLive("existing.example.com", ver); err != nil {
		t.Fatal(err)
	}
	called := false
	materialize := func(context.Context, string) error { called = true; return nil }
	if err := ensureLiveLink(context.Background(), "existing.example.com", materialize); err != nil {
		t.Fatalf("ensureLiveLink: %v", err)
	}
	if called {
		t.Error("materialize was called for an already-materialized host")
	}
}

// No materialize hook configured (no control token — the common case for any
// box that hasn't opted in): falls back to the direct calls, today's
// long-standing behavior.
func TestEnsureLiveLinkFallsBackWithoutMaterialize(t *testing.T) {
	ensureLiveLinkTestEnv(t)
	if err := ensureLiveLink(context.Background(), "fresh.example.com", nil); err != nil {
		t.Fatalf("ensureLiveLink: %v", err)
	}
	if _, err := os.Stat(liveLink("fresh.example.com")); err != nil {
		t.Fatalf("live symlink was not created: %v", err)
	}
}

// Missing symlink + materialize configured: routed through materialize, not
// the direct (sandboxed-and-failing, in production) calls.
func TestEnsureLiveLinkUsesMaterializeWhenConfigured(t *testing.T) {
	ensureLiveLinkTestEnv(t)
	var gotHost string
	materialize := func(_ context.Context, host string) error {
		gotHost = host
		// The real controld handler stages + points live; a fake only needs
		// to prove IT was the one asked, and that its result is honored.
		ver, err := ensureSelfStaged()
		if err != nil {
			return err
		}
		return pointLive(host, ver)
	}
	if err := ensureLiveLink(context.Background(), "seeded.11ch.at", materialize); err != nil {
		t.Fatalf("ensureLiveLink: %v", err)
	}
	if gotHost != "seeded.11ch.at" {
		t.Fatalf("materialize called with %q, want seeded.11ch.at", gotHost)
	}
	if _, err := os.Stat(liveLink("seeded.11ch.at")); err != nil {
		t.Fatalf("live symlink was not created via materialize: %v", err)
	}
}

// A materialize failure (e.g. controld unreachable) must propagate — the
// exact live symptom found 2026-07-13 (a 503 "instance failed to wake"),
// never silently swallowed into "looks fine, wakes anyway."
func TestEnsureLiveLinkPropagatesMaterializeError(t *testing.T) {
	ensureLiveLinkTestEnv(t)
	wantErr := fmt.Errorf("controld materialize seeded.11ch.at: HTTP 502")
	materialize := func(context.Context, string) error { return wantErr }
	err := ensureLiveLink(context.Background(), "seeded.11ch.at", materialize)
	if err == nil {
		t.Fatal("expected an error, got nil")
	}
	if _, statErr := os.Stat(liveLink("seeded.11ch.at")); statErr == nil {
		t.Fatal("live symlink should not exist after a failed materialize")
	}
}