package host

import (
	"bytes"
	"context"
	"crypto/ed25519"
	"encoding/base64"
	"encoding/json"
	"errors"
	"io"
	"net/http"
	"net/http/httptest"
	"net/url"
	"os"
	"path/filepath"
	"reflect"
	"slices"
	"strings"
	"testing"
	"time"

	"github.com/carlosframework/platform/internal/activator"
	"github.com/carlosframework/platform/internal/appconfig"
	"github.com/carlosframework/platform/internal/console"
	"github.com/carlosframework/platform/internal/provision"
	"github.com/carlosframework/platform/internal/registry"
	"github.com/carlosframework/platform/internal/release"
	"github.com/carlosframework/platform/internal/rollout"
	"github.com/carlosframework/platform/internal/status"
)

// writeFailureBook writes recs to dd's failure book through the book itself,
// then retries each host in retried, as an armed run would have left them.
func writeFailureBook(t *testing.T, dd string, recs []rollout.FailureRecord, retried map[string][]string) {
	t.Helper()
	b, err := rollout.OpenFailureBook(dd)
	if err != nil {
		t.Fatal(err)
	}
	for _, r := range recs {
		if _, err := b.Record(r); err != nil {
			t.Fatal(err)
		}
		for _, h := range retried[r.Build] {
			if _, err := b.RetryHost(r.Account, r.App, r.Build, h); err != nil {
				t.Fatal(err)
			}
		}
	}
}

// bookSnapshot is the book file's bytes and modification time, so a test can
// show a declined box never wrote it.
type bookSnapshot struct {
	raw []byte
	mod time.Time
}

func snapshotBook(t *testing.T, dd string) bookSnapshot {
	t.Helper()
	p := filepath.Join(dd, rollout.FailureFile)
	raw, err := os.ReadFile(p)
	if err != nil {
		t.Fatal(err)
	}
	fi, err := os.Stat(p)
	if err != nil {
		t.Fatal(err)
	}
	return bookSnapshot{raw, fi.ModTime()}
}

func (s bookSnapshot) unchanged(t *testing.T, dd string) {
	t.Helper()
	now := snapshotBook(t, dd)
	if !bytes.Equal(now.raw, s.raw) || !now.mod.Equal(s.mod) {
		t.Fatalf("a declined box wrote the failure book:\nbefore %s\nafter  %s", s.raw, now.raw)
	}
}

// declineWithBook arms svc as runAgent does on a box with no delegated cgroup
// subtree: swapArming declined the flag, the Service was built without
// generations, and armRollout runs with CARLOS_SWAP=overlap set. The store
// must never be opened.
func declineWithBook(t *testing.T, svc *activator.Service, reg *registry.Registry, dd string) {
	t.Helper()
	_, pub, err := release.GenerateKey()
	if err != nil {
		t.Fatal(err)
	}
	declineRollout(rolloutNoDelegation)
	opened := false
	m := armRollout(context.Background(), overlapEnv(pub), svc, reg, dd, nil,
		func(context.Context) (release.Store, error) { opened = true; return nil, errors.New("unused") })
	if m != nil || armedRollout() != nil || opened {
		t.Fatal("a declined box armed the scheduler or opened the store")
	}
	if activator.GenerationsEnabled() || svc.GenerationsOn() {
		t.Fatal("a declined box turned generations on")
	}
}

// managedRoute is a managed, hibernating exec instance of bab/blog installed
// on ver.
func managedRoute(host, ver string) registry.Route {
	rt := execRoute(host, "bab", "blog")
	rt.Hibernate, rt.Version = true, ver
	return rt
}

func isTodaysWake(p activator.WakePlan) bool {
	return p.Refuse == "" && p.Boot.StagedPath == "" && p.Check == nil && !p.FirstWake && p.Fallback == nil
}

// Final review F6 (c): a declined box with no failure records, the book
// absent or empty, is exactly today's box: no activator hook, every leg
// today's, and the status says only why the flag was declined.
func TestDeclinedWithNoRecordsIsToday(t *testing.T) {
	for name, body := range map[string]string{
		"absent book": "",
		"empty book":  `{"schema": 1, "records": []}`,
	} {
		t.Run(name, func(t *testing.T) {
			t.Setenv("CARLOS_OPT_DIR", t.TempDir())
			svc, reg, dd := rolloutFixture(t, false)
			if body != "" {
				if err := os.WriteFile(filepath.Join(dd, rollout.FailureFile), []byte(body), 0o644); err != nil {
					t.Fatal(err)
				}
			}
			armAlways(svc, reg, dd)
			declineWithBook(t, svc, reg, dd)
			if svc.Planner != nil || svc.OnGen != nil || svc.OnDrift != nil || svc.CoalesceRecycle != nil {
				t.Fatal("a declined box with no records set an activator hook")
			}
			if heldHolds() != nil || rolloutUnavailable() != "" {
				t.Fatal("a declined box with no records holds or refuses something")
			}
			if r := rolloutStatusProbe(); r == nil || r.Unavailable != rolloutNoDelegation || r.FailureBook != "" {
				t.Fatalf("status = %+v, want only the decline code %q", r, rolloutNoDelegation)
			}
			legsAreTodays(t, svc, reg)
			if body == "" {
				if _, err := os.Stat(filepath.Join(dd, rollout.FailureFile)); !errors.Is(err, os.ErrNotExist) {
					t.Fatalf("a declined box created the failure book: %v", err)
				}
			}
		})
	}
}

// Final review F6 (a): a declined box with records refuses a managed wake
// onto a build they hold for that instance, with fixed text, and wakes
// everything else as today. The book is never written.
func TestDeclinedBoxRefusesAWakeOntoAHeldBuild(t *testing.T) {
	optDir := t.TempDir()
	t.Setenv("CARLOS_OPT_DIR", optDir)
	svc, reg, dd := rolloutFixture(t, false)
	acct := release.RouteAccount(managedRoute("x", ""))
	writeFailureBook(t, dd, []rollout.FailureRecord{{
		Account: acct, App: "blog", Build: "v2", Host: "sib.example.com", Cause: "adopt", Mode: "hold", Reason: "health",
		HeldDown: []string{"down.example.com", "downgood.example.com"},
	}}, map[string][]string{"v2": {"retried.example.com"}})
	before := snapshotBook(t, dd)
	declineWithBook(t, svc, reg, dd)
	if svc.Planner == nil {
		t.Fatal("no Planner: a wake onto a held build runs from the live link")
	}
	if svc.OnGen != nil || svc.OnDrift != nil || svc.CoalesceRecycle != nil || rolloutUnavailable() != "" {
		t.Fatal("honouring records changed more than the wake: the box must otherwise stay on today's paths")
	}

	good := managedRoute("good.example.com", "v2")
	good.GoodVersion, good.GoodDigest = "v2", "sha256:good"
	// A held instance has a good build (v1) to have fallen back to: spec
	// §8.5's bootstrap rule refuses on a record only then.
	blocked := managedRoute("a.example.com", "v2")
	blocked.GoodVersion, blocked.GoodDigest = "v1", "sha256:v1"
	// No good build, installed build blocked, not held down: boots as today.
	noGood := managedRoute("nogood.example.com", "v2")
	// Held down with no good build: still refused.
	downNoGood := managedRoute("down.example.com", "v2")
	downGood := managedRoute("downgood.example.com", "v2")
	downGood.GoodVersion, downGood.GoodDigest = "v2", "sha256:good"
	unit := managedRoute("unit.example.com", "v2")
	unit.Unit = "blog.service"
	// Installed v1, but root has already repointed live/<host> at v2 and
	// not yet stamped the row: the wake would exec v2.
	repointed := managedRoute("repointed.example.com", "v1")
	repointed.GoodVersion, repointed.GoodDigest = "v1", "sha256:v1"
	// The same repoint on an instance with no good build boots as today.
	repointedNoGood := managedRoute("repointednogood.example.com", "v1")
	if err := release.StageArtifact(optDir, acct, "v2", "blog", []byte("v2")); err != nil {
		t.Fatal(err)
	}
	for _, h := range []string{repointed.Host, repointedNoGood.Host} {
		if err := release.PointLive(optDir, h, acct, "v2", "blog"); err != nil {
			t.Fatal(err)
		}
	}
	for _, tc := range []struct {
		rt     registry.Route
		refuse string
	}{
		{blocked, refuseNoSwaps},
		{noGood, ""},
		{repointedNoGood, ""},
		{downNoGood, "build v2 failed in hold mode on this instance, so it stays down until you roll back or retry the build"},
		{downGood, "build v2 failed in hold mode on this instance, so it stays down until you roll back or retry the build"},
		{repointed, refuseNoSwaps},
		{good, ""},
		{managedRoute("retried.example.com", "v2"), ""},
		{managedRoute("old.example.com", "v1"), ""},
		{managedRoute("new.example.com", ""), ""},
		{unit, ""},
	} {
		p := svc.Planner(context.Background(), tc.rt)
		if tc.refuse == "" {
			if !isTodaysWake(p) {
				t.Errorf("%s: planned %+v, want today's wake", tc.rt.Host, p)
			}
			continue
		}
		if p.Refuse != tc.refuse {
			t.Errorf("%s: refusal %q, want %q", tc.rt.Host, p.Refuse, tc.refuse)
		}
	}
	if r := rolloutStatusProbe(); r == nil || r.Unavailable != rolloutNoDelegation || r.FailureBook != rolloutBookHonoured {
		t.Fatalf("status = %+v, want the decline code %q and the book %q", r, rolloutNoDelegation, rolloutBookHonoured)
	}
	before.unchanged(t, dd)
}

// Final review F6 (b), the legs handed fakes: the restart leg and config
// sync's restart leave a held instance running, and every other host gets
// today's bounce. A held instance's config ack never claims a restart.
func TestDeclinedLegsLeaveAHeldInstanceRunning(t *testing.T) {
	t.Setenv("CARLOS_OPT_DIR", t.TempDir())
	svc, reg, dd := rolloutFixture(t, false)
	acct := release.RouteAccount(managedRoute("x", ""))
	writeFailureBook(t, dd, []rollout.FailureRecord{{Account: acct, App: "blog", Build: "v2", Host: "sib.example.com"}}, nil)
	before := snapshotBook(t, dd)
	held, sleeper, free := managedRoute("held.example.com", "v2"), managedRoute("sleeper.example.com", "v2"), managedRoute("free.example.com", "v1")
	// Held and asleep only because they have a good build to have kept; an
	// instance with none boots its blocked build as today, so it is free.
	held.GoodVersion, held.GoodDigest = "v1", "sha256:v1"
	sleeper.GoodVersion, sleeper.GoodDigest = "v1", "sha256:v1"
	noGood := managedRoute("nogood.example.com", "v2")
	for _, rt := range []registry.Route{held, sleeper, free, noGood} {
		if err := reg.Upsert(rt); err != nil {
			t.Fatal(err)
		}
	}
	declineWithBook(t, svc, reg, dd)

	// The restart leg.
	ctx, s := genKit(t)
	var bounced []string
	bounce := func(h string) bool { bounced = append(bounced, h); return true }
	mem := newRestartMemory()
	routes := []registry.Route{held, free, noGood}
	askRestart(t, ctx, s, "bab", "blog")
	restartRequestLeg(ctx, routes, s, t.TempDir(), bounce, nil, mem)
	askRestart(t, ctx, s, "bab", "blog")
	if !restartRequestLeg(ctx, routes, s, t.TempDir(), bounce, nil, mem) {
		t.Fatal("the restart leg did not ask for the root pass")
	}
	slices.Sort(bounced)
	if !slices.Equal(bounced, []string{"free.example.com", "nogood.example.com"}) {
		t.Fatalf("restart bounced %v, want only the instances no record holds", bounced)
	}

	// Config sync, end to end over the real pass.
	store := release.NewDirStore(t.TempDir())
	privText, _, err := appconfig.GenerateKey()
	if err != nil {
		t.Fatal(err)
	}
	priv, err := appconfig.ParsePrivateKey(privText, "test")
	if err != nil {
		t.Fatal(err)
	}
	if err := appconfig.SetEnv(ctx, store, "bab", "blog", map[string]string{"LOG_LEVEL": "debug"}); err != nil {
		t.Fatal(err)
	}
	var restarted []string
	cb, ok := rolloutConfigCallbacks(ctx, store, configLegs{
		awake:   func(h string) bool { return h != sleeper.Host },
		restart: func(h string) error { restarted = append(restarted, h); return nil },
	})
	if !ok {
		t.Fatal("config sync took today's SyncExec, which would bounce the held instance")
	}
	acks := map[string]appconfig.ConfigAck{}
	cb.Ack = func(a appconfig.ConfigAck) { acks[a.Host] = a }
	if err := appconfig.SyncExecWithCallbacks(ctx, reg, store, priv, t.TempDir(), "", cb); err != nil {
		t.Fatal(err)
	}
	slices.Sort(restarted)
	if !slices.Equal(restarted, []string{free.Host, noGood.Host}) {
		t.Fatalf("config sync restarted %v, want only the instances no record holds", restarted)
	}
	if a := acks[held.Host]; a.Restarted || a.Hibernating || a.RestartError != rolloutRestartRefused {
		t.Fatalf("held awake ack = %+v, want RestartError %q", a, rolloutRestartRefused)
	}
	if a := acks[sleeper.Host]; a.Restarted || !a.Hibernating || a.RestartError != "" {
		t.Fatalf("held sleeper ack = %+v, want delivered on wake", a)
	}
	if a := acks[noGood.Host]; !a.Restarted || a.Hibernating || a.RestartError != "" {
		t.Fatalf("no-good ack = %+v, want today's restarted", a)
	}
	if a := acks[free.Host]; !a.Restarted || a.Hibernating || a.RestartError != "" {
		t.Fatalf("free ack = %+v, want today's restarted", a)
	}
	before.unchanged(t, dd)
}

// Final review F6 (a) and (b) on a real activator, the converge leg and the
// adopt bounce as fastConvergeLoop and the adopt pass build them. Root
// installs a build a failure record holds: neither leg stops the instance
// still serving the old build, and when something else does stop it, the
// next wake is refused rather than exec the held build from the live link.
func TestDeclinedBoxKeepsAHeldBuildOffARealActivator(t *testing.T) {
	prev := activator.GenerationsEnabled()
	activator.SetGenerations(false)
	t.Cleanup(func() { activator.SetGenerations(prev) })
	resetRollout(t)
	env := newFastFixture(t)
	t.Setenv("CARLOS_OPT_DIR", env.optDir)
	host := "warm.test"
	// GoodVersion v1: a record holds a build only for an instance that has a
	// good build (the bootstrap rule), and this one has served v1.
	rt := registry.Route{Host: host, Kind: "instance", App: "fastapp", Account: "bab", Channel: "edge",
		Status: "ready", Hibernate: true, Version: "v1", GoodVersion: "v1", GoodDigest: "sha256:v1"}
	if err := env.reg.Upsert(rt); err != nil {
		t.Fatal(err)
	}
	env.stageLive(t, host, "v1")
	writeFailureBook(t, env.dataDir, []rollout.FailureRecord{{Account: release.RouteAccount(rt), App: "fastapp", Build: "v2", Host: "sibling.test"}}, nil)
	backend := &versionedBackend{optDir: env.optDir}
	svc := activator.New(activator.Config{Bucket: "test-bucket", Idle: time.Hour}, backend, env.reg)
	svc.Routes = env.reg
	t.Cleanup(func() {
		sctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
		defer cancel()
		svc.Shutdown(sctx)
	})
	declineWithBook(t, svc, env.reg, env.dataDir)
	request := func() int {
		rec := httptest.NewRecorder()
		proxy := http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusNoContent) })
		if !svc.ServeHTTP(rec, httptest.NewRequest("GET", "http://"+host+"/", nil), rt, proxy) {
			t.Fatal("the activator did not take the request")
		}
		return rec.Code
	}
	if code := request(); code != http.StatusNoContent {
		t.Fatalf("wake onto v1 = %d", code)
	}

	// Root installs v2: the link first, then the stamp.
	env.stageLive(t, host, "v2")
	rt.Version = "v2"
	if err := env.reg.Upsert(rt); err != nil {
		t.Fatal(err)
	}
	leg := convergeLegFor(svc, env.reg)
	leg()
	restart := makeRestartRoute(env.reg, func(string) {}, unitOnly(func(u string) error {
		t.Fatalf("named-unit arm fired for %q", u)
		return nil
	}), adoptBounceFn(svc))
	if err := restart(host); err != nil {
		t.Fatal(err)
	}
	if !svc.Owns(host) {
		t.Fatal("a leg stopped the instance serving v1 into a wake onto held v2")
	}
	if code := request(); code != http.StatusNoContent {
		t.Fatalf("request to the running instance = %d", code)
	}
	if got := backend.started(); !reflect.DeepEqual(got, []string{"v1"}) {
		t.Fatalf("spawned builds = %v, want only v1", got)
	}

	// The sweep's own drift cycle asks the launch gate too (Task 9 fix round
	// 1); whatever else stops the instance (here a bounce), its next wake is
	// refused.
	svc.Bounce(host)
	deadline := time.Now().Add(10 * time.Second)
	for svc.Owns(host) {
		if time.Now().After(deadline) {
			t.Fatal("the bounce never stopped the instance")
		}
		time.Sleep(5 * time.Millisecond)
	}
	if code := request(); code == http.StatusNoContent {
		t.Fatal("the wake onto held v2 was served")
	}
	if got := backend.started(); !reflect.DeepEqual(got, []string{"v1"}) {
		t.Fatalf("spawned builds = %v: held v2 was exec'd from the live link", got)
	}
}

// Task 5 rule 5: a declined box honours the records reconciled from an
// intent an armed run left, and saves them; a save that fails is logged and
// the records are honoured all the same.
func TestDeclinedBoxHonoursAReconciledIntent(t *testing.T) {
	for _, canSave := range []bool{true, false} {
		t.Run(map[bool]string{true: "saved", false: "not saved"}[canSave], func(t *testing.T) {
			if !canSave && os.Geteuid() == 0 {
				t.Skip("running as root: a read-only directory does not fail a save")
			}
			t.Setenv("CARLOS_OPT_DIR", t.TempDir())
			svc, reg, dd := rolloutFixture(t, false)
			acct := release.RouteAccount(managedRoute("x", ""))
			leaveIntent(t, dd, "down.example.com", rollout.Intent{Account: acct, App: "blog", Build: "v2", Mode: "hold", ID: "a9"})
			if !canSave {
				if err := os.Chmod(dd, 0o500); err != nil {
					t.Fatal(err)
				}
				t.Cleanup(func() { os.Chmod(dd, 0o700) })
			}
			declineWithBook(t, svc, reg, dd)
			if heldHolds() == nil || svc.Planner == nil || rolloutUnavailable() != "" {
				t.Fatalf("the reconciled record was not honoured (unavailable %q)", rolloutUnavailable())
			}
			if p := svc.Planner(context.Background(), managedRoute("down.example.com", "v2")); p.Refuse == "" {
				t.Fatalf("plan %+v: want the held-down instance refused", p)
			}
			os.Chmod(dd, 0o700)
			b, err := rollout.OpenFailureBook(dd)
			if err != nil {
				t.Fatal(err)
			}
			if saved := b.Reconciled() == 0; saved != canSave {
				t.Fatalf("book saved %v, want %v", saved, canSave)
			}
		})
	}
}

// Final review F6 (d): a declined box whose failure book cannot be read
// fails closed, as refuseManaged does on an armed box: every managed wake is
// refused with the fixed text, and the bounce legs stand down. The status
// keeps the decline code and says why starts are refused. The book is not
// rewritten.
func TestDeclinedBoxWithAnUnreadableBookFailsClosed(t *testing.T) {
	for name, body := range map[string]string{
		"not json":   "{not json",
		"bad schema": `{"schema": 9}`,
	} {
		t.Run(name, func(t *testing.T) {
			t.Setenv("CARLOS_OPT_DIR", t.TempDir())
			svc, reg, dd := rolloutFixture(t, false)
			if err := os.WriteFile(filepath.Join(dd, rollout.FailureFile), []byte(body), 0o644); err != nil {
				t.Fatal(err)
			}
			before := snapshotBook(t, dd)
			declineWithBook(t, svc, reg, dd)
			if svc.Planner == nil {
				t.Fatal("no Planner: an unreadable book let managed wakes run from the live link")
			}
			if p := svc.Planner(context.Background(), managedRoute("a.example.com", "v1")); p.Refuse != refuseTexts[rolloutBookUnreadable] {
				t.Fatalf("managed wake planned %+v, want the fixed refusal %q", p, refuseTexts[rolloutBookUnreadable])
			}
			unit := managedRoute("u.example.com", "v1")
			unit.Unit = "blog.service"
			if p := svc.Planner(context.Background(), unit); !isTodaysWake(p) {
				t.Fatalf("an unmanaged wake was changed: %+v", p)
			}
			if adoptBounceFn(svc) != nil || fastConvergeLeg(svc, reg) != nil {
				t.Fatal("a bounce leg still stops instances whose next wake is refused")
			}
			if svc.OnDrift == nil || svc.CoalesceRecycle == nil || svc.CoalesceRecycle("bot.internal") {
				t.Fatal("the activator's own moves onto the live link are still armed")
			}
			if svc.OnGen != nil || heldHolds() != nil {
				t.Fatal("an unreadable book was honoured as records")
			}
			if r := rolloutStatusProbe(); r == nil || r.Unavailable != rolloutNoDelegation || r.FailureBook != rolloutBookUnreadable {
				t.Fatalf("status = %+v, want the decline code %q and the book %q", r, rolloutNoDelegation, rolloutBookUnreadable)
			}
			before.unchanged(t, dd)
		})
	}
}

// liveLaunchBackend is fakeWakeBackend's lease and restore around a REAL
// ExecBackend start, so a wake runs the production live-link spawn. before,
// when set, runs between the plan and the start: root's repoint landing
// there.
type liveLaunchBackend struct {
	fakeWakeBackend
	eb     *activator.ExecBackend
	before func()
}

func (b *liveLaunchBackend) Start(ctx context.Context, rt registry.Route) (activator.Proc, string, error) {
	if b.before != nil {
		b.before()
	}
	return b.eb.Start(ctx, rt)
}

func (b *liveLaunchBackend) LiveVersion(host string) (string, error) { return b.eb.LiveVersion(host) }

func (b *liveLaunchBackend) SetLiveExecGate(g func(registry.Route, string) error) {
	b.eb.SetLiveExecGate(g)
}

// Task 9 (Astra R1 #6, the R2/R3 declined residuals; plan review #6), brief
// (a): the plan-time refusal is repeated at the real launch. The wake plans
// with the live link on the good build v1, which no record holds; root
// repoints the link at v2, which a record holds for this instance, before
// the start. The start resolves the link, asks the gate about v2, and execs
// nothing.
func TestDeclinedBoxRefusesAHeldBuildAtLaunch(t *testing.T) {
	prev := activator.GenerationsEnabled()
	activator.SetGenerations(false)
	t.Cleanup(func() { activator.SetGenerations(prev) })
	resetRollout(t)
	env := newFastFixture(t)
	t.Setenv("CARLOS_OPT_DIR", env.optDir)
	host := "warm.test"
	run := t.TempDir()
	rt := registry.Route{Host: host, Kind: "instance", App: "fastapp", Account: "bab", Channel: "edge",
		Status: "ready", Hibernate: true, Version: "v1", GoodVersion: "v1", GoodDigest: "sha256:v1",
		Socket: filepath.Join(run, host+".sock"), DBPath: filepath.Join(env.dataDir, host+".db")}
	if err := env.reg.Upsert(rt); err != nil {
		t.Fatal(err)
	}
	env.stageLive(t, host, "v1")
	mark := filepath.Join(t.TempDir(), "v2.ran")
	if err := release.StageArtifact(env.optDir, "bab", "v2", "fastapp", []byte("#!/bin/sh\necho ran > '"+mark+"'\nexec sleep 30\n")); err != nil {
		t.Fatal(err)
	}
	writeFailureBook(t, env.dataDir, []rollout.FailureRecord{{Account: release.RouteAccount(rt), App: "fastapp", Build: "v2", Host: "sibling.test"}}, nil)
	eb := &activator.ExecBackend{Cfg: activator.Config{Bucket: "test-bucket", Idle: time.Hour},
		OptDir: env.optDir, DataDir: t.TempDir(), RunDirOverride: run}
	backend := &liveLaunchBackend{eb: eb}
	svc := activator.New(activator.Config{Bucket: "test-bucket", Idle: time.Hour}, backend, env.reg)
	svc.Routes = env.reg
	t.Cleanup(func() {
		sctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
		defer cancel()
		svc.Shutdown(sctx)
	})
	declineWithBook(t, svc, env.reg, env.dataDir)
	if eb.LiveExecGate == nil {
		t.Fatal("no LiveExecGate: the launch execs whatever the live link names by then")
	}
	if p := svc.Planner(context.Background(), rt); !isTodaysWake(p) {
		t.Fatalf("plan with the link on v1 = %+v, want today's wake", p)
	}
	backend.before = func() {
		if err := release.PointLive(env.optDir, host, "bab", "v2", "fastapp"); err != nil {
			t.Error(err)
		}
	}
	rec := httptest.NewRecorder()
	proxy := http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusNoContent) })
	if !svc.ServeHTTP(rec, httptest.NewRequest("GET", "http://"+host+"/", nil), rt, proxy) {
		t.Fatal("the activator did not take the request")
	}
	if rec.Code == http.StatusNoContent {
		t.Fatal("the wake onto held v2 was served")
	}
	time.Sleep(200 * time.Millisecond)
	if _, err := os.Stat(mark); err == nil {
		t.Fatal("held v2 was exec'd: the gate at the launch did not see the repoint")
	}
}

// Task 9 rule 2 and Correction 3: a declined box honours hold-ran markers
// as the armed planner does (decideHoldRan, planCheck), refusing where the
// armed box would refuse. A book holding only a marker is honoured. The
// gate at the launch answers as the plan does.
func TestDeclinedBoxHonoursHoldRanMarkers(t *testing.T) {
	held := rollout.HeldDownRefusal("v2")
	route := func(host, installed, good string) registry.Route {
		rt := managedRoute(host, installed)
		if good != "" {
			rt.GoodVersion, rt.GoodDigest = good, "sha256:"+good
		}
		return rt
	}
	acct := release.RouteAccount(managedRoute("x", ""))
	mark := func(t *testing.T, dd string, marks map[string]string) {
		t.Helper()
		b, err := rollout.OpenFailureBook(dd)
		if err != nil {
			t.Fatal(err)
		}
		for host, build := range marks {
			app := "blog"
			if host == "other.example.com" {
				app = "shop" // another app's marker under the same host name
			}
			if err := b.MarkHoldRan(host, rollout.HoldRan{Account: acct, App: app, Build: build}); err != nil {
				t.Fatal(err)
			}
		}
	}
	type want struct {
		rt     registry.Route
		refuse string
	}
	check := func(t *testing.T, svc *activator.Service, cases []want) {
		t.Helper()
		h := heldHolds()
		for _, tc := range cases {
			p := svc.Planner(context.Background(), tc.rt)
			if tc.refuse == "" && !isTodaysWake(p) {
				t.Errorf("%s (installed %s, good %s): planned %+v, want today's wake", tc.rt.Host, tc.rt.Version, tc.rt.GoodVersion, p)
			}
			if tc.refuse != "" && p.Refuse != tc.refuse {
				t.Errorf("%s (installed %s, good %s): refusal %q, want %q", tc.rt.Host, tc.rt.Version, tc.rt.GoodVersion, p.Refuse, tc.refuse)
			}
			got := ""
			if err := h.gate(tc.rt, tc.rt.Version); err != nil {
				got = err.Error()
			}
			if got != tc.refuse {
				t.Errorf("%s (installed %s, good %s): gate %q, want %q", tc.rt.Host, tc.rt.Version, tc.rt.GoodVersion, got, tc.refuse)
			}
		}
	}
	t.Run("only a marker", func(t *testing.T) {
		t.Setenv("CARLOS_OPT_DIR", t.TempDir())
		svc, reg, dd := rolloutFixture(t, false)
		mark(t, dd, map[string]string{"m.example.com": "v2", "mg.example.com": "v1", "other.example.com": "v2"})
		before := snapshotBook(t, dd)
		declineWithBook(t, svc, reg, dd)
		if heldHolds() == nil || svc.Planner == nil {
			t.Fatal("a book holding only a marker was not honoured")
		}
		if r := rolloutStatusProbe(); r == nil || r.FailureBook != rolloutBookHonoured {
			t.Fatalf("status = %+v, want the book %q", r, rolloutBookHonoured)
		}
		check(t, svc, []want{
			// The installed good build is refused while a marker of another
			// build stands: only a signed rollback clears it (Task 10).
			{route("m.example.com", "v1", "v1"), held},
			// The marked build starts: no record holds it.
			{route("m.example.com", "v2", "v1"), ""},
			// Any other build starts.
			{route("m.example.com", "v3", "v1"), ""},
			// With no good build there is nothing to refuse in its favour.
			{route("m.example.com", "v2", ""), ""},
			// A marker naming the installed good build itself lets it start.
			{route("mg.example.com", "v1", "v1"), ""},
			// Another app's marker under the host's name says nothing here.
			{route("other.example.com", "v1", "v1"), ""},
			{route("free.example.com", "v1", "v1"), ""},
		})
		before.unchanged(t, dd)
	})
	t.Run("a marker and records", func(t *testing.T) {
		t.Setenv("CARLOS_OPT_DIR", t.TempDir())
		svc, reg, dd := rolloutFixture(t, false)
		writeFailureBook(t, dd, []rollout.FailureRecord{
			{Account: acct, App: "blog", Build: "v1", Host: "sib.example.com"},
			{Account: acct, App: "blog", Build: "v2", Host: "sib.example.com"},
			{Account: acct, App: "blog", Build: "v3", Host: "sib.example.com"},
			{Account: acct, App: "blog", Build: "v4", Host: "sib.example.com", HeldDown: []string{"m.example.com"}},
		}, nil)
		mark(t, dd, map[string]string{"m.example.com": "v2", "mg.example.com": "v1"})
		declineWithBook(t, svc, reg, dd)
		check(t, svc, []want{
			// The marked build, blocked by a record while the route has a good
			// build.
			{route("m.example.com", "v2", "v1"), held},
			// The marked build that is the installed good build: served, so a
			// sibling's record does not hold it.
			{route("mg.example.com", "v1", "v1"), ""},
			// With no good build a record alone refuses nothing.
			{route("m.example.com", "v2", ""), ""},
			// Any other build a record blocks.
			{route("m.example.com", "v3", "v1"), held},
			// A hold-down still comes first, in its own build's words.
			{route("m.example.com", "v4", "v1"), rollout.HeldDownRefusal("v4")},
			{route("m.example.com", "v5", "v1"), ""},
		})
	})
}

// Task 9: a claims-backed wake on a declined box is gated against the build
// its launch snapshot resolved from the live link, before anything is asked
// of root. A box that honours nothing is today's.
func TestDeclinedBoxGatesAClaimsBackedWake(t *testing.T) {
	f := newAdminTokenFixture(t, "box-1")
	if err := os.WriteFile(f.envPath(), []byte("CARLOS_ADMIN_TOKEN=tok-0123456789abcdef0123456789abcdef\n"), 0o600); err != nil {
		t.Fatal(err)
	}
	rt := f.rt
	rt.Kind, rt.Status, rt.Channel, rt.Account, rt.App, rt.Hibernate, rt.Version = "instance", "ready", "edge", "bab", "jam", true, "v1"
	rt.GoodVersion, rt.GoodDigest = "v0", "sha256:v0"
	launch := claimServingLaunchSource(f.backend, f.tokens)
	if _, err := launch(t.Context(), rt); err != nil {
		t.Fatalf("launch with nothing honoured = %v, want today's", err)
	}
	svc, reg, dd := rolloutFixture(t, false)
	writeFailureBook(t, dd, []rollout.FailureRecord{{Account: release.RouteAccount(rt), App: "jam", Build: "v1", Host: "sib.example.com"}}, nil)
	declineWithBook(t, svc, reg, dd)
	if _, err := launch(t.Context(), rt); err == nil || err.Error() != refuseNoSwaps {
		t.Fatalf("launch of held v1 = %v, want %q", err, refuseNoSwaps)
	}
	served := rt
	served.GoodVersion, served.GoodDigest = "v1", "sha256:v1"
	if _, err := launch(t.Context(), served); err != nil {
		t.Fatalf("launch of the instance's own good build = %v, want it allowed", err)
	}
}

// Task 9 (brief (e)): with no records and no markers, or the flag off,
// LiveExecGate stays nil, so every spawn is today's.
func TestNoLiveExecGateUnlessADeclinedBoxHonoursSomething(t *testing.T) {
	for name, env := range map[string]func(string) string{"flag off": noEnv, "declined, empty book": nil} {
		t.Run(name, func(t *testing.T) {
			t.Setenv("CARLOS_OPT_DIR", t.TempDir())
			eb := &activator.ExecBackend{}
			svc, reg, dd := rolloutFixtureWith(t, false, &activator.SplitBackend{Exec: eb})
			if env == nil {
				declineWithBook(t, svc, reg, dd)
			} else {
				armRollout(context.Background(), env, svc, reg, dd, nil, unusedStore)
			}
			if eb.LiveExecGate != nil {
				t.Fatal("LiveExecGate set on a box that honours nothing")
			}
			legsAreTodays(t, svc, reg)
		})
	}
	t.Run("declined, a record", func(t *testing.T) {
		t.Setenv("CARLOS_OPT_DIR", t.TempDir())
		eb := &activator.ExecBackend{}
		svc, reg, dd := rolloutFixtureWith(t, false, &activator.SplitBackend{Exec: eb})
		writeFailureBook(t, dd, []rollout.FailureRecord{{Account: "a", App: "blog", Build: "v2", Host: "sib.example.com"}}, nil)
		declineWithBook(t, svc, reg, dd)
		if eb.LiveExecGate == nil {
			t.Fatal("a declined box honouring a record left LiveExecGate nil")
		}
	})
}

// Task 9 fix round 1 (ruling 4a): the marked-build rule holds whichever of
// the stamp and the live link names the marked build. The live link names
// the marked good build v1 while the stamp says v3 (root repointing): that
// is the build the marker says may run, and the instance served it, so a
// sibling's record does not hold it either.
func TestDeclinedBoxLetsTheMarkedGoodBuildStartFromTheLiveLink(t *testing.T) {
	optDir := t.TempDir()
	t.Setenv("CARLOS_OPT_DIR", optDir)
	svc, reg, dd := rolloutFixture(t, false)
	acct := release.RouteAccount(managedRoute("x", ""))
	writeFailureBook(t, dd, []rollout.FailureRecord{{Account: acct, App: "blog", Build: "v1", Host: "sib.example.com"}}, nil)
	b, err := rollout.OpenFailureBook(dd)
	if err != nil {
		t.Fatal(err)
	}
	if err := b.MarkHoldRan("mg.example.com", rollout.HoldRan{Account: acct, App: "blog", Build: "v1"}); err != nil {
		t.Fatal(err)
	}
	if err := release.StageArtifact(optDir, acct, "v1", "blog", []byte("v1")); err != nil {
		t.Fatal(err)
	}
	if err := release.PointLive(optDir, "mg.example.com", acct, "v1", "blog"); err != nil {
		t.Fatal(err)
	}
	declineWithBook(t, svc, reg, dd)
	rt := managedRoute("mg.example.com", "v3")
	rt.GoodVersion, rt.GoodDigest = "v1", "sha256:v1"
	if p := svc.Planner(context.Background(), rt); !isTodaysWake(p) {
		t.Fatalf("plan with the link on the marked good build = %+v, want today's wake", p)
	}
	if err := heldHolds().gate(rt, "v1"); err != nil {
		t.Fatalf("gate refused the marked good build the link names: %v", err)
	}
}

// Task 9 fix round 1 (ruling 4c): a claims move launch (a moves probe or
// landing) on a declined box is gated as a claims-backed wake is, after its
// snapshot resolves the link.
func TestDeclinedBoxGatesAClaimsMoveLaunch(t *testing.T) {
	f := newAdminTokenFixture(t, "box-1")
	if err := os.WriteFile(f.envPath(), []byte("CARLOS_ADMIN_TOKEN=tok-0123456789abcdef0123456789abcdef\n"), 0o600); err != nil {
		t.Fatal(err)
	}
	t.Setenv("CARLOS_SECRETS_KEY", base64.StdEncoding.EncodeToString(bytes.Repeat([]byte{7}, 32)))
	rt := f.rt
	rt.Kind, rt.Status, rt.Channel, rt.Account, rt.App, rt.Hibernate, rt.Version = "instance", "ready", "edge", "bab", "jam", true, "v1"
	rt.GoodVersion, rt.GoodDigest = "v0", "sha256:v0"
	launch := claimMoveLaunchSource(f.backend)
	if _, _, err := launch(t.Context(), rt); err != nil {
		t.Fatalf("move launch with nothing honoured = %v, want today's", err)
	}
	svc, reg, dd := rolloutFixture(t, false)
	writeFailureBook(t, dd, []rollout.FailureRecord{{Account: release.RouteAccount(rt), App: "jam", Build: "v1", Host: "sib.example.com"}}, nil)
	declineWithBook(t, svc, reg, dd)
	if _, _, err := launch(t.Context(), rt); err == nil || err.Error() != refuseNoSwaps {
		t.Fatalf("move launch of held v1 = %v, want %q", err, refuseNoSwaps)
	}
	served := rt
	served.GoodVersion, served.GoodDigest = "v1", "sha256:v1"
	if _, _, err := launch(t.Context(), served); err != nil {
		t.Fatalf("move launch of the instance's own good build = %v, want it allowed", err)
	}
}

// heldRouteIn upserts a managed, hibernating instance of bab/blog into reg,
// installed on installed with good as its good build.
func heldRouteIn(t *testing.T, reg *registry.Registry, host, installed, good string) registry.Route {
	t.Helper()
	rt := managedRoute(host, installed)
	if good != "" {
		rt.GoodVersion, rt.GoodDigest = good, "sha256:"+good
	}
	if err := reg.Upsert(rt); err != nil {
		t.Fatal(err)
	}
	return rt
}

// declinedHostReport is rolloutStatusProbe's entry for host, failing the
// test when the report is not a declined one.
func declinedHostReport(t *testing.T, host string) (status.HostRollout, bool) {
	t.Helper()
	r := rolloutStatusProbe()
	if r == nil || r.Unavailable != rolloutNoDelegation || r.FailureBook != rolloutBookHonoured {
		t.Fatalf("status = %+v, want the decline code %q and the book %q", r, rolloutNoDelegation, rolloutBookHonoured)
	}
	hr, ok := r.Hosts[host]
	return hr, ok
}

// Task 10 (ruling 194, plan review #5): a declined box publishes its holds,
// with the build and its restriction's epoch, and processes a verified
// signed retry against the book it honours. A grant for another build, or
// for an epoch the restriction has since moved past, clears nothing; the
// current grant clears the hold-down in memory, where the planner and the
// launch gate see it at once, and on disk.
func TestDeclinedBoxProcessesAVerifiedSignedRetry(t *testing.T) {
	t.Setenv("CARLOS_OPT_DIR", t.TempDir())
	svc, reg, dd := rolloutFixture(t, false)
	const host = "h.example.com"
	rt := heldRouteIn(t, reg, host, "v2", "v1")
	acct := release.RouteAccount(rt)
	writeFailureBook(t, dd, []rollout.FailureRecord{{
		Account: acct, App: "blog", Build: "v2", Host: "sib.example.com", Mode: "hold", Reason: "health",
	}}, nil)
	b, err := rollout.OpenFailureBook(dd)
	if err != nil {
		t.Fatal(err)
	}
	stale := b.RestrictionEpoch(acct, "blog", "v2", host)
	// The armed run saw v2 fail again, holding this host down, after a
	// grant for the first restriction could have been signed.
	if _, err := b.Record(rollout.FailureRecord{Account: acct, App: "blog", Build: "v2", Host: "sib2.example.com", Mode: "hold", Reason: "health", HeldDown: []string{host}}); err != nil {
		t.Fatal(err)
	}
	epoch := b.RestrictionEpoch(acct, "blog", "v2", host)
	if epoch == 0 || epoch == stale {
		t.Fatalf("epochs %d then %d: the second failure did not move the restriction", stale, epoch)
	}
	declineWithBook(t, svc, reg, dd)
	ctx := context.Background()
	held := rollout.HeldDownRefusal("v2")
	if p := svc.Planner(ctx, rt); p.Refuse != held {
		t.Fatalf("plan before the retry = %+v, want %q", p, held)
	}
	hr, ok := declinedHostReport(t, host)
	if !ok || hr.Account != acct || hr.App != "blog" || hr.Installed != "v2" || hr.HeldBy != "v2" || hr.HeldEpoch != epoch || hr.Attempt != status.AttemptDown ||
		hr.HeldBook == "" || hr.HeldBook != heldHolds().book.ID() || hr.RetryV != status.HostRetryV {
		t.Fatalf("published %+v (%v), want v2 holding %s down at epoch %d of the box's book", hr, ok, host, epoch)
	}

	mem, priv := retryMemory(t)
	mem.retryHost = rolloutRetryHost()
	if mem.retryHost == nil {
		t.Fatal("a declined box honouring its book has no retry path")
	}
	_, s := genKit(t)
	routes, err := reg.List()
	if err != nil {
		t.Fatal(err)
	}
	bounce := func(h string) bool { t.Errorf("%s was bounced for a retry", h); return true }
	restartRequestLeg(ctx, routes, s, dd, bounce, nil, mem) // baseline
	askRetry(t, ctx, s, priv, acct, "blog", host, "v9", hr.HeldBook, epoch)
	askRetry(t, ctx, s, priv, acct, "blog", host, "v2", hr.HeldBook, stale)
	restartRequestLeg(ctx, routes, s, dd, bounce, nil, mem)
	if p := svc.Planner(ctx, rt); p.Refuse != held {
		t.Fatalf("plan after a grant for another build and a stale grant = %+v, want %q still", p, held)
	}
	askRetry(t, ctx, s, priv, acct, "blog", host, "v2", hr.HeldBook, epoch)
	restartRequestLeg(ctx, routes, s, dd, bounce, nil, mem)
	if p := svc.Planner(ctx, rt); !isTodaysWake(p) {
		t.Fatalf("plan after the signed retry = %+v, want today's wake onto v2", p)
	}
	if err := heldHolds().gate(rt, "v2"); err != nil {
		t.Fatalf("the launch gate still refuses v2 after the retry: %v", err)
	}
	if hr, ok := declinedHostReport(t, host); ok {
		t.Fatalf("published %+v after the retry, want no hold", hr)
	}
	b, err = rollout.OpenFailureBook(dd)
	if err != nil {
		t.Fatal(err)
	}
	if b.HeldDown(acct, "blog", "v2", host) || b.Blocks(acct, "blog", "v2", host, false) {
		t.Fatal("the retry was not saved: the book on disk still holds the host")
	}
}

// writePreRuleBook writes dd's failure book as an agent before the
// crash-epoch hard limit left it (readiness round 2, Task 1): schema 2 with
// a book identity and per-record epochs, no generations and no
// durable_epochs. v2's record holds host down.
func writePreRuleBook(t *testing.T, dd, acct, host string) {
	t.Helper()
	raw, err := json.Marshal(map[string]any{
		"schema": 2,
		"book":   "old",
		"seq":    10,
		"records": []map[string]any{{"account": acct, "app": "blog", "build": "v2", "host": "sib.example.com",
			"mode": "hold", "reason": "health", "held_down": []string{host}, "epochs": map[string]uint64{"": 11, host: 11}}},
	})
	if err != nil {
		t.Fatal(err)
	}
	if err := os.WriteFile(filepath.Join(dd, rollout.FailureFile), raw, 0o644); err != nil {
		t.Fatal(err)
	}
}

// Readiness round 2, Task 1 (plan 10): a declined box saves a book that
// owes a save at startup, whatever reconciliation found (here a book an
// earlier agent wrote, whose identity the open rotates), and publishes its
// generation at once. When that save fails it publishes the hold with no
// generation, and the next periodic pass, with no pointer or grant
// arriving, saves the book once the disk recovers and publishes it.
func TestDeclinedBoxPublishesAGenerationOnceItsBookIsSaved(t *testing.T) {
	for _, diskFails := range []bool{false, true} {
		t.Run(map[bool]string{false: "saved at startup", true: "saved by the periodic pass"}[diskFails], func(t *testing.T) {
			if diskFails && os.Geteuid() == 0 {
				t.Skip("running as root: a read-only directory does not fail a save")
			}
			t.Setenv("CARLOS_OPT_DIR", t.TempDir())
			svc, reg, dd := rolloutFixture(t, false)
			const host = "h.example.com"
			rt := heldRouteIn(t, reg, host, "v2", "v1")
			acct := release.RouteAccount(rt)
			writePreRuleBook(t, dd, acct, host)
			if diskFails {
				if err := os.Chmod(dd, 0o500); err != nil {
					t.Fatal(err)
				}
				t.Cleanup(func() { os.Chmod(dd, 0o700) })
			}
			declineWithBook(t, svc, reg, dd)
			hr, ok := declinedHostReport(t, host)
			if !ok || hr.HeldBy != "v2" || hr.Attempt != status.AttemptDown || hr.HeldBook == "old" || hr.HeldBook == "" {
				t.Fatalf("published %+v (%v), want v2 holding %s down under a new book identity", hr, ok, host)
			}
			if diskFails {
				if hr.HeldEpoch != 0 {
					t.Fatalf("published generation %d while the book owes a save, want 0", hr.HeldEpoch)
				}
				if err := os.Chmod(dd, 0o700); err != nil {
					t.Fatal(err)
				}
				if hr, _ := declinedHostReport(t, host); hr.HeldEpoch != 0 {
					t.Fatalf("published generation %d before any save landed, want 0", hr.HeldEpoch)
				}
				p := &pointerReads{ptrs: map[string]*release.ChannelPointer{}, errs: map[string]error{"blog/stable": errors.New("no pointer")}}
				heldHolds().pass(context.Background(), p.read)
				hr, _ = declinedHostReport(t, host)
			}
			if hr.HeldEpoch == 0 {
				t.Fatalf("published %+v: want a generation once the book is saved", hr)
			}
			b, err := rollout.OpenFailureBook(dd)
			if err != nil {
				t.Fatal(err)
			}
			if b.ID() != hr.HeldBook || b.PublishedEpoch(acct, "blog", "v2", host) != hr.HeldEpoch {
				t.Fatalf("the book on disk is %q at %d, want the published %q at %d",
					b.ID(), b.PublishedEpoch(acct, "blog", "v2", host), hr.HeldBook, hr.HeldEpoch)
			}
		})
	}
}

// pointerReads is a fake signed pointer read keyed "<app>/<channel>", which
// counts the reads it is asked for.
type pointerReads struct {
	ptrs  map[string]*release.ChannelPointer
	errs  map[string]error
	reads []string
}

func (p *pointerReads) read(_ context.Context, _, app, channel string) (*release.ChannelPointer, error) {
	k := app + "/" + channel
	p.reads = append(p.reads, k)
	if err := p.errs[k]; err != nil {
		return nil, err
	}
	return p.ptrs[k], nil
}

// Task 10 rule 2 (`carlos promote --retry`): the declined box's pointer
// pass clears a record when its channel's signed pointer names the record's
// build with a newer PromotedAt. The pointer the build failed under, a
// pointer naming another build and a failed read clear nothing; an app with
// nothing to clear costs no read.
func TestDeclinedBoxPointerPassClearsARecordOnPromoteRetry(t *testing.T) {
	t.Setenv("CARLOS_OPT_DIR", t.TempDir())
	svc, reg, dd := rolloutFixture(t, false)
	rt := heldRouteIn(t, reg, "h.example.com", "v2", "v1")
	other := execRoute("s.example.com", "bab", "shop")
	other.Hibernate, other.Version, other.GoodVersion, other.GoodDigest = true, "z2", "z1", "sha256:z1"
	if err := reg.Upsert(other); err != nil {
		t.Fatal(err)
	}
	acct := release.RouteAccount(rt)
	failedUnder := time.Date(2026, 10, 1, 12, 0, 0, 0, time.UTC)
	writeFailureBook(t, dd, []rollout.FailureRecord{{
		Account: acct, App: "blog", Build: "v2", Host: "sib.example.com", Reason: "health", PromotedAt: failedUnder,
	}}, nil)
	declineWithBook(t, svc, reg, dd)
	ctx := context.Background()
	routes, err := reg.List()
	if err != nil {
		t.Fatal(err)
	}
	h := heldHolds()
	stillHeld := func(when string) {
		t.Helper()
		if p := svc.Planner(ctx, rt); p.Refuse != refuseNoSwaps {
			t.Fatalf("%s: plan = %+v, want %q", when, p, refuseNoSwaps)
		}
	}
	stillHeld("before any pointer")
	if hr, ok := declinedHostReport(t, rt.Host); !ok || hr.HeldBy != "v2" || hr.Attempt != status.AttemptFailed || hr.HeldEpoch == 0 {
		t.Fatalf("published %+v (%v), want v2 holding it as failed, with an epoch", hr, ok)
	}

	p := &pointerReads{ptrs: map[string]*release.ChannelPointer{}, errs: map[string]error{}}
	p.errs["blog/stable"] = errors.New("bucket blip")
	h.applyPointers(ctx, p.read, routes)
	stillHeld("after a failed read")
	delete(p.errs, "blog/stable")
	p.ptrs["blog/stable"] = &release.ChannelPointer{App: "blog", Channel: "stable", Version: "v2", PromotedAt: failedUnder}
	h.applyPointers(ctx, p.read, routes)
	stillHeld("after the pointer v2 failed under")
	p.ptrs["blog/stable"] = &release.ChannelPointer{App: "blog", Channel: "stable", Version: "v3", PromotedAt: failedUnder.Add(time.Hour)}
	h.applyPointers(ctx, p.read, routes)
	stillHeld("after a pointer naming another build")
	p.ptrs["blog/stable"] = &release.ChannelPointer{App: "blog", Channel: "stable", Version: "v2", PromotedAt: failedUnder.Add(time.Minute)}
	h.applyPointers(ctx, p.read, routes)
	if pl := svc.Planner(ctx, rt); !isTodaysWake(pl) {
		t.Fatalf("plan after promote --retry = %+v, want today's wake onto v2", pl)
	}
	for _, r := range p.reads {
		if strings.HasPrefix(r, "shop/") {
			t.Fatalf("reads %v: an app with nothing to clear was read", p.reads)
		}
	}
	b, err := rollout.OpenFailureBook(dd)
	if err != nil {
		t.Fatal(err)
	}
	if _, ok := b.Lookup(acct, "blog", "v2"); ok {
		t.Fatal("the cleared record was not saved")
	}
}

// Task 10 Correction 3 (Task 3's rule, ruling 205): the pointer pass also
// clears a hold-ran marker on a signed rollback, for an app with markers and
// no records: the route installs its good build and the channel's pointer
// names that build with a PromotedAt after the marker's At. The planner then
// lets the good build start. An older pointer, and a route not installing
// its good build, clear nothing.
func TestDeclinedBoxPointerPassClearsAMarkerOnASignedRollback(t *testing.T) {
	t.Setenv("CARLOS_OPT_DIR", t.TempDir())
	svc, reg, dd := rolloutFixture(t, false)
	rolled := heldRouteIn(t, reg, "m.example.com", "v1", "v1")
	notRolled := heldRouteIn(t, reg, "n.example.com", "v3", "v1")
	acct := release.RouteAccount(rolled)
	ranAt := time.Date(2026, 10, 1, 12, 0, 0, 0, time.UTC)
	b, err := rollout.OpenFailureBook(dd)
	if err != nil {
		t.Fatal(err)
	}
	for _, host := range []string{rolled.Host, notRolled.Host} {
		if err := b.MarkHoldRan(host, rollout.HoldRan{Account: acct, App: "blog", Build: "v2", At: ranAt}); err != nil {
			t.Fatal(err)
		}
	}
	epoch := b.RestrictionEpoch(acct, "blog", "v2", rolled.Host)
	declineWithBook(t, svc, reg, dd)
	ctx := context.Background()
	routes, err := reg.List()
	if err != nil {
		t.Fatal(err)
	}
	held := rollout.HeldDownRefusal("v2")
	if p := svc.Planner(ctx, rolled); p.Refuse != held {
		t.Fatalf("plan before the rollback = %+v, want %q", p, held)
	}
	if hr, ok := declinedHostReport(t, rolled.Host); !ok || hr.HeldBy != "v2" || hr.HeldEpoch != epoch || hr.Attempt != status.AttemptDown {
		t.Fatalf("published %+v (%v), want the marker's v2 holding the good build down at epoch %d", hr, ok, epoch)
	}

	h := heldHolds()
	p := &pointerReads{ptrs: map[string]*release.ChannelPointer{}}
	p.ptrs["blog/stable"] = &release.ChannelPointer{App: "blog", Channel: "stable", Version: "v1", PromotedAt: ranAt.Add(-time.Minute)}
	h.applyPointers(ctx, p.read, routes)
	if len(p.reads) == 0 {
		t.Fatal("an app with only markers was not read")
	}
	if pl := svc.Planner(ctx, rolled); pl.Refuse != held {
		t.Fatalf("plan after a pointer older than the marker = %+v, want %q still", pl, held)
	}
	p.ptrs["blog/stable"] = &release.ChannelPointer{App: "blog", Channel: "stable", Version: "v1", PromotedAt: ranAt.Add(time.Minute)}
	h.applyPointers(ctx, p.read, routes)
	if pl := svc.Planner(ctx, rolled); !isTodaysWake(pl) {
		t.Fatalf("plan after the signed rollback = %+v, want today's wake onto the good build", pl)
	}
	if hr, ok := declinedHostReport(t, rolled.Host); ok {
		t.Fatalf("published %+v after the rollback, want no hold", hr)
	}
	b, err = rollout.OpenFailureBook(dd)
	if err != nil {
		t.Fatal(err)
	}
	if _, ok := b.HoldRanOn(rolled.Host); ok {
		t.Fatal("the cleared marker was not saved")
	}
	if _, ok := b.HoldRanOn(notRolled.Host); !ok {
		t.Fatal("a marker on a route not installing its good build was cleared")
	}
}

// retryConsole is a console with a root key over store, its ops account
// owning app, with a CLI token for its owner: the shape the root package's
// e2e tests log in through (e2eDeviceLogin).
func retryConsole(t *testing.T, store release.Store, app, host string) (base, token string, pub ed25519.PublicKey) {
	t.Helper()
	seed, pubB64, err := release.GenerateKey()
	if err != nil {
		t.Fatal(err)
	}
	priv, err := release.ParseSigningKey(seed, "test")
	if err != nil {
		t.Fatal(err)
	}
	if pub, err = release.ParsePublicKey(pubB64, "test"); err != nil {
		t.Fatal(err)
	}
	dir := t.TempDir()
	cst, err := console.Open(filepath.Join(dir, "console.db"))
	if err != nil {
		t.Fatal(err)
	}
	t.Cleanup(func() { cst.Close() })
	var srv *console.Server
	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { srv.ServeHTTP(w, r) }))
	t.Cleanup(ts.Close)
	if srv, err = console.NewServer(console.Config{Origin: ts.URL, Keymail: "http://keymail.invalid",
		Registry: filepath.Join(dir, "registry.db"), Releases: store, ReleasePub: pub, Key: priv, AppScheme: "http"}, cst); err != nil {
		t.Fatal(err)
	}
	const addr = "paul@keymail.dev"
	now := time.Now()
	if err := cst.AddMember(console.OpsAccountID, addr, console.RoleOwner, now); err != nil {
		t.Fatal(err)
	}
	if _, err := cst.CreateAppIn(console.OpsAccountID, app, addr, now); err != nil {
		t.Fatal(err)
	}
	if err := provision.CreateRecord(context.Background(), store, provision.Record{Schema: 1, Host: host, Slug: strings.SplitN(host, ".", 2)[0],
		App: app, Account: release.OpsAccount, Region: "ie", Channel: "stable", CreatedAt: now.UTC(), CreatedBy: addr}); err != nil {
		t.Fatal(err)
	}
	post := func(u string, body any, cookie string, form url.Values) map[string]any {
		t.Helper()
		var req *http.Request
		if form != nil {
			req, _ = http.NewRequest(http.MethodPost, u, strings.NewReader(form.Encode()))
			req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
			req.Header.Set("Cookie", "carlos_console_session="+cookie)
		} else {
			raw, _ := json.Marshal(body)
			req, _ = http.NewRequest(http.MethodPost, u, bytes.NewReader(raw))
			req.Header.Set("Content-Type", "application/json")
		}
		res, err := http.DefaultClient.Do(req)
		if err != nil {
			t.Fatal(err)
		}
		defer res.Body.Close()
		raw, _ := io.ReadAll(res.Body)
		if res.StatusCode != http.StatusOK {
			t.Fatalf("POST %s = %d %s", u, res.StatusCode, raw)
		}
		out := map[string]any{}
		_ = json.Unmarshal(raw, &out)
		return out
	}
	start := post(ts.URL+"/auth/cli/start", map[string]string{"label": "test@harness"}, "", nil)
	session, err := cst.CreateSession(addr, "", now)
	if err != nil {
		t.Fatal(err)
	}
	if err := cst.RecordStepUp(session, addr, now, now); err != nil {
		t.Fatal(err)
	}
	post(ts.URL+"/auth/cli/approve", nil, session, url.Values{"code": {start["user_code"].(string)}, "action": {"approve"}, "csrf": {session}})
	poll := post(ts.URL+"/auth/cli/poll", map[string]string{"device_code": start["device_code"].(string)}, "", nil)
	token, _ = poll["token"].(string)
	if token == "" {
		t.Fatalf("no CLI token: %v", poll)
	}
	return ts.URL, token, pub
}

// Task 10 Correction 1 (plan review #5), from the API to the box: the
// declined box publishes its held host, the console's retry door signs a
// grant for it from that report, and the declined box's restart leg
// verifies the grant and clears the hold, so the next wake is today's.
func TestDeclinedBoxRetryFromTheConsoleDoorToTheRestartLeg(t *testing.T) {
	t.Setenv("CARLOS_OPT_DIR", t.TempDir())
	svc, reg, dd := rolloutFixture(t, false)
	const host = "h.blog.example.com"
	rt := heldRouteIn(t, reg, host, "v2", "v1")
	acct := release.RouteAccount(rt)
	writeFailureBook(t, dd, []rollout.FailureRecord{{
		Account: acct, App: "blog", Build: "v2", Host: "sib.example.com", Mode: "hold", Reason: "health", HeldDown: []string{host},
	}}, nil)
	declineWithBook(t, svc, reg, dd)
	ctx, s := genKit(t)
	base, token, pub := retryConsole(t, s, "blog", host)

	// The box's restart memory, seeded before the request, as at arming.
	mem := newRestartMemory()
	mem.pub = pub
	mem.retryHost = rolloutRetryHost()
	routes, err := reg.List()
	if err != nil {
		t.Fatal(err)
	}
	bounce := func(h string) bool { t.Errorf("%s was bounced for a retry", h); return true }
	restartRequestLeg(ctx, routes, s, dd, bounce, nil, mem)

	// The box's status, as the status loop publishes it.
	box := status.CollectBox(nil, status.Probes{MachineID: "box1"}, time.Now())
	box.Metrics = &status.BoxMetrics{RouteVersions: map[string]string{host: rt.Version}, Rollout: rolloutStatusProbe()}
	if err := status.WriteBox(ctx, s, box); err != nil {
		t.Fatal(err)
	}
	req, _ := http.NewRequest(http.MethodPost, base+"/api/cli/apps/"+release.OpsAccount+"/blog/instances/"+host+"/retry", strings.NewReader(`{}`))
	req.Header.Set("Content-Type", "application/json")
	req.Header.Set("Authorization", "Bearer "+token)
	res, err := http.DefaultClient.Do(req)
	if err != nil {
		t.Fatal(err)
	}
	raw, _ := io.ReadAll(res.Body)
	res.Body.Close()
	if res.StatusCode != http.StatusOK {
		t.Fatalf("the console's retry door = %d %s, want a signed grant for the declined box's hold", res.StatusCode, raw)
	}

	restartRequestLeg(ctx, routes, s, dd, bounce, nil, mem)
	if p := svc.Planner(ctx, rt); !isTodaysWake(p) {
		t.Fatalf("plan after the console's retry = %+v, want today's wake onto v2", p)
	}
	b, err := rollout.OpenFailureBook(dd)
	if err != nil {
		t.Fatal(err)
	}
	if b.HeldDown(acct, "blog", "v2", host) {
		t.Fatal("the console's retry was not saved")
	}
}