package host

import (
	"bytes"
	"context"
	"crypto/ed25519"
	"crypto/sha256"
	"encoding/hex"
	"encoding/json"
	"errors"
	"fmt"
	"os"
	"path/filepath"
	"slices"
	"strings"
	"testing"
	"time"

	"github.com/carlosframework/platform/internal/moves"
	"github.com/carlosframework/platform/internal/provision"
	"github.com/carlosframework/platform/internal/registry"
	"github.com/carlosframework/platform/internal/release"
)

// applyFixture is a fake box: temp opt/data/etc roots, a DirStore with a
// signed pointer, and a sysOps that records every call and answers like a
// healthy systemd. The candidate "binary" is answered by the fake run.
type applyFixture struct {
	t                   *testing.T
	a                   *systemApplier
	store               *release.DirStore
	priv                ed25519.PrivateKey
	pub                 ed25519.PublicKey
	calls               []string          // "systemctl is-active carlos-edge", "run <base> version", …
	answer              map[string]string // "version:<path>" → stdout; "plan:<path>" → JSON plan
	now                 time.Time
	edgeUp              bool
	nrestarts           string
	activeEnter         string
	probeErr            error
	edgeStartedThisBoot bool
}

func newApplyFixture(t *testing.T, channel, build string) *applyFixture {
	t.Helper()
	pub, priv, err := ed25519.GenerateKey(nil)
	if err != nil {
		t.Fatal(err)
	}
	f := &applyFixture{t: t, store: release.NewDirStore(t.TempDir()), priv: priv, answer: map[string]string{},
		now: time.Date(2026, 9, 6, 11, 0, 0, 0, time.UTC), edgeUp: true, nrestarts: "0", activeEnter: "1000", edgeStartedThisBoot: true}
	f.pub = pub
	f.a = &systemApplier{
		openStore: func(context.Context) (release.Store, ed25519.PublicKey, error) { return f.store, f.pub, nil },
		channel:   channel,
		optDir:    t.TempDir(), dataDir: t.TempDir(), etcDir: t.TempDir(), build: build,
		platform: "linux-arm64",
	}
	f.a.ops = sysOps{
		systemctl: func(args ...string) (string, error) {
			f.calls = append(f.calls, "systemctl "+strings.Join(args, " "))
			switch {
			case args[0] == "is-active" && args[1] == systemEdgeUnit:
				if f.edgeUp {
					return "active\n", nil
				}
				return "failed\n", errors.New("inactive")
			case args[0] == "show":
				return f.nrestarts + "\n" + f.activeEnter + "\n", nil
			}
			return "", nil
		},
		run: func(ctx context.Context, timeout time.Duration, name string, args ...string) (string, error) {
			f.calls = append(f.calls, "run "+filepath.Base(name)+" "+strings.Join(args, " "))
			if len(args) == 1 && args[0] == "version" {
				if v, ok := f.answer["version:"+name]; ok {
					return v, nil
				}
				// With no scripted answer the "binary" speaks for its own
				// bytes ("#!/bin/sh\necho carlos vN"), so a rename over it
				// changes what it reports — which is what the install does.
				if b, err := os.ReadFile(name); err == nil {
					if _, rest, ok := strings.Cut(string(b), "echo "); ok {
						return strings.SplitN(rest, "\n", 2)[0] + "\n", nil
					}
				}
				return "", errors.New("no such binary")
			}
			if len(args) >= 3 && args[0] == "bootstrap" && args[1] == "-root" {
				plan, ok := f.answer["plan:"+name]
				if !ok {
					return "", errors.New("bootstrap failed")
				}
				units := strings.Split(f.answer["units:"+name], " ")
				dir := filepath.Join(args[2], "etc", "systemd", "system")
				os.MkdirAll(dir, 0o755)
				for _, u := range units {
					if u != "" {
						os.WriteFile(filepath.Join(dir, u), []byte("[Unit]\n# "+filepath.Base(name)+"\n"), 0o644)
					}
				}
				os.MkdirAll(filepath.Join(args[2], "etc", "sysusers.d"), 0o755)
				os.WriteFile(filepath.Join(args[2], "etc", "sysusers.d", "carlos.conf"), []byte("g carlos-sockets -\n"), 0o644)
				os.MkdirAll(filepath.Join(args[2], "etc", "tmpfiles.d"), 0o755)
				os.WriteFile(filepath.Join(args[2], "etc", "tmpfiles.d", "carlos.conf"), []byte("d /run/carlos\n"), 0o644)
				os.MkdirAll(filepath.Join(args[2], "data"), 0o755)
				os.WriteFile(filepath.Join(args[2], "data", "registry.db"), nil, 0o640)
				return plan, nil
			}
			// The two config appliers the install shells out to.
			if b := filepath.Base(name); b == "systemd-sysusers" || b == "systemd-tmpfiles" {
				return "", nil
			}
			return "", errors.New("unexpected command")
		},
		now:                 func() time.Time { return f.now },
		sleep:               func(d time.Duration) { f.now = f.now.Add(d) },
		probeEdge:           func() error { return f.probeErr },
		edgeStartedThisBoot: func() bool { return f.edgeStartedThisBoot },
		logf:                t.Logf,
	}
	// The layout guard opens the registry only for a candidate that
	// predates the layout, and closes it; an empty one in the fake data dir
	// is a flat box.
	f.a.openRegistry = func() (*registry.Registry, error) {
		return registry.Open(filepath.Join(f.a.dataDir, "registry.db"))
	}
	os.MkdirAll(systemRootDir(f.a.optDir), 0o755)
	os.MkdirAll(filepath.Join(f.a.etcDir, "systemd", "system"), 0o755)
	// The on-disk binary answers `version` with build.
	os.WriteFile(systemBinaryPath(f.a.optDir), []byte("#!/bin/sh\necho carlos "+build+"\n"), 0o755)
	f.answer["version:"+systemBinaryPath(f.a.optDir)] = "carlos " + build + "\n"
	f.answer["plan:"+systemBinaryPath(f.a.optDir)] = `{"enabled":["carlos-edge.service","carlos-adopt.timer","carlos-adopt.path","carlos-stage.path","carlos-restart.path","carlos-system-update.path","carlos-system-recover.service"],"inert":["carlos-restore-verify.timer"]}`
	f.answer["units:"+systemBinaryPath(f.a.optDir)] = "carlos-edge.service carlos-adopt.timer carlos-adopt.path carlos-stage.path carlos-restart.path carlos-system-update.path carlos-system-update.service carlos-system-recover.service carlos-instance-control.service carlos-old-only.timer"
	f.a.participation = updateParticipationFixture(t)
	return f
}

// promote signs a pointer for f.a.channel naming version with body as the
// linux-arm64 artifact, stages body into the agent's cache, and writes the
// request the agent would have written. Returns the request.
func (f *applyFixture) promote(version string, body []byte, noop bool) systemRequest {
	f.t.Helper()
	sum := sha256.Sum256(body)
	rel := release.SystemRelease{Version: version, Artifacts: map[string]release.SystemArtifact{
		"linux-arm64": {Key: release.SystemArtifactKey(version, "linux-arm64"), SHA256: hex.EncodeToString(sum[:])}}}
	f.store.Put(context.Background(), rel.Artifacts["linux-arm64"].Key, body, "")
	p, err := release.PromoteSystem(context.Background(), f.store, release.RootSigner(f.priv), "router", f.a.channel, rel, release.SystemPromoteOptions{})
	if err != nil {
		f.t.Fatal(err)
	}
	req := systemRequest{Version: version, PromotedAt: p.PromotedAt, NoOp: noop, SeenAt: f.now}
	os.MkdirAll(systemDataDir(f.a.dataDir), 0o755)
	writeJSONDurable(systemRequestPath(f.a.dataDir), req, 0o644)
	if !noop {
		os.MkdirAll(filepath.Dir(systemCachePath(f.a.dataDir, version)), 0o755)
		os.WriteFile(systemCachePath(f.a.dataDir, version), body, 0o644)
	}
	return req
}

func (f *applyFixture) result() *systemResult { r, _ := readSystemResult(f.a.optDir); return r }

func (f *applyFixture) called(prefix string) bool {
	for _, c := range f.calls {
		if strings.HasPrefix(c, prefix) {
			return true
		}
	}
	return false
}

func TestApplyNoRequestIsNothingToDo(t *testing.T) {
	f := newApplyFixture(t, "stable", "v1")
	if err := f.a.Run(context.Background(), false); err != nil {
		t.Fatal(err)
	}
	if f.result() != nil || len(f.calls) != 0 {
		t.Fatalf("result=%+v calls=%v", f.result(), f.calls)
	}
}

func TestApplyLockKeepsTwoRunsApart(t *testing.T) {
	f := newApplyFixture(t, "stable", "v1")
	f.promote("v2", []byte("x"), false)
	// Hold the lock as another process would.
	held, err := os.OpenFile(f.a.lockPath(), os.O_CREATE|os.O_RDWR, 0o600)
	if err != nil {
		t.Fatal(err)
	}
	defer held.Close()
	if err := flockExclusiveNonblock(held); err != nil {
		t.Fatal(err)
	}
	if err := f.a.Run(context.Background(), false); err != nil {
		t.Fatalf("a locked-out path run must exit 0 quietly: %v", err)
	}
	if f.result() != nil {
		t.Fatal("a locked-out run wrote a result")
	}
}

func TestApplyRequestIsAHintNotAnInstruction(t *testing.T) {
	f := newApplyFixture(t, "stable", "v1")
	// A symlinked parent under /data.
	os.MkdirAll(filepath.Join(f.a.dataDir, "elsewhere"), 0o755)
	os.Symlink(filepath.Join(f.a.dataDir, "elsewhere"), filepath.Join(f.a.dataDir, "system"))
	os.MkdirAll(filepath.Join(f.a.dataDir, "elsewhere", "router"), 0o755)
	os.WriteFile(filepath.Join(f.a.dataDir, "elsewhere", "router", "requested.json"), []byte(`{"version":"v2"}`), 0o644)
	if _, err := f.a.readRequest(); err == nil {
		t.Fatal("followed a symlinked parent under /data")
	}
	os.Remove(filepath.Join(f.a.dataDir, "system"))
	// A symlinked request file.
	os.MkdirAll(systemDataDir(f.a.dataDir), 0o755)
	os.WriteFile(filepath.Join(f.a.dataDir, "real.json"), []byte(`{"version":"v2"}`), 0o644)
	os.Symlink(filepath.Join(f.a.dataDir, "real.json"), systemRequestPath(f.a.dataDir))
	if _, err := f.a.readRequest(); err == nil {
		t.Fatal("followed a symlinked request")
	}
	os.Remove(systemRequestPath(f.a.dataDir))
	// A traversal version.
	writeJSONDurable(systemRequestPath(f.a.dataDir), systemRequest{Version: "../../etc/passwd", PromotedAt: f.now}, 0o644)
	if _, err := f.a.readRequest(); err == nil {
		t.Fatal("accepted a traversal version")
	}
	// Oversized.
	os.WriteFile(systemRequestPath(f.a.dataDir), []byte(`{"version":"`+strings.Repeat("a", 5000)+`"}`), 0o644)
	if _, err := f.a.readRequest(); err == nil {
		t.Fatal("accepted a 5 KiB request")
	}
}

func TestApplyChannelComesFromRootNotTheRequest(t *testing.T) {
	// The box is on stable; a valid CANARY pointer exists and the agent
	// requests its version. Root reads control/system/router/stable.json
	// (missing) and refuses — a signed canary promote does not authorise a
	// stable box.
	f := newApplyFixture(t, "stable", "v1")
	f.a.channel = "canary"
	req := f.promote("v2", []byte("x"), false) // signs the CANARY pointer
	f.a.channel = "stable"
	if err := f.a.Run(context.Background(), false); err != nil {
		t.Fatal(err)
	}
	r := f.result()
	if r == nil || r.Outcome != systemOutcomeRefused || !r.matches("stable", "v2", req.PromotedAt) {
		t.Fatalf("result = %+v", r)
	}
	if _, err := os.Stat(systemBinaryPath(f.a.optDir) + ".prev"); err == nil {
		t.Fatal("touched the box")
	}
}

func TestApplyStaleTupleIsRefusedAgainstTheRequestsTuple(t *testing.T) {
	t.Parallel()
	f := newApplyFixture(t, "stable", "v1")
	old := f.promote("v2", []byte("x"), false)
	time.Sleep(1100 * time.Millisecond)
	f.promote("v3", []byte("y"), false)
	// Put the OLD request back: the agent's spread started under v2.
	writeJSONDurable(systemRequestPath(f.a.dataDir), old, 0o644)
	if err := f.a.Run(context.Background(), false); err != nil {
		t.Fatal(err)
	}
	r := f.result()
	if r == nil || r.Outcome != systemOutcomeRefused || r.Version != "v2" || !r.PromotedAt.Equal(old.PromotedAt) {
		t.Fatalf("result = %+v (must name the REQUEST's tuple so v3 is not suppressed)", r)
	}
}

func TestApplyReplayGuardIsPerChannelAndEqualIsAllowed(t *testing.T) {
	f := newApplyFixture(t, "stable", "v1")
	req := f.promote("v2", []byte("x"), true)
	// Older than the watermark → refused, seen unchanged.
	writeJSONDurable(filepath.Join(systemRootDir(f.a.optDir), "seen.json"), systemSeen{"stable": req.PromotedAt.Add(time.Hour), "canary": time.Time{}}, 0o644)
	if err := f.a.Run(context.Background(), false); err != nil {
		t.Fatal(err)
	}
	if r := f.result(); r == nil || r.Outcome != systemOutcomeRefused || !strings.Contains(r.Reason, "replay") {
		t.Fatalf("result = %+v", r)
	}
	// Equal → allowed (a retry of the same promote).
	writeJSONDurable(filepath.Join(systemRootDir(f.a.optDir), "seen.json"), systemSeen{"stable": req.PromotedAt}, 0o644)
	if err := f.a.Run(context.Background(), false); err != nil {
		t.Fatal(err)
	}
	if r := f.result(); r == nil || r.Outcome != systemOutcomeNoted {
		t.Fatalf("result = %+v", r)
	}
	// Another channel's newer watermark does not block this channel.
	writeJSONDurable(filepath.Join(systemRootDir(f.a.optDir), "seen.json"), systemSeen{"canary": req.PromotedAt.Add(time.Hour)}, 0o644)
	if err := f.a.Run(context.Background(), false); err != nil {
		t.Fatal(err)
	}
	if r := f.result(); r == nil || r.Outcome != systemOutcomeNoted {
		t.Fatalf("result = %+v", r)
	}
	seen, _ := readSystemSeen(f.a.optDir)
	if !seen["stable"].Equal(req.PromotedAt) {
		t.Fatalf("seen not advanced: %v", seen)
	}
}

func TestApplyNoOpNotesAndTouchesNothing(t *testing.T) {
	f := newApplyFixture(t, "stable", "v2")
	f.promote("v2", []byte("x"), true)
	if err := f.a.Run(context.Background(), false); err != nil {
		t.Fatal(err)
	}
	if r := f.result(); r == nil || r.Outcome != systemOutcomeNoted {
		t.Fatalf("result = %+v", r)
	}
	if _, err := os.Stat(f.a.stagingDir()); err == nil {
		t.Fatal("a no-op created staging/")
	}
	if len(f.calls) != 0 {
		t.Fatalf("a no-op ran commands: %v", f.calls)
	}
}

func TestApplyStoreErrorIsDeferredNotRefused(t *testing.T) {
	f := newApplyFixture(t, "stable", "v1")
	req := f.promote("v2", []byte("x"), false)
	f.a.openStore = func(context.Context) (release.Store, ed25519.PublicKey, error) {
		return unreadableStore{f.store}, f.pub, nil
	}
	if err := f.a.Run(context.Background(), false); err != nil {
		t.Fatal(err)
	}
	if r := f.result(); r == nil || r.Outcome != systemOutcomeDeferred || !r.matches("stable", "v2", req.PromotedAt) {
		t.Fatalf("result = %+v", r)
	}
}

// unreadableStore is the bucket-outage fixture for reads: the door is
// restarting, so the pointer cannot be fetched at all.
type unreadableStore struct{ release.Store }

func (unreadableStore) Get(context.Context, string) ([]byte, string, error) {
	return nil, "", errors.New("door restarting")
}

func TestApplyStagesTheCandidateFromARootOwnedCopy(t *testing.T) {
	f := newApplyFixture(t, "stable", "v1")
	body := []byte("#!/bin/sh\necho carlos v2\n")
	f.promote("v2", body, false)
	f.answer["version:"+filepath.Join(f.a.stagingDir(), "carlos")] = "carlos v2\n"
	p, err := f.a.verifyPointer(context.Background())
	if err != nil {
		t.Fatal(err)
	}
	req, _ := f.a.readRequest()
	staged, err := f.a.stageCandidate(context.Background(), p, req)
	if err != nil {
		t.Fatal(err)
	}
	st, _ := os.Stat(staged)
	if st.Mode().Perm() != 0o755 {
		t.Fatalf("staged mode = %v after verification", st.Mode().Perm())
	}
	if dst, _ := os.Stat(f.a.stagingDir()); dst.Mode().Perm() != 0o700 {
		t.Fatalf("staging dir mode = %v", dst.Mode().Perm())
	}
	// Swapping the cache after staging changes nothing about the copy.
	os.WriteFile(systemCachePath(f.a.dataDir, "v2"), []byte("evil"), 0o644)
	got, _ := os.ReadFile(staged)
	if string(got) != string(body) {
		t.Fatal("staged copy follows the cache")
	}
	// A sha mismatch is refused BEFORE the copy is made executable and
	// staging/ is cleared.
	f2 := newApplyFixture(t, "stable", "v1")
	f2.promote("v2", body, false)
	os.WriteFile(systemCachePath(f2.a.dataDir, "v2"), []byte("evil"), 0o644)
	p2, _ := f2.a.verifyPointer(context.Background())
	req2, _ := f2.a.readRequest()
	if _, err := f2.a.stageCandidate(context.Background(), p2, req2); err == nil || !strings.Contains(err.Error(), "sha256") {
		t.Fatalf("err = %v", err)
	}
	if st, err := os.Stat(filepath.Join(f2.a.stagingDir(), "carlos")); err == nil && st.Mode().Perm() != 0o600 {
		t.Fatalf("an unverified copy was made executable: mode = %v", st.Mode().Perm())
	}
	// A candidate that lies about its version is refused.
	f3 := newApplyFixture(t, "stable", "v1")
	f3.promote("v2", body, false)
	f3.answer["version:"+filepath.Join(f3.a.stagingDir(), "carlos")] = "carlos v9\n"
	p3, _ := f3.a.verifyPointer(context.Background())
	req3, _ := f3.a.readRequest()
	if _, err := f3.a.stageCandidate(context.Background(), p3, req3); err == nil || !strings.Contains(err.Error(), "reports") {
		t.Fatalf("err = %v", err)
	}
	// A symlinked cache file is refused.
	f4 := newApplyFixture(t, "stable", "v1")
	f4.promote("v2", body, false)
	os.Remove(systemCachePath(f4.a.dataDir, "v2"))
	os.WriteFile(filepath.Join(f4.a.dataDir, "real"), body, 0o644)
	os.Symlink(filepath.Join(f4.a.dataDir, "real"), systemCachePath(f4.a.dataDir, "v2"))
	p4, _ := f4.a.verifyPointer(context.Background())
	req4, _ := f4.a.readRequest()
	if _, err := f4.a.stageCandidate(context.Background(), p4, req4); err == nil {
		t.Fatal("followed a symlinked cache file")
	}
}

// applyHappy sets up a v1 box with a v2 candidate whose fake bootstrap
// plan adds carlos-new-only.path and drops carlos-old-only.timer.
func applyHappy(t *testing.T) (*applyFixture, systemRequest) {
	return applyHappyVersion(t, "v2")
}

// applyHappyVersion is applyHappy with the candidate's version named, for
// the guards that place a version against release.LayoutFloor.
func applyHappyVersion(t *testing.T, version string) (*applyFixture, systemRequest) {
	f := newApplyFixture(t, "stable", "v1")
	body := []byte("#!/bin/sh\necho carlos " + version + "\n")
	req := f.promote(version, body, false)
	staged := filepath.Join(f.a.stagingDir(), "carlos")
	f.answer["version:"+staged] = "carlos " + version + "\n"
	f.answer["plan:"+staged] = `{"enabled":["carlos-edge.service","carlos-adopt.timer","carlos-adopt.path","carlos-stage.path","carlos-restart.path","carlos-system-update.path","carlos-system-recover.service","carlos-new-only.path"],"inert":["carlos-restore-verify.timer"]}`
	f.answer["units:"+staged] = "carlos-edge.service carlos-adopt.timer carlos-adopt.path carlos-stage.path carlos-restart.path carlos-system-update.path carlos-system-update.service carlos-system-recover.service carlos-instance-control.service carlos-new-only.path"
	// The unit v1 owns and v2 drops is on the box, so removing it is a
	// thing that can be seen to happen — or not to.
	os.WriteFile(filepath.Join(f.a.etcDir, "systemd", "system", "carlos-old-only.timer"), []byte("[Unit]\n# v1\n"), 0o644)
	// .prev, once made, answers as v1 and stages v1's units.
	prev := systemPrevPath(f.a.optDir)
	f.answer["version:"+prev] = "carlos v1\n"
	f.answer["plan:"+prev] = f.answer["plan:"+systemBinaryPath(f.a.optDir)]
	f.answer["units:"+prev] = f.answer["units:"+systemBinaryPath(f.a.optDir)]
	// The on-disk binary answers from its own bytes, so it says v1 until
	// the rename and v2 after it — a scripted answer would make the run a
	// no-op before it started.
	delete(f.answer, "version:"+systemBinaryPath(f.a.optDir))
	return f, req
}

func TestApplyInstallsGatesArmsAndRecords(t *testing.T) {
	f, req := applyHappy(t)
	if err := f.a.Run(context.Background(), false); err != nil {
		t.Fatal(err)
	}
	r := f.result()
	if r == nil || r.Outcome != systemOutcomeApplied || !r.matches("stable", "v2", req.PromotedAt) {
		t.Fatalf("result = %+v", r)
	}
	ap, _ := readSystemApplied(f.a.optDir)
	if ap == nil || ap.Version != "v2" || ap.Channel != "stable" {
		t.Fatalf("applied = %+v", ap)
	}
	if j, _ := readSystemJournal(f.a.optDir); j != nil {
		t.Fatalf("journal left open: %+v", j)
	}
	if _, err := os.Stat(f.a.stagingDir()); err == nil {
		t.Fatal("staging/ left behind")
	}
	bin, _ := os.ReadFile(systemBinaryPath(f.a.optDir))
	if !strings.Contains(string(bin), "carlos v2") {
		t.Fatal("binary not swapped")
	}
	prev, _ := os.ReadFile(systemPrevPath(f.a.optDir))
	if !strings.Contains(string(prev), "carlos v1") {
		t.Fatal(".prev is not the old binary")
	}
	units := filepath.Join(f.a.etcDir, "systemd", "system")
	if _, err := os.Stat(filepath.Join(units, "carlos-new-only.path")); err != nil {
		t.Error("new unit not installed")
	}
	if _, err := os.Stat(filepath.Join(units, "carlos-old-only.timer")); err == nil {
		t.Error("removed unit still installed")
	}
	if _, err := os.Stat(filepath.Join(f.a.etcDir, "sysusers.d", "carlos.conf")); err != nil {
		t.Error("sysusers not installed")
	}
	if _, err := os.Stat(filepath.Join(f.a.etcDir, "tmpfiles.d", "carlos.conf")); err != nil {
		t.Error("tmpfiles not installed")
	}
	if _, err := os.Stat(filepath.Join(f.a.etcDir, "data", "registry.db")); err == nil {
		t.Error("staging data/ copied")
	}
	// Order of the systemctl calls is the spec.
	joined := strings.Join(f.calls, "\n")
	order := []string{
		"systemctl stop carlos-adopt.timer carlos-adopt.path carlos-stage.path carlos-restart.path carlos-adopt.service carlos-config-sync.service carlos-stage.service carlos-restart.service",
		"systemctl disable --now carlos-old-only.timer",
		"run systemd-sysusers",
		"run systemd-tmpfiles --create",
		"systemctl daemon-reload",
		"systemctl reset-failed carlos-edge",
		"systemctl restart carlos-edge",
		"systemctl is-active carlos-edge",
		"systemctl daemon-reload",
		"systemctl enable carlos-edge.service carlos-adopt.timer carlos-adopt.path carlos-stage.path carlos-restart.path carlos-system-update.path carlos-system-recover.service carlos-new-only.path",
		"systemctl start carlos-adopt.timer carlos-adopt.path carlos-stage.path carlos-restart.path carlos-system-update.path carlos-new-only.path",
	}
	at := 0
	for _, want := range order {
		i := strings.Index(joined[at:], want)
		if i < 0 {
			t.Fatalf("call order broken: %q not found after position %d in:\n%s", want, at, joined)
		}
		at += i + len(want)
	}
	for _, c := range f.calls {
		if strings.HasPrefix(c, "systemctl start ") && strings.Contains(c, ".service") {
			t.Fatalf("apply started a service: %s", c)
		}
	}
	if strings.Contains(joined, "systemctl enable --now") {
		t.Fatal("enable --now is never used")
	}
}

func TestApplyGateSoakCatchesAFlap(t *testing.T) {
	f, req := applyHappy(t)
	// Edge comes up, then restarts once during the soak.
	restarts := 0
	orig := f.a.ops.systemctl
	f.a.ops.systemctl = func(args ...string) (string, error) {
		if args[0] == "show" {
			restarts++
			if restarts > 1 {
				return "1\n2000\n", nil
			}
		}
		return orig(args...)
	}
	err := f.a.Run(context.Background(), false)
	if err == nil {
		t.Fatal("a flapping edge passed the gate")
	}
	r := f.result()
	if r == nil || !r.matches("stable", "v2", req.PromotedAt) || r.Outcome == systemOutcomeApplied {
		t.Fatalf("result = %+v", r)
	}
}

func TestApplyPrevIsRefreshedOnlyWhenTheDiskIsKnownGood(t *testing.T) {
	// applied.json names v0 but the disk says v1 (a rollback-failed left a
	// bad v1 in place, .prev is the good v0): .prev must be left alone.
	f, _ := applyHappy(t)
	os.WriteFile(systemPrevPath(f.a.optDir), []byte("#!/bin/sh\necho carlos v0\n"), 0o755)
	f.answer["version:"+systemPrevPath(f.a.optDir)] = "carlos v0\n"
	writeJSONDurable(filepath.Join(systemRootDir(f.a.optDir), "applied.json"), systemApplied{Version: "v0", Channel: "stable", At: f.now}, 0o644)
	_ = f.a.Run(context.Background(), false)
	prev, _ := os.ReadFile(systemPrevPath(f.a.optDir))
	if !strings.Contains(string(prev), "carlos v0") {
		t.Fatal(".prev was overwritten by a binary that never passed a gate")
	}
}

func TestApplyPrevUnitsComeFromPrevNotFromDisk(t *testing.T) {
	f, _ := applyHappy(t)
	os.WriteFile(systemPrevPath(f.a.optDir), []byte("#!/bin/sh\necho carlos v0\n"), 0o755)
	f.answer["version:"+systemPrevPath(f.a.optDir)] = "carlos v0\n"
	f.answer["plan:"+systemPrevPath(f.a.optDir)] = `{"enabled":["carlos-edge.service"],"inert":[]}`
	f.answer["units:"+systemPrevPath(f.a.optDir)] = "carlos-edge.service carlos-v0-only.timer"
	writeJSONDurable(filepath.Join(systemRootDir(f.a.optDir), "applied.json"), systemApplied{Version: "v0", Channel: "stable", At: f.now}, 0o644)
	// Inspect the journal mid-flight by failing the gate.
	f.edgeUp = false
	_ = f.a.Run(context.Background(), false)
	// units_removed must be computed against .prev's set (v0-only.timer),
	// not the on-disk v1's (old-only.timer).
	found := false
	for _, c := range f.calls {
		if c == "run carlos.prev bootstrap -root "+filepath.Join(f.a.stagingDir(), "prev")+" -plan-json" {
			found = true
		}
	}
	if !found {
		t.Fatalf("prev units not staged from carlos.prev:\n%s", strings.Join(f.calls, "\n"))
	}
	// And units_removed follows from that set: v0's timer goes, and the
	// on-disk v1's — a build nothing here is restoring — is left alone.
	if !f.called("systemctl disable --now carlos-v0-only.timer") {
		t.Errorf("carlos-v0-only.timer not removed:\n%s", strings.Join(f.calls, "\n"))
	}
	if f.called("systemctl disable --now carlos-old-only.timer") {
		t.Errorf("removed a unit computed against the on-disk build:\n%s", strings.Join(f.calls, "\n"))
	}
}

func TestApplyPrevIsRefreshedWhenAppliedNamesTheDisk(t *testing.T) {
	// The ordinary box: applied.json names the binary that is on disk, so
	// that binary passed a gate here and is the one to fall back to.
	f, _ := applyHappy(t)
	os.WriteFile(systemPrevPath(f.a.optDir), []byte("#!/bin/sh\necho carlos v0\n"), 0o755)
	writeJSONDurable(filepath.Join(systemRootDir(f.a.optDir), "applied.json"), systemApplied{Version: "v1", Channel: "stable", At: f.now}, 0o644)
	if err := f.a.Run(context.Background(), false); err != nil {
		t.Fatal(err)
	}
	prev, _ := os.ReadFile(systemPrevPath(f.a.optDir))
	if !strings.Contains(string(prev), "carlos v1") {
		t.Fatalf(".prev = %q, want the v1 the box had applied", prev)
	}
}

func TestApplyPrevIsLeftAloneWhenNothingWasEverApplied(t *testing.T) {
	// No applied.json, but a .prev exists: the box's history is unknown, so
	// the untested disk binary does not get to become the fallback.
	f, _ := applyHappy(t)
	os.WriteFile(systemPrevPath(f.a.optDir), []byte("#!/bin/sh\necho carlos v0\n"), 0o755)
	f.answer["version:"+systemPrevPath(f.a.optDir)] = "carlos v0\n"
	if err := f.a.Run(context.Background(), false); err != nil {
		t.Fatal(err)
	}
	prev, _ := os.ReadFile(systemPrevPath(f.a.optDir))
	if !strings.Contains(string(prev), "carlos v0") {
		t.Fatalf(".prev = %q, want the v0 that was already there", prev)
	}
}

func TestApplyGateDemandsZeroRestartsNotJustAStillOne(t *testing.T) {
	// The edge crashed once and then settled: nothing moves across the
	// soak, but NRestarts is not 0 and the build is not one to keep.
	f, req := applyHappy(t)
	f.nrestarts = "1"
	if err := f.a.Run(context.Background(), false); err == nil {
		t.Fatal("an edge that had already crash-restarted passed the gate")
	}
	r := f.result()
	if r == nil || !r.matches("stable", "v2", req.PromotedAt) || r.Outcome == systemOutcomeApplied {
		t.Fatalf("result = %+v", r)
	}
}

func TestApplyRecordsAnArmingFailureAgainstTheApply(t *testing.T) {
	// The gate passed, so the roll stands; but `enable` is one batched
	// call, and a box left unarmed must not look like a clean apply.
	f, req := applyHappy(t)
	orig := f.a.ops.systemctl
	f.a.ops.systemctl = func(args ...string) (string, error) {
		if args[0] == "enable" {
			orig(args...)
			return "", errors.New("Failed to enable unit carlos-new-only.path")
		}
		return orig(args...)
	}
	if err := f.a.Run(context.Background(), false); err != nil {
		t.Fatal(err)
	}
	r := f.result()
	if r == nil || r.Outcome != systemOutcomeApplied || !r.matches("stable", "v2", req.PromotedAt) {
		t.Fatalf("result = %+v", r)
	}
	if !strings.Contains(r.Reason, "armed with errors") {
		t.Fatalf("reason = %q, want the arming failure recorded", r.Reason)
	}
	if ap, _ := readSystemApplied(f.a.optDir); ap == nil || ap.Version != "v2" {
		t.Fatalf("applied = %+v (the gate passed; the roll stands)", ap)
	}
}

func TestApplyRollsBackWhenTheGateFails(t *testing.T) {
	f, req := applyHappy(t)
	// v2 never comes up; v1 does — which is what makes this a rollback and
	// not a rollback-failed. The fake edge answers for whatever binary is
	// on disk when the restart is issued.
	f.edgeUp = false
	orig := f.a.ops.systemctl
	f.a.ops.systemctl = func(args ...string) (string, error) {
		if args[0] == "restart" {
			if b, _ := os.ReadFile(systemBinaryPath(f.a.optDir)); strings.Contains(string(b), "carlos v1") {
				f.edgeUp = true
			}
		}
		return orig(args...)
	}
	err := f.a.Run(context.Background(), false)
	if err == nil {
		t.Fatal("a failed gate must exit non-zero")
	}
	r := f.result()
	if r == nil || r.Outcome != systemOutcomeRolledBack || !r.matches("stable", "v2", req.PromotedAt) || !strings.Contains(r.Reason, "gate") {
		t.Fatalf("result = %+v", r)
	}
	bin, _ := os.ReadFile(systemBinaryPath(f.a.optDir))
	if !strings.Contains(string(bin), "carlos v1") {
		t.Fatal("binary not restored")
	}
	if _, err := os.Stat(systemPrevPath(f.a.optDir)); err != nil {
		t.Fatal(".prev consumed by the rollback")
	}
	units := filepath.Join(f.a.etcDir, "systemd", "system")
	if _, err := os.Stat(filepath.Join(units, "carlos-new-only.path")); err == nil {
		t.Error("added unit survived the rollback")
	}
	if _, err := os.Stat(filepath.Join(units, "carlos-old-only.timer")); err != nil {
		t.Error("removed unit not restored")
	}
	if ap, _ := readSystemApplied(f.a.optDir); ap != nil {
		t.Fatalf("applied.json written by a rollback: %+v", ap)
	}
	if j, _ := readSystemJournal(f.a.optDir); j != nil {
		t.Fatal("journal left open after rollback")
	}
	joined := strings.Join(f.calls, "\n")
	if !strings.Contains(joined, "systemctl disable --now carlos-new-only.path") {
		t.Error("added unit not disabled")
	}
	// The install already staged carlos.prev; the rollback reuses it rather
	// than re-running a bootstrap on a box that is already unhealthy.
	if n := strings.Count(joined, "run carlos.prev bootstrap"); n != 1 {
		t.Errorf("carlos.prev bootstrapped %d times, want 1:\n%s", n, joined)
	}
	// reset-failed precedes the rollback's restart too, and the previous
	// plan is armed afterwards.
	i := strings.LastIndex(joined, "systemctl reset-failed carlos-edge")
	k := strings.LastIndex(joined, "systemctl restart carlos-edge")
	m := strings.LastIndex(joined, "systemctl start carlos-adopt.timer")
	if !(i < k && k < m) {
		t.Fatalf("rollback order wrong:\n%s", joined)
	}
}

// The relay rides the platform binary, so a self-update has to bounce it
// onto the new build the way it bounces the edge — after the edge, and with
// reset-failed then restart. `restart`, not `try-restart`: an unarmed box is
// kept relay-less by the unit's own ConditionPathExists (bootstrap starts
// the unit outright and relies on the same thing), while try-restart after a
// reset-failed cannot revive a relay that crash-looped at boot, because
// reset-failed has just made it inactive and try-restart skips an inactive
// unit (relay spec §3).
func TestApplyRestartsTheRelayAfterTheEdge(t *testing.T) {
	f, _ := applyHappy(t)
	if err := f.a.Run(context.Background(), false); err != nil {
		t.Fatal(err)
	}
	joined := strings.Join(f.calls, "\n")
	edge := strings.Index(joined, "systemctl restart carlos-edge")
	reset := strings.Index(joined, "systemctl reset-failed carlos-relay")
	restart := strings.Index(joined, "systemctl restart carlos-relay")
	switch {
	case reset < 0 || restart < 0:
		t.Fatalf("relay never restarted:\n%s", joined)
	case !(edge < reset && reset < restart):
		t.Fatalf("want restart carlos-edge, then reset-failed then restart carlos-relay:\n%s", joined)
	}
	// try-restart is the bug this test now guards against: it would leave a
	// failed relay down until the next update.
	if strings.Contains(joined, "systemctl try-restart carlos-relay") {
		t.Errorf("relay try-restarted, which cannot revive a failed unit:\n%s", joined)
	}
}

// Discussion #256: the root instance-control service execs the platform
// binary and stays up for weeks, so an install that leaves it running the
// previous build leaves every nested wake failing at namespace setup. It
// is restarted after the new units are in place (daemon-reload) and before
// the edge, so the new agent never meets an old control service.
func TestApplyRestartsTheInstanceControlServiceBeforeTheEdge(t *testing.T) {
	f, _ := applyHappy(t)
	if err := f.a.Run(context.Background(), false); err != nil {
		t.Fatal(err)
	}
	joined := strings.Join(f.calls, "\n")
	reload := strings.Index(joined, "systemctl daemon-reload")
	reset := strings.Index(joined, "systemctl reset-failed carlos-instance-control.service")
	restart := strings.Index(joined, "systemctl restart carlos-instance-control.service")
	edge := strings.Index(joined, "systemctl restart carlos-edge")
	switch {
	case reset < 0 || restart < 0:
		t.Fatalf("instance-control never restarted:\n%s", joined)
	case !(reload < reset && reset < restart && restart < edge):
		t.Fatalf("want daemon-reload, then reset-failed and restart carlos-instance-control.service, then restart carlos-edge:\n%s", joined)
	}
	if strings.Contains(joined, "systemctl try-restart carlos-instance-control.service") {
		t.Errorf("try-restart cannot start an inactive (socket-activated, never connected) service:\n%s", joined)
	}
}

// A control service that will not restart fails the install, and the
// rollback restarts it again onto the previous binary (so the restart is
// attempted twice over the run), rather than leaving a new edge beside a
// control service of unknown vintage.
func TestApplyInstanceControlRestartFailureRollsBack(t *testing.T) {
	f, req := applyHappy(t)
	orig := f.a.ops.systemctl
	failed := false
	f.a.ops.systemctl = func(args ...string) (string, error) {
		out, err := orig(args...)
		if args[0] == "restart" && args[1] == systemControlUnit && !failed {
			failed = true
			return "", errors.New("Job for carlos-instance-control.service failed")
		}
		return out, err
	}
	if err := f.a.Run(context.Background(), false); err == nil {
		t.Fatal("a control service that would not restart passed the install")
	}
	r := f.result()
	if r == nil || r.Outcome != systemOutcomeRolledBack || !r.matches("stable", "v2", req.PromotedAt) || !strings.Contains(r.Reason, "instance-control restart") {
		t.Fatalf("result = %+v", r)
	}
	if n := strings.Count(strings.Join(f.calls, "\n"), "systemctl restart carlos-instance-control.service"); n != 2 {
		t.Errorf("control service restarted %d times, want 2 (install then rollback):\n%s", n, strings.Join(f.calls, "\n"))
	}
	if bin, _ := os.ReadFile(systemBinaryPath(f.a.optDir)); !strings.Contains(string(bin), "carlos v1") {
		t.Fatal("binary not rolled back")
	}
}

// The rollback of a failed gate restarts the control service onto the
// previous binary too, before the edge, for the same reason the relay is.
func TestApplyRollbackRestartsTheInstanceControlService(t *testing.T) {
	f, _ := applyHappy(t)
	f.edgeUp = false
	orig := f.a.ops.systemctl
	f.a.ops.systemctl = func(args ...string) (string, error) {
		if args[0] == "restart" && args[1] == systemEdgeUnit {
			if b, _ := os.ReadFile(systemBinaryPath(f.a.optDir)); strings.Contains(string(b), "carlos v1") {
				f.edgeUp = true
			}
		}
		return orig(args...)
	}
	if err := f.a.Run(context.Background(), false); err == nil {
		t.Fatal("a failed gate must exit non-zero")
	}
	joined := strings.Join(f.calls, "\n")
	edge := strings.LastIndex(joined, "systemctl restart carlos-edge")
	ctl := strings.LastIndex(joined, "systemctl restart carlos-instance-control.service")
	if ctl < 0 || ctl > edge {
		t.Fatalf("the rollback left the control service on the withdrawn build, or restarted it after the edge:\n%s", joined)
	}
	if n := strings.Count(joined, "systemctl restart carlos-instance-control.service"); n != 2 {
		t.Errorf("control service restarted %d times, want 2 (apply then rollback):\n%s", n, joined)
	}
}

// A candidate that ships no control service (a downgrade below the
// isolated-exec releases) has nothing to restart: installUnits has just
// disabled and removed the unit, and a restart would fail the install.
func TestApplyDoesNotRestartAControlServiceTheCandidateLacks(t *testing.T) {
	f, req := applyHappy(t)
	staged := filepath.Join(f.a.stagingDir(), "carlos")
	f.answer["units:"+staged] = strings.ReplaceAll(f.answer["units:"+staged], " carlos-instance-control.service", "")
	if err := f.a.Run(context.Background(), false); err != nil {
		t.Fatal(err)
	}
	if r := f.result(); r == nil || r.Outcome != systemOutcomeApplied || !r.matches("stable", "v2", req.PromotedAt) {
		t.Fatalf("result = %+v", r)
	}
	joined := strings.Join(f.calls, "\n")
	if !strings.Contains(joined, "systemctl disable --now carlos-instance-control.service") {
		t.Fatalf("the unit the candidate lacks was not disabled:\n%s", joined)
	}
	if strings.Contains(joined, "systemctl restart carlos-instance-control.service") {
		t.Fatalf("restarted a unit the candidate removed:\n%s", joined)
	}
}

// A relay that cannot be restarted is not a failed update. The commonest
// case is the ordinary one — a box with no masters file, whose unit systemd
// skips on its condition and may well answer about — and the edge, the
// binary and the gate are all fine.
func TestApplyRelayRestartFailureDoesNotFailTheUpdate(t *testing.T) {
	f, req := applyHappy(t)
	orig := f.a.ops.systemctl
	f.a.ops.systemctl = func(args ...string) (string, error) {
		if args[0] == "restart" && args[1] == systemRelayUnit {
			orig(args...) // still recorded
			return "", errors.New("Unit carlos-relay.service is not loaded")
		}
		return orig(args...)
	}
	if err := f.a.Run(context.Background(), false); err != nil {
		t.Fatalf("a relay that would not restart failed the whole update: %v", err)
	}
	r := f.result()
	if r == nil || r.Outcome != systemOutcomeApplied || !r.matches("stable", "v2", req.PromotedAt) {
		t.Fatalf("result = %+v", r)
	}
	if !f.called("systemctl restart " + systemRelayUnit) {
		t.Fatalf("restart never attempted:\n%s", strings.Join(f.calls, "\n"))
	}
}

// And the rollback bounces it too: the box is going back to the previous
// binary, and leaving the relay on the withdrawn one is the same bug the
// edge's own rollback restart exists to avoid.
func TestApplyRollbackRestartsTheRelay(t *testing.T) {
	f, _ := applyHappy(t)
	f.edgeUp = false
	orig := f.a.ops.systemctl
	f.a.ops.systemctl = func(args ...string) (string, error) {
		if args[0] == "restart" {
			if b, _ := os.ReadFile(systemBinaryPath(f.a.optDir)); strings.Contains(string(b), "carlos v1") {
				f.edgeUp = true
			}
		}
		return orig(args...)
	}
	if err := f.a.Run(context.Background(), false); err == nil {
		t.Fatal("a failed gate must exit non-zero")
	}
	joined := strings.Join(f.calls, "\n")
	edge := strings.LastIndex(joined, "systemctl restart carlos-edge")
	relay := strings.LastIndex(joined, "systemctl restart carlos-relay")
	if relay < 0 || relay < edge {
		t.Fatalf("the rollback left the relay on the withdrawn build:\n%s", joined)
	}
	// Twice over the whole run: once for the apply's restart, once for the
	// rollback's — not just the apply's, which would pass a LastIndex check
	// by accident if the rollback did nothing.
	if n := strings.Count(joined, "systemctl restart carlos-relay"); n != 2 {
		t.Errorf("relay restarted %d times, want 2 (apply then rollback):\n%s", n, joined)
	}
}

func TestApplyRollbackFailedLeavesPrevAndAppliedAlone(t *testing.T) {
	t.Parallel()
	f, req := applyHappy(t)
	writeJSONDurable(filepath.Join(systemRootDir(f.a.optDir), "applied.json"), systemApplied{Version: "v1", Channel: "stable", At: f.now}, 0o644)
	f.edgeUp = false // never comes up, on either binary
	if err := f.a.Run(context.Background(), false); err == nil {
		t.Fatal("must exit non-zero")
	}
	r := f.result()
	if r == nil || r.Outcome != systemOutcomeRollbackFailed || !r.matches("stable", "v2", req.PromotedAt) {
		t.Fatalf("result = %+v", r)
	}
	ap, _ := readSystemApplied(f.a.optDir)
	if ap == nil || ap.Version != "v1" {
		t.Fatalf("applied = %+v, must still name v1", ap)
	}
	prev, _ := os.ReadFile(systemPrevPath(f.a.optDir))
	if !strings.Contains(string(prev), "carlos v1") {
		t.Fatal(".prev damaged")
	}
	// A second apply of a NEW version must not refresh .prev from the
	// unproven disk binary. The box a failed rollback leaves behind is the
	// one that makes that matter: applied.json still names v1, and the v2
	// that never passed a gate is what is on disk.
	os.WriteFile(systemBinaryPath(f.a.optDir), []byte("#!/bin/sh\necho carlos v2\n"), 0o755)
	f.answer["version:"+systemBinaryPath(f.a.optDir)] = "carlos v2\n"
	os.Remove(systemRequestPath(f.a.dataDir))
	f.promote("v3", []byte("#!/bin/sh\necho carlos v3\n"), false)
	f.answer["version:"+filepath.Join(f.a.stagingDir(), "carlos")] = "carlos v3\n"
	f.answer["plan:"+filepath.Join(f.a.stagingDir(), "carlos")] = f.answer["plan:"+systemPrevPath(f.a.optDir)]
	f.answer["units:"+filepath.Join(f.a.stagingDir(), "carlos")] = f.answer["units:"+systemPrevPath(f.a.optDir)]
	_ = f.a.Run(context.Background(), false)
	prev2, _ := os.ReadFile(systemPrevPath(f.a.optDir))
	if string(prev2) != string(prev) {
		t.Fatal(".prev refreshed after rollback-failed")
	}
}

func TestRecoveryRollsBackFilesOnlyAtBootAndRestartsAtRuntime(t *testing.T) {
	for _, phase := range []string{systemPhaseInstalling, systemPhaseInstalled, systemPhaseUnits, systemPhaseRestarted} {
		t.Run(phase, func(t *testing.T) {
			f, _ := applyHappy(t)
			// Simulate: v2 installed, journal open at phase, box rebooted.
			os.WriteFile(systemPrevPath(f.a.optDir), []byte("#!/bin/sh\necho carlos v1\n"), 0o755)
			os.WriteFile(systemBinaryPath(f.a.optDir), []byte("#!/bin/sh\necho carlos v2\n"), 0o755)
			os.WriteFile(filepath.Join(f.a.etcDir, "systemd", "system", "carlos-new-only.path"), []byte("x"), 0o644)
			writeJSONDurable(filepath.Join(systemRootDir(f.a.optDir), "journal.json"),
				systemJournal{Phase: phase, Target: "v2", PrevBuild: "v1", UnitsAdded: []string{"carlos-new-only.path"}, UnitsRemoved: []string{"carlos-old-only.timer"}, StartedAt: f.now}, 0o600)
			f.edgeStartedThisBoot = false
			if err := f.a.Run(context.Background(), true); err != nil {
				t.Fatal(err)
			}
			bin, _ := os.ReadFile(systemBinaryPath(f.a.optDir))
			if !strings.Contains(string(bin), "carlos v1") {
				t.Fatal("binary not restored")
			}
			if _, err := os.Stat(filepath.Join(f.a.etcDir, "systemd", "system", "carlos-new-only.path")); err == nil {
				t.Error("added unit survived")
			}
			if _, err := os.Stat(filepath.Join(f.a.etcDir, "systemd", "system", "carlos-old-only.timer")); err != nil {
				t.Error("removed unit not restored")
			}
			joined := strings.Join(f.calls, "\n")
			if strings.Contains(joined, "systemctl restart carlos-edge") || strings.Contains(joined, "systemctl start ") || strings.Contains(joined, "systemctl restart carlos-instance-control") {
				t.Fatalf("boot recovery restarted or started something:\n%s", joined)
			}
			if !strings.Contains(joined, "systemctl enable carlos-edge.service") {
				t.Fatal("previous plan not enabled for the boot")
			}
			if r := f.result(); r == nil || r.Outcome != systemOutcomeRolledBack || !strings.Contains(r.Reason, phase) {
				t.Fatalf("result = %+v", r)
			}
			if j, _ := readSystemJournal(f.a.optDir); j != nil {
				t.Fatal("journal left open")
			}
		})
	}
	// Runtime (OnFailure=): same, plus a non-blocking restart and the
	// previous plan's timers/paths started.
	f, _ := applyHappy(t)
	os.WriteFile(systemPrevPath(f.a.optDir), []byte("#!/bin/sh\necho carlos v1\n"), 0o755)
	writeJSONDurable(filepath.Join(systemRootDir(f.a.optDir), "journal.json"),
		systemJournal{Phase: systemPhaseUnits, Target: "v2", PrevBuild: "v1", StartedAt: f.now}, 0o600)
	f.edgeStartedThisBoot = true
	if err := f.a.Run(context.Background(), true); err != nil {
		t.Fatal(err)
	}
	joined := strings.Join(f.calls, "\n")
	for _, want := range []string{"systemctl stop carlos-adopt.timer", "systemctl reset-failed carlos-edge", "systemctl restart --no-block carlos-edge", "systemctl start carlos-adopt.timer"} {
		if !strings.Contains(joined, want) {
			t.Errorf("runtime recovery lacks %q:\n%s", want, joined)
		}
	}
	// The control service goes back to the previous binary with the edge
	// (#256), and before it; at boot (above) neither is touched.
	ctl := strings.Index(joined, "systemctl restart carlos-instance-control.service")
	edge := strings.Index(joined, "systemctl restart --no-block carlos-edge")
	if ctl < 0 || ctl > edge {
		t.Errorf("runtime recovery does not restart the control service before the edge:\n%s", joined)
	}
}

func TestRecoveryFinishesForwardFromGated(t *testing.T) {
	f, req := applyHappy(t)
	os.WriteFile(systemBinaryPath(f.a.optDir), []byte("#!/bin/sh\necho carlos v2\n"), 0o755)
	writeJSONDurable(systemRequestPath(f.a.dataDir), req, 0o644)
	writeJSONDurable(filepath.Join(systemRootDir(f.a.optDir), "journal.json"),
		systemJournal{Phase: systemPhaseGated, Target: "v2", PrevBuild: "v1", StartedAt: f.now}, 0o600)
	if err := f.a.Run(context.Background(), true); err != nil {
		t.Fatal(err)
	}
	bin, _ := os.ReadFile(systemBinaryPath(f.a.optDir))
	if !strings.Contains(string(bin), "carlos v2") {
		t.Fatal("a gated apply was undone")
	}
	ap, _ := readSystemApplied(f.a.optDir)
	if ap == nil || ap.Version != "v2" {
		t.Fatalf("applied = %+v", ap)
	}
	if r := f.result(); r == nil || r.Outcome != systemOutcomeApplied {
		t.Fatalf("result = %+v", r)
	}
}

func TestRecoveryRemovesTheJournalLast(t *testing.T) {
	// The restore is done and the result is the last thing before the
	// journal goes: a restart that fails after the files are back leaves
	// both the journal and no result at all for the next run to redo.
	f, _ := applyHappy(t)
	os.WriteFile(systemPrevPath(f.a.optDir), []byte("#!/bin/sh\necho carlos v1\n"), 0o755)
	writeJSONDurable(filepath.Join(systemRootDir(f.a.optDir), "journal.json"),
		systemJournal{Phase: systemPhaseInstalled, Target: "v2", PrevBuild: "v1", StartedAt: f.now}, 0o600)
	f.edgeStartedThisBoot = true
	orig := f.a.ops.systemctl
	f.a.ops.systemctl = func(args ...string) (string, error) {
		if args[0] == "restart" {
			return "", errors.New("job for carlos-edge.service failed")
		}
		return orig(args...)
	}
	if err := f.a.Run(context.Background(), true); err == nil {
		t.Fatal("a failed recovery must exit non-zero")
	}
	if j, _ := readSystemJournal(f.a.optDir); j == nil {
		t.Fatal("journal removed although recovery did not complete")
	}
	if r := f.result(); r != nil {
		t.Fatalf("result written before the restart was issued: %+v", r)
	}
	bin, _ := os.ReadFile(systemBinaryPath(f.a.optDir))
	if !strings.Contains(string(bin), "carlos v1") {
		t.Fatal("the restore itself did not happen")
	}
}

func TestRecoveryKeepsTheJournalWhenTheRestoreFails(t *testing.T) {
	// systemd will not reload, so the units are not back: the journal stays
	// open and the next run recovers again.
	f, _ := applyHappy(t)
	os.WriteFile(systemPrevPath(f.a.optDir), []byte("#!/bin/sh\necho carlos v1\n"), 0o755)
	writeJSONDurable(filepath.Join(systemRootDir(f.a.optDir), "journal.json"),
		systemJournal{Phase: systemPhaseInstalled, Target: "v2", PrevBuild: "v1", StartedAt: f.now}, 0o600)
	orig := f.a.ops.systemctl
	f.a.ops.systemctl = func(args ...string) (string, error) {
		if args[0] == "daemon-reload" {
			return "", errors.New("systemd busy")
		}
		return orig(args...)
	}
	if err := f.a.Run(context.Background(), true); err == nil {
		t.Fatal("a failed recovery must exit non-zero")
	}
	if j, _ := readSystemJournal(f.a.optDir); j == nil {
		t.Fatal("journal removed although recovery did not complete")
	}
}

func TestRecoveryAnswersTheJournalsVersionNotALaterRequest(t *testing.T) {
	// The agent has moved on to v3 while a v2 apply lay interrupted. The
	// result belongs to v2 — v3 has not been tried at all.
	f, _ := applyHappy(t)
	started := f.now
	os.WriteFile(systemPrevPath(f.a.optDir), []byte("#!/bin/sh\necho carlos v1\n"), 0o755)
	writeJSONDurable(filepath.Join(systemRootDir(f.a.optDir), "journal.json"),
		systemJournal{Phase: systemPhaseUnits, Target: "v2", PrevBuild: "v1", StartedAt: started}, 0o600)
	v3 := f.promote("v3", []byte("#!/bin/sh\necho carlos v3\n"), false)
	if err := f.a.Run(context.Background(), true); err != nil {
		t.Fatal(err)
	}
	r := f.result()
	if r == nil || r.Version != "v2" || !r.PromotedAt.Equal(started) {
		t.Fatalf("result = %+v, want the v2 the journal names at %s", r, started)
	}
	if r.PromotedAt.Equal(v3.PromotedAt) {
		t.Fatal("the recovery answered for the request the agent wrote later")
	}
}

func TestRecoveryAnswersTheInterruptedPromoteNotTheRePromote(t *testing.T) {
	t.Parallel()
	// v2 was interrupted; the operator re-promoted the same v2. The
	// recovery's result must name the promote it recovered, or the retry
	// arrives at a box that has already recorded it as rolled back.
	f, first := applyHappy(t)
	os.WriteFile(systemPrevPath(f.a.optDir), []byte("#!/bin/sh\necho carlos v1\n"), 0o755)
	writeJSONDurable(filepath.Join(systemRootDir(f.a.optDir), "journal.json"),
		systemJournal{Phase: systemPhaseUnits, Target: "v2", PromotedAt: first.PromotedAt, PrevBuild: "v1", StartedAt: f.now}, 0o600)
	time.Sleep(1100 * time.Millisecond) // promoted_at has second resolution
	again := f.promote("v2", []byte("#!/bin/sh\necho carlos v2\n"), false)
	if again.PromotedAt.Equal(first.PromotedAt) {
		t.Fatal("fixture: the re-promote must move promoted_at")
	}
	if err := f.a.Run(context.Background(), true); err != nil {
		t.Fatal(err)
	}
	r := f.result()
	if r == nil || r.Outcome != systemOutcomeRolledBack || !r.matches("stable", "v2", first.PromotedAt) {
		t.Fatalf("result = %+v, want the promote the journal names (%s)", r, first.PromotedAt)
	}
	// And the re-promote is a tuple of its own: the terminal-tuple guard
	// does not refuse it.
	if err := f.a.Run(context.Background(), false); err != nil {
		t.Fatal(err)
	}
	if r := f.result(); r == nil || r.Outcome != systemOutcomeApplied || !r.matches("stable", "v2", again.PromotedAt) {
		t.Fatalf("result = %+v, the operator's re-promote must be applied", r)
	}
}

func TestRecoveryFromGatedRollsBackWhenTheBinaryIsNotTheGatedOne(t *testing.T) {
	// The journal says v2 passed its gate, but v1 is on disk: whatever put
	// it there, this is a pre-gate state and going forward would record an
	// apply of a build that is not installed.
	f, _ := applyHappy(t)
	os.WriteFile(systemPrevPath(f.a.optDir), []byte("#!/bin/sh\necho carlos v1\n"), 0o755)
	writeJSONDurable(filepath.Join(systemRootDir(f.a.optDir), "journal.json"),
		systemJournal{Phase: systemPhaseGated, Target: "v2", PrevBuild: "v1", StartedAt: f.now}, 0o600)
	if err := f.a.Run(context.Background(), true); err != nil {
		t.Fatal(err)
	}
	if ap, _ := readSystemApplied(f.a.optDir); ap != nil {
		t.Fatalf("applied.json written for a build that is not on disk: %+v", ap)
	}
	if r := f.result(); r == nil || r.Outcome != systemOutcomeRolledBack {
		t.Fatalf("result = %+v", r)
	}
}

func TestRollbackThatCannotRestoreKeepsTheJournalForTheNextRun(t *testing.T) {
	f, _ := applyHappy(t)
	f.edgeUp = false
	prevDir := filepath.Join(f.a.stagingDir(), "prev")
	prevPlan := f.answer["plan:"+systemPrevPath(f.a.optDir)]
	stops := 0
	orig := f.a.ops.systemctl
	f.a.ops.systemctl = func(args ...string) (string, error) {
		// The rollback's own quiesce: by then the staging is gone and
		// carlos.prev will not bootstrap, so the restore stops after the
		// binary and before the units.
		if args[0] == "stop" {
			if stops++; stops == 2 {
				os.RemoveAll(prevDir)
				delete(f.answer, "plan:"+systemPrevPath(f.a.optDir))
			}
		}
		return orig(args...)
	}
	if err := f.a.Run(context.Background(), false); err == nil {
		t.Fatal("a failed rollback must exit non-zero")
	}
	if r := f.result(); r == nil || r.Outcome != systemOutcomeRollbackFailed {
		t.Fatalf("result = %+v", r)
	}
	if j, _ := readSystemJournal(f.a.optDir); j == nil {
		t.Fatal("journal removed although the restore never finished")
	}
	units := filepath.Join(f.a.etcDir, "systemd", "system")
	if _, err := os.Stat(filepath.Join(units, "carlos-new-only.path")); err != nil {
		t.Fatal("precondition: the failed build's units are what is still installed")
	}
	// The next run recovers what the rollback could not: the bootstrap
	// works again, and the previous build's units go back.
	f.answer["plan:"+systemPrevPath(f.a.optDir)] = prevPlan
	if err := f.a.Run(context.Background(), true); err != nil {
		t.Fatal(err)
	}
	if _, err := os.Stat(filepath.Join(units, "carlos-new-only.path")); err == nil {
		t.Error("added unit survived the recovery")
	}
	if _, err := os.Stat(filepath.Join(units, "carlos-old-only.timer")); err != nil {
		t.Error("removed unit not restored by the recovery")
	}
	if j, _ := readSystemJournal(f.a.optDir); j != nil {
		t.Error("journal left open after a complete recovery")
	}
}

func TestARecoveredRunDoesNotThenApplyTheRequest(t *testing.T) {
	// The v2 request is still on disk, and the recovery has just rolled v2
	// back. Applying it again in the same run would overwrite the result
	// the agent is waiting for and restart the edge a second time.
	f, _ := applyHappy(t)
	os.WriteFile(systemPrevPath(f.a.optDir), []byte("#!/bin/sh\necho carlos v1\n"), 0o755)
	os.WriteFile(systemBinaryPath(f.a.optDir), []byte("#!/bin/sh\necho carlos v2\n"), 0o755)
	writeJSONDurable(filepath.Join(systemRootDir(f.a.optDir), "journal.json"),
		systemJournal{Phase: systemPhaseUnits, Target: "v2", PrevBuild: "v1", StartedAt: f.now}, 0o600)
	if err := f.a.Run(context.Background(), false); err != nil {
		t.Fatal(err)
	}
	if r := f.result(); r == nil || r.Outcome != systemOutcomeRolledBack {
		t.Fatalf("result = %+v", r)
	}
	if ap, _ := readSystemApplied(f.a.optDir); ap != nil {
		t.Fatalf("the request was applied on top of the recovery: %+v", ap)
	}
	bin, _ := os.ReadFile(systemBinaryPath(f.a.optDir))
	if !strings.Contains(string(bin), "carlos v1") {
		t.Fatal("the binary is not the one the recovery restored")
	}
}

func TestARolledBackTupleIsNotAppliedAgain(t *testing.T) {
	f, req := applyHappy(t)
	results := filepath.Join(systemRootDir(f.a.optDir), "result.json")
	writeJSONDurable(results, systemResult{Channel: "stable", Version: req.Version, PromotedAt: req.PromotedAt,
		Outcome: systemOutcomeRolledBack, Reason: "gate: carlos-edge is failed", At: f.now}, 0o644)
	if err := f.a.Run(context.Background(), false); err != nil {
		t.Fatal(err)
	}
	if len(f.calls) != 0 {
		t.Fatalf("a build that already rolled back here was applied again: %v", f.calls)
	}
	if r := f.result(); r == nil || r.Outcome != systemOutcomeRolledBack {
		t.Fatalf("result = %+v, the standing outcome must be left alone", r)
	}
	// A refusal is not terminal: the same tuple is tried again.
	writeJSONDurable(results, systemResult{Channel: "stable", Version: req.Version, PromotedAt: req.PromotedAt,
		Outcome: systemOutcomeRefused, Reason: "seen.json unreadable", At: f.now}, 0o644)
	if err := f.a.Run(context.Background(), false); err != nil {
		t.Fatal(err)
	}
	if r := f.result(); r == nil || r.Outcome != systemOutcomeApplied {
		t.Fatalf("result = %+v, a refused tuple must stay retryable", r)
	}
}

// --recover runs at boot, before the network is up and on a box whose
// environment may be no more than /etc/carlos/host.env: it must never need
// a deploy store or CARLOS_RELEASE_PUBKEY. The seam says so out loud —
// opening the store at all fails the test.
func TestRecoverNeedsNeitherStoreNorPubkey(t *testing.T) {
	f, _ := applyHappy(t)
	f.a.openStore = func(context.Context) (release.Store, ed25519.PublicKey, error) {
		t.Error("--recover opened the deploy store")
		return nil, nil, errors.New("no store on this box")
	}
	os.WriteFile(systemPrevPath(f.a.optDir), []byte("#!/bin/sh\necho carlos v1\n"), 0o755)
	os.WriteFile(systemBinaryPath(f.a.optDir), []byte("#!/bin/sh\necho carlos v2\n"), 0o755)
	writeJSONDurable(filepath.Join(systemRootDir(f.a.optDir), "journal.json"),
		systemJournal{Phase: systemPhaseUnits, Target: "v2", PrevBuild: "v1", StartedAt: f.now}, 0o600)
	if err := f.a.Run(context.Background(), true); err != nil {
		t.Fatal(err)
	}
	bin, _ := os.ReadFile(systemBinaryPath(f.a.optDir))
	if !strings.Contains(string(bin), "carlos v1") {
		t.Fatalf("binary = %q, want the .prev restored", bin)
	}
	// And a --recover with nothing to recover — the every-boot case — is
	// just as storeless.
	f2, _ := applyHappy(t)
	f2.a.openStore = func(context.Context) (release.Store, ed25519.PublicKey, error) {
		t.Error("a no-op --recover opened the deploy store")
		return nil, nil, errors.New("no store on this box")
	}
	if err := f2.a.Run(context.Background(), true); err != nil {
		t.Fatal(err)
	}
	if f2.result() != nil {
		t.Fatalf("a --recover with no journal wrote a result: %+v", f2.result())
	}
}

// --recover waits for the apply lock, but not forever. carlos-system-recover
// is ordered Before=carlos-edge.service, so a recover start job queued while
// an apply is mid-`systemctl restart carlos-edge` has systemd holding the
// restart for the recovery — and a recovery blocking on the lock would hold
// it for the apply. Both would sit there until their unit timeouts. It waits
// half a minute and stands down; the apply recovers at its own start.
func TestRecoverGivesUpOnALockAnApplyHolds(t *testing.T) {
	f, _ := applyHappy(t)
	writeJSONDurable(filepath.Join(systemRootDir(f.a.optDir), "journal.json"),
		systemJournal{Phase: systemPhaseUnits, Target: "v2", PrevBuild: "v1", StartedAt: f.now}, 0o600)
	os.WriteFile(systemPrevPath(f.a.optDir), []byte("#!/bin/sh\necho carlos v1\n"), 0o755)
	held, err := os.OpenFile(f.a.lockPath(), os.O_CREATE|os.O_RDWR, 0o600)
	if err != nil {
		t.Fatal(err)
	}
	defer held.Close()
	if err := flockExclusiveNonblock(held); err != nil {
		t.Skipf("flock unsupported on this platform: %v", err)
	}
	var logged []string
	f.a.ops.logf = func(format string, args ...any) { logged = append(logged, format) }
	start := f.now
	if err := f.a.Run(context.Background(), true); err != nil {
		t.Fatalf("a locked-out --recover must exit 0: %v", err)
	}
	if waited := f.now.Sub(start); waited != systemRecoverLockWait {
		t.Errorf("waited %s, want %s", waited, systemRecoverLockWait)
	}
	if len(logged) != 1 || !strings.Contains(logged[0], "it recovers at its own start") {
		t.Errorf("log = %v", logged)
	}
	if j, _ := readSystemJournal(f.a.optDir); j == nil {
		t.Error("a locked-out --recover closed the journal it never read")
	}
	if len(f.calls) != 0 {
		t.Errorf("a locked-out --recover touched the box: %v", f.calls)
	}
	// The lock is free again: the next run gets it and does the work.
	held.Close()
	if err := f.a.Run(context.Background(), true); err != nil {
		t.Fatal(err)
	}
	if j, _ := readSystemJournal(f.a.optDir); j != nil {
		t.Error("the recovery that got the lock left the journal open")
	}
}

// The carlos.prev backup is NOT journaled, though everything else that
// touches the box is. copyFileDurable is whole-or-nothing, so there is
// nothing to recover from — and a journal here is actively harmful: the
// exec shim in carlos-system-recover.service reads it at the next boot and
// would run a .prev one release behind the box for a whole apply.
//
// A sentinel journal makes the absence visible: decidePrev neither writes
// over it nor removes it. (Nothing puts a journal there in real life —
// recoverIfNeeded has closed any open one long before install runs.)
func TestPrevBackupIsNotJournaled(t *testing.T) {
	f, _ := applyHappy(t)
	writeJSONDurable(filepath.Join(systemRootDir(f.a.optDir), "journal.json"),
		systemJournal{Phase: "sentinel", Target: "v9", PrevBuild: "v1", StartedAt: f.now}, 0o600)
	writeJSONDurable(filepath.Join(systemRootDir(f.a.optDir), "applied.json"),
		systemApplied{Version: "v1", Channel: "stable", At: f.now}, 0o644)
	ok, err := f.a.decidePrev(context.Background())
	if err != nil || !ok {
		t.Fatalf("decidePrev = %v, %v", ok, err)
	}
	prev, _ := os.ReadFile(systemPrevPath(f.a.optDir))
	if !strings.Contains(string(prev), "carlos v1") {
		t.Fatalf(".prev = %q, want the known-good v1 backed up", prev)
	}
	j, err := readSystemJournal(f.a.optDir)
	if err != nil {
		t.Fatal(err)
	}
	if j == nil || j.Phase != "sentinel" {
		t.Fatalf("journal = %+v, want the sentinel untouched: the backup writes no journal", j)
	}
}

// A journal written by the version that DID journal the backup outlives the
// upgrade to the one that does not: recovery still knows the phase, cleans
// the half-written copy, and carries on to the request it was started for.
func TestRecoveryStillHandlesALegacyBackingUpJournal(t *testing.T) {
	f, req := applyHappy(t)
	tmp := systemPrevPath(f.a.optDir) + ".tmp"
	os.WriteFile(tmp, []byte("half a binary"), 0o755)
	writeJSONDurable(filepath.Join(systemRootDir(f.a.optDir), "journal.json"),
		systemJournal{Phase: systemPhaseBackingUp, PrevBuild: "v1", StartedAt: f.now}, 0o600)
	if err := f.a.Run(context.Background(), false); err != nil {
		t.Fatal(err)
	}
	if _, err := os.Stat(tmp); err == nil {
		t.Error("the half-written .prev copy was left behind")
	}
	if j, _ := readSystemJournal(f.a.optDir); j != nil {
		t.Error("the legacy journal was left open")
	}
	if r := f.result(); r == nil || !r.matches("stable", req.Version, req.PromotedAt) || r.Outcome != systemOutcomeApplied {
		t.Fatalf("result = %+v, want the run to carry on to its request", r)
	}
}

// TestApplyArmsTheTargetBuildsList is the forward-compatibility half of
// discussion #118. The applier is the OLD binary: when the target build
// introduces a unit that must be started at apply time rather than at the
// next boot, only the target build's own plan knows that. So arm() starts
// what plan.Armed names, even a unit this binary's own rule would skip.
//
// Without this, a unit new to the box is enabled and left inactive — which
// is exactly how carlos-relay.service reached the Ireland box on 2026-09-07
// enabled, gated, armed with a masters file, and not running.
func TestApplyArmsTheTargetBuildsList(t *testing.T) {
	_, started := armCalls(t, bootstrapPlan{
		Enabled: []string{"carlos-edge.service", "carlos-future.service"},
		// A service this binary's armedUnits would never pick.
		Armed: []string{"carlos-future.service"},
	})
	if !slices.Contains(started, "carlos-future.service") {
		t.Errorf("apply did not start the unit the target build asked for: started %v", started)
	}
	if slices.Contains(started, "carlos-edge.service") {
		t.Errorf("apply started the edge, which runs this very code: started %v", started)
	}
}

// The plan's armed list comes from another build, so arm() does not take it
// on trust: the edge is never started whatever the plan says, and a unit the
// plan arms but does not enable is dropped rather than passed to systemctl,
// because one bad name in the single batched `start` would lose every unit
// in it.
func TestApplyRefusesToArmTheEdgeOrAnUnenabledUnit(t *testing.T) {
	_, started := armCalls(t, bootstrapPlan{
		Enabled: []string{"carlos-adopt.timer", "carlos-edge.service"},
		Armed:   []string{"carlos-adopt.timer", "carlos-edge.service", "carlos-never-installed.service"},
	})
	if slices.Contains(started, "carlos-edge.service") {
		t.Errorf("the edge was started from the plan's armed list: started %v", started)
	}
	if slices.Contains(started, "carlos-never-installed.service") {
		t.Errorf("a unit the plan never enabled was handed to systemctl: started %v", started)
	}
	if !slices.Contains(started, "carlos-adopt.timer") {
		t.Errorf("the good unit was lost along with the bad ones: started %v", started)
	}
}

// A plan.json from a build older than the Armed field has no armed key. That
// is what a rollback stages (carlos.prev's plan), so the fallback is not a
// theoretical path — arm() must decide locally exactly as it did before.
func TestApplyArmsLocallyWhenPlanPredatesArmed(t *testing.T) {
	var plan bootstrapPlan
	if err := json.Unmarshal([]byte(`{"enabled":["carlos-adopt.timer","carlos-edge.service","carlos-relay.service"]}`), &plan); err != nil {
		t.Fatal(err)
	}
	if plan.Armed != nil {
		t.Fatalf("armed should be absent, got %v", plan.Armed)
	}
	_, started := armCalls(t, plan)
	// Membership, not substring: the relay is batched onto the same start
	// line as the timer, and its bare name also appears on the enable line.
	// A substring check is how the first version of this test passed
	// against a fallback changed to drop the relay — the exact #118 failure
	// mode it exists to catch.
	for _, want := range []string{"carlos-adopt.timer", "carlos-relay.service"} {
		if !slices.Contains(started, want) {
			t.Errorf("fallback did not arm %s: started %v", want, started)
		}
	}
	if slices.Contains(started, "carlos-edge.service") {
		t.Errorf("fallback started the edge: started %v", started)
	}
}

// armCalls runs arm(plan) against a recording systemd and returns the units
// it enabled and the units it started. Both are read as SETS out of the two
// batched systemctl lines: arm puts every unit on one `enable` and every
// started unit on one `start`, so a substring check over the joined log
// cannot tell "start carlos-relay.service" from the same name appearing in
// the enable line, and cannot see a name that is merely batched beside
// another. The first version of these tests got both wrong.
func armCalls(t *testing.T, plan bootstrapPlan) (enabled, started []string) {
	t.Helper()
	a := &systemApplier{ops: sysOps{
		systemctl: func(args ...string) (string, error) {
			switch args[0] {
			case "enable":
				enabled = append(enabled, args[1:]...)
			case "start":
				started = append(started, args[1:]...)
			}
			return "", nil
		},
		logf: func(string, ...any) {},
	}}
	if err := a.arm(plan); err != nil {
		t.Fatal(err)
	}
	return enabled, started
}

// seedRoutes writes rows into the fixture's registry and closes it, so the
// guard's own open is the only one held when Run reads it.
func (f *applyFixture) seedRoutes(routes ...registry.Route) {
	f.t.Helper()
	reg, err := registry.Open(filepath.Join(f.a.dataDir, "registry.db"))
	if err != nil {
		f.t.Fatal(err)
	}
	defer reg.Close()
	for _, rt := range routes {
		if err := reg.Upsert(rt); err != nil {
			f.t.Fatal(err)
		}
	}
}

func flatRow(dataDir, host string) registry.Route {
	return registry.Route{Host: host, Kind: "instance", App: "tito", Channel: "stable", Hibernate: true, Status: "ready",
		DBPath: provision.DefaultDBPath(dataDir, host), Socket: provision.DefaultSocket(host)}
}

func nestedRow(dataDir, host string) registry.Route {
	rt := flatRow(dataDir, host)
	rt.DBPath, rt.Socket = provision.NestedDBPath(dataDir, host), provision.NestedSocket(host)
	return rt
}

// postLayout gives the fixture's candidate the mount unit the layout ships
// with, which is what makes it a release that understands nested rows.
func (f *applyFixture) postLayout() {
	staged := filepath.Join(f.a.stagingDir(), "carlos")
	f.answer["units:"+staged] += " data-instances.mount"
}

// A pre-layout candidate would remove the mount units and then serve a
// nested row from an empty mountpoint directory, so the apply refuses it
// before anything on the box changes, and names the row in the way.
func TestApplyRefusesAPreLayoutCandidateOverANestedRow(t *testing.T) {
	f, req := applyHappy(t)
	f.seedRoutes(flatRow(f.a.dataDir, "flat.example.com"), nestedRow(f.a.dataDir, "acme.example.com"))
	if err := f.a.Run(context.Background(), false); err != nil {
		t.Fatal(err)
	}
	r := f.result()
	if r == nil || r.Outcome != systemOutcomeRefused || !r.matches("stable", "v2", req.PromotedAt) {
		t.Fatalf("result = %+v", r)
	}
	want := "refused: v2 predates the instance layout, and this box still has: acme.example.com. " +
		"revert the layout and wait for every row to report flat before installing it"
	if r.Reason != want {
		t.Fatalf("reason = %q\nwant     %q", r.Reason, want)
	}
	if j, _ := readSystemJournal(f.a.optDir); j != nil {
		t.Fatalf("a refused apply wrote a journal: %+v", j)
	}
	if bin, _ := os.ReadFile(systemBinaryPath(f.a.optDir)); !strings.Contains(string(bin), "carlos v1") {
		t.Fatal("a refused apply swapped the binary")
	}
	if f.called("systemctl stop") || f.called("systemctl restart") {
		t.Fatalf("a refused apply touched systemd: %v", f.calls)
	}
}

func TestApplyInstallsAPreLayoutCandidateOverFlatRows(t *testing.T) {
	f, req := applyHappy(t)
	f.seedRoutes(flatRow(f.a.dataDir, "flat.example.com"), flatRow(f.a.dataDir, "acme.example.com"))
	if err := f.a.Run(context.Background(), false); err != nil {
		t.Fatal(err)
	}
	if r := f.result(); r == nil || r.Outcome != systemOutcomeApplied || !r.matches("stable", "v2", req.PromotedAt) {
		t.Fatalf("result = %+v", r)
	}
}

// A candidate that ships the mount units understands nested rows, so the
// guard never even opens the registry for it.
func TestApplyInstallsAPostLayoutCandidateOverNestedRows(t *testing.T) {
	f, req := applyHappy(t)
	f.postLayout()
	f.seedRoutes(nestedRow(f.a.dataDir, "acme.example.com"))
	opened := false
	f.a.openRegistry = func() (*registry.Registry, error) {
		opened = true
		return nil, errors.New("not reached")
	}
	if err := f.a.Run(context.Background(), false); err != nil {
		t.Fatal(err)
	}
	if r := f.result(); r == nil || r.Outcome != systemOutcomeApplied || !r.matches("stable", "v2", req.PromotedAt) {
		t.Fatalf("result = %+v", r)
	}
	if opened {
		t.Error("the guard opened the registry for a candidate that ships the layout")
	}
}

// A candidate that ships the mount units but reports a version below
// release.LayoutFloor (v0.20.0 to v0.24.2: mounts, but an edge unit on
// which nested rows cannot wake) is refused over a nested row all the same.
// This is the box's own line of defence against `carlos system update
// --file`, which the console's guard does not judge. Flat rows let it in.
func TestApplyRefusesAMountedButBelowFloorCandidateOverANestedRow(t *testing.T) {
	t.Parallel()
	for _, tc := range []struct {
		name   string
		rows   []registry.Route
		want   string
		reason string
	}{
		{"nested row", nil, systemOutcomeRefused, "v0.24.2 predates the instance layout, and this box still has: acme.example.com"},
		{"flat rows", nil, systemOutcomeApplied, ""},
	} {
		t.Run(tc.name, func(t *testing.T) {
			f, req := applyHappyVersion(t, "v0.24.2")
			f.postLayout()
			dd := f.a.dataDir
			if tc.name == "nested row" {
				f.seedRoutes(flatRow(dd, "flat.example.com"), nestedRow(dd, "acme.example.com"))
			} else {
				f.seedRoutes(flatRow(dd, "flat.example.com"), flatRow(dd, "acme.example.com"))
			}
			if err := f.a.Run(context.Background(), false); err != nil {
				t.Fatal(err)
			}
			r := f.result()
			if r == nil || r.Outcome != tc.want || !r.matches("stable", "v0.24.2", req.PromotedAt) {
				t.Fatalf("result = %+v, want %s", r, tc.want)
			}
			if tc.reason != "" && !strings.Contains(r.Reason, tc.reason) {
				t.Fatalf("reason = %q, want it to contain %q", r.Reason, tc.reason)
			}
		})
	}
	// At the floor the unit set is what counts, and it ships the mounts.
	f, req := applyHappyVersion(t, "v0.25.0")
	f.postLayout()
	f.seedRoutes(nestedRow(f.a.dataDir, "acme.example.com"))
	if err := f.a.Run(context.Background(), false); err != nil {
		t.Fatal(err)
	}
	if r := f.result(); r == nil || r.Outcome != systemOutcomeApplied || !r.matches("stable", "v0.25.0", req.PromotedAt) {
		t.Fatalf("v0.25.0 over a nested row = %+v, want applied", r)
	}
}

// A registry the guard cannot read cannot vouch that nothing is nested.
func TestApplyRefusesAPreLayoutCandidateWhenTheRegistryWillNotOpen(t *testing.T) {
	f, _ := applyHappy(t)
	f.a.openRegistry = func() (*registry.Registry, error) { return nil, errors.New("registry.db: disk I/O error") }
	if err := f.a.Run(context.Background(), false); err != nil {
		t.Fatal(err)
	}
	r := f.result()
	if r == nil || r.Outcome != systemOutcomeRefused || !strings.Contains(r.Reason, "registry.db: disk I/O error") ||
		!strings.Contains(r.Reason, "predates the instance layout") {
		t.Fatalf("result = %+v", r)
	}
	if j, _ := readSystemJournal(f.a.optDir); j != nil {
		t.Fatalf("a refused apply wrote a journal: %+v", j)
	}
}

// Every row a pre-layout release would mishandle is named once: nested by
// path shape whatever its flags, fenced by a journal, and a journal whose
// row is gone. Flat rows are not named.
func TestLayoutReversalIncompleteNamesEveryRowInTheWay(t *testing.T) {
	t.Parallel()
	dd := t.TempDir()
	reg, err := registry.Open(filepath.Join(dd, "registry.db"))
	if err != nil {
		t.Fatal(err)
	}
	defer reg.Close()
	unflagged := nestedRow(dd, "unflagged.example.com")
	unflagged.Hibernate = false
	for _, rt := range []registry.Route{
		flatRow(dd, "flat.example.com"),
		nestedRow(dd, "nested.example.com"),
		unflagged,
		flatRow(dd, "moving.example.com"),
	} {
		if err := reg.Upsert(rt); err != nil {
			t.Fatal(err)
		}
	}
	os.MkdirAll(filepath.Join(dd, "layout"), 0o700)
	for _, h := range []string{"moving.example.com", "gone.example.com"} {
		os.WriteFile(provision.LayoutJournalPath(dd, h), []byte("{}"), 0o600)
	}
	// A journal's temp file is not a journal.
	os.WriteFile(filepath.Join(dd, "layout", ".other.example.com.tmp-1"), []byte("{}"), 0o600)

	got := layoutReversalIncomplete(reg, dd)
	want := []string{"gone.example.com", "moving.example.com", "nested.example.com", "unflagged.example.com"}
	if !slices.Equal(got, want) {
		t.Fatalf("reasons = %q, want %q", got, want)
	}

	flatOnly := t.TempDir()
	reg2, err := registry.Open(filepath.Join(flatOnly, "registry.db"))
	if err != nil {
		t.Fatal(err)
	}
	defer reg2.Close()
	if err := reg2.Upsert(flatRow(flatOnly, "flat.example.com")); err != nil {
		t.Fatal(err)
	}
	if got := layoutReversalIncomplete(reg2, flatOnly); len(got) != 0 {
		t.Fatalf("a flat box reported %q", got)
	}
}

// ---- the moves protocol floor (moves spec §11, Task 13) ----

// floorFixture is applyHappy with release numbers the floor can compare: a
// box running `running` (applied.json names it, so the install refreshes
// carlos.prev from it) and a signed candidate at target. The candidate
// ships the layout's mount unit, so the layout guard never opens the
// registry and every registry open in these tests is the floor's own.
func floorFixture(t *testing.T, running, target string) (*applyFixture, systemRequest) {
	t.Helper()
	f := newApplyFixture(t, "stable", running)
	req := f.promote(target, []byte("#!/bin/sh\necho carlos "+target+"\n"), false)
	bin, staged, prev := systemBinaryPath(f.a.optDir), filepath.Join(f.a.stagingDir(), "carlos"), systemPrevPath(f.a.optDir)
	f.answer["version:"+staged] = "carlos " + target + "\n"
	f.answer["plan:"+staged] = f.answer["plan:"+bin]
	f.answer["units:"+staged] = f.answer["units:"+bin] + " " + layoutMountUnit
	f.answer["plan:"+prev] = f.answer["plan:"+bin]
	f.answer["units:"+prev] = f.answer["units:"+bin]
	// Every binary answers `version` from its own bytes from here on.
	delete(f.answer, "version:"+bin)
	if err := writeJSONDurable(f.a.file("applied.json"), systemApplied{Version: running, Channel: "stable", At: f.now}, 0o644); err != nil {
		t.Fatal(err)
	}
	return f, req
}

// holdRow seeds a row and holds it under a move, as the agent's SetHold does.
func (f *applyFixture) holdRow(host string) {
	f.t.Helper()
	f.seedRoutes(flatRow(f.a.dataDir, host))
	reg, err := registry.Open(filepath.Join(f.a.dataDir, "registry.db"))
	if err != nil {
		f.t.Fatal(err)
	}
	defer reg.Close()
	if err := reg.SetHold(host, "moving", "op-1"); err != nil {
		f.t.Fatal(err)
	}
}

// onDisk is what the box's carlos binary says it is.
func (f *applyFixture) onDisk() string { return f.a.onDiskBuild(context.Background()) }

// brokenBox puts a failed install on disk (running), a carlos.prev at prev
// with its sidecar (none when sidecar is ""), and the journal an install
// leaves open after its restart.
func (f *applyFixture) brokenBox(running, prev, sidecar string) systemJournal {
	f.t.Helper()
	os.WriteFile(systemBinaryPath(f.a.optDir), []byte("#!/bin/sh\necho carlos "+running+"\n"), 0o755)
	os.WriteFile(systemPrevPath(f.a.optDir), []byte("#!/bin/sh\necho carlos "+prev+"\n"), 0o755)
	if sidecar != "" {
		if err := writePrevVersion(f.a.optDir, sidecar); err != nil {
			f.t.Fatal(err)
		}
	}
	j := systemJournal{Phase: systemPhaseRestarted, Target: running, PrevBuild: prev, StartedAt: f.now}
	if err := f.a.journal(j); err != nil {
		f.t.Fatal(err)
	}
	return j
}

func TestSystemApplyArmsTheFloorBeforeStaging(t *testing.T) {
	f, _ := floorFixture(t, "v0.28.0", "v0.29.0")
	putFloor(t, f.store, "v0.30.0")
	staged := filepath.Join(f.a.stagingDir(), "carlos")
	run := f.a.ops.run
	f.a.ops.run = func(ctx context.Context, d time.Duration, name string, args ...string) (string, error) {
		if name == staged {
			t.Errorf("a below-floor candidate was staged and run: %s %v", name, args)
		}
		return run(ctx, d, name, args...)
	}
	before, _ := os.ReadFile(systemBinaryPath(f.a.optDir))
	if err := f.a.Run(context.Background(), false); err != nil {
		t.Fatal(err)
	}
	if lf, err := readLocalFloor(f.a.dataDir); err != nil || lf == nil || lf.MinRelease != "v0.30.0" {
		t.Fatalf("local floor = %+v, %v", lf, err)
	}
	if b, _ := os.ReadFile(filepath.Join(f.a.dataDir, moves.FloorMinReleaseFile)); string(b) != "v0.30.0\n" {
		t.Fatalf("floor.min-release = %q", b)
	}
	r := f.result()
	if r == nil || r.Outcome != systemOutcomeRefused || !strings.Contains(r.Reason, "moves protocol floor: v0.29.0 is below v0.30.0") {
		t.Fatalf("result = %+v", r)
	}
	if after, _ := os.ReadFile(systemBinaryPath(f.a.optDir)); !bytes.Equal(before, after) {
		t.Fatal("the binary changed")
	}
	if _, err := os.Stat(systemPrevPath(f.a.optDir)); err == nil {
		t.Fatal("a refused install refreshed carlos.prev")
	}
	if j, _ := readSystemJournal(f.a.optDir); j != nil {
		t.Fatal("a refused install opened a journal")
	}
}

// Boot recovery runs before the network: the floor it enforces is the one
// on disk, and the version it judges is carlos.prev's sidecar.
func TestBootRecoveryUsesTheArmedFloorOffline(t *testing.T) {
	for _, tc := range []struct {
		name, sidecar string
		restored      bool
	}{
		{"below the floor", "v0.29.0", false},
		{"at the floor", "v0.30.0", true},
		{"no sidecar", "", false},
	} {
		t.Run(tc.name, func(t *testing.T) {
			f, _ := floorFixture(t, "v0.31.0", "v0.32.0")
			f.a.openStore = func(context.Context) (release.Store, ed25519.PublicKey, error) {
				return unreadableStore{f.store}, f.pub, nil
			}
			armLocalFloor(t, f.a.dataDir, "v0.30.0")
			prev := tc.sidecar
			if prev == "" {
				prev = "v0.30.0"
			}
			f.brokenBox("v0.31.0", prev, tc.sidecar)
			f.edgeStartedThisBoot = false
			err := f.a.Run(context.Background(), true)
			j, _ := readSystemJournal(f.a.optDir)
			if tc.restored {
				if err != nil || f.onDisk() != "v0.30.0" || j != nil {
					t.Fatalf("err %v, on disk %q, journal %+v", err, f.onDisk(), j)
				}
				return
			}
			if err == nil || !strings.Contains(err.Error(), "moves protocol floor") {
				t.Fatalf("recovery err = %v", err)
			}
			if f.onDisk() != "v0.31.0" || j == nil {
				t.Fatalf("a refused recovery changed the box: on disk %q, journal %+v", f.onDisk(), j)
			}
		})
	}
}

// decidePrev learns carlos.prev's version and judges it before the journal
// is written. The proof of "before": region HA defers this install at the
// step just ahead of the journal, and the two files are already there.
func TestDecidePrevWritesTheSidecarAndVerdictBeforeTheJournal(t *testing.T) {
	t.Parallel()
	for _, tc := range []struct {
		name, running string
		held, verdict bool
	}{
		{"at or above the floor", "v0.31.0", false, true},
		{"below the floor", "v0.29.0", false, false},
		{"a held row", "v0.31.0", true, false},
	} {
		t.Run(tc.name, func(t *testing.T) {
			f, _ := floorFixture(t, tc.running, "v0.32.0")
			putFloor(t, f.store, "v0.30.0")
			if tc.held {
				f.holdRow("app.example.com")
			}
			// A verdict from an earlier judgement must not outlive this one.
			writeFileDurable(filepath.Join(f.a.optDir, moves.PrevAllowedFile), []byte("v0.30.0\n"), 0o644)
			f.a.regionHA = true
			os.MkdirAll(filepath.Join(f.a.dataDir, hostexecDir, "op-9"), 0o755)
			if err := f.a.Run(context.Background(), false); err != nil {
				t.Fatal(err)
			}
			if r := f.result(); r == nil || r.Outcome != systemOutcomeDeferred {
				t.Fatalf("result = %+v", r)
			}
			if j, _ := readSystemJournal(f.a.optDir); j != nil {
				t.Fatal("journal written")
			}
			if b, _ := os.ReadFile(filepath.Join(f.a.optDir, moves.PrevVersionFile)); string(b) != tc.running+"\n" {
				t.Fatalf("sidecar = %q, want %s", b, tc.running)
			}
			b, err := os.ReadFile(filepath.Join(f.a.optDir, moves.PrevAllowedFile))
			if tc.verdict != (err == nil) || (tc.verdict && string(b) != "v0.30.0\n") {
				t.Fatalf("verdict = %q, %v; want present %v", b, err, tc.verdict)
			}
		})
	}
}

// Replacing carlos.prev withdraws its predecessor's sidecar and verdict
// first, so a copy that dies part way leaves a carlos.prev nothing vouches
// for, never the new bytes under the old verdict.
func TestDecidePrevForgetsTheOldVerdictBeforeTheCopy(t *testing.T) {
	f, _ := floorFixture(t, "v0.31.0", "v0.32.0")
	putFloor(t, f.store, "v0.30.0")
	os.WriteFile(systemPrevPath(f.a.optDir), []byte("#!/bin/sh\necho carlos v0.30.0\n"), 0o755)
	writePrevVersion(f.a.optDir, "v0.30.0")
	writeFileDurable(filepath.Join(f.a.optDir, moves.PrevAllowedFile), []byte("v0.30.0\n"), 0o644)
	// Prevent publication without depending on the writer's temporary name.
	if err := os.Remove(systemPrevPath(f.a.optDir)); err != nil {
		t.Fatal(err)
	}
	if err := os.Mkdir(systemPrevPath(f.a.optDir), 0o755); err != nil {
		t.Fatal(err)
	}
	if err := f.a.Run(context.Background(), false); err != nil {
		t.Fatal(err)
	}
	if r := f.result(); r == nil || r.Outcome != systemOutcomeRefused {
		t.Fatalf("result = %+v", r)
	}
	for _, name := range []string{moves.PrevAllowedFile, moves.PrevVersionFile} {
		if _, err := os.Stat(filepath.Join(f.a.optDir, name)); err == nil {
			t.Errorf("%s survived a failed replacement of carlos.prev", name)
		}
	}
}

// Moves spec §11 (round-6 finding 5): while a row is held, no activating
// path takes the box below the release it runs, whatever the floor allows.
func TestHeldParticipantRefusesEveryDowngrade(t *testing.T) {
	const downgrade = "a participant in a held operation does not downgrade"
	for _, held := range []bool{true, false} {
		t.Run(fmt.Sprintf("held=%v", held), func(t *testing.T) {
			// install
			f, _ := floorFixture(t, "v0.32.0", "v0.31.0")
			putFloor(t, f.store, "v0.30.0")
			if held {
				f.holdRow("app.example.com")
			}
			f.a.Run(context.Background(), false)
			r := f.result()
			if held && (r == nil || r.Outcome != systemOutcomeRefused || !strings.Contains(r.Reason, downgrade) || f.onDisk() != "v0.32.0") {
				t.Fatalf("held install: result %+v, on disk %q", r, f.onDisk())
			}
			if !held && (r == nil || r.Outcome != systemOutcomeApplied || f.onDisk() != "v0.31.0") {
				t.Fatalf("unheld install: result %+v, on disk %q", r, f.onDisk())
			}

			// rollback
			f, req := floorFixture(t, "v0.32.0", "v0.33.0")
			armLocalFloor(t, f.a.dataDir, "v0.30.0")
			if held {
				f.holdRow("app.example.com")
			}
			j := f.brokenBox("v0.32.0", "v0.31.0", "v0.31.0")
			f.a.rollback(context.Background(), req, "gate: test", &j)
			r = f.result()
			if held && (r == nil || r.Outcome != systemOutcomeRollbackFailed || !strings.Contains(r.Reason, downgrade) || f.onDisk() != "v0.32.0") {
				t.Fatalf("held rollback: result %+v, on disk %q", r, f.onDisk())
			}
			if !held && (r == nil || r.Outcome != systemOutcomeRolledBack || f.onDisk() != "v0.31.0") {
				t.Fatalf("unheld rollback: result %+v, on disk %q", r, f.onDisk())
			}

			// restoreFiles, which recovery calls without rollback
			f, _ = floorFixture(t, "v0.32.0", "v0.33.0")
			armLocalFloor(t, f.a.dataDir, "v0.30.0")
			if held {
				f.holdRow("app.example.com")
			}
			j = f.brokenBox("v0.32.0", "v0.31.0", "v0.31.0")
			_, err := f.a.restoreFiles(context.Background(), &j)
			if held && (err == nil || !strings.Contains(err.Error(), downgrade) || f.onDisk() != "v0.32.0") {
				t.Fatalf("held restore: err %v, on disk %q", err, f.onDisk())
			}
			if !held && (err != nil || f.onDisk() != "v0.31.0") {
				t.Fatalf("unheld restore: err %v, on disk %q", err, f.onDisk())
			}
		})
	}
}

// D13: the floor is armed by publishing its object, not by the switch. A
// deployment that never published it behaves as main, held rows and all,
// and never opens the registry for the floor. Once armed, turning the
// switch off does not disarm it.
func TestFloorIsDormantWithoutTheObject(t *testing.T) {
	t.Setenv(moves.SwitchEnv, "")
	counting := func(f *applyFixture) *int {
		n := 0
		open := f.a.openRegistry
		f.a.openRegistry = func() (*registry.Registry, error) { n++; return open() }
		return &n
	}

	f, _ := floorFixture(t, "v0.32.0", "v0.31.0")
	f.holdRow("app.example.com")
	opened := counting(f)
	if err := f.a.Run(context.Background(), false); err != nil {
		t.Fatal(err)
	}
	if r := f.result(); r == nil || r.Outcome != systemOutcomeApplied || f.onDisk() != "v0.31.0" {
		t.Fatalf("install: result %+v, on disk %q", r, f.onDisk())
	}
	if *opened != 0 {
		t.Fatalf("an unarmed install opened the registry %d times", *opened)
	}
	// The activation guard creates .moves for its lock on every box (fix
	// round 2); an unarmed install writes no floor into it.
	for _, name := range []string{moves.FloorFile, moves.FloorMinReleaseFile} {
		if _, err := os.Stat(filepath.Join(f.a.dataDir, name)); err == nil {
			t.Fatalf("an unarmed install wrote %s", name)
		}
	}
	if _, err := os.Stat(filepath.Join(f.a.optDir, moves.PrevAllowedFile)); err == nil {
		t.Fatal("an unarmed install wrote a verdict")
	}

	f, req := floorFixture(t, "v0.32.0", "v0.33.0")
	f.holdRow("app.example.com")
	opened = counting(f)
	j := f.brokenBox("v0.32.0", "v0.31.0", "")
	f.a.rollback(context.Background(), req, "gate: test", &j)
	if r := f.result(); r == nil || r.Outcome != systemOutcomeRolledBack || f.onDisk() != "v0.31.0" {
		t.Fatalf("rollback: result %+v, on disk %q", r, f.onDisk())
	}
	if *opened != 0 {
		t.Fatalf("an unarmed rollback opened the registry %d times", *opened)
	}

	f, _ = floorFixture(t, "v0.32.0", "v0.33.0")
	f.holdRow("app.example.com")
	opened = counting(f)
	j = f.brokenBox("v0.32.0", "v0.31.0", "")
	if _, err := f.a.restoreFiles(context.Background(), &j); err != nil || f.onDisk() != "v0.31.0" {
		t.Fatalf("restore: err %v, on disk %q", err, f.onDisk())
	}
	if *opened != 0 {
		t.Fatalf("an unarmed restore opened the registry %d times", *opened)
	}

	// Armed on disk, the switch still off: a below-floor carlos.prev stays put.
	f, _ = floorFixture(t, "v0.32.0", "v0.33.0")
	armLocalFloor(t, f.a.dataDir, "v0.30.0")
	j = f.brokenBox("v0.32.0", "v0.29.0", "v0.29.0")
	if _, err := f.a.restoreFiles(context.Background(), &j); err == nil || !strings.Contains(err.Error(), "moves protocol floor: v0.29.0 is below v0.30.0") {
		t.Fatalf("armed restore with the switch off: err %v", err)
	}
	if f.onDisk() != "v0.32.0" {
		t.Fatalf("on disk %q", f.onDisk())
	}
}

// duringStaging runs change when the apply runs the staged candidate's
// `version`, which is the middle of staging: after the early activation
// check, before the install puts anything in place.
func (f *applyFixture) duringStaging(change func()) {
	staged := filepath.Join(f.a.stagingDir(), "carlos")
	run := f.a.ops.run
	once := false
	f.a.ops.run = func(ctx context.Context, d time.Duration, name string, args ...string) (string, error) {
		if name == staged && !once {
			once = true
			change()
		}
		return run(ctx, d, name, args...)
	}
}

// Fix round 1, finding 1: the early check can go stale while the candidate
// is staged. The install checks again at the rename, under the floor lock
// and with the held rows frozen, and a refusal there leaves the box as it
// was: the binary untouched, no journal, and the quiesced passes re-armed.
func TestActivationIsCheckedAgainAtTheRename(t *testing.T) {
	for _, tc := range []struct {
		name, running, target, want string
		change                      func(f *applyFixture)
	}{
		{"a floor raised during staging", "v0.31.0", "v0.32.0", "moves protocol floor: v0.32.0 is below v0.33.0",
			func(f *applyFixture) { armLocalFloor(t, f.a.dataDir, "v0.33.0") }},
		{"a hold set during staging", "v0.32.0", "v0.31.0", "a participant in a held operation does not downgrade",
			func(f *applyFixture) { f.holdRow("app.example.com") }},
	} {
		t.Run(tc.name, func(t *testing.T) {
			f, _ := floorFixture(t, tc.running, tc.target)
			putFloor(t, f.store, "v0.30.0")
			f.duringStaging(func() { tc.change(f) })
			if err := f.a.Run(context.Background(), false); err != nil {
				t.Fatal(err)
			}
			r := f.result()
			if r == nil || r.Outcome != systemOutcomeRefused || !strings.Contains(r.Reason, tc.want) {
				t.Fatalf("result = %+v", r)
			}
			if f.onDisk() != tc.running {
				t.Fatalf("on disk %q, want %s", f.onDisk(), tc.running)
			}
			if j, _ := readSystemJournal(f.a.optDir); j != nil {
				t.Fatalf("journal left open: %+v", j)
			}
			joined := strings.Join(f.calls, "\n")
			if !strings.Contains(joined, "systemctl start carlos-adopt.timer") {
				t.Fatalf("the quiesced passes were not re-armed:\n%s", joined)
			}
			if strings.Contains(joined, "systemctl restart carlos-edge") {
				t.Fatalf("a refused install restarted the edge:\n%s", joined)
			}
		})
	}
}

// Fix round 1, finding 2: a gated journal is finished forward only when its
// target is at or above the floor armed now. Below it, recovery goes back
// to carlos.prev, which restoreFiles judges in turn.
func TestForwardRecoveryChecksTheFloor(t *testing.T) {
	for _, tc := range []struct {
		name, prev, want string
		ok               bool
	}{
		{"carlos.prev below the floor too", "v0.30.0", "v0.31.0", false},
		{"carlos.prev at the floor", "v0.32.0", "v0.32.0", true},
	} {
		t.Run(tc.name, func(t *testing.T) {
			f, _ := floorFixture(t, "v0.30.0", "v0.31.0")
			f.brokenBox("v0.31.0", tc.prev, tc.prev)
			f.a.journal(systemJournal{Phase: systemPhaseGated, Target: "v0.31.0", PrevBuild: tc.prev, StartedAt: f.now})
			armLocalFloor(t, f.a.dataDir, "v0.32.0")
			f.edgeStartedThisBoot = false
			err := f.a.Run(context.Background(), true)
			if ap, _ := readSystemApplied(f.a.optDir); ap != nil && ap.Version == "v0.31.0" {
				t.Fatalf("a below-floor target was finished forward: %+v", ap)
			}
			j, _ := readSystemJournal(f.a.optDir)
			if f.onDisk() != tc.want {
				t.Fatalf("on disk %q, want %s", f.onDisk(), tc.want)
			}
			if tc.ok && (err != nil || j != nil) {
				t.Fatalf("err %v, journal %+v", err, j)
			}
			if !tc.ok && (err == nil || !strings.Contains(err.Error(), "moves protocol floor") || j == nil) {
				t.Fatalf("err %v, journal %+v", err, j)
			}
		})
	}
}

// restoreFiles puts carlos.prev in place under the floor lock: a floor
// writer holding it holds the restore off.
func TestRestoreWaitsForTheFloorLock(t *testing.T) {
	f, _ := floorFixture(t, "v0.32.0", "v0.33.0")
	armLocalFloor(t, f.a.dataDir, "v0.30.0")
	j := f.brokenBox("v0.32.0", "v0.31.0", "v0.31.0")
	held, free := make(chan struct{}), make(chan struct{})
	go withFloorLock(f.a.dataDir, false, func(*movesDir) error {
		close(held)
		<-free
		return nil
	})
	<-held
	done := make(chan error, 1)
	go func() { _, err := f.a.restoreFiles(context.Background(), &j); done <- err }()
	time.Sleep(300 * time.Millisecond)
	// The bytes, not f.onDisk: the fake ops are the restore's while it runs.
	if b, _ := os.ReadFile(systemBinaryPath(f.a.optDir)); !strings.Contains(string(b), "v0.32.0") {
		close(free)
		t.Fatalf("restored while the floor lock was held: %q", b)
	}
	close(free)
	if err := <-done; err != nil || f.onDisk() != "v0.31.0" {
		t.Fatalf("after the lock: err %v, on disk %q", err, f.onDisk())
	}
}

// The install's rename waits for the floor lock: a floor writer that takes
// it after the journal (at the quiesce) holds the binary where it is.
func TestInstallRenameWaitsForTheFloorLock(t *testing.T) {
	f, _ := floorFixture(t, "v0.31.0", "v0.32.0")
	putFloor(t, f.store, "v0.30.0")
	var duringLock []byte
	held := make(chan struct{})
	freed := make(chan struct{})
	systemctl := f.a.ops.systemctl
	once := false
	f.a.ops.systemctl = func(args ...string) (string, error) {
		if args[0] == "stop" && !once {
			once = true
			go func() {
				defer close(freed)
				withFloorLock(f.a.dataDir, false, func(*movesDir) error {
					close(held)
					time.Sleep(time.Second)
					duringLock, _ = os.ReadFile(systemBinaryPath(f.a.optDir))
					return nil
				})
			}()
			<-held
		}
		return systemctl(args...)
	}
	if err := f.a.Run(context.Background(), false); err != nil {
		t.Fatal(err)
	}
	<-freed
	if !strings.Contains(string(duringLock), "carlos v0.31.0") {
		t.Fatalf("the binary was replaced while the floor lock was held: %q", duringLock)
	}
	if r := f.result(); r == nil || r.Outcome != systemOutcomeApplied || f.onDisk() != "v0.32.0" {
		t.Fatalf("result %+v, on disk %q", r, f.onDisk())
	}
}

// ---- fix round 2: one invariant. No executable is renamed into place, and
// no journal is finished, unless the floor and hold check ran under
// floor.lock and WithHoldsFrozen immediately before that step. ----

// unitActivity fakes systemctl's view of the root passes' triggers, so a
// test can see which were active before a step and after it.
func (f *applyFixture) unitActivity(active map[string]bool) {
	systemctl := f.a.ops.systemctl
	f.a.ops.systemctl = func(args ...string) (string, error) {
		switch args[0] {
		case "is-active":
			if len(args) > 1 && args[1] != systemEdgeUnit {
				var out []string
				for _, u := range args[1:] {
					if active[u] {
						out = append(out, "active")
					} else {
						out = append(out, "inactive")
					}
				}
				f.calls = append(f.calls, "systemctl "+strings.Join(args, " "))
				return strings.Join(out, "\n") + "\n", nil
			}
		case "stop":
			for _, u := range args[1:] {
				active[u] = false
			}
		case "start":
			for _, u := range args[1:] {
				active[u] = true
			}
		}
		return systemctl(args...)
	}
}

func copyActivity(m map[string]bool) map[string]bool {
	c := map[string]bool{}
	for k, v := range m {
		c[k] = v
	}
	return c
}

// First arming, during staging: the agent arms a floor on a box that had
// none when the apply began. The rename refuses.
func TestFirstFloorArmedDuringStagingRefusesTheRename(t *testing.T) {
	f, _ := floorFixture(t, "v0.31.0", "v0.32.0")
	f.duringStaging(func() {
		if err := writeLocalFloor(f.a.dataDir, moves.Floor{Min: 1, MinRelease: "v0.33.0"}); err != nil {
			t.Error(err)
		}
	})
	if err := f.a.Run(context.Background(), false); err != nil {
		t.Fatal(err)
	}
	if r := f.result(); r == nil || r.Outcome != systemOutcomeRefused || !strings.Contains(r.Reason, "moves protocol floor: v0.32.0 is below v0.33.0") {
		t.Fatalf("result = %+v", r)
	}
	if f.onDisk() != "v0.31.0" {
		t.Fatalf("on disk %q", f.onDisk())
	}
}

// First arming, between the rule and the rename: the guard holds floor.lock
// even on a box with no floor yet (it creates .moves for it), so the agent's
// first arming waits until the rename is done and cannot slip in between.
func TestFirstArmingWaitsForTheActivationLock(t *testing.T) {
	t.Parallel()
	f, _ := floorFixture(t, "v0.31.0", "v0.32.0")
	armed := make(chan error, 1)
	f.a.beforeSwap = func() {
		go func() { armed <- writeLocalFloor(f.a.dataDir, moves.Floor{Min: 1, MinRelease: "v0.40.0"}) }()
		time.Sleep(300 * time.Millisecond)
		if _, err := os.Stat(filepath.Join(f.a.dataDir, moves.FloorFile)); err == nil {
			t.Error("a first floor was armed between the activation check and the rename")
		}
	}
	if err := f.a.Run(context.Background(), false); err != nil {
		t.Fatal(err)
	}
	if r := f.result(); r == nil || r.Outcome != systemOutcomeApplied {
		t.Fatalf("result = %+v", r)
	}
	if err := <-armed; err != nil {
		t.Fatal(err)
	}
	if lf, _ := readLocalFloor(f.a.dataDir); lf == nil || lf.MinRelease != "v0.40.0" {
		t.Fatalf("the agent's arming after the rename = %+v", lf)
	}
}

// Forward recovery re-checks at the step it guards: a floor raised while
// recovery stages the gated build's plan stops it being finished forward.
func TestForwardRecoveryRechecksAfterStagePlan(t *testing.T) {
	f, _ := floorFixture(t, "v0.30.0", "v0.31.0")
	f.brokenBox("v0.31.0", "v0.30.0", "v0.30.0")
	f.a.journal(systemJournal{Phase: systemPhaseGated, Target: "v0.31.0", PrevBuild: "v0.30.0", StartedAt: f.now})
	armLocalFloor(t, f.a.dataDir, "v0.30.0")
	bin := systemBinaryPath(f.a.optDir)
	run := f.a.ops.run
	f.a.ops.run = func(ctx context.Context, d time.Duration, name string, args ...string) (string, error) {
		if name == bin && len(args) > 0 && args[0] == "bootstrap" {
			if err := writeLocalFloor(f.a.dataDir, moves.Floor{Min: 1, MinRelease: "v0.32.0"}); err != nil {
				t.Error(err)
			}
		}
		return run(ctx, d, name, args...)
	}
	f.edgeStartedThisBoot = false
	err := f.a.Run(context.Background(), true)
	if ap, _ := readSystemApplied(f.a.optDir); ap != nil && ap.Version == "v0.31.0" {
		t.Fatalf("finished forward below a floor raised during stagePlan: %+v", ap)
	}
	if err == nil || !strings.Contains(err.Error(), "moves protocol floor") {
		t.Fatalf("err = %v", err)
	}
	if j, _ := readSystemJournal(f.a.optDir); j == nil {
		t.Fatal("journal removed")
	}
}

// A restore refused after the quiesce (the floor rose during the copy) puts
// the root passes' triggers back as they were, and keeps the journal.
func TestRefusedRestoreLeavesTheRootPassesAsTheyWere(t *testing.T) {
	f, _ := floorFixture(t, "v0.32.0", "v0.33.0")
	armLocalFloor(t, f.a.dataDir, "v0.30.0")
	j := f.brokenBox("v0.32.0", "v0.31.0", "v0.31.0")
	active := map[string]bool{"carlos-adopt.timer": true, "carlos-adopt.path": true, "carlos-stage.path": false, "carlos-restart.path": true}
	before := copyActivity(active)
	f.unitActivity(active)
	systemctl := f.a.ops.systemctl
	f.a.ops.systemctl = func(args ...string) (string, error) {
		out, err := systemctl(args...)
		if args[0] == "stop" {
			armLocalFloor(t, f.a.dataDir, "v0.33.0")
		}
		return out, err
	}
	_, err := f.a.restoreFiles(context.Background(), &j)
	if err == nil || !strings.Contains(err.Error(), "moves protocol floor: v0.31.0 is below v0.33.0") {
		t.Fatalf("err = %v", err)
	}
	for u, was := range before {
		if active[u] != was {
			t.Errorf("%s active = %v after the refusal, was %v", u, active[u], was)
		}
	}
	if f.onDisk() != "v0.32.0" {
		t.Fatalf("on disk %q", f.onDisk())
	}
	if _, err := os.Stat(systemBinaryPath(f.a.optDir) + ".restore"); err == nil {
		t.Fatal("the restore copy was left behind")
	}
	if j, _ := readSystemJournal(f.a.optDir); j == nil {
		t.Fatal("the recovery journal was dropped")
	}
}

// An install refused at the rename starts the paused triggers again first,
// and drops the journal only once they are started.
func TestRefusedInstallRearmsBeforeDroppingTheJournal(t *testing.T) {
	f, _ := floorFixture(t, "v0.31.0", "v0.32.0")
	putFloor(t, f.store, "v0.30.0")
	active := map[string]bool{"carlos-adopt.timer": true, "carlos-adopt.path": true, "carlos-stage.path": true, "carlos-restart.path": false}
	before := copyActivity(active)
	f.unitActivity(active)
	systemctl := f.a.ops.systemctl
	f.a.ops.systemctl = func(args ...string) (string, error) {
		if args[0] == "start" {
			if j, _ := readSystemJournal(f.a.optDir); j == nil {
				t.Error("the journal was dropped before the paused passes were started again")
			}
		}
		return systemctl(args...)
	}
	f.duringStaging(func() { armLocalFloor(t, f.a.dataDir, "v0.33.0") })
	if err := f.a.Run(context.Background(), false); err != nil {
		t.Fatal(err)
	}
	if r := f.result(); r == nil || r.Outcome != systemOutcomeRefused {
		t.Fatalf("result = %+v", r)
	}
	for u, was := range before {
		if active[u] != was {
			t.Errorf("%s active = %v, was %v", u, active[u], was)
		}
	}
	if j, _ := readSystemJournal(f.a.optDir); j != nil {
		t.Fatalf("journal left: %+v", j)
	}
}

// When starting them again fails, the run says so and fails, the retry
// state stays on disk, and the next run (here the OnFailure recovery)
// starts them. The journal does not stay: its units would pick carlos.prev,
// and an older carlos.prev would restore itself over a binary that never
// changed.
func TestRefusedInstallWhoseRearmFailsSaysSoAndRetries(t *testing.T) {
	f, _ := floorFixture(t, "v0.31.0", "v0.32.0")
	putFloor(t, f.store, "v0.30.0")
	active := map[string]bool{"carlos-adopt.timer": true, "carlos-adopt.path": true, "carlos-stage.path": true, "carlos-restart.path": true}
	before := copyActivity(active)
	f.unitActivity(active)
	failStart := true
	systemctl := f.a.ops.systemctl
	f.a.ops.systemctl = func(args ...string) (string, error) {
		if args[0] == "start" && failStart {
			return "", errors.New("unit masked")
		}
		return systemctl(args...)
	}
	f.duringStaging(func() { armLocalFloor(t, f.a.dataDir, "v0.33.0") })
	err := f.a.Run(context.Background(), false)
	if err == nil || !strings.Contains(err.Error(), "could not be started again") {
		t.Fatalf("Run err = %v", err)
	}
	r := f.result()
	if r == nil || r.Outcome != systemOutcomeRefused || !strings.Contains(r.Reason, "could not be started again: unit masked") {
		t.Fatalf("result = %+v", r)
	}
	if _, err := os.Stat(f.a.file(rearmPendingFile)); err != nil {
		t.Fatalf("no retry state: %v", err)
	}
	if j, _ := readSystemJournal(f.a.optDir); j != nil {
		t.Fatalf("journal kept: %+v", j)
	}
	if f.onDisk() != "v0.31.0" {
		t.Fatalf("on disk %q", f.onDisk())
	}

	failStart = false
	if err := f.a.Run(context.Background(), true); err != nil {
		t.Fatal(err)
	}
	if _, err := os.Stat(f.a.file(rearmPendingFile)); err == nil {
		t.Fatal("retry state left after the passes started")
	}
	for u, was := range before {
		if active[u] != was {
			t.Errorf("%s active = %v, was %v", u, active[u], was)
		}
	}
	if f.onDisk() != "v0.31.0" {
		t.Fatalf("the retry changed the binary: %q", f.onDisk())
	}
}

// F15's dormancy rule for the lock: on a box with no floor, an activation
// that cannot create .moves for its lock (a read-only data dir here) logs
// it and goes ahead as main's would.
func TestUnarmedBoxWithoutALockDirUpdatesAsMain(t *testing.T) {
	if os.Geteuid() == 0 {
		t.Skip("root ignores the read-only mode")
	}
	f, _ := floorFixture(t, "v0.31.0", "v0.32.0")
	if err := os.Chmod(f.a.dataDir, 0o555); err != nil {
		t.Fatal(err)
	}
	t.Cleanup(func() { os.Chmod(f.a.dataDir, 0o755) })
	if err := f.a.Run(context.Background(), false); err != nil {
		t.Fatal(err)
	}
	if r := f.result(); r == nil || r.Outcome != systemOutcomeApplied || f.onDisk() != "v0.32.0" {
		t.Fatalf("result %+v, on disk %q", r, f.onDisk())
	}
}

// ---- fix round 3: one guard, both locks, no unlocked path on contention ----

// shortFloorLockWait makes a contended floor.lock time out quickly.
func shortFloorLockWait(t *testing.T) {
	old := floorLockWait
	floorLockWait = 200 * time.Millisecond
	t.Cleanup(func() { floorLockWait = old })
}

// holdFloorLock takes floor.lock as a writer would, without publishing
// anything, until the returned func is called.
func holdFloorLock(t *testing.T, dataDir string) func() {
	t.Helper()
	held, free, done := make(chan struct{}), make(chan struct{}), make(chan struct{})
	go func() {
		defer close(done)
		if err := withFloorLock(dataDir, true, func(*movesDir) error {
			close(held)
			<-free
			return nil
		}); err != nil {
			t.Error(err)
		}
	}()
	<-held
	return func() { close(free); <-done }
}

// A writer holding floor.lock has not necessarily published yet: a reader
// that times out on it must not go ahead unlocked. The apply is deferred,
// and nothing is activated.
func TestContendedFloorLockDefersTheInstall(t *testing.T) {
	shortFloorLockWait(t)
	f, _ := floorFixture(t, "v0.31.0", "v0.32.0")
	unlock := holdFloorLock(t, f.a.dataDir)
	defer unlock()
	if err := f.a.Run(context.Background(), false); err != nil {
		t.Fatal(err)
	}
	r := f.result()
	if r == nil || r.Outcome != systemOutcomeDeferred || !strings.Contains(r.Reason, "another writer holds the floor lock") {
		t.Fatalf("result = %+v", r)
	}
	if f.onDisk() != "v0.31.0" {
		t.Fatalf("activated under a contended lock: on disk %q", f.onDisk())
	}
	if j, _ := readSystemJournal(f.a.optDir); j != nil {
		t.Fatalf("journal left: %+v", j)
	}
}

// With a floor armed, the guard freezes the holds even where the version
// being made current is the one already on disk (a journal finish).
func TestForwardFinishFreezesTheHolds(t *testing.T) {
	f, _ := floorFixture(t, "v0.30.0", "v0.31.0")
	f.brokenBox("v0.31.0", "v0.30.0", "v0.30.0")
	f.a.journal(systemJournal{Phase: systemPhaseGated, Target: "v0.31.0", PrevBuild: "v0.30.0", StartedAt: f.now})
	armLocalFloor(t, f.a.dataDir, "v0.30.0")
	frozen := 0
	open := f.a.openRegistry
	f.a.openRegistry = func() (*registry.Registry, error) { frozen++; return open() }
	f.edgeStartedThisBoot = false
	if err := f.a.Run(context.Background(), true); err != nil {
		t.Fatal(err)
	}
	if ap, _ := readSystemApplied(f.a.optDir); ap == nil || ap.Version != "v0.31.0" {
		t.Fatalf("not finished forward: %+v", ap)
	}
	if frozen == 0 {
		t.Fatal("the forward finish did not freeze the holds")
	}
}

// The backward recovery's own finish (result and journal removal) is
// guarded too: a floor raised after the restore's rename, before the
// finish, keeps the journal.
func TestBackwardRecoveryFinishRechecksTheFloor(t *testing.T) {
	f, _ := floorFixture(t, "v0.31.0", "v0.32.0")
	armLocalFloor(t, f.a.dataDir, "v0.30.0")
	f.brokenBox("v0.31.0", "v0.30.0", "v0.30.0")
	systemctl := f.a.ops.systemctl
	f.a.ops.systemctl = func(args ...string) (string, error) {
		if args[0] == "restart" {
			armLocalFloor(t, f.a.dataDir, "v0.31.0")
		}
		return systemctl(args...)
	}
	f.edgeStartedThisBoot = true
	err := f.a.Run(context.Background(), true)
	if err == nil || !strings.Contains(err.Error(), "moves protocol floor: v0.30.0 is below v0.31.0") {
		t.Fatalf("err = %v", err)
	}
	if j, _ := readSystemJournal(f.a.optDir); j == nil {
		t.Fatal("the journal was finished below a floor raised before the finish")
	}
}

// A restore refused before its rename whose rearm then fails records the
// units it paused (as they were before the quiesce) in rearm-pending.json.
// The next recovery starts them first, and only then refuses again.
func TestRefusedRestoreWhoseRearmFailsIsRetriedFirst(t *testing.T) {
	f, _ := floorFixture(t, "v0.32.0", "v0.33.0")
	armLocalFloor(t, f.a.dataDir, "v0.30.0")
	j := f.brokenBox("v0.32.0", "v0.31.0", "v0.31.0")
	active := map[string]bool{"carlos-adopt.timer": true, "carlos-adopt.path": true, "carlos-stage.path": false, "carlos-restart.path": true}
	before := copyActivity(active)
	f.unitActivity(active)
	failStart := true
	systemctl := f.a.ops.systemctl
	var order []string
	f.a.ops.systemctl = func(args ...string) (string, error) {
		if args[0] == "start" {
			order = append(order, "start")
			if failStart {
				return "", errors.New("unit masked")
			}
		}
		out, err := systemctl(args...)
		if args[0] == "stop" && failStart {
			armLocalFloor(t, f.a.dataDir, "v0.33.0")
		}
		return out, err
	}
	if _, err := f.a.restoreFiles(context.Background(), &j); err == nil || !strings.Contains(err.Error(), "could not be started again") {
		t.Fatalf("err = %v", err)
	}
	var p rearmPending
	if err := readJSONFile(f.a.file(rearmPendingFile), &p); err != nil {
		t.Fatalf("no retry state: %v", err)
	}
	if want := []string{"carlos-adopt.timer", "carlos-adopt.path", "carlos-restart.path"}; !slices.Equal(p.Units, want) {
		t.Fatalf("pending units = %v, want %v (as they were before the quiesce)", p.Units, want)
	}

	failStart = false
	f.edgeStartedThisBoot = false
	err := f.a.Run(context.Background(), true)
	if err == nil || !strings.Contains(err.Error(), "moves protocol floor") {
		t.Fatalf("recovery err = %v", err)
	}
	if len(order) < 2 || order[len(order)-1] != "start" {
		t.Fatalf("the recovery did not start the pending passes: %v", order)
	}
	if _, err := os.Stat(f.a.file(rearmPendingFile)); err == nil {
		t.Fatal("retry state left after the passes started")
	}
	for u, was := range before {
		if active[u] != was {
			t.Errorf("%s active = %v, was %v", u, active[u], was)
		}
	}
	if j, _ := readSystemJournal(f.a.optDir); j == nil {
		t.Fatal("journal dropped")
	}
}