package provision

import (
	"context"
	"database/sql"
	"encoding/json"
	"errors"
	"os"
	"reflect"
	"strings"
	"testing"

	"github.com/carlosframework/platform/internal/registry"
	"github.com/carlosframework/platform/internal/release"
	"github.com/carlosframework/platform/internal/seeds"
)

// rollbackRecorder is the brief's fake: it records the box effects in the
// order they happen.
type rollbackRecorder struct {
	steps    []string
	proceed  bool
	fenceErr error
	gen      uint64
	marker   string // the seed id the local database's marker names
	// retired is each RetireKey call, "<db path> <host> <env before>". It
	// is kept apart from steps, whose sequences predate the hook.
	retired []string
}

func (r *rollbackRecorder) hooks(h *SeedHooks) {
	h.NextFenceGen = func() (uint64, error) { r.gen++; return r.gen, nil }
	h.FenceForRollback = func(_ context.Context, host string, gen uint64) (bool, error) {
		r.steps = append(r.steps, "fence")
		return r.proceed, r.fenceErr
	}
	h.LiftFence = func(host string, gen uint64) { r.steps = append(r.steps, "lift") }
	h.RemoveLocal = func(rt registry.Route) error { r.steps = append(r.steps, "remove-local:"+rt.Kind); return nil }
	h.DeleteReplica = func(_ context.Context, host string) error { r.steps = append(r.steps, "delete-replica"); return nil }
	h.LocalSeedID = func(string) (string, error) { return r.marker, nil }
	h.RetireKey = func(db, host, before string) error {
		r.retired = append(r.retired, db+" "+host+" "+before)
		return nil
	}
}

func TestRollbackAfterAbortRestoresTheRowExactly(t *testing.T) {
	s, sqid := shopSetup(t)
	rec := importRecord(t, s, sqid)
	reg := regWith(t, addrRow(rec, sqid))
	original, _ := reg.Get(rec.Host)
	cfg := importCfg(t, nil)
	rr := &rollbackRecorder{proceed: true}
	rr.hooks(cfg.Seeds)
	cfg.Seeds.Data.Put(ctx, seeds.SnapshotKey(rec.Host, rec.Seed.ID), []byte("db"), "")
	other, _ := seeds.NewID()
	cfg.Seeds.Data.Put(ctx, seeds.SnapshotKey(rec.Host, other), []byte("db"), "")
	Reconcile(ctx, s, reg, cfg) // swap
	if _, err := SetRecordSeed(ctx, s, rec.Account, rec.App, rec.Host, SeedAbort(rec.Seed.ID)); err != nil {
		t.Fatal(err)
	}
	if err := Reconcile(ctx, s, reg, cfg); err != nil {
		t.Fatal(err)
	}
	if got := mustGet(t, reg, rec.Host); !reflect.DeepEqual(*got, *original) {
		t.Fatalf("restored %+v, want %+v", got, original)
	}
	if j, _ := reg.ImportJournal(rec.Host); j != nil {
		t.Fatalf("journal = %+v", j)
	}
	want := []string{"fence", "remove-local:instance", "lift", "delete-replica"}
	if !reflect.DeepEqual(rr.steps, want) {
		t.Fatalf("steps = %q, want %q", rr.steps, want)
	}
	if _, _, err := cfg.Seeds.Data.Get(ctx, seeds.SnapshotKey(rec.Host, rec.Seed.ID)); !errors.Is(err, os.ErrNotExist) {
		t.Fatal("this seed's snapshot survived")
	}
	if _, _, err := cfg.Seeds.Data.Get(ctx, seeds.SnapshotKey(rec.Host, other)); err != nil {
		t.Fatal("another seed's snapshot was deleted")
	}
	r, _ := GetRecord(ctx, s, rec.Account, rec.App, rec.Host)
	if r.Seed.State != seeds.StateRolledBack {
		t.Fatalf("record = %s", r.Seed.State)
	}
}

func TestRollbackForAGoneOrReplacedRecord(t *testing.T) {
	for _, mutate := range []string{"deleted", "another seed"} {
		t.Run(mutate, func(t *testing.T) {
			s, sqid := shopSetup(t)
			rec := importRecord(t, s, sqid)
			reg := regWith(t, addrRow(rec, sqid))
			original, _ := reg.Get(rec.Host)
			cfg := importCfg(t, nil)
			rr := &rollbackRecorder{proceed: true}
			rr.hooks(cfg.Seeds)
			Reconcile(ctx, s, reg, cfg)
			if err := DeleteRecord(ctx, s, rec.Account, rec.App, rec.Host); err != nil {
				t.Fatal(err)
			}
			var next Record
			if mutate == "another seed" {
				next = importRecord(t, s, sqid) // the same host, a new seed id, state ready
			}
			if err := Reconcile(ctx, s, reg, cfg); err != nil {
				t.Fatal(err)
			}
			if got := mustGet(t, reg, rec.Host); !reflect.DeepEqual(*got, *original) {
				t.Fatalf("restored %+v", got)
			}
			if j, _ := reg.ImportJournal(rec.Host); j != nil {
				t.Fatalf("journal = %+v", j)
			}
			// A record naming another seed holds the host: the host's
			// replica is not this rollback's to delete, and that record
			// keeps its own state.
			wantSteps := []string{"fence", "remove-local:instance", "lift", "delete-replica"}
			if mutate == "another seed" {
				wantSteps = wantSteps[:3]
				if st := seedState(t, s, next); st.ID != next.Seed.ID || st.State != seeds.StateReady {
					t.Fatalf("the new record's seed = %+v", st)
				}
			}
			if !reflect.DeepEqual(rr.steps, wantSteps) {
				t.Fatalf("steps = %q, want %q", rr.steps, wantSteps)
			}
		})
	}
}

func TestRollbackDefersWhenTheFenceIsNotOurs(t *testing.T) {
	s, sqid := shopSetup(t)
	rec := importRecord(t, s, sqid)
	reg := regWith(t, addrRow(rec, sqid))
	cfg := importCfg(t, nil)
	rr := &rollbackRecorder{proceed: false}
	rr.hooks(cfg.Seeds)
	Reconcile(ctx, s, reg, cfg)
	SetRecordSeed(ctx, s, rec.Account, rec.App, rec.Host, SeedAbort(rec.Seed.ID))
	Reconcile(ctx, s, reg, cfg)
	j, _ := reg.ImportJournal(rec.Host)
	got := mustGet(t, reg, rec.Host)
	if j == nil || j.State != registry.JournalRollingBack || got.ImportGate != registry.ImportGateClosed || got.Kind != "instance" {
		t.Fatalf("after a deferred fence: journal %+v row %+v; want rolling-back, gate closed, row kept", j, got)
	}
	if !reflect.DeepEqual(rr.steps, []string{"fence"}) {
		t.Fatalf("steps = %q: a deferred fence went on to remove or delete", rr.steps)
	}
	// Next pass the fence proceeds, with the SAME generation.
	gen := j.FenceGen
	rr.proceed = true
	Reconcile(ctx, s, reg, cfg)
	if rr.gen != gen {
		t.Fatalf("a resumed rollback minted a new generation: %d, journal had %d", rr.gen, gen)
	}
	if j, _ := reg.ImportJournal(rec.Host); j != nil {
		t.Fatalf("journal = %+v", j)
	}
}

// A fence hook that answers with an error defers, whatever proceed says,
// and this rollback's generation is lifted (review Minor 1).
func TestRollbackDefersOnAFenceError(t *testing.T) {
	x := abortedImport(t)
	rr := &rollbackRecorder{proceed: true, fenceErr: errors.New("stop unproven")}
	rr.hooks(x.cfg.Seeds)
	x.pass(t)
	if j, _ := x.reg.ImportJournal(x.rec.Host); j == nil || j.State != registry.JournalRollingBack {
		t.Fatalf("journal = %+v", j)
	}
	if !reflect.DeepEqual(rr.steps, []string{"fence", "lift"}) {
		t.Fatalf("steps = %q", rr.steps)
	}
}

func TestCleanupPendingNeverFences(t *testing.T) {
	s, sqid := shopSetup(t)
	rec := importRecord(t, s, sqid)
	reg := regWith(t, addrRow(rec, sqid))
	cfg := importCfg(t, nil)
	rr := &rollbackRecorder{proceed: true}
	rr.hooks(cfg.Seeds)
	Reconcile(ctx, s, reg, cfg)
	SetRecordSeed(ctx, s, rec.Account, rec.App, rec.Host, SeedAbort(rec.Seed.ID))
	// Crash after the route restore, before the bucket cleanup.
	if err := reg.CloseImportGate(rec.Host, rec.Seed.ID, 9); err != nil {
		t.Fatal(err)
	}
	if err := reg.RestoreImportRoute(rec.Host, rec.Seed.ID); err != nil {
		t.Fatal(err)
	}
	rr.steps = nil
	if err := Reconcile(ctx, s, reg, cfg); err != nil {
		t.Fatal(err)
	}
	if !reflect.DeepEqual(rr.steps, []string{"delete-replica"}) {
		t.Fatalf("steps = %q; a cleanup-pending resume fenced, stopped or removed something", rr.steps)
	}
	if rr.gen != 0 {
		t.Fatalf("a cleanup-pending resume minted a fence generation")
	}
}

func TestJournalDoesNotRollBackOnAReadFailure(t *testing.T) {
	for _, how := range []string{"unparseable body", "read error"} {
		t.Run(how, func(t *testing.T) {
			s, sqid := shopSetup(t)
			rec := importRecord(t, s, sqid)
			reg := regWith(t, addrRow(rec, sqid))
			cfg := importCfg(t, nil)
			rr := &rollbackRecorder{proceed: true}
			rr.hooks(cfg.Seeds)
			Reconcile(ctx, s, reg, cfg)
			var store release.Store = s
			key := RecordKey(rec.Account, rec.App, rec.Host)
			if how == "unparseable body" {
				// The record body becomes unreadable: protected, never "gone".
				putReplace(t, s, key, []byte("{not json"))
			} else {
				// preflight m7: a Get that fails (readFailed), not a bad body.
				store = getFailsFor{s, key}
			}
			if err := Reconcile(ctx, store, reg, cfg); err != nil {
				t.Fatal(err)
			}
			if len(rr.steps) != 0 || rr.gen != 0 {
				t.Fatalf("an unreadable record started a rollback: steps %q", rr.steps)
			}
			if j, _ := reg.ImportJournal(rec.Host); j == nil || j.State != registry.JournalSwapped {
				t.Fatalf("journal = %+v", j)
			}
			if got := mustGet(t, reg, rec.Host); got.Kind != "instance" {
				t.Fatal("the removal phase touched a journaled host")
			}
		})
	}
}

// getFailsFor fails every read of one key with something other than "not
// found".
type getFailsFor struct {
	release.Store
	key string
}

func (g getFailsFor) Get(ctx context.Context, key string) ([]byte, string, error) {
	if key == g.key {
		return nil, "", errors.New("connection reset")
	}
	return g.Store.Get(ctx, key)
}

// ---- the box side, with the step order checked as it happens -------------

// crashed is the panic a fault raises to kill the "process" mid-pass.
type crashed struct{}

// rollbackBox is the box's side of the rollback: an in-memory fence (lost
// when the process dies), the import's local data, and the host's replica.
// Every hook checks, when it is called, the order the spec requires: the
// fence comes after the gate is closed and under the journal's own
// generation; nothing local is removed unless the host is fenced and the
// journal is still rolling back; nothing in the bucket is deleted before
// the original route is back.
type rollbackBox struct {
	t        *testing.T
	reg      *registry.Registry
	original registry.Route
	steps    []string
	proceed  bool
	gen      uint64
	fenced   map[string]uint64
	local    bool
	replica  bool
	fault    string // the step that fails, once
	crash    bool   // the fault kills the process (a panic) rather than returning an error
	noRoute  bool   // the import's row was removed: no route comes back
	marker   string // the seed id the local database's <db>.seed names
	markErr  error  // reading that marker fails
	// retired is each RetireKey call's env-before value, kept apart from
	// steps; key is whether the import's minted key is still on the box.
	retired []string
	key     bool
}

func newRollbackBox(t *testing.T, reg *registry.Registry, original registry.Route) *rollbackBox {
	return &rollbackBox{t: t, reg: reg, original: original, proceed: true, fenced: map[string]uint64{}, local: true, replica: true, key: true}
}

func (b *rollbackBox) trip(step string) error {
	if b.fault != step {
		return nil
	}
	b.fault = ""
	if b.crash {
		panic(crashed{})
	}
	return errors.New("injected failure at " + step)
}

// restart is the process dying: the activator's fences are in memory.
func (b *rollbackBox) restart() { b.fenced = map[string]uint64{} }

func (b *rollbackBox) journal() *registry.ImportJournal {
	j, err := b.reg.ImportJournal(b.original.Host)
	if err != nil {
		b.t.Fatal(err)
	}
	return j
}

func (b *rollbackBox) row() *registry.Route {
	rt, err := b.reg.Get(b.original.Host)
	if err != nil {
		b.t.Fatal(err)
	}
	return rt
}

// wantRestored: nothing is deleted from the bucket before the original
// route is back and the journal says cleanup-pending (or is gone, for the
// no-journal finish).
func (b *rollbackBox) wantRestored(what string) {
	switch rt := b.row(); {
	case b.noRoute:
		if rt != nil {
			b.t.Errorf("%s with a route at the host: %+v", what, rt)
		}
	case rt == nil || !reflect.DeepEqual(*rt, b.original):
		b.t.Errorf("%s before the original route was back: row %+v", what, rt)
	}
	if j := b.journal(); j != nil && j.State != registry.JournalCleanupPending {
		b.t.Errorf("%s with the journal at %s", what, j.State)
	}
	if len(b.fenced) != 0 {
		b.t.Errorf("%s with a fence held: the cleanup never fences", what)
	}
}

func (b *rollbackBox) hooks(h *SeedHooks) {
	h.NextFenceGen = func() (uint64, error) {
		b.steps = append(b.steps, "gen")
		b.gen++
		return b.gen, nil
	}
	h.FenceForRollback = func(_ context.Context, host string, gen uint64) (bool, error) {
		b.steps = append(b.steps, "fence")
		if j := b.journal(); j == nil || j.State != registry.JournalRollingBack || j.FenceGen != gen {
			b.t.Errorf("fenced with generation %d under journal %+v", gen, j)
		}
		if rt := b.row(); rt != nil && rt.ImportGate != registry.ImportGateClosed {
			b.t.Errorf("fenced before the gate was closed: %+v", rt)
		}
		if err := b.trip("fence"); err != nil {
			return false, err
		}
		if held, ok := b.fenced[host]; ok {
			return held == gen, nil // ours from an earlier pass, or another operation's
		}
		if !b.proceed {
			return false, nil
		}
		b.fenced[host] = gen
		return true, nil
	}
	h.LiftFence = func(host string, gen uint64) {
		b.steps = append(b.steps, "lift")
		_ = b.trip("lift") // a lift cannot fail; only a crash stops it
		if b.fenced[host] == gen {
			delete(b.fenced, host)
		}
	}
	h.RemoveLocal = func(rt registry.Route) error {
		b.steps = append(b.steps, "remove-local")
		if _, ok := b.fenced[rt.Host]; !ok {
			b.t.Errorf("removed local data of a host that is not fenced and stopped")
		}
		if j := b.journal(); j == nil || j.State != registry.JournalRollingBack {
			b.t.Errorf("removed local data with the journal at %+v", j)
		}
		// The closed import row, or (its row gone) the host's default
		// nested database path.
		if rt.DBPath != NestedDBPath("/data", rt.Host) || (rt.ImportGate != registry.ImportGateClosed && !b.noRoute) {
			b.t.Errorf("removed local data for a row that is not the closed import row: %+v", rt)
		}
		if err := b.trip("remove-local"); err != nil {
			return err
		}
		b.local = false
		return b.trip("remove-local-done")
	}
	h.RetireKey = func(dbPath, host, before string) error {
		b.retired = append(b.retired, before)
		if _, ok := b.fenced[host]; !ok {
			b.t.Errorf("retired the instance key of a host that is not fenced and stopped")
		}
		if j := b.journal(); j == nil || j.State != registry.JournalRollingBack || j.EnvBefore != before {
			b.t.Errorf("retired the instance key by %q under journal %+v", before, j)
		}
		if dbPath != NestedDBPath("/data", host) {
			b.t.Errorf("retired the instance key beside %s", dbPath)
		}
		// Before the local data: its marker is the missing-row path's
		// evidence that the installation is the import's. (A resumed pass
		// retires again after the data went; that is a no-op.)
		if b.key && !b.local {
			b.t.Errorf("the instance key was first retired after the local data was removed")
		}
		if err := b.trip("retire-key"); err != nil {
			return err
		}
		b.key = false
		return nil
	}
	h.LocalSeedID = func(dbPath string) (string, error) {
		b.steps = append(b.steps, "read-marker")
		if dbPath != NestedDBPath("/data", b.original.Host) {
			b.t.Errorf("read the marker beside %s", dbPath)
		}
		if b.markErr != nil {
			return "", b.markErr
		}
		if !b.local {
			return "", nil
		}
		return b.marker, nil
	}
	h.DeleteReplica = func(_ context.Context, host string) error {
		b.steps = append(b.steps, "delete-replica")
		b.wantRestored("deleted the replica")
		if err := b.trip("delete-replica"); err != nil {
			return err
		}
		b.replica = false
		return nil
	}
}

// faultData is the region data bucket, with the snapshot delete checked and
// faultable.
type faultData struct {
	release.Store
	box *rollbackBox
}

func (d faultData) Delete(ctx context.Context, key string) error {
	if strings.HasPrefix(key, seeds.Prefix) {
		d.box.steps = append(d.box.steps, "delete-snapshot")
		d.box.wantRestored("deleted the snapshot")
		if err := d.box.trip("delete-snapshot"); err != nil {
			return err
		}
	}
	return d.Store.Delete(ctx, key)
}

// faultRecords is the release bucket, with the record CAS faultable.
type faultRecords struct {
	release.Store
	box *rollbackBox
	key string
}

func (f faultRecords) Put(ctx context.Context, key string, body []byte, ifMatch string) (string, error) {
	if key == f.key {
		if err := f.box.trip("cas"); err != nil {
			return "", err
		}
	}
	return f.Store.Put(ctx, key, body, ifMatch)
}

// rollbackScene is an import this box swapped, verified once (its local
// database placed and replicated) and then aborted.
type rollbackScene struct {
	raw   *release.DirStore
	store release.Store
	reg   *registry.Registry
	path  string
	rec   Record
	other string // another seed of the host, with its own snapshot
	cfg   ReconcileConfig
	box   *rollbackBox
}

func abortedImport(t *testing.T) *rollbackScene {
	t.Helper()
	s, sqid := shopSetup(t)
	rec := importRecord(t, s, sqid)
	reg, path := regAt(t, addrRow(rec, sqid))
	box := newRollbackBox(t, reg, *mustGet(t, reg, rec.Host))
	box.marker = rec.Seed.ID // the verify wake placed this seed
	cfg := importCfg(t, nil)
	box.hooks(cfg.Seeds)
	data := cfg.Seeds.Data
	cfg.Seeds.Data = faultData{data, box}
	other, _ := seeds.NewID()
	for _, id := range []string{rec.Seed.ID, other} {
		if _, err := data.Put(ctx, seeds.SnapshotKey(rec.Host, id), []byte("db"), ""); err != nil {
			t.Fatal(err)
		}
	}
	x := &rollbackScene{raw: s, store: faultRecords{s, box, RecordKey(rec.Account, rec.App, rec.Host)},
		reg: reg, path: path, rec: rec, other: other, cfg: cfg, box: box}
	x.pass(t)
	if got := mustGet(t, reg, rec.Host); !registry.IsImportRow(*got, registry.ImportJournal{Host: rec.Host, SeedID: rec.Seed.ID}) {
		t.Fatalf("the swap did not happen: %+v", got)
	}
	if _, err := SetRecordSeed(ctx, s, rec.Account, rec.App, rec.Host, SeedAbort(rec.Seed.ID)); err != nil {
		t.Fatal(err)
	}
	box.steps = nil
	return x
}

// pass is one Reconcile. A fault that crashes the process ends the pass
// where it stands, and the box restarts.
func (x *rollbackScene) pass(t *testing.T) {
	t.Helper()
	defer func() {
		if r := recover(); r != nil {
			if _, ok := r.(crashed); !ok {
				panic(r)
			}
			x.box.restart()
		}
	}()
	if err := Reconcile(ctx, x.store, x.reg, x.cfg); err != nil {
		t.Fatal(err)
	}
}

func (x *rollbackScene) journalState(t *testing.T) string {
	t.Helper()
	j, err := x.reg.ImportJournal(x.rec.Host)
	if err != nil {
		t.Fatal(err)
	}
	if j == nil {
		return ""
	}
	return j.State
}

// wantRolledBack is the clean end state: the original route exactly, no
// journal, this seed's snapshot gone and the other kept, no local data, no
// replica, no fence, and the record rolled back.
func (x *rollbackScene) wantRolledBack(t *testing.T) {
	t.Helper()
	if got := mustGet(t, x.reg, x.rec.Host); !reflect.DeepEqual(*got, x.box.original) {
		t.Fatalf("row = %+v, want the original %+v", got, x.box.original)
	}
	if st := x.journalState(t); st != "" {
		t.Fatalf("journal still at %s", st)
	}
	data := x.cfg.Seeds.Data
	if _, _, err := data.Get(ctx, seeds.SnapshotKey(x.rec.Host, x.rec.Seed.ID)); !errors.Is(err, os.ErrNotExist) {
		t.Fatalf("this seed's snapshot: %v", err)
	}
	if _, _, err := data.Get(ctx, seeds.SnapshotKey(x.rec.Host, x.other)); err != nil {
		t.Fatalf("another seed's snapshot: %v", err)
	}
	if x.box.local || x.box.replica || len(x.box.fenced) != 0 {
		t.Fatalf("local %v replica %v fenced %v", x.box.local, x.box.replica, x.box.fenced)
	}
	// The key the import's verify minted is retired, by what the swap
	// recorded: this host had no env file before the import (importCfg).
	if x.box.key || len(x.box.retired) == 0 {
		t.Fatalf("the import's instance key was not retired (calls %q)", x.box.retired)
	}
	for _, before := range x.box.retired {
		if before != registry.EnvBeforeAbsent {
			t.Fatalf("retired by %q, want what the swap recorded (%q)", before, registry.EnvBeforeAbsent)
		}
	}
	if st := seedState(t, x.raw, x.rec); st.ID != x.rec.Seed.ID || st.State != seeds.StateRolledBack {
		t.Fatalf("record seed = %+v, want rolled-back", st)
	}
}

func (x *rollbackScene) trigger(t *testing.T, ddl string) {
	t.Helper()
	addTrigger(t, x.path, ddl)
}

func count(steps []string, step string) int {
	n := 0
	for _, s := range steps {
		if s == step {
			n++
		}
	}
	return n
}

// Spec §9 and preflight M13: a failure between any two of the rollback's
// steps, as an error the process survives or as a crash that loses the
// in-memory fence, is resumed by the next passes to the same clean end,
// with nothing lost and nothing done twice that must not be: one fence
// generation per close, no local removal or fence once the route is back,
// and no bucket delete before it is.
//
// The three registry-trigger rows "crash" as a failed transaction followed
// by a restart, not as a panic inside the transaction: SQLite rolls the
// transaction back either way, so the state left is the same.
func TestRollbackResumesAfterAFailureBetweenAnyTwoSteps(t *testing.T) {
	const raise = ` BEGIN SELECT RAISE(ABORT, 'injected'); END`
	for _, f := range []struct {
		name      string
		hook      string // a box hook or a bucket write that fails once
		trigger   string // or a registry write that fails, until dropped
		after     string // the journal's state after the failed pass
		crashOnly bool
	}{
		{name: "closing the gate", trigger: `CREATE TRIGGER fault BEFORE UPDATE OF state ON import_journal WHEN NEW.state = 'rolling-back'` + raise, after: registry.JournalSwapped},
		{name: "fencing", hook: "fence", after: registry.JournalRollingBack},
		{name: "retiring the instance key", hook: "retire-key", after: registry.JournalRollingBack},
		{name: "removing the local data", hook: "remove-local", after: registry.JournalRollingBack},
		{name: "after removing the local data", hook: "remove-local-done", after: registry.JournalRollingBack},
		{name: "restoring the route", trigger: `CREATE TRIGGER fault BEFORE UPDATE OF state ON import_journal WHEN NEW.state = 'cleanup-pending'` + raise, after: registry.JournalRollingBack},
		{name: "lifting the fence", hook: "lift", after: registry.JournalCleanupPending, crashOnly: true},
		{name: "deleting the snapshot", hook: "delete-snapshot", after: registry.JournalCleanupPending},
		{name: "deleting the replica", hook: "delete-replica", after: registry.JournalCleanupPending},
		{name: "deleting the journal row", trigger: `CREATE TRIGGER fault BEFORE DELETE ON import_journal` + raise, after: registry.JournalCleanupPending},
		{name: "marking the record rolled back", hook: "cas", after: ""},
	} {
		for _, crash := range []bool{false, true} {
			if f.crashOnly && !crash {
				continue
			}
			name := f.name + "/error"
			if crash {
				name = f.name + "/crash"
			}
			t.Run(name, func(t *testing.T) {
				x := abortedImport(t)
				x.box.fault, x.box.crash = f.hook, crash
				if f.trigger != "" {
					x.trigger(t, f.trigger)
				}
				x.pass(t)
				if x.box.fault != "" {
					t.Fatalf("the fault at %s was never reached: steps %q", f.hook, x.box.steps)
				}
				if st := x.journalState(t); st != f.after {
					t.Fatalf("after the failed pass the journal is %q, want %q", st, f.after)
				}
				if f.trigger != "" {
					x.trigger(t, `DROP TRIGGER fault`)
					if crash {
						x.box.restart()
					}
				}
				if f.after == "" {
					if st := seedState(t, x.raw, x.rec); st.State != seeds.StateAborted {
						t.Fatalf("record = %s, want aborted with no journal row", st.State)
					}
				}
				restoredAt := -1
				for i := 0; i < 3; i++ {
					if restoredAt < 0 && x.journalState(t) != registry.JournalSwapped && x.journalState(t) != registry.JournalRollingBack {
						restoredAt = len(x.box.steps)
					}
					x.pass(t)
				}
				x.wantRolledBack(t)
				gens := 1
				if f.after == registry.JournalSwapped {
					gens = 2 // the failed close's generation was never stored
				}
				if n := count(x.box.steps, "gen"); n != gens {
					t.Fatalf("minted %d fence generations, want %d: %q", n, gens, x.box.steps)
				}
				if restoredAt >= 0 {
					for _, s := range x.box.steps[restoredAt:] {
						if s == "fence" || s == "remove-local" || s == "gen" {
							t.Fatalf("%s after the route was restored: %q", s, x.box.steps)
						}
					}
				}
			})
		}
	}
}

// Preflight M13: the record is deleted and the fence defers. The gated row
// is Provisioned and its record gone, which is what the removal phase
// deletes, but the journal owns it: it survives, closed, every pass.
func TestRollbackRecordDeletedFenceDeferredRowSurvivesRemoval(t *testing.T) {
	x := abortedImport(t)
	if err := DeleteRecord(ctx, x.raw, x.rec.Account, x.rec.App, x.rec.Host); err != nil {
		t.Fatal(err)
	}
	x.box.proceed = false
	for i := 0; i < 2; i++ {
		x.pass(t)
		got := mustGet(t, x.reg, x.rec.Host)
		if got.Kind != "instance" || got.ImportGate != registry.ImportGateClosed || !got.Provisioned {
			t.Fatalf("pass %d: row = %+v, want the closed import row", i, got)
		}
		if st := x.journalState(t); st != registry.JournalRollingBack {
			t.Fatalf("pass %d: journal = %s", i, st)
		}
	}
	if !x.box.local || !x.box.replica || count(x.box.steps, "remove-local") != 0 {
		t.Fatalf("a deferred rollback removed or deleted something: %q", x.box.steps)
	}
	x.box.proceed = true
	x.pass(t)
	if got := mustGet(t, x.reg, x.rec.Host); !reflect.DeepEqual(*got, x.box.original) {
		t.Fatalf("row = %+v", got)
	}
	if x.journalState(t) != "" || x.box.local || x.box.replica {
		t.Fatalf("not finished: journal %q local %v replica %v", x.journalState(t), x.box.local, x.box.replica)
	}
}

// The rollback stops the instance before it removes or deletes anything,
// restores the route before the bucket is touched, and the removal phase,
// which runs on the route list read before the rollback, leaves the
// restored legacy row alone (the journaled-host skip): here the record is
// gone, so without the skip the stale Provisioned import row would have
// its host removed, taking the legacy row with it.
func TestRollbackOrderAndTheRemovalPhaseKeepsTheRestoredRow(t *testing.T) {
	x := abortedImport(t)
	if err := DeleteRecord(ctx, x.raw, x.rec.Account, x.rec.App, x.rec.Host); err != nil {
		t.Fatal(err)
	}
	x.pass(t)
	want := []string{"gen", "fence", "remove-local", "lift", "delete-snapshot", "delete-replica"}
	if !reflect.DeepEqual(x.box.steps, want) {
		t.Fatalf("steps = %q, want %q", x.box.steps, want)
	}
	if got := mustGet(t, x.reg, x.rec.Host); !reflect.DeepEqual(*got, x.box.original) {
		t.Fatalf("row after the pass = %+v, want the original", got)
	}
	if x.journalState(t) != "" {
		t.Fatalf("journal = %s", x.journalState(t))
	}
}

// insertJournal writes a journal row straight into the registry file, as
// a cold restore of an older snapshot would bring it back.
func insertJournal(t *testing.T, path string, j registry.ImportJournal) {
	t.Helper()
	orig, err := json.Marshal(j.Original)
	if err != nil {
		t.Fatal(err)
	}
	db, err := sql.Open("sqlite", path)
	if err != nil {
		t.Fatal(err)
	}
	defer db.Close()
	if _, err := db.Exec(`INSERT INTO import_journal (host, seed_id, original, record_key, state, fence_gen, at) VALUES (?,?,?,?,?,?,?)`,
		j.Host, j.SeedID, string(orig), j.RecordKey, j.State, j.FenceGen, j.At); err != nil {
		t.Fatal(err)
	}
}

// The Task 4 review's stale journal row: the import was rolled back and
// its record finished, the host was created again, and then a journal row
// for the old import comes back (a cold restore of an older snapshot).
// Whatever state the row comes back in, nothing the new instance owns is
// touched: not its row, its record, its process, its local data or its
// replica.
func TestRollbackOfAStaleJournalTouchesNothingTheNewInstanceOwns(t *testing.T) {
	for _, owner := range []string{"the same app", "another app"} {
		for _, state := range []string{registry.JournalSwapped, registry.JournalRollingBack, registry.JournalCleanupPending} {
			t.Run(owner+"/"+state, func(t *testing.T) {
				x := abortedImport(t)
				stale, _ := x.reg.ImportJournal(x.rec.Host)
				x.pass(t)
				x.wantRolledBack(t)
				// The console finishes A and the host is created again.
				if err := DeleteRecord(ctx, x.raw, x.rec.Account, x.rec.App, x.rec.Host); err != nil {
					t.Fatal(err)
				}
				next := shopRecord(x.rec.Account)
				if owner == "another app" {
					next.App = "other"
					if err := PutManifest(ctx, x.raw, next.Account, "other", Manifest{Schema: 1, App: "other", Domain: "other.example.com"}); err != nil {
						t.Fatal(err)
					}
				}
				if err := CreateRecord(ctx, x.raw, next); err != nil {
					t.Fatal(err)
				}
				// The new instance's row, as the box serves it.
				if err := x.reg.Delete(next.Host); err != nil {
					t.Fatal(err)
				}
				x.pass(t)
				newRow := mustGet(t, x.reg, next.Host)
				if newRow.App != next.App || !newRow.Provisioned || newRow.ImportGate != "" {
					t.Fatalf("the new instance was not minted: %+v", newRow)
				}
				x.box.local, x.box.replica = true, true // its own data now
				x.box.original = *newRow                // what any bucket delete must find at the host
				newRec, _ := GetRecord(ctx, x.raw, next.Account, next.App, next.Host)

				j := *stale
				j.State = state
				if state != registry.JournalSwapped {
					j.FenceGen = 7
				}
				insertJournal(t, x.path, j)
				x.box.steps = nil
				logged := captureLog(t)
				x.pass(t)
				x.pass(t)

				if got := mustGet(t, x.reg, next.Host); !reflect.DeepEqual(*got, *newRow) {
					t.Fatalf("the new instance's row changed:\n got %+v\nwant %+v", got, newRow)
				}
				if got, _ := GetRecord(ctx, x.raw, next.Account, next.App, next.Host); !reflect.DeepEqual(got, newRec) {
					t.Fatalf("the new instance's record changed: %+v", got)
				}
				for _, s := range x.box.steps {
					// A lift names only the stale row's own generation,
					// which no fence on the host holds.
					if s != "delete-snapshot" && s != "lift" {
						t.Fatalf("the stale journal reached the box: %q\n%s", x.box.steps, logged)
					}
				}
				if !x.box.local || !x.box.replica {
					t.Fatalf("the new instance's data went: local %v replica %v", x.box.local, x.box.replica)
				}
				// A stale row that would act on the route is held; one at
				// cleanup-pending finishes its own seed's cleanup and goes.
				if st := x.journalState(t); (state == registry.JournalCleanupPending) != (st == "") {
					t.Fatalf("journal = %q after a stale %s row", st, state)
				}
				why := "the route at the host is not the import's"
				if state == registry.JournalCleanupPending {
					why = "the host's replica is left alone"
				}
				if !strings.Contains(logged.String(), why) {
					t.Fatalf("not held for the reason %q:\n%s", why, logged)
				}
			})
		}
	}
}

// The rollback tolerates a missing import row (the Task 4 review's carry):
// something removed it, so no route is put back, and the cleanup carries
// on. Its local data is still removed under the fence (fix rounds 1 and 2):
// the database's marker names this seed and the host has no row.
func TestRollbackOfAMissingRowSkipsTheRestore(t *testing.T) {
	x := abortedImport(t)
	if err := x.reg.Delete(x.rec.Host); err != nil {
		t.Fatal(err)
	}
	x.box.noRoute = true
	x.pass(t)
	if rt, _ := x.reg.Get(x.rec.Host); rt != nil {
		t.Fatalf("a route came back: %+v", rt)
	}
	want := []string{"gen", "fence", "read-marker", "remove-local", "lift", "delete-snapshot", "delete-replica"}
	if !reflect.DeepEqual(x.box.steps, want) {
		t.Fatalf("steps = %q, want %q", x.box.steps, want)
	}
	if x.journalState(t) != "" || x.box.replica || x.box.local {
		t.Fatalf("journal %q replica %v local %v", x.journalState(t), x.box.replica, x.box.local)
	}
	if st := seedState(t, x.raw, x.rec); st.State != seeds.StateRolledBack {
		t.Fatalf("record = %s", st.State)
	}
}

// Step 6 deletes the host's replica only when it can tell the replica is
// this import's: a record it cannot read, a host listing it cannot finish,
// or a record naming this seed in a state that is not abandoned all defer
// the cleanup; the journal stays at cleanup-pending.
func TestRollbackDefersTheReplicaDeleteWhenItCannotTell(t *testing.T) {
	for _, c := range []string{"the record read fails", "the record names this seed, placed"} {
		t.Run(c, func(t *testing.T) {
			x := abortedImport(t)
			if err := x.reg.CloseImportGate(x.rec.Host, x.rec.Seed.ID, 9); err != nil {
				t.Fatal(err)
			}
			if err := x.reg.RestoreImportRoute(x.rec.Host, x.rec.Seed.ID); err != nil {
				t.Fatal(err)
			}
			key := RecordKey(x.rec.Account, x.rec.App, x.rec.Host)
			switch c {
			case "the record read fails":
				// Only the step-6 re-read fails: the pass's own read works.
				x.store = &failNthGet{Store: x.store, key: key, n: 2}
			default:
				placed := x.rec
				seed := *x.rec.Seed
				seed.State = seeds.StatePlaced
				placed.Seed = &seed
				putReplace(t, x.raw, key, mustJSON(t, placed))
			}
			logged := captureLog(t)
			x.pass(t)
			if st := x.journalState(t); st != registry.JournalCleanupPending {
				t.Fatalf("journal = %q", st)
			}
			if !strings.Contains(logged.String(), "cleanup deferred") {
				t.Fatalf("not deferred:\n%s", logged)
			}
			if !x.box.replica || count(x.box.steps, "delete-replica") != 0 {
				t.Fatalf("the replica was deleted: %q", x.box.steps)
			}
		})
	}
}

// failNthGet fails every read of one key from the nth on.
type failNthGet struct {
	release.Store
	key  string
	n    int
	seen int
}

func (f *failNthGet) Get(ctx context.Context, key string) ([]byte, string, error) {
	if key == f.key {
		f.seen++
		if f.seen >= f.n {
			return nil, "", errors.New("connection reset")
		}
	}
	return f.Store.Get(ctx, key)
}

// Without every box hook the rollback does nothing at all: the import
// stays journaled, gated and untouched.
func TestRollbackNeedsEveryHook(t *testing.T) {
	x := abortedImport(t)
	x.cfg.Seeds.DeleteReplica = nil
	row := *mustGet(t, x.reg, x.rec.Host)
	x.pass(t)
	if got := mustGet(t, x.reg, x.rec.Host); !reflect.DeepEqual(*got, row) {
		t.Fatalf("row changed: %+v", got)
	}
	if st := x.journalState(t); st != registry.JournalSwapped || len(x.box.steps) != 0 {
		t.Fatalf("journal %q steps %q", st, x.box.steps)
	}
}

// The route is read again under the fence: a row that stopped being the
// import's while the host was being fenced (removed and replaced by
// another, here) is not removed, the fence is lifted, and the rollback
// holds with the journal at rolling-back.
func TestRollbackRereadsTheRouteUnderTheFence(t *testing.T) {
	x := abortedImport(t)
	other := addrRow(x.rec, x.rec.Account)
	other.Addr = "127.0.0.1:9999"
	fence := x.cfg.Seeds.FenceForRollback
	x.cfg.Seeds.FenceForRollback = func(ctx context.Context, host string, gen uint64) (bool, error) {
		proceed, err := fence(ctx, host, gen)
		if err := x.reg.Delete(host); err != nil {
			t.Fatal(err)
		}
		if err := x.reg.Upsert(other); err != nil {
			t.Fatal(err)
		}
		return proceed, err
	}
	x.pass(t)
	placed := mustGet(t, x.reg, x.rec.Host)
	if placed.Addr != other.Addr || placed.ImportGate != "" {
		t.Fatalf("the replacing row changed: %+v", placed)
	}
	if count(x.box.steps, "remove-local") != 0 || !x.box.local || len(x.box.fenced) != 0 {
		t.Fatalf("steps %q local %v fenced %v", x.box.steps, x.box.local, x.box.fenced)
	}
	if st := x.journalState(t); st != registry.JournalRollingBack {
		t.Fatalf("journal = %q", st)
	}
}

// Preflight M3: a journal row whose seed id fails seeds.Check names no
// snapshot and no replica of this import: the cleanup deletes nothing in
// the bucket and only retires the journal row.
func TestRollbackCleanupOfAnInvalidSeedIDDeletesNothing(t *testing.T) {
	s, sqid := shopSetup(t)
	rec := importRecord(t, s, sqid)
	reg, path := regAt(t, addrRow(rec, sqid))
	original := *mustGet(t, reg, rec.Host)
	insertJournal(t, path, registry.ImportJournal{Host: rec.Host, SeedID: "../not-a-seed", Original: original,
		RecordKey: RecordKey(rec.Account, rec.App, rec.Host), State: registry.JournalCleanupPending, FenceGen: 3, At: 1})
	cfg := importCfg(t, nil)
	b := newRollbackBox(t, reg, original)
	b.hooks(cfg.Seeds)
	cfg.Seeds.Data = faultData{cfg.Seeds.Data, b}
	if err := Reconcile(ctx, s, reg, cfg); err != nil {
		t.Fatal(err)
	}
	if len(b.steps) != 0 || !b.replica {
		t.Fatalf("steps %q replica %v", b.steps, b.replica)
	}
	if j, _ := reg.ImportJournal(rec.Host); j != nil {
		t.Fatalf("journal = %+v", j)
	}
	if got := mustGet(t, reg, rec.Host); !reflect.DeepEqual(*got, original) {
		t.Fatalf("row = %+v", got)
	}
}

// Fix round 2: with the import's row gone, its local database is decided
// from the file. Removed when its marker names this seed (whoever holds
// the host's record, here another app's); left when the marker names
// another seed or there is none; and a marker that cannot be read lifts
// the fence and defers. The replica still follows step 6's record test:
// another record holds the host, so it is kept.
func TestRollbackOfAMissingRowDecidesTheLocalDataFromItsMarker(t *testing.T) {
	otherSeed, _ := seeds.NewID()
	for _, c := range []struct {
		name    string
		marker  func(x *rollbackScene) string
		markErr error
		removed bool
		defers  bool
		logged  string
	}{
		{name: "the marker names this seed", marker: func(x *rollbackScene) string { return x.rec.Seed.ID }, removed: true,
			logged: "its local data is removed"},
		{name: "the marker names another seed", marker: func(*rollbackScene) string { return otherSeed },
			logged: "its seed marker names \"" + otherSeed + "\", not this seed"},
		{name: "no marker", marker: func(*rollbackScene) string { return "" },
			logged: "its seed marker names \"\", not this seed"},
		{name: "the marker cannot be read", markErr: errors.New("permission denied"), defers: true,
			logged: "the seed marker beside"},
	} {
		t.Run(c.name, func(t *testing.T) {
			x := abortedImport(t)
			if err := x.reg.Delete(x.rec.Host); err != nil {
				t.Fatal(err)
			}
			x.box.noRoute = true
			if c.marker != nil {
				x.box.marker = c.marker(x)
			}
			x.box.markErr = c.markErr
			other := shopRecord(x.rec.Account)
			other.App = "other"
			if err := CreateRecord(ctx, x.raw, other); err != nil {
				t.Fatal(err)
			}
			logged := captureLog(t)
			x.pass(t)
			if !strings.Contains(logged.String(), c.logged) {
				t.Fatalf("not logged %q:\n%s", c.logged, logged)
			}
			if x.box.local == c.removed || (count(x.box.steps, "remove-local") == 1) != c.removed {
				t.Fatalf("local %v steps %q; want removed %v", x.box.local, x.box.steps, c.removed)
			}
			// The instance key follows the database, by the same marker:
			// retired with it, left with it (merge review S1).
			if x.box.key == c.removed || (len(x.box.retired) == 1) != c.removed {
				t.Fatalf("key left %v, retire calls %q; want retired %v", x.box.key, x.box.retired, c.removed)
			}
			if len(x.box.fenced) != 0 {
				t.Fatalf("a fence was left: %v", x.box.fenced)
			}
			if c.defers {
				if st := x.journalState(t); st != registry.JournalRollingBack {
					t.Fatalf("journal = %q, want the rollback waiting at rolling-back", st)
				}
				if want := []string{"gen", "fence", "read-marker", "lift"}; !reflect.DeepEqual(x.box.steps, want) {
					t.Fatalf("steps = %q, want %q", x.box.steps, want)
				}
				return
			}
			if x.journalState(t) != "" || !x.box.replica {
				t.Fatalf("journal %q replica %v: the cleanup did not finish, or deleted another record's replica",
					x.journalState(t), x.box.replica)
			}
		})
	}
}

// Review Minor 1: a fence hook that fails is followed by a lift of this
// rollback's generation, in case it fenced before it failed.
func TestRollbackLiftsTheFenceWhenTheFenceHookFails(t *testing.T) {
	x := abortedImport(t)
	fence := x.cfg.Seeds.FenceForRollback
	x.cfg.Seeds.FenceForRollback = func(ctx context.Context, host string, gen uint64) (bool, error) {
		_, _ = fence(ctx, host, gen) // installs it
		return true, errors.New("stop evidence unreadable")
	}
	x.pass(t)
	if len(x.box.fenced) != 0 || count(x.box.steps, "remove-local") != 0 {
		t.Fatalf("fenced %v steps %q", x.box.fenced, x.box.steps)
	}
}

// Fix round 1, I1, the provision half of the end-to-end check: with the
// import's row gone (and its marker naming this seed), the local removal
// is aimed at the host's default
// nested database path, which is exactly where a plain instance created at
// the host afterwards is minted. (The activator half, seedpurge_linux_test.go,
// shows a plain wake at that path finds no file once PurgeSeedLocal ran.)
func TestRollbackOfAMissingRowPurgesWhereTheNextInstanceIsMinted(t *testing.T) {
	s, sqid := shopSetup(t)
	rec := importRecord(t, s, sqid)
	reg := regWith(t, addrRow(rec, sqid))
	cfg := importCfg(t, nil)
	rr := &rollbackRecorder{proceed: true}
	rr.hooks(cfg.Seeds)
	rr.marker = rec.Seed.ID
	var purged []string
	cfg.Seeds.RemoveLocal = func(rt registry.Route) error { purged = append(purged, rt.DBPath); return nil }
	if err := Reconcile(ctx, s, reg, cfg); err != nil {
		t.Fatal(err)
	}
	if _, err := SetRecordSeed(ctx, s, rec.Account, rec.App, rec.Host, SeedAbort(rec.Seed.ID)); err != nil {
		t.Fatal(err)
	}
	if err := reg.Delete(rec.Host); err != nil {
		t.Fatal(err)
	}
	if err := Reconcile(ctx, s, reg, cfg); err != nil {
		t.Fatal(err)
	}
	if want := []string{NestedDBPath(cfg.DataDir, rec.Host)}; !reflect.DeepEqual(purged, want) {
		t.Fatalf("purged %q, want %q", purged, want)
	}
	if err := DeleteRecord(ctx, s, rec.Account, rec.App, rec.Host); err != nil {
		t.Fatal(err)
	}
	if err := CreateRecord(ctx, s, shopRecord(sqid)); err != nil {
		t.Fatal(err)
	}
	if err := Reconcile(ctx, s, reg, cfg); err != nil {
		t.Fatal(err)
	}
	if next := mustGet(t, reg, rec.Host); next.DBPath != purged[0] || next.SeedSource != "" {
		t.Fatalf("the plain instance's row = %+v, want it at %s", next, purged[0])
	}
}

// Review Minor 1: a fence this rollback kept after a failed step is lifted
// when, by the next pass, the row at the host is no longer the import's.
func TestRollbackLiftsAKeptFenceWhenTheRowIsNoLongerTheImports(t *testing.T) {
	x := abortedImport(t)
	x.box.fault = "remove-local" // fails once; the fence is kept for the resume
	x.pass(t)
	if len(x.box.fenced) != 1 {
		t.Fatalf("fenced %v, want this rollback's fence kept", x.box.fenced)
	}
	other := addrRow(x.rec, x.rec.Account)
	other.Addr = "127.0.0.1:9999"
	if err := x.reg.Delete(x.rec.Host); err != nil {
		t.Fatal(err)
	}
	if err := x.reg.Upsert(other); err != nil {
		t.Fatal(err)
	}
	x.pass(t)
	if len(x.box.fenced) != 0 || x.journalState(t) != registry.JournalRollingBack {
		t.Fatalf("fenced %v journal %q", x.box.fenced, x.journalState(t))
	}
}

// Fix round 2: a row that appears at the host while the marker is read
// (not expected while journaled) keeps the local database, lifts the
// fence, and leaves the rollback waiting for the next pass to judge it.
func TestRollbackOfAMissingRowLeavesTheDataWhenARowAppears(t *testing.T) {
	x := abortedImport(t)
	if err := x.reg.Delete(x.rec.Host); err != nil {
		t.Fatal(err)
	}
	x.box.noRoute = true
	read := x.cfg.Seeds.LocalSeedID
	x.cfg.Seeds.LocalSeedID = func(dbPath string) (string, error) {
		if err := x.reg.Upsert(addrRow(x.rec, x.rec.Account)); err != nil {
			t.Fatal(err)
		}
		return read(dbPath)
	}
	x.pass(t)
	if !x.box.local || count(x.box.steps, "remove-local") != 0 || len(x.box.fenced) != 0 {
		t.Fatalf("local %v steps %q fenced %v", x.box.local, x.box.steps, x.box.fenced)
	}
	if st := x.journalState(t); st != registry.JournalRollingBack {
		t.Fatalf("journal = %q", st)
	}
}

// A generation that cannot be minted (the box's source failed, or answered
// 0, which is no fence's) closes nothing: the journal stays swapped with
// no generation stored, so a later pass mints a real one. A 0 stored there
// would read as "ours already" on any unfenced host.
func TestRollbackWaitsForAFenceGeneration(t *testing.T) {
	for name, mint := range map[string]func() (uint64, error){
		"an error": func() (uint64, error) { return 0, errors.New("registry busy") },
		"zero":     func() (uint64, error) { return 0, nil },
	} {
		t.Run(name, func(t *testing.T) {
			x := abortedImport(t)
			x.cfg.Seeds.NextFenceGen = mint
			x.pass(t)
			j, _ := x.reg.ImportJournal(x.rec.Host)
			if j == nil || j.State != registry.JournalSwapped || j.FenceGen != 0 {
				t.Fatalf("journal = %+v, want swapped with no generation", j)
			}
			if got := mustGet(t, x.reg, x.rec.Host); got.ImportGate == registry.ImportGateClosed {
				t.Fatalf("the gate closed without a generation: %+v", got)
			}
			if len(x.box.steps) != 0 {
				t.Fatalf("steps = %q", x.box.steps)
			}
		})
	}
}

// Merge review S1: the swap records what the host's own env file held
// before the import, in the journal, and swaps only when it could read it.
func TestSwapRecordsTheEnvFileBeforeTheImport(t *testing.T) {
	for _, before := range []string{registry.EnvBeforeAbsent, registry.EnvBeforeNoKey, registry.EnvBeforeKey} {
		t.Run(before, func(t *testing.T) {
			s, sqid := shopSetup(t)
			rec := importRecord(t, s, sqid)
			reg := regWith(t, addrRow(rec, sqid))
			cfg := importCfg(t, nil)
			var asked []string
			cfg.Seeds.EnvBefore = func(db, host string) (string, error) {
				// Asked before the import's row exists.
				if rt := mustGet(t, reg, rec.Host); rt.ImportGate != "" || rt.Kind != "service" && rt.Addr == "" {
					t.Errorf("the env file was read after the swap: %+v", rt)
				}
				asked = append(asked, db+" "+host)
				return before, nil
			}
			if err := Reconcile(ctx, s, reg, cfg); err != nil {
				t.Fatal(err)
			}
			j, err := reg.ImportJournal(rec.Host)
			if err != nil || j == nil || j.EnvBefore != before {
				t.Fatalf("journal = %+v, %v; want env before %q", j, err, before)
			}
			if want := []string{NestedDBPath("/data", rec.Host) + " " + rec.Host}; !reflect.DeepEqual(asked, want) {
				t.Fatalf("asked %q, want %q", asked, want)
			}
		})
	}
	for name, hook := range map[string]func(string, string) (string, error){
		"the file cannot be read": func(string, string) (string, error) { return "", errors.New("a symlink") },
		"no reader":               nil,
	} {
		t.Run(name, func(t *testing.T) {
			s, sqid := shopSetup(t)
			rec := importRecord(t, s, sqid)
			reg := regWith(t, addrRow(rec, sqid))
			original := *mustGet(t, reg, rec.Host)
			cfg := importCfg(t, nil)
			cfg.Seeds.EnvBefore = hook
			if err := Reconcile(ctx, s, reg, cfg); err != nil {
				t.Fatal(err)
			}
			if j, _ := reg.ImportJournal(rec.Host); j != nil {
				t.Fatalf("swapped without knowing what the env file held: %+v", j)
			}
			if got := mustGet(t, reg, rec.Host); !reflect.DeepEqual(*got, original) {
				t.Fatalf("row = %+v, want the original untouched", got)
			}
			if st := seedState(t, s, rec); st.State != seeds.StateReady {
				t.Fatalf("record = %s, want ready: a deferred swap refuses nothing", st.State)
			}
		})
	}
}

// Merge review S1: the rollback hands the journal's record of the env file
// to the key's retirement, for the import's own row, whatever it says. A
// key that was there before the import is the hook's to leave (the
// activator's RetireSeedKey; its own tests cover the file).
func TestRollbackRetiresTheKeyByWhatTheSwapRecorded(t *testing.T) {
	for _, before := range []string{registry.EnvBeforeAbsent, registry.EnvBeforeNoKey, registry.EnvBeforeKey} {
		t.Run(before, func(t *testing.T) {
			s, sqid := shopSetup(t)
			rec := importRecord(t, s, sqid)
			reg := regWith(t, addrRow(rec, sqid))
			cfg := importCfg(t, nil)
			cfg.Seeds.EnvBefore = func(string, string) (string, error) { return before, nil }
			rr := &rollbackRecorder{proceed: true}
			rr.hooks(cfg.Seeds)
			Reconcile(ctx, s, reg, cfg) // swap
			if _, err := SetRecordSeed(ctx, s, rec.Account, rec.App, rec.Host, SeedAbort(rec.Seed.ID)); err != nil {
				t.Fatal(err)
			}
			if err := Reconcile(ctx, s, reg, cfg); err != nil {
				t.Fatal(err)
			}
			want := []string{NestedDBPath("/data", rec.Host) + " " + rec.Host + " " + before}
			if !reflect.DeepEqual(rr.retired, want) {
				t.Fatalf("retired %q, want %q", rr.retired, want)
			}
		})
	}
}

// A retirement that fails keeps the rollback where it was, fenced, with
// the local data and the gated row in place; the next pass tries again.
func TestRollbackWaitsWhileTheKeyCannotBeRetired(t *testing.T) {
	x := abortedImport(t)
	retire := x.cfg.Seeds.RetireKey
	fail := true
	x.cfg.Seeds.RetireKey = func(db, host, before string) error {
		if fail {
			return errors.New("the env file is a symlink")
		}
		return retire(db, host, before)
	}
	x.pass(t)
	if st := x.journalState(t); st != registry.JournalRollingBack {
		t.Fatalf("journal = %q, want rolling-back", st)
	}
	if !x.box.local || !x.box.key || count(x.box.steps, "remove-local") != 0 {
		t.Fatalf("local %v key %v steps %q: the rollback went on past a key it could not retire", x.box.local, x.box.key, x.box.steps)
	}
	if got := mustGet(t, x.reg, x.rec.Host); got.ImportGate != registry.ImportGateClosed {
		t.Fatalf("row = %+v, want the closed import row", got)
	}
	fail = false
	x.pass(t)
	x.wantRolledBack(t)
}