package activator

import (
	"bytes"
	"context"
	"encoding/base64"
	"errors"
	"log"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"sync/atomic"
	"syscall"
	"testing"
	"time"

	"github.com/carlosframework/platform/internal/provision"
	"github.com/carlosframework/platform/internal/registry"
)

// keyLines returns every CARLOS_INSTANCE_KEY value in raw, in order.
func keyLines(raw []byte) []string {
	var out []string
	for _, line := range strings.Split(string(raw), "\n") {
		if v, ok := strings.CutPrefix(line, InstanceKeyVar+"="); ok {
			out = append(out, v)
		}
	}
	return out
}

// syncProbe counts the key's syncs and fails them on demand. Set the
// error fields only between calls.
type syncProbe struct {
	file, dir       int
	fileErr, dirErr error
}

func probeInstanceKeySync(t *testing.T) *syncProbe {
	t.Helper()
	p := &syncProbe{}
	adminTokenMu.Lock()
	old := instanceKeySync
	instanceKeySync.file = func(f *os.File) error {
		p.file++
		if p.fileErr != nil {
			return p.fileErr
		}
		return f.Sync()
	}
	instanceKeySync.dir = func(d string) error {
		p.dir++
		if p.dirErr != nil {
			return p.dirErr
		}
		return syncDir(d)
	}
	adminTokenMu.Unlock()
	t.Cleanup(func() {
		adminTokenMu.Lock()
		instanceKeySync = old
		adminTokenMu.Unlock()
	})
	return p
}

func TestEnsureInstanceKeyMintsOnceInTheAgreedFormat(t *testing.T) {
	p := filepath.Join(t.TempDir(), "shop.example.com.env")
	got, err := ensureInstanceKeyNoConfig(p)
	if err != nil || got != InstanceKeyMinted {
		t.Fatalf("first ensure = %v, %v; want minted", got, err)
	}
	raw, _ := os.ReadFile(p)
	keys := keyLines(raw)
	if len(keys) != 1 {
		t.Fatalf("file holds %d keys, want 1", len(keys))
	}
	dec, err := base64.StdEncoding.Strict().DecodeString(keys[0])
	if err != nil || len(dec) != 32 || len(keys[0]) != 44 {
		t.Fatalf("key is not 32 bytes in padded standard base64 (len %d, decode err %v)", len(keys[0]), err)
	}
	if !bytes.HasSuffix(raw, []byte("\n")) {
		t.Fatal("the key line has no newline: the next append would join it")
	}
	if st, _ := os.Stat(p); st.Mode().Perm() != 0o600 {
		t.Fatalf("mode = %o, want 600", st.Mode().Perm())
	}
	// The parser every launch path uses yields the exact characters minted.
	env, _ := readEnvFile(p)
	if env[InstanceKeyVar] != keys[0] {
		t.Fatal("readEnvFile does not yield the minted key verbatim")
	}
	again, err := ensureInstanceKeyNoConfig(p)
	if err != nil || again != InstanceKeyPresent {
		t.Fatalf("second ensure = %v, %v; want present", again, err)
	}
	raw2, _ := os.ReadFile(p)
	if !bytes.Equal(raw, raw2) {
		t.Fatal("second ensure changed the file")
	}
}

// Never overwrite: a key already there, of any shape, is kept byte for byte
// and only reported.
func TestEnsureInstanceKeyNeverRewritesAnExistingKey(t *testing.T) {
	for _, existing := range []string{
		"CARLOS_INSTANCE_KEY=hand-written-not-base64\n",
		"A=1\nCARLOS_INSTANCE_KEY=" + strings.Repeat("A", 43) + "=", // well formed, no trailing newline
	} {
		p := filepath.Join(t.TempDir(), "shop.example.com.env")
		if err := os.WriteFile(p, []byte(existing), 0o600); err != nil {
			t.Fatal(err)
		}
		var buf bytes.Buffer
		log.SetOutput(&buf)
		got, err := ensureInstanceKeyNoConfig(p)
		log.SetOutput(os.Stderr)
		if err != nil || got != InstanceKeyPresent {
			t.Fatalf("ensure over %q = %v, %v; want present", existing, got, err)
		}
		if raw, _ := os.ReadFile(p); string(raw) != existing {
			t.Fatalf("file changed: %q -> %q", existing, raw)
		}
		if strings.Contains(buf.String(), "hand-written-not-base64") {
			t.Fatal("the shape warning logged the key's value")
		}
	}
}

func TestEnsureInstanceKeyAppendsOnAFreshLineAndKeepsEveryOtherLine(t *testing.T) {
	p := filepath.Join(t.TempDir(), "shop.example.com.env")
	before := "CARLOS_ADMIN_TOKEN=" + strings.Repeat("a", 64) + "\n# comment\nDATABASE_URL=sqlite" // no trailing newline
	if err := os.WriteFile(p, []byte(before), 0o644); err != nil {
		t.Fatal(err)
	}
	if got, err := ensureInstanceKeyNoConfig(p); err != nil || got != InstanceKeyMinted {
		t.Fatalf("ensure = %v, %v", got, err)
	}
	raw, _ := os.ReadFile(p)
	if !strings.HasPrefix(string(raw), before+"\n"+InstanceKeyVar+"=") {
		t.Fatalf("file = %q: the key must land on its own line after the untouched original", raw)
	}
	env, _ := readEnvFile(p)
	if env["DATABASE_URL"] != "sqlite" || len(env["CARLOS_ADMIN_TOKEN"]) != 64 {
		t.Fatal("an existing line was lost")
	}
	if st, _ := os.Stat(p); st.Mode().Perm() != 0o600 {
		t.Fatalf("mode = %o, want the 0644 file tightened to 600", st.Mode().Perm())
	}
}

// A key the app's configuration supplies is the app's key: minting one into
// <host>.env would override it (mergeEnv) and change the key under its data.
// The decision is the effective env's, so an empty CARLOS_INSTANCE_KEY= in
// the own file does not hide the configured key (finding 4).
func TestEnsureInstanceKeyDefersToTheRenderedConfiguration(t *testing.T) {
	for _, own := range []string{"A=1\n", "A=1\n" + InstanceKeyVar + "=\n"} {
		dir := t.TempDir()
		p := filepath.Join(dir, "shop.example.com.env")
		if err := os.WriteFile(filepath.Join(dir, "shop.example.com.config.env"), []byte(InstanceKeyVar+"=from-config\n"), 0o600); err != nil {
			t.Fatal(err)
		}
		if err := os.WriteFile(p, []byte(own), 0o600); err != nil {
			t.Fatal(err)
		}
		got, err := ensureInstanceKeyNoConfig(p)
		if err != nil || got != InstanceKeyFromConfig {
			t.Fatalf("own %q: ensure = %v, %v; want from-config", own, got, err)
		}
		if raw, _ := os.ReadFile(p); string(raw) != own {
			t.Fatalf("own file changed: %q", raw)
		}
	}
}

func TestBackfillInstanceKeyAppendsOnlyToAnExistingEnvFile(t *testing.T) {
	b := &ExecBackend{DataDir: t.TempDir()}
	fresh := registry.Route{Host: "new.example.com", Socket: "/run/carlos/new.sock"}
	got, err := b.BackfillInstanceKey(t.Context(), fresh)
	if err != nil || got != InstanceKeyNoEnvFile {
		t.Fatalf("backfill with no env file = %v, %v; want no-env-file", got, err)
	}
	if _, err := os.Lstat(filepath.Join(b.DataDir, fresh.Host+".env")); !os.IsNotExist(err) {
		t.Fatalf("backfill created an env file: %v", err)
	}
	old := registry.Route{Host: "shop.example.com", Socket: "/run/carlos/shop.sock"}
	envPath := filepath.Join(b.DataDir, old.Host+".env")
	tok := "CARLOS_ADMIN_TOKEN=" + strings.Repeat("b", 64) + "\n"
	if err := os.WriteFile(envPath, []byte(tok), 0o600); err != nil {
		t.Fatal(err)
	}
	if got, err := b.BackfillInstanceKey(t.Context(), old); err != nil || got != InstanceKeyMinted {
		t.Fatalf("backfill = %v, %v; want minted", got, err)
	}
	raw, _ := os.ReadFile(envPath)
	if !strings.HasPrefix(string(raw), tok) || len(keyLines(raw)) != 1 {
		t.Fatalf("file = %q", raw)
	}
	// Idempotent: a second backfill (the next agent start) writes nothing.
	if got, err := b.BackfillInstanceKey(t.Context(), old); err != nil || got != InstanceKeyPresent {
		t.Fatalf("second backfill = %v, %v; want present", got, err)
	}
	if raw2, _ := os.ReadFile(envPath); !bytes.Equal(raw, raw2) {
		t.Fatal("second backfill changed the file")
	}
}

// Review focus 1: a hand-edited file with two keys. Readers take the last
// (parseEnvBytes), and the ensure writes nothing.
func TestEnsureInstanceKeyLeavesAFileWithTwoKeysAlone(t *testing.T) {
	p := filepath.Join(t.TempDir(), "shop.example.com.env")
	body := InstanceKeyVar + "=first\n" + InstanceKeyVar + "=second\n"
	if err := os.WriteFile(p, []byte(body), 0o600); err != nil {
		t.Fatal(err)
	}
	if got, err := ensureInstanceKeyNoConfig(p); err != nil || got != InstanceKeyPresent {
		t.Fatalf("ensure = %v, %v; want present", got, err)
	}
	if raw, _ := os.ReadFile(p); string(raw) != body {
		t.Fatalf("file changed: %q", raw)
	}
	if env, _ := readEnvFile(p); env[InstanceKeyVar] != "second" {
		t.Fatalf("readers see %q, want the last line's value", env[InstanceKeyVar])
	}
}

// Review focus 2: an empty value is no key. One is appended after it, and
// readers take the appended one.
func TestEnsureInstanceKeyTreatsAnEmptyValueAsAbsent(t *testing.T) {
	p := filepath.Join(t.TempDir(), "shop.example.com.env")
	if err := os.WriteFile(p, []byte(InstanceKeyVar+"=\n"), 0o600); err != nil {
		t.Fatal(err)
	}
	if got, err := ensureInstanceKeyNoConfig(p); err != nil || got != InstanceKeyMinted {
		t.Fatalf("ensure = %v, %v; want minted", got, err)
	}
	env, _ := readEnvFile(p)
	if !wellFormedInstanceKey(env[InstanceKeyVar]) {
		t.Fatal("readers do not see the appended key")
	}
}

// Review focus 3: a seed, a visitor's wake and the backfill can ask at
// once. Exactly one key is written.
func TestEnsureInstanceKeyMintsOneKeyUnderConcurrentCallers(t *testing.T) {
	p := filepath.Join(t.TempDir(), "shop.example.com.env")
	var wg sync.WaitGroup
	for i := 0; i < 8; i++ {
		wg.Add(1)
		go func() {
			defer wg.Done()
			if _, err := ensureInstanceKeyNoConfig(p); err != nil {
				t.Error(err)
			}
		}()
	}
	wg.Wait()
	if raw, _ := os.ReadFile(p); len(keyLines(raw)) != 1 {
		t.Fatalf("%d keys written", len(keyLines(raw)))
	}
}

// Review focus 5: an empty CARLOS_INSTANCE_KEY in the rendered
// configuration supplies nothing, so the platform mints one.
func TestEnsureInstanceKeyMintsWhenTheConfigurationValueIsEmpty(t *testing.T) {
	dir := t.TempDir()
	if err := os.WriteFile(filepath.Join(dir, "shop.example.com.config.env"), []byte(InstanceKeyVar+"=\n"), 0o600); err != nil {
		t.Fatal(err)
	}
	if got, err := ensureInstanceKeyNoConfig(filepath.Join(dir, "shop.example.com.env")); err != nil || got != InstanceKeyMinted {
		t.Fatalf("ensure = %v, %v; want minted", got, err)
	}
}

// Finding 2: a mint whose file or directory sync fails reports failure,
// and so does every retry until a sync succeeds. The retry finds the key
// present, keeps its bytes, and still completes the chmod and both syncs
// before it reports success.
func TestEnsureInstanceKeyRedoesAFailedSyncBeforeReportingSuccess(t *testing.T) {
	for _, tc := range []struct {
		name string
		fail func(*syncProbe, error)
	}{
		{"file sync", func(p *syncProbe, err error) { p.fileErr = err }},
		{"directory sync", func(p *syncProbe, err error) { p.dirErr = err }},
	} {
		t.Run(tc.name, func(t *testing.T) {
			probe := probeInstanceKeySync(t)
			p := filepath.Join(t.TempDir(), "shop.example.com.env")
			tc.fail(probe, errors.New("injected: EIO"))
			if _, err := ensureInstanceKeyNoConfig(p); err == nil {
				t.Fatal("a mint whose sync failed reported success")
			}
			raw, _ := os.ReadFile(p)
			if len(keyLines(raw)) != 1 {
				t.Fatalf("the failed mint left %d keys, want its 1", len(keyLines(raw)))
			}
			if _, err := ensureInstanceKeyNoConfig(p); err == nil {
				t.Fatal("a retry reported success while the sync still fails")
			}
			if err := os.Chmod(p, 0o644); err != nil {
				t.Fatal(err)
			}
			tc.fail(probe, nil)
			before := *probe
			got, err := ensureInstanceKeyNoConfig(p)
			if err != nil || got != InstanceKeyPresent {
				t.Fatalf("retry = %v, %v; want present", got, err)
			}
			if probe.file != before.file+1 || probe.dir != before.dir+1 {
				t.Fatalf("the retry synced the file %d and the directory %d times, want once each", probe.file-before.file, probe.dir-before.dir)
			}
			if raw2, _ := os.ReadFile(p); !bytes.Equal(raw, raw2) {
				t.Fatal("a retry rewrote the key")
			}
			if st, _ := os.Stat(p); st.Mode().Perm() != 0o600 {
				t.Fatalf("mode = %o, want the present path to set 600", st.Mode().Perm())
			}
		})
	}
}

// Ruling 26: a replica row is never minted a key, on any path; the only
// key it may hold is its home's (replicaKeyGate).
func TestReplicaRowsGetNoPlatformKey(t *testing.T) {
	b := &ExecBackend{DataDir: t.TempDir()}
	rt := registry.Route{Host: "shop-au.example.com", Kind: "instance", Socket: "/run/carlos/shop-au.sock", Replica: true}
	envPath := filepath.Join(b.DataDir, rt.Host+".env")
	if err := os.WriteFile(envPath, []byte("CARLOS_ADMIN_TOKEN=t\n"), 0o600); err != nil {
		t.Fatal(err)
	}
	if got, err := b.ensureInstanceKeyFor(t.Context(), rt, b.pathsFor(rt)); err != nil || got != InstanceKeyReplica {
		t.Fatalf("launch ensure = %v, %v; want replica", got, err)
	}
	if got, err := b.BackfillInstanceKey(t.Context(), rt); err != nil || got != InstanceKeyReplica {
		t.Fatalf("backfill = %v, %v; want replica", got, err)
	}
	if raw, _ := os.ReadFile(envPath); len(keyLines(raw)) != 0 {
		t.Fatal("a replica row was given a platform key")
	}
}

// The flat ensure never follows a symlink planted at <host>.env (spec §7.2,
// the hazard TestEnsureAdminTokenRefusesToChmodThroughASymlink guards). On
// every path, mint, existing key, backfill and the claim settle, it refuses
// and leaves the link's target's mode and bytes exactly as they were.
func TestEnsureInstanceKeyRefusesToFollowASymlink(t *testing.T) {
	wellFormed := InstanceKeyVar + "=" + strings.Repeat("A", 43) + "=\n"
	for _, tc := range []struct {
		name   string
		target string // the target's contents
		mode   os.FileMode
		call   func(b *ExecBackend, rt registry.Route, link string) error
	}{
		{"mint", "A=1\n", 0o644, func(_ *ExecBackend, _ registry.Route, link string) error {
			_, err := ensureInstanceKeyNoConfig(link)
			return err
		}},
		{"existing key", wellFormed, 0o644, func(_ *ExecBackend, _ registry.Route, link string) error {
			_, err := ensureInstanceKeyNoConfig(link)
			return err
		}},
		{"backfill mint", "A=1\n", 0o644, func(b *ExecBackend, rt registry.Route, _ string) error {
			_, err := b.BackfillInstanceKey(t.Context(), rt)
			return err
		}},
		{"backfill existing key", wellFormed, 0o644, func(b *ExecBackend, rt registry.Route, _ string) error {
			_, err := b.BackfillInstanceKey(t.Context(), rt)
			return err
		}},
		{"claim settle", wellFormed, 0o600, func(b *ExecBackend, rt registry.Route, _ string) error {
			return b.settleClaimInstanceEnv(rt)
		}},
	} {
		t.Run(tc.name, func(t *testing.T) {
			b := &ExecBackend{DataDir: t.TempDir()}
			rt := registry.Route{Host: "shop.example.com", Socket: "/run/carlos/shop.sock"}
			target := filepath.Join(t.TempDir(), "someone-elses-file")
			if err := os.WriteFile(target, []byte(tc.target), tc.mode); err != nil {
				t.Fatal(err)
			}
			if err := os.Chmod(target, tc.mode); err != nil {
				t.Fatal(err)
			}
			link := filepath.Join(b.DataDir, rt.Host+".env")
			if err := os.Symlink(target, link); err != nil {
				t.Skipf("this filesystem will not make symlinks: %v", err)
			}
			err := tc.call(b, rt, link)
			if !errors.Is(err, errOwnEnvSymlink) {
				t.Fatalf("err = %v, want errOwnEnvSymlink", err)
			}
			if strings.Contains(err.Error(), strings.Repeat("A", 43)) {
				t.Fatal("the refusal put the key in its error")
			}
			st, err := os.Stat(target)
			if err != nil {
				t.Fatal(err)
			}
			if st.Mode().Perm() != tc.mode {
				t.Fatalf("mode of the symlink's TARGET = %o, want the untouched %o", st.Mode().Perm(), tc.mode)
			}
			if raw, _ := os.ReadFile(target); string(raw) != tc.target {
				t.Fatalf("the symlink's TARGET changed: %q -> %q", tc.target, raw)
			}
		})
	}
}

// A dangling link is refused too: the mint's O_CREATE must not create the
// file it points at.
func TestEnsureInstanceKeyRefusesADanglingSymlink(t *testing.T) {
	dir := t.TempDir()
	target := filepath.Join(t.TempDir(), "not-yet")
	link := filepath.Join(dir, "shop.example.com.env")
	if err := os.Symlink(target, link); err != nil {
		t.Skipf("this filesystem will not make symlinks: %v", err)
	}
	if _, err := ensureInstanceKeyNoConfig(link); !errors.Is(err, errOwnEnvSymlink) {
		t.Fatalf("err = %v, want errOwnEnvSymlink", err)
	}
	if _, err := os.Lstat(target); !os.IsNotExist(err) {
		t.Fatalf("the mint created the link's target: %v", err)
	}
}

// The refusal is the open's own, not a check before it: the helper refuses
// a link for every flag set the ensure uses, so no Lstat-to-open window
// exists on unix.
func TestOpenOwnEnvNoFollowRefusesALinkForEveryFlagSet(t *testing.T) {
	dir := t.TempDir()
	target := filepath.Join(dir, "target")
	if err := os.WriteFile(target, []byte("A=1\n"), 0o644); err != nil {
		t.Fatal(err)
	}
	link := filepath.Join(dir, "h.env")
	if err := os.Symlink(target, link); err != nil {
		t.Skipf("this filesystem will not make symlinks: %v", err)
	}
	for _, flags := range []int{os.O_RDONLY, os.O_APPEND | os.O_WRONLY, os.O_APPEND | os.O_WRONLY | os.O_CREATE} {
		perm := os.FileMode(0)
		if flags&os.O_CREATE != 0 {
			perm = 0o600
		}
		f, err := openOwnEnvNoFollow(link, flags, perm)
		if err == nil {
			f.Close()
			t.Fatalf("flags %#x: opened through a symlink", flags)
		}
		if !errors.Is(err, errOwnEnvSymlink) {
			t.Fatalf("flags %#x: err = %v, want errOwnEnvSymlink", flags, err)
		}
	}
	// An ordinary file still opens.
	f, err := openOwnEnvNoFollow(target, os.O_RDONLY, 0)
	if err != nil {
		t.Fatal(err)
	}
	f.Close()
}

// setInstanceKeySeams swaps the append and the before-open hook for one
// test, under adminTokenMu as their readers require. A nil argument
// leaves that seam as it is.
func setInstanceKeySeams(t *testing.T, write func(*os.File, []byte) (int, error), beforeOpen func(string)) {
	t.Helper()
	adminTokenMu.Lock()
	oldWrite, oldHook := instanceKeyWrite, instanceKeyHooks.beforeOpen
	if write != nil {
		instanceKeyWrite = write
	}
	if beforeOpen != nil {
		instanceKeyHooks.beforeOpen = beforeOpen
	}
	adminTokenMu.Unlock()
	t.Cleanup(func() {
		adminTokenMu.Lock()
		instanceKeyWrite, instanceKeyHooks.beforeOpen = oldWrite, oldHook
		adminTokenMu.Unlock()
	})
}

// A mint whose append fails part way (a full disk) is undone: the file is
// byte for byte what it was, so no short key line survives to be read as
// present, and the next call mints a full key.
func TestEnsureInstanceKeyUndoesAPartialAppend(t *testing.T) {
	for _, tc := range []struct {
		name   string
		before *string // nil: no file before the call
		write  func(f *os.File, b []byte) (int, error)
	}{
		{"nothing written, ENOSPC", ptr("A=1\n"), func(*os.File, []byte) (int, error) { return 0, syscall.ENOSPC }},
		{"short key line, ENOSPC", ptr("A=1\n"), func(f *os.File, b []byte) (int, error) {
			n, _ := f.Write(b[:len(InstanceKeyVar)+5])
			return n, syscall.ENOSPC
		}},
		{"short after the fresh-line prefix, ENOSPC", ptr("A=1"), func(f *os.File, b []byte) (int, error) {
			n, _ := f.Write(b[:3])
			return n, syscall.ENOSPC
		}},
		{"short with no error", ptr("A=1\n"), func(f *os.File, b []byte) (int, error) {
			return f.Write(b[:len(b)-1])
		}},
		{"new file, short key line", nil, func(f *os.File, b []byte) (int, error) {
			n, _ := f.Write(b[:10])
			return n, syscall.ENOSPC
		}},
	} {
		t.Run(tc.name, func(t *testing.T) {
			probe := probeInstanceKeySync(t)
			p := filepath.Join(t.TempDir(), "shop.example.com.env")
			if tc.before != nil {
				if err := os.WriteFile(p, []byte(*tc.before), 0o600); err != nil {
					t.Fatal(err)
				}
			}
			setInstanceKeySeams(t, tc.write, nil)
			if _, err := ensureInstanceKeyNoConfig(p); err == nil {
				t.Fatal("a mint whose append failed reported success")
			}
			raw, err := os.ReadFile(p)
			if err != nil {
				t.Fatal(err)
			}
			want := ""
			if tc.before != nil {
				want = *tc.before
			}
			if string(raw) != want {
				t.Fatalf("after the failed append the file is %q, want it undone to %q", raw, want)
			}
			if probe.file != 1 {
				t.Fatalf("the undo synced the file %d times, want once", probe.file)
			}
			setInstanceKeySeams(t, func(f *os.File, b []byte) (int, error) { return f.Write(b) }, nil)
			got, err := ensureInstanceKeyNoConfig(p)
			if err != nil || got != InstanceKeyMinted {
				t.Fatalf("next ensure = %v, %v; want minted", got, err)
			}
			raw, _ = os.ReadFile(p)
			keys := keyLines(raw)
			if len(keys) != 1 || !wellFormedInstanceKey(keys[0]) {
				t.Fatalf("file = %q: want one full key", raw)
			}
			if !strings.HasPrefix(string(raw), want) {
				t.Fatalf("file = %q: the original bytes were lost", raw)
			}
		})
	}
}

func ptr(s string) *string { return &s }

// The append open refuses a link on its own, not only because the read
// before it did: a link swapped in after a clean read (the hook) is
// refused on the mint, the existing-key and the backfill paths, and its
// target is left exactly as it was.
func TestEnsureInstanceKeyOpenRefusesALinkSwappedInAfterTheRead(t *testing.T) {
	wellFormed := InstanceKeyVar + "=" + strings.Repeat("A", 43) + "=\n"
	for _, tc := range []struct {
		name string
		own  string
		call func(b *ExecBackend, rt registry.Route, path string) error
	}{
		{"mint", "A=1\n", func(_ *ExecBackend, _ registry.Route, path string) error {
			_, err := ensureInstanceKeyNoConfig(path)
			return err
		}},
		{"existing key", wellFormed, func(_ *ExecBackend, _ registry.Route, path string) error {
			_, err := ensureInstanceKeyNoConfig(path)
			return err
		}},
		{"backfill mint", "A=1\n", func(b *ExecBackend, rt registry.Route, _ string) error {
			_, err := b.BackfillInstanceKey(t.Context(), rt)
			return err
		}},
	} {
		t.Run(tc.name, func(t *testing.T) {
			b := &ExecBackend{DataDir: t.TempDir()}
			rt := registry.Route{Host: "shop.example.com", Socket: "/run/carlos/shop.sock"}
			path := filepath.Join(b.DataDir, rt.Host+".env")
			if err := os.WriteFile(path, []byte(tc.own), 0o600); err != nil {
				t.Fatal(err)
			}
			target := filepath.Join(t.TempDir(), "someone-elses-file")
			const targetBody = "B=2\n"
			if err := os.WriteFile(target, []byte(targetBody), 0o644); err != nil {
				t.Fatal(err)
			}
			if err := os.Chmod(target, 0o644); err != nil {
				t.Fatal(err)
			}
			swapped := 0
			setInstanceKeySeams(t, nil, func(p string) {
				if p != path {
					return
				}
				swapped++
				if err := os.Remove(p); err != nil {
					t.Error(err)
				}
				if err := os.Symlink(target, p); err != nil {
					t.Error(err)
				}
			})
			err := tc.call(b, rt, path)
			if swapped != 1 {
				t.Fatalf("the hook ran %d times, want once", swapped)
			}
			if !errors.Is(err, errOwnEnvSymlink) {
				t.Fatalf("err = %v, want errOwnEnvSymlink", err)
			}
			st, err := os.Stat(target)
			if err != nil {
				t.Fatal(err)
			}
			if st.Mode().Perm() != 0o644 {
				t.Fatalf("mode of the symlink's TARGET = %o, want the untouched 644", st.Mode().Perm())
			}
			if raw, _ := os.ReadFile(target); string(raw) != targetBody {
				t.Fatalf("the symlink's TARGET changed: %q", raw)
			}
		})
	}
}

// A refusal names the path once.
func TestEnsureInstanceKeySymlinkErrorNamesThePathOnce(t *testing.T) {
	dir := t.TempDir()
	target := filepath.Join(dir, "target")
	if err := os.WriteFile(target, []byte("A=1\n"), 0o644); err != nil {
		t.Fatal(err)
	}
	link := filepath.Join(dir, "shop.example.com.env")
	if err := os.Symlink(target, link); err != nil {
		t.Skipf("this filesystem will not make symlinks: %v", err)
	}
	_, err := ensureInstanceKeyNoConfig(link)
	if !errors.Is(err, errOwnEnvSymlink) {
		t.Fatalf("err = %v, want errOwnEnvSymlink", err)
	}
	if n := strings.Count(err.Error(), link); n != 1 {
		t.Fatalf("error %q names the path %d times, want once", err, n)
	}
}

// startRecorded starts host on b with the recording binary and returns the
// environment the child saw, or nil when no child ran (none recorded within a
// second, or none is left running that could). The route's socket is a temp
// path, so the row classifies as foreign; startRecordedRoute takes the row as
// given.
func startRecorded(t *testing.T, b *ExecBackend, host string) (map[string]string, error) {
	t.Helper()
	return startRecordedRoute(t, b, registry.Route{Host: host, Kind: "instance", Socket: tmpSock(t)})
}

func startRecordedRoute(t *testing.T, b *ExecBackend, rt registry.Route) (map[string]string, error) {
	t.Helper()
	_, envPath := stageRecordingLiveFullEnv(t, b, rt.Host, "v1")
	ctx, cancel := recordedCtx(2*time.Second, envPath)
	defer cancel()
	proc, _, err := b.Start(ctx, rt) // the fake binary never binds: the health wait fails, as in the sibling tests
	if proc != nil {
		defer proc.Stop(context.Background())
	}
	raw := awaitChildFile(envPath, time.Second)
	if raw == "" {
		return nil, err
	}
	child := map[string]string{}
	for _, line := range strings.Split(raw, "\n") {
		if k, v, ok := strings.Cut(line, "="); ok {
			child[k] = v
		}
	}
	return child, err
}

// Start hands the child the key it minted, and a second Start (a restart, a
// redeploy) hands it the same one.
func TestExecBackendStartHandsTheChildTheMintedInstanceKey(t *testing.T) {
	b := testExecBackend(t)
	host := "shop.example.com"
	first, _ := startRecorded(t, b, host)
	minted, err := readEnvFile(filepath.Join(b.DataDir, host+".env"))
	if err != nil {
		t.Fatal(err)
	}
	if first == nil || !wellFormedInstanceKey(minted[InstanceKeyVar]) || first[InstanceKeyVar] != minted[InstanceKeyVar] {
		t.Fatal("the child was not handed the key minted into <host>.env")
	}
	if second, _ := startRecorded(t, b, host); second == nil || second[InstanceKeyVar] != minted[InstanceKeyVar] {
		t.Fatal("a second start handed the child a different key")
	}
}

// Task 3 review (M4): the tests above launch through the foreign arm, since
// a temp socket is neither default. A row that classifies as flat goes
// through the same spawn body, and this pins it end to end: Start mints
// into <DataDir>/<host>.env and hands the child that key.
func TestExecBackendStartHandsAFlatRowsChildTheMintedInstanceKey(t *testing.T) {
	b := testExecBackend(t)
	b.RunDirOverride = shortRunDir(t)
	host := "shop.example.com"
	rt := flatGenRoute(b, host)
	if l, why := b.Layout(rt); l != LayoutFlat {
		t.Fatalf("fixture classifies %q (%s), want flat", l, why)
	}
	child, _ := startRecordedRoute(t, b, rt)
	minted, err := readEnvFile(filepath.Join(b.DataDir, host+".env"))
	if err != nil {
		t.Fatal(err)
	}
	if child == nil || !wellFormedInstanceKey(minted[InstanceKeyVar]) || child[InstanceKeyVar] != minted[InstanceKeyVar] {
		t.Fatal("a flat row's child was not handed the key minted into <host>.env")
	}
}

// Finding 2 through a launch: while the key's sync fails, Start fails and
// no child runs. Once the sync succeeds, the child is handed the key the
// failed attempt wrote, never a second one.
func TestExecBackendStartStartsNoChildUntilTheKeyIsSynced(t *testing.T) {
	for _, tc := range []struct {
		name string
		fail func(*syncProbe, error)
	}{
		{"file sync", func(p *syncProbe, err error) { p.fileErr = err }},
		{"directory sync", func(p *syncProbe, err error) { p.dirErr = err }},
	} {
		t.Run(tc.name, func(t *testing.T) {
			probe := probeInstanceKeySync(t)
			b := testExecBackend(t)
			host := "shop.example.com"
			tc.fail(probe, errors.New("injected: EIO"))
			for i := 0; i < 2; i++ {
				child, err := startRecorded(t, b, host)
				if err == nil || !strings.Contains(err.Error(), "instance key for "+host) {
					t.Fatalf("start %d with a failing sync: err = %v, want the instance key's error", i, err)
				}
				if child != nil {
					t.Fatalf("start %d ran a child before the key was durable", i)
				}
			}
			written, _ := readEnvFile(filepath.Join(b.DataDir, host+".env"))
			tc.fail(probe, nil)
			child, _ := startRecorded(t, b, host)
			if child == nil || child[InstanceKeyVar] == "" || child[InstanceKeyVar] != written[InstanceKeyVar] {
				t.Fatal("after the sync succeeded the child was not handed the key the failed attempt wrote")
			}
		})
	}
}

// Finding 4 through a launch: <host>.env and <host>.config.env together.
// An empty override never hands the child an empty key while the
// configuration supplies one, and no key is minted over the configured one.
// With neither supplying one, the child gets the minted key.
func TestExecBackendStartNeverHandsTheChildAnEmptyKeyWhileOneExists(t *testing.T) {
	tok := "CARLOS_ADMIN_TOKEN=" + strings.Repeat("c", 64) + "\n"
	for _, tc := range []struct {
		name, own, config string
		configured        bool
	}{
		{"empty override, configured key", tok + InstanceKeyVar + "=\n", InstanceKeyVar + "=from-config\n", true},
		{"no override, configured key", tok, InstanceKeyVar + "=from-config\n", true},
		{"empty override, empty configuration", tok + InstanceKeyVar + "=\n", InstanceKeyVar + "=\n", false},
	} {
		t.Run(tc.name, func(t *testing.T) {
			b := testExecBackend(t)
			host := "shop.example.com"
			own := filepath.Join(b.DataDir, host+".env")
			if err := os.WriteFile(own, []byte(tc.own), 0o600); err != nil {
				t.Fatal(err)
			}
			if err := os.WriteFile(filepath.Join(b.DataDir, host+".config.env"), []byte(tc.config), 0o600); err != nil {
				t.Fatal(err)
			}
			child, _ := startRecorded(t, b, host)
			if child == nil {
				t.Fatal("no child ran")
			}
			raw, _ := os.ReadFile(own)
			if tc.configured {
				if child[InstanceKeyVar] != "from-config" {
					t.Fatalf("the child was handed %q, want the configured key", child[InstanceKeyVar])
				}
				if string(raw) != tc.own {
					t.Fatal("a key was minted over the configured one")
				}
				return
			}
			env, _ := readEnvFile(own)
			if !wellFormedInstanceKey(child[InstanceKeyVar]) || child[InstanceKeyVar] != env[InstanceKeyVar] {
				t.Fatal("the child was not handed the minted key")
			}
		})
	}
}

// Coverage (Astra round 1): the first process of a fresh flat instance is
// often its seed. The seed mints the key, sees it, and the first start
// hands the instance the same one.
func TestExecBackendSeedMintsTheKeyForAFreshFlatInstance(t *testing.T) {
	b := testExecBackend(t)
	host := "seeded.example.com"
	_, envPath := stageRecordingLiveFullEnv(t, b, host, "v1")
	if err := b.Seed(context.Background(), registry.Route{Host: host, Kind: "instance", Seed: `["seed"]`}); err != nil {
		t.Fatalf("Seed: %v", err)
	}
	seed := map[string]string{}
	for _, line := range strings.Split(waitForFile(t, envPath, time.Second), "\n") {
		if k, v, ok := strings.Cut(line, "="); ok {
			seed[k] = v
		}
	}
	own, _ := readEnvFile(filepath.Join(b.DataDir, host+".env"))
	if !wellFormedInstanceKey(own[InstanceKeyVar]) || seed[InstanceKeyVar] != own[InstanceKeyVar] {
		t.Fatal("the seed did not mint the key and see it")
	}
	if child, _ := startRecorded(t, b, host); child == nil || child[InstanceKeyVar] != own[InstanceKeyVar] {
		t.Fatal("the first start handed the instance a different key from its seed's")
	}
}

// Coverage (Astra round 1): a hibernation wake restores the database and
// leaves <host>.env alone, so the start after the restore hands the
// instance the key it had.
func TestHibernationRestoreKeepsTheInstanceKey(t *testing.T) {
	b := testExecBackend(t)
	host := "sleepy.example.com"
	if child, _ := startRecorded(t, b, host); child == nil {
		t.Fatal("no child ran")
	}
	envPath := filepath.Join(b.DataDir, host+".env")
	before, _ := os.ReadFile(envPath)
	key := parseEnvBytes(before)[InstanceKeyVar]
	if !wellFormedInstanceKey(key) {
		t.Fatal("the first start minted no key")
	}
	old := restoreReplica
	restoreReplica = func(_ context.Context, dbPath, bucket, prefix, region string) (bool, error) {
		return true, writeMinimalSQLite(dbPath)
	}
	t.Cleanup(func() { restoreReplica = old })
	_ = os.Remove(filepath.Join(b.DataDir, host+".db")) // hibernated: the database is gone
	rt := registry.Route{Host: host, Kind: "instance", Socket: tmpSock(t)}
	if fresh, err := b.Restore(context.Background(), rt); err != nil || fresh {
		t.Fatalf("Restore = fresh %v, %v; want a restored database", fresh, err)
	}
	if after, _ := os.ReadFile(envPath); !bytes.Equal(before, after) {
		t.Fatal("the restore changed <host>.env")
	}
	if child, _ := startRecorded(t, b, host); child == nil || child[InstanceKeyVar] != key {
		t.Fatal("the start after a restore handed the instance a different key")
	}
}

// refuseAfterFirstClassification makes b classify a row as it does today
// the first time it is asked and as layout (fenced or conflict) every time
// after: a launch dispatcher's own check passes, and the launch body's
// pathsFor sees a layout journal or a colliding peer that appeared since.
func refuseAfterFirstClassification(b *ExecBackend, layout Layout) {
	var calls atomic.Int32
	switch layout {
	case LayoutFenced:
		b.LayoutFacts.Journal = func(string) bool { return calls.Add(1) > 1 }
	case LayoutConflict:
		b.LayoutFacts.SameID = func(a, b string) bool { return true }
		b.LayoutFacts.Peers = func() ([]registry.Route, error) {
			if calls.Add(1) == 1 {
				return nil, nil
			}
			return []registry.Route{{Host: "other.example.com", Kind: "instance", Hibernate: true, Socket: "/nonexistent/other.sock"}}, nil
		}
	}
}

// Review (Task 2): the mint refuses a fenced or conflicting row on the
// paths the launch body hands it, not only on the dispatcher's earlier
// classification. A row that becomes fenced or conflicting between Start's
// or Seed's check and the body's own pathsFor fails the launch with the
// key's error, gets no key, and runs no child.
func TestLaunchBodiesMintNoKeyForARowRefusedSinceTheDispatch(t *testing.T) {
	for _, layout := range []Layout{LayoutFenced, LayoutConflict} {
		for _, launch := range []string{"start", "seed"} {
			t.Run(string(layout)+"/"+launch, func(t *testing.T) {
				b := testExecBackend(t)
				host := "shop.example.com"
				_, envPath := stageRecordingLiveFullEnv(t, b, host, "v1")
				refuseAfterFirstClassification(b, layout)
				rt := registry.Route{Host: host, Kind: "instance", Socket: tmpSock(t), Seed: `["seed"]`}
				ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
				defer cancel()
				var err error
				if launch == "start" {
					var proc Proc
					proc, _, err = b.Start(ctx, rt)
					if proc != nil {
						defer proc.Stop(context.Background())
					}
				} else {
					err = b.Seed(ctx, rt)
				}
				if err == nil || !strings.Contains(err.Error(), "instance key for "+host) || !strings.Contains(err.Error(), "not minted") {
					t.Fatalf("%s of a row %s since the dispatch: err = %v, want the key's refusal", launch, layout, err)
				}
				// The host is named once, by the wrapper. (A fenced reason
				// also names it inside the journal path, so the count is
				// of the "<host>: " prefix.)
				if n := strings.Count(err.Error(), host+": "); n != 1 {
					t.Fatalf("the refusal names the host %d times: %v", n, err)
				}
				raw, _ := os.ReadFile(filepath.Join(b.DataDir, host+".env"))
				if n := len(keyLines(raw)); n != 0 {
					t.Fatalf("a %s row was minted %d key(s)", layout, n)
				}
				if _, serr := os.Stat(envPath); !errors.Is(serr, os.ErrNotExist) {
					t.Fatalf("a child ran for a %s row (stat %v)", layout, serr)
				}
			})
		}
	}
}

// The refusal itself, for every layout pathsFor can report it on: a flat
// row and a nested one, each fenced and conflicting. Nothing is written in
// the data directory or the instance directory. (A nested row's prepare
// cannot reach the mint in a test once fenced: its fenced paths name
// /run/carlos, which prepare creates first. prepareAt hands the mint its
// paths unchanged, so this is the refusal it gets.)
func TestEnsureInstanceKeyForRefusesFencedAndConflictRows(t *testing.T) {
	for _, nested := range []bool{false, true} {
		for _, layout := range []Layout{LayoutFenced, LayoutConflict} {
			name := "flat"
			if nested {
				name = "nested"
			}
			t.Run(name+"/"+string(layout), func(t *testing.T) {
				var b *ExecBackend
				var rt registry.Route
				if nested {
					b, rt = nestedBackend(t, &fakeControl{inv: "inv1"}, &fakeUnit{})
				} else {
					b = testExecBackend(t)
					rt = registry.Route{Host: "shop.example.com", Kind: "instance", Hibernate: true,
						DBPath: provision.DefaultDBPath(b.DataDir, "shop.example.com"), Socket: filepath.Join(provision.SocketDir, "shop.example.com.sock")}
				}
				if l, _ := b.Layout(rt); (nested && l != LayoutNested) || (!nested && l != LayoutFlat) {
					t.Fatalf("fixture classifies as %s", l)
				}
				refuseAfterFirstClassification(b, layout)
				_, _ = b.Layout(rt) // the dispatcher's check
				p := b.pathsFor(rt)
				if p.layout != layout {
					t.Fatalf("paths classify as %s, want %s", p.layout, layout)
				}
				if _, err := b.ensureInstanceKeyFor(t.Context(), rt, p); err == nil || !strings.Contains(err.Error(), "not minted") {
					t.Fatalf("ensure for a %s row: err = %v, want a refusal", layout, err)
				} else if strings.HasPrefix(err.Error(), rt.Host) {
					t.Fatalf("the refusal names the host itself (%v); its callers do", err)
				}
				for _, f := range []string{
					filepath.Join(b.DataDir, rt.Host+".env"),
					filepath.Join(provision.InstanceDir(b.DataDir, rt.Host), rt.Host+".env"),
				} {
					if _, err := os.Lstat(f); !errors.Is(err, os.ErrNotExist) {
						t.Fatalf("%s exists after a refused ensure (lstat %v)", f, err)
					}
				}
			})
		}
	}
}

// The backfill refuses the same rows (instancekey.go BackfillInstanceKey):
// an existing env file without a key is left byte for byte.
func TestBackfillInstanceKeyRefusesFencedAndConflictRows(t *testing.T) {
	for _, layout := range []Layout{LayoutFenced, LayoutConflict} {
		t.Run(string(layout), func(t *testing.T) {
			b := testExecBackend(t)
			host := "shop.example.com"
			rt := registry.Route{Host: host, Kind: "instance", Hibernate: true,
				DBPath: provision.DefaultDBPath(b.DataDir, host), Socket: filepath.Join(provision.SocketDir, host+".sock")}
			own := filepath.Join(b.DataDir, host+".env")
			tok := "CARLOS_ADMIN_TOKEN=" + strings.Repeat("d", 64) + "\n"
			if err := os.WriteFile(own, []byte(tok), 0o600); err != nil {
				t.Fatal(err)
			}
			refuseAfterFirstClassification(b, layout)
			_, _ = b.Layout(rt)
			if _, err := b.BackfillInstanceKey(t.Context(), rt); err == nil || !strings.Contains(err.Error(), "not backfilled") {
				t.Fatalf("backfill of a %s row: err = %v, want a refusal", layout, err)
			} else if strings.HasPrefix(err.Error(), host) {
				t.Fatalf("the refusal names the host itself (%v); the agent's backfill does", err)
			}
			if raw, _ := os.ReadFile(own); string(raw) != tok {
				t.Fatalf("the backfill changed a %s row's env file", layout)
			}
		})
	}
}

// v0.26.1 hotfix: the agent's backfill now runs beside the activator
// rather than before it, so a backfill and a start's own mint can reach
// the same host at once. Both read the files and ask a slow bucket outside
// adminTokenMu, and both are held there until each has done so, so each
// decides from a first read that found no key. They still decide and
// append under the lock, create-only, so the host ends with exactly one
// key, appended whole after the bytes that were there, and exactly one of
// the two reports it minted.
func TestBackfillAndAStartsMintRacingOnOneHostWriteOneKey(t *testing.T) {
	const rounds = 40
	tok := "CARLOS_ADMIN_TOKEN=" + strings.Repeat("b", 64) + "\n"
	for i := range rounds {
		b := &ExecBackend{DataDir: t.TempDir()}
		b.ConfiguredKey = func(ctx context.Context, _ registry.Route) (string, error) {
			select {
			case <-time.After(5 * time.Millisecond): // a slow bucket that sets no key
				return "", nil
			case <-ctx.Done():
				return "", ctx.Err()
			}
		}
		rt := registry.Route{Host: "shop.example.com", Kind: "instance", Status: "ready", Socket: "/run/carlos/shop.sock"}
		own := filepath.Join(b.DataDir, rt.Host+".env")
		if err := os.WriteFile(own, []byte(tok), 0o600); err != nil {
			t.Fatal(err)
		}
		// Both callers pass phase one before either takes the lock.
		var phase sync.WaitGroup
		phase.Add(2)
		instanceKeyHooks.betweenPhases = func(string) { phase.Done(); phase.Wait() }
		var wg sync.WaitGroup
		var backfilled, started InstanceKeyOutcome
		var berr, serr error
		wg.Add(2)
		go func() {
			defer wg.Done()
			backfilled, berr = b.BackfillInstanceKey(t.Context(), rt)
		}()
		go func() {
			defer wg.Done()
			started, serr = b.ensureInstanceKeyFor(t.Context(), rt, b.pathsFor(rt))
		}()
		wg.Wait()
		instanceKeyHooks.betweenPhases = nil
		if berr != nil || serr != nil {
			t.Fatalf("round %d: backfill %v, %v; start %v, %v", i, backfilled, berr, started, serr)
		}
		minted := 0
		for _, o := range []InstanceKeyOutcome{backfilled, started} {
			switch o {
			case InstanceKeyMinted:
				minted++
			case InstanceKeyPresent:
			default:
				t.Fatalf("round %d: outcome %v, want minted or present", i, o)
			}
		}
		raw, _ := os.ReadFile(own)
		keys := keyLines(raw)
		if minted != 1 || len(keys) != 1 || !wellFormedInstanceKey(keys[0]) || string(raw) != tok+InstanceKeyVar+"="+keys[0]+"\n" {
			t.Fatalf("round %d: %d minted, %d keys, file %d bytes; want one whole key appended once", i, minted, len(keys), len(raw))
		}
	}
}