//go:build linux

package host

import (
	"bytes"
	"context"
	"errors"
	"net/http/httptest"
	"os"
	"path/filepath"
	"reflect"
	"strings"
	"sync/atomic"
	"syscall"
	"testing"
	"time"

	"github.com/carlosframework/platform/internal/activator"
	"github.com/carlosframework/platform/internal/cgroup"
	"github.com/carlosframework/platform/internal/genselect"
	"github.com/carlosframework/platform/internal/hostbox"
	"github.com/carlosframework/platform/internal/hostclaim"
	"github.com/carlosframework/platform/internal/hostguard"
	"github.com/carlosframework/platform/internal/hostwrite"
	"github.com/carlosframework/platform/internal/lease"
	"github.com/carlosframework/platform/internal/lease/leasetest"
	"github.com/carlosframework/platform/internal/provision"
	"github.com/carlosframework/platform/internal/registry"
	"github.com/superfly/ltx"
	"golang.org/x/sys/unix"
)

// This fixture ends at Restore: a legacy holder has no C8 evidence yet.
// It uses the real selector, lease against local S3, retry wrapper, admission,
// activation, prelaunch journal, selected Work, stop coordinator and retirement
// transport. Initial installation, kernel checks and launch preflight are
// explicit seams; it does not claim an installed-unit or privileged witness.
type servingPreparation struct {
	assembly      ClaimAssembly
	source        hostbox.WorkSource
	writer        hostbox.Writer
	f             *servingRootFixture
	owners        *claimServingOwners
	wake          *ClaimWakeAssembly
	runtime       *activator.ClaimRuntime
	local         claimStopLocal
	events        []string
	baseline      *genselect.Plan
	enrollFailure error
	afterEnroll   func()
}

func newServingPreparation(t *testing.T, f *servingRootFixture, boot, controller string, configure ...func(*ClaimWakeAssembly)) *servingPreparation {
	t.Helper()
	if boot == "boot" {
		var err error
		boot, err = f.p.ledger.fs.(*servingParkFS).currentBoot()
		if err != nil {
			t.Fatal(err)
		}
	}
	p := &servingPreparation{f: f, local: claimStopLocal{Actor: f.actor, BootID: boot, ControllerID: controller, PID: 77, ProcessStart: controller + "-start"}}
	f.f.peer = StopPeer{UID: 1001, PID: p.local.PID, ProcessStart: p.local.ProcessStart, BootID: boot, ControllerID: controller}
	launchFixture := plan9D5Fixture(t)
	launchFixture.launch.Args = []string{"--socket", provision.NestedSocket(f.f.hostName()), "--db", f.db}
	launchFixture.launch.Limit = cgroup.TierDefault
	launchFixture.launch.Env = []string{"CARLOS_ADMIN_TOKEN=test"}
	sources := launchFixture.owners.Sources
	p.owners = &claimServingOwners{Data: f.f.data, Bucket: f.f.bucket, Region: f.actor.Region, Box: f.actor.Box, Sources: sources, Now: func() time.Time { return f.f.now }, NewID: f.f.newID}
	producers := &ClaimProducerSet{}
	w := &ClaimWakeAssembly{LeaseRegion: "eu-west-1", Manager: new(cgroup.Manager), Inputs: p.owners, RequireServingEnrollment: true}
	p.wake = w
	root := servingHandoverRoot(t, f)
	hook := coldAdmissionHook(f.f.guard, handoverRecoveryIdentity(t, f.actor), servingRootSocket(t, root))
	w.AfterAdmission = func(ctx context.Context, prior, admitted hostclaim.Observed) error {
		p.events = append(p.events, "admit")
		err := hook(ctx, prior, admitted)
		if err != nil {
			old := prior.Claim.Holder
			receipt, receiptErr := f.f.guard.CompletedColdPredecessor(ctx, prior.Host, hostguard.Captured{Incarnation: old.Incarnation, Stamp: old.Stamp}, old.Session, prior.Claim.Op.ID, prior.Claim.Op.Phase)
			inventory, invErr := root.Reconcile.Inventory.ReadInstallation(ctx, prior.Host, old.Incarnation, old.Stamp)
			var st syscall.Stat_t
			statErr := syscall.Stat(filepath.Dir(f.db), &st)
			t.Logf("handover diagnosis prior=%+v next=%+v receipt=%+v receiptErr=%v inventory=%+v invErr=%v liveDev=%d liveIno=%d statErr=%v", old, admitted.Claim.Holder, receipt.Target, receiptErr, inventory, invErr, st.Dev, st.Ino, statErr)
		}
		return err
	}
	w.Prelaunch = func(ctx context.Context, o hostclaim.Observed, work hostwrite.Work, in ClaimLaunchInputs) (*activator.RoleLaunch, error) {
		p.events = append(p.events, "prelaunch")
		target := recoveryTuple(o, p.local)
		err := work.With(ctx, in.Identity, func() error {
			return f.f.journal.Prelaunch(ctx, hostbox.LaunchInventory{Target: target, ControllerPID: p.local.PID, ProcessStart: p.local.ProcessStart, NoState: true, Groups: []hostbox.LaunchGroup{{RelativePath: "carlos/" + claimContainmentName(target), Device: 1, Inode: target.Session}}})
		})
		return &activator.RoleLaunch{Target: target, Journal: f.f.journal}, err
	}
	roles := &ClaimRoleAssembly{Inventory: f.p.root.Journals.Inventory, ManagedRoots: map[string]string{"data": f.f.guard.DataDir}, Identity: handoverRecoveryIdentity(t, f.actor)}
	w.Roles = roles
	roles.Root = fakeLaunchRoot{protocol: hostbox.RoleLaunchProtocol}
	w.PrepareState = func(ctx context.Context, o hostclaim.Observed, work hostwrite.Work, in ClaimLaunchInputs) error {
		p.events = append(p.events, "state")
		return roles.PrepareState(ctx, o, work, in)
	}
	w.Launch = func(context.Context, hostclaim.Observed, hostwrite.Work, ClaimLaunchInputs, *activator.RoleLaunch) (*activator.RoleLaunch, error) {
		t.Fatal("tenant launch in restore witness")
		return nil, hostbox.ErrUnsupported
	}
	w.Producers = func(_ context.Context, o hostclaim.Observed) (*ClaimProducers, error) {
		return producers.Ensure(recoveryTuple(o, p.local), p.local.ProcessStart, f.f.journal.RecordProducer)
	}
	w.Retain = func(hostclaim.Observed, activator.Proc, activator.Rep, error) {}
	w.ObserveLoss = func(hostclaim.Observed, error) {}
	w.Retries = activator.ServingRetries{Claims: f.f.claims, Box: f.actor.Box, Enrollment: f.actor.Enrollment, BootID: boot, ControllerID: controller, Now: func() time.Time { return f.f.now }, NewID: f.f.newID, Inputs: p.owners.launchDigest}
	a := ClaimAssembly{Claims: f.f.claims, Launches: f.f.journal, Wake: w, Producers: producers, stopLocal: func(context.Context) (claimStopLocal, error) { return p.local, nil }}
	a.collectStop = func(ctx context.Context, req StopRequest, drain StopDrainer) (StopResult, error) {
		f.f.requests = append(f.f.requests, req)
		coordinator := f.p.root.Stop.Coordinator
		coordinator.Drainer = drain
		coordinator.RetainNoProcess = f.p.retainLegacyNoProcess
		if req.WriterSession != 0 {
			coordinator.Kernel = f.f.kernel
		}
		result, err := coordinator.FenceAndStop(ctx, req)
		if err != nil {
			inventory, inventoryErr := coordinator.Inventory.ReadStopInventory(ctx, req.Host)
			t.Logf("stop diagnosis request=%+v inventory=%+v inventoryErr=%v stopErr=%v", req, inventory, inventoryErr, err)
			return result, err
		}
		return validateTransportStop(req, result)
	}
	a.bindClaimWakeStop()
	retirement := &claimServingArtifactRetirement{Claims: f.f.claims, Guard: f.f.guard, DataDir: f.f.guard.DataDir, Box: f.actor.Box, Region: f.actor.Region, Root: hostbox.ServingParkClient{Call: servingParkSocket(t, servingTransportRoot(t, f))}}
	w.RetireArtifacts = retirement.Retire
	w.RecoverArtifacts = func(ctx context.Context, o hostclaim.Observed, in ClaimLaunchInputs) error {
		p.events = append(p.events, "recover")
		return retirement.Recover(ctx, o, in)
	}
	w.BeforeLegacyInstall = retirement.BeforeLegacyInstall
	w.BeforeInstallLocked = func(ctx context.Context, o hostclaim.Observed, in ClaimLaunchInputs) error {
		if err := retirement.EnrollLocked(ctx, o, in); err != nil {
			return err
		}
		p.events = append(p.events, "enrolled")
		if p.afterEnroll != nil {
			p.afterEnroll()
		}
		return p.enrollFailure
	}
	source := hostbox.WorkSource{Guard: f.f.guard, Box: f.actor.Box, Region: f.actor.Region, Now: func() time.Time { return f.f.now }, Policy: func(context.Context, *hostclaim.Claim, hostbox.Selection, hostwrite.Identity) error { return nil }, Local: func(context.Context, *hostclaim.Claim, hostbox.Selection) error { return nil }}
	writer := hostbox.Writer{Data: f.f.data, Bucket: f.f.bucket, Claims: f.f.claims, Box: f.actor.Box, Region: f.actor.Region, AdmissionEvidence: p.owners.admissionEvidence}
	var err error
	for _, fn := range configure {
		fn(w)
	}
	p.assembly, p.source, p.writer = a, source, writer
	p.runtime, err = w.Build(t.Context(), source, writer)
	if err != nil {
		t.Fatal(err)
	}
	t.Cleanup(func() {
		ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
		defer cancel()
		if err := w.Close(ctx); err != nil {
			t.Error(err)
		}
	})
	return p
}

func (p *servingPreparation) prepare(t *testing.T) (hostclaim.Observed, activator.ClaimStart) {
	t.Helper()
	f := p.f
	rt := registry.Route{Host: f.f.hostName(), DBPath: f.db, Socket: provision.NestedSocket(f.f.hostName())}
	o := f.f.read()
	// Prerequisites' isolated launch preflight requires an installed /data
	// namespace. Drive its selection, stop and real lease in the same order;
	// Admit and Prepare below are the unmodified production runtime closures.
	var err error
	if f.f.guard.DataDir == "/data" {
		if err = p.runtime.Prerequisites(t.Context(), rt, o, recoveryPolicy); err != nil {
			t.Fatal("prerequisites", err)
		}
		p.events = append(p.events, "prerequisites")
		pending := p.wake.get(rt.Host)
		if pending == nil || pending.held == nil {
			t.Fatal("prerequisites returned without a lease")
		}
		_, selected := pending.snapshot()
		p.baseline = selected.Baseline
	} else {
		selected, err := p.owners.Select(t.Context(), rt, o, recoveryPolicy)
		if err != nil {
			t.Fatal(err)
		}
		p.events = append(p.events, "select")
		if selected.Empty || selected.Baseline == nil {
			t.Fatal("baseline restore replaced by empty install")
		}
		p.baseline = selected.Baseline
		if err = p.wake.stopPredecessor(t.Context(), o); err != nil {
			t.Fatal("stop predecessor", err)
		}
		p.events = append(p.events, "stop")
		held, err := lease.Acquire(t.Context(), f.f.bucket, o.Claim.Holder.Location, p.wake.LeaseRegion)
		if err != nil {
			t.Fatal(err)
		}
		p.events = append(p.events, "lease")
		live, cancel := context.WithCancel(t.Context())
		done := make(chan struct{})
		go func() { <-live.Done(); close(done) }()
		p.wake.pending[rt.Host] = &claimWakeAttempt{owner: p.wake, observed: o, inputs: selected, held: held, ctx: live, cancel: cancel, done: done}
	}
	admitted, err := p.runtime.Admit(t.Context(), rt, f.f.read(), recoveryPolicy)
	if err != nil {
		t.Fatal("admit", err)
	}
	r := admitted.Claim.Holder.WakeRetry
	if r == nil || r.Attempt == nil || r.Attempt.Session != admitted.Claim.Holder.Session || r.Attempt.ControllerID != p.local.ControllerID {
		t.Fatalf("retry not bound: %+v", r)
	}
	start, err := p.runtime.Prepare(t.Context(), rt, admitted, recoveryPolicy)
	if err != nil {
		t.Fatal("prepare", err)
	}
	return admitted, start
}

func TestFilesSeedOrdinaryPreparationRestore(t *testing.T) {
	for _, failure := range []string{"", "enrolled-refusal", "missing-marker", "cancel-after-enrollment", "disk-after-enrollment", "invalid-sqlite", "cancel-after-output", "restore-checksum"} {
		for _, restart := range []string{"same-controller", "controller", "reboot"} {
			t.Run(failure+"/"+restart, func(t *testing.T) { runFilesSeedOrdinaryPreparation(t, failure, restart) })
		}
	}
}

func runFilesSeedOrdinaryPreparation(t *testing.T, failure, restart string, mounted ...bool) {
	runFilesSeedPreparationWithLive(t, failure, restart, mounted, nil)
}
func runFilesSeedPreparationWithLive(t *testing.T, failure, restart string, mounted []bool, live []filesSeedLiveWitness) {
	if failure == "real-seed" {
		t.Setenv("CARLOS_HOST_CLAIM", "1")
		t.Setenv("CARLOS_HOST_CLAIM_ORIGINS", "1")
	}
	t.Setenv("CARLOS_HOST_CLAIM", "1")
	t.Setenv("CARLOS_HOST_CLAIM_ORIGINS", "1")
	t.Setenv("CARLOS_MOVES_PHASE1", "")
	var f *servingRootFixture
	if len(mounted) > 0 && mounted[0] {
		shared := false
		for _, witness := range live {
			shared = shared || witness.shared
		}
		if !shared {
			t.Cleanup(func() {
				if err := os.RemoveAll("/data"); err != nil {
					t.Error(err)
				}
			})
			t.Cleanup(func() {
				if err := os.RemoveAll("/var/lib/carlos-claims"); err != nil {
					t.Error(err)
				}
			})
		}
		if failure == "restore-enospc" {
			if err := os.Mkdir("/data", 0700); err != nil {
				t.Fatal(err)
			}
			if err := unix.Mount("tmpfs", "/data", "tmpfs", unix.MS_NOSUID|unix.MS_NODEV, "size=2g"); err != nil {
				t.Fatal(err)
			}
			t.Cleanup(func() {
				if err := unix.Unmount("/data", 0); err != nil {
					t.Error(err)
				}
			})
		}
		state, err := openClaimServingParkAuthority()
		if err != nil {
			t.Fatal("production private store", err)
		}
		t.Cleanup(func() {
			if err := state.Close(); err != nil {
				t.Error(err)
			}
		})
		host := recoveryHost
		for _, witness := range live {
			if witness.host != "" {
				host = witness.host
			}
		}
		f, _, _ = newServingLegacyRootSetup(t, false, "/data", host)
		if !shared {
			instances := filepath.Join(f.f.guard.DataDir, "instances")
			if err := unix.Mount(instances, instances, "", unix.MS_BIND, ""); err != nil {
				t.Fatal(err)
			}
			t.Cleanup(func() {
				if err := unix.Unmount(instances, 0); err != nil {
					t.Error(err)
				}
			})
			if err := unix.Mount("", instances, "", unix.MS_BIND|unix.MS_REMOUNT|unix.MS_NOSUID|unix.MS_NODEV|unix.MS_NOEXEC, ""); err != nil {
				t.Fatal(err)
			}
			if parent, _, err := newServingParkFS(f.f.guard.DataDir, state).parent(); err == nil {
				parent.Close()
				t.Fatal("production filesystem accepted a mount without nosymfollow")
			}
			if err := unix.Mount("", instances, "", unix.MS_BIND|unix.MS_REMOUNT|unix.MS_NOSYMFOLLOW|unix.MS_NOSUID|unix.MS_NODEV|unix.MS_NOEXEC, ""); err != nil {
				t.Fatal(err)
			}
			link := filepath.Join(instances, ".nosymfollow-witness")
			if err := os.Symlink(f.db, link); err != nil {
				t.Fatal(err)
			}
			fd, err := unix.Open(link, unix.O_RDONLY|unix.O_CLOEXEC, 0)
			if fd >= 0 {
				unix.Close(fd)
			}
			if !errors.Is(err, syscall.ELOOP) {
				t.Fatalf("kernel followed symlink on nosymfollow mount: %v", err)
			}
			if err := os.Remove(link); err != nil {
				t.Fatal(err)
			}
		}
		// Use the production UID and mount checks in this isolated namespace.
		f.p.ledger.state = state
		f.p.ledger.fs = newServingParkFS(f.f.guard.DataDir, state)
		source, err := os.Open(filepath.Dir(f.db))
		if err != nil {
			t.Fatal(err)
		}
		sourceID, err := servingParkDirID(source)
		source.Close()
		if err != nil {
			t.Fatal(err)
		}
		outside, err := os.Open(f.f.guard.DataDir)
		if err != nil {
			t.Fatal(err)
		}
		outsideID, err := servingParkDirID(outside)
		outside.Close()
		if err != nil {
			t.Fatal(err)
		}
		if sourceID.Device != outsideID.Device || sourceID.Mount == outsideID.Mount {
			t.Fatal("fixture did not create same-device cross-mount boundary")
		}
		if err := os.Rename(f.db, filepath.Join(f.f.guard.DataDir, "old-flat-park")); !errors.Is(err, syscall.EXDEV) {
			t.Fatalf("old destination must fail with EXDEV: %v", err)
		}
	} else {
		f, _, _ = newServingLegacyRootSetup(t, false)
	}
	extraBytes := 0
	if failure == "cancel-after-output" {
		extraBytes = 2 << 20
	}
	selected := plan12RestoreBaseline(t, f.f, extraBytes)
	raw, err := f.f.data.GetVersion(t.Context(), f.f.bucket, selected.Files[0].Key, selected.Files[0].VersionID)
	if err != nil {
		t.Fatal(err)
	}
	bucket := f.f.data.(*leasetest.VersionedBucket)
	key := strings.Replace(selected.Files[0].Key, "/g-src/", "/", 1)
	bucket.Put(f.f.bucket, key, []byte("older invalid snapshot"))
	version := bucket.Put(f.f.bucket, key, raw)
	// Keep a valid LTX envelope so failures occur inside the production restore,
	// after legacy retirement, rather than at selected-object decoding.
	if failure == "invalid-sqlite" {
		var spec ltx.FileSpec
		if _, err := spec.ReadFrom(bytes.NewReader(raw)); err != nil {
			t.Fatal(err)
		}
		copy(spec.Pages[0].Data, []byte("not a sqlite db!"))
		var encoded bytes.Buffer
		if _, err := spec.WriteTo(&encoded); err != nil {
			t.Fatal(err)
		}
		version = bucket.Put(f.f.bucket, key, encoded.Bytes())
	}
	var deltaKey, deltaVersion string
	var deltaGood []byte
	if failure == "restore-checksum" {
		var spec ltx.FileSpec
		if _, err := spec.ReadFrom(bytes.NewReader(raw)); err != nil {
			t.Fatal(err)
		}
		checksum := ltx.ChecksumFlag
		for _, page := range spec.Pages {
			checksum ^= ltx.ChecksumPage(page.Header.Pgno, page.Data)
		}
		checksum |= ltx.ChecksumFlag
		// Delta trailers cannot validate the resulting database without applying
		// the snapshot. Keep the envelope valid and fail only the final check.
		spec.Header.Flags = 0
		spec.Trailer.PostApplyChecksum = checksum
		var snapshot bytes.Buffer
		if _, err := spec.WriteTo(&snapshot); err != nil {
			t.Fatal(err)
		}
		raw = append([]byte(nil), snapshot.Bytes()...)
		version = bucket.Put(f.f.bucket, key, raw)
		spec.Header.MinTXID, spec.Header.MaxTXID = 2, 2
		spec.Header.PreApplyChecksum = checksum
		var encoded bytes.Buffer
		if _, err := spec.WriteTo(&encoded); err != nil {
			t.Fatal(err)
		}
		deltaGood = append([]byte(nil), encoded.Bytes()...)
		encoded.Reset()
		spec.Trailer.PostApplyChecksum ^= 1
		if _, err := spec.WriteTo(&encoded); err != nil {
			t.Fatal(err)
		}
		deltaKey = filepath.Dir(filepath.Dir(key)) + "/0000/" + ltx.FormatFilename(2, 2)
		deltaVersion = bucket.Put(f.f.bucket, deltaKey, encoded.Bytes())
	}
	handler, _ := leasetest.NewS3Server(bucket)
	server := httptest.NewServer(handler)
	t.Cleanup(server.Close)
	for key, value := range map[string]string{"AWS_ACCESS_KEY_ID": "test-key", "AWS_SECRET_ACCESS_KEY": "test-secret", "AWS_SESSION_TOKEN": "", "AWS_EC2_METADATA_DISABLED": "true", "AWS_CONFIG_FILE": filepath.Join(t.TempDir(), "absent"), "AWS_SHARED_CREDENTIALS_FILE": filepath.Join(t.TempDir(), "absent"), "CARLOS_S3_ENDPOINT": server.URL} {
		t.Setenv(key, value)
	}
	sentinels := map[string]string{filepath.Join(filepath.Dir(f.db), "files", "payload"): "live files", filepath.Join(filepath.Dir(f.db), ".env"): "environment"}
	for path, body := range sentinels {
		if err = os.MkdirAll(filepath.Dir(path), 0700); err != nil {
			t.Fatal(err)
		}
		if err = os.WriteFile(path, []byte(body), 0600); err != nil {
			t.Fatal(err)
		}
	}
	if failure == "real-seed" {
		f.actor.Bucket = f.f.bucket
		f.actor.Cloud.Region = "eu-west-1"
		f.actor.HomeRegion = "ie"
		f.actor.FleetHomeRegion = "ie"
	}
	p := newServingPreparation(t, f, "boot", "controller")
	if failure == "real-seed" {
		runFilesSeedRealPreparation(t, p, live...)
		return
	}
	var legacy servingParkRecord
	var baseline *genselect.Plan
	for i := 0; i < 2; i++ {
		if i == 1 {
			if failure == "invalid-sqlite" {
				version = bucket.Put(f.f.bucket, key, raw)
			}
			if failure == "restore-checksum" {
				deltaVersion = bucket.Put(f.f.bucket, deltaKey, deltaGood)
			}
			f.f.now = f.f.now.Add(time.Minute)
		}
		p.events = nil
		admitted, start := p.prepare(t)
		wantFiles := 1
		if failure == "restore-checksum" {
			wantFiles = 2
		}
		if len(p.baseline.Files) != wantFiles || p.baseline.Files[0].Key != key || p.baseline.Files[0].VersionID != version {
			t.Fatalf("wrong exact baseline: %+v", p.baseline)
		}
		if failure == "restore-checksum" && (p.baseline.Files[1].Key != deltaKey || p.baseline.Files[1].VersionID != deltaVersion) {
			t.Fatalf("wrong exact delta: %+v", p.baseline)
		}
		if baseline != nil && failure != "restore-checksum" && failure != "invalid-sqlite" && !reflect.DeepEqual(baseline, p.baseline) {
			t.Fatal("baseline changed")
		}
		baseline = p.baseline
		p.enrollFailure = nil
		cancelCtx, cancel := context.WithCancel(t.Context())
		var restoreCtx context.Context = cancelCtx
		var outputCancel *servingRestoreOutputCancel
		if i == 0 && failure == "cancel-after-output" {
			outputCancel = &servingRestoreOutputCancel{Context: cancelCtx, cancel: cancel, directory: filepath.Dir(f.db)}
			restoreCtx = outputCancel
		}
		p.afterEnroll = nil
		if i == 0 {
			switch failure {
			case "enrolled-refusal":
				p.enrollFailure = os.ErrPermission
			case "disk-after-enrollment":
				p.enrollFailure = syscall.ENOSPC
			case "cancel-after-enrollment":
				p.afterEnroll = cancel
			case "restore-enospc":
				p.afterEnroll = func() {
					if err := unix.Mount("tmpfs", "/data", "tmpfs", unix.MS_REMOUNT|unix.MS_NOSUID|unix.MS_NODEV, "size=1m"); err != nil {
						t.Fatal(err)
					}
					err := os.WriteFile(filepath.Join(filepath.Dir(f.db), ".disk-full-witness"), make([]byte, 1<<20), 0600)
					if !errors.Is(err, syscall.ENOSPC) {
						t.Fatalf("small private tmpfs did not fill: %v", err)
					}
				}
			}
		}
		restoreErr := start.Restore(restoreCtx)
		if i == 0 && failure == "restore-enospc" {
			if !errors.Is(restoreErr, syscall.ENOSPC) || !strings.Contains(restoreErr.Error(), "restore selected generation") {
				t.Fatalf("not a production restore ENOSPC: %v", restoreErr)
			}
			if err := os.Remove(filepath.Join(filepath.Dir(f.db), ".disk-full-witness")); err != nil {
				t.Fatal(err)
			}
			if err := unix.Mount("tmpfs", "/data", "tmpfs", unix.MS_REMOUNT|unix.MS_NOSUID|unix.MS_NODEV, "size=2g"); err != nil {
				t.Fatal(err)
			}
		}
		if outputCancel != nil && !outputCancel.observed.Load() {
			t.Fatalf("cancellation never observed real restore output: %v", restoreErr)
		}
		cancel()
		failed := i == 0 && failure != "" && failure != "missing-marker"
		if failed {
			if restoreErr == nil {
				t.Fatal("missed restore failure")
			}
			if p.enrollFailure != nil && !errors.Is(restoreErr, p.enrollFailure) {
				t.Fatalf("wrong failure: %v", restoreErr)
			}
			if (failure == "cancel-after-enrollment" || failure == "cancel-after-output") && !errors.Is(restoreErr, context.Canceled) {
				t.Fatalf("wrong cancellation: %v", restoreErr)
			}
			if failure == "invalid-sqlite" && !strings.Contains(restoreErr.Error(), "check restored database") {
				t.Fatalf("failure before database check: %v", restoreErr)
			}
			if failure == "restore-checksum" && !strings.Contains(restoreErr.Error(), "restored database checksum") {
				t.Fatalf("failure before restored checksum: %v", restoreErr)
			}
			if _, err = os.Stat(f.db); !os.IsNotExist(err) {
				t.Fatalf("failed restore left database: %v", err)
			}
			temps, err := filepath.Glob(filepath.Join(filepath.Dir(f.db), ".restore-*"))
			if err != nil || len(temps) != 0 {
				t.Fatalf("restore scratch survived: %v %v", temps, err)
			}
		} else {
			if restoreErr != nil {
				t.Fatal("restore", restoreErr)
			}
			if got := mwReadDB(t, f.db); got != "selected recovery baseline" {
				t.Fatal(got)
			}
			marker, err := hostbox.ReadGenerationMarker(f.db + ".gen")
			if err != nil || marker != admitted.Claim.Holder.Generation {
				t.Fatalf("marker %q %v", marker, err)
			}
			if i == 0 && failure == "missing-marker" {
				if err = os.Remove(f.db + ".gen"); err != nil {
					t.Fatal(err)
				}
			}
		}
		want := []string{"select", "stop", "lease", "admit", "prelaunch", "recover", "state", "enrolled"}
		if f.f.guard.DataDir == "/data" {
			want = append([]string{"prerequisites"}, want[3:]...)
		}
		if !reflect.DeepEqual(p.events, want) {
			t.Fatalf("preparation order: %v", p.events)
		}
		current, err := servingWriterIdentity(f.f.read(), f.actor.Box, f.actor.Region)
		if err != nil {
			t.Fatal(err)
		}
		slot, err := f.p.ledger.slot(servingParkAuthorization{Successor: current})
		if err != nil {
			t.Fatal(err)
		}
		record, err := f.p.ledger.read(slot.ID)
		if err != nil || !record.Complete || !record.Authorization.Legacy {
			t.Fatalf("legacy slot: %+v %v", record, err)
		}
		if i == 0 {
			legacy = record
		} else if !reflect.DeepEqual(legacy, record) {
			t.Fatal("never-launched restore displaced legacy park or changed deadline")
		}
		if i == 0 && restart != "same-controller" {
			// Lose only the controller's lease and memory. Leave its retry and
			// prelaunch evidence unsettled for the next controller's stop path.
			attempt := p.wake.get(recoveryHost)
			if err = p.wake.release(t.Context(), recoveryHost, attempt); err != nil {
				t.Fatal(err)
			}
			// This is ordinary retry recovery, not Task 1R's future seed rerun.
			nextBoot := "boot"
			if restart == "reboot" {
				nextBoot = "boot-2"
			}
			next := newServingPreparation(t, f, nextBoot, "controller-2")
			err = next.runtime.Prerequisites(t.Context(), registry.Route{Host: f.f.hostName(), DBPath: f.db}, f.f.read(), recoveryPolicy)
			if !errors.Is(err, hostbox.ErrChanged) {
				t.Fatalf("retry settlement: %v", err)
			}
			if r := f.f.retry(); r.State != "open" || r.Attempt != nil || r.AttemptsReserved != 1 {
				t.Fatalf("settlement refunded/stranded: %+v", r)
			}
			p = next
		} else {
			if err = p.runtime.FailedAdmission(t.Context(), admitted, errors.New("restore-only witness")); err != nil {
				t.Fatal("settlement", err)
			}
		}
	}
	for suffix, want := range map[string]string{"": "legacy bytes", "-wal": "legacy WAL", "-shm": "legacy SHM"} {
		body, err := os.ReadFile(filepath.Join(f.f.guard.DataDir, "instances", ".serving-parked", legacy.Authorization.ID, filepath.Base(f.db)+suffix))
		if err != nil || string(body) != want {
			t.Fatalf("park bytes %q %v", body, err)
		}
	}
	for path, want := range sentinels {
		body, err := os.ReadFile(path)
		if err != nil || string(body) != want {
			t.Fatalf("unrelated bytes: %s %v", path, err)
		}
	}
	after, err := f.f.data.GetVersion(t.Context(), f.f.bucket, key, version)
	if err != nil || string(after) != string(raw) {
		t.Fatal("remote baseline changed", err)
	}
	if r := f.f.retry(); r.State != "open" || r.Attempt != nil || r.AttemptsReserved != 2 {
		t.Fatalf("final retry: %+v", r)
	}
	if restart != "same-controller" && len(f.f.kernel.recovered) == 0 {
		t.Fatal("restart did not recover original launch tuple")
	}
}

func TestFilesSeedLegacyStopRefusesForeignWriterBeforeRoot(t *testing.T) {
	f, _, _ := newServingLegacyRootSetup(t, false)
	o, err := hostclaim.Update(t.Context(), f.f.claims, f.f.read(), func(c *hostclaim.Claim) error { c.Holder.WriterBox = "foreign-box"; return nil })
	if err != nil {
		t.Fatal(err)
	}
	calls := 0
	a := ClaimAssembly{Claims: f.f.claims, collectStop: func(context.Context, StopRequest, StopDrainer) (StopResult, error) { calls++; return StopResult{}, nil }}
	local := claimStopLocal{Actor: f.actor, BootID: "boot", ControllerID: "controller", PID: 77, ProcessStart: "start"}
	before, _ := f.disk.Names()
	_, err = a.collectCapturedStop(t.Context(), o, local, claimLaunchOwner{BootID: local.BootID, ControllerID: local.ControllerID}, false)
	if !errors.Is(err, hostbox.ErrChanged) || calls != 0 {
		t.Fatalf("foreign legacy writer: calls %d error %v", calls, err)
	}
	after, _ := f.disk.Names()
	if !reflect.DeepEqual(before, after) {
		t.Fatal("foreign legacy stop wrote private evidence")
	}
}

func TestServingParkNestedBindMount(t *testing.T) {
	if !plan14RootFixture(t) {
		return
	}
	for _, failure := range []string{"", "enrolled-refusal", "missing-marker", "cancel-after-enrollment", "disk-after-enrollment", "invalid-sqlite", "cancel-after-output", "restore-checksum", "restore-enospc"} {
		for _, restart := range []string{"same-controller", "controller", "reboot"} {
			t.Run(failure+"/"+restart, func(t *testing.T) { runFilesSeedOrdinaryPreparation(t, failure, restart, true) })
		}
	}
}

// Cancel only once the real restore has written a temporary database. This
// context seam injects a fault without replacing Restore or its filesystem.
type servingRestoreOutputCancel struct {
	context.Context
	cancel    context.CancelFunc
	directory string
	observed  atomic.Bool
}

func (c *servingRestoreOutputCancel) Err() error {
	if !c.observed.Load() {
		names, _ := filepath.Glob(filepath.Join(c.directory, ".restore-*"))
		for _, name := range names {
			if !strings.HasSuffix(name, ".db") && !strings.HasSuffix(name, ".db.tmp") {
				continue
			}
			if info, err := os.Stat(name); err == nil && info.Mode().IsRegular() && info.Size() > 0 {
				c.observed.Store(true)
				c.cancel()
				break
			}
		}
	}
	return c.Context.Err()
}

func (c *servingRestoreOutputCancel) Done() <-chan struct{} {
	_ = c.Err()
	return c.Context.Done()
}