//go:build linux && rootrunner

package host

import (
	"bytes"
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"net"
	"net/http"
	"os"
	"os/exec"
	"os/user"
	"path/filepath"
	"reflect"
	"strconv"
	"strings"
	"sync/atomic"
	"syscall"
	"testing"
	"time"

	"github.com/carlosframework/platform/internal/activator"
	"github.com/carlosframework/platform/internal/cgroup"
	"github.com/carlosframework/platform/internal/hostbox"
	"github.com/carlosframework/platform/internal/hostclaim"
	"github.com/carlosframework/platform/internal/hostguard"
	"github.com/carlosframework/platform/internal/instancectl"
	"github.com/carlosframework/platform/internal/registry"
	"github.com/carlosframework/platform/internal/rootstate"
	"golang.org/x/sys/unix"
)

// The control socket coordinates the test only. Every effect request uses the
// production journal client and its separately authenticated Unix connection.
type servingPeerCommand struct {
	Action         string
	Current, Prior servingArtifactIdentity
	Target         hostguard.FenceTarget
	ID             string
	Previous       hostclaim.Observed
	Admitted       hostclaim.Observed
	Actor          hostbox.Actor
	Stop           StopRequest
}

type servingPeerReply struct {
	Peer          StopPeer
	Authorization servingParkAuthorization
	Launch        hostbox.LaunchInventory
	Stopped       StopResult
	Error         string
}

// Invoked only by the disposable edge unit, or its same-UID sibling control.
func TestRootServingParkPeerPayload(t *testing.T) {
	control, socket := os.Getenv("SERVING_PEER_CONTROL"), os.Getenv("SERVING_PEER_SOCKET")
	if control == "" || socket == "" || os.Geteuid() == 0 {
		t.Fatal("invoke through TestRootServingParkAuthenticatedPeer")
	}
	call := rootJournalClient(socket)
	if os.Getenv("SERVING_PEER_SIBLING") == "1" {
		var q servingPeerCommand
		if err := json.NewDecoder(os.Stdin).Decode(&q); err != nil {
			t.Fatal(err)
		}
		if _, err := (hostbox.ServingParkClient{Call: call}).Park(t.Context(), q.Current, q.Prior); err == nil {
			t.Fatal("same-UID sibling was authorized")
		}
		return
	}
	conn, err := net.DialTimeout("unix", control, 5*time.Second)
	if err != nil {
		t.Fatal(err)
	}
	defer conn.Close()
	if err := conn.SetDeadline(time.Now().Add(2 * time.Minute)); err != nil {
		t.Fatal(err)
	}
	start, err := claimConstructionProcessStart()
	if err != nil {
		t.Fatal(err)
	}
	controller, err := (productionStopControllers{}).VerifiedController(t.Context(), os.Getpid())
	if err != nil {
		t.Fatal(err)
	}
	boot, err := os.ReadFile("/proc/sys/kernel/random/boot_id")
	if err != nil {
		t.Fatal(err)
	}
	peer := StopPeer{UID: uint32(os.Geteuid()), PID: os.Getpid(), ProcessStart: start, BootID: strings.TrimSpace(string(boot)), ControllerID: controller}
	encode, decode := json.NewEncoder(conn), json.NewDecoder(conn)
	if err := encode.Encode(servingPeerReply{Peer: peer}); err != nil {
		t.Fatal(err)
	}
	client := hostbox.ServingParkClient{Call: call}
	var manager *cgroup.Manager
	var seedServer *filesSeedServer
	defer func() {
		if seedServer != nil {
			seedServer.close(t.Context())
		}
	}()
	for {
		var q servingPeerCommand
		if err := decode.Decode(&q); errors.Is(err, io.EOF) {
			return
		} else if err != nil {
			t.Fatal(err)
		}
		var reply servingPeerReply
		var callErr error
		switch q.Action {
		case "control-request":
			transport := &http.Transport{DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) {
				return (&net.Dialer{}).DialContext(ctx, "unix", socket)
			}}
			response, err := (&http.Client{Transport: transport, Timeout: 3 * time.Second}).Get("http://root/claim/retire")
			if err != nil {
				callErr = err
			} else {
				response.Body.Close()
				if response.StatusCode != http.StatusMethodNotAllowed {
					callErr = fmt.Errorf("control status %d", response.StatusCode)
				}
			}
			transport.CloseIdleConnections()
		case "seed-listen":
			ready := make(chan struct{})
			close(ready)
			seedServer, callErr = startFilesSeedSocket(t.Context(), ready, filesSeedSocket, func(ctx context.Context, c *net.UnixConn) error {
				return authenticateFilesSeedPeer(ctx, c, os.Geteuid(), false)
			}, func(ctx context.Context, host string, out io.Writer) error {
				_, err := io.WriteString(out, "accepted "+host+"\n")
				return err
			})
		case "seed-flat-client":
			binary, err := os.Executable()
			if err != nil {
				t.Fatal(err)
			}
			cmd := exec.CommandContext(t.Context(), binary, "-test.run=^TestRootFilesSeedClientPayload$", "-test.v")
			cmd.Env = append(os.Environ(), "FILES_SEED_EXPECT_REFUSAL=1", "FILES_SEED_CLIENT=1")
			if out, err := cmd.CombinedOutput(); err != nil {
				callErr = fmt.Errorf("flat client: %w: %s", err, out)
			}
		case "seed-manager":
			manager, callErr = filesSeedManager(t.Context())
		case "seed-capability":
			callErr = NewClaimRoleRootClient(socket).KindCapability(t.Context(), "files-seed")
		case "stop":
			reply.Stopped, callErr = CollectClaimStop(t.Context(), socket, q.Stop, stopDrainFunc(func(_ context.Context, req StopRequest, fence string) (DrainAcknowledgement, error) {
				return DrainClaimAcknowledgement(fence, req.ControllerID, peer.ProcessStart, nil), nil
			}))
		case "handover":
			hook := coldHandoverWith(handoverRecoveryIdentity(t, q.Actor), socket, func(context.Context, hostclaim.Observed) (hostguard.FenceReceipt, error) {
				return hostguard.FenceReceipt{Target: q.Target}, nil
			})
			callErr = hook(t.Context(), q.Previous, q.Admitted)
		case "park":
			reply.Authorization, callErr = client.Park(t.Context(), q.Current, q.Prior)
		case "complete":
			callErr = client.Complete(t.Context(), q.ID, q.Current)
		case "sibling":
			binary, err := os.Executable()
			if err != nil {
				t.Fatal(err)
			}
			cmd := exec.CommandContext(t.Context(), binary, "-test.run=^TestRootServingParkPeerPayload$", "-test.v", "-test.timeout=20s")
			cmd.Env = append(os.Environ(), "SERVING_PEER_SIBLING=1")
			body, err := json.Marshal(q)
			if err != nil {
				t.Fatal(err)
			}
			cmd.Stdin = strings.NewReader(string(body))
			out, err := cmd.CombinedOutput()
			if err != nil {
				callErr = fmt.Errorf("sibling: %w: %s", err, out)
			}
		case "prelaunch":
			if manager == nil {
				manager = cgroup.Setup(t.Logf)
				if manager == nil {
					t.Fatal("edge unit has no delegated cgroup")
				}
			}
			target := q.Target
			target.BootID, target.ControllerID = peer.BootID, peer.ControllerID
			roles := &ClaimRoleAssembly{Manager: manager}
			parent, group, err := roles.preparePrelaunchGroup(claimContainmentName(target), cgroup.TierDefault)
			if err != nil {
				t.Fatal(err)
			}
			t.Cleanup(func() { _ = parent.Close() })
			launch := hostbox.LaunchInventory{Target: target, ControllerPID: peer.PID, ProcessStart: peer.ProcessStart, NoState: true, Groups: []hostbox.LaunchGroup{group}}
			journal, _ := hostbox.RemoteJournals(call)
			// Each forged observation must be refused before any launch record.
			for _, field := range []string{"pid", "start", "controller", "boot", "inode", "path"} {
				bad := launch
				bad.Groups = append([]hostbox.LaunchGroup(nil), launch.Groups...)
				switch field {
				case "pid":
					bad.ControllerPID++
				case "start":
					bad.ProcessStart += "0"
				case "controller":
					bad.Target.ControllerID = "forged"
				case "boot":
					bad.Target.BootID = "forged"
				case "inode":
					bad.Groups[0].Inode++
				case "path":
					bad.Groups[0].RelativePath += "-foreign"
				}
				if err := journal.Prelaunch(t.Context(), bad); err == nil {
					t.Fatalf("forged prelaunch %s was authorized", field)
				}
			}
			callErr = journal.Prelaunch(t.Context(), launch)
			reply.Launch = launch
		default:
			t.Fatalf("unknown test action %q", q.Action)
		}
		if callErr != nil {
			reply.Error = callErr.Error()
		}
		if err := encode.Encode(reply); err != nil {
			t.Fatal(err)
		}
	}
}

// This witness needs an unused carlos-edge.service name because production
// authentication resolves that exact unit. It refuses existing deployments.
// Claim, installation, predecessor stop/start and roster/census data are fixtures;
// peer authentication, prelaunch containment, park authority and IO are real.
func TestRootServingParkAuthenticatedPeer(t *testing.T) {
	runServingParkAuthenticatedPeer(t, false, false)
}

func TestRootServingParkAuthenticatedHandover(t *testing.T) {
	runServingParkAuthenticatedPeer(t, true, false)
}

func runServingParkAuthenticatedPeer(t *testing.T, handover, writer bool) {
	t.Helper()
	runServingParkPeerNamespace(t, handover, writer, "")
}

func TestRootServingNamespaceStartup(t *testing.T) {
	runServingParkPeerNamespace(t, true, false, "parent")
}

func TestRootServingNamespaceUnrecordedStartup(t *testing.T) {
	runServingParkPeerNamespace(t, true, false, "fresh")
}

func runServingParkPeerNamespace(t *testing.T, handover, writer bool, namespace string) {
	t.Helper()
	servingRootRunner(t)
	pid1, err := os.ReadFile("/proc/1/comm")
	if err != nil || strings.TrimSpace(string(pid1)) != "systemd" {
		t.Fatal("root runner needs systemd as PID 1")
	}
	probe, probeErr := exec.CommandContext(t.Context(), "systemctl", "show", "carlos-edge.service", "-p", "LoadState", "--value").Output()
	if strings.TrimSpace(string(probe)) != "not-found" {
		t.Fatalf("refusing existing or unreadable carlos-edge.service: %s %v", probe, probeErr)
	}
	// The runner is disposable, but only a user created here is removed here.
	account, err := user.Lookup("carlos")
	if _, absent := err.(user.UnknownUserError); absent {
		if _, groupErr := user.LookupGroup("carlos"); groupErr == nil {
			t.Fatal("refusing pre-existing carlos group without its user")
		} else if _, unknown := groupErr.(user.UnknownGroupError); !unknown {
			t.Fatal(groupErr)
		}
		out, createErr := exec.CommandContext(t.Context(), "useradd", "--system", "--no-create-home", "--user-group", "--shell", "/usr/sbin/nologin", "carlos").CombinedOutput()
		if createErr != nil {
			t.Fatalf("create runner user: %v %s", createErr, out)
		}
		t.Cleanup(func() {
			ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
			defer cancel()
			if out, err := exec.CommandContext(ctx, "userdel", "carlos").CombinedOutput(); err != nil {
				t.Errorf("remove runner user: %v %s", err, out)
				return
			}
			if _, err := user.LookupGroup("carlos"); err == nil {
				if out, err := exec.CommandContext(ctx, "groupdel", "carlos").CombinedOutput(); err != nil {
					t.Errorf("remove runner group: %v %s", err, out)
				}
			}
		})
		account, err = user.Lookup("carlos")
	}
	if err != nil || account == nil || account.Uid == "0" {
		t.Fatalf("runner service user: %v", err)
	}
	root, err := os.MkdirTemp("/var/lib", "carlos-rootrunner-peer-")
	if err != nil {
		t.Fatal(err)
	}
	t.Cleanup(func() {
		if err := os.RemoveAll(root); err != nil {
			t.Error(err)
		}
	})
	if err := os.Chmod(root, 0755); err != nil {
		t.Fatal(err)
	}
	binary, err := os.Executable()
	if err != nil {
		t.Fatal(err)
	}
	body, err := os.ReadFile(binary)
	if err != nil {
		t.Fatal(err)
	}
	installed := filepath.Join(root, "runner.test")
	if err := os.WriteFile(installed, body, 0755); err != nil {
		t.Fatal(err)
	}
	if namespace == "seed-cli" {
		rootFilesSeedDelegatedCLI(t, installed)
		return
	}
	if namespace == "seed-two-host" {
		rootFilesSeedTwoHost(t, installed)
		return
	}
	if namespace == "seed-isolated" {
		rootFilesSeedIsolated(t, installed)
		return
	}
	f := newServingRootFixture(t, true)
	servingRunnerMount(t, f.f.guard.DataDir)
	for name, body := range map[string]string{"files/upload": "tenant upload", "env": "tenant configuration"} {
		path := filepath.Join(filepath.Dir(f.db), name)
		if err := os.MkdirAll(filepath.Dir(path), 0700); err != nil {
			t.Fatal(err)
		}
		if err := os.WriteFile(path, []byte(body), 0600); err != nil {
			t.Fatal(err)
		}
	}
	if handover {
		servingHandoverRoot(t, f)
	} else {
		f.install(f.admitted)
	}
	state, err := rootstate.Open(filepath.Join(root, "parks"), true, false)
	if err != nil {
		t.Fatal(err)
	}
	t.Cleanup(func() { _ = state.Close() })
	f.p.ledger.state = state
	f.p.ledger.fs = newServingParkFS(f.f.guard.DataDir, state)
	coordinator := f.f.root(nil)
	coordinator.Evidence = f.f.claims
	coordinator.Inventory = f.p.root.Journals.Inventory
	production, err := NewProductionClaimStopHandler(coordinator, f.f.journal, productionStopControllers{}, f.p.root.Roots)
	if err != nil {
		t.Fatal(err)
	}
	f.p.root.Stop = production.(ClaimStopHTTP)
	if namespace == "seed-killed" {
		f.p.root.Stop.Coordinator.Kernel = servingSeedRecoveryTrace{LinuxStopKernel: f.p.root.Stop.Coordinator.Kernel.(LinuxStopKernel), t: t}
	}
	kernelResults := make(chan error, 8)
	if writer {
		kernel := f.p.root.Stop.Coordinator.Kernel
		f.p.root.Stop.Coordinator.Kernel = stopKernelFunc(func(ctx context.Context, target hostguard.FenceTarget, inventory StopInventory, ack DrainAcknowledgement) (StopKernelEvidence, error) {
			evidence, err := kernel.VerifyStopped(ctx, target, inventory, ack)
			kernelResults <- err
			return evidence, err
		})
	}
	f.p.root.Reconcile.Box, f.p.root.Reconcile.Enrollment = f.actor.Box, f.actor.Enrollment
	var admitted atomic.Bool
	admitted.Store(true)
	f.p.root.RequireNewWork = func(context.Context) error {
		if !admitted.Load() {
			return hostbox.ErrUnsupported
		}
		return nil
	}
	h := servingTransportRoot(t, f)
	path := filepath.Join(root, "root.sock")
	listener, err := net.Listen("unix", path)
	if err != nil {
		t.Fatal(err)
	}
	if err := os.Chmod(path, 0666); err != nil {
		listener.Close()
		t.Fatal(err)
	}
	var handler http.Handler = h
	if strings.HasPrefix(namespace, "seed-") {
		capability := newLaunchFixture(t, hostclaim.Op{Kind: hostclaim.KindServing, Exec: "box-1"}).root
		capability.Authenticate = h.Stop.Authenticate
		capability.SeedController = verifyFilesSeedDelegation
		handler = ClaimRootWithLaunch{Root: h, Launch: capability}
	}
	if namespace == "seed-construction" {
		filesSeedInstanceControlChild(t, installed, root, listener.(*net.UnixListener))
	} else {
		server := &instancectl.Server{ClaimStop: handler, PeerOK: func(net.Conn) error { return nil }, Logf: t.Logf}
		ctx, cancel := context.WithCancel(context.Background())
		if lifecycle, ok := handler.(interface{ Start(context.Context) }); ok {
			lifecycle.Start(ctx)
		}
		done := make(chan error, 1)
		go func() { done <- server.Serve(ctx, listener) }()
		t.Cleanup(func() {
			cancel()
			if err := <-done; err != nil {
				t.Error(err)
			}
		})
	}
	controlPath := filepath.Join(root, "control.sock")
	control, err := net.ListenUnix("unix", &net.UnixAddr{Name: controlPath, Net: "unix"})
	if err != nil {
		t.Fatal(err)
	}
	t.Cleanup(func() { control.Close() })
	if err := os.Chmod(controlPath, 0666); err != nil {
		t.Fatal(err)
	}
	raw, err := unitsFS.ReadFile("units/carlos-edge.service")
	if err != nil {
		t.Fatal(err)
	}
	var lines []string
	for _, line := range strings.Split(string(raw), "\n") {
		if strings.HasPrefix(line, "Requires=") || strings.HasPrefix(line, "After=") || strings.HasPrefix(line, "Wants=") || strings.HasPrefix(line, "ExecStart=") || strings.HasPrefix(line, "ExecReload=") || strings.HasPrefix(line, "EnvironmentFile=") || strings.HasPrefix(line, "ReadWritePaths=") || strings.HasPrefix(line, "Restart=") || strings.HasPrefix(line, "Slice=") {
			continue
		}
		lines = append(lines, line)
	}
	// Add a Service section because the embedded unit ends in Install.
	content := strings.Join(lines, "\n") + fmt.Sprintf("\n[Service]\nType=simple\nRestart=no\nTimeoutStopSec=10s\nExecStart=%s -test.run=^TestRootServingParkPeerPayload$ -test.v -test.timeout=2m\nEnvironment=SERVING_PEER_CONTROL=%s SERVING_PEER_SOCKET=%s\nIPAddressDeny=any\n", installed, controlPath, path)
	if namespace == "seed-live" {
		if err := os.Mkdir("/run/carlos", 0755); err != nil {
			t.Fatal("refusing existing runtime directory", err)
		}
		t.Cleanup(func() { os.RemoveAll("/run/carlos") })
		uid, _ := strconv.Atoi(account.Uid)
		gid, _ := strconv.Atoi(account.Gid)
		if err := os.Chown("/run/carlos", uid, gid); err != nil {
			t.Fatal(err)
		}
		content += "ReadWritePaths=/run/carlos\n"
	}
	unitPath := "/run/systemd/system/carlos-edge.service"
	unit, err := os.OpenFile(unitPath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0644)
	if err != nil {
		t.Fatal(err)
	}
	t.Cleanup(func() {
		ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
		defer cancel()
		if out, err := exec.CommandContext(ctx, "systemctl", "stop", "carlos-edge.service").CombinedOutput(); err != nil {
			t.Errorf("stop test edge: %v %s", err, out)
		}
		if err := os.Remove(unitPath); err != nil {
			t.Error(err)
		}
		if out, err := exec.CommandContext(ctx, "systemctl", "daemon-reload").CombinedOutput(); err != nil {
			t.Errorf("reload test edge: %v %s", err, out)
		}
	})
	_, writeErr := io.WriteString(unit, content)
	if err := errors.Join(writeErr, unit.Close()); err != nil {
		t.Fatal(err)
	}
	servingRunnerSystemctl(t, "daemon-reload")
	connect := func() (net.Conn, *json.Encoder, *json.Decoder, StopPeer) {
		t.Helper()
		servingRunnerSystemctl(t, "start", "carlos-edge.service")
		if err := control.SetDeadline(time.Now().Add(15 * time.Second)); err != nil {
			t.Fatal(err)
		}
		conn, err := control.Accept()
		if err != nil {
			t.Fatal(err)
		}
		t.Cleanup(func() { conn.Close() })
		if err := conn.SetDeadline(time.Now().Add(90 * time.Second)); err != nil {
			t.Fatal(err)
		}
		enc, dec := json.NewEncoder(conn), json.NewDecoder(conn)
		var hello servingPeerReply
		if err := dec.Decode(&hello); err != nil {
			t.Fatal(err)
		}
		peer, err := h.Stop.Authenticate(t.Context(), conn)
		if err != nil || peer != hello.Peer || strconv.FormatUint(uint64(peer.UID), 10) != account.Uid {
			t.Fatalf("production peer: %+v %+v %v", peer, hello.Peer, err)
		}
		return conn, enc, dec, peer
	}
	old, _, _, first := connect()
	servingRunnerSystemctl(t, "stop", "carlos-edge.service")
	_, encode, decode, next := connect()
	if first == next {
		t.Fatal("edge restart kept the process tuple")
	}
	if _, err := f.p.parkServingDatabase(t.Context(), old, f.current, f.prior); err == nil {
		t.Fatal("old controller connection authorized retirement")
	}
	command := func(q servingPeerCommand, refused bool) servingPeerReply {
		t.Helper()
		t.Logf("authenticated peer request: action=%s expect-refusal=%v", q.Action, refused)
		if err := encode.Encode(q); err != nil {
			t.Fatal(err)
		}
		var reply servingPeerReply
		if err := decode.Decode(&reply); err != nil {
			t.Fatal(err)
		}
		if (reply.Error != "") != refused {
			t.Fatalf("%s: expected refusal=%v: %s", q.Action, refused, reply.Error)
		}
		return reply
	}
	if namespace == "seed-construction" {
		command(servingPeerCommand{Action: "control-request"}, false)
		command(servingPeerCommand{Action: "seed-manager"}, false)
		command(servingPeerCommand{Action: "seed-capability"}, false)
		return
	}
	if strings.HasPrefix(namespace, "seed-") {
		command(servingPeerCommand{Action: "seed-capability"}, true)
		command(servingPeerCommand{Action: "seed-manager"}, false)
		command(servingPeerCommand{Action: "seed-capability"}, false)
		if namespace == "seed-killed" {
			observed := f.f.read()
			attempt := hostclaim.WakeRetryAttempt{ID: "killed-attempt", Box: f.actor.Box, Enrollment: f.actor.Enrollment, BootID: next.BootID, ControllerID: next.ControllerID, Session: observed.Claim.Holder.Session, InvocationID: "killed-invocation", LaunchInputs: "seed-inputs"}
			observed, err = hostclaim.Update(t.Context(), f.f.claims, observed, hostclaim.ReserveWakeRetryAttempt(hostclaim.WakeRetryReservation{HolderIncarnation: observed.Claim.Holder.Incarnation, EpisodeID: "killed-seed", Attempt: attempt, Now: f.f.now}))
			if err != nil {
				t.Fatal(err)
			}
			target := recoveryTuple(observed, claimStopLocal{Actor: f.actor})
			pre := command(servingPeerCommand{Action: "prelaunch", Target: target}, false)
			former := next
			servingRunnerSystemctl(t, "kill", "--signal=KILL", "carlos-edge.service")
			// Start waits for systemd to reap the killed MainPID; no graceful cleanup ran.
			servingRunnerSystemctl(t, "stop", "carlos-edge.service")
			_, encode, decode, next = connect()
			if former.PID == next.PID || former.ControllerID == next.ControllerID {
				t.Fatal("kill did not replace controller")
			}
			command(servingPeerCommand{Action: "seed-manager"}, false)
			command(servingPeerCommand{Action: "seed-capability"}, false)
			rt := registry.Route{Host: observed.Host, DBPath: f.db}
			files := &movesFiles{DataDir: f.f.guard.DataDir, EnvFile: func(host, db string) (claimEnvFile, error) {
				return claimEnvFile{Nested: true, Dir: filepath.Dir(db), Name: host + ".env"}, nil
			}}
			service := &activator.Service{}
			exclusion, err := service.AcquireSeedExclusion(t.Context(), rt.Host, nil)
			if err != nil {
				t.Fatal(err)
			}
			defer exclusion.Release()
			stops := 0
			retries := activator.ServingRetries{Claims: f.f.claims, Box: f.actor.Box, Enrollment: f.actor.Enrollment, BootID: next.BootID, ControllerID: next.ControllerID, Now: func() time.Time { return f.f.now }, NewID: f.f.newID, Inputs: func(context.Context, registry.Route) (string, error) { return "seed-inputs", nil }, ExactHealth: func(context.Context, *exec.Cmd, hostclaim.WakeRetryAttempt) error { return hostbox.ErrChanged }, StopAttempt: func(ctx context.Context, o hostclaim.Observed, a hostclaim.WakeRetryAttempt) error {
				stops++
				req := StopRequest{Region: f.actor.Region, Recover: true, Host: o.Host, Incarnation: o.Claim.Holder.Incarnation, InstallationStamp: o.Claim.Holder.Stamp, WriterSession: o.Claim.Holder.Session, Box: f.actor.Box, Enrollment: f.actor.Enrollment, BootID: a.BootID, ControllerID: a.ControllerID, ExpectedClaimETag: o.ETag, ExpectedDirectoryRevision: pre.Launch.Target.DirectoryRevision, RosterRevision: pre.Launch.Target.RosterRevision}
				command(servingPeerCommand{Action: "stop", Stop: req}, false)
				return nil
			}}
			settled, err := settleFilesSeedRerun(t.Context(), rt, files, retries, observed, exclusion)
			if err != nil {
				t.Fatal("killed rerun settlement", err)
			}
			retry := settled.Claim.Holder.WakeRetry
			if retry == nil || retry.State != "open" || retry.AttemptsReserved != 1 || retry.Attempt != nil || stops != 1 {
				t.Fatal("retry debt not settled exactly", retry, stops)
			}
			again, err := settleFilesSeedRerun(t.Context(), rt, files, retries, settled, exclusion)
			if err != nil || again.ETag != settled.ETag || stops != 1 {
				t.Fatal("rerun repeated stop or reservation", err)
			}
			return
		}
		if namespace == "seed-live" {
			command(servingPeerCommand{Action: "seed-listen"}, false)
			uid, _ := strconv.Atoi(account.Uid)
			gid, _ := strconv.Atoi(account.Gid)
			cmd := exec.CommandContext(t.Context(), installed, "-test.run=^TestRootFilesSeedClientPayload$", "-test.v")
			cmd.Dir = root
			cmd.Env = append(os.Environ(), "FILES_SEED_CLIENT=1")
			cmd.SysProcAttr = &syscall.SysProcAttr{Credential: &syscall.Credential{Uid: uint32(uid), Gid: uint32(gid)}}
			if out, err := cmd.CombinedOutput(); err != nil {
				t.Fatalf("external client: %v %s", err, out)
			}
			command(servingPeerCommand{Action: "seed-flat-client"}, false)
			c, err := net.Dial("unix", filesSeedSocket)
			if err != nil {
				t.Fatal(err)
			}
			json.NewEncoder(c).Encode(filesSeedRequest{Verb: "seed"})
			c.SetReadDeadline(time.Now().Add(time.Second))
			if _, err := c.Read(make([]byte, 1)); err == nil {
				t.Fatal("root client accepted")
			}
			c.Close()
		}
		if err := verifyFilesSeedDelegation(t.Context(), first); err == nil {
			t.Fatal("former controller accepted")
		}
		forged := next
		forged.UID = 0
		if err := verifyFilesSeedDelegation(t.Context(), forged); err == nil {
			t.Fatal("root peer accepted")
		}
		forged = next
		forged.ControllerID = "changed"
		if err := verifyFilesSeedDelegation(t.Context(), forged); err == nil {
			t.Fatal("changed controller accepted")
		}
		return
	}
	q := servingPeerCommand{Action: "park", Current: f.current, Prior: f.prior}
	if _, err := (hostbox.ServingParkClient{Call: rootJournalClient(path)}).Park(t.Context(), f.current, f.prior); err == nil {
		t.Fatal("root caller authorized retirement")
	}
	q.Action = "sibling"
	command(q, false)
	q.Action = "park"
	admitted.Store(false)
	command(q, true)
	admitted.Store(true)
	bad := q
	bad.Current.Generation = "foreign-generation"
	command(bad, true)
	before, err := os.ReadFile(f.db)
	if err != nil || string(before) != "served bytes" {
		t.Fatal("refusal changed source", err)
	}
	names, err := state.Names()
	if err != nil {
		t.Fatal(err)
	}
	for _, name := range names {
		if name != "namespace.lock" {
			t.Fatalf("refusal wrote park authority: %s", name)
		}
	}
	if handover {
		transfer := servingPeerCommand{Action: "handover", Previous: f.predecessor, Admitted: f.f.read(), Actor: f.actor, Target: f.receipt.Target}
		captured := hostguard.Captured{Incarnation: f.prior.Incarnation, Stamp: f.prior.Stamp}
		oldInventory, _, err := f.p.root.Journals.Inventory.Snapshot(t.Context(), f.current.Host, captured)
		if err != nil {
			t.Fatal(err)
		}
		admitted.Store(false)
		command(transfer, true)
		admitted.Store(true)
		bad := transfer
		bad.Target.FenceID = "foreign-receipt"
		command(bad, true)
		command(transfer, false)
		command(transfer, false)
		installed, err := f.f.guard.Capture(f.current.Host)
		if err != nil || installed != (hostguard.Captured{Incarnation: f.current.Incarnation, Stamp: f.current.Stamp}) {
			t.Fatalf("authenticated handover installation: %+v %v", installed, err)
		}
		retained, _, err := f.p.root.Journals.Inventory.Snapshot(t.Context(), f.current.Host, captured)
		if err != nil || !reflect.DeepEqual(retained, oldInventory) {
			t.Fatal("handover changed predecessor inventory", err)
		}
		for suffix, want := range map[string]string{"": "served bytes", ".gen": "G", "-wal": "WAL", "-shm": "SHM"} {
			body, err := os.ReadFile(f.db + suffix)
			if err != nil || string(body) != want {
				t.Fatal("handover changed source", suffix, err)
			}
		}
	}
	target := recoveryTuple(f.f.read(), claimStopLocal{Actor: f.actor})
	pre := command(servingPeerCommand{Action: "prelaunch", Target: target}, false)
	actual, err := f.f.journal.Read(t.Context(), pre.Launch.Target)
	if err != nil || actual.ControllerPID != next.PID || actual.ProcessStart != next.ProcessStart || len(actual.Groups) != 1 || actual.Groups[0] != pre.Launch.Groups[0] {
		t.Fatalf("prelaunch not retained: %+v %v", actual, err)
	}
	if namespace != "" {
		fs := f.p.ledger.fs.(*servingParkFS)
		parkPath := filepath.Join(fs.data, "instances", ".serving-parked")
		if err := os.Mkdir(parkPath, 0700); err != nil {
			t.Fatal(err)
		}
		if err := os.WriteFile(filepath.Join(parkPath, "unknown"), []byte("retained unknown bytes"), 0600); err != nil {
			t.Fatal(err)
		}
		// Parent evidence comes from authenticated installation checks. Lose
		// either its reply or the later fresh-root identity write.
		fs.state = quarantineWriteFault{Store: state, before: func(name string, body []byte) error {
			var q servingParkQuarantine
			if namespace == "fresh" && name == "namespace-quarantine.json" && servingParkDecode(body, &q) == nil && q.Fresh.Inode != 0 {
				return unix.ENOSPC
			}
			return nil
		}, after: func(name string, _ []byte) error {
			if namespace == "parent" && name == "namespace-parent.json" {
				return unix.EIO
			}
			return nil
		}}
		command(q, true)
		fs.state = state
		if namespace == "parent" {
			if _, err := state.Read("namespace-quarantine.json"); !errors.Is(err, os.ErrNotExist) {
				t.Fatal("lost parent reply did not stop before quarantine", err)
			}
		} else if err := os.WriteFile(filepath.Join(parkPath, "uncertain"), []byte("unrecorded bytes"), 0600); err != nil {
			t.Fatal(err)
		}
		before := f.f.read()
		restarted := *f.p
		ledger := *f.p.ledger
		ledger.fs = newServingParkFS(fs.data, state)
		restarted.ledger = &ledger
		page, err := restarted.recoverServingParks(t.Context(), "", servingParkRecoveryLimit)
		if err != nil || len(page.Results) != 0 || page.More {
			t.Fatalf("root namespace startup: %+v %v", page, err)
		}
		var quarantine servingParkQuarantine
		body, err := state.Read("namespace-quarantine.json")
		if err != nil || servingParkDecode(body, &quarantine) != nil || !quarantine.Complete {
			t.Fatal("startup did not complete quarantine", err)
		}
		if quarantine.Parent.Mount != quarantine.Obstruction.Mount || quarantine.Parent.Mount != quarantine.Fresh.Mount {
			t.Fatal("startup crossed the bind mount")
		}
		if namespace == "fresh" {
			if !quarantine.UnrecordedMoved || quarantine.Unrecorded.Mount != quarantine.Parent.Mount || quarantine.Unrecorded == quarantine.Fresh {
				t.Fatal("startup adopted unrecorded root")
			}
			body, err = os.ReadFile(filepath.Join(filepath.Dir(parkPath), quarantine.unrecordedName(), "uncertain"))
			if err != nil || string(body) != "unrecorded bytes" {
				t.Fatal("startup changed unrecorded bytes", err)
			}
		}
		body, err = os.ReadFile(filepath.Join(filepath.Dir(parkPath), quarantine.name(), "unknown"))
		if err != nil || string(body) != "retained unknown bytes" || !reflect.DeepEqual(before, f.f.read()) {
			t.Fatal("startup changed unknown bytes or claim", err)
		}
		body, err = os.ReadFile(f.db)
		if err != nil || string(body) != "served bytes" {
			t.Fatal("namespace recovery changed source", err)
		}
	}
	parked := command(q, false).Authorization
	if parked.ID == "" {
		t.Fatal("empty park identity")
	}
	command(servingPeerCommand{Action: "complete", ID: parked.ID, Current: f.current}, true)
	f.tags(f.current.Stamp)
	command(servingPeerCommand{Action: "complete", ID: parked.ID, Current: f.current}, false)
	r, err := f.p.ledger.read(parked.ID)
	if err != nil || !r.Complete || r.Disposition != servingParkRetain {
		t.Fatalf("completed park: %+v %v", r, err)
	}
	if err := f.p.ledger.fs.CheckParked(&r); err != nil {
		t.Fatal(err)
	}
	for suffix, want := range map[string]string{"": "served bytes", ".gen": "G", "-wal": "WAL", "-shm": "SHM"} {
		if _, err := os.Lstat(f.db + suffix); !errors.Is(err, os.ErrNotExist) {
			t.Fatal("park left source", suffix, err)
		}
		body, err := os.ReadFile(filepath.Join(f.f.guard.DataDir, "instances", ".serving-parked", parked.ID, filepath.Base(f.db)+suffix))
		if err != nil || string(body) != want {
			t.Fatal("park changed bytes", suffix, err)
		}
	}
	if replay := command(q, false).Authorization; replay != parked {
		t.Fatal("authenticated replay changed retirement")
	}
	for name, want := range map[string]string{"files/upload": "tenant upload", "env": "tenant configuration"} {
		body, err := os.ReadFile(filepath.Join(filepath.Dir(f.db), name))
		if err != nil || string(body) != want {
			t.Fatal("retirement changed preserved state", name, err)
		}
	}
	if writer {
		runServingPeerStoppedWriter(t, f, installed, pre.Launch, next, command, kernelResults)
	}
}

// Transfer the serving installation on the nested mount without moving its
// state directory, then park the database through the root transport.
func TestRootServingParkColdHandoverBoundary(t *testing.T) {
	servingRootRunner(t)
	runServingColdHandoverBoundary(t, servingRunnerMount)
}

// Keep root errors in the test log; the public transport intentionally returns
// only uncertainty after a durable request may have made filesystem changes.
type servingPeerParkTrace struct {
	servingParkAuthority
	root *servingParkRoot
	t    *testing.T
}

func (p servingPeerParkTrace) parkServingDatabase(ctx context.Context, conn net.Conn, current, prior servingArtifactIdentity) (servingParkAuthorization, error) {
	a, err := p.servingParkAuthority.parkServingDatabase(ctx, conn, current, prior)
	p.t.Logf("authenticated peer park authority: %v", err)
	if err != nil {
		check := p.root.withAuthority(ctx, conn, current, func(o hostclaim.Observed, view hostguard.FenceView, actor hostbox.Actor, dir servingArtifactDirectory, tags uint64) error {
			selected, e := servingPredecessor(o.Claim, current, tags, prior.Generation)
			p.t.Logf("park diagnostic predecessor: exact=%v error=%v", selected == prior, e)
			receipts, e := servingRootStops(ctx, view, prior, actor)
			p.t.Logf("park diagnostic receipts: count=%d error=%v", len(receipts), e)
			source, e := p.root.ledger.fs.Capture(servingParkAuthorization{TagsFrom: tags, Predecessor: prior, OriginalSuccessor: current, Successor: current, Directory: dir, ID: servingRetirementID(prior, current, false)})
			p.t.Logf("park diagnostic source capture: %v", e)
			if e == nil {
				disposition, e := servingGenerationParkDisposition(ctx, p.root.ledger.journal, prior, receipts, source.Files[0].Present)
				p.t.Logf("park diagnostic disposition: %v error=%v", disposition, e)
			}
			return nil
		})
		p.t.Logf("park diagnostic root installation: %v", check)
	}
	return a, err
}

func (p servingPeerParkTrace) completeServingDatabasePark(ctx context.Context, conn net.Conn, id string, current servingArtifactIdentity) error {
	err := p.servingParkAuthority.completeServingDatabasePark(ctx, conn, id, current)
	p.t.Logf("authenticated peer completion authority: %v", err)
	return err
}

func TestRootFilesSeedDelegatedController(t *testing.T) {
	runServingParkPeerNamespace(t, false, false, "seed-delegation")
}

func TestRootFilesSeedLiveTransport(t *testing.T) {
	runServingParkPeerNamespace(t, false, false, "seed-live")
}

func TestRootFilesSeedClientPayload(t *testing.T) {
	if os.Getenv("FILES_SEED_CLIENT") != "1" {
		t.Fatal("invoke through live transport witness")
	}
	t.Setenv("CARLOS_HOST_CLAIM", "1")
	t.Setenv("CARLOS_HOST_CLAIM_ORIGINS", "1")
	t.Setenv("CARLOS_MOVES_PHASE1", "")
	var out bytes.Buffer
	err := runSystemFiles(t.Context(), []string{"seed", "--host", recoveryHost}, &out)
	if os.Getenv("FILES_SEED_EXPECT_REFUSAL") == "1" {
		if err == nil {
			t.Fatal("tenant admitted")
		}
		return
	}
	if err != nil || out.String() != "accepted "+recoveryHost+"\n" {
		t.Fatal(err, out.String())
	}
	// A same-UID impostor can own a pathname but cannot impersonate MainPID.
	held := filesSeedSocket + ".held"
	if err := os.Rename(filesSeedSocket, held); err != nil {
		t.Fatal(err)
	}
	defer func() {
		if err := os.Rename(held, filesSeedSocket); err != nil {
			t.Error(err)
		}
	}()
	fake, err := net.ListenUnix("unix", &net.UnixAddr{Name: filesSeedSocket, Net: "unix"})
	if err != nil {
		t.Fatal(err)
	}
	defer fake.Close()
	if err := os.Chmod(filesSeedSocket, 0600); err != nil {
		t.Fatal(err)
	}
	if err := runSystemFiles(t.Context(), []string{"seed"}, io.Discard); !errors.Is(err, ErrClaimStopIdentity) {
		t.Fatal("impostor accepted", err)
	}
}

func TestRootFilesSeedKilledControllerRerun(t *testing.T) {
	runServingParkPeerNamespace(t, false, false, "seed-killed")
}

type servingSeedRecoveryTrace struct {
	LinuxStopKernel
	t *testing.T
}

func (k servingSeedRecoveryTrace) RecoverStopped(ctx context.Context, target hostguard.FenceTarget, in StopInventory) (StopKernelEvidence, DrainAcknowledgement, error) {
	evidence, ack, err := k.LinuxStopKernel.RecoverStopped(ctx, target, in)
	k.t.Logf("killed controller root recovery: %v", err)
	return evidence, ack, err
}