//go:build linux && rootrunner package host import ( "bytes" "context" "encoding/json" "errors" "fmt" "os" "os/exec" "path/filepath" "strings" "syscall" "testing" "time" "github.com/carlosframework/platform/internal/hostbox" "github.com/carlosframework/platform/internal/hostguard" "github.com/carlosframework/platform/internal/provision" "github.com/carlosframework/platform/internal/rootstate" "golang.org/x/sys/unix" ) func servingRootRunner(t *testing.T) { t.Helper() mapping, err := os.ReadFile("/proc/self/uid_map") if err != nil || os.Geteuid() != 0 || strings.Join(strings.Fields(string(mapping)), " ") != "0 0 4294967295" { t.Fatalf("root runner needs real root in the initial user namespace: %v", err) } if t.Name() == "TestRootServingParkUnitPayload" || os.Getenv("PLAN14_ROOT_FIXTURE") == t.Name() { // Unit and chroot children use the parent-owned fixture filesystem. return } // The runner's /tmp is a small RAM filesystem. Serving now enforces the // real data-disk floor, so positive fixtures need the instance's disk. data, err := os.MkdirTemp("/var/lib", "carlos-serving-test-") if err != nil { t.Fatal(err) } t.Cleanup(func() { if err := os.RemoveAll(data); err != nil { t.Error(err) } }) if !strings.HasPrefix(t.Name(), "TestRootReboot") { scratch := servingRunnerScratchFilesystem(t, data, 2<<30) t.Setenv("TMPDIR", scratch) } else { t.Setenv("TMPDIR", data) } } // This includes loopback S3 and Unix-socket fixtures; it makes no remote calls. // Installation, peer/census admission and prelaunch cgroups remain test seams. func TestRootServingParkNestedBindRestore(t *testing.T) { servingRootRunner(t) if os.Getenv("PLAN14_ROOT_FIXTURE") == t.Name() { if err := unix.Mount("", "/", "", unix.MS_REC|unix.MS_PRIVATE, ""); err != nil { t.Fatal(err) } if !plan14RootFixture(t) { t.Fatal("root child did not enter its scratch root") } for _, failure := range []string{"", "enrolled-refusal", "missing-marker", "cancel-after-enrollment", "disk-after-enrollment", "invalid-sqlite", "cancel-after-output", "restore-checksum", "restore-enospc"} { for _, restart := range []string{"same-controller", "controller", "reboot"} { t.Run(failure+"/"+restart, func(t *testing.T) { runFilesSeedOrdinaryPreparation(t, failure, restart, true) }) } } return } root := t.TempDir() for _, name := range []string{"scratch", "proc", "foreign-dir"} { if err := os.Mkdir(filepath.Join(root, name), 0755); err != nil { t.Fatal(err) } } if err := os.WriteFile(filepath.Join(root, "foreign"), nil, 0644); err != nil { t.Fatal(err) } binary, err := os.Executable() if err != nil { t.Fatal(err) } cmd := exec.CommandContext(t.Context(), binary, "-test.run=^"+t.Name()+"$", "-test.v", "-test.timeout=3m") cmd.Env = append(os.Environ(), "PLAN14_ROOT_FIXTURE="+t.Name(), "PLAN14_ROOT_DIR="+root) cmd.SysProcAttr = &syscall.SysProcAttr{Cloneflags: unix.CLONE_NEWNS} if out, err := cmd.CombinedOutput(); err != nil { t.Fatalf("real-root mounted restore: %v\n%s", err, out) } else { t.Log(string(out)) } } func servingRunnerMount(t *testing.T, data string) { t.Helper() instances := filepath.Join(data, "instances") if err := os.MkdirAll(instances, 0700); err != nil { t.Fatal(err) } if err := unix.Mount(instances, instances, "", unix.MS_BIND, ""); err != nil { t.Fatal(err) } if err := unix.Mount("", instances, "", unix.MS_PRIVATE|unix.MS_REC, ""); err != nil { _ = unix.Unmount(instances, 0) t.Fatal(err) } t.Cleanup(func() { if err := unix.Unmount(instances, 0); err != nil { t.Error(err) } }) if err := unix.Mount("", instances, "", unix.MS_BIND|unix.MS_REMOUNT|unix.MS_NOSYMFOLLOW|unix.MS_NOSUID|unix.MS_NODEV|unix.MS_NOEXEC, ""); err != nil { t.Fatal(err) } } // Exercise the production filesystem and private store. The authorization is // a fixture, so this helper proves no peer, census or installation admission. func servingRunnerPark(t *testing.T, data string, state rootstate.Store) servingParkRecord { t.Helper() current := servingArtifactIdentity{Host: "runner.example", Box: "runner", Region: "test", Incarnation: "inc", Location: "instances/runner.example", Generation: "G2", Stamp: 2, Session: 2} db := provision.NestedDBPath(data, current.Host) if err := os.MkdirAll(filepath.Dir(db), 0700); err != nil { t.Fatal(err) } for suffix, body := range map[string]string{"": "database", "-wal": "WAL", "-shm": "SHM"} { if err := os.WriteFile(db+suffix, []byte(body), 0600); err != nil { t.Fatal(err) } } dir, err := os.Open(filepath.Dir(db)) if err != nil { t.Fatal(err) } id, err := servingParkDirID(dir) dir.Close() if err != nil { t.Fatal(err) } fs := newServingParkFS(data, state) a := servingParkAuthorization{Successor: current, OriginalSuccessor: current, Directory: id, Legacy: true} a.ID = servingRetirementID(a.Predecessor, current, true) // The helper's installation authorization remains a fixture. Capture the // parent through the production recorder so reboot can inspect its proof. if err := fs.recordNamespaceParent(t.Context(), current, id); err != nil { t.Fatal(err) } source, err := fs.Capture(a) if err != nil { t.Fatal(err) } // Stop authorization remains explicitly fixture-backed. It is encoded in // the production record format so the reboot inspector reads the ledger. at := time.Now().UTC() target := hostguard.FenceTarget{Host: current.Host, Box: current.Box, Region: current.Region, Incarnation: current.Incarnation, Stamp: 1, Session: 1, FenceID: "runner-stop", Enrollment: "runner-enrollment", BootID: source.BootID, ControllerID: "runner-controller", DirectoryRevision: "1", RosterRevision: 1, ClaimETag: "runner-claim", ClaimGen: 1} receipt := StopReceipt{V: 1, ID: target.FenceID, Kind: StopReceiptStopped, Region: target.Region, Host: target.Host, Box: target.Box, Incarnation: target.Incarnation, InstallationStamp: target.Stamp, WriterSession: target.Session, FenceID: target.FenceID, Enrollment: target.Enrollment, BootID: target.BootID, ControllerID: target.ControllerID, DirectoryRevision: testReceiptNumeral(target), Binding: target.DirectoryRevision, ClaimETag: target.ClaimETag, ClaimGen: target.ClaimGen, EvidenceAt: at} r := servingParkRecord{Version: 1, Authorization: a, Source: source, Disposition: servingParkRetain, Receipts: []hostguard.FenceReceipt{{Target: target, Bytes: mustMarshalStopReceipt(receipt), EvidenceAt: at}}} save := func() error { l := servingParkLedger{state: state} if err := l.save(r); err != nil { return err } return l.write("fixture.json", r) } if err = save(); err != nil { t.Fatal(err) } if err = fs.Prepare(&r, save); err != nil { t.Fatal(err) } if err = fs.Move(&r, save); err != nil { t.Fatal(err) } if err = fs.CheckParked(&r); err != nil { t.Fatal(err) } r.Parked, r.ParkedAt = true, time.Now().UTC() r.ReapAfter = r.ParkedAt.Add(7 * 24 * time.Hour) if err = save(); err != nil { t.Fatal(err) } ledger := servingParkLedger{state: state} if err := ledger.write("slot-"+servingParkKey(current.Box, current.Host)+".json", servingParkSlot{Version: 1, Box: current.Box, Host: current.Host, ID: a.ID, Pending: a.ID, Last: a.ID}); err != nil { t.Fatal(err) } if err := ledger.write(servingLegacyKey("legacy", current), servingLegacyConsumption{Authorization: a, Source: source, Receipts: r.Receipts}); err != nil { t.Fatal(err) } return r } // This child runs only through the installed test unit below. The production // hardening is retained, with a test binary and scratch deployment roots. func TestRootServingParkUnitPayload(t *testing.T) { servingRootRunner(t) data, authority := os.Getenv("SERVING_RUNNER_DATA"), os.Getenv("SERVING_RUNNER_AUTHORITY") mode := os.Getenv("SERVING_RUNNER_MODE") if data == "" || authority == "" || (mode != "on" && mode != "off") { t.Fatal("invoke through TestRootServingParkInstalledUnit") } if mode == "off" { if err := os.WriteFile(filepath.Join(data, "must-not-write"), nil, 0600); !errors.Is(err, syscall.EROFS) { t.Fatalf("claims-off data write: %v", err) } if err := os.WriteFile(filepath.Join(authority, "must-not-write"), nil, 0600); !errors.Is(err, syscall.EROFS) { t.Fatalf("claims-off authority write: %v", err) } return } state, err := rootstate.Open(authority, false, false) if err != nil { t.Fatal(err) } defer state.Close() r := servingRunnerPark(t, data, state) fs := newServingParkFS(data, state) // Exercise persistent handles inside the unchanged hardened unit. A // simulated boot lets the read-only checker run; it enables no effects. fs.boot = func() (string, error) { return servingOtherBoot, nil } if _, err := fs.inspectRebootTopology(t.Context()); err != nil { t.Fatal("installed-unit topology inspection", err) } beforeRecord, err := state.Read("retirement-" + r.Authorization.ID + ".json") if err != nil { t.Fatal(err) } if _, err := fs.inspectRebootRecord(t.Context(), r.Authorization.ID); err != nil { t.Fatal("installed-unit record inspection", err) } servingRunnerPrepareReboot(t, fs, r) afterRecord, err := state.Read("retirement-" + r.Authorization.ID + ".json") if err != nil || !bytes.Equal(beforeRecord, afterRecord) { t.Fatal("record inspection changed history", err) } fs.boot = servingParkBootID r.Deleting, r.Cause = true, "expiry" save := func() error { return (&servingParkLedger{state: state}).write("fixture.json", r) } fs.unlink = func(fd int, name string, flags int) error { err := unix.Unlinkat(fd, name, flags) if err == nil && flags == unix.AT_REMOVEDIR { return unix.EIO } return err } if err := fs.Delete(&r, save); !errors.Is(err, unix.EIO) { t.Fatalf("lost empty-directory removal reply: %v", err) } body, err := state.Read("fixture.json") if err != nil || servingParkDecode(body, &r) != nil || !r.EventRemoveIntent || r.EventRemoved { t.Fatal("missing durable event removal intent", err) } fs = newServingParkFS(data, state) if err = fs.Delete(&r, save); err != nil || !r.EventRemoved { t.Fatal("installed unit did not recover empty event removal", err) } if _, err = os.Lstat(filepath.Join(data, "instances", ".serving-parked", r.Authorization.ID)); !errors.Is(err, os.ErrNotExist) { t.Fatal("recorded empty event survived", err) } for _, item := range r.Source.Files { if item.Present && !item.Deleted { t.Fatal("installed unit did not delete recorded artifact") } } } func servingRunnerSystemctl(t *testing.T, args ...string) string { t.Helper() ctx, cancel := context.WithTimeout(t.Context(), 45*time.Second) defer cancel() out, err := exec.CommandContext(ctx, "systemctl", args...).CombinedOutput() if err != nil { if len(args) > 1 && args[0] == "start" { journal, _ := exec.CommandContext(ctx, "journalctl", "-u", args[1], "--no-pager", "-n", "60").CombinedOutput() t.Log(string(journal)) } t.Fatalf("systemctl %v: %v\n%s", args, err, out) } return string(out) } func TestRootServingParkInstalledUnit(t *testing.T) { servingRootRunner(t) pid1, err := os.ReadFile("/proc/1/comm") if err != nil || strings.TrimSpace(string(pid1)) != "systemd" { t.Fatal("root runner needs systemd as PID 1") } root, err := os.MkdirTemp("/var/lib", "carlos-rootrunner-") if err != nil { t.Fatal(err) } t.Cleanup(func() { if err := os.RemoveAll(root); err != nil { t.Error(err) } }) data := filepath.Join(root, "data") opt := filepath.Join(root, "opt") // Create only absent deployment directories and remove only those on exit. for _, path := range []string{data, opt, "/run/carlos-control", "/run/carlos", "/var/lib/carlos-claims", "/var/lib/carlos-release", rootReleasePrivate} { if err := os.Mkdir(path, 0700); err == nil { if path == data || path == opt { continue } p := path t.Cleanup(func() { if err := os.Remove(p); err != nil && !errors.Is(err, os.ErrNotExist) { t.Error(err) } }) } else if !errors.Is(err, os.ErrExist) { t.Fatal(err) } } authority := filepath.Join("/var/lib/carlos-claims", filepath.Base(root)) if err := os.Mkdir(authority, 0700); err != nil { t.Fatal(err) } t.Cleanup(func() { if err := os.RemoveAll(authority); err != nil { t.Error(err) } }) servingRunnerMount(t, data) binary, err := os.Executable() if err != nil { t.Fatal(err) } body, err := os.ReadFile(binary) if err != nil { t.Fatal(err) } installed := filepath.Join(root, "runner.test") if err := os.WriteFile(installed, body, 0700); err != nil { t.Fatal(err) } raw, err := unitsFS.ReadFile("units/" + claimControlUnit) if err != nil { t.Fatal(err) } for _, mode := range []string{"off", "on"} { t.Run(mode, func(t *testing.T) { unit := filepath.Base(root) + "-" + mode + ".service" content := claimUnitContent(claimControlUnit, raw, claimUnitSettings{enabled: mode == "on", data: data, opt: opt}) var lines []string for _, line := range strings.Split(string(content), "\n") { if strings.HasPrefix(line, "Requires=") || strings.HasPrefix(line, "After=") || strings.HasPrefix(line, "ExecStart=") || strings.HasPrefix(line, "EnvironmentFile=") { continue } lines = append(lines, line) } content = []byte(strings.Join(lines, "\n") + fmt.Sprintf("\nType=oneshot\nTimeoutStartSec=45s\nExecStart=%s -test.run=^TestRootServingParkUnitPayload$ -test.v -test.timeout=30s\nEnvironment=SERVING_RUNNER_DATA=%s SERVING_RUNNER_AUTHORITY=%s SERVING_RUNNER_MODE=%s\nIPAddressDeny=any\n", installed, data, authority, mode)) path := filepath.Join("/run/systemd/system", unit) file, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0644) if err != nil { t.Fatal(err) } _, writeErr := file.Write(content) closeErr := file.Close() t.Cleanup(func() { ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() if out, err := exec.CommandContext(ctx, "systemctl", "stop", unit).CombinedOutput(); err != nil { t.Errorf("stop test unit: %v %s", err, out) } if err := os.Remove(path); err != nil { t.Error(err) } if out, err := exec.CommandContext(ctx, "systemctl", "daemon-reload").CombinedOutput(); err != nil { t.Errorf("reload after cleanup: %v %s", err, out) } }) if err := errors.Join(writeErr, closeErr); err != nil { t.Fatal(err) } servingRunnerSystemctl(t, "daemon-reload") servingRunnerSystemctl(t, "start", unit) if got := strings.TrimSpace(servingRunnerSystemctl(t, "show", unit, "--property=Result", "--value")); got != "success" { t.Fatal(got) } if got := strings.TrimSpace(servingRunnerSystemctl(t, "show", unit, "--property=ExecMainStatus", "--value")); got != "0" { t.Fatal(got) } }) } } const servingRebootRoot = "/var/lib/carlos-rootrunner-serving-reboot" type servingRebootWitness struct { Boot string Record servingParkRecord } // The runner reboots between these phases. Raw history stays unchanged while // fresh persistent evidence permits current-boot completion and reaping. func TestRootRebootServingParkChangedMountPhase1(t *testing.T) { servingRootRunner(t) if err := os.Mkdir(servingRebootRoot, 0700); err != nil { t.Fatal(err) } data := filepath.Join(servingRebootRoot, "data") var grow func() if strings.Contains(t.Name(), "GrownDisk") { grow = servingRunnerGrownDisk(t, data, true) } servingRunnerMount(t, data) state, err := rootstate.Open(filepath.Join(servingRebootRoot, "authority"), true, false) if err != nil { t.Fatal(err) } defer state.Close() r := servingRunnerPark(t, data, state) servingRunnerTopologyDiagnostic(t, newServingParkFS(data, state)) boot, err := os.ReadFile("/proc/sys/kernel/random/boot_id") if err != nil { t.Fatal(err) } if err := (&servingParkLedger{state: state}).write("reboot.json", servingRebootWitness{Boot: strings.TrimSpace(string(boot)), Record: r}); err != nil { t.Fatal(err) } if grow != nil { grow() } } func TestRootRebootServingParkChangedMountPhase2(t *testing.T) { servingRootRunner(t) state, err := rootstate.Open(filepath.Join(servingRebootRoot, "authority"), false, false) if err != nil { t.Fatal(err) } defer state.Close() body, err := state.Read("reboot.json") if err != nil { t.Fatal(err) } var saved servingRebootWitness if err := json.Unmarshal(body, &saved); err != nil { t.Fatal(err) } boot, err := os.ReadFile("/proc/sys/kernel/random/boot_id") if err != nil || saved.Boot == "" || strings.TrimSpace(string(boot)) == saved.Boot { t.Fatal("phase 2 requires an actual intervening reboot") } t.Cleanup(func() { if err := os.RemoveAll(servingRebootRoot); err != nil { t.Error(err) } }) data := filepath.Join(servingRebootRoot, "data") if strings.Contains(t.Name(), "GrownDisk") { servingRunnerGrownDisk(t, data, false) } // Mount IDs can be reused across boots. Add bind layers until this fixture // has a different ID, instead of assuming reboot alone changes the number. fs := newServingParkFS(data, state) for i := 0; i < 8; i++ { servingRunnerMount(t, data) parent, id, err := fs.parent() if err != nil { t.Fatal(err) } parent.Close() if id.Mount != saved.Record.Source.Parent.Mount { break } if i == 7 { t.Fatal("could not create changed mount topology") } } before, err := state.Read("namespace.json") if err != nil { t.Fatal(err) } originalNamespace := append([]byte(nil), before...) // Prove independent recognition after a real reboot before exercising the // old-authority refusals below. No namespace or retirement is rewritten. parentBefore, err := state.Read("namespace-parent.json") if err != nil { t.Fatal(err) } servingRunnerTopologyDiagnostic(t, fs) observation, err := fs.inspectRebootTopology(t.Context()) if err != nil || observation.BootID != strings.TrimSpace(string(boot)) || observation.Previous.BootID != saved.Boot || observation.CurrentParent.Mount == observation.Previous.Parent.Mount { t.Fatal("persistent topology inspection after actual reboot", observation, err) } for _, replacement := range []string{"directory-bind", "filesystem"} { t.Run(replacement, func(t *testing.T) { path := filepath.Join(data, "instances", ".serving-parked") if replacement == "directory-bind" { other := filepath.Join(data, "replacement") if err := os.Mkdir(other, 0700); err != nil { t.Fatal(err) } if err := unix.Mount(other, path, "", unix.MS_BIND, ""); err != nil { t.Fatal(err) } } else if err := unix.Mount("tmpfs", path, "tmpfs", unix.MS_NOSUID|unix.MS_NODEV, "size=1m,mode=0700"); err != nil { t.Fatal(err) } t.Cleanup(func() { if err := unix.Unmount(path, 0); err != nil { t.Error(err) } }) if _, err := fs.inspectRebootTopology(t.Context()); err == nil { t.Fatal("replacement mount accepted across reboot") } if body, err := state.Read("namespace.json"); err != nil || !bytes.Equal(body, before) { t.Fatal("replacement refusal changed authority", err) } }) } if _, err := fs.inspectRebootTopology(t.Context()); err != nil { t.Fatal("original mount after replacement controls", err) } parentAfter, err := state.Read("namespace-parent.json") if err != nil || !bytes.Equal(parentBefore, parentAfter) { t.Fatal("inspection rewrote parent authority", err) } beforeRecord, err := state.Read("retirement-" + saved.Record.Authorization.ID + ".json") if err != nil { t.Fatal(err) } recordObservation, err := fs.inspectRebootRecord(t.Context(), saved.Record.Authorization.ID) if err != nil || recordObservation.Source.Mount != observation.CurrentParent.Mount || recordObservation.Event.Mount != observation.CurrentRoot.Mount { t.Fatal("persistent source/event/artifacts after reboot", recordObservation, err) } // Model a reused file inode by changing only the saved persistent handle. // The same device/inode/size must not suffice to identify an old artifact. corrupted := saved.Record corrupted.Source.Files[0].Persistent.Handle = "aa" ledger := servingParkLedger{state: state} if err := ledger.save(corrupted); err != nil { t.Fatal(err) } if _, err := fs.inspectRebootRecord(t.Context(), corrupted.Authorization.ID); err == nil { t.Fatal("changed persistent artifact handle accepted") } if err := state.Write("retirement-"+saved.Record.Authorization.ID+".json", beforeRecord); err != nil { t.Fatal(err) } if _, err := fs.inspectRebootRecord(t.Context(), saved.Record.Authorization.ID); err != nil { t.Fatal("restored read-only control", err) } servingRunnerPrepareReboot(t, fs, saved.Record) afterRecord, err := state.Read("retirement-" + saved.Record.Authorization.ID + ".json") if err != nil || !bytes.Equal(beforeRecord, afterRecord) { t.Fatal("record inspection changed history", err) } t.Log("persistent parent/root/source/event/file handles match across actual reboot; retirement authority remains unchanged") if err := fs.CheckParked(&saved.Record); !errors.Is(err, hostbox.ErrChanged) { t.Fatalf("old mount authority reused: %v", err) } r := saved.Record r.Deleting, r.Cause = true, "expiry" writes := 0 if err := fs.Delete(&r, func() error { writes++; return nil }); !errors.Is(err, hostbox.ErrChanged) || writes != 0 { t.Fatalf("stale mount deleted data: writes=%d err=%v", writes, err) } after, err := state.Read("namespace.json") if err != nil || !bytes.Equal(before, after) { t.Fatal("refusal changed namespace authority", err) } for suffix, want := range map[string]string{"": "database", "-wal": "WAL", "-shm": "SHM"} { path := filepath.Join(data, "instances", ".serving-parked", r.Authorization.ID, r.Authorization.Successor.Host+".db"+suffix) got, err := os.ReadFile(path) if err != nil || string(got) != want { t.Fatalf("reboot lost retained %s: %q %v", suffix, got, err) } } // The kernel need not recycle the number in this run. Normalize only this // fixture's saved device and mount numbers to the actual new mount, leaving the real // pre-reboot BootID untouched. A mount-number-only guard would now accept it. parent, actual, err := fs.parent() if err != nil { t.Fatal(err) } parent.Close() var namespace servingParkNamespace if servingParkDecode(before, &namespace) != nil || namespace.BootID != saved.Boot { t.Fatal("phase 1 did not persist its actual boot") } if actual.Inode != saved.Record.Source.Parent.Inode { t.Fatal("reboot fixture changed its underlying parent identity") } namespace.Parent.Mount, namespace.Root.Mount = actual.Mount, actual.Mount namespace.Parent.Device, namespace.Root.Device = actual.Device, actual.Device if err := (&servingParkLedger{state: state}).write("namespace.json", namespace); err != nil { t.Fatal(err) } r = saved.Record r.Authorization.Directory.Mount = actual.Mount r.Source.Directory.Mount, r.Source.Parent.Mount = actual.Mount, actual.Mount r.Event.Mount, r.Namespace.Mount = actual.Mount, actual.Mount r.Authorization.Directory.Device = actual.Device r.Source.Directory.Device, r.Source.Parent.Device = actual.Device, actual.Device r.Event.Device, r.Namespace.Device = actual.Device, actual.Device for i := range r.Source.Files { if r.Source.Files[i].Present { r.Source.Files[i].Mount = actual.Mount r.Source.Files[i].Device = actual.Device } } before, err = state.Read("namespace.json") if err != nil { t.Fatal(err) } if err := fs.CheckParked(&r); !errors.Is(err, hostbox.ErrChanged) { t.Fatal("recycled mount number accepted across actual reboot", err) } writes = 0 if err := fs.Delete(&r, func() error { writes++; return nil }); !errors.Is(err, hostbox.ErrChanged) || writes != 0 { t.Fatal("recycled mount number authorized deletion", writes, err) } after, err = state.Read("namespace.json") if err != nil || !bytes.Equal(before, after) { t.Fatal("recycled-number refusal rewrote authority", err) } for suffix, want := range map[string]string{"": "database", "-wal": "WAL", "-shm": "SHM"} { body, err := os.ReadFile(filepath.Join(data, "instances", ".serving-parked", r.Authorization.ID, r.Authorization.Successor.Host+".db"+suffix)) if err != nil || string(body) != want { t.Fatal("recycled-number refusal changed parked data", suffix, err) } } // Read-only control: with only the fixture boot changed, every physical // identity must pass. Otherwise the previous refusals could have another // cause and would not witness the boot boundary. Never delete this control. namespace.BootID = strings.TrimSpace(string(boot)) if err := (&servingParkLedger{state: state}).write("namespace.json", namespace); err != nil { t.Fatal(err) } r.Source.BootID = strings.TrimSpace(string(boot)) controlErr := fs.CheckParked(&r) if err := state.Write("namespace.json", before); err != nil { t.Fatal(err) } if controlErr != nil { t.Fatal("numeric-reuse control has another identity mismatch", controlErr) } if err := state.Write("namespace.json", originalNamespace); err != nil { t.Fatal(err) } ledger.fs, ledger.now = fs, time.Now observe, err := fs.inspectRebootRecord(t.Context(), saved.Record.Authorization.ID) if err != nil { t.Fatal(err) } a := saved.Record.activeAuthorization() active := &servingParkRebootAuthority{ClaimETag: "runner-claim", Actor: hostbox.Actor{Box: a.Successor.Box, Region: a.Successor.Region, Enrollment: "runner-enrollment", Revision: 1, Binding: "1", Eligible: true}, Directory: observe.Source, Tags: a.TagsFrom} if err := ledger.withNamespace(t.Context(), func() error { return ledger.commitRebootLocked(t.Context(), saved.Record, active) }); err != nil { t.Fatal("commit after actual reboot", err) } current, err := ledger.read(a.ID) if err != nil { t.Fatal(err) } if current.Source.BootID != strings.TrimSpace(string(boot)) || current.Source.Parent.Mount != actual.Mount { t.Fatal("wrong committed topology") } if err := fs.CheckParked(¤t); err != nil { t.Fatal("current park check", err) } if err := ledger.completeLocked(t.Context(), a.ID, a.Successor); err != nil { t.Fatal("complete", err) } current, err = ledger.read(a.ID) if err != nil { t.Fatal(err) } ledger.now = func() time.Time { return current.ReapAfter } reapErr := ledger.reapLocked(t.Context(), a.ID) if strings.Contains(t.Name(), "GrownDisk") { if !errors.Is(reapErr, hostbox.ErrChanged) || current.Source.Volume != "" { t.Fatal("serial-less loop gained deletion authority", reapErr) } for _, item := range current.Source.Files { if item.Present { if _, err := os.Stat(filepath.Join(data, "instances", ".serving-parked", a.ID, item.Name)); err != nil { t.Fatal("serial-less park lost bytes", err) } } } t.Log("grown filesystem recovered and completed; no independent loop volume serial, so reaping retained every artifact") } else if reapErr != nil { t.Fatal("verified reap after reboot", reapErr) } historical, err := ledger.readHistorical(a.ID) if err != nil || historical.Authorization != saved.Record.Authorization || historical.Source.BootID != saved.Record.Source.BootID || historical.ReapAfter != saved.Record.ReapAfter { t.Fatal("changed history", err) } if !strings.Contains(t.Name(), "GrownDisk") { t.Log("committed current mount identities and reaped the exact parked artifacts after an actual reboot") } } // Installation, census and stopped-writer permission remain fixture evidence. // This checks durable preparation and replay under real unit/mount constraints. func servingRunnerPrepareReboot(t *testing.T, fs *servingParkFS, r servingParkRecord) { t.Helper() ledger := servingParkLedger{state: fs.state, fs: fs, now: time.Now} observe, err := fs.inspectRebootRecord(t.Context(), r.Authorization.ID) if err != nil { t.Fatal(err) } a := r.activeAuthorization() active := &servingParkRebootAuthority{ClaimETag: "runner-claim", Actor: hostbox.Actor{Box: a.Successor.Box, Region: a.Successor.Region, Enrollment: "runner-enrollment", Revision: 1, Binding: "1", Eligible: true}, Directory: observe.Source, Tags: a.TagsFrom} prepare := func() error { return ledger.withNamespace(t.Context(), func() error { return ledger.prepareRebootLocked(t.Context(), r, active) }) } if err := prepare(); err != nil { t.Fatal("durable reboot preparation", err) } name := servingNamespacePreparationName(a.ID, observe.Topology) before, err := fs.state.Read(name) if err != nil { t.Fatal(err) } if err := prepare(); err != nil { t.Fatal("durable reboot preparation replay", err) } after, err := fs.state.Read(name) if err != nil || !bytes.Equal(before, after) { t.Fatal("preparation replay changed evidence", err) } if err := fs.recoverRecordedNamespace(t.Context()); err == nil { t.Fatal("preparation granted old-boot namespace authority") } } // Run the existing privileged acceptance batch and the actual reboot in one // runner allocation. Each batch test still runs by its original top-level name. func TestRootRebootServingParkAcceptancePhase1(t *testing.T) { servingRootRunner(t) binary, err := os.Executable() if err != nil { t.Fatal(err) } pattern := "^(TestRootServingParkAuthenticatedPeer|TestRootServingParkAuthenticatedHandover|TestRootServingParkStoppedWriterHandover|TestRootServingParkColdHandoverBoundary|TestRootServingParkNestedBindRestore|TestRootServingParkInstalledUnit|TestRootServingParkQuarantine|TestRootServingNamespaceStartup|TestRootServingParkUnrecordedReplacement|TestRootServingNamespaceUnrecordedStartup|TestRootServingParkDisplacedRoot|TestRootServingParkCollection)$" cmd := exec.CommandContext(t.Context(), binary, "-test.run="+pattern, "-test.v", "-test.count=1", "-test.timeout=8m") cmd.Env = os.Environ() out, err := cmd.CombinedOutput() t.Log(string(out)) if err != nil { t.Error(err) } TestRootRebootServingParkChangedMountPhase1(t) } func TestRootRebootServingParkAcceptancePhase2(t *testing.T) { TestRootRebootServingParkChangedMountPhase2(t) } func servingRunnerTopologyDiagnostic(t *testing.T, fs *servingParkFS) { t.Helper() body, err := fs.state.Read("namespace.json") if err != nil { t.Fatal(err) } t.Logf("recorded topology: %s", body) parent, pid, err := fs.parent() if err != nil { t.Fatal(err) } defer parent.Close() root, rid, err := fs.privateDir(parent, ".serving-parked") if err != nil { t.Fatal(err) } defer root.Close() boot, err := fs.currentBoot() if err != nil { t.Fatal(err) } topology, err := fs.captureTopology(parent, root, servingParkNamespace{Parent: pid, Root: rid, BootID: boot}) if err != nil { t.Fatal(err) } t.Logf("current topology: boot=%s parent=%+v root=%+v persistent=%+v", boot, pid, rid, topology) out, err := exec.CommandContext(t.Context(), "findmnt", "-n", "-o", "UUID,MAJ:MIN,SOURCE", "-T", filepath.Join(fs.data, "instances")).CombinedOutput() if err != nil { t.Fatal(err, string(out)) } t.Logf("current filesystem UUID/device/source: %s", out) } func TestRootFilesSeedRealOriginPreparation(t *testing.T) { runRootFilesSeedRealOriginPreparation(t, false) } func TestRootFilesSeedRealOriginTenant(t *testing.T) { runRootFilesSeedRealOriginPreparation(t, true) } func TestRootFilesSeedWholeExecutorKilledRerun(t *testing.T) { runRootFilesSeedRealOriginPreparation(t, true) } func runRootFilesSeedRealOriginPreparation(t *testing.T, live bool) { servingRootRunner(t) if os.Getenv("PLAN14_ROOT_FIXTURE") == t.Name() { if err := unix.Mount("", "/", "", unix.MS_REC|unix.MS_PRIVATE, ""); err != nil { t.Fatal(err) } if !plan14RootFixture(t) { t.Fatal("root child did not enter its scratch root") } if live { runFilesSeedPreparationWithLive(t, "real-seed", "same-controller", []bool{true}, []filesSeedLiveWitness{rootFilesSeedWitnessFor(t, "/tenant.test")}) } else { runFilesSeedOrdinaryPreparation(t, "real-seed", "same-controller", true) } return } root := t.TempDir() for _, name := range []string{"scratch", "proc", "foreign-dir"} { if err := os.Mkdir(filepath.Join(root, name), 0755); err != nil { t.Fatal(err) } } if err := os.WriteFile(filepath.Join(root, "foreign"), nil, 0644); err != nil { t.Fatal(err) } binary, err := os.Executable() if err != nil { t.Fatal(err) } if live { body, err := os.ReadFile(binary) if err != nil { t.Fatal(err) } if err := os.WriteFile(filepath.Join(root, "tenant.test"), body, 0755); err != nil { t.Fatal(err) } } cmd := exec.CommandContext(t.Context(), binary, "-test.run=^"+t.Name()+"$", "-test.v", "-test.timeout=3m") cmd.Env = append(os.Environ(), "PLAN14_ROOT_FIXTURE="+t.Name(), "PLAN14_ROOT_DIR="+root) cmd.SysProcAttr = &syscall.SysProcAttr{Cloneflags: unix.CLONE_NEWNS} if out, err := cmd.CombinedOutput(); err != nil { t.Fatalf("real-root mounted restore: %v\n%s", err, out) } else { t.Log(string(out)) } }