//go:build linux && rootrunner package host import ( "context" "encoding/json" "os" "os/exec" "path/filepath" "testing" "time" "github.com/carlosframework/platform/internal/hostbox" "github.com/carlosframework/platform/internal/hostclaim" "github.com/carlosframework/platform/internal/hostguard" "github.com/carlosframework/platform/internal/provision" "github.com/carlosframework/platform/internal/registry" "github.com/carlosframework/platform/internal/release" "github.com/carlosframework/platform/internal/rootparticipation" "github.com/carlosframework/platform/internal/rootstate" "golang.org/x/sys/unix" ) func TestPlan14GenerationRequiredArtifactsRootRunner(t *testing.T) { plan14RequiredArtifactsRootRunner(t, false) } func TestPlan14BootLocalRecoveryRootRunner(t *testing.T) { plan14RequiredArtifactsRootRunner(t, true) } func plan14RequiredArtifactsRootRunner(t *testing.T, local bool) { const childKey = "PLAN14_E30_REQUIRED_CHILD" dir := os.Getenv(childKey) if dir == "" { generationRunnerRegistryUser(t) var err error dir, err = os.MkdirTemp("/run", "plan14-generation-rows-") if err != nil { t.Fatal(err) } t.Cleanup(func() { if err := os.RemoveAll(dir); err != nil { t.Error(err) } }) if err := os.Chmod(dir, 0755); err != nil { t.Fatal(err) } binary, err := os.Executable() if err != nil { t.Fatal(err) } cmd := exec.CommandContext(t.Context(), "unshare", "--mount", "--pid", "--fork", "--mount-proc", binary, "-test.run=^"+t.Name()+"$", "-test.v", "-test.timeout=2m") cmd.Dir = dir cmd.Env = append(os.Environ(), childKey+"="+dir) if out, err := cmd.CombinedOutput(); err != nil { t.Fatalf("isolated required-artifact census: %v\n%s", err, out) } else { t.Logf("%s", out) } return } if os.Geteuid() != 0 { t.Fatal("designated runner must execute as root") } if err := unix.Mount("", "/", "", unix.MS_REC|unix.MS_PRIVATE, ""); err != nil { t.Fatal(err) } private := filepath.Join(dir, "varlib") if err := os.Mkdir(private, 0755); err != nil { t.Fatal(err) } if err := unix.Mount(private, "/var/lib", "", unix.MS_BIND, ""); err != nil { t.Fatal(err) } t.Cleanup(func() { if err := unix.Unmount("/var/lib", 0); err != nil { t.Error(err) } }) t.Setenv("CARLOS_HOST_CLAIM", "true") t.Setenv("CARLOS_OPT_DIR", dir) t.Setenv("CARLOS_DATA_DIR", filepath.Join(dir, "data")) t.Setenv("CARLOS_DEPLOYMENT_BUCKET", "") t.Setenv("CARLOS_DEPLOYMENT_DIR", filepath.Join(dir, "deployment")) for _, path := range []string{filepath.Join(dir, "versions"), dataDir(), os.Getenv("CARLOS_DEPLOYMENT_DIR")} { if err := os.Mkdir(path, 0755); err != nil { t.Fatal(err) } } uid, gid := generationRunnerRegistryUser(t) if err := os.Chown(dataDir(), uid, gid); err != nil { t.Fatal(err) } reg, err := openRegistryChild(filepath.Join(dataDir(), "registry.db"), uid, gid) if err != nil { t.Fatal(err) } // Join the fixture writer before the production kernel scan. if err := reg.Close(); err != nil { t.Fatal(err) } for _, path := range []string{rootparticipation.Namespace, "/var/lib/carlos-claims/authority/fences", release.RootGenerationsDir} { state, err := rootstate.Open(path, true, false) if err != nil { t.Fatal(err) } state.Close() } write := func(path, name string, v any) { t.Helper() state, err := rootstate.Open(path, false, false) if err != nil { t.Fatal(err) } defer state.Close() raw, err := json.Marshal(v) if err == nil { err = state.Write(name, raw) } if err != nil { t.Fatal(err) } } write(rootparticipation.Namespace, "census.json", rootparticipation.Census{Schema: 1, Box: "runner", Enrollment: "en", Migration: 1, Revision: 1, LaunchRevision: 1, Complete: true}) if info, err := os.Stat("/var/lib/carlos-claims"); err != nil || info.Mode().Perm() != 0700 { t.Fatalf("private journal directory: %v %v", info, err) } generationRunnerEmptyRegistry(t, dir, uid, gid) generationRunnerCreation(t, dir) rec := provision.Record{Schema: 1, Host: "retained.example", Account: "bab", App: "shop", Channel: "stable", CreatedAt: time.Date(2026, 10, 1, 0, 0, 0, 0, time.UTC)} store, err := openDeployStore(t.Context()) if err != nil { t.Fatal(err) } plan14PutRecord(t, store, rec) inc, err := hostclaim.Incarnation(rec.Account, rec.App, rec.Host, rec.CreatedAt) if err != nil { t.Fatal(err) } in := hostguard.RootInstallation{Artifacts: hostguard.InstallationArtifacts{Schema: 1, StateDirectory: true, Timers: "backup"}, Host: rec.Host, Incarnation: inc, Stamp: 5, Account: rec.Account, App: rec.App, Record: provision.RecordKey(rec.Account, rec.App, rec.Host), Box: "runner", Enrollment: "en", Migration: 1, InventoryRevision: 1, Empty: true} write("/var/lib/carlos-claims/authority/fences", in.Host+".json", map[string]any{ "Schema": 1, "Installation": in, "Retired": []hostguard.Captured{{Incarnation: inc, Stamp: 2}}, "State": map[string]any{"v": 1, "host": in.Host, "incarnation": inc, "active_stamp": 5, "retired_stamps": []uint64{2}}, }) path := provision.InstanceDir(dataDir(), in.Host) if err := os.MkdirAll(path, 0700); err != nil { t.Fatal(err) } var st unix.Stat_t if err := unix.Stat(path, &st); err != nil { t.Fatal(err) } rel, err := filepath.Rel(dataDir(), path) if err != nil { t.Fatal(err) } payload := hostbox.StopInventory{Complete: true, Host: in.Host, Incarnation: inc, InstallationStamp: 2, Box: in.Box, Enrollment: in.Enrollment, DirectoryRevision: "1", Entries: []hostbox.StopInventoryEntry{ {Kind: "route", ArtifactID: "route", ManagedRoot: "registry", RelativePath: in.Host, RowIdentity: "row", Ownership: "owner", Materialization: "materialized"}, {Identity: inventoryIdentityFixture(t, dataDir(), rel), Kind: "path", ArtifactID: "state", ManagedRoot: "data", RelativePath: rel, Ownership: "owner", Materialization: "materialized", Device: uint64(st.Dev), Inode: st.Ino, Type: unix.S_IFDIR}, }} journal, err := hostbox.OpenLaunchJournal("/var/lib/carlos-claims/journals.db") if err != nil { t.Fatal(err) } t.Cleanup(func() { journal.Close() }) service, err := hostbox.NewJournalService(journal) if err != nil { t.Fatal(err) } current := payload current.InstallationStamp, current.Entries = in.Stamp, nil if err := service.Inventory.WriteStopInventory(t.Context(), current); err != nil { t.Fatal(err) } put := func(p hostbox.StopInventory) { t.Helper() if err := service.Inventory.WriteStopInventory(t.Context(), p); err != nil { t.Fatal(err) } } payload.Entries = append(payload.Entries, hostbox.StopInventoryEntry{Kind: "timer", ArtifactID: "timer", ManagedRoot: "registry", RelativePath: in.Host + "/backup", RowIdentity: "timer", Ownership: "owner", Materialization: "materialized"}) put(payload) owner := registry.ClaimOwnership{ClaimIncarnation: inc, ClaimStamp: 2, ClaimOwner: "owner", ClaimRowID: "row"} route := registry.Route{Host: in.Host, ClaimOwnership: owner} timer := registry.Timer{Host: in.Host, Name: "backup", Kind: "cron", Registration: "timer", ClaimOwnership: owner} timer.ClaimRowID = "timer" rows := func(rt registry.Route, timers []registry.Timer) { t.Helper() writer, err := openRegistryChild(filepath.Join(dataDir(), "registry.db"), uid, gid) if err != nil { t.Fatal(err) } defer writer.Close() if err := writer.Delete(in.Host); err != nil { t.Fatal(err) } if err := writer.ReplaceTimers(in.Host, timers); err != nil { t.Fatal(err) } if rt.Host != "" { if err := writer.Upsert(rt); err != nil { t.Fatal(err) } } } rows(route, []registry.Timer{timer}) certificate := func() release.GenerationCompleteness { t.Helper() state, err := rootstate.Open(release.RootGenerationsDir, false, false) if err != nil { t.Fatal(err) } defer state.Close() c, err := (&release.RootGenerations{Store: state}).Completeness() if err != nil { t.Fatal(err) } return c } source := productionRootGenerationPins() if local { t.Setenv("CARLOS_DEPLOYMENT_DIR", "") t.Setenv("CARLOS_DEPLOYMENT_BUCKET", "") if err := registerProductionLaunchVerifier(); err != nil { t.Fatal(err) } source = generationPinSource{open: func() (rootstate.Store, error) { return rootstate.Open(release.RootGenerationsDir, false, false) }, reconcile: (generationReconciler{read: productionLocalGenerationInventory, kernel: productionGenerationKernelCensus}).reconcile} } pinsUnderRelease := func() (release.StagedPaths, error) { var pins release.StagedPaths err := withClaimRootRelease(t.Context(), true, func() (rootstate.Store, error) { return rootstate.Open(rootReleasePrivate, false, false) }, func() error { var err error pins, err = source.StagedPins(t.Context(), dir) return err }) return pins, err } floorIdle := func() error { l, err := rootparticipation.Open() if err != nil { return err } defer l.State.Close() return l.WithLaunchExclusion(t.Context(), func(ctx context.Context) error { return l.WithTransaction(ctx, rootFloorDowngradeIdle) }) } accept := func() { t.Helper() if pins, err := pinsUnderRelease(); err != nil || len(pins) != 0 || !certificate().Complete { snapshot, readErr := generationInventoryAt(t.Context(), rootparticipation.Namespace, "/var/lib/carlos-claims/authority/fences", "/var/lib/carlos-claims/journals.db") t.Logf("root inventory: %v", readErr) if readErr == nil { snapshot, readErr = generationRequiredInventory(t.Context(), store, snapshot, productionGenerationCreation) t.Logf("required artifacts: %v", readErr) } if readErr == nil { _, readErr = productionGenerationRegistryInventory(t.Context(), snapshot) t.Logf("registry artifacts: %v", readErr) } _, _, kernelErr := productionGenerationKernelCensus(t.Context()) t.Logf("kernel census: %v", kernelErr) t.Fatalf("production required artifacts: %v %v", pins, err) } } accept() if local && floorIdle() != nil { t.Fatal("local floor recovery refused complete offline obligations") } if local { state, err := rootstate.Open(rootparticipation.Namespace, false, false) if err != nil { t.Fatal(err) } defer state.Close() l := &rootparticipation.Ledger{State: state, VerifyLaunches: (rootLaunchVerifier{exclude: participationRelease, open: func() (rootstate.Store, error) { return rootstate.Open(release.RootGenerationsDir, false, false) }, reconcile: generationReconciler{read: productionLocalGenerationInventory, kernel: productionGenerationKernelCensus}}).verify, ExcludeLaunches: participationRelease.With} a := &systemApplier{participation: l, dataDir: dataDir()} if err := a.controllersRecovered(context.Background()); err != nil { t.Fatal("boot participation recovery without deployment storage", err) } } for _, omit := range []string{"route", "state", "timer", "historical-state", "timer-both"} { t.Run(omit, func(t *testing.T) { initial := certificate() omitEntry := omit if omit == "historical-state" { omitEntry = "state" changed := rec changed.Kind, changed.Addr = "service", "127.0.0.1:8080" plan14PutRecord(t, store, changed) } if omit == "timer-both" { omitEntry = "timer" rows(route, nil) } missing := payload missing.Entries = nil for _, entry := range payload.Entries { if entry.ArtifactID != omitEntry { missing.Entries = append(missing.Entries, entry) } } put(missing) if local && floorIdle() == nil { t.Fatal("local floor recovery accepted omitted obligation", omit) } if pins, err := pinsUnderRelease(); err == nil || pins != nil { t.Fatalf("production omitted %s certified: %v %v", omit, pins, err) } if c := certificate(); c.Complete || c.Revision <= initial.Revision { t.Fatal("omission retained the completeness certificate") } if err := generationPayloadFiles(t.Context(), dataDir(), []hostbox.GenerationPayload{{Inventory: payload}}); err != nil { t.Fatalf("omission damaged retained payload: %v", err) } put(payload) plan14PutRecord(t, store, rec) rows(route, []registry.Timer{timer}) accept() }) } for _, attack := range []string{"unknown-route-host", "unknown-timer-host", "route-missing", "route-owner", "timer-missing", "timer-registration", "extra-timer", "malformed-database"} { t.Run(attack, func(t *testing.T) { initial := certificate() rt, tm := route, timer switch attack { case "route-missing": rt = registry.Route{} case "route-owner": rt.ClaimOwner = "forged" case "timer-registration": tm.Registration = "forged" } timers := []registry.Timer{tm} if attack == "timer-missing" { timers = nil } if attack == "extra-timer" { extra := timer extra.Name, extra.Registration, extra.ClaimRowID = "extra", "extra", "extra" timers = append(timers, extra) } cleanupUnknown := func() {} rows(rt, timers) if attack == "unknown-route-host" || attack == "unknown-timer-host" { writer, err := openRegistryChild(filepath.Join(dataDir(), "registry.db"), uid, gid) if err != nil { t.Fatal(err) } if attack == "unknown-route-host" { err = writer.Upsert(registry.Route{Host: "omitted.example"}) } else { err = writer.ReplaceTimers("omitted.example", []registry.Timer{{Host: "omitted.example", Name: "paused", Kind: "cron", NextAt: 0}}) } closeErr := writer.Close() if err != nil || closeErr != nil { t.Fatalf("unknown host setup: %v %v", err, closeErr) } cleanupUnknown = func() { writer, err := openRegistryChild(filepath.Join(dataDir(), "registry.db"), uid, gid) if err != nil { t.Fatal(err) } defer writer.Close() if err := writer.Delete("omitted.example"); err != nil { t.Fatal(err) } if err := writer.ReplaceTimers("omitted.example", nil); err != nil { t.Fatal(err) } } } path := filepath.Join(dataDir(), "registry.db") if attack == "malformed-database" { if err := os.Rename(path, path+".saved"); err != nil { t.Fatal(err) } if err := os.WriteFile(path, []byte("not sqlite"), 0644); err != nil { t.Fatal(err) } if err := os.Chown(path, uid, gid); err != nil { t.Fatal(err) } } if pins, err := pinsUnderRelease(); err == nil || pins != nil { t.Fatalf("production %s certified: %v %v", attack, pins, err) } if c := certificate(); c.Complete || c.Revision <= initial.Revision { t.Fatal("row mismatch retained certificate") } if attack == "malformed-database" { if err := os.Remove(path); err != nil { t.Fatal(err) } if err := os.Rename(path+".saved", path); err != nil { t.Fatal(err) } } cleanupUnknown() rows(route, []registry.Timer{timer}) accept() }) } } func generationRunnerEmptyRegistry(t *testing.T, opt string, uid, gid int) { t.Helper() journal, err := hostbox.OpenLaunchJournal("/var/lib/carlos-claims/journals.db") if err != nil { t.Fatal(err) } if _, err := hostbox.NewJournalService(journal); err != nil { t.Fatal(err) } if err := journal.Close(); err != nil { t.Fatal(err) } source := productionRootGenerationPins() check := func(t *testing.T, refuse bool) { t.Helper() err := withClaimRootRelease(t.Context(), true, func() (rootstate.Store, error) { return rootstate.Open(rootReleasePrivate, false, false) }, func() error { pins, err := source.StagedPins(t.Context(), opt) if len(pins) != 0 { t.Fatalf("empty fixture has pins: %v", pins) } return err }) if (err != nil) != refuse { t.Fatalf("empty-root registry census: refuse=%v error=%v", refuse, err) } state, err := rootstate.Open(release.RootGenerationsDir, false, false) if err != nil { t.Fatal(err) } defer state.Close() c, err := (&release.RootGenerations{Store: state}).Completeness() if err != nil || c.Complete == refuse { t.Fatalf("empty-root certificate: %+v %v", c, err) } } check(t, false) for _, kind := range []string{"route", "timer"} { t.Run("empty-root-unknown-"+kind, func(t *testing.T) { writer, err := openRegistryChild(filepath.Join(dataDir(), "registry.db"), uid, gid) if err != nil { t.Fatal(err) } if kind == "route" { err = writer.Upsert(registry.Route{Host: "omitted.example"}) } else { err = writer.ReplaceTimers("omitted.example", []registry.Timer{{Host: "omitted.example", Name: "paused", Kind: "cron", NextAt: 0}}) } closeErr := writer.Close() if err != nil || closeErr != nil { t.Fatalf("unknown host setup: %v %v", err, closeErr) } check(t, true) writer, err = openRegistryChild(filepath.Join(dataDir(), "registry.db"), uid, gid) if err != nil { t.Fatal(err) } if err := writer.Delete("omitted.example"); err != nil { t.Fatal(err) } if err := writer.ReplaceTimers("omitted.example", nil); err != nil { t.Fatal(err) } if err := writer.Close(); err != nil { t.Fatal(err) } check(t, false) }) } } func generationRunnerCreation(t *testing.T, opt string) { t.Helper() ctx := t.Context() store, err := openDeployStore(ctx) if err != nil { t.Fatal(err) } rec := provision.Record{Schema: 1, Host: "new.example", Account: "bab", App: "shop", Channel: "stable", CreatedAt: time.Date(2026, 10, 1, 0, 0, 0, 0, time.UTC)} plan14PutRecord(t, store, rec) inc, err := hostclaim.Incarnation(rec.Account, rec.App, rec.Host, rec.CreatedAt) if err != nil { t.Fatal(err) } in := hostguard.RootInstallation{Artifacts: hostguard.InstallationArtifacts{Schema: 1, StateDirectory: true}, Host: rec.Host, Account: rec.Account, App: rec.App, Record: provision.RecordKey(rec.Account, rec.App, rec.Host), Incarnation: inc, Stamp: 1, Box: "runner", Enrollment: "en", Migration: 1, InventoryRevision: 1, Empty: true} fences, err := rootstate.Open("/var/lib/carlos-claims/authority/fences", false, false) if err != nil { t.Fatal(err) } defer fences.Close() raw, err := json.Marshal(map[string]any{"Schema": 1, "Installation": in, "State": map[string]any{"v": 1, "host": in.Host, "incarnation": inc, "active_stamp": in.Stamp}}) if err != nil { t.Fatal(err) } if err := fences.Write(in.Host+".json", raw); err != nil { t.Fatal(err) } journal, err := hostbox.OpenLaunchJournal("/var/lib/carlos-claims/journals.db") if err != nil { t.Fatal(err) } service, err := hostbox.NewJournalService(journal) if err != nil { t.Fatal(err) } if err := service.Inventory.WriteStopInventory(ctx, hostbox.StopInventory{Complete: true, Host: in.Host, Incarnation: inc, InstallationStamp: in.Stamp, Box: in.Box, Enrollment: in.Enrollment, DirectoryRevision: "1"}); err != nil { t.Fatal(err) } if err := journal.Close(); err != nil { t.Fatal(err) } historyState, err := rootstate.Open(release.RootHistoryDir, true, false) if err != nil { t.Fatal(err) } defer historyState.Close() history := &release.RootHistory{Store: historyState} binding := release.HistoryBinding{Host: in.Host, Account: in.Account, App: in.App} check := func(refuse bool) { t.Helper() err := withClaimRootRelease(ctx, true, func() (rootstate.Store, error) { return rootstate.Open(rootReleasePrivate, false, false) }, func() error { pins, err := productionRootGenerationPins().StagedPins(ctx, opt) if len(pins) != 0 { t.Fatalf("creation fixture has pins: %v", pins) } return err }) if (err != nil) != refuse { t.Fatalf("creation census: refuse=%v error=%v", refuse, err) } state, err := rootstate.Open(release.RootGenerationsDir, false, false) if err != nil { t.Fatal(err) } defer state.Close() c, err := (&release.RootGenerations{Store: state}).Completeness() if err != nil || c.Complete == refuse { t.Fatalf("creation certificate: %+v %v", c, err) } } check(true) if err := history.RecordCreation(binding, rootHistoryInstallation(in)); err != nil { t.Fatal(err) } check(false) names, err := historyState.Names() if err != nil || len(names) != 1 { t.Fatalf("creation files: %v %v", names, err) } saved, err := historyState.Read(names[0]) if err != nil { t.Fatal(err) } for _, attack := range []string{"erased", "corrupt", "consumed", "wrong-installation"} { t.Run("creation-"+attack, func(t *testing.T) { switch attack { case "erased", "wrong-installation": if err := historyState.Remove(names[0]); err != nil { t.Fatal(err) } if attack == "wrong-installation" { if err := history.RecordCreation(binding, "other"); err != nil { t.Fatal(err) } } case "corrupt": if err := historyState.Write(names[0], []byte("{")); err != nil { t.Fatal(err) } case "consumed": if err := history.ConsumeCreation(binding); err != nil { t.Fatal(err) } } check(true) if err := historyState.Write(names[0], saved); err != nil { t.Fatal(err) } check(false) }) } // This fixture owns the isolated stores. Remove its complete fixture before // the caller installs its separate predecessor case. if err := fences.Remove(in.Host + ".json"); err != nil { t.Fatal(err) } if err := historyState.Remove(names[0]); err != nil { t.Fatal(err) } if err := os.Remove("/var/lib/carlos-claims/journals.db"); err != nil { t.Fatal(err) } if err := store.Delete(ctx, in.Record); err != nil { t.Fatal(err) } }