//go:build linux package host import ( "context" "encoding/json" "errors" "os" "os/exec" "path/filepath" "testing" "time" "github.com/carlosframework/platform/internal/hostbox" "github.com/carlosframework/platform/internal/hostguard" "github.com/carlosframework/platform/internal/registry" "github.com/carlosframework/platform/internal/release" "github.com/carlosframework/platform/internal/rootparticipation" "github.com/carlosframework/platform/internal/rootstate" "golang.org/x/sys/unix" ) func generationParkedFixture(t *testing.T, currentBoot ...bool) (generationInventory, hostbox.RetirementJournal, string) { t.Helper() root := t.TempDir() t.Setenv("CARLOS_DATA_DIR", root) host := "parked.example" in := hostbox.StopInventory{Complete: true, Host: host, Incarnation: "inc", InstallationStamp: 1, Box: "box", Enrollment: "en", DirectoryRevision: "1"} in.Entries = []hostbox.StopInventoryEntry{ {Kind: "route", ArtifactID: "route", ManagedRoot: "registry", RelativePath: host, Ownership: "owner", RowIdentity: "row", Materialization: "materialized"}, {Kind: "timer", ArtifactID: "timer", ManagedRoot: "registry", RelativePath: host + "/backup", Ownership: "owner", RowIdentity: "timer", Materialization: "materialized"}, } path := filepath.Join(root, "instances", host) if err := os.MkdirAll(path, 0700); err != nil { t.Fatal(err) } if err := os.WriteFile(filepath.Join(path, "payload"), []byte("retained"), 0600); err != nil { t.Fatal(err) } var st unix.Stat_t if err := unix.Stat(path, &st); err != nil { t.Fatal(err) } in.Entries = append(in.Entries, hostbox.StopInventoryEntry{Identity: inventoryIdentityFixture(t, root, "instances/"+host), Kind: "path", ArtifactID: "state", ManagedRoot: "data", RelativePath: "instances/" + host, Ownership: "owner", Materialization: "materialized", Device: uint64(st.Dev), Inode: st.Ino, Type: unix.S_IFDIR}) target := hostguard.FenceTarget{Host: host, Incarnation: in.Incarnation, Stamp: 1, Session: 1, Box: in.Box, Enrollment: in.Enrollment, DirectoryRevision: in.DirectoryRevision, RosterRevision: 1, FenceID: "retire", BootID: in.Entries[len(in.Entries)-1].Identity.Boot, ControllerID: "controller", Op: "op", Phase: "retiring", ObligationID: "park"} receipt := StopReceipt{V: 1, ID: "receipt", Kind: "stopped", Host: host, Incarnation: target.Incarnation, InstallationStamp: target.Stamp, WriterSession: target.Session, Box: target.Box, Enrollment: target.Enrollment, DirectoryRevision: testReceiptNumeral(target), Binding: target.DirectoryRevision, FenceID: target.FenceID, BootID: target.BootID, ControllerID: target.ControllerID, OperationID: target.Op, Phase: target.Phase, ObligationID: target.ObligationID, EvidenceAt: time.Unix(100, 0).UTC()} raw, err := json.Marshal(receipt) if err != nil { t.Fatal(err) } inventory := hostbox.RetirementInventory{Host: host, Captured: hostguard.Captured{Incarnation: in.Incarnation, Stamp: 1}, Ownership: "owner"} for _, e := range in.Entries { inventory.Artifacts = append(inventory.Artifacts, hostbox.Artifact{Kind: hostbox.ArtifactKind(e.Kind), Root: e.ManagedRoot, Name: e.RelativePath, Owner: e.Ownership, RowID: e.RowIdentity, Identity: e.Identity, Dev: e.Device, Ino: e.Inode, Type: e.Type}) } j, err := hostbox.NewRetirementJournal(target.FenceID, inventory) if err != nil { t.Fatal(err) } j.Target = &target j.StopReceipt = raw j.Complete = true for i := range j.Steps { j.Steps[i].Done = true } view := hostguard.FenceView{Schema: 1, Host: host, Installation: hostguard.RootInstallation{Host: host, Incarnation: in.Incarnation, Stamp: 1, Box: in.Box, Enrollment: in.Enrollment, Migration: 1, InventoryRevision: 1, Account: "bab", App: "shop", Record: "apps/bab/shop/instances/" + host + ".json", Artifacts: hostguard.InstallationArtifacts{Schema: 1, StateDirectory: true, Timers: "backup"}}, Receipts: []hostguard.FenceReceipt{{Target: target, Bytes: raw, EvidenceAt: receipt.EvidenceAt}}} snapshot := generationInventory{Census: rootparticipation.Census{Schema: 1, Box: in.Box, Enrollment: in.Enrollment, Migration: 1, Revision: 1, LaunchRevision: 1, Complete: true}, Installations: []hostguard.FenceView{view}, Payloads: []hostbox.GenerationPayload{{Inventory: in, Revision: 1}}, Launches: []hostbox.LaunchInventory{{Target: target, WritersClosed: true}}} return snapshot, j, root } // Decode the durable snapshot shape, including the newly retained journal list. func generationParkedSnapshot(t *testing.T, snapshot generationInventory, journals []hostbox.RetirementJournal) generationInventory { t.Helper() raw, err := json.Marshal(snapshot) if err != nil { t.Fatal(err) } var fields map[string]json.RawMessage if err = json.Unmarshal(raw, &fields); err != nil { t.Fatal(err) } fields["Retirements"], err = json.Marshal(journals) if err != nil { t.Fatal(err) } raw, err = json.Marshal(fields) if err != nil { t.Fatal(err) } if err = json.Unmarshal(raw, &snapshot); err != nil { t.Fatal(err) } return snapshot } func TestPlan14GenerationParkedPayload(t *testing.T) { for _, attack := range []string{"complete", "predecessor", "pending", "unfinished-step", "wrong-inode", "wrong-owner", "wrong-target", "missing-receipt", "wrong-receipt", "missing-artifact", "extra-artifact", "duplicate-journal", "missing-parked", "replaced-parked", "symlink-parked", "live-row", "live-timer", "successor", "changed-during-kernel"} { t.Run(attack, func(t *testing.T) { snapshot, j, root := generationParkedFixture(t) original := filepath.Join(root, "instances", j.Inventory.Host) parked := filepath.Join(root, ".parked", j.Inventory.Captured.Incarnation, j.Inventory.Host, j.ID, "data", "instances", j.Inventory.Host) if err := os.MkdirAll(filepath.Dir(parked), 0700); err != nil { t.Fatal(err) } if err := os.Rename(original, parked); err != nil { t.Fatal(err) } reg, err := registry.Open(filepath.Join(root, "registry.db")) if err != nil { t.Fatal(err) } defer reg.Close() t.Setenv("CARLOS_HOST_CLAIM", "true") switch attack { case "predecessor", "successor": snapshot.Installations[0].Retired = []hostguard.Captured{j.Inventory.Captured} snapshot.Installations[0].Installation.Stamp = 2 snapshot.Installations[0].Installation.Empty = true current := snapshot.Payloads[0] current.Inventory.InstallationStamp = 2 current.Inventory.Entries = nil if attack == "successor" { snapshot.Installations[0].Installation.Empty = false current.Inventory.Entries = []hostbox.StopInventoryEntry{{Kind: "route", ArtifactID: "route", ManagedRoot: "registry", RelativePath: j.Inventory.Host, Ownership: "successor", RowIdentity: "next", Materialization: "materialized"}} snapshot.Launches = append(snapshot.Launches, hostbox.LaunchInventory{Target: hostguard.FenceTarget{Host: j.Inventory.Host, Incarnation: "inc", Stamp: 2, Box: "box", Enrollment: "en"}, WritersClosed: true}) if err := reg.Upsert(registry.Route{Host: j.Inventory.Host, ClaimOwnership: registry.ClaimOwnership{ClaimIncarnation: "inc", ClaimStamp: 2, ClaimOwner: "successor", ClaimRowID: "next"}}); err != nil { t.Fatal(err) } } snapshot.Payloads = append(snapshot.Payloads, current) case "pending": j.Complete = false case "unfinished-step": j.Steps[0].Done = false case "wrong-inode": j.Inventory.Artifacts[0].Ino++ case "wrong-owner": j.Inventory.Ownership = "other" case "wrong-target": j.Target.Session++ case "missing-receipt": snapshot.Installations[0].Receipts = nil case "wrong-receipt": j.StopReceipt = []byte("forged") case "missing-artifact": j.Inventory.Artifacts = j.Inventory.Artifacts[:len(j.Inventory.Artifacts)-1] case "extra-artifact": j.Inventory.Artifacts = append(j.Inventory.Artifacts, hostbox.Artifact{Kind: hostbox.ArtifactTimer, Root: "registry", Name: j.Inventory.Host + "/unknown", Owner: "owner", RowID: "extra"}) case "missing-parked": if err := os.Rename(parked, parked+".old"); err != nil { t.Fatal(err) } case "replaced-parked", "symlink-parked": if err := os.Rename(parked, parked+".old"); err != nil { t.Fatal(err) } if attack == "replaced-parked" { err = os.Mkdir(parked, 0700) } else { err = os.Symlink(parked+".old", parked) } if err != nil { t.Fatal(err) } case "live-row": if err := reg.Upsert(registry.Route{Host: j.Inventory.Host, ClaimOwnership: registry.ClaimOwnership{ClaimIncarnation: "inc", ClaimStamp: 1, ClaimOwner: "owner", ClaimRowID: "row"}}); err != nil { t.Fatal(err) } case "live-timer": if err := reg.ReplaceTimers(j.Inventory.Host, []registry.Timer{{Host: j.Inventory.Host, Name: "backup", Kind: "cron", Registration: "timer", ClaimOwnership: registry.ClaimOwnership{ClaimIncarnation: "inc", ClaimStamp: 1, ClaimOwner: "owner", ClaimRowID: "timer"}}}); err != nil { t.Fatal(err) } } journals := []hostbox.RetirementJournal{j} if attack == "duplicate-journal" { journals = append(journals, j) } snapshot = generationParkedSnapshot(t, snapshot, journals) v := generationReconciler{read: func(ctx context.Context) (generationInventory, error) { return generationRegistryInventory(ctx, reg, snapshot) }, kernel: func(context.Context) (string, string, error) { if attack == "changed-during-kernel" { j.Complete = false snapshot = generationParkedSnapshot(t, snapshot, []hostbox.RetirementJournal{j}) } return "boot", generationInventoryDigest("kernel"), nil }} got, err := v.reconcile(t.Context(), nil) good := attack == "complete" || attack == "predecessor" || attack == "successor" if good && (err != nil || got.Installations == "") { t.Fatalf("completed root retirement did not reconcile: %+v %v", got, err) } if !good && err == nil { t.Fatal("invalid retirement certified a snapshot") } }) } } type generationParkAuthority struct{} func (generationParkAuthority) WithCleanup(_ context.Context, _ hostguard.FenceTarget, effect func() error) error { return effect() } type generationCrashParker struct { hostbox.TypedParker crash bool } func (p generationCrashParker) Park(ctx context.Context, id string, a hostbox.Artifact) error { if err := p.TypedParker.Park(ctx, id, a); err != nil { return err } if p.crash { os.Exit(38) } return nil } type generationCrashRows struct { hostbox.RegistryRows crash bool } func (r generationCrashRows) RetireRowsJournal(ctx context.Context, id, owner string, rows []hostbox.Artifact, archive func(json.RawMessage) error) error { if err := r.RegistryRows.RetireRowsJournal(ctx, id, owner, rows, archive); err != nil { return err } if r.crash { os.Exit(38) } return nil } func generationRunPark(t *testing.T, root string, j hostbox.RetirementJournal, reg *registry.Registry, crash string) { t.Helper() j.Complete = false for i := range j.Steps { j.Steps[i].Done = false } p := hostbox.TypedParker{Roots: map[string]string{"data": root}, ParkedRoots: map[string]string{"data": root}, Incarnation: j.Inventory.Captured.Incarnation, Host: j.Inventory.Host, Retirement: j.ID} r := hostbox.TypedRetirement{JournalDir: filepath.Join(root, "retire"), Authority: generationParkAuthority{}, Filesystem: generationCrashParker{p, crash == "park"}, Rows: generationCrashRows{hostbox.RegistryRows{Registry: reg, Inventory: j.Inventory}, crash == "rows"}, Ack: func(context.Context, hostbox.RetirementJournal) error { if crash == "complete" { os.Exit(38) } return nil }} if err := r.Run(t.Context(), *j.Target, j); err != nil { t.Fatal(err) } } func TestPlan14GenerationParkedCrash(t *testing.T) { if root := os.Getenv("PLAN14_PARK_CRASH_ROOT"); root != "" { var j hostbox.RetirementJournal raw, err := os.ReadFile(filepath.Join(root, "intent.json")) if err != nil { t.Fatal(err) } if err = json.Unmarshal(raw, &j); err != nil { t.Fatal(err) } reg, err := registry.Open(filepath.Join(root, "registry.db")) if err != nil { t.Fatal(err) } defer reg.Close() generationRunPark(t, root, j, reg, os.Getenv("PLAN14_PARK_CRASH_POINT")) t.Fatal("crash boundary did not exit") } for _, point := range []string{"park", "rows", "complete"} { t.Run(point, func(t *testing.T) { snapshot, j, root := generationParkedFixture(t) t.Setenv("CARLOS_HOST_CLAIM", "true") reg, err := registry.Open(filepath.Join(root, "registry.db")) if err != nil { t.Fatal(err) } own := registry.ClaimOwnership{ClaimIncarnation: "inc", ClaimStamp: 1, ClaimOwner: "owner", ClaimRowID: "row"} if err := reg.Upsert(registry.Route{Host: j.Inventory.Host, ClaimOwnership: own}); err != nil { t.Fatal(err) } own.ClaimRowID = "timer" if err := reg.ReplaceTimers(j.Inventory.Host, []registry.Timer{{Host: j.Inventory.Host, Name: "backup", Kind: "cron", Registration: "timer", ClaimOwnership: own}}); err != nil { t.Fatal(err) } reg.Close() raw, err := json.Marshal(j) if err != nil { t.Fatal(err) } if err := os.WriteFile(filepath.Join(root, "intent.json"), raw, 0600); err != nil { t.Fatal(err) } cmd := exec.CommandContext(t.Context(), os.Args[0], "-test.run=^TestPlan14GenerationParkedCrash$", "-test.timeout=20s") cmd.Dir = root cmd.Env = append(os.Environ(), "PLAN14_PARK_CRASH_ROOT="+root, "PLAN14_PARK_CRASH_POINT="+point) out, err := cmd.CombinedOutput() var exited *exec.ExitError if !errors.As(err, &exited) || exited.ExitCode() != 38 { t.Fatalf("child did not crash at %s: %v %s", point, err, out) } reg, err = registry.Open(filepath.Join(root, "registry.db")) if err != nil { t.Fatal(err) } defer reg.Close() read := func(ctx context.Context) (generationInventory, error) { records, err := generationRetirementRecords(ctx, &rollbackDisk{dir: filepath.Join(root, "retire")}) if err != nil { return generationInventory{}, err } return generationRegistryInventory(ctx, reg, generationParkedSnapshot(t, snapshot, records)) } v := generationReconciler{read: read, kernel: func(context.Context) (string, string, error) { return "boot", generationInventoryDigest("kernel"), nil }} _, err = v.reconcile(t.Context(), nil) if (err == nil) != (point == "complete") { t.Fatalf("interrupted retirement at %s: %v", point, err) } generationRunPark(t, root, j, reg, "") if _, err := v.reconcile(t.Context(), nil); err != nil { t.Fatal("resumed retirement", err) } generationRunPark(t, root, j, reg, "") if _, err := v.reconcile(t.Context(), nil); err != nil { t.Fatal("replayed retirement", err) } }) } } func TestPlan14GenerationParkedReadback(t *testing.T) { _, j, _ := generationParkedFixture(t) for _, attack := range []string{"complete", "sync-failure", "unknown-field", "trailing", "wrong-name", "malformed", "unknown-file", "reader-failure"} { t.Run(attack, func(t *testing.T) { dir := t.TempDir() raw, err := json.Marshal(j) if err != nil { t.Fatal(err) } name := j.ID + ".json" switch attack { case "unknown-field": raw = append([]byte(`{"unexpected":true,`), raw[1:]...) case "trailing": raw = append(raw, []byte(` {}`)...) case "wrong-name": name = "other.json" case "unknown-file": name = "other" case "malformed": raw = []byte(`{"complete":`) } if err := os.WriteFile(filepath.Join(dir, name), raw, 0600); err != nil { t.Fatal(err) } open := func(path string) (rootstate.Store, error) { if attack == "reader-failure" { return nil, os.ErrPermission } return &rollbackDisk{dir: path}, nil } calls := 0 sync := func(string) error { calls++ if attack == "sync-failure" { return os.ErrPermission } return nil } got, err := generationReadRetirements(t.Context(), dir, open, sync) if attack == "complete" { if err != nil || len(got) != 1 || calls != 1 { t.Fatalf("complete read: %v %v %d", got, err, calls) } return } if err == nil || got != nil { t.Fatal("invalid retirement read granted evidence", got, err) } if attack == "sync-failure" { got, err = generationReadRetirements(t.Context(), dir, open, sync) if err == nil || got != nil || calls != 2 { t.Fatal("failed sync was not retried", got, err, calls) } got, err = generationReadRetirements(t.Context(), dir, open, func(string) error { return nil }) if err != nil || len(got) != 1 { t.Fatal("durable reread failed", got, err) } } }) } } func TestPlan14GenerationParkedKeepsReservation(t *testing.T) { snapshot, j, root := generationParkedFixture(t, true) reg, err := registry.Open(filepath.Join(root, "registry.db")) if err != nil { t.Fatal(err) } defer reg.Close() t.Setenv("CARLOS_HOST_CLAIM", "true") own := registry.ClaimOwnership{ClaimIncarnation: "inc", ClaimStamp: 1, ClaimOwner: "owner", ClaimRowID: "row"} if err := reg.Upsert(registry.Route{Host: j.Inventory.Host, ClaimOwnership: own}); err != nil { t.Fatal(err) } own.ClaimRowID = "timer" if err := reg.ReplaceTimers(j.Inventory.Host, []registry.Timer{{Host: j.Inventory.Host, Name: "backup", Kind: "cron", Registration: "timer", ClaimOwnership: own}}); err != nil { t.Fatal(err) } generationRunPark(t, root, j, reg, "") opt := t.TempDir() path := release.VersionPath(opt, "bab", "v1", "shop") if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil { t.Fatal(err) } if err := os.WriteFile(path, []byte("reserved build"), 0755); err != nil { t.Fatal(err) } identity := func(path string) rootstate.FileIdentity { t.Helper() var s unix.Stat_t if err := unix.Stat(path, &s); err != nil { t.Fatal(err) } return rootstate.FileIdentity{Device: uint64(s.Dev), Inode: s.Ino} } in := snapshot.Installations[0].Installation binding := release.GenerationBinding{Binding: release.HistoryBinding{Host: in.Host, Account: in.Account, App: in.App}, Target: generationOwner(*j.Target), Installation: rootHistoryInstallation(in), Record: in.Record, RecordETag: "record", Attempt: "pending", Inputs: "inputs", Invocation: "invocation", ControllerPID: 42, ControllerStart: "start", Version: "v1", Digest: generationInventoryDigest("build"), Path: path, Tree: filepath.Dir(path), Executable: identity(path), TreeIdentity: identity(filepath.Dir(path))} snapshot.Launches[0].ControllerPID = 42 snapshot.Launches[0].ProcessStart = "start" disk := &rollbackDisk{dir: t.TempDir()} ledger := &release.RootGenerations{Store: disk} reserved, err := ledger.Reserve(binding) if err != nil { t.Fatal(err) } read := func(ctx context.Context) (generationInventory, error) { records, err := generationRetirementRecords(ctx, &rollbackDisk{dir: filepath.Join(root, "retire")}) if err != nil { return generationInventory{}, err } return generationRegistryInventory(ctx, reg, generationParkedSnapshot(t, snapshot, records)) } reconciler := generationReconciler{read: read, kernel: func(context.Context) (string, string, error) { return "boot", generationInventoryDigest("kernel"), nil }} pins, err := (release.RootGenerationPins{Ledger: ledger, Reconcile: reconciler.reconcile}).StagedPins(t.Context(), opt) if err != nil { t.Fatal(err) } if pinned, err := pins.Refers(path); err != nil || !pinned { t.Fatal("completed retirement dropped pending reservation", pinned, err) } if got, err := ledger.Read(binding); err != nil || got != reserved { t.Fatal("payload retirement changed generation authority", got, err) } } func TestPlan14GenerationParkedNestedMountLocation(t *testing.T) { snapshot, _, root := generationParkedFixture(t) entry := &snapshot.Payloads[0].Inventory.Entries[2] entry.Identity.MountPoint = filepath.Join(root, "instances") in := snapshot.Payloads[0].Inventory inventory := hostbox.RetirementInventory{Host: in.Host, Captured: hostguard.Captured{Incarnation: in.Incarnation, Stamp: in.InstallationStamp}, Ownership: "owner"} for _, e := range in.Entries { inventory.Artifacts = append(inventory.Artifacts, hostbox.Artifact{Kind: hostbox.ArtifactKind(e.Kind), Root: e.ManagedRoot, Name: e.RelativePath, Owner: e.Ownership, RowID: e.RowIdentity, Identity: e.Identity, Dev: e.Device, Ino: e.Inode, Type: e.Type}) } j, err := hostbox.NewRetirementJournal("retire", inventory) if err != nil { t.Fatal(err) } r := snapshot.Installations[0].Receipts[0] j.Target, j.StopReceipt, j.Complete = &r.Target, r.Bytes, true for i := range j.Steps { j.Steps[i].Done = true } snapshot.Retirements = []hostbox.RetirementJournal{j} physical, err := generationPhysicalPayloads(t.Context(), snapshot) if err != nil { t.Fatal(err) } got := physical[0].Inventory.Entries[2] want := filepath.Join("instances", ".parked", in.Incarnation, in.Host, j.ID, "data", entry.RelativePath) if got.RelativePath != want || got.Identity != entry.Identity { t.Fatalf("parked payload=%+v, want %s with unchanged identity", got, want) } }