//go:build linux package host import ( "context" "encoding/json" "net" "os" "path/filepath" "reflect" "testing" "time" "github.com/carlosframework/platform/internal/hostbox" "github.com/carlosframework/platform/internal/hostclaim" "github.com/carlosframework/platform/internal/hostguard" "github.com/carlosframework/platform/internal/hostwrite" "github.com/carlosframework/platform/internal/instancectl" "github.com/carlosframework/platform/internal/provision" "github.com/carlosframework/platform/internal/registry" ) // Initial installation remains a fixture. Every later ownership transfer uses // the production serving handover and retains the original source directory. func servingHandoverRoot(t *testing.T, f *servingRootFixture) ClaimRootHTTP { t.Helper() inventory := f.p.root.Journals.Inventory captured, err := f.f.guard.Capture(f.f.read().Host) if err != nil { t.Fatal(err) } in, _, err := inventory.Snapshot(t.Context(), f.f.read().Host, captured) if err != nil { t.Fatal(err) } if f.p.root.Reconcile.Registry == nil { owner := registry.ClaimOwnership{ClaimIncarnation: in.Incarnation, ClaimStamp: in.InstallationStamp, ClaimOwner: in.Entries[0].Ownership, ClaimRowID: "prior-route"} reg := restartTestRegistry(t) holder := f.f.read().Claim.Holder row := registry.Route{Host: in.Host, Account: holder.Account, App: holder.App, Kind: "instance", DBPath: f.db, Socket: provision.NestedSocket(in.Host), ClaimOwnership: owner} if err := reg.Upsert(row); err != nil { t.Fatal(err) } in.Entries = append(in.Entries, hostbox.StopInventoryEntry{Kind: "route", ArtifactID: owner.ClaimRowID, ManagedRoot: "registry", RelativePath: in.Host, RowIdentity: owner.ClaimRowID, Ownership: owner.ClaimOwner, Materialization: "materialized"}) if err := inventory.WriteStopInventory(t.Context(), in); err != nil { t.Fatal(err) } _, revision, err := inventory.Snapshot(t.Context(), in.Host, captured) if err != nil { t.Fatal(err) } raw, err := f.rootDisk.Read(in.Host + ".json") if err != nil { t.Fatal(err) } var saved map[string]any if err = json.Unmarshal(raw, &saved); err != nil { t.Fatal(err) } saved["Installation"].(map[string]any)["InventoryRevision"] = revision raw, err = json.Marshal(saved) if err != nil { t.Fatal(err) } if err = f.rootDisk.Write(in.Host+".json", raw); err != nil { t.Fatal(err) } record := provision.Record{Schema: 1, Host: in.Host, Account: holder.Account, App: holder.App, Region: f.actor.Region, CreatedAt: holder.CreatedAt, Channel: "edge"} body, err := json.Marshal(record) if err != nil { t.Fatal(err) } if _, err = f.f.claims.Put(t.Context(), holder.Key, body, ""); err != nil { t.Fatal(err) } f.p.root.Reconcile.Registry = reg f.p.root.Reconcile.Inventory = inventory f.p.root.Reconcile.Box, f.p.root.Reconcile.Enrollment = f.actor.Box, f.actor.Enrollment f.p.root.Reconcile.Retirement.JournalDir = filepath.Join(t.TempDir(), "handover") f.p.root.Launches = f.f.journal } f.f.guard.Root = hostguard.NewRootAuthorityWithInventory(f.rootDisk, func(ctx context.Context, host string, c hostguard.Captured) (uint64, error) { _, rev, err := inventory.Snapshot(ctx, host, c) return rev, err }) f.p.root.Stop.Coordinator.Guard = f.f.guard f.f.evidence = f.f.claims f.p.root.Stop.Coordinator.Evidence = f.f.claims return servingTransportRoot(t, f) } func TestServingParkColdHandoverPreservesState(t *testing.T) { runServingColdHandoverBoundary(t, nil) } func TestServingHandoverThenPrelaunchParkReplay(t *testing.T) { f := newServingRootFixture(t, true) root := servingHandoverRoot(t, f) socket := servingRootSocket(t, root) hook := coldHandoverWith(handoverRecoveryIdentity(t, f.actor), socket, func(context.Context, hostclaim.Observed) (hostguard.FenceReceipt, error) { return hostguard.FenceReceipt{Target: f.receipt.Target}, nil }) admitted := f.f.read() for i := 0; i < 2; i++ { if err := hook(t.Context(), f.predecessor, admitted); err != nil { t.Fatal("handover", err) } } installed, err := f.f.guard.Capture(f.current.Host) if err != nil || installed != (hostguard.Captured{Incarnation: f.current.Incarnation, Stamp: f.current.Stamp}) { t.Fatalf("installation: %+v %v", installed, err) } target := recoveryTuple(f.f.read(), claimStopLocal{Actor: f.actor, BootID: f.receipt.Target.BootID, ControllerID: "controller"}) if err := f.f.journal.Prelaunch(t.Context(), hostbox.LaunchInventory{Target: target, ControllerPID: 77, ProcessStart: "edge-start", NoState: true, Groups: []hostbox.LaunchGroup{{RelativePath: "carlos/next", Device: 1, Inode: 99}}}); err != nil { t.Fatal(err) } client := hostbox.ServingParkClient{Call: rootJournalClient(socket)} a, err := client.Park(t.Context(), f.current, f.prior) if err != nil { t.Fatal("park after prelaunch", err) } f.tags(f.current.Stamp) if err := client.Complete(t.Context(), a.ID, f.current); err != nil { t.Fatal(err) } before, err := f.p.ledger.read(a.ID) if err != nil || !before.Complete { t.Fatal("completion", err) } again, err := client.Park(t.Context(), f.current, f.prior) if err != nil || again != a { t.Fatalf("replay after handover and prelaunch: %+v %v", again, err) } after, err := f.p.ledger.read(a.ID) if err != nil || !reflect.DeepEqual(before, after) { t.Fatal("replay changed record", err) } if err := f.p.ledger.fs.CheckParked(&after); err != nil { t.Fatal(err) } } func runServingColdHandoverBoundary(t *testing.T, mount func(*testing.T, string), revisions ...uint64) { t.Helper() f := newServingRootFixture(t, true) if mount != nil { mount(t, f.f.guard.DataDir) } root := servingHandoverRoot(t, f) if len(revisions) != 0 { f.actor.Revision = revisions[0] } stateDir := filepath.Dir(f.db) expected := map[string]string{filepath.Base(f.db): "served bytes", filepath.Base(f.db) + ".gen": "G", filepath.Base(f.db) + "-wal": "WAL", filepath.Base(f.db) + "-shm": "SHM", "files/upload": "tenant upload", "env": "tenant configuration"} for _, name := range []string{"files/upload", "env"} { p := filepath.Join(stateDir, name) if err := os.MkdirAll(filepath.Dir(p), 0700); err != nil { t.Fatal(err) } if err := os.WriteFile(p, []byte(expected[name]), 0600); err != nil { t.Fatal(err) } } before := f.f.read() old, _, err := root.Reconcile.Inventory.Snapshot(t.Context(), before.Host, hostguard.Captured{Incarnation: f.prior.Incarnation, Stamp: f.prior.Stamp}) if err != nil { t.Fatal(err) } next := hostbox.ColdSuccessor{Host: f.current.Host, Incarnation: f.current.Incarnation, Box: f.actor.Box, Enrollment: f.actor.Enrollment, DirectoryRevision: f.actor.Binding, ETag: before.ETag, Generation: f.current.Generation, Stamp: f.current.Stamp, Session: f.current.Session} handover := root.servingHandover() for i := 0; i < 2; i++ { if err = handover.Run(t.Context(), f.receipt.Target, next); err != nil { t.Fatal("serving handover", err) } } for name, want := range expected { body, err := os.ReadFile(filepath.Join(stateDir, name)) if err != nil || string(body) != want { t.Fatalf("handover changed %s: %q %v", name, body, err) } } retained, _, err := root.Reconcile.Inventory.Snapshot(t.Context(), before.Host, hostguard.Captured{Incarnation: f.prior.Incarnation, Stamp: f.prior.Stamp}) if err != nil || !reflect.DeepEqual(old, retained) { t.Fatal("predecessor inventory changed", err) } if f.f.read().ETag != before.ETag { t.Fatal("handover rewrote claim") } a, err := f.park() if err != nil { t.Fatal("database retirement after handover", err) } f.tags(f.current.Stamp) if err = f.p.completeServingDatabasePark(t.Context(), f.conn, a.ID, f.current); err != nil { t.Fatal(err) } for _, name := range []string{"files/upload", "env"} { body, err := os.ReadFile(filepath.Join(stateDir, name)) if err != nil || string(body) != expected[name] { t.Fatalf("retirement changed %s: %q %v", name, body, err) } } } func TestServingHandoverLegacy(t *testing.T) { f, _, coordinator := newServingLegacyRootSetup(t, false) root := servingHandoverRoot(t, f) boot, err := servingParkBootID() if err != nil { t.Fatal(err) } f.f.peer.BootID = boot o := f.f.read() req := StopRequest{Region: f.actor.Region, Host: o.Host, Incarnation: o.Claim.Holder.Incarnation, InstallationStamp: o.Claim.Holder.Stamp, Box: f.actor.Box, Enrollment: f.actor.Enrollment, BootID: boot, ControllerID: "controller", ExpectedClaimETag: o.ETag, ExpectedDirectoryRevision: "7", RosterRevision: 7} if _, err := coordinator.FenceAndStop(t.Context(), req); err != nil { t.Fatal(err) } admitted, err := hostclaim.Update(t.Context(), f.f.claims, f.f.read(), hostclaim.AdmitWriter(hostclaim.Admission{ObservedSession: 0, Box: f.actor.Box, Region: f.actor.Region, PredecessorRegion: f.actor.Region, Mode: "cold", NewGeneration: "G"})) if err != nil { t.Fatal(err) } f.current, err = servingWriterIdentity(admitted, f.actor.Box, f.actor.Region) if err != nil { t.Fatal(err) } saved, err := f.p.ledger.legacyStop(f.current) if err != nil { t.Fatal(err) } f.actor.Revision = 8 n := hostbox.ColdSuccessor{Host: f.current.Host, Incarnation: f.current.Incarnation, Box: f.actor.Box, Enrollment: f.actor.Enrollment, DirectoryRevision: f.actor.Binding, ETag: f.f.read().ETag, Generation: f.current.Generation, Stamp: f.current.Stamp, Session: f.current.Session} if err := root.servingHandover().Run(t.Context(), saved.Result.Target, n); err != nil { t.Fatalf("legacy transfer: %v prior=%+v next=%+v", err, saved.Result.Target, n) } if _, err := f.p.parkLegacyServingDatabase(t.Context(), f.conn, f.current, servingLegacyPredecessor{}); err != nil { t.Fatal("legacy park after transfer", err) } } func TestServingHandoverRefusesForeignAuthority(t *testing.T) { for _, attack := range []string{"census", "actor", "successor", "directory-revision", "operation", "predecessor", "receipt", "root-revision", "registry-path", "registry-account", "source-replaced", "extra-timer", "prelaunch"} { t.Run(attack, func(t *testing.T) { f := newServingRootFixture(t, true) root := servingHandoverRoot(t, f) p := f.receipt.Target n := hostbox.ColdSuccessor{Host: f.current.Host, Incarnation: f.current.Incarnation, Box: f.actor.Box, Enrollment: f.actor.Enrollment, DirectoryRevision: "7", ETag: f.f.read().ETag, Generation: f.current.Generation, Stamp: f.current.Stamp, Session: f.current.Session} row, err := root.Reconcile.Registry.Get(n.Host) if err != nil { t.Fatal(err) } switch attack { case "census": root.RequireNewWork = func(context.Context) error { return hostbox.ErrChanged } case "actor": f.actor.Eligible = false case "successor": n.Generation = "foreign" case "directory-revision": n.DirectoryRevision = "8" case "operation": n.Op = "move" case "predecessor": p.Session++ case "receipt": p.FenceID = "unknown" case "root-revision": f.mutateRoot(func(m map[string]any) { m["Installation"].(map[string]any)["InventoryRevision"] = 999 }) case "registry-path": row.DBPath = filepath.Join(t.TempDir(), "victim.db") if err = root.Reconcile.Registry.Upsert(*row); err != nil { t.Fatal(err) } case "registry-account": row.Account = "bac" if err = root.Reconcile.Registry.Upsert(*row); err != nil { t.Fatal(err) } case "source-replaced": dir := filepath.Dir(f.db) if err = os.Rename(dir, dir+"-original"); err != nil { t.Fatal(err) } if err = os.Mkdir(dir, 0700); err != nil { t.Fatal(err) } case "extra-timer": if err = root.Reconcile.Registry.UpsertTimer(registry.Timer{Host: n.Host, Name: "foreign", Kind: "cron", Spec: "0 * * * *"}); err != nil { t.Fatal(err) } case "prelaunch": target := recoveryTuple(f.f.read(), claimStopLocal{Actor: f.actor, BootID: f.receipt.Target.BootID, ControllerID: "controller"}) if err = f.f.journal.Prelaunch(t.Context(), hostbox.LaunchInventory{Target: target, ControllerPID: 77, ProcessStart: "edge-start", NoState: true, Groups: []hostbox.LaunchGroup{{RelativePath: "carlos/next", Device: 1, Inode: 99}}}); err != nil { t.Fatal(err) } } before, err := root.Reconcile.Registry.Get(n.Host) if err != nil { t.Fatal(err) } if err = root.servingHandover().Run(t.Context(), p, n); err == nil { t.Fatal("accepted", attack) } after, err := root.Reconcile.Registry.Get(n.Host) if err != nil || !reflect.DeepEqual(before, after) { t.Fatal("refusal changed row", err) } source := f.db if attack == "source-replaced" { source = filepath.Join(filepath.Dir(f.db)+"-original", filepath.Base(f.db)) } body, err := os.ReadFile(source) if err != nil || string(body) != "served bytes" { t.Fatal("refusal changed source", err) } }) } } // The stop coordinator writes the receipt used by the next handover. Process // and authentication observations remain fixtures; the rootrunner covers them. func TestServingStoppedWriterSecondHandover(t *testing.T) { f := newServingRootFixture(t, true) root := servingHandoverRoot(t, f) f.f.peer = StopPeer{UID: 1001, PID: 77, ProcessStart: "edge-start", BootID: f.receipt.Target.BootID, ControllerID: "controller"} coordinator := f.f.root(nil) coordinator.Evidence, coordinator.Inventory = f.f.claims, root.Journals.Inventory root.Stop.Coordinator = coordinator root.Stop.Authenticate = func(context.Context, net.Conn) (StopPeer, error) { return f.f.peer, nil } f.p.root.Stop.Coordinator = coordinator socket := servingRootSocket(t, root) transfer := coldHandoverWith(handoverRecoveryIdentity(t, f.actor), socket, func(context.Context, hostclaim.Observed) (hostguard.FenceReceipt, error) { return f.receipt, nil }) if err := transfer(t.Context(), f.predecessor, f.f.read()); err != nil { t.Fatal(err) } target := recoveryTuple(f.f.read(), claimStopLocal{Actor: f.actor, BootID: f.receipt.Target.BootID, ControllerID: "controller"}) if err := f.f.journal.Prelaunch(t.Context(), hostbox.LaunchInventory{Target: target, ControllerPID: 77, ProcessStart: "edge-start", NoState: true, Groups: []hostbox.LaunchGroup{{RelativePath: "carlos/next", Device: 1, Inode: 99}}}); err != nil { t.Fatal(err) } client := hostbox.ServingParkClient{Call: rootJournalClient(socket)} first, err := client.Park(t.Context(), f.current, f.prior) if err != nil { t.Fatal(err) } f.tags(f.current.Stamp) if err := client.Complete(t.Context(), first.ID, f.current); err != nil { t.Fatal(err) } for suffix, bytes := range map[string]string{"": "second writer bytes", ".gen": f.current.Generation, "-wal": "second WAL", "-shm": "second SHM"} { if err := os.WriteFile(f.db+suffix, []byte(bytes), 0600); err != nil { t.Fatal(err) } } r := hostbox.LaunchReservation{ID: "second:main", Role: "main", InvocationID: "second", Inputs: "digest", Target: target, Group: hostbox.LaunchGroup{RelativePath: instancectl.ClaimUnitControlGroup(target.Host)}, Executable: hostbox.LaunchFile{Device: 1, Inode: 3}} if err := f.f.journal.IntendUnit(t.Context(), r); err != nil { t.Fatal(err) } if err := f.f.journal.StartedUnit(t.Context(), r, hostbox.LaunchGroup{RelativePath: r.Group.RelativePath, Device: 1, Inode: 4, InvocationID: "second-unit"}, 99, "tenant-start"); err != nil { t.Fatal(err) } o := f.f.read() req := StopRequest{Region: f.actor.Region, Host: o.Host, Incarnation: f.current.Incarnation, InstallationStamp: f.current.Stamp, WriterSession: f.current.Session, Box: f.actor.Box, Enrollment: f.actor.Enrollment, BootID: f.receipt.Target.BootID, ControllerID: "controller", ExpectedClaimETag: o.ETag, ExpectedDirectoryRevision: "7", RosterRevision: 7} drain := stopDrainFunc(func(_ context.Context, req StopRequest, fence string) (DrainAcknowledgement, error) { return DrainClaimAcknowledgement(fence, req.ControllerID, "edge-start", nil), nil }) if _, err := CollectClaimStop(t.Context(), socket, req, drain); err == nil { t.Fatal("accepted an unsettled unit") } if err := f.f.journal.SettleUnit(t.Context(), target, r.ID); err != nil { t.Fatal(err) } stopped, err := CollectClaimStop(t.Context(), socket, req, drain) if err != nil { t.Fatal("stop through transport", err) } f.receipt, err = f.f.guard.FenceReceiptFor(t.Context(), stopped.Target) if err != nil || string(f.receipt.Bytes) != string(stopped.Bytes) { t.Fatal("durable receipt", err) } previous := f.f.read() h := previous.Claim.Holder admitted, err := hostclaim.Update(t.Context(), f.f.claims, previous, hostclaim.AdmitWriter(hostclaim.Admission{ObservedSession: h.Session, ObservedGeneration: h.Generation, Box: f.actor.Box, Region: f.actor.Region, PredecessorRegion: f.actor.Region, Mode: "cold", NewGeneration: "G3"})) if err != nil { t.Fatal(err) } prior := f.current f.current, err = servingWriterIdentity(admitted, f.actor.Box, f.actor.Region) if err != nil { t.Fatal(err) } if err := transfer(t.Context(), previous, admitted); err != nil { t.Fatal("second handover", err) } parked, err := client.Park(t.Context(), f.current, prior) if err != nil { t.Fatal("second park", err) } f.tags(f.current.Stamp) if err := client.Complete(t.Context(), parked.ID, f.current); err != nil { t.Fatal(err) } record, err := f.p.ledger.read(parked.ID) if err != nil || !record.Complete || record.Disposition != servingParkRetain || len(record.Receipts) != 1 || string(record.Receipts[0].Bytes) != string(stopped.Bytes) { t.Fatal("second park did not consume exact transport stop receipt", err) } old, err := f.p.ledger.read(first.ID) if err != nil || !old.Closed || old.Cause != "replacement" { t.Fatal("previous park was not replaced", err) } for suffix, want := range map[string]string{"": "second writer bytes", ".gen": prior.Generation, "-wal": "second WAL", "-shm": "second SHM"} { body, err := os.ReadFile(filepath.Join(f.f.guard.DataDir, "instances", ".serving-parked", parked.ID, filepath.Base(f.db)+suffix)) if err != nil || string(body) != want { t.Fatal("second park changed bytes", suffix, err) } } if replay, err := client.Park(t.Context(), f.current, prior); err != nil || replay != parked { t.Fatal("second park replay", err) } if after, err := f.p.ledger.read(parked.ID); err != nil || !reflect.DeepEqual(record, after) { t.Fatal("second park replay changed record", err) } servingRestoreAfterStoppedWriter(t, f, client) if after, err := f.p.ledger.read(parked.ID); err != nil || !reflect.DeepEqual(record, after) { t.Fatal("restore changed retained park", err) } } // This connects production ColdRestore and retirement enrollment to the stopped // writer's park. The exact-version LTX baseline and local policy are fixtures; // full wake preparation and C8 remain separate acceptance gates. func servingRestoreAfterStoppedWriter(t *testing.T, f *servingRootFixture, parker servingDatabaseParker) { t.Helper() plan := plan12RestoreBaseline(t, f.f) before, err := f.f.data.GetVersion(t.Context(), f.f.bucket, plan.Files[0].Key, plan.Files[0].VersionID) if err != nil { t.Fatal(err) } admitted := f.f.read() in := ClaimLaunchInputs{DBPath: f.db} r := &claimServingArtifactRetirement{Claims: f.f.claims, Guard: f.f.guard, DataDir: f.f.guard.DataDir, Box: f.actor.Box, Region: f.actor.Region, Root: parker} source := hostbox.WorkSource{Guard: f.f.guard, Box: f.actor.Box, Region: f.actor.Region, Now: func() time.Time { return f.f.now }, Policy: func(_ context.Context, c *hostclaim.Claim, selected hostbox.Selection, _ hostwrite.Identity) error { return hostbox.Serving(c, selected) }, Local: func(context.Context, *hostclaim.Claim, hostbox.Selection) error { return nil }} work, err := source.Select(t.Context(), admitted.Host, "restore") if err != nil { t.Fatal("select restore work", err) } if err := r.Recover(t.Context(), admitted, in); err != nil { t.Fatal("recover completed retirement", err) } enrolled := 0 restore := hostbox.ColdRestore{Reader: f.f.data, Bucket: f.f.bucket, Generation: f.current.Generation, Work: work, BeforeLegacyInstall: func(ctx context.Context, _, _ string) error { return r.BeforeLegacyInstall(ctx, admitted, in, work) }, BeforeInstallLocked: func(ctx context.Context, _, _ string) error { if err := r.EnrollLocked(ctx, admitted, in); err != nil { return err } enrolled++ return nil }, RetireArtifacts: func(ctx context.Context, generation string) error { return r.Retire(ctx, admitted, in, generation) }, } if err := restore.Install(t.Context(), f.db, plan); err != nil { t.Fatal("restore after stopped writer", err) } if enrolled != 1 || mwReadDB(t, f.db) != "selected recovery baseline" { t.Fatal("restore missed enrollment or restored the wrong baseline") } if marker, err := hostbox.ReadGenerationMarker(f.db + ".gen"); err != nil || marker != f.current.Generation { t.Fatal("restored generation", marker, err) } after, err := f.f.data.GetVersion(t.Context(), f.f.bucket, plan.Files[0].Key, plan.Files[0].VersionID) if err != nil || string(after) != string(before) { t.Fatal("restore changed remote baseline", err) } }