//go:build linux package hostbox import ( "context" "errors" "os" "path/filepath" "testing" "time" "github.com/carlosframework/platform/internal/hostclaim" "github.com/carlosframework/platform/internal/hostguard" "github.com/carlosframework/platform/internal/registry" ) func TestColdHandoverParksFilesRestoresOwnedRowsAndNeverReplaysOverStartedSuccessor(t *testing.T) { coldHandoverTimers(t, false) } func TestPlan14ColdHandoverAbandonedTimers(t *testing.T) { coldHandoverTimers(t, true) } func coldHandoverTimers(t *testing.T, abandoned bool) { t.Setenv("CARLOS_HOST_CLAIM", "1") source, store, initial := workFixture(t) ctx := t.Context() admit := func(old hostclaim.Observed, generation string) hostclaim.Observed { next, err := hostclaim.Update(ctx, store, old, hostclaim.AdmitWriter(hostclaim.Admission{ObservedOpID: old.Claim.Op.ID, ObservedPhase: old.Claim.Op.Phase, ObservedSession: old.Claim.Holder.Session, ObservedGeneration: old.Claim.Holder.Generation, Box: "box", Region: "ie", Mode: "cold", NewGeneration: generation})) if err != nil { t.Fatal(err) } return next } old := admit(initial, "first") if err := source.Guard.Activate(ctx, old.Host, "box", nil); err != nil { t.Fatal(err) } prior := hostguard.FenceTarget{FenceID: "fence", Region: "ie", Host: old.Host, Incarnation: old.Claim.Holder.Incarnation, Stamp: old.Claim.Holder.Stamp, Session: old.Claim.Holder.Session, Box: "box", Enrollment: "enrollment", BootID: "boot", ControllerID: "controller", ClaimETag: old.ETag, DirectoryRevision: "1", Op: old.Claim.Op.ID, Phase: old.Claim.Op.Phase} if _, err := source.Guard.BeginFence(ctx, prior); err != nil { t.Fatal(err) } if err := source.Guard.CompleteFence(ctx, prior, []byte("verified-root-receipt"), time.Now().UTC(), func(context.Context, hostguard.FenceTarget, []byte, time.Time) error { return nil }); err != nil { t.Fatal(err) } base := t.TempDir() reg, err := registry.Open(filepath.Join(base, "registry.db")) if err != nil { t.Fatal(err) } defer reg.Close() inventory, err := OpenStopInventoryDB(filepath.Join(base, "inventory.db")) if err != nil { t.Fatal(err) } defer inventory.Close() launches, err := OpenLaunchJournal(filepath.Join(base, "launches.db")) if err != nil { t.Fatal(err) } defer launches.Close() owner := registry.ClaimOwnership{ClaimIncarnation: prior.Incarnation, ClaimStamp: prior.Stamp, ClaimOwner: "old-owner", ClaimRowID: "old-route"} dbPath := filepath.Join(source.Guard.DataDir, old.Host+".db") if err = os.WriteFile(dbPath, []byte("untrusted-local-database"), 0600); err != nil { t.Fatal(err) } if err = reg.Upsert(registry.Route{Host: old.Host, Kind: "instance", Socket: "/run/example.sock", DBPath: dbPath, ClaimOwnership: owner, HoldOp: "held-operation"}); err != nil { t.Fatal(err) } if err = reg.UpsertTimer(registry.Timer{Host: old.Host, Name: "daily", Kind: "cron", Spec: "0 0 * * *", NextAt: 123, Attempts: 2}); err != nil { t.Fatal(err) } timer, err := reg.GetTimer(old.Host, "daily") if err != nil { t.Fatal(err) } parker := TypedParker{CapturePersistent: true, Roots: map[string]string{"data": source.Guard.DataDir}, ParkedRoot: source.Guard.DataDir, Incarnation: prior.Incarnation, Host: prior.Host, Retirement: "capture"} artifact, err := parker.CaptureArtifact("data", old.Host+".db", owner.ClaimOwner, ArtifactPath) if err != nil { t.Fatal(err) } in := StopInventory{Complete: true, Host: old.Host, Incarnation: prior.Incarnation, InstallationStamp: prior.Stamp, Box: "box", Enrollment: "enrollment", DirectoryRevision: "1", Entries: []StopInventoryEntry{{Kind: "route", ArtifactID: owner.ClaimRowID, ManagedRoot: "registry", RelativePath: old.Host, RowIdentity: owner.ClaimRowID, Ownership: owner.ClaimOwner, Materialization: "materialized"}, {Kind: "path", ArtifactID: "database", ManagedRoot: "data", RelativePath: artifact.Name, Ownership: artifact.Owner, Materialization: "materialized", Identity: artifact.Identity, Device: artifact.Dev, Inode: artifact.Ino, Type: artifact.Type}}} if abandoned { for _, name := range []string{"abandoned", "withdrawn"} { in.Entries = append(in.Entries, StopInventoryEntry{Kind: "timer", ArtifactID: name, ManagedRoot: "registry", RelativePath: old.Host + "/" + name, Ownership: owner.ClaimOwner, RowIdentity: name, Materialization: "intended", TimerRetired: name == "withdrawn"}) } } if err = inventory.WriteStopInventory(ctx, in); err != nil { t.Fatal(err) } successor := admit(old, "second") next := ColdSuccessor{Host: old.Host, Incarnation: prior.Incarnation, Stamp: successor.Claim.Holder.Stamp, Session: successor.Claim.Holder.Session, Generation: successor.Claim.Holder.Generation, Box: "box", Enrollment: "enrollment", DirectoryRevision: "1", ETag: successor.ETag, Op: successor.Claim.Op.ID, Phase: successor.Claim.Op.Phase} handover := ColdHandover{Guard: source.Guard, Claims: store, Registry: reg, Inventory: inventory, Launches: launches, Roots: map[string]string{"data": source.Guard.DataDir}, JournalDir: filepath.Join(base, "handover"), VerifyReceipt: func(context.Context, hostguard.FenceReceipt) error { return nil }} intentCrash := errors.New("crash after durable intent before successor barrier") handover.afterIntent = func() error { return intentCrash } if err = handover.Run(ctx, prior, next); !errors.Is(err, intentCrash) { t.Fatal(err) } pending, err := source.Guard.PendingFences(ctx) if err != nil || len(pending) != 0 { t.Fatalf("barrier exists before durable intent recovery: %+v %v", pending, err) } handover.afterIntent = nil crash := errors.New("crash after old installation parked") handover.afterPark = func() error { return crash } if err = handover.Recover(ctx); !errors.Is(err, crash) { t.Fatal(err) } if err = source.Guard.Activate(ctx, old.Host, "box", nil); err != nil { t.Fatal(err) } if err = source.Guard.With(ctx, old.Host, hostguard.Captured{Incarnation: next.Incarnation, Stamp: next.Stamp}, func(*hostclaim.Claim) error { return nil }, func() error { return nil }); !errors.Is(err, hostguard.ErrFenced) { t.Fatalf("crash exposed successor: %v", err) } if err = reg.Close(); err != nil { t.Fatal(err) } reg, err = registry.Open(filepath.Join(base, "registry.db")) if err != nil { t.Fatal(err) } defer reg.Close() if err = inventory.Close(); err != nil { t.Fatal(err) } inventory, err = OpenStopInventoryDB(filepath.Join(base, "inventory.db")) if err != nil { t.Fatal(err) } defer inventory.Close() handover.Registry = reg handover.Inventory = inventory handover.afterPark = nil if err = handover.Recover(ctx); err != nil { t.Fatal(err) } if _, err = os.Stat(dbPath); !os.IsNotExist(err) { t.Fatalf("old DB reused: %v", err) } got, err := reg.Get(old.Host) if err != nil || got.ClaimStamp != next.Stamp || got.ClaimRowID == owner.ClaimRowID || got.HoldOp != "held-operation" { t.Fatalf("route ownership not restored: %+v %v", got, err) } after, err := reg.GetTimer(old.Host, "daily") if err != nil || after.Registration != timer.Registration || after.NextAt != timer.NextAt || after.Attempts != timer.Attempts || after.ClaimRowID == timer.ClaimRowID || after.ClaimStamp != next.Stamp { t.Fatalf("timer bookkeeping not retained: %+v %v", after, err) } retained, err := inventory.ReadInstallation(ctx, old.Host, prior.Incarnation, prior.Stamp) if err != nil || len(retained.Entries) != len(in.Entries) { t.Fatal("predecessor inventory discarded") } if err = os.WriteFile(dbPath, []byte("new-runtime-data"), 0600); err != nil { t.Fatal(err) } if err = reg.SetVersions(old.Host, "new-live-version", ""); err != nil { t.Fatal(err) } if err = handover.Run(ctx, prior, next); err != nil { t.Fatal(err) } if err = handover.Recover(ctx); err != nil { t.Fatal(err) } body, err := os.ReadFile(dbPath) if err != nil || string(body) != "new-runtime-data" { t.Fatal("completed replay touched live successor") } got, err = reg.Get(old.Host) if err != nil || got.Version != "new-live-version" { t.Fatal("completed replay overwrote new route") } }