//go:build linux package hostbox import ( "context" "database/sql" "errors" "fmt" "os" "path/filepath" "reflect" "testing" "time" "github.com/carlosframework/platform/internal/hostclaim" "github.com/carlosframework/platform/internal/hostguard" "github.com/carlosframework/platform/internal/provision" "github.com/carlosframework/platform/internal/registry" ) func servingHandoverFixture(t *testing.T, deviceDelta ...uint64) (ServingHandover, hostguard.FenceTarget, ColdSuccessor) { t.Helper() return servingHandoverInitialFixture(t, false, false, deviceDelta...) } func servingHandoverInitialFixture(t *testing.T, first, omitPath bool, deviceDelta ...uint64) (ServingHandover, hostguard.FenceTarget, ColdSuccessor) { t.Helper() t.Setenv("CARLOS_HOST_CLAIM", "1") source, store, initial := workFixture(t) admit := func(o hostclaim.Observed, g string) hostclaim.Observed { n, err := hostclaim.Update(t.Context(), store, o, hostclaim.AdmitWriter(hostclaim.Admission{ObservedSession: o.Claim.Holder.Session, ObservedGeneration: o.Claim.Holder.Generation, Box: "box", Region: "ie", Mode: "cold", NewGeneration: g})) if err != nil { t.Fatal(err) } return n } old := initial if !first { old = admit(initial, "first") } base := t.TempDir() disk := &plan14ColdDisk{dir: t.TempDir()} reg, err := registry.Open(filepath.Join(base, "registry.db")) if err != nil { t.Fatal(err) } t.Cleanup(func() { reg.Close() }) inv, err := OpenStopInventoryDB(filepath.Join(base, "inventory.db")) if err != nil { t.Fatal(err) } t.Cleanup(func() { inv.Close() }) launches, err := OpenLaunchJournal(filepath.Join(base, "launch.db")) if err != nil { t.Fatal(err) } t.Cleanup(func() { launches.Close() }) guard := source.Guard guard.Root = hostguard.NewRootAuthorityWithInventory(disk, func(ctx context.Context, host string, c hostguard.Captured) (uint64, error) { _, r, e := inv.Snapshot(ctx, host, c) return r, e }) holder := old.Claim.Holder owner := ownershipFor(old.Host, holder.Incarnation, holder.Stamp, "box") row := registry.Route{Host: old.Host, Kind: "instance", Account: holder.Account, App: holder.App, DBPath: provision.NestedDBPath(guard.DataDir, old.Host), Socket: provision.NestedSocket(old.Host), ClaimOwnership: registry.ClaimOwnership{ClaimIncarnation: holder.Incarnation, ClaimStamp: holder.Stamp, ClaimOwner: owner, ClaimRowID: "old-route"}} if err = reg.Upsert(row); err != nil { t.Fatal(err) } dir := filepath.Dir(row.DBPath) if err = os.MkdirAll(dir, 0700); err != nil { t.Fatal(err) } if err = os.WriteFile(filepath.Join(dir, "env"), []byte("configuration"), 0600); err != nil { t.Fatal(err) } roots := map[string]string{"data": guard.DataDir} rel, _ := filepath.Rel(guard.DataDir, dir) a, err := (TypedParker{Roots: roots, ParkedRoots: roots, Incarnation: holder.Incarnation, Host: old.Host, Retirement: "capture"}).CaptureArtifact("data", rel, owner, ArtifactPath) if err != nil { t.Fatal(err) } if len(deviceDelta) > 0 { a.Dev += deviceDelta[0] } in := StopInventory{Complete: true, Host: old.Host, Incarnation: holder.Incarnation, InstallationStamp: holder.Stamp, Box: "box", Enrollment: "enrollment", DirectoryRevision: "1", Entries: []StopInventoryEntry{ {Kind: "route", ArtifactID: "old-route", ManagedRoot: "registry", RelativePath: old.Host, RowIdentity: "old-route", Ownership: owner, Materialization: "materialized"}, {Kind: "path", ArtifactID: "state", ManagedRoot: "data", RelativePath: rel, Ownership: owner, Materialization: "materialized", Device: a.Dev, Inode: a.Ino, Type: a.Type}, }} if omitPath { in.Entries = in.Entries[:1] } if err = inv.WriteStopInventory(t.Context(), in); err != nil { t.Fatal(err) } _, rev, err := inv.Snapshot(t.Context(), old.Host, hostguard.Captured{Incarnation: holder.Incarnation, Stamp: holder.Stamp}) if err != nil { t.Fatal(err) } installation := hostguard.RootInstallation{Host: old.Host, Incarnation: holder.Incarnation, Stamp: holder.Stamp, Account: holder.Account, App: holder.App, Record: holder.Key, Box: "box", Enrollment: "enrollment", Migration: 1, InventoryRevision: rev} if err = guard.EstablishRoot(t.Context(), installation, func(*hostclaim.Claim) error { return nil }); err != nil { t.Fatal(err) } prior := hostguard.FenceTarget{FenceID: "stop", Region: "ie", Host: old.Host, Incarnation: holder.Incarnation, Stamp: holder.Stamp, Session: holder.Session, Box: "box", Enrollment: "enrollment", BootID: "boot", ControllerID: "controller", ClaimETag: old.ETag, DirectoryRevision: "1"} if !first { if _, err = guard.BeginFence(t.Context(), prior); err != nil { t.Fatal(err) } if err = guard.CompleteFence(t.Context(), prior, []byte("verified stop"), time.Now().UTC(), func(context.Context, hostguard.FenceTarget, []byte, time.Time) error { return nil }); err != nil { t.Fatal(err) } } else { prior.FenceID = "" } successor := admit(old, "second") n := successor.Claim.Holder next := ColdSuccessor{Host: old.Host, Incarnation: n.Incarnation, Stamp: n.Stamp, Session: n.Session, Generation: n.Generation, Box: "box", Enrollment: "enrollment", DirectoryRevision: "1", ETag: successor.ETag} h := ServingHandover{ColdHandover: ColdHandover{Guard: guard, Claims: store, Registry: reg, Inventory: inv, Launches: launches, Roots: roots, JournalDir: filepath.Join(base, "handover"), VerifyReceipt: func(_ context.Context, r hostguard.FenceReceipt) error { if r.Target != prior || string(r.Bytes) != "verified stop" { return ErrChanged } return nil }}, VerifyDirectory: func(_ context.Context, p hostguard.FenceTarget, e StopInventoryEntry, a Artifact) error { if p.BootID != "boot" || a.Dev != e.Device { return ErrChanged } return nil }, Validate: func(_ context.Context, c *hostclaim.Claim, p hostguard.FenceTarget, _ ColdSuccessor, _ StopInventory, i hostguard.RootInstallation) error { if p != prior || i != installation || !reflect.DeepEqual(c, successor.Claim) { return ErrChanged } return nil }} return h, prior, next } func TestServingHandoverRecoversEveryBoundary(t *testing.T) { for _, point := range []string{"intent", "barrier", "inventory", "rows", "complete-journal"} { t.Run(point, func(t *testing.T) { h, p, n := servingHandoverFixture(t) old, _, err := h.Inventory.Snapshot(t.Context(), p.Host, hostguard.Captured{Incarnation: p.Incarnation, Stamp: p.Stamp}) if err != nil { t.Fatal(err) } crash := errors.New("injected interruption") h.checkpoint = func(got string) error { if got == point { return crash } return nil } if err = h.Run(t.Context(), p, n); !errors.Is(err, crash) { t.Fatal(err) } _ = h.Guard.Activate(t.Context(), p.Host, p.Box, nil) allowed := func(*hostclaim.Claim) error { return nil } if err = h.Guard.With(t.Context(), p.Host, hostguard.Captured{Incarnation: n.Incarnation, Stamp: n.Stamp}, allowed, func() error { t.Fatal("successor ran during partial handover"); return nil }); err == nil { t.Fatal("partial transfer authorized") } h.checkpoint = nil if err = h.Recover(t.Context()); err != nil { t.Fatal("recover", err) } if err = h.Run(t.Context(), p, n); err != nil { t.Fatal("replay", err) } if err = h.Guard.With(t.Context(), p.Host, hostguard.Captured{Incarnation: n.Incarnation, Stamp: n.Stamp}, allowed, func() error { return nil }); err != nil { t.Fatal("completed transfer", err) } successor, err := h.Inventory.ReadInstallation(t.Context(), n.Host, n.Incarnation, n.Stamp) if err != nil { t.Fatal(err) } for _, e := range successor.Entries { if e.Kind == "path" && (e.Identity.Boot == "" || VerifyInventoryIdentity(h.Roots[e.ManagedRoot], e) != nil) { t.Fatal("successor lost verified directory identity", e) } } retained, _, err := h.Inventory.Snapshot(t.Context(), p.Host, hostguard.Captured{Incarnation: p.Incarnation, Stamp: p.Stamp}) if err != nil || !reflect.DeepEqual(retained, old) { t.Fatal("old inventory", err) } b, err := os.ReadFile(filepath.Join(filepath.Dir(provision.NestedDBPath(h.Guard.DataDir, p.Host)), "env")) if err != nil || string(b) != "configuration" { t.Fatal("source lost", err) } }) } } func TestServingHandoverRefusesOldDestructiveRecovery(t *testing.T) { h, p, n := servingHandoverFixture(t) old := h.ColdHandover old.OperationsOnly = true if err := old.Run(t.Context(), p, n); !errors.Is(err, ErrUnsupported) { t.Fatal(err) } if _, err := os.Stat(filepath.Dir(provision.NestedDBPath(h.Guard.DataDir, p.Host))); err != nil { t.Fatal(err) } } func TestServingHandoverStaleJournalDoesNotBlockHealthy(t *testing.T) { h, p, n := servingHandoverFixture(t) h.checkpoint = func(point string) error { if point == "intent" { return errors.New("interrupt") } return nil } if h.Run(t.Context(), p, n) == nil { t.Fatal("no interrupted intent") } stale := filepath.Join(h.JournalDir, "000-stale.json") if err := os.WriteFile(stale, []byte("stale journal"), 0600); err != nil { t.Fatal(err) } h.checkpoint = nil if h.Recover(t.Context()) == nil { t.Fatal("stale journal was not reported") } if b, err := os.ReadFile(stale); err != nil || string(b) != "stale journal" { t.Fatal("stale evidence changed", err) } if err := h.Guard.With(t.Context(), p.Host, hostguard.Captured{Incarnation: n.Incarnation, Stamp: n.Stamp}, func(*hostclaim.Claim) error { return nil }, func() error { return nil }); err != nil { t.Fatal("healthy journal not recovered", err) } } func TestServingHandoverRebindsOnlyProvenDirectory(t *testing.T) { for _, proven := range []bool{false, true} { t.Run(fmt.Sprint(proven), func(t *testing.T) { h, prior, next := servingHandoverFixture(t, 1) in, _, err := h.Inventory.Snapshot(t.Context(), prior.Host, hostguard.Captured{Incarnation: prior.Incarnation, Stamp: prior.Stamp}) if err != nil { t.Fatal(err) } var live uint64 for _, e := range in.Entries { if e.Kind == "path" { live = e.Device - 1 } } calls := 0 h.VerifyDirectory = func(_ context.Context, p hostguard.FenceTarget, e StopInventoryEntry, a Artifact) error { calls++ if p != prior || a.Dev != live || e.Device != live+1 || !proven { return ErrChanged } return nil } err = h.Run(t.Context(), prior, next) if !proven { if err == nil { t.Fatal("unproven device accepted") } return } if err != nil { t.Fatal(err) } out, err := h.Inventory.ReadInstallation(t.Context(), next.Host, next.Incarnation, next.Stamp) if err != nil { t.Fatal(err) } for _, e := range out.Entries { if e.Kind == "path" && e.Device != live { t.Fatal("successor kept stale device") } } if calls < 2 { t.Fatal("proof not rechecked", calls) } }) } } func TestServingHandoverEqualNumbersRequireProof(t *testing.T) { for _, boundary := range []string{"initial", "intent", "barrier", "inventory-write", "complete-journal"} { t.Run(boundary, func(t *testing.T) { h, p, n := servingHandoverFixture(t) changed := boundary == "initial" calls := 0 h.VerifyDirectory = func(_ context.Context, _ hostguard.FenceTarget, e StopInventoryEntry, a Artifact) error { calls++ if e.Device != a.Dev || e.Inode != a.Ino || e.Type != a.Type { t.Fatal("numeric identities differ") } if changed { return ErrChanged } return nil } if boundary == "inventory-write" { validate := h.Validate h.Validate = func(ctx context.Context, c *hostclaim.Claim, p hostguard.FenceTarget, n ColdSuccessor, in StopInventory, i hostguard.RootInstallation) error { err := validate(ctx, c, p, n, in, i) if calls == 3 { changed = true } return err } } else if boundary != "initial" { h.checkpoint = func(point string) error { if point == boundary { return errors.New("interrupted") } return nil } if h.Run(t.Context(), p, n) == nil { t.Fatal("interruption missed") } h.checkpoint = nil changed = true } before, err := h.Registry.Get(p.Host) if err != nil { t.Fatal(err) } inventory, invErr := h.Inventory.ReadInstallation(t.Context(), n.Host, n.Incarnation, n.Stamp) if err := h.Run(t.Context(), p, n); !errors.Is(err, ErrChanged) { t.Fatal("unproven directory accepted", err) } if calls == 0 { t.Fatal("proof skipped") } after, err := h.Registry.Get(p.Host) if err != nil || !reflect.DeepEqual(before, after) { t.Fatal("registry changed", err) } got, gotErr := h.Inventory.ReadInstallation(t.Context(), n.Host, n.Incarnation, n.Stamp) if !reflect.DeepEqual(inventory, got) || !errors.Is(gotErr, invErr) { t.Fatal("inventory changed", gotErr) } if boundary != "complete-journal" && !errors.Is(gotErr, sql.ErrNoRows) { t.Fatal("successor inventory exists", gotErr) } }) } } func TestServingHandoverFreshRouteWithoutState(t *testing.T) { for _, first := range []bool{true, false} { t.Run(fmt.Sprint(first), func(t *testing.T) { h, p, n := servingHandoverInitialFixture(t, first, true) err := h.Run(t.Context(), p, n) if !first { if !errors.Is(err, ErrTypedInventory) { t.Fatal("running predecessor without state", err) } return } if err != nil { t.Fatal("first route handover", err) } if err := h.Run(t.Context(), p, n); err != nil { t.Fatal("replay", err) } in, err := h.Inventory.ReadInstallation(t.Context(), n.Host, n.Incarnation, n.Stamp) if err != nil || len(in.Entries) != 1 || in.Entries[0].Kind != "route" { t.Fatal("unexpected state authority", in, err) } }) } } func TestServingHandoverBarrierIsNotPendingStop(t *testing.T) { h, prior, next := servingHandoverFixture(t) crashed := errors.New("crash after durable barrier") h.checkpoint = func(point string) error { if point == "barrier" { return crashed } return nil } if err := h.Run(t.Context(), prior, next); !errors.Is(err, crashed) { t.Fatal(err) } all, err := h.Guard.PendingFences(t.Context()) if err != nil || len(all) != 1 { t.Fatal(all, err) } stops, err := h.Guard.PendingStopFences(t.Context()) if err != nil || len(stops) != 0 { t.Fatal("handover exposed as a stop", stops, err) } h.checkpoint = nil if err := h.Recover(t.Context()); err != nil { t.Fatal("recover handover", err) } all, err = h.Guard.PendingFences(t.Context()) if err != nil || len(all) != 0 { t.Fatal(all, err) } }