//go:build linux package hostbox import ( "bytes" "context" "errors" "os" "path/filepath" "testing" "time" "golang.org/x/sys/unix" "github.com/carlosframework/platform/internal/hostclaim" "github.com/carlosframework/platform/internal/hostguard" "github.com/carlosframework/platform/internal/provision" "github.com/carlosframework/platform/internal/registry" ) // intendedFixture is a prior installation whose PrepareState persisted the // state-dir intent and then failed or crashed before its final inventory write. type intendedFixture struct { handover ColdHandover prior hostguard.FenceTarget next ColdSuccessor inventory *StopInventoryDB reg *registry.Registry data string stateRel string state string id string } func newIntendedFixture(t *testing.T, edit func(*StopInventoryEntry)) intendedFixture { t.Helper() t.Setenv("CARLOS_HOST_CLAIM", "1") source, store, initial := workFixture(t) ctx := t.Context() admit := func(old hostclaim.Observed, generation string) hostclaim.Observed { next, err := hostclaim.Update(ctx, store, old, hostclaim.AdmitWriter(hostclaim.Admission{ObservedOpID: old.Claim.Op.ID, ObservedPhase: old.Claim.Op.Phase, ObservedSession: old.Claim.Holder.Session, ObservedGeneration: old.Claim.Holder.Generation, Box: "box", Region: "ie", Mode: "cold", NewGeneration: generation})) if err != nil { t.Fatal(err) } return next } old := admit(initial, "first") if err := source.Guard.Activate(ctx, old.Host, "box", nil); err != nil { t.Fatal(err) } prior := hostguard.FenceTarget{FenceID: "fence", Region: "ie", Host: old.Host, Incarnation: old.Claim.Holder.Incarnation, Stamp: old.Claim.Holder.Stamp, Session: old.Claim.Holder.Session, Box: "box", Enrollment: "enrollment", BootID: "boot", ControllerID: "controller", ClaimETag: old.ETag, DirectoryRevision: "1", Op: old.Claim.Op.ID, Phase: old.Claim.Op.Phase} if _, err := source.Guard.BeginFence(ctx, prior); err != nil { t.Fatal(err) } if err := source.Guard.CompleteFence(ctx, prior, []byte("verified-root-receipt"), time.Now().UTC(), func(context.Context, hostguard.FenceTarget, []byte, time.Time) error { return nil }); err != nil { t.Fatal(err) } base := t.TempDir() reg, err := registry.Open(filepath.Join(base, "registry.db")) if err != nil { t.Fatal(err) } t.Cleanup(func() { reg.Close() }) inventory, err := OpenStopInventoryDB(filepath.Join(base, "inventory.db")) if err != nil { t.Fatal(err) } t.Cleanup(func() { inventory.Close() }) launches, err := OpenLaunchJournal(filepath.Join(base, "launches.db")) if err != nil { t.Fatal(err) } t.Cleanup(func() { launches.Close() }) owner := registry.ClaimOwnership{ClaimIncarnation: prior.Incarnation, ClaimStamp: prior.Stamp, ClaimOwner: "old-owner", ClaimRowID: "old-route"} data := source.Guard.DataDir dbPath := provision.NestedDBPath(data, old.Host) statePath := filepath.Dir(dbPath) if err = os.MkdirAll(filepath.Dir(statePath), 0700); err != nil { t.Fatal(err) } stateRel, err := filepath.Rel(data, statePath) if err != nil { t.Fatal(err) } if err = reg.Upsert(registry.Route{Host: old.Host, Kind: "instance", Socket: provision.NestedSocket(old.Host), DBPath: dbPath, ClaimOwnership: owner}); err != nil { t.Fatal(err) } if err = reg.UpsertTimer(registry.Timer{Host: old.Host, Name: "daily", Kind: "cron", Spec: "0 0 * * *", NextAt: 123, Attempts: 2}); err != nil { t.Fatal(err) } // The intent PrepareState writes before Mkdirat: typed, no identity. state := StopInventoryEntry{Kind: "path", ArtifactID: "state-dir", ManagedRoot: "data", RelativePath: stateRel, Ownership: owner.ClaimOwner, Materialization: "intended", Type: unix.S_IFDIR} if edit != nil { edit(&state) } in := StopInventory{Complete: true, Host: old.Host, Incarnation: prior.Incarnation, InstallationStamp: prior.Stamp, Box: "box", Enrollment: "enrollment", DirectoryRevision: "1", Entries: []StopInventoryEntry{ {Kind: "route", ArtifactID: owner.ClaimRowID, ManagedRoot: "registry", RelativePath: old.Host, RowIdentity: owner.ClaimRowID, Ownership: owner.ClaimOwner, Materialization: "materialized"}, state, }} if err = inventory.WriteStopInventory(ctx, in); err != nil { t.Fatal(err) } successor := admit(old, "second") next := ColdSuccessor{Host: old.Host, Incarnation: prior.Incarnation, Stamp: successor.Claim.Holder.Stamp, Session: successor.Claim.Holder.Session, Generation: successor.Claim.Holder.Generation, Box: "box", Enrollment: "enrollment", DirectoryRevision: "1", ETag: successor.ETag, Op: successor.Claim.Op.ID, Phase: successor.Claim.Op.Phase} handover := ColdHandover{Guard: source.Guard, Claims: store, Registry: reg, Inventory: inventory, Launches: launches, Roots: map[string]string{"data": data}, JournalDir: filepath.Join(base, "handover"), VerifyReceipt: func(context.Context, hostguard.FenceReceipt) error { return nil }} return intendedFixture{handover: handover, prior: prior, next: next, inventory: inventory, reg: reg, data: data, stateRel: stateRel, state: statePath, id: "handover-" + ownershipFor(next.Host, next.Incarnation, next.Stamp, next.Box)} } func (f intendedFixture) parked(retirement, name string) string { return filepath.Join(f.data, ".parked", f.prior.Incarnation, f.prior.Host, f.id+retirement, "data", name) } func (f intendedFixture) intendedJournal() string { return filepath.Join(f.handover.JournalDir, "retired", f.id+"-intended.json") } func (f intendedFixture) obstructMainRetirement(t *testing.T) string { t.Helper() blocked := filepath.Join(f.handover.JournalDir, "retired", f.id+"-old.json") if err := os.MkdirAll(blocked, 0700); err != nil { t.Fatal(err) } return blocked } func (f intendedFixture) requirePriorRoute(t *testing.T) { t.Helper() if route, err := f.reg.Get(f.prior.Host); err != nil || route == nil || route.ClaimStamp != f.prior.Stamp || route.ClaimOwner != "old-owner" { t.Fatalf("refusal changed prior route: %+v %v", route, err) } } func (f intendedFixture) requireUnmoved(t *testing.T) { t.Helper() f.requirePriorRoute(t) if _, err := os.Lstat(f.intendedJournal()); !errors.Is(err, os.ErrNotExist) { t.Fatalf("refusal recorded an intended capture: %v", err) } } func (f intendedFixture) requireComplete(t *testing.T) { t.Helper() if _, err := os.Lstat(f.state); !errors.Is(err, os.ErrNotExist) { t.Fatalf("intended state dir remains: %v", err) } if route, err := f.reg.Get(f.prior.Host); err != nil || route == nil || route.ClaimStamp != f.next.Stamp { t.Fatalf("route not handed to successor: %+v %v", route, err) } } // Mkdirat failed (ENOSPC/EACCES) after the intent was durable: nothing exists // to move, so the next admission's handover must complete. func TestColdHandoverResumesAbsentIntendedStateDir(t *testing.T) { for name, typ := range map[string]uint32{"typed": unix.S_IFDIR, "untyped": 0} { t.Run(name, func(t *testing.T) { f := newIntendedFixture(t, func(e *StopInventoryEntry) { e.Type = typ }) ctx := t.Context() if err := f.handover.Run(ctx, f.prior, f.next); err != nil { t.Fatalf("absent intended path stranded handover: %v", err) } f.requireComplete(t) if _, err := os.Lstat(f.intendedJournal()); !errors.Is(err, os.ErrNotExist) { t.Fatalf("absent intent captured an artifact: %v", err) } if _, err := os.Lstat(f.parked("-intended", f.stateRel)); !errors.Is(err, os.ErrNotExist) { t.Fatalf("absent intent parked something: %v", err) } if err := f.handover.Run(ctx, f.prior, f.next); err != nil { t.Fatal(err) } if err := f.handover.Recover(ctx); err != nil { t.Fatal(err) } f.requireComplete(t) }) } } // Crash after Mkdirat and before the materialized write: the empty directory // is parked by its captured inode, never unlinked, and replay never recaptures. func TestColdHandoverParksTypedEmptyIntendedStateDirAndReplays(t *testing.T) { f := newIntendedFixture(t, nil) ctx := t.Context() if err := os.Mkdir(f.state, 0700); err != nil { t.Fatal(err) } created, err := os.Lstat(f.state) if err != nil { t.Fatal(err) } // Fail the main retirement after the intended park, as an interrupted run. blocked := f.obstructMainRetirement(t) if err = f.handover.Run(ctx, f.prior, f.next); err == nil { t.Fatalf("obstructed main retirement: %v", err) } f.requirePriorRoute(t) parked := f.parked("-intended", f.stateRel) got, err := os.Lstat(parked) if err != nil || !os.SameFile(created, got) { t.Fatalf("intended dir not parked by inode: %v", err) } if _, err = os.Lstat(f.state); !errors.Is(err, os.ErrNotExist) { t.Fatalf("intended dir still at source: %v", err) } body, err := os.ReadFile(f.intendedJournal()) if err != nil { t.Fatal(err) } var journal RetirementJournal if err = decodeStrict(body, &journal); err != nil || !journal.Complete || len(journal.Inventory.Artifacts) != 1 || journal.Target == nil || *journal.Target != f.prior { t.Fatalf("intended capture not journalled under the exact prior: %+v %v", journal, err) } // A same-named directory appearing after the durable decision is unknown. if err = os.Mkdir(f.state, 0700); err != nil { t.Fatal(err) } if err = os.Remove(blocked); err != nil { t.Fatal(err) } if err = f.handover.Run(ctx, f.prior, f.next); !errors.Is(err, ErrChanged) { t.Fatalf("late same-named directory accepted: %v", err) } if _, err = os.Lstat(f.state); err != nil { t.Fatalf("late directory touched: %v", err) } if err = os.Remove(f.state); err != nil { t.Fatal(err) } if err = f.handover.Recover(ctx); err != nil { t.Fatal(err) } f.requireComplete(t) if got, err = os.Lstat(parked); err != nil || !os.SameFile(created, got) { t.Fatalf("parked intended dir not preserved: %v", err) } if again, err := os.ReadFile(f.intendedJournal()); err != nil || !bytes.Equal(again, body) { t.Fatalf("replay rewrote intended capture: %v", err) } mainBody, err := os.ReadFile(filepath.Join(f.handover.JournalDir, "retired", f.id+"-old.json")) if err != nil { t.Fatal(err) } var rows RetirementJournal if err = decodeStrict(mainBody, &rows); err != nil || !rows.Complete || len(rows.Archives) == 0 { t.Fatalf("row archives missing: %v", err) } for range 2 { if err = f.handover.Run(ctx, f.prior, f.next); err != nil { t.Fatal(err) } if err = f.handover.Recover(ctx); err != nil { t.Fatal(err) } } if got, err = os.Lstat(parked); err != nil || !os.SameFile(created, got) { t.Fatalf("replay disturbed parked intended dir: %v", err) } if again, err := os.ReadFile(filepath.Join(f.handover.JournalDir, "retired", f.id+"-old.json")); err != nil || !bytes.Equal(again, mainBody) { t.Fatalf("replay rewrote row archives: %v", err) } } // Ownership of a present intended name is only provable for an empty // directory of the recorded type with no successor claim on it. Every other // shape refuses before any artifact moves, and keeps refusing. func TestColdHandoverRefusesUnprovableIntendedStateDir(t *testing.T) { cases := []struct { name string edit func(*StopInventoryEntry) setup func(*testing.T, intendedFixture) want error check func(*testing.T, intendedFixture) }{ { name: "nonempty", setup: func(t *testing.T, f intendedFixture) { if err := os.Mkdir(f.state, 0700); err != nil { t.Fatal(err) } if err := os.WriteFile(filepath.Join(f.state, "data.db"), []byte("unknown"), 0600); err != nil { t.Fatal(err) } }, want: ErrChanged, check: func(t *testing.T, f intendedFixture) { if body, err := os.ReadFile(filepath.Join(f.state, "data.db")); err != nil || string(body) != "unknown" { t.Fatalf("nonempty intended dir touched: %v", err) } }, }, { name: "symlink", setup: func(t *testing.T, f intendedFixture) { if err := os.Mkdir(filepath.Join(f.data, "elsewhere"), 0700); err != nil { t.Fatal(err) } if err := os.Symlink("elsewhere", f.state); err != nil { t.Fatal(err) } }, want: ErrChanged, check: func(t *testing.T, f intendedFixture) { if info, err := os.Lstat(f.state); err != nil || info.Mode()&os.ModeSymlink == 0 { t.Fatalf("symlink touched: %v", err) } if _, err := os.Lstat(filepath.Join(f.data, "elsewhere")); err != nil { t.Fatalf("symlink target touched: %v", err) } }, }, { name: "regular file", setup: func(t *testing.T, f intendedFixture) { if err := os.WriteFile(f.state, []byte("unknown"), 0600); err != nil { t.Fatal(err) } }, want: ErrChanged, check: func(t *testing.T, f intendedFixture) { if body, err := os.ReadFile(f.state); err != nil || string(body) != "unknown" { t.Fatalf("regular file touched: %v", err) } }, }, { name: "untyped intent", edit: func(e *StopInventoryEntry) { e.Type = 0 }, want: ErrChanged, }, { name: "wrong recorded type", edit: func(e *StopInventoryEntry) { e.Type = unix.S_IFREG }, want: ErrChanged, }, { name: "intent with identity", edit: func(e *StopInventoryEntry) { e.Device, e.Inode = 1, 1 }, want: ErrTypedInventory, }, { name: "successor state", setup: func(t *testing.T, f intendedFixture) { if err := os.Mkdir(f.state, 0700); err != nil { t.Fatal(err) } successor := StopInventory{Complete: true, Host: f.next.Host, Incarnation: f.next.Incarnation, InstallationStamp: f.next.Stamp, Box: f.next.Box, Enrollment: f.next.Enrollment, DirectoryRevision: f.next.DirectoryRevision, Entries: []StopInventoryEntry{{Kind: "path", ArtifactID: "successor-state", ManagedRoot: "data", RelativePath: f.stateRel, Ownership: "successor-owner", Materialization: "intended", Type: unix.S_IFDIR}}} if err := f.inventory.WriteStopInventory(t.Context(), successor); err != nil { t.Fatal(err) } }, want: ErrChanged, }, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { f := newIntendedFixture(t, tc.edit) setup := tc.setup if setup == nil { setup = func(t *testing.T, f intendedFixture) { if err := os.Mkdir(f.state, 0700); err != nil { t.Fatal(err) } } } setup(t, f) before, err := os.Lstat(f.state) if err != nil { t.Fatal(err) } for range 2 { if err = f.handover.Run(t.Context(), f.prior, f.next); !errors.Is(err, tc.want) { t.Fatalf("got %v, want %v", err, tc.want) } f.requireUnmoved(t) if after, err := os.Lstat(f.state); err != nil || !os.SameFile(before, after) { t.Fatalf("refused intended name changed: %v", err) } if _, err = os.Lstat(f.parked("-intended", f.stateRel)); !errors.Is(err, os.ErrNotExist) { t.Fatalf("refused intended name parked: %v", err) } } if tc.check != nil { tc.check(t, f) } }) } } func TestColdHandoverAbsentIntentionDecisionSurvivesInterruptedRetirement(t *testing.T) { f := newIntendedFixture(t, nil) ctx := t.Context() blocked := f.obstructMainRetirement(t) if err := f.handover.Run(ctx, f.prior, f.next); err == nil { t.Fatalf("first retirement: %v", err) } f.requirePriorRoute(t) if err := os.Remove(blocked); err != nil { t.Fatal(err) } if err := os.Mkdir(f.state, 0700); err != nil { t.Fatal(err) } unknown, err := os.Lstat(f.state) if err != nil { t.Fatal(err) } if err := f.handover.Recover(ctx); !errors.Is(err, ErrChanged) { t.Fatalf("accepted newly appearing intended path: %v", err) } got, err := os.Lstat(f.state) if err != nil || !os.SameFile(unknown, got) { t.Fatalf("unknown path moved: %v", err) } f.requireUnmoved(t) if err := os.Remove(f.state); err != nil { t.Fatal(err) } if err := f.handover.Recover(ctx); err != nil { t.Fatal(err) } f.requireComplete(t) } func TestColdHandoverEmptyDirectoryUsesCapturedIdentity(t *testing.T) { root := t.TempDir() p := TypedParker{Roots: map[string]string{"data": root}, ParkedRoot: root, Host: "host", Incarnation: "inc", Retirement: "retire"} path := filepath.Join(root, "state") if err := os.Mkdir(path, 0700); err != nil { t.Fatal(err) } a, err := p.CaptureArtifact("data", "state", "owner", ArtifactPath) if err != nil { t.Fatal(err) } if err := p.VerifyEmptyDirectory(a); err != nil { t.Fatal(err) } if err := os.Rename(path, filepath.Join(root, "captured")); err != nil { t.Fatal(err) } if err := os.Mkdir(path, 0700); err != nil { t.Fatal(err) } if err := p.VerifyEmptyDirectory(a); !errors.Is(err, ErrChanged) { t.Fatalf("substitute empty directory accepted: %v", err) } if err := os.Remove(path); err != nil { t.Fatal(err) } if err := os.Symlink("captured", path); err != nil { t.Fatal(err) } if err := p.VerifyEmptyDirectory(a); err == nil { t.Fatal("followed symlink to captured directory") } if err := os.Remove(path); err != nil { t.Fatal(err) } if err := os.Rename(filepath.Join(root, "captured"), path); err != nil { t.Fatal(err) } if err := os.WriteFile(filepath.Join(path, "unknown"), nil, 0600); err != nil { t.Fatal(err) } if err := p.VerifyEmptyDirectory(a); !errors.Is(err, ErrChanged) { t.Fatalf("nonempty directory accepted: %v", err) } } func TestClaimReviewHandoverRefusesCompletedSuccessorProducer(t *testing.T) { f := newIntendedFixture(t, nil) target := f.prior target.Stamp = f.next.Stamp target.Session = f.next.Session j := f.handover.Launches if err := j.Prelaunch(t.Context(), LaunchInventory{ControllerPID: 123, ProcessStart: "456", Target: target, Groups: []LaunchGroup{{RelativePath: "host", Device: 1, Inode: 1}}, NoState: true}); err != nil { t.Fatal(err) } if err := j.RecordProducer(t.Context(), target, "delivery"); err != nil { t.Fatal(err) } if _, err := j.CompleteProducer(t.Context(), target, "delivery", ProducerCompletion{}); err != nil { t.Fatal(err) } if err := f.handover.Run(t.Context(), f.prior, f.next); !errors.Is(err, ErrChanged) { t.Fatalf("handover allowed completed successor: %v", err) } f.requireUnmoved(t) } func TestPlan9Fix2ColdHandoverRefusesForeignInventory(t *testing.T) { for _, root := range []string{"opt", "units", "data"} { t.Run(root, func(t *testing.T) { f := newIntendedFixture(t, nil) victim := filepath.Join(f.data, "foreign") if err := os.WriteFile(victim, []byte("root-only"), 0600); err != nil { t.Fatal(err) } f.handover.Roots[root] = f.data parker := TypedParker{Roots: f.handover.Roots, ParkedRoot: f.data, Incarnation: f.prior.Incarnation, Host: f.prior.Host, Retirement: "capture"} a, err := parker.CaptureArtifact(root, "foreign", "old-owner", ArtifactPath) if err != nil { t.Fatal(err) } in, err := f.inventory.ReadExactStopInventory(t.Context(), f.prior) if err != nil { t.Fatal(err) } in.Entries = append(in.Entries, StopInventoryEntry{Kind: "path", ManagedRoot: root, RelativePath: a.Name, ArtifactID: "foreign", Ownership: a.Owner, Device: a.Dev, Inode: a.Ino, Type: a.Type, Materialization: "materialized"}) if err = f.inventory.WriteStopInventory(t.Context(), in); err != nil { t.Fatal(err) } if err = f.handover.Run(t.Context(), f.prior, f.next); !errors.Is(err, ErrTypedInventory) { t.Fatalf("foreign inventory: %v", err) } if raw, err := os.ReadFile(victim); err != nil || string(raw) != "root-only" { t.Fatalf("foreign file moved: %v", err) } }) } }