//go:build unix package hostguard import ( "context" "encoding/json" "errors" "os" "path/filepath" "sync" "sync/atomic" "testing" "time" "github.com/carlosframework/platform/internal/rootstate" ) type rootTestStore struct { rootstate.Store dir string mu sync.Mutex onLock func() } func (s *rootTestStore) Read(n string) ([]byte, error) { return os.ReadFile(filepath.Join(s.dir, n)) } func (s *rootTestStore) Write(n string, b []byte) error { return os.WriteFile(filepath.Join(s.dir, n), b, 0600) } func (s *rootTestStore) Names() ([]string, error) { entries, e := os.ReadDir(s.dir) var names []string for _, n := range entries { names = append(names, n.Name()) } return names, e } func (s *rootTestStore) Lock(context.Context, string) (func(), error) { s.mu.Lock() if s.onLock != nil { s.onLock() } return s.mu.Unlock, nil } func (s *rootTestStore) Close() error { return nil } func TestPlan14RootFences(t *testing.T) { for _, attack := range []string{"delete", "rename", "replace lock", "forged receipt"} { t.Run(attack, func(t *testing.T) { f, target, captured := fenceFixture(t) store := &rootTestStore{dir: t.TempDir()} root := NewRootAuthority(store) f.g.Root = root f.obs.Claim.Holder.Account, f.obs.Claim.Holder.App = "bab", "console" installation := RootInstallation{Host: target.Host, Incarnation: captured.Incarnation, Stamp: captured.Stamp, Account: "bab", App: "console", Record: "record@etag", Box: target.Box, Enrollment: target.Enrollment, Migration: 1} if err := f.g.EstablishRoot(t.Context(), installation, allow); err != nil { t.Fatal(err) } if err := f.g.With(t.Context(), target.Host, captured, allow, func() error { return nil }); err != nil { t.Fatal(err) } if _, err := f.g.BeginFence(t.Context(), target); err != nil { t.Fatal(err) } before, err := store.Read(target.Host + ".json") if err != nil { t.Fatal(err) } path := filepath.Join(f.g.DataDir, "hostguard", target.Host+".json") switch attack { case "delete": if err := os.Remove(path); err != nil { t.Fatal(err) } case "rename": if err := os.Rename(filepath.Dir(path), filepath.Dir(path)+".old"); err != nil { t.Fatal(err) } case "replace lock": if err := os.Rename(f.g.LockDir, f.g.LockDir+".old"); err != nil { t.Fatal(err) } if err := os.Mkdir(f.g.LockDir, 0700); err != nil { t.Fatal(err) } case "forged receipt": if err := os.WriteFile(path, []byte(`{"v":1,"host":"shop.example","incarnation":"inc","active_stamp":100}`), 0660); err != nil { t.Fatal(err) } } effects := 0 if err := f.g.With(t.Context(), target.Host, captured, allow, func() error { effects++; return nil }); !errors.Is(err, ErrFenced) || effects != 0 { t.Fatalf("fence bypass: %v %d", err, effects) } pending, err := f.g.PendingFences(t.Context()) if err != nil || len(pending) != 1 || pending[0] != target { t.Fatalf("lost recovery: %+v %v", pending, err) } stops, err := f.g.PendingStopFences(t.Context()) if err != nil || len(stops) != 1 || stops[0] != target { t.Fatalf("ordinary stop missing from recovery: %+v %v", stops, err) } after, err := store.Read(target.Host + ".json") if err != nil || string(before) != string(after) { t.Fatal("root evidence changed", err) } got, err := f.g.Capture(target.Host) if err != nil || got != captured { t.Fatal("capture used agent state", got, err) } }) } } func TestPlan14RootFenceView(t *testing.T) { f, target, captured := fenceFixture(t) store := &rootTestStore{dir: t.TempDir()} f.g.Root = NewRootAuthority(store) f.obs.Claim.Holder.Account, f.obs.Claim.Holder.App = "bab", "console" in := RootInstallation{Host: target.Host, Incarnation: captured.Incarnation, Stamp: captured.Stamp, Account: "bab", App: "console", Record: "record@etag", Box: target.Box, Enrollment: target.Enrollment, Migration: 1} if err := f.g.EstablishRoot(t.Context(), in, allow); err != nil { t.Fatal(err) } if _, err := f.g.BeginFence(t.Context(), target); err != nil { t.Fatal(err) } view, err := f.g.Root.View(t.Context(), target.Host, captured) if err != nil { t.Fatal(err) } view.Pending.FenceID = "forged" again, err := f.g.Root.View(t.Context(), target.Host, captured) if err != nil || again.Pending.FenceID != target.FenceID { t.Fatal("mutable view", err) } agent := f.g agent.Root = nil agent.FenceView = f.g.Root.View effects := 0 if err := agent.With(t.Context(), target.Host, captured, allow, func() error { effects++; return nil }); !errors.Is(err, ErrFenced) || effects != 0 { t.Fatal("agent bypassed root view", err, effects) } if _, err := f.g.Root.View(t.Context(), target.Host, Captured{Incarnation: "wrong", Stamp: 1}); err == nil { t.Fatal("wrong tuple accepted") } agent.FenceView = func(context.Context, string, Captured) (FenceView, error) { return FenceView{}, nil } if err := agent.With(t.Context(), target.Host, captured, allow, func() error { effects++; return nil }); err == nil || effects != 0 { t.Fatal("old server granted work") } } func TestPlan14InventoryRegistrationUnderAgentGuardDeadlocks(t *testing.T) { f, target, captured := fenceFixture(t) store := &rootTestStore{dir: t.TempDir()} root := f.g root.Root = NewRootAuthority(store) f.obs.Claim.Holder.Account, f.obs.Claim.Holder.App = "bab", "console" in := RootInstallation{Host: target.Host, Incarnation: captured.Incarnation, Stamp: captured.Stamp, Account: "bab", App: "console", Record: "record@etag", Box: target.Box, Enrollment: target.Enrollment, Migration: 1} if err := root.EstablishRoot(t.Context(), in, allow); err != nil { t.Fatal(err) } // Match the inventory-write call inside ReconcileAdapter.Install's work.With. agent := f.g agent.FenceView = root.Root.View ctx, cancel := context.WithTimeout(t.Context(), 100*time.Millisecond) defer cancel() requested := false err := agent.With(ctx, target.Host, captured, allow, func() error { requested = true return root.EstablishRoot(ctx, in, allow) }) if !requested || !errors.Is(err, context.DeadlineExceeded) { t.Fatalf("expected the root registration to wait for the agent's held legacy lock: %v, requested=%v", err, requested) } } func TestPlan14FenceViewRechecksAfterUnlockedRead(t *testing.T) { f, target, captured := fenceFixture(t) root := f.g root.Root = NewRootAuthority(&rootTestStore{dir: t.TempDir()}) f.obs.Claim.Holder.Account, f.obs.Claim.Holder.App = "bab", "console" in := RootInstallation{Host: target.Host, Incarnation: captured.Incarnation, Stamp: captured.Stamp, Account: "bab", App: "console", Record: "record@etag", Box: target.Box, Enrollment: target.Enrollment, Migration: 1} if err := root.EstablishRoot(t.Context(), in, allow); err != nil { t.Fatal(err) } path := filepath.Join(f.g.DataDir, "hostguard", target.Host+".json") before, err := os.ReadFile(path) if err != nil { t.Fatal(err) } claimBefore, err := json.Marshal(f.obs) if err != nil { t.Fatal(err) } agent := f.g fencedBeforeAdmission := false agent.FenceView = func(ctx context.Context, host string, c Captured) (FenceView, error) { view, err := root.Root.View(ctx, host, c) if err != nil { return FenceView{}, err } // Root attempts its fence after the snapshot, before admission. With // the read inside the legacy lock, it cannot commit until admission // finishes. Moving only the read outside permits this same-tuple race. wait, cancel := context.WithTimeout(ctx, 100*time.Millisecond) defer cancel() _, err = root.BeginFence(wait, target) if err == nil { fencedBeforeAdmission = true } else if !errors.Is(err, context.DeadlineExceeded) { return FenceView{}, err } return view, nil } effects := 0 err = agent.With(t.Context(), target.Host, captured, allow, func() error { effects++; return nil }) if fencedBeforeAdmission { if !errors.Is(err, ErrFenced) || effects != 0 { t.Errorf("stale root snapshot admitted an already fenced session: err=%v effects=%d", err, effects) } } else { if (!errors.Is(err, ErrFenced) || effects != 0) && (err != nil || effects != 1) { t.Fatalf("unfenced admission: err=%v effects=%d", err, effects) } if _, err := root.BeginFence(t.Context(), target); err != nil { t.Fatal(err) } } view, err := root.Root.View(t.Context(), target.Host, captured) if errors.Is(err, ErrFenced) { r, e := root.Root.read(target.Host) if e != nil || !r.quarantined || r.installation != in { t.Fatalf("quarantine changed installation: %+v %v", r, e) } return } if err != nil || view.Pending == nil || *view.Pending != target || view.Installation != in { t.Fatalf("fence changed installation identity: %+v %v", view, err) } after, err := os.ReadFile(path) if err != nil || string(before) != string(after) { t.Fatalf("fence changed agent installation record: %v", err) } claimAfter, err := json.Marshal(f.obs) if err != nil || string(claimBefore) != string(claimAfter) { t.Fatalf("fence changed claim observation: %v", err) } if err := root.With(t.Context(), target.Host, captured, allow, func() error { t.Fatal("root admitted fenced work"); return nil }); !errors.Is(err, ErrFenced) { t.Fatalf("root fence check: %v", err) } } func TestPlan14FenceViewWhileRootWaitsForAgent(t *testing.T) { f, target, captured := fenceFixture(t) root := f.g store := &rootTestStore{dir: t.TempDir()} root.Root = NewRootAuthority(store) f.obs.Claim.Holder.Account, f.obs.Claim.Holder.App = "bab", "console" in := RootInstallation{Host: target.Host, Incarnation: captured.Incarnation, Stamp: captured.Stamp, Account: "bab", App: "console", Record: "record@etag", Box: target.Box, Enrollment: target.Enrollment, Migration: 1} if err := root.EstablishRoot(t.Context(), in, allow); err != nil { t.Fatal(err) } ctx, cancel := context.WithTimeout(t.Context(), 2*time.Second) defer cancel() unlock, err := acquire(ctx, f.g.LockDir, target.Host) if err != nil { t.Fatal(err) } defer func() { if unlock != nil { unlock() } }() locked := make(chan struct{}, 1) var acquisitions atomic.Int32 store.onLock = func() { acquisitions.Add(1); locked <- struct{}{} } done := make(chan error, 1) go func() { _, err := root.BeginFence(ctx, target); done <- err }() select { case <-locked: case <-ctx.Done(): t.Fatal(ctx.Err()) } viewDone := make(chan error, 1) go func() { _, err := root.Root.View(ctx, target.Host, captured); viewDone <- err }() select { case err := <-viewDone: if err != nil { t.Fatal("honest contention fenced the view", err) } case <-ctx.Done(): t.Fatal("fence view waited on root holding a lock needed by the agent") } if n := acquisitions.Load(); n != 2 { t.Fatalf("root mutation and view must each acquire the root lock: %d acquisitions", n) } unlock() unlock = nil select { case err := <-done: if !errors.Is(err, context.DeadlineExceeded) { t.Fatal(err) } case <-ctx.Done(): t.Fatal("root update did not finish after agent unlocked") } } func TestPlan14RootInventoryCommitRetainsDenials(t *testing.T) { f, target, captured := fenceFixture(t) f.obs.Claim.Holder.Account, f.obs.Claim.Holder.App = "bab", "console" store := &rootTestStore{dir: t.TempDir()} f.g.Root = NewRootAuthority(store) in := RootInstallation{Host: target.Host, Incarnation: captured.Incarnation, Stamp: captured.Stamp, Account: "bab", App: "console", Record: "record", Box: target.Box, Enrollment: target.Enrollment, Migration: 1, InventoryRevision: 1} effects := 0 commit := func() error { effects++; return nil } if err := f.g.CommitRootInstallation(t.Context(), in, allow, commit); err != nil || effects != 1 { t.Fatalf("initial root inventory: %v %d", err, effects) } if _, err := f.g.BeginFence(t.Context(), target); err != nil { t.Fatal(err) } in.InventoryRevision++ if err := f.g.CommitRootInstallation(t.Context(), in, allow, commit); !errors.Is(err, ErrFenced) || effects != 1 { t.Fatalf("fenced inventory commit: %v %d", err, effects) } view, err := f.g.Root.View(t.Context(), target.Host, captured) if err != nil || view.Installation.InventoryRevision != 1 || view.Pending == nil || *view.Pending != target { t.Fatalf("inventory erased fence: %+v %v", view, err) } } func TestPlan14ColdHandoverRefreshesRootInventory(t *testing.T) { testRootColdHandover(t, false) } func TestPlan14MigrationPredecessorReadmission(t *testing.T) { testRootColdHandover(t, true) } func testRootColdHandover(t *testing.T, migration bool) { f, prior, captured := fenceFixture(t) f.obs.Claim.Holder.Account, f.obs.Claim.Holder.App = "bab", "app" store := &rootTestStore{dir: t.TempDir()} f.g.Root = NewRootAuthorityWithInventory(store, func(_ context.Context, host string, c Captured) (uint64, error) { if host != prior.Host || c.Stamp != 17 { return 0, ErrStampMismatch } return 2, nil }) in := RootInstallation{Host: prior.Host, Incarnation: prior.Incarnation, Stamp: prior.Stamp, Account: "bab", App: "app", Record: "record", Box: prior.Box, Enrollment: prior.Enrollment, Migration: 1, InventoryRevision: 5} if err := f.g.EstablishRoot(t.Context(), in, allow); err != nil { t.Fatal(err) } if migration { if err := f.g.Root.RetainMigrationDeny(t.Context(), in, prior.Session); err != nil { t.Fatal(err) } } if _, err := f.g.BeginFence(t.Context(), prior); err != nil { t.Fatal(err) } at := time.Now().UTC() if err := f.g.CompleteFence(t.Context(), prior, []byte("verified stop"), at, func(context.Context, FenceTarget, []byte, time.Time) error { return nil }); err != nil { t.Fatal(err) } next := prior next.FenceID = "cold-next" next.Stamp = 17 next.Session = 17 f.obs.Claim.Holder.Stamp, f.obs.Claim.Holder.Session = 17, 17 if err := f.g.BeginColdHandover(t.Context(), prior, next, allow); err != nil { t.Fatal(err) } if err := f.g.Activate(t.Context(), prior.Host, prior.Box, nil); err == nil { t.Fatal("activation cleared cold barrier") } if migration { fail := errors.New("interrupted handover") if err := f.g.CompleteColdHandover(t.Context(), next, false, allow, func() error { return fail }); !errors.Is(err, fail) { t.Fatal(err) } if err := f.g.With(t.Context(), prior.Host, Captured{Incarnation: prior.Incarnation, Stamp: 17}, allow, func() error { t.Fatal("pending successor ran"); return nil }); err == nil { t.Fatal("pending handover allowed") } } completed := 0 if err := f.g.CompleteColdHandover(t.Context(), next, false, allow, func() error { completed++; return nil }); err != nil { t.Fatal(err) } if err := f.g.CompleteColdHandover(t.Context(), next, true, allow, func() error { completed++; return nil }); err != nil || completed != 1 { t.Fatal("handover replay", err, completed) } view, err := f.g.Root.View(t.Context(), prior.Host, Captured{Incarnation: prior.Incarnation, Stamp: 17}) if err != nil || view.Installation.InventoryRevision != 2 || view.Barrier != nil { t.Fatalf("successor snapshot: %+v %v", view, err) } effects := 0 if err := f.g.With(t.Context(), prior.Host, view.Captured, allow, func() error { effects++; return nil }); err != nil || effects != 1 { t.Fatalf("successor refused: %v", err) } if err := f.g.With(t.Context(), prior.Host, captured, allow, func() error { effects++; return nil }); err == nil || effects != 1 { t.Fatal("predecessor replay admitted") } receipt, err := f.g.FenceReceiptFor(t.Context(), prior) if err != nil || !receipt.EvidenceAt.Equal(at) { t.Fatalf("old stop evidence lost: %+v %v", receipt, err) } } func TestPlan14InstallationCommitRecovery(t *testing.T) { f, target, captured := fenceFixture(t) store := &rootTestStore{dir: t.TempDir()} f.g.Root = NewRootAuthority(store) f.obs.Claim.Holder.Account, f.obs.Claim.Holder.App = "bab", "console" in := RootInstallation{Host: target.Host, Incarnation: captured.Incarnation, Stamp: captured.Stamp, Account: "bab", App: "console", Record: "record", Box: target.Box, Enrollment: target.Enrollment, Migration: 1, InventoryRevision: 1} crash := errors.New("simulated crash after inventory commit") intent := []byte("typed inventory request") err := f.g.CommitRootInventory(t.Context(), in, intent, allow, nil, func(RootInstallation) error { return crash }) if !errors.Is(err, crash) { t.Fatal(err) } f.g.Root = NewRootAuthority(store) if err := f.g.With(t.Context(), target.Host, captured, allow, func() error { t.Fatal("pending installation admitted"); return nil }); err == nil { t.Fatal("admitted") } pending, err := f.g.Root.PendingInstallations() if err != nil || len(pending) != 1 || string(pending[0]) != string(intent) { t.Fatal(pending, err) } if _, err = f.g.Root.View(t.Context(), target.Host, captured); err == nil { t.Fatal("pending view available") } if err = f.g.CommitRootInventory(t.Context(), in, []byte("other"), allow, nil, func(RootInstallation) error { t.Fatal("changed intent committed"); return nil }); err == nil { t.Fatal("changed intent accepted") } if err = f.g.CommitRootInventory(t.Context(), in, intent, allow, nil, func(RootInstallation) error { return nil }); err != nil { t.Fatal(err) } if err = f.g.With(t.Context(), target.Host, captured, allow, func() error { return nil }); err != nil { t.Fatal(err) } pending, err = f.g.Root.PendingInstallations() if err != nil || len(pending) != 0 { t.Fatal(pending, err) } } func TestPlan14RootNoInstallationNeedsExplicitEmptyState(t *testing.T) { f, target, captured := fenceFixture(t) f.obs.Claim.Holder.Account, f.obs.Claim.Holder.App = "bab", "app" store := &rootTestStore{dir: t.TempDir()} f.g.Root = NewRootAuthority(store) effects := 0 effect := func() error { effects++; return nil } if err := f.g.WithNoInstallation(t.Context(), target.Host, captured.Incarnation, allow, effect); err == nil || effects != 0 { t.Fatal("missing root record authorized absence", err) } in := RootInstallation{Host: target.Host, Incarnation: captured.Incarnation, Stamp: captured.Stamp, Account: "bab", App: "app", Record: "record", Box: target.Box, Enrollment: target.Enrollment, Migration: 1, Empty: true} if err := f.g.EstablishRoot(t.Context(), in, allow); err != nil { t.Fatal(err) } if err := f.g.WithNoInstallation(t.Context(), target.Host, captured.Incarnation, allow, effect); err != nil || effects != 1 { t.Fatal("explicit empty refused", err) } if _, err := f.g.BeginFence(t.Context(), target); err != nil { t.Fatal(err) } if err := f.g.WithNoInstallation(t.Context(), target.Host, captured.Incarnation, allow, effect); err == nil || effects != 1 { t.Fatal("empty bypassed fence", err) } } func TestPlan14AdmissionFenceUsesRootView(t *testing.T) { f, target, captured := fenceFixture(t) store := &rootTestStore{dir: t.TempDir()} root := f.g root.Root = NewRootAuthority(store) f.obs.Claim.Holder.Account, f.obs.Claim.Holder.App = "bab", "console" in := RootInstallation{Host: target.Host, Incarnation: captured.Incarnation, Stamp: captured.Stamp, Account: "bab", App: "console", Record: "record", Box: target.Box, Enrollment: target.Enrollment, Migration: 1} if err := root.EstablishRoot(t.Context(), in, allow); err != nil { t.Fatal(err) } agent := f.g agent.FenceView = root.Root.View for _, g := range []Guard{root, agent} { if closed, err := g.AdmissionFenced(target.Host, f.obs.Claim, captured); err != nil || closed { t.Fatal(closed, err) } } if _, err := root.BeginFence(t.Context(), target); err != nil { t.Fatal(err) } if err := os.Remove(filepath.Join(agent.DataDir, "hostguard", target.Host+".json")); err != nil { t.Fatal(err) } for _, g := range []Guard{root, agent} { ctx, cancel := context.WithTimeout(t.Context(), time.Second) err := g.WithLocal(ctx, target.Host, func() error { closed, err := g.AdmissionFenced(target.Host, f.obs.Claim, captured) if err == nil && !closed { return errors.New("root fence was ignored") } return err }) cancel() if err != nil { t.Fatal(err) } } agent.FenceView = func(context.Context, string, Captured) (FenceView, error) { return FenceView{}, nil } if _, err := agent.AdmissionFenced(target.Host, f.obs.Claim, captured); err == nil { t.Fatal("malformed view granted admission") } v, err := root.Root.View(t.Context(), target.Host, captured) if err != nil { t.Fatal(err) } v.Pending = nil v.Retired = []Captured{captured} if !errors.Is(v.admit(target.Host, captured, f.obs.Claim.Holder.Session), ErrRetiredStamp) { t.Fatal("retired view granted admission") } }