package host import ( "context" "github.com/carlosframework/platform/internal/hostbox" "github.com/carlosframework/platform/internal/provision" "net" "os/exec" "testing" "time" "github.com/carlosframework/platform/internal/activator" "github.com/carlosframework/platform/internal/hostclaim" "github.com/carlosframework/platform/internal/hostguard" "github.com/carlosframework/platform/internal/registry" ) func TestPlan14RecoveryInventorySourceGap(t *testing.T) { for _, attack := range []string{"none", "record moved", "claim moved", "fenced", "peer"} { t.Run(attack, func(t *testing.T) { f, o, h := plan12Handover(t, hostclaim.KindRestoring, "stop") holder := o.Claim.Holder authority := f.guard authority.Root = hostguard.NewRootAuthority(&rollbackDisk{dir: t.TempDir()}) in := hostguard.RootInstallation{Host: o.Host, Incarnation: holder.Incarnation, Stamp: holder.Stamp, Migration: 1, InventoryRevision: 1, Account: holder.Account, App: holder.App, Record: holder.Key, Box: "box-1", Enrollment: "enrollment-1"} if err := authority.EstablishRoot(t.Context(), in, func(*hostclaim.Claim) error { return nil }); err != nil { t.Fatal(err) } h.Guard.FenceView = authority.Root.View plan14PutRecord(t, f.claims, provision.Record{Schema: 1, Host: o.Host, Account: holder.Account, App: holder.App, Channel: "stable", CreatedAt: f.now}) actor := hostbox.Actor{Box: "box-1", Enrollment: "enrollment-1", Revision: 7, Binding: "7"} root := ClaimRootHTTP{Stop: ClaimStopHTTP{Coordinator: ClaimStopCoordinator{Guard: authority, Evidence: f.claims}, Authenticate: func(context.Context, net.Conn) (StopPeer, error) { return f.peer, nil }}, InventoryActor: func(context.Context) (hostbox.Actor, error) { return actor, nil }, Roots: map[string]string{"data": f.guard.DataDir}, Reconcile: hostbox.ReconcileAdapter{Inventory: h.Inventory, Registry: h.Registry, Box: actor.Box, Enrollment: actor.Enrollment}} // Inventory reads use the same authenticated Unix endpoint as production. journals, err := hostbox.NewJournalService(f.journal) if err != nil { t.Fatal(err) } journals.Inventory = h.Inventory root.Journals = journals socket := startRetirementRecoveryTransport(t, root) _, h.Inventory = hostbox.RemoteJournals(rootJournalClient(socket)) if attack == "peer" { root.Stop.Authenticate = func(context.Context, net.Conn) (StopPeer, error) { return StopPeer{}, hostbox.ErrChanged } socket = startRetirementRecoveryTransport(t, root) } h.RecoveryRebind = func(ctx context.Context, q recoveryRebindRequest) error { err := recoveryRebindClient(socket)(ctx, q) if err != nil { return err } if attack == "fenced" { _, err = root.Stop.Coordinator.Guard.BeginFence(ctx, hostguard.FenceTarget{Host: q.Next.Host, Incarnation: q.Next.Incarnation, Stamp: q.Next.Stamp, Session: q.Next.Session, Op: q.Next.Op, Phase: q.Next.Phase, Box: q.Next.Box, Enrollment: q.Next.Enrollment, DirectoryRevision: q.Next.DirectoryRevision, ClaimETag: q.Next.ETag, BootID: "boot-1", ControllerID: "controller-1", FenceID: "new-fence"}) if err != nil { return err } } if attack == "record moved" { rec := provision.Record{Schema: 1, Host: o.Host, Account: holder.Account, App: holder.App, Channel: "stable", CreatedAt: f.now, RecordSeq: 2} plan14PutRecord(t, f.claims, rec) } if attack == "claim moved" { _, err = hostclaim.Update(ctx, f.claims, f.read(), func(c *hostclaim.Claim) error { c.Op.Journal["moved"] = "yes"; return nil }) } return err } before, err := h.Registry.Get(o.Host) if err != nil { t.Fatal(err) } c := f.controller("boot-1", "controller-1", 4101, "start-1") c.wake.Inputs = plan9D2Inputs{recoveryInputs{f}} c.wake.Operations = &plan9D2Attempts{} c.wake.ProbeReady = func(context.Context, *exec.Cmd, registry.Route, ClaimLaunchInputs, string) error { return nil } wireClaimAdmission(c.wake, "box-1", func(context.Context, hostclaim.Observed, hostclaim.Observed) error { t.Fatal("wrong handover") return nil }, h, nil, nil) if err = c.wake.stopPredecessor(t.Context(), o); err != nil { t.Fatal(err) } live, cancel := context.WithCancel(t.Context()) defer cancel() done := make(chan struct{}) go func() { <-live.Done(); close(done) }() c.wake.mu.Lock() c.wake.pending[o.Host] = &claimWakeAttempt{observed: o, held: recoveryHeld{}, ctx: live, cancel: cancel, done: done} c.wake.mu.Unlock() ctx, stop := context.WithTimeout(t.Context(), 3*time.Second) defer stop() _, err = c.runtime.Admit(ctx, registry.Route{Host: o.Host}, o, activator.StartPolicy{Probe: true}) if attack != "none" { if err == nil { t.Fatal("stale or unauthenticated rebind accepted") } after, e := h.Registry.Get(o.Host) if e != nil || after.ClaimOwnership != before.ClaimOwnership { t.Fatal("stale result rebound rows", e) } return } if err != nil { t.Fatal("root recovery admission failed", err) } current := f.read() if current.Claim.Holder.Stamp <= holder.Stamp { t.Fatal("fixture did not reach the new claim stamp", err) } after, e := h.Registry.Get(o.Host) if e != nil || after.ClaimOwnership == before.ClaimOwnership { t.Fatal("rows not rebound after root commit", e) } view, e := authority.Root.View(t.Context(), o.Host, hostguard.Captured{Incarnation: holder.Incarnation, Stamp: holder.Stamp}) if e != nil || view.Installation.Stamp != current.Claim.Holder.Stamp { t.Fatal("root installation not rebound", view, e) } t.Logf("fresh claim stamp %d, retained root stamp %d; recovery succeeds: %v", current.Claim.Holder.Stamp, view.Installation.Stamp, err) }) } }