//go:build linux package host import ( "encoding/json" "errors" "strings" "testing" "time" "github.com/carlosframework/platform/internal/hostbox" "github.com/carlosframework/platform/internal/hostclaim" "github.com/carlosframework/platform/internal/hostguard" "github.com/carlosframework/platform/internal/hostops" "github.com/carlosframework/platform/internal/provision" "github.com/carlosframework/platform/internal/registry" "github.com/carlosframework/platform/internal/rootparticipation" "github.com/carlosframework/platform/internal/rootstate" ) var errRootInterrupted = errors.New("root interrupted before its commit") func (w *assemblyWorld) pending() int { w.t.Helper() pending, err := w.assembly.Guard.Root.PendingInstallations() if err != nil { w.t.Fatal(err) } return len(pending) } // interruptedRegistration is the agent's state registration for rec through // root's commitRootInventory, with root dying after its intent is durable // and before it records the commit. func (w *assemblyWorld) interruptedRegistration(agent *bindingAgent, rec provision.Record, route registry.Route, actor hostbox.Actor) { w.t.Helper() before := w.pending() *w.interrupt = 1 err := agent.registerState(rec.Host, route.DBPath, actor) *w.interrupt = -1 if err == nil || w.pending() != before+1 { w.t.Fatalf("registration was not interrupted after its intent: %v, pending %d", err, w.pending()) } } // interruptedRebind leaves a recovery rebind intent for rec's installation // stamped with stamp: the claim moves to a recovered successor stamp, and the // guard's RebindRootInventory (the write rebindRecoveryInventory makes) // persists the intent and is interrupted before its commit. func (w *assemblyWorld) interruptedRebind(rec provision.Record, stamp string) recoveryRebindRequest { w.t.Helper() ctx := w.t.Context() o := w.claim(rec.Host) prior := hostguard.Captured{Incarnation: o.Claim.Holder.Incarnation, Stamp: o.Claim.Holder.Stamp} _, revision := w.inventory(rec.Host) next, err := hostclaim.Update(ctx, w.store, o, func(c *hostclaim.Claim) error { c.Holder.Stamp, c.Holder.Session, c.Holder.Generation, c.Holder.WriterBox = prior.Stamp+1, prior.Stamp+1, "recovered", "box-1" c.Holder.Admissions = []hostclaim.AdmissionRecord{{Region: "ie", Session: prior.Stamp + 1, Box: "box-1", Generation: "recovered", Mode: "cold"}} return nil }) if err != nil { w.t.Fatal(err) } h := next.Claim.Holder _, recordETag, err := w.store.Get(ctx, h.Key) if err != nil { w.t.Fatal(err) } q := recoveryRebindRequest{V: 1, Prior: prior, Revision: revision, RecordKey: h.Key, RecordETag: recordETag, Next: hostbox.ColdSuccessor{Host: rec.Host, Incarnation: h.Incarnation, Stamp: h.Stamp, Session: h.Session, Generation: h.Generation, Box: "box-1", Enrollment: w.roster.row().Enrollment, DirectoryRevision: stamp, ETag: next.ETag}} raw, err := json.Marshal(recoveryRebindIntent{Recovery: &q}) if err != nil { w.t.Fatal(err) } installation := hostguard.RootInstallation{Host: rec.Host, Incarnation: h.Incarnation, Stamp: h.Stamp, Account: h.Account, App: h.App, Record: h.Key, Box: "box-1", Enrollment: q.Next.Enrollment, Migration: 1, InventoryRevision: 1} before := w.pending() err = w.assembly.Guard.RebindRootInventory(ctx, prior, revision, installation, raw, func(*hostclaim.Claim) error { return nil }, func() error { return errRootInterrupted }) if !errors.Is(err, errRootInterrupted) || w.pending() != before+1 { w.t.Fatalf("rebind intent: %v, pending %d", err, w.pending()) } return q } // migrationRequired gives root a completed Plan 14 migration for actor, // seen while w.migrating is set, so OpenProductionClaimAssembly runs its // startup inventory recovery and its migration census // (verifyRootMigrationStartup) with participation required. The // participation ledger and the migration record are root-owned directories // on a box; here they are test directories. The census's last step, the // installed floor binaries' check, needs root-owned binaries this host does // not have and is the one step stood in for. func (w *assemblyWorld) migrationRequired(actor hostbox.Actor) { w.t.Helper() census := rootparticipation.Census{Schema: 1, Box: actor.Box, Enrollment: actor.Enrollment, Migration: 1, Revision: 1, LaunchRevision: 1, Complete: true} participation := &rollbackDisk{dir: w.t.TempDir()} raw, _ := json.Marshal(census) if err := participation.Write("census.json", raw); err != nil { w.t.Fatal(err) } now := time.Now().UTC() record := rootMigrationRecord{Schema: 1, Identity: rootMigrationIdentity{Deployment: actor.ControlBucket, Box: actor.Box, Enrollment: actor.Enrollment, Generation: 1}, Phase: "complete", Snapshot: rootMigrationSnapshot{Complete: true, DirectoryRevision: 1, Exclusion: "exclusion", Items: []rootMigrationItem{}, Report: hostops.CensusReport{Box: actor.Box, Enrollment: actor.Enrollment, BootID: "boot", Reservation: "migration", DirectoryRevision: 1, CapturedAt: now, Complete: true, Installations: []hostops.CensusInstallation{}, Seeds: &hostops.CensusSeeds{Journaled: []string{}, Gated: []string{}}}}} state := &rollbackDisk{dir: w.t.TempDir()} raw, _ = json.Marshal(record) if err := state.Write("migration.json", raw); err != nil { w.t.Fatal(err) } original, originalState, originalFloors := openRootMigrationParticipation, openRootMigrationState, rootMigrationStartupFloors w.t.Cleanup(func() { openRootMigrationParticipation, openRootMigrationState, rootMigrationStartupFloors = original, originalState, originalFloors }) openRootMigrationParticipation = func() (rootparticipation.Ledger, bool, error) { if !w.migrating { return rootparticipation.Ledger{}, false, nil } return rootparticipation.Ledger{State: participation}, true, nil } openRootMigrationState = func() (rootstate.Store, error) { return state, nil } rootMigrationStartupFloors = func(string, rootFloorIdentity) error { return nil } } // plantInventoryIntent leaves q pending at root for installation, as a root // that died after persisting the intent and before its commit leaves it: // the guard's real CommitRootInventoryArtifacts persists the intent, and the // commit fails before any inventory is written. func (w *assemblyWorld) plantInventoryIntent(q hostbox.JournalRequest, installation hostguard.RootInstallation) { w.t.Helper() raw, err := json.Marshal(q) if err != nil { w.t.Fatal(err) } before := w.pending() err = w.assembly.Guard.CommitRootInventoryArtifacts(w.t.Context(), installation, raw, func(*hostclaim.Claim) error { return nil }, nil, func(hostguard.RootInstallation) error { return errRootInterrupted }, nil) if !errors.Is(err, errRootInterrupted) || w.pending() != before+1 { w.t.Fatalf("planted intent: %v, pending %d", err, w.pending()) } } // inventoryRequest is an inventory write for rec's current installation as // the agent would send it: inventory in, at revision, against the claim and // record as they are now. func (w *assemblyWorld) inventoryRequest(rec provision.Record, in hostbox.StopInventory, revision uint64, id string) hostbox.JournalRequest { w.t.Helper() o := w.claim(rec.Host) h := o.Claim.Holder _, recordETag, err := w.store.Get(w.t.Context(), h.Key) if err != nil { w.t.Fatal(err) } return hostbox.JournalRequest{V: 3, ID: id, Op: "inventory-write", Revision: revision, RecordKey: h.Key, RecordETag: recordETag, Inventory: in, Target: hostguard.FenceTarget{Host: rec.Host, Incarnation: h.Incarnation, Stamp: h.Stamp, Session: h.Session, Box: "box-1", Enrollment: w.roster.row().Enrollment, ClaimETag: o.ETag, Op: o.Claim.Op.ID, Phase: o.Claim.Op.Phase}} } // plantedFirstInstall leaves rec's first inventory write pending at root, // stamped stamp, with no inventory written. func (w *assemblyWorld) plantedFirstInstall(rec provision.Record, stamp string) { w.t.Helper() h := w.claim(rec.Host).Claim.Holder enrollment := w.roster.row().Enrollment q := w.inventoryRequest(rec, hostbox.StopInventory{Complete: true, Host: rec.Host, Incarnation: h.Incarnation, InstallationStamp: h.Stamp, Box: "box-1", Enrollment: enrollment, DirectoryRevision: stamp}, 0, "planted-"+rec.Host) w.plantInventoryIntent(q, hostguard.RootInstallation{Empty: true, Host: rec.Host, Incarnation: h.Incarnation, Stamp: h.Stamp, Account: h.Account, App: h.App, Record: h.Key, Box: "box-1", Enrollment: enrollment, Migration: 1, InventoryRevision: 1}) } // targetStampedWrite is agent's inventory write of rec's current inventory, // unchanged, through the agent's own journal client, with its target's // DirectoryRevision set to stamp: an agent-chosen request field that root // must not take on trust. The request is prepared (nonce, issue time, ID) // by the client exactly as any agent write is. func (w *assemblyWorld) targetStampedWrite(agent *bindingAgent, rec provision.Record, stamp string) error { w.t.Helper() ctx := w.t.Context() h := w.claim(rec.Host).Claim.Holder _, etag, err := w.store.Get(ctx, h.Key) if err != nil { w.t.Fatal(err) } captured := hostguard.Captured{Incarnation: h.Incarnation, Stamp: h.Stamp} in, revision, err := agent.remote.Snapshot(ctx, rec.Host, captured) if err != nil { w.t.Fatal(err) } client := hostbox.ReconcileAdapter{Reconciler: hostbox.Reconciler{Guard: agent.guard, Claims: w.store}, Box: agent.pinned.Box, Enrollment: agent.pinned.Enrollment} expected, err := client.CaptureInventory(ctx, hostbox.RecordSelection{Host: rec.Host, Key: h.Key, ETag: etag, Incarnation: h.Incarnation, Family: h.Family}, captured, revision) if err != nil { w.t.Fatal(err) } expected.Target.DirectoryRevision = stamp _, err = agent.remote.WriteExpected(ctx, in, expected) return err } // N3 (roster binding task 5b, fix round 1 finding 1): interrupted intents // are left pending at B0 (an inventory write whose commit landed before the // record write was lost, a first install likewise, a first install whose // journal never committed, and a recovery rebind); the box is quarantined // and root restarts. Each intent is settled and logged, root comes up (never // errRootMigrationBusy), and its stop service answers a stop of the B0 // installation. After readmission and another restart root is still up and // the B0 installation is refused at wake. The control: an interrupted intent // whose binding did not change replays and completes. // // Two restarts settle pending intents in production: the startup inventory // recovery inside OpenProductionClaimAssembly, followed there by the // migration census (verifyRootMigrationStartup), either of which aborts the // assembly; and the service's RecoverPending. "startup" restarts the real // assembly with migration participation required through the whole // construction, so the census inspects every settled installation; // "service" restarts without participation and settles through the root // handler's RecoverPending. func TestRootStartupSettlesObsoleteIntents(t *testing.T) { for _, path := range []string{"startup", "service"} { t.Run(path, func(t *testing.T) { ctx := t.Context() w := newAssemblyWorld(t) b0 := w.fresh() if path == "startup" { w.migrationRequired(b0) } w.mustStart() agent := w.agent(b0) recA, routeA := w.newHost("a") recD, routeD := w.newHost("d") recE, routeE := w.newHost("e") for _, h := range []struct { rec provision.Record route registry.Route }{{recA, routeA}, {recD, routeD}, {recE, routeE}} { if err := agent.install(h.rec, h.route); err != nil { t.Fatal(err) } } recover := func() error { if path == "startup" { w.migrating = true return w.restart() } if err := w.restart(); err != nil { return err } return w.root.RecoverPending(ctx) } // Control: same binding, interrupted, replays and completes. (The // startup inventory recovery replays through root's creation // history, a root-owned directory; the control runs through the // service, before participation is required.) w.roster.holdChurn() _, before := w.inventory(recE.Host) w.interruptedRegistration(agent, recE, routeE, w.fresh()) if err := w.restart(); err != nil { t.Fatal("restart with a same-binding intent:", err) } if err := w.root.RecoverPending(ctx); err != nil { t.Fatal("recovery of a same-binding intent:", err) } if _, after := w.inventory(recE.Host); w.pending() != 0 || after != before+1 { t.Fatalf("same-binding intent: pending %d, inventory revision %d -> %d", w.pending(), before, after) } // Obsolete intents at B0, then quarantine and restart. agent = w.agent(b0) _, beforeA := w.inventory(recA.Host) w.interruptedRegistration(agent, recA, routeA, w.fresh()) if _, after := w.inventory(recA.Host); after != beforeA+1 { t.Fatalf("interrupted registration did not land in the inventory: %d -> %d", beforeA, after) } recF, routeF := w.newHost("f") *w.interrupt = 1 err := agent.install(recF, routeF) *w.interrupt = -1 if err == nil || w.pending() != 2 { t.Fatalf("first install was not interrupted after its intent: %v, pending %d", err, w.pending()) } recG, _ := w.newHost("g") w.plantedFirstInstall(recG, b0.Binding) w.interruptedRebind(recD, b0.Binding) w.roster.quarantine("loss_1") logs := captureLog(t) if err := recover(); err != nil { t.Fatal("quarantined restart with obsolete intents:", err) } if w.pending() != 0 { t.Fatalf("obsolete intents left pending: %d", w.pending()) } for _, kind := range []string{"inventory", "rebind"} { if !strings.Contains(logs(), "obsolete pending "+kind+" intent: stamped "+b0.Binding) { t.Fatalf("no settlement log for the %s intent: %s", kind, logs()) } } if strings.Contains(logs(), "not settled") { t.Fatalf("a settlement failed: %s", logs()) } // Guard and inventory agree for every settled installation, and // each still refuses execution. views, err := w.assembly.Guard.Root.MigrationInstallations(ctx) if err != nil { t.Fatal("migration census after settlement:", err) } settled := map[string]hostguard.FenceView{} for _, v := range views { if v.Settled { settled[v.Host] = v } } for _, rec := range []provision.Record{recA, recD, recF, recG} { v, ok := settled[rec.Host] if !ok { t.Fatalf("%s not settled: %+v", rec.Host, views) } _, revision, err := w.assembly.Journals.Inventory.Snapshot(ctx, rec.Host, v.Captured) if err != nil || revision != v.Installation.InventoryRevision { t.Fatalf("%s: guard revision %d, inventory %d %v", rec.Host, v.Installation.InventoryRevision, revision, err) } if _, err := w.assembly.Guard.Root.View(ctx, rec.Host, v.Captured); !errors.Is(err, hostguard.ErrFenced) { t.Fatalf("%s: settled installation's execution view: %v", rec.Host, err) } } if v := settled[recA.Host]; v.Installation.InventoryRevision != beforeA+1 { t.Fatalf("landed registration not recorded: %d", v.Installation.InventoryRevision) } if v := settled[recF.Host]; v.Installation.InventoryRevision != 1 { t.Fatalf("landed first install not recorded: %d", v.Installation.InventoryRevision) } if v := settled[recG.Host]; v.Installation.InventoryRevision != 0 { t.Fatalf("unlanded first install: %d", v.Installation.InventoryRevision) } if _, err := w.root.InventoryActor(ctx); err == nil { t.Fatal("quarantined root admitted new work") } if _, err := w.noProcessStop(recA.Host, b0.Binding, w.roster.read().Roster.Revision); err != nil { t.Fatal("stop service after settlement:", err) } // Readmission and another restart: up, the B0 installation refused. w.roster.readmit() if err := recover(); err != nil { t.Fatal("readmitted restart:", err) } b1 := w.fresh() if err := w.agent(b1).registerState(recE.Host, routeE.DBPath, b1); !errors.Is(err, hostbox.ErrTypedInventory) { t.Fatalf("the B0 installation at wake after readmission: %v", err) } }) } } // Fix round 1, finding 2: root validates an agent-sent target stamp against // the inventory's validated binding before it persists any intent, and // startup decides obsolescence on validated binding evidence only. // Contradictory: a request whose inventory names B0 (valid) and whose // target names another binding is refused before any intent, and the // installation keeps working. Control: the same request with the target // naming B0, interrupted after its intent, replays and completes at the // next start. Legacy: an intent persisted with a contradictory target (as // the unchecked root accepted one) is not settled as obsolete at startup. func TestRootInventoryRefusesContradictoryTarget(t *testing.T) { ctx := t.Context() w := newAssemblyWorld(t) w.mustStart() b0 := w.fresh() recA, routeA := w.newHost("a") recB, routeB := w.newHost("b") for _, h := range []struct { rec provision.Record route registry.Route }{{recA, routeA}, {recB, routeB}} { if err := w.agent(b0).install(h.rec, h.route); err != nil { t.Fatal(err) } } other := strings.Repeat("ab", 32) if other == b0.Binding { t.Fatal("fixture binding collision") } restart := func(what string) string { t.Helper() logs := captureLog(t) if err := w.restart(); err != nil { t.Fatal(what, err) } if err := w.root.RecoverPending(ctx); err != nil { t.Fatal(what, err) } return logs() } // Contradictory target: refused before the intent. _, before := w.inventory(recA.Host) *w.interrupt = 1 err := w.targetStampedWrite(w.agent(b0), recA, other) consumed := *w.interrupt == 0 *w.interrupt = -1 if err == nil || w.pending() != 0 || consumed { t.Fatalf("contradictory target: %v, pending %d, root wrote %v", err, w.pending(), consumed) } if _, after := w.inventory(recA.Host); after != before { t.Fatalf("contradictory target changed the inventory: %d -> %d", before, after) } if logs := restart("restart after a contradictory target:"); strings.Contains(logs, "obsolete pending") { t.Fatalf("an unchanged binding was settled as obsolete: %s", logs) } if err := w.agent(b0).registerState(recA.Host, routeA.DBPath, b0); err != nil { t.Fatal("the installation after a refused contradictory target:", err) } // Control: the target names the validated binding; interrupted, then // replayed at the next start. _, before = w.inventory(recB.Host) *w.interrupt = 1 err = w.targetStampedWrite(w.agent(b0), recB, b0.Binding) *w.interrupt = -1 if err == nil || w.pending() != 1 { t.Fatalf("same-binding target was not interrupted after its intent: %v, pending %d", err, w.pending()) } if logs := restart("restart with a same-binding target:"); strings.Contains(logs, "obsolete pending") { t.Fatalf("a same-binding intent was settled: %s", logs) } if _, after := w.inventory(recB.Host); w.pending() != 0 || after != before+1 { t.Fatalf("same-binding target: pending %d, inventory revision %d -> %d", w.pending(), before, after) } // Legacy: an intent persisted with a contradictory target. Startup does // not take its target as obsolescence evidence; the replay is refused // and the intent stays pending for the ordinary recovery. in, revision := w.inventory(recA.Host) q := w.inventoryRequest(recA, in, revision, "legacy-"+recA.Host) q.Target.DirectoryRevision = other h := w.claim(recA.Host).Claim.Holder installed, err := w.assembly.Guard.Root.Installation(ctx, recA.Host, hostguard.Captured{Incarnation: h.Incarnation, Stamp: h.Stamp}) if err != nil { t.Fatal(err) } installed.InventoryRevision = revision + 1 w.plantInventoryIntent(q, installed) if logs := restart("restart with a legacy contradictory intent:"); strings.Contains(logs, "obsolete pending") { t.Fatalf("a contradictory target was taken as obsolescence evidence: %s", logs) } if w.pending() != 1 { t.Fatalf("legacy contradictory intent: pending %d", w.pending()) } } // Fix round 2: an intent whose claim advanced before recovery is abandoned // (the guard's exact-intent and fresh-claim checks), and the abandoned // installation ends settled like an obsolete one, so it never blocks // cleanup startup. The binding never changes here. A landed inventory write // whose claim then moved to an operation session, and a rebind intent whose // claim then advanced, are abandoned by the service's RecoverPending. Root // then restarts twice with migration participation required through the // whole OpenProductionClaimAssembly: its startup census inspects both // abandoned installations, and the terminal state persists across restarts. // (Abandonment runs in the service, as it can in production: the startup // recovery's replays need root's creation history and the guard's lock // directory, root-owned paths this test host does not have.) func TestRootStartupInspectsSupersededIntents(t *testing.T) { ctx := t.Context() w := newAssemblyWorld(t) b0 := w.fresh() w.migrationRequired(b0) w.mustStart() agent := w.agent(b0) recA, routeA := w.newHost("a") recD, routeD := w.newHost("d") if err := agent.install(recA, routeA); err != nil { t.Fatal(err) } if err := agent.install(recD, routeD); err != nil { t.Fatal(err) } // A: a landed inventory write, then the claim moves on. _, beforeA := w.inventory(recA.Host) w.interruptedRegistration(agent, recA, routeA, b0) moved := w.operationSession(recA.Host) // D: a rebind intent, then the claim moves on. w.interruptedRebind(recD, b0.Binding) if _, err := hostclaim.Update(ctx, w.store, w.claim(recD.Host), func(c *hostclaim.Claim) error { c.Holder.Generation = "advanced" return nil }); err != nil { t.Fatal(err) } if err := w.restart(); err != nil { t.Fatal(err) } if err := w.root.RecoverPending(ctx); err != nil { t.Fatal("service recovery of superseded intents:", err) } if w.pending() != 0 { t.Fatalf("superseded intents left pending: %d", w.pending()) } for _, round := range []string{"first restart", "second restart"} { w.migrating = true if err := w.restart(); err != nil { t.Fatalf("%s with migration participation required: %v", round, err) } if w.fresh().Binding != b0.Binding { t.Fatal("the binding changed") } } views, err := w.assembly.Guard.Root.MigrationInstallations(ctx) if err != nil { t.Fatal("migration census:", err) } settled := map[string]hostguard.FenceView{} for _, v := range views { if v.Settled { settled[v.Host] = v } } for _, rec := range []provision.Record{recA, recD} { v, ok := settled[rec.Host] if !ok { t.Fatalf("%s not settled: %+v", rec.Host, views) } _, revision, err := w.assembly.Journals.Inventory.Snapshot(ctx, rec.Host, v.Captured) if err != nil || revision != v.Installation.InventoryRevision { t.Fatalf("%s: guard revision %d, inventory %d %v", rec.Host, v.Installation.InventoryRevision, revision, err) } if _, err := w.assembly.Guard.Root.View(ctx, rec.Host, v.Captured); !errors.Is(err, hostguard.ErrFenced) { t.Fatalf("%s: abandoned installation's execution view: %v", rec.Host, err) } } if v := settled[recA.Host]; v.Installation.InventoryRevision != beforeA+1 { t.Fatalf("landed write not recorded: %d", v.Installation.InventoryRevision) } // Startup proceeded over these views; journal-migrate's collector, the // census's other consumer, still refuses them (census condition 1). if err := refuseSettledInstallations(views); !errors.Is(err, hostguard.ErrFenced) { t.Fatal("the collector would admit settled installations:", err) } // The stop service is up and stops A's moved-on operation session. if _, err := w.root.Stop.Coordinator.FenceAndStop(ctx, w.priorStop(moved, b0.Binding)); err != nil { t.Fatal("stop service after abandonment:", err) } }