//go:build linux package host import ( "bytes" "context" "crypto/ed25519" "errors" "fmt" "os" "path/filepath" "testing" "time" "github.com/carlosframework/platform/internal/genselect" "github.com/carlosframework/platform/internal/hostactors" "github.com/carlosframework/platform/internal/hostbox" "github.com/carlosframework/platform/internal/hostclaim" "github.com/carlosframework/platform/internal/hostguard" "github.com/carlosframework/platform/internal/hostops" "github.com/carlosframework/platform/internal/hostwrite" "github.com/carlosframework/platform/internal/park" "github.com/carlosframework/platform/internal/provision" "github.com/carlosframework/platform/internal/registry" "github.com/carlosframework/platform/internal/release" "github.com/carlosframework/platform/internal/rootstate" ) // bindingAssemblyOwners opens the claim owners the way the production // provider does in instance-control mode: the identity is productionClaimRoster // over the signed roster, and the cleanup enrollment is the startup read // verifyClaimCleanupEnrollment makes, once per open. Owners that the paths // under test never call are the fix4 unused dependencies. type bindingAssemblyOwners struct { base ClaimOwnerBindings dir hostactors.Directory } func (o bindingAssemblyOwners) OpenClaimOwners(ctx context.Context, _ release.Store, r ClaimRuntimeResources) (ClaimOwnerBindings, error) { out := o.base if r.Mode != "instance-control" { return out, hostwrite.ErrUnavailable } var err error out.cleanupEnrollment, err = verifyClaimCleanupEnrollment(ctx, out.Identity, o.dir) return out, err } // assemblyWorld is one box under the real instance-control construction: // OpenProductionClaimAssembly over a signed roster, with root's handler // built by the production claimRootHTTP and its stop coordinator by // rootStopCoordinator. Root's state stores are directories (the fix4 // seams), the guard's lock directory is a temporary one, and // NewProductionClaimStopHandler's process bindings (the carlos user, the // edge unit's peer, the kernel observation) are fixtures: that handler // needs systemd and a carlos account this test host does not have. The // agent half installs through root's real commitRootInventory. type assemblyWorld struct { t *testing.T roster *bindingRosterFixture key ed25519.PrivateKey store release.Store reg *registry.Registry r ClaimRuntimeResources owners bindingAssemblyOwners locks string kernel *receiptKernel ids int assembly *ClaimAssembly root ClaimRootHTTP authority string journal string // retire is root's retirement and cold-handover journal namespace, // /var/lib/carlos-claims/retire on a box (claimRetireJournalDir). retire string now time.Time // interrupt, while set, fails root's next record write after the // numbered one (a crash after the intent is durable). interrupt *int // migrating, while set, makes root see a completed migration // (migrationRequired). migrating bool } // interruptibleDisk is a root authority directory whose writes can fail // from the Nth on: root's process dying with its intent on disk. type interruptibleDisk struct { *rollbackDisk after *int } func (d interruptibleDisk) Write(k string, b []byte) error { if *d.after == 0 { return errors.New("root interrupted") } if *d.after > 0 { *d.after-- } return d.rollbackDisk.Write(k, b) } func newAssemblyWorld(t *testing.T) *assemblyWorld { t.Helper() fixtureWithoutRootMigrationCensus(t) w := &assemblyWorld{t: t, roster: newBindingRosterFixture(t, "box-1", "ie"), key: ed25519.NewKeyFromSeed(bytes.Repeat([]byte{9}, ed25519.SeedSize)), kernel: &receiptKernel{}, now: time.Now().UTC().Truncate(time.Second)} w.store = w.roster.store w.roster.bindCensusKey(w.key) w.reg = restartTestRegistry(t) w.r = ClaimRuntimeResources{Mode: "instance-control", Registry: w.reg, DataDir: t.TempDir(), OptDir: t.TempDir(), UnitsDir: t.TempDir()} for _, dir := range []string{".hostclaims", "hostguard", "instances"} { if err := os.Mkdir(filepath.Join(w.r.DataDir, dir), 0700); err != nil { t.Fatal(err) } } w.locks, w.authority, w.journal, w.retire = t.TempDir(), t.TempDir(), filepath.Join(t.TempDir(), "journals.db"), t.TempDir() // Root's journal namespace is trusted only when no other principal can // write it (hostbox checkTrustedJournalDir). if err := os.Chmod(w.retire, 0700); err != nil { t.Fatal(err) } never := -1 w.interrupt = &never originalAuthority, originalJournal := openClaimRootAuthority, openClaimRootJournal t.Cleanup(func() { openClaimRootAuthority, openClaimRootJournal = originalAuthority, originalJournal }) openClaimRootAuthority = func() (rootstate.Store, error) { return interruptibleDisk{rollbackDisk: &rollbackDisk{dir: w.authority}, after: w.interrupt}, nil } openClaimRootJournal = func() (*hostbox.LaunchJournal, error) { return hostbox.OpenLaunchJournal(w.journal) } unused := fix4UnusedDependencies{} w.owners = bindingAssemblyOwners{dir: w.roster.dir, base: ClaimOwnerBindings{ Identity: w.roster.identity(), RetirementAuthority: &hostops.ProductionAuthority{Store: w.store, Directory: w.roster.dir, Now: time.Now}, EvidenceKey: w.key, Local: unused, Controllers: unused, Data: unused, Inputs: unused, Evidence: unused, InputPolicy: func(context.Context, *hostclaim.Claim, hostbox.Selection, hostwrite.Identity) error { return nil }, MovesPolicy: func(context.Context, *hostclaim.Claim, hostbox.Selection, hostwrite.Identity) error { return nil }, LeaseSet: func(context.Context, string, string) ([]string, error) { return nil, nil }, View: func(context.Context, hostclaim.Observed) (park.View, error) { return park.View{}, nil }, Authorize: func(context.Context, hostclaim.Observed, hostclaim.Obligation) error { return nil }, AdmissionEvidence: func(context.Context, hostclaim.Observed, hostclaim.Admission, *genselect.Plan) error { return nil }, PublicationEvidence: func(context.Context, hostclaim.Observed) error { return nil }, Close: func(context.Context) error { return nil }, }} return w } // start is root's instance-control start: OpenProductionClaimAssembly, then // NewProductionRootClaimStop's construction of the stop coordinator and the // root handler. It returns the construction error unchanged. func (w *assemblyWorld) start() error { w.t.Helper() ctx := w.t.Context() a, err := OpenProductionClaimAssembly(ctx, w.store, w.r, w.owners) if err != nil { return err } w.t.Cleanup(func() { a.Close(context.Background()) }) a.Guard.LockDir = w.locks actor, err := a.actorForRoot(ctx) if err != nil { return err } c := a.rootStopCoordinator(w.store) c.CarlosUID, c.Now, c.Kernel = 1001, time.Now, w.kernel c.NewID = func() string { w.ids++; return fmt.Sprintf("root-%d", w.ids) } c.Peers = stopPeerFunc(func(context.Context) (StopPeer, error) { return StopPeer{UID: 1001, PID: 42, ProcessStart: "start-42", BootID: "boot-1", ControllerID: "controller-1"}, nil }) c.Drainer = stopDrainFunc(func(_ context.Context, _ StopRequest, fence string) (DrainAcknowledgement, error) { return DrainClaimAcknowledgement(fence, "controller-1", "start-42", nil), nil }) w.assembly, w.root = a, a.claimRootHTTP(actor, ClaimStopHTTP{Coordinator: c}) w.root.Reconcile.Retirement.JournalDir = w.retire return nil } func (w *assemblyWorld) mustStart() { w.t.Helper() if err := w.start(); err != nil { w.t.Fatal("instance-control start:", err) } } // restart closes root's assembly and starts it again over the same state. func (w *assemblyWorld) restart() error { w.t.Helper() if err := w.assembly.Close(w.t.Context()); err != nil { w.t.Fatal(err) } return w.start() } func (w *assemblyWorld) fresh() hostbox.Actor { w.t.Helper() a, err := w.owners.base.Identity.Verify(w.t.Context()) if err != nil { w.t.Fatal(err) } return a } // newHost creates a claimed instance (a console create completed onto this // box) and its provision record. func (w *assemblyWorld) newHost(name string) (provision.Record, registry.Route) { w.t.Helper() ctx := w.t.Context() host := name + ".bab.oncarlos.com" in := hostclaim.NewHolder{Family: "instance", Key: "apps/bab/jam/instances/" + host + ".json", Account: "bab", App: "jam", Host: host, Region: "ie", RecordRegion: "ie", CreatedAt: w.now} body := []byte(`{"schema":1,"host":"` + host + `","account":"bab","app":"jam","region":"ie","created_at":"` + w.now.Format(time.RFC3339) + `"}`) absent, err := hostclaim.Read(ctx, w.store, host) if err != nil { w.t.Fatal(err) } o, err := hostclaim.Update(ctx, w.store, absent, hostclaim.BeginCreate(in, body, "create", "console", w.now)) if err != nil { w.t.Fatal(err) } if _, err = hostclaim.Update(ctx, w.store, o, hostclaim.CompleteCreate("create", "box-1")); err != nil { w.t.Fatal(err) } rec := provision.Record{Schema: 1, Host: host, Account: "bab", App: "jam", Channel: "stable", CreatedAt: w.now} plan14PutRecord(w.t, w.store, rec) return rec, registry.Route{Host: host, Account: "bab", App: "jam", Kind: "instance", DBPath: provision.NestedDBPath(w.r.DataDir, host), Socket: provision.NestedSocket(host)} } func (w *assemblyWorld) claim(host string) hostclaim.Observed { w.t.Helper() o, err := hostclaim.Read(w.t.Context(), w.store, host) if err != nil { w.t.Fatal(err) } return o } // bindingAgent is the agent half of one agent start: its pin is the // binding Verify returned when it started, set into the adapter and the // census exactly as ClaimAssembly.Build does, and its inventory writes go to // root's real commitRootInventory. type bindingAgent struct { w *assemblyWorld pinned hostbox.Actor adapter hostbox.ReconcileAdapter remote *hostbox.StopInventoryDB source hostbox.WorkSource guard hostguard.Guard census *hostbox.EmptyInstallationCensus } func (w *assemblyWorld) agent(pinned hostbox.Actor) *bindingAgent { w.t.Helper() if !hostactors.ValidBinding(pinned.Binding) { w.t.Fatalf("agent pin %q", pinned.Binding) } root, journals := w.root, w.assembly.Journals _, remote := hostbox.RemoteJournals(func(ctx context.Context, q hostbox.JournalRequest) (hostbox.JournalResponse, error) { if q.Op == "inventory-write" { return root.commitRootInventory(ctx, q) } return journals.Execute(ctx, pinned.Box+"/"+pinned.Enrollment, q) }) guard := hostguard.Guard{DataDir: w.r.DataDir, LockDir: w.locks, Claims: hostguard.StoreReader{Store: w.store}, FenceView: w.assembly.Guard.Root.View} census := &hostbox.EmptyInstallationCensus{Inventory: journals.Inventory, Registry: w.reg, Launches: w.assembly.Launches, Box: pinned.Box, Enrollment: pinned.Enrollment, DirectoryRevision: pinned.Binding, Roots: w.assembly.ManagedRoots, SharedPaths: map[string][]string{"data": {"hostguard"}}} // The production policy, unit-shape refusal included (G10). policy, _ := witnessPolicy(w.t, w.reg) source := hostbox.WorkSource{Guard: guard, Box: pinned.Box, Region: "ie", Now: time.Now, Policy: policy, Local: func(context.Context, *hostclaim.Claim, hostbox.Selection) error { return nil }} adapter := hostbox.ReconcileAdapter{Registry: w.reg, Inventory: remote, Box: pinned.Box, Enrollment: pinned.Enrollment, DirectoryRevision: pinned.Binding, Census: census, Reconciler: hostbox.Reconciler{Claims: w.store, Guard: guard, Box: pinned.Box, Region: "ie", Now: time.Now, Writes: hostwrite.Config{Enabled: true, Selector: source}, Local: park.Local{Guard: guard, Claims: w.store, DataDir: guard.DataDir, Box: pinned.Box}, CheckRecord: func(context.Context, hostbox.RecordSelection) error { return nil }, CheckLocal: func(context.Context, hostclaim.Observed, hostbox.RecordSelection) error { return nil }}} return &bindingAgent{w: w, pinned: pinned, adapter: adapter, remote: remote, source: source, guard: guard, census: census} } // install is the agent's install of rec on this box. func (a *bindingAgent) install(rec provision.Record, route registry.Route) error { a.w.t.Helper() holder := a.w.claim(rec.Host).Claim.Holder ctx, cancel := context.WithTimeout(a.w.t.Context(), 5*time.Second) defer cancel() return a.adapter.Install(ctx, provision.ClaimRecord{Host: rec.Host, Key: holder.Key, Account: rec.Account, App: rec.App, Family: "instance", CreatedAt: rec.CreatedAt, Value: rec, Route: route}, func(owner registry.ClaimOwnership) error { route.ClaimOwnership = owner return a.w.reg.Upsert(route) }) } // registerState is the agent's second inventory write for host, state // registration at wake, compared at root with actor. func (a *bindingAgent) registerState(host, dbPath string, actor hostbox.Actor) error { a.w.t.Helper() ctx := a.w.t.Context() work, err := a.source.Select(ctx, host, "spawn") if err != nil { return err } roles := &ClaimRoleAssembly{Inventory: a.remote, ManagedRoots: a.w.assembly.ManagedRoots, Guard: a.guard, Claims: a.w.store} return roles.prepareStateRemote(ctx, a.w.claim(host), work, ClaimLaunchInputs{DBPath: dbPath}, actor) } func (w *assemblyWorld) inventory(host string) (hostbox.StopInventory, uint64) { w.t.Helper() h := w.claim(host).Claim.Holder in, revision, err := w.assembly.Journals.Inventory.Snapshot(w.t.Context(), host, hostguard.Captured{Incarnation: h.Incarnation, Stamp: h.Stamp}) if err != nil { w.t.Fatal(err) } return in, revision } // noProcessStop is the agent's session-zero stop of host's installation, // stamped with stamp and the roster counter it read. func (w *assemblyWorld) noProcessStop(host, stamp string, revision uint64) (StopResult, error) { w.t.Helper() o := w.claim(host) h := o.Claim.Holder enrollment := w.roster.row().Enrollment return w.root.Stop.Coordinator.FenceAndStop(w.t.Context(), StopRequest{Region: "ie", Host: host, Incarnation: h.Incarnation, InstallationStamp: h.Stamp, Box: "box-1", Enrollment: enrollment, BootID: "boot-1", ControllerID: "controller-1", ExpectedClaimETag: o.ETag, ExpectedDirectoryRevision: stamp, RosterRevision: revision}) } // L4 (roster binding R2): readmission through the real instance-control // assembly, without and then with a restart. Root's InventoryActor is the // fresh identity; the cleanup identity is the historical one. func TestClaimBindingReadmissionWithoutRestart(t *testing.T) { ctx := t.Context() w := newAssemblyWorld(t) w.mustStart() b0 := w.fresh() if startup := w.assembly.cleanupEnrollment.actor; startup.Binding != b0.Binding { t.Fatalf("cleanup identity %q, fresh %q", startup.Binding, b0.Binding) } old := w.agent(b0) recA, routeA := w.newHost("a") if err := old.install(recA, routeA); err != nil { t.Fatal("install at B0:", err) } if in, _ := w.inventory(recA.Host); in.DirectoryRevision != b0.Binding { t.Fatalf("installed stamp %q", in.DirectoryRevision) } // Quarantine: new work is refused at once, historical cleanup works. w.roster.quarantine("loss_1") if _, err := w.root.InventoryActor(ctx); err == nil { t.Fatal("InventoryActor admitted a quarantined box") } recB, routeB := w.newHost("b") if err := old.install(recB, routeB); err == nil { t.Fatal("root accepted an install while quarantined") } stopped, err := w.noProcessStop(recA.Host, b0.Binding, w.roster.read().Roster.Revision) if err != nil { t.Fatal("stop of the B0 installation while quarantined:", err) } if r := decodeStopReceipt(t, stopped.Bytes); r.Binding != b0.Binding { t.Fatalf("quarantined stop receipt binding %q", r.Binding) } signer := ClaimStopCoordinator{Now: time.Now} if err := w.assembly.cleanupEnrollment.configureSigner(ctx, &signer, w.key); err != nil { t.Fatal("configureSigner while quarantined:", err) } // Readmission (direct signed write; see readmit): the fresh identity // returns B1 without a restart. w.roster.readmit() b1, err := w.root.InventoryActor(ctx) if err != nil || !hostactors.ValidBinding(b1.Binding) || b1.Binding == b0.Binding { t.Fatalf("InventoryActor after readmission: %q (B0 %q) %v", b1.Binding, b0.Binding, err) } if err := old.install(recB, routeB); err == nil { t.Fatal("root accepted the old agent's B0 install after readmission") } if row, err := w.reg.Get(recB.Host); err != nil || row != nil { t.Fatalf("refused install left a route: %+v %v", row, err) } renewed := w.agent(w.fresh()) if renewed.pinned.Binding != b1.Binding { t.Fatal("restarted agent did not pin B1") } if err := renewed.install(recB, routeB); err != nil { t.Fatal("install at B1:", err) } if in, _ := w.inventory(recB.Host); in.DirectoryRevision != b1.Binding { t.Fatalf("B1 install stamp %q", in.DirectoryRevision) } if err := renewed.registerState(recA.Host, routeA.DBPath, b1); !errors.Is(err, hostbox.ErrTypedInventory) { t.Fatalf("state registration of the B0 installation after readmission: %v", err) } // Restart: the cleanup identity is B1 now; a fresh install passes and // the B0 installation is still refused. if err := w.restart(); err != nil { t.Fatal(err) } if startup := w.assembly.cleanupEnrollment.actor; startup.Binding != b1.Binding { t.Fatalf("restarted cleanup identity %q", startup.Binding) } agent := w.agent(w.fresh()) recC, routeC := w.newHost("c") if err := agent.install(recC, routeC); err != nil { t.Fatal("fresh install after restart:", err) } if err := agent.registerState(recA.Host, routeA.DBPath, w.fresh()); !errors.Is(err, hostbox.ErrTypedInventory) { t.Fatalf("B0 installation after restart: %v", err) } if err := agent.registerState(recB.Host, routeB.DBPath, w.fresh()); err != nil { t.Fatal("B1 installation's state registration after restart:", err) } } // L2 and L3 through the real instance-control assembly: root's handler is // built by OpenProductionClaimAssembly and claimRootHTTP, its InventoryActor // the fresh identity; installs reach its commitRootInventory. Six real // roster writes and a held create leave every comparison passing, without // and after a root restart. func TestClaimBindingAssemblySurvivesRosterWritesWithoutRestart(t *testing.T) { testClaimBindingAssemblySurvivesRosterWrites(t, false) } func TestClaimBindingAssemblySurvivesRosterWritesAfterRestart(t *testing.T) { testClaimBindingAssemblySurvivesRosterWrites(t, true) } func testClaimBindingAssemblySurvivesRosterWrites(t *testing.T, restart bool) { ctx := t.Context() w := newAssemblyWorld(t) w.mustStart() pinned := w.fresh() agent := w.agent(pinned) rec, route := w.newHost("a") if err := agent.install(rec, route); err != nil { t.Fatal(err) } before := w.roster.read().Roster.Revision w.roster.sixWrites() if restart { if err := w.restart(); err != nil { t.Fatal("restart after six roster writes:", err) } agent = w.agent(w.fresh()) } held := w.roster.reserveHold() actor, err := w.root.InventoryActor(ctx) if err != nil || actor.Revision != before+7 || !hostbox.SameBinding(actor, pinned) { t.Fatalf("InventoryActor with a hold held: revision %d (before %d), same binding %v, %v", actor.Revision, before, hostbox.SameBinding(actor, pinned), err) } if agent.pinned.Binding != pinned.Binding || w.assembly.cleanupEnrollment.actor.Binding != pinned.Binding { t.Fatal("a pin moved with unrelated roster writes") } h := w.claim(rec.Host).Claim.Holder if err := agent.census.CheckLocal(ctx, w.claim(rec.Host), hostbox.RecordSelection{Host: rec.Host, Incarnation: h.Incarnation}); err != nil { t.Fatal("agent census:", err) } if err := w.root.Reconcile.Census.CheckLocal(ctx, w.claim(rec.Host), hostbox.RecordSelection{Host: rec.Host, Incarnation: h.Incarnation}); err != nil { t.Fatal("root census:", err) } if err := agent.registerState(rec.Host, route.DBPath, actor); err != nil { t.Fatal("state registration through root after roster writes:", err) } if in, revision := w.inventory(rec.Host); in.DirectoryRevision != pinned.Binding || revision < 2 { t.Fatalf("after state registration: stamp %q revision %d", in.DirectoryRevision, revision) } if err := claimBoundIdentity(w.owners.base.Identity.Verify, pinned)(ctx); err != nil { t.Fatal("bound identity:", err) } w.roster.releaseHold(held) // A binding change refuses new work at root at once. w.roster.quarantine("loss_1") w.roster.readmit() if err := agent.registerState(rec.Host, route.DBPath, w.fresh()); !errors.Is(err, hostbox.ErrTypedInventory) { t.Fatalf("state registration after readmission: %v", err) } recB, routeB := w.newHost("b") if err := agent.install(recB, routeB); err == nil { t.Fatal("root accepted a B0 install after readmission") } }