package host import ( "context" "fmt" "testing" "time" "github.com/carlosframework/platform/internal/hostbox" "github.com/carlosframework/platform/internal/hostclaim" "github.com/carlosframework/platform/internal/hostguard" "github.com/carlosframework/platform/internal/provision" ) func plan14InstallRegistration(t *testing.T, kind, phase string) (*recoveryFixture, ClaimRootHTTP, hostbox.JournalRequest) { t.Helper() f := newRecoveryFixture(t) if _, err := hostclaim.Update(t.Context(), f.claims, f.read(), func(c *hostclaim.Claim) error { if kind != hostclaim.KindServing { c.Op = hostclaim.Op{ID: "install-op", Kind: kind, Phase: phase, Exec: "box-1", Participants: []string{"box-0", "box-1"}, Intent: map[string]string{"source": "box-0", "target": "box-1", "to": "ie", "region": "ie", "source_box": "box-0", "target_box": "box-1"}, Journal: map[string]string{"install_stamp": fmt.Sprint(c.Holder.Stamp)}} } return nil }); err != nil { t.Fatal(err) } o := f.read() h := o.Claim.Holder rec := provision.Record{Schema: 1, Host: o.Host, Account: h.Account, App: h.App, Channel: "stable", Region: "ie", CreatedAt: f.now} if kind == hostclaim.KindLanding { rec.Move = &provision.MoveState{ID: o.Claim.Op.ID, Incarnation: h.Incarnation, From: "uk", To: "ie", State: provision.MoveLanding, LandingPhase: phase} } plan14PutRecord(t, f.claims, rec) _, etag, err := f.claims.Get(t.Context(), h.Key) if err != nil { t.Fatal(err) } reg := restartTestRegistry(t) journals, err := hostbox.NewJournalService(f.journal) if err != nil { t.Fatal(err) } guard := f.guard guard.Root = hostguard.NewRootAuthority(&rollbackDisk{dir: t.TempDir()}) actor := hostbox.Actor{Box: "box-1", Enrollment: "en", Region: "ie", Revision: 1, Binding: "1"} roots := map[string]string{"data": guard.DataDir, "opt": t.TempDir(), "units": t.TempDir()} census := &hostbox.EmptyInstallationCensus{Inventory: journals.Inventory, Registry: reg, Launches: f.journal, Box: actor.Box, Enrollment: actor.Enrollment, DirectoryRevision: "1", Roots: roots, SharedPaths: map[string][]string{"data": {"hostguard"}}} root := ClaimRootHTTP{Journals: journals, Launches: f.journal, Roots: roots, InventoryActor: func(context.Context) (hostbox.Actor, error) { return actor, nil }, Reconcile: hostbox.ReconcileAdapter{Registry: reg, Inventory: journals.Inventory, Census: census}, Stop: ClaimStopHTTP{Coordinator: ClaimStopCoordinator{Guard: guard, Evidence: f.claims}}} q := hostbox.JournalRequest{V: 3, ID: "registration-1", Op: "inventory-write", RecordKey: h.Key, RecordETag: etag, Target: hostguard.FenceTarget{Host: o.Host, Incarnation: h.Incarnation, Stamp: h.Stamp, Session: h.Session, Box: actor.Box, Enrollment: actor.Enrollment, ClaimETag: o.ETag, Op: o.Claim.Op.ID, Phase: o.Claim.Op.Phase}, Inventory: hostbox.StopInventory{Complete: true, Host: o.Host, Incarnation: h.Incarnation, InstallationStamp: h.Stamp, Box: actor.Box, Enrollment: actor.Enrollment, DirectoryRevision: "1"}} _, client := hostbox.RemoteJournals(func(_ context.Context, prepared hostbox.JournalRequest) (hostbox.JournalResponse, error) { q = prepared return hostbox.JournalResponse{V: 1, ID: prepared.ID, Revision: 1}, nil }) if _, err := client.WriteExpected(t.Context(), q.Inventory, hostbox.InventoryExpectation{RecordKey: q.RecordKey, RecordETag: q.RecordETag, Target: q.Target}); err != nil { t.Fatal(err) } return f, root, q } func TestPlan14FirstRegistrationOperations(t *testing.T) { t.Parallel() for _, kind := range []string{hostclaim.KindServing, hostclaim.KindRelocating, hostclaim.KindLanding} { for _, attack := range []string{"none", "phase", "box", "session", "enrollment", "record", "op", "stamp", "agent row"} { t.Run(kind+"/"+attack, func(t *testing.T) { phase := hostclaim.PhaseInstalling if kind == hostclaim.KindLanding { phase = provision.LandChecking } f, root, q := plan14InstallRegistration(t, kind, phase) switch attack { case "phase": q.Target.Phase = "wrong" case "box": q.Target.Box = "box-0" case "session": q.Target.Session++ case "enrollment": q.Target.Enrollment = "other" case "record": q.RecordETag = "other" case "op": q.Target.Op = "other" case "stamp": q.Target.Stamp++ case "agent row": rt, _, err := rootClaimRecord(t.Context(), f.claims, f.read().Claim, root.Roots["data"]) if err != nil { t.Fatal(err) } if err = root.Reconcile.Registry.Upsert(rt); err != nil { t.Fatal(err) } } out, err := root.commitRootInventory(t.Context(), q) if attack != "none" { if err == nil { t.Fatal("unauthorized registration") } return } if err != nil { t.Fatal(err) } if out.Revision != 1 { t.Fatal(out) } if _, err = root.commitRootInventory(t.Context(), q); err != nil { t.Fatal("replay", err) } if kind == hostclaim.KindLanding { row, err := root.Reconcile.Registry.Get(q.Target.Host) if err != nil || row == nil || row.HoldOp != q.Target.Op || row.Status != "landing" || row.DBPath != provision.DefaultDBPath(root.Roots["data"], q.Target.Host) || !row.ClaimOwnership.Valid() { t.Fatalf("root held row: %+v %v", row, err) } } }) } } } func TestPlan14FirstRegistrationRefusesLaterPhases(t *testing.T) { t.Parallel() for kind, phases := range map[string][]string{hostclaim.KindLanding: {provision.LandConfigured, provision.LandRestored, provision.LandProbing}, hostclaim.KindRelocating: {hostclaim.PhaseRelocatingProbing, hostclaim.PhaseProbeFailed}, hostclaim.KindRestoring: {provision.LandConfigured}, hostclaim.KindReverting: {provision.LandConfigured}} { for _, phase := range phases { t.Run(kind+"/"+phase, func(t *testing.T) { _, root, q := plan14InstallRegistration(t, kind, phase) if _, err := root.commitRootInventory(t.Context(), q); err == nil { t.Fatal("skipped first registration") } }) } } } func plan14PreparedInventory(t *testing.T, root ClaimRootHTTP, q hostbox.JournalRequest) hostbox.JournalRequest { t.Helper() _, client := hostbox.RemoteJournals(func(_ context.Context, p hostbox.JournalRequest) (hostbox.JournalResponse, error) { q = p return hostbox.JournalResponse{V: 1, ID: p.ID, Revision: p.Revision + 1}, nil }) if _, err := client.WriteExpected(t.Context(), q.Inventory, hostbox.InventoryExpectation{Revision: q.Revision, RecordKey: q.RecordKey, RecordETag: q.RecordETag, Target: q.Target}); err != nil { t.Fatal(err) } return q } func TestPlan14LandingConfiguredAdoptsOnlyRootRow(t *testing.T) { t.Parallel() for _, attack := range []string{"none", "row id", "row missing", "op", "session", "record", "enrollment", "probe session"} { t.Run(attack, func(t *testing.T) { f, root, q := plan14InstallRegistration(t, hostclaim.KindLanding, provision.LandChecking) if _, err := root.commitRootInventory(t.Context(), q); err != nil { t.Fatal(err) } prior, err := root.Reconcile.Registry.Get(q.Target.Host) if err != nil { t.Fatal(err) } o, err := hostclaim.Update(t.Context(), f.claims, f.read(), func(c *hostclaim.Claim) error { c.Op.Phase = provision.LandConfigured if attack == "probe session" { c.Op.Phase = provision.LandProbing } if attack == "op" { c.Op.ID = "different" } if attack == "session" { c.Holder.Session++ } return nil }) if err != nil { t.Fatal(err) } rec, _, err := provision.FindRecordByHost(t.Context(), f.claims, o.Host) if err != nil { t.Fatal(err) } if attack == "probe session" { h := o.Claim.Holder o, err = hostclaim.Update(t.Context(), f.claims, o, hostclaim.AdmitWriter(hostclaim.Admission{ObservedOpID: o.Claim.Op.ID, ObservedPhase: o.Claim.Op.Phase, ObservedSession: h.Session, ObservedGeneration: h.Generation, Box: "box-1", Mode: "cold", NewGeneration: "landing-generation", KeepStamp: true, Coverage: "sealed", Region: "ie", PredecessorRegion: "uk"})) if err != nil { t.Fatal(err) } } rec.Move.LandingPhase = o.Claim.Op.Phase rec.Move.ID = o.Claim.Op.ID plan14PutRecord(t, f.claims, *rec) _, q.RecordETag, err = f.claims.Get(t.Context(), q.RecordKey) if err != nil { t.Fatal(err) } q.Target.ClaimETag, q.Target.Phase, q.Target.Op, q.Target.Session = o.ETag, o.Claim.Op.Phase, o.Claim.Op.ID, o.Claim.Holder.Session q.Inventory, q.Revision, err = root.Journals.Inventory.Snapshot(t.Context(), o.Host, hostguard.Captured{Incarnation: q.Target.Incarnation, Stamp: q.Target.Stamp}) if err != nil { t.Fatal(err) } switch attack { case "row id": prior.ClaimRowID = "forged" if err = root.Reconcile.Registry.Delete(o.Host); err == nil { err = root.Reconcile.Registry.Upsert(*prior) } case "row missing": err = root.Reconcile.Registry.Delete(o.Host) case "record": q.RecordETag = "stale" case "enrollment": q.Target.Enrollment = "stale" } if err != nil { t.Fatal(err) } q = plan14PreparedInventory(t, root, q) out, err := root.commitRootInventory(t.Context(), q) if attack != "none" && attack != "probe session" { if err == nil { t.Fatal("adopted changed installation") } return } if err != nil { t.Fatal(err) } if out.Revision != 2 { t.Fatal(out) } row, err := root.Reconcile.Registry.Get(o.Host) if err != nil || row.ClaimOwnership != prior.ClaimOwnership { t.Fatalf("identity changed: %+v %v", row, err) } if _, err := root.commitRootInventory(t.Context(), q); err != nil { t.Fatal("configured replay", err) } }) } } func TestPlan14LandingRegistrationOutsideHostLock(t *testing.T) { t.Parallel() for _, stale := range []bool{false, true} { t.Run(fmt.Sprint(stale), func(t *testing.T) { f, root, _ := plan14InstallRegistration(t, hostclaim.KindLanding, provision.LandChecking) if err := f.guard.Activate(t.Context(), f.read().Host, "box-1", nil); err != nil { t.Fatal(err) } previous, err := f.guard.Capture(f.read().Host) if err != nil { t.Fatal(err) } if _, err := hostclaim.Update(t.Context(), f.claims, f.read(), func(c *hostclaim.Claim) error { c.Holder.Stamp++; return nil }); err != nil { t.Fatal(err) } guard := f.guard guard.FenceView = root.Stop.Coordinator.Guard.Root.View _, remote := hostbox.RemoteJournals(func(ctx context.Context, q hostbox.JournalRequest) (hostbox.JournalResponse, error) { if q.Op != "inventory-write" { return root.Journals.Execute(ctx, "box-1/en", q) } out, err := root.commitRootInventory(ctx, q) if err == nil && stale { rec, _, e := provision.FindRecordByHost(ctx, f.claims, q.Target.Host) if e != nil { return out, e } rec.RecordSeq++ plan14PutRecord(t, f.claims, *rec) } return out, err }) a := hostbox.ReconcileAdapter{Inventory: remote, Box: "box-1", Enrollment: "en", DirectoryRevision: "1", Reconciler: hostbox.Reconciler{Claims: f.claims, Guard: guard}} ctx, cancel := context.WithTimeout(t.Context(), 2*time.Second) defer cancel() err = a.RegisterLanding(ctx, f.read()) if stale { if err == nil { t.Fatal("activated after record changed during RPC") } return } if err != nil { t.Fatal("registration blocked or refused", err) } if err := a.RegisterLanding(ctx, f.read()); err != nil { t.Fatal("checking retry", err) } captured, err := f.guard.Capture(f.read().Host) if err != nil || captured != previous { t.Fatalf("registration hid previous stay: %+v, want %+v: %v", captured, previous, err) } if err := guard.Activate(ctx, f.read().Host, "box-1", nil); err != nil { t.Fatal("ordered activation", err) } }) } } type plan14RegistrationCrash struct { *rollbackDisk writes int } func (s *plan14RegistrationCrash) Write(name string, body []byte) error { s.writes++ if s.writes == 2 { return fmt.Errorf("injected root commit crash") } return s.rollbackDisk.Write(name, body) } func TestPlan14LandingRegistrationRecoversRootCommit(t *testing.T) { _, root, q := plan14InstallRegistration(t, hostclaim.KindLanding, provision.LandChecking) store := &plan14RegistrationCrash{rollbackDisk: &rollbackDisk{dir: t.TempDir()}} root.Stop.Coordinator.Guard.Root = hostguard.NewRootAuthority(store) if _, err := root.commitRootInventory(t.Context(), q); err == nil { t.Fatal("missed injected crash") } before, err := root.Reconcile.Registry.Get(q.Target.Host) if err != nil || before == nil { t.Fatalf("row before recovery: %v %v", before, err) } if err := root.recoverRootInventory(t.Context()); err != nil { t.Fatal(err) } after, err := root.Reconcile.Registry.Get(q.Target.Host) if err != nil || after == nil || before.ClaimOwnership != after.ClaimOwnership { t.Fatalf("recovery reminted row: %v %v", after, err) } view, err := root.Stop.Coordinator.Guard.Root.View(t.Context(), q.Target.Host, hostguard.Captured{Incarnation: q.Target.Incarnation, Stamp: q.Target.Stamp}) if err != nil || view.Installation.InventoryRevision != 1 || view.Installation.Registration.RowID != before.ClaimRowID { t.Fatalf("root recovery: %+v %v", view, err) } } func TestPlan14FirstRegistrationUsesExistingInstallAuthority(t *testing.T) { t.Parallel() for _, mode := range []string{"never started", "writer", "missing writer", "foreign writer", "replacement", "existing install"} { t.Run(mode, func(t *testing.T) { kind, phase := hostclaim.KindServing, "" if mode == "replacement" || mode == "existing install" { kind, phase = hostclaim.KindRelocating, hostclaim.PhaseInstalled } f, root, q := plan14InstallRegistration(t, kind, phase) o, err := hostclaim.Update(t.Context(), f.claims, f.read(), func(c *hostclaim.Claim) error { switch mode { case "writer": c.Holder.WriterBox, c.Holder.Session = "box-1", 1 case "missing writer": c.Holder.WriterBox, c.Holder.Session = "", 1 case "foreign writer": c.Holder.WriterBox, c.Holder.Session = "box-0", 1 case "existing install": c.Op.Journal["local_install"] = "already-installed" } return nil }) if err != nil { t.Fatal(err) } q.Target.ClaimETag, q.Target.Session = o.ETag, o.Claim.Holder.Session q = plan14PreparedInventory(t, root, q) _, err = root.commitRootInventory(t.Context(), q) refuse := mode == "missing writer" || mode == "foreign writer" || mode == "existing install" if (err != nil) != refuse { t.Fatalf("refuse=%v: %v", refuse, err) } }) } }