package hostops import ( "context" "crypto/ed25519" "crypto/rand" "errors" "path/filepath" "strings" "testing" "time" "github.com/carlosframework/platform/internal/hostactors" "github.com/carlosframework/platform/internal/provision" "github.com/carlosframework/platform/internal/registry" "github.com/carlosframework/platform/internal/release" "github.com/carlosframework/platform/internal/seeds" ) // activation is a deployment at the moment claim activation publishes its // backfill: a signed roster of two boxes, the collecting manifest (the // record-writer freeze), and one signed census report per box. type activation struct { t *testing.T ctx context.Context f *fixture sg release.Signer d hostactors.Directory roster hostactors.Roster boxKeys map[string]ed25519.PrivateKey records []Record } func newActivation(t *testing.T) *activation { t.Helper() a := &activation{t: t, ctx: context.Background(), f: newFixture(t), boxKeys: map[string]ed25519.PrivateKey{}} pub, key, err := ed25519.GenerateKey(rand.Reader) if err != nil { t.Fatal(err) } a.sg = release.RootSigner(key) a.roster = hostactors.Roster{V: 1, DeploymentBucket: "bucket", Revision: 1, Regions: map[string]hostactors.Region{"ie": {}}, Boxes: map[string]hostactors.Box{}, Consoles: map[string]hostactors.Console{}, Holds: map[string]hostactors.Reservation{}, Activations: map[string]hostactors.Reservation{}} for i, ref := range []string{"box-a", "box-b"} { boxPub, boxKey, err := ed25519.GenerateKey(rand.Reader) if err != nil { t.Fatal(err) } a.boxKeys[ref] = boxKey a.roster.Boxes[ref] = hostactors.Box{CensusPublicKey: boxPub, Enrollment: strings.Repeat(string(rune('a'+i)), 32), Namespace: "platform", BoxID: ref, Label: ref, Region: "ie", Pool: "default", Cloud: hostactors.Cloud{Provider: "aws", Account: "123", Region: "ie", InstanceID: "i-" + ref}, EvidenceSHA256: strings.Repeat("c", 64), State: "active"} } raw, err := release.SignRootEnvelope(a.sg, a.roster) if err != nil { t.Fatal(err) } if _, err := a.f.store.Put(a.ctx, hostactors.Key, raw, ""); err != nil { t.Fatal(err) } a.d = hostactors.Directory{Store: a.f.store, PublicKey: pub, Bucket: "bucket"} return a } // record writes one instance record, with seed as its seed (nil for none). func (a *activation) record(host string, seed *provision.SeedSource) { a.t.Helper() r, err := PrepareInstance(provision.Record{Schema: 1, Account: "bab", App: "shop", Host: host, Slug: strings.SplitN(host, ".", 2)[0], Channel: "edge", Region: "ie", Acquisition: "acquisition", Seed: seed}, a.f.now, "ie") if err != nil { a.t.Fatal(err) } if _, err := a.f.store.Put(a.ctx, r.Holder.Key, r.Body, ""); err != nil { a.t.Fatal(err) } if r, err = ReadRecord(a.ctx, a.f.store, r.Holder.Key); err != nil { a.t.Fatalf("the fixture's record does not read: %v", err) } a.records = append(a.records, r) } // publish writes the collecting manifest over the records, each box's // signed report carrying its seed evidence (a missing entry is a report // with none), and runs activation's Publish under the manifest's etag. func (a *activation) publish(evidence map[string]*CensusSeeds) error { a.t.Helper() p := BackfillPlan{V: 1, ID: "backfill", Phase: "collecting", DirectoryRevision: 1, CreatedAt: a.f.now, Hosts: map[string]BackfillHost{}} for _, r := range a.records { p.Hosts[r.Holder.Host] = BackfillHost{Key: r.Holder.Key, ETag: r.ETag, Acquisition: "acquisition", Region: "ie", Box: "box-a"} } raw, err := release.SignRootEnvelope(a.sg, p) if err != nil { a.t.Fatal(err) } tag, err := a.f.store.Put(a.ctx, BackfillKey, raw, "") if err != nil { a.t.Fatal(err) } for ref, box := range a.roster.Boxes { report := CensusReport{Box: ref, Enrollment: box.Enrollment, BootID: "boot", Reservation: p.ID, DirectoryRevision: 1, CapturedAt: a.f.now, Complete: true, Installations: []CensusInstallation{}, Seeds: evidence[ref]} if err := PublishCensusReport(a.ctx, a.f.store, a.d, report, a.boxKeys[ref]); err != nil { a.t.Fatalf("publish %s's census report: %v", ref, err) } } b := &Backfill{Store: a.f.store, Directory: a.d} return b.Publish(a.ctx, a.sg, tag) } // phase is the backfill manifest's phase now. func (a *activation) phase() string { a.t.Helper() b := &Backfill{Store: a.f.store, Directory: a.d} p, _, err := b.readPlan(a.ctx) if err != nil { a.t.Fatal(err) } return p.Phase } func noSeeds() map[string]*CensusSeeds { return map[string]*CensusSeeds{"box-a": {Journaled: []string{}, Gated: []string{}}, "box-b": {Journaled: []string{}, Gated: []string{}}} } func testSeed(kind, state string) *provision.SeedSource { id, _ := seeds.NewID() s := &provision.SeedSource{ID: id, Kind: kind, State: state, SHA256: strings.Repeat("a", 64), Bytes: 10, By: "t", At: time.Date(2026, 10, 5, 11, 0, 0, 0, time.UTC)} if kind == seeds.KindImport { s.Replaces, s.VerifyHash = "127.0.0.1:8401", strings.Repeat("b", 64) } else { s.From = "src.shop.example.com" } return s } // refused asserts err is activation's seed refusal over base, naming what. func refused(t *testing.T, err, base error, what string) { t.Helper() var se *SeedActivationError if !errors.As(err, &se) || !errors.Is(err, base) || !strings.Contains(err.Error(), what) { t.Fatalf("Publish = %v; want a seed refusal over %v naming %q", err, base, what) } } // Merge review S2 (a): with every box reporting no seed state and no // record carrying an unfinished seed, activation publishes as before. A // finished seed does not hold it. func TestActivationPublishesWithNoUnfinishedSeed(t *testing.T) { a := newActivation(t) a.record("plain.shop.example.com", nil) a.record("copy.shop.example.com", testSeed(seeds.KindReplica, seeds.StatePlaced)) a.record("moved.shop.example.com", testSeed(seeds.KindImport, seeds.StateCommitted)) if err := a.publish(noSeeds()); err != nil { t.Fatalf("Publish: %v", err) } if got := a.phase(); got != "adopting" { t.Fatalf("phase = %q, want adopting", got) } } // Activation refuses over each unfinished seed a record can carry, under // the collecting freeze, and publishes nothing. That includes the two // states ruled in after the first fix: a --from seed at ready, which // nothing can place once claims are on, and an import at refused, which a // claim console cannot delete. Only a placed copy and a committed import // publish. func TestActivationRefusesAnUnfinishedSeedRecord(t *testing.T) { for _, c := range []struct { kind, state string refuse bool }{ {seeds.KindReplica, seeds.StateReserving, true}, {seeds.KindImport, seeds.StateAwaitingUpload, true}, {seeds.KindImport, seeds.StateUploaded, true}, {seeds.KindImport, seeds.StateReady, true}, {seeds.KindImport, seeds.StateSwapped, true}, {seeds.KindImport, seeds.StatePlaced, true}, {seeds.KindImport, seeds.StateAborted, true}, {seeds.KindReplica, seeds.StateAborted, true}, {seeds.KindImport, seeds.StateRolledBack, true}, {seeds.KindReplica, seeds.StateRolledBack, true}, {seeds.KindReplica, seeds.StateReady, true}, {seeds.KindImport, seeds.StateRefused, true}, {seeds.KindReplica, seeds.StateCommitted, true}, {"later", seeds.StatePlaced, true}, {"later", seeds.StateCommitted, true}, {seeds.KindReplica, seeds.StatePlaced, false}, {seeds.KindImport, seeds.StateCommitted, false}, } { t.Run(c.kind+" "+c.state, func(t *testing.T) { a := newActivation(t) a.record("plain.shop.example.com", nil) a.record("seeded.shop.example.com", testSeed(c.kind, c.state)) err := a.publish(noSeeds()) if !c.refuse { if err != nil || a.phase() != "adopting" { t.Fatalf("Publish = %v, phase %q; want published", err, a.phase()) } return } refused(t, err, ErrPending, "seeded.shop.example.com has an unfinished seed ("+c.kind+", "+c.state+")") if got := a.phase(); got != "collecting" { t.Fatalf("phase = %q after a refusal, want collecting", got) } }) } } // Activation refuses when a box's signed report carries no seed state: it // fails closed, whatever the other boxes and the records say. func TestActivationRefusesWithoutABoxesSeedEvidence(t *testing.T) { for name, evidence := range map[string]map[string]*CensusSeeds{ "no box says": {}, "one box says": {"box-a": {Journaled: []string{}, Gated: []string{}}}, "the other one": {"box-b": {Journaled: []string{}, Gated: []string{}}}, } { t.Run(name, func(t *testing.T) { a := newActivation(t) a.record("plain.shop.example.com", nil) err := a.publish(evidence) refused(t, err, ErrIdentity, "reported no seed state") if got := a.phase(); got != "collecting" { t.Fatalf("phase = %q after a refusal, want collecting", got) } }) } // A collector that cannot read its registry leaves the evidence out. if seeds, err := CollectCensusSeeds(failingSeedState{}); err == nil || seeds != nil { t.Fatalf("CollectCensusSeeds over a registry that cannot be read = %+v, %v", seeds, err) } } type failingSeedState struct{} func (failingSeedState) List() ([]registry.Route, error) { return nil, errors.New("registry busy") } func (failingSeedState) ImportJournalHosts() ([]string, error) { return nil, errors.New("registry busy") } // importOnBox is a box's registry holding one import, taken as far as // state: swapped (the gated row and a journal row), rolling-back (the gate // closed), or cleanup-pending (the original route back, the journal row // still there). func importOnBox(t *testing.T, host, state string) *registry.Registry { t.Helper() reg, err := registry.Open(filepath.Join(t.TempDir(), "registry.db")) if err != nil { t.Fatal(err) } t.Cleanup(func() { reg.Close() }) if err := reg.Upsert(registry.Route{Host: host, Kind: "service", Addr: "127.0.0.1:8401", Status: "ready", App: "shop", Account: "bab", CreatedAt: 1700000000}); err != nil { t.Fatal(err) } original, err := reg.Get(host) if err != nil || original == nil { t.Fatal(original, err) } id, _ := seeds.NewID() next := registry.Route{Host: host, Kind: "instance", Socket: "/run/carlos/instances/x/app.sock", DBPath: "/data/instances/x/" + host + ".db", Status: "ready", App: "shop", Account: "bab", Hibernate: true, Provisioned: true, ImportGate: strings.Repeat("b", 64), SeedSource: seeds.RowSeed{ID: id, Kind: seeds.KindImport, State: seeds.StateSwapped}.Encode(), CreatedAt: 1700000001} if err := reg.SwapForImport(registry.ImportJournal{Host: host, SeedID: id, Original: *original, RecordKey: provision.RecordKey("bab", "shop", host), At: 1, EnvBefore: registry.EnvBeforeAbsent}, next); err != nil { t.Fatal(err) } if state == registry.JournalSwapped { return reg } if err := reg.CloseImportGate(host, id, 7); err != nil { t.Fatal(err) } if state == registry.JournalRollingBack { return reg } if err := reg.RestoreImportRoute(host, id); err != nil { t.Fatal(err) } return reg } // The ruling's four states, with the box's evidence read from a real // registry by the collector's own function. swapped and placed are an // import the record still shows; rolling-back and cleanup-pending are // shown here with the record already gone, where only the box's signed // evidence can say an import is still in flight. func TestActivationRefusesAnImportInFlightOnABox(t *testing.T) { const host = "old.shop.example.com" for _, c := range []struct { name string record string // the record's seed state, "" for no record journal string wantJournaled bool wantGated bool }{ {"swapped", seeds.StateSwapped, registry.JournalSwapped, true, true}, {"placed", seeds.StatePlaced, registry.JournalSwapped, true, true}, {"rolling-back", "", registry.JournalRollingBack, true, true}, {"cleanup-pending", "", registry.JournalCleanupPending, true, false}, } { t.Run(c.name, func(t *testing.T) { a := newActivation(t) a.record("plain.shop.example.com", nil) if c.record != "" { a.record(host, testSeed(seeds.KindImport, c.record)) } evidence := noSeeds() got, err := CollectCensusSeeds(importOnBox(t, host, c.journal)) if err != nil { t.Fatal(err) } if (len(got.Journaled) == 1 && got.Journaled[0] == host) != c.wantJournaled || (len(got.Gated) == 1 && got.Gated[0] == host) != c.wantGated { t.Fatalf("the box's evidence = %+v; want journaled %v, gated %v", got, c.wantJournaled, c.wantGated) } evidence["box-b"] = got refused(t, a.publish(evidence), ErrPending, "box box-b still holds an import journal row for "+host) if got := a.phase(); got != "collecting" { t.Fatalf("phase = %q after a refusal, want collecting", got) } }) } // A gate with no journal row (a committed import whose box has not yet // cleared its gate cannot be told from one that never will) refuses too. a := newActivation(t) a.record("plain.shop.example.com", nil) evidence := noSeeds() evidence["box-a"] = &CensusSeeds{Journaled: []string{}, Gated: []string{host}} refused(t, a.publish(evidence), ErrPending, "box box-a still gates "+host+" for an import") } // A box with no import reports two empty lists, which is not "did not // say": the collector's answer for a clean registry lets activation go on. func TestCollectCensusSeedsOnACleanBox(t *testing.T) { reg, err := registry.Open(filepath.Join(t.TempDir(), "registry.db")) if err != nil { t.Fatal(err) } defer reg.Close() if err := reg.Upsert(registry.Route{Host: "plain.shop.example.com", Kind: "instance", Socket: "/run/x.sock", App: "shop", Account: "bab", Hibernate: true}); err != nil { t.Fatal(err) } got, err := CollectCensusSeeds(reg) if err != nil || got == nil || got.Journaled == nil || got.Gated == nil || len(got.Journaled)+len(got.Gated) != 0 { t.Fatalf("CollectCensusSeeds = %+v, %v; want two empty lists", got, err) } a := newActivation(t) a.record("plain.shop.example.com", nil) if err := a.publish(map[string]*CensusSeeds{"box-a": got, "box-b": got}); err != nil { t.Fatalf("Publish with the collector's clean evidence: %v", err) } }