package hostops import ( "bytes" "crypto/ed25519" "crypto/sha256" "encoding/base64" "encoding/hex" "encoding/json" "errors" "fmt" "os" "slices" "testing" "time" "github.com/carlosframework/platform/internal/hostclaim" "github.com/carlosframework/platform/internal/release" ) const ( doneOne = "one.shop.example.com" doneTwo = "two.shop.example.com" ) // doneOtherKey signs the plan a wrong signer wrote. Fixed, so a mutation // check can name it. var doneOtherKey = ed25519.NewKeyFromSeed(bytes.Repeat([]byte{9}, ed25519.SeedSize)) // keyless re-signs the roster with these boxes' census keys removed: the dev // roster's shape, where sophie has no key. func (a *activation) keyless(refs ...string) { a.t.Helper() for _, ref := range refs { box := a.roster.Boxes[ref] box.CensusPublicKey = nil a.roster.Boxes[ref] = box } _, tag, err := a.f.store.Get(a.ctx, "control/host-actors.json") if err != nil { a.t.Fatal(err) } raw, err := release.SignRootEnvelope(a.sg, a.roster) if err != nil { a.t.Fatal(err) } if _, err := a.f.store.Put(a.ctx, "control/host-actors.json", raw, tag); err != nil { a.t.Fatal(err) } } // collecting writes backfill-collect's plan over the records (no box, no // region), edited by edit when it is not nil, signed by signer. func (a *activation) collecting(signer release.Signer, edit func(*BackfillPlan)) string { a.t.Helper() p := BackfillPlan{V: 1, ID: "backfill", Phase: "collecting", DirectoryRevision: 1, CreatedAt: a.f.now, Hosts: map[string]BackfillHost{}} for _, r := range a.records { p.Hosts[r.Holder.Host] = BackfillHost{Key: r.Holder.Key, ETag: r.ETag, Acquisition: r.Holder.Acquisition} } if edit != nil { edit(&p) } raw, err := release.SignRootEnvelope(signer, p) if err != nil { a.t.Fatal(err) } _, before, err := a.f.store.Get(a.ctx, BackfillKey) if err != nil && !errors.Is(err, os.ErrNotExist) { a.t.Fatal(err) } tag, err := a.f.store.Put(a.ctx, BackfillKey, raw, before) if err != nil { a.t.Fatal(err) } return tag } // emptyReport is a box's complete census report for the reservation, as // the migration census writes it on an empty box. func (a *activation) emptyReport(ref string) CensusReport { return CensusReport{Box: ref, Enrollment: a.roster.Boxes[ref].Enrollment, BootID: "boot", Reservation: "backfill", DirectoryRevision: 1, CapturedAt: a.f.now, Complete: true, Installations: []CensusInstallation{}, Seeds: &CensusSeeds{Journaled: []string{}, Gated: []string{}}} } // rawReport signs report with key and writes it at the box's report key, // bypassing PublishCensusReport's own checks, as a broken box could. func (a *activation) rawReport(report CensusReport, key ed25519.PrivateKey) { a.t.Helper() canonical, err := json.Marshal(report) if err != nil { a.t.Fatal(err) } raw, err := json.Marshal(CensusEnvelope{Report: report, Signature: ed25519.Sign(key, canonical)}) if err != nil { a.t.Fatal(err) } k, err := reportKey("backfill", report.Box) if err != nil { a.t.Fatal(err) } if _, err := a.f.store.Put(a.ctx, k, raw, ""); err != nil { a.t.Fatal(err) } } func (a *activation) installation(host string) CensusInstallation { a.t.Helper() for _, r := range a.records { if r.Holder.Host == host { inc, err := r.Incarnation() if err != nil { a.t.Fatal(err) } return CensusInstallation{Host: host, Account: r.Holder.Account, App: r.Holder.App, Family: r.Holder.Family, Key: r.Holder.Key, Incarnation: inc, Route: true} } } a.t.Fatalf("no record for %s", host) return CensusInstallation{} } // The empty cohort: two records frozen, box-a (keyed) reports nothing, // box-b has no census key. Done writes done under If-Match with box-a's // digest only, returns box-b as unreported, and lifts the create freeze. // The console's adoption pass is a no-op on a done plan, adoption refuses // it, and a second done is refused. func TestBackfillDoneClosesAnEmptyCohort(t *testing.T) { a := newActivation(t) a.record(doneOne, nil) a.record(doneTwo, nil) a.keyless("box-b") tag := a.collecting(a.sg, nil) if err := CheckCreateFreeze(a.ctx, a.f.store, a.d.PublicKey); !errors.Is(err, ErrPending) { t.Fatalf("freeze while collecting = %v", err) } if err := PublishCensusReport(a.ctx, a.f.store, a.d, a.emptyReport("box-a"), a.boxKeys["box-a"]); err != nil { t.Fatal(err) } before := a.plan() b := &Backfill{Store: a.f.store, Directory: a.d} unreported, err := b.Done(a.ctx, a.sg, tag) if err != nil || !slices.Equal(unreported, []string{"box-b"}) { t.Fatalf("Done = %v, %v; want box-b unreported", unreported, err) } p := a.plan() key, _ := reportKey("backfill", "box-a") body, _, err := a.f.store.Get(a.ctx, key) if err != nil { t.Fatal(err) } sum := sha256.Sum256(body) if p.Phase != "done" || len(p.Reports) != 1 || p.Reports["box-a"] != hex.EncodeToString(sum[:]) || p.ID != before.ID || !p.CreatedAt.Equal(before.CreatedAt) || p.DirectoryRevision != before.DirectoryRevision { t.Fatalf("done plan = %+v", p) } for host, h := range before.Hosts { if p.Hosts[host] != h { t.Fatalf("%s changed: %+v -> %+v", host, h, p.Hosts[host]) } } if len(p.Hosts) != 2 { t.Fatalf("hosts = %+v", p.Hosts) } if err := CheckCreateFreeze(a.ctx, a.f.store, a.d.PublicKey); err != nil { t.Fatalf("freeze after done = %v", err) } serial := 0 c := NewBackfillController(a.f.store, a.d, func() time.Time { return a.f.now }, func() string { serial++; return fmt.Sprintf("op-%d", serial) }) if err := c.ReconcileClaims(a.ctx); err != nil { t.Fatalf("adoption pass over a done plan = %v", err) } for _, host := range []string{doneOne, doneTwo} { if obs, err := hostclaim.Read(a.ctx, a.f.store, host); err != nil || !obs.Absent { t.Fatalf("%s gained a claim: %+v %v", host, obs.Claim, err) } } if _, _, _, _, err := b.Adoption(a.ctx, a.records[0]); !errors.Is(err, ErrContract) { t.Fatalf("Adoption on a done plan = %v", err) } _, doneTag, err := a.f.store.Get(a.ctx, BackfillKey) if err != nil { t.Fatal(err) } if _, err := b.Done(a.ctx, a.sg, doneTag); !errors.Is(err, ErrDonePhase) { t.Fatalf("second Done = %v", err) } } // The dev trial's real shape: every instance record deleted before the // window, so the collecting plan freezes zero hosts. Its hosts are an empty // object, never null, and Done accepts an empty listing against it. func TestBackfillDoneWithZeroHosts(t *testing.T) { a := newActivation(t) a.keyless("box-b") tag := a.collecting(a.sg, nil) raw, _, err := a.f.store.Get(a.ctx, BackfillKey) if err != nil { t.Fatal(err) } var wire struct { Payload string `json:"payload"` } if err := json.Unmarshal(raw, &wire); err != nil { t.Fatal(err) } payload, err := base64.StdEncoding.DecodeString(wire.Payload) if err != nil || !bytes.Contains(payload, []byte(`"hosts":{}`)) { t.Fatalf("zero-host plan payload = %s, %v; want \"hosts\":{}", payload, err) } if p := a.plan(); p.Hosts == nil || len(p.Hosts) != 0 { t.Fatalf("zero-host plan hosts = %#v", p.Hosts) } if err := PublishCensusReport(a.ctx, a.f.store, a.d, a.emptyReport("box-a"), a.boxKeys["box-a"]); err != nil { t.Fatal(err) } b := &Backfill{Store: a.f.store, Directory: a.d} unreported, err := b.Done(a.ctx, a.sg, tag) if err != nil || !slices.Equal(unreported, []string{"box-b"}) { t.Fatalf("Done over zero hosts = %v, %v", unreported, err) } if p := a.plan(); p.Phase != "done" || p.Hosts == nil || len(p.Hosts) != 0 || len(p.Reports) != 1 { t.Fatalf("done plan = %#v", p) } if err := CheckCreateFreeze(a.ctx, a.f.store, a.d.PublicKey); err != nil { t.Fatal(err) } // A record that appears after a zero-host collect is refused. b2 := newActivation(t) b2.keyless("box-b") tag = b2.collecting(b2.sg, nil) if err := PublishCensusReport(b2.ctx, b2.f.store, b2.d, b2.emptyReport("box-a"), b2.boxKeys["box-a"]); err != nil { t.Fatal(err) } b2.record(doneOne, nil) if _, err := (&Backfill{Store: b2.f.store, Directory: b2.d}).Done(b2.ctx, b2.sg, tag); !errors.Is(err, ErrDoneRecords) { t.Fatalf("Done with a record the zero-host plan does not name = %v", err) } } // Every condition is its own refusal, and a refusal writes nothing: the // plan stays collecting, byte for byte, and creates stay frozen. func TestBackfillDoneRefusals(t *testing.T) { type setup struct { keyless []string // boxes with no census key (default box-b) edit func(*BackfillPlan) // the collecting plan signer bool // sign the plan with doneOtherKey reports func(a *activation) // default: box-a's empty report after func(a *activation) // changes after the reports tag func(t string) string // the etag Done is given } for name, c := range map[string]struct { s setup want []error }{ "etag": {setup{tag: func(string) string { return "other" }}, []error{ErrDoneETag}}, "envelope": {setup{signer: true}, []error{ErrDoneEnvelope}}, "phase": {setup{edit: func(p *BackfillPlan) { p.Phase = "adopting"; p.Reports = map[string]string{} }}, []error{ErrDonePhase}}, "record etag": {setup{after: func(a *activation) { r := a.records[0] if _, err := a.f.store.Put(a.ctx, r.Holder.Key, append(append([]byte(nil), r.Body...), '\n'), r.ETag); err != nil { a.t.Fatal(err) } }}, []error{ErrDoneRecords}}, "record added": {setup{after: func(a *activation) { a.record("three.shop.example.com", nil) }}, []error{ErrDoneRecords}}, "record gone": {setup{after: func(a *activation) { if err := release.DeleteIfMatch(a.ctx, a.f.store, a.records[1].Holder.Key, a.records[1].ETag); err != nil { a.t.Fatal(err) } }}, []error{ErrDoneRecords}}, "acquisition": {setup{edit: func(p *BackfillPlan) { h := p.Hosts[doneOne] h.Acquisition = "other" p.Hosts[doneOne] = h }}, []error{ErrDoneRecords}}, "claim": {setup{after: func(a *activation) { key, err := hostclaim.Key(doneOne) if err != nil { a.t.Fatal(err) } if _, err := a.f.store.Put(a.ctx, key, []byte(`{}`), ""); err != nil { a.t.Fatal(err) } }}, []error{ErrDoneClaims}}, "admission": {setup{after: func(a *activation) { key, err := AdmissionKey("bab", "shop") if err != nil { a.t.Fatal(err) } if _, err := a.f.store.Put(a.ctx, key, []byte(`{}`), ""); err != nil { a.t.Fatal(err) } }}, []error{ErrDoneAdmission}}, "no report": {setup{reports: func(*activation) {}}, []error{ErrDoneReport, os.ErrNotExist}}, "keyed box without a report": {setup{keyless: []string{}}, []error{ErrDoneReport, os.ErrNotExist}}, "no keyed box": {setup{keyless: []string{"box-a", "box-b"}, reports: func(*activation) {}}, []error{ErrDoneReport}}, "report signature": {setup{reports: func(a *activation) { a.rawReport(a.emptyReport("box-a"), a.boxKeys["box-b"]) }}, []error{ErrDoneReport}}, "report revision": {setup{reports: func(a *activation) { r := a.emptyReport("box-a") r.DirectoryRevision = 2 a.rawReport(r, a.boxKeys["box-a"]) }}, []error{ErrDoneReport}}, "report reservation": {setup{reports: func(a *activation) { r := a.emptyReport("box-a") r.Reservation = "other" a.rawReport(r, a.boxKeys["box-a"]) }}, []error{ErrDoneReport}}, "report incomplete": {setup{reports: func(a *activation) { r := a.emptyReport("box-a") r.Complete = false a.rawReport(r, a.boxKeys["box-a"]) }}, []error{ErrDoneReport}}, "report captured before the plan": {setup{reports: func(a *activation) { r := a.emptyReport("box-a") r.CapturedAt = a.f.now.Add(-time.Second) if err := PublishCensusReport(a.ctx, a.f.store, a.d, r, a.boxKeys["box-a"]); err != nil { a.t.Fatal(err) } }}, []error{ErrDoneStale}}, "installations": {setup{reports: func(a *activation) { r := a.emptyReport("box-a") r.Installations = []CensusInstallation{a.installation(doneOne)} if err := PublishCensusReport(a.ctx, a.f.store, a.d, r, a.boxKeys["box-a"]); err != nil { a.t.Fatal(err) } }}, []error{ErrDoneInstallations}}, "roster moved": {setup{after: func(a *activation) { a.roster.Revision = 2 a.keyless() }}, []error{ErrPending}}, } { t.Run(name, func(t *testing.T) { a := newActivation(t) a.record(doneOne, nil) a.record(doneTwo, nil) keyless := []string{"box-b"} if c.s.keyless != nil { keyless = c.s.keyless } a.keyless(keyless...) signer := a.sg if c.s.signer { signer = release.RootSigner(doneOtherKey) } tag := a.collecting(signer, c.s.edit) if c.s.reports != nil { c.s.reports(a) } else if err := PublishCensusReport(a.ctx, a.f.store, a.d, a.emptyReport("box-a"), a.boxKeys["box-a"]); err != nil { t.Fatal(err) } if c.s.after != nil { c.s.after(a) } if c.s.tag != nil { tag = c.s.tag(tag) } before, beforeTag, err := a.f.store.Get(a.ctx, BackfillKey) if err != nil { t.Fatal(err) } b := &Backfill{Store: a.f.store, Directory: a.d} unreported, err := b.Done(a.ctx, a.sg, tag) for _, want := range c.want { if !errors.Is(err, want) { t.Fatalf("Done = %v, %v; want %v", unreported, err, want) } } after, afterTag, err := a.f.store.Get(a.ctx, BackfillKey) if err != nil || !bytes.Equal(before, after) || beforeTag != afterTag { t.Fatalf("a refused Done changed the plan (%v)", err) } if err := CheckCreateFreeze(a.ctx, a.f.store, a.d.PublicKey); err == nil { t.Fatal("a refused Done lifted the create freeze") } }) } } // Done's copy of the census's per-box report check must accept and refuse // exactly what ProductionAuthority.census does (census is accepted-legacy // and stays untouched). Each variant is box-a's report; box-b's is valid. func TestDoneVerifiesReportsAsTheCensusDoes(t *testing.T) { for name, c := range map[string]struct { edit func(a *activation, r *CensusReport) key string // signing box key (default box-a) }{ "valid": {}, "valid with a host": {edit: func(a *activation, r *CensusReport) { r.Installations = []CensusInstallation{a.installation(doneOne)} }}, "other box's key": {key: "box-b"}, "other box": {edit: func(_ *activation, r *CensusReport) { r.Box = "box-b" }}, "other enrollment": {edit: func(_ *activation, r *CensusReport) { r.Enrollment = "x" }}, "other reservation": {edit: func(_ *activation, r *CensusReport) { r.Reservation = "other" }}, "other revision": {edit: func(_ *activation, r *CensusReport) { r.DirectoryRevision = 2 }}, "no boot": {edit: func(_ *activation, r *CensusReport) { r.BootID = "" }}, "no capture time": {edit: func(_ *activation, r *CensusReport) { r.CapturedAt = time.Time{} }}, "nil installations": {edit: func(_ *activation, r *CensusReport) { r.Installations = nil }}, "incomplete": {edit: func(_ *activation, r *CensusReport) { r.Complete = false }}, "unknown host": {edit: func(a *activation, r *CensusReport) { i := a.installation(doneOne) i.Host = "x.example.com" r.Installations = []CensusInstallation{i} }}, "host twice": {edit: func(a *activation, r *CensusReport) { i := a.installation(doneOne) r.Installations = []CensusInstallation{i, i} }}, "neither route nor local": {edit: func(a *activation, r *CensusReport) { i := a.installation(doneOne) i.Route = false r.Installations = []CensusInstallation{i} }}, "other account": {edit: func(a *activation, r *CensusReport) { i := a.installation(doneOne) i.Account = "zzz" r.Installations = []CensusInstallation{i} }}, "other key": {edit: func(a *activation, r *CensusReport) { i := a.installation(doneOne) i.Key = "apps/x" r.Installations = []CensusInstallation{i} }}, "other family": {edit: func(a *activation, r *CensusReport) { i := a.installation(doneOne) i.Family = "sidecar" r.Installations = []CensusInstallation{i} }}, "other incarnation": {edit: func(a *activation, r *CensusReport) { i := a.installation(doneOne) i.Incarnation = "x" r.Installations = []CensusInstallation{i} }}, } { t.Run(name, func(t *testing.T) { a := newActivation(t) a.record(doneOne, nil) a.collecting(a.sg, nil) if err := PublishCensusReport(a.ctx, a.f.store, a.d, a.emptyReport("box-b"), a.boxKeys["box-b"]); err != nil { t.Fatal(err) } r := a.emptyReport("box-a") if c.edit != nil { c.edit(a, &r) } key := a.boxKeys["box-a"] if c.key != "" { key = a.boxKeys[c.key] } k, _ := reportKey("backfill", "box-a") canonical, _ := json.Marshal(r) raw, _ := json.Marshal(CensusEnvelope{Report: r, Signature: ed25519.Sign(key, canonical)}) tag, err := a.f.store.Put(a.ctx, k, raw, "") if err != nil { t.Fatal(err) } _, censusErr := ProductionAuthority{Store: a.f.store, Directory: a.d}.census(a.ctx, "", "", "backfill", 0, nil) records := map[string]Record{} for _, rec := range a.records { records[rec.Holder.Host] = rec } _, doneErr := verifyCensusReport(raw, tag, "box-a", a.roster.Boxes["box-a"], "backfill", a.roster.Revision, records) if (censusErr == nil) != (doneErr == nil) { t.Fatalf("census = %v, Done's check = %v", censusErr, doneErr) } if name == "valid" || name == "valid with a host" { if doneErr != nil { t.Fatalf("valid report refused: %v", doneErr) } } else if doneErr == nil { t.Fatal("variant accepted") } }) } } // doneCohort is the done tests' empty cohort: hosts records (zero or two), // box-b keyless, box-a's empty report published, the collecting plan's etag // returned. func doneCohort(t *testing.T, hosts int) (*activation, string) { t.Helper() a := newActivation(t) for _, host := range []string{doneOne, doneTwo}[:hosts] { a.record(host, nil) } a.keyless("box-b") tag := a.collecting(a.sg, nil) if err := PublishCensusReport(a.ctx, a.f.store, a.d, a.emptyReport("box-a"), a.boxKeys["box-a"]); err != nil { t.Fatal(err) } return a, tag } // A record created while Done reads the reports (the first listing has // already passed) is caught by the second listing, immediately before the // write: Done refuses and writes nothing. func TestBackfillDoneRefusesARecordCreatedDuringReportRetrieval(t *testing.T) { for _, hosts := range []int{0, 2} { t.Run(fmt.Sprintf("%d hosts", hosts), func(t *testing.T) { a, tag := doneCohort(t, hosts) reportAt, err := reportKey("backfill", "box-a") if err != nil { t.Fatal(err) } fired := false a.f.store.afterGet = func(key string) { if key != reportAt { return } a.f.store.afterGet = nil fired = true a.record("three.shop.example.com", nil) } before, beforeTag, err := a.f.store.Get(a.ctx, BackfillKey) if err != nil { t.Fatal(err) } _, err = (&Backfill{Store: a.f.store, Directory: a.d}).Done(a.ctx, a.sg, tag) if !fired { t.Fatal("the report was never read") } if !errors.Is(err, ErrDoneRecords) || errors.Is(err, ErrDoneRecordsChanged) { t.Fatalf("Done = %v; want the records refusal before the write", err) } after, afterTag, err := a.f.store.Get(a.ctx, BackfillKey) if err != nil || !bytes.Equal(before, after) || beforeTag != afterTag { t.Fatalf("a refused Done changed the plan (%v)", err) } if err := CheckCreateFreeze(a.ctx, a.f.store, a.d.PublicKey); !errors.Is(err, ErrPending) { t.Fatalf("freeze after a refused Done = %v", err) } }) } } // A record created after Done's write, before its third listing, is not in // the done plan. Done has written done and cannot take it back; it says // so with ErrDoneRecordsChanged (which the claims run reports under the // records refusal). func TestBackfillDoneReportsARecordCreatedAfterTheWrite(t *testing.T) { for _, hosts := range []int{0, 2} { t.Run(fmt.Sprintf("%d hosts", hosts), func(t *testing.T) { a, tag := doneCohort(t, hosts) fired := false a.f.store.afterPut = func(key string) { if key != BackfillKey { return } a.f.store.afterPut = nil fired = true a.record("three.shop.example.com", nil) } _, err := (&Backfill{Store: a.f.store, Directory: a.d}).Done(a.ctx, a.sg, tag) if !fired { t.Fatal("Done never wrote the plan") } if !errors.Is(err, ErrDoneRecordsChanged) || !errors.Is(err, ErrDoneRecords) { t.Fatalf("Done = %v; want ErrDoneRecordsChanged", err) } if p := a.plan(); p.Phase != "done" || len(p.Hosts) != hosts { t.Fatalf("plan = %+v; want done over the %d frozen hosts", p, hosts) } }) } }