package hostops import ( "context" "crypto/ed25519" "crypto/rand" "crypto/sha256" "encoding/hex" "encoding/json" "errors" "reflect" "strings" "testing" "time" "github.com/carlosframework/platform/internal/domains" "github.com/carlosframework/platform/internal/hostactors" "github.com/carlosframework/platform/internal/hostclaim" "github.com/carlosframework/platform/internal/release" ) func TestReconcileFrozenBackfillAndQuarantine(t *testing.T) { testReconcileFrozenBackfill(t, false, "") } func TestReconcileFrozenBackfillRepairsLegacyHolder(t *testing.T) { testReconcileFrozenBackfill(t, true, "") } func testReconcileFrozenBackfill(t *testing.T, legacy bool, scenario string) { t.Helper() ctx := context.Background() f := newFixture(t) pub, key, err := ed25519.GenerateKey(rand.Reader) if err != nil { t.Fatal(err) } sg := release.RootSigner(key) roster := hostactors.Roster{V: 1, DeploymentBucket: "bucket", Revision: 1, Regions: map[string]hostactors.Region{"ie": {}, "us": {}}, Boxes: map[string]hostactors.Box{}, Consoles: map[string]hostactors.Console{}, Holds: map[string]hostactors.Reservation{}, Activations: map[string]hostactors.Reservation{}} boxKeys := map[string]ed25519.PrivateKey{} for i, ref := range []string{"box-a", "box-b"} { boxPub, boxKey, e := ed25519.GenerateKey(rand.Reader) if e != nil { t.Fatal(e) } boxKeys[ref] = boxKey region := "ie" en := strings.Repeat("a", 32) if i == 1 { region = "us" en = strings.Repeat("b", 32) } roster.Boxes[ref] = hostactors.Box{CensusPublicKey: boxPub, Enrollment: en, Namespace: "platform", BoxID: ref, Label: ref, Region: region, Pool: "default", Cloud: hostactors.Cloud{Provider: "aws", Account: "123", Region: region, InstanceID: "i-" + ref}, EvidenceSHA256: strings.Repeat("c", 64), State: "active"} } if scenario == "two-box-region" || scenario == "two-owners" { box := roster.Boxes["box-b"] box.Region, box.Cloud.Region = "ie", "ie" roster.Boxes["box-b"] = box } if scenario == "inactive-owner" || scenario == "wrong-region" { box := roster.Boxes["box-a"] if scenario == "inactive-owner" { box.State = "retired" } else { box.Region, box.Cloud.Region = "us", "us" } roster.Boxes["box-a"] = box } put := func(k string, v any, signed bool) string { t.Helper() var raw []byte var e error if signed { raw, e = release.SignRootEnvelope(sg, v) } else { raw, e = json.Marshal(v) } if e != nil { t.Fatal(e) } tag, e := f.store.Put(ctx, k, raw, "") if e != nil { t.Fatal(e) } return tag } put(hostactors.Key, roster, true) d := hostactors.Directory{Store: f.store, PublicKey: pub, Bucket: "bucket"} c := NewBackfillController(f.store, d, func() time.Time { return f.now }, f.c.NewID) r := f.record(t, "legacy.example.com") var body map[string]any if err = json.Unmarshal(r.Body, &body); err != nil { t.Fatal(err) } body["acq"] = "ownership-acquisition" put(r.Holder.Key, body, false) r, err = ReadRecord(ctx, f.store, r.Holder.Key) if err != nil { t.Fatal(err) } inc, err := r.Incarnation() if err != nil { t.Fatal(err) } put(domains.ClaimKey(r.Holder.Host), domains.Claim{Schema: 1, Host: r.Holder.Host, Account: "bab", App: "jam", Acquisition: "ownership-acquisition", RecordIncarnation: inc, OwnershipState: "committed"}, false) if err = c.ReconcileClaims(ctx); err != nil { t.Fatalf("unfrozen adoption: %v", err) } obs, err := hostclaim.Read(ctx, f.store, r.Holder.Host) if err != nil || !obs.Absent { t.Fatal(obs, err) } var prior hostclaim.Observed if legacy { nh := r.Holder nh.Acquisition = "" nh.Region = "ie" prior, err = hostclaim.Update(ctx, f.store, obs, func(claim *hostclaim.Claim) error { if err := hostclaim.Adopt(nh, "box-a", []string{"us"}, []string{"box-b"}, "legacy-discovery", f.c.NewID)(claim); err != nil { return err } claim.Holder.Stamp = 17 claim.Holder.Generation = "legacy-generation" return nil }) if err != nil { t.Fatal(err) } } p := BackfillPlan{V: 1, ID: "backfill", Phase: "adopting", DirectoryRevision: 1, CreatedAt: f.now, Reports: map[string]string{}, Hosts: map[string]BackfillHost{r.Holder.Host: {Key: r.Holder.Key, ETag: r.ETag, Acquisition: "ownership-acquisition", Region: "ie", Box: "box-a"}}} for ref, b := range roster.Boxes { report := CensusReport{Box: ref, Enrollment: b.Enrollment, BootID: "boot", Reservation: p.ID, DirectoryRevision: 1, CapturedAt: f.now, Complete: true, Installations: []CensusInstallation{}} if ref == "box-a" || scenario == "two-owners" { report.Installations = append(report.Installations, CensusInstallation{Host: r.Holder.Host, Account: "bab", App: "jam", Family: "instance", Key: r.Holder.Key, Incarnation: inc, Local: true, Route: true}) } key, _ := reportKey(p.ID, ref) body, e := json.Marshal(report) if e != nil { t.Fatal(e) } put(key, CensusEnvelope{Report: report, Signature: ed25519.Sign(boxKeys[ref], body)}, false) raw, _, err := f.store.Get(ctx, key) if err != nil { t.Fatal(err) } sum := sha256.Sum256(raw) p.Reports[ref] = hex.EncodeToString(sum[:]) } put(BackfillKey, p, true) qtag := put(HostsQuarantineKey, HostsQuarantine{V: 1, Hosts: map[string]string{r.Holder.Host: "conflict"}}, false) if err = c.ReconcileClaims(ctx); !errors.Is(err, ErrPending) { t.Fatalf("quarantine: %v", err) } raw, _ := json.Marshal(HostsQuarantine{V: 1, Hosts: map[string]string{}}) if _, err = f.store.Put(ctx, HostsQuarantineKey, raw, qtag); err != nil { t.Fatal(err) } // A digest listed by a root manifest cannot turn a forged box report into // an authenticated census. Test the verifier before its cache is populated. reportPath, _ := reportKey(p.ID, "box-a") authentic, reportTag, e := f.store.Get(ctx, reportPath) if e != nil { t.Fatal(e) } var forged CensusEnvelope if e = json.Unmarshal(authentic, &forged); e != nil { t.Fatal(e) } forged.Signature[0] ^= 1 forgedRaw, _ := json.Marshal(forged) forgedTag, e := f.store.Put(ctx, reportPath, forgedRaw, reportTag) if e != nil { t.Fatal(e) } sum := sha256.Sum256(forgedRaw) p.Reports["box-a"] = hex.EncodeToString(sum[:]) _, manifestTag, e := f.store.Get(ctx, BackfillKey) if e != nil { t.Fatal(e) } signed, _ := release.SignRootEnvelope(sg, p) manifestTag, e = f.store.Put(ctx, BackfillKey, signed, manifestTag) if e != nil { t.Fatal(e) } if e = c.ReconcileClaims(ctx); e == nil { t.Fatal("accepted forged census") } if _, e = f.store.Put(ctx, reportPath, authentic, forgedTag); e != nil { t.Fatal(e) } sum = sha256.Sum256(authentic) p.Reports["box-a"] = hex.EncodeToString(sum[:]) signed, _ = release.SignRootEnvelope(sg, p) if _, e = f.store.Put(ctx, BackfillKey, signed, manifestTag); e != nil { t.Fatal(e) } if scenario == "two-owners" || scenario == "inactive-owner" || scenario == "wrong-region" { if err = c.ReconcileClaims(ctx); !errors.Is(err, ErrIdentity) { t.Fatalf("ineligible census owner: %v", err) } obs, err = hostclaim.Read(ctx, f.store, r.Holder.Host) if err != nil || !obs.Absent { t.Fatalf("ineligible owner adopted: %+v %v", obs, err) } return } if err = c.ReconcileClaims(ctx); err != nil { t.Fatal(err) } obs, err = hostclaim.Read(ctx, f.store, r.Holder.Host) if err != nil { t.Fatal(err) } if obs.Absent || obs.Claim.Holder.Acquisition != "ownership-acquisition" || obs.Claim.Op.Exec != "box-a" || obs.Claim.Touched.Complete || len(obs.Claim.Obligations) != 2 { t.Fatalf("bad backfill: %+v", obs.Claim) } wantDestinations := map[string]hostclaim.Obligation{ "park-replica": {Region: "us", Owner: "region:us"}, "park-local": {Box: "box-b", Owner: "box:box-b"}, } for _, obligation := range obs.Claim.Obligations { want, ok := wantDestinations[obligation.Kind] if !ok || obligation.Region != want.Region || obligation.Box != want.Box || obligation.Owner != want.Owner { t.Fatalf("wrong discovery destination: %+v; want %+v", obligation, want) } delete(wantDestinations, obligation.Kind) } if len(wantDestinations) != 0 { t.Fatalf("missing discovery destinations: %+v", wantDestinations) } if legacy { want := *prior.Claim holder := *want.Holder holder.Acquisition = "ownership-acquisition" want.Holder = &holder want.Gen++ if obs.ETag == prior.ETag || !reflect.DeepEqual(obs.Claim, &want) { t.Fatalf("legacy repair changed more than acquisition and claim generation: got %+v (holder %+v), want %+v (holder %+v)", obs.Claim, obs.Claim.Holder, &want, want.Holder) } } before := obs.ETag if err = c.ReconcileClaims(ctx); err != nil { t.Fatal(err) } obs, err = hostclaim.Read(ctx, f.store, r.Holder.Host) if err != nil || obs.ETag != before { t.Fatal("non-idempotent", err) } } func TestReconcileDiscardPartialCensus(t *testing.T) { f := newFixture(t) r := f.record(t, "legacy.example.com") if _, err := f.store.Put(f.ctx, r.Holder.Key, r.Body, ""); err != nil { t.Fatal(err) } fail := errors.New("second listing failed") f.store.afterList = func(prefix string, keys []string) error { if strings.Contains(prefix, "sidecars/") { return fail } return nil } calls := 0 err := ReconcileClaims(f.ctx, f.c, func(context.Context, string) (hostclaim.Observed, error) { calls++; return hostclaim.Observed{}, nil }) if !errors.Is(err, fail) || calls != 0 { t.Fatal(err, calls) } } func TestClaimDeleteDoorNeverAdopts(t *testing.T) { f := newFixture(t) r := f.record(t, "legacy.example.com") if _, err := f.store.Put(f.ctx, r.Holder.Key, r.Body, ""); err != nil { t.Fatal(err) } inc, err := r.Incarnation() if err != nil { t.Fatal(err) } if _, err = f.c.BeginRetire(f.ctx, r.Holder.Key, "test", inc); !errors.Is(err, ErrPending) { t.Fatal(err) } obs, err := hostclaim.Read(f.ctx, f.store, r.Holder.Host) if err != nil || !obs.Absent { t.Fatalf("door adopted: %+v %v", obs, err) } } func TestBackfillAuthorityCannotGainMutationMethods(t *testing.T) { c := NewBackfillController(nil, hostactors.Directory{}, time.Now, func() string { return "id" }) if _, ok := c.Authority.(interface { StartCreate(context.Context, hostclaim.Observed, Record, string, string) (hostclaim.Observed, error) }); ok { t.Fatal("backfill gained create authority") } if _, ok := c.Authority.(interface { StartDeleteClaim(context.Context, hostclaim.Observed, Record, string) (hostclaim.Observed, error) }); ok { t.Fatal("backfill gained delete authority") } if _, ok := c.Authority.(interface { RecordEffects(context.Context, hostclaim.Observed, Record) error }); ok { t.Fatal("backfill gained effect authority") } } func TestReconcileBackfillCensusOwnerEligibility(t *testing.T) { for _, scenario := range []string{"two-box-region", "two-owners", "inactive-owner", "wrong-region"} { t.Run(scenario, func(t *testing.T) { testReconcileFrozenBackfill(t, false, scenario) }) } } func TestReconcileBackfillPlanPhases(t *testing.T) { for _, phase := range []string{"absent", "collecting", "done", "future", "adopting", "bad-signature", "read-error"} { t.Run(phase, func(t *testing.T) { f := newFixture(t) pub, key, err := ed25519.GenerateKey(rand.Reader) if err != nil { t.Fatal(err) } if phase != "absent" && phase != "read-error" { p := BackfillPlan{V: 1, ID: "backfill", Phase: phase, CreatedAt: f.now, Hosts: map[string]BackfillHost{}} raw, err := release.SignRootEnvelope(release.RootSigner(key), p) if err != nil { t.Fatal(err) } if _, err = f.store.Put(f.ctx, BackfillKey, raw, ""); err != nil { t.Fatal(err) } } if phase == "bad-signature" { pub = make(ed25519.PublicKey, ed25519.PublicKeySize) } if phase == "read-error" { f.store.failGet = BackfillKey } c := NewBackfillController(f.store, hostactors.Directory{Store: f.store, PublicKey: pub, Bucket: "bucket"}, f.c.Now, f.c.NewID) lists, adopts := 0, 0 f.store.afterList = func(string, []string) error { lists++; return nil } err = ReconcileClaims(f.ctx, c, func(context.Context, string) (hostclaim.Observed, error) { adopts++; return hostclaim.Observed{}, nil }) wantError := phase == "adopting" || phase == "bad-signature" || phase == "read-error" if (err != nil) != wantError || lists != 0 || adopts != 0 { t.Fatalf("phase %s: err=%v lists=%d adopts=%d", phase, err, lists, adopts) } if !wantError && phase != "absent" { if _, _, _, _, err := c.Authority.Adoption(f.ctx, Record{}); err == nil { t.Fatal("inactive plan granted direct adoption") } } }) } }