package hostops import ( "context" "crypto/ed25519" "crypto/rand" "encoding/json" "errors" "strings" "testing" "time" "github.com/carlosframework/platform/internal/hostactors" "github.com/carlosframework/platform/internal/hostclaim" "github.com/carlosframework/platform/internal/provision" "github.com/carlosframework/platform/internal/release" ) const moveStartID = "mv_abcdefghijklmnopqrstuvwxyz" // moveStartStore counts claim CASes and can run a hook on the first record // read, which BeginClaimMove makes after its admission-free prechecks. type moveStartStore struct { release.Store claimKey, recordKey string claimPuts int onRecordRead func() } func (s *moveStartStore) Get(ctx context.Context, key string) ([]byte, string, error) { if key == s.recordKey && s.onRecordRead != nil { f := s.onRecordRead s.onRecordRead = nil f() } return s.Store.Get(ctx, key) } func (s *moveStartStore) Put(ctx context.Context, key string, body []byte, match string) (string, error) { if key == s.claimKey { s.claimPuts++ } return s.Store.Put(ctx, key, body, match) } type moveStartFixture struct { raw *moveStartStore w *ControlWriter authority ProductionAuthority host, inc string now time.Time enIE string enUK string pub ed25519.PublicKey } // productionData is ConfigureClaimRuntime's data handle over f.raw: the // mutation boundary over the conditional record store, checked by the // production checker with the real ControlWriter over the same raw store. func (f *moveStartFixture) productionData(t *testing.T) release.Store { t.Helper() records, err := NewConditionalRecordStore(f.raw, f.pub) if err != nil { t.Fatal(err) } check, err := NewProductionMutationCheck(f.raw, f.w) if err != nil { t.Fatal(err) } data, err := NewMutationBoundary(true, records, check) if err != nil { t.Fatal(err) } return data } func moveRosterBox(id, region, enrollment, instance string) hostactors.Box { return hostactors.Box{Enrollment: enrollment, Namespace: "platform", BoxID: id, Label: id, Region: region, Pool: "default", Cloud: hostactors.Cloud{Provider: "aws", Account: "123", Region: region, InstanceID: instance}, EvidenceSHA256: strings.Repeat("c", 64), State: "active"} } func newMoveStartFixture(t *testing.T, mutateRoster func(*hostactors.Roster)) *moveStartFixture { t.Helper() ctx := context.Background() now := time.Date(2026, 10, 7, 12, 0, 0, 0, time.UTC) host := "shop.example.com" f := &moveStartFixture{host: host, now: now, enIE: strings.Repeat("a", 32), enUK: strings.Repeat("b", 32)} claimKey, _ := hostclaim.Key(host) f.raw = &moveStartStore{Store: release.NewDirStore(t.TempDir()), claimKey: claimKey, recordKey: provision.RecordKey("bab", "app", host)} pub, key, err := ed25519.GenerateKey(rand.Reader) if err != nil { t.Fatal(err) } f.pub = pub roster := hostactors.Roster{V: 1, DeploymentBucket: "bucket", Revision: 3, Boxes: map[string]hostactors.Box{ "ie-box": moveRosterBox("ie-box", "ie", f.enIE, "i-ie"), "uk-box": moveRosterBox("uk-box", "uk", f.enUK, "i-uk"), }, Consoles: map[string]hostactors.Console{}, Holds: map[string]hostactors.Reservation{}, Activations: map[string]hostactors.Reservation{}} if mutateRoster != nil { mutateRoster(&roster) } signed, err := release.SignRootEnvelope(release.RootSigner(key), roster) if err != nil { t.Fatal(err) } if _, err = f.raw.Store.Put(ctx, hostactors.Key, signed, ""); err != nil { t.Fatal(err) } if f.inc, err = hostclaim.Incarnation("bab", "app", host, now); err != nil { t.Fatal(err) } rec, _ := json.Marshal(provision.Record{Schema: 1, Host: host, Slug: "app", Account: "bab", App: "app", Channel: "edge", Region: "ie", Acquisition: "acq", CreatedAt: now}) if _, err = f.raw.Store.Put(ctx, f.raw.recordKey, rec, ""); err != nil { t.Fatal(err) } c := hostclaim.Claim{V: 2, Host: host, Gen: 4, Holder: &hostclaim.Holder{Family: "instance", Account: "bab", App: "app", Key: f.raw.recordKey, Acquisition: "acq", Incarnation: f.inc, CreatedAt: now, Location: "instances/" + host}, Op: hostclaim.Op{Kind: hostclaim.KindServing, Exec: "ie-box"}, Touched: hostclaim.Touched{Complete: true, Regions: []string{"ie"}, Boxes: []string{"ie-box"}}} cb, _ := json.Marshal(c) if _, err = f.raw.Store.Put(ctx, claimKey, cb, ""); err != nil { t.Fatal(err) } admission, _ := json.Marshal(AppAdmission{V: 1, AppID: "canonical", Revision: 1, Epoch: 1, State: "open", Registrations: map[string]Registration{}, Effects: map[string]Effect{}}) if _, err = f.raw.Store.Put(ctx, "apps/bab/app/admission.json", admission, ""); err != nil { t.Fatal(err) } clock := func() time.Time { return f.now } if f.w, err = NewControlWriter(f.raw, pub, clock); err != nil { t.Fatal(err) } f.authority = ProductionAuthority{Store: f.raw, Control: f.w, Directory: hostactors.Directory{Store: f.raw, PublicKey: pub, Bucket: "bucket"}, ConsoleID: "console", Enrollment: strings.Repeat("e", 32), Now: clock} return f } func (f *moveStartFixture) read(t *testing.T) hostclaim.Observed { t.Helper() o, err := hostclaim.Read(context.Background(), f.raw, f.host) if err != nil { t.Fatal(err) } return o } func (f *moveStartFixture) begin(h *StartHandle, o hostclaim.Observed, target, to string) (hostclaim.Observed, error) { f.raw.claimPuts = 0 return BeginClaimMove(context.Background(), f.raw, f.authority.Directory, h, o, moveStartID, "operator", target, to, f.now) } // BeginClaimMove persists admission_id from the writer-minted exact "move" // registration alongside app_id, and both enrollments from the signed // roster, in one claim CAS. func TestBeginClaimMovePersistsAdmissionAndRosterEnrollment(t *testing.T) { ctx := context.Background() f := newMoveStartFixture(t, nil) o := f.read(t) h, err := f.authority.RegisterMove(ctx, o, "canonical", moveStartID) if err != nil { t.Fatal(err) } moving, err := f.begin(h, o, "uk-box", "uk") if err != nil { t.Fatal(err) } in := moving.Claim.Op.Intent want := map[string]string{"to": "uk", "source": "ie-box", "target": "uk-box", "app_id": "canonical", "admission_id": moveStartID, "source_enrollment": f.enIE, "target_enrollment": f.enUK} for k, v := range want { if in[k] != v { t.Fatalf("intent %s = %q, want %q (%+v)", k, in[k], v, in) } } if len(in) != len(want) || in["source_box"] != "" || in["source_region"] != "" { t.Fatalf("extra intent keys: %+v", in) } if f.raw.claimPuts != 1 || moving.Claim.Op.Kind != hostclaim.KindMoving || moving.Claim.Gen != o.Claim.Gen+1 { t.Fatalf("claim CASes=%d op=%+v", f.raw.claimPuts, moving.Claim.Op) } a, err := ReadAdmission(ctx, f.raw, "bab", "app", "canonical") if err != nil { t.Fatal(err) } reg := a.Value.Registrations[h.ID()] if reg.Kind != RegistrationMove || reg.Host != f.host || reg.Incarnation != f.inc || reg.Claim != SavedObservation(o) { t.Fatalf("registration not the exact move registration: %+v", reg) } // The record start is one issued effect of the same registration and // passes the lifecycle record writer's admission reread. rec, err := StartClaimMoveRecord(ctx, conditionalRecordStore{raw: f.raw}, h, moving, provision.MoveStart{ID: moveStartID, Incarnation: f.inc, From: "ie", To: "uk", Cut: "auto", By: "operator", Now: f.now}) if err != nil { t.Fatal(err) } if rec.Move == nil || rec.Move.ID != moveStartID || rec.Move.State != provision.MovePreparing { t.Fatalf("record not started: %+v", rec.Move) } // The start CAS output converts under C7 only with the admission it // carries: the conversion's required keys are present. if in["source"] == "" || in["source_enrollment"] == "" || in["app_id"] == "" || in["admission_id"] == "" { t.Fatal("start output is not convertible") } } func TestBeginClaimMoveRefusesWithoutExactLiveAdmission(t *testing.T) { for _, tc := range []string{"closed-before-cas", "drained-before-cas", "expired", "other-observation", "wrong-kind", "foreign-handle", "nil-handle"} { t.Run(tc, func(t *testing.T) { ctx := context.Background() f := newMoveStartFixture(t, nil) o := f.read(t) var h *StartHandle var err error if tc == "wrong-kind" { // A writer-minted registration under the move id, of another kind. reg := Registration{Kind: "wake", Host: f.host, Owner: RegistrationOwner{Console: "console", Enrollment: strings.Repeat("e", 32), Controller: "controller"}, AppID: "canonical", Epoch: 1, Claim: SavedObservation(o), Incarnation: f.inc, CreatedAt: f.now} h, err = f.w.RegisterStart(ctx, "bab", "app", "canonical", moveStartID, reg, f.now) } else { h, err = f.authority.RegisterMove(ctx, o, "canonical", moveStartID) } if err != nil { t.Fatal(err) } switch tc { case "closed-before-cas": // Closed between the prechecks and the claim CAS: the final // admission reread is inside the transition. f.raw.onRecordRead = func() { if _, err := f.w.BeginExclusive(ctx, "bab", "app", "canonical", AppOperation{ID: "delete", Kind: OpDeleting, AppID: "canonical"}, RegistrationOwner{Console: "console", Enrollment: strings.Repeat("e", 32), Controller: "controller"}); err != nil { t.Error(err) } } case "drained-before-cas": f.raw.onRecordRead = func() { f.now = f.now.Add(RegistrationTTL) } case "expired": f.now = f.now.Add(RegistrationTTL) case "other-observation": // A registration minted for an older claim observation. var err error if o, err = hostclaim.Update(ctx, f.raw.Store, o, func(c *hostclaim.Claim) error { return nil }); err != nil { t.Fatal(err) } case "foreign-handle": h = &StartHandle{w: f.w, account: "bab", app: "app", appID: "canonical", id: h.ID(), owner: RegistrationOwner{Console: "other", Enrollment: strings.Repeat("e", 32), Controller: "forged"}, epoch: 1} case "nil-handle": h = nil } before := f.read(t) if _, err := f.begin(h, o, "uk-box", "uk"); err == nil { t.Fatal("began a move without its exact live admission") } after := f.read(t) if f.raw.claimPuts != 0 || after.ETag != before.ETag { t.Fatalf("refusal wrote the claim: puts=%d", f.raw.claimPuts) } }) } } func TestBeginClaimMoveTakesEnrollmentsOnlyFromTheSignedRoster(t *testing.T) { for _, tc := range []string{"target-wrong-region", "source-wrong-region", "target-quarantined", "unknown-target", "same-box", "fenced-source"} { t.Run(tc, func(t *testing.T) { ctx := context.Background() var mutate func(*hostactors.Roster) target, to := "uk-box", "uk" switch tc { case "target-wrong-region": to = "de" case "source-wrong-region": mutate = func(r *hostactors.Roster) { b := r.Boxes["ie-box"] b.Region, b.Cloud.Region = "fr", "fr" r.Boxes["ie-box"] = b } case "target-quarantined": mutate = func(r *hostactors.Roster) { b := r.Boxes["uk-box"] b.State, b.QuarantineRevision, b.LossIDs = "quarantined", 1, []string{"loss"} r.Boxes["uk-box"] = b } case "unknown-target": target = "us-box" case "same-box": target, to = "ie-box", "ie" } f := newMoveStartFixture(t, mutate) o := f.read(t) if tc == "fenced-source" { // Any fence object for the source's enrollment fails Lookup closed. if _, err := f.raw.Store.Put(ctx, "control/box-fences/"+f.enIE+".json", []byte(`{}`), ""); err != nil { t.Fatal(err) } } h, err := f.authority.RegisterMove(ctx, o, "canonical", moveStartID) if err != nil { t.Fatal(err) } if _, err := f.begin(h, o, target, to); err == nil { t.Fatal("accepted") } if f.raw.claimPuts != 0 { t.Fatal("refusal wrote the claim") } }) } } // The registration id is the move id, so the settle sweep keeps it past its // TTL while the move holds the claim; recovery's admission reread depends on // it (moveAdmission). func TestClaimMoveRegistrationOutlivesItsTTL(t *testing.T) { ctx := context.Background() f := newMoveStartFixture(t, nil) o := f.read(t) h, err := f.authority.RegisterMove(ctx, o, "canonical", moveStartID) if err != nil { t.Fatal(err) } if _, err = f.begin(h, o, "uk-box", "uk"); err != nil { t.Fatal(err) } f.now = f.now.Add(RegistrationTTL + 2*RegistrationSkew) if err := f.w.SettleRegistration(ctx, "bab", "app", "canonical", moveStartID, f.now); !errors.Is(err, ErrPending) { t.Fatalf("settled a move that still holds the claim: %v", err) } a, err := ReadAdmission(ctx, f.raw, "bab", "app", "canonical") if err != nil { t.Fatal(err) } if _, ok := a.Value.Registrations[moveStartID]; !ok { t.Fatal("move registration drained under a moving claim") } if _, err := f.authority.RegisterMove(ctx, o, "canonical", "not-a-move-id"); !errors.Is(err, ErrIdentity) { t.Fatalf("registered under a non-move id: %v", err) } } // The record start runs only under the registration the claim names. func TestStartClaimMoveRecordIsBoundToItsRegistration(t *testing.T) { ctx := context.Background() f := newMoveStartFixture(t, nil) o := f.read(t) h, err := f.authority.RegisterMove(ctx, o, "canonical", moveStartID) if err != nil { t.Fatal(err) } moving, err := f.begin(h, o, "uk-box", "uk") if err != nil { t.Fatal(err) } moving.Claim.Op.Intent["admission_id"] = "other" if _, err := StartClaimMoveRecord(ctx, conditionalRecordStore{raw: f.raw}, h, moving, provision.MoveStart{ID: moveStartID, Incarnation: f.inc, From: "ie", To: "uk", Cut: "auto", Now: f.now}); !errors.Is(err, ErrIdentity) { t.Fatalf("record start under another registration: %v", err) } } // The start CAS and the record start through the production data handle: // the claim CAS needs the registration's lifecycle grant and the record Put // its issued effect. A caller holding only the data handle (no grant, or a // grant but no issued effect) writes neither. func TestClaimMoveStartThroughProductionBoundary(t *testing.T) { ctx := context.Background() f := newMoveStartFixture(t, nil) data := f.productionData(t) dir := hostactors.Directory{Store: data, PublicKey: f.pub, Bucket: "bucket"} authority := f.authority authority.Store, authority.Directory = data, dir o := f.read(t) h, err := authority.RegisterMove(ctx, o, "canonical", moveStartID) if err != nil { t.Fatal(err) } recordTag := func() string { t.Helper() _, tag, err := provision.GetRecordETag(ctx, f.raw, "bab", "app", f.host) if err != nil { t.Fatal(err) } return tag } f.raw.claimPuts = 0 admitted := hostclaim.MoveAdmission{AppID: "canonical", AdmissionID: moveStartID, SourceEnrollment: f.enIE, TargetEnrollment: f.enUK} if _, err := hostclaim.Update(ctx, data, o, hostclaim.BeginAdmittedMove(moveStartID, "operator", "ie-box", "uk-box", "uk", f.now, admitted)); !errors.Is(err, ErrPending) { t.Fatalf("unbound start CAS: %v", err) } if f.raw.claimPuts != 0 || f.read(t).ETag != o.ETag { t.Fatal("unbound start CAS reached the claim") } moving, err := BeginClaimMove(ctx, data, dir, h, o, moveStartID, "operator", "uk-box", "uk", f.now) if err != nil { t.Fatal(err) } if f.raw.claimPuts != 1 || moving.Claim.Op.Kind != hostclaim.KindMoving || f.read(t).ETag != moving.ETag { t.Fatalf("granted start CAS: puts=%d op=%+v", f.raw.claimPuts, moving.Claim.Op) } start := provision.MoveStart{ID: moveStartID, Incarnation: f.inc, From: "ie", To: "uk", Cut: "auto", By: "operator", Now: f.now} before := recordTag() for name, c := range map[string]context.Context{"no-grant": ctx, "grant-without-effect": h.Lifecycle(ctx)} { if _, err := setLifecycleRecordMove(c, data, moving, provision.MoveStartTransition(start)); !errors.Is(err, ErrPending) { t.Fatalf("%s: unbound record start: %v", name, err) } } if recordTag() != before { t.Fatal("unbound record start wrote the record") } rec, err := StartClaimMoveRecord(ctx, data, h, moving, start) if err != nil { t.Fatal(err) } if rec.Move == nil || rec.Move.ID != moveStartID || rec.Move.State != provision.MovePreparing { t.Fatalf("record not started: %+v", rec.Move) } a, err := ReadAdmission(ctx, f.raw, "bab", "app", "canonical") if err != nil { t.Fatal(err) } if e := a.Value.Effects[moveStartID]; e.State != "complete" || e.OperationID != moveStartID || e.Resource != f.raw.recordKey || e.Owner != h.Owner() { t.Fatalf("record start effect: %+v", e) } // A replay of the completed effect writes nothing and reads the result. started := recordTag() if rec, err = StartClaimMoveRecord(ctx, data, h, moving, start); err != nil || rec.Move.ID != moveStartID || recordTag() != started { t.Fatalf("replay: %v", err) } }