package hostops import ( "context" "crypto/ed25519" "crypto/rand" "crypto/sha256" "encoding/hex" "encoding/json" "errors" "strings" "testing" "time" "github.com/carlosframework/platform/internal/boxid" "github.com/carlosframework/platform/internal/hostclaim" "github.com/carlosframework/platform/internal/provision" "github.com/carlosframework/platform/internal/release" "github.com/carlosframework/platform/internal/termfence" ) type c7ReplayAuthority struct { home string record termfence.Record region string } func (a c7ReplayAuthority) CheckOneBoxRegion(_ context.Context, source, region string) error { if source != a.record.Box || region != a.region { return hostclaim.ErrFence } return nil } func (a c7ReplayAuthority) ValidateFence(_ context.Context, f termfence.Record) (hostclaim.FenceBinding, error) { if f != a.record { return hostclaim.FenceBinding{}, hostclaim.ErrFence } return hostclaim.FenceBinding{FleetHomeRegion: a.home, Region: a.region, RosterRevision: 1, RosterETag: "roster"}, nil } type c7DecisionHook struct { release.Store key string after func() error } func (s *c7DecisionHook) Put(ctx context.Context, key string, b []byte, match string) (string, error) { tag, err := s.Store.Put(ctx, key, b, match) if err == nil && key == s.key && s.after != nil { f := s.after s.after = nil if e := f(); e != nil { return "", e } } return tag, err } type c7World struct { raw release.Store host, key string inc string ref hostclaim.FenceRef authority c7ReplayAuthority now time.Time clock time.Time // the ControlWriter's clock pub ed25519.PublicKey control *ControlWriter mh *StartHandle // the move registration's handle, as the Moves coordinator holds it } var c7Owner = RegistrationOwner{Console: "console", Enrollment: strings.Repeat("e", 32), Controller: "coordinator"} // newC7World is a moving claim with the intent BeginClaimMove writes, its // record in cutting/archived, the exact move registration (owned by // c7Owner, live) with its handle, and the source's termination fence. func newC7World(t *testing.T, mutate func(*hostclaim.Claim, *provision.Record, *AppAdmission)) *c7World { t.Helper() ctx := context.Background() w := &c7World{raw: release.NewDirStore(t.TempDir()), host: "app.example", now: time.Now().UTC()} w.clock = w.now pub, _, err := ed25519.GenerateKey(rand.Reader) if err != nil { t.Fatal(err) } w.pub = pub if w.control, err = NewControlWriter(w.raw, pub, func() time.Time { return w.clock }); err != nil { t.Fatal(err) } // In-package harness: the handle a RegisterMove call returned for the // seeded registration below. w.mh = &StartHandle{w: w.control, account: "bab", app: "app", appID: "canonical", id: "move", owner: c7Owner, epoch: 1} en := strings.Repeat("a", 32) if w.inc, err = hostclaim.Incarnation("bab", "app", w.host, w.now); err != nil { t.Fatal(err) } w.key = provision.RecordKey("bab", "app", w.host) c := hostclaim.Claim{V: 2, Host: w.host, Gen: 1, Holder: &hostclaim.Holder{Family: "instance", Account: "bab", App: "app", Key: w.key, Acquisition: "acq", Incarnation: w.inc, CreatedAt: w.now, Location: "instances/" + w.host, Stamp: 7, Generation: "generation"}, Op: hostclaim.Op{Kind: hostclaim.KindMoving, ID: "move", Phase: "releasing", Exec: "source", Participants: []string{"source", "target"}, Intent: map[string]string{"to": "uk", "source": "source", "target": "target", "app_id": "canonical", "admission_id": "move", "source_enrollment": en, "target_enrollment": strings.Repeat("b", 32)}}, Obligations: []hostclaim.Obligation{{ID: "ack", Op: "move", Kind: "park-local", Box: "target", Owner: "box:target", Retirement: "retire"}, {ID: "keep", Op: "move", Kind: "park-replica", Region: "uk", Owner: "region:uk", Retirement: "retire-region"}}} rec := provision.Record{Schema: 1, Host: w.host, Slug: "app", Account: "bab", App: "app", Acquisition: "acq", Channel: "edge", Region: "ie", CreatedAt: w.now, Move: &provision.MoveState{ID: "move", Incarnation: w.inc, From: "ie", To: "uk", State: provision.MoveCutting, CutPhase: provision.CutArchived}} adm := AppAdmission{V: 1, AppID: "canonical", Epoch: 1, Revision: 1, State: "open", Effects: map[string]Effect{}, Registrations: map[string]Registration{"move": {Kind: RegistrationMove, Host: w.host, Owner: c7Owner, AppID: "canonical", Epoch: 1, Claim: SavedClaim{ETag: "serving", Gen: 1}, Expires: w.now.Add(RegistrationTTL), CreatedAt: w.now, Incarnation: w.inc}}} if mutate != nil { mutate(&c, &rec, &adm) } cb, _ := json.Marshal(c) ck, _ := hostclaim.Key(w.host) if _, err = w.raw.Put(ctx, ck, cb, ""); err != nil { t.Fatal(err) } rb, _ := json.Marshal(rec) if _, err = w.raw.Put(ctx, w.key, rb, ""); err != nil { t.Fatal(err) } ab, _ := json.Marshal(adm) if _, err = w.raw.Put(ctx, "apps/bab/app/admission.json", ab, ""); err != nil { t.Fatal(err) } f := termfence.Record{V: 1, Box: "source", Enrollment: en, Cloud: boxid.Cloud{Provider: "aws", Account: "123", Region: "eu-west-1", InstanceID: "i-source"}, State: "terminated", ObservedAt: w.now.Format(time.RFC3339), Via: "describe", RequestID: "request", WrittenBy: termfence.WrittenBy{Box: "worker", Enrollment: strings.Repeat("c", 32)}} fb, _ := json.Marshal(f) sum := sha256.Sum256(fb) w.ref = hostclaim.FenceRef{Enrollment: en, Key: "control/box-fences/" + en + ".json", SHA256: hex.EncodeToString(sum[:])} if _, err = w.raw.Put(ctx, w.ref.Key, fb, ""); err != nil { t.Fatal(err) } w.authority = c7ReplayAuthority{record: f, region: "ie"} return w } func (w *c7World) claim(t *testing.T) hostclaim.Observed { t.Helper() o, err := hostclaim.Read(context.Background(), w.raw, w.host) if err != nil { t.Fatal(err) } return o } // productionData is ConfigureClaimRuntime's data handle: the mutation // boundary over the conditional record store over under (w.raw or a fault // hook on it), checked by the production checker with the real ControlWriter. func (w *c7World) productionData(t *testing.T, under release.Store) release.Store { t.Helper() records, err := NewConditionalRecordStore(under, w.pub, w.control.homeRegion) if err != nil { t.Fatal(err) } check, err := NewProductionMutationCheck(w.raw, w.control) if err != nil { t.Fatal(err) } data, err := NewMutationBoundary(true, records, check) if err != nil { t.Fatal(err) } return data } func (w *c7World) effects(t *testing.T, action string) []Effect { t.Helper() a, err := ReadAdmission(context.Background(), w.raw, "bab", "app", "canonical") if err != nil { t.Fatal(err) } var out []Effect for _, e := range a.Value.Effects { if action == "" || e.Action == action { out = append(out, e) } } return out } func (w *c7World) recordTag(t *testing.T) string { t.Helper() _, tag, err := provision.GetRecordETag(context.Background(), w.raw, "bab", "app", w.host) if err != nil { t.Fatal(err) } return tag } // A correctly held call through the production data handle: the record // decision is one issued effect of the move registration, settled complete, // and the conversion CAS passes the boundary under its lifecycle grant. func TestC7RecoveryHeldCallSettlesItsDecisionEffect(t *testing.T) { ctx := context.Background() w := newC7World(t, nil) data := w.productionData(t, w.raw) if err := RecoverTerminatedMovingHost(ctx, data, w.mh, w.authority, w.host, w.ref, w.now); err != nil { t.Fatal(err) } if c := w.claim(t).Claim; c.Op.Kind != hostclaim.KindRelocating || c.Op.Intent["source_box"] != "source" || c.Op.Intent["region"] != "ie" { t.Fatalf("not converted: %+v", c.Op) } d := w.effects(t, "move-relocating-decision") if len(d) != 1 || d[0].State != "complete" || d[0].ResultReference != "move" || d[0].OperationID != "move" || d[0].Owner != c7Owner || d[0].Resource != w.key { t.Fatalf("decision effect: %+v", d) } } // RecoverTerminatedMovingHost through the production data handle, held by a // writer-minted move registration. Unbound calls refuse before the record // decision; the data handle alone writes neither the decision nor the // conversion; a crash after the durable decision replays with no second // decision and keeps a chore acknowledged in between. func TestC7RecoveryThroughProductionBoundary(t *testing.T) { ctx := context.Background() w := newC7World(t, func(_ *hostclaim.Claim, _ *provision.Record, adm *AppAdmission) { delete(adm.Registrations, "move") }) o := w.claim(t) reg := Registration{Kind: RegistrationMove, Host: w.host, Owner: c7Owner, AppID: "canonical", Epoch: 1, Claim: SavedObservation(o), Incarnation: w.inc, CreatedAt: w.now} mh, err := w.control.RegisterStart(ctx, "bab", "app", "canonical", "move", reg, w.clock) if err != nil { t.Fatal(err) } reg.Kind = "wake" other, err := w.control.RegisterStart(ctx, "bab", "app", "canonical", "wake-1", reg, w.clock) if err != nil { t.Fatal(err) } hook := &c7DecisionHook{Store: w.raw, key: w.key} data := w.productionData(t, hook) claimBefore, recordBefore := o.ETag, w.recordTag(t) unchanged := func(name string) { t.Helper() if w.claim(t).ETag != claimBefore || w.recordTag(t) != recordBefore || len(w.effects(t, "")) != 0 { t.Fatalf("%s: wrote before refusing", name) } } forged := &StartHandle{w: w.control, account: "bab", app: "app", appID: "canonical", id: "move", owner: RegistrationOwner{Console: "console", Enrollment: strings.Repeat("e", 32), Controller: "forged"}, epoch: 1} for _, tc := range []struct { name string h *StartHandle }{{"no-handle", nil}, {"other-registration", other}, {"forged-owner", forged}, {"expired", mh}} { if tc.name == "expired" { w.clock = w.now.Add(RegistrationTTL) } err := RecoverTerminatedMovingHost(ctx, data, tc.h, w.authority, w.host, w.ref, w.now) w.clock = w.now if err == nil { t.Fatalf("%s: recovered without the move registration", tc.name) } unchanged(tc.name) } // The data handle alone: no record decision without an issued effect // (a lifecycle grant is not one), and no claim CAS without a grant. moving, holder, err := movingIdentities(o.Claim) if err != nil { t.Fatal(err) } fence, _ := json.Marshal(w.ref) decide := provision.MoveRelocatingDecision(provision.MoveGuard{ID: "move", Incarnation: w.inc}, "source", w.ref.Enrollment, "ie", string(fence), o.Claim.Gen, o.ETag, moving, holder, w.now) for name, c := range map[string]context.Context{"no-grant": ctx, "grant-without-effect": mh.Lifecycle(ctx)} { if _, err := setLifecycleRecordMove(c, data, o, decide); !errors.Is(err, ErrPending) { t.Fatalf("%s: unbound record decision: %v", name, err) } } if _, err := hostclaim.Update(ctx, data, o, func(*hostclaim.Claim) error { return nil }); !errors.Is(err, ErrPending) { t.Fatalf("ungranted claim CAS: %v", err) } unchanged("data handle alone") // Held: the decision lands, then its response is lost. hook.after = func() error { return errors.New("crash: response lost after the durable record decision") } if err := RecoverTerminatedMovingHost(ctx, data, mh, w.authority, w.host, w.ref, w.now); err == nil { t.Fatal("missing crash") } if w.claim(t).ETag != claimBefore { t.Fatal("converted despite the crash") } rec, err := provision.GetRecord(ctx, w.raw, "bab", "app", w.host) if err != nil { t.Fatal(err) } if rec.Move.State != provision.MoveRelocating || !rec.Move.Relocation.DecidedAt.Equal(w.now) { t.Fatalf("decision not durable: %+v", rec.Move) } // The write reached the store and its outcome was lost: uncertain, never // "not-applied". if d := w.effects(t, "move-relocating-decision"); len(d) != 1 || d[0].State != "uncertain" || d[0].OperationID != "move" || d[0].Owner != c7Owner { t.Fatalf("decision effect after the crash: %+v", d) } if _, err := hostclaim.ConvertMovingToRelocating(ctx, data, w.claim(t), "move", w.ref, w.authority, movingRecordDecision{store: data, placement: w.authority}); !errors.Is(err, ErrPending) { t.Fatalf("conversion without the registration's grant: %v", err) } // Seeded between the runs: the target box acknowledges its chore (its // own write, not this console's data handle). if _, err := hostclaim.Update(ctx, w.raw, w.claim(t), hostclaim.CompleteObligation("move", "ack")); err != nil { t.Fatal(err) } if err := RecoverTerminatedMovingHost(ctx, data, mh, w.authority, w.host, w.ref, w.now.Add(time.Minute)); err != nil { t.Fatal(err) } c := w.claim(t).Claim if c.Op.Kind != hostclaim.KindRelocating || c.Holder.Stamp != 7 || len(c.Obligations) != 1 || c.Obligations[0].ID != "keep" || c.Op.Journal["admission_id"] != "move" { t.Fatalf("replay lost the acknowledgement or the contract: %+v", c) } if rec, err = provision.GetRecord(ctx, w.raw, "bab", "app", w.host); err != nil || !rec.Move.Relocation.DecidedAt.Equal(w.now) { t.Fatalf("replay redecided: %v %+v", err, rec.Move) } if d := w.effects(t, "move-relocating-decision"); len(d) != 1 { t.Fatalf("replay issued a second decision: %+v", d) } converted := w.claim(t).ETag if err := RecoverTerminatedMovingHost(ctx, data, mh, w.authority, w.host, w.ref, w.now.Add(time.Hour)); err != nil { t.Fatal("converted replay", err) } if err := RecoverTerminatedMovingHost(ctx, data, nil, w.authority, w.host, w.ref, w.now.Add(time.Hour)); err == nil { t.Fatal("unbound converted replay") } if w.claim(t).ETag != converted { t.Fatal("converted replay wrote the claim") } } func TestC7DecisionReplayPreservesInterveningChoreAcknowledgement(t *testing.T) { for _, changeOperation := range []bool{false, true} { t.Run(map[bool]string{false: "chore", true: "changed-operation"}[changeOperation], func(t *testing.T) { ctx := context.Background() w := newC7World(t, nil) hook := &c7DecisionHook{Store: w.raw, key: w.key} hook.after = func() error { o, e := hostclaim.Read(ctx, w.raw, w.host) if e != nil { return e } _, e = hostclaim.Update(ctx, w.raw, o, func(next *hostclaim.Claim) error { if changeOperation { next.Op.Intent["to"] = "de" return nil } return hostclaim.CompleteObligation("move", "ack")(next) }) if e != nil { return e } return errors.New("crash after durable record decision") } wrapped := conditionalRecordStore{raw: hook} if err := RecoverTerminatedMovingHost(ctx, wrapped, w.mh, w.authority, w.host, w.ref, w.now); err == nil { t.Fatal("missing crash") } before := w.claim(t) err := RecoverTerminatedMovingHost(ctx, wrapped, w.mh, w.authority, w.host, w.ref, w.now.Add(time.Minute)) after := w.claim(t) if changeOperation { if err == nil || after.ETag != before.ETag { t.Fatal("changed operation converted", err) } return } if err != nil { t.Fatal(err) } c := after.Claim if c.Op.Kind != hostclaim.KindRelocating || c.Op.Exec != "none" || c.Holder.Stamp != 7 || c.Holder.Generation != "generation" || len(c.Obligations) != 1 || c.Obligations[0].ID != "keep" { t.Fatalf("lost durable acknowledgement or holder: %+v", c) } if c.Op.Intent["source_box"] != "source" || c.Op.Intent["region"] != "ie" || c.Op.Intent["move_to_region"] != "uk" || c.Op.Journal["admission_id"] != "move" || c.Op.Journal["conversion_decision_sha256"] == "" { t.Fatalf("conversion lost its contract: %+v", c.Op) } rec, err := provision.GetRecord(ctx, w.raw, "bab", "app", w.host) if err != nil { t.Fatal(err) } if rec.Move.State != provision.MoveRelocating || rec.Move.Relocation.SourceBox != "source" || rec.Move.Relocation.Region != "ie" || !rec.Move.Relocation.DecidedAt.Equal(w.now) { t.Fatalf("record decision: %+v", rec.Move) } if err = RecoverTerminatedMovingHost(ctx, wrapped, w.mh, w.authority, w.host, w.ref, w.now.Add(time.Hour)); err != nil { t.Fatal("converted replay", err) } saved := w.claim(t) if _, e := hostclaim.Update(ctx, w.raw, saved, func(c *hostclaim.Claim) error { c.Op.Journal["conversion_decision_sha256"] = strings.Repeat("f", 64) return nil }); e != nil { t.Fatal(e) } if err = RecoverTerminatedMovingHost(ctx, wrapped, w.mh, w.authority, w.host, w.ref, w.now.Add(2*time.Hour)); err == nil { t.Fatal("unbound converted claim replayed") } }) } } // Every refusal leaves both the record and the claim as they were. The // region is the record's and must equal the fence binding; the claim's // intent is never a region input. func TestC7RecoveryRefusesBeforeAnyWrite(t *testing.T) { for _, tc := range []string{"fence-region", "record-region", "claim-region-ignored", "no-admission", "admission-kind", "admission-epoch", "admission-incarnation", "missing-admission-id", "admission-not-op", "legacy-source-box", "wrong-enrollment", "flipped"} { t.Run(tc, func(t *testing.T) { ctx := context.Background() w := newC7World(t, func(c *hostclaim.Claim, rec *provision.Record, adm *AppAdmission) { reg := adm.Registrations["move"] switch tc { case "record-region": rec.Region, rec.Move.From = "fr", "fr" case "claim-region-ignored": // A pre-C7 claim region cannot steer the decision. c.Op.Intent["source_region"] = "fr" case "no-admission": delete(adm.Registrations, "move") case "admission-kind": reg.Kind = "create" case "admission-epoch": adm.Epoch = 2 case "admission-incarnation": reg.Incarnation = strings.Repeat("9", 64) case "missing-admission-id": delete(c.Op.Intent, "admission_id") case "admission-not-op": // A live move registration under another id is drainable. c.Op.Intent["admission_id"] = "other" adm.Registrations["other"] = reg case "legacy-source-box": c.Op.Intent["source_box"] = c.Op.Intent["source"] delete(c.Op.Intent, "source") case "wrong-enrollment": c.Op.Intent["source_enrollment"] = strings.Repeat("d", 32) case "flipped": rec.Region, rec.Move.State, rec.Move.CutPhase = "uk", provision.MoveLanding, provision.CutFlipped } if _, ok := adm.Registrations["move"]; ok { adm.Registrations["move"] = reg } }) if tc == "fence-region" { w.authority.region = "uk" } claimBefore := w.claim(t) _, recordTag, err := provision.GetRecordETag(ctx, w.raw, "bab", "app", w.host) if err != nil { t.Fatal(err) } err = RecoverTerminatedMovingHost(ctx, conditionalRecordStore{raw: w.raw}, w.mh, w.authority, w.host, w.ref, w.now) if tc == "claim-region-ignored" { if err != nil { t.Fatal("a stale claim region blocked a valid conversion", err) } return } if err == nil { t.Fatal("accepted") } _, recordAfter, _ := provision.GetRecordETag(ctx, w.raw, "bab", "app", w.host) if w.claim(t).ETag != claimBefore.ETag || recordAfter != recordTag { t.Fatal("refusal wrote the claim or the record") } }) } } func TestC7TargetParkedAtWaitsForRegionAndBoxChores(t *testing.T) { for _, kind := range []string{"none", "box", "region", "participant", "executor", "region-mismatch"} { t.Run(kind, func(t *testing.T) { w := newC7World(t, func(c *hostclaim.Claim, r *provision.Record, _ *AppAdmission) { c.Op.Kind, c.Op.Phase, c.Op.Exec = hostclaim.KindRestoring, hostclaim.PhaseReleased, "none" c.Op.Participants = []string{"source"} c.Lost = []string{"target"} c.Obligations = nil switch kind { case "box": c.Obligations = []hostclaim.Obligation{{ID: "local", Op: "move", Kind: "park-local", Box: "target", Owner: "box:target", Retirement: "r"}} case "region": c.Obligations = []hostclaim.Obligation{{ID: "region", Op: "move", Kind: "park-replica", Region: "uk", Owner: "region:uk", Retirement: "r"}} case "participant": c.Op.Participants = append(c.Op.Participants, "target") case "executor": c.Op.Exec = "target" case "region-mismatch": c.Op.Intent["to"] = "ie" } r.Move.State = provision.MoveRollingBack r.Move.Rollback = &provision.MoveRollback{ID: "rb_abcdefghijklmnopqrstuvwxyz", TargetPhase: provision.RbTargetReleased} }) stamp := func() error { return w.control.RunRecovery(t.Context(), func(ctx context.Context) error { return runMoveRecovery(ctx, w.claim(t), func(ctx context.Context, g *MoveRecoveryGrant) error { return g.StampLostMoveTargetParked(ctx) }) }) } err := stamp() if kind != "none" { if !errors.Is(err, ErrPending) { t.Fatalf("pending: %v", err) } } else if err != nil { t.Fatal(err) } after, err := provision.GetRecord(t.Context(), w.raw, "bab", "app", w.host) if err != nil { t.Fatal(err) } if (after.Move.Rollback.TargetParkedAt != nil) != (kind == "none") { t.Fatal("wrong park stamp") } if kind == "none" { tag := w.recordTag(t) w.clock = w.now.Add(time.Hour) if err := stamp(); err != nil { t.Fatal(err) } if w.recordTag(t) != tag { t.Fatal("changed original park time") } } }) } }