package console import ( "context" "encoding/json" "errors" "fmt" "net/http" "reflect" "strings" "testing" "time" "github.com/carlosframework/platform/internal/hostactors" "github.com/carlosframework/platform/internal/hostclaim" "github.com/carlosframework/platform/internal/moves" "github.com/carlosframework/platform/internal/objstore" "github.com/carlosframework/platform/internal/provision" "github.com/carlosframework/platform/internal/release" "github.com/carlosframework/platform/internal/svcrole" ) // The residence-move doors (issue #331): the console writes guarded // transitions and serial bumps; the state machine's 400/409 matrix is the // tenant's whole interface, so every cell is pinned. func TestAPIInstanceMoveVerbs(t *testing.T) { fx, token := instanceFixture(t, "code-inst-move") m := titoManifest() m.Regions = nil m.Pool = "" // pooled records are refused at the move door — separate rail, pinned below enableInstances(t, fx, "titogo", m) u := instancesURL(fx, "bab", "titogo") put := func(key, body string) { t.Helper() if _, err := fx.rel.Put(context.Background(), key, []byte(body), ""); err != nil { t.Fatal(err) } } // Create BEFORE arming: once uk-box publishes a placement, the #329 // gate would refuse region ie (not armed) — the record predates arming, // exactly like every real record on a fleet that grows regions later. if code, _, out := rawReq(t, http.MethodPost, u, token, `{"host":"acme.go.tito.io","region":"ie"}`); code != http.StatusCreated { t.Fatalf("create = %d %s", code, out) } put("control/edges/uk-box.json", `{"v":1,"box":"uk-box","eip":"1.2.3.4","placement":"uk","data_bucket":"data-uk","data_bucket_region":"eu-west-2","hosts":{}}`) moveURL := u + "/acme.go.tito.io/move" // A target with no staging bucket → 400 naming the set that can take one. // The move door's bar is the target's published data_bucket, because // that is exactly what internal/host's moveTargets demands before a move // can even be prepared. if code, _, out := rawReq(t, http.MethodPost, moveURL, token, `{"region":"se"}`); code != http.StatusBadRequest || !strings.Contains(out, "Regions that can: uk") { t.Fatalf("move to a region with no staging bucket = %d %s, want 400 naming the set", code, out) } // Same region → 409 (the transition's own conflict). if code, _, out := rawReq(t, http.MethodPost, u+"/acme.go.tito.io/move", token, `{"region":"ie"}`); code != http.StatusBadRequest && code != http.StatusConflict { t.Fatalf("move to current region = %d %s, want a refusal", code, out) } else if code == http.StatusBadRequest && !strings.Contains(out, "armed") { // "ie" is not armed either — refused at the armed gate, also fine. _ = out } // Cut before any move → 409. if code, _, out := rawReq(t, http.MethodPost, moveURL+"/cut", token, ""); code != http.StatusConflict { t.Fatalf("cut with no move = %d %s, want 409", code, out) } // The poll door answers "none" before any move — 200, never a 404. code, _, out := rawReq(t, http.MethodGet, moveURL, token, "") if code != http.StatusOK || !strings.Contains(out, `"state":"none"`) { t.Fatalf("move status with no move = %d %s, want 200 none", code, out) } // Happy path: request → the FLAT view (titogo's consumer contract): // state, target, updated_at — never the record. code, _, out = rawReq(t, http.MethodPost, moveURL, token, `{"region":"uk"}`) if code != http.StatusAccepted { t.Fatalf("move request = %d %s, want 202", code, out) } var mv struct { State string `json:"state"` Target string `json:"target"` UpdatedAt int64 `json:"updated_at"` } if err := json.Unmarshal([]byte(out), &mv); err != nil { t.Fatal(err) } if mv.State != provision.MovePreparing || mv.Target != "uk" || mv.UpdatedAt == 0 { t.Fatalf("after request: %+v", mv) } // The record itself still carries the nested truth (and region ie). if code, _, out := rawReq(t, http.MethodGet, u+"/acme.go.tito.io", token, ""); code != http.StatusOK || !strings.Contains(out, `"region":"ie"`) || !strings.Contains(out, provision.MovePreparing) { t.Fatalf("record after request: %d %s", code, out) } // The poll door mirrors it. if code, _, out := rawReq(t, http.MethodGet, moveURL, token, ""); code != http.StatusOK || !strings.Contains(out, `"target":"uk"`) { t.Fatalf("move status mid-move = %d %s", code, out) } // Second request while in flight → 409. if code, _, out := rawReq(t, http.MethodPost, moveURL, token, `{"region":"uk"}`); code != http.StatusConflict { t.Fatalf("second move = %d %s, want 409", code, out) } // Cut from preparing → 409 (only ready cuts). if code, _, out := rawReq(t, http.MethodPost, moveURL+"/cut", token, ""); code != http.StatusConflict { t.Fatalf("cut from preparing = %d %s, want 409", code, out) } // Abort from preparing → 202 aborting (two-phase: the SOURCE box // deletes staging, then clears the field). if code, _, out := rawReq(t, http.MethodDelete, moveURL, token, ""); code != http.StatusAccepted || !strings.Contains(out, provision.MoveAborting) { t.Fatalf("abort = %d %s, want 202 aborting", code, out) } // A second abort while aborting → 409; a new request while aborting → // 409 (still in flight). if code, _, _ := rawReq(t, http.MethodDelete, moveURL, token, ""); code != http.StatusConflict { t.Fatalf("double abort = %d, want 409", code) } if code, _, _ := rawReq(t, http.MethodPost, moveURL, token, `{"region":"uk"}`); code != http.StatusConflict { t.Fatalf("request while aborting = %d, want 409", code) } // The box's half clears it; then a fresh request proceeds. if _, err := provision.SetRecordMove(context.Background(), fx.rel, "bab", "titogo", "acme.go.tito.io", provision.MoveAbortComplete()); err != nil { t.Fatal(err) } code, _, out = rawReq(t, http.MethodGet, u+"/acme.go.tito.io", token, "") if code != http.StatusOK || strings.Contains(out, `"move"`) { t.Fatalf("after abort-complete: %d %s, want no move key", code, out) } if code, _, _ := rawReq(t, http.MethodPost, moveURL, token, `{"region":"uk"}`); code != http.StatusAccepted { t.Fatal("re-request") } if _, err := provision.SetRecordMove(context.Background(), fx.rel, "bab", "titogo", "acme.go.tito.io", provision.MovePromoteReady(time.Now())); err != nil { t.Fatal(err) } if code, _, out := rawReq(t, http.MethodPost, moveURL+"/cut", token, ""); code != http.StatusAccepted { t.Fatalf("cut from ready = %d %s, want 202", code, out) } if code, _, out := rawReq(t, http.MethodDelete, moveURL, token, ""); code != http.StatusConflict { t.Fatalf("abort from cutting = %d %s, want 409 (completes forward)", code, out) } // The serial was bumped by the successful writes. if _, _, err := fx.rel.Get(context.Background(), "control/serial"); err != nil { t.Fatalf("control/serial not written: %v", err) } // 404 discipline: unknown host is the byte-identical not-found. if code, _, _ := rawReq(t, http.MethodPost, u+"/ghost.go.tito.io/move", token, `{"region":"uk"}`); code != http.StatusNotFound { t.Fatalf("move on unknown host = %d, want 404", code) } } // The class rails: only an exec-backed instance route moves. func TestAPIInstanceMoveClassRails(t *testing.T) { fx, token := instanceFixture(t, "code-inst-move-rails") letStateUpstreams(t, fx, "alice@keymail.dev") m := titoManifest() m.Regions = nil m.Pool = "" enableInstances(t, fx, "titogo", m) u := instancesURL(fx, "bab", "titogo") if _, err := fx.rel.Put(context.Background(), "control/edges/uk-box.json", []byte(`{"v":1,"box":"uk-box","eip":"1.2.3.4","placement":"uk","hosts":{}}`), ""); err != nil { t.Fatal(err) } mk := func(body string) string { t.Helper() code, _, out := rawReq(t, http.MethodPost, u, token, body) if code != http.StatusCreated { t.Fatalf("create %s = %d %s", body, code, out) } var rec provision.Record if err := json.Unmarshal([]byte(out), &rec); err != nil { t.Fatal(err) } return rec.Host } for _, tc := range []struct { name, body, wantIn string }{ {"static", `{"host":"st.go.tito.io","kind":"static"}`, "kind static"}, {"addr-backed", `{"host":"ad.go.tito.io","addrs":["10.0.0.1:80"],"kind":"service"}`, "kind service"}, } { t.Run(tc.name, func(t *testing.T) { host := mk(tc.body) code, _, out := rawReq(t, http.MethodPost, u+"/"+host+"/move", token, `{"region":"uk"}`) if code != http.StatusBadRequest || !strings.Contains(out, tc.wantIn) { t.Fatalf("move %s = %d %s, want 400 mentioning %q", tc.name, code, out, tc.wantIn) } }) } // Fleet-placed: region carries fleet/. host := mk(`{"host":"fl.go.tito.io"}`) if err := provision.SetRecordRegion(context.Background(), fx.rel, "bab", "titogo", host, "fleet/pi"); err != nil { t.Fatal(err) } code, _, out := rawReq(t, http.MethodPost, u+"/"+host+"/move", token, `{"region":"uk"}`) if code != http.StatusBadRequest || !strings.Contains(out, "fleet") { t.Fatalf("move fleet-placed = %d %s, want 400 mentioning fleet", code, out) } _ = fmt.Sprint() _ = provisionPooledMoveRefusal(t, fx, token, u) // A replica (G14): refused at the request, while the status door still // answers, since the gate it shares must not strand a replica's move. rep := provision.Record{Schema: 1, Host: "rep.go.tito.io", Slug: "rep", App: "titogo", Account: "bab", Region: "ie", Role: "replica", Channel: "stable", CreatedAt: time.Now().UTC()} if err := provision.CreateRecord(context.Background(), fx.rel, rep); err != nil { t.Fatal(err) } code, _, out = rawReq(t, http.MethodPost, u+"/rep.go.tito.io/move", token, `{"region":"uk"}`) if code != http.StatusBadRequest || !strings.Contains(out, "replica") { t.Fatalf("move replica = %d %s, want 400 mentioning the replica rail", code, out) } if code, _, out = rawReq(t, http.MethodGet, u+"/rep.go.tito.io/move", token, ""); code != http.StatusOK { t.Fatalf("move status of a replica = %d %s, want 200", code, out) } } // provisionPooledMoveRefusal pins the pool rail: a pooled record's data // lives on the pool's box, and v1 refuses the class at the door. func provisionPooledMoveRefusal(t *testing.T, fx *consoleFixture, token, u string) bool { t.Helper() rec := provision.Record{Schema: 1, Host: "pooled.go.tito.io", Slug: "pooled", App: "titogo", Account: "bab", Pool: "dedicated-x", Channel: "stable", CreatedAt: time.Now().UTC()} if err := provision.CreateRecord(context.Background(), fx.rel, rec); err != nil { t.Fatal(err) } code, _, out := rawReq(t, http.MethodPost, u+"/pooled.go.tito.io/move", token, `{"region":"uk"}`) if code != http.StatusBadRequest || !strings.Contains(out, "pool") { t.Fatalf("move pooled = %d %s, want 400 mentioning the pool rail", code, out) } return true } // The armed-regions endpoint (issue #346): codes + armed flags from the // edges files (the create door's own predicate), the home region under its // NAME, and the deployment default — the picker contract. func TestAPIRegions(t *testing.T) { fx, token := instanceFixture(t, "code-inst-regions") m := titoManifest() m.Regions = nil m.Pool = "" enableInstances(t, fx, "titogo", m) base := fx.ts.URL + "/api/cli/apps/bab/titogo/regions" put := func(key, body string) { t.Helper() if _, err := fx.rel.Put(context.Background(), key, []byte(body), ""); err != nil { t.Fatal(err) } } // No named home, no placement-publishing boxes: empty list, empty // default — consumers hide their pickers. code, _, out := rawReq(t, http.MethodGet, base, token, "") if code != http.StatusOK { t.Fatalf("regions = %d %s", code, out) } var rl struct { Regions []struct { Code string `json:"code"` Armed bool `json:"armed"` } `json:"regions"` Default string `json:"default"` } if err := json.Unmarshal([]byte(out), &rl); err != nil { t.Fatal(err) } if len(rl.Regions) != 0 || rl.Default != "" { t.Fatalf("empty deployment answered %+v", rl) } // A named home + one armed box + one EIP-less placement. fx.srv.cfg.HomeRegion = "ie" put("control/edges/uk-box.json", `{"v":1,"box":"uk-box","eip":"1.2.3.4","placement":"uk","hosts":{}}`) put("control/edges/se-box.json", `{"v":1,"box":"se-box","eip":"","placement":"se","hosts":{}}`) code, _, out = rawReq(t, http.MethodGet, base, token, "") if code != http.StatusOK { t.Fatalf("regions = %d %s", code, out) } if err := json.Unmarshal([]byte(out), &rl); err != nil { t.Fatal(err) } want := "[{ie true} {se false} {uk true}]" got := fmt.Sprint(func() (r []struct { Code string Armed bool }) { for _, x := range rl.Regions { r = append(r, struct { Code string Armed bool }{x.Code, x.Armed}) } return }()) if got != want || rl.Default != "ie" { t.Fatalf("regions = %s default=%q, want %s / ie", got, rl.Default, want) } } // The home region's NAME normalizes to the record's "" at the create door // (issue #346): a picker offering "ie" creates exactly what an unconstrained // create always made, and the fleet's scope rules never see the code. func TestAPIInstanceCreateNormalizesHomeRegion(t *testing.T) { fx, token := instanceFixture(t, "code-inst-homenorm") m := titoManifest() m.Regions = nil m.Pool = "" enableInstances(t, fx, "titogo", m) fx.srv.cfg.HomeRegion = "ie" if _, err := fx.rel.Put(context.Background(), "control/edges/uk-box.json", []byte(`{"v":1,"box":"uk-box","eip":"1.2.3.4","placement":"uk","hosts":{}}`), ""); err != nil { t.Fatal(err) } u := instancesURL(fx, "bab", "titogo") // "ie" (the home's name, NOT in the armed edges set) is accepted and // stored as "". code, _, out := rawReq(t, http.MethodPost, u, token, `{"host":"home.go.tito.io","region":"ie"}`) if code != http.StatusCreated { t.Fatalf("create region ie (home name) = %d %s, want 201", code, out) } rec, err := provision.GetRecord(context.Background(), fx.rel, "bab", "titogo", "home.go.tito.io") if err != nil { t.Fatal(err) } if rec.Region != "" { t.Fatalf("record region = %q, want the normalized unconstrained value", rec.Region) } // A move targeting the home name is refused plainly (no staging lane). if code, _, out := rawReq(t, http.MethodPost, u+"/home.go.tito.io/move", token, `{"region":"ie"}`); code != http.StatusBadRequest || !strings.Contains(out, "home region") { t.Fatalf("move to home = %d %s, want the plain refusal", code, out) } } // ---- Phase 1 (CARLOS_MOVES_PHASE1): the operator door and the claim CAS ---- // operatorInstanceFixture is instanceFixture with the token's address an // operator and the Phase 1 switch on. func operatorInstanceFixture(t *testing.T, code string) (*consoleFixture, string) { t.Helper() fx, token := instanceFixture(t, code) if err := fx.st.SetOperator("alice@keymail.dev", true); err != nil { t.Fatal(err) } enableMovesPhase1(fx) return fx, token } // enableMovesPhase1 turns the switch on for a fixture built with it off: // the config field, and the routes NewServer registers only when it is on. func enableMovesPhase1(fx *consoleFixture) { fx.srv.cfg.MovesEnabled = true fx.srv.registerMovesPhase1Routes() } // errorText is a JSON error body's sentence, "" when the body is not one. func errorText(body string) string { var e struct { Error string `json:"error"` } if json.Unmarshal([]byte(body), &e) != nil { return "" } return e.Error } // writeServingClaim writes the host's first claim: serving under box, held // by the titogo instance of incarnation inc, at the root location the // activator's lease lives at (the only one Phase 1 moves, ruling T5-lease). Update validates what it // writes, so an invalid fixture dies here and not in the door. func writeServingClaim(t *testing.T, rel release.Store, host, inc string, created time.Time, box, region string) { t.Helper() _, err := hostclaim.Update(context.Background(), rel, hostclaim.Observed{Host: host, Absent: true}, func(c *hostclaim.Claim) error { c.Holder = &hostclaim.Holder{Family: "instance", Key: provision.RecordKey("bab", "titogo", host), Account: "bab", App: "titogo", Incarnation: inc, CreatedAt: created, Location: "instances/" + host, Stamp: 1} c.Op = hostclaim.Op{Kind: hostclaim.KindServing, Exec: box} c.Touched = hostclaim.Touched{Complete: true, Regions: []string{region}, Boxes: []string{box}} return nil }) if err != nil { t.Fatal(err) } } // claimFor writes acme's serving claim for the record's own incarnation. func claimFor(t *testing.T, fx *consoleFixture, host, box, region string) (inc string) { t.Helper() rec, err := provision.GetRecord(context.Background(), fx.rel, "bab", "titogo", host) if err != nil { t.Fatal(err) } inc, err = hostclaim.Incarnation("bab", "titogo", host, rec.CreatedAt) if err != nil { t.Fatal(err) } writeServingClaim(t, fx.rel, host, inc, rec.CreatedAt, box, region) return inc } func claimBytes(t *testing.T, fx *consoleFixture, host string) string { t.Helper() b, _, err := fx.rel.Get(context.Background(), "control/hosts/"+host+".json") if err != nil { return "" } return string(b) } // The switch-off contract (index D13): with CARLOS_MOVES_PHASE1 off the // door is main's, byte for byte. A member's token still moves, the legacy // sentences keep their wording, no claim is written, and the resolve route // answers the family's 404. func TestMoveDoorSwitchOffIsLegacy(t *testing.T) { fx, token := instanceFixture(t, "code-inst-move-off") m := titoManifest() m.Regions = nil m.Pool = "" enableInstances(t, fx, "titogo", m) u := instancesURL(fx, "bab", "titogo") if code, _, out := rawReq(t, http.MethodPost, u, token, `{"host":"acme.go.tito.io","region":"ie"}`); code != http.StatusCreated { t.Fatalf("create = %d %s", code, out) } if _, err := fx.rel.Put(context.Background(), "control/edges/uk-box.json", []byte(`{"v":1,"box":"uk-box","eip":"1.2.3.4","placement":"uk","data_bucket":"data-uk","hosts":{}}`), ""); err != nil { t.Fatal(err) } moveURL := u + "/acme.go.tito.io/move" if code, _, out := rawReq(t, http.MethodPost, moveURL, token, `{"region":"se"}`); code != http.StatusBadRequest || !strings.Contains(out, "cannot take a move in this deployment \u2014 it publishes no staging bucket") { t.Fatalf("legacy refusal = %d %s, want main's sentence", code, out) } code, _, out := rawReq(t, http.MethodPost, moveURL, token, `{"region":"uk","cut":"manual"}`) if code != http.StatusAccepted || !strings.Contains(out, provision.MovePreparing) { t.Fatalf("member request with the switch off = %d %s, want 202", code, out) } rec, err := provision.GetRecord(context.Background(), fx.rel, "bab", "titogo", "acme.go.tito.io") if err != nil { t.Fatal(err) } if !rec.Move.Legacy() || rec.Move.Cut != "" { t.Fatalf("switch-off move = %+v, want main's id-less journal", rec.Move) } if claimBytes(t, fx, "acme.go.tito.io") != "" { t.Fatal("the legacy door wrote a host claim") } // The transition refusals keep main's bytes, prefix and all. if code, _, out := rawReq(t, http.MethodPost, moveURL+"/cut", token, ""); code != http.StatusConflict || errorText(out) != "provision: move state conflict: cut is only legal from ready" { t.Fatalf("legacy cut from preparing = %d %s, want main's sentence", code, out) } // The resolve route does not exist with the switch off: every way in // answers exactly what an unrouted sibling path answers on main. unrouted := moveURL + "/no-such-door" same := func(what, method, bearer, body string) { t.Helper() wc, wh, wb := rawReq(t, method, unrouted, bearer, body) gc, gh, gb := rawReq(t, method, moveURL+"/resolve", bearer, body) if gc != wc || gb != wb || gh.Get("Content-Type") != wh.Get("Content-Type") || gh.Get("Allow") != wh.Get("Allow") { t.Errorf("%s: resolve = %d %q %q, an unrouted path = %d %q %q", what, gc, gh.Get("Content-Type"), gb, wc, wh.Get("Content-Type"), wb) } } same("unauthenticated POST", http.MethodPost, "", `{"retry":true}`) same("member POST", http.MethodPost, token, `{"retry":true}`) same("member GET", http.MethodGet, token, "") if err := fx.st.SetOperator("alice@keymail.dev", true); err != nil { t.Fatal(err) } same("operator POST", http.MethodPost, token, `{"retry":true}`) same("unauthenticated GET", http.MethodGet, "", "") } // The fleet's refusals (§7.1, §11, D15), each before the claim CAS. func TestMoveDoorRefusesTheFleet(t *testing.T) { floor := `{"min":1,"min_release":"v0.23.0"}` unarmed := ukBox("uk-box") unarmed.FloorUnarmed = true unverified := ukBox("uk-box") unverified.Unverified = true // A box from before moves: its edges file has no moves_protocol, which // decodes as 0, the same as an explicit 0 (§14.3). preProtocol := ukBox("uk-box") preProtocol.NoProtocol = true for _, c := range []struct { name, floor string boxes []fleetBox broken string // a key whose read fails code int want string }{ {"two boxes in the target", "", []fleetBox{ieBox("ie-box"), ukBox("uk-a"), ukBox("uk-b")}, "", http.StatusBadRequest, "uk has more than one box. Moves between regions with several boxes are not supported yet."}, {"a box that has not armed the floor", floor, []fleetBox{ieBox("ie-box"), unarmed}, "", http.StatusConflict, "a box in ie or uk is below the moves protocol floor. roll the platform first"}, {"a pre-protocol box, no floor published", "", []fleetBox{ieBox("ie-box"), preProtocol}, "", http.StatusConflict, "a box in ie or uk is below the moves protocol floor. roll the platform first"}, {"no verified box in the target", "", []fleetBox{ieBox("ie-box"), unverified}, "", http.StatusBadRequest, "no box in uk publishes an edges file yet"}, {"no eligible box in the source", "", []fleetBox{ukBox("uk-box"), {Box: "se-box", Placement: "se", AWS: "eu-north-1", EIP: "1.2.3.5", Bucket: "data-se"}}, "", http.StatusBadRequest, "no eligible box in ie publishes an edges file"}, {"an unreadable edges file", "", []fleetBox{ieBox("ie-box"), ukBox("uk-box"), {Box: "se-box", Placement: "se", AWS: "eu-north-1", EIP: "1.2.3.5", Bucket: "data-se"}}, edgesFilePrefix + "se-box.json", http.StatusConflict, "the fleet's edges files could not all be read. try again"}, {"a floor above this console", `{"min":2,"min_release":"v0.23.0"}`, []fleetBox{ieBox2("ie-box"), ukBox2("uk-box")}, "", http.StatusConflict, "this console is below the moves protocol floor. roll the console first"}, {"an unreadable floor", floor, []fleetBox{ieBox("ie-box"), ukBox("uk-box")}, moves.FloorKey, http.StatusServiceUnavailable, "could not read the moves protocol floor. try again"}, } { t.Run(c.name, func(t *testing.T) { fx, token := operatorInstanceFixture(t, "code-inst-move-fleet") m := titoManifest() m.Regions = nil m.Pool = "" enableInstances(t, fx, "titogo", m) u := instancesURL(fx, "bab", "titogo") if code, _, out := rawReq(t, http.MethodPost, u, token, `{"host":"acme.go.tito.io","region":"ie"}`); code != http.StatusCreated { t.Fatalf("create = %d %s", code, out) } if c.floor != "" { if _, err := fx.rel.Put(context.Background(), moves.FloorKey, []byte(c.floor), ""); err != nil { t.Fatal(err) } } armMoveFleet(t, fx, c.boxes...) claimFor(t, fx, "acme.go.tito.io", "ie-box", "ie") before := claimBytes(t, fx, "acme.go.tito.io") if c.broken != "" { fx.srv.cfg.Releases = failingGet{Store: fx.rel, key: c.broken} } code, _, out := rawReq(t, http.MethodPost, u+"/acme.go.tito.io/move", token, `{"region":"uk"}`) fx.srv.cfg.Releases = fx.rel if code != c.code || !strings.Contains(out, c.want) { t.Fatalf("request = %d %s, want %d %q", code, out, c.code, c.want) } if claimBytes(t, fx, "acme.go.tito.io") != before { t.Fatal("a refused request wrote the claim") } }) } } // Mutations are operator-only (§12, codex 11); a member's token and the // Instance-role token keep the status GET only (M18). func TestMoveMutationsAreOperatorOnly(t *testing.T) { fx, token := instanceFixture(t, "code-inst-move-op") enableMovesPhase1(fx) m := titoManifest() m.Regions = nil m.Pool = "" enableInstances(t, fx, "titogo", m) u := instancesURL(fx, "bab", "titogo") if code, _, out := rawReq(t, http.MethodPost, u, token, `{"host":"acme.go.tito.io","region":"ie"}`); code != http.StatusCreated { t.Fatalf("create = %d %s", code, out) } router := serviceToken(t, fx.st, OpsAccountID, "router", "titogo", svcrole.Instance) moveURL := u + "/acme.go.tito.io/move" want := canonicalNotFound(t) for _, who := range []struct{ name, token string }{{"a member", token}, {"an instance token", router}} { for _, c := range []struct{ method, path, body string }{ {http.MethodPost, moveURL, `{"region":"uk"}`}, {http.MethodPost, moveURL + "/cut", ""}, {http.MethodDelete, moveURL, ""}, {http.MethodPost, moveURL + "/resolve", `{"retry":true}`}, } { code, h, body := rawReq(t, c.method, c.path, who.token, c.body) assertNotFound(t, c.method+" "+c.path+" as "+who.name, want, code, h, body) } if code, _, out := rawReq(t, http.MethodGet, moveURL, who.token, ""); code != http.StatusOK || !strings.Contains(out, `"state":"none"`) { t.Fatalf("status as %s = %d %s", who.name, code, out) } } if claimBytes(t, fx, "acme.go.tito.io") != "" { t.Fatal("a refused mutation wrote a claim") } // An operator reaches the resolve door. A retry with no move in flight // is the guarded transition's own 409; a body that asks for nothing the // door does yet is "nothing to resolve yet". if err := fx.st.SetOperator("alice@keymail.dev", true); err != nil { t.Fatal(err) } if code, _, out := rawReq(t, http.MethodPost, moveURL+"/resolve", token, `{"retry":true}`); code != http.StatusConflict || errorText(out) != "no journaled move in flight" { t.Fatalf("resolve --retry as an operator with no move = %d %s", code, out) } if code, _, out := rawReq(t, http.MethodPost, moveURL+"/resolve", token, `{}`); code != http.StatusConflict || errorText(out) != "nothing to resolve yet" { t.Fatalf("resolve with nothing asked as an operator = %d %s", code, out) } } // Review Focus 4: a refused request writes no claim. func TestMoveDoorRefusesWithoutClaimWrite(t *testing.T) { fx, token := operatorInstanceFixture(t, "code-inst-move-ref") m := titoManifest() m.Regions = []string{"ie", "uk"} m.Pool = "" enableInstances(t, fx, "titogo", m) u := instancesURL(fx, "bab", "titogo") if code, _, out := rawReq(t, http.MethodPost, u, token, `{"host":"acme.go.tito.io","region":"ie"}`); code != http.StatusCreated { t.Fatalf("create = %d %s", code, out) } if _, err := fx.rel.Put(context.Background(), moves.FloorKey, []byte(`{"min":1,"min_release":"v0.23.0"}`), ""); err != nil { t.Fatal(err) } armMoveFleet(t, fx, ieBox("ie-box"), ukBox("uk-box"), fleetBox{Box: "se-box", Placement: "se", AWS: "eu-north-1", EIP: "1.2.3.5", Bucket: "data-se"}) moveURL := u + "/acme.go.tito.io/move" // No claim yet: 409, no claim written. if code, _, out := rawReq(t, http.MethodPost, moveURL, token, `{"region":"uk"}`); code != http.StatusConflict || !strings.Contains(out, "no host claim") { t.Fatalf("claimless = %d %s", code, out) } if claimBytes(t, fx, "acme.go.tito.io") != "" { t.Fatal("a refused request wrote a claim") } inc := claimFor(t, fx, "acme.go.tito.io", "ie-box", "ie") before := claimBytes(t, fx, "acme.go.tito.io") // Current region, a region outside the app's, a bad cut: refused, and // none writes. for _, c := range []struct { body, want string code int }{ {`{"region":"ie"}`, "already in ie", http.StatusConflict}, {`{"region":"se"}`, "region se is not one of this app's regions: ie, uk", http.StatusBadRequest}, {`{"region":"uk","cut":"later"}`, `cut must be \"auto\" or \"manual\"`, http.StatusBadRequest}, } { code, _, out := rawReq(t, http.MethodPost, moveURL, token, c.body) if code != c.code || !strings.Contains(out, c.want) { t.Fatalf("%s = %d %s, want %d %q", c.body, code, out, c.code, c.want) } if claimBytes(t, fx, "acme.go.tito.io") != before { t.Fatalf("%s wrote the claim", c.body) } } // The real thing: the claim goes moving with the id the record carries. code, _, out := rawReq(t, http.MethodPost, moveURL, token, `{"region":"uk","cut":"manual"}`) if code != http.StatusAccepted { t.Fatalf("request = %d %s", code, out) } rec, err := provision.GetRecord(context.Background(), fx.rel, "bab", "titogo", "acme.go.tito.io") if err != nil { t.Fatal(err) } obs, err := hostclaim.Read(context.Background(), fx.rel, "acme.go.tito.io") if err != nil { t.Fatal(err) } op := obs.Claim.Op if op.Kind != hostclaim.KindMoving || op.ID != rec.Move.ID || !moves.ValidOpID(op.ID) || op.Exec != "ie-box" || op.Intent["target"] != "uk-box" || op.By != "alice@keymail.dev" { t.Fatalf("claim op %+v", op) } if rec.Move.Cut != "manual" || rec.Move.Incarnation != inc || rec.Move.From != "ie" || rec.Move.To != "uk" || rec.Move.RequestedBy != "alice@keymail.dev" { t.Fatalf("record move %+v", rec.Move) } // A second request: the claim is not serving, 409 naming the kind. if code, _, out := rawReq(t, http.MethodPost, moveURL, token, `{"region":"uk"}`); code != http.StatusConflict || !strings.Contains(out, "under a moving operation") { t.Fatalf("second request = %d %s", code, out) } } // §7.1 binds the source to the claim's executor (round-5 finding 7), and // the claim's holder must be this incarnation. func TestMoveDoorBindsTheClaimToTheSourceAndTheIncarnation(t *testing.T) { for _, c := range []struct { name, exec, want string otherInc, free bool }{ {name: "an executor that is not the source's eligible box", exec: "old-box", want: "the source box old-box is not the eligible box of ie"}, {name: "another incarnation", exec: "ie-box", otherInc: true, want: "another incarnation"}, {name: "a free claim with no holder", free: true, want: "the instance has no host claim yet. it is adopted on its next reconcile pass"}, } { t.Run(c.name, func(t *testing.T) { fx, token := operatorInstanceFixture(t, "code-inst-move-bind") m := titoManifest() m.Regions = nil m.Pool = "" enableInstances(t, fx, "titogo", m) u := instancesURL(fx, "bab", "titogo") if code, _, out := rawReq(t, http.MethodPost, u, token, `{"host":"acme.go.tito.io","region":"ie"}`); code != http.StatusCreated { t.Fatalf("create = %d %s", code, out) } armMoveFleet(t, fx, ieBox("ie-box"), ukBox("uk-box")) rec, err := provision.GetRecord(context.Background(), fx.rel, "bab", "titogo", "acme.go.tito.io") if err != nil { t.Fatal(err) } created := rec.CreatedAt if c.otherInc { created = created.Add(-time.Hour) } inc, err := hostclaim.Incarnation("bab", "titogo", "acme.go.tito.io", created) if err != nil { t.Fatal(err) } if c.free { if _, err := hostclaim.Update(context.Background(), fx.rel, hostclaim.Observed{Host: "acme.go.tito.io", Absent: true}, func(*hostclaim.Claim) error { return nil }); err != nil { t.Fatal(err) } } else { writeServingClaim(t, fx.rel, "acme.go.tito.io", inc, created, c.exec, "ie") } before := claimBytes(t, fx, "acme.go.tito.io") code, _, out := rawReq(t, http.MethodPost, u+"/acme.go.tito.io/move", token, `{"region":"uk"}`) if code != http.StatusConflict || !strings.Contains(out, c.want) { t.Fatalf("request = %d %s, want 409 %q", code, out, c.want) } if claimBytes(t, fx, "acme.go.tito.io") != before { t.Fatal("a refused request wrote the claim") } if rec, _ := provision.GetRecord(context.Background(), fx.rel, "bab", "titogo", "acme.go.tito.io"); rec.Move != nil { t.Fatalf("a refused request wrote the record: %+v", rec.Move) } }) } } // moveDoorWithClaim is a Phase 1 door fixture that has everything but the // request: the instance in ie, the fleet armed (and enrolled in the roster), // and a serving claim under ie-box. It returns the move URL. func moveDoorWithClaim(t *testing.T, code string) (fx *consoleFixture, token, moveURL string) { t.Helper() fx, token = operatorInstanceFixture(t, code) m := titoManifest() m.Regions = nil m.Pool = "" enableInstances(t, fx, "titogo", m) u := instancesURL(fx, "bab", "titogo") if c, _, out := rawReq(t, http.MethodPost, u, token, `{"host":"acme.go.tito.io","region":"ie"}`); c != http.StatusCreated { t.Fatalf("create = %d %s", c, out) } armMoveFleet(t, fx, ieBox("ie-box"), ukBox("uk-box")) claimFor(t, fx, "acme.go.tito.io", "ie-box", "ie") return fx, token, u + "/acme.go.tito.io/move" } // Without the host claim the door's start is main's plain BeginMove: the op // carries the move's own three keys and nothing else. The admission and // both enrollments are the host-claim door's (TestMoveDoorAdmitsTheMoveUnderTheHostClaim); // a console with no claim runtime has no admission to record, and no C7 // conversion to pin. func TestMoveDoorPlainStartCarriesTheMoveKeys(t *testing.T) { fx, token, moveURL := moveDoorWithClaim(t, "code-inst-move-enr") if c, _, out := rawReq(t, http.MethodPost, moveURL, token, `{"region":"uk","cut":"manual"}`); c != http.StatusAccepted { t.Fatalf("request = %d %s", c, out) } obs, err := hostclaim.Read(context.Background(), fx.rel, "acme.go.tito.io") if err != nil { t.Fatal(err) } want := map[string]string{"to": "uk", "source": "ie-box", "target": "uk-box"} if !reflect.DeepEqual(obs.Claim.Op.Intent, want) { t.Fatalf("claim intent = %v, want %v", obs.Claim.Op.Intent, want) } } // brokenFenceStore fails every read of a termination fence and nothing else. type brokenFenceStore struct{ release.Store } func (b brokenFenceStore) Get(ctx context.Context, key string) ([]byte, string, error) { if strings.HasPrefix(key, "control/box-fences/") { return nil, "", errors.New("secret-detail: fence store down") } return b.Store.Get(ctx, key) } // writeMoveFence stores a termination fence for box, as its own regional // worker would have: observed terminated, for the roster's cloud identity. // A box never witnesses its own termination, so the fence is written by a // separate active peer in the same region and account, which this adds to // the current roster. A body key "written_by" overrides the writer. func writeMoveFence(t *testing.T, fx *consoleFixture, b hostactors.Box, body map[string]any) { t.Helper() ctx := context.Background() raw, _, err := fx.rel.Get(ctx, hostactors.Key) if err != nil { t.Fatal(err) } var roster hostactors.Roster if err := release.VerifyRootEnvelope(fx.pub, raw, &roster); err != nil { t.Fatal(err) } writer := moveRosterBox(fleetBox{Box: "fence-writer-" + b.BoxID, Placement: b.Region, AWS: b.Cloud.Region}) writer.Cloud.Account = b.Cloud.Account roster.Boxes[writer.BoxID] = writer putMoveRoster(t, fx, roster) f := map[string]any{"v": 1, "box": b.BoxID, "enrollment": b.Enrollment, "cloud": b.Cloud, "state": "terminated", "observed_at": time.Now().UTC(), "via": "describe", "request_id": "request", "written_by": map[string]string{"box": writer.BoxID, "enrollment": writer.Enrollment}} for k, v := range body { f[k] = v } raw, err = json.Marshal(f) if err != nil { t.Fatal(err) } if err := release.CreateExact(ctx, fx.rel, "control/box-fences/"+b.Enrollment+".json", raw); err != nil { t.Fatal(err) } } // Ruling E2: from ONE roster observation, a source must be active, carry an // enrollment, and have no termination fence, or the door refuses 409. A // roster or fence that cannot be read is a 503 with a fixed sentence, never // the error's own text. Every refusal writes neither the claim nor the // record. func TestMoveDoorRefusesAnIneligibleSource(t *testing.T) { const ineligible = "the source box is not eligible to move from" const unreadable = "could not read the host roster. try again" ie, uk := ieBox("ie-box"), ukBox("uk-box") rosterWith := func(mut func(b *hostactors.Box)) hostactors.Roster { b := moveRosterBox(ie) mut(&b) return hostactors.Roster{Boxes: map[string]hostactors.Box{"ie-box": b, "uk-box": moveRosterBox(uk)}} } for _, c := range []struct { name string setup func(t *testing.T, fx *consoleFixture) code int want string }{ {"no roster entry for the source", func(t *testing.T, fx *consoleFixture) { putMoveRoster(t, fx, hostactors.Roster{Boxes: map[string]hostactors.Box{"uk-box": moveRosterBox(uk)}}) }, http.StatusConflict, ineligible}, {"an entry for another box only", func(t *testing.T, fx *consoleFixture) { putMoveRoster(t, fx, hostactors.Roster{Boxes: map[string]hostactors.Box{"old-box": moveRosterBox(fleetBox{Box: "old-box", Placement: "ie", AWS: "eu-west-1"})}}) }, http.StatusConflict, ineligible}, {"a retired source", func(t *testing.T, fx *consoleFixture) { putMoveRoster(t, fx, rosterWith(func(b *hostactors.Box) { b.State = "retired" })) }, http.StatusConflict, ineligible}, {"a quarantined source", func(t *testing.T, fx *consoleFixture) { putMoveRoster(t, fx, rosterWith(func(b *hostactors.Box) { b.State, b.QuarantineRevision, b.LossIDs = "quarantined", 1, []string{"loss-1"} })) }, http.StatusConflict, ineligible}, {"a fenced source", func(t *testing.T, fx *consoleFixture) { writeMoveFence(t, fx, moveRosterBox(ie), nil) }, http.StatusConflict, ineligible}, {"a fence that cannot be read", func(t *testing.T, fx *consoleFixture) { fx.srv.cfg.Releases = brokenFenceStore{fx.srv.cfg.Releases} }, http.StatusServiceUnavailable, unreadable}, {"a fence record that does not validate", func(t *testing.T, fx *consoleFixture) { writeMoveFence(t, fx, moveRosterBox(ie), map[string]any{"state": "stopped"}) }, http.StatusServiceUnavailable, unreadable}, {"no roster object", func(t *testing.T, fx *consoleFixture) { if err := fx.rel.Delete(context.Background(), hostactors.Key); err != nil { t.Fatal(err) } }, http.StatusServiceUnavailable, unreadable}, {"a roster that does not verify", func(t *testing.T, fx *consoleFixture) { putMoveRoster(t, fx, hostactors.Roster{Boxes: map[string]hostactors.Box{"ie-box": moveRosterBox(ie)}}) if _, err := fx.rel.Put(context.Background(), hostactors.Key, []byte(`{"secret-detail":"ed25519: bad"}`), mustETag(t, fx, hostactors.Key)); err != nil { t.Fatal(err) } }, http.StatusServiceUnavailable, unreadable}, {"a roster whose source has an empty enrollment", func(t *testing.T, fx *consoleFixture) { putMoveRoster(t, fx, rosterWith(func(b *hostactors.Box) { b.Enrollment = "" })) }, http.StatusServiceUnavailable, unreadable}, {"a roster for another bucket", func(t *testing.T, fx *consoleFixture) { putMoveRoster(t, fx, hostactors.Roster{Boxes: map[string]hostactors.Box{"ie-box": moveRosterBox(ie)}}) fx.srv.cfg.Bucket = "another-bucket" }, http.StatusServiceUnavailable, unreadable}, } { t.Run(c.name, func(t *testing.T) { fx, token, moveURL := moveDoorWithClaim(t, "code-inst-move-enr-closed") c.setup(t, fx) before := claimBytes(t, fx, "acme.go.tito.io") code, _, out := rawReq(t, http.MethodPost, moveURL, token, `{"region":"uk"}`) if code != c.code || errorText(out) != c.want { t.Fatalf("request = %d %s, want %d %q", code, out, c.code, c.want) } if strings.Contains(out, "secret-detail") || strings.Contains(out, "roster authority") { t.Fatalf("the body leaks an error's text: %s", out) } if claimBytes(t, fx, "acme.go.tito.io") != before { t.Fatal("a refused request wrote the claim") } if rec, _ := provision.GetRecord(context.Background(), fx.rel, "bab", "titogo", "acme.go.tito.io"); rec.Move != nil { t.Fatalf("a refused request wrote the record: %+v", rec.Move) } }) } } // The lookup the door uses: by box reference (never by label), active only, // with a non-empty enrollment. A verified roster cannot carry an empty // enrollment, so that case is proved here and answers 503 at the door. func TestSourceEnrollmentOfTheRoster(t *testing.T) { r := hostactors.Roster{Boxes: map[string]hostactors.Box{ "ie-box": {BoxID: "ie-box", Label: "dublin", Enrollment: "e1", State: "active"}, "blank": {BoxID: "blank", State: "active"}, "retired": {BoxID: "retired", Enrollment: "e2", State: "retired"}, "quar": {BoxID: "quar", Enrollment: "e3", State: "quarantined"}, "nostate": {BoxID: "nostate", Enrollment: "e4"}, }} if got, ok := sourceEnrollment(r, "ie-box"); !ok || got != "e1" { t.Fatalf("enrolled source = %q, %v", got, ok) } for _, box := range []string{"blank", "retired", "quar", "nostate", "dublin", "absent", ""} { if got, ok := sourceEnrollment(r, box); ok || got != "" { t.Fatalf("source %q = %q, %v, want none", box, got, ok) } } } func mustETag(t *testing.T, fx *consoleFixture, key string) string { t.Helper() _, etag, err := fx.rel.Get(context.Background(), key) if err != nil { t.Fatal(err) } return etag } // Private storage is the instance's BINDING (round-4 finding 13), not the // app's create-time policy, and both policy reads fail closed. func TestMoveDoorRefusesABoundStoreNotAPolicy(t *testing.T) { fx, token := operatorInstanceFixture(t, "code-inst-move-store") m := titoManifest() m.Regions = nil m.Pool = "" enableInstances(t, fx, "titogo", m) u := instancesURL(fx, "bab", "titogo") if code, _, out := rawReq(t, http.MethodPost, u, token, `{"host":"acme.go.tito.io","region":"ie"}`); code != http.StatusCreated { t.Fatalf("create = %d %s", code, out) } armMoveFleet(t, fx, ieBox("ie-box"), ukBox("uk-box")) claimFor(t, fx, "acme.go.tito.io", "ie-box", "ie") before := claimBytes(t, fx, "acme.go.tito.io") moveURL := u + "/acme.go.tito.io/move" ctx := context.Background() if _, err := objstore.PutIn(ctx, fx.rel, "bab", "titogo", "acme", &objstore.Record{}, ""); err != nil { t.Fatal(err) } rel := fx.srv.cfg.Releases for _, c := range []struct{ name, key string }{ {"the storage binding", objstore.KeyIn("bab", "titogo", "acme")}, {"the manifest", provision.ManifestKey("bab", "titogo")}, } { fx.srv.cfg.Releases = failingGet{Store: rel, key: c.key} code, _, out := rawReq(t, http.MethodPost, moveURL, token, `{"region":"uk"}`) fx.srv.cfg.Releases = rel if code != http.StatusServiceUnavailable { t.Fatalf("%s unreadable = %d %s, want 503", c.name, code, out) } if claimBytes(t, fx, "acme.go.tito.io") != before { t.Fatalf("%s unreadable: the claim was written", c.name) } } // A store under the slug with the policy off: M-C17, no claim written. code, _, out := rawReq(t, http.MethodPost, moveURL, token, `{"region":"uk"}`) if code != http.StatusBadRequest || !strings.Contains(out, "acme.go.tito.io uses private storage. Instances with private storage cannot move yet.") { t.Fatalf("bound store = %d %s, want M-C17", code, out) } if claimBytes(t, fx, "acme.go.tito.io") != before { t.Fatal("a refused request wrote the claim") } // The policy on and no store: admitted. if err := fx.rel.Delete(ctx, objstore.KeyIn("bab", "titogo", "acme")); err != nil { t.Fatal(err) } m.PrivateStorage = true enableInstances(t, fx, "titogo", m) if code, _, out := rawReq(t, http.MethodPost, moveURL, token, `{"region":"uk"}`); code != http.StatusAccepted { t.Fatalf("policy on, no store = %d %s, want 202", code, out) } } // The legacy matrix with the Phase 1 rules: an operator's token, a serving // claim, a production-shaped fleet, guarded cut and abort. func TestAPIInstanceMoveVerbsPhase1(t *testing.T) { fx, token := operatorInstanceFixture(t, "code-inst-move-p1") m := titoManifest() m.Regions = nil m.Pool = "" enableInstances(t, fx, "titogo", m) u := instancesURL(fx, "bab", "titogo") ctx := context.Background() if code, _, out := rawReq(t, http.MethodPost, u, token, `{"host":"acme.go.tito.io","region":"ie"}`); code != http.StatusCreated { t.Fatalf("create = %d %s", code, out) } armMoveFleet(t, fx, ieBox("ie-box"), ukBox("uk-box")) inc := claimFor(t, fx, "acme.go.tito.io", "ie-box", "ie") moveURL := u + "/acme.go.tito.io/move" guard := func() provision.MoveGuard { t.Helper() rec, err := provision.GetRecord(ctx, fx.rel, "bab", "titogo", "acme.go.tito.io") if err != nil || rec.Move == nil { t.Fatalf("record move: %v %+v", err, rec) } return provision.MoveGuard{ID: rec.Move.ID, Incarnation: rec.Move.Incarnation} } // A target with no staging bucket: 400 naming the set that can take one. if code, _, out := rawReq(t, http.MethodPost, moveURL, token, `{"region":"se"}`); code != http.StatusBadRequest || !strings.Contains(out, "it publishes no staging bucket. Regions that can: ie, uk") { t.Fatalf("move to a region with no staging bucket = %d %s", code, out) } // Cut and abort before any move: the guarded transitions' own 409. for _, c := range []struct{ method, path string }{{http.MethodPost, moveURL + "/cut"}, {http.MethodDelete, moveURL}} { if code, _, out := rawReq(t, c.method, c.path, token, ""); code != http.StatusConflict || errorText(out) != "no journaled move in flight" { t.Fatalf("%s %s with no move = %d %s", c.method, c.path, code, out) } } if code, _, out := rawReq(t, http.MethodGet, moveURL, token, ""); code != http.StatusOK || !strings.Contains(out, `"state":"none"`) { t.Fatalf("status with no move = %d %s", code, out) } // Request: the flat view. code, _, out := rawReq(t, http.MethodPost, moveURL, token, `{"region":"uk"}`) if code != http.StatusAccepted || !strings.Contains(out, `"state":"preparing"`) || !strings.Contains(out, `"target":"uk"`) { t.Fatalf("request = %d %s", code, out) } g := guard() if g.Incarnation != inc { t.Fatalf("guard %+v, want incarnation %s", g, inc) } // Second request while in flight, cut from preparing: 409. if code, _, out := rawReq(t, http.MethodPost, moveURL, token, `{"region":"uk"}`); code != http.StatusConflict { t.Fatalf("second request = %d %s", code, out) } if code, _, out := rawReq(t, http.MethodPost, moveURL+"/cut", token, ""); code != http.StatusConflict || errorText(out) != "cut is only legal from ready" { t.Fatalf("cut from preparing = %d %s", code, out) } // Abort from preparing: 202 aborting; again: 409; a request: 409. if code, _, out := rawReq(t, http.MethodDelete, moveURL, token, ""); code != http.StatusAccepted || !strings.Contains(out, provision.MoveAborting) { t.Fatalf("abort = %d %s", code, out) } if code, _, out := rawReq(t, http.MethodDelete, moveURL, token, ""); code != http.StatusConflict || errorText(out) != "already aborting" { t.Fatalf("double abort = %d %s", code, out) } if code, _, _ := rawReq(t, http.MethodPost, moveURL, token, `{"region":"uk"}`); code != http.StatusConflict { t.Fatalf("request while aborting = %d", code) } // The source's half (Task 14): the record clears, the claim is released. releaseMove := func(g provision.MoveGuard) { t.Helper() if _, err := provision.SetRecordMove(ctx, fx.rel, "bab", "titogo", "acme.go.tito.io", provision.MoveGuardedAbortComplete(g, time.Now())); err != nil { t.Fatal(err) } obs, err := hostclaim.Read(ctx, fx.rel, "acme.go.tito.io") if err != nil { t.Fatal(err) } if _, err := hostclaim.Update(ctx, fx.rel, obs, hostclaim.ReleaseMove(g.ID, "ie-box")); err != nil { t.Fatal(err) } } releaseMove(g) // A fresh request proceeds with a new id; ready; cut; abort from // cutting/requested is legal under the guard. if code, _, out := rawReq(t, http.MethodPost, moveURL, token, `{"region":"uk"}`); code != http.StatusAccepted { t.Fatalf("re-request = %d %s", code, out) } g2 := guard() if g2.ID == g.ID { t.Fatal("a second move reused the first move's id") } if _, err := provision.SetRecordMove(ctx, fx.rel, "bab", "titogo", "acme.go.tito.io", provision.MoveGuardedPromoteReady(g2, time.Now(), provision.MoveInputs{})); err != nil { t.Fatal(err) } if code, _, out := rawReq(t, http.MethodPost, moveURL+"/cut", token, ""); code != http.StatusAccepted || !strings.Contains(out, provision.MoveCutting) { t.Fatalf("cut from ready = %d %s", code, out) } if rec, _ := provision.GetRecord(ctx, fx.rel, "bab", "titogo", "acme.go.tito.io"); rec.Move.CutPhase != provision.CutRequested { t.Fatalf("cut phase %q, want requested", rec.Move.CutPhase) } if code, _, out := rawReq(t, http.MethodDelete, moveURL, token, ""); code != http.StatusAccepted || !strings.Contains(out, provision.MoveAborting) { t.Fatalf("abort from cutting/requested = %d %s, want 202", code, out) } releaseMove(g2) // Past the admission: abort is refused with M-C13 (preflight L5), the // cut completes forward or moves back on its own. if code, _, out := rawReq(t, http.MethodPost, moveURL, token, `{"region":"uk"}`); code != http.StatusAccepted { t.Fatalf("third request = %d %s", code, out) } g3 := guard() if _, err := provision.SetRecordMove(ctx, fx.rel, "bab", "titogo", "acme.go.tito.io", func(rec *provision.Record) error { rec.Move.State, rec.Move.CutPhase = provision.MoveCutting, provision.CutAdmitted return nil }); err != nil { t.Fatal(err) } if code, _, out := rawReq(t, http.MethodDelete, moveURL, token, ""); code != http.StatusConflict || errorText(out) != "This move has started its cut. It will finish or move back on its own." { t.Fatalf("abort after admission = %d %s, want 409", code, out) } _ = g3 if _, _, err := fx.rel.Get(ctx, "control/serial"); err != nil { t.Fatalf("control/serial not written: %v", err) } if code, _, _ := rawReq(t, http.MethodPost, u+"/ghost.go.tito.io/move", token, `{"region":"uk"}`); code != http.StatusNotFound { t.Fatalf("move on unknown host = %d, want 404", code) } } // A move journaled by an older console has no id: with the switch on, the // cut names it (MV-N7) and abort takes the legacy transition (F7(a)). func TestMoveDoorLegacyRecordUnderTheSwitch(t *testing.T) { fx, token := operatorInstanceFixture(t, "code-inst-move-leg") m := titoManifest() m.Regions = nil m.Pool = "" enableInstances(t, fx, "titogo", m) u := instancesURL(fx, "bab", "titogo") ctx := context.Background() if code, _, out := rawReq(t, http.MethodPost, u, token, `{"host":"acme.go.tito.io","region":"ie"}`); code != http.StatusCreated { t.Fatalf("create = %d %s", code, out) } if _, err := provision.SetRecordMove(ctx, fx.rel, "bab", "titogo", "acme.go.tito.io", provision.MoveRequest("uk", time.Now())); err != nil { t.Fatal(err) } if _, err := provision.SetRecordMove(ctx, fx.rel, "bab", "titogo", "acme.go.tito.io", provision.MovePromoteReady(time.Now())); err != nil { t.Fatal(err) } moveURL := u + "/acme.go.tito.io/move" // A request while the legacy move is in flight is refused by the // record's own transition, tried before the claim CAS: no claim write. armMoveFleet(t, fx, ieBox("ie-box"), ukBox("uk-box")) claimFor(t, fx, "acme.go.tito.io", "ie-box", "ie") before := claimBytes(t, fx, "acme.go.tito.io") if code, _, out := rawReq(t, http.MethodPost, moveURL, token, `{"region":"uk"}`); code != http.StatusConflict || errorText(out) != "a move to uk is already ready" { t.Fatalf("request over a legacy move = %d %s, want 409", code, out) } if claimBytes(t, fx, "acme.go.tito.io") != before { t.Fatal("a request refused by the record wrote the claim") } if code, _, out := rawReq(t, http.MethodPost, moveURL+"/cut", token, ""); code != http.StatusConflict || !strings.Contains(out, "this move was started by an older console and has no id. abort it and request it again") { t.Fatalf("cut of a legacy move = %d %s, want MV-N7", code, out) } if code, _, out := rawReq(t, http.MethodDelete, moveURL, token, ""); code != http.StatusAccepted || !strings.Contains(out, provision.MoveAborting) { t.Fatalf("abort of a legacy move = %d %s, want 202", code, out) } // A legacy move past its cut: the legacy refusal, reworded under the // switch (no prefix, no em dash). if _, err := provision.SetRecordMove(ctx, fx.rel, "bab", "titogo", "acme.go.tito.io", provision.MoveAbortComplete()); err != nil { t.Fatal(err) } for _, mut := range []func(*provision.Record) error{provision.MoveRequest("uk", time.Now()), provision.MovePromoteReady(time.Now()), provision.MoveMarkCutting()} { if _, err := provision.SetRecordMove(ctx, fx.rel, "bab", "titogo", "acme.go.tito.io", mut); err != nil { t.Fatal(err) } } if code, _, out := rawReq(t, http.MethodDelete, moveURL, token, ""); code != http.StatusConflict || errorText(out) != "past the point of no return. the cut completes forward" { t.Fatalf("abort of a cutting legacy move = %d %s", code, out) } } // interleaveOnGet runs fn once, just before the at'th read of key: a // deterministic stand-in for another writer landing between two reads. type interleaveOnGet struct { release.Store key string at, n int fn func() } func (f *interleaveOnGet) Get(ctx context.Context, key string) ([]byte, string, error) { if key == f.key { f.n++ if f.n == f.at { f.fn() } } return f.Store.Get(ctx, key) } // F7(a)'s fallback is bound to the legacy move the door read: if that move // finished and another began before the write, the abort is refused and // the replacement is left alone. func TestMoveDoorLegacyAbortRefusesAReplacement(t *testing.T) { fx, token := operatorInstanceFixture(t, "code-inst-move-legrep") m := titoManifest() m.Regions = nil m.Pool = "" enableInstances(t, fx, "titogo", m) u := instancesURL(fx, "bab", "titogo") ctx := context.Background() const host = "acme.go.tito.io" if code, _, out := rawReq(t, http.MethodPost, u, token, `{"host":"acme.go.tito.io","region":"ie"}`); code != http.StatusCreated { t.Fatalf("create = %d %s", code, out) } first := time.Now().Add(-time.Hour).UTC() second := first.Add(time.Minute) if _, err := provision.SetRecordMove(ctx, fx.rel, "bab", "titogo", host, provision.MoveRequest("uk", first)); err != nil { t.Fatal(err) } rel := fx.rel // The door's first read sees the first move; before its write reads the // record again, that move is aborted and cleared and a second begins. fx.srv.cfg.Releases = &interleaveOnGet{Store: rel, key: provision.RecordKey("bab", "titogo", host), at: 2, fn: func() { for _, mut := range []func(*provision.Record) error{provision.MoveAbort(), provision.MoveAbortComplete(), provision.MoveRequest("uk", second)} { if _, err := provision.SetRecordMove(ctx, rel, "bab", "titogo", host, mut); err != nil { t.Error(err) } } }} code, _, out := rawReq(t, http.MethodDelete, u+"/"+host+"/move", token, "") fx.srv.cfg.Releases = rel if code != http.StatusConflict || errorText(out) != "the move changed since this request read it. read its state and try again" { t.Fatalf("delayed legacy abort = %d %s, want 409", code, out) } rec, err := provision.GetRecord(ctx, rel, "bab", "titogo", host) if err != nil { t.Fatal(err) } if rec.Move == nil || rec.Move.State != provision.MovePreparing || !rec.Move.RequestedAt.Equal(second) { t.Fatalf("the replacement move = %+v, want it preparing and untouched", rec.Move) } } // recordWriteFault fails the first write of key. With land, the write // reaches the store first, so only the answer is lost. With blind, every // later read of key fails too, so the door cannot tell which happened. type recordWriteFault struct { release.Store key string land, blind bool failed bool } func (f *recordWriteFault) Put(ctx context.Context, key string, body []byte, ifMatch string) (string, error) { if key == f.key && !f.failed { f.failed = true if f.land { if _, err := f.Store.Put(ctx, key, body, ifMatch); err != nil { return "", err } } return "", errors.New("write timeout") } return f.Store.Put(ctx, key, body, ifMatch) } func (f *recordWriteFault) Get(ctx context.Context, key string) ([]byte, string, error) { if key == f.key && f.failed && f.blind { return nil, "", errors.New("read timeout") } return f.Store.Get(ctx, key) } // The claim CAS succeeded and the record write did not: the door rereads // the record and either releases the claim (the move is definitely not // journaled), answers success (it is), or, when it cannot tell, leaves the // claim and promises nothing. func TestMoveDoorCompensatesAFailedRecordWrite(t *testing.T) { const host = "acme.go.tito.io" for _, c := range []struct { name string land, blind bool code int want string // the error sentence; "" for success claimKind string }{ {name: "not written: the claim is released", code: http.StatusBadGateway, want: "could not record the move. try again", claimKind: hostclaim.KindServing}, {name: "written, answer lost: success", land: true, code: http.StatusAccepted, claimKind: hostclaim.KindMoving}, {name: "unknown: the claim is left", blind: true, code: http.StatusBadGateway, want: "could not confirm whether the move was recorded. read its state before trying again", claimKind: hostclaim.KindMoving}, } { t.Run(c.name, func(t *testing.T) { fx, token := operatorInstanceFixture(t, "code-inst-move-comp") m := titoManifest() m.Regions = nil m.Pool = "" enableInstances(t, fx, "titogo", m) u := instancesURL(fx, "bab", "titogo") ctx := context.Background() if code, _, out := rawReq(t, http.MethodPost, u, token, `{"host":"acme.go.tito.io","region":"ie"}`); code != http.StatusCreated { t.Fatalf("create = %d %s", code, out) } armMoveFleet(t, fx, ieBox("ie-box"), ukBox("uk-box")) claimFor(t, fx, host, "ie-box", "ie") rel := fx.rel fault := &recordWriteFault{Store: rel, key: provision.RecordKey("bab", "titogo", host), land: c.land, blind: c.blind} fx.srv.cfg.Releases = fault code, _, out := rawReq(t, http.MethodPost, u+"/"+host+"/move", token, `{"region":"uk"}`) fx.srv.cfg.Releases = rel if !fault.failed { t.Fatal("the fault never fired") } if code != c.code || (c.want != "" && errorText(out) != c.want) { t.Fatalf("request = %d %s, want %d %q", code, out, c.code, c.want) } obs, err := hostclaim.Read(ctx, rel, host) if err != nil { t.Fatal(err) } if obs.Claim.Op.Kind != c.claimKind || obs.Claim.Op.Exec != "ie-box" { t.Fatalf("claim op = %+v, want %s under ie-box", obs.Claim.Op, c.claimKind) } rec, err := provision.GetRecord(ctx, rel, "bab", "titogo", host) if err != nil { t.Fatal(err) } switch c.claimKind { case hostclaim.KindServing: // Released with evidence, and the host can be asked again. if rec.Move != nil || obs.Claim.Op.Journal["released_move"] == "" { t.Fatalf("record move %+v, claim op %+v", rec.Move, obs.Claim.Op) } if code, _, out := rawReq(t, http.MethodPost, u+"/"+host+"/move", token, `{"region":"uk"}`); code != http.StatusAccepted { t.Fatalf("retry after a release = %d %s", code, out) } default: if c.land && (rec.Move == nil || rec.Move.ID != obs.Claim.Op.ID) { t.Fatalf("record move %+v, claim op %+v: want one id", rec.Move, obs.Claim.Op) } } }) } } // The move door's own refusals and the shared gates' sentences it surfaces // (F13): with the switch off, main's bytes; with it on, the Phase 1 // sentences at the move door and main's bytes at every other door. func TestMoveDoorGateSentences(t *testing.T) { const dash = "\u2014" hib := false records := []provision.Record{ {Host: "st.go.tito.io", Slug: "st", Kind: "static"}, {Host: "ad.go.tito.io", Slug: "ad", Addrs: []string{"10.0.0.1:80"}}, {Host: "fl.go.tito.io", Slug: "fl", Region: "fleet/pi"}, {Host: "po.go.tito.io", Slug: "po", Pool: "dedicated-x"}, {Host: "nh.go.tito.io", Slug: "nh", Hibernate: &hib}, } type sentence struct{ host, legacy, phase1 string } rails := []sentence{ {"st.go.tito.io", "only exec-backed instances move " + dash + " this route is kind static", "only exec-backed instances move. this route is kind static"}, {"ad.go.tito.io", "an addr-backed route's process is not platform-managed " + dash + " nothing to move", "an addr-backed route's process is not platform-managed. there is nothing to move"}, {"fl.go.tito.io", "fleet-placed instances keep state-does-not-follow semantics " + dash + " move is for platform regions", "fleet-placed instances keep state-does-not-follow semantics. moves are for platform regions"}, {"po.go.tito.io", "pooled instances do not move in v1 " + dash + " their data lives on the pool's own box", "pooled instances do not move yet. their data lives on the pool's own box"}, {"nh.go.tito.io", "a non-hibernating instance runs outside the activator " + dash + " nothing manages its data, nothing can move it", "a non-hibernating instance runs outside the activator. nothing manages its data, so nothing can move it"}, } const ( limitLegacy = "too many instance requests " + dash + " wait a minute and try again" limitPhase1 = "too many instance requests. wait a minute and try again" bucketLegacy = "no deployment bucket on this host " + dash + " the console is running without one" bucketPhase1 = "this console has no deployment bucket, so it cannot answer this" ) ambiguousLegacy := `account "bab" is ambiguous ` + dash + ` pass --account-id with one of: bab ("ops"), bac ("bab")` ambiguousPhase1 := `account "bab" is ambiguous. pass --account-id with one of: bab ("ops"), bac ("bab")` for _, on := range []bool{false, true} { t.Run(fmt.Sprintf("switch %v", on), func(t *testing.T) { fx, token := operatorInstanceFixture(t, fmt.Sprintf("code-inst-move-copy-%v", on)) if !on { fx.srv.cfg.MovesEnabled = false } ctx := context.Background() for _, rec := range records { rec.Schema, rec.App, rec.Account, rec.Channel, rec.CreatedAt = 1, "titogo", "bab", "stable", time.Now().UTC() if err := provision.CreateRecord(ctx, fx.rel, rec); err != nil { t.Fatal(err) } } u := instancesURL(fx, "bab", "titogo") pick := func(s sentence) string { if on { return s.phase1 } return s.legacy } check := func(what, method, url, body string, wantCode int, want string) { t.Helper() code, _, out := rawReq(t, method, url, token, body) if code != wantCode || errorText(out) != want { t.Errorf("%s = %d %s, want %d %q", what, code, out, wantCode, want) } } // The class rails, on every move verb that runs them. for _, rail := range rails { for _, v := range []struct{ method, suffix, body string }{ {http.MethodPost, "/move", `{"region":"uk"}`}, {http.MethodGet, "/move", ""}, {http.MethodPost, "/move/cut", ""}, {http.MethodDelete, "/move", ""}, } { check(rail.host+" "+v.method+v.suffix, v.method, u+"/"+rail.host+v.suffix, v.body, http.StatusBadRequest, pick(rail)) } } // The shared instance gate: no deployment bucket. rel := fx.srv.cfg.Releases fx.srv.cfg.Releases = nil check("move, no bucket", http.MethodPost, u+"/st.go.tito.io/move", `{"region":"uk"}`, http.StatusServiceUnavailable, pick(sentence{"", bucketLegacy, bucketPhase1})) check("instance GET, no bucket", http.MethodGet, u+"/st.go.tito.io", "", http.StatusServiceUnavailable, bucketLegacy) fx.srv.cfg.Releases = rel // The account resolver: an ambiguous account segment. if err := fx.st.CreateAccount(2, "bac", "bab", "alice@keymail.dev", time.Now()); err != nil { t.Fatal(err) } check("move, ambiguous account", http.MethodPost, u+"/st.go.tito.io/move", `{"region":"uk"}`, http.StatusConflict, pick(sentence{"", ambiguousLegacy, ambiguousPhase1})) check("instance GET, ambiguous account", http.MethodGet, u+"/st.go.tito.io", "", http.StatusConflict, ambiguousLegacy) check("channels GET, ambiguous account", http.MethodGet, fx.ts.URL+"/api/cli/apps/bab/titogo/channels", "", http.StatusConflict, ambiguousLegacy) check("app create, ambiguous account", http.MethodPost, fx.ts.URL+"/api/cli/apps/bab/new", `{"name":"other"}`, http.StatusConflict, ambiguousLegacy) u = u + "?exact=sqid" // The shared instance gate: the rate limit, spent last. for i := 0; i < cliInstanceBurst; i++ { fx.srv.cli.instances.allow(hashToken(token), time.Now()) } base := instancesURL(fx, "bab", "titogo") check("move, rate limited", http.MethodPost, base+"/st.go.tito.io/move?exact=sqid", `{"region":"uk"}`, http.StatusTooManyRequests, pick(sentence{"", limitLegacy, limitPhase1})) check("instance GET, rate limited", http.MethodGet, base+"/st.go.tito.io?exact=sqid", "", http.StatusTooManyRequests, limitLegacy) }) } } // Fix round 2 of Task 5 (ruling T5-lease): Phase 1 cuts only a holder whose // location is the activator's own lease prefix, so the door refuses a // scoped holder (instances//) at request time with the // recorded sentence (MV-N167), before any claim write. It does so whether // or not this console knows the activator's prefix; a root holder passes. func TestMoveDoorRefusesAScopedHolder(t *testing.T) { for _, prefix := range []string{"", "instances"} { t.Run("prefix "+prefix, func(t *testing.T) { fx, token := operatorInstanceFixture(t, "code-inst-move-scoped") fx.srv.cfg.HibernatePrefix = prefix m := titoManifest() m.Regions = nil m.Pool = "" enableInstances(t, fx, "titogo", m) u := instancesURL(fx, "bab", "titogo") if code, _, out := rawReq(t, http.MethodPost, u, token, `{"host":"acme.go.tito.io","region":"ie"}`); code != http.StatusCreated { t.Fatalf("create = %d %s", code, out) } armMoveFleet(t, fx, ieBox("ie-box"), ukBox("uk-box")) inc := claimFor(t, fx, "acme.go.tito.io", "ie-box", "ie") rewriteClaim(t, fx, "acme.go.tito.io", func(c *hostclaim.Claim) { c.Holder.Location = "instances/acme.go.tito.io/" + inc }) before := claimBytes(t, fx, "acme.go.tito.io") code, _, out := rawReq(t, http.MethodPost, u+"/acme.go.tito.io/move", token, `{"region":"uk"}`) if code != http.StatusConflict || errorText(out) != provision.RefusedReplicaLayout { t.Fatalf("a scoped holder = %d %s, want 409 %q", code, out, provision.RefusedReplicaLayout) } if claimBytes(t, fx, "acme.go.tito.io") != before { t.Fatal("a refused request wrote the claim") } rewriteClaim(t, fx, "acme.go.tito.io", func(c *hostclaim.Claim) { c.Holder.Location = "instances/acme.go.tito.io" }) if code, _, out := rawReq(t, http.MethodPost, u+"/acme.go.tito.io/move", token, `{"region":"uk"}`); code != http.StatusAccepted { t.Fatalf("a root holder = %d %s, want 202", code, out) } }) } } // rewriteClaim edits the host's claim in place, as a test fixture. func rewriteClaim(t *testing.T, fx *consoleFixture, host string, edit func(*hostclaim.Claim)) { t.Helper() obs, err := hostclaim.Read(context.Background(), fx.rel, host) if err != nil || obs.Absent { t.Fatalf("claim: %v absent=%v", err, obs.Absent) } if _, err := hostclaim.Update(context.Background(), fx.rel, obs, func(c *hostclaim.Claim) error { edit(c); return nil }); err != nil { t.Fatal(err) } } // Plan 4 Task 1 (ruling J20): the rollback, retry and declare-lost refusals // reach an operator as their recorded sentences, taken from the transitions // themselves, not the fixed sentence. func TestPhase1ConflictPassesPlan4Refusals(t *testing.T) { g := provision.MoveGuard{ID: "mv_abcdefghijklmnopqrstuvwxyz", Incarnation: strings.Repeat("ab", 32)} now := time.Date(2026, 10, 6, 9, 0, 0, 0, time.UTC) const rb = "rb_abcdefghijklmnopqrstuvwxyz" committed := &provision.Record{Region: "uk", Move: &provision.MoveState{ID: g.ID, Incarnation: g.Incarnation, From: "ie", To: "uk", State: provision.MoveCommitted}} notStuck := &provision.Record{Region: "ie", Move: &provision.MoveState{ID: g.ID, Incarnation: g.Incarnation, From: "ie", To: "uk", State: provision.MoveReverting}} cleaned := &provision.Record{Region: "ie", Move: &provision.MoveState{ID: g.ID, Incarnation: g.Incarnation, From: "ie", To: "uk", State: provision.MoveRollingBack, Rollback: &provision.MoveRollback{ID: rb, TargetPhase: provision.RbTargetCleaned}}} for want, err := range map[string]error{ provision.RefusedCommitted: provision.MoveRollBack(g, rb, provision.RollbackLandingDeadline, now)(committed), provision.RefusedNotStuck: provision.MoveRetry(g)(notStuck), provision.RefusedNothingToDeclare: provision.MoveRollbackTargetLost(g, "control/losses/x.json", now)(cleaned), } { if !errors.Is(err, provision.ErrMoveConflict) { t.Fatalf("%q: %v", want, err) } if got := phase1Conflict(err); got != want { t.Fatalf("phase1Conflict(%v) = %q, want %q", err, got, want) } } // Ruling J9 (fix round 1): the four reasons Plan 4 records are spoken as // their recorded sentences too, not the fixed one. for _, reason := range []string{provision.RollbackDeclaredLost, provision.StuckBaselineMismatch, provision.StuckRecoveryProbeAttempts, provision.StuckRevertProbeBudget, provision.StuckReturnFilesOccupied, provision.StuckReturnInstallMismatch, provision.StuckReturnUnknownStamp, provision.StuckReturnTargetDatabase, provision.StuckReturnTargetChanged, provision.StuckReturnTargetFiles} { err := fmt.Errorf("%w: %s", provision.ErrMoveConflict, reason) if got := phase1Conflict(err); got != reason { t.Fatalf("phase1Conflict(%v) = %q, want %q", err, got, reason) } } } // stuckMove is a move journal stuck in the given way, as the box's own // transitions leave one (MoveMarkStuck), for the resolve door to act on. func stuckMove(t *testing.T, fx *consoleFixture, setup func(*provision.MoveState), reason string) provision.MoveGuard { t.Helper() g := provision.MoveGuard{ID: "mv_abcdefghijklmnopqrstuvwxyz", Incarnation: strings.Repeat("ab", 32)} ctx := context.Background() if _, err := provision.SetRecordMove(ctx, fx.rel, "bab", "titogo", "acme.go.tito.io", func(rec *provision.Record) error { rec.Move = &provision.MoveState{ID: g.ID, Incarnation: g.Incarnation, From: "ie", To: "uk", Cut: "auto", RequestedAt: time.Now().UTC()} setup(rec.Move) return nil }); err != nil { t.Fatal(err) } if reason != "" { if _, err := provision.SetRecordMove(ctx, fx.rel, "bab", "titogo", "acme.go.tito.io", provision.MoveMarkStuck(g, reason)); err != nil { t.Fatal(err) } } return g } // resolveRetryBody is the resolve door's retry request. const resolveRetryBody = `{"retry":true}` // TestResolveRetryFromStuckOnly (Plan 4 Task 7, §7.10): the resolve door's // {"retry": true} runs provision.MoveRetry under the guard from the record, // and answers 409 unless the move is stuck. The retry clears the stuck // reason only; the box resumes at the phase StuckAt names. The answer is // the flat view, which carries no reason. func TestResolveRetryFromStuckOnly(t *testing.T) { fx, token := operatorInstanceFixture(t, "code-inst-move-retry") m := titoManifest() m.Regions = nil m.Pool = "" enableInstances(t, fx, "titogo", m) u := instancesURL(fx, "bab", "titogo") ctx := context.Background() if code, _, out := rawReq(t, http.MethodPost, u, token, `{"host":"acme.go.tito.io","region":"ie"}`); code != http.StatusCreated { t.Fatalf("create = %d %s", code, out) } resolveURL := u + "/acme.go.tito.io/move/resolve" const rb = "rb_abcdefghijklmnopqrstuvwxyz" read := func() *provision.Record { t.Helper() rec, err := provision.GetRecord(ctx, fx.rel, "bab", "titogo", "acme.go.tito.io") if err != nil { t.Fatal(err) } return rec } moveBytes := func() string { t.Helper() b, err := json.Marshal(read().Move) if err != nil { t.Fatal(err) } return string(b) } clear := func() { t.Helper() if _, err := provision.SetRecordMove(ctx, fx.rel, "bab", "titogo", "acme.go.tito.io", func(rec *provision.Record) error { rec.Move = nil; return nil }); err != nil { t.Fatal(err) } } serial := func() string { t.Helper() b, _, err := fx.rel.Get(ctx, "control/serial") if err != nil { return "" } return string(b) } // No move in flight: the guarded transition's own 409, nothing written. if code, _, out := rawReq(t, http.MethodPost, resolveURL, token, resolveRetryBody); code != http.StatusConflict || errorText(out) != "no journaled move in flight" { t.Fatalf("retry with no move = %d %s", code, out) } // A move that is not stuck is refused with the recorded sentence and // writes nothing, in every state a move can be in. for _, c := range []struct { name string setup func(*provision.MoveState) }{ {"preparing", func(m *provision.MoveState) { m.State = provision.MovePreparing }}, {"ready", func(m *provision.MoveState) { m.State = provision.MoveReady }}, {"cutting", func(m *provision.MoveState) { m.State, m.CutPhase = provision.MoveCutting, provision.CutAdmitted }}, {"aborting", func(m *provision.MoveState) { m.State = provision.MoveAborting }}, {"landing", func(m *provision.MoveState) { m.State = provision.MoveLanding }}, {"committed", func(m *provision.MoveState) { m.State = provision.MoveCommitted }}, {"rolling back", func(m *provision.MoveState) { m.State, m.Rollback = provision.MoveRollingBack, &provision.MoveRollback{ID: rb, SourcePhase: provision.RbSourceVerified} }}, {"reverting", func(m *provision.MoveState) { m.State = provision.MoveReverting }}, } { stuckMove(t, fx, c.setup, "") before, ser := moveBytes(), serial() if code, _, out := rawReq(t, http.MethodPost, resolveURL, token, resolveRetryBody); code != http.StatusConflict || errorText(out) != provision.RefusedNotStuck { t.Fatalf("retry of a move that is %s = %d %s, want %q", c.name, code, out, provision.RefusedNotStuck) } if moveBytes() != before || serial() != ser { t.Fatalf("a refused retry of a move that is %s wrote", c.name) } clear() } // A body that asks for no retry changes nothing, and a stuck move stays // stuck. A body that is not JSON is the family's 400. stuckMove(t, fx, func(m *provision.MoveState) { m.State, m.Rollback = provision.MoveRollingBack, &provision.MoveRollback{ID: rb, Reason: provision.RollbackLandingDeadline, TargetPhase: provision.RbTargetCleaned, SourcePhase: provision.RbSourceVerified} }, provision.StuckBaselineMismatch) before := moveBytes() for _, body := range []string{`{}`, `{"retry":false}`} { if code, _, out := rawReq(t, http.MethodPost, resolveURL, token, body); code != http.StatusConflict || errorText(out) != "nothing to resolve yet" { t.Fatalf("resolve %s = %d %s", body, code, out) } } if code, _, out := rawReq(t, http.MethodPost, resolveURL, token, `{"retry":`); code != http.StatusBadRequest { t.Fatalf("resolve with a broken body = %d %s", code, out) } if moveBytes() != before { t.Fatal("a resolve that asked for no retry wrote") } clear() // Stuck rollbacks: each returns to the source phase StuckAt names, with // the stuck reason cleared and every other field kept. for _, c := range []struct { name, reason, at string }{ {"a baseline mismatch", provision.StuckBaselineMismatch, provision.RbSourceVerified}, {"a recovery probe out of attempts", provision.StuckRecoveryProbeAttempts, provision.RbSourceProbing}, } { stuckMove(t, fx, func(m *provision.MoveState) { m.State, m.Reason = provision.MoveRollingBack, provision.RollbackLandingDeadline m.Rollback = &provision.MoveRollback{ID: rb, Reason: provision.RollbackLandingDeadline, TargetPhase: provision.RbTargetCleaned, SourcePhase: c.at} }, c.reason) stuck := read().Move if stuck.State != provision.MoveStuck || stuck.StuckReason != c.reason || stuck.Rollback.StuckAt != c.at { t.Fatalf("%s: setup %+v %+v", c.name, stuck, stuck.Rollback) } ser := serial() code, _, out := rawReq(t, http.MethodPost, resolveURL, token, resolveRetryBody) if code != http.StatusAccepted { t.Fatalf("retry after %s = %d %s", c.name, code, out) } var view map[string]any if err := json.Unmarshal([]byte(out), &view); err != nil { t.Fatal(err) } if len(view) != 6 || view["state"] != provision.MoveRollingBack || view["target"] != "uk" || view["updated_at"] == nil || view["id"] != "mv_abcdefghijklmnopqrstuvwxyz" || view["protocol"] != "phase1" || view["phase"] != provision.RbTargetCleaned+"/"+c.at { t.Fatalf("retry after %s answered %s, want the flat view in rolling_back", c.name, out) } got := read().Move want := *stuck want.State, want.StuckReason, want.UpdatedAt = provision.MoveRollingBack, "", got.UpdatedAt rbWant := *stuck.Rollback rbWant.SourcePhase, rbWant.StuckAt = c.at, "" want.Rollback = &rbWant if !reflect.DeepEqual(*got, want) { t.Fatalf("retry after %s:\n got %+v / %+v\nwant %+v / %+v", c.name, *got, *got.Rollback, want, rbWant) } if serial() == ser { t.Fatalf("retry after %s did not bump the serial, so the box would not notice", c.name) } // Not stuck any more: a second retry is refused. if code, _, out := rawReq(t, http.MethodPost, resolveURL, token, resolveRetryBody); code != http.StatusConflict || errorText(out) != provision.RefusedNotStuck { t.Fatalf("second retry after %s = %d %s", c.name, code, out) } clear() } // A stuck revert has no rollback to return to: it goes back to // reverting, and keeps the cause the revert recorded. stuckMove(t, fx, func(m *provision.MoveState) { m.State, m.Reason = provision.MoveReverting, provision.RevertCutDeadline }, provision.StuckRevertProbeBudget) if got := read().Move; got.State != provision.MoveStuck || got.StuckReason != provision.StuckRevertProbeBudget { t.Fatalf("revert setup %+v", got) } code, _, out := rawReq(t, http.MethodPost, resolveURL, token, resolveRetryBody) if code != http.StatusAccepted || !strings.Contains(out, `"state":"reverting"`) { t.Fatalf("retry of a stuck revert = %d %s", code, out) } if got := read().Move; got.State != provision.MoveReverting || got.StuckReason != "" || got.Reason != provision.RevertCutDeadline || got.Rollback != nil { t.Fatalf("a retried revert %+v", got) } } // ---- Plan 6 Task 2: the status door's shape and GET …/move?detail=1 ---- // moveAnswer is a move door's answer as a CLI reads it: the flat view // (preflight L2: id and last; L4: the Phase 1 marker) and, for an operator's // detail=1, the detail object. type moveAnswer struct { State string `json:"state"` Target string `json:"target"` UpdatedAt int64 `json:"updated_at"` ID string `json:"id"` Last *provision.MoveOutcome `json:"last"` Protocol string `json:"protocol"` Detail *struct { Move *provision.MoveState `json:"move"` Claim *struct { Kind string `json:"kind"` Phase string `json:"phase"` Exec string `json:"exec"` Participants []string `json:"participants"` Target string `json:"target"` } `json:"claim"` TargetReport *moves.TargetReport `json:"target_report"` SourceReport *struct { Box string `json:"box"` MoveID string `json:"move_id"` S1 moves.Check `json:"s1"` S2 moves.Check `json:"s2"` Env []struct { Name string `json:"name"` Kind string `json:"kind"` } `json:"env"` } `json:"source_report"` Evidence struct { Source string `json:"source"` Target string `json:"target"` } `json:"evidence"` Log []string `json:"log"` Timings map[string]float64 `json:"timings"` WaitingForBox string `json:"waiting_for_box"` From string `json:"from"` Region string `json:"region"` } `json:"detail"` } func readMoveAnswer(t *testing.T, method, u, token, body string, wantCode int) (moveAnswer, string) { t.Helper() code, _, out := rawReq(t, method, u, token, body) if code != wantCode { t.Fatalf("%s %s = %d %s, want %d", method, u, code, out, wantCode) } var a moveAnswer if err := json.Unmarshal([]byte(out), &a); err != nil { t.Fatalf("%s %s: %v in %s", method, u, err, out) } return a, out } // setMove writes the record's move through the record writer, as a test // fixture (never a direct Put of the record). func setMove(t *testing.T, fx *consoleFixture, mut func(*provision.Record)) { t.Helper() if _, err := provision.SetRecordMove(context.Background(), fx.rel, "bab", "titogo", "acme.go.tito.io", func(rec *provision.Record) error { mut(rec) return nil }); err != nil { t.Fatal(err) } } func recordMove(t *testing.T, fx *consoleFixture) *provision.MoveState { t.Helper() rec, err := provision.GetRecord(context.Background(), fx.rel, "bab", "titogo", "acme.go.tito.io") if err != nil || rec.Move == nil { t.Fatalf("record move: %v", err) } return rec.Move } // Preflight L2 and L4: with the switch on every move answer carries the // move id and the Phase 1 marker, and after a close the flat view carries // the outcome (D11's last), for every caller of the flat view, the // Instance-role token included. func TestMoveViewCarriesTheIDAndTheOutcome(t *testing.T) { fx, token, moveURL := moveDoorWithClaim(t, "code-inst-move-view") router := serviceToken(t, fx.st, OpsAccountID, "router", "titogo", svcrole.Instance) a, out := readMoveAnswer(t, http.MethodGet, moveURL, token, "", http.StatusOK) if a.State != "none" || a.Protocol != "phase1" || a.ID != "" || a.Last != nil || strings.Contains(out, `"detail"`) { t.Fatalf("status before any move = %s", out) } a, out = readMoveAnswer(t, http.MethodPost, moveURL, token, `{"region":"uk","cut":"manual"}`, http.StatusAccepted) id := recordMove(t, fx).ID if a.ID != id || !provision.ValidMoveID(id) || a.Protocol != "phase1" || a.State != provision.MovePreparing { t.Fatalf("request answered %s, want id %s and the marker", out, id) } for _, who := range []string{token, router} { if a, out := readMoveAnswer(t, http.MethodGet, moveURL, who, "", http.StatusOK); a.ID != id || a.Protocol != "phase1" || a.Last != nil { t.Fatalf("status in flight = %s", out) } } if a, out := readMoveAnswer(t, http.MethodDelete, moveURL, token, "", http.StatusAccepted); a.ID != id || a.State != provision.MoveAborting || a.Protocol != "phase1" { t.Fatalf("abort answered %s", out) } at := time.Now().UTC().Truncate(time.Second) if _, err := provision.SetRecordMove(context.Background(), fx.rel, "bab", "titogo", "acme.go.tito.io", provision.MoveGuardedAbortComplete(provision.MoveGuard{ID: id, Incarnation: recordMove(t, fx).Incarnation}, at)); err != nil { t.Fatal(err) } for _, who := range []string{token, router} { a, out := readMoveAnswer(t, http.MethodGet, moveURL, who, "", http.StatusOK) if a.State != "none" || a.ID != "" || a.Last == nil || a.Last.ID != id || a.Last.Outcome != "aborted" || !a.Last.At.Equal(at) || a.Protocol != "phase1" { t.Fatalf("status after the close = %s", out) } } } // With the switch off the status door is main's, byte for byte: no id, no // last, no marker, and detail=1 is not read. func TestMoveViewSwitchOffIsMainsShape(t *testing.T) { fx, token := instanceFixture(t, "code-inst-move-view-off") if err := fx.st.SetOperator("alice@keymail.dev", true); err != nil { t.Fatal(err) } m := titoManifest() m.Regions = nil m.Pool = "" enableInstances(t, fx, "titogo", m) u := instancesURL(fx, "bab", "titogo") if c, _, out := rawReq(t, http.MethodPost, u, token, `{"host":"acme.go.tito.io","region":"ie"}`); c != http.StatusCreated { t.Fatalf("create = %d %s", c, out) } moveURL := u + "/acme.go.tito.io/move" g := provision.MoveGuard{ID: "mv_abcdefghijklmnopqrstuvwxyz", Incarnation: strings.Repeat("ab", 32)} setMove(t, fx, func(rec *provision.Record) { rec.Move = &provision.MoveState{ID: g.ID, Incarnation: g.Incarnation, From: "ie", To: "uk", State: provision.MoveAborting, RequestedAt: time.Now().UTC()} }) if _, err := provision.SetRecordMove(context.Background(), fx.rel, "bab", "titogo", "acme.go.tito.io", provision.MoveGuardedAbortComplete(g, time.Now())); err != nil { t.Fatal(err) } for _, q := range []string{"", "?detail=1"} { code, _, out := rawReq(t, http.MethodGet, moveURL+q, token, "") if code != http.StatusOK || strings.TrimSpace(out) != `{"state":"none"}` { t.Fatalf("switch-off status%s = %d %q, want main's bytes", q, code, out) } } } // M18 (Task 2 rulings): detail=1 is the operator's. A member's token and // the Instance-role token get the flat view, with no detail and no error. func TestMoveDetailIsOperatorOnly(t *testing.T) { fx, token, moveURL := moveDoorWithClaim(t, "code-inst-move-dop") router := serviceToken(t, fx.st, OpsAccountID, "router", "titogo", svcrole.Instance) readMoveAnswer(t, http.MethodPost, moveURL, token, `{"region":"uk","cut":"manual"}`, http.StatusAccepted) id := recordMove(t, fx).ID a, out := readMoveAnswer(t, http.MethodGet, moveURL+"?detail=1", token, "", http.StatusOK) if a.Detail == nil || a.Detail.Move == nil || a.Detail.Move.ID != id || a.Detail.Claim == nil || a.Detail.Claim.Kind != hostclaim.KindMoving || a.Detail.Claim.Exec != "ie-box" || a.Detail.Claim.Target != "uk-box" || a.Protocol != "phase1" || a.ID != id { t.Fatalf("an operator's detail = %s", out) } // The CLI's sentences name the regions as the caller does (Task 1a): // the move's origin and the record's region now. if a.Detail.From != "ie" || a.Detail.Region != "ie" { t.Fatalf("detail from %q region %q, want ie and ie: %s", a.Detail.From, a.Detail.Region, out) } flat, _ := readMoveAnswer(t, http.MethodGet, moveURL, token, "", http.StatusOK) if err := fx.st.SetOperator("alice@keymail.dev", false); err != nil { t.Fatal(err) } for _, who := range []struct{ name, token string }{{"a member", token}, {"an instance token", router}} { a, out := readMoveAnswer(t, http.MethodGet, moveURL+"?detail=1", who.token, "", http.StatusOK) if a.Detail != nil || strings.Contains(out, `"detail"`) { t.Fatalf("detail=1 as %s = %s, want the flat view", who.name, out) } if a.State != flat.State || a.ID != flat.ID || a.Target != flat.Target || a.Protocol != flat.Protocol || a.UpdatedAt != flat.UpdatedAt { t.Fatalf("detail=1 as %s = %s, want the flat view %+v", who.name, out, flat) } } } // Review Focus 2: the detail answer carries the reports' check ids, the // evidence keys and the log keys, and never a launch environment value, an // evidence or log body, a CARLOS_ value or a secret. func TestMoveDetailLeaksNothing(t *testing.T) { fx, token, moveURL := moveDoorWithClaim(t, "code-inst-move-leak") ctx := context.Background() readMoveAnswer(t, http.MethodPost, moveURL, token, `{"region":"uk","cut":"manual"}`, http.StatusAccepted) mv := recordMove(t, fx) prefix := moves.ReportPrefix("bab", "titogo", "acme.go.tito.io", mv.ID) srcKey, dstKey := moves.StoppedKey(prefix, "source"), moves.StoppedKey(prefix, "target") const rb = "rb_abcdefghijklmnopqrstuvwxyz" setMove(t, fx, func(rec *provision.Record) { rec.Move.State, rec.Move.CutPhase = provision.MoveRollingBack, provision.CutArchived rec.Move.Source = &provision.MoveSource{FinalTXID: 7, DBSHA256: strings.Repeat("d", 64), Evidence: srcKey} rec.Move.Rollback = &provision.MoveRollback{ID: rb, Reason: provision.RollbackLandingDeadline, TargetPhase: provision.RbTargetReleased, Evidence: dstKey} }) secretsKey := strings.Repeat("A", 43) + "=" env := map[string]string{ "DATABASE_URL": "postgres://app:hunter2-db-canary@db.internal/app", "STRIPE_SECRET_KEY": "sk_live_stripe_canary_value", "CARLOS_SECRETS_KEY": secretsKey, "CARLOS_APP_TOKEN": "carlos-app-token-canary", } hk, err := moves.ReportKeyFromSecrets(secretsKey) if err != nil { t.Fatal(err) } src := moves.SourceReport{V: 1, Box: "ie-box", MoveID: mv.ID, At: time.Now().UTC(), Env: moves.ReportEnv(hk, env), S1: moves.Check{OK: true}, S2: moves.Check{OK: true}, Release: "v0.25.0"} // Malformed reports: raw values under a name, one of them a secret that // is 64 hex characters (the shape of an HMAC), a digest that is a raw // secret, and a target report whose env input is one. The console can // tell none of them from an HMAC, so it passes no env value at all. const hexSecret = "9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08" const hexDigestSecret = "sha256:60303ae22b998861bce3b28f33eec1be758a213c86c93c076dbe9f558c11c752" const hexTargetSecret = "fd61a03af4f77d870fc21e05e7e80678095c92d808cfb3b5c279ee04c74aca13" src.Env.Vars["PLAIN_LEAK"] = "raw-env-value-canary" src.Env.Vars["HEX_API_SECRET"] = hexSecret malformed := src malformed.Env.Digest = hexDigestSecret dst := moves.TargetReport{V: 1, Box: "uk-box", Region: "uk", MoveID: mv.ID, At: time.Now().UTC(), Checks: map[string]moves.Check{ moves.CheckProtocol: {OK: true}, moves.CheckEnv: {OK: false, Reason: "the launch environment differs"}, }, Inputs: provision.MoveInputs{EnvReport: hexTargetSecret, Release: "v0.25.0"}} putJSON := func(key string, v any) { t.Helper() b, err := json.Marshal(v) if err != nil { t.Fatal(err) } if _, err := fx.rel.Put(ctx, key, b, ""); err != nil { t.Fatal(err) } } putJSON(moves.SourceReportKey(prefix), malformed) putJSON(moves.TargetReportKey(prefix), dst) putJSON(srcKey, map[string]any{"v": 1, "role": "source", "note": "evidence-body-canary-source", "env": env["DATABASE_URL"]}) putJSON(dstKey, map[string]any{"v": 1, "role": "target", "note": "evidence-body-canary-target"}) logKeys := []string{moves.LogKeyFor(prefix, "requested"), moves.LogKeyFor(prefix, "rolling_back")} for _, k := range logKeys { putJSON(k, map[string]any{"event": "x", "note": "log-body-canary", "actor": "alice@keymail.dev"}) } a, out := readMoveAnswer(t, http.MethodGet, moveURL+"?detail=1", token, "", http.StatusOK) if a.Detail == nil || a.Detail.TargetReport == nil || a.Detail.SourceReport == nil { t.Fatalf("detail = %s, want both reports", out) } leaks := []string{ env["DATABASE_URL"], "hunter2-db-canary", env["STRIPE_SECRET_KEY"], secretsKey, env["CARLOS_APP_TOKEN"], "evidence-body-canary", "log-body-canary", token, hexDigestSecret, hexTargetSecret, strings.TrimPrefix(hexDigestSecret, "sha256:"), src.Env.Digest, } // No value of either report's env section: every one of the source's // (the HMACs, the raw canary, the hex secret) and the target's input. for _, v := range malformed.Env.Vars { leaks = append(leaks, v) } for _, leak := range leaks { if strings.Contains(out, leak) { t.Errorf("the detail answer holds %q:\n%s", leak, out) } } for _, want := range []string{moves.CheckProtocol, moves.CheckEnv, `"s1"`, `"s2"`, srcKey, dstKey, "v0.25.0"} { if !strings.Contains(out, want) { t.Errorf("the detail answer lacks %q:\n%s", want, out) } } // Each env entry is its name and its kind, nothing else. got := map[string]string{} for _, e := range a.Detail.SourceReport.Env { got[e.Name] = e.Kind } want := map[string]string{} for name := range malformed.Env.Vars { want[name] = moves.Class(name) } if !reflect.DeepEqual(got, want) { t.Errorf("source env = %v, want the names and kinds %v", got, want) } var raw struct { Detail struct { SourceReport map[string]json.RawMessage `json:"source_report"` } `json:"detail"` } if err := json.Unmarshal([]byte(out), &raw); err != nil { t.Fatal(err) } var entries []map[string]any if err := json.Unmarshal(raw.Detail.SourceReport["env"], &entries); err != nil { t.Fatalf("source env is not a list of entries: %v in %s", err, raw.Detail.SourceReport["env"]) } for _, e := range entries { if len(e) != 2 { t.Errorf("an env entry carries more than its name and kind: %v", e) } } if a.Detail.Evidence.Source != srcKey || a.Detail.Evidence.Target != dstKey { t.Errorf("evidence = %+v, want the two keys", a.Detail.Evidence) } if !reflect.DeepEqual(a.Detail.Log, logKeys) { t.Errorf("log = %v, want %v", a.Detail.Log, logKeys) } } // Plan 6 Task 4: once a move has closed done or rolled_back, detail=1 shows // its timings, read from the terminal log event's Fields: the box or the // console wrote them from the record's own journal times when it built the // entry (moves.Timings), and the door never works them out a second way. // Only the timing fields are shown, never the rest of the event; a move with // no terminal event logged yet, or one still in flight, shows none; a log // read that fails answers the reports' fixed sentence. func TestMoveDetailShowsTheTimings(t *testing.T) { fx, token, moveURL := moveDoorWithClaim(t, "code-inst-move-timings") ctx := context.Background() readMoveAnswer(t, http.MethodPost, moveURL, token, `{"region":"uk","cut":"manual"}`, http.StatusAccepted) mv := recordMove(t, fx) if a, out := readMoveAnswer(t, http.MethodGet, moveURL+"?detail=1", token, "", http.StatusOK); a.Detail == nil || a.Detail.Timings != nil { t.Fatalf("a move in flight shows timings: %s", out) } prefix := moves.ReportPrefix("bab", "titogo", "acme.go.tito.io", mv.ID) setMove(t, fx, func(rec *provision.Record) { rec.Move, rec.LastMove = nil, &provision.MoveOutcome{ID: mv.ID, Outcome: "done", At: time.Now().UTC(), OfflineS: 70.5} }) if a, out := readMoveAnswer(t, http.MethodGet, moveURL+"?detail=1", token, "", http.StatusOK); a.Detail == nil || a.Detail.Timings != nil { t.Fatalf("timings before the terminal event is logged: %s", out) } const digest = "c0ffee00c0ffee00c0ffee00c0ffee00c0ffee00c0ffee00c0ffee00c0ffee00" ev := moves.LogEvent{V: 1, Op: mv.ID, Host: "acme.go.tito.io", Event: "done", Identity: "done", Actor: "uk-box", At: time.Now().UTC(), Fields: map[string]string{"from": "ie", "to": "uk", "db_sha256": digest, "prepare_s": "300", "cut_s": "40", "cut_budget_s": "95", "land_s": "30.5", "land_budget_s": "50", "offline_s": "70.5"}} if _, err := moves.WriteLog(ctx, fx.rel, prefix, ev); err != nil { t.Fatal(err) } a, out := readMoveAnswer(t, http.MethodGet, moveURL+"?detail=1", token, "", http.StatusOK) want := map[string]float64{"prepare_s": 300, "cut_s": 40, "cut_budget_s": 95, "land_s": 30.5, "land_budget_s": 50, "offline_s": 70.5} if a.Detail == nil || !reflect.DeepEqual(a.Detail.Timings, want) { t.Fatalf("detail timings = %s, want %v", out, want) } if strings.Contains(out, digest) { t.Fatalf("the detail shows the event's other fields: %s", out) } // A rolled_back close reads its own event, not the done one. setMove(t, fx, func(rec *provision.Record) { rec.LastMove.Outcome = "rolled_back" }) if a, out := readMoveAnswer(t, http.MethodGet, moveURL+"?detail=1", token, "", http.StatusOK); a.Detail == nil || a.Detail.Timings != nil { t.Fatalf("a rolled_back close shows the done event's timings: %s", out) } rb := ev rb.Event, rb.Identity, rb.Fields = "rolled_back", "rolled_back", map[string]string{"offline_s": "210", "rollback_s": "135", "release_budget_s": "900"} if _, err := moves.WriteLog(ctx, fx.rel, prefix, rb); err != nil { t.Fatal(err) } if a, out := readMoveAnswer(t, http.MethodGet, moveURL+"?detail=1", token, "", http.StatusOK); a.Detail == nil || !reflect.DeepEqual(a.Detail.Timings, map[string]float64{"offline_s": 210, "rollback_s": 135, "release_budget_s": 900}) { t.Fatalf("rolled_back timings = %s", out) } } // Relocating (spec rev 7.20 §7.10, D16) and Paul's J3 (preflight L18): a // move whose claim is relocating/pinning with exec "none" in a region with // no live enrolled box reports waiting_for_box = that region: the source's // before the flip, the target's after a committed landing was converted. A // converted committed landing answers state relocating, in the flat view // too. "Live enrolled" is boxchoice.Eligible over the live edges snapshot, // with the roster read through hostactors.Directory: an active entry with // an enrollment and no termination fence. // waitingFor is the detail's waiting_for_box, or a marker when the answer // has no detail at all. func waitingFor(a moveAnswer) string { if a.Detail == nil { return "" } return a.Detail.WaitingForBox } func TestMoveDetailShowsWaitingForBox(t *testing.T) { relocate := func(region, box string) func(*hostclaim.Claim) { return func(c *hostclaim.Claim) { c.Op.Kind, c.Op.Phase, c.Op.Exec, c.Op.Participants = hostclaim.KindRelocating, "pinning", "none", nil c.Op.Intent = map[string]string{"region": region, "source_box": box} } } // Every phase hostclaim lets run on no box (validateRelocating). noBoxPhases := []string{"starting", "releasing", "pinning"} inPhase := func(phase string) func(*hostclaim.Claim) { return func(c *hostclaim.Claim) { c.Op.Phase, c.Op.Exec = phase, "none" } } t.Run("before the flip", func(t *testing.T) { fx, token, moveURL := moveDoorWithClaim(t, "code-inst-move-wait-src") readMoveAnswer(t, http.MethodPost, moveURL, token, `{"region":"uk","cut":"manual"}`, http.StatusAccepted) mv := recordMove(t, fx) setMove(t, fx, func(rec *provision.Record) { rec.Move.Relocation = &provision.MoveRelocationDecision{SourceBox: "ie-box", Region: "ie", PreviousState: rec.Move.State, DecidedAt: time.Now().UTC()} rec.Move.State = provision.MoveRelocating }) rewriteClaim(t, fx, "acme.go.tito.io", relocate("ie", "ie-box")) // ie-box was terminated: its fence is verified, so ie has no live // enrolled box even though its edges file is fresh. writeMoveFence(t, fx, moveRosterBox(ieBox("ie-box")), nil) router := serviceToken(t, fx.st, OpsAccountID, "router", "titogo", svcrole.Instance) if a, out := readMoveAnswer(t, http.MethodGet, moveURL, router, "", http.StatusOK); a.State != provision.MoveRelocating || a.ID != mv.ID { t.Fatalf("flat view = %s", out) } for _, phase := range noBoxPhases { rewriteClaim(t, fx, "acme.go.tito.io", inPhase(phase)) a, out := readMoveAnswer(t, http.MethodGet, moveURL+"?detail=1", token, "", http.StatusOK) if a.State != provision.MoveRelocating || waitingFor(a) != "ie" { t.Fatalf("detail in %s/none with no box in ie = %s, want waiting_for_box ie", phase, out) } } // A live enrolled box joins ie: nothing is waiting. armMoveFleet(t, fx, ieBox("ie-box2")) putMoveRoster(t, fx, hostactors.Roster{Boxes: map[string]hostactors.Box{"ie-box2": moveRosterBox(ieBox("ie-box2")), "uk-box": moveRosterBox(ukBox("uk-box"))}}) if a, out := readMoveAnswer(t, http.MethodGet, moveURL+"?detail=1", token, "", http.StatusOK); a.State != provision.MoveRelocating || waitingFor(a) != "" { t.Fatalf("detail with a live box in ie = %s", out) } // The same box, live but not enrolled in the roster, is not one. putMoveRoster(t, fx, hostactors.Roster{Boxes: map[string]hostactors.Box{"uk-box": moveRosterBox(ukBox("uk-box"))}}) if a, out := readMoveAnswer(t, http.MethodGet, moveURL+"?detail=1", token, "", http.StatusOK); waitingFor(a) != "ie" { t.Fatalf("detail with an unenrolled box in ie = %s, want waiting_for_box ie", out) } }) t.Run("after the commit (J3)", func(t *testing.T) { fx, token, moveURL := moveDoorWithClaim(t, "code-inst-move-wait-dst") readMoveAnswer(t, http.MethodPost, moveURL, token, `{"region":"uk","cut":"manual"}`, http.StatusAccepted) now := time.Now().UTC() setMove(t, fx, func(rec *provision.Record) { rec.Region = "uk" rec.Move.State, rec.Move.CutPhase, rec.Move.FlippedAt, rec.Move.CommittedAt = provision.MoveCommitted, provision.CutFlipped, &now, &now }) // Not converted: the record's own state. if a, out := readMoveAnswer(t, http.MethodGet, moveURL+"?detail=1", token, "", http.StatusOK); a.State != provision.MoveCommitted || waitingFor(a) != "" { t.Fatalf("a committed landing = %s", out) } rewriteClaim(t, fx, "acme.go.tito.io", relocate("uk", "uk-box")) writeMoveFence(t, fx, moveRosterBox(ukBox("uk-box")), nil) router := serviceToken(t, fx.st, OpsAccountID, "router", "titogo", svcrole.Instance) if a, out := readMoveAnswer(t, http.MethodGet, moveURL, router, "", http.StatusOK); a.State != provision.MoveRelocating { t.Fatalf("flat view of a converted landing = %s, want relocating", out) } for _, phase := range noBoxPhases { rewriteClaim(t, fx, "acme.go.tito.io", inPhase(phase)) a, out := readMoveAnswer(t, http.MethodGet, moveURL+"?detail=1", token, "", http.StatusOK) if a.State != provision.MoveRelocating || waitingFor(a) != "uk" || (a.Detail != nil && a.Detail.Move.State != provision.MoveCommitted) { t.Fatalf("detail of a converted landing in %s/none with no box in uk = %s, want waiting_for_box uk", phase, out) } } // A live enrolled box joins uk: still relocating, nothing waiting. armMoveFleet(t, fx, ukBox("uk-box2")) putMoveRoster(t, fx, hostactors.Roster{Boxes: map[string]hostactors.Box{"ie-box": moveRosterBox(ieBox("ie-box")), "uk-box2": moveRosterBox(ukBox("uk-box2"))}}) if a, out := readMoveAnswer(t, http.MethodGet, moveURL+"?detail=1", token, "", http.StatusOK); a.State != provision.MoveRelocating || waitingFor(a) != "" { t.Fatalf("detail with a live box in uk = %s", out) } // A phase that runs on a box is not waiting for one. putMoveRoster(t, fx, hostactors.Roster{Boxes: map[string]hostactors.Box{"ie-box": moveRosterBox(ieBox("ie-box"))}}) rewriteClaim(t, fx, "acme.go.tito.io", func(c *hostclaim.Claim) { c.Op.Phase, c.Op.Exec, c.Op.Intent["target_box"] = "installing", "uk-box2", "uk-box2" }) if a, out := readMoveAnswer(t, http.MethodGet, moveURL+"?detail=1", token, "", http.StatusOK); a.State != provision.MoveRelocating || waitingFor(a) != "" { t.Fatalf("detail of a relocation that is installing = %s", out) } rewriteClaim(t, fx, "acme.go.tito.io", func(c *hostclaim.Claim) { c.Op.Phase, c.Op.Exec = "releasing", "uk-box" delete(c.Op.Intent, "target_box") }) if a, out := readMoveAnswer(t, http.MethodGet, moveURL+"?detail=1", token, "", http.StatusOK); a.State != provision.MoveRelocating || waitingFor(a) != "" { t.Fatalf("detail of a relocation releasing on its source box = %s", out) } }) } // Preflight L5 (claim-off half): once the cut has started (cutting past // requested, landing, committed, rolling back, reverting) the abort door // answers M-C13 and writes nothing. Before it, abort is the guarded // transition; a stuck move keeps provision's own refusal. func TestMoveAbortAfterTheCutStartedSaysSo(t *testing.T) { const mc13 = "This move has started its cut. It will finish or move back on its own." fx, token, moveURL := moveDoorWithClaim(t, "code-inst-move-mc13") readMoveAnswer(t, http.MethodPost, moveURL, token, `{"region":"uk","cut":"manual"}`, http.StatusAccepted) for _, c := range []struct { name string mut func(*provision.MoveState) }{ {"cutting/admitted", func(m *provision.MoveState) { m.State, m.CutPhase = provision.MoveCutting, provision.CutAdmitted }}, {"cutting/flipped", func(m *provision.MoveState) { m.State, m.CutPhase = provision.MoveCutting, provision.CutFlipped }}, {"landing", func(m *provision.MoveState) { m.State = provision.MoveLanding }}, {"committed", func(m *provision.MoveState) { m.State = provision.MoveCommitted }}, {"rolling back", func(m *provision.MoveState) { m.State = provision.MoveRollingBack }}, {"reverting", func(m *provision.MoveState) { m.State = provision.MoveReverting }}, } { setMove(t, fx, func(rec *provision.Record) { c.mut(rec.Move) }) before, _ := json.Marshal(recordMove(t, fx)) if code, _, out := rawReq(t, http.MethodDelete, moveURL, token, ""); code != http.StatusConflict || errorText(out) != mc13 { t.Fatalf("abort in %s = %d %s, want M-C13", c.name, code, out) } if after, _ := json.Marshal(recordMove(t, fx)); string(after) != string(before) { t.Fatalf("a refused abort in %s wrote", c.name) } } setMove(t, fx, func(rec *provision.Record) { rec.Move.State, rec.Move.CutPhase, rec.Move.StuckReason = provision.MoveStuck, provision.CutArchived, provision.StuckBaselineMismatch }) if code, _, out := rawReq(t, http.MethodDelete, moveURL, token, ""); code != http.StatusConflict || errorText(out) == mc13 { t.Fatalf("abort of a stuck move = %d %s, want provision's refusal", code, out) } setMove(t, fx, func(rec *provision.Record) { rec.Move.State, rec.Move.CutPhase, rec.Move.StuckReason = provision.MoveCutting, provision.CutRequested, "" }) if a, out := readMoveAnswer(t, http.MethodDelete, moveURL, token, "", http.StatusAccepted); a.State != provision.MoveAborting { t.Fatalf("abort from cutting/requested = %s", out) } } // Task 1a fix round 1 (L19): with the switch on the flat view carries the // move's phase, so a poller reads detail=1 only when the state or the phase // changes. func TestMoveViewCarriesThePhase(t *testing.T) { fx, token, moveURL := moveDoorWithClaim(t, "code-inst-move-phase") readMoveAnswer(t, http.MethodPost, moveURL, token, `{"region":"uk","cut":"manual"}`, http.StatusAccepted) const rb = "rb_abcdefghijklmnopqrstuvwxyz" for _, c := range []struct { mut func(*provision.MoveState) want string }{ {func(m *provision.MoveState) { m.State = provision.MovePreparing }, ""}, {func(m *provision.MoveState) { m.State, m.CutPhase = provision.MoveCutting, provision.CutAdmitted }, provision.CutAdmitted}, {func(m *provision.MoveState) { m.State, m.LandingPhase = provision.MoveLanding, provision.LandProbing }, provision.LandProbing}, {func(m *provision.MoveState) { m.State, m.Rollback = provision.MoveRollingBack, &provision.MoveRollback{ID: rb, TargetPhase: provision.RbTargetReleasing, SourcePhase: provision.RbSourceVerified} }, provision.RbTargetReleasing + "/" + provision.RbSourceVerified}, } { setMove(t, fx, func(rec *provision.Record) { c.mut(rec.Move) }) code, _, out := rawReq(t, http.MethodGet, moveURL, token, "") var v map[string]any if code != http.StatusOK || json.Unmarshal([]byte(out), &v) != nil { t.Fatalf("status = %d %s", code, out) } got, has := v["phase"] if (c.want == "" && has) || (c.want != "" && got != c.want) { t.Fatalf("phase = %v (present %v), want %q: %s", got, has, c.want, out) } } } // Task 1b (preflight L7, MV-N11): a console without the host claim cannot run // the claim's loss protocol, so the resolve door answers a declare-lost with // MV-N11 and writes nothing; a retry is unchanged. func TestResolveDeclareLostWithoutTheClaim(t *testing.T) { fx, token, moveURL := moveDoorWithClaim(t, "code-inst-move-dl-off") readMoveAnswer(t, http.MethodPost, moveURL, token, `{"region":"uk","cut":"manual"}`, http.StatusAccepted) before, _ := json.Marshal(recordMove(t, fx)) code, _, out := rawReq(t, http.MethodPost, moveURL+"/resolve", token, `{"declare_lost":"uk-box"}`) if code != http.StatusConflict || errorText(out) != "declare-lost is not enabled in this deployment yet" { t.Fatalf("declare-lost without the claim = %d %s", code, out) } if after, _ := json.Marshal(recordMove(t, fx)); string(after) != string(before) { t.Fatal("a refused declare-lost wrote") } if code, _, out := rawReq(t, http.MethodPost, moveURL+"/resolve", token, `{"retry":true}`); code != http.StatusConflict || errorText(out) != provision.RefusedNotStuck { t.Fatalf("retry = %d %s", code, out) } }