package console import ( "context" "encoding/json" "errors" "net/http" "strings" "testing" "time" "github.com/carlosframework/platform/internal/boxchoice" "github.com/carlosframework/platform/internal/hostactors" "github.com/carlosframework/platform/internal/hostclaim" "github.com/carlosframework/platform/internal/hostops" "github.com/carlosframework/platform/internal/provision" ) // roomyEdges gives each box's edges file fresh, roomy headroom, then edit // (when set) changes uk-box's. func roomyEdges(t *testing.T, fx *consoleFixture, edit func(map[string]any), boxes ...string) { t.Helper() ctx := context.Background() for _, box := range boxes { key := edgesFilePrefix + box + ".json" raw, etag, err := fx.rel.Get(ctx, key) if err != nil { t.Fatal(err) } var m map[string]any if err := json.Unmarshal(raw, &m); err != nil { t.Fatal(err) } m["headroom"] = map[string]any{"mem_free_mib": 4000, "mem_total_mib": 8000, "disk_free_mib": 50000, "disk_total_mib": 100000, "awake": 0} m["generation"] = time.Now().UTC().Format(time.RFC3339) if box == "uk-box" && edit != nil { edit(m) } body, _ := json.Marshal(m) if _, err := fx.rel.Put(ctx, key, body, etag); err != nil { t.Fatal(err) } } } // haRoster signs a roster naming both regions, every box of fleet, and // each box's role from roles. func haRoster(t *testing.T, fx *consoleFixture, roles map[string]string, fleet ...fleetBox) { t.Helper() haRosterIn(t, fx, roles, []string{"ie", "uk"}, fleet...) } // haRosterIn is haRoster naming only the regions in codes (the roster's // Regions), which decides whether it places boxes in a region. func haRosterIn(t *testing.T, fx *consoleFixture, roles map[string]string, codes []string, fleet ...fleetBox) { t.Helper() boxes := map[string]hostactors.Box{} for _, b := range fleet { rb := moveRosterBox(b) rb.Role = roles[b.Box] boxes[b.Box] = rb } regions := map[string]hostactors.Region{} for _, c := range codes { regions[c] = map[string]hostactors.Region{ "ie": {Account: moveTestAccount, AWSRegion: "eu-west-1"}, "uk": {Account: moveTestAccount, AWSRegion: "eu-west-2"}, }[c] } putMoveRoster(t, fx, hostactors.Roster{Boxes: boxes, Regions: regions}) } // bindRegionHA turns region HA on for fx's claim runtime, with the region // HA chooser bound as the create door binds it. func bindRegionHA(fx *consoleFixture) { fx.srv.cfg.RegionHA = true a := fx.srv.cfg.ClaimAuthority a.Namespace = "platform" a.Executors = fx.srv.executorChooser(a.Directory) } // haMoveFleet is armMoveFleet's ie-box and uk-box (and any more boxes), // each with fresh, roomy headroom, a signed roster naming both regions // with each box's role from roles, and, with ha, a region HA chooser. func haMoveFleet(t *testing.T, ha bool, roles map[string]string, more ...fleetBox) *consoleFixture { t.Helper() return haFleetOf(t, ha, roles, []string{"ie", "uk"}, append([]fleetBox{ieBox("ie-box"), ukBox("uk-box")}, more...)...) } // haFleetOf is haMoveFleet for exactly fleet, with a roster naming only // the regions in codes. func haFleetOf(t *testing.T, ha bool, roles map[string]string, codes []string, fleet ...fleetBox) *consoleFixture { t.Helper() fx := newConsoleFixture(t) armMoveFleet(t, fx, fleet...) names := []string{} for _, b := range fleet { names = append(names, b.Box) } roomyEdges(t, fx, nil, names...) haRosterIn(t, fx, roles, codes, fleet...) fx.srv.cfg.HostClaim = true if ha { d := hostactors.Directory{Store: fx.rel, PublicKey: fx.pub, Bucket: moveTestBucket} fx.srv.cfg.ClaimAuthority = &hostops.ProductionAuthority{Directory: d} bindRegionHA(fx) } return fx } // Region HA spec 2 ยง7 (H27), census ruling region-ha-2a rev 2 item 4: the // move door's room check only refuses. With room, the selection is the // one region HA off makes; with none, it is the no-room refusal with its // alert; and it never narrows the candidates to pick a destination main's // selection would not. func TestMoveRoomCheckOnlyRefuses(t *testing.T) { const host, app = "acme.go.tito.io", "bab/titogo" ctx := context.Background() t.Run("room: the selection region HA off makes", func(t *testing.T) { off, err := haMoveFleet(t, false, nil).srv.selectMoveBoxes(ctx, host, "uk", "ie", app) if err != nil { t.Fatal(err) } on, err := haMoveFleet(t, true, nil).srv.selectMoveBoxes(ctx, host, "uk", "ie", app) if err != nil || on != off || on.Target != "uk-box" || on.Source != "ie-box" { t.Fatalf("region HA on %+v %v, off %+v", on, err, off) } }) t.Run("no room: the only destination is legacy", func(t *testing.T) { pages := pagesTo(t) fx := haMoveFleet(t, true, map[string]string{"uk-box": hostactors.RoleLegacy}) _, err := fx.srv.selectMoveBoxes(ctx, host, "uk", "ie", app) var full *noCapacityError if !errors.As(err, &full) || full.Region != "uk" { t.Fatalf("err = %v, want no room in uk", err) } if ref := claimResponseRefusal(err); ref.Status != http.StatusServiceUnavailable || ref.Msg != noCapacityRefusal("uk") { t.Fatalf("refusal %+v", ref) } expectPages(t, pages, noRoomSubject("uk")) }) t.Run("a legacy box beside a default one is not filtered out", func(t *testing.T) { roles := map[string]string{"uk-box": hostactors.RoleLegacy} off, err := haMoveFleet(t, false, roles, ukBox("uk-box-2")).srv.selectMoveBoxes(ctx, host, "uk", "ie", app) if err != nil || !off.Many { t.Fatalf("region HA off: %+v %v, want main's several-boxes refusal", off, err) } on, err := haMoveFleet(t, true, roles, ukBox("uk-box-2")).srv.selectMoveBoxes(ctx, host, "uk", "ie", app) if err != nil || on != off { t.Fatalf("region HA on %+v %v, want region HA off's %+v", on, err, off) } }) t.Run("a platform source still moves out", func(t *testing.T) { fx := haMoveFleet(t, true, map[string]string{"ie-box": hostactors.RolePlatform}) got, err := fx.srv.selectMoveBoxes(ctx, host, "uk", "ie", app) if err != nil || got.Source != "ie-box" || got.Target != "uk-box" { t.Fatalf("selection %+v %v", got, err) } }) } // Astra 2a round 1 finding 4: with host claims on and region HA off, the // move door is main's. A persisted role excludes nothing, and the door // reads no roster, so an unreadable one cannot refuse the move. func TestMoveDestinationWithRegionHAOffIsMains(t *testing.T) { const host, app = "acme.go.tito.io", "bab/titogo" fx := haMoveFleet(t, false, map[string]string{"uk-box": hostactors.RoleLegacy}) got, err := fx.srv.selectMoveBoxes(context.Background(), host, "uk", "ie", app) if err != nil || got.Target != "uk-box" { t.Fatalf("selection %+v %v", got, err) } fx.srv.cfg.Releases = failingGet{Store: fx.rel, key: hostactors.Key} if got, err := fx.srv.selectMoveBoxes(context.Background(), host, "uk", "ie", app); err != nil || got.Target != "uk-box" { t.Fatalf("with the roster unreadable: %+v %v", got, err) } } // haMoveDoor is moveDoorWithClaim's world on a host-claim console with the // claim runtime armed: roomy edges files (uk-box's changed by edit), a // roster naming both regions and, with ha, region HA on. func haMoveDoor(t *testing.T, code string, ha bool, edit func(map[string]any)) (*consoleFixture, string, string) { t.Helper() fx, token, moveURL := moveDoorWithClaim(t, code) roomyEdges(t, fx, edit, "ie-box", "uk-box") haRoster(t, fx, nil, ieBox("ie-box"), ukBox("uk-box")) armClaimMoves(t, fx, true, true, nil) if ha { bindRegionHA(fx) } return fx, token, moveURL } // haMoveDoorOf is moveDoorWithClaim's world over exactly fleet (the claim // is under fleet's first box, ie-box), with roomy edges files, each box's // role from roles and region HA on. func haMoveDoorOf(t *testing.T, code string, roles map[string]string, fleet ...fleetBox) (*consoleFixture, string, string) { t.Helper() fx, token := operatorInstanceFixture(t, code) m := titoManifest() m.Regions = nil m.Pool = "" enableInstances(t, fx, "titogo", m) u := instancesURL(fx, "bab", "titogo") if c, _, out := rawReq(t, http.MethodPost, u, token, `{"host":"acme.go.tito.io","region":"ie"}`); c != http.StatusCreated { t.Fatalf("create = %d %s", c, out) } armMoveFleet(t, fx, fleet...) claimFor(t, fx, "acme.go.tito.io", "ie-box", "ie") names := []string{} for _, b := range fleet { names = append(names, b.Box) } roomyEdges(t, fx, nil, names...) haRoster(t, fx, roles, fleet...) armClaimMoves(t, fx, true, true, nil) bindRegionHA(fx) return fx, token, u + "/acme.go.tito.io/move" } // Astra 2a round 2 finding 1: through the whole HTTP door, a destination // with no box that has room (no edges file, or only blank public // addresses) is the no-room refusal, alert included, and writes nothing: // not the placement checks' 400. With region HA off it is main's 400; with // room the move is admitted to uk-box either way. func TestMoveDoorWithNoRoomInTheDestinationIsNoRoom(t *testing.T) { for _, c := range []struct { name string edit func(map[string]any) }{ {"only a blank public address", func(m map[string]any) { m["eip"] = "" }}, {"no edges file", nil}, } { t.Run(c.name, func(t *testing.T) { noFile := c.edit == nil world := func(t *testing.T, code string, ha bool) (*consoleFixture, string, string) { fx, token, moveURL := haMoveDoor(t, code, ha, c.edit) if noFile { // A box that never reported: no edges file and no status // snapshot. (A fresh snapshot with no edges file is an // incomplete read, pinned below.) for _, key := range []string{edgesFilePrefix + "uk-box.json", "status/boxes/uk-box.json"} { if err := fx.rel.Delete(context.Background(), key); err != nil { t.Fatal(err) } } } return fx, token, moveURL } _, token, moveURL := world(t, "code-ha-move-off", false) if code, _, out := rawReq(t, http.MethodPost, moveURL, token, `{"region":"uk"}`); code != http.StatusBadRequest { t.Fatalf("region HA off: move = %d %s, want main's 400", code, out) } pages := pagesTo(t) fx, token, moveURL := world(t, "code-ha-move-on", true) claim := claimBytes(t, fx, "acme.go.tito.io") code, hdr, out := rawReq(t, http.MethodPost, moveURL, token, `{"region":"uk"}`) if code != http.StatusServiceUnavailable || !strings.Contains(out, noCapacityRefusal("uk")) { t.Fatalf("move = %d %s", code, out) } if got := hdr.Get("Retry-After"); got != noCapacityRetryAfter { t.Fatalf("Retry-After = %q, want %q", got, noCapacityRetryAfter) } expectPages(t, pages, noRoomSubject("uk")) if claimBytes(t, fx, "acme.go.tito.io") != claim { t.Fatal("a refused move wrote the claim") } a, err := hostops.ReadAdmission(context.Background(), fx.rel, "bab", "titogo", claimMoveAppID) if err != nil || len(a.Value.Registrations) != 0 { t.Fatalf("a refused move registered: %+v %v", a.Value.Registrations, err) } if rec, err := provision.GetRecord(context.Background(), fx.rel, "bab", "titogo", "acme.go.tito.io"); err != nil || rec.Move != nil { t.Fatalf("a refused move journaled: %+v %v", rec.Move, err) } }) } t.Run("room", func(t *testing.T) { fx, token, moveURL := haMoveDoor(t, "code-ha-move-room", true, nil) if code, _, out := rawReq(t, http.MethodPost, moveURL, token, `{"region":"uk"}`); code != http.StatusAccepted { t.Fatalf("move = %d %s", code, out) } o, err := hostclaim.Read(context.Background(), fx.rel, "acme.go.tito.io") if err != nil || o.Claim == nil || o.Claim.Op.Intent["target"] != "uk-box" { t.Fatalf("claim %+v %v, want a move to uk-box", o.Claim, err) } }) } // Census ruling region-ha-2a rev 2 item 3: an incomplete placement read // (here a malformed edges file in the destination) is not no room. The // move door answers the existing 503 "temporarily unavailable", pages // nobody, and writes nothing. func TestMoveDoorWithAnIncompleteReadIsUnavailableNotNoRoom(t *testing.T) { for _, c := range []struct { name string breakIt func(*testing.T, *consoleFixture) }{ {"a malformed edges file", func(t *testing.T, fx *consoleFixture) { key := edgesFilePrefix + "uk-box.json" _, etag, err := fx.rel.Get(context.Background(), key) if err != nil { t.Fatal(err) } if _, err := fx.rel.Put(context.Background(), key, []byte("{not json"), etag); err != nil { t.Fatal(err) } }}, {"a fresh status snapshot and no edges file", func(t *testing.T, fx *consoleFixture) { if err := fx.rel.Delete(context.Background(), edgesFilePrefix+"uk-box.json"); err != nil { t.Fatal(err) } }}, } { t.Run(c.name, func(t *testing.T) { moveDoorIncompleteRead(t, c.breakIt) }) } } func moveDoorIncompleteRead(t *testing.T, breakIt func(*testing.T, *consoleFixture)) { pages := pagesTo(t) fx, token, moveURL := haMoveDoor(t, "code-ha-move-incomplete", true, nil) ctx := context.Background() breakIt(t, fx) claim := claimBytes(t, fx, "acme.go.tito.io") code, hdr, out := rawReq(t, http.MethodPost, moveURL, token, `{"region":"uk"}`) if code != http.StatusServiceUnavailable || !strings.Contains(out, claimResponseUnavailable) || strings.Contains(out, noCapacityRefusal("uk")) { t.Fatalf("move = %d %s, want the temporarily unavailable 503", code, out) } if got := hdr.Get("Retry-After"); got == noCapacityRetryAfter { t.Fatalf("Retry-After = %q, the no-room value", got) } expectPages(t, pages) if claimBytes(t, fx, "acme.go.tito.io") != claim { t.Fatal("a refused move wrote the claim") } if rec, err := provision.GetRecord(ctx, fx.rel, "bab", "titogo", "acme.go.tito.io"); err != nil || rec.Move != nil { t.Fatalf("a refused move journaled: %+v %v", rec.Move, err) } } // Census ruling region-ha-2a (fix round 1): the room check refuses only // where placement is known to have no room. A roster that does not place // the destination (hostops.ErrContract) and a destination with no // candidates are main's: the selection equals the one region HA off makes, // and nobody is paged. func TestMoveRoomCheckRefusesOnlyWhereRoomIsKnownMissing(t *testing.T) { const host, app = "acme.go.tito.io", "bab/titogo" ctx := context.Background() for _, c := range []struct { name string codes []string fleet []fleetBox want moveSelection }{ {"the roster does not place the destination", []string{"ie"}, []fleetBox{ieBox("ie-box"), ukBox("uk-box")}, moveSelection{Target: "uk-box", Source: "ie-box"}}, {"the destination has no candidates", []string{"ie", "uk"}, []fleetBox{ieBox("ie-box"), func() fleetBox { b := ukBox("uk-box"); b.Unverified = true; return b }()}, moveSelection{Source: "ie-box"}}, } { t.Run(c.name, func(t *testing.T) { pages := pagesTo(t) off, err := haFleetOf(t, false, nil, c.codes, c.fleet...).srv.selectMoveBoxes(ctx, host, "uk", "ie", app) if err != nil || off.Target != c.want.Target || off.Source != c.want.Source { t.Fatalf("region HA off: %+v %v, want target %q source %q", off, err, c.want.Target, c.want.Source) } // Every field, the floor's included, is main's. on, err := haFleetOf(t, true, nil, c.codes, c.fleet...).srv.selectMoveBoxes(ctx, host, "uk", "ie", app) if err != nil || on != off { t.Fatalf("region HA on: %+v %v, want region HA off's %+v", on, err, off) } expectPages(t, pages) }) } t.Run("the door answers main's 400 for a destination with no candidates", func(t *testing.T) { pages := pagesTo(t) unverified := ukBox("uk-box") unverified.Unverified = true _, token, moveURL := haMoveDoorOf(t, "code-ha-move-nocands", nil, ieBox("ie-box"), unverified) code, _, out := rawReq(t, http.MethodPost, moveURL, token, `{"region":"uk"}`) if code != http.StatusBadRequest || !strings.Contains(out, "no box in uk publishes an edges file yet") { t.Fatalf("move = %d %s, want main's 400", code, out) } expectPages(t, pages) }) } // The door's own selection branch (apiinstancemove.go, after // selectMoveBoxes): a destination whose placeable box is not a candidate // (uk-box-2 has room but no verified identity) while its only candidate is // excluded by role passes roomIn and is refused by moveRoomCheck, with the // no-room answer, its Retry-After and its alert, and writes nothing. func TestMoveDoorRefusesWhenNoCandidateHasRoom(t *testing.T) { pages := pagesTo(t) stranger := ukBox("uk-box-2") stranger.Unverified = true fx, token, moveURL := haMoveDoorOf(t, "code-ha-move-nocand-room", map[string]string{"uk-box": hostactors.RoleLegacy}, ieBox("ie-box"), ukBox("uk-box"), stranger) claim := claimBytes(t, fx, "acme.go.tito.io") code, hdr, out := rawReq(t, http.MethodPost, moveURL, token, `{"region":"uk"}`) if code != http.StatusServiceUnavailable || !strings.Contains(out, noCapacityRefusal("uk")) { t.Fatalf("move = %d %s", code, out) } if got := hdr.Get("Retry-After"); got != noCapacityRetryAfter { t.Fatalf("Retry-After = %q, want %q", got, noCapacityRetryAfter) } expectPages(t, pages, noRoomSubject("uk")) if claimBytes(t, fx, "acme.go.tito.io") != claim { t.Fatal("a refused move wrote the claim") } if rec, err := provision.GetRecord(context.Background(), fx.rel, "bab", "titogo", "acme.go.tito.io"); err != nil || rec.Move != nil { t.Fatalf("a refused move journaled: %+v %v", rec.Move, err) } } // Fix round 1: a placement read moveRoomCheck cannot complete (the second // read the door makes, after roomIn's) is the claim doors' temporarily // unavailable 503, with no page and no Retry-After of no room. It is not // the moves protocol floor's answer, and an incomplete snapshot is still // hostops.ErrEvidenceIncomplete under the wrapper. func TestMoveRoomCheckReadFailureIsUnavailable(t *testing.T) { for _, c := range []struct { name string fail func(boxchoice.Snapshot, error) (boxchoice.Snapshot, error) }{ {"an error", func(boxchoice.Snapshot, error) (boxchoice.Snapshot, error) { return boxchoice.Snapshot{}, errors.New("edges store down") }}, {"an incomplete snapshot", func(snap boxchoice.Snapshot, err error) (boxchoice.Snapshot, error) { snap.Complete = false return snap, err }}, } { t.Run(c.name, func(t *testing.T) { pages := pagesTo(t) fx, token, moveURL := haMoveDoor(t, "code-ha-move-read", true, nil) edges, calls := fx.srv.cfg.ClaimAuthority.Executors.Edges, 0 fx.srv.cfg.ClaimAuthority.Executors.Edges = func(ctx context.Context) (boxchoice.Snapshot, error) { snap, err := edges(ctx) if calls++; calls >= 2 { // roomIn's read passes; moveRoomCheck's does not return c.fail(snap, err) } return snap, err } claim := claimBytes(t, fx, "acme.go.tito.io") code, hdr, out := rawReq(t, http.MethodPost, moveURL, token, `{"region":"uk"}`) if code != http.StatusServiceUnavailable || !strings.Contains(out, claimResponseUnavailable) { t.Fatalf("move = %d %s, want the temporarily unavailable 503", code, out) } if calls != 2 || hdr.Get("Retry-After") == noCapacityRetryAfter { t.Fatalf("chooser reads = %d, Retry-After = %q", calls, hdr.Get("Retry-After")) } expectPages(t, pages) if claimBytes(t, fx, "acme.go.tito.io") != claim { t.Fatal("a refused move wrote the claim") } }) } t.Run("an incomplete snapshot stays ErrEvidenceIncomplete", func(t *testing.T) { fx := haMoveFleet(t, true, nil) edges := fx.srv.cfg.ClaimAuthority.Executors.Edges fx.srv.cfg.ClaimAuthority.Executors.Edges = func(ctx context.Context) (boxchoice.Snapshot, error) { snap, err := edges(ctx) snap.Complete = false return snap, err } _, err := fx.srv.selectMoveBoxes(context.Background(), "acme.go.tito.io", "uk", "ie", "bab/titogo") var check *roomCheckError if !errors.As(err, &check) || !errors.Is(err, hostops.ErrEvidenceIncomplete) { t.Fatalf("err = %v, want a room check error wrapping ErrEvidenceIncomplete", err) } }) }