package console import ( "context" "encoding/json" "errors" "fmt" "net/http" "net/http/httptest" "strings" "testing" "time" "github.com/carlosframework/platform/internal/boxchoice" "github.com/carlosframework/platform/internal/hostclaim" "github.com/carlosframework/platform/internal/hostops" "github.com/carlosframework/platform/internal/objstore" "github.com/carlosframework/platform/internal/provision" "github.com/carlosframework/platform/internal/status" ) func TestNoCapacityAlertsRepeatAfterAnHour(t *testing.T) { var a noCapacityAlerts t0 := time.Date(2026, 10, 16, 12, 0, 0, 0, time.UTC) if !a.due("uk", t0) || a.due("uk", t0.Add(59*time.Minute)) || !a.due("ie", t0) || !a.due("uk", t0.Add(time.Hour)) { t.Fatal("once per region an hour") } } func TestClaimRefusalAnswersNoRoom(t *testing.T) { ref := claimResponseRefusal(fmt.Errorf("begin: %w", &noCapacityError{Region: "uk"})) if ref.Status != http.StatusServiceUnavailable || ref.Msg != noCapacityRefusal("uk") { t.Fatalf("refusal %+v", ref) } var err error = &noCapacityError{Region: "uk"} if !errors.Is(err, hostops.ErrNoEligibleBox) { t.Fatal("a no-room refusal is not ErrNoEligibleBox") } } // pagesTo replaces the alert sender for the test and returns what it sent. func pagesTo(t *testing.T) chan string { t.Helper() pages := make(chan string, 16) old := pageNoCapacity pageNoCapacity = func(_ context.Context, level, subject, _ string) error { pages <- level + " " + subject return nil } t.Cleanup(func() { pageNoCapacity = old }) return pages } func expectPages(t *testing.T, pages chan string, want ...string) { t.Helper() for _, w := range want { select { case got := <-pages: if got != w { t.Fatalf("page %q, want %q", got, w) } case <-time.After(5 * time.Second): t.Fatalf("no page %q", w) } } select { case got := <-pages: t.Fatalf("an extra page: %q", got) case <-time.After(100 * time.Millisecond): } } func noRoomSubject(region string) string { subject, _ := noCapacityAlert(region) return "warning " + subject } // haClaimCase is a host-claim console with region HA on: its create doors // choose through the region HA chooser over the edges files the test // publishes (publishHABox). home names the console's home region ("" for // none): a first route is placed in the home. func haClaimCase(t *testing.T, home string) *claimCase { t.Helper() cx := newClaimCaseWith(t, func(c *Config) { c.HomeRegion = home }) cx.srv.cfg.RegionHA = true a := cx.srv.cfg.ClaimAuthority a.Executors = cx.srv.executorChooser(a.Directory) return cx } // publishHABox publishes (or republishes) a roster box's edges file and // status snapshot: placeable in ie unless mut changes it, and stale when // stale is set. func publishHABox(t *testing.T, cx *claimCase, box string, stale bool, mut func(map[string]any)) { t.Helper() ctx := context.Background() m := map[string]any{ "v": 1, "box": box, "box_id": box, "eip": "203.0.113.7", "region": "eu-west-1", "placement": "ie", "hibernation_configured": true, "serves_shared_pool": true, "accepts_inbound": true, "secrets_key_fp": cx.srv.expectedSecretsFP(), "capabilities": []string{"region-ha-1"}, "headroom": map[string]any{"mem_free_mib": 4000, "mem_total_mib": 8000, "disk_free_mib": 50000, "disk_total_mib": 100000, "awake": 0}, "generation": time.Now().UTC().Format(time.RFC3339), "hosts": map[string]any{}, } if mut != nil { mut(m) } body, err := json.Marshal(m) if err != nil { t.Fatal(err) } key := "control/edges/" + box + ".json" _, tag, err := cx.rel.Get(ctx, key) if err != nil { tag = "" } if _, err := cx.rel.Put(ctx, key, body, tag); err != nil { t.Fatal(err) } at := time.Now() if stale { at = at.Add(-time.Hour) } if err := status.WriteBox(ctx, cx.rel, status.Box{Label: box, GeneratedAt: at}); err != nil { t.Fatal(err) } } // registrations counts account/app's registered operations. func registrations(t *testing.T, cx *claimCase, account, app string) int { t.Helper() a, err := hostops.ReadCanonicalAdmission(context.Background(), cx.rel, account, app) if err != nil { t.Fatal(err) } return len(a.Value.Registrations) } // refusedBeforeRegistering checks what H27 promises of a refused create: // nothing registered, no claim, and nothing recovery writes afterwards. func refusedBeforeRegistering(t *testing.T, cx *claimCase, host string, registered int) { t.Helper() ctx := context.Background() if n := registrations(t, cx, "bab", "titogo"); n != registered { t.Fatalf("registrations %d after the refusal, want %d: the door registered before choosing", n, registered) } if obs, err := hostclaim.Read(ctx, cx.rel, host); err != nil || !obs.Absent { t.Fatalf("claim after the refusal: %+v %v", obs, err) } if err := cx.srv.claimRecoveryPass(ctx); err != nil { t.Fatalf("recovery: %v", err) } if obs, err := hostclaim.Read(ctx, cx.rel, host); err != nil || !obs.Absent { t.Fatalf("claim after recovery: %+v %v", obs, err) } if n := registrations(t, cx, "bab", "titogo"); n != registered { t.Fatalf("registrations %d after recovery, want %d", n, registered) } } // wroteNothing checks that nothing reached the bucket or a backend since // mark: no lease, no registration, no record and no store write. func wroteNothing(t *testing.T, cx *claimCase, mark int) { t.Helper() if events := cx.rec.since(mark); len(events) != 0 { t.Fatalf("a refused request wrote %v", events) } } // Region HA spec 2 §7 (H27): every create door that chooses a box refuses // with a 503 when none is eligible, before it registers anything, and the // operator hears of it once per region an hour. Every box here is stale. func TestCreateDoorsRefuseWithNoRoomBeforeRegistering(t *testing.T) { t.Run("instance create", func(t *testing.T) { pages := pagesTo(t) cx := haClaimCase(t, "") cx.instanceWorld(t) publishHABox(t, cx, "box-ie-tito", true, nil) before := registrations(t, cx, "bab", "titogo") mark := cx.rec.mark() for range 3 { // a retry loop: refused every time, paged once r := cx.apiReply(t, http.MethodPost, "/api/cli/apps/bab/titogo/instances", `{"host":"acme.go.tito.io","region":"ie"}`) if r.Status != http.StatusServiceUnavailable || !strings.Contains(r.Body, noCapacityRefusal("ie")) { t.Fatalf("create = %d %s", r.Status, r.Body) } } wroteNothing(t, cx, mark) refusedBeforeRegistering(t, cx, "acme.go.tito.io", before) expectPages(t, pages, noRoomSubject("ie")) }) t.Run("standalone sidecar create", func(t *testing.T) { pages := pagesTo(t) cx := haClaimCase(t, "") cx.sidecarWorld(t) before := registrations(t, cx, "bab", "titogo") mark := cx.rec.mark() r := cx.apiReply(t, http.MethodPost, "/api/cli/apps/bab/titogo/sidecars", `{"name":"bot","region":"ie"}`) if r.Status != http.StatusServiceUnavailable || !strings.Contains(r.Body, noCapacityRefusal("ie")) { t.Fatalf("sidecar create = %d %s", r.Status, r.Body) } wroteNothing(t, cx, mark) refusedBeforeRegistering(t, cx, provision.SidecarHost("bot", "titogo", "bab"), before) expectPages(t, pages, noRoomSubject("ie")) }) t.Run("domain attach, a first route", func(t *testing.T) { pages := pagesTo(t) cx := haClaimCase(t, "ie") shipVersion(t, cx.consoleFixture, "titogo", "v1") seedChannel(t, cx.consoleFixture, "titogo", "edge", "v1") before := registrations(t, cx, "bab", "titogo") mark := cx.rec.mark() r := cx.apiReply(t, http.MethodPost, "/api/cli/apps/bab/titogo/domains", `{"host":"first.titogo-claim.example","kind":"static"}`) if r.Status != http.StatusServiceUnavailable || !strings.Contains(r.Body, noCapacityRefusal("ie")) { t.Fatalf("attach = %d %s", r.Status, r.Body) } wroteNothing(t, cx, mark) refusedBeforeRegistering(t, cx, "first.titogo-claim.example", before) expectPages(t, pages, noRoomSubject("ie")) }) t.Run("browser domain attach, a first route", func(t *testing.T) { pages := pagesTo(t) cx := haClaimCase(t, "ie") shipVersion(t, cx.consoleFixture, "titogo", "v1") seedChannel(t, cx.consoleFixture, "titogo", "edge", "v1") before := registrations(t, cx, "bab", "titogo") mark := cx.rec.mark() r := cx.formReply(t, "/a/bab/apps/titogo/domains/attach", "host", "first.titogo-claim.example", "kind", "static") if r.Status != http.StatusServiceUnavailable || !strings.Contains(r.Body, noCapacityRefusal("ie")) { t.Fatalf("browser attach = %d %s", r.Status, r.Body) } wroteNothing(t, cx, mark) refusedBeforeRegistering(t, cx, "first.titogo-claim.example", before) expectPages(t, pages, noRoomSubject("ie")) }) } // appDerivedWorld is titogo with no pool and no region allow-list, whose // app region is ie: a create that names no region takes the app's. func appDerivedWorld(t *testing.T, cx *claimCase) { t.Helper() m := titoManifest() m.Regions = nil m.Pool = "" enableInstances(t, cx.consoleFixture, "titogo", m) cx.srv.appRegionOf = func(int64, string) (string, string, error) { return "ie", "app", nil } } // Region HA spec 2 §7 (H27, Astra 2a round 1 finding 3): with region HA // on, an app-derived region is judged by the chooser's eligible set. Every // box stale is the no-room refusal, alert included, never the region // refusal's 400. func TestAppRegionWithEveryBoxStaleIsNoRoom(t *testing.T) { pages := pagesTo(t) cx := haClaimCase(t, "") appDerivedWorld(t, cx) publishHABox(t, cx, "box-ie-default", true, nil) before := registrations(t, cx, "bab", "titogo") r := cx.apiReply(t, http.MethodPost, "/api/cli/apps/bab/titogo/instances", `{"host":"acme.go.tito.io"}`) if r.Status != http.StatusServiceUnavailable || !strings.Contains(r.Body, noCapacityRefusal("ie")) { t.Fatalf("create = %d %s", r.Status, r.Body) } refusedBeforeRegistering(t, cx, "acme.go.tito.io", before) expectPages(t, pages, noRoomSubject("ie")) } // ... and a misconfigured sibling (a box in ie holding another secrets // key, outside the roster) no longer refuses a region whose eligible set // holds another box: the create goes ahead on that box. With region HA off // the sibling still refuses, as on main. func TestAppRegionWithAMisconfiguredSiblingStillCreates(t *testing.T) { cx := haClaimCase(t, "") appDerivedWorld(t, cx) publishHABox(t, cx, "box-ie-default", false, nil) publishHABox(t, cx, "box-ie-stray", false, func(m map[string]any) { m["secrets_key_fp"] = "sha256:another-key" }) off := cx.srv.placementChooser("") cx.srv.cfg.RegionHA = false r := cx.apiReply(t, http.MethodPost, "/api/cli/apps/bab/titogo/instances", `{"host":"other.go.tito.io"}`) if r.Status != http.StatusBadRequest { t.Fatalf("region HA off: create = %d %s, want main's region refusal", r.Status, r.Body) } cx.srv.cfg.RegionHA = true if off == nil { t.Fatal("no chooser with region HA on") } r = cx.apiReply(t, http.MethodPost, "/api/cli/apps/bab/titogo/instances", `{"host":"acme.go.tito.io"}`) if r.Status != http.StatusCreated && r.Status != http.StatusAccepted { t.Fatalf("create = %d %s", r.Status, r.Body) } obs, err := hostclaim.Read(context.Background(), cx.rel, "acme.go.tito.io") if err != nil || obs.Absent { t.Fatalf("claim: %+v %v", obs, err) } if exec, chosen := obs.Claim.Op.Exec, obs.Claim.Op.Intent["selected_executor"]; exec != "box-ie-default" && chosen != "box-ie-default" { t.Fatalf("executor %q / %q, want box-ie-default", exec, chosen) } } // putSeedStore gives titogo's canary environment a store a seed made, and // returns the store record's bytes. func putSeedStore(t *testing.T, cx *claimCase) (key string, body []byte) { t.Helper() ctx := context.Background() at := time.Now().UTC() if _, err := objstore.PutIn(ctx, cx.rel, "bab", "titogo", "canary", &objstore.Record{ Declared: &objstore.Declared{EnvPrefix: "S3", At: at, By: "bob@keymail.dev", Seed: newSeedID(t)}, Grant: &objstore.GrantBlock{Backend: "mem", Bucket: "carlos-bab-titogo-canary", Source: objstore.GrantSourceAccountApproval, At: at, By: "operator@keymail.dev"}, }, ""); err != nil { t.Fatal(err) } key = objstore.KeyIn("bab", "titogo", "canary") body, _, err := cx.rel.Get(ctx, key) if err != nil { t.Fatal(err) } return key, body } // Astra 2a round 2 finding 2: the instance create door chooses before its // first write. A create naming an environment whose store a seed made would // take that store (bindEnvironmentCreate). Refused for want of room, it // leaves the store's bytes as they were and writes nothing at all: when // every box is stale (the region check refuses), and when the region's last // placeable box goes between the region check and the choice (the early // choice refuses, before the binding). With room, the same create takes the // store, so the binding is on the path. func TestRefusedCreateLeavesTheEnvironmentsStoreAlone(t *testing.T) { const body = `{"host":"acme.go.tito.io","region":"ie","environment":"canary"}` refused := func(t *testing.T, cx *claimCase, pages chan string) { t.Helper() key, stored := putSeedStore(t, cx) before := registrations(t, cx, "bab", "titogo") mark := cx.rec.mark() r := cx.apiReply(t, http.MethodPost, "/api/cli/apps/bab/titogo/instances", body) if r.Status != http.StatusServiceUnavailable || !strings.Contains(r.Body, noCapacityRefusal("ie")) { t.Fatalf("create = %d %s", r.Status, r.Body) } wroteNothing(t, cx, mark) if now, _, err := cx.rel.Get(context.Background(), key); err != nil || string(now) != string(stored) { t.Fatalf("the store after the refusal:\n%s\nwant\n%s (%v)", now, stored, err) } refusedBeforeRegistering(t, cx, "acme.go.tito.io", before) expectPages(t, pages, noRoomSubject("ie")) } t.Run("every box stale", func(t *testing.T) { pages := pagesTo(t) cx := haClaimCase(t, "") cx.instanceWorld(t) publishHABox(t, cx, "box-ie-tito", true, nil) refused(t, cx, pages) publishHABox(t, cx, "box-ie-tito", false, nil) r := cx.apiReply(t, http.MethodPost, "/api/cli/apps/bab/titogo/instances", body) if r.Status != http.StatusCreated && r.Status != http.StatusAccepted { t.Fatalf("create with room = %d %s", r.Status, r.Body) } store, _, err := objstore.GetIn(context.Background(), cx.rel, "bab", "titogo", "canary") if err != nil || store == nil || store.Declared == nil || store.Declared.Seed != "" { t.Fatalf("the store after a create with room: %+v %v, want the seed's mark cleared", store, err) } }) t.Run("the last box goes before the choice", func(t *testing.T) { pages := pagesTo(t) cx := haClaimCase(t, "") cx.instanceWorld(t) publishHABox(t, cx, "box-ie-tito", false, nil) x := cx.srv.cfg.ClaimAuthority.Executors edges, reads := x.Edges, 0 x.Edges = func(ctx context.Context) (boxchoice.Snapshot, error) { if reads++; reads > 1 { return boxchoice.Snapshot{Complete: true}, nil } return edges(ctx) } refused(t, cx, pages) }) } // Astra 2a round 2 finding 1, at the create door: with region HA on, a // region whose boxes publish no public address (or no edges file) is a // region with no room, not the armed-placement gate's 400. With region HA // off the gate answers, as on main. func TestCreateRegionWithNoReachableBoxIsNoRoom(t *testing.T) { const body = `{"host":"acme.go.tito.io","region":"ie"}` pages := pagesTo(t) cx := haClaimCase(t, "") cx.instanceWorld(t) publishHABox(t, cx, "box-ie-tito", false, func(m map[string]any) { m["eip"] = "" }) cx.srv.cfg.RegionHA = false if r := cx.apiReply(t, http.MethodPost, "/api/cli/apps/bab/titogo/instances", body); r.Status != http.StatusBadRequest { t.Fatalf("region HA off: create = %d %s, want main's 400", r.Status, r.Body) } cx.srv.cfg.RegionHA = true before := registrations(t, cx, "bab", "titogo") mark := cx.rec.mark() r := cx.apiReply(t, http.MethodPost, "/api/cli/apps/bab/titogo/instances", body) if r.Status != http.StatusServiceUnavailable || !strings.Contains(r.Body, noCapacityRefusal("ie")) { t.Fatalf("create = %d %s", r.Status, r.Body) } wroteNothing(t, cx, mark) refusedBeforeRegistering(t, cx, "acme.go.tito.io", before) expectPages(t, pages, noRoomSubject("ie")) } // Census ruling region-ha-2a rev 2, item 3: the early choice takes no lease // and writes nothing, and its executor is the one the authority chooses on // its own (the create's choice before the early choice existed). A choice // the context already carries is kept: nothing is chosen again. func TestChooseFirstWritesNothingAndChoosesOnce(t *testing.T) { cx := haClaimCase(t, "") appDerivedWorld(t, cx) publishHABox(t, cx, "box-ie-default", false, nil) ctx := context.Background() host := "acme.go.tito.io" build := func(now time.Time, acquisition string) (hostops.Record, error) { rec := provision.Record{Schema: 1, Host: host, App: "titogo", Account: "bab", Channel: "stable", Kind: "static", CreatedBy: "bob@keymail.dev", Acquisition: acquisition} return hostops.PrepareInstance(rec, now, cx.srv.cfg.claimPlacementRegion("ie")) } probe, err := build(time.Now().UTC(), "") if err != nil { t.Fatal(err) } before, err := hostclaim.Read(ctx, cx.rel, host) if err != nil { t.Fatal(err) } want, chosen, err := cx.srv.cfg.ClaimAuthority.SelectExecutor(ctx, before, probe) if err != nil || !chosen || want.Box != "box-ie-default" { t.Fatalf("the authority's own choice: %+v %v %v", want, chosen, err) } mark := cx.rec.mark() got, err := cx.srv.chooseCreateFirst(ctx, host, build) if err != nil { t.Fatal(err) } wroteNothing(t, cx, mark) if sel, ok := hostops.ChosenExecutor(got, host); !ok || sel != want { t.Fatalf("early choice %+v (%v), want the authority's %+v", sel, ok, want) } again, err := cx.srv.chooseExecutorFirst(got, host, before, func(time.Time, string) (hostops.Record, error) { t.Fatal("chose again") return hostops.Record{}, nil }) if err != nil || again != got { t.Fatalf("a carried choice was not kept: %v", err) } if sel, chosen, err := cx.srv.cfg.ClaimAuthority.SelectExecutor(got, before, probe); err != nil || !chosen || sel != want { t.Fatalf("the create's selection %+v %v %v, want the door's %+v", sel, chosen, err, want) } } // Census ruling region-ha-2a item 5 (amadan #236): the early choice // carries the claim observation it was decided on, and beginClaimCreate // keeps it only if its own read of the claim is that observation. A claim // write landing between the choice and begin refuses the create with // main's retryable busy answer: the old choice is never used, nothing is // chosen again, and nothing is registered or written. func TestAStaleEarlyChoiceIsNeverUsed(t *testing.T) { cx := haClaimCase(t, "") cx.instanceWorld(t) publishHABox(t, cx, "box-ie-tito", false, nil) ctx := context.Background() const host = "acme.go.tito.io" acct, err := cx.st.AccountBySqid("bab") if err != nil || acct == nil { t.Fatalf("account: %+v %v", acct, err) } build := func(now time.Time, acquisition string) (hostops.Record, error) { rec := provision.Record{Schema: 1, Host: host, App: "titogo", Account: "bab", Channel: "stable", Kind: "static", Pool: "tito", Region: "ie", CreatedBy: "bob@keymail.dev", Acquisition: acquisition} return hostops.PrepareInstance(rec, now, cx.srv.cfg.claimPlacementRegion("ie")) } chosen, err := cx.srv.chooseCreateFirst(ctx, host, build) if _, ok := hostops.ChosenExecutor(chosen, host); err != nil || !ok { t.Fatalf("early choice: %v (chosen %v)", err, ok) } // A claim write lands between the choice and begin. created := time.Now().UTC().Truncate(time.Second) inc, err := hostclaim.Incarnation("bab", "titogo", host, created) if err != nil { t.Fatal(err) } writeServingClaim(t, cx.rel, host, inc, created, "box-ie-tito", "ie") written, _, err := cx.rel.Get(ctx, "control/hosts/"+host+".json") if err != nil { t.Fatal(err) } before := registrations(t, cx, "bab", "titogo") mark := cx.rec.mark() // The build runs only to choose again or, past RegisterStart, to write // the record: either is the old choice's create going on. req, err := cx.srv.beginClaimCreate(chosen, acct, "titogo", host, "bob@keymail.dev", func(time.Time, string) (hostops.Record, error) { t.Fatal("begin went on after the claim changed") return hostops.Record{}, nil }) if req != nil { req.Close() } if !errors.Is(err, errChoiceStale) || !errors.Is(err, hostclaim.ErrConflict) { t.Fatalf("begin on a changed claim = %v, want the stale-choice refusal", err) } if ref := claimResponseRefusal(err); ref.Status != http.StatusConflict || ref.Msg != claimRetireBusyMsg { t.Fatalf("refusal %+v, want main's retryable busy answer", ref) } wroteNothing(t, cx, mark) if n := registrations(t, cx, "bab", "titogo"); n != before { t.Fatalf("registrations %d, want %d", n, before) } if now, _, err := cx.rel.Get(ctx, "control/hosts/"+host+".json"); err != nil || string(now) != string(written) { t.Fatalf("the claim changed under the refusal: %v", err) } } // Region HA 2a, Task 6's carry (controller ruling): PlaceableBoxes and // Capacity choose nothing without a namespace, so every caller uses // placementChooser, scoped as ProductionAuthority.SelectExecutor scopes the // create. With a placeable box in each pool, the scoped chooser counts it, // in the shared pool and in a named one; the unscoped chooser the authority // holds counts nothing, which is why no door asks it. func TestPlacementChooserIsScoped(t *testing.T) { ctx := context.Background() cx := haClaimCase(t, "") publishHABox(t, cx, "box-ie-default", false, nil) publishHABox(t, cx, "box-ie-tito", false, nil) ie := cx.srv.cfg.claimPlacementRegion("ie") for pool, want := range map[string]string{"": "box-ie-default", "tito": "box-ie-tito"} { x := cx.srv.placementChooser(pool) if x == nil { t.Fatalf("pool %q: no chooser with region HA on", pool) } if x.Namespace == "" || x.Namespace != cx.srv.cfg.ClaimAuthority.Namespace { t.Fatalf("pool %q: chooser namespace %q, want the authority's %q", pool, x.Namespace, cx.srv.cfg.ClaimAuthority.Namespace) } n, _, err := x.Capacity(ctx, ie) if err != nil || n == 0 { t.Fatalf("pool %q: capacity %d %v, want a placeable box", pool, n, err) } boxes, err := x.PlaceableBoxes(ctx, ie) if err != nil || len(boxes) != 1 || boxes[0] != want { t.Fatalf("pool %q: placeable %v %v, want [%s]", pool, boxes, err, want) } if err := cx.srv.roomIn(ctx, "ie", pool); err != nil { t.Fatalf("pool %q: roomIn with a placeable box: %v", pool, err) } } if n, _, err := cx.srv.cfg.ClaimAuthority.Executors.Capacity(ctx, ie); err != nil || n != 0 { t.Fatalf("the unscoped chooser counted %d (%v): the scope is not what makes the count", n, err) } cx.srv.cfg.RegionHA = false if cx.srv.placementChooser("") != nil || cx.srv.roomIn(ctx, "ie", "") != nil { t.Fatal("region HA off: a chooser or a refusal") } } // Region HA 2a Task 11 fix round 1 (ruling: an incomplete placement read // is not "no room"). A malformed edges file beside a placeable box makes // the read incomplete: every choosing door answers main's temporarily // unavailable 503 (on the app-derived path C28's could-not-confirm), never // RHA-C17, pages nobody, and writes nothing. func TestAnIncompletePlacementReadIsNotNoRoom(t *testing.T) { malformed := func(t *testing.T, cx *claimCase) { t.Helper() if _, err := cx.rel.Put(context.Background(), "control/edges/box-x.json", []byte("{"), ""); err != nil { t.Fatal(err) } } unavailable := func(t *testing.T, r claimReply, msg string) { t.Helper() if r.Status != http.StatusServiceUnavailable || !strings.Contains(r.Body, msg) || strings.Contains(r.Body, noCapacityRefusal("ie")) { t.Fatalf("answer %d %s, want 503 %q and not the no-room refusal", r.Status, r.Body, msg) } } t.Run("instance create, an explicit region", func(t *testing.T) { pages := pagesTo(t) cx := haClaimCase(t, "") cx.instanceWorld(t) publishHABox(t, cx, "box-ie-tito", false, nil) malformed(t, cx) before := registrations(t, cx, "bab", "titogo") mark := cx.rec.mark() status, hdr, body := rawReq(t, http.MethodPost, cx.ts.URL+"/api/cli/apps/bab/titogo/instances", cx.bearer, `{"host":"acme.go.tito.io","region":"ie"}`) unavailable(t, claimReply{Status: status, Body: body}, claimResponseUnavailable) if got := hdr.Get("Retry-After"); got != "2" { t.Fatalf("Retry-After %q, want main's 2", got) } wroteNothing(t, cx, mark) refusedBeforeRegistering(t, cx, "acme.go.tito.io", before) expectPages(t, pages) }) t.Run("standalone sidecar create", func(t *testing.T) { pages := pagesTo(t) cx := haClaimCase(t, "") cx.sidecarWorld(t) publishHABox(t, cx, "box-ie-tito", false, nil) malformed(t, cx) before := registrations(t, cx, "bab", "titogo") mark := cx.rec.mark() unavailable(t, cx.apiReply(t, http.MethodPost, "/api/cli/apps/bab/titogo/sidecars", `{"name":"bot","region":"ie"}`), claimResponseUnavailable) wroteNothing(t, cx, mark) refusedBeforeRegistering(t, cx, provision.SidecarHost("bot", "titogo", "bab"), before) expectPages(t, pages) }) t.Run("domain attach, a first route", func(t *testing.T) { pages := pagesTo(t) cx := haClaimCase(t, "ie") shipVersion(t, cx.consoleFixture, "titogo", "v1") seedChannel(t, cx.consoleFixture, "titogo", "edge", "v1") publishHABox(t, cx, "box-ie-default", false, nil) malformed(t, cx) before := registrations(t, cx, "bab", "titogo") mark := cx.rec.mark() unavailable(t, cx.apiReply(t, http.MethodPost, "/api/cli/apps/bab/titogo/domains", `{"host":"first.titogo-claim.example","kind":"static"}`), claimResponseUnavailable) wroteNothing(t, cx, mark) refusedBeforeRegistering(t, cx, "first.titogo-claim.example", before) expectPages(t, pages) }) t.Run("instance create, the app's region", func(t *testing.T) { pages := pagesTo(t) cx := haClaimCase(t, "") appDerivedWorld(t, cx) publishHABox(t, cx, "box-ie-default", false, nil) malformed(t, cx) before := registrations(t, cx, "bab", "titogo") unavailable(t, cx.apiReply(t, http.MethodPost, "/api/cli/apps/bab/titogo/instances", `{"host":"acme.go.tito.io"}`), msgAppRegionUnconfirmed("ie")) refusedBeforeRegistering(t, cx, "acme.go.tito.io", before) expectPages(t, pages) }) } // Region HA 2a Task 11 fix round 1 (ruling): the no-capacity alert is // recorded as sent only once a send succeeds, so a failed send is retried // on the next refusal rather than silenced for an hour; a sent alert // still silences the region for the hour. func TestNoCapacityAlertIsRecordedOnlyWhenSent(t *testing.T) { var a noCapacityAlerts t0 := time.Date(2026, 10, 16, 12, 0, 0, 0, time.UTC) if !a.due("uk", t0) { t.Fatal("the first alert is not due") } if a.due("uk", t0.Add(time.Second)) { t.Fatal("a second alert while the first is being sent") } a.failed("uk", t0) if !a.due("uk", t0.Add(2*time.Second)) { t.Fatal("a failed send silenced the region") } a.failed("uk", t0) // an older send's failure does not free a newer one if a.due("uk", t0.Add(3*time.Second)) { t.Fatal("a stale failure freed the region") } sent := make(chan string, 16) calls := 0 old := pageNoCapacity pageNoCapacity = func(_ context.Context, level, subject, _ string) error { calls++ sent <- subject if calls == 1 { return errors.New("the mail relay is down") } return nil } t.Cleanup(func() { pageNoCapacity = old }) s := &Server{} ctx := context.Background() want, _ := noCapacityAlert("ie") if err := s.noRoom(ctx, "ie"); !errors.Is(err, hostops.ErrNoEligibleBox) { t.Fatalf("noRoom = %v", err) } if got := <-sent; got != want { t.Fatalf("page %q", got) } // The failed send is given back once it returns: a later refusal pages. deadline := time.Now().Add(5 * time.Second) for second := false; !second; { if time.Now().After(deadline) { t.Fatal("a failed alert was never sent again") } _ = s.noRoom(ctx, "ie") select { case got := <-sent: if got != want { t.Fatalf("page %q", got) } second = true case <-time.After(20 * time.Millisecond): } } // That one was sent: the region is quiet for the hour. for range 3 { _ = s.noRoom(ctx, "ie") } select { case got := <-sent: t.Fatalf("an extra page after a sent alert: %q", got) case <-time.After(200 * time.Millisecond): } } // Region HA 2a Task 11 fix round 1 (ruling): the no-room 503 carries // Retry-After: 60, matching RHA-C17's "try again in a few minutes", on // every choosing door; every other claim refusal keeps its own. func TestNoRoomCarriesRetryAfterAMinute(t *testing.T) { pagesTo(t) for _, d := range []struct { name, home, path, body string world func(*testing.T, *claimCase) }{ {"instance create", "", "/api/cli/apps/bab/titogo/instances", `{"host":"acme.go.tito.io","region":"ie"}`, func(t *testing.T, cx *claimCase) { cx.instanceWorld(t) }}, {"standalone sidecar create", "", "/api/cli/apps/bab/titogo/sidecars", `{"name":"bot","region":"ie"}`, func(t *testing.T, cx *claimCase) { cx.sidecarWorld(t) }}, {"domain attach, a first route", "ie", "/api/cli/apps/bab/titogo/domains", `{"host":"first.titogo-claim.example","kind":"static"}`, func(t *testing.T, cx *claimCase) { shipVersion(t, cx.consoleFixture, "titogo", "v1") seedChannel(t, cx.consoleFixture, "titogo", "edge", "v1") }}, } { cx := haClaimCase(t, d.home) d.world(t, cx) publishHABox(t, cx, "box-ie-tito", true, nil) publishHABox(t, cx, "box-ie-default", true, nil) status, hdr, body := rawReq(t, http.MethodPost, cx.ts.URL+d.path, cx.bearer, d.body) if status != http.StatusServiceUnavailable || !strings.Contains(body, noCapacityRefusal("ie")) || hdr.Get("Retry-After") != "60" { t.Fatalf("%s: %d Retry-After %q %s, want the no-room 503 with Retry-After 60", d.name, status, hdr.Get("Retry-After"), body) } } cx := haClaimCase(t, "") // The browser's attach door renders the same refusal. req := httptest.NewRequest(http.MethodPost, "/a/bab/apps/titogo/domains/attach", nil) rec := httptest.NewRecorder() cx.srv.renderDoorRefusal(rec, req, claimResponseRefusal(&noCapacityError{Region: "ie"})) if rec.Code != http.StatusServiceUnavailable || rec.Header().Get("Retry-After") != "60" { t.Fatalf("browser refusal %d Retry-After %q", rec.Code, rec.Header().Get("Retry-After")) } // Other refusals are as they were. rec = httptest.NewRecorder() writeClaimResponseError(rec, hostclaim.ErrConflict) if rec.Header().Get("Retry-After") != "2" { t.Fatalf("busy refusal Retry-After %q, want 2", rec.Header().Get("Retry-After")) } rec = httptest.NewRecorder() writeClaimResponseError(rec, hostops.ErrEvidenceIncomplete) if rec.Code != http.StatusServiceUnavailable || rec.Header().Get("Retry-After") != "2" { t.Fatalf("unavailable refusal %d Retry-After %q, want 503 and 2", rec.Code, rec.Header().Get("Retry-After")) } rec = httptest.NewRecorder() cx.srv.renderDoorRefusal(rec, req, &doorRefusal{http.StatusServiceUnavailable, claimResponseUnavailable}) if rec.Header().Get("Retry-After") != "" { t.Fatalf("browser unavailable refusal Retry-After %q, want none (main's)", rec.Header().Get("Retry-After")) } }