package console // The relay's doors (relay spec §5), through the real mux and a real bearer // token. What is pinned here, in the order it would hurt if lost: a stranger // probing the relay learns nothing — its refusal is byte-for-byte the STORE // door's refusal, so the two cannot be told apart by anything but the path; // a deployment with no armed region refuses to enable and NAMES the variable // an operator would set, rather than minting a key nothing can verify; status // tells the truth about a region whose relay has gone quiet, because "stale" // is the only way a member with no box access finds out; and rotate — the one // door with no account in its path — answers a caller who is not on // CARLOS_RELAY_OPERATORS with the same 404 an unknown account gets, never a // 403 that would confirm this deployment runs a relay at all. import ( "context" "maps" "net/http" "strings" "testing" "time" "github.com/carlosframework/platform/internal/appconfig" "github.com/carlosframework/platform/internal/provision" "github.com/carlosframework/platform/internal/registry" "github.com/carlosframework/platform/internal/relay" ) // relayURL is the app-scoped family's root: enable POSTs here, status GETs // here, and disable/adopted hang off it. func relayURL(fx *consoleFixture, app string) string { return fx.ts.URL + "/api/cli/apps/" + fx.sqid + "/" + app + "/relay" } // seedHeartbeat writes one armed relay's heartbeat as the box itself would, // with its last SUCCESSFUL authority read `age` in the past — the clock // status reads, and the one a dead relay cannot move. func seedHeartbeat(t *testing.T, fx *consoleFixture, box, region string, age time.Duration, kids ...string) { t.Helper() seedHeartbeatRetiring(t, fx, box, region, age, nil, kids...) } // seedHeartbeatRetiring is seedHeartbeat with the retiring map set, for the // one test that covers the hop from a relay's heartbeat to the status reply. func seedHeartbeatRetiring(t *testing.T, fx *consoleFixture, box, region string, age time.Duration, retiring map[string]time.Time, kids ...string) { t.Helper() now := time.Now() hb := relay.Heartbeat{ Box: box, Region: region, ReadAt: now.Add(-age), WrittenAt: now.Add(-age), Listeners: []string{"udp:3478", "tcp:3478"}, Kids: kids, Retiring: retiring, } if err := relay.WriteHeartbeat(context.Background(), fx.rel, hb); err != nil { t.Fatalf("seed heartbeat: %v", err) } } // region pulls one row out of a status reply's region_status list. func regionRow(t *testing.T, out map[string]any, i int) map[string]any { t.Helper() rows, ok := out["region_status"].([]any) if !ok || len(rows) <= i { t.Fatalf("region_status = %#v, want at least %d row(s)", out["region_status"], i+1) } row, ok := rows[i].(map[string]any) if !ok { t.Fatalf("region_status[%d] = %#v, want an object", i, rows[i]) } return row } // ---- enable, status, disable ------------------------------------------------- // TestAPIRelayEnableThenStatusThenDisable is the member's whole path through // the doors: turn it on, read what was issued and how the region looks, turn // it off. Each assertion is on something only the door can get wrong — the // operations themselves are pinned in relayops_test.go. func TestAPIRelayEnableThenStatusThenDisable(t *testing.T) { fx, _ := relayFixture(t, "code-relay-door") status, out := postJSON(t, relayURL(fx, "titogo"), map[string]any{}, cliToken(t, fx, "alice@keymail.dev", "code-relay-door-2")) if status != http.StatusOK { t.Fatalf("enable = %d %v, want 200", status, out) } if out["enabled"] != true { t.Fatalf("enabled = %v, want true", out["enabled"]) } inc := str(t, out, "incarnation") if len(inc) != 16 { t.Fatalf("incarnation = %q, want 16 hex characters", inc) } if got := strList(t, out, "regions"); len(got) != 1 || got[0] != "eu-west-1" { t.Fatalf("regions = %v, want the one armed region", got) } // The username suffix is echoed, the key never is: the suffix is the same // public identity the app's own CARLOS_RELAY carries, and a member needs // it to match a call's credential against a rotation. issued, ok := out["issued"].(map[string]any) if !ok || !strings.Contains(issued["eu-west-1"].(string), inc) { t.Fatalf("issued = %#v, want a suffix naming the incarnation", out["issued"]) } // Nothing else: no key, no envelope, no variable. The record holds the // digest of what was sealed and the console cannot open it anyway, but a // door that grew a "value" field would be the one place a key could leak // back out of this system. for _, k := range []string{"value", "key", "envelope", "secret", "master"} { if _, ok := out[k]; ok { t.Fatalf("the enable reply carries %q — nothing key-shaped may cross this door", k) } } // A relay that read the authority six minutes ago is STALE, and the lease // it serves on is still live (fifteen minutes) — two different clocks, and // a member debugging a failing call needs both. token := cliToken(t, fx, "alice@keymail.dev", "code-relay-door-3") seedHeartbeat(t, fx, "box1", "eu-west-1", 6*time.Minute, "k1") status, out = getJSON(t, relayURL(fx, "titogo"), token) if status != http.StatusOK { t.Fatalf("status = %d %v, want 200", status, out) } row := regionRow(t, out, 0) if row["region"] != "eu-west-1" || row["box"] != "box1" { t.Fatalf("region row = %#v, want the seeded box", row) } if row["stale"] != true { t.Fatalf("stale = %v on a six-minute-old heartbeat, want true", row["stale"]) } if row["lease_live"] != true { t.Fatalf("lease_live = %v six minutes in, want true (the lease runs 15)", row["lease_live"]) } if age, _ := row["heartbeat_age"].(string); !strings.HasPrefix(age, "6m") { t.Fatalf("heartbeat_age = %v, want about six minutes", row["heartbeat_age"]) } if kids := strList(t, row, "kids"); len(kids) != 1 || kids[0] != "k1" { t.Fatalf("kids = %v, want the loaded master", kids) } status, out = postJSON(t, relayURL(fx, "titogo")+"/disable", map[string]any{}, token) if status != http.StatusOK { t.Fatalf("disable = %d %v, want 200", status, out) } if out["enabled"] != false { t.Fatalf("enabled = %v after a disable, want false", out["enabled"]) } // The disable is not COMPLETE until the region's own relay heartbeats an // authority written at or after it (spec §5). The seeded heartbeat is six // minutes older than the publish, so the region is still pending. if pending := strList(t, out, "disable_pending"); len(pending) != 1 || pending[0] != "eu-west-1" { t.Fatalf("disable_pending = %v, want the region whose relay has not caught up", pending) } if rec := relayRecord(t, fx, "titogo"); rec == nil || rec.Enabled { t.Fatalf("record = %+v after the disable door, want it off", rec) } } // TestAPIRelayStatusBeforeEnableIs404: "never enabled" is a state, not a // failure, and it is a sentence rather than an empty record — handleAPIStoreGet's // same choice. It speaks only because apiAccountApp has already proved // membership above it. func TestAPIRelayStatusBeforeEnableIs404(t *testing.T) { fx, _ := relayFixture(t, "code-relay-unenabled") token := cliToken(t, fx, "alice@keymail.dev", "code-relay-unenabled-2") status, out := getJSON(t, relayURL(fx, "titogo"), token) if status != http.StatusNotFound { t.Fatalf("status with no record = %d %v, want 404", status, out) } if !strings.Contains(str(t, out, "error"), "carlos relay enable") { t.Fatalf("error = %q, want the sentence naming the verb that fixes it", out["error"]) } } // TestAPIRelayEnableWithoutRegionsIs503: a deployment that arms no region // refuses, and the refusal names CARLOS_RELAY_REGIONS. A 500 or a silent // success would both be worse than this: the first tells a member nothing, // the second would mint an identity no relay would ever accept. func TestAPIRelayEnableWithoutRegionsIs503(t *testing.T) { fx, _ := relayFixture(t, "code-relay-noregions") fx.srv.cfg.RelayRegions = nil token := cliToken(t, fx, "alice@keymail.dev", "code-relay-noregions-2") status, out := postJSON(t, relayURL(fx, "titogo"), map[string]any{}, token) if status != http.StatusServiceUnavailable { t.Fatalf("enable with no armed region = %d %v, want 503", status, out) } if str(t, out, "error") != errRelayNoRegions.Error() { t.Fatalf("error = %q, want the sentinel verbatim", out["error"]) } if rec := relayRecord(t, fx, "titogo"); rec != nil { t.Fatalf("a refused enable wrote a record: %+v", rec) } } // TestAPIRelayEnableNamingAnUnarmedRegionIs400: a region the caller typed // that this deployment does not run is the CALLER's mistake, so it is a 400 // naming the armed ones — not the 502 an unmatched error would read as. func TestAPIRelayEnableNamingAnUnarmedRegionIs400(t *testing.T) { fx, _ := relayFixture(t, "code-relay-badregion") token := cliToken(t, fx, "alice@keymail.dev", "code-relay-badregion-2") status, out := postJSON(t, relayURL(fx, "titogo"), map[string]any{"regions": []string{"us-east-1"}}, token) if status != http.StatusBadRequest { t.Fatalf("enable in an unarmed region = %d %v, want 400", status, out) } if msg := str(t, out, "error"); !strings.Contains(msg, "us-east-1") || !strings.Contains(msg, "eu-west-1") { t.Fatalf("error = %q, want the region asked for and the ones armed", msg) } } // TestAPIRelayAdoptedUnknownHostIs404: the ack must name a route of THIS app. // Without that the record would grow a cutover for anything a caller typed, // and status would report identities against routes that do not exist. func TestAPIRelayAdoptedUnknownHostIs404(t *testing.T) { fx, _ := relayFixture(t, "code-relay-adopted") token := cliToken(t, fx, "alice@keymail.dev", "code-relay-adopted-2") if status, out := postJSON(t, relayURL(fx, "titogo"), map[string]any{}, token); status != http.StatusOK { t.Fatalf("enable = %d %v, want 200", status, out) } status, out := postJSON(t, relayURL(fx, "titogo")+"/adopted", map[string]any{"host": "nowhere.example"}, token) if status != http.StatusNotFound { t.Fatalf("adopted for an unknown host = %d %v, want 404", status, out) } if msg := str(t, out, "error"); !strings.Contains(msg, "nowhere.example") { t.Fatalf("error = %q, want the host named", msg) } // An empty host is the caller's mistake before any lookup happens. if status, _ := postJSON(t, relayURL(fx, "titogo")+"/adopted", map[string]any{"host": " "}, token); status != http.StatusBadRequest { t.Fatalf("adopted with no host = %d, want 400", status) } if rec := relayRecord(t, fx, "titogo"); rec != nil && len(rec.Cutovers) != 0 { t.Fatalf("a refused ack wrote a cutover: %+v", rec.Cutovers) } } // TestAPIRelayAdoptedRefusesARouteThePlatformRestarts: the member verb // exists for the ONE route shape this platform cannot restart. A unit or // socket-exec route is bounced by the config sync itself, so its cutover is // an observation — the ack that sync writes, naming what the host actually // rendered. A member's word here would stamp the identities the console // ISSUED over that observation, and a route whose restart quietly failed // would then read as cut over and let the old master be retired under it. func TestAPIRelayAdoptedRefusesARouteThePlatformRestarts(t *testing.T) { fx, _ := relayFixture(t, "code-relay-adopted-shape") token := cliToken(t, fx, "alice@keymail.dev", "code-relay-adopted-shape-2") fx.addRoute(t, registry.Route{Host: "s.titogo.oncarlos.com", App: "titogo", Kind: "service", Socket: "/run/titogo-s.sock", Status: "ready"}) if status, out := postJSON(t, relayURL(fx, "titogo"), map[string]any{}, token); status != http.StatusOK { t.Fatalf("enable = %d %v, want 200", status, out) } status, out := postJSON(t, relayURL(fx, "titogo")+"/adopted", map[string]any{"host": "s.titogo.oncarlos.com"}, token) if status != http.StatusBadRequest { t.Fatalf("adopted for a socket-exec route = %d %v, want 400", status, out) } if msg := str(t, out, "error"); !strings.Contains(msg, "the platform restarts this route itself; its adoption is observed, not declared") { t.Fatalf("error = %q, want the reason in those words", msg) } if rec := relayRecord(t, fx, "titogo"); rec != nil && len(rec.Cutovers) != 0 { t.Fatalf("a refused ack wrote a cutover: %+v", rec.Cutovers) } } // TestAPIRelayMultiUpstreamRouteIsRefusedThenReadsAsTCPExecFile: a record // names its upstream EITHER as the legacy Addr or as the Addrs set, never // both (provision.ValidateShape), so a multi-upstream external route has an // empty Addr. Read unfolded, it looked like a route with no address at all — // which the delivery check reads as deliverable — so the enable spec §5 step // 2 exists to refuse went through, status called the route "unknown", and // `carlos relay adopted` accepted it. Both halves are pinned here: the // refusal names the host, and once the bundle opts in, the same route reads // as the tcp-exec-file it is. func TestAPIRelayMultiUpstreamRouteIsRefusedThenReadsAsTCPExecFile(t *testing.T) { fx, _ := relayFixture(t, "code-relay-addrs") ctx := context.Background() token := cliToken(t, fx, "alice@keymail.dev", "code-relay-addrs-2") if err := provision.CreateRecord(ctx, fx.rel, provision.Record{ Schema: 1, Host: "m.titogo.oncarlos.com", Slug: "m", App: "titogo", Account: fx.sqid, Kind: "service", Channel: "stable", Addrs: []string{"10.0.0.5:8080", "10.0.0.6:8080"}, CreatedAt: time.Now().UTC(), }); err != nil { t.Fatalf("seed a multi-upstream record: %v", err) } status, out := postJSON(t, relayURL(fx, "titogo"), map[string]any{}, token) if status != http.StatusBadRequest { t.Fatalf("enable over a multi-upstream route with no file delivery = %d %v, want 400", status, out) } if msg := str(t, out, "error"); !strings.Contains(msg, "m.titogo.oncarlos.com") { t.Fatalf("error = %q, want the host named", msg) } if err := appconfig.SetExecDelivery(ctx, fx.rel, fx.sqid, "titogo", appconfig.ExecDeliveryFile); err != nil { t.Fatal(err) } if status, out := postJSON(t, relayURL(fx, "titogo"), map[string]any{}, token); status != http.StatusOK { t.Fatalf("enable after opting into file delivery = %d %v, want 200", status, out) } _, out = getJSON(t, relayURL(fx, "titogo"), token) wantRoute(t, relayRouteRows(t, out), "m.titogo.oncarlos.com", "tcp-exec-file", "written, awaiting delivery") } // ---- the gate ---------------------------------------------------------------- // TestAPIRelayGateIsByteIdenticalWithTheStore: every app-scoped relay door // refuses a stranger, an account that does not exist and an app that was // never claimed with ONE answer — and that answer is byte-for-byte what the // STORE door gives the same caller. The cross-door comparison is the point: // if the relay's 404 ever drifted from its neighbour's, the difference itself // would tell a token holder which of a deployment's apps have a relay. func TestAPIRelayGateIsByteIdenticalWithTheStore(t *testing.T) { fx, _ := relayFixture(t, "code-relay-gate") token := cliToken(t, fx, "alice@keymail.dev", "code-relay-gate-member") eve := cliToken(t, fx, "eve@keymail.dev", "code-relay-gate-eve") // The store door's refusal for the same caller, as the yardstick. _, wantBody := getRaw(t, storeURL(fx, "titogo"), eve) doors := []struct { name string call func(u, bearer string) (int, string) }{ {"GET", func(u, bearer string) (int, string) { return getRaw(t, u, bearer) }}, {"POST", func(u, bearer string) (int, string) { return postRaw(t, u, map[string]any{}, bearer) }}, {"POST disable", func(u, bearer string) (int, string) { return postRaw(t, u+"/disable", map[string]any{}, bearer) }}, {"POST adopted", func(u, bearer string) (int, string) { return postRaw(t, u+"/adopted", map[string]any{"host": "x"}, bearer) }}, } for _, d := range doors { t.Run(d.name, func(t *testing.T) { // (a) a member of nothing, against a real account and a real app. statusA, bodyA := d.call(relayURL(fx, "titogo"), eve) // (b) a member, against an account that does not exist. statusB, bodyB := d.call(fx.ts.URL+"/api/cli/apps/zzzz/titogo/relay", token) // (c) a member of the account, against an app it never claimed. statusC, bodyC := d.call(relayURL(fx, "ghost-app"), token) if statusA != http.StatusNotFound || statusB != http.StatusNotFound || statusC != http.StatusNotFound { t.Fatalf("%s gate = %d / %d / %d, want 404 for all three", d.name, statusA, statusB, statusC) } if bodyA != bodyB || bodyB != bodyC { t.Fatalf("%s refusals are not byte-identical:\n(a) %q\n(b) %q\n(c) %q", d.name, bodyA, bodyB, bodyC) } if bodyA != wantBody { t.Fatalf("%s refusal body = %q, want the store door's own refusal %q", d.name, bodyA, wantBody) } }) } if rec := relayRecord(t, fx, "titogo"); rec != nil { t.Fatalf("a refused request wrote a record: %+v", rec) } // The token gate runs first, before any name is resolved. if status, _ := getJSON(t, relayURL(fx, "titogo"), ""); status != http.StatusUnauthorized { t.Fatalf("status with no token = %d, want 401", status) } if status, _ := postJSON(t, relayURL(fx, "titogo"), map[string]any{}, ""); status != http.StatusUnauthorized { t.Fatalf("enable with no token = %d, want 401", status) } } // ---- rotate ------------------------------------------------------------------ // TestAPIRelayRotateNeedsAnOperatorAndSaysNothingOtherwise: the one door with // no account segment. A member in good standing who is not on // CARLOS_RELAY_OPERATORS gets http.NotFound's own bytes — not a 403 naming // the variable, which would tell any valid token that this deployment runs a // relay worth rotating. Listed, the same caller rotates and is told which kid // the region issues from now. func TestAPIRelayRotateNeedsAnOperatorAndSaysNothingOtherwise(t *testing.T) { fx, _ := relayFixture(t, "code-relay-rotate-door") token := cliToken(t, fx, "alice@keymail.dev", "code-relay-rotate-door-2") rotate := fx.ts.URL + "/api/cli/relay/rotate" status, body := postRaw(t, rotate, map[string]any{"region": "eu-west-1"}, token) if status != http.StatusNotFound { t.Fatalf("rotate by a non-operator = %d %q, want 404", status, body) } if body != "404 page not found\n" { t.Fatalf("rotate refusal body = %q, want the stdlib http.NotFound output", body) } if status, _ := postRaw(t, rotate, map[string]any{"region": "eu-west-1"}, ""); status != http.StatusUnauthorized { t.Fatalf("rotate with no token = %d, want 401", status) } if status, out := postJSON(t, relayURL(fx, "titogo"), map[string]any{}, token); status != http.StatusOK { t.Fatalf("enable = %d %v, want 200", status, out) } fx.srv.cfg.RelayOperators = []string{"alice@keymail.dev"} // A rotation refuses until every live relay in the region has loaded the // staged kid, and that refusal is the operator's to fix — a 400 naming the // box, not a 502. t.Setenv(relayMastersEnv("eu-west-1"), "k1:"+relayTestMaster(1)+",k2:"+relayTestMaster(2)) seedHeartbeat(t, fx, "box1", "eu-west-1", time.Minute, "k1") status, out := postJSON(t, rotate, map[string]any{"region": "eu-west-1"}, token) if status != http.StatusBadRequest { t.Fatalf("rotate before the box loaded the kid = %d %v, want 400", status, out) } if msg := str(t, out, "error"); !strings.Contains(msg, "box1") || !strings.Contains(msg, "k2") { t.Fatalf("error = %q, want the box and the kid named", msg) } seedHeartbeat(t, fx, "box1", "eu-west-1", time.Minute, "k1", "k2") status, out = postJSON(t, rotate, map[string]any{"region": "eu-west-1"}, token) if status != http.StatusOK { t.Fatalf("rotate = %d %v, want 200", status, out) } if out["kid"] != "k2" || out["region"] != "eu-west-1" { t.Fatalf("rotate reply = %v, want the region on its new kid", out) } if apps, _ := out["apps"].(float64); apps != 1 { t.Fatalf("apps = %v, want the one enabled app re-issued", out["apps"]) } retire, err := time.Parse(time.RFC3339, str(t, out, "retire_no_earlier_than")) if err != nil { t.Fatalf("retire_no_earlier_than = %q: %v", out["retire_no_earlier_than"], err) } if d := time.Until(retire); d < 27*time.Hour || d > 29*time.Hour { t.Fatalf("retire_no_earlier_than is %s away, want about 28h", d) } // An empty region is the caller's own mistake, before anything moves. if status, _ := postJSON(t, rotate, map[string]any{}, token); status != http.StatusBadRequest { t.Fatalf("rotate with no region = %d, want 400", status) } } // ---- routes: what each shape is, and whose move it is ------------------------ // relayRouteRows indexes a status reply's routes by host, so an assertion // names the route it is about rather than a position in a list. func relayRouteRows(t *testing.T, out map[string]any) map[string]map[string]any { t.Helper() raw, _ := out["routes"].([]any) rows := map[string]map[string]any{} for _, r := range raw { row, ok := r.(map[string]any) if !ok { t.Fatalf("route row = %#v, want an object", r) } rows[str(t, row, "host")] = row } return rows } // wantRoute asserts one route row's shape and that its delivery sentence // contains each fragment. func wantRoute(t *testing.T, rows map[string]map[string]any, host, shape string, deliveryHas ...string) { t.Helper() row, ok := rows[host] if !ok { t.Fatalf("no route row for %s (have %v)", host, rows) } if got := str(t, row, "shape"); got != shape { t.Fatalf("%s shape = %q, want %q", host, got, shape) } got := str(t, row, "delivery") for _, want := range deliveryHas { if !strings.Contains(got, want) { t.Fatalf("%s delivery = %q, want it to contain %q", host, got, want) } } } // TestAPIRelayStatusNamesEachRouteShapeAndWhoseMoveItIs is the delivery half // of `carlos relay status`, which is the only sight a member with no box // access has of it. Four routes, four different answers: // // - a unit and a socket-exec are restarted by the config sync, so the key // arrives without anyone being asked and the row says only that it was // written; // - a TCP-address route with file delivery is the one shape this platform // cannot restart, so its row says whose move it is — spec §5's // "awaiting `carlos relay adopted`" — rather than leaving a member to // wonder why a rotation never completes; // - a route with NO config delivery, added after the enable that would have // refused it, is named as the fault it is. `adopted` refuses it too: a // cutover there would claim a process holds a key nothing ever wrote for // it, and a retirement would later be checked against that claim. // // Then the ack itself: recorded, it reads as adopted; after a rotation // re-issues the app under a new kid, the SAME ack must stop reading as // adopted, because it named identities that no longer exist. func TestAPIRelayStatusNamesEachRouteShapeAndWhoseMoveItIs(t *testing.T) { fx, _ := relayFixture(t, "code-relay-shapes") ctx := context.Background() token := cliToken(t, fx, "alice@keymail.dev", "code-relay-shapes-2") // One named bundle opts into file delivery; the app's default does not. // That is what lets two otherwise identical TCP routes differ, and it is // the per-route precedence relayExecDelivery applies (#195). if err := appconfig.SetExecDeliveryAt(ctx, fx.rel, fx.sqid, "titogo", "prod", appconfig.ExecDeliveryFile); err != nil { t.Fatal(err) } fx.addRoute(t, registry.Route{Host: "a.titogo.oncarlos.com", App: "titogo", Kind: "service", Unit: "titogo-a.service", Status: "ready"}) fx.addRoute(t, registry.Route{Host: "b.titogo.oncarlos.com", App: "titogo", Kind: "service", Socket: "/run/titogo-b.sock", Status: "ready"}) fx.addRoute(t, registry.Route{Host: "c.titogo.oncarlos.com", App: "titogo", Kind: "service", Addr: "10.0.0.5:8080", Environment: "prod", Status: "ready"}) if status, out := postJSON(t, relayURL(fx, "titogo"), map[string]any{}, token); status != http.StatusOK { t.Fatalf("enable = %d %v, want 200", status, out) } // Added AFTER the enable, which is the only way it can exist: enable // itself refuses an app that already has one. fx.addRoute(t, registry.Route{Host: "d.titogo.oncarlos.com", App: "titogo", Kind: "service", Addr: "10.0.0.6:8080", Status: "ready"}) status, out := getJSON(t, relayURL(fx, "titogo"), token) if status != http.StatusOK { t.Fatalf("status = %d %v, want 200", status, out) } rows := relayRouteRows(t, out) if len(rows) != 4 { t.Fatalf("routes = %v, want four rows", rows) } wantRoute(t, rows, "a.titogo.oncarlos.com", "unit", "written, awaiting delivery") wantRoute(t, rows, "b.titogo.oncarlos.com", "socket-exec", "written, awaiting delivery") // No box has synced yet, so no config-ack exists — c reads exactly like // a and b until one does (Task 12: a tcp-exec-file route's "file // delivered, awaiting adopted" sentence is earned by an ack, not // assumed from its shape alone). wantRoute(t, rows, "c.titogo.oncarlos.com", "tcp-exec-file", "written, awaiting delivery") wantRoute(t, rows, "d.titogo.oncarlos.com", "no-delivery", "no config delivery", "exec_delivery file") // Now a box's config sync delivers the file and writes its ack // (appconfig.WriteConfigAck, seeded here exactly as syncExecHost would) // — only then does c's row say whose move it is. issued := relayRecord(t, fx, "titogo").IssuedSuffixes if err := appconfig.WriteConfigAck(ctx, fx.rel, appconfig.ConfigAck{ Box: "box1", Host: "c.titogo.oncarlos.com", Shape: "tcp-exec-file", At: time.Now(), RelaySuffixes: maps.Clone(issued), }); err != nil { t.Fatalf("seed ack c: %v", err) } _, out = getJSON(t, relayURL(fx, "titogo"), token) rows = relayRouteRows(t, out) wantRoute(t, rows, "c.titogo.oncarlos.com", "tcp-exec-file", "carlos relay adopted --host c.titogo.oncarlos.com") // The undeliverable route cannot be acknowledged away. status, out = postJSON(t, relayURL(fx, "titogo")+"/adopted", map[string]any{"host": "d.titogo.oncarlos.com"}, token) if status != http.StatusBadRequest { t.Fatalf("adopted for a no-delivery route = %d %v, want 400", status, out) } if msg := str(t, out, "error"); !strings.Contains(msg, "no config delivery") { t.Fatalf("error = %q, want the reason named", msg) } // The deliverable one can, and then reads as adopted. if status, out := postJSON(t, relayURL(fx, "titogo")+"/adopted", map[string]any{"host": "c.titogo.oncarlos.com"}, token); status != http.StatusOK { t.Fatalf("adopted = %d %v, want 200", status, out) } _, out = getJSON(t, relayURL(fx, "titogo"), token) rows = relayRouteRows(t, out) wantRoute(t, rows, "c.titogo.oncarlos.com", "tcp-exec-file", "adopted by member at") if rows["c.titogo.oncarlos.com"]["cutover"] != true { t.Fatalf("cutover = %v after an ack, want true", rows["c.titogo.oncarlos.com"]["cutover"]) } // A rotation re-issues the app under a new kid. The ack named the OLD // identities, so it must stop reading as adopted — otherwise a member // would believe a route runs a key it has never seen, and the old master // would be retired out from under it. t.Setenv(relayMastersEnv("eu-west-1"), "k1:"+relayTestMaster(1)+",k2:"+relayTestMaster(2)) seedHeartbeat(t, fx, "box1", "eu-west-1", time.Minute, "k1", "k2") if _, err := fx.srv.relayRotate(ctx, "eu-west-1"); err != nil { t.Fatalf("rotate: %v", err) } _, out = getJSON(t, relayURL(fx, "titogo"), token) rows = relayRouteRows(t, out) wantRoute(t, rows, "c.titogo.oncarlos.com", "tcp-exec-file", "the identities have changed since", "carlos relay adopted --host c.titogo.oncarlos.com") if rows["c.titogo.oncarlos.com"]["cutover"] != false { t.Fatalf("cutover = %v after a rotation, want false", rows["c.titogo.oncarlos.com"]["cutover"]) } if rows["c.titogo.oncarlos.com"]["acked_at"] == nil { t.Fatal("acked_at was dropped — the ack still happened, it just no longer counts") } } // TestAPIRelayRotateStagingRefusalsAreClientErrors pins the two refusal // sentences relayLooksLikeClientError matches by wording rather than by // sentinel: nothing new staged, and no live relay in the region at all. // Both are the OPERATOR's to fix by finishing the runbook, so both are a 400 // naming what is wrong — and if either sentence is ever reworded without // updating the matcher, they become a 502 that reads as "the console broke" // and this test is what says so. func TestAPIRelayRotateStagingRefusalsAreClientErrors(t *testing.T) { fx, _ := relayFixture(t, "code-relay-staging") token := cliToken(t, fx, "alice@keymail.dev", "code-relay-staging-2") fx.srv.cfg.RelayOperators = []string{"alice@keymail.dev"} rotate := fx.ts.URL + "/api/cli/relay/rotate" // (a) nothing new staged: the only entry is already the issuer. seedHeartbeat(t, fx, "box1", "eu-west-1", time.Minute, "k1") status, out := postJSON(t, rotate, map[string]any{"region": "eu-west-1"}, token) if status != http.StatusBadRequest { t.Fatalf("rotate with nothing new staged = %d %v, want 400", status, out) } if msg := str(t, out, "error"); !strings.Contains(msg, "no new master is staged") { t.Fatalf("error = %q, want the nothing-staged sentence", msg) } // (b) a new master staged, but every heartbeat in the region is stale. // Issuing now would mint credentials nothing can verify, and a dead relay // must not be able to consent by silence. t.Setenv(relayMastersEnv("eu-west-1"), "k1:"+relayTestMaster(1)+",k2:"+relayTestMaster(2)) seedHeartbeat(t, fx, "box1", "eu-west-1", 6*time.Minute, "k1", "k2") status, out = postJSON(t, rotate, map[string]any{"region": "eu-west-1"}, token) if status != http.StatusBadRequest { t.Fatalf("rotate with no live relay = %d %v, want 400", status, out) } if msg := str(t, out, "error"); !strings.Contains(msg, "has heartbeated in the last") { t.Fatalf("error = %q, want the no-live-relay sentence", msg) } } // TestAPIRelayStatusReadsConfigAcks is Task 12's console half: three // unit-backed routes, one config-ack each, seeded exactly as config sync // itself writes them (control/config-ack//.json, // appconfig.WriteConfigAck) rather than through any sync machinery — the // door's whole job here is reading and matching what's already in the // bucket, not producing it. An ack naming the issued identities and a // successful restart reads as adopted; one naming them with a restart // error reads as failed; one naming identities from BEFORE this app's // enable (a stale ack, the shape a rotation or a re-enable would leave // behind) must not read as adopted at all — see relayMatchingAck's own // doc comment for why. func TestAPIRelayStatusReadsConfigAcks(t *testing.T) { fx, _ := relayFixture(t, "code-relay-config-acks") ctx := context.Background() token := cliToken(t, fx, "alice@keymail.dev", "code-relay-config-acks-2") fx.addRoute(t, registry.Route{Host: "a.titogo.oncarlos.com", App: "titogo", Kind: "service", Unit: "titogo-a.service", Status: "ready"}) fx.addRoute(t, registry.Route{Host: "b.titogo.oncarlos.com", App: "titogo", Kind: "service", Unit: "titogo-b.service", Status: "ready"}) fx.addRoute(t, registry.Route{Host: "c.titogo.oncarlos.com", App: "titogo", Kind: "service", Unit: "titogo-c.service", Status: "ready"}) fx.addRoute(t, registry.Route{Host: "d.titogo.oncarlos.com", App: "titogo", Kind: "service", Unit: "titogo-d.service", Status: "ready"}) if status, out := postJSON(t, relayURL(fx, "titogo"), map[string]any{}, token); status != http.StatusOK { t.Fatalf("enable = %d %v, want 200", status, out) } issued := relayRecord(t, fx, "titogo").IssuedSuffixes if issued["eu-west-1"] == "" { t.Fatalf("enable issued nothing for eu-west-1: %v", issued) } now := time.Now() if err := appconfig.WriteConfigAck(ctx, fx.rel, appconfig.ConfigAck{ Box: "box1", Host: "a.titogo.oncarlos.com", Shape: "unit", At: now, Restarted: true, RelaySuffixes: maps.Clone(issued), }); err != nil { t.Fatalf("seed ack a: %v", err) } if err := appconfig.WriteConfigAck(ctx, fx.rel, appconfig.ConfigAck{ Box: "box1", Host: "b.titogo.oncarlos.com", Shape: "unit", At: now, RestartError: "systemctl try-restart carlos-app@b.titogo.oncarlos.com: exit status 1", RelaySuffixes: maps.Clone(issued), }); err != nil { t.Fatalf("seed ack b: %v", err) } if err := appconfig.WriteConfigAck(ctx, fx.rel, appconfig.ConfigAck{ Box: "box1", Host: "c.titogo.oncarlos.com", Shape: "unit", At: now, Restarted: true, RelaySuffixes: map[string]string{"eu-west-1": "abc:titogo:stale0000000:k0"}, }); err != nil { t.Fatalf("seed ack c: %v", err) } if err := appconfig.WriteConfigAck(ctx, fx.rel, appconfig.ConfigAck{ Box: "box1", Host: "d.titogo.oncarlos.com", Shape: "unit", At: now, Hibernating: true, RelaySuffixes: maps.Clone(issued), }); err != nil { t.Fatalf("seed ack d: %v", err) } _, out := getJSON(t, relayURL(fx, "titogo"), token) rows := relayRouteRows(t, out) wantRoute(t, rows, "a.titogo.oncarlos.com", "unit", "adopted at", "restarted") wantRoute(t, rows, "b.titogo.oncarlos.com", "unit", "restart failed", "systemctl try-restart carlos-app@b.titogo.oncarlos.com: exit status 1") if got := str(t, rows["c.titogo.oncarlos.com"], "delivery"); strings.Contains(got, "adopted") { t.Fatalf("c delivery = %q, a stale-suffix ack must not read as adopted", got) } if got := str(t, rows["c.titogo.oncarlos.com"], "delivery"); got != "written, awaiting delivery" { t.Fatalf("c delivery = %q, want the base written-awaiting-delivery sentence", got) } wantRoute(t, rows, "d.titogo.oncarlos.com", "unit", "hibernating, delivered on wake") } // TestAPIRelayStatusNeverAdoptsOnEmptyIssuedSuffixes is the review's finding // 2 (2026-09-07): maps.Equal(nil, nil) is true, so without an explicit // guard an enabled record still at phase "config" — IssuedSuffixes not // written yet, the state an enable interrupted between spec §5 steps 4 and // 5 persists in until the reconcile finishes it — would match ANY route's // ordinary pre-relay ack (no CARLOS_RELAY entry at all, RelaySuffixes nil) // and print an adoption for identities the console has never actually // issued. func TestAPIRelayStatusNeverAdoptsOnEmptyIssuedSuffixes(t *testing.T) { fx, _ := relayFixture(t, "code-relay-empty-issued") ctx := context.Background() token := cliToken(t, fx, "alice@keymail.dev", "code-relay-empty-issued-2") fx.addRoute(t, registry.Route{Host: "a.titogo.oncarlos.com", App: "titogo", Kind: "service", Unit: "titogo-a.service", Status: "ready"}) if status, out := postJSON(t, relayURL(fx, "titogo"), map[string]any{}, token); status != http.StatusOK { t.Fatalf("enable = %d %v, want 200", status, out) } // Roll the finished enable back to the interrupted-mid-walk shape by // hand: still enabled, phase "config", nothing issued yet. rec, etag, err := relay.GetRecord(ctx, fx.rel, fx.sqid, "titogo") if err != nil { t.Fatalf("GetRecord: %v", err) } rec.Phase = "config" rec.IssuedSuffixes = nil if _, err := relay.PutRecord(ctx, fx.rel, fx.sqid, "titogo", rec, etag); err != nil { t.Fatalf("PutRecord: %v", err) } // An ordinary ack — the shape a route synced before this app ever // enabled the relay carries: RelaySuffixes nil, no error naming // anything wrong with it. if err := appconfig.WriteConfigAck(ctx, fx.rel, appconfig.ConfigAck{ Box: "box1", Host: "a.titogo.oncarlos.com", Shape: "unit", At: time.Now(), Restarted: true, }); err != nil { t.Fatalf("seed ack: %v", err) } _, out := getJSON(t, relayURL(fx, "titogo"), token) rows := relayRouteRows(t, out) if got := str(t, rows["a.titogo.oncarlos.com"], "delivery"); strings.Contains(got, "adopted") { t.Fatalf("delivery = %q, an empty-IssuedSuffixes record must never read as adopted", got) } } // TestAPIRelayStatusCarriesRetiringMasters covers the hop from a relay's // heartbeat to the status reply (discussion #120). Without it the console // half of the retiring display can be deleted with the whole suite green, // and an operator mid-rotation silently reads "nothing is retiring" — which // is indistinguishable from the true answer and is the wrong one to act on. func TestAPIRelayStatusCarriesRetiringMasters(t *testing.T) { fx, _ := relayFixture(t, "code-relay-retiring") token := cliToken(t, fx, "alice@keymail.dev", "code-relay-retiring-2") if status, out := postJSON(t, relayURL(fx, "titogo"), map[string]any{}, token); status != http.StatusOK { t.Fatalf("enable = %d %v, want 200", status, out) } retire := time.Date(2026, 9, 9, 12, 0, 0, 0, time.UTC) seedHeartbeatRetiring(t, fx, "box1", "eu-west-1", time.Minute, map[string]time.Time{"k1": retire}, "k1", "k2") status, out := getJSON(t, relayURL(fx, "titogo"), token) if status != http.StatusOK { t.Fatalf("status = %d %v, want 200", status, out) } row := regionRow(t, out, 0) got, ok := row["retiring"].(map[string]any) if !ok { t.Fatalf("region row carries no retiring map: %#v", row) } if got["k1"] != retire.Format(time.RFC3339) { t.Errorf("retiring[k1] = %v, want %s", got["k1"], retire.Format(time.RFC3339)) } // k2 is loaded but carries no retirement date, so it must be absent // rather than present with a zero time. if _, ok := got["k2"]; ok { t.Errorf("k2 has no retirement date and must not appear: %#v", got) } }