package console // The narrowed restart door (apirestartscope.go, discussion #16). What is // pinned here, in the order it would hurt if lost: a selector that names // nothing is REFUSED rather than widened into an app-wide restart; the request // that does land goes to the log old boxes never read, leaving the app-wide // object untouched; a dry run writes and announces nothing; and the answer // names the routes it addressed, which is the whole complaint the flags exist // to answer. import ( "context" "encoding/json" "net/http" "slices" "strings" "testing" "github.com/carlosframework/platform/internal/registry" "github.com/carlosframework/platform/internal/release" "github.com/carlosframework/platform/internal/svcrole" ) func scopeURL(fx *consoleFixture, app string) string { return fx.ts.URL + "/api/cli/apps/bab/" + app + "/restart/scope" } // scopeFixture is a console whose registry holds a small titogo: two routes on // the relay bundle, one on router, one bound to nothing, one asleep, plus a // neighbour app and another account's row to prove the tenant boundary. func scopeFixture(t *testing.T, code string) (*consoleFixture, string) { t.Helper() fx, token := restartFixture(t, code) reg, err := registry.Open(fx.regPath) if err != nil { t.Fatal(err) } for _, rt := range []registry.Route{ {Host: "connect.go.tito.io", Kind: "instance", Status: "ready", App: "titogo", Account: "bab", Addr: "127.0.0.1:9001", Environment: "relay"}, {Host: "relay2.go.tito.io", Kind: "instance", Status: "ready", App: "titogo", Account: "bab", Addr: "127.0.0.1:9002", Environment: "relay", Unit: "relay2.service"}, {Host: "platform.ie.go.tito.io", Kind: "instance", Status: "ready", App: "titogo", Account: "bab", Addr: "127.0.0.1:9003", Environment: "router"}, {Host: "bill.go.tito.io", Kind: "instance", Status: "ready", App: "titogo", Account: "bab", Addr: "127.0.0.1:9004"}, {Host: "sleepy.go.tito.io", Kind: "instance", Status: "ready", App: "titogo", Account: "bab", Addr: "127.0.0.1:9005", Environment: "relay", Hibernate: true}, {Host: "neighbour.example.com", Kind: "instance", Status: "ready", App: "kass", Account: "bab", Addr: "127.0.0.1:9006", Environment: "relay"}, {Host: "foreign.example.com", Kind: "instance", Status: "ready", App: "titogo", Account: "zzzz", Addr: "127.0.0.1:9007", Environment: "relay"}, } { if err := reg.Upsert(rt); err != nil { t.Fatal(err) } } return fx, token } // scopeLog reads the narrowed request log every NEW box watches. func scopeLog(t *testing.T, fx *consoleFixture, app string) *release.ScopedRestartGeneration { t.Helper() g, found, err := release.LoadRestartScopes(context.Background(), fx.rel, release.OpsAccount, app) if err != nil { t.Fatalf("read narrowed request log: %v", err) } if !found { return &release.ScopedRestartGeneration{} } return g } type scopeAnswer struct { Account string `json:"account"` App string `json:"app"` Scope struct { Kind string `json:"kind"` Value string `json:"value"` } `json:"scope"` DryRun bool `json:"dry_run"` Requested bool `json:"requested"` Routes []struct { Host string `json:"host"` Environment string `json:"environment"` Unit string `json:"unit"` Hibernate bool `json:"hibernate"` } `json:"routes"` } func postScope(t *testing.T, fx *consoleFixture, token, body string) (int, scopeAnswer, []byte) { t.Helper() status, out := postBody(t, scopeURL(fx, "titogo"), token, body) var res scopeAnswer if status == http.StatusOK { if err := json.Unmarshal(out, &res); err != nil { t.Fatalf("decode %s: %v", out, err) } } return status, res, out } func scopeHosts(res scopeAnswer) []string { out := make([]string, 0, len(res.Routes)) for _, r := range res.Routes { out = append(out, r.Host) } return out } // TestAPIRestartScopeEnvironmentAddressesOnlyThatBundle is the fix: a request // for one environment names the routes reading it — this app's, this account's // — and nothing else. The neighbouring app on the same bundle name and the // other account's identically-bound route are both absent, and that is the // tenant boundary, not a coincidence of the fixture. func TestAPIRestartScopeEnvironmentAddressesOnlyThatBundle(t *testing.T) { fx, token := scopeFixture(t, "code-scope-env") status, res, out := postScope(t, fx, token, `{"environment":"relay"}`) if status != http.StatusOK { t.Fatalf("restart scope = %d %s, want 200", status, out) } if !res.Requested || res.Scope.Kind != "environment" || res.Scope.Value != "relay" { t.Fatalf("answer = %+v, want a requested environment scope", res) } want := []string{"connect.go.tito.io", "relay2.go.tito.io", "sleepy.go.tito.io"} if got := scopeHosts(res); !slices.Equal(got, want) { t.Fatalf("addressed %v, want %v", got, want) } // The sleeping route is NAMED and flagged rather than filtered out: an // operator has to be able to see that it is in the set and will not cycle // now, which is correct rather than a failure. var sleepySeen bool for _, r := range res.Routes { if r.Host == "sleepy.go.tito.io" { sleepySeen = r.Hibernate } if r.Host == "relay2.go.tito.io" && r.Unit != "relay2.service" { t.Fatalf("route %+v lost the unit that owns its process", r) } } if !sleepySeen { t.Fatal("the hibernating route came back unflagged — an operator would read it as one that just cycled") } g := scopeLog(t, fx, "titogo") if len(g.Requests) != 1 || g.Requests[0].Scope.Value != "relay" { t.Fatalf("request log = %+v, want the one narrowed request", g.Requests) } if got := serial(t, fx); got != 1 { t.Fatalf("serial after a narrowed request = %d, want 1 (otherwise boxes wait out the hourly floor)", got) } } func TestAPIRestartScopeRecordsMessageAndTarget(t *testing.T) { fx, token := scopeFixture(t, "code-scope-message") status, _, out := postScope(t, fx, token, `{"environment":"relay","message":" cycle the relay workers "}`) if status != http.StatusOK { t.Fatalf("restart scope = %d %s, want 200", status, out) } events, err := fx.st.Events(OpsAccountID, "titogo", 1) if err != nil || len(events) != 1 { t.Fatalf("Events = %+v, %v", events, err) } got := events[0] if got.Action != "restart" || got.TargetType != "environment" || got.Target != "relay" || got.Message != "cycle the relay workers" || got.Detail != "requested a restart of environment relay (cli)" { t.Fatalf("activity = %+v", got) } } func TestAPIRestartScopeRejectsInvalidMessageBeforeWriting(t *testing.T) { fx, token := scopeFixture(t, "code-scope-message-invalid") status, _, out := postScope(t, fx, token, `{"host":"bill.go.tito.io","message":"first\nsecond"}`) if status != http.StatusBadRequest { t.Fatalf("restart scope = %d %s, want 400", status, out) } if len(scopeLog(t, fx, "titogo").Requests) != 0 || restartGenEtag(t, fx, "titogo") != "" { t.Fatal("an invalid message still wrote a restart request") } if events, err := fx.st.Events(OpsAccountID, "titogo", 10); err != nil || len(events) != 0 { t.Fatalf("activity after invalid message = %+v, %v", events, err) } } // TestAPIRestartScopeLeavesTheAppWideObjectAlone is the additive property at // the door: a narrowed request must not move the object every OLD box watches, // or the fleet's un-upgraded boxes would each cycle the whole app. func TestAPIRestartScopeLeavesTheAppWideObjectAlone(t *testing.T) { fx, token := scopeFixture(t, "code-scope-additive") if status, _, out := postScope(t, fx, token, `{"host":"bill.go.tito.io"}`); status != http.StatusOK { t.Fatalf("restart scope = %d %s, want 200", status, out) } if got := restartGenEtag(t, fx, "titogo"); got != "" { t.Fatalf("the app-wide restart generation exists (%q) after a narrowed request — every old box would cycle everything", got) } } // TestAPIRestartScopeRefusesAnUnknownEnvironment: the refusal that keeps a // typo from becoming the bug being fixed. It names what the app actually has, // because a refusal exists to let somebody spot their own mistake. func TestAPIRestartScopeRefusesAnUnknownEnvironment(t *testing.T) { fx, token := scopeFixture(t, "code-scope-typo") status, _, out := postScope(t, fx, token, `{"environment":"relayy"}`) if status != http.StatusBadRequest { t.Fatalf("a typo'd environment = %d %s, want 400", status, out) } body := string(out) for _, want := range []string{"relayy", "relay", "router"} { if !strings.Contains(body, want) { t.Fatalf("refusal %s does not name %q — it must say what it looked for and what exists", body, want) } } if len(scopeLog(t, fx, "titogo").Requests) != 0 || restartGenEtag(t, fx, "titogo") != "" { t.Fatal("a refused selector still wrote a request") } if got := serial(t, fx); got != 0 { t.Fatalf("serial = %d after a refusal, want it untouched", got) } } // TestAPIRestartScopeRefusesAnUnknownHost: the same rule for the other flag. func TestAPIRestartScopeRefusesAnUnknownHost(t *testing.T) { fx, token := scopeFixture(t, "code-scope-host-typo") status, _, out := postScope(t, fx, token, `{"host":"bil.go.tito.io"}`) if status != http.StatusBadRequest { t.Fatalf("an unknown host = %d %s, want 400", status, out) } if !strings.Contains(string(out), "bill.go.tito.io") { t.Fatalf("refusal %s does not list the hosts this app has", out) } // Another account's route is not a host this caller may name, and the // refusal must not leak that it exists. if strings.Contains(string(out), "foreign.example.com") { t.Fatalf("refusal %s named another account's route", out) } } // TestAPIRestartScopeRefusesAMismatchedPair is the precedence rule's teeth. // --host wins over --environment, but only after the host is checked against // it: quietly ignoring the wider flag would hide that one of the two is a // typo, and this is not a verb to guess on. func TestAPIRestartScopeRefusesAMismatchedPair(t *testing.T) { fx, token := scopeFixture(t, "code-scope-mismatch") status, _, out := postScope(t, fx, token, `{"host":"platform.ie.go.tito.io","environment":"relay"}`) if status != http.StatusBadRequest { t.Fatalf("a mismatched pair = %d %s, want 400", status, out) } if !strings.Contains(string(out), "router") || !strings.Contains(string(out), "relay") { t.Fatalf("refusal %s must name what the host actually reads and what was asked for", out) } if len(scopeLog(t, fx, "titogo").Requests) != 0 { t.Fatal("a mismatched pair still wrote a request") } } // TestAPIRestartScopeHostWinsOverAMatchingEnvironment: the pair is accepted // when it agrees, and the narrower flag is what lands. func TestAPIRestartScopeHostWinsOverAMatchingEnvironment(t *testing.T) { fx, token := scopeFixture(t, "code-scope-pair") status, res, out := postScope(t, fx, token, `{"host":"connect.go.tito.io","environment":"relay"}`) if status != http.StatusOK { t.Fatalf("a matching pair = %d %s, want 200", status, out) } if res.Scope.Kind != "host" || res.Scope.Value != "connect.go.tito.io" { t.Fatalf("scope = %+v, want the host to win", res.Scope) } if got := scopeHosts(res); !slices.Equal(got, []string{"connect.go.tito.io"}) { t.Fatalf("addressed %v, want the one route", got) } } // TestAPIRestartScopeDryRunTouchesNothing: the answer an operator wanted // before cycling anything — the set, and no restart. func TestAPIRestartScopeDryRunTouchesNothing(t *testing.T) { fx, token := scopeFixture(t, "code-scope-dry") status, res, out := postScope(t, fx, token, `{"environment":"relay","dry_run":true}`) if status != http.StatusOK { t.Fatalf("dry run = %d %s, want 200", status, out) } if res.Requested || !res.DryRun { t.Fatalf("answer = %+v, want dry_run true and requested false", res) } if got := scopeHosts(res); len(got) != 3 { t.Fatalf("dry run addressed %v, want the same three routes the real request would", got) } if len(scopeLog(t, fx, "titogo").Requests) != 0 || restartGenEtag(t, fx, "titogo") != "" { t.Fatal("a dry run wrote a request") } if got := serial(t, fx); got != 0 { t.Fatalf("serial = %d after a dry run, want it untouched — a dry run must not even wake the fleet", got) } } // TestAPIRestartScopeDryRunOfTheWholeApp: with no selector, the dry run is how // somebody sees what a plain `carlos restart --app` would cycle. That is the // blast radius the reporter said they would not run the verb to discover. func TestAPIRestartScopeDryRunOfTheWholeApp(t *testing.T) { fx, token := scopeFixture(t, "code-scope-dry-app") status, res, out := postScope(t, fx, token, `{"dry_run":true}`) if status != http.StatusOK { t.Fatalf("app-wide dry run = %d %s, want 200", status, out) } if res.Scope.Kind != "app" { t.Fatalf("scope = %+v, want the whole app", res.Scope) } want := []string{"bill.go.tito.io", "connect.go.tito.io", "platform.ie.go.tito.io", "relay2.go.tito.io", "sleepy.go.tito.io"} if got := scopeHosts(res); !slices.Equal(got, want) { t.Fatalf("addressed %v, want every route of the app: %v", got, want) } } // TestAPIRestartScopeRefusesAWriteWithNoSelector is the door's own // never-widen rule, turned on itself. // // This path exists BECAUSE the app-wide door widens a request it cannot read. // A body with no selector in it — a bare `{}`, or a client that renamed a // field — resolves to the whole app, and performing that here would be the // nineteen-route accident committed through the door built to prevent it. So // the app-wide case is answered only as a --dry-run (which is how somebody // sees the blast radius before typing the real verb), and a WRITE with no // narrowing is refused, naming the door that does own it. func TestAPIRestartScopeRefusesAWriteWithNoSelector(t *testing.T) { fx, token := scopeFixture(t, "code-scope-appwide") status, _, out := postScope(t, fx, token, `{}`) if status != http.StatusBadRequest { t.Fatalf("an app-wide write through the scope door = %d %s, want 400", status, out) } if !strings.Contains(string(out), "carlos restart --app") { t.Fatalf("refusal %s does not name the door that cycles the whole app", out) } if got := restartGenEtag(t, fx, "titogo"); got != "" { t.Fatalf("a refused app-wide write still moved the object every box watches (%q) — that is the whole app cycled", got) } if len(scopeLog(t, fx, "titogo").Requests) != 0 { t.Fatal("a refused app-wide write still appended to the narrowed log") } if got := serial(t, fx); got != 0 { t.Fatalf("serial = %d after a refusal, want it untouched", got) } // The dry run of the same body is still answered: that is the one form of // app-wide this door is for. if status, res, out := postScope(t, fx, token, `{"dry_run":true}`); status != http.StatusOK || res.Requested { t.Fatalf("app-wide dry run = %d %s (%+v), want 200 and nothing requested", status, out, res) } } // TestAPIRestartScopeRefusesAFieldItDoesNotKnow is the same rule one layer // out. A misspelled or renamed selector key that was merely IGNORED would // leave a body with no selector in it, which is a whole-app restart requested // by a client that asked for one route. The refusal names the field, because // the fix is a spelling. func TestAPIRestartScopeRefusesAFieldItDoesNotKnow(t *testing.T) { fx, token := scopeFixture(t, "code-scope-unknown-field") status, _, out := postScope(t, fx, token, `{"hosts":"relay2.go.tito.io"}`) if status != http.StatusBadRequest { t.Fatalf(`{"hosts":...} = %d %s, want 400 — an ignored key here is an app-wide restart`, status, out) } if !strings.Contains(string(out), "hosts") { t.Fatalf("refusal %s does not name the field, so nobody can find the typo", out) } if restartGenEtag(t, fx, "titogo") != "" || len(scopeLog(t, fx, "titogo").Requests) != 0 { t.Fatal("a body this door refused still wrote a request") } } // TestAPIRestartScopeGateIsTheFamilys404: an app the account never claimed, a // stranger's token and a credential bound to another app are ONE answer, byte // for byte the answer the app-wide door gives — this repo's rule for these // gates is byte-identical, not merely same-status, because a differently // shaped 404 is an oracle that says "this door exists for you, that app does // not". func TestAPIRestartScopeGateIsTheFamilys404(t *testing.T) { fx, _ := scopeFixture(t, "code-scope-gate") // The old door's refusal, read from the old door rather than spelled out // here, so the two cannot drift apart without this failing. eve := cliToken(t, fx, "eve@keymail.dev", "code-scope-gate-eve") oldStatus, oldBody := postBody(t, fx.ts.URL+"/api/cli/apps/bab/titogo/restart", eve, "") if oldStatus != http.StatusNotFound { t.Fatalf("the app-wide door answered a stranger %d, want 404", oldStatus) } for _, c := range []struct { name string url string token string }{ {"a stranger", scopeURL(fx, "titogo"), eve}, // An app-scoped service credential reaching PAST its own app. The // binding is checked before the app is even looked up, so "not your // app" and "no such app" stay the same answer. {"a credential bound to another app", scopeURL(fx, "titogo"), serviceToken(t, fx.st, OpsAccountID, "kassbot", "kass", svcrole.Operate)}, {"an app this account never claimed", scopeURL(fx, "nosuchapp"), cliToken(t, fx, "alice@keymail.dev", "code-scope-gate-alice")}, } { status, body := postBody(t, c.url, c.token, `{"environment":"relay"}`) if status != oldStatus || string(body) != string(oldBody) { t.Fatalf("%s got %d %q, want the app-wide door's own %d %q", c.name, status, body, oldStatus, oldBody) } } if len(scopeLog(t, fx, "titogo").Requests) != 0 || restartGenEtag(t, fx, "titogo") != "" { t.Fatal("a refused caller still wrote a request") } if got := serial(t, fx); got != 0 { t.Fatalf("serial = %d after refusals, want it untouched", got) } } // TestAPIRestartScopeServiceCredentialReachesItsOwnApp is the other half of // the case above: the 404 there is about the app binding, not about services // being turned away from this door. An Operate credential bound to titogo // narrows a restart of titogo exactly as a member does. func TestAPIRestartScopeServiceCredentialReachesItsOwnApp(t *testing.T) { fx, _ := scopeFixture(t, "code-scope-svc-own") tok := serviceToken(t, fx.st, OpsAccountID, "titogobot", "titogo", svcrole.Operate) status, res, out := postScope(t, fx, tok, `{"host":"bill.go.tito.io"}`) if status != http.StatusOK { t.Fatalf("a credential bound to titogo = %d %s, want 200", status, out) } if !res.Requested || res.Scope.Kind != "host" || res.Scope.Value != "bill.go.tito.io" { t.Fatalf("answer = %+v, want the narrowed request", res) } } // TestAPIRestartScopeNeedsARegistry: a console that cannot see routes cannot // resolve a selector or report a set, and must not answer "requested" for // something it never checked. The app-wide door still works there, and the // refusal says so. func TestAPIRestartScopeNeedsARegistry(t *testing.T) { fx, token := scopeFixture(t, "code-scope-noreg") fx.srv.cfg.Registry = "" status, _, out := postScope(t, fx, token, `{"environment":"relay"}`) if status != http.StatusServiceUnavailable { t.Fatalf("a registry-less console answered %d %s, want 503", status, out) } if !strings.Contains(string(out), "carlos restart --app") { t.Fatalf("refusal %s does not name the form that still works", out) } } // TestAPIRestartScopeRefusesAMalformedSelector keeps a hostile value out // before the registry is even opened. func TestAPIRestartScopeRefusesAMalformedSelector(t *testing.T) { fx, token := scopeFixture(t, "code-scope-malformed") for _, body := range []string{ `{"host":"../../etc/passwd"}`, `{"environment":"NOT VALID"}`, `{"environment":"default"}`, } { if status, _, out := postScope(t, fx, token, body); status != http.StatusBadRequest { t.Fatalf("%s = %d %s, want 400", body, status, out) } } }