package console // TestClaimConsoleMutationInvariant is the runtime half of the claim // mutation invariant (plan 5 rev 4): // // With HostClaim on, no console code path mutates app-scoped state (bucket // keys under the app's prefixes, its SQL rows, its backend and SES // resources) without a live admission registration for that incarnation, // or the exclusive operation that owns it. // // TestClaimConsoleRouteCensus (claim_routes_test.go) pins WHICH entries // exist, from source. This test pins what each one DOES, at runtime: // // 1. Every registered entry, route or worker, from s.claimEntries.snapshot() // on an enabled console, has exactly one fixture, and every fixture names // a registered entry. A new door without a fixture fails here. // 2. A fixture is exactly one of: read (reason), nonApp (reason), or door. // There is no default. // 3. A door runs twice per affected incarnation. With that incarnation // closed under a foreign exclusive operation, the same authenticated, // valid request leaves no raw effect on any affected app: no bucket key // the production classifier scopes to it, no SQL row change, no outbound HTTP, SES, DNS, backend // or versioned call at all, no write to its admission. Then, with every // incarnation open, the same request must succeed, write its admission // (the gate saw it) and reach at least one raw app effect. A 401, 404 or // a refused body reaches no effect, so it can never count as coverage. // 4. A read or nonApp entry has its own probe (claim_invariant_probes_test.go, // the tables in claim_invariant_probes_*.go): a valid authenticated request // against a staged resource, accepted only on the exact success status // with an existing resource in the answer, no auth redirect, no error flash // and a request that matched this very entry; a mutator also proves its // effect from the world (a registry row, a bucket object, a SQL row, an // account, a token). It must leave no app-scoped raw effect. A request the // handler rejected (any status of 400 or more, an auth redirect, a 303 back // with an error flash) is never coverage, and the detectors say so. // An operation a deployment intentionally does not support is a refused // fixture instead: the exact status and message, from claimUnsupported. // 5. Every app-scoped writer the boundaries expose (bucket and versioned // mutators by reflection over the installed boundary, every mutating // backend method, every gated SQL table) is reached by at least one // door's open run, or carries a written reason in claimWriterReasons. // // Effects are observed BELOW the boundaries (claim_invariant_world_test.go), // so only what production let through is counted. import ( "context" "encoding/json" "fmt" "html" "net/http" "reflect" "slices" "sort" "strings" "sync" "sync/atomic" "testing" "time" "github.com/carlosframework/platform/internal/dbreads" "github.com/carlosframework/platform/internal/dnsrec" "github.com/carlosframework/platform/internal/hostops" "github.com/carlosframework/platform/internal/mail" "github.com/carlosframework/platform/internal/objstore" "github.com/carlosframework/platform/internal/release" ) // claimFixture classifies one registered entry. Exactly one field is set. type claimFixture struct { read string // why this entry cannot mutate any state at all nonApp string // why this entry mutates only deployment, account or person state door *claimDoor // an app-scoped mutator, driven for real refused *claimRefusalFixture // an intentionally unsupported app mutator, driven for real } // claimRefusalFixture is an app-scoped door whose operation is intentionally // unsupported on an enabled console (claimUnsupported). It is driven like a // door, foreign and open, with the owner's real authenticated, valid // request, and both runs must answer exactly status with exactly msg and // leave no app or control effect at all. An auth refusal, a 404 or a // generic error is never one of these: the census only admits 409 with a // message from claimUnsupported. type claimRefusalFixture struct { apps []claimAppRef setup func(t *testing.T, cx *claimCase) // do sends the request and returns the final status and body. do func(t *testing.T, cx *claimCase) (int, string) status int msg string } // claimOfflineMoveUnsupportedMsg is MV-O20, what every offline move door // answers on a console with host claims or Phase 1 moves on (offlineGate). const claimOfflineMoveUnsupportedMsg = "offline moves are not available while host claims or Phase 1 moves are on" // claimUnsupported is the closed list of operations an enabled console // refuses on purpose, keyed by the exact message production answers. Only // these may be a refused fixture; every other app mutator is a door that // must hold the gate and reach its effect. var claimUnsupported = map[string]string{ claimRunUnsupportedMsg: "carlos run launches a cmd unit that the claim's isolated root protocol has no reservation for (#250); " + "every run route refuses before it reads or writes anything", claimHostResolveUnsupportedMsg: "host-loss declaration/readmission changes claim roots; the existing service has no scoped lifecycle grant in this slice", claimMoveUnsupportedMsg: "instance move request, cut and abort rewrite the record's move journal, " + "which only the unbuilt host-claim Moves protocol may do under its own operation", claimOfflineMoveUnsupportedMsg: "the offline operator move (MV-O20) is only for consoles with host claims and Phase 1 moves off; " + "every offline door, the status view included, refuses after its operator gate and before it reads evidence or writes the record", claimRefusalText(errClaimAliasDetachUnsupported): "removing an alias-only marker needs its own conditional effect operation; " + "a first-route detach retires its record and is a door", claimRefusalText(errClaimAliasCapabilitiesUnsupported): "wildcard and catch-all claims are written only by the legacy capability path", claimRenameUnsupportedMsg: "rename rewrites the app's identity (SQL rows, prefixes, admission key); " + "no operation gates the old and new names together, so every rename is refused", claimTransferUnsupportedMsg: "account transfer rewrites the app's identity under another account; " + "no operation gates both, so every transfer is refused", claimMoveResolveUnavailableMsg: "the operator's door out of a stuck move (CARLOS_MOVES_PHASE1) has only its gate until " + "the Moves resolution body lands; it answers this sentence and writes nothing", claimDBReadResidencyMsg: "a forwarded db read needs the root signer, which a host-claim console never uses for it " + "(db region forward spec §6): the reads route reads the record and answers main's residency sentence, writing nothing", seedClaimConsoleRefusal().Msg: "seeded instances need nested boxes, and the claim-armed path waits for #250 (seeded-instances spec §3.4); " + "every import door refuses before it reads the record, and the legacy route assignment before it reads anything", } // claimDBReadResidencyMsg is main's residency sentence for the claim world's // instance (in ie) on a console in the home region: what the forwarded db // read answers on a host-claim console. claimDBReadBody is a valid read. var ( claimDBReadResidencyMsg = claimDBHost + " keeps its data in region ie. This console can only read data kept in the home region." claimDBReadBody = func() string { key, err := dbreads.NewReplyKey() if err != nil { panic(err) } return `{"op":"query","sql":"SELECT 1","reply_key":"` + dbreads.EncodeReplyKey(key.PublicKey()) + `"}` }() ) // claimLifecycleRoot is the classifier's NonApp reason for a lifecycle CAS // root (hostops.ClassifyMutation): a host claim moved, which a refusal must // never do. const claimLifecycleRoot = "lifecycle CAS root" // claimDoor is one app-scoped mutator and the request that drives it. type claimDoor struct { // apps are the incarnations the request affects. apps[0] is the one the // request names; it must write its admission in the open run. apps []claimAppRef // setup stages the world with every incarnation open, outside every // measured window. It may drive other doors. setup func(t *testing.T, cx *claimCase) // do sends the authenticated, valid request (and any passkey step-up it // asks for) and returns the final status. do func(t *testing.T, cx *claimCase) int // check, when set, proves from the world (not the status) that the // handler's mutation path was reached: it runs after every foreign run // (open=false, the mutation must NOT have happened) and after the open // run (open=true, it must have). A non-empty answer is a problem. A // rejected probe (bad session, CSRF or id) leaves the world unchanged, so // it fails the open check however it answered. check func(t *testing.T, cx *claimCase, open bool) string } // claimWindow is what one measured request left behind. type claimWindow struct { status int events []claimRawEvent sqlChanged map[claimAppRef][]string } // claimWriterReasons names app-scoped writers that no door's open run // reaches in this fixture, each with the reason. Empty is the claim that // every writer is reached. A new boundary method or gated table fails the // coverage step until a door reaches it or a reason is written here. var claimWriterReasons = map[string]string{ "bucket/PutFrom": "the only console streaming caller is geoippublish.go, which writes deployment GeoIP blobs, not app state; app-prefixed fallback writes are checked by Put and the streaming new-door witness", "bucket/Delete": "the data boundary never sends an unconditional delete: Delete reads the exact revision, " + "authorizes it and issues DeleteIfMatch, which bucket/DeleteIfMatch counts", "versioned/PutIfAbsent": "no console door calls it: the console imports neither internal/park nor internal/hostbox, " + "the only code that puts versioned objects, and its own versioned callers are the db door's reads and the purge sweep's DeleteVersion", "versioned/CopyVersion": "no console door calls it: only internal/park (archive copies) and internal/hostbox call CopyVersion, " + "and the console imports neither", "sql/owners": "legacy per-app grants, written only by cmd/console's operator grant and revoke verbs " + "(Store.GrantAt and Revoke); enabled operator DBs are guarded too and refuse these unbound legacy grants; no route or worker writes them", "sql/mail_counts": "retired by the senders migration: nothing writes it, the one-shot backfill only reads it, " + "and DropLegacyMailCounts drops it", } // claimCoordinators are boundary methods that hold the gate for a caller and // then run its function: they perform no raw write themselves, so they are // not raw sinks. Every write inside the function reaches a real sink method, // which the inventory counts. The reason is mandatory. var claimCoordinators = map[string]string{ "bucket/RunSQLIngest": "coordinator for the SQL adapter: admits one shared ingest and issues its receipt-bound effect; raw SQL is independently covered by the existing SQL door fixtures", "bucket/RunAppMutation": "coordinator: registers or begins the exclusive app mutation, then runs the caller's function; " + "it writes nothing raw itself (its admission and effect writes go through the typed control writer, " + "and the function's own writes reach Put, Delete or DeleteIfMatch)", } // claimCaseParallelism bounds how many door, refusal and probe subtests run // at once. Each one builds its own console and measures what a request left // behind by waiting for quiet windows (quiesce). Too many at once starve the // CPU, and a handler goroutine that misses its window is measured with the // wrong request; the bound keeps that contention down. const claimCaseParallelism = 8 // claimParallelCases counts the claim cases running under t.Parallel. A // fixture that swaps process-wide state (a package variable, the log // output, a shared client's transport) calls claimMustRunAlone, which // fails the case if any parallel case is running, itself included. A swap // added to a fixture that is not listed serial therefore fails loudly // instead of leaking into a neighbour's measurement. var claimParallelCases atomic.Int32 func claimMustRunAlone(t *testing.T, what string) { t.Helper() if n := claimParallelCases.Load(); n > 0 { t.Fatalf("%s swaps process-wide state but runs beside %d parallel claim case(s): list it serial (claimSerialDoors, or the probe's serial field)", what, n) } } // claimParallelSlot takes a slot for a parallel claim case and counts it // as running until its own cleanup is done. func claimParallelSlot(t *testing.T, slots chan struct{}) { slots <- struct{}{} // Registered first, so it runs last: the slot is held until the case's // own cleanup (servers, store) is done. t.Cleanup(func() { <-slots }) claimParallelCases.Add(1) t.Cleanup(func() { claimParallelCases.Add(-1) }) } // claimSerialDoors are the door and refusal fixtures that must not run in // parallel, each with the reason. They run first, one at a time, before any // parallel door starts. A fixture whose staging touches process-wide state // (the environment, the log output, a package variable) belongs here. var claimSerialDoors = map[string]string{ "route POST /api/cli/relay/rotate": "relayEnabled and relayStageMaster stage the region's relay masters " + "through t.Setenv: process environment, which t.Parallel forbids", "route POST /api/cli/apps/{account}/{app}/relay": "relayArmed stages the region's relay master through t.Setenv: " + "process environment, which t.Parallel forbids", "route POST /api/cli/apps/{account}/{app}/relay/adopted": "relayEnabled stages the region's relay master through t.Setenv: " + "process environment, which t.Parallel forbids", "route POST /api/cli/apps/{account}/{app}/relay/disable": "relayEnabled stages the region's relay master through t.Setenv: " + "process environment, which t.Parallel forbids", "worker StartRelay": "relayEnabled stages the region's relay master through t.Setenv: " + "process environment, which t.Parallel forbids", "worker StartMailEventPush": "swaps mailPushClient.Transport, a package variable, for its recorder", } // claimSerialDoorCensus: every serial entry names a door or refusal fixture // and carries its reason. func claimSerialDoorCensus(fixtures map[string]claimFixture, serial map[string]string) []string { var problems []string for key, reason := range serial { f, ok := fixtures[key] switch { case !ok || (f.door == nil && f.refused == nil): problems = append(problems, fmt.Sprintf("claimSerialDoors names %s, which is not a door or refusal fixture", key)) case strings.TrimSpace(reason) == "": problems = append(problems, fmt.Sprintf("claimSerialDoors names %s with no reason", key)) } } sort.Strings(problems) return problems } func TestClaimConsoleMutationInvariant(t *testing.T) { census := newClaimCase(t) if !census.srv.cfg.HostClaim { t.Fatal("the invariant console is not enabled") } if problems := claimFixtureCensus(census.srv.claimEntries.snapshot(), claimFixtures); len(problems) != 0 { for _, p := range problems { t.Error(p) } t.FailNow() } if problems := claimProbeCensus(claimFixtures, claimProbes); len(problems) != 0 { for _, p := range problems { t.Error(p) } t.FailNow() } if problems := claimVariantCensus(census.srv.claimEntries.snapshot(), claimFixtures, claimLifecycleVariants); len(problems) != 0 { for _, p := range problems { t.Error(p) } t.FailNow() } inventory := claimWriterInventory(census.srv) keys := make([]string, 0, len(claimFixtures)) for k := range claimFixtures { keys = append(keys, k) } sort.Strings(keys) if problems := claimSerialDoorCensus(claimFixtures, claimSerialDoors); len(problems) != 0 { for _, p := range problems { t.Error(p) } t.FailNow() } // Doors and refusals, each on its own console. The group returns only // when every one of them has finished, parallel ones included, so the // writer check below sees every door's open run. var ( reachedMu sync.Mutex reached = map[string]bool{} ) t.Run("doors", func(t *testing.T) { slots := make(chan struct{}, claimCaseParallelism) run := func(t *testing.T, f claimFixture) { if f.refused != nil { for _, p := range runClaimRefusal(t, f.refused) { t.Error(p) } return } problems, writers := runClaimDoor(t, f.door) for _, p := range problems { t.Error(p) } reachedMu.Lock() for _, w := range writers { reached[w] = true } reachedMu.Unlock() } // The serial ones first, one at a time, while nothing else runs: // a parallel subtest only starts once this function has returned. for _, key := range keys { if f := claimFixtures[key]; (f.door != nil || f.refused != nil) && claimSerialDoors[key] != "" { t.Run(key, func(t *testing.T) { run(t, f) }) } } for _, key := range keys { if f := claimFixtures[key]; (f.door != nil || f.refused != nil) && claimSerialDoors[key] == "" { t.Run(key, func(t *testing.T) { t.Parallel() claimParallelSlot(t, slots) run(t, f) }) } } }) // Lifecycle variants: further actions on an entry above whose only // effect is a host claim written under their own registration grant // (claim_invariant_wakereset_test.go). runClaimLifecycleVariants(t) // Reads and nonApp entries are probed with their own valid request on // their own console (claim_invariant_probes_test.go): a request the // handler turned away is not coverage, and a mutator proves its effect // from the world. They run in parallel too, after every door, except the // probes marked serial (claimProbe.serial), which run alone first. t.Run("probes", func(t *testing.T) { runClaimProbes(t, claimFixtures, claimProbes, claimCaseParallelism) }) var unreached []string for w := range inventory { if !reached[w] && claimWriterReasons[w] == "" { unreached = append(unreached, w) } } for w := range claimWriterReasons { if !inventory[w] { t.Errorf("claimWriterReasons names %s, which no boundary exposes any more", w) } } sort.Strings(unreached) for _, w := range unreached { t.Errorf("app-scoped writer %s is reached by no door's open run and has no reason in claimWriterReasons", w) } } // claimFixtureCensus compares the registered entries with the fixture table, // exactly, and checks each fixture is one kind with its reason or door. func claimFixtureCensus(entries []claimEntry, fixtures map[string]claimFixture) []string { var problems []string got := map[string]bool{} for _, e := range entries { key := e.Kind + " " + e.Pattern got[key] = true if _, ok := fixtures[key]; !ok { problems = append(problems, fmt.Sprintf("registered %s (handler %s) has no claim fixture: add a door, or a read or nonApp reason", key, e.Handler)) } } for key, f := range fixtures { if !got[key] { problems = append(problems, fmt.Sprintf("claim fixture %s names no registered entry", key)) } kinds := 0 if strings.TrimSpace(f.read) != "" { kinds++ } if strings.TrimSpace(f.nonApp) != "" { kinds++ } if f.door != nil { kinds++ if len(f.door.apps) == 0 || f.door.do == nil { problems = append(problems, fmt.Sprintf("claim fixture %s is a door with no affected app or no request", key)) } } if r := f.refused; r != nil { kinds++ switch { case len(r.apps) == 0 || r.do == nil: problems = append(problems, fmt.Sprintf("claim fixture %s is a refusal with no affected app or no request", key)) case r.status != http.StatusConflict: problems = append(problems, fmt.Sprintf("claim fixture %s is a refusal answering %d: only the explicit 409 of an intentionally "+ "unsupported operation can be one (never 401, 403, 404 or a generic error)", key, r.status)) case strings.TrimSpace(claimUnsupported[r.msg]) == "": problems = append(problems, fmt.Sprintf("claim fixture %s is a refusal with message %q, which is not an intentionally "+ "unsupported operation in claimUnsupported: make it a door", key, r.msg)) } } if kinds != 1 { problems = append(problems, fmt.Sprintf("claim fixture %s must be exactly one of read, nonApp, door or refused (has %d)", key, kinds)) } } sort.Strings(problems) return problems } // runClaimDoor drives one door: setup, one foreign-exclusive run per // affected incarnation, then the open run. It returns the problems and the // writers the open run reached. func runClaimDoor(t *testing.T, d *claimDoor, opts ...func(*Config)) ([]string, []string) { t.Helper() cx := newClaimCaseWith(t, opts...) if d.setup != nil { d.setup(t, cx) } cx.quiesce() var problems []string for _, ref := range d.apps { restore := cx.foreign(t, ref) w := cx.measure(t, d) problems = append(problems, claimForeignProblems(ref, d.apps, w)...) if d.check != nil { if p := d.check(t, cx, false); p != "" { problems = append(problems, "foreign exclusive on "+ref.String()+": "+p) } } restore() } w := cx.measure(t, d) problems = append(problems, claimOpenProblems(d.apps, w)...) if d.check != nil { if p := d.check(t, cx, true); p != "" { problems = append(problems, "open run: "+p) } } return problems, claimWritersOf(d.apps, w) } // runClaimRefusal drives one intentionally unsupported door like a door: // setup, one foreign-exclusive run per affected incarnation, then the open // run. Every run must answer the exact refusal and leave nothing behind. func runClaimRefusal(t *testing.T, f *claimRefusalFixture, opts ...func(*Config)) []string { t.Helper() cx := newClaimCaseWith(t, opts...) if f.setup != nil { f.setup(t, cx) } cx.quiesce() var problems []string run := func(label string) { var body string w := cx.measureRun(t, f.apps, func() int { status, out := f.do(t, cx) body = out return status }) problems = append(problems, claimRefusalProblems(label, f, w, body)...) } for _, ref := range f.apps { restore := cx.foreign(t, ref) run("foreign exclusive on " + ref.String()) restore() } run("open run") return problems } // claimRefusalProblems: the run answered exactly the unsupported refusal // and reached no app effect, no control object and no host claim. func claimRefusalProblems(label string, f *claimRefusalFixture, w claimWindow, body string) []string { var problems []string say := func(format string, args ...any) { problems = append(problems, label+": "+fmt.Sprintf(format, args...)) } if w.status != f.status { say("the authenticated valid request answered %d, want the explicit %d refusal", w.status, f.status) } if !claimRefusalSays(body, f.msg) { if len(body) > 300 { body = body[:300] + "..." } say("the answer %q is not exactly the unsupported refusal %q", body, f.msg) } for _, e := range w.events { switch { case e.Control: say("the refusal wrote control object %s", e.Key) case e.Err != nil: say("raw %s reached a key the production classifier refuses (%v)", e, e.Err) case e.Unscoped: say("raw %s reached an outbound sink or a data bucket", e) case len(e.Scopes) != 0: say("raw %s is app-scoped (%v)", e, e.Scopes) case e.NonApp == claimLifecycleRoot: say("raw %s moved a lifecycle root", e) } } for ref, changed := range w.sqlChanged { say("SQL rows of %s changed in %v", ref, changed) } return problems } // claimRefusalSays: a JSON error answer carries exactly msg; a rendered // page carries msg as its escaped text. func claimRefusalSays(body, msg string) bool { var reply struct { Error *string `json:"error"` } if json.Unmarshal([]byte(body), &reply) == nil { return reply.Error != nil && *reply.Error == msg } return strings.Contains(body, html.EscapeString(msg)) } // measure sends the door's request once and returns what it left behind. func (cx *claimCase) measure(t *testing.T, d *claimDoor) claimWindow { t.Helper() return cx.measureRun(t, d.apps, func() int { return d.do(t, cx) }) } // measureRun sends one request through do and returns what it left behind // on apps. The CLI rate limiters are fresh for every run: a 429 is not a // gate. func (cx *claimCase) measureRun(t *testing.T, apps []claimAppRef, do func() int) claimWindow { t.Helper() cx.srv.cli = newCLILimits() before := map[claimAppRef]map[string]string{} for _, ref := range apps { before[ref] = cx.sqlDigest(t, ref) } m := cx.rec.mark() status := do() cx.quiesce() w := claimWindow{status: status, events: cx.rec.since(m), sqlChanged: map[claimAppRef][]string{}} for _, ref := range apps { if changed := claimChangedTables(before[ref], cx.sqlDigest(t, ref)); len(changed) != 0 { w.sqlChanged[ref] = changed } } return w } // claimForeignProblems: with foreign holding an exclusive operation on one // affected incarnation, nothing may reach any affected app. func claimForeignProblems(foreign claimAppRef, refs []claimAppRef, w claimWindow) []string { var problems []string say := func(format string, args ...any) { problems = append(problems, fmt.Sprintf("foreign exclusive on %s: ", foreign)+fmt.Sprintf(format, args...)) } for _, e := range w.events { switch { case e.Control && e.touches([]claimAppRef{foreign}): say("the request wrote %s's canonical control object %s", foreign, e.Key) case e.Control: // The shared mail gate, or another affected app's registration // taken and released around the refused preflight: gate state, // not an app effect. case e.Err != nil: say("raw %s reached a key the production classifier refuses (%v)", e, e.Err) case e.Unscoped: say("raw %s reached an outbound sink or a data bucket", e) case e.NonApp == claimLifecycleRoot: say("raw %s moved a lifecycle root while an affected app was foreign-owned", e) case e.touches(refs): say("raw %s reached an affected app", e) } } for _, ref := range refs { if changed := w.sqlChanged[ref]; len(changed) != 0 { say("SQL rows of %s changed in %v", ref, changed) } } return problems } // claimOpenProblems: with every incarnation open, the request must be // admitted by the gate and reach at least one raw app effect. func claimOpenProblems(refs []claimAppRef, w claimWindow) []string { var problems []string primary := refs[0] if w.status == 0 || w.status >= 400 { problems = append(problems, fmt.Sprintf("open run: the authenticated valid request answered %d", w.status)) } admitted, effect := false, false for _, e := range w.events { switch { case e.admissionWrite(primary): admitted = true case e.Control: case e.Err != nil: problems = append(problems, fmt.Sprintf("open run: raw %s reached a key the production classifier refuses (%v)", e, e.Err)) case e.Unscoped: effect = true case e.touches(refs): effect = true } } for _, ref := range refs { if len(w.sqlChanged[ref]) != 0 { effect = true } } if !admitted { problems = append(problems, fmt.Sprintf("open run: no write to %s's admission, so the request never registered with the gate", primary)) } if !effect { problems = append(problems, "open run: the request reached no raw app effect (bucket, SQL, versioned, HTTP, SES, DNS or backend)") } return problems } // claimWritersOf names the writers an open run reached. func claimWritersOf(refs []claimAppRef, w claimWindow) []string { set := map[string]bool{} for _, e := range w.events { switch { case e.Control || e.Err != nil: case !e.Unscoped && !e.touches(refs): case e.Surface == "bucket": set["bucket/"+e.Method] = true case strings.HasPrefix(e.Surface, "versioned:"): set["versioned/"+e.Method] = true default: set[e.Surface+"/"+e.Method] = true } } for _, changed := range w.sqlChanged { for _, table := range changed { set["sql/"+table] = true } } out := make([]string, 0, len(set)) for k := range set { out = append(out, k) } sort.Strings(out) return out } // claimWriterInventory is every app-scoped writer the installed boundaries // expose. The bucket and versioned method sets are read off the INSTALLED // boundary by reflection, so a writer method added to it joins the list. func claimWriterInventory(s *Server) map[string]bool { inv := map[string]bool{"http/POST": true} methods := func(prefix string, v any, reads ...string) { rt := reflect.TypeOf(v) for i := 0; i < rt.NumMethod(); i++ { name := rt.Method(i).Name if slices.Contains(reads, name) { continue } if reason, ok := claimCoordinators[prefix+name]; ok && strings.TrimSpace(reason) != "" { continue } inv[prefix+name] = true } } methods("bucket/", s.cfg.Releases, "Get", "List", "ListDirs", "Open", "ListMod", "ListSizes", "ListPage", "BucketShapedListDirs", "GetIfChanged", "ListETags") methods("versioned/", s.cfg.DBVersions, "ListVersions", "OpenVersion", "GetVersion", "OpenVersionRange") for _, b := range []struct { surface string iface reflect.Type reads []string }{ {"objstore.Provisioner", reflect.TypeFor[objstore.Provisioner](), nil}, {"objstore.Scanner", reflect.TypeFor[objstore.Scanner](), []string{"Capacity", "Enforcement", "Provenance", "Status", "Verdict"}}, {"mail.Provisioner", reflect.TypeFor[mail.Provisioner](), []string{"IdentityStatus"}}, {"dnsrec.Writer", reflect.TypeFor[dnsrec.Writer](), nil}, } { for i := 0; i < b.iface.NumMethod(); i++ { if name := b.iface.Method(i).Name; !slices.Contains(b.reads, name) { inv[b.surface+"/"+name] = true } } } for name, tb := range claimSQLTables { if tb.NonApp == "" { inv["sql/"+name] = true } } return inv } // ---- reads and nonApp entries ---------------------------------------------- // The generic request this section used to send every entry ({} for a body, // "claim-probe" for an id) is gone: runClaimProbe (claim_invariant_probes_test.go) // sends each one its own valid request and judges the answer and the world. // ---- the enabled constructor ------------------------------------------------ // TestClaimConstructorInstallsEveryBoundary: the console built with the flag // on from the start never hands a handler a raw writer. func TestClaimConstructorInstallsEveryBoundary(t *testing.T) { cx := newClaimCase(t) s := cx.srv if !s.cfg.HostClaim || !cx.st.claimGated() { t.Fatalf("HostClaim %v, SQL gate armed %v: want both", s.cfg.HostClaim, cx.st.claimGated()) } for _, c := range []struct { name string installed any raw any }{ {"Releases", s.cfg.Releases, cx.rawStore}, {"DBVersions", s.cfg.DBVersions, cx.rawVersions}, {"StoreProvisioner", s.cfg.StoreProvisioner, cx.rawProv}, {"StoreScanner", s.cfg.StoreScanner, cx.rawScanner}, {"MailProvisioner", s.cfg.MailProvisioner, cx.rawMail}, {"DNSWriter", s.cfg.DNSWriter, cx.rawDNS}, } { if c.installed == nil || c.installed == c.raw { t.Errorf("Config.%s is the raw writer (or nil) after ConfigureClaimRuntime: %T", c.name, c.installed) } rt := reflect.TypeOf(c.installed) if rt != nil && rt.Kind() == reflect.Pointer { rt = rt.Elem() } if rt != nil && rt.Kind() == reflect.Struct { for i := 0; i < rt.NumField(); i++ { if rt.Field(i).Anonymous { t.Errorf("Config.%s (%s) embeds %s: a promoted method would bypass the gate", c.name, rt, rt.Field(i).Type) } } } } ctx := context.Background() m := cx.rec.mark() if _, err := s.cfg.Releases.Put(ctx, "apps/bab/titogo/claim-ungated.json", []byte(`{}`), ""); err == nil { t.Error("an ungated app-key Put through Config.Releases succeeded") } if _, err := s.cfg.Releases.Put(ctx, admissionKeyOf("bab", "titogo"), []byte(`{}`), "x"); err == nil { t.Error("Config.Releases wrote a canonical admission object") } if err := s.cfg.Releases.Delete(ctx, admissionKeyOf("bab", "titogo")); err == nil { t.Error("Config.Releases deleted a canonical admission object") } if _, err := s.cfg.StoreProvisioner.EnsureBucket(ctx, "carlos-bab-titogo"); err == nil { t.Error("an ungated EnsureBucket through Config.StoreProvisioner succeeded") } if _, err := s.cfg.DBVersions.PutIfAbsent(ctx, claimFxBucket, "apps/bab/titogo/claim-ungated.json", []byte(`{}`)); err == nil { t.Error("an ungated PutIfAbsent through Config.DBVersions succeeded") } if _, err := s.store.db.ExecContext(ctx, `UPDATE apps SET claim_revision = claim_revision + 1 WHERE app = 'titogo'`); err == nil { t.Error("an ungated UPDATE of an app row succeeded on the console's store") } cx.quiesce() if events := cx.rec.since(m); len(events) != 0 { t.Errorf("ungated writes reached the raw writers: %v", events) } } // ---- new-door witnesses ----------------------------------------------------- // claimWitnessHandler answers 401 to anything but the case's owner token, // so the witnesses below are authenticated doors like every real one. func claimWitnessHandler(cx *claimCase, write func(r *http.Request) error) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { if _, ok := cx.st.CLIPrincipal(bearer(r), time.Now()); !ok { http.Error(w, "not signed in", http.StatusUnauthorized) return } if err := write(r); err != nil { http.Error(w, err.Error(), http.StatusConflict) return } w.WriteHeader(http.StatusOK) } } // claimWitnessRefusalHandler is an intentionally unsupported witness door: // 401 to anything but the owner token, otherwise write (if any) and then // production's exact move refusal. func claimWitnessRefusalHandler(cx *claimCase, write func(r *http.Request) error) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { if _, ok := cx.st.CLIPrincipal(bearer(r), time.Now()); !ok { http.Error(w, "not signed in", http.StatusUnauthorized) return } if write != nil { _ = write(r) } jsonError(w, http.StatusConflict, claimMoveUnsupportedMsg) } } var claimWitnessApps = []claimAppRef{{Account: "bab", App: "titogo"}} func claimWitnessDoor(path string, write func(cx *claimCase) func(r *http.Request) error, authed bool) *claimDoor { return &claimDoor{ apps: claimWitnessApps, setup: func(t *testing.T, cx *claimCase) { cx.srv.registerRoute("POST "+path, claimWitnessHandler(cx, write(cx))) }, do: func(t *testing.T, cx *claimCase) int { token := "" if authed { token = cx.bearer } status, _, _ := rawReq(t, http.MethodPost, cx.ts.URL+path, token, "{}") return status }, } } // The structural witnesses: each one is a door added after this test was // written. The first must fail the census, the ungated ones must fail their // open run, the gated one must pass both runs, and the detector must see a // leak when one is shown to it. func TestClaimConsoleMutationInvariantWitnesses(t *testing.T) { const witnessKey = "titogo-claim-witness.json" appKey := "apps/bab/titogo/" + witnessKey t.Run("missing fixture", func(t *testing.T) { cx := newClaimCase(t) cx.srv.registerRoute("POST /claim-witness/missing", cx.srv.handleHome) problems := claimFixtureCensus(cx.srv.claimEntries.snapshot(), claimFixtures) if !slices.ContainsFunc(problems, func(p string) bool { return strings.Contains(p, "route POST /claim-witness/missing") }) { t.Fatalf("a route with no fixture passed the census: %v", problems) } withFixture := map[string]claimFixture{} for k, v := range claimFixtures { withFixture[k] = v } withFixture["route POST /claim-witness/missing"] = claimFixture{} if p := claimFixtureCensus(cx.srv.claimEntries.snapshot(), withFixture); !slices.ContainsFunc(p, func(p string) bool { return strings.Contains(p, "must be exactly one of") }) { t.Fatalf("a fixture with no classification passed the census: %v", p) } }) ungated := map[string]func(cx *claimCase) func(r *http.Request) error{ "streaming bucket": func(cx *claimCase) func(r *http.Request) error { return func(r *http.Request) error { _, err := release.PutObjectFrom(r.Context(), cx.srv.cfg.Releases, appKey, strings.NewReader("{}"), 2, "") return err } }, "bucket": func(cx *claimCase) func(r *http.Request) error { return func(r *http.Request) error { _, err := cx.srv.cfg.Releases.Put(r.Context(), appKey, []byte(`{"ungated":true}`), "") return err } }, "sql": func(cx *claimCase) func(r *http.Request) error { return func(r *http.Request) error { _, err := cx.srv.store.db.ExecContext(r.Context(), `INSERT INTO audit (app, address, action, created_at, account_id) VALUES ('titogo', ?, 'claim-witness', 1, ?)`, claimFxOwner, OpsAccountID) return err } }, "backend": func(cx *claimCase) func(r *http.Request) error { return func(r *http.Request) error { _, err := cx.srv.cfg.StoreProvisioner.EnsureBucket(r.Context(), "carlos-bab-titogo") return err } }, "versioned": func(cx *claimCase) func(r *http.Request) error { return func(r *http.Request) error { _, err := cx.srv.cfg.DBVersions.PutIfAbsent(r.Context(), claimFxBucket, appKey, []byte(`{}`)) return err } }, } for name, write := range ungated { t.Run("fixture but ungated "+name+" mutator", func(t *testing.T) { problems, _ := runClaimDoor(t, claimWitnessDoor("/claim-witness/ungated-"+name, write, true)) if !slices.ContainsFunc(problems, func(p string) bool { return strings.Contains(p, "reached no raw app effect") }) { t.Fatalf("an ungated %s mutator passed the invariant: %v", name, problems) } }) } gated := func(cx *claimCase) func(r *http.Request) error { return func(r *http.Request) error { acct := &Account{ID: OpsAccountID, Sqid: "bab"} return cx.srv.claimAppMutation(r.Context(), acct, "titogo", "claim-witness", func(ctx context.Context) error { _, err := cx.srv.cfg.Releases.Put(ctx, appKey, []byte(`{"gated":true}`), "") return err }) } } t.Run("admitted passes, foreign closed refuses", func(t *testing.T) { problems, writers := runClaimDoor(t, claimWitnessDoor("/claim-witness/gated", gated, true)) if len(problems) != 0 { t.Fatalf("a door that holds the gate failed the invariant: %v", problems) } if !slices.Contains(writers, "bucket/Put") { t.Fatalf("the gated witness reached writers %v, want bucket/Put", writers) } }) t.Run("an unauthenticated request is not coverage", func(t *testing.T) { problems, _ := runClaimDoor(t, claimWitnessDoor("/claim-witness/unauthenticated", gated, false)) if !slices.ContainsFunc(problems, func(p string) bool { return strings.Contains(p, "answered 401") }) || !slices.ContainsFunc(problems, func(p string) bool { return strings.Contains(p, "reached no raw app effect") }) { t.Fatalf("a 401 counted as coverage: %v", problems) } }) // The refused mode: only the owner's real request answered with the // exact unsupported refusal and no effect at all passes. refusal := func(path string, authed bool, write func(cx *claimCase) func(r *http.Request) error) *claimRefusalFixture { return &claimRefusalFixture{ apps: claimWitnessApps, setup: func(t *testing.T, cx *claimCase) { var w func(r *http.Request) error if write != nil { w = write(cx) } cx.srv.registerRoute("POST "+path, claimWitnessRefusalHandler(cx, w)) }, do: func(t *testing.T, cx *claimCase) (int, string) { token := "" if authed { token = cx.bearer } status, _, out := rawReq(t, http.MethodPost, cx.ts.URL+path, token, "{}") return status, out }, status: http.StatusConflict, msg: claimMoveUnsupportedMsg, } } t.Run("an exact authenticated refusal passes", func(t *testing.T) { if problems := runClaimRefusal(t, refusal("/claim-witness/refused", true, nil)); len(problems) != 0 { t.Fatalf("an exact unsupported refusal with no effect failed: %v", problems) } }) t.Run("an auth refusal is not an explicit refusal", func(t *testing.T) { problems := runClaimRefusal(t, refusal("/claim-witness/refused-unauthenticated", false, nil)) has := func(s string) bool { return slices.ContainsFunc(problems, func(p string) bool { return strings.Contains(p, s) }) } if !has("open run: the authenticated valid request answered 401") || !has("is not exactly the unsupported refusal") { t.Fatalf("a 401 counted as an explicit refusal: %v", problems) } }) t.Run("a refusal that reached an effect is not a refusal", func(t *testing.T) { problems := runClaimRefusal(t, refusal("/claim-witness/refused-after-write", true, gated)) if !slices.ContainsFunc(problems, func(p string) bool { return strings.Contains(p, "open run: the refusal wrote control object") }) || !slices.ContainsFunc(problems, func(p string) bool { return strings.Contains(p, "open run: raw bucket Put "+appKey+" is app-scoped") }) { t.Fatalf("a refusal that wrote passed: %v", problems) } }) t.Run("the census admits only exact unsupported refusals", func(t *testing.T) { do := func(*testing.T, *claimCase) (int, string) { return 0, "" } for name, c := range map[string]struct { f claimRefusalFixture want string }{ "401": {claimRefusalFixture{apps: claimWitnessApps, do: do, status: http.StatusUnauthorized, msg: claimMoveUnsupportedMsg}, "answering 401"}, "404": {claimRefusalFixture{apps: claimWitnessApps, do: do, status: http.StatusNotFound, msg: claimMoveUnsupportedMsg}, "answering 404"}, "generic 409": {claimRefusalFixture{apps: claimWitnessApps, do: do, status: http.StatusConflict, msg: claimRetireBusyMsg}, "not an intentionally unsupported operation"}, "no request": {claimRefusalFixture{apps: claimWitnessApps, status: http.StatusConflict, msg: claimMoveUnsupportedMsg}, "no affected app or no request"}, } { f := c.f problems := claimFixtureCensus(nil, map[string]claimFixture{"route POST /claim-witness/refused": {refused: &f}}) if !slices.ContainsFunc(problems, func(p string) bool { return strings.Contains(p, c.want) }) { t.Errorf("%s: the census admitted the refusal: %v", name, problems) } } }) // The mutation check for the token-revoke census entry. The real door // (claimLegacyAppTokenRevokeDoor) passes in TestClaimConsoleMutationInvariant; // here the same staging and the same valid request are aimed at a route // that revokes the same token WITHOUT admission, which is exactly what // RevokeCLIToken did before it was admitted. The census must fail it. t.Run("the SQL trigger refuses a raw token revoke", func(t *testing.T) { const path = "/claim-witness/ungated-token-revoke" d := claimLegacyAppTokenRevokeDoor(path) stage := d.setup d.setup = func(t *testing.T, cx *claimCase) { stage(t, cx) cx.srv.registerRoute("POST "+path, claimWitnessHandler(cx, func(r *http.Request) error { return cx.st.claimRawRevokeCLIToken(claimFxOwner, cx.vars["apptok"]) })) } d.do = func(t *testing.T, cx *claimCase) int { status, _, _ := rawReq(t, http.MethodPost, cx.ts.URL+path, cx.bearer, "{}") return status } problems, _ := runClaimDoor(t, d) has := func(s string) bool { return slices.ContainsFunc(problems, func(p string) bool { return strings.Contains(p, s) }) } if !has("never registered with the gate") || !has("still live after the open run") { t.Fatalf("an ungated legacy app token revoke passed the census: %v", problems) } }) t.Run("an ungated token-revoke route fails the census", func(t *testing.T) { const path = "/claim-witness/ungated-token-revoke" d := claimLegacyAppTokenRevokeDoor(path) stage := d.setup d.setup = func(t *testing.T, cx *claimCase) { stage(t, cx) cx.srv.registerRoute("POST "+path, claimWitnessHandler(cx, func(r *http.Request) error { // Deliberately simulate a forgotten gate that reaches raw SQL. // This test-only bypass proves the census detects a LANDED // mutation independently of the production SQL trigger. scope := []claimSQLScope{{AccountID: OpsAccountID, App: "kass", AppID: cx.appIDs[claimAppRef{Account: "bab", App: "kass"}], Mode: claimModeEffect}} return cx.st.withClaimSQL(gateCtx(t), scope, allowAllClaimSQL, func(st *Store) error { return st.claimRawRevokeCLIToken(claimFxOwner, cx.vars["apptok"]) }) })) } d.do = func(t *testing.T, cx *claimCase) int { status, _, _ := rawReq(t, http.MethodPost, cx.ts.URL+path, cx.bearer, "{}") return status } problems, _ := runClaimDoor(t, d) has := func(s string) bool { return slices.ContainsFunc(problems, func(p string) bool { return strings.Contains(p, s) }) } if !has("never registered with the gate") || !has("revoked while its app was closed") || !has("SQL rows of bab/kass changed in [cli_tokens]") { t.Fatalf("an ungated legacy app token revoke passed the census: %v", problems) } }) t.Run("a rejected token-revoke probe is not coverage", func(t *testing.T) { // The signed-in owner with a valid CSRF token, but an id that names // no row: the handler answers a redirect (303, below 400) and // revokes nothing. Status alone would pass it. d := claimLegacyAppTokenRevokeDoor("/auth/cli/tokens/revoke") d.do = claimForm("/auth/cli/tokens/revoke", "id", "no-such-listing-id") problems, _ := runClaimDoor(t, d) has := func(s string) bool { return slices.ContainsFunc(problems, func(p string) bool { return strings.Contains(p, s) }) } if !has("never registered with the gate") || !has("reached no raw app effect") || !has("still live after the open run") { t.Fatalf("a rejected probe counted as coverage: %v", problems) } }) t.Run("a person token revoke needs no admission", func(t *testing.T) { cx := newClaimCase(t) raw := mintUserToken(t, cx.st, claimFxOwner, time.Now()) id := hashToken(raw) restore := cx.foreign(t, claimAppRef{Account: "bab", App: "kass"}) defer restore() w := cx.measureRun(t, claimWitnessApps, func() int { return cx.form(t, "/auth/cli/tokens/revoke", "id", id) }) if w.status == 0 || w.status >= 400 { t.Fatalf("the person token revoke answered %d", w.status) } toks, err := cx.st.CLITokens(claimFxOwner) if err != nil { t.Fatal(err) } for _, tok := range toks { if tok.ID == id { t.Fatal("the person token is still live: its revoke needed an admission it must not need") } } }) // RevokeCLITokenValue deletes by hash alone, so it removes a legacy app // token as readily as a person's. The logout door in front of it only // admits person tokens (cliUser), but the method does not rely on that: a // row that names an app is admitted for that app first. t.Run("an app token revoked by its own value is admitted", func(t *testing.T) { cx := newClaimCase(t) raw, err := cx.st.NewAppToken(OpsAccountID, "kass", claimFxOwner, "census app token", time.Now()) if err != nil { t.Fatal(err) } live := func() bool { _, ok := cx.st.CLIPrincipal(raw, time.Now()) return ok } if !live() { t.Fatal("the staged app token is not live") } restore := cx.foreign(t, claimAppRef{Account: "bab", App: "kass"}) err = cx.st.RevokeCLITokenValueContext(context.Background(), raw) restore() if err == nil { t.Fatal("an app token was revoked by value while its app was closed under a foreign exclusive operation") } if !live() { t.Fatal("the refused revoke still removed the app token's row") } if err := cx.st.RevokeCLITokenValueContext(context.Background(), raw); err != nil { t.Fatalf("the open revoke by value: %v", err) } if live() { t.Fatal("the app token is still live after the open revoke by value") } if err := cx.st.RevokeCLITokenValueContext(context.Background(), raw); err != nil { t.Fatalf("revoking a token that is already gone is not an error: %v", err) } }) t.Run("a person token revoked by value needs no admission", func(t *testing.T) { cx := newClaimCase(t) raw := mintUserToken(t, cx.st, claimFxOwner, time.Now()) restore := cx.foreign(t, claimAppRef{Account: "bab", App: "kass"}) defer restore() if err := cx.st.RevokeCLITokenValueContext(context.Background(), raw); err != nil { t.Fatalf("a person's logout needed an app admission: %v", err) } if _, ok := cx.st.CLITokenAddress(raw, time.Now()); ok { t.Fatal("the person token is still live after its own revoke") } }) t.Run("the foreign detector sees each kind of leak", func(t *testing.T) { ref := claimAppRef{Account: "bab", App: "titogo"} refs := []claimAppRef{ref} for name, w := range map[string]claimWindow{ "bucket": {events: []claimRawEvent{{Surface: "bucket", Method: "Put", Key: appKey, Scopes: []hostops.AppScope{ref.scope()}}}}, "admission": {events: []claimRawEvent{{Surface: "bucket", Method: "Put", Key: admissionKeyOf("bab", "titogo"), Control: true, Scopes: []hostops.AppScope{ref.scope()}}}}, "http": {events: []claimRawEvent{{Surface: "http", Method: "POST", Key: "https://app.example/mail/events", Unscoped: true}}}, "backend": {events: []claimRawEvent{{Surface: "objstore.Provisioner", Method: "EnsureBucket", Key: "carlos-bab-titogo", Unscoped: true}}}, "versioned": {events: []claimRawEvent{{Surface: "versioned:data", Method: "DeleteVersion", Key: "instances/acme.go.tito.io/db", Unscoped: true}}}, "sql": {sqlChanged: map[claimAppRef][]string{ref: {"audit"}}}, } { if len(claimForeignProblems(ref, refs, w)) == 0 { t.Errorf("a %s leak under a foreign exclusive went unreported", name) } } quiet := claimWindow{status: http.StatusConflict, events: []claimRawEvent{ {Surface: "bucket", Method: "Put", Key: hostops.MailGateKey, Control: true}, {Surface: "bucket", Method: "Put", Key: "status/boxes/box.json", NonApp: "deployment status"}, }} if p := claimForeignProblems(ref, refs, quiet); len(p) != 0 { t.Errorf("non-app effects were reported as app leaks: %v", p) } if p := claimOpenProblems(refs, quiet); len(p) == 0 { t.Error("an open run with no admission write and no app effect passed") } }) } func TestClaimLegacyAppTokenRevokeRegression(t *testing.T) { problems, _ := runClaimDoor(t, claimLegacyAppTokenRevokeDoor("/auth/cli/tokens/revoke")) for _, problem := range problems { t.Error(problem) } }