package engines import ( "fmt" "os" "sort" "strings" "testing" ) // Witness scenarios, in hostlife's shape (internal/sim/hostlife/witness_test.go). // // The random sweep reaches some interleavings rarely. A witness is one // fixed, hand-scheduled scenario for one rule: it forces exactly the // interleaving the rule exists for, as a sequence of protocol steps, and // draws nothing from the world's RNG. It must hold (no invariant violated) // with the rules in place, and violate its named invariant when the rule's // mutation is applied, alone or in the stated combination. // // A witness calls w.check() at each step boundary, as doStep does in the // sweep; event invariants (INV-P at delivery) are recorded inside the step. type witness struct { rule string // the rule it is for needs string // the mutation set under which it must violate: the rule alone, or a combination the rule belongs to want string // the failure the scenario names: a substring of the first violation; "+X" parts must also be recorded note string // what it shows run func(t *testing.T, w *world) } var witnesses = []witness{ {"set-before-repoint", "set-before-repoint", "INV-I1+INV-P", "R2-1: adoption of a declaring build over a public route; between the repoint and the restart's gate a public request socket-activates the live unit under 0 (INV-I1 at the start) and is served by the declaring build (INV-P)", wSetBeforeRepoint}, {"set-before-repoint", "set-before-repoint", "INV-I1", "R2-1: between the repoint and the restart's gate, Restart=always starts the live unit, a declaring build, under 0; the restart then fences and replaces it, so only the check at the start sees it", wSetBeforeRepointSystemd}, {"fence-needs-reload", "fence-needs-reload", "INV-I1", "R10-1: the edge's reload keeps failing; a failed reload that advances the fenced counter lets adoption repoint and start the declaring build under the public map", wFenceNeedsReload}, {"fence-drains-admitted", "fence-drains-admitted", "INV-P", "R3-1: a request admitted at 0 waits in the activator's queue across the adoption; a fence that does not drain it lets it resume into the declaring build", wFenceDrainsAdmitted}, {"fence-box-wide", "fence-box-wide", "INV-P", "R13-2: a request admitted through a service row aimed at the engine's backend; the row is deleted, the request held across a declaring adoption; a fence scoped to the group's rows leaves it, and it resumes into the engine", wFenceBoxWide}, {"gate-adopt", "gate-adopt,set-before-repoint", "INV-I1", "redundant alone: adoption's set-before is the same gate reached earlier, so with it in place the restart's gate finds the group fenced; with both gone adoption starts the declaring build under 0", wGateAdopt}, {"gate-swap", "gate-swap", "INV-I1", "R7-1: a generation swap's incoming generation is a declaring build; ungated, it starts under 0", wGateSwap}, {"gate-fallback", "gate-fallback", "INV-I1+INV-P", "R9-1: after --public the first wake fails and the wake plan falls back to a declaring GoodVersion; ungated, it starts under 0 (INV-I1) and the triggering public request is served by it (INV-P)", wGateFallback}, {"gate-recovery", "gate-recovery", "INV-I1", "R9-1: wake recovery picks a declaring GoodVersion; ungated, it starts under 0", wGateRecovery}, {"gate-seed", "gate-seed", "INV-I1", "a seed of a declaring build; ungated, it starts under 0", wGateSeed}, {"gate-cmd", "gate-cmd", "INV-I1", "a carlos run command of a declaring build; ungated, it starts under 0", wGateCmd}, {"gate-waits-when-written", "gate-waits-when-written", "INV-I1", "R10-1: adoption times out with 1 written and the reload failing; a gate that waits only when it wrote the mark lets wake recovery start a declaring GoodVersion under the public map", wGateWaitsWhenWritten}, {"unreadable-declares", "unreadable-declares", "INV-I1", "a declaring build whose manifest cannot be read, launched under 0; read as non-declaring, it starts under 0", wUnreadableDeclares}, {"live-link-only-adoption", "live-link-only-adoption", "INV-I1", "R11: something other than adoption moves the live link to a declaring build; systemd's Restart=always starts it, ungated, under 0", wLiveLinkOnly}, {"mark-on-final-upstream", "mark-on-final-upstream", "INV-P", "R1-2: a public request through an alias whose final upstream is the engine; checked on the alias row, it reaches the engine", wMarkOnFinalUpstreamAlias}, {"mark-on-final-upstream", "mark-on-final-upstream", "INV-P", "R1-2: a public request through a static row's path map whose final upstream is the engine; checked on the static row, it reaches the engine", wMarkOnFinalUpstreamPath}, // Task 3: classification, minted groups, the operator clear {"classify-null-refused", "classify-null-refused", "INV-P", "R2-2: the engine runs a declaring build whose live manifest cannot be read; the upgrade leaves it NULL; an edge that serves NULL like 0 admits a public request into the engine", wClassifyNullRefused}, {"classify-staged", "classify-staged", "INV-I1+INV-P", "R8-1: an awake engine is repointed to a plain build without a restart, so the declaring build still runs; an upgrade that reads only the live manifest classifies the group 0, under the running engine (INV-I1) and a public request reaches it (INV-P)", wClassifyStaged}, {"classify-staged", "classify-staged,dedicated-launch-check", "INV-I1", "R8-1, app scope: two instances of one app; the declaring build is staged once, by the other instance; this one runs it from that copy, is repointed to a plain build without a restart and upgraded; read per group, or without the staged clause, it classifies 0 under the running engine. Since Task 4 the launch check refuses the second instance's declaring adoption (another group's row on the box), so it needs that check gone too, and the edge's dedicated refusal keeps the public out (INV-I1 only)", wClassifyStagedApp}, {"minted-follow-owner", "minted-follow-owner,dedicated-launch-check,dedicated-edge-refusal", "INV-P", "R5-1: an app minted a platform row while public; adoption of a declaring build marks and fences the app; a minted row that is its own group stays 0 and a public request by the minted host reaches the engine", wMintedFollowOwner}, {"minted-follow-owner", "minted-follow-owner", "INV-P", "R5-1, by the grant: --grant doc-engine on the minted row of a public app; a minted row that is its own group takes the mark alone, and a public request by the app's own host reaches the granted app", wMintedFollowOwnerGrant}, {"grant-marks-group", "grant-marks-group", "INV-P", "R6-1: --grant doc-engine on the minted row of a plain release; a grant that marks only when it is on the owner's own row leaves the group 0, and a public request by the minted host reaches the granted app", wGrantMarksGroup}, {"static-minted-public", "static-minted-public", "INV-AV", "R6-2: a static app with a minted row and a canary; an upgrade that classifies minted static rows like instance rows leaves them NULL, and they never answer (liveness)", wStaticMintedPublic}, {"owner-nonempty-unique", "owner-nonempty-unique,dedicated-edge-refusal", "INV-P", "R7-2: an engine instance's minted row was minted before owners were recorded; a fallback that matches empty sockets puts it in the group of a later plain instance of the app, so --public on that instance writes 0 on the engine's minted row. Since Task 4 its availability effect is gone (a misgrouped row on a box that homes a closed group is refused by design either way) and its safety effect needs the edge's dedicated refusal gone too: a public request by the minted host reaches the engine", wOwnerNonEmptyUnique}, {"public-checks-live", "public-checks-live", "INV-I1+INV-P", "--public on a group whose live release declares; without step 1 the clear writes 0, and systemd's socket activation starts the declaring live build ungated (INV-I1) for a public request (INV-P)", wPublicChecksLive}, {"public-stops-group", "public-stops-group,public-confirms-empty", "INV-I1+INV-P", "R7-1/R8-2: redundant alone: without the stop, step 3 finds the declaring generation still serving and refuses; with both gone, a clear after a rollout that repointed the live link to a plain build and only scheduled the swap writes 0 under the still-serving declaring generation", wPublicStopsGroup}, {"public-confirms-empty", "public-confirms-empty", "INV-I1+INV-P", "the clear's stop does not complete and leaves the declaring generation running; without step 3 the clear writes 0 under it (INV-I1) and a public request reaches it (INV-P)", wPublicConfirmsEmpty}, {"public-holds-lock", "public-holds-lock", "INV-I1+INV-P", "--public checks a plain live release; without the adoption lock, adoption repoints to a declaring build before the stop; the clear writes 0 over a declaring live link, and socket activation starts it ungated", wPublicHoldsLock}, // Task 4: the dedicated engine box and ingress {"ports-excluded", "ports-excluded", "INV-P", "R11-1: the engine route holds a port grant from a plain release, so the box renders its listener; adoption of a declaring build marks the group; a box that keeps the listener lets a public connection to the port, which the firewall admits as granted, reach the engine past the edge, with no capability pass in between", wPortsExcluded}, {"ports-excluded", "ports-excluded", "INV-P", "R11-1, by the grant: the public engine route on a plain build holds a port grant and its listener; --grant doc-engine marks the group; a box that keeps the listener lets a public connection reach the granted group past the edge, with no capability pass in between", wPortsExcludedGrant}, {"dedicated-launch-check", "dedicated-launch-check,dedicated-edge-refusal", "INV-P", "R12-2: redundant alone: a service row aimed at the engine's backend exists before a declaring adoption; the launch check refuses the adoption; without it the edge's refusal still refuses the row once the group is internal; with both gone a public request through the row reaches the engine", wDedicatedLaunchCheck}, {"dedicated-edge-refusal", "dedicated-edge-refusal", "INV-P", "R12-2: a service row aimed at the engine's backend is added after the engine runs internal, where no launch check sees it; an edge that serves it lets a public request reach the engine", wDedicatedEdgeRefusal}, {"ingress-precondition", "ingress-precondition", "INV-P", "R13-1: a service row on the calling box aimed at the engine box's address; a firewall that admits it lets a public request through the calling box's edge reach the engine directly", wIngressPrivate}, {"ingress-precondition", "ingress-precondition", "INV-P", "R12-1: the engine binds a port of its own on every interface; a firewall that admits it lets a public connection reach the engine", wOwnPort}, // Task 5: calls, identity, grants and arming {"cgroup-identity", "cgroup-identity", "INV-ID", "R1-1: on an armed box a sibling instance's process connects to the granted instance's engine socket (mode 0666); a socket whose path is the identity records the granted app as the caller", wCgroupIdentity}, {"door-uid", "door-uid", "INV-ID", "R2-3: a flat tenant running as carlos reads a granted app's door key and calls an engine through the door; a door that takes the key from uid carlos records the granted app as the caller", wDoorUID}, {"hop-caller-signed", "hop-caller-signed", "INV-ID", "a public client sends the engine box a request carrying X-Carlos-Hop-Caller naming a granted app; an engine box that takes the caller from the header rather than a verified hop records that app as the caller", wHopCallerSigned}, {"engine-box-recheck", "engine-box-recheck", "INV-G", "G4: a revoke is read everywhere; the calling box is rebuilt (no floor) and a bucket writer at once replays the record the revoke superseded, still inside its ten minutes; the calling box admits on it; an engine box that trusts the calling box's admission serves the revoked app although it has read the revoke", wEngineBoxRecheck}, {"exact-route-engine", "exact-route-engine", "INV-G", "R4-2: an app holds engine A's grant only; A's hostname is a static row whose path map points at engine B's route; an engine call resolved through the path map reaches B", wExactRouteEngine}, {"floor-persisted", "floor-persisted", "INV-G", "R1-4: a grant is revoked and read everywhere; a bucket writer at once replays the older record, still inside its ten minutes, which the floor refuses; both edges restart; a floor not persisted admits the replay, and the revoke both boxes had read is no longer in force", wFloorPersisted}, {"stale-clears-grants", "stale-clears-grants,grant-freshness-bound", "INV-G", "§6.4: redundant alone: a table's record was signed no later than its last good read, so the signed_at bound empties a stale table first. The bucket is unreachable when a revoke is published; ten minutes pass with no good read; with both checks gone the stale table still grants and serves the revoked app", wStaleClearsGrants}, {"armed-refuses-unisolated", "armed-refuses-unisolated", "INV-ARM", "R3-2/R4-1: on an armed box with engine sockets open, a foreign instance starts and a worker spawns; a box that does not refuse them runs unisolated tenant code beside open sockets", wArmedRefusesUnisolated}, {"armed-refuses-unisolated", "armed-refuses-unisolated", "INV-ARM", "R4-1: on an armed box, an app's worker spawns into carlos-exec@ (worker alone)", wArmedRefusesWorker}, {"armed-refuses-unisolated", "armed-refuses-unisolated", "INV-ARM", "R3-2: on an armed box, a flat instance starts (flat alone)", wArmedRefusesFlat}, {"arm-checks-box", "arm-checks-box", "INV-ARM", "R5-2: an app's worker runs in carlos-exec@ on an unarmed box; arming that does not check the box opens engine sockets beside it", wArmChecksBox}, // Final review {"new-group-fenced", "new-group-fenced", "INV-I1+INV-P", "a public request is admitted through a service row aimed at address X and held; the row is deleted; a declaring instance is provisioned at X; a new row's NULL taken as fenced lets its first launch skip the reload, so the held request is never drained and reaches the engine", wNewGroupFenced}, {"grant-freshness-bound", "grant-freshness-bound", "INV-G", "R5: the console re-signs (revision 2) and both edges read it; it revokes shop (revision 3) and a bucket writer puts revision 2 back before any edge reads 3; the floor admits it (equal revision); a mirror with no signed_at bound keeps the revoked grant until revision 2 expires, so a call ten minutes and one poll later reaches the engine", wGrantFreshnessBound}, {"clear-launch-lock", "clear-launch-lock", "INV-I1+INV-P", "--public on an internal group whose live release is plain; after its empty check and before its write of 0, a carlos run command of a declaring build passes the gate (the group is still fenced as internal); a launch that ignores the group's lock starts it, and --public writes 0 under it", wClearLaunchLock}, } // TestWitnesses is the deterministic half of the mutation evidence. // // With no mutation (the gate) it proves every witness's claim in full: the // scenario holds with every rule in place, holds under every proper subset of // its stated mutation set (so a rule a combination names is shown necessary, // not assumed), and under the whole set violates the failure the scenario // names, not merely some invariant. // // Under ENGINESIM_MUTATE=m (the harness) it runs the witnesses of the rules // in m; a witness whose set is within m must violate with its named failure. // ENGINESIM_WITNESS=rule runs one rule's witnesses with their trace. func TestWitnesses(t *testing.T) { only := os.Getenv("ENGINESIM_WITNESS") if mutated == "" { ran := 0 for _, x := range witnesses { if only != "" && x.rule != only { continue } ran++ check(t, x, "", false, only != "") for _, s := range properSubsets(x.needs) { check(t, x, s, false, only != "") } check(t, x, x.needs, true, only != "") } if only != "" && ran == 0 { t.Fatalf("no witness for rule %q", only) } return } set := mutationSet(mutated) for _, x := range witnesses { if only != "" && x.rule != only { continue } if !set[x.rule] { continue } needs := true for _, m := range strings.Split(x.needs, ",") { if !set[m] { needs = false } } if needs { check(t, x, mutated, true, only != "") continue } // the mutation set is not the witness's: report, enforce nothing report(t, x, mutated, only != "") } } // TestWitnessRulesKnown: every witness names a rule in allRules, and so does // every member of its mutation set. func TestWitnessRulesKnown(t *testing.T) { known := map[string]bool{} for _, r := range allRules { known[r] = true } for _, x := range witnesses { if !known[x.rule] { t.Errorf("witness for unknown rule %q", x.rule) } for _, m := range strings.Split(x.needs, ",") { if !known[m] { t.Errorf("witness %s needs unknown rule %q", x.rule, m) } } } } func report(t *testing.T, x witness, m string, trace bool) { saved := mutated mutated = m defer func() { mutated = saved }() w := newWitness() x.run(t, w) tag := "held" if len(w.violations) > 0 { tag = "violated: " + w.violations[0] } t.Logf("witness %s under %q (needs %q): %s", x.rule, m, x.needs, tag) if trace { dump(t, w) } } // check runs one witness under mutation m, reports it, and fails the test if // the outcome is not the stated one: held when no violation is expected, the // named failure when one is. func check(t *testing.T, x witness, m string, expectViolation bool, trace bool) { saved := mutated mutated = m defer func() { mutated = saved }() w := newWitness() x.run(t, w) violated := len(w.violations) > 0 tag := "held" if violated { tag = "violated: " + w.violations[0] } t.Logf("witness %s under %q (needs %q): %s", x.rule, m, x.needs, tag) switch { case expectViolation && !violated: t.Errorf("witness %s held under %q, which it must violate", x.rule, m) dump(t, w) case expectViolation && !wantMet(w.violations, x.want): t.Errorf("witness %s under %q violated, but not its named failure %q", x.rule, m, x.want) dump(t, w) case !expectViolation && violated: t.Errorf("witness %s violated under %q, where it must hold", x.rule, m) dump(t, w) } if trace { dump(t, w) } } // wantMet: the first violation names want's first part, and every further // "+" part is named by some recorded violation. func wantMet(vs []string, want string) bool { parts := strings.Split(want, "+") if len(vs) == 0 || !namedFailure(vs[0], parts[0]) { return false } for _, f := range parts[1:] { found := false for _, v := range vs { found = found || namedFailure(v, f) } if !found { return false } } return true } // namedFailure: the violation names the witness's failure. want is one // failure or several separated by "|"; a failure that ends in a digit must not // run on into another (INV-I1 is not INV-I10). func namedFailure(v, want string) bool { for _, f := range strings.Split(want, "|") { for i := 0; ; { j := strings.Index(v[i:], f) if j < 0 { break } end := i + j + len(f) last := f[len(f)-1] if !(last >= '0' && last <= '9') || end == len(v) || v[end] < '0' || v[end] > '9' { return true } i = i + j + 1 } } return false } func mutationSet(m string) map[string]bool { set := map[string]bool{} for _, x := range strings.Split(m, ",") { if x != "" { set[x] = true } } return set } // properSubsets: every non-empty proper subset of a comma-separated set. func properSubsets(needs string) []string { xs := strings.Split(needs, ",") var out []string for mask := 1; mask < (1< 0 { t.Fatalf("witness: the fixture violates: %v", w.violations) } return b } // ---------- the witnesses ---------- // wSetBeforeRepoint (R2-1): adoption repoints the live link to a declaring // build; before its restart a public request arrives, socket activation // starts the live unit (no activator, no gate) and serves it. With the mark // set and fenced before the repoint, the request is refused. func wSetBeforeRepoint(t *testing.T, w *world) { b := w.publicEngine(t) w.stopGroup(b, eng) w.adopt(b, eng, vEngine, func() { if r := w.admitPublic(b, engHost, "/"); r != nil { w.socketActivate(b, eng) w.deliver(r) } }) w.check() } // wSetBeforeRepointSystemd (R2-1): adoption repoints the live link to a // declaring build; before its restart, Restart=always starts the live unit // (no activator, no gate). No request is involved. func wSetBeforeRepointSystemd(t *testing.T, w *world) { b := w.publicEngine(t) w.adopt(b, eng, vEngine, func() { w.systemdRestart(b, eng) }) w.check() } // wFenceDrainsAdmitted (R3-1): a request admitted at 0 waits in the // activator's queue while adoption marks, fences, repoints and restarts; it // then resumes. func wFenceDrainsAdmitted(t *testing.T, w *world) { b := w.publicEngine(t) r := w.admitPublic(b, engHost, "/") if r == nil { t.Fatalf("witness: a public request to a public route was refused") } r.Held = true w.adopt(b, eng, vEngine, nil) w.check() w.deliver(r) w.check() } // r10Sequence (R10-1): the edge's reload fails; adoption of a declaring // build writes 1 and times out without repointing; then wake recovery picks // the declaring GoodVersion. func r10Sequence(t *testing.T, w *world) { b := w.publicEngine(t) b.Good[eng] = vEngine b.ReloadFails = true w.logf("witness: the edge's reload starts failing") w.adopt(b, eng, vEngine, nil) w.check() w.stopGroup(b, eng) w.wake(b, eng, "recovery", nil) w.check() } func wFenceNeedsReload(t *testing.T, w *world) { r10Sequence(t, w) } func wGateWaitsWhenWritten(t *testing.T, w *world) { r10Sequence(t, w) } // wFenceBoxWide (R13-2): a service row aimed at the engine's backend admits a // public request; the row is deleted and the request held; adoption of a // declaring build fences the group; the request resumes on the proxy it // already holds. func wFenceBoxWide(t *testing.T, w *world) { b := w.publicEngine(t) const svc = "svc.example" b.Rows[svc] = &row{Host: svc, Kind: "service", Backend: b.Rows[engHost].Backend, PathTo: map[string]string{}} b.Mark[svc] = markPublic w.reload(b) w.check() r := w.admitPublic(b, svc, "/") if r == nil { t.Fatalf("witness: the service row refused a public request: %s", w.boxesState()) } r.Held = true delete(b.Rows, svc) delete(b.Mark, svc) w.logf("witness: row %s deleted; request %d still holds its proxy", svc, r.ID) w.adopt(b, eng, vEngine, nil) w.check() w.deliver(r) w.check() } // wGateAdopt: adoption of a declaring build with no other start. func wGateAdopt(t *testing.T, w *world) { b := w.publicEngine(t) w.adopt(b, eng, vEngine, nil) w.check() } // wGateSwap (R7-1), wGateSeed, wGateCmd, wGateRecovery: a declaring build // started by that path under 0. func gatePath(path string) func(*testing.T, *world) { return func(t *testing.T, w *world) { b := w.publicEngine(t) if path == "recovery" { b.Good[eng] = vEngine w.stopGroup(b, eng) w.wake(b, eng, "recovery", nil) } else { w.launch(b, eng, vEngine, path) } w.check() } } func wGateSwap(t *testing.T, w *world) { gatePath("swap")(t, w) } func wGateSeed(t *testing.T, w *world) { gatePath("seed")(t, w) } func wGateCmd(t *testing.T, w *world) { gatePath("cmd")(t, w) } func wGateRecovery(t *testing.T, w *world) { gatePath("recovery")(t, w) } // oraclePublic makes box 0's engine group public on a plain live release. func oraclePublic(w *world, b *box) { b.Live[eng] = vPlain w.clearPublic(b, eng) } // wGateFallback (R9-1): the group ran a declaring build, was adopted to a // plain one (the mark stays 1) and cleared with --public; a public request // wakes it, the first wake fails and the plan falls back to the declaring // GoodVersion. func wGateFallback(t *testing.T, w *world) { b := w.boxes[0] b.Live[eng], b.Good[eng] = vEngine, vEngine if !w.launch(b, eng, vEngine, "seed") { t.Fatalf("witness: the engine did not start under its fenced mark: %s", w.boxesState()) } w.check() w.adopt(b, eng, vPlain, nil) w.check() b.Good[eng] = vEngine // the plain build has not proved itself yet if !w.clearPublic(b, eng) { t.Fatalf("witness: --public refused on a plain live release: %s", w.boxesState()) } r := w.admitPublic(b, engHost, "/") if r == nil { t.Fatalf("witness: a public request after --public was refused: %s", w.boxesState()) } r.Held = true w.wake(b, eng, "fallback", r) w.check() } // wUnreadableDeclares: a build that declares but whose manifest cannot be // read, seeded under 0. func wUnreadableDeclares(t *testing.T, w *world) { b := w.publicEngine(t) w.launch(b, eng, vUnreadable, "seed") w.check() } // wLiveLinkOnly (R11): something other than adoption moves the live link to // a declaring build, then systemd restarts the unit from the live link. func wLiveLinkOnly(t *testing.T, w *world) { b := w.publicEngine(t) w.moveLive(b, eng, vEngine, "a rollout tool") w.check() w.systemdRestart(b, eng) w.check() } // markOnFinalUpstream (R1-2): the engine runs fenced as internal; one public // request arrives by host and path, a row that resolves to the engine's row. func markOnFinalUpstream(t *testing.T, w *world, host, path string) { b := w.boxes[0] b.Live[eng], b.Good[eng] = vEngine, vEngine if !w.launch(b, eng, vEngine, "seed") { t.Fatalf("witness: the engine did not start under its fenced mark: %s", w.boxesState()) } w.check() if r := w.admitPublic(b, host, path); r != nil { w.deliver(r) } w.check() } func wMarkOnFinalUpstreamAlias(t *testing.T, w *world) { markOnFinalUpstream(t, w, engAlias, "/") } func wMarkOnFinalUpstreamPath(t *testing.T, w *world) { markOnFinalUpstream(t, w, engSite, engSitePath) } // ---------- Task 3: classification, minted groups, the operator clear ---------- // requestBy sends one public request by host and delivers it if admitted. func (w *world) requestBy(b *box, host string) { if r := w.admitPublic(b, host, "/"); r != nil { w.deliver(r) } } // wClassifyNullRefused (R2-2): the engine's live release declares but its // manifest cannot be read; the upgrade can classify it neither way, so it // stays NULL; a public request arrives. func wClassifyNullRefused(t *testing.T, w *world) { b := w.boxes[0] if !w.adopt(b, eng, vUnreadable, nil) { t.Fatalf("witness: adoption of the engine refused: %s", w.boxesState()) } w.check() w.upgrade(b) w.check() if m := w.groupMark(b, eng); m != markNull { t.Fatalf("witness: the upgrade classified an unreadable live release %v, want NULL", m) } w.requestBy(b, engHost) w.check() } // wClassifyStaged (R8-1): the engine runs a declaring build; adoption of a // plain build repoints the awake route without a restart, so the declaring // build keeps running; then the agent upgrades and a public request arrives. func wClassifyStaged(t *testing.T, w *world) { b := w.boxes[0] if !w.adopt(b, eng, vEngine, nil) { t.Fatalf("witness: adoption of the engine refused: %s", w.boxesState()) } w.check() if !w.repointAwake(b, eng, vPlain) { t.Fatalf("witness: the awake repoint was refused: %s", w.boxesState()) } w.check() w.upgrade(b) w.check() w.requestBy(b, engHost) w.check() } // wClassifyStagedApp (R8-1, app scope): tcp-a stages the declaring build // first; tcp-b, another instance of the app, adopts the same build (staged // for the app already, so recorded under tcp-a only), is repointed to a // plain build without a restart, and the agent upgrades. // // Since Task 4 the launch check refuses tcp-b's declaring adoption beside // tcp-a (a proxying row of another group): the model has no era before the // upgrade in which launches pass no gate, so this scenario needs the launch // check gone too. The edge's dedicated refusal then still refuses tcp-b's // public request (tcp-a is homed internal), so it violates INV-I1 only. func wClassifyStagedApp(t *testing.T, w *world) { b := w.engineFreeBox(t) if !w.provision(b, tcpA, tcpApp, "10.0.0.7:9101", vEngine) || !w.provision(b, tcpB, tcpApp, "10.0.0.7:9102", vPlain) { t.Fatalf("witness: provisioning refused: %s", w.boxesState()) } if !w.adopt(b, tcpB, vEngine, nil) { if on("dedicated-launch-check") { w.logf("witness: the launch check refuses %s's declaring adoption beside %s; on a dedicated box the scenario cannot arise", tcpB, tcpA) return } t.Fatalf("witness: adoption of the declaring build refused: %s", w.boxesState()) } w.check() for _, v := range b.Staged[tcpB] { if v == vEngine { t.Fatalf("witness: the declaring build is recorded under %s too; the scenario needs it staged for the other instance only", tcpB) } } if !w.repointAwake(b, tcpB, vPlain) { t.Fatalf("witness: the awake repoint was refused: %s", w.boxesState()) } w.check() w.upgrade(b) w.check() w.requestBy(b, tcpB) w.check() } const ( tcpApp = "acct/tcp" tcpA = "tcp-a" tcpB = "tcp-b" tcpC = "tcp-c" tcpD = "tcp-d" mintedB = "tcp-b.acct.apps.example" mintedC = "tcp-c.acct.apps.example" ) // tcpPublic provisions a TCP-backed instance tcp-b of a plain build and mints // its platform row with the owner recorded; both answer publicly. func (w *world) tcpPublic(t *testing.T, b *box) { if !w.provision(b, tcpB, tcpApp, "10.0.0.7:9102", vPlain) { t.Fatalf("witness: provisioning %s refused: %s", tcpB, w.boxesState()) } w.mint(b, tcpB, mintedB, true) w.reload(b) w.check() if r := w.admitPublic(b, mintedB, "/"); r == nil { t.Fatalf("witness: the minted row of a public app refused a public request: %s", w.boxesState()) } else { w.deliver(r) } w.check() if len(w.violations) > 0 { t.Fatalf("witness: the fixture violates: %v", w.violations) } } // wMintedFollowOwner (R5-1): a public app with a minted row adopts a // declaring build; a public request arrives by the minted host. // // Since Task 4 a minted row that is its own group is a proxying row of // another group: the launch check refuses the adoption beside it, and the // edge's dedicated refusal refuses it once the app is internal. The witness // needs both gone with minted-follow-owner. func wMintedFollowOwner(t *testing.T, w *world) { b := w.engineFreeBox(t) w.tcpPublic(t, b) if !w.adopt(b, tcpB, vEngine, nil) { if on("minted-follow-owner") || !on("dedicated-launch-check") { t.Fatalf("witness: adoption of the declaring build refused: %s", w.boxesState()) } w.logf("witness: the minted row, its own group, is another group's row beside %s; the launch check refuses the adoption", tcpB) return } w.check() w.requestBy(b, mintedB) w.check() } // wMintedFollowOwnerGrant (R5-1, by the grant): a public app of a plain // build with a minted row; the operator grants doc-engine on the minted row; // a public request arrives by the app's own host. func wMintedFollowOwnerGrant(t *testing.T, w *world) { b := w.engineFreeBox(t) w.tcpPublic(t, b) w.grant(b, mintedB) w.check() w.requestBy(b, tcpB) w.check() } // wGrantMarksGroup (R6-1): a public app of a plain build with a minted row; // the operator grants doc-engine on the minted row; a public request arrives // by the minted host. func wGrantMarksGroup(t *testing.T, w *world) { b := w.engineFreeBox(t) w.tcpPublic(t, b) w.grant(b, mintedB) w.check() w.requestBy(b, mintedB) w.check() } // wStaticMintedPublic (R6-2): a static app with its minted row and a canary // on its own channel; the agent upgrades; at the end every public row must // answer. func wStaticMintedPublic(t *testing.T, w *world) { b := w.boxes[0] w.addStaticApp(b, "site") w.reload(b) w.check() w.upgrade(b) w.check() w.final() } // wOwnerNonEmptyUnique (R7-2): app acct/tcp's TCP instance tcp-b runs a // declaring build, internal; its platform row was minted before owners were // recorded, so the edge finds its owner by backend. tcp-a, a later plain // instance of the app, is cleared with --public. A public request arrives by // tcp-b's minted host. // // Task 3's version ended in INV-AV: tcp-b's public minted row, misgrouped // into a closed group, never answered. Since Task 4 a closed group homes its // box, so every row outside it, the misgrouped row included (its true group // is tcp-b), is refused by design and INV-AV does not count it. The safety // effect remains, behind the edge's dedicated refusal. func wOwnerNonEmptyUnique(t *testing.T, w *world) { b := w.engineFreeBox(t) if !w.provision(b, tcpB, tcpApp, "10.0.0.7:9102", vEngine) { t.Fatalf("witness: provisioning %s refused: %s", tcpB, w.boxesState()) } w.mint(b, tcpB, mintedB, false) w.reload(b) w.check() if !w.provision(b, tcpA, tcpApp, "10.0.0.7:9101", vPlain) { t.Fatalf("witness: provisioning %s refused: %s", tcpA, w.boxesState()) } w.check() if !w.clearPublic(b, tcpA) { t.Fatalf("witness: --public refused on %s's plain live release: %s", tcpA, w.boxesState()) } w.requestBy(b, mintedB) w.check() } // wPublicChecksLive: the engine's live release declares; the operator runs // --public; a public request arrives and socket activation starts the unit. func wPublicChecksLive(t *testing.T, w *world) { b := w.boxes[0] if !w.adopt(b, eng, vEngine, nil) { t.Fatalf("witness: adoption of the engine refused: %s", w.boxesState()) } w.check() w.clearPublic(b, eng) r := w.admitPublic(b, engHost, "/") w.socketActivate(b, eng) if r != nil { w.deliver(r) } w.check() } // wPublicStopsGroup (R7-1, R8-2): the engine runs a declaring generation; a // generation rollout of a plain build repoints the live link and only // schedules the swap; the operator runs --public; a public request arrives. func wPublicStopsGroup(t *testing.T, w *world) { b := w.boxes[0] if !w.adopt(b, eng, vEngine, nil) { t.Fatalf("witness: adoption of the engine refused: %s", w.boxesState()) } w.check() if !w.repointAwake(b, eng, vPlain) { t.Fatalf("witness: the rollout's repoint was refused: %s", w.boxesState()) } w.check() w.clearPublic(b, eng) w.requestBy(b, engHost) w.check() } // wPublicConfirmsEmpty: the engine runs a declaring generation; a rollout of // a plain build repoints the live link and only schedules the swap; the // operator runs --public and its stop leaves that generation running; a // public request arrives. func wPublicConfirmsEmpty(t *testing.T, w *world) { b := w.boxes[0] if !w.adopt(b, eng, vEngine, nil) { t.Fatalf("witness: adoption of the engine refused: %s", w.boxesState()) } w.check() if !w.repointAwake(b, eng, vPlain) { t.Fatalf("witness: the rollout's repoint was refused: %s", w.boxesState()) } w.check() w.clearStopMisses = true w.clearPublic(b, eng) w.clearStopMisses = false w.requestBy(b, engHost) w.check() } // TestGrantRefusedOnUnresolvedMintedRow: --grant doc-engine on a minted row // whose owner is not recorded and cannot be found uniquely (here two // instances of its app share its address) is refused; the row stays NULL at // the edge and refused, and no group is marked. func TestGrantRefusedOnUnresolvedMintedRow(t *testing.T) { w := newWitness() b := w.engineFreeBox(t) for _, h := range []string{tcpC, tcpD} { if !w.provision(b, h, tcpApp, "10.0.0.7:9103", vPlain) { t.Fatalf("provisioning %s refused: %s", h, w.boxesState()) } } w.mint(b, tcpC, mintedC, false) // and one whose recorded owner has no row b.Rows["gone"] = &row{Host: "gone", Kind: "instance", App: tcpApp, Backend: "10.0.0.7:9104", PathTo: map[string]string{}} w.mint(b, "gone", "gone.acct.apps.example", true) delete(b.Rows, "gone") w.reload(b) wrote := b.Wrote for _, h := range []string{mintedC, "gone.acct.apps.example"} { if w.grant(b, h) { t.Errorf("--grant on %s, whose owner cannot be resolved, succeeded", h) } if b.Rows[h].Grant { t.Errorf("--grant on %s recorded the grant", h) } if b.Serving[h] != markNull { t.Errorf("the edge serves %s %v, want NULL", h, b.Serving[h]) } if r := w.admitPublic(b, h, "/"); r != nil { t.Errorf("a public request to %s was admitted", h) } } if b.Wrote != wrote { t.Errorf("a refused grant changed the registry (%d to %d)", wrote, b.Wrote) } for _, g := range []string{tcpC, tcpD} { if m := w.groupMark(b, g); m != markPublic { t.Errorf("group %s is %v after a refused grant, want public", g, m) } } if len(w.violations) > 0 { t.Errorf("violations: %v", w.violations) } if t.Failed() { dump(t, w) } } // TestRevokeRunsTheBarrier: --grant on a minted row times out while the // edge's reload fails (1 written, not fenced); a public request is admitted // and held; --revoke through the minted row also times out and the mark stays // 1; adoption of a declaring build is refused for want of the fence. Once // the reload recovers, the fence drains the held request, the declaring // build starts, and nothing public reaches it. With the reload working, // --revoke returns once the edge has loaded it, and the mark is still 1. func TestRevokeRunsTheBarrier(t *testing.T) { w := newWitness() b := w.engineFreeBox(t) w.tcpPublic(t, b) b.ReloadFails = true if w.grant(b, mintedB) { t.Fatal("--grant returned loaded while the edge's reload fails") } if m := w.groupMark(b, tcpB); m != markInternal { t.Fatalf("after the timed-out grant the group is %v, want internal", m) } held := w.admitPublic(b, tcpB, "/") if held == nil { t.Fatalf("the edge (still serving 0) refused a public request: %s", w.boxesState()) } held.Held = true if w.revoke(b, mintedB) { t.Fatal("--revoke returned loaded while the edge's reload fails") } if b.Rows[mintedB].Grant { t.Fatal("--revoke left the grant recorded") } if m := w.groupMark(b, tcpB); m != markInternal { t.Fatalf("--revoke changed the mark to %v; only --public writes 0", m) } if w.adopt(b, tcpB, vEngine, nil) { t.Fatalf("adoption of a declaring build passed the gate without a fence: %s", w.boxesState()) } w.check() b.ReloadFails = false if !w.adopt(b, tcpB, vEngine, nil) { t.Fatalf("adoption refused after the reload recovered: %s", w.boxesState()) } w.check() w.deliver(held) if held.Reached != 0 { t.Fatalf("the request admitted before the fence reached process %d", held.Reached) } for _, h := range []string{tcpB, mintedB} { if r := w.admitPublic(b, h, "/"); r != nil { t.Fatalf("a public request to %s was admitted after the fence", h) } } // with the reload working, revoke waits for the edge and returns loaded w.grant(b, tcpB) if !w.revoke(b, tcpB) || b.Fenced < b.Wrote { t.Fatalf("--revoke with a working reload did not return loaded: %s", w.boxesState()) } if m := w.groupMark(b, tcpB); m != markInternal || !w.fenced(b, tcpB) { t.Fatalf("after --revoke the group is %v (fenced %v), want internal and fenced", m, w.fenced(b, tcpB)) } if len(w.violations) > 0 { t.Fatalf("violations: %v", w.violations) } if t.Failed() { dump(t, w) } } // wPublicHoldsLock: the engine was adopted to a plain build (the mark stays // 1); the operator runs --public; after its checks pass and before it stops // the group, adoption of a declaring build runs; then a public request // arrives and socket activation starts the unit. func wPublicHoldsLock(t *testing.T, w *world) { b := w.boxes[0] if !w.adopt(b, eng, vEngine, nil) || !w.adopt(b, eng, vPlain, nil) { t.Fatalf("witness: adoption refused: %s", w.boxesState()) } w.check() w.clearHook = func(hb *box, g, stage string) { if stage == "locked" { w.adopt(hb, g, vEngine, nil) } } w.clearPublic(b, eng) w.clearHook = nil r := w.admitPublic(b, engHost, "/") w.socketActivate(b, eng) if r != nil { w.deliver(r) } w.check() } // TestI1Oracle pins INV-I1 on hand-built states of box 0's engine group with // a declaring process running: the fence condition alone catches a group that // is NULL, unloaded by the edge, or written and swapped but unacknowledged; // the served-map condition alone catches a counter advanced without a swap. // A NULL group counts as closed once the edge has fenced the change that made // it NULL: the edge refuses NULL like 1 (spec §5.1 "The column"), and the // upgrade leaves a group NULL under a running declaring build whose live // manifest cannot be read (Task 3). A new row's NULL, which no change closed // and no reload has drained, never counts (final review, finding 1). func TestI1Oracle(t *testing.T) { cases := []struct { name string setup func(w *world, b *box) want string // "" holds; else a substring of the reason }{ {"fenced as internal", func(w *world, b *box) {}, ""}, {"NULL written over 0, not yet fenced", func(w *world, b *box) { oraclePublic(w, b) w.writeMark(b, eng, markNull) }, "not fenced"}, {"NULL written over 0 and fenced (the edge treats NULL like 1)", func(w *world, b *box) { oraclePublic(w, b) w.writeMark(b, eng, markNull) w.reload(b) }, ""}, {"a new row, NULL, with no fence generation", func(w *world, b *box) { for _, h := range []string{eng, engHost} { delete(b.Rows, h) delete(b.Mark, h) } delete(b.SetAt, eng) b.Rows[eng] = &row{Host: eng, Kind: "instance", App: "acct/eng", Backend: "/run/carlos/eng.sock", PathTo: map[string]string{}} b.Wrote++ w.reload(b) }, "not fenced"}, {"NULL on one row, 0 served on the other", func(w *world, b *box) { oraclePublic(w, b) b.Mark[engHost] = markNull b.Serving[engHost] = markNull }, "not fenced"}, {"written 1, absent from the edge's map", func(w *world, b *box) { oraclePublic(w, b) w.writeMark(b, eng, markInternal) delete(b.EdgeRows, engHost) delete(b.Serving, engHost) }, "not fenced"}, {"written 1 and swapped in, fence not acknowledged", func(w *world, b *box) { oraclePublic(w, b) w.writeMark(b, eng, markInternal) b.Serving[engHost] = markInternal }, "not fenced"}, {"counter advanced without a swap", func(w *world, b *box) { oraclePublic(w, b) w.writeMark(b, eng, markInternal) b.Fenced = b.Wrote }, "serves " + eng + " public"}, } for _, c := range cases { w := newWitness() b := w.boxes[0] c.setup(w, b) p := &proc{ID: 99, Group: eng, Ver: vEngine, Slice: "inst:" + eng} b.Procs = append(b.Procs, p) why := w.i1Broken(b, p) w.check() switch { case c.want == "" && (why != "" || len(w.violations) > 0): t.Errorf("%s: want held, got %q %v", c.name, why, w.violations) case c.want != "" && !strings.Contains(why, c.want): t.Errorf("%s: reason %q, want it to name %q", c.name, why, c.want) case c.want != "" && (len(w.violations) != 1 || invariantOf(w.violations[0]) != "INV-I1"): t.Errorf("%s: check recorded %v, want one INV-I1", c.name, w.violations) } // a plain build in the same state breaks nothing p.Ver = vPlain if why := w.i1Broken(b, p); why != "" { t.Errorf("%s: a non-declaring process breaks I1: %q", c.name, why) } } } // ---------- Task 4: the dedicated engine box and ingress ---------- const ( svcHost = "svc.example" // a service row of the witnesses tcpE = "tcp-e" // a TCP-backed engine on box 0 tcpEAddr = "10.0.0.7:9200" ) // engineFreeBox is box 0 with the engine's rows removed, the edge reloaded: // a box that homes no internal group, for scenarios that need instances of // their own (the dedicated-box rule refuses every other proxying row on a box // that homes the engine, and refuses a declaring launch beside one). func (w *world) engineFreeBox(t *testing.T) *box { b := w.boxes[0] for _, h := range []string{eng, engHost, engAlias, engSite} { delete(b.Rows, h) delete(b.Mark, h) } delete(b.SetAt, eng) b.Wrote++ w.logf("witness: box0's engine rows removed (registry change %d)", b.Wrote) w.reload(b) w.check() if len(w.violations) > 0 { t.Fatalf("witness: the fixture violates: %v", w.violations) } return b } // runningEngine starts the engine group on box 0 on the declaring build, // fenced as internal (its starting mark), and returns its process. func (w *world) runningEngine(t *testing.T) (*box, *proc) { b := w.boxes[0] b.Live[eng], b.Good[eng] = vEngine, vEngine if !w.launch(b, eng, vEngine, "seed") { t.Fatalf("witness: the engine did not start under its fenced mark: %s", w.boxesState()) } w.check() if len(w.violations) > 0 { t.Fatalf("witness: the fixture violates: %v", w.violations) } return b, b.Procs[len(b.Procs)-1] } // portsEngine is the public engine route shipping a plain release declaring // ports: the box renders its listener, and a public connection to the port // reaches the plain build. It returns the port. func (w *world) portsEngine(t *testing.T) (*box, string) { b := w.publicEngine(t) if !w.ship(b, eng, vPlain, true) { t.Fatalf("witness: a plain release declaring ports was refused: %s", w.boxesState()) } port := b.PortGrant[eng] if port == "" || b.Listen[port] != eng { t.Fatalf("witness: no listener rendered for the granted port: %s", w.boxesState()) } if p := w.directConnect(b, port, true); p == nil { t.Fatalf("witness: the granted port of a public route reached nothing: %s", w.boxesState()) } w.check() return b, port } // listenerGone fails the witness if the write that closed the group left // port's listener in place, while ports-excluded holds: the write removes it // itself, before any fence and with no capability pass (final review, minor // 2). func (w *world) listenerGone(t *testing.T, b *box, port, after string) { if on("ports-excluded") && b.Listen[port] != "" { t.Fatalf("witness: the listener on %s is still there right after %s: %s", port, after, w.boxesState()) } } // wPortsExcluded (R11-1): the public engine route ships a plain release // declaring ports, and the box renders its listener; then adoption of a // declaring build; at once, with no capability pass, a public connection // arrives on the granted port. func wPortsExcluded(t *testing.T, w *world) { b, port := w.portsEngine(t) if !w.adopt(b, eng, vEngine, nil) { t.Fatalf("witness: adoption of the engine refused: %s", w.boxesState()) } w.listenerGone(t, b, port, "the adoption") w.check() w.directConnect(b, port, true) w.check() } // wPortsExcludedGrant (R11-1, by the grant): the public engine route serves a // plain build on its ports listener; the operator grants doc-engine on it; at // once, with no capability pass, a public connection arrives on the port. func wPortsExcludedGrant(t *testing.T, w *world) { b, port := w.portsEngine(t) w.grant(b, engHost) w.listenerGone(t, b, port, "the grant") w.check() w.directConnect(b, port, true) w.check() } // TestPortsListenerGoneOnNull (final review, minor 2): the upgrade leaves a // route with no live release NULL; the write of NULL removes its ports // listener at once, before the capability pass runs, and a public connection // to the port is refused by the firewall. func TestPortsListenerGoneOnNull(t *testing.T) { w := newWitness() b := w.boxes[1] b.PortGrant["shop"] = "40001" w.capsPass(b) if b.Listen["40001"] != "shop" { t.Fatalf("fixture: no listener for shop's grant: %s", w.boxesState()) } w.upgrade(b) if m := w.groupMark(b, "shop"); m != markNull { t.Fatalf("fixture: shop is %v after the upgrade, want NULL", m) } if b.Listen["40001"] != "" { t.Errorf("the listener is still there right after the upgrade's NULL: %s", w.boxesState()) } if p := w.directConnect(b, "40001", true); p != nil { t.Errorf("a public connection to the NULL route's port reached process %d", p.ID) } w.check() if len(w.violations) > 0 { t.Errorf("violations: %v", w.violations) } if t.Failed() { dump(t, w) } } // wDedicatedLaunchCheck (R12-2): the public engine route has a service row // beside it aimed at its backend; adoption of a declaring build; a public // request arrives through the service row. func wDedicatedLaunchCheck(t *testing.T, w *world) { b := w.publicEngine(t) w.addServiceRow(b, svcHost, b.Rows[eng].Backend) w.reload(b) w.check() ok := w.adopt(b, eng, vEngine, nil) if ok && on("dedicated-launch-check") { t.Fatalf("witness: a declaring adoption passed the launch check beside a service row: %s", w.boxesState()) } w.check() w.requestBy(b, svcHost) w.check() } // wDedicatedEdgeRefusal (R12-2): the engine runs internal; a service row // aimed at its backend is added afterwards; the edge polls; a public request // arrives through the service row. func wDedicatedEdgeRefusal(t *testing.T, w *world) { b, _ := w.runningEngine(t) w.addServiceRow(b, svcHost, b.Rows[eng].Backend) w.reload(b) w.check() w.requestBy(b, svcHost) w.check() } // wIngressPrivate (R13-1): a TCP-backed engine runs internal on box 0; a // service row on box 1 is aimed at box 0's address and the engine's port; a // public request arrives at box 1's edge through it. func wIngressPrivate(t *testing.T, w *world) { b0, b1 := w.engineFreeBox(t), w.boxes[1] if !w.provision(b0, tcpE, tcpApp, tcpEAddr, vEngine) { t.Fatalf("witness: provisioning the TCP engine refused: %s", w.boxesState()) } w.check() w.addServiceRow(b1, svcHost, tcpEAddr) w.reload(b1) w.check() r := w.admitPublic(b1, svcHost, "/") if r == nil { t.Fatalf("witness: box1's edge refused its own service row: %s", w.boxesState()) } w.deliver(r) w.check() } // wOwnPort (R12-1): the engine runs internal and binds a port of its own on // every interface; a public connection arrives on that port. func wOwnPort(t *testing.T, w *world) { b, p := w.runningEngine(t) w.bindOwnPort(b, p) w.directConnect(b, ownPort(p), true) w.check() } // TestNullRouteHomesTheBox: a NULL route homes its box like a route marked // 1, whether or not it has a release. Here box 1's shop has no release, so // the upgrade leaves it NULL; a service row aimed at its backend must get the // internal refusal. Were it served, a request admitted through it and held, // with the row then deleted, would outlive a declaring `carlos run` in the // NULL group: the gate writes 1 over NULL, a group already fenced, so no // reload bumps the epoch, and the launch check finds no row to refuse. func TestNullRouteHomesTheBox(t *testing.T) { w := newWitness() b := w.boxes[1] w.upgrade(b) if m := w.groupMark(b, "shop"); m != markNull || b.Live["shop"] != 0 { t.Fatalf("fixture: shop is %v with live v%d, want NULL with no release", m, b.Live["shop"]) } w.addServiceRow(b, svcHost, b.Rows["shop"].Backend) w.reload(b) r := w.admitPublic(b, svcHost, "/") if r != nil { r.Held = true w.deleteRow(b, svcHost) w.launch(b, "shop", vEngine, "cmd") w.deliver(r) } w.check() if r != nil { t.Errorf("a service row beside a NULL route was served (request %d)", r.ID) } if len(w.violations) > 0 { t.Errorf("violations: %v", w.violations) } if t.Failed() { dump(t, w) } } // TestShipRefusesDocEngineWithPorts (spec §8): `carlos ship` refuses a // release declaring doc_engine and ports together, and nothing is adopted or // granted; a plain release declaring ports ships, and its port listens. func TestShipRefusesDocEngineWithPorts(t *testing.T) { w := newWitness() b := w.publicEngine(t) wrote := b.Wrote if w.ship(b, eng, vEngine, true) { t.Fatal("a release declaring doc_engine and ports shipped") } if b.Live[eng] != vPlain || b.PortGrant[eng] != "" || len(b.Listen) != 0 || b.Wrote != wrote { t.Fatalf("a refused ship changed the box: %s", w.boxesState()) } if !w.ship(b, eng, vPlain, true) || b.Listen[b.PortGrant[eng]] != eng { t.Fatalf("a plain release declaring ports did not ship with its listener: %s", w.boxesState()) } if len(w.violations) > 0 { t.Fatalf("violations: %v", w.violations) } } // TestEngineBoxKeepsStaticRows (spec §8): on a box homing an internal group, // a service row aimed at the engine and another app's instance get the // internal refusal, a declaring launch beside them is refused, and the // static row keeps answering. func TestEngineBoxKeepsStaticRows(t *testing.T) { w := newWitness() b, _ := w.runningEngine(t) w.addServiceRow(b, svcHost, b.Rows[eng].Backend) b.Rows["other"] = &row{Host: "other", Kind: "instance", App: "acct/other", Backend: "10.0.0.7:9300", PathTo: map[string]string{}} b.Mark["other"] = markPublic w.reload(b) for _, h := range []string{svcHost, "other"} { if _, outcome := w.admit(b, h, "/"); outcome != refused { t.Errorf("%s on the engine's box: %s, want the internal refusal", h, outcome) } } if _, outcome := w.admit(b, engSite, "/"); outcome != answered { t.Errorf("the static row %s on the engine's box: %s, want answered", engSite, outcome) } if w.launch(b, eng, vEngine, "seed") { t.Errorf("a declaring launch beside other proxying rows was not refused") } w.check() if len(w.violations) > 0 { t.Errorf("violations: %v", w.violations) } if t.Failed() { dump(t, w) } } // wNewGroupFenced (final review, finding 1): on a box that homes nothing, a // service row aimed at address X admits a public request, which is held; the // row is deleted; a declaring instance is provisioned at X. Its row is new // and NULL, but the edge has never loaded it: its first launch must write 1 // and wait for a fence, whose epoch bump drains the held request. func wNewGroupFenced(t *testing.T, w *world) { b := w.engineFreeBox(t) w.addServiceRow(b, svcHost, tcpEAddr) w.reload(b) w.check() r := w.admitPublic(b, svcHost, "/") if r == nil { t.Fatalf("witness: the service row refused a public request: %s", w.boxesState()) } r.Held = true w.deleteRow(b, svcHost) if !w.provision(b, tcpE, tcpApp, tcpEAddr, vEngine) { t.Fatalf("witness: provisioning the declaring instance refused: %s", w.boxesState()) } w.check() w.deliver(r) w.check() } // wClearLaunchLock (final review, finding 2): the engine was adopted to a // plain build (the mark stays 1); the operator runs --public; after its empty // check, a carlos run command of the declaring build tries to start; then a // public request arrives. func wClearLaunchLock(t *testing.T, w *world) { b := w.boxes[0] if !w.adopt(b, eng, vEngine, nil) || !w.adopt(b, eng, vPlain, nil) { t.Fatalf("witness: adoption refused: %s", w.boxesState()) } w.check() w.clearHook = func(hb *box, g, stage string) { if stage == "confirmed" { w.launch(hb, g, vEngine, "cmd") } } w.clearPublic(b, eng) w.clearHook = nil w.requestBy(b, engHost) w.check() } // wGrantFreshnessBound (final review, finding 3; ruling R5): thirty ticks in, // the console re-signs (revision 2, granting shop) and both edges read it; // the console revokes shop (revision 3) and a bucket writer puts revision 2 // back before any edge reads 3. Each edge reads revision 2 again (equal to // its floor: the same signed bytes). Shop calls at once (within the bound), // then again ten minutes and one poll after the revoke. func wGrantFreshnessBound(t *testing.T, w *world) { w.runningEngine(t) shop := w.armedShop(t) w.advance(30) w.publish(w.console.Rev + 1) // the periodic re-sign: revision 2, signed at 30 w.pollAll() w.consoleRevoke(shopEng) // revision 3, at 30 w.replay(2) w.pollAll() w.check() w.engineCall(w.boxes[1], shop, false, "carlos", "shop", eng, "/") w.check() w.advance(staleTicks + pollTicks + 1) w.pollAll() w.engineCall(w.boxes[1], shop, false, "carlos", "shop", eng, "/") w.check() } // TestReplayBoundedBySignedAt (ruling R5): wGrantFreshnessBound's replay, with // every rule in place, keeps the revoked grant on both edges until the // replayed record's signed_at plus ten minutes, and not one tick longer. func TestReplayBoundedBySignedAt(t *testing.T) { w := newWitness() w.runningEngine(t) shop := w.armedShop(t) w.advance(30) w.publish(w.console.Rev + 1) signed := w.now() w.pollAll() w.consoleRevoke(shopEng) w.replay(2) w.advance(staleTicks - 1) // 59 ticks after revision 2 was signed w.pollAll() if !w.engineCall(w.boxes[1], shop, false, "carlos", "shop", eng, "/") { t.Errorf("at signed_at+%d the replayed record should still grant (the bound is not tighter than R5): %s", w.now()-signed, w.mirrorsState()) } w.advance(1) w.pollAll() if w.engineCall(w.boxes[1], shop, false, "carlos", "shop", eng, "/") { t.Errorf("at signed_at+%d the replayed record still grants: %s", w.now()-signed, w.mirrorsState()) } w.check() if len(w.violations) > 0 || len(w.residuals) > 0 { t.Errorf("violations %v residuals %v", w.violations, w.residuals) } if t.Failed() { dump(t, w) } } // ---------- Task 5: calls, identity, grants and arming ---------- // shopEng is the grant every Task 5 world starts with: shop may call eng. const shopEng = "acct/shop/eng" // armedShop starts shop on box 1 on the plain build and arms box 1, so // shop's engine socket is open. It returns shop's process. func (w *world) armedShop(t *testing.T) *proc { b := w.boxes[1] b.Live["shop"], b.Good["shop"] = vPlain, vPlain if !w.launch(b, "shop", vPlain, "seed") { t.Fatalf("witness: shop did not start: %s", w.boxesState()) } if !w.arm(b) { t.Fatalf("witness: arming box 1 with only nested shop refused: %s", w.boxesState()) } w.check() if len(w.violations) > 0 { t.Fatalf("witness: the fixture violates: %v", w.violations) } return w.newest(b, "shop") } // pollAll is one mirror poll on every box. func (w *world) pollAll() { for _, b := range w.boxes { w.mirrorTick(b) } } // wCgroupIdentity (R1-1): box 1 runs shop and a sibling instance, other, // and is armed; other's process connects to shop's engine socket. func wCgroupIdentity(t *testing.T, w *world) { w.runningEngine(t) b := w.boxes[1] if !w.provision(b, "other", "acct/other", "127.0.0.1:9002", vPlain) { t.Fatalf("witness: provisioning other refused: %s", w.boxesState()) } w.armedShop(t) op := w.newest(b, "other") if op == nil { t.Fatalf("witness: other is not running: %s", w.boxesState()) } w.engineCall(b, op, false, "carlos", "shop", eng, "/") w.check() } // wDoorUID (R2-3): a flat tenant on unarmed box 1, running as carlos, // presents shop's door key with Host: eng. func wDoorUID(t *testing.T, w *world) { w.runningEngine(t) b := w.boxes[1] if !w.launchUnisolated(b, "flat") { t.Fatalf("witness: a flat start on an unarmed box refused: %s", w.boxesState()) } w.check() w.engineCall(b, w.newest(b, "flat"), true, "carlos", "shop", eng, "/") w.check() } // wHopCallerSigned: a public client sends box 0 a request for the engine's // hostname carrying X-Carlos-Hop-Caller: acct/shop/eng and no hop MAC. func wHopCallerSigned(t *testing.T, w *world) { b, _ := w.runningEngine(t) w.clientHop(b, engHost, "/", callerOf("acct/shop", eng)) w.check() } // wEngineBoxRecheck (G4): shop's grant is revoked and read everywhere; box // 1's data directory is lost (a rebuilt box: no floor), and a bucket writer // at once replays revision 1, signed moments ago; box 1 accepts it, box 0's // floor refuses it; shop calls eng. func wEngineBoxRecheck(t *testing.T, w *world) { w.runningEngine(t) shop := w.armedShop(t) b1 := w.boxes[1] w.consoleRevoke(shopEng) w.pollAll() w.rebuild(b1) w.replay(1) w.pollAll() w.check() if !w.tableGrants(b1, shopEng) { t.Fatalf("witness: the rebuilt calling box does not hold the replayed grant: %s", w.mirrorsState()) } w.engineCall(b1, shop, false, "carlos", "shop", eng, "/") w.check() } // wExactRouteEngine (R4-2): shop holds engine site's grant and not eng's; // site's hostname is box 0's static row, whose path map sends /convert to // eng's route; shop calls site at /convert. func wExactRouteEngine(t *testing.T, w *world) { w.runningEngine(t) shop := w.armedShop(t) w.consoleGrant("acct/shop/" + site) w.consoleRevoke(shopEng) w.pollAll() w.check() w.engineCall(w.boxes[1], shop, false, "carlos", "shop", site, engSitePath) w.check() } // wFloorPersisted (R1-4): shop's grant is revoked and read everywhere; a // bucket writer at once replays revision 1 (signed moments ago), which both // floors refuse; both edges restart; each polls; shop calls eng. func wFloorPersisted(t *testing.T, w *world) { w.runningEngine(t) shop := w.armedShop(t) w.consoleRevoke(shopEng) w.pollAll() w.replay(1) w.pollAll() for _, b := range w.boxes { w.restartEdge(b) } w.pollAll() w.check() w.engineCall(w.boxes[1], shop, false, "carlos", "shop", eng, "/") w.check() } // wStaleClearsGrants (§6.4): the bucket is unreachable; shop's grant is // revoked; ten minutes pass with no good read; shop calls eng. func wStaleClearsGrants(t *testing.T, w *world) { w.runningEngine(t) shop := w.armedShop(t) w.setBucketDown(true) w.consoleRevoke(shopEng) w.advance(staleTicks) w.pollAll() w.check() w.engineCall(w.boxes[1], shop, false, "carlos", "shop", eng, "/") w.check() } // armedRefuses: box 1 is armed with shop's socket open; each kind of // unisolated launch is tried in turn. func armedRefuses(kinds ...string) func(*testing.T, *world) { return func(t *testing.T, w *world) { w.armedShop(t) b := w.boxes[1] for _, k := range kinds { ok := w.launchUnisolated(b, k) if ok && on("armed-refuses-unisolated") { t.Fatalf("witness: an armed box admitted a %s launch: %s", k, w.boxesState()) } w.check() } } } func wArmedRefusesUnisolated(t *testing.T, w *world) { armedRefuses("foreign", "worker")(t, w) } func wArmedRefusesWorker(t *testing.T, w *world) { armedRefuses("worker")(t, w) } func wArmedRefusesFlat(t *testing.T, w *world) { armedRefuses("flat")(t, w) } // wArmChecksBox (R5-2): an app's worker runs in carlos-exec@ on unarmed box // 1 beside nested shop; the operator arms box 1. func wArmChecksBox(t *testing.T, w *world) { b := w.boxes[1] b.Live["shop"], b.Good["shop"] = vPlain, vPlain w.launch(b, "shop", vPlain, "seed") if !w.launchUnisolated(b, "worker") { t.Fatalf("witness: a worker spawn on an unarmed box refused: %s", w.boxesState()) } w.check() ok := w.arm(b) if ok && on("arm-checks-box") { t.Fatalf("witness: arming a box with a running worker succeeded: %s", w.boxesState()) } w.check() } // TestResidualsAreCountedNotFailed: each of the spec's two stated residuals, // as a fixed scenario. With its flag off the scenario is a // violation (the sequence really breaks the invariant the residual // concedes); with the flag on the world records residual: and no // violation. The default world never sets a flag (TestEngines asserts the // sweep records no residual). func TestResidualsAreCountedNotFailed(t *testing.T) { cases := []struct { name string set func(w *world) run func(t *testing.T, w *world) }{ {"unwrittenManifestAcrossUpgrade", func(w *world) { w.unwrittenManifestAcrossUpgrade = true }, residualUnwrittenManifest}, {"flatEraTamper", func(w *world) { w.flatEraTamper = true }, residualFlatEraTamper}, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { off := newWitness() c.run(t, off) if len(off.violations) == 0 || len(off.residuals) > 0 { t.Errorf("flag off: want a violation and no residual, got violations %v residuals %v", off.violations, off.residuals) dump(t, off) } on := newWitness() c.set(on) c.run(t, on) if len(on.violations) > 0 || len(on.residuals) == 0 { t.Errorf("flag on: want residuals and no violation, got violations %v residuals %v", on.violations, on.residuals) dump(t, on) } for _, r := range on.residuals { if !strings.HasPrefix(r, "residual:"+c.name) { t.Errorf("flag on: residual %q is not residual:%s", r, c.name) } } }) } } // residualUnwrittenManifest (§5.1 "Classification before serving"): the // engine runs a declaring build whose staged manifest was never written; // an awake repoint moves its live link to a plain build without a restart; // the box takes the upgrade, which classifies the group 0 under the // running engine; a public request arrives. func residualUnwrittenManifest(t *testing.T, w *world) { b := w.boxes[0] b.Live[eng], b.Good[eng] = vUnreadable, vUnreadable if !w.launch(b, eng, vUnreadable, "seed") { t.Fatalf("scenario: the engine did not start: %s", w.boxesState()) } if !w.repointAwake(b, eng, vPlain) { t.Fatalf("scenario: the awake repoint refused: %s", w.boxesState()) } w.check() if w.residualAtUpgrade(b) == "" { t.Fatalf("scenario: the upgrade residual is not set up: %s", w.boxesState()) } w.upgrade(b) w.check() w.requestBy(b, engHost) w.check() } // residualFlatEraTamper (§6.2): a flat tenant on unarmed box 1 plants code // in shop's instance directory; the operator moves the flat route off and // arms the box (arming vouches for the present, not the past); shop's next // start runs the planted code in shop's slice, which calls eng. func residualFlatEraTamper(t *testing.T, w *world) { w.runningEngine(t) b := w.boxes[1] b.Live["shop"], b.Good["shop"] = vPlain, vPlain if !w.launchUnisolated(b, "flat") { t.Fatalf("scenario: a flat start on an unarmed box refused: %s", w.boxesState()) } w.tamper(b, w.newest(b, "flat"), "shop") w.moveToNested(b, "flat") if !w.arm(b) { t.Fatalf("scenario: arming refused after the flat route moved: %s", w.boxesState()) } if !w.launch(b, "shop", vPlain, "seed") { t.Fatalf("scenario: shop did not start: %s", w.boxesState()) } w.check() w.engineCall(b, w.newest(b, "shop"), false, "carlos", "shop", eng, "/") w.check() } // TestGrantedCallsReach is the positive control for the Task 5 witnesses, // which hold by refusing: with every rule on, a granted caller reaches the // engine through the socket (across a hop) and through the door from an // enrolled host-app uid (spec §8), and a revoke is in force once read. func TestGrantedCallsReach(t *testing.T) { w := newWitness() w.runningEngine(t) shop := w.armedShop(t) b1 := w.boxes[1] if !w.engineCall(b1, shop, false, "carlos", "shop", eng, "/") { t.Errorf("shop's own process through shop's socket did not reach eng") } user := &proc{Group: "shop", Slice: "user:u-shop"} if !w.engineCall(b1, user, true, "u-shop", "shop", eng, "/") { t.Errorf("the door from shop's enrolled uid with shop's key did not reach eng") } if w.engineCall(b1, user, true, "carlos", "shop", eng, "/") { t.Errorf("the same key from uid carlos reached eng") } w.consoleRevoke(shopEng) w.pollAll() if w.engineCall(b1, shop, false, "carlos", "shop", eng, "/") { t.Errorf("a revoke read by both boxes did not stop the call") } w.check() if len(w.violations) > 0 { t.Errorf("violations: %v", w.violations) } if t.Failed() { dump(t, w) } } // TestArmRefusesAndAdmits (spec §8): arming refuses a box with a flat // route, a foreign route, a route with workers, or a non-empty carlos-exec@ // cgroup; it succeeds on the engine's box (a unit-backed engine) and on a box // of nested callers; on an armed box every unisolated launch is refused // before a process starts; disarm closes the engine sockets, and then // launches resume. func TestArmRefusesAndAdmits(t *testing.T) { for _, kind := range unisolatedKinds { w := newWitness() b := w.boxes[1] w.launchUnisolated(b, kind) if w.arm(b) { t.Errorf("%s: arming succeeded with the %s tenant running", kind, kind) } w.stopGroup(b, kind) // the cgroup is empty; the route remains if w.arm(b) { t.Errorf("%s: arming succeeded with a %s route on the box", kind, kind) } w.moveToNested(b, kind) if !w.arm(b) { t.Errorf("%s: arming refused once the route moved to nested", kind) } } w := newWitness() w.runningEngine(t) if !w.arm(w.boxes[0]) { t.Errorf("arming the engine's box (unit-backed engine) refused") } w.armedShop(t) b := w.boxes[1] n := len(b.Procs) for _, kind := range unisolatedKinds { if w.launchUnisolated(b, kind) { t.Errorf("an armed box admitted a %s launch", kind) } } if len(b.Procs) != n { t.Errorf("a refused launch started a process: %d -> %d", n, len(b.Procs)) } w.disarm(b) if b.Sockets || b.Armed { t.Errorf("disarm left sockets %v armed %v", b.Sockets, b.Armed) } if !w.launchUnisolated(b, "worker") { t.Errorf("a worker spawn after disarm refused") } w.check() if len(w.violations) > 0 { t.Errorf("violations: %v", w.violations) } if t.Failed() { dump(t, w) } } // withMutation runs f with the mutation set m, restoring the previous set. func withMutation(m string, f func()) { saved := mutated mutated = m defer func() { mutated = saved }() f() } // TestExpiryKeepsRevokeAge: the grant oracle keeps expiry, staleness and // revoke age apart (final review, finding 4). With the table's own checks off // (stale-clears-grants and grant-freshness-bound), the edges keep serving a // cached grant, and the oracle alone decides: // - a key revoked long before the newest record expires is a violation by // its revoke's age, whatever the expiry; // - a key the newest record grants is a violation the moment that record // expires: an expired record admits nothing, with no grace; // - a key nobody revoked is a violation once the admitting box has had no // good read for ten minutes, whatever the revoke's age (there is none), // and not a tick earlier. func TestExpiryKeepsRevokeAge(t *testing.T) { withMutation("stale-clears-grants,grant-freshness-bound", func() { w := newWitness() w.runningEngine(t) shop := w.armedShop(t) w.setBucketDown(true) w.consoleRevoke(shopEng) // tick 0; revision 2, never read w.advance(lifetimeTicks) // revision 2 expires now w.engineCall(w.boxes[1], shop, false, "carlos", "shop", eng, "/") if len(w.violations) != 1 || invariantOf(w.violations[0]) != "INV-G" || !strings.Contains(w.violations[0], "since tick 0") { t.Errorf("a key revoked at tick 0, called across the record's expiry: want INV-G since tick 0, got %v", w.violations) dump(t, w) } w = newWitness() w.runningEngine(t) shop = w.armedShop(t) w.setBucketDown(true) w.advance(lifetimeTicks) // revision 1, which grants shop, expires now w.engineCall(w.boxes[1], shop, false, "carlos", "shop", eng, "/") if len(w.violations) != 1 || invariantOf(w.violations[0]) != "INV-G" || !strings.Contains(w.violations[0], "expired at 25920") { t.Errorf("a granted key at its record's expiry: want INV-G at once, naming the expiry, got %v", w.violations) dump(t, w) } w = newWitness() w.runningEngine(t) shop = w.armedShop(t) w.setBucketDown(true) w.advance(staleTicks - 1) // no good read for 59 ticks; shop still granted w.engineCall(w.boxes[1], shop, false, "carlos", "shop", eng, "/") if len(w.violations) != 0 { t.Errorf("a granted key 59 ticks after the last good read is within the bound: got %v", w.violations) dump(t, w) } w.advance(1) w.engineCall(w.boxes[1], shop, false, "carlos", "shop", eng, "/") if len(w.violations) != 1 || invariantOf(w.violations[0]) != "INV-G" || !strings.Contains(w.violations[0], "no good read") { t.Errorf("a granted key ten minutes after the last good read: want INV-G for the stale read, got %v", w.violations) dump(t, w) } }) } // upgradeResidualFixture is residualUnwrittenManifest up to the upgrade, // with the flag on: the residual is recorded and nothing violates. It // returns the engine's box and the tagged process. func upgradeResidualFixture(t *testing.T) (*world, *box, *proc) { w := newWitness() w.unwrittenManifestAcrossUpgrade = true b := w.boxes[0] b.Live[eng], b.Good[eng] = vUnreadable, vUnreadable w.launch(b, eng, vUnreadable, "seed") p := w.newest(b, eng) w.repointAwake(b, eng, vPlain) w.upgrade(b) w.check() if len(w.residuals) == 0 || len(w.violations) > 0 || p.Residual == "" { t.Fatalf("fixture: want the upgrade residual recorded on process %d, got residuals %v violations %v", p.ID, w.residuals, w.violations) } return w, b, p } // TestResidualCoversOnlyItsOwnBreaches: a process tagged with the // unwritten-manifest residual is not a licence. While its group is still // open, a breach by a path the upgrade's 0 did not open (a public connection // to a port the engine bound itself, through a broken firewall) is a // violation. Once the group is closed again the window has ended: a later 0 // written by a broken --public, under that same process, is a violation. func TestResidualCoversOnlyItsOwnBreaches(t *testing.T) { t.Run("another path", func(t *testing.T) { w, b, p := upgradeResidualFixture(t) withMutation("ingress-precondition", func() { w.bindOwnPort(b, p) w.directConnect(b, ownPort(p), true) }) w.check() if len(w.violations) != 1 || invariantOf(w.violations[0]) != "INV-P" { t.Errorf("want INV-P through the broken firewall, got violations %v residuals %v", w.violations, w.residuals) dump(t, w) } }) t.Run("after the window", func(t *testing.T) { w, b, p := upgradeResidualFixture(t) w.writeMark(b, eng, markInternal) // a gate's 1: the group is closed again w.reload(b) w.check() if p.Residual != "" || len(w.violations) > 0 { t.Fatalf("closing the group: want the window ended and no violation, got tag %q violations %v", p.Residual, w.violations) } withMutation("public-confirms-empty", func() { w.clearStopMisses = true w.operatorPublic(b, eng) // the stop misses process p; step 3 is gone w.reload(b) w.check() }) if len(w.violations) == 0 || invariantOf(w.violations[0]) != "INV-I1" { t.Errorf("a broken --public under the formerly tagged process: want INV-I1, got violations %v residuals %v", w.violations, w.residuals) dump(t, w) } }) } // flatEraFixture is residualFlatEraTamper's setup with the flag on, the // tamper planted in group target's instance directory instead of shop's: a // flat tenant on unarmed box 1 plants it, its route moves off, the box is // armed, and shop and other (a sibling instance) start. It returns box 1. func flatEraFixture(t *testing.T, target string) (*world, *box) { w := newWitness() w.flatEraTamper = true w.runningEngine(t) b := w.boxes[1] if !w.provision(b, "other", "acct/other", "127.0.0.1:9002", vPlain) { t.Fatalf("fixture: provisioning other refused: %s", w.boxesState()) } b.Live["shop"], b.Good["shop"] = vPlain, vPlain if !w.launchUnisolated(b, "flat") { t.Fatalf("fixture: a flat start on an unarmed box refused: %s", w.boxesState()) } w.tamper(b, w.newest(b, "flat"), target) w.moveToNested(b, "flat") if !w.arm(b) { t.Fatalf("fixture: arming refused after the flat route moved: %s", w.boxesState()) } for _, g := range []string{"shop", "other"} { w.stopGroup(b, g) if !w.launch(b, g, vPlain, "seed") { t.Fatalf("fixture: %s did not start: %s", g, w.boxesState()) } } w.check() if len(w.violations) > 0 { t.Fatalf("fixture: violations %v", w.violations) } return w, b } // TestFlatEraTamperKeepsUnrelatedViolations: the flat-era residual covers a // call whose recorded caller is the compromised instance itself, made by its // own process in its own slice (final review, finding 5). Combined with an // unrelated mutation, the unrelated breach stays a violation: // - cgroup-identity off: planted code in other borrows shop's engine // socket; the engine box records shop, which is not the compromised // instance; // - hop-caller-signed off: planted code in shop sends its own socket's call // with an X-Carlos-Hop-Caller naming another granted app, which the // engine box takes. // // And the positive control: planted code in shop calling through shop's own // socket is the residual, not a violation. func TestFlatEraTamperKeepsUnrelatedViolations(t *testing.T) { t.Run("own socket: the residual", func(t *testing.T) { w, b := flatEraFixture(t, "shop") w.engineCall(b, w.newest(b, "shop"), false, "carlos", "shop", eng, "/") w.check() if len(w.violations) > 0 || len(w.residuals) != 1 || !strings.HasPrefix(w.residuals[0], "residual:flatEraTamper:") { t.Errorf("want residual:flatEraTamper and no violation, got violations %v residuals %v", w.violations, w.residuals) dump(t, w) } }) t.Run("cgroup-identity off: a borrowed socket", func(t *testing.T) { w, b := flatEraFixture(t, "other") withMutation("cgroup-identity", func() { w.engineCall(b, w.newest(b, "other"), false, "carlos", "shop", eng, "/") }) w.check() if len(w.violations) != 1 || invariantOf(w.violations[0]) != "INV-ID" || len(w.residuals) > 0 { t.Errorf("want INV-ID and no residual, got violations %v residuals %v", w.violations, w.residuals) dump(t, w) } }) t.Run("hop-caller-signed off: a forged hop caller", func(t *testing.T) { w, b := flatEraFixture(t, "shop") w.consoleGrant("acct/tcp/" + eng) w.pollAll() forged := callerOf("acct/tcp", eng) withMutation("hop-caller-signed", func() { w.placeCall(call{from: b, p: w.newest(b, "shop"), uid: "carlos", host: "shop", engine: eng, path: "/", forged: &forged}) }) w.check() if len(w.violations) != 1 || invariantOf(w.violations[0]) != "INV-ID" || len(w.residuals) > 0 { t.Errorf("want INV-ID and no residual, got violations %v residuals %v", w.violations, w.residuals) dump(t, w) } }) } // TestWitnessesHoldUnmutated: every witness holds with every rule in place. // A witness that violates unmutated is a model bug, and a witness that // passes under its mutation for the wrong reason starts here (Review Focus 1). func TestWitnessesHoldUnmutated(t *testing.T) { saved := mutated mutated = "" defer func() { mutated = saved }() if len(witnesses) == 0 { t.Fatal("no witnesses") } for _, x := range witnesses { w := newWitness() x.run(t, w) if len(w.violations) > 0 { t.Errorf("witness %s (needs %q) violates with no mutation: %v", x.rule, x.needs, w.violations) dump(t, w) } } } // expectedOutcome is one line of expected-outcomes.txt: `rule missed reason` // or `rule rare N reason` (hostlife's format, read by mutate.sh). type expectedOutcome struct { rule, outcome string depth int reason string } func readExpectedOutcomes(t *testing.T) []expectedOutcome { t.Helper() data, err := os.ReadFile("expected-outcomes.txt") if err != nil { t.Fatalf("mutate.sh reads expected-outcomes.txt beside it: %v", err) } var out []expectedOutcome for i, line := range strings.Split(string(data), "\n") { if strings.HasPrefix(line, "#") || strings.TrimSpace(line) == "" { continue } f := strings.Fields(line) e := expectedOutcome{rule: f[0]} if len(f) > 1 { e.outcome = f[1] } rest := f[min(2, len(f)):] if e.outcome == "rare" { if len(f) < 3 { t.Errorf("expected-outcomes.txt:%d: rare needs a seed count: %q", i+1, line) continue } if _, err := fmt.Sscanf(f[2], "%d", &e.depth); err != nil || e.depth <= 0 { t.Errorf("expected-outcomes.txt:%d: rare needs a positive seed count: %q", i+1, line) } rest = f[3:] } e.reason = strings.Join(rest, " ") out = append(out, e) } return out } // TestEveryRuleHasWitness: each rule in allRules has a witness or a line in // expected-outcomes.txt that says why it has none or why it is not caught // alone; every line there is well formed and names a known rule; and a rule // listed `missed` has no witness that its mutation alone must violate (the // harness would report it STALE). func TestEveryRuleHasWitness(t *testing.T) { known := map[string]bool{} for _, r := range allRules { known[r] = true } expected := map[string]expectedOutcome{} for _, e := range readExpectedOutcomes(t) { if !known[e.rule] { t.Errorf("expected-outcomes.txt names unknown rule %q", e.rule) } if _, dup := expected[e.rule]; dup { t.Errorf("expected-outcomes.txt lists %q twice", e.rule) } if e.outcome != "missed" && e.outcome != "rare" { t.Errorf("expected-outcomes.txt: %s: outcome %q, want missed or rare N", e.rule, e.outcome) } if e.reason == "" { t.Errorf("expected-outcomes.txt: %s gives no reason", e.rule) } expected[e.rule] = e } witnessed := map[string]bool{} for _, x := range witnesses { witnessed[x.rule] = true if e, ok := expected[x.rule]; ok && e.outcome == "missed" && x.needs == x.rule { t.Errorf("%s is listed missed, but a witness must violate under its mutation alone", x.rule) } } for _, r := range allRules { if !witnessed[r] { if _, ok := expected[r]; !ok { t.Errorf("rule %q has neither a witness nor an expected-outcomes.txt line", r) } } } }