package release import ( "bytes" "context" "crypto/ed25519" "encoding/json" "errors" "fmt" "os" "path/filepath" "strings" "testing" "time" ) // D3-held (controller ruling): a held stage applies D3 exactly as adoption // does (ProofGate.admit before staging, ProofGate.bind before publishing) // and fails closed. Where adoption would refuse, the held stage returns // ErrHeldNotProven before publishing and live/ is untouched; with no // policy, or a policy that does not list the app, nothing changes. // heldProofEnv is heldFixture with the held host already live on "v0" (so // "untouched" means something), a policy path, a bake directory, a prover // key and a gate. type heldProofEnv struct { ctx context.Context store Store opt, bake string policy string gate *ProofGate proverPub string prover ed25519.PrivateKey pinnedSHA string } func newHeldProofEnv(t *testing.T) *heldProofEnv { t.Helper() ctx, s, opt := heldFixture(t) if err := StageArtifact(opt, "bab", "v0", "console", []byte("earlier build")); err != nil { t.Fatal(err) } if err := PointLive(opt, heldTestHost, "bab", "v0", "console"); err != nil { t.Fatal(err) } prover, _, pubB64 := proverKey(t) e := &heldProofEnv{ctx: ctx, store: s, opt: opt, bake: t.TempDir(), policy: filepath.Join(t.TempDir(), "proof-policy.json"), proverPub: pubB64, prover: prover, pinnedSHA: etagOf([]byte("pinned build"))} e.gate = NewProofGate(e.policy, e.bake) return e } func (e *heldProofEnv) writePolicy(t *testing.T, body string) { t.Helper() if err := os.WriteFile(e.policy, []byte(body), 0o644); err != nil { t.Fatal(err) } } func (e *heldProofEnv) require(t *testing.T, app string) { e.writePolicy(t, fmt.Sprintf(`{"schema":1,"require":[{"app":%q,"prover":"dev","key":%q}]}`, app, e.proverPub)) } func (e *heldProofEnv) prove(t *testing.T, sha string) { t.Helper() p, err := SignProof(e.prover, Proof{App: "console", Platform: adoptPlatform, SHA256: sha, Version: "v1", Prover: "dev", ActiveSince: time.Date(2026, 10, 6, 9, 0, 0, 0, time.UTC), ProvenAt: time.Date(2026, 10, 6, 10, 0, 0, 0, time.UTC)}) if err != nil { t.Fatal(err) } b, err := json.MarshalIndent(p, "", " ") if err != nil { t.Fatal(err) } if _, err := e.store.Put(e.ctx, ProofKey("bab", "console", sha, "dev"), b, ""); err != nil { t.Fatal(err) } } // staleCopyDown is stageStaleCopyKept for v1: a copy of other bytes staged // while v1's canonical manifest cannot be read, so only D3's bind can catch // it. A held stage reaches bind on such a copy only on pointer trust (want // "", a build promoted before records, settings spec ยง4.1): under v1's // promotion record the stage binds the record's digest and refuses the copy // before bind (TestHeldStageRecordRefusesAStaleCopyItCannotCheck). So v1's // record is removed first, and the digestless canary/x pointer is the // stage's evidence. func (e *heldProofEnv) staleCopyDown(t *testing.T, stale []byte) Store { t.Helper() if err := e.store.Delete(e.ctx, PromotionRecordKey("bab", "console", "v1")); err != nil { t.Fatal(err) } return stageStaleCopyKept(t, e.ctx, e.store, e.opt, "bab", "console", "v1", stale) } func (e *heldProofEnv) live() string { v, _ := LiveVersion(e.opt, heldTestHost); return v } // materialize runs MaterializeHeld with a guard that records whether it // was asked to publish, and returns the error, whether the guard ran, and // what was printed. func (e *heldProofEnv) materialize(t *testing.T, s Store) (error, bool, string) { t.Helper() var ( err error guarded bool out bytes.Buffer ) withCapturedStdout(t, &out, func() { _, err = MaterializeHeld(e.ctx, s, heldPub, nr(t), e.opt, heldRoute(), "v1", e.gate, func(publish func() error) error { guarded = true return publish() }) }) return err, guarded, out.String() } func (e *heldProofEnv) refusedUntouched(t *testing.T, err error, guarded bool) { t.Helper() if !errors.Is(err, ErrHeldNotProven) { t.Fatalf("MaterializeHeld: %v, want ErrHeldNotProven", err) } if guarded { t.Fatal("the held stage asked its authority to publish an unproven release") } if e.live() != "v0" { t.Fatalf("live = %q, want v0 untouched", e.live()) } } func TestHeldStageProofCoveredUnprovenIsRefused(t *testing.T) { e := newHeldProofEnv(t) e.require(t, "console") err, guarded, out := e.materialize(t, e.store) e.refusedUntouched(t, err, guarded) if !strings.Contains(out, "is not proven on dev") { t.Fatalf("printed %q, want adoption's not-proven line", out) } // admit runs before staging: an unproven build is never downloaded. if _, err := os.Stat(VersionPath(e.opt, "bab", "v1", "console")); !errors.Is(err, os.ErrNotExist) { t.Fatalf("v1 was staged for an unproven held stage: %v", err) } // A missing proof is not a refusal record, as in adoption. for _, base := range []string{e.bake, HeldRefusalBase(e.bake)} { if r, _ := ReadRefusal(base, heldTestHost); r != nil { t.Fatalf("a missing proof wrote a refusal: %+v", r) } } } func TestHeldStageProofBadProofIsRefused(t *testing.T) { e := newHeldProofEnv(t) e.require(t, "console") other, _, _ := proverKey(t) p, err := SignProof(other, Proof{App: "console", Platform: adoptPlatform, SHA256: e.pinnedSHA, Version: "v1", Prover: "dev", ActiveSince: time.Date(2026, 10, 6, 9, 0, 0, 0, time.UTC), ProvenAt: time.Date(2026, 10, 6, 10, 0, 0, 0, time.UTC)}) if err != nil { t.Fatal(err) } b, _ := json.Marshal(p) if _, err := e.store.Put(e.ctx, ProofKey("bab", "console", e.pinnedSHA, "dev"), b, ""); err != nil { t.Fatal(err) } err, guarded, _ := e.materialize(t, e.store) e.refusedUntouched(t, err, guarded) } func TestHeldStageProofCoveredAdmittedPublishes(t *testing.T) { e := newHeldProofEnv(t) e.require(t, "console") e.prove(t, e.pinnedSHA) if err, guarded, _ := e.materialize(t, e.store); err != nil || !guarded { t.Fatalf("MaterializeHeld: %v (guarded %v)", err, guarded) } if e.live() != "v1" { t.Fatalf("live = %q, want the proven v1", e.live()) } } func TestHeldStageProofHealthyHistoryPublishes(t *testing.T) { e := newHeldProofEnv(t) e.require(t, "console") if err := RecordHealthy(e.bake, "bab", "console", HealthyBuild{SHA256: e.pinnedSHA, Version: "v1", How: "settle", At: time.Now()}); err != nil { t.Fatal(err) } cs := &proofReadCountingStore{Store: e.store} if err, _, _ := e.materialize(t, cs); err != nil { t.Fatalf("MaterializeHeld: %v", err) } if e.live() != "v1" || cs.proofGets() != 0 { t.Fatalf("live = %q, %d proof GETs; want v1 from the healthy history and no proof read", e.live(), cs.proofGets()) } } func TestHeldStageProofUncoveredIsUnchanged(t *testing.T) { for name, setup := range map[string]func(t *testing.T, e *heldProofEnv){ "no policy file": func(*testing.T, *heldProofEnv) {}, "policy lists another app": func(t *testing.T, e *heldProofEnv) { e.require(t, "other") }, } { t.Run(name, func(t *testing.T) { e := newHeldProofEnv(t) setup(t, e) cs := &proofReadCountingStore{Store: e.store} if err, guarded, _ := e.materialize(t, cs); err != nil || !guarded { t.Fatalf("MaterializeHeld: %v (guarded %v)", err, guarded) } if e.live() != "v1" { t.Fatalf("live = %q, want v1", e.live()) } if n := cs.proofGets(); n != 0 { t.Fatalf("%d proof GETs for an app no policy covers, want 0", n) } }) } } // A policy that does not load is closed for every held route, named unit // or not: the held path cannot vouch for the route's unit binding. func TestHeldStageProofBrokenPolicyIsRefused(t *testing.T) { e := newHeldProofEnv(t) e.writePolicy(t, `{"schema":1,"require":[`) err, guarded, _ := e.materialize(t, e.store) e.refusedUntouched(t, err, guarded) } // bind: the manifest's sha256 is proven, but the staged file holds other // bytes (a kept stale copy: see stageStaleCopyKept). Refused before // publishing, with adoption's not-proven record. func TestHeldStageProofBindingRefusesOtherStagedBytes(t *testing.T) { e := newHeldProofEnv(t) e.require(t, "console") e.prove(t, e.pinnedSHA) stale := []byte("not the shipped bytes") down := e.staleCopyDown(t, stale) // bind runs inside the publication (fix round 1), so the guard is asked // and the refusal comes from within it. err, _, out := e.materialize(t, down) if !errors.Is(err, ErrHeldNotProven) || e.live() != "v0" { t.Fatalf("MaterializeHeld: %v, live %q; want ErrHeldNotProven and v0", err, e.live()) } if !strings.Contains(out, "not pointed live") { t.Fatalf("printed %q, want the binding refusal", out) } r, _ := ReadRefusal(HeldRefusalBase(e.bake), heldTestHost) if r == nil || r.Kind != RefusalNotProven || !strings.Contains(r.Reason, etagOf(stale)) || r.RestoredVersion != "v0" { t.Fatalf("refusal = %+v, want a not-proven record naming the staged hash, still on v0", r) } } // PublishHeld called on its own (after StageHeld) runs the same check. func TestHeldStageProofPublishHeldChecksToo(t *testing.T) { e := newHeldProofEnv(t) if _, err := StageHeld(e.ctx, e.store, heldPub, nr(t), e.opt, heldRoute(), "v1"); err != nil { t.Fatal(err) } e.require(t, "console") var out bytes.Buffer var err error withCapturedStdout(t, &out, func() { err = PublishHeld(e.ctx, e.store, e.opt, heldRoute(), "v1", e.gate, openGuard) }) if !errors.Is(err, ErrHeldNotProven) || e.live() != "v0" { t.Fatalf("PublishHeld: %v, live %q; want ErrHeldNotProven and v0", err, e.live()) } } // A held stage with no gate cannot publish: D3 is not optional here. func TestHeldStageProofNeedsAGate(t *testing.T) { e := newHeldProofEnv(t) if _, err := MaterializeHeld(e.ctx, e.store, heldPub, nr(t), e.opt, heldRoute(), "v1", nil, openGuard); err == nil { t.Fatal("MaterializeHeld published with no proof gate") } if err := PublishHeld(e.ctx, e.store, e.opt, heldRoute(), "v1", nil, openGuard); err == nil { t.Fatal("PublishHeld published with no proof gate") } if e.live() != "v0" { t.Fatalf("live = %q, want v0", e.live()) } if _, err := os.Stat(VersionPath(e.opt, "bab", "v1", "console")); !errors.Is(err, os.ErrNotExist) { t.Fatalf("v1 was staged with no gate: %v", err) } } // The root relay: an unproven request is removed, never published and // never acknowledged; a proven one goes live. func TestServeHeldStageRequestsAppliesTheProofGate(t *testing.T) { e := newHeldProofEnv(t) e.require(t, "console") dd := t.TempDir() path := writeHeldRequest(t, dd, heldTestHost+".json", goodHeldRequest()) var out bytes.Buffer withCapturedStdout(t, &out, func() { if err := ServeHeldStageRequests(e.ctx, e.store, heldPub, nr(t), e.opt, dd, e.gate, passAuth(nil)); err != nil { t.Fatalf("ServeHeldStageRequests: %v", err) } }) gone(t, path) gone(t, filepath.Join(HeldStagedDir(dd), heldTestHost+".json")) if e.live() != "v0" { t.Fatalf("live = %q, want v0 untouched", e.live()) } if !strings.Contains(out.String(), "refused") { t.Fatalf("printed %q, want the refusal", out.String()) } e.prove(t, e.pinnedSHA) e.gate = NewProofGate(e.policy, e.bake) // a fresh root pass: no cached miss writeHeldRequest(t, dd, heldTestHost+".json", goodHeldRequest()) withCapturedStdout(t, &out, func() { if err := ServeHeldStageRequests(e.ctx, e.store, heldPub, nr(t), e.opt, dd, e.gate, passAuth(nil)); err != nil { t.Fatalf("ServeHeldStageRequests: %v", err) } }) if e.live() != "v1" { t.Fatalf("live = %q, want the proven v1", e.live()) } } // materializeWith is materialize with the caller's guard. func (e *heldProofEnv) materializeWith(t *testing.T, guard func(publish func() error) error) (error, string) { t.Helper() var ( err error out bytes.Buffer ) withCapturedStdout(t, &out, func() { _, err = MaterializeHeld(e.ctx, e.store, heldPub, nr(t), e.opt, heldRoute(), "v1", e.gate, guard) }) return err, out.String() } // Fix round 1, finding 1: the authority's re-check (bucket reads, the // registry freeze) runs between staging and publication, and the staged // bytes can be replaced meanwhile. bind runs inside the publication, after // the re-check, so bytes replaced while the guard ran are refused. func TestHeldStageProofBindsInsideThePublication(t *testing.T) { e := newHeldProofEnv(t) e.require(t, "console") e.prove(t, e.pinnedSHA) stale := []byte("replaced while the guard re-checked") err, _ := e.materializeWith(t, func(publish func() error) error { if err := StageArtifact(e.opt, "bab", "v1", "console", stale); err != nil { t.Fatal(err) } return publish() }) if !errors.Is(err, ErrHeldNotProven) || e.live() != "v0" { t.Fatalf("MaterializeHeld: %v, live %q; want ErrHeldNotProven and v0", err, e.live()) } } // Fix round 1, finding 1: bytes replaced between bind and the link swap // are caught by a re-hash at the swap, and the link goes back to what it // was (or away, when there was nothing). func TestHeldStageProofRehashesAtTheSwap(t *testing.T) { for name, hadLive := range map[string]bool{"was on v0": true, "had no link": false} { t.Run(name, func(t *testing.T) { e := newHeldProofEnv(t) if !hadLive { if err := os.Remove(LiveLink(e.opt, heldTestHost)); err != nil { t.Fatal(err) } } e.require(t, "console") e.prove(t, e.pinnedSHA) prev := heldBeforeSwap t.Cleanup(func() { heldBeforeSwap = prev }) swapped := false heldBeforeSwap = func() { swapped = true if err := StageArtifact(e.opt, "bab", "v1", "console", []byte("replaced after bind")); err != nil { t.Fatal(err) } } err, guarded, _ := e.materialize(t, e.store) if !swapped || !guarded { t.Fatalf("the seam ran %v, the guard ran %v", swapped, guarded) } want := "v0" if !hadLive { want = "" } if !errors.Is(err, ErrHeldNotProven) || e.live() != want { t.Fatalf("MaterializeHeld: %v, live %q; want ErrHeldNotProven and %q", err, e.live(), want) } }) } } // Fix round 1, finding 2: a held binding refusal carries its move's op. func TestHeldStageProofRefusalCarriesTheMove(t *testing.T) { e := newHeldProofEnv(t) e.require(t, "console") e.prove(t, e.pinnedSHA) down := e.staleCopyDown(t, []byte("not the shipped bytes")) e.materialize(t, down) r, _ := ReadRefusal(HeldRefusalBase(e.bake), heldTestHost) if r == nil || r.MoveOp != heldTestOp || r.Kind != RefusalNotProven { t.Fatalf("refusal = %+v, want a not-proven record naming op %s", r, heldTestOp) } // Final review finding 3: in the held namespace, never adoption's. if r, _ := ReadRefusal(e.bake, heldTestHost); r != nil { t.Fatalf("the held refusal reached adoption's records: %+v", r) } } // Fix round 1, finding 2: an adoption refusal already standing for the host // is never overwritten by a held one. Since final review finding 3 the held // one is recorded beside it, in its own namespace. func TestHeldStageProofLeavesAnAdoptionRefusal(t *testing.T) { e := newHeldProofEnv(t) e.require(t, "console") e.prove(t, e.pinnedSHA) adoption := Refusal{Host: heldTestHost, App: "console", Version: "v0", SHA256: "aa", Reason: "did not come up", PointerPromotedAt: time.Date(2026, 10, 1, 0, 0, 0, 0, time.UTC), RefusedAt: time.Date(2026, 10, 1, 0, 1, 0, 0, time.UTC)} if err := WriteRefusal(e.bake, adoption); err != nil { t.Fatal(err) } before, _ := os.ReadFile(filepath.Join(RefusalDir(e.bake), heldTestHost+".json")) down := e.staleCopyDown(t, []byte("not the shipped bytes")) err, _, _ := e.materialize(t, down) if !errors.Is(err, ErrHeldNotProven) { t.Fatalf("MaterializeHeld: %v, want ErrHeldNotProven", err) } after, _ := os.ReadFile(filepath.Join(RefusalDir(e.bake), heldTestHost+".json")) if !bytes.Equal(before, after) { t.Fatalf("the adoption refusal changed:\n%s\nwant\n%s", after, before) } if r, _ := ReadRefusal(HeldRefusalBase(e.bake), heldTestHost); r == nil || r.MoveOp != heldTestOp { t.Fatalf("the held refusal = %+v, want it recorded beside the adoption one", r) } } // Fix round 1, finding 2: a later held stage of the same move that // succeeds retires that move's refusal. func TestHeldStageProofSuccessRetiresTheMovesRefusal(t *testing.T) { e := newHeldProofEnv(t) e.require(t, "console") e.prove(t, e.pinnedSHA) down := e.staleCopyDown(t, []byte("not the shipped bytes")) e.materialize(t, down) if r, _ := ReadRefusal(HeldRefusalBase(e.bake), heldTestHost); r == nil { t.Fatal("no refusal to retire") } if err := StageArtifact(e.opt, "bab", "v1", "console", []byte("pinned build")); err != nil { t.Fatal(err) } if err, _, _ := e.materialize(t, e.store); err != nil || e.live() != "v1" { t.Fatalf("retry: %v, live %q", err, e.live()) } if r, _ := ReadRefusal(HeldRefusalBase(e.bake), heldTestHost); r != nil { t.Fatalf("the move's refusal outlived its successful retry: %+v", r) } } // RetireHeldRefusals removes a held refusal whose move is over and nothing // else: not a live move's, never an adoption refusal. func TestRetireHeldRefusals(t *testing.T) { bake := t.TempDir() at := time.Date(2026, 10, 7, 0, 0, 0, 0, time.UTC) for _, r := range []Refusal{ {Host: "live.example.com", App: "a", Version: "v1", RefusedAt: at, Kind: RefusalNotProven}, {Host: "over.example.com", App: "a", Version: "v1", RefusedAt: at, Kind: RefusalNotProven}, } { if err := heldRefusalWriter(bake, "mv_"+strings.TrimSuffix(r.Host, ".example.com"))(r); err != nil { t.Fatal(err) } } if err := WriteRefusal(bake, Refusal{Host: "adopted.example.com", App: "a", Version: "v1", RefusedAt: at, Kind: RefusalNotProven}); err != nil { t.Fatal(err) } retired, err := RetireHeldRefusals(bake, func(r Refusal) bool { return r.MoveOp == "mv_live" }) if err != nil || len(retired) != 1 || retired[0] != "over.example.com" { t.Fatalf("retired %v, %v; want over.example.com", retired, err) } left, _ := ListHeldRefusals(bake) if len(left) != 1 || left[0].Host != "live.example.com" { t.Fatalf("held left %+v", left) } if adopted, _ := ListRefusals(bake); len(adopted) != 1 || adopted[0].Host != "adopted.example.com" { t.Fatalf("adoption's records %+v", adopted) } // With no refusal directory it reads nothing and fails nothing. if retired, err := RetireHeldRefusals(t.TempDir(), func(Refusal) bool { return false }); err != nil || len(retired) != 0 { t.Fatalf("empty: %v, %v", retired, err) } }