package release import ( "bytes" "context" "encoding/json" "fmt" "os" "path/filepath" "testing" "time" ) // Jitter is in [0, 0.25), so a hold of H is eligible somewhere in // [firstSeen+H, firstSeen+1.25H). 91h is safely past stable's worst case // (72h * 1.25 = 90h); 71h is safely inside the best case. const ( insideStableHold = 71 * time.Hour pastStableHold = 91 * time.Hour ) func TestBakeHoldBlocksNewStableUntilHeld(t *testing.T) { path := filepath.Join(t.TempDir(), "first-seen.json") st := LoadFirstSeen(path) t0 := time.Now() if !st.Observe("bab", "myapp", "stable", "v1", t0) { t.Fatalf("first Observe reported no change") } if hold := Hold(st, "bab", "myapp", "stable", "v1", DefaultHolds, false, t0); hold == "" { t.Fatalf("brand-new stable head not held") } if hold := Hold(st, "bab", "myapp", "stable", "v1", DefaultHolds, false, t0.Add(insideStableHold)); hold == "" { t.Fatalf("stable head released before its 72h window") } if hold := Hold(st, "bab", "myapp", "stable", "v1", DefaultHolds, false, t0.Add(pastStableHold)); hold != "" { t.Fatalf("stable head still held past its window: %s", hold) } // A version with NO first-seen record fails closed. if hold := Hold(st, "bab", "myapp", "stable", "vGhost", DefaultHolds, false, t0.Add(pastStableHold)); hold == "" { t.Fatalf("unseen version not held (must fail closed)") } // The local root-only override releases immediately. if hold := Hold(st, "bab", "myapp", "stable", "v1", DefaultHolds, true, t0); hold != "" { t.Fatalf("ignore=true still held: %s", hold) } // canary/* has no window: eligible the moment it is first seen. st.Observe("bab", "myapp", "canary/x", "v1", t0) if hold := Hold(st, "bab", "myapp", "canary/x", "v1", DefaultHolds, false, t0); hold != "" { t.Fatalf("canary held: %s", hold) } } // TestRollbackNeverBakes is THE key property: a pointer moved BACK to a // previously-seen version applies immediately — its first-seen record // predates the hold (that's how it got installed the first time), and // Observe never resets an existing timestamp. Evacuating a bad release must // never itself sit in a bake window. func TestRollbackNeverBakes(t *testing.T) { s := NewDirStore(t.TempDir()) ctx := context.Background() priv, pub := testKeys(t) shipRelease(t, s, "bab", "myapp", "v1", "linux-amd64", []byte("one")) shipRelease(t, s, "bab", "myapp", "v2", "linux-amd64", []byte("two")) st := LoadFirstSeen(filepath.Join(t.TempDir(), "first-seen.json")) t0 := time.Now() // v1 promoted to stable; the box first sees it at t0, bakes, installs. if err := Promote(ctx, s, priv, "bab", "myapp", "stable", "v1"); err != nil { t.Fatalf("Promote v1: %v", err) } ptr, err := Channel(ctx, s, pub, "bab", "myapp", "stable") if err != nil { t.Fatalf("Channel: %v", err) } st.Observe("bab", "myapp", "stable", ptr.Version, t0) // Time passes; v1's window elapses; v2 is promoted and first seen NOW. t1 := t0.Add(pastStableHold) if err := Promote(ctx, s, priv, "bab", "myapp", "stable", "v2"); err != nil { t.Fatalf("Promote v2: %v", err) } ptr, _ = Channel(ctx, s, pub, "bab", "myapp", "stable") st.Observe("bab", "myapp", "stable", ptr.Version, t1) if hold := Hold(st, "bab", "myapp", "stable", ptr.Version, DefaultHolds, false, t1); hold == "" { t.Fatalf("fresh v2 not held") } // v2 turns out bad — rollback. The pointer is back at v1, and v1 applies // IMMEDIATELY: its first-seen record is t0, long past the hold. Observe // must not reset it. if err := Rollback(ctx, s, priv, "bab", "myapp", "stable"); err != nil { t.Fatalf("Rollback: %v", err) } ptr, err = Channel(ctx, s, pub, "bab", "myapp", "stable") if err != nil { t.Fatalf("Channel after rollback: %v", err) } if ptr.Version != "v1" { t.Fatalf("rollback pointer = %s, want v1", ptr.Version) } st.Observe("bab", "myapp", "stable", ptr.Version, t1) // re-seen at head: no reset if hold := Hold(st, "bab", "myapp", "stable", ptr.Version, DefaultHolds, false, t1); hold != "" { t.Fatalf("rolled-back v1 held: %s (evacuation must never bake)", hold) } // And v2 stays held — the rollback didn't launder it. if hold := Hold(st, "bab", "myapp", "stable", "v2", DefaultHolds, false, t1); hold == "" { t.Fatalf("bad v2 no longer held after rollback") } } func TestLoadFirstSeenCorruptFailsClosed(t *testing.T) { path := filepath.Join(t.TempDir(), "first-seen.json") if err := os.WriteFile(path, []byte("{ this is not json"), 0o644); err != nil { t.Fatal(err) } st := LoadFirstSeen(path) // A corrupt file is an empty clock: everything bakes from now — a wiped // clock delays updates, it never lets one through early. if hold := Hold(st, "bab", "myapp", "stable", "v1", DefaultHolds, false, time.Now()); hold == "" { t.Fatalf("corrupt clock did not fail closed") } // Missing file: same. st2 := LoadFirstSeen(filepath.Join(t.TempDir(), "absent.json")) if hold := Hold(st2, "bab", "myapp", "stable", "v1", DefaultHolds, false, time.Now()); hold == "" { t.Fatalf("missing clock did not fail closed") } } func TestFirstSeenSaveLoadRoundTrip(t *testing.T) { path := filepath.Join(t.TempDir(), "first-seen.json") st := LoadFirstSeen(path) t0 := time.Now() st.Observe("bab", "myapp", "stable", "v1", t0) if st.Jitter < 0 || st.Jitter >= 0.25 { t.Fatalf("minted jitter %v outside [0, 0.25)", st.Jitter) } if err := st.Save(); err != nil { t.Fatalf("Save: %v", err) } st2 := LoadFirstSeen(path) if st2.Jitter != st.Jitter { t.Fatalf("jitter not persisted: %v != %v", st2.Jitter, st.Jitter) } at, ok := st2.EligibleAt("bab", "myapp", "stable", "v1", DefaultHolds) if !ok { t.Fatalf("first-seen record lost on reload") } want := t0.UTC().Truncate(time.Second).Add(time.Duration(float64(72*time.Hour) * (1 + st.Jitter))) if !at.Equal(want) { t.Fatalf("EligibleAt = %v, want %v", at, want) } } func TestHoldForWildcardAndUnknown(t *testing.T) { if d := HoldFor(DefaultHolds, "canary/any-branch"); d != 0 { t.Fatalf("canary/* = %v, want 0", d) } if d := HoldFor(DefaultHolds, "beta"); d != 24*time.Hour { t.Fatalf("beta = %v, want 24h", d) } if d := HoldFor(DefaultHolds, "stable"); d != 72*time.Hour { t.Fatalf("stable = %v, want 72h", d) } // An unknown channel fails closed to the longest standard window. if d := HoldFor(DefaultHolds, "mystery"); d != 72*time.Hour { t.Fatalf("unknown channel = %v, want 72h (fail closed)", d) } // A per-deployment override map replaces the defaults wholesale. custom := map[string]time.Duration{"stable": time.Hour} if d := HoldFor(custom, "stable"); d != time.Hour { t.Fatalf("override stable = %v, want 1h", d) } } // The bake clock is per (account, app, channel, version). Keyed on the app // name alone, one account's 72h stable observation would satisfy ANOTHER // account's identically-named app on the same box — its supply-chain window // silently skipped, which is the one thing this clock exists to prevent. func TestBakeClockIsPerAccount(t *testing.T) { st := LoadFirstSeen(filepath.Join(t.TempDir(), "clock.json")) long := time.Now().Add(-100 * time.Hour) // Account bab saw v1 at stable 100h ago; account kqmv has never seen it. st.Observe("bab", "blog", "stable", "v1", long) if hold := Hold(st, "bab", "blog", "stable", "v1", DefaultHolds, false, time.Now()); hold != "" { t.Fatalf("the account that saw it 100h ago is held: %s", hold) } if hold := Hold(st, "kqmv", "blog", "stable", "v1", DefaultHolds, false, time.Now()); hold == "" { t.Fatal("another account's identically-named app skipped its bake window " + "on the strength of the first account's clock") } } // The clock key must be injective. Channels may contain "/", so joining // segments with "/" would make ops app "kass" + channel "canary/x" collide // with account "kass" + app "canary" + channel "x" — one account's // observation silently satisfying another's window, which is the same bug // TestBakeClockIsPerAccount closes, reached by a different route. func TestBakeClockKeyIsInjective(t *testing.T) { st := LoadFirstSeen(filepath.Join(t.TempDir(), "clock.json")) long := time.Now().Add(-100 * time.Hour) // ops app "kass" on channel "canary/x". st.Observe(OpsAccount, "kass", "canary/x", "v1", long) // Account "kass", app "canary", channel "x" — three different segments // that concatenate the same way, and must NOT resolve to that record. if _, ok := st.EligibleAt("kass", "canary", "x", "v1", DefaultHolds); ok { t.Fatal("the clock key collides across (account, app, channel) boundaries") } if hold := Hold(st, "kass", "canary", "x", "v1", DefaultHolds, false, time.Now()); hold == "" { t.Fatal("a colliding key let another account skip its bake window") } // The record it was actually written under still resolves. if hold := Hold(st, OpsAccount, "kass", "canary/x", "v1", DefaultHolds, false, time.Now()); hold != "" { t.Fatalf("the real observation stopped resolving: %s", hold) } } // A clock written before accounts existed has bare "/" keys. // Reading those as the ops account keeps every existing box's observations — // otherwise the whole fleet's windows restart on upgrade, and nothing // installs for 72h. func TestBakeClockReadsPreAccountRecordsAsOps(t *testing.T) { path := filepath.Join(t.TempDir(), "clock.json") long := time.Now().Add(-100 * time.Hour).UTC().Truncate(time.Second) legacy := map[string]any{ "jitter": 0.1, "channels": map[string]any{ "blog/stable": map[string]any{"v1": long}, }, } raw, err := json.MarshalIndent(legacy, "", " ") if err != nil { t.Fatal(err) } if err := os.WriteFile(path, raw, 0o644); err != nil { t.Fatal(err) } st := LoadFirstSeen(path) if hold := Hold(st, OpsAccount, "blog", "stable", "v1", DefaultHolds, false, time.Now()); hold != "" { t.Fatalf("a pre-accounts observation was lost on upgrade: %s", hold) } // A non-ops account gets no free ride from it. if hold := Hold(st, "kqmv", "blog", "stable", "v1", DefaultHolds, false, time.Now()); hold == "" { t.Fatal("a pre-accounts record satisfied a different account's window") } } func TestLoadFirstSeenMigrating(t *testing.T) { now := time.Date(2026, 8, 3, 12, 0, 0, 0, time.UTC) t.Run("prefers the new path when it exists", func(t *testing.T) { dir := t.TempDir() newPath := filepath.Join(dir, "new.json") oldPath := filepath.Join(dir, "old.json") stNew := LoadFirstSeen(newPath) stNew.Observe("acct", "app", "stable", "v-new", now) if err := stNew.Save(); err != nil { t.Fatal(err) } stOld := LoadFirstSeen(oldPath) stOld.Observe("acct", "app", "stable", "v-old", now) if err := stOld.Save(); err != nil { t.Fatal(err) } st := LoadFirstSeenMigrating(newPath, oldPath) if _, ok := st.EligibleAt("acct", "app", "stable", "v-new", nil); !ok { t.Fatal("new path's observation missing") } if _, ok := st.EligibleAt("acct", "app", "stable", "v-old", nil); ok { t.Fatal("legacy content leaked in despite the new path existing") } }) t.Run("falls back to legacy content, saves to the new path, leaves legacy untouched", func(t *testing.T) { dir := t.TempDir() newPath := filepath.Join(dir, "new.json") oldPath := filepath.Join(dir, "old.json") stOld := LoadFirstSeen(oldPath) stOld.Observe("acct", "app", "stable", "v1", now) if err := stOld.Save(); err != nil { t.Fatal(err) } legacyRaw, err := os.ReadFile(oldPath) if err != nil { t.Fatal(err) } st := LoadFirstSeenMigrating(newPath, oldPath) if _, ok := st.EligibleAt("acct", "app", "stable", "v1", nil); !ok { t.Fatal("legacy observation not carried over") } if err := st.Save(); err != nil { t.Fatal(err) } if _, err := os.Stat(newPath); err != nil { t.Fatalf("Save did not write the new path: %v", err) } afterRaw, err := os.ReadFile(oldPath) if err != nil || !bytes.Equal(legacyRaw, afterRaw) { t.Fatalf("legacy file must be byte-identical after migration (err %v)", err) } }) t.Run("both missing is a fresh clock at the new path", func(t *testing.T) { dir := t.TempDir() newPath := filepath.Join(dir, "new.json") st := LoadFirstSeenMigrating(newPath, filepath.Join(dir, "absent.json")) st.Observe("acct", "app", "stable", "v1", now) if err := st.Save(); err != nil { t.Fatal(err) } if _, err := os.Stat(newPath); err != nil { t.Fatalf("Save must create the new path: %v", err) } }) } // TestFirstSeenSaveReplacesFileItCannotWriteInPlace stands in for the real // production race (2026-08-03 review, CRITICAL): carlos-edge.service writes // this path as user `carlos`, the root adopt one-shot writes it as root — // whichever creates the file first locks the other uid out, because an // in-place O_TRUNC write needs write permission on the FILE, and a 0644 file // owned by the other uid has none for anyone else. A file this process // cannot open for writing (0o444, no write bit for anyone, including its own // owner) reproduces the same "cannot write in place" shape without needing a // second uid: rename(2) only requires write permission on the containing // directory, never on the file being replaced, so a Save implemented as // temp-file+rename must still succeed here where a truncating write cannot. func TestFirstSeenSaveReplacesFileItCannotWriteInPlace(t *testing.T) { path := filepath.Join(t.TempDir(), "clock.json") if err := os.WriteFile(path, []byte("{}"), 0o444); err != nil { t.Fatal(err) } st := LoadFirstSeen(path) st.Observe("acct", "app", "stable", "v1", time.Now()) if err := st.Save(); err != nil { t.Fatalf("Save must replace via rename, not an in-place write: %v", err) } // The temp name is minted fresh per attempt (CreateTemp's // ".carlos-write-*"), not a fixed ".tmp" — glob for the // pattern rather than stat-ing one fixed name. if matches, err := filepath.Glob(filepath.Join(filepath.Dir(path), ".carlos-write-*")); err != nil { t.Fatalf("glob for stray temp files: %v", err) } else if len(matches) != 0 { t.Fatalf("Save left a temp file behind: %v", matches) } raw, err := os.ReadFile(path) if err != nil { t.Fatal(err) } var reloaded FirstSeen if err := json.Unmarshal(raw, &reloaded); err != nil { t.Fatalf("Save produced invalid/partial content: %v", err) } if _, ok := reloaded.Channels[channelClockKey("acct", "app", "stable")]["v1"]; !ok { t.Fatal("observation missing after Save") } } // The pipelines spec defines edge as "picked up as it lands — no human in // the loop, by definition". Without its own entry, HoldFor fail-closes edge // to 72h — longer than beta, and a spec violation (2026-08-04 console bake // controls spec, §1/§8). func TestHoldForEdgeIsZero(t *testing.T) { if d := HoldFor(DefaultHolds, "edge"); d != 0 { t.Fatalf("edge hold = %v, want 0", d) } } // The box-side production memory (2026-08-04 console bake controls spec §4). // Scenario table, because the security content IS the state machine: the // default is off; on is instant; off ratchets on the box's own clock; a // DELETED flag never downgrades a box that saw true; a stale replayed off // cannot roll back a newer on. func TestProductionMemoryRatchet(t *testing.T) { base := time.Date(2026, 8, 4, 12, 0, 0, 0, time.UTC) flag := func(prod bool, setAt time.Time) *ProductionFlag { return &ProductionFlag{Schema: 1, Account: "bab", App: "kass", Production: prod, SetAt: setAt} } newClock := func() *FirstSeen { st := LoadFirstSeen(filepath.Join(t.TempDir(), "clock.json")) st.Jitter = 0 // determinism: eligibility is exactly firstSeen + hold return st } t.Run("default is off", func(t *testing.T) { st := newClock() if st.ProductionBake("bab", "kass", nil, true, DefaultHolds, base) { t.Fatal("an app with no flag ever seen bakes") } }) t.Run("unverifiable flag fails toward baking", func(t *testing.T) { st := newClock() if !st.ProductionBake("bab", "kass", fmt.Errorf("sig"), true, DefaultHolds, base) { t.Fatal("a corrupt flag read as the default instead of production") } }) t.Run("on is instant, deletion is not a downgrade", func(t *testing.T) { st := newClock() if !st.ObserveProduction("bab", "kass", flag(true, base), base) { t.Fatal("first observation did not change the clock") } if !st.ProductionBake("bab", "kass", nil, true, DefaultHolds, base) { t.Fatal("production: true did not apply immediately") } // The object vanishes from the bucket (flag == nil every later pass): // the box holds its verdict. st.ObserveProduction("bab", "kass", nil, base.Add(time.Hour)) if !st.ProductionBake("bab", "kass", nil, true, DefaultHolds, base.Add(time.Hour)) { t.Fatal("deleting the flag object downgraded a box that saw true") } }) t.Run("off ratchets for the stable window", func(t *testing.T) { st := newClock() st.ObserveProduction("bab", "kass", flag(true, base), base) off := flag(false, base.Add(time.Hour)) st.ObserveProduction("bab", "kass", off, base.Add(2*time.Hour)) // box sees it an hour late hold := HoldFor(DefaultHolds, RungStable) // Still baking just inside the window, measured from the box's OWN // sighting (base+2h), not the flag's set_at (base+1h). if !st.ProductionBake("bab", "kass", nil, true, DefaultHolds, base.Add(2*time.Hour).Add(hold-time.Minute)) { t.Fatal("off was honored before the ratchet elapsed") } if st.ProductionBake("bab", "kass", nil, true, DefaultHolds, base.Add(2*time.Hour).Add(hold+time.Minute)) { t.Fatal("off was not honored after the ratchet elapsed") } }) t.Run("no bakeable head skips the ratchet", func(t *testing.T) { st := newClock() st.ObserveProduction("bab", "kass", flag(true, base), base) st.ObserveProduction("bab", "kass", flag(false, base.Add(time.Hour)), base.Add(time.Hour)) if st.ProductionBake("bab", "kass", nil, false, DefaultHolds, base.Add(time.Hour)) { t.Fatal("carve-out: nothing at a held head, yet off still ratcheted") } }) t.Run("a stale off cannot be replayed over a newer on", func(t *testing.T) { st := newClock() staleOff := flag(false, base) st.ObserveProduction("bab", "kass", staleOff, base) st.ObserveProduction("bab", "kass", flag(true, base.Add(time.Hour)), base.Add(time.Hour)) // Attacker re-writes the old signed off object; the box refuses to // regress below the newest set_at it has honored. if st.ObserveProduction("bab", "kass", staleOff, base.Add(2*time.Hour)) { t.Fatal("a replayed stale flag changed the clock") } if !st.ProductionBake("bab", "kass", nil, true, DefaultHolds, base.Add(100*time.Hour)) { t.Fatal("a replayed stale off eventually turned the bake off") } }) t.Run("memory survives a save/load round trip", func(t *testing.T) { st := newClock() st.ObserveProduction("bab", "kass", flag(true, base), base) if err := st.Save(); err != nil { t.Fatal(err) } st2 := LoadFirstSeen(st.path) if !st2.ProductionBake("bab", "kass", nil, true, DefaultHolds, base) { t.Fatal("production memory did not survive persistence") } }) // Task-4-review fix 2: prodKey must fold the ops account the same way // channelClockKey does, or an app addressed as "" (the pre-accounts // bare shape some callers still use) and the same app addressed as // OpsAccount silently keep SEPARATE production memories. t.Run("prodKey folds the ops account like channelClockKey", func(t *testing.T) { st := newClock() st.ObserveProduction(OpsAccount, "kass", flag(true, base), base) if !st.ProductionBake("", "kass", nil, true, DefaultHolds, base) { t.Fatal(`observed under OpsAccount, not found evaluating under ""`) } st2 := newClock() st2.ObserveProduction("", "kass", flag(true, base), base) if !st2.ProductionBake(OpsAccount, "kass", nil, true, DefaultHolds, base) { t.Fatal(`observed under "", not found evaluating under OpsAccount`) } // No free ride for an unrelated account: "bab"/"kass" (the ops // account) must not satisfy "zzz"/"kass". if st2.ProductionBake("zzz", "kass", nil, true, DefaultHolds, base) { t.Fatal("an unrelated account satisfied the ops account's production memory") } }) // Task-4-review fix 3a: re-reading the SAME off object (the every-pass // re-read, not a fresh sighting) must not change anything — in // particular must not push the ratchet clock forward from the box's // FIRST sighting to whatever pass happens to re-read it. t.Run("equal set_at re-read does not reset the ratchet clock", func(t *testing.T) { st := newClock() st.ObserveProduction("bab", "kass", flag(true, base), base) off := flag(false, base.Add(time.Hour)) st.ObserveProduction("bab", "kass", off, base.Add(2*time.Hour)) // first sighting if st.ObserveProduction("bab", "kass", off, base.Add(3*time.Hour)) { t.Fatal("re-reading the identical off object reported a change") } hold := HoldFor(DefaultHolds, RungStable) // Still measured from the FIRST sighting (base+2h), not the re-read // (base+3h) — a re-read one hour later must not buy the flag one // more hour of ratchet. if !st.ProductionBake("bab", "kass", nil, true, DefaultHolds, base.Add(2*time.Hour).Add(hold-time.Minute)) { t.Fatal("off honored before the ratchet from its first sighting elapsed") } if st.ProductionBake("bab", "kass", nil, true, DefaultHolds, base.Add(2*time.Hour).Add(hold+time.Minute)) { t.Fatal("re-reading the same object pushed the ratchet clock forward") } }) // Task-4-review fix 3b: an in-progress off-ratchet (not just a bare // "on" latch) must itself survive Save/LoadFirstSeen. t.Run("off-state ratchet survives a save/load round trip", func(t *testing.T) { st := newClock() st.ObserveProduction("bab", "kass", flag(true, base), base) st.ObserveProduction("bab", "kass", flag(false, base.Add(time.Hour)), base.Add(time.Hour)) if err := st.Save(); err != nil { t.Fatal(err) } st2 := LoadFirstSeen(st.path) hold := HoldFor(DefaultHolds, RungStable) if !st2.ProductionBake("bab", "kass", nil, true, DefaultHolds, base.Add(time.Hour).Add(hold-time.Minute)) { t.Fatal("off-state ratchet lost on reload: should still be baking mid-window") } if st2.ProductionBake("bab", "kass", nil, true, DefaultHolds, base.Add(time.Hour).Add(hold+time.Minute)) { t.Fatal("off-state ratchet lost on reload: should be off past the window") } }) // Task-4-review fix 3c: the ratchet is jittered exactly like any other // hold (HoldFor's formula), not a bare, un-jittered 72h. t.Run("the ratchet is jittered like any other hold", func(t *testing.T) { st := LoadFirstSeen(filepath.Join(t.TempDir(), "clock.json")) st.Jitter = 0.2 st.ObserveProduction("bab", "kass", flag(true, base), base) st.ObserveProduction("bab", "kass", flag(false, base.Add(time.Hour)), base.Add(time.Hour)) hold := HoldFor(DefaultHolds, RungStable) // 72h, unjittered if !st.ProductionBake("bab", "kass", nil, true, DefaultHolds, base.Add(time.Hour).Add(hold).Add(time.Minute)) { t.Fatal("jitter not applied: off honored right at the un-jittered window") } jittered := time.Duration(float64(hold) * 1.2) if st.ProductionBake("bab", "kass", nil, true, DefaultHolds, base.Add(time.Hour).Add(jittered).Add(time.Minute)) { t.Fatal("jitter not applied: off still honored past the jittered window") } }) // Task-4-review fix 3d: an app declared off and NEVER on has nothing to // ratchet — it must never bake, immediately and forever. t.Run("an app declared off and never on never bakes", func(t *testing.T) { st := newClock() st.ObserveProduction("bab", "kass", flag(false, base), base) if st.ProductionBake("bab", "kass", nil, true, DefaultHolds, base) { t.Fatal("an app declared off, never on, baked immediately") } if st.ProductionBake("bab", "kass", nil, true, DefaultHolds, base.Add(1000*time.Hour)) { t.Fatal("an app declared off, never on, baked eventually") } }) } // Task-4-review fix 1 (Critical): LoadFirstSeen must distinguish a file that // read but did not PARSE from a file that is simply absent. Before this fix // both collapsed to the same "empty clock" result, which for the CHANNEL // clock is fine (fails closed either way — everything re-bakes from now), // but for the PRODUCTION memory silently failed OPEN: a corrupt clock file // (mem == nil for every app) made ProductionBake return false, instantly // un-baking every production app on the box. A corrupt file must instead // fail TOWARD baking (st.tainted), while an absent file — a genuinely fresh // box — still gets the ordinary false default. func TestCorruptClockFailsTowardBakingNotAwayFromIt(t *testing.T) { base := time.Date(2026, 8, 4, 12, 0, 0, 0, time.UTC) path := filepath.Join(t.TempDir(), "clock.json") if err := os.WriteFile(path, []byte("{ this is not json"), 0o644); err != nil { t.Fatal(err) } st := LoadFirstSeen(path) if !st.ProductionBake("bab", "kass", nil, true, DefaultHolds, base) { t.Fatal("a corrupt clock file did not fail toward baking") } // A genuinely fresh box (no file at all) is a different case entirely — // still the ordinary false default, not tainted. fresh := LoadFirstSeen(filepath.Join(t.TempDir(), "absent.json")) if fresh.ProductionBake("bab", "kass", nil, true, DefaultHolds, base) { t.Fatal("a fresh box with no clock file baked by default") } // A file that EXISTS but can't be read at all (not "did not parse") is // a different os.ReadFile failure mode than os.ErrNotExist — a // directory sitting at the clock path is the easiest way to reproduce // it portably (EISDIR), and review found the first cut of this fix only // tainted the "read OK, parse failed" branch, leaving this one to // silently fail open. dirPath := filepath.Join(t.TempDir(), "clock.json") if err := os.Mkdir(dirPath, 0o755); err != nil { t.Fatal(err) } unreadable := LoadFirstSeen(dirPath) if !unreadable.ProductionBake("bab", "kass", nil, true, DefaultHolds, base) { t.Fatal("an unreadable (directory-at-path) clock file did not fail toward baking") } } func TestHoldMemoryRatchet(t *testing.T) { f := &FirstSeen{Jitter: 0} now := time.Date(2026, 8, 20, 12, 0, 0, 0, time.UTC) pipe := func(bake string, setAt time.Time) *Pipeline { return &Pipeline{Schema: 1, Channels: []PipelineChannel{ {Name: "edge"}, {Name: "production", Bake: bake}}, SetAt: setAt} } // First sighting: production baselines at the fail-closed 72h, NOT its declared 24h. f.ObservePipeline("bae", "web", pipe("24h", now), now) holds := f.HoldsForApp("bae", "web", pipe("24h", now)) if holds["production"] != 72*time.Hour { t.Fatalf("first-sighting window = %v; want fail-closed 72h baseline", holds["production"]) } if holds["edge"] != 0 { t.Fatalf("edge window = %v; want 0 (legacy baseline)", holds["edge"]) } // After the old window elapses on the box's own clock, the pending 24h lands. later := now.Add(72*time.Hour + time.Second) f.ObservePipeline("bae", "web", pipe("24h", now), later) if h := f.HoldsForApp("bae", "web", pipe("24h", now)); h["production"] != 24*time.Hour { t.Fatalf("post-ratchet window = %v; want 24h", h["production"]) } // Tightening is immediate. f.ObservePipeline("bae", "web", pipe("96h", now.Add(80*time.Hour)), later.Add(time.Minute)) if h := f.HoldsForApp("bae", "web", nil); h != nil { t.Fatalf("HoldsForApp(nil pipeline) = %v; want nil", h) } if h := f.HoldsForApp("bae", "web", pipe("96h", now)); h["production"] != 96*time.Hour { t.Fatalf("tighten = %v; want 96h immediately", h["production"]) } } func TestHoldMemoryReplayFloorAndReReadKeepsClock(t *testing.T) { f := &FirstSeen{Jitter: 0} now := time.Date(2026, 8, 20, 12, 0, 0, 0, time.UTC) p1 := &Pipeline{Schema: 1, Channels: []PipelineChannel{{Name: "stable", Bake: "1h"}}, SetAt: now} f.ObservePipeline("bae", "web", p1, now) // Re-reading the same object an hour later must NOT restart the pending clock… f.ObservePipeline("bae", "web", p1, now.Add(time.Hour)) // …so the loosening lands 72h after the FIRST sighting, not the second. f.ObservePipeline("bae", "web", p1, now.Add(72*time.Hour+time.Second)) if h := f.HoldsForApp("bae", "web", p1); h["stable"] != time.Hour { t.Fatalf("window = %v; want 1h landed 72h after first sighting", h["stable"]) } // A replayed OLDER SetAt can never loosen further. old := &Pipeline{Schema: 1, Channels: []PipelineChannel{{Name: "stable", Bake: "0s"}}, SetAt: now.Add(-time.Hour)} f.ObservePipeline("bae", "web", old, now.Add(73*time.Hour)) if h := f.HoldsForApp("bae", "web", p1); h["stable"] != time.Hour { t.Fatalf("replayed old object loosened the window to %v", h["stable"]) } } func TestHoldMemorySurvivesSaveLoadAndOldBinaryDrop(t *testing.T) { // Additive-field discipline: marshal a FirstSeen with Holds, unmarshal, // memory intact; unmarshal a PRE-CHANGE JSON (no holds key) into the new // struct — no error, nil map, and ObservePipeline re-latches from scratch // (the closed direction: a restarted clock only ever delays loosening). f := &FirstSeen{Jitter: 0, Holds: map[string]*holdMemory{ channelClockKey("bae", "web", "production"): {Window: 24 * time.Hour, SetAt: time.Now().UTC()}}} b, err := json.Marshal(f) if err != nil { t.Fatal(err) } var back FirstSeen if err := json.Unmarshal(b, &back); err != nil { t.Fatal(err) } if back.Holds[channelClockKey("bae", "web", "production")].Window != 24*time.Hour { t.Fatal("holds memory lost across marshal round trip") } var legacy FirstSeen if err := json.Unmarshal([]byte(`{"jitter":0,"channels":{}}`), &legacy); err != nil || legacy.Holds != nil { t.Fatalf("legacy clock parse = %v, holds %v", err, legacy.Holds) } } // TestHoldMemoryZeroBakeRatchetsIn is the review fix for the critical bug: // Pending's zero value is indistinguishable from "nothing pending", and a // declared bake of exactly ZERO is a legal value — without PendingAt as // its own sentinel, a brand-new channel with no bake declared would latch // its 72h fail-closed baseline and NEVER ratchet down, because // mem.Pending (freshly zeroed) already equals w=0 on the very first call. func TestHoldMemoryZeroBakeRatchetsIn(t *testing.T) { f := &FirstSeen{Jitter: 0} now := time.Date(2026, 8, 20, 12, 0, 0, 0, time.UTC) pipe := func(setAt time.Time) *Pipeline { return &Pipeline{Schema: 1, Channels: []PipelineChannel{ {Name: "dev"}, {Name: "live"}}, SetAt: setAt} } f.ObservePipeline("bae", "web", pipe(now), now) if h := f.HoldsForApp("bae", "web", pipe(now)); h["dev"] != 72*time.Hour || h["live"] != 72*time.Hour { t.Fatalf("first-sighting windows = %v; want fail-closed 72h baseline on both channels", h) } later := now.Add(72*time.Hour + time.Second) if !f.ObservePipeline("bae", "web", pipe(now), later) { t.Fatal("re-observe past the elapsed baseline did not report changed") } h := f.HoldsForApp("bae", "web", pipe(now)) if h["dev"] != 0 || h["live"] != 0 { t.Fatalf("post-ratchet windows = %v; want 0 (declared zero bake) on both channels", h) } } // TestHoldsForAppBeforeObserveReturnsBaselines is the review fix for // HoldsForApp's fallback-agreement claim: before any ObservePipeline call, // every declared channel must still report the exact baseline // ObservePipeline would have latched on first sighting — including a rung // name like "edge" whose baseline is 0, which HoldFor's own unknown-channel // fallback (72h) would silently get wrong if left out of the map. func TestHoldsForAppBeforeObserveReturnsBaselines(t *testing.T) { f := &FirstSeen{Jitter: 0} p := &Pipeline{Schema: 1, Channels: []PipelineChannel{ {Name: "edge"}, {Name: "custom"}}, SetAt: time.Now()} h := f.HoldsForApp("bae", "web", p) if h["edge"] != 0 { t.Fatalf("edge baseline = %v; want 0 (rung default) before any observation", h["edge"]) } if h["custom"] != 72*time.Hour { t.Fatalf("custom baseline = %v; want 72h fail-closed before any observation", h["custom"]) } } // TestObservePipelineChangedContract pins the `changed` return value — // PassFilter's sole trigger for a changed-gated Save — across the cases // that matter: a genuine first sighting, a replay that changes nothing, a // pending promotion actually landing, and p == nil with nothing to // promote. func TestObservePipelineChangedContract(t *testing.T) { now := time.Date(2026, 8, 20, 12, 0, 0, 0, time.UTC) t.Run("first sighting reports changed", func(t *testing.T) { f := &FirstSeen{Jitter: 0} p := &Pipeline{Schema: 1, Channels: []PipelineChannel{{Name: "stable", Bake: "96h"}}, SetAt: now} if !f.ObservePipeline("bae", "web", p, now) { t.Fatal("first sighting did not report changed") } }) t.Run("replay of the same object with nothing pending reports no change", func(t *testing.T) { f := &FirstSeen{Jitter: 0} // 96h >= the 72h baseline: tightens immediately, no pending created. p := &Pipeline{Schema: 1, Channels: []PipelineChannel{{Name: "stable", Bake: "96h"}}, SetAt: now} f.ObservePipeline("bae", "web", p, now) if f.ObservePipeline("bae", "web", p, now.Add(time.Hour)) { t.Fatal("replay of the identical object (nothing pending) reported changed") } }) t.Run("pending promotion firing reports changed", func(t *testing.T) { f := &FirstSeen{Jitter: 0} // 1h < the 72h baseline: goes pending at the first sighting. p := &Pipeline{Schema: 1, Channels: []PipelineChannel{{Name: "stable", Bake: "1h"}}, SetAt: now} f.ObservePipeline("bae", "web", p, now) if !f.ObservePipeline("bae", "web", p, now.Add(72*time.Hour+time.Second)) { t.Fatal("pending promotion elapsing did not report changed") } }) t.Run("p == nil stamps the deletion once, then reports no change", func(t *testing.T) { f := &FirstSeen{Jitter: 0} // 72h == stable's legacy default: tightens to exactly the baseline, // nothing pending — so absence has no WINDOW to ratchet down. The // FIRST absence is still a change: it stamps DeletedAt, which is the // box's own record that this app was pipelined until just now, and // which the adopt pass's bake verdict rides on for one cycle (PR #258 // review, C1 — for an app-defined channel name the legacy default is // LONGER than the enforced window, so deletion lands instantly and // this stamp is the only evidence left). A stamp that is not // persisted is no evidence at all, so it must report changed. p := &Pipeline{Schema: 1, Channels: []PipelineChannel{{Name: "stable", Bake: "72h"}}, SetAt: now} f.ObservePipeline("bae", "web", p, now) if !f.ObservePipeline("bae", "web", nil, now.Add(time.Hour)) { t.Fatal("the first p == nil did not report changed — the deletion stamp would never be saved") } // Every pass after it is the steady state a fleet actually sits in, // and must not force a Save: re-observed absence changes nothing. if f.ObservePipeline("bae", "web", nil, now.Add(2*time.Hour)) { t.Fatal("re-observed absence reported changed — the stamp must not be restamped") } }) t.Run("p == nil with no memory at all reports no change", func(t *testing.T) { // Every legacy app on every pass lands here — the byte-identical // no-behaviour-change case the deletion ratchet must not disturb. f := &FirstSeen{Jitter: 0} if f.ObservePipeline("bae", "web", nil, now) { t.Fatal("p == nil with no memory reported changed") } if len(f.Holds) != 0 { t.Fatalf("p == nil minted memory out of nothing: %v", f.Holds) } }) } // TestHoldMemoryDeletionRatchetsDownToLegacy: deleting pipeline.json is a // bucket write, no signing key required — so absence must be treated as a // POLICY DECLARATION of the legacy defaults, ratcheted like any other: a // channel whose legacy default is narrower than what's enforced goes pending // for one old-window cycle; deletion never collapses a window instantly, and // never sticks forever. Re-observed absence must not restart the clock, and // absence carries no SetAt, so the remembered replay floor stays untouched. func TestHoldMemoryDeletionRatchetsDownToLegacy(t *testing.T) { f := &FirstSeen{Jitter: 0} now := time.Date(2026, 8, 22, 12, 0, 0, 0, time.UTC) p := &Pipeline{Schema: 1, Channels: []PipelineChannel{{Name: "edge", Bake: "24h"}}, SetAt: now} f.ObservePipeline("bae", "web", p, now) key := channelClockKey("bae", "web", "edge") if f.Holds[key].Window != 24*time.Hour { t.Fatalf("window = %v, want 24h tightened immediately", f.Holds[key].Window) } setAt := f.Holds[key].SetAt // The deletion: edge's legacy default (0) is narrower than the enforced // 24h, so it goes PENDING — the 24h stays enforced for one full cycle. del := now.Add(time.Hour) if !f.ObservePipeline("bae", "web", nil, del) { t.Fatal("first observation of absence did not report changed — the pending clock would never persist") } mem := f.Holds[key] if mem.Window != 24*time.Hour { t.Fatalf("window = %v the pass after deletion, want the ratcheted 24h still enforced", mem.Window) } if mem.Pending != 0 || !mem.PendingAt.Equal(del) { t.Fatalf("pending = %v at %v, want the legacy 0 pending from the deletion sighting %v", mem.Pending, mem.PendingAt, del) } if !mem.SetAt.Equal(setAt) { t.Fatalf("SetAt = %v after absence, want %v untouched — absence carries no SetAt", mem.SetAt, setAt) } // Re-observation of absence must NOT restart the clock — same rule as a // re-read of the same object. if f.ObservePipeline("bae", "web", nil, del.Add(time.Hour)) { t.Fatal("re-observation of absence reported changed") } if !f.Holds[key].PendingAt.Equal(del) { t.Fatalf("re-observation restarted the pending clock: %v, want %v", f.Holds[key].PendingAt, del) } // After the OLD window elapses on the box's own clock, legacy lands. if !f.ObservePipeline("bae", "web", nil, del.Add(24*time.Hour+time.Second)) { t.Fatal("the pending legacy window elapsing did not report changed") } if f.Holds[key].Window != 0 { t.Fatalf("window = %v after the old window elapsed, want edge's legacy 0", f.Holds[key].Window) } // Quiescent thereafter — nothing left to change, so the changed-gated // Save is not forced every pass forever. if f.ObservePipeline("bae", "web", nil, del.Add(48*time.Hour)) { t.Fatal("absence at the legacy windows reported changed") } // And a later replayed pipeline OLDER than the remembered SetAt still // cannot tick the memory — deletion did not soften the replay floor. old := &Pipeline{Schema: 1, Channels: []PipelineChannel{{Name: "edge", Bake: "1h"}}, SetAt: setAt.Add(-time.Hour)} f.ObservePipeline("bae", "web", old, del.Add(49*time.Hour)) if f.Holds[key].Window != 0 || !f.Holds[key].PendingAt.IsZero() { t.Fatalf("replayed older pipeline ticked the memory after deletion: %+v", f.Holds[key]) } } // TestHoldMemoryDeletionTightensBelowLegacyImmediately: a channel whose // enforced window sits BELOW its legacy default (a fully ratcheted-in short // bake) tightens back to the legacy default the moment the pipeline is // deleted — widening is always immediate, exactly as for a declared window. func TestHoldMemoryDeletionTightensBelowLegacyImmediately(t *testing.T) { f := &FirstSeen{Jitter: 0} now := time.Date(2026, 8, 22, 12, 0, 0, 0, time.UTC) key := channelClockKey("bae", "web", "stable") f.Holds = map[string]*holdMemory{key: {Window: time.Hour, SetAt: now}} if !f.ObservePipeline("bae", "web", nil, now.Add(time.Minute)) { t.Fatal("deletion tightening did not report changed") } mem := f.Holds[key] if mem.Window != 72*time.Hour { t.Fatalf("window = %v after deletion, want stable's fail-closed 72h immediately", mem.Window) } if mem.Pending != 0 || !mem.PendingAt.IsZero() { t.Fatalf("pending = %v at %v after a tighten, want cleared", mem.Pending, mem.PendingAt) } } // TestLiveRatchetIsWiderThanARaisedWindow pins the distinction PR #258's // review turned on (C1): "is this box still running a pipeline ratchet for // this channel" is NOT "does the memory raise the window above legacy". An // app-defined name resolves to HoldFor's 72h fallback, so an ordinary // production channel enforcing 24h raises nothing while being fully // pipelined — and after a deletion its window lands on that same 72h // instantly, leaving a memory that looks exactly like a legacy app's. The // DeletedAt stamp is what carries the verdict across that gap, and it runs // out. func TestLiveRatchetIsWiderThanARaisedWindow(t *testing.T) { now := time.Date(2026, 8, 22, 12, 0, 0, 0, time.UTC) legacyProduction := 72 * time.Hour // HoldFor's unknown-name fallback // Ratcheted DOWN below legacy: raises nothing, still live. mem := &holdMemory{Window: 24 * time.Hour, SetAt: now} if !mem.liveRatchet("production", 0, now) { t.Error("a window ratcheted BELOW legacy did not read as a live ratchet — this is the C1 collapse") } // Raised above legacy: live, the case that always worked. if !(&holdMemory{Window: 24 * time.Hour}).liveRatchet("edge", 0, now) { t.Error("a window raised above legacy did not read as a live ratchet") } // Exactly legacy with a pending loosening: live. if !(&holdMemory{Window: legacyProduction, Pending: time.Hour, PendingAt: now}).liveRatchet("production", 0, now) { t.Error("a pending loosening did not read as a live ratchet") } // Exactly legacy, nothing pending, never deleted: a legacy app's memory. if (&holdMemory{Window: legacyProduction, SetAt: now}).liveRatchet("production", 0, now) { t.Error("a memory sitting at legacy with nothing in flight read as a live ratchet") } // Deleted, cycle still running / run out. deleted := &holdMemory{Window: legacyProduction, SetAt: now, DeletedAt: now} if !deleted.liveRatchet("production", 0, now.Add(71*time.Hour)) { t.Error("a deletion inside its own cycle did not read as a live ratchet") } if deleted.liveRatchet("production", 0, now.Add(73*time.Hour)) { t.Error("a deletion whose cycle has run out still read as live — the app must return to legacy") } // nil is never live (the {"holds":{"x":null}} shape). if (*holdMemory)(nil).liveRatchet("production", 0, now) { t.Error("a nil memory read as a live ratchet") } } // TestHoldMemoryDeletionStampClearedByAVerifiedPipeline: the stamp says "this // app was pipelined until just now". A verified pipeline naming the channel // says it IS pipelined, on the object's own word, so the stamp has no job // left — and must not go on ticking a cycle that would later expire under a // live pipeline. func TestHoldMemoryDeletionStampClearedByAVerifiedPipeline(t *testing.T) { f := &FirstSeen{Jitter: 0} now := time.Date(2026, 8, 22, 12, 0, 0, 0, time.UTC) key := channelClockKey("bae", "web", "production") p := &Pipeline{Schema: 1, Channels: []PipelineChannel{{Name: "production", Bake: "24h"}}, SetAt: now} f.ObservePipeline("bae", "web", p, now) f.ObservePipeline("bae", "web", nil, now.Add(time.Hour)) if f.Holds[key].DeletedAt.IsZero() { t.Fatal("absence did not stamp DeletedAt") } stamp := f.Holds[key].DeletedAt // Re-observed absence must not restamp it. f.ObservePipeline("bae", "web", nil, now.Add(2*time.Hour)) if !f.Holds[key].DeletedAt.Equal(stamp) { t.Fatalf("re-observed absence restamped DeletedAt: %v, want %v", f.Holds[key].DeletedAt, stamp) } // The object comes back — same SetAt, the byte-identical restore. if !f.ObservePipeline("bae", "web", p, now.Add(3*time.Hour)) { t.Fatal("a restored pipeline that clears the deletion stamp did not report changed") } if !f.Holds[key].DeletedAt.IsZero() { t.Fatalf("DeletedAt = %v after the pipeline was restored, want cleared", f.Holds[key].DeletedAt) } } // TestHoldMemoryRestoredPipelineCancelsAbsencePending: the re-review probe. // Delete the pipeline (absence stamps the legacy default pending), then // restore the object BYTE-IDENTICALLY — same SetAt: a bucket-versioning // restore, an undo, a replay. The replay floor rightly skips the policy // block, but the absence-stamped pending must not survive a live verified // pipeline declaring more: unlanded it is cancelled, and one old-window // cycle later the enforced window is still the declared 24h, never the // legacy 0 the deletion briefly put in flight. func TestHoldMemoryRestoredPipelineCancelsAbsencePending(t *testing.T) { f := &FirstSeen{Jitter: 0} now := time.Date(2026, 8, 22, 12, 0, 0, 0, time.UTC) p := &Pipeline{Schema: 1, Channels: []PipelineChannel{{Name: "edge", Bake: "24h"}}, SetAt: now} f.ObservePipeline("bae", "web", p, now) key := channelClockKey("bae", "web", "edge") // The deletion: legacy 0 goes pending under the enforced 24h. del := now.Add(time.Hour) f.ObservePipeline("bae", "web", nil, del) if mem := f.Holds[key]; mem.Pending != 0 || !mem.PendingAt.Equal(del) { t.Fatalf("absence did not stamp the legacy pending: %+v", mem) } // The restore, byte-identical (same SetAt). The pending is no longer // the live policy — it must be cancelled, and that is a change. if !f.ObservePipeline("bae", "web", p, del.Add(time.Hour)) { t.Fatal("cancelling the absence pending did not report changed — it would never persist") } if mem := f.Holds[key]; mem.Window != 24*time.Hour || mem.Pending != 0 || !mem.PendingAt.IsZero() { t.Fatalf("memory after the restore = %+v, want 24h enforced with nothing pending", mem) } // Past the old window, still observing the restored pipeline: the // declared 24h holds — the legacy 0 never lands. if f.ObservePipeline("bae", "web", p, del.Add(25*time.Hour)) { t.Fatal("a quiescent restored pipeline reported changed") } if mem := f.Holds[key]; mem.Window != 24*time.Hour || !mem.PendingAt.IsZero() { t.Fatalf("memory past the old window = %+v, want the declared 24h still enforced", mem) } } // TestObservePipelineNullHoldsEntryDoesNotPanic: {"holds":{"x":null}} parses // cleanly into a nil *holdMemory. Every loop over Holds must skip it, never // dereference it — a panic here takes down the adopt pass fleet-wide. func TestObservePipelineNullHoldsEntryDoesNotPanic(t *testing.T) { path := filepath.Join(t.TempDir(), "clock.json") blob := `{"jitter":0,"channels":{},"holds":{"` + channelClockKey("bae", "web", "edge") + `":null}}` if err := os.WriteFile(path, []byte(blob), 0o644); err != nil { t.Fatal(err) } f := LoadFirstSeen(path) if f.tainted { t.Fatal("a null holds entry is legal JSON, not a tainted clock") } now := time.Date(2026, 8, 22, 12, 0, 0, 0, time.UTC) f.ObservePipeline("bae", "web", nil, now) // must not panic p := &Pipeline{Schema: 1, Channels: []PipelineChannel{{Name: "edge", Bake: "24h"}}, SetAt: now} f.ObservePipeline("bae", "web", p, now) // must not panic; re-latches over the null if f.Holds[channelClockKey("bae", "web", "edge")].Window != 24*time.Hour { t.Fatal("observation over a null entry did not re-latch the memory") } }