package release import ( "bytes" "context" "crypto/ed25519" "encoding/base64" "encoding/json" "errors" "os" "path/filepath" "slices" "strconv" "strings" "testing" "time" "github.com/carlosframework/platform/internal/caps" "github.com/carlosframework/platform/internal/registry" ) func testRegistry(t *testing.T) *registry.Registry { t.Helper() reg, err := registry.Open(filepath.Join(t.TempDir(), "registry.db")) if err != nil { t.Fatalf("open registry: %v", err) } return reg } func addRoute(t *testing.T, reg *registry.Registry, host, app, channel string, hibernate bool) { t.Helper() if err := reg.Upsert(registry.Route{ Host: host, Kind: "instance", App: app, Channel: channel, Hibernate: hibernate, }); err != nil { t.Fatalf("upsert %s: %v", host, err) } } func TestPassRepointsAwakeAndHibernating(t *testing.T) { s := NewDirStore(t.TempDir()) ctx := context.Background() priv, pub := testKeys(t) opt := t.TempDir() reg := testRegistry(t) // canary/* has no bake window, so the pass applies immediately. shipRelease(t, s, "bab", "myapp", "v1", adoptPlatform, []byte("build one")) if err := Promote(ctx, s, priv, "bab", "myapp", "canary/x", "v1"); err != nil { t.Fatalf("Promote: %v", err) } addRoute(t, reg, "awake.example.com", "myapp", "canary/x", false) addRoute(t, reg, "sleepy.example.com", "myapp", "canary/x", true) var restarted []string restart := func(host string) error { restarted = append(restarted, host); return nil } if err := Pass(ctx, reg, s, pub, opt, restart); err != nil { t.Fatalf("Pass: %v", err) } for _, host := range []string{"awake.example.com", "sleepy.example.com"} { if v, _ := LiveVersion(opt, host); v != "v1" { t.Fatalf("%s live = %q, want v1", host, v) } rt, err := reg.Get(host) if err != nil || rt == nil || rt.Version != "v1" { t.Fatalf("%s registry version = %+v, %v", host, rt, err) } } // Only the awake host was restarted; the hibernating host is NEVER woken // — its next wake execs the new binary via the live symlink. if len(restarted) != 1 || restarted[0] != "awake.example.com" { t.Fatalf("restarted = %v, want only awake.example.com", restarted) } // The staged artifact matches the shipped bytes (sha256-gated download). body, err := os.ReadFile(VersionPath(opt, "bab", "v1", "myapp")) if err != nil || string(body) != "build one" { t.Fatalf("staged bytes = %q, %v", body, err) } } // TestPassRestartHibernatingOptIn pins both sides of the RestartHibernating // door. Default (RunAdopt's shape, no option): hibernate-flagged routes get // the pure repoint and the callback never sees them — byte-identical to the // behavior TestPassRepointsAwakeAndHibernating above pins. Opted in (the // agent's in-process loops): the callback is consulted for sleepers too, so a // WARM exec child — minted Hibernate:true, but running right now — can be // cycled onto the binary this pass just pointed it at. // // The pass itself still starts nothing either way: all the option changes is // whether the callback is asked. Not waking a sleeper is the CALLBACK's // promise (Bounce returns false when nothing is awake; try-restart is a no-op // on a stopped unit), which is why handing it a sleeping host is safe. func TestPassRestartHibernatingOptIn(t *testing.T) { for _, tc := range []struct { name string optIn bool want []string reason string }{ {"default: sleepers are not offered to the callback", false, []string{"awake.example.com"}, "the root adopt pass has no children of its own to cycle"}, {"opted in: sleepers are offered too", true, []string{"awake.example.com", "sleepy.example.com"}, "a warm exec child must bounce onto its new binary"}, } { t.Run(tc.name, func(t *testing.T) { s := NewDirStore(t.TempDir()) ctx := context.Background() priv, pub := testKeys(t) opt := t.TempDir() reg := testRegistry(t) shipRelease(t, s, "bab", "myapp", "v1", adoptPlatform, []byte("build one")) if err := Promote(ctx, s, priv, "bab", "myapp", "canary/x", "v1"); err != nil { t.Fatalf("Promote: %v", err) } addRoute(t, reg, "awake.example.com", "myapp", "canary/x", false) addRoute(t, reg, "sleepy.example.com", "myapp", "canary/x", true) var restarted []string restart := func(host string) error { restarted = append(restarted, host); return nil } var opts []PassOption if tc.optIn { opts = append(opts, RestartHibernating(restart)) } if err := Pass(ctx, reg, s, pub, opt, restart, opts...); err != nil { t.Fatalf("Pass: %v", err) } if strings.Join(restarted, ",") != strings.Join(tc.want, ",") { t.Fatalf("restarted = %v, want %v (%s)", restarted, tc.want, tc.reason) } // Either way the repoint happened for both — the option changes who // is offered to the callback, never which hosts adopt. for _, host := range []string{"awake.example.com", "sleepy.example.com"} { if v, _ := LiveVersion(opt, host); v != "v1" { t.Fatalf("%s live = %q, want v1", host, v) } } }) } } // TestPassRestartHibernatingKeepsSleepersLast pins the ordering the option // must not disturb: awake hosts are still adopted BEFORE any sleeper, so an // awake host can discover a boot-broken build and poison it before a // hibernating host is moved onto it. Handing sleepers a callback must not // promote them into the first loop. func TestPassRestartHibernatingKeepsSleepersLast(t *testing.T) { s := NewDirStore(t.TempDir()) ctx := context.Background() priv, pub := testKeys(t) opt := t.TempDir() reg := testRegistry(t) shipRelease(t, s, "bab", "myapp", "v1", adoptPlatform, []byte("build one")) if err := Promote(ctx, s, priv, "bab", "myapp", "canary/x", "v1"); err != nil { t.Fatalf("Promote: %v", err) } // Named so a plain registry ordering (alphabetical by host) would put the // sleeper first — only the awake/hibernating split can produce the order // this test wants. addRoute(t, reg, "aaa-sleepy.example.com", "myapp", "canary/x", true) addRoute(t, reg, "zzz-awake.example.com", "myapp", "canary/x", false) var order []string restart := func(host string) error { order = append(order, host); return nil } if err := Pass(ctx, reg, s, pub, opt, restart, RestartHibernating(restart)); err != nil { t.Fatalf("Pass: %v", err) } want := []string{"zzz-awake.example.com", "aaa-sleepy.example.com"} if strings.Join(order, ",") != strings.Join(want, ",") { t.Fatalf("restart order = %v, want %v (awake hosts must still go first)", order, want) } } // TestPassRestartHibernatingPoisonsLikeAnyOtherHost: a sleeper's failed // restart takes the same rollback the awake path takes — the symlink goes // back and the version is poisoned for the rest of the pass. Nothing about // hibernation makes a broken build safe to leave pointed at. func TestPassRestartHibernatingPoisonsLikeAnyOtherHost(t *testing.T) { s := NewDirStore(t.TempDir()) ctx := context.Background() priv, pub := testKeys(t) opt := t.TempDir() reg := testRegistry(t) shipRelease(t, s, "bab", "myapp", "v1", adoptPlatform, []byte("build one")) shipRelease(t, s, "bab", "myapp", "v2", adoptPlatform, []byte("build two")) if err := Promote(ctx, s, priv, "bab", "myapp", "canary/x", "v1"); err != nil { t.Fatalf("Promote v1: %v", err) } addRoute(t, reg, "sleepy.example.com", "myapp", "canary/x", true) ok := func(string) error { return nil } if err := Pass(ctx, reg, s, pub, opt, ok, RestartHibernating(ok)); err != nil { t.Fatalf("Pass v1: %v", err) } if err := Promote(ctx, s, priv, "bab", "myapp", "canary/x", "v2"); err != nil { t.Fatalf("Promote v2: %v", err) } boom := func(string) error { return errors.New("bounce failed") } if err := Pass(ctx, reg, s, pub, opt, boom, RestartHibernating(boom)); err != nil { t.Fatalf("Pass v2: %v", err) } if v, _ := LiveVersion(opt, "sleepy.example.com"); v != "v1" { t.Fatalf("live = %q, want the symlink rolled back to v1", v) } } // TestPassBakeDirMovesTheClock proves the BakeDir option's whole point // (issue #13): the bake clock lands under the given dir, not optDir's // legacy /opt-dir location. func TestPassBakeDirMovesTheClock(t *testing.T) { s := NewDirStore(t.TempDir()) ctx := context.Background() priv, pub := testKeys(t) opt := t.TempDir() bakeDir := t.TempDir() reg := testRegistry(t) shipRelease(t, s, "bab", "myapp", "v1", adoptPlatform, []byte("build one")) if err := Promote(ctx, s, priv, "bab", "myapp", "canary/x", "v1"); err != nil { t.Fatalf("Promote: %v", err) } addRoute(t, reg, "awake.example.com", "myapp", "canary/x", false) if err := Pass(ctx, reg, s, pub, opt, func(string) error { return nil }, BakeDir(bakeDir)); err != nil { t.Fatalf("Pass: %v", err) } if _, err := os.Stat(filepath.Join(bakeDir, "channel-first-seen.json")); err != nil { t.Fatalf("BakeDir option did not land the clock in bakeDir: %v", err) } if _, err := os.Stat(FirstSeenPath(opt)); !os.IsNotExist(err) { t.Fatalf("bake clock also written to the legacy opt dir despite BakeDir (stat err %v)", err) } } // TestPassMigratesBakeClockDurably proves the migration off the legacy path // completes in ONE pass even when nothing new was observed (2026-08-03 // review, IMPORTANT): Save is changed-gated, so with the head already // recorded in the legacy clock, Observe reports no change this pass and the // ordinary "tick and save" path would never write the new location — a box // could run for days with the new path absent, and deleting the legacy file // in that window would silently reset the channel's bake window. The legacy // clock here already has this exact head, so the ONLY way the new path ends // up populated is the migration forcing its own first Save. func TestPassMigratesBakeClockDurably(t *testing.T) { s := NewDirStore(t.TempDir()) ctx := context.Background() priv, pub := testKeys(t) opt := t.TempDir() bakeDir := t.TempDir() reg := testRegistry(t) shipRelease(t, s, "bab", "myapp", "v1", adoptPlatform, []byte("build one")) if err := Promote(ctx, s, priv, "bab", "myapp", "canary/x", "v1"); err != nil { t.Fatalf("Promote: %v", err) } addRoute(t, reg, "awake.example.com", "myapp", "canary/x", false) legacy := LoadFirstSeen(FirstSeenPath(opt)) legacy.Observe("bab", "myapp", "canary/x", "v1", time.Now()) if err := legacy.Save(); err != nil { t.Fatalf("seed legacy clock: %v", err) } if err := Pass(ctx, reg, s, pub, opt, func(string) error { return nil }, BakeDir(bakeDir)); err != nil { t.Fatalf("Pass: %v", err) } st := LoadFirstSeen(filepath.Join(bakeDir, "channel-first-seen.json")) if _, ok := st.EligibleAt("bab", "myapp", "canary/x", "v1", DefaultHolds); !ok { t.Fatal("migrated content did not land in the new path after one pass with no new head observed") } } func TestPassPoisonedBuildSkipsRemainingHosts(t *testing.T) { s := NewDirStore(t.TempDir()) ctx := context.Background() priv, pub := testKeys(t) opt := t.TempDir() reg := testRegistry(t) // Both hosts start on v1 (staged locally), channel head moves to v2. shipRelease(t, s, "bab", "myapp", "v1", adoptPlatform, []byte("good")) shipRelease(t, s, "bab", "myapp", "v2", adoptPlatform, []byte("bad")) if err := StageArtifact(opt, "bab", "v1", "myapp", []byte("good")); err != nil { t.Fatal(err) } for _, h := range []string{"a.example.com", "b.example.com"} { addRoute(t, reg, h, "myapp", "canary/x", false) if err := PointLive(opt, h, "bab", "v1", "myapp"); err != nil { t.Fatal(err) } } if err := Promote(ctx, s, priv, "bab", "myapp", "canary/x", "v2"); err != nil { t.Fatalf("Promote v2: %v", err) } // v2 fails its restart on the first host — it must be poisoned for the // rest of the pass, never cycling every host in turn. var calls int restart := func(host string) error { calls++; return errors.New("unit failed") } if err := Pass(ctx, reg, s, pub, opt, restart); err != nil { t.Fatalf("Pass: %v", err) } if calls != 1 { t.Fatalf("restart called %d times, want 1 (poisoned build must not bounce every host)", calls) } // The failed host was rolled back to what it ran; the second never moved. for _, h := range []string{"a.example.com", "b.example.com"} { if v, _ := LiveVersion(opt, h); v != "v1" { t.Fatalf("%s live = %q, want v1", h, v) } } } func TestPassStableHeldByBakeWindow(t *testing.T) { s := NewDirStore(t.TempDir()) ctx := context.Background() priv, pub := testKeys(t) opt := t.TempDir() reg := testRegistry(t) shipRelease(t, s, "bab", "myapp", "v1", adoptPlatform, []byte("bin")) if err := Promote(ctx, s, priv, "bab", "myapp", "stable", "v1"); err != nil { t.Fatalf("Promote: %v", err) } // The app is DECLARED production (2026-08-04 spec §2): the window is no // longer unconditional — it is what the flag buys. if err := SetProduction(ctx, s, priv, "bab", "myapp", true, "test"); err != nil { t.Fatalf("SetProduction: %v", err) } addRoute(t, reg, "h.example.com", "myapp", "", false) // "" = stable if err := Pass(ctx, reg, s, pub, opt, func(string) error { return nil }); err != nil { t.Fatalf("Pass: %v", err) } // A brand-new stable head is inside its 72h supply-chain window: the // clock started this pass, nothing installs. if v, _ := LiveVersion(opt, "h.example.com"); v != "" { t.Fatalf("live = %q, want unmoved (bake hold)", v) } // The pass persisted the first sighting so the window is measured from // the box's own clock. st := LoadFirstSeen(FirstSeenPath(opt)) if _, ok := st.EligibleAt("bab", "myapp", "stable", "v1", DefaultHolds); !ok { t.Fatalf("Pass did not tick the first-seen clock") } } func TestPassSkipsUnadoptableRoutes(t *testing.T) { s := NewDirStore(t.TempDir()) ctx := context.Background() priv, pub := testKeys(t) opt := t.TempDir() reg := testRegistry(t) shipRelease(t, s, "bab", "myapp", "v1", adoptPlatform, []byte("bin")) if err := Promote(ctx, s, priv, "bab", "myapp", "canary/x", "v1"); err != nil { t.Fatalf("Promote: %v", err) } // A static route, an app-less instance, and a mid-provision instance: // none may be moved by an adoption pass. if err := reg.Upsert(registry.Route{Host: "site.example.com", Kind: "static", App: "myapp", Channel: "canary/x"}); err != nil { t.Fatal(err) } addRoute(t, reg, "noapp.example.com", "", "canary/x", false) if err := reg.Upsert(registry.Route{Host: "prov.example.com", Kind: "instance", App: "myapp", Channel: "canary/x", Status: "creating"}); err != nil { t.Fatal(err) } if err := Pass(ctx, reg, s, pub, opt, func(string) error { return nil }); err != nil { t.Fatalf("Pass: %v", err) } for _, h := range []string{"site.example.com", "noapp.example.com", "prov.example.com"} { if v, _ := LiveVersion(opt, h); v != "" { t.Fatalf("%s was moved (%q) but is not adoptable", h, v) } } } // TestPassSkipsPlatformMintedRoutes: a platform domain is an ALIAS for an // app's canonical route, not a lifecycle host of its own. Adopting one would // stage a second copy of the artifacts, mint a live/ symlink no unit // ever execs, try-restart a carlos-app@ unit that doesn't exist, and // stamp Version onto a row the console then renders as if it were a real // instance. The canonical route it aliases is adopted normally — that IS the // deploy. func TestPassSkipsPlatformMintedRoutes(t *testing.T) { s := NewDirStore(t.TempDir()) ctx := context.Background() priv, pub := testKeys(t) opt := t.TempDir() reg := testRegistry(t) shipRelease(t, s, "bab", "myapp", "v1", adoptPlatform, []byte("bin")) if err := Promote(ctx, s, priv, "bab", "myapp", "canary/x", "v1"); err != nil { t.Fatalf("Promote: %v", err) } // The canonical route and its platform alias, identical but for the flag. addRoute(t, reg, "myapp.example.com", "myapp", "canary/x", false) if err := reg.Upsert(registry.Route{Host: "myapp.bab.oncarlos.test", Kind: "instance", App: "myapp", Channel: "canary/x", Status: "ready", Platform: true}); err != nil { t.Fatal(err) } var restarted []string if err := Pass(ctx, reg, s, pub, opt, func(h string) error { restarted = append(restarted, h) return nil }); err != nil { t.Fatalf("Pass: %v", err) } if v, _ := LiveVersion(opt, "myapp.bab.oncarlos.test"); v != "" { t.Errorf("platform alias was adopted (live = %q) — aliases are not lifecycle hosts", v) } for _, h := range restarted { if h == "myapp.bab.oncarlos.test" { t.Error("adopt try-restarted a platform alias's nonexistent unit") } } if rt, _ := reg.Get("myapp.bab.oncarlos.test"); rt != nil && rt.Version != "" { t.Errorf("adopt stamped Version %q onto a platform alias row", rt.Version) } // The canonical route still adopts: the alias must not have stolen its // pass (canaries bypass bake, so this lands immediately). if v, _ := LiveVersion(opt, "myapp.example.com"); v != "v1" { t.Errorf("canonical route live = %q, want v1 — skipping aliases must not skip real hosts", v) } } // The test that proves per-account app names actually work end to end: two // routes, the SAME app name, different accounts, different channel pointers. // Each host must adopt its own account's version. Before accounts, adopt // grouped by (app, channel) and would have installed one of them on both. func TestPassResolvesPerAccount(t *testing.T) { s := NewDirStore(t.TempDir()) ctx := context.Background() priv, pub := testKeys(t) opt := t.TempDir() reg := testRegistry(t) shipRelease(t, s, "bab", "blog", "v1", adoptPlatform, []byte("ops build")) shipRelease(t, s, "kqmv", "blog", "v2", adoptPlatform, []byte("kass build")) if err := Promote(ctx, s, priv, "bab", "blog", "canary/x", "v1"); err != nil { t.Fatal(err) } if err := Promote(ctx, s, priv, "kqmv", "blog", "canary/x", "v2"); err != nil { t.Fatal(err) } for _, rt := range []registry.Route{ {Host: "ops.example.com", Kind: "instance", App: "blog", Channel: "canary/x", Account: "bab", Status: "ready"}, {Host: "kass.example.com", Kind: "instance", App: "blog", Channel: "canary/x", Account: "kqmv", Status: "ready"}, } { if err := reg.Upsert(rt); err != nil { t.Fatal(err) } } if err := Pass(ctx, reg, s, pub, opt, func(string) error { return nil }); err != nil { t.Fatalf("Pass: %v", err) } for host, want := range map[string]string{ "ops.example.com": "v1", "kass.example.com": "v2", } { if got, _ := LiveVersion(opt, host); got != want { t.Errorf("%s live = %q, want %q — each account follows its OWN pointer", host, got, want) } } } // Same app name, same VERSION STRING, different accounts, different bytes. // The on-box version store must keep them apart, or one account's binary gets // executed under another account's host — the same collision in a different // layer from the channel-pointer one above. func TestPassStagesPerAccountForIdenticalVersions(t *testing.T) { s := NewDirStore(t.TempDir()) ctx := context.Background() priv, pub := testKeys(t) opt := t.TempDir() reg := testRegistry(t) shipRelease(t, s, "bab", "blog", "v1", adoptPlatform, []byte("ops bytes")) shipRelease(t, s, "kqmv", "blog", "v1", adoptPlatform, []byte("kass bytes")) for _, acct := range []string{"bab", "kqmv"} { if err := Promote(ctx, s, priv, acct, "blog", "canary/x", "v1"); err != nil { t.Fatal(err) } } for _, rt := range []registry.Route{ {Host: "ops.example.com", Kind: "instance", App: "blog", Channel: "canary/x", Account: "bab", Status: "ready"}, {Host: "kass.example.com", Kind: "instance", App: "blog", Channel: "canary/x", Account: "kqmv", Status: "ready"}, } { if err := reg.Upsert(rt); err != nil { t.Fatal(err) } } if err := Pass(ctx, reg, s, pub, opt, func(string) error { return nil }); err != nil { t.Fatalf("Pass: %v", err) } for host, want := range map[string]string{ "ops.example.com": "ops bytes", "kass.example.com": "kass bytes", } { target, err := os.Readlink(LiveLink(opt, host)) if err != nil { t.Fatalf("%s: read live link: %v", host, err) } got, err := os.ReadFile(target) if err != nil { t.Fatalf("%s: read staged binary: %v", host, err) } if string(got) != want { t.Errorf("%s runs %q, want %q — one account's build was staged over another's", host, got, want) } } } // The ops account keeps the pre-accounts on-disk layout byte for byte, so // rolling this code out does not restage and restart every existing host. func TestOpsVersionPathIsUnchanged(t *testing.T) { opt := "/opt/carlos" if got := VersionPath(opt, "bab", "v1", "kass"); got != "/opt/carlos/versions/v1/kass" { t.Errorf("ops VersionPath = %q, want the pre-accounts path", got) } if got := VersionPath(opt, "kqmv", "v1", "kass"); got != "/opt/carlos/versions/v1/kqmv/kass" { t.Errorf("non-ops VersionPath = %q", got) } } func TestPassMissingPlatformArtifactSkips(t *testing.T) { s := NewDirStore(t.TempDir()) ctx := context.Background() priv, pub := testKeys(t) opt := t.TempDir() reg := testRegistry(t) // Shipped for some other platform only: this box must skip, never // install the wrong arch. shipRelease(t, s, "bab", "myapp", "v1", "plan9-mips", []byte("bin")) if err := Promote(ctx, s, priv, "bab", "myapp", "canary/x", "v1"); err != nil { t.Fatalf("Promote: %v", err) } addRoute(t, reg, "h.example.com", "myapp", "canary/x", false) if err := Pass(ctx, reg, s, pub, opt, func(string) error { return nil }); err != nil { t.Fatalf("Pass: %v", err) } if v, _ := LiveVersion(opt, "h.example.com"); v != "" { t.Fatalf("live = %q, want unmoved (no artifact for this platform)", v) } } func TestPassFilterOnlyTouchesRoutesExtraAccepts(t *testing.T) { s := NewDirStore(t.TempDir()) ctx := context.Background() priv, pub := testKeys(t) opt := t.TempDir() reg := testRegistry(t) shipRelease(t, s, "bab", "myapp", "v1", adoptPlatform, []byte("build one")) if err := Promote(ctx, s, priv, "bab", "myapp", "canary/x", "v1"); err != nil { t.Fatalf("Promote canary/x: %v", err) } if err := Promote(ctx, s, priv, "bab", "myapp", "stable", "v1"); err != nil { t.Fatalf("Promote stable: %v", err) } addRoute(t, reg, "canary.example.com", "myapp", "canary/x", false) addRoute(t, reg, "stable.example.com", "myapp", "stable", false) canaryOnly := func(rt registry.Route) bool { return rt.Channel == "canary/x" } restart := func(string) error { return nil } if err := PassFilter(ctx, reg, s, pub, opt, restart, canaryOnly); err != nil { t.Fatalf("PassFilter: %v", err) } if v, _ := LiveVersion(opt, "canary.example.com"); v != "v1" { t.Errorf("canary.example.com live = %q, want v1 (extra accepts it)", v) } if v, _ := LiveVersion(opt, "stable.example.com"); v != "" { t.Errorf("stable.example.com live = %q, want unmoved — PassFilter must not touch a route extra rejects", v) } } // WithPointerGate (S3 change-index design §5): a group the gate skips is out // of the pass, its pointer unread; every other group is read as before and // its outcome reported once. func TestPassFilterPointerGate(t *testing.T) { s := NewDirStore(t.TempDir()) ctx := context.Background() priv, pub := testKeys(t) opt := t.TempDir() reg := testRegistry(t) shipRelease(t, s, "bab", "myapp", "v1", adoptPlatform, []byte("build one")) for _, ch := range []string{"canary/x", "stable"} { if err := Promote(ctx, s, priv, "bab", "myapp", ch, "v1"); err != nil { t.Fatalf("Promote %s: %v", ch, err) } } addRoute(t, reg, "canary.example.com", "myapp", "canary/x", false) addRoute(t, reg, "stable.example.com", "myapp", "stable", false) addRoute(t, reg, "beta.example.com", "myapp", "beta", false) // nothing promoted asked := map[string][]string{} read := map[string]string{} gate := func(acct, app, channel string, routes []registry.Route) (bool, func(*ChannelPointer, error)) { for _, rt := range routes { asked[channel] = append(asked[channel], rt.Host) } if channel == "canary/x" { return true, nil } return false, func(ptr *ChannelPointer, err error) { switch { case err != nil: read[channel] += "error;" default: read[channel] += ptr.Version + ";" } } } if err := PassFilter(ctx, reg, s, pub, opt, func(string) error { return nil }, nil, WithPointerGate(gate)); err != nil { t.Fatalf("PassFilter: %v", err) } want := map[string][]string{"canary/x": {"canary.example.com"}, "stable": {"stable.example.com"}, "beta": {"beta.example.com"}} if len(asked) != len(want) { t.Fatalf("the gate was asked about %v, want %v", asked, want) } for ch, hosts := range want { if !slices.Equal(asked[ch], hosts) { t.Fatalf("the gate was asked about %s with %v, want %v", ch, asked[ch], hosts) } } if v, _ := LiveVersion(opt, "canary.example.com"); v != "" { t.Errorf("a skipped group adopted: canary.example.com live = %q", v) } if v, _ := LiveVersion(opt, "stable.example.com"); v != "v1" { t.Errorf("stable.example.com live = %q, want v1", v) } if read["stable"] != "v1;" || read["beta"] != "error;" || read["canary/x"] != "" { t.Errorf("read outcomes %v, want stable v1 once, beta's error once, nothing for the skipped group", read) } } // writeBackdatedPointer signs and writes a channel pointer with an arbitrary // PromotedAt, bypassing Promote/signPointer's own "stamp now" behavior — the // only way to get an EXPIRED canary pointer into a store under test, since // every public promotion path stamps the real clock. Kept in this internal // test package (never exported) precisely because forging a pointer's lease // is not something production code should ever be able to do. func writeBackdatedPointer(t *testing.T, s Store, priv ed25519.PrivateKey, acct, app, channel, version string, promotedAt time.Time) { t.Helper() p := &ChannelPointer{Schema: 1, App: app, Channel: channel, Version: version, PromotedAt: promotedAt} payload, err := json.MarshalIndent(p, "", " ") if err != nil { t.Fatalf("marshal backdated pointer: %v", err) } b64 := base64.StdEncoding.EncodeToString(payload) sig := ed25519.Sign(priv, []byte(b64)) wrapper, err := json.MarshalIndent(signedWrapper{ Payload: b64, Sig: base64.StdEncoding.EncodeToString(sig), }, "", " ") if err != nil { t.Fatalf("marshal backdated wrapper: %v", err) } if _, err := s.Put(context.Background(), ChannelKey(acct, app, channel), wrapper, ""); err != nil { t.Fatalf("write backdated pointer: %v", err) } } // TestPassAdoptsAnExpiredCanary pins spec §5's core promise: expiry is // VISIBILITY, not enforcement. A canary/* route whose pointer is well past // CanaryTTL must still adopt exactly like a fresh one — nothing about // expiry may block a repoint, a restart, or a rollback/promote (the same // signed-pointer plumbing every other channel uses; expiry checks nothing // about the ladder or the CAS path). func TestPassAdoptsAnExpiredCanary(t *testing.T) { s := NewDirStore(t.TempDir()) ctx := context.Background() priv, pub := testKeys(t) opt := t.TempDir() reg := testRegistry(t) shipRelease(t, s, "bab", "myapp", "v1", adoptPlatform, []byte("build one")) writeBackdatedPointer(t, s, priv, "bab", "myapp", "canary/x", "v1", time.Now().Add(-(CanaryTTL + 30*24*time.Hour))) // long expired addRoute(t, reg, "old-canary.example.com", "myapp", "canary/x", false) var restarted []string restart := func(host string) error { restarted = append(restarted, host); return nil } if err := Pass(ctx, reg, s, pub, opt, restart); err != nil { t.Fatalf("Pass: %v", err) } if v, _ := LiveVersion(opt, "old-canary.example.com"); v != "v1" { t.Fatalf("an expired canary route did not adopt: live = %q, want v1", v) } if len(restarted) != 1 || restarted[0] != "old-canary.example.com" { t.Fatalf("an expired canary route was not restarted: restarted = %v", restarted) } rt, err := reg.Get("old-canary.example.com") if err != nil || rt == nil || rt.Version != "v1" { t.Fatalf("registry version after adopting an expired canary = %+v, %v", rt, err) } // Promotion off (or onto) an expired canary channel is unaffected too — // expiry is computed from PromotedAt at read time, never stored, so // nothing about a pointer's own expiry state can gate a write to it. shipRelease(t, s, "bab", "myapp", "v2", adoptPlatform, []byte("build two")) if err := Promote(ctx, s, priv, "bab", "myapp", "canary/x", "v2"); err != nil { t.Fatalf("promoting a new version onto an expired canary channel: %v", err) } p, err := Channel(ctx, s, pub, "bab", "myapp", "canary/x") if err != nil { t.Fatalf("Channel after re-promoting an expired canary: %v", err) } if p.Version != "v2" { t.Fatalf("channel = %q after promote, want v2", p.Version) } // And the re-promotion refreshed the lease — CanaryExpired.go's whole // point: an actively-updated canary never expires. if CanaryExpired(p, time.Now()) { t.Error("re-promoting an expired canary must refresh its lease, not leave it expired") } } // TestPassFilterDoesNotNagWithoutTheOption pins the fast-loop half of the // nag's scoping (2026-08-03 review): internal/host/agent.go's fastConvergeLoop // calls PassFilter every 2-3 seconds for its fast set, and NEVER // passes NagExpiredCanaries — so an expired canary that is sitting still // must produce ZERO nag lines from that path, not one every few seconds. // This is PassFilter called the way that loop calls it (without the nag // option) against the same expired fixture TestPassAdoptsAnExpiredCanary // uses. func TestPassFilterDoesNotNagWithoutTheOption(t *testing.T) { s := NewDirStore(t.TempDir()) ctx := context.Background() priv, pub := testKeys(t) opt := t.TempDir() reg := testRegistry(t) shipRelease(t, s, "bab", "myapp", "v1", adoptPlatform, []byte("build one")) writeBackdatedPointer(t, s, priv, "bab", "myapp", "canary/x", "v1", time.Now().Add(-(CanaryTTL + 30*24*time.Hour))) addRoute(t, reg, "old-canary.example.com", "myapp", "canary/x", false) var buf bytes.Buffer withCapturedStdout(t, &buf, func() { if err := PassFilter(ctx, reg, s, pub, opt, func(string) error { return nil }, isCanaryTestRoute); err != nil { t.Fatalf("PassFilter: %v", err) } }) if strings.Contains(buf.String(), "expired") { t.Errorf("PassFilter without NagExpiredCanaries printed a nag line:\n%s", buf.String()) } } // TestPassNagsOncePerExpiredCanaryRoute pins the root path's half: RunAdopt // (carlos-adopt.service, hourly) passes NagExpiredCanaries(os.Stdout) to // Pass, and that call must print ONE nag line PER EXPIRED CANARY ROUTE it // sees — not one per hourly pass regardless of how many routes follow the // expired channel. Three routes (two awake, one hibernating — the awake and // hibernating lists are walked separately in Pass/PassFilter, so this is // also the regression test for "hibernating routes get skipped" and // "a route is visited twice") on ONE expired canary channel must produce // exactly three lines, each naming its own host. // // (2026-08-03 review, MINOR 6: this test used to be named // TestPassNagsExactlyOnceWithTheOption and only ever exercised one route, // so "not once per host revisit" was asserted by name only, never proven.) func TestPassNagsOncePerExpiredCanaryRoute(t *testing.T) { s := NewDirStore(t.TempDir()) ctx := context.Background() priv, pub := testKeys(t) opt := t.TempDir() reg := testRegistry(t) shipRelease(t, s, "bab", "myapp", "v1", adoptPlatform, []byte("build one")) writeBackdatedPointer(t, s, priv, "bab", "myapp", "canary/x", "v1", time.Now().Add(-(CanaryTTL + 30*24*time.Hour))) addRoute(t, reg, "awake-a.example.com", "myapp", "canary/x", false) addRoute(t, reg, "awake-b.example.com", "myapp", "canary/x", false) addRoute(t, reg, "sleepy.example.com", "myapp", "canary/x", true) var buf bytes.Buffer if err := Pass(ctx, reg, s, pub, opt, func(string) error { return nil }, NagExpiredCanaries(&buf)); err != nil { t.Fatalf("Pass: %v", err) } out := buf.String() if got := strings.Count(out, "expired"); got != 3 { t.Errorf("nag lines containing \"expired\" = %d, want exactly 3 (one per route, awake and hibernating alike):\n%s", got, out) } for _, host := range []string{"awake-a.example.com", "awake-b.example.com", "sleepy.example.com"} { if !strings.Contains(out, host) { t.Errorf("nag output does not name %s:\n%s", host, out) } } } // TestNagReportsTimePastTTLNotTotalAge pins the 2026-08-03 review's // IMPORTANT 2 fix: the nag must report how long PAST CanaryTTL a canary is, // not its total age. A pointer promoted CanaryTTL+1h ago has been EXPIRED // for about an hour, not 169 hours (7 days 1 hour, its total age) — the // second number is real but answers a different question, and printing it // under "has been expired" is the bug the review caught. It must also // disagree with nothing the console shows for the same pointer // (internal/console/ladder_test.go's TestLadderExpiredCanaryIsBadgedWithAge // exercises the same release.CanaryAge/FormatCanaryAge pair). func TestNagReportsTimePastTTLNotTotalAge(t *testing.T) { s := NewDirStore(t.TempDir()) ctx := context.Background() priv, pub := testKeys(t) opt := t.TempDir() reg := testRegistry(t) shipRelease(t, s, "bab", "myapp", "v1", adoptPlatform, []byte("build one")) writeBackdatedPointer(t, s, priv, "bab", "myapp", "canary/x", "v1", time.Now().Add(-(CanaryTTL + time.Hour))) addRoute(t, reg, "old-canary.example.com", "myapp", "canary/x", false) var buf bytes.Buffer if err := Pass(ctx, reg, s, pub, opt, func(string) error { return nil }, NagExpiredCanaries(&buf)); err != nil { t.Fatalf("Pass: %v", err) } out := buf.String() if !strings.Contains(out, "1h") { t.Errorf("nag does not say 1h (time past TTL) for a canary expired ~1h ago:\n%s", out) } if strings.Contains(out, "169h") { t.Errorf("nag reports total age (169h) instead of time past TTL:\n%s", out) } } // TestPassDoesNotNagAFreshCanaryEvenWithTheOption: the option opts a CALL // into nagging; it does not turn every canary into a nag target. A fresh // canary must stay silent whether or not NagExpiredCanaries is set. func TestPassDoesNotNagAFreshCanaryEvenWithTheOption(t *testing.T) { s := NewDirStore(t.TempDir()) ctx := context.Background() priv, pub := testKeys(t) opt := t.TempDir() reg := testRegistry(t) shipRelease(t, s, "bab", "myapp", "v1", adoptPlatform, []byte("build one")) if err := Promote(ctx, s, priv, "bab", "myapp", "canary/x", "v1"); err != nil { t.Fatalf("Promote: %v", err) } addRoute(t, reg, "fresh-canary.example.com", "myapp", "canary/x", false) var buf bytes.Buffer if err := Pass(ctx, reg, s, pub, opt, func(string) error { return nil }, NagExpiredCanaries(&buf)); err != nil { t.Fatalf("Pass: %v", err) } if strings.Contains(buf.String(), "expired") { t.Errorf("a freshly-promoted canary produced a nag line:\n%s", buf.String()) } } // isCanaryTestRoute mirrors host.isCanaryRoute (unexported, different // package) for the one fast-loop-shaped test above that needs a canary-only // filter without importing internal/host. func isCanaryTestRoute(rt registry.Route) bool { return strings.HasPrefix(rt.Channel, "canary/") } // withCapturedStdout redirects os.Stdout to buf for the duration of fn — // used ONLY to prove PassFilter's default (no NagExpiredCanaries) path // prints nothing canary-related even via the package's ordinary fmt.Printf // progress lines, which is the honest thing to assert against (adoptHost's // other progress lines still go to os.Stdout; NagExpiredCanaries is what // redirects the NAG line specifically to an injected writer instead). func withCapturedStdout(t *testing.T, buf *bytes.Buffer, fn func()) { t.Helper() r, w, err := os.Pipe() if err != nil { t.Fatalf("os.Pipe: %v", err) } orig := os.Stdout os.Stdout = w defer func() { os.Stdout = orig }() done := make(chan struct{}) go func() { buf.ReadFrom(r) close(done) }() fn() w.Close() <-done } // adoptRefusesManifest runs the whole fail-closed scenario for a stored // manifest whose capability block this binary will not accept: a box already // serving v1, a channel that has moved to v2, and v2's manifest bytes written // DIRECTLY to the store (bypassing Ship, which is exactly what a newer // platform binary would leave behind). It asserts the four things that make // the refusal safe — the live symlink unmoved, nothing staged, no restart, // the registry unchanged — and returns what the pass printed. // // Shared by the two tests below because the two refusals arrive by DIFFERENT // routes and both have to land here: one from caps.Set's strict decoder // (json.Unmarshal fails), one from ValidCapabilities (the block decodes // perfectly and is only then refused). A gate that caught the first and not // the second would look correct in a single test. func adoptRefusesManifest(t *testing.T, capsJSON string) string { t.Helper() s := NewDirStore(t.TempDir()) ctx := context.Background() priv, pub := testKeys(t) opt := t.TempDir() reg := testRegistry(t) // The box is up and running on v1. shipRelease(t, s, "bab", "myapp", "v1", adoptPlatform, []byte("good")) if err := StageArtifact(opt, "bab", "v1", "myapp", []byte("good")); err != nil { t.Fatal(err) } addRoute(t, reg, "h.example.com", "myapp", "canary/x", false) if err := PointLive(opt, "h.example.com", "bab", "v1", "myapp"); err != nil { t.Fatal(err) } // v2 ships and the channel moves onto it while it is still a manifest this // binary understands... body := []byte("newer") shipRelease(t, s, "bab", "myapp", "v2", adoptPlatform, body) if err := Promote(ctx, s, priv, "bab", "myapp", "canary/x", "v2"); err != nil { t.Fatalf("Promote v2: %v", err) } // ...and is then replaced by the bytes a NEWER binary would have written: // same artifacts, same sha, plus a capability block this one refuses. key := ManifestKey("bab", "myapp", "v2") _, etag, err := s.Get(ctx, key) if err != nil { t.Fatalf("read v2 manifest: %v", err) } future := []byte(`{ "schema": 1, "app": "myapp", "version": "v2", "kind": "binary", "artifacts": { "` + adoptPlatform + `": { "file": "` + adoptPlatform + `", "sha256": "` + etagOf(body) + `", "size": ` + strconv.Itoa(len(body)) + ` } }, "capabilities": ` + capsJSON + ` }`) if _, err := s.Put(ctx, key, future, etag); err != nil { t.Fatalf("replace v2 manifest: %v", err) } var out bytes.Buffer var restarted []string withCapturedStdout(t, &out, func() { if err := Pass(ctx, reg, s, pub, opt, func(h string) error { restarted = append(restarted, h) return nil }); err != nil { t.Fatalf("Pass: %v", err) } }) if v, _ := LiveVersion(opt, "h.example.com"); v != "v1" { t.Fatalf("live = %q, want v1 — the box must keep serving what it has", v) } if _, err := os.Stat(VersionPath(opt, "bab", "v2", "myapp")); !os.IsNotExist(err) { t.Fatalf("v2 was staged (stat err = %v); a refused version must never reach disk", err) } if len(restarted) != 0 { t.Fatalf("restarted = %v, want none", restarted) } if rt, _ := reg.Get("h.example.com"); rt == nil || rt.Version == "v2" { t.Fatalf("registry recorded v2 as adopted: %+v", rt) } return out.String() } // TestAdoptRefusesTheVersionWhenAManifestDeclaresAnUnknownName is the // fail-closed direction, and it is the test that matters most in this task. // // The box's answer to a capability name from a vocabulary it does not have // must be to refuse the VERSION: nothing staged, the live symlink unmoved, // one printed line naming the capability and the fix. The box keeps serving // what it has, which for a daemon is the safe direction; the alternative — // materializing a unit that is missing something the app says it needs, and // looking healthy — is the config-that-lies failure mode this whole design // exists to avoid. // // This refusal comes from caps.Set's strict DECODER: json.Unmarshal never // completes, so the manifest never becomes a value at all. func TestAdoptRefusesTheVersionWhenAManifestDeclaresAnUnknownName(t *testing.T) { line := adoptRefusesManifest(t, `{ "v": 1, "daemon": true, "gpu": true }`) if !strings.Contains(line, "gpu") { t.Fatalf("printed %q, want a line naming the capability", line) } if !strings.Contains(line, "update the platform binary") { t.Fatalf("printed %q, want a line naming the fix", line) } } // TestAdoptRefusesTheVersionWhenAManifestIsSemanticallyInvalid closes the // other half of the gate, and it is the half a decoder-only test would miss: // this block decodes PERFECTLY — every name is known, "v" is 1, the shape is // right — and is refused only by ValidCapabilities, on a port that is not a // capability at all. It is reached because validateManifest runs on every // GetManifest, not only at ship; delete that one line and this test goes red // while the unknown-name test above stays green. func TestAdoptRefusesTheVersionWhenAManifestIsSemanticallyInvalid(t *testing.T) { line := adoptRefusesManifest(t, `{ "v": 1, "daemon": true, "ports": [ { "port": 8080 } ] }`) if !strings.Contains(line, "8080") { t.Fatalf("printed %q, want a line naming the offending port", line) } if !strings.Contains(line, "capabilities") { t.Fatalf("printed %q, want a line naming what was refused", line) } } // TestEnsureStagedWritesTheSiblingManifest pins the fresh-download half of // Task 3: after ensureStaged fetches and stages a version for the first // time, ReadStagedManifest returns a manifest whose App/Version match and // whose Capabilities round-trip byte for byte — this is what lets the // converge pass read a running version's capabilities off disk, with no // bucket round-trip on every tick. func TestEnsureStagedWritesTheSiblingManifest(t *testing.T) { s := NewDirStore(t.TempDir()) ctx := context.Background() opt := t.TempDir() m := Manifest{ App: "keymail", Version: "v1abc", Kind: "binary", Artifacts: map[string]Artifact{adoptPlatform: {}}, Capabilities: &caps.Set{V: 1, Command: "relay", Daemon: true, RuntimeDir: "keymail"}, } if err := Ship(ctx, s, "bab", m, map[string][]byte{adoptPlatform: []byte("relay binary")}); err != nil { t.Fatalf("Ship: %v", err) } inHand, err := ensureStaged(ctx, s, opt, "bab", "keymail", "v1abc", "keymail.example.com") if err != nil { t.Fatalf("ensureStaged: %v", err) } // The manifest ensureStaged HAS IN HAND is what adoptHost's converge // builds its plan from (Task 7's first pin) — not a re-read of the // sibling it just wrote. if inHand == nil || inHand.Capabilities == nil || inHand.Capabilities.Command != "relay" { t.Fatalf("ensureStaged returned %+v, want the fetched manifest with its capability block", inHand) } got, err := ReadStagedManifest(opt, "bab", "v1abc", "keymail") if err != nil { t.Fatalf("ReadStagedManifest: %v", err) } if got == nil || got.App != "keymail" || got.Version != "v1abc" { t.Fatalf("got = %+v, want App/Version to match", got) } if got.Capabilities == nil || got.Capabilities.Command != "relay" || !got.Capabilities.Daemon || got.Capabilities.RuntimeDir != "keymail" { t.Fatalf("Capabilities did not round-trip: %+v", got.Capabilities) } } // TestEnsureStagedBackfillsASiblingForAnAlreadyStagedBinary pins the named // finding: a box that staged a version's binary before this feature existed // (or before the sibling write otherwise happened) must have the sibling // back-filled on the next pass WITHOUT the artifact being re-downloaded — // otherwise the converge pass is permanently blind on exactly the versions // already running, since a staged binary never touches ensureStaged's // download branch again on its own. func TestEnsureStagedBackfillsASiblingForAnAlreadyStagedBinary(t *testing.T) { s := NewDirStore(t.TempDir()) ctx := context.Background() opt := t.TempDir() m := Manifest{ App: "keymail", Version: "v1abc", Kind: "binary", Artifacts: map[string]Artifact{adoptPlatform: {}}, Capabilities: &caps.Set{V: 1, Daemon: true, RuntimeDir: "keymail"}, } if err := Ship(ctx, s, "bab", m, map[string][]byte{adoptPlatform: []byte("relay binary")}); err != nil { t.Fatalf("Ship: %v", err) } // Stage the binary BY HAND, as a pre-Task-3 box would have — no sibling. if err := StageArtifact(opt, "bab", "v1abc", "keymail", []byte("relay binary")); err != nil { t.Fatalf("StageArtifact: %v", err) } binPath := VersionPath(opt, "bab", "v1abc", "keymail") beforeInfo, err := os.Stat(binPath) if err != nil { t.Fatalf("stat before backfill: %v", err) } beforeBytes, err := os.ReadFile(binPath) if err != nil { t.Fatalf("read before backfill: %v", err) } if m2, err := ReadStagedManifest(opt, "bab", "v1abc", "keymail"); err != nil || m2 != nil { t.Fatalf("sibling should be absent before backfill: %+v, %v", m2, err) } inHand, err := ensureStaged(ctx, s, opt, "bab", "keymail", "v1abc", "keymail.example.com") if err != nil { t.Fatalf("ensureStaged (backfill): %v", err) } if inHand == nil || inHand.Capabilities == nil || !inHand.Capabilities.Daemon { t.Fatalf("back-fill returned %+v, want the fetched manifest in hand", inHand) } got, err := ReadStagedManifest(opt, "bab", "v1abc", "keymail") if err != nil { t.Fatalf("ReadStagedManifest after backfill: %v", err) } if got == nil || got.Capabilities == nil || !got.Capabilities.Daemon || got.Capabilities.RuntimeDir != "keymail" { t.Fatalf("backfilled sibling missing capabilities: %+v", got) } afterInfo, err := os.Stat(binPath) if err != nil { t.Fatalf("stat after backfill: %v", err) } afterBytes, err := os.ReadFile(binPath) if err != nil { t.Fatalf("read after backfill: %v", err) } if !beforeInfo.ModTime().Equal(afterInfo.ModTime()) { t.Fatalf("backfill re-touched the binary: mtime %v -> %v", beforeInfo.ModTime(), afterInfo.ModTime()) } if string(beforeBytes) != string(afterBytes) { t.Fatalf("backfill re-downloaded the binary bytes") } } // TestEnsureStagedSiblingWriteFailureIsNotFatalToStagingTheBinary is // killing test 2 from the 2026-08-03 review: a mutant that makes the // marshal/StageManifest failure at the end of ensureStaged's fresh-download // path fatal must be caught. The binary must stage successfully even when // its sibling cannot be written — a missing sibling is the safe leave-alone // state, and backfillManifest gets another chance at it every later pass. // // StageManifest's own write is blocked by pre-creating its final rename // target AS A DIRECTORY — rename then fails to replace it, without ever // touching StageArtifact's own (different) binary path. func TestEnsureStagedSiblingWriteFailureIsNotFatalToStagingTheBinary(t *testing.T) { s := NewDirStore(t.TempDir()) ctx := context.Background() opt := t.TempDir() m := Manifest{ App: "keymail", Version: "v1abc", Kind: "binary", Artifacts: map[string]Artifact{adoptPlatform: {}}, Capabilities: &caps.Set{V: 1, Daemon: true, RuntimeDir: "keymail"}, } if err := Ship(ctx, s, "bab", m, map[string][]byte{adoptPlatform: []byte("relay binary")}); err != nil { t.Fatalf("Ship: %v", err) } tmpBlock := StagedManifestPath(opt, "bab", "v1abc", "keymail") if err := os.MkdirAll(tmpBlock, 0o755); err != nil { t.Fatalf("pre-create tmp block: %v", err) } inHand, err := ensureStaged(ctx, s, opt, "bab", "keymail", "v1abc", "h.example.com") if err != nil { t.Fatalf("ensureStaged returned an error for a failed SIBLING write: %v — the binary staged fine and that must not be fatal", err) } // Task 7's first pin, in its sharpest form: the sibling write FAILED, so // a converge that re-read the sibling would see (nil, nil) and leave the // previous version's drop-ins in place under a new binary. The manifest // this function returns is the fetched one and is unaffected. if inHand == nil || inHand.Capabilities == nil || !inHand.Capabilities.Daemon { t.Fatalf("ensureStaged returned %+v after a failed sibling write, want the fetched manifest still in hand", inHand) } body, err := os.ReadFile(VersionPath(opt, "bab", "v1abc", "keymail")) if err != nil || string(body) != "relay binary" { t.Fatalf("binary did not stage despite the sibling write failing: %q, %v", body, err) } if got, err := ReadStagedManifest(opt, "bab", "v1abc", "keymail"); err == nil || got != nil { t.Fatalf("sibling = %+v, %v; want blocked (the write genuinely failed)", got, err) } } // TestEnsureStagedBackfillFetchFailureIsNotFatal is killing test 3 / I2's // regression test (2026-08-03 review): ensureStaged's back-fill fetch (a // GetManifest for an ALREADY-STAGED binary) must not be fatal to adoption. // The store here has no release shipped at all, so GetManifest is // guaranteed to fail — simulating a bucket hiccup for a version that is // already running and needs no adoption. A mutant that makes this fetch // failure fatal again would hard-fail (and, on a canary route's 2-3-second // fast loop, journal-flood) adoption of a binary that doesn't need it. func TestEnsureStagedBackfillFetchFailureIsNotFatal(t *testing.T) { s := NewDirStore(t.TempDir()) // empty — nothing ever shipped ctx := context.Background() opt := t.TempDir() if err := StageArtifact(opt, "bab", "v1abc", "keymail", []byte("relay binary")); err != nil { t.Fatalf("StageArtifact: %v", err) } inHand, err := ensureStaged(ctx, s, opt, "bab", "keymail", "v1abc", "h.example.com") if err != nil { t.Fatalf("ensureStaged returned an error for an already-staged binary's failed back-fill fetch: %v — adoption of an already-running version must proceed", err) } // This is the ONE path with no manifest to give. It stays (nil, nil) so // ordinary adoption proceeds; a capability-armed pass turns that nil into // a refusal of the version rather than a silent leave-alone — see // TestArmedPassRefusesAVersionWhoseManifestItCouldNotObtain. if inHand != nil { t.Fatalf("ensureStaged returned %+v, want nil (no manifest could be obtained)", inHand) } if got, err := ReadStagedManifest(opt, "bab", "v1abc", "keymail"); err != nil || got != nil { t.Fatalf("sibling = %+v, %v; want absent (the fetch failed, nothing to write)", got, err) } } // TestPassBackfillsTheLiveVersionsSiblingOnAnUnchangedPass pins C1, the // critical finding from the 2026-08-03 review: ensureStaged's own back-fill // only runs from the call inside adoptHost that sits BEHIND the cur == ver // short-circuit, so a version that is already live — with no repoint due // this pass — never reaches it on its own. Since the migration path is // entirely "a box's CURRENT version was staged before this feature // existed," without a separate per-pass repair call the sibling for // exactly the version actually running would stay missing forever. This // test exercises the real call site (Pass, not ensureStaged directly): the // route is already on v1abc, nothing repoints, and the sibling must still // appear. func TestPassBackfillsTheLiveVersionsSiblingOnAnUnchangedPass(t *testing.T) { s := NewDirStore(t.TempDir()) ctx := context.Background() priv, pub := testKeys(t) opt := t.TempDir() reg := testRegistry(t) m := Manifest{ App: "keymail", Version: "v1abc", Kind: "binary", Artifacts: map[string]Artifact{adoptPlatform: {}}, Capabilities: &caps.Set{V: 1, Daemon: true, RuntimeDir: "keymail"}, } if err := Ship(ctx, s, "bab", m, map[string][]byte{adoptPlatform: []byte("relay binary")}); err != nil { t.Fatalf("Ship: %v", err) } if err := Promote(ctx, s, priv, "bab", "keymail", "canary/x", "v1abc"); err != nil { t.Fatalf("Promote: %v", err) } // The binary is staged and pointed live BY HAND, exactly as a // pre-Task-3 box would be — no sibling, and no version change due this // pass (the route is already on v1abc). if err := StageArtifact(opt, "bab", "v1abc", "keymail", []byte("relay binary")); err != nil { t.Fatalf("StageArtifact: %v", err) } addRoute(t, reg, "h.example.com", "keymail", "canary/x", false) if err := PointLive(opt, "h.example.com", "bab", "v1abc", "keymail"); err != nil { t.Fatalf("PointLive: %v", err) } if m2, err := ReadStagedManifest(opt, "bab", "v1abc", "keymail"); err != nil || m2 != nil { t.Fatalf("sibling should be absent before the pass: %+v, %v", m2, err) } if err := Pass(ctx, reg, s, pub, opt, func(string) error { return nil }); err != nil { t.Fatalf("Pass: %v", err) } got, err := ReadStagedManifest(opt, "bab", "v1abc", "keymail") if err != nil { t.Fatalf("ReadStagedManifest after Pass: %v", err) } if got == nil || got.Capabilities == nil || !got.Capabilities.Daemon { t.Fatalf("Pass did not back-fill the live version's sibling: %+v", got) } if v, _ := LiveVersion(opt, "h.example.com"); v != "v1abc" { t.Fatalf("live version moved unexpectedly: %q", v) } } // ---- Task 7: the capability converge inside adoptHost ----------------------- // capsRecorder is a tiny in-memory systemd for the capability tests in this // package, plus the one thing internal/caps's own fake cannot observe: WHERE // THE LIVE SYMLINK POINTED at the moment each verb ran. That is how the // ordering rule — converge before PointLive — is asserted as a fact about // this pass rather than as a reading of the code. type capsRecorder struct { opt, host string calls []string live []string // live[i] is LiveVersion when calls[i] ran liveAt map[string]string // verb+unit -> LiveVersion at its FIRST call active map[string]bool enabled map[string]bool fail map[string]error // "verb unit" -> error onCall func(call string) // optional: fires after a call is recorded } func newCapsRecorder(opt, host string) *capsRecorder { return &capsRecorder{ opt: opt, host: host, liveAt: map[string]string{}, active: map[string]bool{}, enabled: map[string]bool{}, fail: map[string]error{}, } } func (r *capsRecorder) run(args ...string) error { call := strings.Join(args, " ") v, _ := LiveVersion(r.opt, r.host) r.calls = append(r.calls, call) r.live = append(r.live, v) if _, seen := r.liveAt[call]; !seen { r.liveAt[call] = v } if r.onCall != nil { r.onCall(call) } if err := r.fail[call]; err != nil { return err } if len(args) >= 2 { switch args[0] { case "start", "restart": r.active[args[1]] = true case "stop": r.active[args[1]] = false case "enable": r.enabled[args[1]] = true } } return nil } func (r *capsRecorder) query(args ...string) bool { if len(args) < 2 { return false } switch args[0] { case "is-active": return r.active[args[1]] case "is-enabled": return r.enabled[args[1]] } return false } func (r *capsRecorder) called(call string) bool { for _, c := range r.calls { if c == call { return true } } return false } // lastLiveAt is where the live symlink pointed at the LAST occurrence of a // call. liveAt records only the first, which is blind to everything a // rollback does after the same verb has already run once — the ordering // inside the rollback (symlink first, drop-ins second) is exactly what needs // pinning, so it needs the last one. func (r *capsRecorder) lastLiveAt(call string) (string, bool) { for i := len(r.calls) - 1; i >= 0; i-- { if r.calls[i] == call { return r.live[i], true } } return "", false } // serviceCycles is every verb that STARTS the service unit — the count that // says whether a version change cycled the app once or twice. func (r *capsRecorder) serviceCycles(host string) (n int, lastLive string) { for i, c := range r.calls { if c == "start "+caps.ServiceUnit(host) || c == "restart "+caps.ServiceUnit(host) { n++ lastLive = r.live[i] } } return n, lastLive } // reset clears the call log between passes. func (r *capsRecorder) reset() { r.calls, r.live = nil, nil r.liveAt = map[string]string{} } // capsTestOps wires every caps seam at a temp dir, so a Pass in this package // touches no systemd, no /etc, no /run and no /var/lib. Logf is left nil on // purpose: converge's own lines then go to os.Stdout with every other adopt // line, which is what withCapturedStdout asserts against. func capsTestOps(t *testing.T, r *capsRecorder) (caps.Ops, string) { t.Helper() etc := t.TempDir() return caps.Ops{ EtcDir: etc, StateRoot: t.TempDir(), RunDir: t.TempDir(), Systemctl: r.run, Query: r.query, EnsureUser: func(string) error { return nil }, }, etc } // addCapRoute adds a route with a backing and a granted capability column — // addRoute plus the two columns Task 7 reads. func addCapRoute(t *testing.T, reg *registry.Registry, host, app, channel, backing, grants string, hibernate, platform bool) { t.Helper() if err := reg.Upsert(registry.Route{ Host: host, Kind: "instance", App: app, Channel: channel, Hibernate: hibernate, Backing: backing, Caps: grants, Platform: platform, }); err != nil { t.Fatalf("upsert %s: %v", host, err) } } // shipCapRelease is shipRelease with a capability block. func shipCapRelease(t *testing.T, s Store, acct, app, version string, set *caps.Set, body []byte) { t.Helper() m := Manifest{ App: app, Version: version, Kind: "binary", Artifacts: map[string]Artifact{adoptPlatform: {}}, Capabilities: set, } if err := Ship(context.Background(), s, acct, m, map[string][]byte{adoptPlatform: body}); err != nil { t.Fatalf("Ship %s/%s: %v", app, version, err) } } func relaySet() *caps.Set { return &caps.Set{ V: 1, Command: "relay", Daemon: true, RuntimeDir: "keymail", Ports: []caps.Port{{Port: 25, Name: "smtp"}}, } } // TestAdoptConvergesCapabilitiesBeforePointLive is the ordering rule of spec // §4d, asserted where it can actually fail: the drop-ins are written and // daemon-reloaded while the live symlink still names the OLD version, and the // daemon's restart happens after it names the new one. Converge after // PointLive would leave a window where the new binary is live under the old // unit shape; converge without a reload would leave bytes systemd never read. func TestAdoptConvergesCapabilitiesBeforePointLive(t *testing.T) { s := NewDirStore(t.TempDir()) ctx := context.Background() priv, pub := testKeys(t) opt := t.TempDir() reg := testRegistry(t) host := "keymail.dev" // The box is up on v1, which declares nothing. shipRelease(t, s, "bab", "keymail", "v1", adoptPlatform, []byte("old relay")) if err := StageArtifact(opt, "bab", "v1", "keymail", []byte("old relay")); err != nil { t.Fatal(err) } addCapRoute(t, reg, host, "keymail", "canary/x", registry.BackingUnit, "port:25", false, false) if err := PointLive(opt, host, "bab", "v1", "keymail"); err != nil { t.Fatal(err) } set := relaySet() shipCapRelease(t, s, "bab", "keymail", "v2", set, []byte("new relay")) if err := Promote(ctx, s, priv, "bab", "keymail", "canary/x", "v2"); err != nil { t.Fatalf("Promote v2: %v", err) } rec := newCapsRecorder(opt, host) o, etc := capsTestOps(t, rec) var restarted []string restart := func(h string) error { restarted = append(restarted, h); return nil } if err := Pass(ctx, reg, s, pub, opt, restart, WithCapabilities(o)); err != nil { t.Fatalf("Pass: %v", err) } if v, _ := LiveVersion(opt, host); v != "v2" { t.Fatalf("live = %q, want v2", v) } if v, seen := rec.liveAt["daemon-reload"]; !seen || v != "v1" { t.Fatalf("daemon-reload observed live=%q (seen=%v), want v1 — the drop-ins must be loaded BEFORE the symlink moves", v, seen) } // The bytes are the renderer's, byte for byte — this test pins the // wiring, not a second copy of the drop-in format. g := caps.Grants{Ports: map[int]bool{25: true}} svcPath, sockPath := caps.DropInPaths(etc, host) wantSvc, err := caps.RenderService("keymail", set, g) if err != nil { t.Fatalf("RenderService: %v", err) } gotSvc, err := os.ReadFile(svcPath) if err != nil || !bytes.Equal(gotSvc, wantSvc) { t.Fatalf("service drop-in = %q (%v), want %q", gotSvc, err, wantSvc) } wantSock, err := caps.RenderSocket(set, g) if err != nil { t.Fatalf("RenderSocket: %v", err) } gotSock, err := os.ReadFile(sockPath) if err != nil || !bytes.Equal(gotSock, wantSock) { t.Fatalf("socket drop-in = %q (%v), want %q", gotSock, err, wantSock) } // A converged DAEMON's restart is caps.Restart, not the caller's // try-restart callback — that is what makes its failure real enough to // roll back (restartRoute swallows every error it sees). if len(restarted) != 0 { t.Fatalf("restart callback called for %v; a converged daemon's restart is caps.Restart's", restarted) } svcRestart := "restart " + caps.ServiceUnit(host) if v, seen := rec.liveAt[svcRestart]; !seen || v != "v2" { t.Fatalf("%q observed live=%q (seen=%v), want v2 — the daemon restarts onto the NEW binary", svcRestart, v, seen) } // And exactly ONCE. Converge runs before the symlink moves, so any start // it performs itself boots the OLD binary under the NEW unit shape — a // combination nobody shipped — and then the caller cycles the app a // second time. Plan.DeferRestart is what collapses that to one cycle. if n, last := rec.serviceCycles(host); n != 1 || last != "v2" { t.Fatalf("service started/restarted %d time(s), last at live=%q; want exactly 1, at v2 (calls: %v)", n, last, rec.calls) } } // TestRestartFailureRestoresBothTheSymlinkAndTheDropIns is the rollback that // keeps a bad build off a mail server: when the daemon will not come back on // the new version, the box must end the pass exactly where it started — old // symlink, old unit text, the version poisoned for the rest of the pass. // Restoring one without the other is the divergence this pairing exists to // prevent (bytes that say :465 under a binary that never asked for it). func TestRestartFailureRestoresBothTheSymlinkAndTheDropIns(t *testing.T) { s := NewDirStore(t.TempDir()) ctx := context.Background() priv, pub := testKeys(t) opt := t.TempDir() reg := testRegistry(t) host := "a.example.com" v1 := &caps.Set{V: 1, Command: "relay", Daemon: true, Ports: []caps.Port{{Port: 25}}} v2 := &caps.Set{V: 1, Command: "relay", Daemon: true, Ports: []caps.Port{{Port: 25}, {Port: 465}}} shipCapRelease(t, s, "bab", "keymail", "v1", v1, []byte("good relay")) shipCapRelease(t, s, "bab", "keymail", "v2", v2, []byte("bad relay")) if err := Promote(ctx, s, priv, "bab", "keymail", "canary/x", "v1"); err != nil { t.Fatalf("Promote v1: %v", err) } addCapRoute(t, reg, host, "keymail", "canary/x", registry.BackingUnit, "port:25,port:465", false, false) // b is exec-backed (materializes nothing) and exists only to prove the // poison: it must never be moved onto the version a's restart rejected. addCapRoute(t, reg, "b.example.com", "keymail", "canary/x", "", "", false, false) rec := newCapsRecorder(opt, host) o, etc := capsTestOps(t, rec) if err := Pass(ctx, reg, s, pub, opt, func(string) error { return nil }, WithCapabilities(o)); err != nil { t.Fatalf("Pass v1: %v", err) } svcPath, sockPath := caps.DropInPaths(etc, host) wantSvc, err := os.ReadFile(svcPath) if err != nil { t.Fatalf("read converged service drop-in: %v", err) } wantSock, err := os.ReadFile(sockPath) if err != nil { t.Fatalf("read converged socket drop-in: %v", err) } if !strings.Contains(string(wantSock), "ListenStream=25") || strings.Contains(string(wantSock), "465") { t.Fatalf("v1 socket drop-in = %q, want :25 only", wantSock) } if err := Promote(ctx, s, priv, "bab", "keymail", "canary/x", "v2"); err != nil { t.Fatalf("Promote v2: %v", err) } rec.fail["restart "+caps.ServiceUnit(host)] = errors.New("relay failed to start") var out bytes.Buffer withCapturedStdout(t, &out, func() { if err := Pass(ctx, reg, s, pub, opt, func(string) error { return nil }, WithCapabilities(o)); err != nil { t.Fatalf("Pass v2: %v", err) } }) if v, _ := LiveVersion(opt, host); v != "v1" { t.Fatalf("live = %q, want v1 — the symlink must be rolled back", v) } gotSvc, _ := os.ReadFile(svcPath) gotSock, _ := os.ReadFile(sockPath) if !bytes.Equal(gotSvc, wantSvc) { t.Fatalf("service drop-in = %q, want v1's %q", gotSvc, wantSvc) } if !bytes.Equal(gotSock, wantSock) { t.Fatalf("socket drop-in = %q, want v1's %q — a rolled-back version must not leave :465 bound", gotSock, wantSock) } // Bytes are not a binding: restoring the socket half has to re-run the // socket sequence, or systemd keeps the listener the rolled-back version // asked for. if !rec.called("restart " + caps.SocketUnit(host)) { t.Fatalf("calls = %v, want the socket unit re-bound during the restore", rec.calls) } // The ordering inside the rollback is load-bearing and easy to get // backwards: the symlink goes back FIRST, so the service the restore // starts comes up on the OLD binary. Restoring first would start v2's // binary under v1's unit text — the failure this whole branch exists to // undo. lastLiveAt, not liveAt: the same verb already ran during the // forward converge. if v, seen := rec.lastLiveAt("start " + caps.ServiceUnit(host)); !seen || v != "v1" { t.Fatalf("the restore's service start observed live=%q (seen=%v), want v1 — the symlink must be rolled back BEFORE the drop-ins are", v, seen) } if v, _ := LiveVersion(opt, "b.example.com"); v != "v1" { t.Fatalf("b live = %q, want v1 — the failed version must be poisoned for the rest of the pass", v) } line := out.String() if !strings.Contains(line, "rolled the symlink back") { t.Fatalf("printed %q, want a line saying the symlink was rolled back", line) } if !strings.Contains(line, "capability drop-ins") { t.Fatalf("printed %q, want a line saying the drop-ins were restored too", line) } } // TestExecBackedRouteWithCapabilitiesMaterializesNothing: Backing == "" is // the whole existing fleet. A release declaring the full relay set must // change no unit text on such a route (its process is an exec child, not a // carlos-app@ unit) while adoption itself proceeds exactly as before. This // is also why no existing test in this package can reach the converge path. func TestExecBackedRouteWithCapabilitiesMaterializesNothing(t *testing.T) { s := NewDirStore(t.TempDir()) ctx := context.Background() priv, pub := testKeys(t) opt := t.TempDir() reg := testRegistry(t) host := "exec.example.com" shipCapRelease(t, s, "bab", "keymail", "v1", relaySet(), []byte("relay")) if err := Promote(ctx, s, priv, "bab", "keymail", "canary/x", "v1"); err != nil { t.Fatalf("Promote: %v", err) } addCapRoute(t, reg, host, "keymail", "canary/x", "", "port:25", false, false) rec := newCapsRecorder(opt, host) o, etc := capsTestOps(t, rec) var restarted []string if err := Pass(ctx, reg, s, pub, opt, func(h string) error { restarted = append(restarted, h) return nil }, WithCapabilities(o)); err != nil { t.Fatalf("Pass: %v", err) } assertEmptyDir(t, etc, "an exec-backed route must materialize no unit text") if len(rec.calls) != 0 { t.Fatalf("systemctl calls = %v, want none", rec.calls) } if v, _ := LiveVersion(opt, host); v != "v1" { t.Fatalf("live = %q, want v1 — adoption itself must proceed", v) } if len(restarted) != 1 || restarted[0] != host { t.Fatalf("restarted = %v, want the ordinary callback for %s", restarted, host) } } // TestPlatformAliasRouteMaterializesNothing: a platform alias is a second // name for one process. A second carlos-app@ unit claiming :25 would break // the real one, so an alias materializes nothing — and, being outside // Adoptable(), is not adopted at all. func TestPlatformAliasRouteMaterializesNothing(t *testing.T) { s := NewDirStore(t.TempDir()) ctx := context.Background() priv, pub := testKeys(t) opt := t.TempDir() reg := testRegistry(t) host := "keymail.bab.apps.example.com" shipCapRelease(t, s, "bab", "keymail", "v1", relaySet(), []byte("relay")) if err := Promote(ctx, s, priv, "bab", "keymail", "canary/x", "v1"); err != nil { t.Fatalf("Promote: %v", err) } addCapRoute(t, reg, host, "keymail", "canary/x", registry.BackingUnit, "port:25", false, true) rec := newCapsRecorder(opt, host) o, etc := capsTestOps(t, rec) if err := Pass(ctx, reg, s, pub, opt, func(string) error { return nil }, WithCapabilities(o)); err != nil { t.Fatalf("Pass: %v", err) } assertEmptyDir(t, etc, "a platform alias must materialize no unit text") if len(rec.calls) != 0 { t.Fatalf("systemctl calls = %v, want none", rec.calls) } if v, _ := LiveVersion(opt, host); v != "" { t.Fatalf("live = %q, want unset — a platform alias is not adopted at all", v) } } // TestPassWithoutWithCapabilitiesIsUnchanged is the byte-identical-to-before // assertion, and the reason this task could land on its own: a unit-backed // route with a full capability manifest, adopted by a Pass that was never // given the option, writes no unit text anywhere and adopts exactly as it // always did. func TestPassWithoutWithCapabilitiesIsUnchanged(t *testing.T) { s := NewDirStore(t.TempDir()) ctx := context.Background() priv, pub := testKeys(t) opt := t.TempDir() reg := testRegistry(t) host := "unit.example.com" etc := t.TempDir() // stands in for /etc/systemd/system; nothing may reach it shipCapRelease(t, s, "bab", "keymail", "v1", relaySet(), []byte("relay")) if err := Promote(ctx, s, priv, "bab", "keymail", "canary/x", "v1"); err != nil { t.Fatalf("Promote: %v", err) } addCapRoute(t, reg, host, "keymail", "canary/x", registry.BackingUnit, "port:25", false, false) var restarted []string if err := Pass(ctx, reg, s, pub, opt, func(h string) error { restarted = append(restarted, h) return nil }); err != nil { t.Fatalf("Pass: %v", err) } assertEmptyDir(t, etc, "a Pass without WithCapabilities must write no unit text") if v, _ := LiveVersion(opt, host); v != "v1" { t.Fatalf("live = %q, want v1", v) } if len(restarted) != 1 || restarted[0] != host { t.Fatalf("restarted = %v, want the ordinary callback for %s", restarted, host) } } // TestUnconvergedRepointOfACapabilityRouteLogsTheWindow pins the 2026-08-03 // ruling on the canary fast loop. That loop runs inside carlos-edge.service // (ProtectSystem=strict) and CANNOT write /etc/systemd/system, so it repoints // a capability-bearing canary and leaves the unit shape to the hourly root // pass — a window of up to an hour where a new binary runs under the previous // version's drop-ins. The window is accepted; being silent about it is not. // // The signal is deliberately the cheapest honest one: "the new version's // manifest declares capabilities at all", not "the new set differs from what // is materialized" — the difference test needs the grants and a render of // both sides, and this line exists to mark the window, not to diff it. func TestUnconvergedRepointOfACapabilityRouteLogsTheWindow(t *testing.T) { s := NewDirStore(t.TempDir()) ctx := context.Background() priv, pub := testKeys(t) opt := t.TempDir() reg := testRegistry(t) shipCapRelease(t, s, "bab", "keymail", "v1", relaySet(), []byte("relay")) shipRelease(t, s, "bab", "plain", "v1", adoptPlatform, []byte("plain")) for _, app := range []string{"keymail", "plain"} { if err := Promote(ctx, s, priv, "bab", app, "canary/x", "v1"); err != nil { t.Fatalf("Promote %s: %v", app, err) } } addCapRoute(t, reg, "caps.example.com", "keymail", "canary/x", registry.BackingUnit, "port:25", false, false) // A unit-backed route whose release declares NOTHING must stay quiet — // otherwise the line fires on every ordinary tenant and nobody reads it. addCapRoute(t, reg, "plain.example.com", "plain", "canary/x", registry.BackingUnit, "", false, false) var out bytes.Buffer withCapturedStdout(t, &out, func() { if err := PassFilter(ctx, reg, s, pub, opt, func(string) error { return nil }, isCanaryTestRoute); err != nil { t.Fatalf("PassFilter: %v", err) } }) line := out.String() if !strings.Contains(line, "caps.example.com: v1 declares capabilities") { t.Fatalf("printed %q, want one line naming the unconverged capability repoint", line) } if !strings.Contains(line, "carlos-adopt.service") { t.Fatalf("printed %q, want the line to name what closes the window", line) } if strings.Contains(line, "plain.example.com: v1 declares") { t.Fatalf("printed %q, want silence for a route whose release declares nothing", line) } } // TestHibernatingUnitRouteConvergesWithoutStarting: a hibernating route // reaches adoptHost with restart == nil. Its drop-ins are unit TEXT, not a // wake, so they converge — but nothing may start the service, or an hourly // timer would defeat hibernation. func TestHibernatingUnitRouteConvergesWithoutStarting(t *testing.T) { s := NewDirStore(t.TempDir()) ctx := context.Background() priv, pub := testKeys(t) opt := t.TempDir() reg := testRegistry(t) host := "sleepy.example.com" // No daemon: daemon + hibernate is a contradiction converge refuses. set := &caps.Set{V: 1, Command: "web", Ports: []caps.Port{{Port: 25}}} shipCapRelease(t, s, "bab", "keymail", "v1", set, []byte("relay")) if err := Promote(ctx, s, priv, "bab", "keymail", "canary/x", "v1"); err != nil { t.Fatalf("Promote: %v", err) } addCapRoute(t, reg, host, "keymail", "canary/x", registry.BackingUnit, "port:25", true, false) rec := newCapsRecorder(opt, host) o, etc := capsTestOps(t, rec) if err := Pass(ctx, reg, s, pub, opt, func(string) error { t.Fatal("a hibernating host must never reach the restart callback") return nil }, WithCapabilities(o)); err != nil { t.Fatalf("Pass: %v", err) } svcPath, _ := caps.DropInPaths(etc, host) if _, err := os.Stat(svcPath); err != nil { t.Fatalf("stat %s: %v — a hibernating route's drop-ins still converge", svcPath, err) } if v, _ := LiveVersion(opt, host); v != "v1" { t.Fatalf("live = %q, want v1", v) } for _, c := range rec.calls { if strings.HasSuffix(c, caps.ServiceUnit(host)) { t.Fatalf("calls = %v, want no verb against the SERVICE unit — a drop-in is not a wake", rec.calls) } } } // TestConvergeRefusalRefusesTheVersion: a refusal from converge falls exactly // where an ensureStaged failure falls — nothing pointed live, the box keeps // serving what it has, one line naming the operator's fix. Pointing live // anyway would run a new binary against a unit shape the platform has just // declined to write. // // The refusal used here is spec §4e's daemon + hibernate contradiction, // because it is refused with nothing written at all. func TestConvergeRefusalRefusesTheVersion(t *testing.T) { s := NewDirStore(t.TempDir()) ctx := context.Background() priv, pub := testKeys(t) opt := t.TempDir() reg := testRegistry(t) host := "sleepy.example.com" shipRelease(t, s, "bab", "keymail", "v1", adoptPlatform, []byte("old")) if err := StageArtifact(opt, "bab", "v1", "keymail", []byte("old")); err != nil { t.Fatal(err) } addCapRoute(t, reg, host, "keymail", "canary/x", registry.BackingUnit, "port:25", true, false) if err := PointLive(opt, host, "bab", "v1", "keymail"); err != nil { t.Fatal(err) } shipCapRelease(t, s, "bab", "keymail", "v2", relaySet(), []byte("new")) if err := Promote(ctx, s, priv, "bab", "keymail", "canary/x", "v2"); err != nil { t.Fatalf("Promote v2: %v", err) } rec := newCapsRecorder(opt, host) o, etc := capsTestOps(t, rec) var out bytes.Buffer withCapturedStdout(t, &out, func() { if err := Pass(ctx, reg, s, pub, opt, func(string) error { return nil }, WithCapabilities(o)); err != nil { t.Fatalf("Pass: %v", err) } }) if v, _ := LiveVersion(opt, host); v != "v1" { t.Fatalf("live = %q, want v1 — a refused converge must not point live", v) } assertEmptyDir(t, etc, "a refusal writes nothing") if len(rec.calls) != 0 { t.Fatalf("systemctl calls = %v, want none", rec.calls) } line := out.String() if !strings.Contains(line, "not pointed live") { t.Fatalf("printed %q, want a line saying the version was not pointed live", line) } if !strings.Contains(line, "-hibernate=false") { t.Fatalf("printed %q, want converge's own line naming the operator's fix", line) } } // TestArmedPassRefusesAVersionWhoseManifestItCouldNotObtain is the second of // Task 7's two pins, in the one place it can bite: the binary is staged (so // ensureStaged succeeds) but its manifest is obtainable neither from the // sibling nor from the bucket. caps.Plan's nil Set means "leave the drop-ins // alone" — correct for a pass surveying the box, and exactly wrong here, // where it would point a new binary at the previous version's unit shape and // say nothing. So an armed pass refuses the version. // // The second half is the compatibility half: the SAME situation on a pass // with no WithCapabilities adopts normally, which is what keeps every // existing caller byte-identical. func TestArmedPassRefusesAVersionWhoseManifestItCouldNotObtain(t *testing.T) { dir := t.TempDir() s := NewDirStore(dir) ctx := context.Background() priv, pub := testKeys(t) opt := t.TempDir() reg := testRegistry(t) host := "keymail.dev" shipRelease(t, s, "bab", "keymail", "v1", adoptPlatform, []byte("old")) if err := StageArtifact(opt, "bab", "v1", "keymail", []byte("old")); err != nil { t.Fatal(err) } if err := StageManifest(opt, "bab", "v1", "keymail", []byte(`{"schema":1,"app":"keymail","version":"v1","kind":"binary"}`)); err != nil { t.Fatal(err) } addCapRoute(t, reg, host, "keymail", "canary/x", registry.BackingUnit, "port:25", false, false) if err := PointLive(opt, host, "bab", "v1", "keymail"); err != nil { t.Fatal(err) } // v2's binary is on disk with no sibling, and its manifest object is then // removed from the store: the back-fill fetch cannot answer either. shipCapRelease(t, s, "bab", "keymail", "v2", relaySet(), []byte("new")) if err := Promote(ctx, s, priv, "bab", "keymail", "canary/x", "v2"); err != nil { t.Fatalf("Promote v2: %v", err) } if err := StageArtifact(opt, "bab", "v2", "keymail", []byte("new")); err != nil { t.Fatal(err) } if err := os.Remove(filepath.Join(dir, filepath.FromSlash(ManifestKey("bab", "keymail", "v2")))); err != nil { t.Fatalf("remove v2 manifest: %v", err) } rec := newCapsRecorder(opt, host) o, etc := capsTestOps(t, rec) var out bytes.Buffer withCapturedStdout(t, &out, func() { if err := Pass(ctx, reg, s, pub, opt, func(string) error { return nil }, WithCapabilities(o)); err != nil { t.Fatalf("armed Pass: %v", err) } }) if v, _ := LiveVersion(opt, host); v != "v1" { t.Fatalf("live = %q, want v1 — an unknown unit shape must refuse the version", v) } assertEmptyDir(t, etc, "a refusal writes nothing") if line := out.String(); !strings.Contains(line, "not pointed live") { t.Fatalf("printed %q, want a line saying the version was not pointed live", line) } // Unarmed, the identical situation adopts exactly as it always did. if err := Pass(ctx, reg, s, pub, opt, func(string) error { return nil }); err != nil { t.Fatalf("unarmed Pass: %v", err) } if v, _ := LiveVersion(opt, host); v != "v2" { t.Fatalf("live = %q after an unarmed pass, want v2 — capabilities off means behaviour unchanged", v) } } // assertEmptyDir fails unless dir has no entries at all. func assertEmptyDir(t *testing.T, dir, why string) { t.Helper() ents, err := os.ReadDir(dir) if err != nil { t.Fatalf("read %s: %v", dir, err) } if len(ents) != 0 { var names []string for _, e := range ents { names = append(names, e.Name()) } t.Fatalf("%s contains %v — %s", dir, names, why) } } // TestAdoptRemovesDropInsWhenTheNewVersionDeclaresNothing is Task 7's second // pin, and the mutant it kills is a one-word one: Plan{Set: m.Capabilities}. // Manifest.Capabilities is a nil pointer for BOTH a legacy manifest and a // fetch that never happened, and caps.Plan reads a nil Set as "leave the // drop-ins exactly as they are". Passing it through verbatim would mean a // capability DROPPED from a manifest is never removed — a relay that stopped // asking for :25 would keep binding it, which is the exact opposite of spec // §4d's removal semantics. A manifest that RESOLVED always carries a Set, // empty when it declares nothing. func TestAdoptRemovesDropInsWhenTheNewVersionDeclaresNothing(t *testing.T) { s := NewDirStore(t.TempDir()) ctx := context.Background() priv, pub := testKeys(t) opt := t.TempDir() reg := testRegistry(t) host := "keymail.dev" shipCapRelease(t, s, "bab", "keymail", "v1", relaySet(), []byte("relay")) if err := Promote(ctx, s, priv, "bab", "keymail", "canary/x", "v1"); err != nil { t.Fatalf("Promote v1: %v", err) } addCapRoute(t, reg, host, "keymail", "canary/x", registry.BackingUnit, "port:25", false, false) rec := newCapsRecorder(opt, host) o, etc := capsTestOps(t, rec) if err := Pass(ctx, reg, s, pub, opt, func(string) error { return nil }, WithCapabilities(o)); err != nil { t.Fatalf("Pass v1: %v", err) } svcPath, sockPath := caps.DropInPaths(etc, host) for _, p := range []string{svcPath, sockPath} { if _, err := os.Stat(p); err != nil { t.Fatalf("stat %s after v1: %v", p, err) } } // v2 declares nothing at all — a plain manifest, exactly like every // release in every production bucket today. shipRelease(t, s, "bab", "keymail", "v2", adoptPlatform, []byte("plain relay")) if err := Promote(ctx, s, priv, "bab", "keymail", "canary/x", "v2"); err != nil { t.Fatalf("Promote v2: %v", err) } if err := Pass(ctx, reg, s, pub, opt, func(string) error { return nil }, WithCapabilities(o)); err != nil { t.Fatalf("Pass v2: %v", err) } if v, _ := LiveVersion(opt, host); v != "v2" { t.Fatalf("live = %q, want v2", v) } for _, p := range []string{svcPath, sockPath} { if _, err := os.Stat(p); !os.IsNotExist(err) { b, _ := os.ReadFile(p) t.Fatalf("%s still exists after a version that declares nothing (%q) — a dropped capability must be removed", p, b) } } // Removing the socket half has to reach systemd, or the listener the // removed file asked for is still bound. if !rec.called("restart " + caps.SocketUnit(host)) { t.Fatalf("calls = %v, want the socket unit re-bound when its drop-in was removed", rec.calls) } } // TestDaemonVersionChangeStartsTheServiceExactlyOnce covers the other half of // the deferred restart: a version change whose SOCKET shape is unchanged, so // nothing in converge's socket ordering runs and step 9's service restart is // the only thing that would have cycled the unit. Deferred, the app is // started exactly once, after the symlink moves — one cycle, on the new // binary, instead of two with the first one running the old. func TestDaemonVersionChangeStartsTheServiceExactlyOnce(t *testing.T) { s := NewDirStore(t.TempDir()) ctx := context.Background() priv, pub := testKeys(t) opt := t.TempDir() reg := testRegistry(t) host := "keymail.dev" v1 := &caps.Set{V: 1, Command: "relay", Daemon: true, Ports: []caps.Port{{Port: 25}}} v2 := &caps.Set{V: 1, Command: "relay-two", Daemon: true, Ports: []caps.Port{{Port: 25}}} shipCapRelease(t, s, "bab", "keymail", "v1", v1, []byte("relay one")) shipCapRelease(t, s, "bab", "keymail", "v2", v2, []byte("relay two")) if err := Promote(ctx, s, priv, "bab", "keymail", "canary/x", "v1"); err != nil { t.Fatalf("Promote v1: %v", err) } addCapRoute(t, reg, host, "keymail", "canary/x", registry.BackingUnit, "port:25", false, false) rec := newCapsRecorder(opt, host) o, _ := capsTestOps(t, rec) restart := func(string) error { return nil } if err := Pass(ctx, reg, s, pub, opt, restart, WithCapabilities(o)); err != nil { t.Fatalf("Pass v1: %v", err) } if n, _ := rec.serviceCycles(host); n != 1 { t.Fatalf("first adoption started the service %d time(s), want 1 (calls: %v)", n, rec.calls) } rec.reset() if err := Promote(ctx, s, priv, "bab", "keymail", "canary/x", "v2"); err != nil { t.Fatalf("Promote v2: %v", err) } if err := Pass(ctx, reg, s, pub, opt, restart, WithCapabilities(o)); err != nil { t.Fatalf("Pass v2: %v", err) } if v, _ := LiveVersion(opt, host); v != "v2" { t.Fatalf("live = %q, want v2", v) } if n, last := rec.serviceCycles(host); n != 1 || last != "v2" { t.Fatalf("service started/restarted %d time(s), last at live=%q; want exactly 1, at v2 (calls: %v)", n, last, rec.calls) } } // TestConvergeFailureAfterItsReloadRestoresTheDropIns is the failure the // converge contract hands to this caller by name: a systemd verb that fails // AFTER the daemon-reload leaves the new version's unit text on disk, loaded, // and deliberately un-rolled-back (Converge reports Changed with the error // and says the version-change caller owns it). Nothing points live, so those // bytes describe a version this box is not running — and they are permanent // if nobody undoes them: the next pass finds no diff, snapshots them as its // own Prev, and every later "restore" restores the wrong shape. // // The failing verb here is `enable` on the socket unit, which is post-reload // and NOT one of the diverging ones Converge rolls back itself. func TestConvergeFailureAfterItsReloadRestoresTheDropIns(t *testing.T) { s := NewDirStore(t.TempDir()) ctx := context.Background() priv, pub := testKeys(t) opt := t.TempDir() reg := testRegistry(t) host := "keymail.dev" // The box is on v1, which declares nothing, so there are no drop-ins yet. shipRelease(t, s, "bab", "keymail", "v1", adoptPlatform, []byte("old")) if err := StageArtifact(opt, "bab", "v1", "keymail", []byte("old")); err != nil { t.Fatal(err) } addCapRoute(t, reg, host, "keymail", "canary/x", registry.BackingUnit, "port:25", false, false) if err := PointLive(opt, host, "bab", "v1", "keymail"); err != nil { t.Fatal(err) } shipCapRelease(t, s, "bab", "keymail", "v2", relaySet(), []byte("new")) if err := Promote(ctx, s, priv, "bab", "keymail", "canary/x", "v2"); err != nil { t.Fatalf("Promote v2: %v", err) } rec := newCapsRecorder(opt, host) o, etc := capsTestOps(t, rec) rec.fail["enable "+caps.SocketUnit(host)] = errors.New("unit masked") if err := Pass(ctx, reg, s, pub, opt, func(string) error { return nil }, WithCapabilities(o)); err != nil { t.Fatalf("Pass: %v", err) } if v, _ := LiveVersion(opt, host); v != "v1" { t.Fatalf("live = %q, want v1", v) } svcPath, sockPath := caps.DropInPaths(etc, host) for _, p := range []string{svcPath, sockPath} { if b, err := os.ReadFile(p); err == nil { t.Fatalf("%s survives a converge that failed after its reload (%q) — v2's unit shape must not be left on disk under v1's binary", p, b) } } } // TestConvergeRefusalLeavesAnEarlierVersionsDropInsAlone is the other side of // that coin, and the reason the restore above is conditional. An ORDINARY // refusal writes nothing — and the refusals that fire earliest fire before // the drop-ins are even read, so the snapshot is EMPTY. caps.Restore reads an // empty snapshot as "this route had no generated drop-ins" and honours it by // REMOVING the ones it finds, so restoring unconditionally on a refusal would // delete the shape the running version is serving under. // // The refusal here is spec §4e's daemon + hibernate contradiction, which // Converge checks before it reads anything. func TestConvergeRefusalLeavesAnEarlierVersionsDropInsAlone(t *testing.T) { s := NewDirStore(t.TempDir()) ctx := context.Background() priv, pub := testKeys(t) opt := t.TempDir() reg := testRegistry(t) host := "sleepy.example.com" // v1 is a hibernating tenant with a granted port and no daemon: a // perfectly legal shape, and it converges. v1 := &caps.Set{V: 1, Command: "web", Ports: []caps.Port{{Port: 25}}} shipCapRelease(t, s, "bab", "keymail", "v1", v1, []byte("web one")) if err := Promote(ctx, s, priv, "bab", "keymail", "canary/x", "v1"); err != nil { t.Fatalf("Promote v1: %v", err) } addCapRoute(t, reg, host, "keymail", "canary/x", registry.BackingUnit, "port:25", true, false) rec := newCapsRecorder(opt, host) o, etc := capsTestOps(t, rec) if err := Pass(ctx, reg, s, pub, opt, func(string) error { return nil }, WithCapabilities(o)); err != nil { t.Fatalf("Pass v1: %v", err) } svcPath, sockPath := caps.DropInPaths(etc, host) wantSvc, err := os.ReadFile(svcPath) if err != nil { t.Fatalf("read v1 service drop-in: %v", err) } wantSock, err := os.ReadFile(sockPath) if err != nil { t.Fatalf("read v1 socket drop-in: %v", err) } // v2 adds `daemon` to a hibernating route — refused before anything is // read or written. shipCapRelease(t, s, "bab", "keymail", "v2", relaySet(), []byte("web two")) if err := Promote(ctx, s, priv, "bab", "keymail", "canary/x", "v2"); err != nil { t.Fatalf("Promote v2: %v", err) } if err := Pass(ctx, reg, s, pub, opt, func(string) error { return nil }, WithCapabilities(o)); err != nil { t.Fatalf("Pass v2: %v", err) } if v, _ := LiveVersion(opt, host); v != "v1" { t.Fatalf("live = %q, want v1 — the refusal must not point live", v) } gotSvc, _ := os.ReadFile(svcPath) gotSock, _ := os.ReadFile(sockPath) if !bytes.Equal(gotSvc, wantSvc) || !bytes.Equal(gotSock, wantSock) { t.Fatalf("drop-ins after a refusal = %q / %q, want v1's %q / %q left exactly as they were", gotSvc, gotSock, wantSvc, wantSock) } } // TestConvergeRefusalDoesNotPoisonTheVersionForOtherHosts is the 2026-08-03 // ruling on the poison. A refusal bounces nothing — no symlink moved, no unit // cycled — so there is no bad build to keep off the next host, which is the // only thing the poison exists for. And the refusal is often HOST-shaped: the // one here is a port a third route's drop-in already holds on this box. // Poisoning on that would let one misconfigured route freeze every other host // out of a version that is fine for them, every pass, until an operator // noticed. // // The ordering is the point: a.example.com is refused FIRST (routes are swept // in host order), so a poison set there is what b.example.com would trip over. func TestConvergeRefusalDoesNotPoisonTheVersionForOtherHosts(t *testing.T) { s := NewDirStore(t.TempDir()) ctx := context.Background() priv, pub := testKeys(t) opt := t.TempDir() reg := testRegistry(t) shipCapRelease(t, s, "bab", "keymail", "v1", relaySet(), []byte("relay")) if err := Promote(ctx, s, priv, "bab", "keymail", "canary/x", "v1"); err != nil { t.Fatalf("Promote: %v", err) } // a holds the grant for :25 and is refused, because something else on // this box already claims it. b declares the same port with NO grant, so // it claims nothing, materializes only its service half, and is fine. addCapRoute(t, reg, "a.example.com", "keymail", "canary/x", registry.BackingUnit, "port:25", false, false) addCapRoute(t, reg, "b.example.com", "keymail", "canary/x", registry.BackingUnit, "", false, false) rec := newCapsRecorder(opt, "a.example.com") o, etc := capsTestOps(t, rec) // A drop-in belonging to a route this pass never sees, holding :25. incumbent := filepath.Join(etc, caps.SocketUnit("zz.example.com")+".d") if err := os.MkdirAll(incumbent, 0o755); err != nil { t.Fatal(err) } if err := os.WriteFile(filepath.Join(incumbent, caps.DropInName), []byte(caps.Marker+"\n[Socket]\nListenStream=25\n"), 0o644); err != nil { t.Fatal(err) } var out bytes.Buffer withCapturedStdout(t, &out, func() { if err := Pass(ctx, reg, s, pub, opt, func(string) error { return nil }, WithCapabilities(o)); err != nil { t.Fatalf("Pass: %v", err) } }) if v, _ := LiveVersion(opt, "a.example.com"); v != "" { t.Fatalf("a live = %q, want unset — its own refusal keeps it off the version", v) } if v, _ := LiveVersion(opt, "b.example.com"); v != "v1" { t.Fatalf("b live = %q, want v1 — another host's refusal must not freeze this one out of the version", v) } line := out.String() if strings.Contains(line, "failed on another host this pass") { t.Fatalf("printed %q, want no poison — a refusal is not a bad build", line) } if !strings.Contains(line, "a.example.com: capabilities for v1 were refused") { t.Fatalf("printed %q, want a's own refusal named", line) } } // TestPointLiveFailureRestoresTheDropIns: converge succeeded, the symlink did // not move. The box is still serving the previous version, and the unit shape // on disk is the new one — a port the running binary never asked for, bound // in public. The drop-ins have to follow the symlink, which stayed put. // // PointLive is failed by parking a non-empty directory at the temp-link path // it renames through, which is as close to a disk-level swap failure as this // package can get without a fake filesystem. func TestPointLiveFailureRestoresTheDropIns(t *testing.T) { s := NewDirStore(t.TempDir()) ctx := context.Background() priv, pub := testKeys(t) opt := t.TempDir() reg := testRegistry(t) host := "keymail.dev" v1 := &caps.Set{V: 1, Command: "relay", Daemon: true, Ports: []caps.Port{{Port: 25}}} v2 := &caps.Set{V: 1, Command: "relay", Daemon: true, Ports: []caps.Port{{Port: 25}, {Port: 465}}} shipCapRelease(t, s, "bab", "keymail", "v1", v1, []byte("relay one")) shipCapRelease(t, s, "bab", "keymail", "v2", v2, []byte("relay two")) if err := Promote(ctx, s, priv, "bab", "keymail", "canary/x", "v1"); err != nil { t.Fatalf("Promote v1: %v", err) } addCapRoute(t, reg, host, "keymail", "canary/x", registry.BackingUnit, "port:25,port:465", false, false) rec := newCapsRecorder(opt, host) o, etc := capsTestOps(t, rec) if err := Pass(ctx, reg, s, pub, opt, func(string) error { return nil }, WithCapabilities(o)); err != nil { t.Fatalf("Pass v1: %v", err) } svcPath, sockPath := caps.DropInPaths(etc, host) wantSvc, _ := os.ReadFile(svcPath) wantSock, _ := os.ReadFile(sockPath) if !strings.Contains(string(wantSock), "ListenStream=25") || strings.Contains(string(wantSock), "465") { t.Fatalf("v1 socket drop-in = %q, want :25 only", wantSock) } // Block the symlink swap. blocker := LiveLink(opt, host) + ".tmp" if err := os.MkdirAll(filepath.Join(blocker, "occupied"), 0o755); err != nil { t.Fatalf("park a blocker at %s: %v", blocker, err) } if err := Promote(ctx, s, priv, "bab", "keymail", "canary/x", "v2"); err != nil { t.Fatalf("Promote v2: %v", err) } var out bytes.Buffer withCapturedStdout(t, &out, func() { if err := Pass(ctx, reg, s, pub, opt, func(string) error { return nil }, WithCapabilities(o)); err != nil { t.Fatalf("Pass v2: %v", err) } }) if v, _ := LiveVersion(opt, host); v != "v1" { t.Fatalf("live = %q, want v1 — the repoint was supposed to fail", v) } gotSvc, _ := os.ReadFile(svcPath) gotSock, _ := os.ReadFile(sockPath) if !bytes.Equal(gotSvc, wantSvc) || !bytes.Equal(gotSock, wantSock) { t.Fatalf("drop-ins = %q / %q, want v1's %q / %q — :465 must not stay bound under a binary that never asked for it", gotSvc, gotSock, wantSvc, wantSock) } if line := out.String(); !strings.Contains(line, "repoint failed") || !strings.Contains(line, "capability drop-ins") { t.Fatalf("printed %q, want a line naming both the failed repoint and the restored drop-ins", line) } } // TestArmedPassConvergesFromTheStagedSiblingWhenTheBucketCannotAnswer pins // the read ensureStaged does on its already-staged path. The binary is on // disk with its sibling beside it and the bucket object is gone: the sibling // IS the manifest in hand there, and dropping that read would send this // straight to a bucket fetch that cannot answer, then to a refusal of a // version whose capabilities are sitting on disk, readable, all along. func TestArmedPassConvergesFromTheStagedSiblingWhenTheBucketCannotAnswer(t *testing.T) { dir := t.TempDir() s := NewDirStore(dir) ctx := context.Background() priv, pub := testKeys(t) opt := t.TempDir() reg := testRegistry(t) host := "keymail.dev" shipRelease(t, s, "bab", "keymail", "v1", adoptPlatform, []byte("old")) if err := StageArtifact(opt, "bab", "v1", "keymail", []byte("old")); err != nil { t.Fatal(err) } addCapRoute(t, reg, host, "keymail", "canary/x", registry.BackingUnit, "port:25", false, false) if err := PointLive(opt, host, "bab", "v1", "keymail"); err != nil { t.Fatal(err) } set := relaySet() shipCapRelease(t, s, "bab", "keymail", "v2", set, []byte("new")) if err := Promote(ctx, s, priv, "bab", "keymail", "canary/x", "v2"); err != nil { t.Fatalf("Promote v2: %v", err) } // v2 is staged the way a box that adopted it once already would have it: // binary AND sibling on disk. Then the bucket object goes away. if err := StageArtifact(opt, "bab", "v2", "keymail", []byte("new")); err != nil { t.Fatal(err) } raw, _, err := s.Get(ctx, ManifestKey("bab", "keymail", "v2")) if err != nil { t.Fatalf("read v2 manifest: %v", err) } if err := StageManifest(opt, "bab", "v2", "keymail", raw); err != nil { t.Fatalf("StageManifest: %v", err) } if err := os.Remove(filepath.Join(dir, filepath.FromSlash(ManifestKey("bab", "keymail", "v2")))); err != nil { t.Fatalf("remove v2 manifest: %v", err) } rec := newCapsRecorder(opt, host) o, etc := capsTestOps(t, rec) if err := Pass(ctx, reg, s, pub, opt, func(string) error { return nil }, WithCapabilities(o)); err != nil { t.Fatalf("Pass: %v", err) } if v, _ := LiveVersion(opt, host); v != "v2" { t.Fatalf("live = %q, want v2 — the sibling answered, so nothing needed the bucket", v) } g := caps.Grants{Ports: map[int]bool{25: true}} svcPath, _ := caps.DropInPaths(etc, host) wantSvc, err := caps.RenderService("keymail", set, g) if err != nil { t.Fatalf("RenderService: %v", err) } gotSvc, err := os.ReadFile(svcPath) if err != nil || !bytes.Equal(gotSvc, wantSvc) { t.Fatalf("service drop-in = %q (%v), want the sibling's capabilities materialized: %q", gotSvc, err, wantSvc) } } // TestRestartFailureSaysSoWhenTheRestoreAlsoFails: the summary line is the // only thing most operators read, and it must not claim a restore that did // not happen. A rollback that could not put the drop-ins back is the worst // state this path can end in — the symlink says v1, the unit text says v2 — // so it is exactly the case that must not be reported as a tidy rollback. // // The restore is failed by making the socket drop-in's directory unwritable // at the moment the daemon restart fails, i.e. after the forward converge has // written it and before the rollback tries to. func TestRestartFailureSaysSoWhenTheRestoreAlsoFails(t *testing.T) { if os.Geteuid() == 0 { t.Skip("running as root: directory permissions do not refuse a write") } s := NewDirStore(t.TempDir()) ctx := context.Background() priv, pub := testKeys(t) opt := t.TempDir() reg := testRegistry(t) host := "keymail.dev" v1 := &caps.Set{V: 1, Command: "relay", Daemon: true, Ports: []caps.Port{{Port: 25}}} v2 := &caps.Set{V: 1, Command: "relay", Daemon: true, Ports: []caps.Port{{Port: 25}, {Port: 465}}} shipCapRelease(t, s, "bab", "keymail", "v1", v1, []byte("relay one")) shipCapRelease(t, s, "bab", "keymail", "v2", v2, []byte("relay two")) if err := Promote(ctx, s, priv, "bab", "keymail", "canary/x", "v1"); err != nil { t.Fatalf("Promote v1: %v", err) } addCapRoute(t, reg, host, "keymail", "canary/x", registry.BackingUnit, "port:25,port:465", false, false) rec := newCapsRecorder(opt, host) o, etc := capsTestOps(t, rec) if err := Pass(ctx, reg, s, pub, opt, func(string) error { return nil }, WithCapabilities(o)); err != nil { t.Fatalf("Pass v1: %v", err) } _, sockPath := caps.DropInPaths(etc, host) sockDir := filepath.Dir(sockPath) t.Cleanup(func() { os.Chmod(sockDir, 0o755) }) rec.fail["restart "+caps.ServiceUnit(host)] = errors.New("relay failed to start") rec.onCall = func(call string) { if call == "restart "+caps.ServiceUnit(host) { if err := os.Chmod(sockDir, 0o555); err != nil { t.Errorf("chmod %s: %v", sockDir, err) } } } if err := Promote(ctx, s, priv, "bab", "keymail", "canary/x", "v2"); err != nil { t.Fatalf("Promote v2: %v", err) } var out bytes.Buffer withCapturedStdout(t, &out, func() { if err := Pass(ctx, reg, s, pub, opt, func(string) error { return nil }, WithCapabilities(o)); err != nil { t.Fatalf("Pass v2: %v", err) } }) line := out.String() if !strings.Contains(line, "could not restore") { t.Fatalf("printed %q, want the summary to say the restore failed", line) } if strings.Contains(line, "and restored the previous version's capability drop-ins") { t.Fatalf("printed %q, want no claim of a restore that did not happen", line) } if v, _ := LiveVersion(opt, host); v != "v1" { t.Fatalf("live = %q, want v1 — the symlink half of the rollback still has to happen", v) } } // TestFirstAdoptionRestartFailureLeavesTheConvergedDropIns: with no previous // version there is no symlink to roll back — the box stays live on the // version whose restart just failed. Handing caps.Restore the empty snapshot // that describes "this route had no drop-ins" would then REMOVE the shape the // live version needs, leaving it unconfigured until the hourly grant-drift // pass rebuilds it. The honest state is the one the symlink names. func TestFirstAdoptionRestartFailureLeavesTheConvergedDropIns(t *testing.T) { s := NewDirStore(t.TempDir()) ctx := context.Background() priv, pub := testKeys(t) opt := t.TempDir() reg := testRegistry(t) host := "keymail.dev" set := relaySet() shipCapRelease(t, s, "bab", "keymail", "v1", set, []byte("relay")) if err := Promote(ctx, s, priv, "bab", "keymail", "canary/x", "v1"); err != nil { t.Fatalf("Promote: %v", err) } addCapRoute(t, reg, host, "keymail", "canary/x", registry.BackingUnit, "port:25", false, false) rec := newCapsRecorder(opt, host) o, etc := capsTestOps(t, rec) rec.fail["restart "+caps.ServiceUnit(host)] = errors.New("relay failed to start") var out bytes.Buffer withCapturedStdout(t, &out, func() { if err := Pass(ctx, reg, s, pub, opt, func(string) error { return nil }, WithCapabilities(o)); err != nil { t.Fatalf("Pass: %v", err) } }) if v, _ := LiveVersion(opt, host); v != "v1" { t.Fatalf("live = %q, want v1 — there is no previous version to roll back to", v) } g := caps.Grants{Ports: map[int]bool{25: true}} svcPath, sockPath := caps.DropInPaths(etc, host) wantSvc, _ := caps.RenderService("keymail", set, g) wantSock, _ := caps.RenderSocket(set, g) gotSvc, err := os.ReadFile(svcPath) if err != nil || !bytes.Equal(gotSvc, wantSvc) { t.Fatalf("service drop-in = %q (%v), want the converged shape the live version needs: %q", gotSvc, err, wantSvc) } gotSock, err := os.ReadFile(sockPath) if err != nil || !bytes.Equal(gotSock, wantSock) { t.Fatalf("socket drop-in = %q (%v), want %q", gotSock, err, wantSock) } if line := out.String(); !strings.Contains(line, "no previous version to roll back to") { t.Fatalf("printed %q, want the line to say why nothing was rolled back", line) } } // ---- the production flag on the adopt path (2026-08-04 spec §6) ------------- // The default: no flag, no window (2026-08-04 spec §2). The whole fleet // runs on canary/* today, so nothing live loses protection — and a // rapid-iteration app's stable is just another channel. func TestPassNoProductionFlagInstallsImmediately(t *testing.T) { s := NewDirStore(t.TempDir()) ctx := context.Background() priv, pub := testKeys(t) opt := t.TempDir() reg := testRegistry(t) shipRelease(t, s, "bab", "myapp", "v1", adoptPlatform, []byte("bin")) if err := Promote(ctx, s, priv, "bab", "myapp", "stable", "v1"); err != nil { t.Fatal(err) } addRoute(t, reg, "h.example.com", "myapp", "", false) if err := Pass(ctx, reg, s, pub, opt, func(string) error { return nil }); err != nil { t.Fatalf("Pass: %v", err) } if v, _ := LiveVersion(opt, "h.example.com"); v != "v1" { t.Fatalf("live = %q, want v1 — no production flag means no window", v) } } // Fast-track: a production app's held head installs the moment its pointer // carries the root-signed waiver (spec §5). func TestPassFastTrackSkipsTheWindow(t *testing.T) { s := NewDirStore(t.TempDir()) ctx := context.Background() priv, pub := testKeys(t) opt := t.TempDir() reg := testRegistry(t) shipRelease(t, s, "bab", "myapp", "v1", adoptPlatform, []byte("bin")) if err := Promote(ctx, s, priv, "bab", "myapp", "stable", "v1"); err != nil { t.Fatal(err) } if err := SetProduction(ctx, s, priv, "bab", "myapp", true, "test"); err != nil { t.Fatal(err) } addRoute(t, reg, "h.example.com", "myapp", "", false) if err := Pass(ctx, reg, s, pub, opt, func(string) error { return nil }); err != nil { t.Fatal(err) } if v, _ := LiveVersion(opt, "h.example.com"); v != "" { t.Fatalf("live = %q, want held (production, no waiver yet)", v) } if _, err := FastTrack(ctx, s, priv, "bab", "myapp", "stable", "test"); err != nil { t.Fatal(err) } if err := Pass(ctx, reg, s, pub, opt, func(string) error { return nil }); err != nil { t.Fatal(err) } if v, _ := LiveVersion(opt, "h.example.com"); v != "v1" { t.Fatalf("live = %q, want v1 — the waiver did not skip the window", v) } } // A production app whose flag OBJECT rots (exists, won't verify) keeps // baking — an attacker who can corrupt but not sign shortens nothing. func TestPassCorruptProductionFlagFailsTowardBaking(t *testing.T) { s := NewDirStore(t.TempDir()) ctx := context.Background() priv, pub := testKeys(t) opt := t.TempDir() reg := testRegistry(t) shipRelease(t, s, "bab", "myapp", "v1", adoptPlatform, []byte("bin")) if err := Promote(ctx, s, priv, "bab", "myapp", "stable", "v1"); err != nil { t.Fatal(err) } if _, err := s.Put(ctx, ProductionKey("bab", "myapp"), []byte("not a wrapper"), ""); err != nil { t.Fatal(err) } addRoute(t, reg, "h.example.com", "myapp", "", false) if err := Pass(ctx, reg, s, pub, opt, func(string) error { return nil }); err != nil { t.Fatalf("Pass: %v", err) } if v, _ := LiveVersion(opt, "h.example.com"); v != "" { t.Fatalf("live = %q, want held — a corrupt flag must read as production", v) } } // ORDERING, pinned: ObserveProduction runs BEFORE this pass's bake verdicts // are taken, so the bucket flag re-latches a wiped memory within ONE pass — // the whole basis of prodMemory's "a mixed-version rollout costs at most one // pass of this memory" comment. Simulates the old binary that re-marshals a // struct with no Production field and thereby deletes the latch. func TestPassReLatchesProductionMemoryInTheSamePass(t *testing.T) { s := NewDirStore(t.TempDir()) ctx := context.Background() priv, pub := testKeys(t) opt := t.TempDir() reg := testRegistry(t) shipRelease(t, s, "bab", "myapp", "v1", adoptPlatform, []byte("bin")) if err := Promote(ctx, s, priv, "bab", "myapp", "stable", "v1"); err != nil { t.Fatal(err) } if err := SetProduction(ctx, s, priv, "bab", "myapp", true, "test"); err != nil { t.Fatal(err) } addRoute(t, reg, "h.example.com", "myapp", "", false) if err := Pass(ctx, reg, s, pub, opt, func(string) error { return nil }); err != nil { t.Fatal(err) } st := LoadFirstSeen(FirstSeenPath(opt)) if len(st.Production) == 0 { t.Fatalf("first pass did not latch the production memory") } // The old binary's write: every channel observation survives, the // production memory (and with it the replay floor) is gone. st.Production = map[string]*prodMemory{} if err := st.Save(); err != nil { t.Fatalf("Save wiped clock: %v", err) } if err := Pass(ctx, reg, s, pub, opt, func(string) error { return nil }); err != nil { t.Fatal(err) } if v, _ := LiveVersion(opt, "h.example.com"); v != "" { t.Fatalf("live = %q, want still held — the bucket flag must re-latch BEFORE this pass's bake verdict, not after it", v) } if st := LoadFirstSeen(FirstSeenPath(opt)); len(st.Production) == 0 { t.Fatalf("the re-latch was not persisted") } } // A clock file that exists but does not parse is rewritten within ONE pass, // even when the pass observes nothing new — the taint-forced Save. Without // it a corrupt file would sit there forever, and every held head on the box // would fail toward baking pass after pass (ProductionBake's st.tainted). func TestPassRewritesATaintedClockInOnePass(t *testing.T) { s := NewDirStore(t.TempDir()) ctx := context.Background() _, pub := testKeys(t) opt := t.TempDir() reg := testRegistry(t) // no routes: nothing new is observed this pass if err := os.WriteFile(FirstSeenPath(opt), []byte("{not json"), 0o644); err != nil { t.Fatal(err) } if st := LoadFirstSeen(FirstSeenPath(opt)); !st.tainted { t.Fatalf("setup: the corrupt clock did not read as tainted") } if err := Pass(ctx, reg, s, pub, opt, func(string) error { return nil }); err != nil { t.Fatalf("Pass: %v", err) } if st := LoadFirstSeen(FirstSeenPath(opt)); st.tainted { t.Fatalf("the pass left the clock file corrupt — a tainted clock must force its own Save") } } // prodReadStore counts Get calls for one exact key. type prodReadStore struct { Store key string n int } func (p *prodReadStore) Get(ctx context.Context, key string) ([]byte, string, error) { if key == p.key { p.n++ } return p.Store.Get(ctx, key) } // The 2-3 SECOND canary fast loop must never pay a production-flag bucket // read: canary/* has no window (DefaultHolds), so no canary-only pass ever // has a HELD head to ask about. The hourly pass pays exactly one read per // (account, app) with a held head — not one per channel, not one per route. func TestCanaryFastLoopPaysNoProductionRead(t *testing.T) { dir := NewDirStore(t.TempDir()) ctx := context.Background() priv, pub := testKeys(t) opt := t.TempDir() reg := testRegistry(t) shipRelease(t, dir, "bab", "myapp", "v1", adoptPlatform, []byte("bin")) for _, ch := range []string{"canary/x", "canary/y", "stable"} { if err := Promote(ctx, dir, priv, "bab", "myapp", ch, "v1"); err != nil { t.Fatalf("Promote %s: %v", ch, err) } } if err := SetProduction(ctx, dir, priv, "bab", "myapp", true, "test"); err != nil { t.Fatal(err) } addRoute(t, reg, "cx.example.com", "myapp", "canary/x", false) addRoute(t, reg, "cy.example.com", "myapp", "canary/y", false) addRoute(t, reg, "s.example.com", "myapp", "stable", false) s := &prodReadStore{Store: dir, key: ProductionKey("bab", "myapp")} if err := PassFilter(ctx, reg, s, pub, opt, func(string) error { return nil }, isCanaryTestRoute); err != nil { t.Fatalf("PassFilter: %v", err) } if s.n != 0 { t.Fatalf("canary-only pass read the production flag %d times, want 0 — the fast loop must not pay bucket reads for a window canary/* does not have", s.n) } // Not vacuous: the full pass, which DOES resolve a held head, reads it — // once, for two canary channels, a stable one and three routes. if err := Pass(ctx, reg, s, pub, opt, func(string) error { return nil }); err != nil { t.Fatalf("Pass: %v", err) } if s.n != 1 { t.Fatalf("full pass read the production flag %d times, want exactly 1 per (account, app)", s.n) } } // A NARROWED pass (the canary fast loop) must never repair a tainted clock: // it fetched no production flags, so its Production map is empty, and writing // it would replace the corrupt file with that emptiness AND clear the taint // that is currently failing closed — before anything re-observed the flags. // The full pass, which holds the flags, does the repair. func TestNarrowedPassNeverRepairsATaintedClock(t *testing.T) { s := NewDirStore(t.TempDir()) ctx := context.Background() priv, pub := testKeys(t) opt := t.TempDir() reg := testRegistry(t) shipRelease(t, s, "bab", "myapp", "v1", adoptPlatform, []byte("bin")) for _, ch := range []string{"canary/x", "stable"} { if err := Promote(ctx, s, priv, "bab", "myapp", ch, "v1"); err != nil { t.Fatalf("Promote %s: %v", ch, err) } } if err := SetProduction(ctx, s, priv, "bab", "myapp", true, "test"); err != nil { t.Fatal(err) } addRoute(t, reg, "cx.example.com", "myapp", "canary/x", false) addRoute(t, reg, "s.example.com", "myapp", "stable", false) if err := os.WriteFile(FirstSeenPath(opt), []byte("{not json"), 0o644); err != nil { t.Fatal(err) } if err := PassFilter(ctx, reg, s, pub, opt, func(string) error { return nil }, isCanaryTestRoute); err != nil { t.Fatalf("PassFilter: %v", err) } if raw, err := os.ReadFile(FirstSeenPath(opt)); err != nil { t.Fatal(err) } else if json.Valid(raw) { t.Fatalf("the canary fast loop overwrote a tainted clock — a pass that fetched no flags cannot repair what it would overwrite; file is now %q", raw) } // The hourly pass has the flags in hand, so it repairs — and the app's // production state survives, re-latched from the bucket in that same pass. if err := Pass(ctx, reg, s, pub, opt, func(string) error { return nil }); err != nil { t.Fatalf("Pass: %v", err) } st := LoadFirstSeen(FirstSeenPath(opt)) if st.tainted { t.Fatalf("the full pass did not repair the tainted clock") } if mem := st.Production[prodKey("bab", "myapp")]; mem == nil || !mem.WasProduction { t.Fatalf("production memory = %+v, want the latch re-established by the repairing pass", mem) } } // The off-ratchet, end to end on the adopt path: flipping production OFF does // not release a held head immediately — the off is honored only after the // stable window has elapsed on this box's own sighting of it (spec §4). This // is the dangerous direction, so it is pinned here and not only in bake_test: // it is also what fails if ProductionBake's hasBakeableHead carve-out is ever // passed a constant false from this call site. func TestPassProductionOffStillRatchets(t *testing.T) { s := NewDirStore(t.TempDir()) ctx := context.Background() priv, pub := testKeys(t) opt := t.TempDir() reg := testRegistry(t) shipRelease(t, s, "bab", "myapp", "v1", adoptPlatform, []byte("bin")) if err := Promote(ctx, s, priv, "bab", "myapp", "stable", "v1"); err != nil { t.Fatal(err) } if err := SetProduction(ctx, s, priv, "bab", "myapp", true, "test"); err != nil { t.Fatal(err) } addRoute(t, reg, "h.example.com", "myapp", "", false) if err := Pass(ctx, reg, s, pub, opt, func(string) error { return nil }); err != nil { t.Fatal(err) } if v, _ := LiveVersion(opt, "h.example.com"); v != "" { t.Fatalf("live = %q, want held (declared production)", v) } // Flip it off. An operator who can flip the flag must not thereby have a // same-minute bypass of the window their held head is sitting in. if err := SetProduction(ctx, s, priv, "bab", "myapp", false, "test"); err != nil { t.Fatal(err) } if err := Pass(ctx, reg, s, pub, opt, func(string) error { return nil }); err != nil { t.Fatal(err) } if v, _ := LiveVersion(opt, "h.example.com"); v != "" { t.Fatalf("live = %q, want STILL held — production: false must ratchet through the stable window, never release a held head on sight", v) } // And the box recorded the off-sighting, which is what the ratchet is // measured from — its own clock, never the flag's timestamp. mem := LoadFirstSeen(FirstSeenPath(opt)).Production[prodKey("bab", "myapp")] if mem == nil || mem.OffFirstSeen.IsZero() || !mem.WasProduction { t.Fatalf("production memory = %+v, want a latched app with an off-sighting recorded", mem) } } // ---- pipeline windows on the adopt path (release pipelines v2 spec §3) ------ // TestPassEnforcesPipelineWindows: an app with pipeline {edge(0), // production(24h)}. Four behaviours in one lifecycle, in order: a release at // the head of "production" is HELD even though the app's production flag is // absent (pipelined apps never consult the flag); the same release at the // head of "edge" adopts immediately; the declared 24h — narrower than the // 72h a brand-new channel baselines at — ratchets in only after the OLD // window has elapsed on this box's own clock, and then governs; and a // FastTrack-re-signed pointer adopts immediately while held (spec §3 // amendment — ratcheting the waiver would make it wait out the very window // it exists to skip). func TestPassEnforcesPipelineWindows(t *testing.T) { s := NewDirStore(t.TempDir()) ctx := context.Background() priv, pub := testKeys(t) opt := t.TempDir() reg := testRegistry(t) // NO production flag is ever set in this test: a pipelined app's windows // come from the pipeline alone. (Legacy behaviour — no flag, no window — // would adopt the production head immediately.) if _, err := SetPipeline(ctx, s, priv, "bab", "myapp", []PipelineChannel{ {Name: "edge"}, {Name: "production", Bake: "24h"}, }, "test"); err != nil { t.Fatalf("SetPipeline: %v", err) } shipRelease(t, s, "bab", "myapp", "v1", adoptPlatform, []byte("bin one")) for _, ch := range []string{"edge", "production"} { if err := Promote(ctx, s, priv, "bab", "myapp", ch, "v1"); err != nil { t.Fatalf("Promote %s: %v", ch, err) } } addRoute(t, reg, "e.example.com", "myapp", "edge", false) addRoute(t, reg, "p.example.com", "myapp", "production", false) // Sub-step 1: the production head is held with no flag in sight. if err := Pass(ctx, reg, s, pub, opt, func(string) error { return nil }); err != nil { t.Fatalf("Pass: %v", err) } if v, _ := LiveVersion(opt, "p.example.com"); v != "" { t.Fatalf("production live = %q, want held — a pipelined app's bake window must apply with no production flag at all", v) } // Sub-step 2: the zero-bake entry channel adopted in that same pass. if v, _ := LiveVersion(opt, "e.example.com"); v != "v1" { t.Fatalf("edge live = %q, want v1 — the pipeline's zero-bake channel adopts immediately", v) } // Sub-step 3: the ratchet lands. The pass above latched the 72h // fail-closed baseline and parked the declared 24h as pending. Backdate // the box's own clocks: the pending sighting past the old window (72h // jittered < 90h < 91h), and v1's first sighting past 24h jittered // (< 30h < 31h) but NOT past 72h — adoption then proves the enforced // window really is the ratcheted-in 24h, not an expired baseline. st := LoadFirstSeen(FirstSeenPath(opt)) key := channelClockKey("bab", "myapp", "production") mem := st.Holds[key] if mem == nil || mem.Window != 72*time.Hour || mem.Pending != 24*time.Hour || mem.PendingAt.IsZero() { t.Fatalf("hold memory = %+v, want the 72h baseline with 24h pending", mem) } mem.PendingAt = mem.PendingAt.Add(-91 * time.Hour) if seen, ok := st.Channels[key]["v1"]; !ok { t.Fatalf("no first sighting recorded for v1 at the production head") } else { st.Channels[key]["v1"] = seen.Add(-31 * time.Hour) } if err := st.Save(); err != nil { t.Fatalf("Save backdated clock: %v", err) } if err := Pass(ctx, reg, s, pub, opt, func(string) error { return nil }); err != nil { t.Fatalf("Pass: %v", err) } if v, _ := LiveVersion(opt, "p.example.com"); v != "v1" { t.Fatalf("production live = %q, want v1 — the ratcheted-in 24h window elapsed on the box's own clock", v) } // And the landing was PERSISTED by that pass: ObservePipeline's changed // flag must reach the Save path, or the promotion would re-run (and the // window claim silently regress) every pass until some other change // happened to force a write. if mem := LoadFirstSeen(FirstSeenPath(opt)).Holds[key]; mem == nil || mem.Window != 24*time.Hour || !mem.PendingAt.IsZero() { t.Fatalf("persisted hold memory = %+v, want the landed 24h window", mem) } // Sub-step 4: fast-track. v2 at the production head is inside the (now // 24h) window; the root-re-signed waiver releases it, unchanged by the // pipeline. shipRelease(t, s, "bab", "myapp", "v2", adoptPlatform, []byte("bin two")) if err := Promote(ctx, s, priv, "bab", "myapp", "production", "v2"); err != nil { t.Fatalf("Promote v2: %v", err) } if err := Pass(ctx, reg, s, pub, opt, func(string) error { return nil }); err != nil { t.Fatalf("Pass: %v", err) } if v, _ := LiveVersion(opt, "p.example.com"); v != "v1" { t.Fatalf("production live = %q, want still v1 — v2 is inside the window until the waiver", v) } if _, err := FastTrack(ctx, s, priv, "bab", "myapp", "production", "test"); err != nil { t.Fatalf("FastTrack: %v", err) } if err := Pass(ctx, reg, s, pub, opt, func(string) error { return nil }); err != nil { t.Fatalf("Pass: %v", err) } if v, _ := LiveVersion(opt, "p.example.com"); v != "v2" { t.Fatalf("production live = %q, want v2 — FastTrack must keep bypassing a pipelined hold", v) } } // TestPassPipelinedAppCanaryKeepsNoWindow: canary/* lives OUTSIDE the // pipeline (spec §2), so a pipelined app's canary routes must keep their // zero window. The trap is structural: HoldsForApp's map carries only the // pipeline's channels, and HoldFor's fallback for a name missing from the // map is the 72h fail-closed default — without canary/* stitched back in at // zero, declaring a pipeline would freeze every canary deploy of the app. func TestPassPipelinedAppCanaryKeepsNoWindow(t *testing.T) { s := NewDirStore(t.TempDir()) ctx := context.Background() priv, pub := testKeys(t) opt := t.TempDir() reg := testRegistry(t) if _, err := SetPipeline(ctx, s, priv, "bab", "myapp", []PipelineChannel{ {Name: "edge"}, {Name: "production", Bake: "24h"}, }, "test"); err != nil { t.Fatalf("SetPipeline: %v", err) } shipRelease(t, s, "bab", "myapp", "v1", adoptPlatform, []byte("bin")) for _, ch := range []string{"canary/x", "production"} { if err := Promote(ctx, s, priv, "bab", "myapp", ch, "v1"); err != nil { t.Fatalf("Promote %s: %v", ch, err) } } addRoute(t, reg, "c.example.com", "myapp", "canary/x", false) addRoute(t, reg, "p.example.com", "myapp", "production", false) if err := Pass(ctx, reg, s, pub, opt, func(string) error { return nil }); err != nil { t.Fatalf("Pass: %v", err) } if v, _ := LiveVersion(opt, "c.example.com"); v != "v1" { t.Fatalf("canary live = %q, want v1 — canary/* is outside the pipeline and must keep its zero window", v) } // Not vacuous: the held production head in the SAME pass proves the // pipelined holds map (not DefaultHolds-with-no-flag, which would have // adopted it) is what governed — including the canary route above. if v, _ := LiveVersion(opt, "p.example.com"); v != "" { t.Fatalf("production live = %q, want held — without the hold this test proves nothing about the map the canary resolved through", v) } } // TestPassCorruptPipelineFailsTowardLegacyHolds: a pipeline object that // EXISTS but will not read or verify is not a pipeline and not an absence — // the app falls back to the legacy path (DefaultHolds + the production // flag), the same fail-closed posture an old binary that cannot parse // pipeline.json takes, whose mitigation is that enabling a nonzero-bake // pipeline also sets production: true (spec §3). And the errored read must // NEVER reach the ratchet memory as a nil: nil means verified-absent, and a // nil fold would land pending promotions — a corrupt object masquerading as // a deletion. func TestPassCorruptPipelineFailsTowardLegacyHolds(t *testing.T) { s := NewDirStore(t.TempDir()) ctx := context.Background() priv, pub := testKeys(t) opt := t.TempDir() reg := testRegistry(t) shipRelease(t, s, "bab", "myapp", "v1", adoptPlatform, []byte("bin")) if err := Promote(ctx, s, priv, "bab", "myapp", "production", "v1"); err != nil { t.Fatalf("Promote: %v", err) } // The flag the console sets alongside any nonzero-bake pipeline. if err := SetProduction(ctx, s, priv, "bab", "myapp", true, "test"); err != nil { t.Fatalf("SetProduction: %v", err) } if _, err := s.Put(ctx, PipelineKey("bab", "myapp"), []byte("not a wrapper"), ""); err != nil { t.Fatalf("corrupt the pipeline object: %v", err) } addRoute(t, reg, "p.example.com", "myapp", "production", false) // Pre-seed ratchet memory with a pending loosening whose old window has // long elapsed, and a v1 sighting past the pending 24h but inside the // enforced 72h. If the errored read were wrongly folded as nil, the // promotion would land and the memory would read 24h below. now := time.Now() st := LoadFirstSeen(FirstSeenPath(opt)) key := channelClockKey("bab", "myapp", "production") st.Holds[key] = &holdMemory{ Window: 72 * time.Hour, SetAt: now.Add(-100 * time.Hour), Pending: 24 * time.Hour, PendingAt: now.Add(-91 * time.Hour), } st.Channels[key] = map[string]time.Time{"v1": now.Add(-31 * time.Hour)} if err := st.Save(); err != nil { t.Fatalf("Save pre-seeded clock: %v", err) } if err := Pass(ctx, reg, s, pub, opt, func(string) error { return nil }); err != nil { t.Fatalf("Pass: %v", err) } if v, _ := LiveVersion(opt, "p.example.com"); v != "" { t.Fatalf("live = %q, want held — an unreadable pipeline must fall to legacy holds with the flag applying, never adopt free", v) } mem := LoadFirstSeen(FirstSeenPath(opt)).Holds[key] if mem == nil || mem.Window != 72*time.Hour || mem.PendingAt.IsZero() { t.Fatalf("hold memory after the errored read = %+v, want untouched (72h enforced, promotion still pending) — an errored read must never tick the ratchet", mem) } } // TestPassErroredPipelineReadKeepsTightenedWindows: the memory floor. A // verified pipeline tightened edge's window to 24h — and DefaultHolds gives // edge ZERO, so a plain fall-back to DefaultHolds on a later errored read // would adopt instantly: one transient GET or verify failure silently // defeating the exact window the bake exists to provide. Spec §3 is // binding: corruption must never loosen — on an errored read, every channel // the box's own ratchet remembers is enforced at max(its DefaultHolds // resolution, the remembered window). func TestPassErroredPipelineReadKeepsTightenedWindows(t *testing.T) { s := NewDirStore(t.TempDir()) ctx := context.Background() priv, pub := testKeys(t) opt := t.TempDir() reg := testRegistry(t) if _, err := SetPipeline(ctx, s, priv, "bab", "myapp", []PipelineChannel{ {Name: "edge", Bake: "24h"}, }, "test"); err != nil { t.Fatalf("SetPipeline: %v", err) } // The flag the console co-writes with any nonzero-bake pipeline. if err := SetProduction(ctx, s, priv, "bab", "myapp", true, "test"); err != nil { t.Fatalf("SetProduction: %v", err) } shipRelease(t, s, "bab", "myapp", "v1", adoptPlatform, []byte("bin")) if err := Promote(ctx, s, priv, "bab", "myapp", "edge", "v1"); err != nil { t.Fatalf("Promote: %v", err) } addRoute(t, reg, "e.example.com", "myapp", "edge", false) // Pass 1, pipeline verified: edge's zero baseline tightens to 24h // immediately (a tighten is never pending), and v1's clock starts — held. if err := Pass(ctx, reg, s, pub, opt, func(string) error { return nil }); err != nil { t.Fatalf("Pass: %v", err) } if v, _ := LiveVersion(opt, "e.example.com"); v != "" { t.Fatalf("live = %q, want held — the verified pipeline tightened edge to 24h", v) } if mem := LoadFirstSeen(FirstSeenPath(opt)).Holds[channelClockKey("bab", "myapp", "edge")]; mem == nil || mem.Window != 24*time.Hour { t.Fatalf("hold memory = %+v, want edge tightened to 24h", mem) } // Corrupt the object in place. The next read ERRORS (exists, won't // verify) — and must not hand edge back its zero legacy window. _, etag, err := s.Get(ctx, PipelineKey("bab", "myapp")) if err != nil { t.Fatalf("read pipeline etag: %v", err) } if _, err := s.Put(ctx, PipelineKey("bab", "myapp"), []byte("not a wrapper"), etag); err != nil { t.Fatalf("corrupt the pipeline object: %v", err) } if err := Pass(ctx, reg, s, pub, opt, func(string) error { return nil }); err != nil { t.Fatalf("Pass: %v", err) } if v, _ := LiveVersion(opt, "e.example.com"); v != "" { t.Fatalf("live = %q, want STILL held — an errored read must keep the ratcheted 24h floor on edge, never fall to its zero legacy window", v) } } // TestPassDeletedPipelineRatchetsDown: deleting pipeline.json is a plain // bucket write — no signing key — so verified absence must never collapse a // ratcheted window instantly. Deletion ratchets DOWN to the legacy defaults // instead: each remembered channel keeps its enforced window for one full // old-window cycle, then lands at HoldFor(DefaultHolds, name) and the app // resumes a legacy-shaped pass. func TestPassDeletedPipelineRatchetsDown(t *testing.T) { s := NewDirStore(t.TempDir()) ctx := context.Background() priv, pub := testKeys(t) opt := t.TempDir() reg := testRegistry(t) if _, err := SetPipeline(ctx, s, priv, "bab", "myapp", []PipelineChannel{ {Name: "edge", Bake: "24h"}, }, "test"); err != nil { t.Fatalf("SetPipeline: %v", err) } // The flag the console co-writes with any nonzero-bake pipeline. if err := SetProduction(ctx, s, priv, "bab", "myapp", true, "test"); err != nil { t.Fatalf("SetProduction: %v", err) } shipRelease(t, s, "bab", "myapp", "v1", adoptPlatform, []byte("bin")) if err := Promote(ctx, s, priv, "bab", "myapp", "edge", "v1"); err != nil { t.Fatalf("Promote: %v", err) } addRoute(t, reg, "e.example.com", "myapp", "edge", false) // Pass 1, pipeline verified: edge tightens to 24h immediately — held. if err := Pass(ctx, reg, s, pub, opt, func(string) error { return nil }); err != nil { t.Fatalf("Pass: %v", err) } if v, _ := LiveVersion(opt, "e.example.com"); v != "" { t.Fatalf("live = %q, want held — the verified pipeline tightened edge to 24h", v) } // The attack: delete the object outright. if err := s.Delete(ctx, PipelineKey("bab", "myapp")); err != nil { t.Fatalf("delete pipeline: %v", err) } // Pass 2: STILL held. Bare DefaultHolds would hand edge its zero legacy // window back and adopt v1 in this very pass. if err := Pass(ctx, reg, s, pub, opt, func(string) error { return nil }); err != nil { t.Fatalf("Pass: %v", err) } if v, _ := LiveVersion(opt, "e.example.com"); v != "" { t.Fatalf("live = %q, want STILL held — deletion must ratchet the 24h window down, never collapse it", v) } key := channelClockKey("bab", "myapp", "edge") st := LoadFirstSeen(FirstSeenPath(opt)) mem := st.Holds[key] if mem == nil || mem.Window != 24*time.Hour || mem.Pending != 0 || mem.PendingAt.IsZero() { t.Fatalf("hold memory = %+v, want 24h enforced with the legacy 0 pending", mem) } pendingAt := mem.PendingAt // Pass 3, absence re-observed: the pending clock must NOT restart. if err := Pass(ctx, reg, s, pub, opt, func(string) error { return nil }); err != nil { t.Fatalf("Pass: %v", err) } if mem := LoadFirstSeen(FirstSeenPath(opt)).Holds[key]; mem == nil || !mem.PendingAt.Equal(pendingAt) { t.Fatalf("re-observed absence restarted the pending clock: %+v, want PendingAt %v", mem, pendingAt) } // Elapse the OLD window on the box's own clock (24h jittered < 30h < // 31h). Next pass: the legacy default lands, edge is back at zero, and a // legacy-shaped pass adopts v1. st = LoadFirstSeen(FirstSeenPath(opt)) st.Holds[key].PendingAt = st.Holds[key].PendingAt.Add(-31 * time.Hour) if seen, ok := st.Channels[key]["v1"]; ok { st.Channels[key]["v1"] = seen.Add(-31 * time.Hour) } if err := st.Save(); err != nil { t.Fatalf("Save backdated clock: %v", err) } if err := Pass(ctx, reg, s, pub, opt, func(string) error { return nil }); err != nil { t.Fatalf("Pass: %v", err) } if v, _ := LiveVersion(opt, "e.example.com"); v != "v1" { t.Fatalf("live = %q, want v1 — the old window elapsed, so legacy holds (edge 0) resume", v) } if mem := LoadFirstSeen(FirstSeenPath(opt)).Holds[key]; mem == nil || mem.Window != 0 || !mem.PendingAt.IsZero() { t.Fatalf("persisted hold memory = %+v, want the landed legacy 0", mem) } } // ratchetedProductionApp arranges the case the whole app-defined-channel // posture turns on, and the one the edge-named tests below cannot reach: a // FLAGLESS app whose pipeline names a channel OUTSIDE the frozen vocabulary // ("production"), with a bake SHORTER than that name's legacy resolution. // HoldFor's fallback for an unrecognised name is the 72h fail-closed default, // so the declared 24h is a LOOSENING that ratchets in over the old window — // and once it has, the box enforces 24h while DefaultHolds still resolves the // same name at 72h. Any "is this app pipelined" test phrased as "does the // memory RAISE a window above legacy" answers NO here, which is exactly the // shape that made an app-defined channel the one place the verdict collapsed. // // Leaves: v1 adopted, the memory ratcheted in at 24h with nothing pending, // and v2 at the production head INSIDE that 24h window — held, on the // pipeline's word alone, with no production flag anywhere. func ratchetedProductionApp(t *testing.T, s Store, ctx context.Context, priv ed25519.PrivateKey, pub ed25519.PublicKey, opt string, reg *registry.Registry) { t.Helper() if _, err := SetPipeline(ctx, s, priv, "bab", "myapp", []PipelineChannel{ {Name: "edge"}, {Name: "production", Bake: "24h"}, }, "test"); err != nil { t.Fatalf("SetPipeline: %v", err) } shipRelease(t, s, "bab", "myapp", "v1", adoptPlatform, []byte("bin one")) if err := Promote(ctx, s, priv, "bab", "myapp", "edge", "v1"); err != nil { t.Fatalf("Promote edge: %v", err) } if err := Promote(ctx, s, priv, "bab", "myapp", "production", "v1"); err != nil { t.Fatalf("Promote production: %v", err) } addRoute(t, reg, "p.example.com", "myapp", "production", false) // Pass 1 latches the 72h fail-closed baseline and parks the declared 24h // as pending — held. if err := Pass(ctx, reg, s, pub, opt, func(string) error { return nil }); err != nil { t.Fatalf("Pass 1: %v", err) } if v, _ := LiveVersion(opt, "p.example.com"); v != "" { t.Fatalf("live = %q, want held at the 72h baseline", v) } // Elapse the OLD window on the box's own clock so the 24h lands, and age // v1 past 24h (but not past 72h) so adoption proves WHICH window governs. key := channelClockKey("bab", "myapp", "production") st := LoadFirstSeen(FirstSeenPath(opt)) mem := st.Holds[key] if mem == nil || mem.Window != 72*time.Hour || mem.Pending != 24*time.Hour { t.Fatalf("hold memory = %+v, want the 72h baseline with 24h pending", mem) } mem.PendingAt = mem.PendingAt.Add(-91 * time.Hour) if seen, ok := st.Channels[key]["v1"]; !ok { t.Fatalf("no first sighting recorded for v1") } else { st.Channels[key]["v1"] = seen.Add(-31 * time.Hour) } if err := st.Save(); err != nil { t.Fatalf("Save backdated clock: %v", err) } if err := Pass(ctx, reg, s, pub, opt, func(string) error { return nil }); err != nil { t.Fatalf("Pass 2: %v", err) } if v, _ := LiveVersion(opt, "p.example.com"); v != "v1" { t.Fatalf("live = %q, want v1 — the ratcheted-in 24h elapsed", v) } if mem := LoadFirstSeen(FirstSeenPath(opt)).Holds[key]; mem == nil || mem.Window != 24*time.Hour || !mem.PendingAt.IsZero() { t.Fatalf("hold memory = %+v, want the landed 24h with nothing pending", mem) } // A fresh head inside the ratcheted window: held while the pipeline reads. shipRelease(t, s, "bab", "myapp", "v2", adoptPlatform, []byte("bin two")) if err := Promote(ctx, s, priv, "bab", "myapp", "production", "v2"); err != nil { t.Fatalf("Promote v2: %v", err) } if err := Pass(ctx, reg, s, pub, opt, func(string) error { return nil }); err != nil { t.Fatalf("Pass 3: %v", err) } if v, _ := LiveVersion(opt, "p.example.com"); v != "v1" { t.Fatalf("live = %q, want still v1 — v2 is inside the ratcheted 24h window", v) } } // TestPassErroredReadOnAppDefinedChannelHoldsWithoutFlag is the C1 probe: the // same collapse the edge-named test below covers, on the naming that actually // hides it. With the memory ratcheted DOWN to 24h against a 72h legacy // resolution, no window in the map is raised above legacy — so a verdict // derived from the OVERLAY finds nothing, falls to the production flag this // app has never had, and adopts v2 the moment one GET of pipeline.json errors. // The evidence has to be the ratchet's own liveness, not the overlay. func TestPassErroredReadOnAppDefinedChannelHoldsWithoutFlag(t *testing.T) { s := NewDirStore(t.TempDir()) ctx := context.Background() priv, pub := testKeys(t) opt := t.TempDir() reg := testRegistry(t) ratchetedProductionApp(t, s, ctx, priv, pub, opt, reg) _, etag, err := s.Get(ctx, PipelineKey("bab", "myapp")) if err != nil { t.Fatalf("read pipeline etag: %v", err) } if _, err := s.Put(ctx, PipelineKey("bab", "myapp"), []byte("not a wrapper"), etag); err != nil { t.Fatalf("corrupt the pipeline object: %v", err) } if err := Pass(ctx, reg, s, pub, opt, func(string) error { return nil }); err != nil { t.Fatalf("Pass: %v", err) } if v, _ := LiveVersion(opt, "p.example.com"); v != "v1" { t.Fatalf("live = %q, want still v1 — an errored read on an app-defined channel must keep the hold, with no production flag in sight", v) } } // TestPassDeletedPipelineOnAppDefinedChannelHoldsWithoutFlag: the same probe // on the other keyless bucket write. Deleting pipeline.json hands the channel // its legacy 72h back — LONGER than the 24h the box was enforcing, so the // window itself never loosens — but the app must not fall to the flagless // legacy VERDICT in the same pass either, or one DELETE adopts a build that // was minutes old. It returns to the legacy path only once the deletion // ratchet has run the old window out on the box's own clock. func TestPassDeletedPipelineOnAppDefinedChannelHoldsWithoutFlag(t *testing.T) { s := NewDirStore(t.TempDir()) ctx := context.Background() priv, pub := testKeys(t) opt := t.TempDir() reg := testRegistry(t) ratchetedProductionApp(t, s, ctx, priv, pub, opt, reg) if err := s.Delete(ctx, PipelineKey("bab", "myapp")); err != nil { t.Fatalf("delete pipeline: %v", err) } if err := Pass(ctx, reg, s, pub, opt, func(string) error { return nil }); err != nil { t.Fatalf("Pass: %v", err) } if v, _ := LiveVersion(opt, "p.example.com"); v != "v1" { t.Fatalf("live = %q, want still v1 — deleting the pipeline must not adopt a held head in the same pass", v) } // And it does return to legacy: elapse the deletion ratchet's own cycle on // the box's clock, and the app is a legacy app again — no floor, no // pipeline evidence, the flagless legacy verdict, so v2 adopts. key := channelClockKey("bab", "myapp", "production") st := LoadFirstSeen(FirstSeenPath(opt)) mem := st.Holds[key] if mem == nil { t.Fatalf("no hold memory for the deleted pipeline's channel") } if mem.DeletedAt.IsZero() { t.Fatalf("hold memory = %+v, want the deletion stamped so its ratchet can run out", mem) } mem.DeletedAt = mem.DeletedAt.Add(-91 * time.Hour) st.Holds[key] = mem if err := st.Save(); err != nil { t.Fatalf("Save backdated clock: %v", err) } if err := Pass(ctx, reg, s, pub, opt, func(string) error { return nil }); err != nil { t.Fatalf("Pass: %v", err) } if v, _ := LiveVersion(opt, "p.example.com"); v != "v2" { t.Fatalf("live = %q, want v2 — once the deletion ratchet has run out the app is legacy again, and a flagless legacy app holds nothing", v) } } // TestPassReplayedSubsetPipelineCannotPoisonTheDeletionStamp: the re-review // probe (PR #258, IMPORTANT-1). The deletion stamp is what carries the bake // verdict across a DELETE for an app-defined channel, so anything that can // leave a STALE stamp standing — one that expires unnoticed and then blocks // its own re-arming — is a way to strip the app of every ratchet it has. // // The chain, all of it keyless bucket writes plus one historical SIGNED // object anybody who ever saw the bucket can keep a copy of: // // 1. DELETE pipeline.json — every remembered channel is stamped. // 2. PUT BACK an OLD, still-validly-signed pipeline that names only a SUBSET // of the app's channels (here: edge, not production). There is no // object-level replay floor — only a per-channel SetAt one — so this is // accepted as a verified read. // 3. Dwell until the un-named channel's stale stamp has run out. // 4. DELETE again. A stamp is never restamped while non-zero, so production // is left with window == legacy, nothing pending and a dead stamp: no // live ratchet, no evidence, and the flagless legacy verdict adopts a // build that is minutes old. // // The fix is that a verified read clears the stamp for EVERY remembered // channel of the app, not only the ones the object names — so step 2 can // only ever DISARM the machinery, never leave a fuse burning behind it, and // step 4 stamps afresh. func TestPassReplayedSubsetPipelineCannotPoisonTheDeletionStamp(t *testing.T) { s := NewDirStore(t.TempDir()) ctx := context.Background() priv, pub := testKeys(t) opt := t.TempDir() reg := testRegistry(t) // The historical object: a legitimately signed, edge-only pipeline from // before production existed. Kept aside, exactly as an attacker with a // copy of yesterday's bucket would. if _, err := SetPipeline(ctx, s, priv, "bab", "myapp", []PipelineChannel{{Name: "edge"}}, "test"); err != nil { t.Fatalf("SetPipeline (historical): %v", err) } historical, _, err := s.Get(ctx, PipelineKey("bab", "myapp")) if err != nil { t.Fatalf("read the historical pipeline: %v", err) } ratchetedProductionApp(t, s, ctx, priv, pub, opt, reg) // 1. Delete: production is stamped, and its held head stays held. if err := s.Delete(ctx, PipelineKey("bab", "myapp")); err != nil { t.Fatalf("delete pipeline: %v", err) } if err := Pass(ctx, reg, s, pub, opt, func(string) error { return nil }); err != nil { t.Fatalf("Pass after delete: %v", err) } if v, _ := LiveVersion(opt, "p.example.com"); v != "v1" { t.Fatalf("live = %q, want still v1 after the delete", v) } // 2. Replay the historical edge-only object (create-only PUT: the key is // absent after the delete, which is precisely the opening). if _, err := s.Put(ctx, PipelineKey("bab", "myapp"), historical, ""); err != nil { t.Fatalf("replay the historical pipeline: %v", err) } if err := Pass(ctx, reg, s, pub, opt, func(string) error { return nil }); err != nil { t.Fatalf("Pass after the replay: %v", err) } if v, _ := LiveVersion(opt, "p.example.com"); v != "v1" { t.Fatalf("live = %q, want still v1 — a verified pipeline is the app being pipelined, not a reason to adopt", v) } // The property that makes step 4 safe: the verified read left NO stamp // standing anywhere under the app, including on the channel it never // named. st := LoadFirstSeen(FirstSeenPath(opt)) for _, name := range []string{"edge", "production"} { if mem := st.Holds[channelClockKey("bab", "myapp", name)]; mem != nil && !mem.DeletedAt.IsZero() { t.Fatalf("%s keeps a deletion stamp (%v) after a verified read — a stale stamp can expire unnoticed and then refuse to re-arm", name, mem.DeletedAt) } } // 3. Dwell. Anything still stamped ages out; v2's own first sighting is // deliberately NOT aged, so the only thing that can adopt it below is the // verdict collapsing. st = LoadFirstSeen(FirstSeenPath(opt)) for _, name := range []string{"edge", "production"} { if mem := st.Holds[channelClockKey("bab", "myapp", name)]; mem != nil && !mem.DeletedAt.IsZero() { mem.DeletedAt = mem.DeletedAt.Add(-91 * time.Hour) } } if err := st.Save(); err != nil { t.Fatalf("Save dwelt clock: %v", err) } // 4. Delete again. The stamp re-arms, so the head is still held. if err := s.Delete(ctx, PipelineKey("bab", "myapp")); err != nil { t.Fatalf("second delete: %v", err) } if err := Pass(ctx, reg, s, pub, opt, func(string) error { return nil }); err != nil { t.Fatalf("Pass after the second delete: %v", err) } if v, _ := LiveVersion(opt, "p.example.com"); v != "v1" { t.Fatalf("live = %q, want STILL v1 — a replayed subset pipeline must not be able to strip the app of its deletion ratchet", v) } } // TestPassErroredPipelineHoldsWithoutProductionFlag: the memory floor is only // half a defence if the app's held head is then ADOPTED anyway. The bake // VERDICT ("does the window apply at all?") must ride on the same evidence // the WINDOWS do — the box's own ratchet memory — not on the production flag // the console co-writes. A co-write is a bucket write the box cannot verify // happened: an app pipelined before that co-write existed, an app whose flag // was deleted, a direct-bucket pipeline write — all leave a flagless app with // a live ratchet memory, and a legacy verdict would adopt its held head the // moment one GET of pipeline.json errored. NO production flag is set here. func TestPassErroredPipelineHoldsWithoutProductionFlag(t *testing.T) { s := NewDirStore(t.TempDir()) ctx := context.Background() priv, pub := testKeys(t) opt := t.TempDir() reg := testRegistry(t) if _, err := SetPipeline(ctx, s, priv, "bab", "myapp", []PipelineChannel{ {Name: "edge", Bake: "24h"}, }, "test"); err != nil { t.Fatalf("SetPipeline: %v", err) } shipRelease(t, s, "bab", "myapp", "v1", adoptPlatform, []byte("bin")) if err := Promote(ctx, s, priv, "bab", "myapp", "edge", "v1"); err != nil { t.Fatalf("Promote: %v", err) } addRoute(t, reg, "e.example.com", "myapp", "edge", false) // Pass 1, pipeline verified: edge's zero baseline tightens to 24h // immediately (a tighten is never pending) — held, flag or no flag. if err := Pass(ctx, reg, s, pub, opt, func(string) error { return nil }); err != nil { t.Fatalf("Pass: %v", err) } if v, _ := LiveVersion(opt, "e.example.com"); v != "" { t.Fatalf("live = %q, want held — a pipelined app's window applies with no production flag at all", v) } // Corrupt the object in place: the next read ERRORS. _, etag, err := s.Get(ctx, PipelineKey("bab", "myapp")) if err != nil { t.Fatalf("read pipeline etag: %v", err) } if _, err := s.Put(ctx, PipelineKey("bab", "myapp"), []byte("not a wrapper"), etag); err != nil { t.Fatalf("corrupt the pipeline object: %v", err) } if err := Pass(ctx, reg, s, pub, opt, func(string) error { return nil }); err != nil { t.Fatalf("Pass: %v", err) } if v, _ := LiveVersion(opt, "e.example.com"); v != "" { t.Fatalf("live = %q, want STILL held — the ratchet memory IS the evidence this app is pipelined; the verdict must not fall back to the production flag", v) } if mem := LoadFirstSeen(FirstSeenPath(opt)).Holds[channelClockKey("bab", "myapp", "edge")]; mem == nil || mem.Window != 24*time.Hour { t.Fatalf("hold memory = %+v, want the 24h floor still remembered — otherwise this test proves nothing", mem) } } // TestPassDeletedPipelineHoldsWithoutProductionFlag: same law on the // verified-absent path. Deleting pipeline.json is a plain bucket write; the // deletion ratchet keeps the 24h window enforced for one old-window cycle, // and the VERDICT must honour it without any production flag — otherwise // deleting one object adopts every held head of a flagless pipelined app in // the very next pass, which is precisely the collapse the ratchet exists to // prevent. func TestPassDeletedPipelineHoldsWithoutProductionFlag(t *testing.T) { s := NewDirStore(t.TempDir()) ctx := context.Background() priv, pub := testKeys(t) opt := t.TempDir() reg := testRegistry(t) if _, err := SetPipeline(ctx, s, priv, "bab", "myapp", []PipelineChannel{ {Name: "edge", Bake: "24h"}, }, "test"); err != nil { t.Fatalf("SetPipeline: %v", err) } shipRelease(t, s, "bab", "myapp", "v1", adoptPlatform, []byte("bin")) if err := Promote(ctx, s, priv, "bab", "myapp", "edge", "v1"); err != nil { t.Fatalf("Promote: %v", err) } addRoute(t, reg, "e.example.com", "myapp", "edge", false) if err := Pass(ctx, reg, s, pub, opt, func(string) error { return nil }); err != nil { t.Fatalf("Pass: %v", err) } if v, _ := LiveVersion(opt, "e.example.com"); v != "" { t.Fatalf("live = %q, want held — the verified pipeline tightened edge to 24h", v) } if err := s.Delete(ctx, PipelineKey("bab", "myapp")); err != nil { t.Fatalf("delete pipeline: %v", err) } if err := Pass(ctx, reg, s, pub, opt, func(string) error { return nil }); err != nil { t.Fatalf("Pass: %v", err) } if v, _ := LiveVersion(opt, "e.example.com"); v != "" { t.Fatalf("live = %q, want STILL held — deletion ratchets the window down over a cycle, and the verdict must ride on that memory, not on a production flag nobody set", v) } key := channelClockKey("bab", "myapp", "edge") if mem := LoadFirstSeen(FirstSeenPath(opt)).Holds[key]; mem == nil || mem.Window != 24*time.Hour || mem.PendingAt.IsZero() { t.Fatalf("hold memory = %+v, want 24h enforced with the legacy 0 pending", mem) } // And the ratchet still LANDS: once the old window has elapsed on the // box's own clock the app is legacy again — memory at the legacy default, // no floor map, and the bare-DefaultHolds verdict adopts. The fix must not // pin a once-pipelined app to a hold forever. st := LoadFirstSeen(FirstSeenPath(opt)) st.Holds[key].PendingAt = st.Holds[key].PendingAt.Add(-31 * time.Hour) if seen, ok := st.Channels[key]["v1"]; ok { st.Channels[key]["v1"] = seen.Add(-31 * time.Hour) } if err := st.Save(); err != nil { t.Fatalf("Save backdated clock: %v", err) } if err := Pass(ctx, reg, s, pub, opt, func(string) error { return nil }); err != nil { t.Fatalf("Pass: %v", err) } if v, _ := LiveVersion(opt, "e.example.com"); v != "v1" { t.Fatalf("live = %q, want v1 — the old window elapsed, so legacy holds (edge 0) resume and the legacy verdict adopts", v) } } // TestPassNoPipelineNoMemoryStaysLegacy pins the other half of the deletion // ratchet's contract: an app with NO pipeline and NO ratchet memory — every // legacy app on every pass — keeps the bare DefaultHolds path byte-identically. // Edge adopts in the first pass, and no holds memory is minted from nothing. func TestPassNoPipelineNoMemoryStaysLegacy(t *testing.T) { s := NewDirStore(t.TempDir()) ctx := context.Background() priv, pub := testKeys(t) opt := t.TempDir() reg := testRegistry(t) shipRelease(t, s, "bab", "myapp", "v1", adoptPlatform, []byte("bin")) if err := Promote(ctx, s, priv, "bab", "myapp", "edge", "v1"); err != nil { t.Fatalf("Promote: %v", err) } addRoute(t, reg, "e.example.com", "myapp", "edge", false) if err := Pass(ctx, reg, s, pub, opt, func(string) error { return nil }); err != nil { t.Fatalf("Pass: %v", err) } if v, _ := LiveVersion(opt, "e.example.com"); v != "v1" { t.Fatalf("live = %q, want v1 — a legacy app's edge adopts immediately, untouched", v) } if holds := LoadFirstSeen(FirstSeenPath(opt)).Holds; len(holds) != 0 { t.Fatalf("holds memory = %v, want none — no pipeline was ever observed", holds) } } // TestPassNullHoldsEntryDoesNotPanic: {"holds":{"x":null}} parses cleanly // into a nil *holdMemory, and the adopt pass's floor loops iterate the map. // An unguarded mem.Window dereference panics the whole pass — fleet blast // radius — on both the errored-read and the verified-absent path. func TestPassNullHoldsEntryDoesNotPanic(t *testing.T) { s := NewDirStore(t.TempDir()) ctx := context.Background() priv, pub := testKeys(t) opt := t.TempDir() reg := testRegistry(t) if _, err := SetPipeline(ctx, s, priv, "bab", "myapp", []PipelineChannel{ {Name: "edge", Bake: "24h"}, }, "test"); err != nil { t.Fatalf("SetPipeline: %v", err) } if err := SetProduction(ctx, s, priv, "bab", "myapp", true, "test"); err != nil { t.Fatalf("SetProduction: %v", err) } shipRelease(t, s, "bab", "myapp", "v1", adoptPlatform, []byte("bin")) if err := Promote(ctx, s, priv, "bab", "myapp", "edge", "v1"); err != nil { t.Fatalf("Promote: %v", err) } addRoute(t, reg, "e.example.com", "myapp", "edge", false) // A clock file whose holds map carries a null entry — legal JSON, loads // clean, nil pointer in the map. blob := `{"jitter":0,"channels":{},"holds":{"` + channelClockKey("bab", "myapp", "edge") + `":null}}` if err := os.WriteFile(FirstSeenPath(opt), []byte(blob), 0o644); err != nil { t.Fatal(err) } // Errored read: corrupt the pipeline object so the memory-floor loop runs. _, etag, err := s.Get(ctx, PipelineKey("bab", "myapp")) if err != nil { t.Fatalf("read pipeline etag: %v", err) } if _, err := s.Put(ctx, PipelineKey("bab", "myapp"), []byte("not a wrapper"), etag); err != nil { t.Fatalf("corrupt the pipeline object: %v", err) } if err := Pass(ctx, reg, s, pub, opt, func(string) error { return nil }); err != nil { t.Fatalf("Pass over a null holds entry (errored read): %v", err) } // Verified absence: same clock shape, object deleted outright. if err := os.WriteFile(FirstSeenPath(opt), []byte(blob), 0o644); err != nil { t.Fatal(err) } if err := s.Delete(ctx, PipelineKey("bab", "myapp")); err != nil { t.Fatalf("delete pipeline: %v", err) } if err := Pass(ctx, reg, s, pub, opt, func(string) error { return nil }); err != nil { t.Fatalf("Pass over a null holds entry (verified absent): %v", err) } } // The 2-3 SECOND canary fast loop must never pay a pipeline bucket read // either: canary/* is outside every pipeline, so no canary-only pass has a // verdict the object could change — the mirror of // TestCanaryFastLoopPaysNoProductionRead. The hourly pass pays exactly one // read per (account, app) with a non-canary head — not one per channel, not // one per route. func TestCanaryFastLoopPaysNoPipelineRead(t *testing.T) { dir := NewDirStore(t.TempDir()) ctx := context.Background() priv, pub := testKeys(t) opt := t.TempDir() reg := testRegistry(t) shipRelease(t, dir, "bab", "myapp", "v1", adoptPlatform, []byte("bin")) for _, ch := range []string{"canary/x", "canary/y", "edge", "stable"} { if err := Promote(ctx, dir, priv, "bab", "myapp", ch, "v1"); err != nil { t.Fatalf("Promote %s: %v", ch, err) } } addRoute(t, reg, "cx.example.com", "myapp", "canary/x", false) addRoute(t, reg, "cy.example.com", "myapp", "canary/y", false) addRoute(t, reg, "e.example.com", "myapp", "edge", false) addRoute(t, reg, "s.example.com", "myapp", "stable", false) s := &prodReadStore{Store: dir, key: PipelineKey("bab", "myapp")} if err := PassFilter(ctx, reg, s, pub, opt, func(string) error { return nil }, isCanaryTestRoute); err != nil { t.Fatalf("PassFilter: %v", err) } if s.n != 0 { t.Fatalf("canary-only pass read the pipeline %d times, want 0 — canary/* is outside every pipeline", s.n) } // Not vacuous: the full pass, with an edge and a stable head resolved, // reads it — once for the whole app. if err := Pass(ctx, reg, s, pub, opt, func(string) error { return nil }); err != nil { t.Fatalf("Pass: %v", err) } if s.n != 1 { t.Fatalf("full pass read the pipeline %d times, want exactly 1 per (account, app)", s.n) } } // ---- WithStartUnit: the rollback must not leave the service down ---- // startUnitFixture is the shared setup of the WithStartUnit tests: one // plain instance route live on "old" (staged locally, symlink pointed), with // "new" promoted at the channel head; canary/* so no bake window applies. // The route carries no Unit column — this package never reads it (the // named-unit shape lives in internal/host's dispatch). What these tests // model is the CALLBACK CONTRACT: a restart that fails the way a named // unit's does (returns the error), and a start capability handed in through // WithStartUnit. func startUnitFixture(t *testing.T, host string) (context.Context, *registry.Registry, Store, ed25519.PublicKey, string) { t.Helper() s := NewDirStore(t.TempDir()) ctx := context.Background() priv, pub := testKeys(t) opt := t.TempDir() reg := testRegistry(t) shipRelease(t, s, "bab", "console", "old", adoptPlatform, []byte("good build")) shipRelease(t, s, "bab", "console", "new", adoptPlatform, []byte("bad build")) if err := StageArtifact(opt, "bab", "old", "console", []byte("good build")); err != nil { t.Fatal(err) } addRoute(t, reg, host, "console", "canary/x", false) if err := PointLive(opt, host, "bab", "old", "console"); err != nil { t.Fatal(err) } if err := Promote(ctx, s, priv, "bab", "console", "canary/x", "new"); err != nil { t.Fatalf("Promote new: %v", err) } return ctx, reg, s, pub, opt } // TestFailedRestartLeavesTheUnitRUNNINGOnTheOldVersion. // // The rollback repointed the symlink and stopped there, which left the box // with a correct symlink and no process. Worse, try-restart is a NO-OP on a // unit that is not running — so the NEXT pass exited 0, stamped SetVersions, // and produced exactly the lying header the whole mechanism exists to // prevent, over a service that was not running at all. func TestFailedRestartLeavesTheUnitRUNNINGOnTheOldVersion(t *testing.T) { host := "console.example.com" ctx, reg, s, pub, opt := startUnitFixture(t, host) var started []string restart := func(string) error { return errors.New("Job for console.service failed") } start := func(h string) error { started = append(started, h); return nil } if err := Pass(ctx, reg, s, pub, opt, restart, WithStartUnit(start)); err != nil { t.Fatalf("Pass: %v", err) } if v, _ := LiveVersion(opt, host); v != "old" { t.Fatalf("symlink = %q, want it rolled back to old", v) } if len(started) != 1 || started[0] != host { t.Fatalf("started = %v, want the unit started once on the restored version — a rollback that leaves the service down is not a rollback", started) } rt, err := reg.Get(host) if err != nil || rt == nil { t.Fatalf("get %s: %+v, %v", host, rt, err) } if rt.Version == "new" { t.Fatal("SetVersions stamped the failed build") } } // TestSecondPassOverAStillBrokenBuildDoesNotStamp: the second-order failure. // With the unit running again after pass 1, pass 2's try-restart is a real // restart that fails again — so it rolls back again and still does not stamp. func TestSecondPassOverAStillBrokenBuildDoesNotStamp(t *testing.T) { host := "console.example.com" ctx, reg, s, pub, opt := startUnitFixture(t, host) // The fake is STATEFUL, and the state is the whole defect narrative: a // failed restart leaves the unit DOWN, and try-restart on a not-running // unit is an exit-0 NO-OP. With the fix reverted (no start on rollback) // pass 1 leaves running=false, pass 2's restart returns nil, and // SetVersions stamps "new" — this test's assertion fires. An // unconditionally-erroring fake could not fail on that revert. running := true restart := func(string) error { if !running { return nil // try-restart is a no-op on a stopped unit } running = false // the failed restart left it down return errors.New("still broken") } start := func(string) error { running = true; return nil } for i := 0; i < 2; i++ { if err := Pass(ctx, reg, s, pub, opt, restart, WithStartUnit(start)); err != nil { t.Fatalf("pass %d: %v", i, err) } } rt, err := reg.Get(host) if err != nil || rt == nil { t.Fatalf("get %s: %+v, %v", host, rt, err) } if rt.Version == "new" { t.Fatal("the second pass stamped a build that has never run") } if v, _ := LiveVersion(opt, host); v != "old" { t.Fatalf("symlink after two passes = %q, want old", v) } } // TestStartFailureStaysPoisonedAndSaysSo: when the start ALSO fails the // refusal stands exactly as before — poisoned, unstamped, rolled back — and // the failure is printed, because at that point the box is genuinely broken // and an operator reading only "rolled the symlink back" would take a host // that is serving nothing for a recovered one. func TestStartFailureStaysPoisonedAndSaysSo(t *testing.T) { host := "console.example.com" ctx, reg, s, pub, opt := startUnitFixture(t, host) restart := func(string) error { return errors.New("Job for console.service failed") } start := func(string) error { return errors.New("Job for console.service failed again") } var out bytes.Buffer withCapturedStdout(t, &out, func() { if err := Pass(ctx, reg, s, pub, opt, restart, WithStartUnit(start)); err != nil { t.Fatalf("Pass: %v", err) } }) if v, _ := LiveVersion(opt, host); v != "old" { t.Fatalf("symlink = %q, want old", v) } rt, err := reg.Get(host) if err != nil || rt == nil { t.Fatalf("get %s: %+v, %v", host, rt, err) } if rt.Version == "new" { t.Fatal("a build whose restart AND start both failed was stamped") } if !strings.Contains(out.String(), "could not start it") { t.Fatalf("printed %q, want the failed start reported", out.String()) } } // TestSuccessfulRestartNeverConsultsStart: the start capability belongs to // the ROLLBACK path alone. A restart that succeeds — the every-deploy case — // must not touch it, or every ordinary adoption would grow a systemctl start. func TestSuccessfulRestartNeverConsultsStart(t *testing.T) { host := "console.example.com" ctx, reg, s, pub, opt := startUnitFixture(t, host) restart := func(string) error { return nil } start := func(h string) error { t.Fatalf("start consulted for %q after a SUCCESSFUL restart", h) return nil } if err := Pass(ctx, reg, s, pub, opt, restart, WithStartUnit(start)); err != nil { t.Fatalf("Pass: %v", err) } if v, _ := LiveVersion(opt, host); v != "new" { t.Fatalf("live = %q, want new", v) } } // TestFailedRollbackStartsNothing: the start is gated on the rollback // having HAPPENED, not on a previous version existing. "Link present, // staged version gone" is a documented real state (materialize.go — an // operator's /opt cleanup, a partially restored volume): PointLive refuses // to repoint at a version directory that is not there, the symlink still // names the refused build, and a start would launch exactly that build. // The refusal is also no longer silent — the pass says the rollback failed. func TestFailedRollbackStartsNothing(t *testing.T) { host := "console.example.com" ctx, reg, s, pub, opt := startUnitFixture(t, host) // The operator's cleanup: old's staged directory is gone, so the // rollback's PointLive will refuse. The live symlink still dangles at // old, which is how adoptHost sees cur = "old" and attempts the update. if err := os.RemoveAll(filepath.Dir(VersionPath(opt, "bab", "old", "console"))); err != nil { t.Fatal(err) } restart := func(string) error { return errors.New("Job for console.service failed") } start := func(h string) error { t.Fatalf("start consulted for %q after a rollback that DID NOT HAPPEN — the symlink still names the refused build", h) return nil } var out bytes.Buffer withCapturedStdout(t, &out, func() { if err := Pass(ctx, reg, s, pub, opt, restart, WithStartUnit(start)); err != nil { t.Fatalf("Pass: %v", err) } }) if !strings.Contains(out.String(), "could not roll the symlink back") { t.Fatalf("printed %q, want the failed rollback reported instead of swallowed", out.String()) } rt, err := reg.Get(host) if err != nil || rt == nil { t.Fatalf("get %s: %+v, %v", host, rt, err) } if rt.Version == "new" { t.Fatal("a build whose restart failed was stamped") } } // TestFirstAdoptionRestartFailureStartsNothing: with no previous version // (cur == "") the symlink could not be rolled back — it still names the // version whose restart just failed — so there is no RESTORED version to // start. Starting there would run the exact build the failure refused. func TestFirstAdoptionRestartFailureStartsNothing(t *testing.T) { s := NewDirStore(t.TempDir()) ctx := context.Background() priv, pub := testKeys(t) opt := t.TempDir() reg := testRegistry(t) host := "console.example.com" shipRelease(t, s, "bab", "console", "new", adoptPlatform, []byte("bad build")) addRoute(t, reg, host, "console", "canary/x", false) if err := Promote(ctx, s, priv, "bab", "console", "canary/x", "new"); err != nil { t.Fatalf("Promote: %v", err) } restart := func(string) error { return errors.New("Job for console.service failed") } start := func(h string) error { t.Fatalf("start consulted for %q on a first adoption — nothing was rolled back, so there is nothing restored to start", h) return nil } if err := Pass(ctx, reg, s, pub, opt, restart, WithStartUnit(start)); err != nil { t.Fatalf("Pass: %v", err) } rt, err := reg.Get(host) if err != nil || rt == nil { t.Fatalf("get %s: %+v, %v", host, rt, err) } if rt.Version == "new" { t.Fatal("a build whose restart failed was stamped") } } // TestRefusedPromotionIsNeitherStagedNorRelayed: a refusal naming the current // pointer stops adoptHost before ensureStaged, so the agent never relays a // root pass for it and nothing restarts. func TestRefusedPromotionIsNeitherStagedNorRelayed(t *testing.T) { host := "console.example.com" ctx, reg, s, pub, opt := startUnitFixture(t, host) bake := t.TempDir() ptr, err := Channel(ctx, s, pub, "bab", "console", "canary/x") if err != nil { t.Fatal(err) } if err := WriteRefusal(bake, Refusal{Host: host, App: "console", Version: ptr.Version, PointerPromotedAt: ptr.PromotedAt, Reason: "exited: status=2"}); err != nil { t.Fatal(err) } restart := func(h string) error { t.Fatalf("restart consulted for refused %s", h); return nil } relayed := false var out bytes.Buffer withCapturedStdout(t, &out, func() { if err := Pass(ctx, reg, s, pub, opt, restart, BakeDir(bake), OnStageBlocked(func(string, string, error) { relayed = true })); err != nil { t.Fatalf("Pass: %v", err) } }) if v, _ := LiveVersion(opt, host); v != "old" { t.Fatalf("live = %q, want old", v) } if relayed { t.Fatal("a refused promotion was relayed to root") } // The ordering this gate exists for: the check sits BEFORE ensureStaged. // Staging succeeds in this fixture, so "relayed" alone cannot tell the // gate's position; the absence of the build and its manifest sibling can. for _, p := range []string{ VersionPath(opt, "bab", "new", "console"), StagedManifestPath(opt, "bab", "new", "console"), } { if _, err := os.Stat(p); !errors.Is(err, os.ErrNotExist) { t.Fatalf("refused version was staged: stat %s = %v, want not-exist", p, err) } } if !strings.Contains(out.String(), "promote --retry") { t.Fatalf("printed %q, want the retry hint", out.String()) } } // TestChannelBackOnTheRestoredBuildClearsTheRefusal (final review F8): v2 // was refused and rolled back to v1, then the channel was promoted back to // v1. The pass finds the box up to date, restarts nothing, and removes the // v2 record that no longer blocks anything. func TestChannelBackOnTheRestoredBuildClearsTheRefusal(t *testing.T) { host := "console.example.com" s := NewDirStore(t.TempDir()) ctx := context.Background() priv, pub := testKeys(t) opt := t.TempDir() reg := testRegistry(t) shipRelease(t, s, "bab", "console", "v1", adoptPlatform, []byte("good build")) shipRelease(t, s, "bab", "console", "v2", adoptPlatform, []byte("bad build")) if err := StageArtifact(opt, "bab", "v1", "console", []byte("good build")); err != nil { t.Fatal(err) } addRoute(t, reg, host, "console", "canary/x", false) if err := PointLive(opt, host, "bab", "v1", "console"); err != nil { t.Fatal(err) } if err := Promote(ctx, s, priv, "bab", "console", "canary/x", "v1"); err != nil { t.Fatal(err) } bake := t.TempDir() if err := WriteRefusal(bake, Refusal{Host: host, App: "console", Version: "v2", PointerPromotedAt: time.Unix(1, 0).UTC(), Reason: "exited: status=2", RestoredVersion: "v1", Recovered: true}); err != nil { t.Fatal(err) } restart := func(h string) error { t.Fatalf("restart consulted for up-to-date %s", h); return nil } if err := Pass(ctx, reg, s, pub, opt, restart, BakeDir(bake)); err != nil { t.Fatalf("Pass: %v", err) } if v, _ := LiveVersion(opt, host); v != "v1" { t.Fatalf("live = %q, want v1", v) } if r, _ := ReadRefusal(bake, host); r != nil { t.Fatalf("stale v2 refusal survived the channel's return to v1: %+v", r) } } // TestUpToDateKeepsARefusalThatStillBlocks: the link is on the refused build // only when its rollback failed; that record still names the pointer and // stays for the watcher. func TestUpToDateKeepsARefusalThatStillBlocks(t *testing.T) { host := "console.example.com" ctx, reg, s, pub, opt := startUnitFixture(t, host) if err := StageArtifact(opt, "bab", "new", "console", []byte("bad build")); err != nil { t.Fatal(err) } if err := PointLive(opt, host, "bab", "new", "console"); err != nil { t.Fatal(err) } ptr, err := Channel(ctx, s, pub, "bab", "console", "canary/x") if err != nil { t.Fatal(err) } bake := t.TempDir() if err := WriteRefusal(bake, Refusal{Host: host, App: "console", Version: ptr.Version, PointerPromotedAt: ptr.PromotedAt, Reason: "exited: status=2", RecoveryError: "could not roll the symlink back"}); err != nil { t.Fatal(err) } if err := Pass(ctx, reg, s, pub, opt, func(string) error { return nil }, BakeDir(bake)); err != nil { t.Fatalf("Pass: %v", err) } if r, _ := ReadRefusal(bake, host); r == nil { t.Fatal("a refusal that still blocks the pointer was removed") } } // TestRePromotionClearsTheBlock: a pointer re-signed for the same version // (what promote --retry does) is adopted, and the stale record is removed. func TestRePromotionClearsTheBlock(t *testing.T) { host := "console.example.com" ctx, reg, s, pub, opt := startUnitFixture(t, host) bake := t.TempDir() if err := WriteRefusal(bake, Refusal{Host: host, App: "console", Version: "new", PointerPromotedAt: time.Unix(1, 0).UTC(), Reason: "older promotion"}); err != nil { t.Fatal(err) } if err := Pass(ctx, reg, s, pub, opt, func(string) error { return nil }, BakeDir(bake)); err != nil { t.Fatalf("Pass: %v", err) } if v, _ := LiveVersion(opt, host); v != "new" { t.Fatalf("live = %q, want new (the record names an older promotion)", v) } if r, _ := ReadRefusal(bake, host); r != nil { t.Fatalf("stale refusal survived a successful adoption: %+v", r) } }