package host import ( "bytes" "context" "crypto/ed25519" "encoding/base64" "errors" "fmt" "log" "net/http" "net/http/httptest" "os" "path/filepath" "runtime" "slices" "sort" "strings" "sync" "sync/atomic" "testing" "time" "github.com/carlosframework/platform/internal/appconfig" "github.com/carlosframework/platform/internal/domains" "github.com/carlosframework/platform/internal/provision" "github.com/carlosframework/platform/internal/registry" "github.com/carlosframework/platform/internal/release" "github.com/carlosframework/platform/internal/status" ) // genKit is the shared setup for the generation-poll tests: a directory // store standing in for the deployment bucket, and a context. No registry // — genPollPass takes routes, not a *registry.Registry, precisely so these // tests need no bolt database (the loop does its own reg.List()). func genKit(t *testing.T) (context.Context, release.Store) { t.Helper() return context.Background(), release.NewDirStore(t.TempDir()) } // execRoute is the socket-exec instance shape appconfig.ExecConfigApps // accepts (execSyncable): a non-platform, ready instance with a socket // upstream. Field-for-field the fixture shape syncexec_test.go uses, so a // change to the predicates fails these tests for the same reason it fails // appconfig's. func execRoute(host, acct, app string) registry.Route { return registry.Route{ Host: host, Kind: "instance", App: app, Account: acct, Status: "ready", Socket: "/run/carlos/" + host + ".sock", } } // unitRoute is the carlos-app@ shape appconfig.UnitConfigApps accepts: the // same syncable instance, backed by a systemd unit whose env file only the // root adopt pass can write. The agent's half of a change to one of these // is the marker file, never a render. func unitRoute(host, acct, app string) registry.Route { rt := execRoute(host, acct, app) rt.Backing = registry.BackingUnit return rt } // countingStore records how many times each key was read, so a test can // pin that an app carrying BOTH route shapes still costs ONE generation // read per tick — the whole economy of the poll set being a union. type countingStore struct { release.Store gets map[string]int } func (c *countingStore) Get(ctx context.Context, key string) ([]byte, string, error) { if c.gets == nil { c.gets = map[string]int{} } c.gets[key]++ return c.Store.Get(ctx, key) } // genEtag reads the app's generation etag straight from the store, so a // test can assert the cache holds what the bucket actually reports rather // than a value the test made up. func genEtag(t *testing.T, ctx context.Context, s release.Store, acct, app string) string { t.Helper() _, etag, err := s.Get(ctx, appconfig.GenKey(acct, app)) if err != nil { t.Fatalf("read generation %s/%s: %v", acct, app, err) } return etag } func touch(t *testing.T, ctx context.Context, s release.Store, acct, app string) { t.Helper() if err := appconfig.TouchGeneration(ctx, s, acct, app); err != nil { t.Fatalf("TouchGeneration %s/%s: %v", acct, app, err) } } // failingKeyStore fails Get for exactly one key — a transient bucket blip // against a single app's generation object while every other key in the // same pass resolves normally (the shape an S3 503 against one object // takes). Same wrapper appconfig's syncexec_test.go uses, for the same // reason. type failingKeyStore struct { release.Store failKey string err error } func (f *failingKeyStore) Get(ctx context.Context, key string) ([]byte, string, error) { if key == f.failKey { return nil, "", f.err } return f.Store.Get(ctx, key) } func TestGenPollPassFirstSightIsSilent(t *testing.T) { ctx, s := genKit(t) touch(t, ctx, s, "bab", "eleven") routes := []registry.Route{execRoute("a.example.com", "bab", "eleven")} cache := newGenCache() execChanged, unitChanged, errs := genPollPass(ctx, routes, s, cache) if execChanged || unitChanged { t.Fatal("first sight of an app must not trigger a sync (boot would re-bounce the world)") } if len(errs) != 0 { t.Fatalf("unexpected read errors: %v", errs) } want := genEtag(t, ctx, s, "bab", "eleven") if got := cache.etags[[2]string{"bab", "eleven"}]; got != want { t.Fatalf("cache etag = %q, want the store's %q", got, want) } } func TestGenPollPassUnchangedIsSilent(t *testing.T) { ctx, s := genKit(t) touch(t, ctx, s, "bab", "eleven") routes := []registry.Route{execRoute("a.example.com", "bab", "eleven")} cache := newGenCache() genPollPass(ctx, routes, s, cache) execChanged, unitChanged, errs := genPollPass(ctx, routes, s, cache) if execChanged || unitChanged { t.Fatal("an unmoved generation etag must not trigger a sync") } if len(errs) != 0 { t.Fatalf("unexpected read errors: %v", errs) } } func TestGenPollPassTouchedGenerationTriggersSync(t *testing.T) { ctx, s := genKit(t) touch(t, ctx, s, "bab", "eleven") routes := []registry.Route{execRoute("a.example.com", "bab", "eleven")} cache := newGenCache() genPollPass(ctx, routes, s, cache) touch(t, ctx, s, "bab", "eleven") execChanged, unitChanged, errs := genPollPass(ctx, routes, s, cache) if !execChanged { t.Fatal("a moved generation etag must trigger a sync") } if unitChanged { t.Fatal("an exec-only app must never ask for a root adopt pass") } if len(errs) != 0 { t.Fatalf("unexpected read errors: %v", errs) } want := genEtag(t, ctx, s, "bab", "eleven") if got := cache.etags[[2]string{"bab", "eleven"}]; got != want { t.Fatalf("cache etag = %q, want the new %q (a stale cache would re-sync forever)", got, want) } if execChanged, unitChanged, _ := genPollPass(ctx, routes, s, cache); execChanged || unitChanged { t.Fatal("the pass after a change must be silent again") } } func TestGenPollPassAbsentGenerationStaysSilent(t *testing.T) { ctx, s := genKit(t) // No touch at all: an app last written by a pre-generation CLI. Legal // forever — the hourly configSyncLoop is its floor. routes := []registry.Route{execRoute("a.example.com", "bab", "eleven")} cache := newGenCache() for i := range 2 { execChanged, unitChanged, errs := genPollPass(ctx, routes, s, cache) if execChanged || unitChanged { t.Fatalf("pass %d: an app with no generation object must never trigger a sync", i) } if len(errs) != 0 { t.Fatalf("pass %d: absent is not a read error: %v", i, errs) } } if got := cache.etags[[2]string{"bab", "eleven"}]; got != genAbsent { t.Fatalf("cache = %q, want the known-absent sentinel %q", got, genAbsent) } } func TestGenPollPassAbsentToPresentTriggersSync(t *testing.T) { ctx, s := genKit(t) routes := []registry.Route{execRoute("a.example.com", "bab", "eleven")} cache := newGenCache() genPollPass(ctx, routes, s, cache) // records known-absent touch(t, ctx, s, "bab", "eleven") // the app's first ever config write execChanged, _, errs := genPollPass(ctx, routes, s, cache) if !execChanged { t.Fatal("absent -> present is the first write to an old app: it must sync") } if len(errs) != 0 { t.Fatalf("unexpected read errors: %v", errs) } } func TestGenPollPassOneChangedAppSyncsOnce(t *testing.T) { ctx, s := genKit(t) touch(t, ctx, s, "bab", "eleven") touch(t, ctx, s, "bab", "titogo") routes := []registry.Route{ execRoute("a.example.com", "bab", "eleven"), execRoute("b.example.com", "bab", "titogo"), } cache := newGenCache() genPollPass(ctx, routes, s, cache) touch(t, ctx, s, "bab", "titogo") execChanged, _, errs := genPollPass(ctx, routes, s, cache) if !execChanged { t.Fatal("one moved app in the set must report the pass as changed") } if len(errs) != 0 { t.Fatalf("unexpected read errors: %v", errs) } // One flag for the whole pass: the caller runs ONE SyncExec, which // resolves every app itself. The unchanged app's cache entry must be // carried forward untouched so it doesn't re-fire next tick. if got, want := cache.etags[[2]string{"bab", "eleven"}], genEtag(t, ctx, s, "bab", "eleven"); got != want { t.Fatalf("unchanged app's cache = %q, want %q", got, want) } if execChanged, _, _ := genPollPass(ctx, routes, s, cache); execChanged { t.Fatal("the next pass must be silent for both apps") } } func TestGenPollPassDropsVanishedApps(t *testing.T) { ctx, s := genKit(t) touch(t, ctx, s, "bab", "eleven") touch(t, ctx, s, "bab", "titogo") both := []registry.Route{ execRoute("a.example.com", "bab", "eleven"), execRoute("b.example.com", "bab", "titogo"), } one := both[:1] cache := newGenCache() genPollPass(ctx, both, s, cache) genPollPass(ctx, one, s, cache) // titogo's route is gone if _, ok := cache.etags[[2]string{"bab", "titogo"}]; ok { t.Fatal("a route that left the registry must leave no cache entry behind") } // The proof that dropping matters: while the app was unrouted its // config moved. The ETAG half must record that silently (first sight // again) rather than comparing against a remembered value — a route // being re-added already gets fresh config from the spawn path. // // The tick itself is NOT silent, and that is the route-shape half // speaking, not the etag one (issue #196): a route entering or leaving // scope changes what belongs on disk — a file to write, or one to reap // — so it earns a pass. The pass is cheap and cannot cost a restart: // SyncExec bounces only a route whose rendered bytes actually changed. // What this test still pins directly is the cache entry above; the // etag half's silence is no longer separately observable from here, // since the shape movement covers it either way. touch(t, ctx, s, "bab", "titogo") _, _, errs := genPollPass(ctx, both, s, cache) if len(errs) != 0 { t.Fatalf("unexpected read errors: %v", errs) } // Settled again: with the route set stable and no bucket movement, the // next tick must be silent. A remembered etag or a mis-stored digest // would still be firing here. execChanged, unitChanged, errs := genPollPass(ctx, both, s, cache) if execChanged || unitChanged { t.Fatal("a settled tick after a re-add must be silent") } if len(errs) != 0 { t.Fatalf("unexpected read errors: %v", errs) } } func TestGenPollPassReadErrorLeavesCacheAndKeepsGoing(t *testing.T) { ctx, dir := genKit(t) touch(t, ctx, dir, "bab", "eleven") touch(t, ctx, dir, "bab", "titogo") routes := []registry.Route{ execRoute("a.example.com", "bab", "eleven"), execRoute("b.example.com", "bab", "titogo"), } cache := newGenCache() genPollPass(ctx, routes, dir, cache) elevenEtag := cache.etags[[2]string{"bab", "eleven"}] // eleven's generation object goes unreadable; titogo's moves. boom := errors.New("bucket said no") s := &failingKeyStore{Store: dir, failKey: appconfig.GenKey("bab", "eleven"), err: boom} touch(t, ctx, dir, "bab", "titogo") execChanged, _, errs := genPollPass(ctx, routes, s, cache) if !execChanged { t.Fatal("one app failing must not stop the others being evaluated") } if len(errs) != 1 { t.Fatalf("read errors = %v, want exactly one (aggregated into one log line per pass)", errs) } if !errors.Is(errs[0], boom) { t.Fatalf("read error = %v, want it to wrap %v", errs[0], boom) } if got := cache.etags[[2]string{"bab", "eleven"}]; got != elevenEtag { t.Fatalf("unreadable app's cache = %q, want it untouched at %q (a cleared entry would re-fire on recovery)", got, elevenEtag) } } func TestGenPollPassUnitFirstSightIsSilent(t *testing.T) { ctx, s := genKit(t) touch(t, ctx, s, "bab", "unitapp") routes := []registry.Route{unitRoute("u.example.com", "bab", "unitapp")} cache := newGenCache() execChanged, unitChanged, errs := genPollPass(ctx, routes, s, cache) if execChanged || unitChanged { t.Fatal("first sight of a unit-backed app must not request a root pass (boot would restart every tenant)") } if len(errs) != 0 { t.Fatalf("unexpected read errors: %v", errs) } want := genEtag(t, ctx, s, "bab", "unitapp") if got := cache.etags[[2]string{"bab", "unitapp"}]; got != want { t.Fatalf("cache etag = %q, want the store's %q", got, want) } } func TestGenPollPassUnitMovementRequestsRootPassOnly(t *testing.T) { ctx, s := genKit(t) touch(t, ctx, s, "bab", "unitapp") touch(t, ctx, s, "bab", "eleven") routes := []registry.Route{ unitRoute("u.example.com", "bab", "unitapp"), execRoute("a.example.com", "bab", "eleven"), } cache := newGenCache() genPollPass(ctx, routes, s, cache) touch(t, ctx, s, "bab", "unitapp") execChanged, unitChanged, errs := genPollPass(ctx, routes, s, cache) if !unitChanged { t.Fatal("a moved generation for a unit-backed app must request the root adopt pass") } if execChanged { t.Fatal("the exec half must stay quiet: no exec-config app moved, so SyncExec would find nothing to write") } if len(errs) != 0 { t.Fatalf("unexpected read errors: %v", errs) } if execChanged, unitChanged, _ := genPollPass(ctx, routes, s, cache); execChanged || unitChanged { t.Fatal("the pass after a change must be silent again (a stale cache would re-touch the marker forever)") } } func TestGenPollPassBothShapesSetBothHalvesOnOneRead(t *testing.T) { ctx, dir := genKit(t) touch(t, ctx, dir, "zed", "mixed") // One app, two routes: the activator spawns one, systemd owns the other. // Its config lives in ONE generation object, so the union must read that // object once per tick and report both halves off it. routes := []registry.Route{ execRoute("mixed-exec.example.com", "zed", "mixed"), unitRoute("mixed-unit.example.com", "zed", "mixed"), } s := &countingStore{Store: dir} cache := newGenCache() genPollPass(ctx, routes, s, cache) touch(t, ctx, dir, "zed", "mixed") execChanged, unitChanged, errs := genPollPass(ctx, routes, s, cache) if !execChanged || !unitChanged { t.Fatalf("execChanged=%v unitChanged=%v, want both — the app has both route shapes", execChanged, unitChanged) } if len(errs) != 0 { t.Fatalf("unexpected read errors: %v", errs) } if got := s.gets[appconfig.GenKey("zed", "mixed")]; got != 2 { t.Fatalf("generation object read %d times over 2 ticks, want 2 — an app in both sets must cost one read per tick", got) } } func TestGenPollPassPrunesUnitAppsToo(t *testing.T) { ctx, s := genKit(t) touch(t, ctx, s, "bab", "unitapp") both := []registry.Route{ execRoute("a.example.com", "bab", "eleven"), unitRoute("u.example.com", "bab", "unitapp"), } one := both[:1] cache := newGenCache() genPollPass(ctx, both, s, cache) // The live set the cache is pruned against is the UNION: a unit app whose // last route left must lose its entry exactly as an exec app does, or its // remembered etag would fire a bogus root-pass request when the route // comes back (adoption already delivers fresh config on the way in). genPollPass(ctx, one, s, cache) if _, ok := cache.etags[[2]string{"bab", "unitapp"}]; ok { t.Fatal("a unit-backed route that left the registry must leave no cache entry behind") } // As in TestGenPollPassDropsVanishedApps: the re-add tick itself moves // the route-shape digest (issue #196), so the silence worth asserting // is the SETTLED tick after it. The dropped cache entry above is what // pins the etag half directly. touch(t, ctx, s, "bab", "unitapp") _, _, errs := genPollPass(ctx, both, s, cache) if len(errs) != 0 { t.Fatalf("unexpected read errors: %v", errs) } execChanged, unitChanged, errs := genPollPass(ctx, both, s, cache) if execChanged || unitChanged { t.Fatal("a settled tick after a unit-backed re-add must be silent") } if len(errs) != 0 { t.Fatalf("unexpected read errors: %v", errs) } } // ---- cold-wake materialize hook (carlos-deploy spec §5) --------------------- // TestMaterializeHookDegradesToNil pins the safe degrade: a box with no // release store, or no (or a broken) release pubkey, gets a NIL hook — which // is today's behavior exactly, the ExecBackend's "no live binary" wake // failure. A non-nil hook built from a half-configured box would turn every // cold wake into a staging error that names the wrong problem. func TestMaterializeHookDegradesToNil(t *testing.T) { reg, err := registry.Open(filepath.Join(t.TempDir(), "registry.db")) if err != nil { t.Fatalf("open registry: %v", err) } pub, _, err := ed25519.GenerateKey(nil) if err != nil { t.Fatalf("generate key: %v", err) } pubB64 := base64.StdEncoding.EncodeToString(pub) for _, c := range []struct{ name, dir, pub string }{ {"no store, no key", "", ""}, {"store but no key", t.TempDir(), ""}, {"store but unparseable key", t.TempDir(), "not-base64!!"}, {"key but no store", "", pubB64}, } { t.Run(c.name, func(t *testing.T) { t.Setenv("CARLOS_DEPLOYMENT_BUCKET", "") t.Setenv("CARLOS_DEPLOYMENT_DIR", c.dir) t.Setenv("CARLOS_RELEASE_PUBKEY", c.pub) if hook := materializeHook(reg, t.TempDir()); hook != nil { t.Fatal("hook must be nil on a box that cannot adopt — nil is today's behavior") } }) } } // TestMaterializeHookStagesAColdRoute exercises the wired path end to end // through the very function the ExecBackend calls: a fully configured box, a // route with nothing staged, and after one hook call the live symlink exists // and names the channel head. This is what retires "no live binary" as a // wake-failure class (issue #77). func TestMaterializeHookStagesAColdRoute(t *testing.T) { dir := t.TempDir() optDir := t.TempDir() s := release.NewDirStore(dir) ctx := context.Background() plat := runtime.GOOS + "-" + runtime.GOARCH pub, priv, err := ed25519.GenerateKey(nil) if err != nil { t.Fatalf("generate key: %v", err) } reg, err := registry.Open(filepath.Join(t.TempDir(), "registry.db")) if err != nil { t.Fatalf("open registry: %v", err) } if err := reg.Upsert(registry.Route{ Host: "cold.example.com", Kind: "instance", App: "coldapp", Channel: "canary/dev", Status: "ready", }); err != nil { t.Fatalf("upsert: %v", err) } m := release.Manifest{ App: "coldapp", Version: "abc1234", Kind: "binary", Artifacts: map[string]release.Artifact{plat: {}}, } if err := release.Ship(ctx, s, "bab", m, map[string][]byte{plat: []byte("cold build")}); err != nil { t.Fatalf("Ship: %v", err) } if err := release.Promote(ctx, s, priv, "bab", "coldapp", "canary/dev", "abc1234"); err != nil { t.Fatalf("Promote: %v", err) } t.Setenv("CARLOS_DEPLOYMENT_BUCKET", "") t.Setenv("CARLOS_DEPLOYMENT_DIR", dir) t.Setenv("CARLOS_RELEASE_PUBKEY", base64.StdEncoding.EncodeToString(pub)) t.Setenv("CARLOS_OPT_DIR", optDir) hook := materializeHook(reg, t.TempDir()) if hook == nil { t.Fatal("a fully configured box must get a hook") } if err := hook(ctx, "cold.example.com"); err != nil { t.Fatalf("hook: %v", err) } if v, err := release.LiveVersion(optDir, "cold.example.com"); err != nil || v != "abc1234" { t.Fatalf("live = %q, %v; want abc1234", v, err) } } // restartTestRegistry is the bolt-backed registry the restart-dispatch tests // reason over — makeRestartRoute's whole job is to ask the registry what // backs a host, so a real registry (not a fake) is what the test needs. func restartTestRegistry(t *testing.T) *registry.Registry { t.Helper() reg, err := registry.Open(filepath.Join(t.TempDir(), "registry.db")) if err != nil { t.Fatalf("open registry: %v", err) } t.Cleanup(func() { if err := reg.Close(); err != nil { t.Error(err) } }) return reg } // TestMakeRestartRouteDispatchesOnBacking pins the correction this split // exists for: a unit tenant is try-restarted, an exec-backed instance is // Bounced. Before the split both took the systemctl path — and systemctl // cannot reach a child of the agent process, so a warm exec instance kept // serving its old binary until it happened to hibernate. func TestMakeRestartRouteDispatchesOnBacking(t *testing.T) { reg := restartTestRegistry(t) if err := reg.Upsert(registry.Route{ Host: "unit.test", Kind: "instance", App: "a", Status: "ready", Backing: registry.BackingUnit, }); err != nil { t.Fatalf("upsert unit route: %v", err) } if err := reg.Upsert(registry.Route{ Host: "exec.test", Kind: "instance", App: "a", Status: "ready", }); err != nil { t.Fatalf("upsert exec route: %v", err) } var restarted, bounced []string restart := makeRestartRoute(reg, func(h string) { restarted = append(restarted, h) }, unitOnly(func(u string) error { t.Fatalf("named-unit arm fired for %q on a route that names no unit", u) return nil }), func(h string) bool { bounced = append(bounced, h); return true }) if err := restart("unit.test"); err != nil { t.Fatalf("restart unit.test: %v", err) } if err := restart("exec.test"); err != nil { t.Fatalf("restart exec.test: %v", err) } if len(restarted) != 1 || restarted[0] != "unit.test" { t.Fatalf("unit path: %v", restarted) } if len(bounced) != 1 || bounced[0] != "exec.test" { t.Fatalf("exec path: %v", bounced) } } // TestMakeRestartRouteUnknownHostFallsBackToUnit: a host the registry cannot // answer for gets the OLD behavior (try-restart, a safe no-op for a // nonexistent unit) — never a bounce of a process we might not own. func TestMakeRestartRouteUnknownHostFallsBackToUnit(t *testing.T) { reg := restartTestRegistry(t) var restarted []string restart := makeRestartRoute(reg, func(h string) { restarted = append(restarted, h) }, unitOnly(func(u string) error { t.Fatalf("named-unit arm fired for %q on a host the registry cannot answer for", u) return nil }), nil) if err := restart("ghost.test"); err != nil { t.Fatalf("restart ghost.test: %v", err) } if len(restarted) != 1 || restarted[0] != "ghost.test" { t.Fatalf("fallback: %v", restarted) } } // TestMakeRestartRouteExecFallsThroughWhenNothingBounced covers the state // issue #14 is about (adopt_advice.go's warning): a route claims exec backing // but a carlos-app@ unit is ACTIVE — an unstamped unit tenant. The // bounce finds nothing of its own to cycle and returns false, and try-restart // must still run, exactly as it did before the dispatch split. Same for a box // with no activator at all (nil bounce), which must not panic either. // // The cost when the route really is a sleeping exec child is one try-restart // of a stopped or absent unit — a no-op, at most one log line per version // change per host, since the callback only fires when the version moves. func TestMakeRestartRouteExecFallsThroughWhenNothingBounced(t *testing.T) { t.Parallel() for _, tc := range []struct { name string bounce func(string) bool }{ {"no activator armed", nil}, {"activator armed but nothing awake", func(string) bool { return false }}, } { t.Run(tc.name, func(t *testing.T) { reg := restartTestRegistry(t) if err := reg.Upsert(registry.Route{ Host: "exec.test", Kind: "instance", App: "a", Status: "ready", }); err != nil { t.Fatalf("upsert exec route: %v", err) } var restarted []string restart := makeRestartRoute(reg, func(h string) { restarted = append(restarted, h) }, unitOnly(func(u string) error { t.Fatalf("named-unit arm fired for %q on a route that names no unit", u) return nil }), tc.bounce) if err := restart("exec.test"); err != nil { t.Fatalf("restart exec.test: %v", err) } if len(restarted) != 1 || restarted[0] != "exec.test" { t.Fatalf("a bounce that cycled nothing must fall through to try-restart: %v", restarted) } }) } } // TestMakeRestartRouteRestartsTheUnitARouteNames pins issue #110's fix: an // addr-backed route that names its own unit (the consoles, `-addr // 127.0.0.1:8090` under a bespoke console.service) is cycled BY THAT NAME, // and neither of the other two arms runs — both would cycle nothing here, // which is exactly how adoption came to stamp a Version the running process // did not have. func TestMakeRestartRouteRestartsTheUnitARouteNames(t *testing.T) { reg := restartTestRegistry(t) if err := reg.Upsert(registry.Route{ Host: "console.carloku.com", Kind: "instance", App: "console", Addr: "127.0.0.1:8090", Status: "ready", Unit: "console.service", }); err != nil { t.Fatalf("upsert console route: %v", err) } var units, restarted, bounced []string restart := makeRestartRoute(reg, func(h string) { restarted = append(restarted, h) }, unitOnly(func(u string) error { units = append(units, u); return nil }), func(h string) bool { bounced = append(bounced, h); return true }) if err := restart("console.carloku.com"); err != nil { t.Fatalf("restart console.carloku.com: %v", err) } if len(units) != 1 || units[0] != "console.service" { t.Fatalf("named-unit arm: %v, want [console.service]", units) } if len(restarted) != 0 { t.Fatalf("a route that names its unit must not also be try-restarted as carlos-app@: %v", restarted) } if len(bounced) != 0 { t.Fatalf("a route that names its unit must not be offered to the activator: %v", bounced) } } // The named-unit arm wins over BACKING, not merely over its absence: a route // carrying both a unit name and unit backing names where its process is, and // the name is the more specific answer. Pinned so a later reordering of the // dispatch has to argue with a test rather than silently cycle // carlos-app@ for a route whose process is somewhere else. func TestMakeRestartRouteUnitNameBeatsBacking(t *testing.T) { reg := restartTestRegistry(t) if err := reg.Upsert(registry.Route{ Host: "odd.test", Kind: "instance", App: "a", Status: "ready", Backing: registry.BackingUnit, Unit: "bespoke.service", }); err != nil { t.Fatalf("upsert: %v", err) } var units, restarted []string restart := makeRestartRoute(reg, func(h string) { restarted = append(restarted, h) }, unitOnly(func(u string) error { units = append(units, u); return nil }), nil) if err := restart("odd.test"); err != nil { t.Fatalf("restart odd.test: %v", err) } if len(units) != 1 || units[0] != "bespoke.service" || len(restarted) != 0 { t.Fatalf("units = %v, host try-restarts = %v; want the named unit alone", units, restarted) } } // The root pass (RunAdopt, carlos-adopt.service) must carry the named-unit // arm, because on a real box it is the pass that actually adopts a console: // the agent runs inside carlos-edge.service's ProtectSystem=strict sandbox // and cannot write /opt/carlos, so its stage is blocked and relayed here. // Everything else about it is unchanged — no activator, so every route it // used to try-restart by host still is. func TestRootRestartRouteRefusesMissingDeploymentBinding(t *testing.T) { t.Parallel() reg := restartTestRegistry(t) if err := reg.Upsert(registry.Route{Host: "console.carloku.com", Kind: "instance", App: "console", Addr: "127.0.0.1:8090", Status: "ready", Unit: "console.service"}); err != nil { t.Fatal(err) } if err := rootRestartRoute(reg, t.TempDir(), nil)("console.carloku.com"); err == nil { t.Fatal("restarted without deployment binding") } } // TestAdoptPassRestartsANamedUnitOnAdoption is the end-to-end: a promote the // pass adopts must reach the route's own unit, not just repoint a symlink and // stamp a version. This is the shape of the deploy that produced issue #110. func TestAdoptPassRestartsANamedUnitOnAdoption(t *testing.T) { dir := t.TempDir() optDir := t.TempDir() s := release.NewDirStore(dir) ctx := context.Background() plat := runtime.GOOS + "-" + runtime.GOARCH pub, priv, err := ed25519.GenerateKey(nil) if err != nil { t.Fatalf("generate key: %v", err) } reg := restartTestRegistry(t) if err := reg.Upsert(registry.Route{ Host: "console.test", Kind: "instance", App: "console", Channel: "canary/console", Addr: "127.0.0.1:8090", Status: "ready", Unit: "console.service", }); err != nil { t.Fatalf("upsert console.test: %v", err) } m := release.Manifest{ App: "console", Version: "v1", Kind: "binary", Artifacts: map[string]release.Artifact{plat: {}}, } if err := release.Ship(ctx, s, "bab", m, map[string][]byte{plat: []byte("build one")}); err != nil { t.Fatalf("Ship: %v", err) } if err := release.Promote(ctx, s, priv, "bab", "console", "canary/console", "v1"); err != nil { t.Fatalf("Promote: %v", err) } var units []string restart := makeRestartRoute(reg, func(h string) { t.Fatalf("carlos-app@ arm fired for %q — the console has no such unit", h) }, unitOnly(func(u string) error { units = append(units, u); return nil }), func(h string) bool { t.Fatalf("bounce arm fired for %q — the activator owns no console child", h) return false }) if err := release.Pass(ctx, reg, s, pub, optDir, restart, release.RestartHibernating(restart)); err != nil { t.Fatalf("Pass: %v", err) } if len(units) != 1 || units[0] != "console.service" { t.Fatalf("adoption cycled %v, want [console.service] — the header would claim v1 over the old process", units) } if v, err := release.LiveVersion(optDir, "console.test"); err != nil || v != "v1" { t.Fatalf("console.test live = %q, %v; want v1", v, err) } } // TestAdoptPassRefusesToStampAVersionAWrongUnitDidNotStart is the honesty // half of issue #110, and the reason the named-unit arm returns its error // where the other two swallow theirs. A well-formed but WRONG unit name // (`systemctl try-restart` exits 5, "not loaded") restarted nothing — exactly // the state the column exists to end — and if that were swallowed adoption // would stamp Version, the edge would serve the new X-Carlos-Version, and // `carlos deploy` would report live over the old process. // // So: no Version stamp, and the failure is in the journal under the unit name // an operator would grep for. The next pass retries. func TestAdoptPassRefusesToStampAVersionAWrongUnitDidNotStart(t *testing.T) { dir := t.TempDir() optDir := t.TempDir() s := release.NewDirStore(dir) ctx := context.Background() plat := runtime.GOOS + "-" + runtime.GOARCH pub, priv, err := ed25519.GenerateKey(nil) if err != nil { t.Fatalf("generate key: %v", err) } reg := restartTestRegistry(t) // The stamp is well-formed — ValidUnitName cannot tell a real unit from an // absent one — but no such unit is loaded on the box. if err := reg.Upsert(registry.Route{ Host: "console.test", Kind: "instance", App: "console", Channel: "canary/console", Addr: "127.0.0.1:8090", Status: "ready", Unit: "consoel.service", }); err != nil { t.Fatalf("upsert console.test: %v", err) } m := release.Manifest{ App: "console", Version: "v1", Kind: "binary", Artifacts: map[string]release.Artifact{plat: {}}, } if err := release.Ship(ctx, s, "bab", m, map[string][]byte{plat: []byte("build one")}); err != nil { t.Fatalf("Ship: %v", err) } if err := release.Promote(ctx, s, priv, "bab", "console", "canary/console", "v1"); err != nil { t.Fatalf("Promote: %v", err) } // A fake systemctl that answers the way the real one does for a unit that // is not loaded: exit 5, with systemd's own words on stdout. bin := t.TempDir() if err := os.WriteFile(filepath.Join(bin, "systemctl"), []byte("#!/bin/sh\necho \"Failed to try-restart $2: Unit $2 not found.\"\nexit 5\n"), 0o755); err != nil { t.Fatal(err) } t.Setenv("PATH", bin+string(os.PathListSeparator)+os.Getenv("PATH")) var journal bytes.Buffer log.SetOutput(&journal) t.Cleanup(func() { log.SetOutput(os.Stderr) }) restart := makeRestartRoute(reg, systemctlTryRestart, agentCheckedUnitRestart(systemdUnits{}, optDir, os.Getenv, nil), nil) if err := release.Pass(ctx, reg, s, pub, optDir, restart); err != nil { t.Fatalf("Pass: %v", err) } rt, err := reg.Get("console.test") if err != nil || rt == nil { t.Fatalf("get console.test: %+v, %v", rt, err) } if rt.Version != "" { t.Fatalf("Version = %q after a restart that started nothing — the header would claim a build the console is not running", rt.Version) } if !strings.Contains(journal.String(), "consoel.service") { t.Fatalf("journal = %q, want the failed try-restart logged under the unit name", journal.String()) } } // TestMakeStartRouteStartsOnlyANamedUnit pins the one-arm shape of the // rollback-start dispatch: only an AWAKE route that NAMES its unit has // anything for it to start. An exec route, a unit-backed route with no name, // a host the registry cannot answer for, and any Hibernate-flagged row all // start NOTHING and return nil — starting carlos-app@, an activator // child, or a sleeping row's unit from the rollback path could wake a // sleeper, which no adopt path may do. func TestMakeStartRouteStartsOnlyANamedUnit(t *testing.T) { reg := restartTestRegistry(t) if err := reg.Upsert(registry.Route{ Host: "console.test", Kind: "instance", App: "console", Addr: "127.0.0.1:8090", Status: "ready", Unit: "console.service", }); err != nil { t.Fatalf("upsert console.test: %v", err) } if err := reg.Upsert(registry.Route{ Host: "exec.test", Kind: "instance", App: "a", Status: "ready", }); err != nil { t.Fatalf("upsert exec.test: %v", err) } if err := reg.Upsert(registry.Route{ Host: "unit.test", Kind: "instance", App: "a", Status: "ready", Backing: registry.BackingUnit, }); err != nil { t.Fatalf("upsert unit.test: %v", err) } // A hibernating row carrying a unit name is the misconfiguration // restart.go documents (provision refuses addr-with-hibernate, so no // legitimate named-unit route sleeps) — and it is exactly the row the // STRUCTURAL guard exists for: even here, the rollback-start starts // nothing. if err := reg.Upsert(registry.Route{ Host: "sleeper.test", Kind: "instance", App: "a", Status: "ready", Unit: "sleeper.service", Hibernate: true, }); err != nil { t.Fatalf("upsert sleeper.test: %v", err) } var units []string start := makeStartRoute(reg, func(u string) error { units = append(units, u); return nil }) for _, h := range []string{"console.test", "exec.test", "unit.test", "ghost.test", "sleeper.test"} { if err := start(h); err != nil { t.Fatalf("start %s: %v", h, err) } } if len(units) != 1 || units[0] != "console.service" { t.Fatalf("started %v, want [console.service] alone", units) } } // TestAdoptPassStartsTheNamedUnitAgainAfterAFailedRestart is the end-to-end // for the rollback-start, through the real release.Pass, the real root // dispatch, and the production systemctl path: a named unit whose // try-restart fails (the new build would not come back) is rolled back to // the version it was running — and STARTED on it, because a failed // try-restart leaves the unit down and try-restart is a no-op on a stopped // unit. Without the start the box keeps a correct symlink over no process, // and the NEXT pass's exit-0 no-op stamps the never-run build. func TestAdoptPassStartsTheNamedUnitAgainAfterAFailedRestart(t *testing.T) { dir := t.TempDir() optDir := t.TempDir() s := release.NewDirStore(dir) ctx := context.Background() plat := runtime.GOOS + "-" + runtime.GOARCH pub, priv, err := ed25519.GenerateKey(nil) if err != nil { t.Fatalf("generate key: %v", err) } reg := restartTestRegistry(t) if err := reg.Upsert(registry.Route{ Host: "console.test", Kind: "instance", App: "console", Channel: "canary/console", Addr: "127.0.0.1:8090", Status: "ready", Unit: "console.service", }); err != nil { t.Fatalf("upsert console.test: %v", err) } for _, v := range []string{"old", "new"} { m := release.Manifest{ App: "console", Version: v, Kind: "binary", Artifacts: map[string]release.Artifact{plat: {}}, } if err := release.Ship(ctx, s, "bab", m, map[string][]byte{plat: []byte("build " + v)}); err != nil { t.Fatalf("Ship %s: %v", v, err) } } if err := release.StageArtifact(optDir, "bab", "old", "console", []byte("build old")); err != nil { t.Fatal(err) } if err := release.PointLive(optDir, "console.test", "bab", "old", "console"); err != nil { t.Fatal(err) } if err := release.Promote(ctx, s, priv, "bab", "console", "canary/console", "new"); err != nil { t.Fatalf("Promote new: %v", err) } // A fake systemctl whose try-restart fails the way a service that will // not come back does, and whose start succeeds — recording the argv of // both, so this covers systemctlStartUnit's production exec.Command and // the ordering, not just the constructors. bin := t.TempDir() callLog := filepath.Join(t.TempDir(), "systemctl-calls") if err := os.WriteFile(filepath.Join(bin, "systemctl"), []byte("#!/bin/sh\necho \"$@\" >> "+callLog+"\ncase \"$1\" in\ntry-restart) echo \"Job for $2 failed.\"; exit 1;;\nesac\nexit 0\n"), 0o755); err != nil { t.Fatal(err) } t.Setenv("PATH", bin+string(os.PathListSeparator)+os.Getenv("PATH")) var journal bytes.Buffer log.SetOutput(&journal) t.Cleanup(func() { log.SetOutput(os.Stderr) }) if err := release.Pass(ctx, reg, s, pub, optDir, makeRestartRoute(reg, systemctlTryRestart, agentCheckedUnitRestart(systemdUnits{}, optDir, os.Getenv, nil), nil), release.WithStartUnit(makeStartRoute(reg, systemctlStartUnit))); err != nil { t.Fatalf("Pass: %v", err) } got, err := os.ReadFile(callLog) if err != nil { t.Fatalf("no systemctl calls at all: %v", err) } // The checked arm's show comes first (its empty fake answer verifies no // binding, so the restart falls back to the plain try-restart). if want := "show console.service -p Id -p FragmentPath -p DropInPaths -p ActiveState -p SubState -p InvocationID -p MainPID -p NRestarts -p ExecStart -p ActiveEnterTimestamp\ntry-restart console.service\nstart console.service\n"; string(got) != want { t.Fatalf("systemctl calls = %q, want %q — the rolled-back unit must be STARTED (try-restart would no-op on it)", got, want) } if v, err := release.LiveVersion(optDir, "console.test"); err != nil || v != "old" { t.Fatalf("live = %q, %v; want old", v, err) } rt, err := reg.Get("console.test") if err != nil || rt == nil { t.Fatalf("get console.test: %+v, %v", rt, err) } if rt.Version == "new" { t.Fatal("a build whose restart failed was stamped") } } // TestAdoptPassBouncesAWarmHibernatingInstance is the end-to-end this task // exists for, through the real release.Pass with the real closure: an exec // instance minted the way provision does it (Hibernate: true, Backing "") is // warm when a new version lands, and it BOUNCES onto that version instead of // serving the old binary until it happens to hibernate. // // Before the RestartHibernating option this was dead code in production: // release.Pass split on the Hibernate FLAG and handed every flagged route // restart=nil, so the dispatch never ran for the routes it was written for. func TestAdoptPassBouncesAWarmHibernatingInstance(t *testing.T) { t.Parallel() dir := t.TempDir() optDir := t.TempDir() s := release.NewDirStore(dir) ctx := context.Background() plat := runtime.GOOS + "-" + runtime.GOARCH pub, priv, err := ed25519.GenerateKey(nil) if err != nil { t.Fatalf("generate key: %v", err) } reg := restartTestRegistry(t) // warm.test is awake; sleepy.test is not. Both are exec-backed hibernators // — the shape provision/reconcile.go mints. for _, h := range []string{"warm.test", "sleepy.test"} { if err := reg.Upsert(registry.Route{ Host: h, Kind: "instance", App: "warmapp", Channel: "canary/dev", Status: "ready", Hibernate: true, }); err != nil { t.Fatalf("upsert %s: %v", h, err) } } m := release.Manifest{ App: "warmapp", Version: "v1", Kind: "binary", Artifacts: map[string]release.Artifact{plat: {}}, } if err := release.Ship(ctx, s, "bab", m, map[string][]byte{plat: []byte("build one")}); err != nil { t.Fatalf("Ship: %v", err) } if err := release.Promote(ctx, s, priv, "bab", "warmapp", "canary/dev", "v1"); err != nil { t.Fatalf("Promote: %v", err) } var tried, bounced []string // The activator's answer: only warm.test has a running child. This is // exactly Service.Bounce's contract — false when the host is not awake, and // it never starts anything. restart := makeRestartRoute(reg, func(h string) { tried = append(tried, h) }, unitOnly(func(u string) error { t.Fatalf("named-unit arm fired for %q on a route that names no unit", u) return nil }), func(h string) bool { bounced = append(bounced, h) return h == "warm.test" }) if err := release.Pass(ctx, reg, s, pub, optDir, restart, release.RestartHibernating(restart)); err != nil { t.Fatalf("Pass: %v", err) } // The warm instance was cycled onto the new binary. if len(bounced) != 2 { t.Fatalf("both hibernators must be offered to the callback, got %v", bounced) } if len(tried) != 1 || tried[0] != "sleepy.test" { t.Fatalf("only the host with nothing awake falls through to try-restart: %v", tried) } // And both adopted: the sleeper gets the repoint and execs the new binary // at its natural next wake — it is never started by the pass. for _, h := range []string{"warm.test", "sleepy.test"} { if v, err := release.LiveVersion(optDir, h); err != nil || v != "v1" { t.Fatalf("%s live = %q, %v; want v1", h, v, err) } } } // ---- fastConvergeLoop: serial poll + fast set (carlos-deploy spec §3-§4) ---- // fastFixture is the shared shape of the fast-converge tick tests: a dir // store standing in for the deployment bucket, a bolt registry, a keypair, // and fresh opt/data dirs — internal/release's matFixture adapted to this // package's conventions (restartTestRegistry, real release helpers). type fastFixture struct { store release.Store pub ed25519.PublicKey priv ed25519.PrivateKey reg *registry.Registry optDir string dataDir string } func newFastFixture(t *testing.T) *fastFixture { t.Helper() pub, priv, err := ed25519.GenerateKey(nil) if err != nil { t.Fatalf("generate key: %v", err) } return &fastFixture{ store: release.NewDirStore(t.TempDir()), pub: pub, priv: priv, reg: restartTestRegistry(t), optDir: t.TempDir(), dataDir: t.TempDir(), } } // deps builds one tick's fastDeps with every seam pointed at the fixture. // fullPass, when non-nil, replaces the production reconcile→adopt→config-sync // set with a counter hook — the serial tests pin WHEN the set runs, not what // it does. The pass options come from fastPassOpts, the same builder // production uses, so a test exercising the sweep exercises the real option // set. func (f *fastFixture) deps(fullPass func()) *fastDeps { restart := func(string) error { return nil } d := &fastDeps{ reg: f.reg, store: f.store, pub: f.pub, optDir: f.optDir, restart: restart, passOpts: fastPassOpts(f.dataDir, func(host, ver string, err error) {}, restart, nil), fullPass: func(context.Context) {}, } if fullPass != nil { d.fullPass = func(context.Context) { fullPass() } } return d } // addRoute mints an adoptable instance route on the given channel — the // shape release.Pass sweeps (kind instance, ready, non-platform). func (f *fastFixture) addRoute(t *testing.T, host, channel string) { t.Helper() if err := f.reg.Upsert(registry.Route{ Host: host, Kind: "instance", App: "fastapp", Channel: channel, Status: "ready", }); err != nil { t.Fatalf("upsert %s: %v", host, err) } } // stageLive stages ver and points host's live symlink at it, so // release.LiveVersion answers ver — "this host has nothing outstanding". func (f *fastFixture) stageLive(t *testing.T, host, ver string) { t.Helper() if err := release.StageArtifact(f.optDir, "bab", ver, "fastapp", []byte("staged build")); err != nil { t.Fatalf("StageArtifact: %v", err) } if err := release.PointLive(f.optDir, host, "bab", ver, "fastapp"); err != nil { t.Fatalf("PointLive: %v", err) } } // mutexProbeStore fails the test if any store call arrives while // reconcileMu is NOT held — the deterministic pin that reconcileSerialized // really runs provision.Reconcile under the package mutex (TryLock succeeds // only when nobody holds it). No goroutines, no sleeps: holding the mutex // for the duration of the store work IS the serialization property. type mutexProbeStore struct { release.Store t *testing.T hits *int } func (s *mutexProbeStore) probe() { *s.hits++ if reconcileMu.TryLock() { reconcileMu.Unlock() s.t.Error("store consulted without reconcileMu held — a concurrent Reconcile pass could interleave and mass-retire fresh routes") } } func (s *mutexProbeStore) Get(ctx context.Context, key string) ([]byte, string, error) { s.probe() return s.Store.Get(ctx, key) } func (s *mutexProbeStore) List(ctx context.Context, prefix string) ([]string, error) { s.probe() return s.Store.List(ctx, prefix) } // TestReconcileSerializedHoldsTheMutex pins finding-1's fix: Reconcile's // removal phase deletes any Provisioned in-scope route absent from its own // top-of-pass record snapshot, so two overlapping passes (instancesLoop's // 5-minute tick vs fastConvergeLoop's serial-triggered full set) could // delete a route the other pass had just minted — and the re-mint comes // back with capability grants revoked. Both loops therefore run Reconcile // only through reconcileSerialized, which must hold reconcileMu across the // whole pass. func TestReconcileSerializedHoldsTheMutex(t *testing.T) { var hits int s := &mutexProbeStore{Store: release.NewDirStore(t.TempDir()), t: t, hits: &hits} reg := restartTestRegistry(t) cfg := provision.ReconcileConfig{DataDir: t.TempDir()} if err := reconcileSerialized(context.Background(), s, reg, cfg); err != nil { t.Fatalf("reconcileSerialized: %v", err) } if hits == 0 { t.Fatal("the probe store was never consulted — this test pinned nothing") } } // TestReconcileSerializedRunsBothRecordFamilies pins the wiring: the sidecar // pass runs from the SAME serialized entry point as the instance pass, under // the same mutex hold. Wired anywhere else it would either race the instance // pass over one registry, or (worse) be forgotten by one of the two loops and // converge only every five minutes — or never. func TestReconcileSerializedRunsBothRecordFamilies(t *testing.T) { ctx := context.Background() s := release.NewDirStore(t.TempDir()) sqid := domains.Sqid(2) manifest := []byte(`{"id":2,"sqid":"` + sqid + `","name":"Eleven","created_at":"2026-08-02T09:00:00Z"}`) if _, err := s.Put(ctx, "accounts/"+sqid+"/account.json", manifest, ""); err != nil { t.Fatal(err) } if err := provision.PutSidecarManifest(ctx, s, sqid, "eleven", provision.SidecarManifest{ Schema: 1, App: "eleven", Args: []string{"sidecar", "run"}, Channel: "stable"}); err != nil { t.Fatal(err) } host := provision.SidecarHost("bot", "eleven", sqid) if err := provision.CreateSidecarRecord(ctx, s, provision.SidecarRecord{ Schema: 1, Name: "bot", Host: host, App: "eleven", Account: sqid, Channel: "stable"}); err != nil { t.Fatal(err) } reg := restartTestRegistry(t) if err := reconcileSerialized(ctx, s, reg, provision.ReconcileConfig{DataDir: t.TempDir()}); err != nil { t.Fatalf("reconcileSerialized: %v", err) } rt, err := reg.Get(host) if err != nil { t.Fatal(err) } if rt == nil { t.Fatal("no sidecar route minted — the sidecar pass must run from reconcileSerialized, beside the instance pass and inside its mutex") } } // TestFastTickActsOnSerialChangeNotBoot pins seed-don't-replay, in both of // its halves. An ABSENT serial object seeds at zero — absence is not // history, there is nothing to replay, and every production bucket starts // without the object, so a deployment's FIRST-EVER bump (0 → 1) must fire // rather than being swallowed as boot noise. A PRESENT object seeds at its // current value without replaying it (boot must not replay history — // adoptLoop's 1-minute first pass covers boot). After either seed, a CHANGE // runs the full pass set exactly once and an unchanged serial runs nothing. func TestFastTickActsOnSerialChangeNotBoot(t *testing.T) { t.Parallel() env := newFastFixture(t) var passes int d := env.deps(func() { passes++ }) now := time.Unix(1754640000, 0) fastTick(context.Background(), d, now) // no serial object at all: seeds at zero, runs nothing if passes != 0 { t.Fatalf("an absent serial object must not run a pass, got %d", passes) } if err := release.BumpSerial(context.Background(), env.store, now); err != nil { t.Fatalf("BumpSerial: %v", err) } fastTick(context.Background(), d, now.Add(2*time.Second)) // the deployment's first-ever bump if passes != 1 { t.Fatalf("the first-ever bump must fire (absence seeds at zero), got %d", passes) } fastTick(context.Background(), d, now.Add(10*time.Second)) // past clamp, unchanged serial if passes != 1 { t.Fatalf("unchanged serial must not re-run, got %d", passes) } // The boot rule for a PRESENT object: a restarted box (fresh state) // seeds at the current value and must not replay it as a change. var reboots int d2 := env.deps(func() { reboots++ }) fastTick(context.Background(), d2, now.Add(12*time.Second)) // first sight of serial=1: SEED only if reboots != 0 { t.Fatalf("boot seed of a present object must not run a pass, got %d", reboots) } if err := release.BumpSerial(context.Background(), env.store, now); err != nil { t.Fatalf("BumpSerial: %v", err) } fastTick(context.Background(), d2, now.Add(14*time.Second)) if reboots != 1 { t.Fatalf("post-boot serial change must run the pass set once, got %d", reboots) } } // TestFastTickClampsFullPasses pins the 5s clamp: a bump landing inside the // window is DEFERRED (the acted serial is not recorded), and the next tick // past the window fires it — a burst of console writes costs one pass set // per 5s, never one per write, and the last write is never lost. func TestFastTickClampsFullPasses(t *testing.T) { env := newFastFixture(t) var passes int d := env.deps(func() { passes++ }) now := time.Unix(1754640000, 0) if err := release.BumpSerial(context.Background(), env.store, now); err != nil { t.Fatalf("BumpSerial: %v", err) } fastTick(context.Background(), d, now) // seed if err := release.BumpSerial(context.Background(), env.store, now); err != nil { t.Fatalf("BumpSerial: %v", err) } fastTick(context.Background(), d, now.Add(2*time.Second)) // pass 1 if err := release.BumpSerial(context.Background(), env.store, now); err != nil { t.Fatalf("BumpSerial: %v", err) } fastTick(context.Background(), d, now.Add(4*time.Second)) // inside 5s clamp: deferred if passes != 1 { t.Fatalf("clamp: got %d passes", passes) } fastTick(context.Background(), d, now.Add(8*time.Second)) // clamp expired: fires if passes != 2 { t.Fatalf("deferred bump must fire after clamp, got %d", passes) } } // TestFastSweepMembership pins who the fast sweep is FOR: canary/* routes // always (a direct-bucket operator promote never bumps the serial), unstaged // routes until they stage, staged non-canary routes never. func TestFastSweepMembership(t *testing.T) { env := newFastFixture(t) // three routes: a canary (always member), a staged edge route (not a // member), an unstaged edge route (member until staged, then drops out). env.addRoute(t, "canary.test", "canary/x") env.addRoute(t, "staged.test", "edge") env.stageLive(t, "staged.test", "abc1234") // mint its live symlink env.addRoute(t, "cold.test", "edge") members := env.deps(nil).fastMembers(time.Unix(1754640000, 0)) sort.Strings(members) if want := []string{"canary.test", "cold.test"}; !slices.Equal(members, want) { t.Fatalf("members = %v, want %v", members, want) } } // TestFastSweepBackoffDoubles pins the per-host backoff that keeps a // permanently-unstageable route (nothing promoted, or a held head) from // costing store reads every 2s forever: 2s doubling to a 5-minute cap. func TestFastSweepBackoffDoubles(t *testing.T) { env := newFastFixture(t) env.addRoute(t, "broken.test", "edge") // nothing promoted: staging can never succeed d := env.deps(nil) now := time.Unix(1754640000, 0) d.noteUnstaged("broken.test", now) if m := d.fastMembers(now.Add(1 * time.Second)); len(m) != 0 { t.Fatalf("inside backoff window, want no members, got %v", m) } if m := d.fastMembers(now.Add(3 * time.Second)); len(m) != 1 { t.Fatalf("past first backoff, want member, got %v", m) } for i := 0; i < 20; i++ { // drive to the cap d.noteUnstaged("broken.test", now) } if d.backoffFor("broken.test") != 5*time.Minute { t.Fatalf("backoff cap = %v, want 5m", d.backoffFor("broken.test")) } } // TestFastSweepBouncesAWarmHibernatingCanary pins, through the real option // builder (fastPassOpts) and the real closure (makeRestartRoute), that the // fast sweep's pass carries BOTH the positional restart and // release.RestartHibernating — dropping the option would silently restore // the dead-branch bug the option exists for (a warm exec hibernator keeps // serving its old binary until it happens to sleep) and no other test would // fail. warm.test (Hibernate: true) is only reachable via the option; // plain.test (Hibernate: false) is only reachable via the positional // restart; both must bounce. func TestFastSweepBouncesAWarmHibernatingCanary(t *testing.T) { t.Parallel() env := newFastFixture(t) ctx := context.Background() plat := runtime.GOOS + "-" + runtime.GOARCH m := release.Manifest{ App: "fastapp", Version: "v1", Kind: "binary", Artifacts: map[string]release.Artifact{plat: {}}, } if err := release.Ship(ctx, env.store, "bab", m, map[string][]byte{plat: []byte("fast build")}); err != nil { t.Fatalf("Ship: %v", err) } if err := release.Promote(ctx, env.store, env.priv, "bab", "fastapp", "canary/dev", "v1"); err != nil { t.Fatalf("Promote: %v", err) } if err := env.reg.Upsert(registry.Route{ Host: "warm.test", Kind: "instance", App: "fastapp", Channel: "canary/dev", Status: "ready", Hibernate: true, }); err != nil { t.Fatalf("upsert warm.test: %v", err) } env.addRoute(t, "plain.test", "canary/dev") var bounced []string restart := makeRestartRoute(env.reg, func(string) {}, unitOnly(func(u string) error { t.Fatalf("named-unit arm fired for %q on a route that names no unit", u) return nil }), func(h string) bool { bounced = append(bounced, h); return true }) d := env.deps(nil) d.restart = restart d.passOpts = fastPassOpts(env.dataDir, func(host, ver string, err error) {}, restart, nil) fastTick(ctx, d, time.Unix(1754640000, 0)) sort.Strings(bounced) if want := []string{"plain.test", "warm.test"}; !slices.Equal(bounced, want) { t.Fatalf("bounced = %v, want %v (warm.test missing = RestartHibernating dropped; plain.test missing = positional restart dropped)", bounced, want) } for _, h := range []string{"warm.test", "plain.test"} { if v, err := release.LiveVersion(env.optDir, h); err != nil || v != "v1" { t.Fatalf("%s live = %q, %v; want v1", h, v, err) } } } // TestRestartHoldGrowsCapsAndResetsOnANewVersion pins the memo arithmetic // behind the flap bound: same version failing again doubles the hold (to a // cap), a different version starts fresh — a recovery promote is new work, // not a continuation of the old failure. func TestRestartHoldGrowsCapsAndResetsOnANewVersion(t *testing.T) { d := &fastDeps{} t0 := time.Now() d.noteRestartFailed("h", "v2", t0) if !d.restartHeld("h", t0.Add(fastRestartBackoffBase-time.Second)) { t.Fatal("first failure must hold for the base step") } if d.restartHeld("h", t0.Add(fastRestartBackoffBase+time.Second)) { t.Fatal("the hold must expire after its step") } d.noteRestartFailed("h", "v2", t0) if !d.restartHeld("h", t0.Add(2*fastRestartBackoffBase-time.Second)) { t.Fatal("a second failure on the SAME version must double the step") } for i := 0; i < 12; i++ { d.noteRestartFailed("h", "v2", t0) } if d.restartHeld("h", t0.Add(fastRestartBackoffCap+time.Second)) { t.Fatalf("the step must cap at %v", fastRestartBackoffCap) } d.noteRestartFailed("h", "v3", t0) if d.restartHeld("h", t0.Add(fastRestartBackoffBase+time.Second)) { t.Fatal("a DIFFERENT version must reset the hold to the base step") } } // TestFastSweepBoundsARestartFlap is the incident this bound exists for: a // canary head whose restart persistently fails, on a box whose fast loop can // actually reach the restart. Before the rollback-start fix that flap // self-terminated after two ticks — by stamping the lie (try-restart exits 0 // on the unit the failed restart left down). With the fix, each sweep pass // is a REAL cycle of the running old service: repoint, restart (kills it), // fail, roll back, start it again. Unbounded, that is a service cycle every // 2 seconds for as long as the broken head stands. The hold bounds it: the // host leaves the sweep for a growing window, a post-hold retry is one cycle // (not one per tick), and a restart that finally succeeds clears the hold at // once. func TestFastSweepBoundsARestartFlap(t *testing.T) { t.Parallel() env := newFastFixture(t) ctx := context.Background() plat := runtime.GOOS + "-" + runtime.GOARCH for _, v := range []string{"old", "new"} { m := release.Manifest{ App: "fastapp", Version: v, Kind: "binary", Artifacts: map[string]release.Artifact{plat: {}}, } if err := release.Ship(ctx, env.store, "bab", m, map[string][]byte{plat: []byte("build " + v)}); err != nil { t.Fatalf("Ship %s: %v", v, err) } } env.addRoute(t, "flap.test", "canary/dev") env.stageLive(t, "flap.test", "old") if err := release.Promote(ctx, env.store, env.priv, "bab", "fastapp", "canary/dev", "new"); err != nil { t.Fatalf("Promote: %v", err) } calls := 0 fail := true base := func(string) error { calls++ if fail { return errors.New("unit would not come back") } return nil } d := env.deps(nil) restart := d.observeRestart(base) d.restart = restart d.passOpts = fastPassOpts(env.dataDir, func(host, ver string, err error) {}, restart, nil) // Tick 1: the sweep attempts the head, the restart fails, the pass rolls // back — and the failure is memo'd. t0 := time.Now() fastTick(ctx, d, t0) if calls != 1 { t.Fatalf("tick 1: restart calls = %d, want 1", calls) } if v, _ := release.LiveVersion(env.optDir, "flap.test"); v != "old" { t.Fatalf("live after the failed tick = %q, want old (rolled back)", v) } // Ticks inside the hold: the host is outside the sweep, the service is // NOT cycled again. Without the hold every one of these is a real kill. for i := 1; i <= 5; i++ { fastTick(ctx, d, t0.Add(time.Duration(i)*fastTickEvery)) } if calls != 1 { t.Fatalf("held ticks: restart calls = %d, want still 1 — the sweep must not cycle a failing service every tick", calls) } // Past the hold: exactly one more attempt, which fails again and doubles // the step. fastTick(ctx, d, t0.Add(fastRestartBackoffBase+2*time.Second)) if calls != 2 { t.Fatalf("post-hold tick: restart calls = %d, want 2", calls) } // The head is fixed now (a recovery promote, say): the next allowed // retry succeeds, adopts, and clears the hold at once. fail = false fastTick(ctx, d, t0.Add(3*fastRestartBackoffBase+4*time.Second)) if calls != 3 { t.Fatalf("recovery tick: restart calls = %d, want 3", calls) } if v, _ := release.LiveVersion(env.optDir, "flap.test"); v != "new" { t.Fatalf("live after recovery = %q, want new", v) } if d.restartHeld("flap.test", t0.Add(24*time.Hour)) { t.Fatal("a SUCCESSFUL restart must clear the hold") } } // ---- store selection: bucket, dir, or the fleet store door (spec F5) ------- // doorHit is one request the fake door saw, reduced to the parts these tests // reason about: which call it was, and the two headers that prove WHICH box // made it (fleetapi sends the tunnel token and channel name on every request). type doorHit struct{ Method, Path, Auth, Name string } // doorFake stands in for the edge's store door. It answers the single GET // these tests drive and records every request, so a test can assert both // halves of the F5 rule: that a bucketless fleet box's store really does // speak to the door under its channel identity, and — the regression pin that // matters more — that a bucket-configured box never sends it anything. type doorFake struct { url string mu sync.Mutex hits []doorHit } func newDoorFake(t *testing.T) *doorFake { t.Helper() d := &doorFake{} srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { d.mu.Lock() d.hits = append(d.hits, doorHit{r.Method, r.URL.Path, r.Header.Get("Authorization"), r.Header.Get("Carlos-Tunnel-Name")}) d.mu.Unlock() if r.Method == http.MethodGet && r.URL.Path == "/store/o/control/serial" { w.Header().Set("ETag", `"e1"`) _, _ = w.Write([]byte("42")) return } http.NotFound(w, r) })) t.Cleanup(srv.Close) d.url = srv.URL return d } func (d *doorFake) seen() []doorHit { d.mu.Lock() defer d.mu.Unlock() return append([]doorHit(nil), d.hits...) } // fleetDoorEnv is the env a fleet box that has been CUT OVER carries: fleet // identity plus tunnel address/credential, and deliberately no bucket and no // AWS credentials at all — the whole point of the door. func fleetDoorEnv(doorURL string) map[string]string { return map[string]string{ "CARLOS_FLEET": "pi", "CARLOS_FLEET_BOX": "pi-box", "CARLOS_TUNNEL_URL": doorURL, "CARLOS_TUNNEL_TOKEN": "tok-123", } } // awsEnvForS3Store makes release.NewS3Store constructible in a test without // network or a developer's real profile — the same shape internal/release's // own S3 tests use. Nothing is ever called on the resulting client here; these // tests only assert which store TYPE the selection built. func awsEnvForS3Store(t *testing.T) { t.Helper() t.Setenv("AWS_ACCESS_KEY_ID", "k") t.Setenv("AWS_SECRET_ACCESS_KEY", "s") t.Setenv("AWS_SESSION_TOKEN", "") t.Setenv("AWS_REGION", "eu-west-1") t.Setenv("AWS_EC2_METADATA_DISABLED", "true") } // TestOpenDeployStoreSelection pins spec F5's whole rule for the store every // converge loop on the box reads through. The migration fallback is the // load-bearing half: a bucket or dir still set must win OUTRIGHT, before fleet // identity is even consulted, or rolling this branch out would silently move // every already-working fleet box onto a door it may not be able to reach. func TestOpenDeployStoreSelection(t *testing.T) { ctx := context.Background() dir := t.TempDir() t.Run("fleet box with no bucket speaks the door", func(t *testing.T) { door := newDoorFake(t) s, err := openDeployStoreEnv(ctx, envFunc(fleetDoorEnv(door.url))) if err != nil { t.Fatalf("openDeployStoreEnv: %v", err) } body, etag, err := s.Get(ctx, "control/serial") if err != nil { t.Fatalf("Get through the door: %v", err) } if string(body) != "42" || etag != `"e1"` { t.Fatalf("Get = %q, %q; want \"42\", \"e1\"", body, etag) } hits := door.seen() if len(hits) != 1 { t.Fatalf("door saw %d requests, want 1: %+v", len(hits), hits) } if hits[0].Path != "/store/o/control/serial" { t.Fatalf("door path = %q, want /store/o/control/serial", hits[0].Path) } // The identity on the wire is the channel identity the edge dials — // "/