package host import ( "context" "errors" "runtime" "slices" "sync/atomic" "testing" "time" "github.com/carlosframework/platform/internal/appconfig" "github.com/carlosframework/platform/internal/registry" "github.com/carlosframework/platform/internal/release" "github.com/carlosframework/platform/internal/telemetry" ) // The box's GET loops read only on change or at the backstop age (S3 // change-index design §5, codex 9): config-gen.json, canary channel // pointers and telemetry.json. // gateClock is a fake clock a test moves by hand. type gateClock struct{ t time.Time } func (c *gateClock) now() time.Time { return c.t } func (c *gateClock) add(d time.Duration) { c.t = c.t.Add(d) } func newGateClock() *gateClock { return &gateClock{t: time.Date(2026, 10, 21, 9, 0, 0, 0, time.UTC)} } func pollWatch(t *testing.T, w *release.ChangeWatch) { t.Helper() if _, err := w.Poll(context.Background()); err != nil { t.Fatalf("poll the change index: %v", err) } } // noIndexStore hides every optional extension of the store it wraps, so a // watch over it is unusable: the fleet store door's shape (spec §3.4). type noIndexStore struct{ release.Store } // flakyGetStore fails Get of one key while fail is set. type flakyGetStore struct { *ciCountingStore key string fail atomic.Bool } func (s *flakyGetStore) Get(ctx context.Context, key string) ([]byte, string, error) { if key == s.key && s.fail.Load() { s.note("Get", key) return nil, "", errors.New("503 slow down") } return s.ciCountingStore.Get(ctx, key) } // genGateKit is an indexed counting store with config generations for the // given apps under account bab, a polled watch over it, and a gated cache on // a fake clock. func genGateKit(t *testing.T, apps ...string) (*ciCountingStore, *release.ChangeWatch, *genCache, *gateClock, []registry.Route) { t.Helper() cs := newCICountingStore(t) var routes []registry.Route for _, app := range apps { touch(t, context.Background(), cs, "bab", app) routes = append(routes, execRoute(app+".example.com", "bab", app)) } w := release.NewChangeWatch(cs) pollWatch(t, w) clock := newGateClock() cache := newGenCache() cache.gate = release.NewReadGate(w, release.ChangeIndexBackstop) cache.now = clock.now cs.reset() return cs, w, cache, clock, routes } func genGets(cs *ciCountingStore, app string) int { return cs.count("Get", appconfig.GenKey("bab", app)) } func TestConfigGenSkipsUnmovedApp(t *testing.T) { t.Parallel() cs, w, cache, clock, routes := genGateKit(t, "shop") ctx := context.Background() genPollPass(ctx, routes, cs, cache) clock.add(3 * time.Second) pollWatch(t, w) if exec, unit, errs := genPollPass(ctx, routes, cs, cache); exec || unit || len(errs) != 0 { t.Fatalf("an unmoved app reported exec %v unit %v errs %v", exec, unit, errs) } if n := genGets(cs, "shop"); n != 1 { t.Fatalf("two ticks with no bump read config-gen.json %d time(s), want 1 (the first): %v", n, cs.got()) } } func TestConfigGenReadsMovedApp(t *testing.T) { t.Parallel() cs, w, cache, clock, routes := genGateKit(t, "shop", "cafe") ctx := context.Background() genPollPass(ctx, routes, cs, cache) touch(t, ctx, cs, "bab", "shop") cs.reset() clock.add(3 * time.Second) pollWatch(t, w) exec, _, errs := genPollPass(ctx, routes, cs, cache) if !exec || len(errs) != 0 { t.Fatalf("a moved app's config: exec %v errs %v, want a sync", exec, errs) } if genGets(cs, "shop") != 1 || genGets(cs, "cafe") != 0 { t.Fatalf("after shop's bump the tick read %v, want shop's config-gen.json only", cs.got()) } } func TestConfigGenReadsAtMaxAge(t *testing.T) { t.Parallel() cs, w, cache, clock, routes := genGateKit(t, "shop") ctx := context.Background() genPollPass(ctx, routes, cs, cache) clock.add(release.ChangeIndexBackstop - time.Second) pollWatch(t, w) genPollPass(ctx, routes, cs, cache) if n := genGets(cs, "shop"); n != 1 { t.Fatalf("a read younger than the backstop was repeated: %d reads", n) } clock.add(time.Second) genPollPass(ctx, routes, cs, cache) if n := genGets(cs, "shop"); n != 2 { t.Fatalf("a read the backstop's age old was not repeated: %d reads, want 2", n) } } func TestConfigGenRetriesAfterError(t *testing.T) { t.Parallel() cs, w, cache, clock, routes := genGateKit(t, "shop") ctx := context.Background() fs := &flakyGetStore{ciCountingStore: cs, key: appconfig.GenKey("bab", "shop")} genPollPass(ctx, routes, fs, cache) touch(t, ctx, cs, "bab", "shop") pollWatch(t, w) cs.reset() fs.fail.Store(true) clock.add(3 * time.Second) if exec, _, errs := genPollPass(ctx, routes, fs, cache); exec || len(errs) != 1 { t.Fatalf("the failed read: exec %v errs %v, want one error and no sync", exec, errs) } // No new bump: the failed read alone makes the next tick read again, // and that read sees the change. fs.fail.Store(false) clock.add(3 * time.Second) pollWatch(t, w) if exec, _, errs := genPollPass(ctx, routes, fs, cache); !exec || len(errs) != 0 { t.Fatalf("the tick after a failed read: exec %v errs %v, want the change synced", exec, errs) } if n := genGets(cs, "shop"); n != 2 { t.Fatalf("config-gen.json read %d time(s) after the bump, want 2 (the failure, the retry)", n) } } // ---- canary pointers ------------------------------------------------------ // canaryKit is a box with one canary route (fastapp on canary/dev, account // bab) whose head is v1, over an indexed counting store. type canaryKit struct { env *fastFixture ds *release.DirStore cs *ciCountingStore d *fastDeps clock *gateClock } const canaryHost = "canary.example.com" func newCanaryKit(t *testing.T, usable bool) *canaryKit { t.Helper() env := newFastFixture(t) cs := newCICountingStore(t) k := &canaryKit{env: env, ds: cs.DirStore, cs: cs, clock: newGateClock()} k.ship(t, "v1") env.addRoute(t, canaryHost, "canary/dev") var store release.Store = cs if !usable { store = noIndexStore{cs} } env.store = store k.d = env.deps(nil) k.d.watch = release.NewChangeWatch(store) k.d.canaryGate = release.NewReadGate(k.d.watch, release.ChangeIndexBackstop) cs.reset() return k } func (k *canaryKit) ship(t *testing.T, ver string) { t.Helper() ctx := context.Background() plat := runtime.GOOS + "-" + runtime.GOARCH m := release.Manifest{App: "fastapp", Version: ver, Kind: "binary", Artifacts: map[string]release.Artifact{plat: {}}} if err := release.Ship(ctx, k.ds, "bab", m, map[string][]byte{plat: []byte("build " + ver)}); err != nil { t.Fatalf("Ship %s: %v", ver, err) } if err := release.Promote(ctx, k.ds, k.env.priv, "bab", "fastapp", "canary/dev", ver); err != nil { t.Fatalf("Promote %s: %v", ver, err) } } // tick runs one fastTick at the kit's clock, checks the canary is still a // member, and returns how many times the pointer has been read so far. func (k *canaryKit) tick(t *testing.T, d time.Duration) int { t.Helper() k.clock.add(d) if m := k.d.fastMembers(k.clock.now()); !slices.Contains(m, canaryHost) { t.Fatalf("the canary route left the sweep: members %v", m) } fastTick(context.Background(), k.d, k.clock.now()) return k.cs.count("Get", release.ChannelKey("bab", "fastapp", "canary/dev")) } func (k *canaryKit) live(t *testing.T) string { t.Helper() v, err := release.LiveVersion(k.env.optDir, canaryHost) if err != nil { t.Fatal(err) } return v } func TestCanaryPointerGated(t *testing.T) { t.Parallel() k := newCanaryKit(t, true) if n := k.tick(t, 0); n != 1 || k.live(t) != "v1" { t.Fatalf("first tick: %d pointer reads, live %q; want 1 and v1", n, k.live(t)) } for i := range 3 { if n := k.tick(t, 2*time.Second); n != 1 { t.Fatalf("quiet tick %d read the pointer again (%d reads)", i, n) } } // A promote bumps the app's scope: the next tick reads and adopts. k.ship(t, "v2") if n := k.tick(t, 2*time.Second); n != 2 || k.live(t) != "v2" { t.Fatalf("after a promote: %d pointer reads, live %q; want 2 and v2", n, k.live(t)) } if n := k.tick(t, 2*time.Second); n != 2 { t.Fatalf("the tick after the adopt read the pointer again (%d reads)", n) } // The age bound: a pointer read the backstop's age ago is read again. if n := k.tick(t, release.ChangeIndexBackstop); n != 3 { t.Fatalf("at the backstop age: %d pointer reads, want 3", n) } } // A canary group whose routes are not live on the head its last read // returned (here the live link was moved off it, as a rolled-back restart // leaves it) keeps today's read on every tick until it is. func TestCanaryPointerReadWhileUnsettled(t *testing.T) { t.Parallel() k := newCanaryKit(t, true) k.tick(t, 0) k.env.stageLive(t, canaryHost, "v0") if n := k.tick(t, 2*time.Second); n != 2 || k.live(t) != "v1" { t.Fatalf("an unsettled group: %d pointer reads, live %q; want 2 and v1", n, k.live(t)) } if n := k.tick(t, 2*time.Second); n != 2 { t.Fatalf("the group settled again but its pointer was read (%d reads)", n) } } // A tick whose own index poll failed reads the canary pointer (amadan #336 // final review, finding 3). Here another caller's index GET hangs holding // the watch's poll slot, and the tick gives up waiting for it: before the // fix that give-up left the frozen observation vouching, and the gate // skipped the read. The next good poll restores the gate. func TestCanaryPointerReadWhenTheTicksIndexPollFails(t *testing.T) { t.Parallel() k := newCanaryKit(t, true) if n := k.tick(t, 0); n != 1 { t.Fatalf("first tick: %d pointer reads, want 1", n) } if n := k.tick(t, 2*time.Second); n != 1 { t.Fatalf("a quiet tick read the pointer (%d reads)", n) } unstall := k.cs.stallIndex() polls := k.cs.count("GetIfChanged", release.ChangeIndexKey) hung := make(chan error, 1) go func() { _, err := k.d.watch.Poll(context.Background()); hung <- err }() for k.cs.count("GetIfChanged", release.ChangeIndexKey) == polls { time.Sleep(time.Millisecond) } if n := k.tick(t, 2*time.Second); n != 2 { unstall() t.Fatalf("a tick whose index poll gave up skipped the pointer read (%d reads, want 2): it trusted a frozen observation", n) } unstall() if err := <-hung; err != nil { t.Fatalf("the hung poll, released: %v", err) } if n := k.tick(t, 2*time.Second); n != 2 { t.Fatalf("after the watch recovered the quiet tick read the pointer (%d reads)", n) } } // Final review R1 finding 6 (amadan #336): two canary channels of one app. // canary/dev's promote fails its restart and rolls back, so its route sits // out the sweep in its restart hold; canary/beta reads its own pointer // meanwhile, which marks the app's generation. When dev's hold ends its // cached head still matches its rolled-back live version, but its own last // read predates the app's generation, so it reads again and retries the // promote. Before, beta's read settled dev until the backstop. func TestCanaryGroupFreshnessIsPerChannel(t *testing.T) { t.Parallel() k := newCanaryKit(t, true) ctx := context.Background() const betaHost = "beta.example.com" if err := release.Promote(ctx, k.ds, k.env.priv, "bab", "fastapp", "canary/beta", "v1"); err != nil { t.Fatal(err) } k.env.addRoute(t, betaHost, "canary/beta") devKey := release.ChannelKey("bab", "fastapp", "canary/dev") betaKey := release.ChannelKey("bab", "fastapp", "canary/beta") tick := func(d time.Duration) (dev, beta int) { k.clock.add(d) fastTick(ctx, k.d, k.clock.now()) return k.cs.count("Get", devKey), k.cs.count("Get", betaKey) } if dev, beta := tick(0); dev != 1 || beta != 1 { t.Fatalf("first tick: dev %d beta %d pointer reads, want 1 each", dev, beta) } if dev, beta := tick(2 * time.Second); dev != 1 || beta != 1 { t.Fatalf("a quiet tick read a pointer: dev %d beta %d", dev, beta) } k.ship(t, "v2") // canary/dev -> v2; the restart fails and rolls back: k.d.restartHold = map[string]*restartFailure{canaryHost: {ver: "v2", step: time.Minute, notBefore: k.clock.now().Add(time.Minute)}} if dev, beta := tick(2 * time.Second); dev != 1 || beta != 2 { t.Fatalf("with dev held: dev %d beta %d pointer reads, want dev untouched and beta read", dev, beta) } if k.live(t) != "v1" { t.Fatalf("fixture: the held route moved to %q", k.live(t)) } delete(k.d.restartHold, canaryHost) // the hold ends dev, beta := tick(2 * time.Second) if dev != 2 { t.Fatalf("dev's pointer read %d time(s) after its hold ended, want 2: beta's read settled dev's group", dev) } if beta != 2 { t.Fatalf("beta's pointer read again (%d) with nothing moved", beta) } if k.live(t) != "v2" { t.Fatalf("dev did not retry the promote: live %q", k.live(t)) } } // A stable route keeps today's path: the gate never skips its pointer. func TestCanaryGateLeavesStableRoutes(t *testing.T) { t.Parallel() k := newCanaryKit(t, true) skip, read := k.d.canaryPointerGate(k.clock.now())("bab", "fastapp", "stable", nil) if skip || read != nil { t.Fatalf("the canary gate took a stable group: skip %v read %v", skip, read != nil) } } // ---- telemetry ------------------------------------------------------------ func telemetryRoutes(t *testing.T, acct string, apps ...string) *registry.Registry { t.Helper() reg := edgesTestRegistry(t) for _, app := range apps { if err := reg.Upsert(registry.Route{Host: app + ".example.com", Kind: "instance", Account: acct, App: app, Status: "ready"}); err != nil { t.Fatal(err) } } return reg } // Armed, an account's telemetry.json is read once per pass, however many of // its apps the box serves. func TestTelemetryAccountReadOncePerPass(t *testing.T) { t.Parallel() ctx := context.Background() cs := newCICountingStore(t) reg := telemetryRoutes(t, "bab", "shop", "cafe", "deli") if err := telemetry.Update(ctx, cs, telemetry.AccountKey("bab"), func(st *telemetry.Settings) { st.Analytics = &telemetry.Analytics{Count: telemetry.CountEdge} }); err != nil { t.Fatal(err) } w := release.NewChangeWatch(cs) pollWatch(t, w) cs.reset() on, err := telemetrySwitchesWith(ctx, cs, reg, newTelemetryReads(w), newGateClock().now()) if err != nil { t.Fatal(err) } if n := cs.count("Get", telemetry.AccountKey("bab")); n != 1 { t.Fatalf("three apps in one account read its telemetry.json %d time(s), want 1: %v", n, cs.got()) } if !on["bab/shop"] || !on["bab/cafe"] || !on["bab/deli"] { t.Fatalf("switches %v, want every app on from the account", on) } } // onceFailingGetStore fails the first Get of one key, then answers. type onceFailingGetStore struct { *ciCountingStore key string failed atomic.Bool } func (s *onceFailingGetStore) Get(ctx context.Context, key string) ([]byte, string, error) { if key == s.key && s.failed.CompareAndSwap(false, true) { s.note("Get", key) return nil, "", errors.New("503 slow down") } return s.ciCountingStore.Get(ctx, key) } // Disarmed, the pass is main's (final review R1, finding 5): each app // resolves both levels on its own, so one transient account read failure // leaves only that app off, never the account's other apps. func TestTelemetryDisarmedResolvesEachAppOnItsOwn(t *testing.T) { t.Parallel() ctx := context.Background() cs := newCICountingStore(t) reg := telemetryRoutes(t, "bab", "shop", "cafe", "deli") if err := telemetry.Update(ctx, cs, telemetry.AccountKey("bab"), func(st *telemetry.Settings) { st.Analytics = &telemetry.Analytics{Count: telemetry.CountEdge} }); err != nil { t.Fatal(err) } fs := &onceFailingGetStore{ciCountingStore: cs, key: telemetry.AccountKey("bab")} cs.reset() on, err := telemetrySwitchesWith(ctx, fs, reg, newTelemetryReads(nil), newGateClock().now()) if err != nil { t.Fatal(err) } if n := cs.count("Get", telemetry.AccountKey("bab")); n != 3 { t.Fatalf("three apps read the account's telemetry.json %d time(s) disarmed, want 3 (one each, as before the index)", n) } if len(on) != 2 { t.Fatalf("switches %v: one failed account read must leave only its own app off", on) } } func TestTelemetryGatedWithAgeBound(t *testing.T) { t.Parallel() ctx := context.Background() cs := newCICountingStore(t) reg := telemetryRoutes(t, "bab", "shop", "cafe") if err := telemetry.Update(ctx, cs, telemetry.AccountKey("bab"), func(st *telemetry.Settings) { st.Analytics = &telemetry.Analytics{Count: telemetry.CountEdge} }); err != nil { t.Fatal(err) } w := release.NewChangeWatch(cs) pollWatch(t, w) r := newTelemetryReads(w) clock := newGateClock() pass := func() map[string]bool { t.Helper() on, err := telemetrySwitchesWith(ctx, cs, reg, r, clock.now()) if err != nil { t.Fatal(err) } return on } cs.reset() if on := pass(); !on["bab/shop"] || !on["bab/cafe"] { t.Fatalf("first pass switches %v", on) } if n := cs.count("Get", ""); n != 3 { t.Fatalf("the first pass made %d reads, want 3 (account, shop, cafe): %v", n, cs.got()) } clock.add(time.Minute) pollWatch(t, w) cs.reset() if on := pass(); !on["bab/shop"] || !on["bab/cafe"] { t.Fatalf("a pass that skipped its reads lost the last good values: %v", on) } if n := cs.count("Get", ""); n != 0 { t.Fatalf("a quiet pass read %v", cs.got()) } // cafe's own setting moves: only cafe's file is read. if err := telemetry.Update(ctx, cs, telemetry.AppKey("bab", "cafe"), func(st *telemetry.Settings) { st.Analytics = &telemetry.Analytics{Count: telemetry.CountOff} }); err != nil { t.Fatal(err) } clock.add(time.Minute) pollWatch(t, w) cs.reset() if on := pass(); !on["bab/shop"] || on["bab/cafe"] { t.Fatalf("after cafe's change switches %v, want shop on, cafe off", on) } if got := cs.got(); !slices.Equal(got, []string{"Get " + telemetry.AppKey("bab", "cafe")}) { t.Fatalf("after cafe's change the pass read %v, want cafe's telemetry.json only", got) } // The age bound: every file is read again at the backstop's age. cs.reset() clock.add(release.ChangeIndexBackstop) pass() if n := cs.count("Get", ""); n != 3 { t.Fatalf("at the backstop age the pass made %d reads, want 3: %v", n, cs.got()) } } // A failed read keeps the last good value and is retried on the next pass. func TestTelemetryKeepsLastGoodOnError(t *testing.T) { t.Parallel() ctx := context.Background() cs := newCICountingStore(t) reg := telemetryRoutes(t, "bab", "shop") if err := telemetry.Update(ctx, cs, telemetry.AccountKey("bab"), func(st *telemetry.Settings) { st.Analytics = &telemetry.Analytics{Count: telemetry.CountEdge} }); err != nil { t.Fatal(err) } fs := &flakyGetStore{ciCountingStore: cs, key: telemetry.AccountKey("bab")} w := release.NewChangeWatch(cs) pollWatch(t, w) r := newTelemetryReads(w) clock := newGateClock() if on, _ := telemetrySwitchesWith(ctx, fs, reg, r, clock.now()); !on["bab/shop"] { t.Fatalf("first pass %v", on) } clock.add(release.ChangeIndexBackstop) // due by age fs.fail.Store(true) cs.reset() if on, _ := telemetrySwitchesWith(ctx, fs, reg, r, clock.now()); !on["bab/shop"] { t.Fatalf("a failed account read dropped the last good value: %v", on) } if n := cs.count("Get", telemetry.AccountKey("bab")); n != 1 { t.Fatalf("the due pass read the account %d time(s), want 1", n) } fs.fail.Store(false) cs.reset() clock.add(time.Second) telemetrySwitchesWith(ctx, fs, reg, r, clock.now()) if n := cs.count("Get", telemetry.AccountKey("bab")); n != 1 { t.Fatalf("the pass after a failed read read the account %d time(s), want 1 (a retry)", n) } } // ---- an unusable watch ---------------------------------------------------- // Review Focus 5: a watch that cannot vouch (here, a store with no // conditional read, the fleet door's shape) makes every gated read due on // every tick. The gates are the production ones, built from the armed watch. func TestGatesDueWhenWatchUnusable(t *testing.T) { cs := newCICountingStore(t) plain := noIndexStore{cs} w := release.NewChangeWatch(plain) if _, err := w.Poll(context.Background()); err == nil { t.Fatal("fixture: a watch over a store with no conditional read polled cleanly") } armedChangeWatch.Store(w) t.Cleanup(func() { armedChangeWatch.Store(nil) }) ctx := context.Background() clock := newGateClock() // config-gen.json touch(t, ctx, cs, "bab", "shop") cache := newGenCache() cache.gate = newChangeGate() cache.now = clock.now routes := []registry.Route{execRoute("shop.example.com", "bab", "shop")} cs.reset() for range 3 { clock.add(3 * time.Second) genPollPass(ctx, routes, plain, cache) } if n := genGets(cs, "shop"); n != 3 { t.Fatalf("config-gen.json read %d time(s) in three ticks, want 3", n) } // telemetry.json, through the production state reg := telemetryRoutes(t, "bab", "shop") cs.reset() for range 3 { clock.add(time.Minute) if _, err := telemetrySwitchesWith(ctx, plain, reg, currentTelemetryReads(), clock.now()); err != nil { t.Fatal(err) } } if a, b := cs.count("Get", telemetry.AccountKey("bab")), cs.count("Get", telemetry.AppKey("bab", "shop")); a != 3 || b != 3 { t.Fatalf("telemetry reads in three passes: account %d app %d, want 3 each", a, b) } // canary pointers k := newCanaryKit(t, false) k.d.canaryGate = newChangeGate() k.d.watch = nil // fastTick's own poll would fail the same way; the gate reads the armed watch for i := range 3 { if n := k.tick(t, 2*time.Second); n != i+1 { t.Fatalf("tick %d: %d pointer reads, want %d", i, n, i+1) } } }