package appconfig import ( "bytes" "context" "errors" "io" "os" "path/filepath" "reflect" "slices" "strings" "sync" "testing" "time" "golang.org/x/sys/unix" "github.com/carlosframework/platform/internal/provision" "github.com/carlosframework/platform/internal/registry" "github.com/carlosframework/platform/internal/relay" "github.com/carlosframework/platform/internal/release" ) // execKit is the shared setup: store, registry, dataDir, keypair. func execKit(t *testing.T) (ctx context.Context, s release.Store, reg *registry.Registry, dataDir, priv, pub string) { t.Helper() s = release.NewDirStore(t.TempDir()) ctx = context.Background() reg = testRegistry(t) dataDir = t.TempDir() var err error priv, pub, err = GenerateKey() if err != nil { t.Fatalf("GenerateKey: %v", err) } return } // captureStdout runs fn with os.Stdout redirected to a pipe and returns // everything fn printed. SyncExec logs via fmt.Printf directly to // os.Stdout, matching every other pass in this package — this is the only // way to pin a log line's presence without restructuring every call site // to take a Writer, so it's used sparingly: only where a test's whole // point IS that a specific line got printed (a `fmt.Printf` deleted by a // future edit would otherwise go unnoticed by every other test, which // only assert on the filesystem). func captureStdout(t *testing.T, fn func()) string { t.Helper() r, w, err := os.Pipe() if err != nil { t.Fatalf("os.Pipe: %v", err) } old := os.Stdout os.Stdout = w defer func() { os.Stdout = old }() fn() if err := w.Close(); err != nil { t.Fatalf("close pipe writer: %v", err) } var buf bytes.Buffer if _, err := io.Copy(&buf, r); err != nil { t.Fatalf("read captured stdout: %v", err) } return buf.String() } // failingKeyStore wraps a release.Store and makes Get fail with a fixed // error for exactly one key — simulating a transient bucket blip against a // single app's default bundle while every other key (other apps' bundles, // including in the SAME pass) resolves normally, the shape a real S3 // SlowDown/503 against one object takes. type failingKeyStore struct { release.Store failKey string err error } func (f *failingKeyStore) Get(ctx context.Context, key string) ([]byte, string, error) { if key == f.failKey { return nil, "", f.err } return f.Store.Get(ctx, key) } func TestSyncExecWritesConfigFileAndBouncesOnChange(t *testing.T) { ctx, s, reg, dd, priv, _ := execKit(t) pk, err := ParsePrivateKey(priv, "test") if err != nil { t.Fatalf("ParsePrivateKey: %v", err) } if err := SetEnv(ctx, s, "bab", "eleven", map[string]string{"LOG_LEVEL": "debug"}); err != nil { t.Fatalf("SetEnv: %v", err) } if err := reg.Upsert(registry.Route{ Host: "inst.example.com", Kind: "instance", App: "eleven", Account: "bab", Status: "ready", Socket: "/run/carlos/inst.sock", }); err != nil { t.Fatalf("Upsert: %v", err) } var bounced []string bounce := func(h string) error { bounced = append(bounced, h); return nil } if err := SyncExec(ctx, reg, s, pk, dd, bounce, "", nil); err != nil { t.Fatalf("SyncExec: %v", err) } got, err := os.ReadFile(filepath.Join(dd, "inst.example.com.config.env")) if err != nil { t.Fatalf("read config env: %v", err) } if string(got) != "LOG_LEVEL=debug\n" { t.Fatalf("config env = %q, want LOG_LEVEL=debug", got) } if fi, _ := os.Stat(filepath.Join(dd, "inst.example.com.config.env")); fi.Mode().Perm() != 0o600 { t.Fatalf("config env mode = %v, want 0600", fi.Mode().Perm()) } if len(bounced) != 1 || bounced[0] != "inst.example.com" { t.Fatalf("bounced = %v, want exactly [inst.example.com]", bounced) } // Second pass, nothing changed: no bounce. bounced = nil if err := SyncExec(ctx, reg, s, pk, dd, bounce, "", nil); err != nil { t.Fatalf("SyncExec (second): %v", err) } if len(bounced) != 0 { t.Fatalf("unchanged pass bounced %v, want none", bounced) } } // TestSyncExecWritesANestedRouteIntoItsInstanceDir pins the isolated-exec // half of config delivery (design §7): a route whose DB and socket are the // nested layout's defaults gets its rendered bundle inside its OWN instance // directory, never at the flat /.config.env path a foreign or // flat row would use. func TestSyncExecWritesANestedRouteIntoItsInstanceDir(t *testing.T) { ctx, s, reg, dd, priv, _ := execKit(t) pk, err := ParsePrivateKey(priv, "test") if err != nil { t.Fatalf("ParsePrivateKey: %v", err) } if err := SetEnv(ctx, s, "bab", "eleven", map[string]string{"LOG_LEVEL": "debug"}); err != nil { t.Fatalf("SetEnv: %v", err) } host := "inst.example.com" // The nested layout's directory pre-exists here exactly as it would on // a real box: the migration pass that flips a row's DBPath/Socket to // the nested defaults is what creates it, before any route can // classify as nested at all. if err := os.MkdirAll(provision.InstanceDir(dd, host), 0o755); err != nil { t.Fatalf("mkdir instance dir: %v", err) } if err := reg.Upsert(registry.Route{ Host: host, Kind: "instance", App: "eleven", Account: "bab", Status: "ready", DBPath: provision.NestedDBPath(dd, host), Socket: provision.NestedSocket(host), }); err != nil { t.Fatalf("Upsert: %v", err) } var bounced []string bounce := func(h string) error { bounced = append(bounced, h); return nil } if err := SyncExec(ctx, reg, s, pk, dd, bounce, "", nil); err != nil { t.Fatalf("SyncExec: %v", err) } nestedPath := filepath.Join(provision.InstanceDir(dd, host), host+".config.env") got, err := os.ReadFile(nestedPath) if err != nil { t.Fatalf("read nested config env: %v", err) } if string(got) != "LOG_LEVEL=debug\n" { t.Fatalf("config env = %q, want LOG_LEVEL=debug", got) } if fi, _ := os.Stat(nestedPath); fi.Mode().Perm() != 0o600 { t.Fatalf("nested config env mode = %v, want 0600", fi.Mode().Perm()) } if _, err := os.Stat(filepath.Join(dd, host+".config.env")); !os.IsNotExist(err) { t.Fatalf("a nested route's config also landed at the flat path, stat err = %v", err) } if len(bounced) != 1 || bounced[0] != host { t.Fatalf("bounced = %v, want exactly [%s]", bounced, host) } } // TestSyncExecSkipsAHostTheLayoutPassHolds pins the second net under the // layout pass: a host with a migration journal is between two directories, // so SyncExec writes no bundle for it at either path, bounces nothing, and // says why, while a sibling on the same pass converges as usual. func TestSyncExecSkipsAHostTheLayoutPassHolds(t *testing.T) { ctx, s, reg, dd, priv, _ := execKit(t) pk, err := ParsePrivateKey(priv, "test") if err != nil { t.Fatalf("ParsePrivateKey: %v", err) } if err := SetEnv(ctx, s, "bab", "eleven", map[string]string{"LOG_LEVEL": "debug"}); err != nil { t.Fatalf("SetEnv: %v", err) } held, sibling := "inst.example.com", "other.example.com" if err := os.MkdirAll(provision.InstanceDir(dd, held), 0o755); err != nil { t.Fatalf("mkdir instance dir: %v", err) } for _, rt := range []registry.Route{ {Host: held, Kind: "instance", App: "eleven", Account: "bab", Status: "ready", DBPath: provision.NestedDBPath(dd, held), Socket: provision.NestedSocket(held)}, {Host: sibling, Kind: "instance", App: "eleven", Account: "bab", Status: "ready", Socket: "/run/carlos/" + sibling + ".sock"}, } { if err := reg.Upsert(rt); err != nil { t.Fatalf("Upsert: %v", err) } } var bounced []string bounce := func(h string) error { bounced = append(bounced, h); return nil } var asked []string fenced := func(h string) bool { asked = append(asked, h); return h == held } out := captureStdout(t, func() { if err := SyncExec(ctx, reg, s, pk, dd, bounce, "", nil, SkipFenced(fenced)); err != nil { t.Fatalf("SyncExec: %v", err) } }) for _, path := range []string{ filepath.Join(provision.InstanceDir(dd, held), held+".config.env"), filepath.Join(dd, held+".config.env"), } { if _, err := os.Lstat(path); !os.IsNotExist(err) { t.Errorf("a held host's bundle was written at %s (err %v)", path, err) } } if got, err := os.ReadFile(filepath.Join(dd, sibling+".config.env")); err != nil || string(got) != "LOG_LEVEL=debug\n" { t.Errorf("sibling bundle = %q, %v", got, err) } if len(bounced) != 1 || bounced[0] != sibling { t.Errorf("bounced = %v, want exactly [%s]", bounced, sibling) } if !strings.Contains(out, held+": layout migration in flight; bundle not written this pass") { t.Errorf("the skip was not reported:\n%s", out) } if !slices.Contains(asked, held) || !slices.Contains(asked, sibling) { t.Errorf("fenced was asked about %v", asked) } } // TestSyncExecFIFOAtANestedBundleDoesNotHangThePass pins the review fix: a // nested row's existing bundle is read through instancefs, so a FIFO a // tenant planted at that name (isolated-exec design §3.3, no tenant-planted // inode may block the agent) must fail fast rather than hang SyncExec's // whole pass forever behind one host's blocking read(2). A sibling host on // the same pass must still converge normally. func TestSyncExecFIFOAtANestedBundleDoesNotHangThePass(t *testing.T) { ctx, s, reg, dd, priv, _ := execKit(t) pk, err := ParsePrivateKey(priv, "test") if err != nil { t.Fatalf("ParsePrivateKey: %v", err) } if err := SetEnv(ctx, s, "bab", "eleven", map[string]string{"LOG_LEVEL": "debug"}); err != nil { t.Fatalf("SetEnv: %v", err) } if err := SetEnv(ctx, s, "bab", "twelve", map[string]string{"LOG_LEVEL": "debug"}); err != nil { t.Fatalf("SetEnv: %v", err) } okHost, fifoHost := "ok.example.com", "fifo.example.com" for _, h := range []string{okHost, fifoHost} { if err := os.MkdirAll(provision.InstanceDir(dd, h), 0o755); err != nil { t.Fatalf("mkdir instance dir for %s: %v", h, err) } } if err := reg.Upsert(registry.Route{ Host: okHost, Kind: "instance", App: "eleven", Account: "bab", Status: "ready", DBPath: provision.NestedDBPath(dd, okHost), Socket: provision.NestedSocket(okHost), }); err != nil { t.Fatalf("Upsert %s: %v", okHost, err) } if err := reg.Upsert(registry.Route{ Host: fifoHost, Kind: "instance", App: "twelve", Account: "bab", Status: "ready", DBPath: provision.NestedDBPath(dd, fifoHost), Socket: provision.NestedSocket(fifoHost), }); err != nil { t.Fatalf("Upsert %s: %v", fifoHost, err) } // Planted where syncExecHost's existed-check reads BEFORE it ever writes // anything: a tenant that got here first, not a crash artifact. fifoPath := filepath.Join(provision.InstanceDir(dd, fifoHost), fifoHost+".config.env") if err := unix.Mkfifo(fifoPath, 0o600); err != nil { t.Fatalf("Mkfifo: %v", err) } var bounced []string bounce := func(h string) error { bounced = append(bounced, h); return nil } done := make(chan error, 1) go func() { done <- SyncExec(ctx, reg, s, pk, dd, bounce, "", nil) }() select { case err := <-done: if err != nil { t.Fatalf("SyncExec: %v", err) } case <-time.After(5 * time.Second): t.Fatal("SyncExec hung behind the FIFO'd host instead of failing that one host fast") } okPath := filepath.Join(provision.InstanceDir(dd, okHost), okHost+".config.env") got, err := os.ReadFile(okPath) if err != nil { t.Fatalf("sibling host's config was not written: %v", err) } if string(got) != "LOG_LEVEL=debug\n" { t.Fatalf("sibling config env = %q, want LOG_LEVEL=debug", got) } if len(bounced) != 1 || bounced[0] != okHost { t.Fatalf("bounced = %v, want exactly [%s] (the FIFO'd host must not bounce)", bounced, okHost) } // The FIFO itself is untouched: syncExecHost's read failed before it // ever reached a write. if fi, err := os.Lstat(fifoPath); err != nil || fi.Mode()&os.ModeNamedPipe == 0 { t.Fatalf("FIFO at %s was replaced or removed: %v %v", fifoPath, fi, err) } } // TestReapConfigEnvWalksInstanceDirs pins reapConfigEnv's nested half: a // stale config file inside a gone host's instance directory is removed, and // a live nested route's own file survives. func TestReapConfigEnvWalksInstanceDirs(t *testing.T) { dd := t.TempDir() reg := testRegistry(t) live, gone := "live.example.com", "gone.example.com" for _, host := range []string{live, gone} { if err := os.MkdirAll(provision.InstanceDir(dd, host), 0o755); err != nil { t.Fatalf("mkdir instance dir for %s: %v", host, err) } p := filepath.Join(provision.InstanceDir(dd, host), host+configEnvSuffix) if err := os.WriteFile(p, []byte("A=1\n"), 0o600); err != nil { t.Fatalf("stage %s: %v", host, err) } } if err := reg.Upsert(registry.Route{ Host: live, Kind: "instance", App: "eleven", Account: "bab", Status: "ready", DBPath: provision.NestedDBPath(dd, live), Socket: provision.NestedSocket(live), }); err != nil { t.Fatalf("Upsert: %v", err) } routes, err := reg.List() if err != nil { t.Fatalf("List: %v", err) } reapConfigEnv(routes, dd) livePath := filepath.Join(provision.InstanceDir(dd, live), live+configEnvSuffix) if _, err := os.Stat(livePath); err != nil { t.Fatalf("live nested route's config was reaped: %v", err) } gonePath := filepath.Join(provision.InstanceDir(dd, gone), gone+configEnvSuffix) if _, err := os.Stat(gonePath); !os.IsNotExist(err) { t.Fatalf("stale nested config survived reap, stat err = %v", err) } } func TestSyncExecRemovesFileWhenManifestDeleted(t *testing.T) { ctx, s, reg, dd, priv, _ := execKit(t) pk, _ := ParsePrivateKey(priv, "test") if err := reg.Upsert(registry.Route{ Host: "inst.example.com", Kind: "instance", App: "eleven", Account: "bab", Status: "ready", Socket: "/run/carlos/inst.sock", }); err != nil { t.Fatalf("Upsert: %v", err) } // A config file sync wrote earlier, whose manifest object is now gone: // only sync writes this namespace, so it must be removed — and the // running instance bounced, or it keeps stale config until next wake. path := filepath.Join(dd, "inst.example.com.config.env") if err := os.WriteFile(path, []byte("OLD=1\n"), 0o600); err != nil { t.Fatalf("stage: %v", err) } var bounced []string if err := SyncExec(ctx, reg, s, pk, dd, func(h string) error { bounced = append(bounced, h) return nil }, "", nil); err != nil { t.Fatalf("SyncExec: %v", err) } if _, err := os.Stat(path); !os.IsNotExist(err) { t.Fatalf("config file survived manifest deletion, stat err = %v", err) } if len(bounced) != 1 { t.Fatalf("bounced = %v, want the stale-config instance bounced once", bounced) } } // TestSyncExecMassRemovalGuardRefusesWhenEveryGroupResolvesAbsent covers // F4's misconfiguration guard: two distinct apps' groups both resolve // "manifest absent" in the same pass (the shape a mangled registry account // column produces — release.RouteAccount falls back to "ops" for every // route at once, so every group's bucket key 404s together). Both // existing config files must survive, and neither instance is bounced — // an unguarded pass would otherwise wipe every config file on the box in // one sweep from what might be a registry fault, not a real deletion. func TestSyncExecMassRemovalGuardRefusesWhenEveryGroupResolvesAbsent(t *testing.T) { ctx, s, reg, dd, priv, _ := execKit(t) pk, _ := ParsePrivateKey(priv, "test") // Neither app ever had SetEnv called: both groups' manifest objects are // genuinely absent from the bucket. routes := []registry.Route{ {Host: "a.example.com", Kind: "instance", App: "eleven", Account: "bab", Status: "ready", Socket: "/run/carlos/a.sock"}, {Host: "b.example.com", Kind: "instance", App: "titogo", Account: "bab", Status: "ready", Socket: "/run/carlos/b.sock"}, } for _, rt := range routes { if err := reg.Upsert(rt); err != nil { t.Fatalf("Upsert %s: %v", rt.Host, err) } } pathA := filepath.Join(dd, "a.example.com.config.env") pathB := filepath.Join(dd, "b.example.com.config.env") if err := os.WriteFile(pathA, []byte("OLD=1\n"), 0o600); err != nil { t.Fatalf("stage a: %v", err) } if err := os.WriteFile(pathB, []byte("OLD=2\n"), 0o600); err != nil { t.Fatalf("stage b: %v", err) } var bounced []string if err := SyncExec(ctx, reg, s, pk, dd, func(h string) error { bounced = append(bounced, h) return nil }, "", nil); err != nil { t.Fatalf("SyncExec: %v", err) } if _, err := os.Stat(pathA); err != nil { t.Fatalf("a's config file was removed despite the mass-removal guard: %v", err) } if _, err := os.Stat(pathB); err != nil { t.Fatalf("b's config file was removed despite the mass-removal guard: %v", err) } if len(bounced) != 0 { t.Fatalf("bounced = %v, want none — the guard must hold back both the removal and the bounce", bounced) } } // TestSyncExecMassRemovalGuardDoesNotBlockNormalConvergence covers the // guard's other edge: with two apps in the pass but only ONE absent, that // one's file is still removed normally — the guard only fires when EVERY // group is absent, so a real single-app deletion on a multi-app box must // keep converging. func TestSyncExecMassRemovalGuardDoesNotBlockNormalConvergence(t *testing.T) { ctx, s, reg, dd, priv, _ := execKit(t) pk, _ := ParsePrivateKey(priv, "test") if err := SetEnv(ctx, s, "bab", "eleven", map[string]string{"LOG_LEVEL": "debug"}); err != nil { t.Fatalf("SetEnv: %v", err) } // titogo's manifest is never set: absent, but eleven's is present, so // this pass is NOT unanimous. routes := []registry.Route{ {Host: "a.example.com", Kind: "instance", App: "eleven", Account: "bab", Status: "ready", Socket: "/run/carlos/a.sock"}, {Host: "b.example.com", Kind: "instance", App: "titogo", Account: "bab", Status: "ready", Socket: "/run/carlos/b.sock"}, } for _, rt := range routes { if err := reg.Upsert(rt); err != nil { t.Fatalf("Upsert %s: %v", rt.Host, err) } } pathA := filepath.Join(dd, "a.example.com.config.env") pathB := filepath.Join(dd, "b.example.com.config.env") if err := os.WriteFile(pathB, []byte("OLD=2\n"), 0o600); err != nil { t.Fatalf("stage b: %v", err) } var bounced []string if err := SyncExec(ctx, reg, s, pk, dd, func(h string) error { bounced = append(bounced, h) return nil }, "", nil); err != nil { t.Fatalf("SyncExec: %v", err) } if _, err := os.Stat(pathA); err != nil { t.Fatalf("a's config file wasn't written: %v", err) } if _, err := os.Stat(pathB); !os.IsNotExist(err) { t.Fatalf("b's config file survived its manifest's absence, stat err = %v", err) } // a's file is a fresh write (its manifest has real content, unlike // b's), so it bounces too — both changes converge normally. if !slices.Contains(bounced, "a.example.com") || !slices.Contains(bounced, "b.example.com") || len(bounced) != 2 { t.Fatalf("bounced = %v, want exactly [a.example.com b.example.com] (order not guaranteed)", bounced) } } // TestSyncExecRendersEachRoutesBoundEnvironment is the exec-path twin of // sync.go's TestSyncRendersEachRoutesBoundEnvironment: two instances of ONE // app, one bound to a named environment and one unbound, must land // DIFFERENT files in the same pass — the bound one gets the default bundle // overlaid by its environment (environment wins per key), the unbound one // gets the default bundle alone. Grouping on (acct, app) alone would render // whichever route came first into both. func TestSyncExecRendersEachRoutesBoundEnvironment(t *testing.T) { ctx, s, reg, dd, priv, _ := execKit(t) pk, err := ParsePrivateKey(priv, "test") if err != nil { t.Fatalf("ParsePrivateKey: %v", err) } if err := SetEnv(ctx, s, "bab", "eleven", map[string]string{ "LOG_LEVEL": "info", "BASE_ONLY": "1", }); err != nil { t.Fatalf("SetEnv: %v", err) } if err := SaveEnvironment(ctx, s, "bab", "eleven", "demo", map[string]Var{ "LOG_LEVEL": {Kind: "env", Value: "debug"}, // overrides the default "DEMO_ONLY": {Kind: "env", Value: "yes"}, // adds }, nil); err != nil { t.Fatalf("SaveEnvironment: %v", err) } routes := []registry.Route{ {Host: "demo.example.com", Kind: "instance", App: "eleven", Account: "bab", Status: "ready", Socket: "/run/carlos/demo.sock", Environment: "demo"}, {Host: "plain.example.com", Kind: "instance", App: "eleven", Account: "bab", Status: "ready", Socket: "/run/carlos/plain.sock"}, } for _, rt := range routes { if err := reg.Upsert(rt); err != nil { t.Fatalf("Upsert %s: %v", rt.Host, err) } } if err := SyncExec(ctx, reg, s, pk, dd, func(string) error { return nil }, "", nil); err != nil { t.Fatalf("SyncExec: %v", err) } want := map[string]string{ "demo.example.com": "BASE_ONLY=1\nDEMO_ONLY=yes\nLOG_LEVEL=debug\n", "plain.example.com": "BASE_ONLY=1\nLOG_LEVEL=info\n", } for host, w := range want { got, err := os.ReadFile(filepath.Join(dd, host+".config.env")) if err != nil { t.Fatalf("read %s config env: %v", host, err) } if string(got) != w { t.Fatalf("%s config env = %q, want %q", host, got, w) } } } // TestSyncExecLeavesFileUntouchedWhenBoundEnvironmentIsMissing pins the // distinction that matters most on this path: a bound environment that is // not in the bucket is an ERROR, never absence. Absence removes the config // file here (sync owns the namespace) — so folding the two together would // strip a live instance's whole config, and bounce it onto the stripped // file, because someone deleted a bundle. The file must survive untouched // and the route must not be bounced. func TestSyncExecLeavesFileUntouchedWhenBoundEnvironmentIsMissing(t *testing.T) { ctx, s, reg, dd, priv, _ := execKit(t) pk, _ := ParsePrivateKey(priv, "test") if err := SetEnv(ctx, s, "bab", "eleven", map[string]string{"LOG_LEVEL": "info"}); err != nil { t.Fatalf("SetEnv: %v", err) } if err := reg.Upsert(registry.Route{ Host: "ghost.example.com", Kind: "instance", App: "eleven", Account: "bab", Status: "ready", Socket: "/run/carlos/g.sock", Environment: "never-created", }); err != nil { t.Fatalf("Upsert: %v", err) } // The last good render, from before the bundle went missing. path := filepath.Join(dd, "ghost.example.com.config.env") const lastGood = "LOG_LEVEL=trace\n" if err := os.WriteFile(path, []byte(lastGood), 0o600); err != nil { t.Fatalf("stage: %v", err) } var bounced []string if err := SyncExec(ctx, reg, s, pk, dd, func(h string) error { bounced = append(bounced, h) return nil }, "", nil); err != nil { t.Fatalf("SyncExec: %v", err) } got, err := os.ReadFile(path) if err != nil { t.Fatalf("config file removed on an unresolved environment: %v", err) } if string(got) != lastGood { t.Fatalf("config env = %q, want it left exactly as-is (%q)", got, lastGood) } if len(bounced) != 0 { t.Fatalf("bounced = %v, want none — an unresolved route is not a changed route", bounced) } } // TestSyncExecOverlayOnlyAppDropsStaleKeys pins the deliberate divergence // from the unit path: SyncExec has NO lostKeys/defaultMissing guard, and // must not grow one. // // This is the exact shape sync.go REFUSES (2026-08-02 review, CRITICAL 1): // a route bound to an environment whose app has no default config.json at // all, so the merged view is the overlay bundle ALONE, dropping a key the // file on disk already holds. On the unit path that refusal is right — // /etc/carlos/apps/.env may have been hand-staged by an operator // before the app's config was ever imported into the bucket, so those keys // are not sync's to delete, and the documented migration step (create a // small override, bind the routes) would otherwise wipe them as root. // // Here it is wrong. Every byte of a .config.env was rendered by a previous // pass of SyncExec from a manifest — the namespace is sync-owned by // construction, nothing hand-staged can be in it — so a key the merged // view no longer contains is stale config that was removed from the // bucket, and dropping it IS the convergence. The instance's own // .env, the one hand-staged file in this directory, is a separate // file layered on top at spawn and never read or written by this sweep. // // Import the guard and this test fails, which is the point. func TestSyncExecOverlayOnlyAppDropsStaleKeys(t *testing.T) { ctx, s, reg, dd, priv, _ := execKit(t) pk, _ := ParsePrivateKey(priv, "test") // No SetEnv for eleven: the app has NO default bundle, only an // environment overlay. STALE is in neither. if err := SaveEnvironment(ctx, s, "bab", "eleven", "demo", map[string]Var{ "KEPT": {Kind: "env", Value: "1"}, }, nil); err != nil { t.Fatalf("SaveEnvironment: %v", err) } if err := reg.Upsert(registry.Route{ Host: "inst.example.com", Kind: "instance", App: "eleven", Account: "bab", Status: "ready", Socket: "/run/carlos/inst.sock", Environment: "demo", }); err != nil { t.Fatalf("Upsert: %v", err) } // A file an earlier pass rendered, back when STALE was still in the // bundle. Sync wrote it, so sync gets to drop it. path := filepath.Join(dd, "inst.example.com.config.env") if err := os.WriteFile(path, []byte("KEPT=1\nSTALE=old\n"), 0o600); err != nil { t.Fatalf("stage: %v", err) } var bounced []string if err := SyncExec(ctx, reg, s, pk, dd, func(h string) error { bounced = append(bounced, h) return nil }, "", nil); err != nil { t.Fatalf("SyncExec: %v", err) } got, err := os.ReadFile(path) if err != nil { t.Fatalf("read config env: %v", err) } if string(got) != "KEPT=1\n" { t.Fatalf("config env = %q, want the overlay ALONE (KEPT=1) — a key-loss guard must not be imported onto this path", got) } if len(bounced) != 1 || bounced[0] != "inst.example.com" { t.Fatalf("bounced = %v, want exactly one bounce for the dropped key", bounced) } } // TestSyncExecMassRemovalGuardIgnoresErroredGroups: an app whose bound // environment is missing is in neither the absent nor the present column, // so it must not sit in the guard's denominator and disarm it. Two // genuinely-absent groups plus one errored group is still "every resolvable // group is absent" — the guard fires, and the errored route's file is left // alone by the error path regardless. func TestSyncExecMassRemovalGuardIgnoresErroredGroups(t *testing.T) { ctx, s, reg, dd, priv, _ := execKit(t) pk, _ := ParsePrivateKey(priv, "test") // keymail has a default bundle but its bound environment is missing: // that group resolves to an error, not to absence. if err := SetEnv(ctx, s, "bab", "keymail", map[string]string{"K": "1"}); err != nil { t.Fatalf("SetEnv: %v", err) } routes := []registry.Route{ {Host: "a.example.com", Kind: "instance", App: "eleven", Account: "bab", Status: "ready", Socket: "/run/carlos/a.sock"}, {Host: "b.example.com", Kind: "instance", App: "titogo", Account: "bab", Status: "ready", Socket: "/run/carlos/b.sock"}, {Host: "c.example.com", Kind: "instance", App: "keymail", Account: "bab", Status: "ready", Socket: "/run/carlos/c.sock", Environment: "gone"}, } for _, rt := range routes { if err := reg.Upsert(rt); err != nil { t.Fatalf("Upsert %s: %v", rt.Host, err) } } for _, h := range []string{"a.example.com", "b.example.com", "c.example.com"} { if err := os.WriteFile(filepath.Join(dd, h+".config.env"), []byte("OLD=1\n"), 0o600); err != nil { t.Fatalf("stage %s: %v", h, err) } } var bounced []string if err := SyncExec(ctx, reg, s, pk, dd, func(h string) error { bounced = append(bounced, h) return nil }, "", nil); err != nil { t.Fatalf("SyncExec: %v", err) } for _, h := range []string{"a.example.com", "b.example.com", "c.example.com"} { if _, err := os.Stat(filepath.Join(dd, h+".config.env")); err != nil { t.Fatalf("%s: config file removed despite the mass-removal guard: %v", h, err) } } if len(bounced) != 0 { t.Fatalf("bounced = %v, want none", bounced) } } func TestSyncExecNeverTouchesInstanceEnvOrOutOfScopeRoutes(t *testing.T) { ctx, s, reg, dd, priv, _ := execKit(t) pk, _ := ParsePrivateKey(priv, "test") if err := SetEnv(ctx, s, "bab", "eleven", map[string]string{"A": "1"}); err != nil { t.Fatalf("SetEnv: %v", err) } // The instance's own env file must never be read, changed, or reaped. own := filepath.Join(dd, "inst.example.com.env") if err := os.WriteFile(own, []byte("CARLOS_ADMIN_TOKEN=tok\n"), 0o600); err != nil { t.Fatalf("stage own env: %v", err) } routes := []registry.Route{ {Host: "inst.example.com", Kind: "instance", App: "eleven", Account: "bab", Status: "ready", Socket: "/run/carlos/inst.sock"}, {Host: "unit.example.com", Kind: "instance", App: "eleven", Account: "bab", Status: "ready", Socket: "/run/carlos/u.sock", Backing: registry.BackingUnit}, {Host: "proxy.example.com", Kind: "instance", App: "eleven", Account: "bab", Status: "ready", Addr: "127.0.0.1:8444"}, {Host: "alias.example.com", Kind: "instance", App: "eleven", Account: "bab", Status: "ready", Socket: "/run/carlos/a.sock", Platform: true}, {Host: "pending.example.com", Kind: "instance", App: "eleven", Account: "bab", Status: "provisioning", Socket: "/run/carlos/p.sock"}, } for _, rt := range routes { if err := reg.Upsert(rt); err != nil { t.Fatalf("Upsert %s: %v", rt.Host, err) } } if err := SyncExec(ctx, reg, s, pk, dd, func(string) error { return nil }, "", nil); err != nil { t.Fatalf("SyncExec: %v", err) } if got, _ := os.ReadFile(own); string(got) != "CARLOS_ADMIN_TOKEN=tok\n" { t.Fatalf("instance env file was touched: %q", got) } if _, err := os.Stat(filepath.Join(dd, "inst.example.com.config.env")); err != nil { t.Fatalf("in-scope route got no config file: %v", err) } for _, h := range []string{"unit.example.com", "proxy.example.com", "alias.example.com", "pending.example.com"} { if _, err := os.Stat(filepath.Join(dd, h+".config.env")); !os.IsNotExist(err) { t.Fatalf("out-of-scope route %s got a config file, stat err = %v", h, err) } } } func TestSyncExecReapsOnlyItsOwnNamespace(t *testing.T) { ctx, s, reg, dd, priv, _ := execKit(t) pk, _ := ParsePrivateKey(priv, "test") if err := SetEnv(ctx, s, "bab", "eleven", map[string]string{"X": "1"}); err != nil { t.Fatalf("SetEnv: %v", err) } if err := reg.Upsert(registry.Route{ Host: "live.example.com", Kind: "instance", App: "eleven", Account: "bab", Status: "ready", Socket: "/run/carlos/l.sock", }); err != nil { t.Fatalf("Upsert: %v", err) } files := map[string]bool{ // name -> should survive "live.example.com.config.env": true, // has a route "gone.example.com.config.env": false, // no route: reaped "gone.example.com.config.env.tmp": false, // stale tmp: reaped "gone.example.com.env": true, // NOT ours: instance file, never reaped "gone.example.com-sidecar.env": true, // NOT ours: sidecar file, never reaped } for name := range files { if err := os.WriteFile(filepath.Join(dd, name), []byte("X=1\n"), 0o600); err != nil { t.Fatalf("stage %s: %v", name, err) } } if err := SyncExec(ctx, reg, s, pk, dd, func(string) error { return nil }, "", nil); err != nil { t.Fatalf("SyncExec: %v", err) } for name, keep := range files { _, err := os.Stat(filepath.Join(dd, name)) if keep && err != nil { t.Fatalf("%s: want kept, got %v", name, err) } if !keep && !os.IsNotExist(err) { t.Fatalf("%s: want reaped, stat err = %v", name, err) } } } // TestSyncExecReapsOnlyItsOwnNamespaceWithAddrShapeInTheMix is the reap // test's issue #18 twin: an opted-in addr route's own file survives reap // (it has a route row, same as any in-scope route), and a stray // *.config.env with NO route row at all is reaped regardless of which // exec shape produced it in some earlier pass — reap keys purely on // (namespace suffix, route existence), never on shape or opt-in. func TestSyncExecReapsOnlyItsOwnNamespaceWithAddrShapeInTheMix(t *testing.T) { ctx, s, reg, dd, priv, _ := execKit(t) pk, _ := ParsePrivateKey(priv, "test") if err := SetEnv(ctx, s, "bab", "titogo", map[string]string{"X": "1"}); err != nil { t.Fatalf("SetEnv: %v", err) } if err := SetExecDelivery(ctx, s, "bab", "titogo", ExecDeliveryFile); err != nil { t.Fatalf("SetExecDelivery: %v", err) } if err := reg.Upsert(registry.Route{ Host: "live.go.tito.io", Kind: "instance", App: "titogo", Account: "bab", Status: "ready", Addr: "127.0.0.1:8444", }); err != nil { t.Fatalf("Upsert: %v", err) } files := map[string]bool{ // name -> should survive "live.go.tito.io.config.env": true, // opted-in addr route, has a route row "gone.go.tito.io.config.env": false, // no route row at all: reaped, whatever shape wrote it } for name := range files { if err := os.WriteFile(filepath.Join(dd, name), []byte("X=1\n"), 0o600); err != nil { t.Fatalf("stage %s: %v", name, err) } } var bounced []string if err := SyncExec(ctx, reg, s, pk, dd, func(h string) error { bounced = append(bounced, h) return nil }, "", nil); err != nil { t.Fatalf("SyncExec: %v", err) } for name, keep := range files { _, err := os.Stat(filepath.Join(dd, name)) if keep && err != nil { t.Fatalf("%s: want kept, got %v", name, err) } if !keep && !os.IsNotExist(err) { t.Fatalf("%s: want reaped, stat err = %v", name, err) } } // The reap pass itself must never bounce — reap's log line says "no // route on this box", never "bounce"; and the addr route's own // convergence (a fresh, unchanged-content write above) must not // bounce either. if len(bounced) != 0 { t.Fatalf("bounced = %v, want none", bounced) } } func TestSyncExecEmptyRegistryRemovesNothing(t *testing.T) { ctx, s, reg, dd, priv, _ := execKit(t) pk, _ := ParsePrivateKey(priv, "test") path := filepath.Join(dd, "orphan.example.com.config.env") if err := os.WriteFile(path, []byte("X=1\n"), 0o600); err != nil { t.Fatalf("stage: %v", err) } if err := SyncExec(ctx, reg, s, pk, dd, func(string) error { return nil }, "", nil); err != nil { t.Fatalf("SyncExec: %v", err) } if _, err := os.Stat(path); err != nil { t.Fatalf("empty registry removed a config file: %v", err) } } func TestSyncExecRefusesNonBareHostname(t *testing.T) { _, _, _, dd, priv, _ := execKit(t) // routes injected directly via the per-route helper below pk, _ := ParsePrivateKey(priv, "test") m := &Manifest{Schema: 1, App: "eleven", Vars: map[string]Var{"A": {Kind: "env", Value: "1"}}} for _, host := range []string{"../escape", "a/b", "", ".", ".."} { syncExecHost(registry.Route{Host: host}, m, "apps/bab/eleven/config.json", pk, dd, func(string) error { return nil }, "", "", nil) } ents, err := os.ReadDir(dd) if err != nil { t.Fatalf("read dataDir: %v", err) } if len(ents) != 0 { t.Fatalf("dataDir = %v, want nothing written for non-bare hostnames", ents) } } // TestSyncExecOptedInAddrRouteMaterializesFile is issue #18's core case: // a TCP-addr route (Socket == "", Addr != "") whose app opted in via // ExecDeliveryFile in its default bundle gets the identical // .config.env a socket-exec route would — same rendering, same // layering guarantees — even though carlos neither spawned nor owns the // process behind it. func TestSyncExecOptedInAddrRouteMaterializesFile(t *testing.T) { ctx, s, reg, dd, priv, _ := execKit(t) pk, err := ParsePrivateKey(priv, "test") if err != nil { t.Fatalf("ParsePrivateKey: %v", err) } if err := SetEnv(ctx, s, "bab", "titogo", map[string]string{"TITOGO_DEMO_BANNER": "1"}); err != nil { t.Fatalf("SetEnv: %v", err) } if err := SetExecDelivery(ctx, s, "bab", "titogo", ExecDeliveryFile); err != nil { t.Fatalf("SetExecDelivery: %v", err) } if err := reg.Upsert(registry.Route{ Host: "demo.go.tito.io", Kind: "instance", App: "titogo", Account: "bab", Status: "ready", Addr: "127.0.0.1:8444", }); err != nil { t.Fatalf("Upsert: %v", err) } if err := SyncExec(ctx, reg, s, pk, dd, func(string) error { return nil }, "", nil); err != nil { t.Fatalf("SyncExec: %v", err) } got, err := os.ReadFile(filepath.Join(dd, "demo.go.tito.io.config.env")) if err != nil { t.Fatalf("opted-in TCP-addr route got no config file: %v", err) } want := execAddrConfigHeader + "TITOGO_DEMO_BANNER=1\n" if string(got) != want { t.Fatalf("config env = %q, want %q (issue #18's required header, then the vars)", got, want) } // Idempotence must survive the header (2026-08-03 re-review, IMPORTANT // 7): the up-to-date comparison has to include the header bytes, or an // unchanged pass rewrites this secrets-bearing root-owned file every // tick forever. The socket shape has this pin; before this assertion, // moving the header prepend after the comparison left the whole suite // green. mtime is the observable: a second pass over unchanged config // must not touch the file. path := filepath.Join(dd, "demo.go.tito.io.config.env") before, err := os.Stat(path) if err != nil { t.Fatal(err) } time.Sleep(10 * time.Millisecond) // ensure a rewrite would move mtime if err := SyncExec(ctx, reg, s, pk, dd, func(string) error { return nil }, "", nil); err != nil { t.Fatalf("SyncExec (second pass): %v", err) } after, err := os.Stat(path) if err != nil { t.Fatal(err) } if !after.ModTime().Equal(before.ModTime()) { t.Fatalf("an unchanged second pass rewrote the config file (mtime %v -> %v)", before.ModTime(), after.ModTime()) } } // TestSyncExecOptedInMultiAddrRouteMaterializesFile pins the HA registry // spelling of the externally managed TCP route. Addrs replaced Addr for HA // routes, including one-upstream routes, and must enter the same opt-in path: // carlos writes their file but never claims to own or bounce the process. func TestSyncExecOptedInMultiAddrRouteMaterializesFile(t *testing.T) { ctx, s, reg, dd, priv, _ := execKit(t) pk, err := ParsePrivateKey(priv, "test") if err != nil { t.Fatalf("ParsePrivateKey: %v", err) } if err := SetEnv(ctx, s, "bab", "woodstar", map[string]string{"PROVISION_TOKEN": "sealed-value"}); err != nil { t.Fatalf("SetEnv: %v", err) } if err := SetExecDelivery(ctx, s, "bab", "woodstar", ExecDeliveryFile); err != nil { t.Fatalf("SetExecDelivery: %v", err) } if err := reg.Upsert(registry.Route{ Host: "woodstar.example.com", Kind: "instance", App: "woodstar", Account: "bab", Status: "ready", Addrs: "127.0.0.1:8451", }); err != nil { t.Fatalf("Upsert: %v", err) } bounces := 0 if err := SyncExec(ctx, reg, s, pk, dd, func(string) error { bounces++ return nil }, "", nil); err != nil { t.Fatalf("SyncExec: %v", err) } got, err := os.ReadFile(filepath.Join(dd, "woodstar.example.com.config.env")) if err != nil { t.Fatalf("opted-in multi-addr route got no config file: %v", err) } want := execAddrConfigHeader + "PROVISION_TOKEN=sealed-value\n" if string(got) != want { t.Fatalf("config env = %q, want %q", got, want) } if bounces != 0 { t.Fatalf("multi-addr route bounced %d times, want zero: its supervisor owns restarts", bounces) } } // TestSyncExecUnoptedAddrRouteIsSkipped pins the un-opted case: the exact // same TCP-addr route shape, but its app never set exec_delivery, is // skipped exactly as before issue #18's third case existed — no file, no // error, nothing to converge or remove. func TestSyncExecUnoptedAddrRouteIsSkipped(t *testing.T) { ctx, s, reg, dd, priv, _ := execKit(t) pk, err := ParsePrivateKey(priv, "test") if err != nil { t.Fatalf("ParsePrivateKey: %v", err) } if err := SetEnv(ctx, s, "bab", "titogo", map[string]string{"TITOGO_DEMO_BANNER": "1"}); err != nil { t.Fatalf("SetEnv: %v", err) } // Deliberately no SetExecDelivery call. if err := reg.Upsert(registry.Route{ Host: "demo.go.tito.io", Kind: "instance", App: "titogo", Account: "bab", Status: "ready", Addr: "127.0.0.1:8444", }); err != nil { t.Fatalf("Upsert: %v", err) } if err := SyncExec(ctx, reg, s, pk, dd, func(string) error { return nil }, "", nil); err != nil { t.Fatalf("SyncExec: %v", err) } if _, err := os.Stat(filepath.Join(dd, "demo.go.tito.io.config.env")); !os.IsNotExist(err) { t.Fatalf("un-opted TCP-addr route got a config file, stat err = %v", err) } } // TestSyncExecUnrecognizedExecDeliveryValueIgnoredNotFatal covers additive // discipline (AGENTS.md): a manifest a NEWER carlos wrote, carrying an // exec_delivery value this binary doesn't understand, must not brick this // box's sync — the whole pass must still succeed, the unknown value must // be treated as "not opted in" (not a fatal error, not silently treated // as "file"), and OTHER apps' routes in the same pass must converge // normally. func TestSyncExecUnrecognizedExecDeliveryValueIgnoredNotFatal(t *testing.T) { ctx, s, reg, dd, priv, _ := execKit(t) pk, err := ParsePrivateKey(priv, "test") if err != nil { t.Fatalf("ParsePrivateKey: %v", err) } // Written directly, bypassing SetExecDelivery's set-time refusal — the // shape of a manifest a newer binary produced. raw := []byte(`{"schema":1,"app":"titogo","vars":{"K":{"kind":"env","value":"1"}},"exec_delivery":"grpc"}`) if _, err := s.Put(ctx, Key("bab", "titogo"), raw, ""); err != nil { t.Fatalf("Put: %v", err) } // A second, ordinary socket-exec app in the same pass, to prove the // unknown value doesn't abort or otherwise disturb the rest of the // sweep. if err := SetEnv(ctx, s, "bab", "eleven", map[string]string{"LOG_LEVEL": "debug"}); err != nil { t.Fatalf("SetEnv: %v", err) } routes := []registry.Route{ {Host: "demo.go.tito.io", Kind: "instance", App: "titogo", Account: "bab", Status: "ready", Addr: "127.0.0.1:8444"}, {Host: "inst.example.com", Kind: "instance", App: "eleven", Account: "bab", Status: "ready", Socket: "/run/carlos/inst.sock"}, } for _, rt := range routes { if err := reg.Upsert(rt); err != nil { t.Fatalf("Upsert %s: %v", rt.Host, err) } } var syncErr error out := captureStdout(t, func() { syncErr = SyncExec(ctx, reg, s, pk, dd, func(string) error { return nil }, "", nil) }) if syncErr != nil { t.Fatalf("SyncExec: %v", syncErr) } if _, err := os.Stat(filepath.Join(dd, "demo.go.tito.io.config.env")); !os.IsNotExist(err) { t.Fatalf("unrecognized exec_delivery value was treated as opted-in, stat err = %v", err) } if _, err := os.Stat(filepath.Join(dd, "inst.example.com.config.env")); err != nil { t.Fatalf("eleven's ordinary socket route did not converge despite titogo's unrecognized value: %v", err) } // MINOR 5 (2026-08-03 review): the whole safety story for this case is // "ignored LOUDLY" — pin the loudness itself, not just the tolerance, // so a future edit that quietly dropped the fmt.Printf would fail a // test instead of only being noticed by someone reading a log by hand. if !strings.Contains(out, "titogo") || !strings.Contains(out, "grpc") { t.Fatalf("stdout = %q, want a line naming the app (titogo) and the unrecognized value (grpc)", out) } } // TestSyncExecOptedInAddrRouteFetchFailureLogsAndLeavesFileAsIs is // IMPORTANT 1 from the 2026-08-03 review: a transient default-bundle read // failure against an ALREADY-opted-in app must not go silent. Before this // fix, checkOptedIn folded def.err != nil into the same "not opted in, // say nothing" branch as a genuinely never-configured app — so an opted-in // app whose bucket blips became indistinguishable, in the log, from an // app that was never opted in at all, on the exact path this feature // exists to make observable. The safety property (the file survives // byte-identical) was already correct; only the silence was the bug. func TestSyncExecOptedInAddrRouteFetchFailureLogsAndLeavesFileAsIs(t *testing.T) { ctx, s, reg, dd, priv, _ := execKit(t) pk, err := ParsePrivateKey(priv, "test") if err != nil { t.Fatalf("ParsePrivateKey: %v", err) } if err := SetEnv(ctx, s, "bab", "titogo", map[string]string{"TITOGO_DEMO_BANNER": "1"}); err != nil { t.Fatalf("SetEnv: %v", err) } if err := SetExecDelivery(ctx, s, "bab", "titogo", ExecDeliveryFile); err != nil { t.Fatalf("SetExecDelivery: %v", err) } if err := reg.Upsert(registry.Route{ Host: "demo.go.tito.io", Kind: "instance", App: "titogo", Account: "bab", Status: "ready", Addr: "127.0.0.1:8444", }); err != nil { t.Fatalf("Upsert: %v", err) } // First pass materializes normally — this is the file a transient // blip must leave exactly as it is. if err := SyncExec(ctx, reg, s, pk, dd, func(string) error { return nil }, "", nil); err != nil { t.Fatalf("SyncExec (initial): %v", err) } path := filepath.Join(dd, "demo.go.tito.io.config.env") original, err := os.ReadFile(path) if err != nil { t.Fatalf("read initial config env: %v", err) } blipErr := errors.New("s3: transient 503 SlowDown") failing := &failingKeyStore{Store: s, failKey: Key("bab", "titogo"), err: blipErr} var bounced []string bounce := func(h string) error { bounced = append(bounced, h); return nil } var syncErr error out := captureStdout(t, func() { syncErr = SyncExec(ctx, reg, failing, pk, dd, bounce, "", nil) }) if syncErr != nil { t.Fatalf("SyncExec (blip): %v", syncErr) } got, err := os.ReadFile(path) if err != nil { t.Fatalf("read config env after the blip: %v", err) } if string(got) != string(original) { t.Fatalf("config env changed across a transient fetch failure: got %q, want the original %q untouched", got, original) } if len(bounced) != 0 { t.Fatalf("bounced = %v, want none — a fetch failure is not a change", bounced) } // The fix: this must no longer be silent. Name the app and the // underlying error, the same "unresolved ... left as-is" contract the // socket path's resolution loop already gives an ordinary fetch // failure. if !strings.Contains(out, "titogo") { t.Fatalf("stdout = %q, want a line naming the app (titogo)", out) } if !strings.Contains(out, "SlowDown") { t.Fatalf("stdout = %q, want the underlying fetch error surfaced", out) } // The mutual exclusion the ruling depends on (2026-08-03 re-review, // IMPORTANT 6): a transient blip must print ONLY the unresolved line — // never the stale-secrets notice, whose remedy is "delete this file by // hand". Dropping !c.known from that notice's guard would turn every // bucket blip into an instruction to rm a live production file // mid-incident, and before this assertion the whole suite stayed green // under exactly that mutation. if strings.Contains(out, "delete it by hand") { t.Fatalf("stdout = %q — a transient fetch failure produced the stale-file removal advice", out) } } // TestSyncExecOptedInAddrRouteIsNeverBounced is the structural pin the // issue's design comment demands: carlos does not own an app-supervised // process, so an opted-in TCP-addr route's bounce callback must not even // be invoked — not "invoked and then skipped by a flag", genuinely never // called. The recorder below would append on any call at all; it must // stay empty across a pass that DOES change the route's file content // (proving the file write path ran) and a second pass where content is // unchanged (proving the still-quiet path also never bounces). func TestSyncExecOptedInAddrRouteIsNeverBounced(t *testing.T) { ctx, s, reg, dd, priv, _ := execKit(t) pk, err := ParsePrivateKey(priv, "test") if err != nil { t.Fatalf("ParsePrivateKey: %v", err) } if err := SetEnv(ctx, s, "bab", "titogo", map[string]string{"TITOGO_DEMO_BANNER": "1"}); err != nil { t.Fatalf("SetEnv: %v", err) } if err := SetExecDelivery(ctx, s, "bab", "titogo", ExecDeliveryFile); err != nil { t.Fatalf("SetExecDelivery: %v", err) } if err := reg.Upsert(registry.Route{ Host: "demo.go.tito.io", Kind: "instance", App: "titogo", Account: "bab", Status: "ready", Addr: "127.0.0.1:8444", }); err != nil { t.Fatalf("Upsert: %v", err) } var bounced []string bounce := func(h string) error { bounced = append(bounced, h) return nil } if err := SyncExec(ctx, reg, s, pk, dd, bounce, "", nil); err != nil { t.Fatalf("SyncExec (initial write): %v", err) } if len(bounced) != 0 { t.Fatalf("bounced = %v, want none — carlos does not own this process", bounced) } // Change the config so the file's content changes on the next pass — // the socket-exec equivalent of this WOULD bounce. if err := SetEnv(ctx, s, "bab", "titogo", map[string]string{"TITOGO_DEMO_BANNER": "2"}); err != nil { t.Fatalf("SetEnv (change): %v", err) } if err := SyncExec(ctx, reg, s, pk, dd, bounce, "", nil); err != nil { t.Fatalf("SyncExec (changed content): %v", err) } if len(bounced) != 0 { t.Fatalf("bounced = %v after a content change, want none", bounced) } got, err := os.ReadFile(filepath.Join(dd, "demo.go.tito.io.config.env")) if err != nil { t.Fatalf("read config env: %v", err) } want := execAddrConfigHeader + "TITOGO_DEMO_BANNER=2\n" if string(got) != want { t.Fatalf("config env = %q, want %q — the changed content, header included — the write path must still run", got, want) } } // TestSyncExecSocketRouteStillBouncesAlongsideOptedInAddrRoute proves the // addr shape's no-bounce rule is scoped to addr routes ONLY: a socket-exec // route for a DIFFERENT app in the same pass must still bounce on change, // unaffected by the addr route sharing the sweep. func TestSyncExecSocketRouteStillBouncesAlongsideOptedInAddrRoute(t *testing.T) { ctx, s, reg, dd, priv, _ := execKit(t) pk, err := ParsePrivateKey(priv, "test") if err != nil { t.Fatalf("ParsePrivateKey: %v", err) } if err := SetEnv(ctx, s, "bab", "titogo", map[string]string{"A": "1"}); err != nil { t.Fatalf("SetEnv: %v", err) } if err := SetExecDelivery(ctx, s, "bab", "titogo", ExecDeliveryFile); err != nil { t.Fatalf("SetExecDelivery: %v", err) } if err := SetEnv(ctx, s, "bab", "eleven", map[string]string{"B": "1"}); err != nil { t.Fatalf("SetEnv: %v", err) } routes := []registry.Route{ {Host: "demo.go.tito.io", Kind: "instance", App: "titogo", Account: "bab", Status: "ready", Addr: "127.0.0.1:8444"}, {Host: "inst.example.com", Kind: "instance", App: "eleven", Account: "bab", Status: "ready", Socket: "/run/carlos/inst.sock"}, } for _, rt := range routes { if err := reg.Upsert(rt); err != nil { t.Fatalf("Upsert %s: %v", rt.Host, err) } } var bounced []string bounce := func(h string) error { bounced = append(bounced, h) return nil } if err := SyncExec(ctx, reg, s, pk, dd, bounce, "", nil); err != nil { t.Fatalf("SyncExec: %v", err) } if !slices.Contains(bounced, "inst.example.com") { t.Fatalf("bounced = %v, want the socket-exec route bounced (unaffected by the addr route sharing the pass)", bounced) } if slices.Contains(bounced, "demo.go.tito.io") { t.Fatalf("bounced = %v, the opted-in addr route must never be bounced", bounced) } if len(bounced) != 1 { t.Fatalf("bounced = %v, want exactly one bounce", bounced) } } // TestSyncExecOptedInAddrRouteHonorsBoundEnvironment: environments- // awareness applies identically to the addr shape — a route bound to a // named environment resolves via LoadMerged (default overlaid by the // environment, environment winning per key), exactly like a socket-exec // route bound the same way. func TestSyncExecOptedInAddrRouteHonorsBoundEnvironment(t *testing.T) { ctx, s, reg, dd, priv, _ := execKit(t) pk, err := ParsePrivateKey(priv, "test") if err != nil { t.Fatalf("ParsePrivateKey: %v", err) } if err := SetEnv(ctx, s, "bab", "titogo", map[string]string{ "LOG_LEVEL": "info", "BASE_ONLY": "1", }); err != nil { t.Fatalf("SetEnv: %v", err) } if err := SetExecDelivery(ctx, s, "bab", "titogo", ExecDeliveryFile); err != nil { t.Fatalf("SetExecDelivery: %v", err) } if err := SaveEnvironment(ctx, s, "bab", "titogo", "demo", map[string]Var{ "LOG_LEVEL": {Kind: "env", Value: "debug"}, "DEMO_ONLY": {Kind: "env", Value: "yes"}, }, nil); err != nil { t.Fatalf("SaveEnvironment: %v", err) } if err := reg.Upsert(registry.Route{ Host: "demo.go.tito.io", Kind: "instance", App: "titogo", Account: "bab", Status: "ready", Addr: "127.0.0.1:8444", Environment: "demo", }); err != nil { t.Fatalf("Upsert: %v", err) } if err := SyncExec(ctx, reg, s, pk, dd, func(string) error { return nil }, "", nil); err != nil { t.Fatalf("SyncExec: %v", err) } got, err := os.ReadFile(filepath.Join(dd, "demo.go.tito.io.config.env")) if err != nil { t.Fatalf("read config env: %v", err) } want := execAddrConfigHeader + "BASE_ONLY=1\nDEMO_ONLY=yes\nLOG_LEVEL=debug\n" if string(got) != want { t.Fatalf("config env = %q, want %q", got, want) } } // TestSyncExecOptedInAddrAppWithNoVarsConvergesToEmptyFile: opting in // (SetExecDelivery alone, no SetEnv) creates the default bundle with an // empty Vars map — found=true, so this is the "configured, then nothing // set" state, not "absent". The route must get an EMPTY file, not have // none written at all and not have the mass-removal guard's absent // bucket confused by it. This is also the reason // "opted-in AND the group resolves absent" is unreachable by // construction: checkOptedIn requires the default bundle to exist // (found=true) before an addr route is ever added to resolved, and a // present-but-empty bundle is exactly resolution{m: }, // never resolution{} (nil, absent). func TestSyncExecOptedInAddrAppWithNoVarsConvergesToEmptyFile(t *testing.T) { ctx, s, reg, dd, priv, _ := execKit(t) pk, err := ParsePrivateKey(priv, "test") if err != nil { t.Fatalf("ParsePrivateKey: %v", err) } if err := SetExecDelivery(ctx, s, "bab", "titogo", ExecDeliveryFile); err != nil { t.Fatalf("SetExecDelivery: %v", err) } if err := reg.Upsert(registry.Route{ Host: "demo.go.tito.io", Kind: "instance", App: "titogo", Account: "bab", Status: "ready", Addr: "127.0.0.1:8444", }); err != nil { t.Fatalf("Upsert: %v", err) } if err := SyncExec(ctx, reg, s, pk, dd, func(string) error { return nil }, "", nil); err != nil { t.Fatalf("SyncExec: %v", err) } got, err := os.ReadFile(filepath.Join(dd, "demo.go.tito.io.config.env")) if err != nil { t.Fatalf("opted-in-but-unconfigured app's route got no file at all (want the header alone, not none): %v", err) } // No vars, but never a truly empty file: the header (issue #18) is // unconditional for this shape, whether or not the app has any vars // set yet. if string(got) != execAddrConfigHeader { t.Fatalf("config env = %q, want just the header (%q)", got, execAddrConfigHeader) } } // TestSyncExecMassRemovalGuardUnaffectedByOptedInAddrAppWithContent // exercises the mass-removal guard sharing its EXACT code path with the // addr shape (no addr-specific carve-out): an opted-in addr app WITH // content sits in the pass alongside a socket app that is genuinely // never configured. Only the socket app's group is absent — the guard // must not fire (only one of two eligible groups is absent, same as // TestSyncExecMassRemovalGuardDoesNotBlockNormalConvergence pins for two // socket apps), so the socket app's stale file is removed normally while // the addr app's file converges with its content, unaffected by sharing // the pass. func TestSyncExecMassRemovalGuardUnaffectedByOptedInAddrAppWithContent(t *testing.T) { ctx, s, reg, dd, priv, _ := execKit(t) pk, err := ParsePrivateKey(priv, "test") if err != nil { t.Fatalf("ParsePrivateKey: %v", err) } if err := SetEnv(ctx, s, "bab", "titogo", map[string]string{"A": "1"}); err != nil { t.Fatalf("SetEnv: %v", err) } if err := SetExecDelivery(ctx, s, "bab", "titogo", ExecDeliveryFile); err != nil { t.Fatalf("SetExecDelivery: %v", err) } // eleven is never configured at all: genuinely absent. routes := []registry.Route{ {Host: "demo.go.tito.io", Kind: "instance", App: "titogo", Account: "bab", Status: "ready", Addr: "127.0.0.1:8444"}, {Host: "b.example.com", Kind: "instance", App: "eleven", Account: "bab", Status: "ready", Socket: "/run/carlos/b.sock"}, } for _, rt := range routes { if err := reg.Upsert(rt); err != nil { t.Fatalf("Upsert %s: %v", rt.Host, err) } } pathEleven := filepath.Join(dd, "b.example.com.config.env") if err := os.WriteFile(pathEleven, []byte("OLD=2\n"), 0o600); err != nil { t.Fatalf("stage eleven: %v", err) } var bounced []string if err := SyncExec(ctx, reg, s, pk, dd, func(h string) error { bounced = append(bounced, h) return nil }, "", nil); err != nil { t.Fatalf("SyncExec: %v", err) } got, err := os.ReadFile(filepath.Join(dd, "demo.go.tito.io.config.env")) if err != nil { t.Fatalf("titogo's config file wasn't written: %v", err) } want := execAddrConfigHeader + "A=1\n" if string(got) != want { t.Fatalf("titogo config env = %q, want %q", got, want) } if _, err := os.Stat(pathEleven); !os.IsNotExist(err) { t.Fatalf("eleven's config file survived its manifest's genuine absence, stat err = %v (guard incorrectly disarmed convergence, or incorrectly held it back)", err) } // eleven's file is a removal (bounceLogged fires); titogo's is a // fresh write, socket-shape rules would bounce it too, but titogo is // the addr shape — it must never appear here. if !slices.Contains(bounced, "b.example.com") { t.Fatalf("bounced = %v, want eleven's removal bounce", bounced) } if slices.Contains(bounced, "demo.go.tito.io") { t.Fatalf("bounced = %v, the opted-in addr route must never be bounced", bounced) } if len(bounced) != 1 { t.Fatalf("bounced = %v, want exactly one bounce", bounced) } } // TestSyncExecOptOutLeavesStaleFileButWarnsEveryPass is IMPORTANT 2 from // the 2026-08-03 review's ruling: turning exec_delivery off does NOT // delete an already-materialized .config.env (deleting it would // change what the app's own supervisor layers in on its next respawn, at // a moment carlos does not control and cannot announce — worse than a // stale file, and consistent with every other ineligibility transition in // this file). But the file holds DECRYPTED secrets and nothing will ever // update it again once opted out, so the silence itself was the bug: // every pass must name the stale file and the manual remedy. func TestSyncExecOptOutLeavesStaleFileButWarnsEveryPass(t *testing.T) { ctx, s, reg, dd, priv, _ := execKit(t) pk, err := ParsePrivateKey(priv, "test") if err != nil { t.Fatalf("ParsePrivateKey: %v", err) } if err := SetEnv(ctx, s, "bab", "titogo", map[string]string{"TITOGO_DEMO_BANNER": "1"}); err != nil { t.Fatalf("SetEnv: %v", err) } if err := SetExecDelivery(ctx, s, "bab", "titogo", ExecDeliveryFile); err != nil { t.Fatalf("SetExecDelivery: %v", err) } if err := reg.Upsert(registry.Route{ Host: "demo.go.tito.io", Kind: "instance", App: "titogo", Account: "bab", Status: "ready", Addr: "127.0.0.1:8444", }); err != nil { t.Fatalf("Upsert: %v", err) } if err := SyncExec(ctx, reg, s, pk, dd, func(string) error { return nil }, "", nil); err != nil { t.Fatalf("SyncExec (opted in): %v", err) } path := filepath.Join(dd, "demo.go.tito.io.config.env") staged, err := os.ReadFile(path) if err != nil { t.Fatalf("read staged config env: %v", err) } if err := SetExecDelivery(ctx, s, "bab", "titogo", ""); err != nil { t.Fatalf("SetExecDelivery (off): %v", err) } var bounced []string bounce := func(h string) error { bounced = append(bounced, h); return nil } var syncErr error out := captureStdout(t, func() { syncErr = SyncExec(ctx, reg, s, pk, dd, bounce, "", nil) }) if syncErr != nil { t.Fatalf("SyncExec (opted out): %v", syncErr) } got, err := os.ReadFile(path) if err != nil { t.Fatalf("stale config env was deleted on opt-out, want it left in place: %v", err) } if string(got) != string(staged) { t.Fatalf("stale config env changed on opt-out: got %q, want the untouched original %q", got, staged) } if len(bounced) != 0 { t.Fatalf("bounced = %v, want none", bounced) } if !strings.Contains(out, "demo.go.tito.io") || !strings.Contains(out, path) { t.Fatalf("stdout = %q, want a line naming the host and the stale file's path", out) } if !strings.Contains(strings.ToLower(out), "secret") { t.Fatalf("stdout = %q, want the warning to say the file holds secrets", out) } // The warning is PER PASS, not one-shot: a second pass with nothing // changed must repeat it, since nothing about the risk (stale // plaintext secrets, forever) has gone away. out2 := captureStdout(t, func() { syncErr = SyncExec(ctx, reg, s, pk, dd, bounce, "", nil) }) if syncErr != nil { t.Fatalf("SyncExec (opted out, second pass): %v", syncErr) } if !strings.Contains(out2, "demo.go.tito.io") { t.Fatalf("second pass stdout = %q, want the warning repeated", out2) } if len(bounced) != 0 { t.Fatalf("bounced = %v after the second pass, want none", bounced) } } // TestSyncExecNoOptOutWarningWhenNeverOptedIn: the stale-file warning // fires only when a file actually exists — an app that was simply NEVER // opted in must stay exactly as quiet as it was before issue #18 existed, // not gain a new per-pass line for a file it never had. func TestSyncExecNoOptOutWarningWhenNeverOptedIn(t *testing.T) { ctx, s, reg, dd, priv, _ := execKit(t) pk, err := ParsePrivateKey(priv, "test") if err != nil { t.Fatalf("ParsePrivateKey: %v", err) } if err := SetEnv(ctx, s, "bab", "titogo", map[string]string{"K": "1"}); err != nil { t.Fatalf("SetEnv: %v", err) } // No SetExecDelivery call at all. if err := reg.Upsert(registry.Route{ Host: "demo.go.tito.io", Kind: "instance", App: "titogo", Account: "bab", Status: "ready", Addr: "127.0.0.1:8444", }); err != nil { t.Fatalf("Upsert: %v", err) } var syncErr error out := captureStdout(t, func() { syncErr = SyncExec(ctx, reg, s, pk, dd, func(string) error { return nil }, "", nil) }) if syncErr != nil { t.Fatalf("SyncExec: %v", syncErr) } if strings.Contains(out, "demo.go.tito.io") { t.Fatalf("stdout = %q, want no mention of a route that never had a file", out) } } // TestSyncExecAddrFileCarriesHeaderSocketFileDoesNot is IMPORTANT 3 from // the 2026-08-03 review: the issue #18 design comment's contract line — // "the file's header comment says so" — pinned directly. The addr shape's // file must carry it; the socket shape's file (same function, same pass) // must NOT, so an existing fleet's socket-exec files never churn or bounce // on upgrade (see execAddrConfigHeader's doc comment for the full reason). func TestSyncExecAddrFileCarriesHeaderSocketFileDoesNot(t *testing.T) { ctx, s, reg, dd, priv, _ := execKit(t) pk, err := ParsePrivateKey(priv, "test") if err != nil { t.Fatalf("ParsePrivateKey: %v", err) } if err := SetEnv(ctx, s, "bab", "titogo", map[string]string{"A": "1"}); err != nil { t.Fatalf("SetEnv: %v", err) } if err := SetExecDelivery(ctx, s, "bab", "titogo", ExecDeliveryFile); err != nil { t.Fatalf("SetExecDelivery: %v", err) } if err := SetEnv(ctx, s, "bab", "eleven", map[string]string{"B": "1"}); err != nil { t.Fatalf("SetEnv: %v", err) } routes := []registry.Route{ {Host: "demo.go.tito.io", Kind: "instance", App: "titogo", Account: "bab", Status: "ready", Addr: "127.0.0.1:8444"}, {Host: "inst.example.com", Kind: "instance", App: "eleven", Account: "bab", Status: "ready", Socket: "/run/carlos/inst.sock"}, } for _, rt := range routes { if err := reg.Upsert(rt); err != nil { t.Fatalf("Upsert %s: %v", rt.Host, err) } } if err := SyncExec(ctx, reg, s, pk, dd, func(string) error { return nil }, "", nil); err != nil { t.Fatalf("SyncExec: %v", err) } addrGot, err := os.ReadFile(filepath.Join(dd, "demo.go.tito.io.config.env")) if err != nil { t.Fatalf("read addr config env: %v", err) } if !strings.HasPrefix(string(addrGot), "#") { t.Fatalf("addr config env = %q, want it to start with a header comment", addrGot) } if !strings.Contains(string(addrGot), "carlos") { t.Fatalf("addr config env = %q, want the header to say carlos wrote it", addrGot) } if !strings.Contains(strings.ToLower(string(addrGot)), "never") || !strings.Contains(strings.ToLower(string(addrGot)), "restart") { t.Fatalf("addr config env = %q, want the header to say carlos never restarts this process", addrGot) } if !strings.Contains(strings.ToLower(string(addrGot)), "supervisor") { t.Fatalf("addr config env = %q, want the header to say the app's supervisor owns restarts", addrGot) } socketGot, err := os.ReadFile(filepath.Join(dd, "inst.example.com.config.env")) if err != nil { t.Fatalf("read socket config env: %v", err) } if want := "B=1\n"; string(socketGot) != want { t.Fatalf("socket config env = %q, want exactly %q — no header on this shape, ever", socketGot, want) } } // TestSyncExecConvergesToEmptyFileWhenAllVarsUnset covers the lifecycle // F8 flagged as missing: a manifest object that still EXISTS but has had // every var unset must converge its route to an EMPTY file, not have the // file removed — removal is reserved for a manifest object that is // entirely ABSENT from the bucket (see syncExecHost's m == nil branch). // Blurring those two states would make "unset every var" behave like // deleting the manifest, which it structurally is not. func TestSyncExecConvergesToEmptyFileWhenAllVarsUnset(t *testing.T) { ctx, s, reg, dd, priv, _ := execKit(t) pk, err := ParsePrivateKey(priv, "test") if err != nil { t.Fatalf("ParsePrivateKey: %v", err) } if err := SetEnv(ctx, s, "bab", "eleven", map[string]string{"LOG_LEVEL": "debug"}); err != nil { t.Fatalf("SetEnv: %v", err) } if err := reg.Upsert(registry.Route{ Host: "inst.example.com", Kind: "instance", App: "eleven", Account: "bab", Status: "ready", Socket: "/run/carlos/inst.sock", }); err != nil { t.Fatalf("Upsert: %v", err) } path := filepath.Join(dd, "inst.example.com.config.env") var bounced []string bounce := func(h string) error { bounced = append(bounced, h); return nil } if err := SyncExec(ctx, reg, s, pk, dd, bounce, "", nil); err != nil { t.Fatalf("SyncExec (set): %v", err) } got, err := os.ReadFile(path) if err != nil { t.Fatalf("read config env: %v", err) } if string(got) != "LOG_LEVEL=debug\n" { t.Fatalf("config env = %q, want LOG_LEVEL=debug", got) } // Unset every var — the manifest object itself still exists (Unset // rewrites it with an empty Vars map; it does not delete the object). if _, err := Unset(ctx, s, "bab", "eleven", []string{"LOG_LEVEL"}); err != nil { t.Fatalf("Unset (all vars): %v", err) } bounced = nil if err := SyncExec(ctx, reg, s, pk, dd, bounce, "", nil); err != nil { t.Fatalf("SyncExec (unset): %v", err) } got, err = os.ReadFile(path) if err != nil { t.Fatalf("config file removed instead of converged to empty: %v", err) } if string(got) != "" { t.Fatalf("config env = %q, want empty", got) } if len(bounced) != 1 || bounced[0] != "inst.example.com" { t.Fatalf("bounced = %v, want exactly one bounce for the change to empty", bounced) } } // TestReapSparesLiveRoutesInstanceEnvWithConfigSuffix covers the PR #16 // deferred namespace-collision hazard: host "x.config"'s hand-staged // instance env is named "x.config.env" (.env for host "x.config"), // which is byte-identical to what host "x"'s SYNC-OWNED file would be // named. With no route "x" at all, the naive reap trims configEnvSuffix off // "x.config.env", sees host "x", finds it not live, and deletes what is // actually route "x.config"'s own instance env — another owner's file. func TestReapSparesLiveRoutesInstanceEnvWithConfigSuffix(t *testing.T) { ctx, s, reg, dd, priv, _ := execKit(t) pk, _ := ParsePrivateKey(priv, "test") path := filepath.Join(dd, "x.config.env") sentinel := []byte("SENTINEL=owned-by-x-config\n") if err := os.WriteFile(path, sentinel, 0o600); err != nil { t.Fatalf("stage: %v", err) } // A live route named "x.config" and, critically, no route "x" at all. if err := reg.Upsert(registry.Route{ Host: "x.config", Kind: "instance", App: "eleven", Account: "bab", Status: "ready", }); err != nil { t.Fatalf("Upsert: %v", err) } out := captureStdout(t, func() { if err := SyncExec(ctx, reg, s, pk, dd, func(string) error { return nil }, "", nil); err != nil { t.Fatalf("SyncExec: %v", err) } }) // Pin the guard's own text, not just the file's survival — a passing // file-survival check alone wouldn't distinguish "the guard fired" from // "reap never ran at all". if !strings.Contains(out, "not reaped") || !strings.Contains(out, "claims this name as its instance env") { t.Fatalf("stdout = %q, want the guard's own refusal text", out) } got, err := os.ReadFile(path) if err != nil { t.Fatalf("x.config's instance env was reaped: %v", err) } if !bytes.Equal(got, sentinel) { t.Fatalf("x.config's instance env bytes changed: got %q, want %q", got, sentinel) } } // TestSyncExecRefusesConfigNamespaceCollision covers the other half of the // same hazard: route "x" IS live (an ordinary socket-exec route) alongside // route "x.config". Converging route x would write/rename into // "x.config.env" — route x.config's hand-staged instance env, not route x's // own namespace. Route x must be refused, loudly, and route x.config's OWN // config file ("x.config.config.env") must still converge normally: the // guard protects one file without collaterally blocking the unrelated // route that happens to produce the colliding name. func TestSyncExecRefusesConfigNamespaceCollision(t *testing.T) { ctx, s, reg, dd, priv, _ := execKit(t) pk, err := ParsePrivateKey(priv, "test") if err != nil { t.Fatalf("ParsePrivateKey: %v", err) } if err := SetEnv(ctx, s, "bab", "eleven", map[string]string{"X": "1"}); err != nil { t.Fatalf("SetEnv (x's app): %v", err) } if err := SetEnv(ctx, s, "bab", "titogo", map[string]string{"Y": "2"}); err != nil { t.Fatalf("SetEnv (x.config's app): %v", err) } if err := reg.Upsert(registry.Route{ Host: "x", Kind: "instance", App: "eleven", Account: "bab", Status: "ready", Socket: "/run/carlos/x.sock", }); err != nil { t.Fatalf("Upsert x: %v", err) } if err := reg.Upsert(registry.Route{ Host: "x.config", Kind: "instance", App: "titogo", Account: "bab", Status: "ready", Socket: "/run/carlos/xc.sock", }); err != nil { t.Fatalf("Upsert x.config: %v", err) } // x.config's hand-staged instance env, pre-staged with sentinel bytes: // its name ("x.config.env") is exactly what route x's own sync-owned // config file would be named. collidingPath := filepath.Join(dd, "x.config.env") sentinel := []byte("SENTINEL=owned-by-x-config\n") if err := os.WriteFile(collidingPath, sentinel, 0o600); err != nil { t.Fatalf("stage: %v", err) } var out string var bounced []string bounce := func(h string) error { bounced = append(bounced, h); return nil } out = captureStdout(t, func() { if err := SyncExec(ctx, reg, s, pk, dd, bounce, "", nil); err != nil { t.Fatalf("SyncExec: %v", err) } }) got, err := os.ReadFile(collidingPath) if err != nil { t.Fatalf("read colliding file: %v", err) } if !bytes.Equal(got, sentinel) { t.Fatalf("x.config's instance env was overwritten by route x's convergence: got %q, want %q", got, sentinel) } if slices.Contains(bounced, "x") { t.Fatalf("bounced = %v, want route x never bounced (it was refused, not converged)", bounced) } // "x.config" alone is a vacuous assertion here — route x.config's own // convergence prints " x.config: config updated" regardless of whether // the guard fired at all. Pin the guard's OWN distinctive text instead. if !strings.Contains(out, "not converging") || !strings.Contains(out, "refusing to write into another owner's namespace") { t.Fatalf("stdout = %q, want the guard's own refusal text", out) } // Route x.config's OWN config file is unrelated to the guard and must // converge exactly as it would with no collision in play. xConfigOwn, err := os.ReadFile(filepath.Join(dd, "x.config.config.env")) if err != nil { t.Fatalf("x.config's own config file did not converge: %v", err) } if string(xConfigOwn) != "Y=2\n" { t.Fatalf("x.config's own config file = %q, want Y=2", xConfigOwn) } } // TestSyncExecFetchFailureLeavesOtherGroupsConverging is the M4 // cross-group half TestSyncExecOptedInAddrRouteFetchFailureLogsAndLeavesFileAsIs // doesn't cover (that test has exactly one group in the pass): one app's // manifest fetch fails while a DIFFERENT app's group, in the SAME pass, // converges normally. A fetch failure must stay scoped to its own group — // it must neither remove/leave-stale the failing group's file (error is not // absence) NOR abort the sweep for every other group on the box. func TestSyncExecFetchFailureLeavesOtherGroupsConverging(t *testing.T) { ctx, s, reg, dd, priv, _ := execKit(t) pk, err := ParsePrivateKey(priv, "test") if err != nil { t.Fatalf("ParsePrivateKey: %v", err) } if err := SetEnv(ctx, s, "bab", "appa", map[string]string{"A": "1"}); err != nil { t.Fatalf("SetEnv appa: %v", err) } if err := SetEnv(ctx, s, "bab", "appb", map[string]string{"B": "2"}); err != nil { t.Fatalf("SetEnv appb: %v", err) } if err := reg.Upsert(registry.Route{ Host: "hosta", Kind: "instance", App: "appa", Account: "bab", Status: "ready", Socket: "/run/carlos/a.sock", }); err != nil { t.Fatalf("Upsert hosta: %v", err) } if err := reg.Upsert(registry.Route{ Host: "hostb", Kind: "instance", App: "appb", Account: "bab", Status: "ready", Socket: "/run/carlos/b.sock", }); err != nil { t.Fatalf("Upsert hostb: %v", err) } stalePath := filepath.Join(dd, "hosta.config.env") stale := []byte("STALE=1\n") if err := os.WriteFile(stalePath, stale, 0o600); err != nil { t.Fatalf("stage: %v", err) } blipErr := errors.New("s3: transient 503 SlowDown") failing := &failingKeyStore{Store: s, failKey: Key("bab", "appa"), err: blipErr} var bounced []string bounce := func(h string) error { bounced = append(bounced, h); return nil } if err := SyncExec(ctx, reg, failing, pk, dd, bounce, "", nil); err != nil { t.Fatalf("SyncExec: %v", err) } got, err := os.ReadFile(stalePath) if err != nil { t.Fatalf("read hosta config: %v", err) } if !bytes.Equal(got, stale) { t.Fatalf("hosta's config changed despite its own group's fetch failure: got %q, want %q", got, stale) } gotB, err := os.ReadFile(filepath.Join(dd, "hostb.config.env")) if err != nil { t.Fatalf("hostb's config didn't converge in the same pass: %v", err) } if string(gotB) != "B=2\n" { t.Fatalf("hostb config = %q, want B=2", gotB) } if !slices.Contains(bounced, "hostb") || slices.Contains(bounced, "hosta") { t.Fatalf("bounced = %v, want exactly [hostb] — appa's route left untouched, appb's route converged", bounced) } } // TestExecConfigApps pins the set the agent's fast generation loop polls: // exactly the apps SyncExec could deliver config to, by route SHAPE — and // nothing else. The opt-in half of the TCP-addr case is deliberately NOT // consulted here (that would cost a bucket read per app, the very thing the // fast loop exists to avoid), so an un-opted addr route is in this list; // see ExecConfigApps' doc comment. func TestExecConfigApps(t *testing.T) { routes := []registry.Route{ // In: the socket-exec shape, unconditionally syncable. {Host: "sock.example.com", Kind: "instance", App: "eleven", Account: "bab", Status: "ready", Socket: "/run/carlos/sock.sock"}, // In: the TCP-addr shape (issue #18), opt-in unknown and unread. {Host: "addr.example.com", Kind: "instance", App: "titogo", Account: "zed", Status: "ready", Addr: "127.0.0.1:9000"}, // In: the HA spelling is the same externally managed TCP shape. {Host: "multi-addr.example.com", Kind: "instance", App: "woodstar", Account: "zed", Status: "ready", Addrs: "127.0.0.1:8451"}, // In, and deduped with sock.example.com: one app, two routes, one // generation object between them. {Host: "sock2.example.com", Kind: "instance", App: "eleven", Account: "bab", Status: "", Socket: "/run/carlos/sock2.sock"}, // In under release.OpsAccount: a route whose Account is unusable as // a bucket segment reads as the ops account everywhere else, and // must here too or the loop watches a key nothing writes. {Host: "legacy.example.com", Kind: "instance", App: "kass", Account: "", Status: "ready", Socket: "/run/carlos/legacy.sock"}, // Out: unit-backed routes belong to the root adopt path. {Host: "unit.example.com", Kind: "instance", App: "unitapp", Account: "bab", Status: "ready", Socket: "/run/carlos/unit.sock", Backing: registry.BackingUnit}, // Out: platform routes are not app config. {Host: "plat.example.com", Kind: "instance", App: "platapp", Account: "bab", Status: "ready", Socket: "/run/carlos/plat.sock", Platform: true}, // Out: a draining route is on its way out; nothing to converge. {Host: "drain.example.com", Kind: "instance", App: "drainapp", Account: "bab", Status: "draining", Socket: "/run/carlos/drain.sock"}, // Out: no app means no config manifest to key on. {Host: "noapp.example.com", Kind: "instance", App: "", Account: "bab", Status: "ready", Socket: "/run/carlos/noapp.sock"}, // Out: neither socket nor addr — no upstream either predicate matches. {Host: "bare.example.com", Kind: "instance", App: "bareapp", Account: "bab", Status: "ready"}, // Out: not an instance route at all. {Host: "proxy.example.com", Kind: "proxy", App: "proxyapp", Account: "bab", Status: "ready", Socket: "/run/carlos/proxy.sock"}, } got := ExecConfigApps(routes) want := [][2]string{ {"bab", "eleven"}, {"bab", "kass"}, {"zed", "titogo"}, {"zed", "woodstar"}, } if !slices.Equal(got, want) { t.Fatalf("ExecConfigApps = %v, want %v (distinct, sorted by account then app)", got, want) } } func TestExecConfigAppsNoRoutes(t *testing.T) { if got := ExecConfigApps(nil); len(got) != 0 { t.Fatalf("ExecConfigApps(nil) = %v, want empty", got) } } // TestUnitConfigApps is TestExecConfigApps' mirror for the other half of the // fast loop's poll set: the apps whose config only the ROOT adopt pass can // deliver. Same fixture shapes, opposite inclusion on Backing — and the // both-shapes app below is the case that matters most, because an app can // hold a socket-exec route and a unit route at once and must appear in BOTH // functions' outputs (the loop then syncs exec in-process AND asks for a // root pass off one generation read). func TestUnitConfigApps(t *testing.T) { routes := []registry.Route{ // In: the carlos-app@ unit shape the root adopt pass renders. {Host: "unit.example.com", Kind: "instance", App: "unitapp", Account: "bab", Status: "ready", Socket: "/run/carlos/unit.sock", Backing: registry.BackingUnit}, // In, and deduped: one app, two unit routes, one generation object. {Host: "unit2.example.com", Kind: "instance", App: "unitapp", Account: "bab", Status: "", Socket: "/run/carlos/unit2.sock", Backing: registry.BackingUnit}, // In under release.OpsAccount, exactly as ExecConfigApps resolves it. {Host: "legacy.example.com", Kind: "instance", App: "kass", Account: "", Status: "ready", Socket: "/run/carlos/legacy.sock", Backing: registry.BackingUnit}, // In BOTH sets: this app's unit route is here, its exec route is in // ExecConfigApps (asserted below). {Host: "mixed-unit.example.com", Kind: "instance", App: "mixed", Account: "zed", Status: "ready", Socket: "/run/carlos/mixed-unit.sock", Backing: registry.BackingUnit}, {Host: "mixed-exec.example.com", Kind: "instance", App: "mixed", Account: "zed", Status: "ready", Socket: "/run/carlos/mixed-exec.sock"}, // Out: exec-backed only — the agent delivers this one itself. {Host: "sock.example.com", Kind: "instance", App: "eleven", Account: "bab", Status: "ready", Socket: "/run/carlos/sock.sock"}, // Out: platform routes are not app config. {Host: "plat.example.com", Kind: "instance", App: "platapp", Account: "bab", Status: "ready", Socket: "/run/carlos/plat.sock", Platform: true, Backing: registry.BackingUnit}, // Out: a draining route is on its way out; nothing to converge. {Host: "drain.example.com", Kind: "instance", App: "drainapp", Account: "bab", Status: "draining", Socket: "/run/carlos/drain.sock", Backing: registry.BackingUnit}, // Out: no app means no config manifest to key on. {Host: "noapp.example.com", Kind: "instance", App: "", Account: "bab", Status: "ready", Socket: "/run/carlos/noapp.sock", Backing: registry.BackingUnit}, // Out: not an instance route at all. {Host: "proxy.example.com", Kind: "proxy", App: "proxyapp", Account: "bab", Status: "ready", Socket: "/run/carlos/proxy.sock", Backing: registry.BackingUnit}, } got := UnitConfigApps(routes) want := [][2]string{ {"bab", "kass"}, {"bab", "unitapp"}, {"zed", "mixed"}, } if !slices.Equal(got, want) { t.Fatalf("UnitConfigApps = %v, want %v (distinct, sorted by account then app)", got, want) } if gotExec := ExecConfigApps(routes); !slices.Contains(gotExec, [2]string{"zed", "mixed"}) { t.Fatalf("ExecConfigApps = %v, want it to also hold {zed mixed} — an app with both route shapes is in both sets", gotExec) } } func TestUnitConfigAppsNoRoutes(t *testing.T) { if got := UnitConfigApps(nil); len(got) != 0 { t.Fatalf("UnitConfigApps(nil) = %v, want empty", got) } } // --- issue #195: the opt-in resolves from the route's bound bundle --- // TestSyncExecOptInResolvesPerEnvironment is issue #195's matrix: which // hosts get a .config.env, for each combination of the app-level // flag and the bound bundle's own. Two TCP-addr exec routes of ONE app, // one bound to "relay" and one unbound — titogo's exact shape, where // turning the app-level flag on to deliver the relay's sealed Apple Wallet // material wrote a file for every router route too. func TestSyncExecOptInResolvesPerEnvironment(t *testing.T) { for _, tc := range []struct { name string appLevel string // "" (never set) | ExecDeliveryFile envLevel string // absent | silent | file | bogus wantRelayFile bool wantUnboundFile bool }{ {"both off", "", "silent", false, false}, {"app on, env silent — inherit on", ExecDeliveryFile, "silent", true, true}, {"app off, env on — the narrow case #195 exists for", "", "file", true, false}, {"app on, env on", ExecDeliveryFile, "file", true, true}, {"app off, env bundle absent", "", "absent", false, false}, // Bound to a bundle that does not exist: the opt-in inherits "on" // from the app default, but the route never reaches a write anyway // — LoadMerged refuses a bound-but-absent bundle, so the group // resolves to an ERROR and the file is left exactly as it is. The // unbound router is untouched by that and still gets its file. {"app on, env bundle absent — resolution errors before delivery", ExecDeliveryFile, "absent", false, true}, {"app on, env states a value this binary doesn't know — that statement decides", ExecDeliveryFile, "bogus", false, true}, } { t.Run(tc.name, func(t *testing.T) { ctx, s, reg, dd, priv, _ := execKit(t) pk, err := ParsePrivateKey(priv, "test") if err != nil { t.Fatalf("ParsePrivateKey: %v", err) } // The default bundle always exists and always has a var, so the // only thing under test is WHETHER a file is written. if err := SetEnv(ctx, s, "bab", "titogo", map[string]string{"SHARED": "1"}); err != nil { t.Fatalf("SetEnv: %v", err) } if tc.appLevel != "" { if err := SetExecDelivery(ctx, s, "bab", "titogo", tc.appLevel); err != nil { t.Fatalf("SetExecDelivery: %v", err) } } switch tc.envLevel { case "absent": // Nothing written: the route is bound to a bundle that does // not exist. case "bogus": // Written past the setter's validation on purpose, and // without SaveEnvironment first (a create, not an update): // this is what an OLDER binary sees when a NEWER one // invents a value, and it must not brick the pass. raw := []byte(`{"schema":1,"app":"titogo","vars":{"SEALED":{"kind":"env","value":"2"}},"exec_delivery":"grpc"}`) if _, err := s.Put(ctx, EnvironmentKey("bab", "titogo", "relay"), raw, ""); err != nil { t.Fatalf("Put: %v", err) } default: if err := SaveEnvironment(ctx, s, "bab", "titogo", "relay", map[string]Var{"SEALED": {Kind: "env", Value: "2"}}, nil); err != nil { t.Fatalf("SaveEnvironment: %v", err) } if tc.envLevel == "file" { if err := SetExecDeliveryAt(ctx, s, "bab", "titogo", "relay", ExecDeliveryFile); err != nil { t.Fatalf("SetExecDeliveryAt: %v", err) } } } for _, rt := range []registry.Route{ {Host: "connect.go.tito.io", Kind: "instance", App: "titogo", Account: "bab", Status: "ready", Addr: "127.0.0.1:9001", Environment: "relay"}, {Host: "router-a.go.tito.io", Kind: "instance", App: "titogo", Account: "bab", Status: "ready", Addr: "127.0.0.1:9002"}, } { if err := reg.Upsert(rt); err != nil { t.Fatalf("Upsert %s: %v", rt.Host, err) } } if err := SyncExec(ctx, reg, s, pk, dd, func(string) error { return nil }, "", nil); err != nil { t.Fatalf("SyncExec: %v", err) } assertConfigFile(t, dd, "connect.go.tito.io", tc.wantRelayFile) assertConfigFile(t, dd, "router-a.go.tito.io", tc.wantUnboundFile) }) } } // assertConfigFile asserts a route's .config.env exists (or not). func assertConfigFile(t *testing.T, dir, host string, want bool) { t.Helper() _, err := os.Stat(filepath.Join(dir, host+configEnvSuffix)) switch { case want && err != nil: t.Fatalf("%s: want a config file, got %v", host, err) case !want && err == nil: t.Fatalf("%s: want NO config file, one was written", host) } } // TestSyncExecEnvScopedOptInDeliversTheMergedBundle: the flag decides // WHETHER a file is written, never what is in it. A route opted in through // its own bundle still gets the default bundle overlaid by that bundle — // the same merge LoadMerged has always produced. func TestSyncExecEnvScopedOptInDeliversTheMergedBundle(t *testing.T) { ctx, s, reg, dd, priv, _ := execKit(t) pk, err := ParsePrivateKey(priv, "test") if err != nil { t.Fatalf("ParsePrivateKey: %v", err) } if err := SetEnv(ctx, s, "bab", "titogo", map[string]string{"SHARED": "1", "OVERRIDDEN": "default"}); err != nil { t.Fatalf("SetEnv: %v", err) } if err := SaveEnvironment(ctx, s, "bab", "titogo", "relay", map[string]Var{ "OVERRIDDEN": {Kind: "env", Value: "relay"}, "WALLET": {Kind: "env", Value: "sealed"}, }, nil); err != nil { t.Fatalf("SaveEnvironment: %v", err) } if err := SetExecDeliveryAt(ctx, s, "bab", "titogo", "relay", ExecDeliveryFile); err != nil { t.Fatalf("SetExecDeliveryAt: %v", err) } if err := reg.Upsert(registry.Route{ Host: "connect.go.tito.io", Kind: "instance", App: "titogo", Account: "bab", Status: "ready", Addr: "127.0.0.1:9001", Environment: "relay", }); err != nil { t.Fatalf("Upsert: %v", err) } if err := SyncExec(ctx, reg, s, pk, dd, func(string) error { return nil }, "", nil); err != nil { t.Fatalf("SyncExec: %v", err) } got, err := os.ReadFile(filepath.Join(dd, "connect.go.tito.io"+configEnvSuffix)) if err != nil { t.Fatalf("read config file: %v", err) } want := execAddrConfigHeader + "OVERRIDDEN=relay\nSHARED=1\nWALLET=sealed\n" if string(got) != want { t.Fatalf("config env = %q, want %q (the merged bundle, unchanged by where the flag lives)", got, want) } // exec_delivery itself must never be rendered into the file: it is a // delivery flag, not a config var. if strings.Contains(string(got), "exec_delivery") || strings.Contains(string(got), "EXEC_DELIVERY") { t.Fatalf("the delivery flag leaked into the rendered file: %q", got) } } // TestSyncExecEnvBundleFetchErrorLeavesFileAlone: an unreadable // environment bundle is NOT a determination that the route is un-opted. // The route's existing file must survive it byte-identical — the same // contract a default-bundle fetch failure already has, extended to the // second bundle the opt-in now consults. func TestSyncExecEnvBundleFetchErrorLeavesFileAlone(t *testing.T) { ctx, s, reg, dd, priv, _ := execKit(t) pk, err := ParsePrivateKey(priv, "test") if err != nil { t.Fatalf("ParsePrivateKey: %v", err) } if err := SetEnv(ctx, s, "bab", "titogo", map[string]string{"SHARED": "1"}); err != nil { t.Fatalf("SetEnv: %v", err) } if err := SaveEnvironment(ctx, s, "bab", "titogo", "relay", map[string]Var{"WALLET": {Kind: "env", Value: "sealed"}}, nil); err != nil { t.Fatalf("SaveEnvironment: %v", err) } if err := SetExecDeliveryAt(ctx, s, "bab", "titogo", "relay", ExecDeliveryFile); err != nil { t.Fatalf("SetExecDeliveryAt: %v", err) } if err := reg.Upsert(registry.Route{ Host: "connect.go.tito.io", Kind: "instance", App: "titogo", Account: "bab", Status: "ready", Addr: "127.0.0.1:9001", Environment: "relay", }); err != nil { t.Fatalf("Upsert: %v", err) } if err := SyncExec(ctx, reg, s, pk, dd, func(string) error { return nil }, "", nil); err != nil { t.Fatalf("SyncExec: %v", err) } path := filepath.Join(dd, "connect.go.tito.io"+configEnvSuffix) before, err := os.ReadFile(path) if err != nil { t.Fatalf("first pass wrote no file: %v", err) } blip := &failingKeyStore{Store: s, failKey: EnvironmentKey("bab", "titogo", "relay"), err: errors.New("503 SlowDown")} out := captureStdout(t, func() { if err := SyncExec(ctx, reg, blip, pk, dd, func(string) error { return nil }, "", nil); err != nil { t.Fatalf("SyncExec (blip): %v", err) } }) after, err := os.ReadFile(path) if err != nil { t.Fatalf("a bundle fetch blip removed a live route's config file: %v", err) } if !bytes.Equal(before, after) { t.Fatalf("config file changed on a fetch blip: %q -> %q", before, after) } if !strings.Contains(out, "unresolved") || !strings.Contains(out, "relay") { t.Fatalf("a fetch blip against the bound bundle must say so loudly and name the environment; got %q", out) } } // ---- Task 11: sidecar config delivery ----------------------------------- // TestSidecarGetsItsConfigEnvFile: execRouteBase is the shape predicate both // exec eligibility cases narrow. A sidecar row (no Socket, no Addr) must // pass it — this is the direct replacement for the hand-staged // -sidecar.env, and the whole reason the file path existed. func TestSidecarGetsItsConfigEnvFile(t *testing.T) { rt := registry.Route{Host: "bot.eleven.bdm.sidecar.internal", Kind: "sidecar", App: "eleven", Status: "ready"} if !execRouteBase(rt) { t.Fatal("a sidecar must receive config delivery — this is the direct replacement for the hand-staged -sidecar.env, and the whole reason the file path existed") } } // TestSidecarConfigStillExcludesPlatformAndUnitRoutes: widening the Kind // test must not loosen any of execRouteBase's other clauses. func TestSidecarConfigStillExcludesPlatformAndUnitRoutes(t *testing.T) { base := registry.Route{Host: "bot.x.sidecar.internal", Kind: "sidecar", App: "eleven", Status: "ready"} plat := base plat.Platform = true if execRouteBase(plat) { t.Fatal("a Platform alias is still excluded") } unit := base unit.Backing = registry.BackingUnit if execRouteBase(unit) { t.Fatal("unit-backed routes are still excluded") } } // TestSyncExecWritesConfigFileForASidecarRoute is the end-to-end proof that // execRouteBase's widening actually reaches the file: a sidecar route // carries neither Socket nor Addr (sidecarRouteFor's contract — nothing // dials a sidecar), so execRouteBase alone is not sufficient to bring it // into SyncExec's resolution loop; inScope must recognize the shape too. // StartWorker (backend_exec.go's instanceEnv) reads exactly this file at // spawn, so a sidecar that never gets it here never gets its env at all. func TestSyncExecWritesConfigFileForASidecarRoute(t *testing.T) { ctx, s, reg, dd, priv, _ := execKit(t) pk, err := ParsePrivateKey(priv, "test") if err != nil { t.Fatalf("ParsePrivateKey: %v", err) } if err := SetEnv(ctx, s, "bdm", "eleven", map[string]string{"BOT_TOKEN": "shh"}); err != nil { t.Fatalf("SetEnv: %v", err) } if err := reg.Upsert(registry.Route{ Host: "bot.eleven.bdm.sidecar.internal", Kind: "sidecar", App: "eleven", Account: "bdm", Status: "ready", }); err != nil { t.Fatalf("Upsert: %v", err) } if err := SyncExec(ctx, reg, s, pk, dd, func(string) error { return nil }, "oncarlos.test", nil); err != nil { t.Fatalf("SyncExec: %v", err) } got, err := os.ReadFile(filepath.Join(dd, "bot.eleven.bdm.sidecar.internal.config.env")) if err != nil { t.Fatalf("sidecar never got a config.env file: %v", err) } if string(got) != "BOT_TOKEN=shh\n" { t.Fatalf("config env = %q, want BOT_TOKEN=shh", got) } } // TestSyncExecSidecarRouteIsNeverBounced pins the safety decision behind // sidecarShape in inScope: Bounce alone cannot stand in for a sidecar's // wake (nothing dials a worker the way a request dials an instance — see // inScope's doc comment), so an automatic bounce on every ordinary // config-sync pass would risk stranding an always-on sidecar or leaving a // hibernating one silent until its schedule next comes due. The bounce // callback must never be invoked for a sidecar route, on either the // initial write or a changed-content pass — restarting one on purpose is // `carlos restart -app X` (host/restart.go), which pairs the stop with an // explicit wake. func TestSyncExecSidecarRouteIsNeverBounced(t *testing.T) { ctx, s, reg, dd, priv, _ := execKit(t) pk, err := ParsePrivateKey(priv, "test") if err != nil { t.Fatalf("ParsePrivateKey: %v", err) } if err := SetEnv(ctx, s, "bdm", "eleven", map[string]string{"BOT_TOKEN": "one"}); err != nil { t.Fatalf("SetEnv: %v", err) } if err := reg.Upsert(registry.Route{ Host: "bot.eleven.bdm.sidecar.internal", Kind: "sidecar", App: "eleven", Account: "bdm", Status: "ready", }); err != nil { t.Fatalf("Upsert: %v", err) } var bounced []string bounce := func(h string) error { bounced = append(bounced, h); return nil } if err := SyncExec(ctx, reg, s, pk, dd, bounce, "oncarlos.test", nil); err != nil { t.Fatalf("SyncExec (initial write): %v", err) } if len(bounced) != 0 { t.Fatalf("bounced = %v, want none — Bounce alone cannot wake a worker back up", bounced) } if err := SetEnv(ctx, s, "bdm", "eleven", map[string]string{"BOT_TOKEN": "two"}); err != nil { t.Fatalf("SetEnv (change): %v", err) } if err := SyncExec(ctx, reg, s, pk, dd, bounce, "oncarlos.test", nil); err != nil { t.Fatalf("SyncExec (changed content): %v", err) } if len(bounced) != 0 { t.Fatalf("bounced = %v after a content change, want none", bounced) } got, err := os.ReadFile(filepath.Join(dd, "bot.eleven.bdm.sidecar.internal.config.env")) if err != nil { t.Fatalf("read config env: %v", err) } if string(got) != "BOT_TOKEN=two\n" { t.Fatalf("config env = %q, want BOT_TOKEN=two — the changed content, no header — the write path must still run", got) } } // TestExecConfigAppsIncludesSidecars (fix round 1, coordinator ruling): a // sidecar route must be in scope for the fast generation-poll loop exactly // as an instance is. Left unwidened, a sidecar-only app's config delivered // by a direct-bucket write — never bumping control/serial, exactly what the // pinfra scripts do — would fall back to the hourly floor while an instance // next to it converges within the fast loop's cadence: a predicate widened // in SyncExec's inScope and not in this sibling, where the symptom is not // an error but work happening late. func TestExecConfigAppsIncludesSidecars(t *testing.T) { routes := []registry.Route{ // In, and the whole point of this test: a SIDECAR-ONLY app — no // socket, no addr route at all — must still be polled. An earlier // version of this test paired the sidecar with an ordinary instance // of the same app, which made the assertion pass whether or not // ExecConfigApps actually looked at the sidecar route at all. {Host: "bot.eleven.bdm.sidecar.internal", Kind: "sidecar", App: "eleven", Account: "bdm", Status: "ready"}, // In BOTH via one dedup: a second app with a sidecar AND a // socket-exec instance must still report as ONE (account, app) pair. {Host: "bot.mixed.bdm.sidecar.internal", Kind: "sidecar", App: "mixed", Account: "bdm", Status: "ready"}, {Host: "inst.mixed.example.com", Kind: "instance", App: "mixed", Account: "bdm", Status: "ready", Socket: "/run/carlos/inst.sock"}, // Out, same exclusions execRouteBase already gives an instance: // Platform and a non-ready Status. {Host: "plat.bot.sidecar.internal", Kind: "sidecar", App: "platapp", Account: "bdm", Status: "ready", Platform: true}, {Host: "creating.bot.sidecar.internal", Kind: "sidecar", App: "creatingapp", Account: "bdm", Status: "creating"}, } got := ExecConfigApps(routes) want := [][2]string{{"bdm", "eleven"}, {"bdm", "mixed"}} if !slices.Equal(got, want) { t.Fatalf("ExecConfigApps = %v, want %v — a sidecar-only app must be polled by the fast loop too, and a mixed-shape app must dedupe to one entry", got, want) } } // The #344 guard, from the 2026-08-28 incident: the domains reconcile // mis-stamped a route's account, the config sync keyed the manifest under // the stamped account, found nothing, and DELETED the tenant's delivered // config — secrets included — bouncing the instance into a crash loop. An // absent manifest may remove the file only when the lookup account agrees // with the account the host itself names; a platform host's label // is not a moving part, the row's account column is. func TestSyncExecHoldsConfigWhenHostNamesADifferentAccount(t *testing.T) { ctx, s, reg, dd, priv, _ := execKit(t) pk, _ := ParsePrivateKey(priv, "test") if err := reg.Upsert(registry.Route{ Host: "birthday-alarm.bde.oncarlos.test", Kind: "instance", App: "birthday-alarm", Account: "bab", // the mis-stamp Status: "ready", Socket: "/run/carlos/birthday.sock", }); err != nil { t.Fatalf("Upsert: %v", err) } // A second, healthy group so the mass-removal guard (which needs EVERY // group absent) stays out of the way — this guard is per-route. if err := reg.Upsert(registry.Route{ Host: "inst.example.com", Kind: "instance", App: "eleven", Account: "bab", Status: "ready", Socket: "/run/carlos/inst.sock", }); err != nil { t.Fatalf("Upsert: %v", err) } if err := SetEnv(ctx, s, "bab", "eleven", map[string]string{"A": "1"}); err != nil { t.Fatalf("SetEnv: %v", err) } path := filepath.Join(dd, "birthday-alarm.bde.oncarlos.test.config.env") if err := os.WriteFile(path, []byte("BIRTHDAYALARM_INSTANCE_KEY=deadbeef\n"), 0o600); err != nil { t.Fatalf("stage: %v", err) } var bounced []string if err := SyncExec(ctx, reg, s, pk, dd, func(h string) error { bounced = append(bounced, h) return nil }, "oncarlos.test", nil); err != nil { t.Fatalf("SyncExec: %v", err) } if _, err := os.Stat(path); err != nil { t.Fatalf("delivered config removed under a mis-stamped account: %v", err) } for _, h := range bounced { if h == "birthday-alarm.bde.oncarlos.test" { t.Fatalf("held route was bounced: %v", bounced) } } // And the guard steps aside when the row and the host agree: the same // absent manifest under the RIGHT account is a real deletion. if err := reg.Upsert(registry.Route{ Host: "birthday-alarm.bab.oncarlos.test", Kind: "instance", App: "birthday-alarm", Account: "bab", Status: "ready", Socket: "/run/carlos/birthday2.sock", }); err != nil { t.Fatalf("Upsert: %v", err) } agreed := filepath.Join(dd, "birthday-alarm.bab.oncarlos.test.config.env") if err := os.WriteFile(agreed, []byte("OLD=1\n"), 0o600); err != nil { t.Fatalf("stage: %v", err) } if err := SyncExec(ctx, reg, s, pk, dd, func(string) error { return nil }, "oncarlos.test", nil); err != nil { t.Fatalf("SyncExec: %v", err) } if _, err := os.Stat(agreed); !os.IsNotExist(err) { t.Fatalf("agreeing host's stale file survived, stat err = %v", err) } } // The #344 guard is NOT a removal branch, and an overlay does not clear it // (task 7 fix round 1 — the brief's own reasoning, "a route with an overlay // never reaches a removal branch anyway," does not transfer to this guard). // A mis-stamp that makes the group's default-bundle READ land under the // wrong account prefix makes the instance layer's own WRITE // (ApplyInstance) land under that same wrong prefix too, so an overlay // resolving here under the mis-stamped account is exactly what a // mis-stamped fleet produces — not evidence the account column is safe to // act on. Sibling of TestSyncExecHoldsConfigWhenHostNamesADifferentAccount, // with an overlay added to the held route. func TestSyncExecHoldsConfigWhenHostNamesADifferentAccountEvenWithAnOverlay(t *testing.T) { ctx, s, reg, dd, priv, _ := execKit(t) pk, err := ParsePrivateKey(priv, "test") if err != nil { t.Fatalf("ParsePrivateKey: %v", err) } if err := reg.Upsert(registry.Route{ Host: "birthday-alarm.bde.oncarlos.test", Kind: "instance", App: "birthday-alarm", Account: "bab", // the mis-stamp Status: "ready", Socket: "/run/carlos/birthday.sock", }); err != nil { t.Fatalf("Upsert: %v", err) } // The overlay lands at apps/bab/birthday-alarm/instances-config/ — the // SAME mis-stamped prefix the group's absent default bundle was looked // up under, which is the whole point: a mangled account column mangles // both. if _, _, err := ApplyInstance(ctx, s, "bab", "birthday-alarm", "birthday-alarm.bde.oncarlos.test", map[string]Var{"TENANT": {Kind: "env", Value: "mis-stamped"}}, nil); err != nil { t.Fatalf("ApplyInstance: %v", err) } // A second, healthy group so the mass-removal guard (which needs EVERY // group absent) stays out of the way — this guard is per-route. if err := reg.Upsert(registry.Route{ Host: "inst.example.com", Kind: "instance", App: "eleven", Account: "bab", Status: "ready", Socket: "/run/carlos/inst.sock", }); err != nil { t.Fatalf("Upsert: %v", err) } if err := SetEnv(ctx, s, "bab", "eleven", map[string]string{"A": "1"}); err != nil { t.Fatalf("SetEnv: %v", err) } path := filepath.Join(dd, "birthday-alarm.bde.oncarlos.test.config.env") if err := os.WriteFile(path, []byte("BIRTHDAYALARM_INSTANCE_KEY=deadbeef\n"), 0o600); err != nil { t.Fatalf("stage: %v", err) } var bounced []string out := captureStdout(t, func() { if err := SyncExec(ctx, reg, s, pk, dd, func(h string) error { bounced = append(bounced, h) return nil }, "oncarlos.test", nil); err != nil { t.Fatalf("SyncExec: %v", err) } }) cur, err := os.ReadFile(path) if err != nil || string(cur) != "BIRTHDAYALARM_INSTANCE_KEY=deadbeef\n" { t.Fatalf("delivered config was replaced by the overlay under a mis-stamped account: %q %v", cur, err) } for _, h := range bounced { if h == "birthday-alarm.bde.oncarlos.test" { t.Fatalf("held route was bounced: %v", bounced) } } if !strings.Contains(out, "birthday-alarm.bde.oncarlos.test") || !strings.Contains(out, "#344") { t.Fatalf("the #344 hold did not fire for a route that has an overlay:\n%s", out) } } // TestSyncExecWritesConfigAckForSocketExecRoute is the review's finding 1 // (2026-09-07): syncExecHost DOES bounce a socket-exec route (bounce is // non-nil for it, only a tcp-exec-file/addr route gets nil), so its ack // must carry that bounce's own result — this pins Restarted true and // RelaySuffixes naming what was actually sealed, the same way syncHost's // unit-path ack does. func TestSyncExecWritesConfigAckForSocketExecRoute(t *testing.T) { ctx, s, reg, dd, _, _ := execKit(t) priv, pub := syncRelayTestKeys(t) rec := &relay.Record{Enabled: true, Incarnation: "inc00003", Regions: []string{"eu-west-1"}} masters := []relay.RegionMaster{{Region: "eu-west-1", Kid: "k1", Master: []byte("0123456789abcdef0123456789abcdef")}} wantSuffixes := sealRelayConfig(t, ctx, s, pub, "bab", "eleven", rec, masters) if err := reg.Upsert(registry.Route{ Host: "inst.example.com", Kind: "instance", App: "eleven", Account: "bab", Status: "ready", Socket: "/run/carlos/inst.sock", }); err != nil { t.Fatalf("Upsert: %v", err) } var acks []ConfigAck ack := func(a ConfigAck) { acks = append(acks, a) } bounce := func(h string) error { return nil } if err := SyncExec(ctx, reg, s, priv, dd, bounce, "", ack); err != nil { t.Fatalf("SyncExec: %v", err) } if len(acks) != 1 { t.Fatalf("acks = %d, want 1: %+v", len(acks), acks) } got := acks[0] if got.Shape != "socket-exec" { t.Fatalf("ack.Shape = %q, want socket-exec", got.Shape) } if !got.Restarted { t.Fatalf("ack.Restarted = false, want true — bounce succeeded and IS this shape's restart (spec §5 step 5)") } if got.RestartError != "" { t.Fatalf("ack.RestartError = %q, want empty", got.RestartError) } if got.RelaySuffixes["eu-west-1"] != wantSuffixes["eu-west-1"] { t.Fatalf("ack.RelaySuffixes[eu-west-1] = %q, want %q", got.RelaySuffixes["eu-west-1"], wantSuffixes["eu-west-1"]) } } // TestSyncExecWritesConfigAckForTCPExecFileRoute is the addr shape's other // half of the same ack path: bounce is nil for it (this platform never // restarts a process it does not own), so Restarted/RestartError must stay // false/empty even though the file itself landed and named real // identities. func TestSyncExecWritesConfigAckForTCPExecFileRoute(t *testing.T) { ctx, s, reg, dd, _, _ := execKit(t) priv, pub := syncRelayTestKeys(t) if err := SetExecDelivery(ctx, s, "bab", "titogo", ExecDeliveryFile); err != nil { t.Fatalf("SetExecDelivery: %v", err) } rec := &relay.Record{Enabled: true, Incarnation: "inc00004", Regions: []string{"eu-west-1"}} masters := []relay.RegionMaster{{Region: "eu-west-1", Kid: "k1", Master: []byte("fedcba9876543210fedcba9876543210")}} wantSuffixes := sealRelayConfig(t, ctx, s, pub, "bab", "titogo", rec, masters) if err := reg.Upsert(registry.Route{ Host: "demo.go.tito.io", Kind: "instance", App: "titogo", Account: "bab", Status: "ready", Addr: "127.0.0.1:8444", }); err != nil { t.Fatalf("Upsert: %v", err) } var acks []ConfigAck ack := func(a ConfigAck) { acks = append(acks, a) } if err := SyncExec(ctx, reg, s, priv, dd, func(string) error { return nil }, "", ack); err != nil { t.Fatalf("SyncExec: %v", err) } if len(acks) != 1 { t.Fatalf("acks = %d, want 1: %+v", len(acks), acks) } got := acks[0] if got.Shape != "tcp-exec-file" { t.Fatalf("ack.Shape = %q, want tcp-exec-file", got.Shape) } if got.Restarted { t.Fatalf("ack.Restarted = true, want false — this platform never restarts this shape's process") } if got.RestartError != "" { t.Fatalf("ack.RestartError = %q, want empty", got.RestartError) } if got.RelaySuffixes["eu-west-1"] != wantSuffixes["eu-west-1"] { t.Fatalf("ack.RelaySuffixes[eu-west-1] = %q, want %q", got.RelaySuffixes["eu-west-1"], wantSuffixes["eu-west-1"]) } } // With overlap swaps armed, config sync reports each delivered revision to // the scheduler before its (no-op) restart, and never for a shape the // platform does not restart (zero-downtime spec §8.1). func TestSyncExecReportsTheDeliveredRevision(t *testing.T) { ctx, s, reg, dd, priv, _ := execKit(t) pk, err := ParsePrivateKey(priv, "test") if err != nil { t.Fatal(err) } if err := SetEnv(ctx, s, "bab", "eleven", map[string]string{"LOG_LEVEL": "debug"}); err != nil { t.Fatal(err) } for _, rt := range []registry.Route{ {Host: "inst.example.com", Kind: "instance", App: "eleven", Account: "bab", Status: "ready", Socket: "/run/carlos/inst.sock"}, {Host: "bot.eleven.bab.sidecar.internal", Kind: "sidecar", App: "eleven", Account: "bab", Status: "ready"}, } { if err := reg.Upsert(rt); err != nil { t.Fatal(err) } } var order, revs []string cb := Callbacks{ Restart: func(h string) error { order = append(order, "restart:"+h); return nil }, Delivered: func(h, rev string) { order = append(order, "delivered:"+h); revs = append(revs, rev) }, } if err := SyncExecWithCallbacks(ctx, reg, s, pk, dd, "", cb); err != nil { t.Fatal(err) } want := []string{"delivered:inst.example.com", "restart:inst.example.com"} if !slices.Equal(order, want) { t.Fatalf("order = %v, want %v (no report for the sidecar)", order, want) } if !strings.HasPrefix(revs[0], "sha256:") { t.Fatalf("rev = %q, want the rendered identity", revs[0]) } order = nil if err := SyncExecWithCallbacks(ctx, reg, s, pk, dd, "", cb); err != nil { t.Fatal(err) } if len(order) != 0 { t.Fatalf("an unchanged file reported %v", order) } if err := SetEnv(ctx, s, "bab", "eleven", map[string]string{"LOG_LEVEL": "info"}); err != nil { t.Fatal(err) } if err := SyncExecWithCallbacks(ctx, reg, s, pk, dd, "", cb); err != nil { t.Fatal(err) } if len(revs) != 2 || revs[1] == revs[0] { t.Fatalf("revs = %v, want a new revision for new content", revs) } } // Final review F7 (1) and (2): with Managed set, the pass splits as the // scheduler does. A managed socket route gets Restart and is reported to // Delivered. A socket route Managed says no to is restarted by Unmanaged, // as config sync restarts it with the flag off, is not reported, and its // ack says it was restarted. A managed native sidecar is reported though // nothing here restarts it (the scheduler's config attempt does); an // unmanaged one is neither reported nor restarted, as before. func TestSyncExecSplitsDeliveryByManagement(t *testing.T) { ctx, s, reg, dd, priv, _ := execKit(t) pk, err := ParsePrivateKey(priv, "test") if err != nil { t.Fatal(err) } if err := SetEnv(ctx, s, "bab", "eleven", map[string]string{"LOG_LEVEL": "debug"}); err != nil { t.Fatal(err) } const ( mInst = "m.example.com" uInst = "u.example.com" mBot = "bot.eleven.bab.sidecar.internal" uBot = "bot2.eleven.bab.sidecar.internal" ) for _, rt := range []registry.Route{ {Host: mInst, Kind: "instance", App: "eleven", Account: "bab", Status: "ready", Socket: "/run/carlos/m.sock"}, {Host: uInst, Kind: "instance", App: "eleven", Account: "bab", Status: "ready", Socket: "/run/carlos/u.sock"}, {Host: mBot, Kind: "sidecar", App: "eleven", Account: "bab", Status: "ready"}, {Host: uBot, Kind: "sidecar", App: "eleven", Account: "bab", Status: "ready"}, } { if err := reg.Upsert(rt); err != nil { t.Fatal(err) } } calls := map[string][]string{} acks := map[string]ConfigAck{} cb := Callbacks{ Restart: func(h string) error { calls[h] = append(calls[h], "restart"); return nil }, Unmanaged: func(h string) error { calls[h] = append(calls[h], "unmanaged"); return nil }, Delivered: func(h, _ string) { calls[h] = append(calls[h], "delivered") }, Ack: func(a ConfigAck) { acks[a.Host] = a }, Managed: func(rt registry.Route) bool { return rt.Host == mInst || rt.Host == mBot }, } if err := SyncExecWithCallbacks(ctx, reg, s, pk, dd, "", cb); err != nil { t.Fatal(err) } want := map[string][]string{ mInst: {"delivered", "restart"}, uInst: {"unmanaged"}, mBot: {"delivered"}, } if !reflect.DeepEqual(calls, want) { t.Fatalf("calls = %v, want %v", calls, want) } if a := acks[uInst]; !a.Restarted || a.RestartError != "" { t.Fatalf("unmanaged ack = %+v, want today's: restarted", a) } if a := acks[mBot]; a.Restarted || a.RestartError != "" { t.Fatalf("managed sidecar ack = %+v: nothing here restarted it", a) } // Managed nil: every socket route gets Restart and Delivered, and no // sidecar is reported, exactly as before. if err := SetEnv(ctx, s, "bab", "eleven", map[string]string{"LOG_LEVEL": "info"}); err != nil { t.Fatal(err) } calls = map[string][]string{} cb.Managed = nil if err := SyncExecWithCallbacks(ctx, reg, s, pk, dd, "", cb); err != nil { t.Fatal(err) } want = map[string][]string{ mInst: {"delivered", "restart"}, uInst: {"delivered", "restart"}, } if !reflect.DeepEqual(calls, want) { t.Fatalf("with Managed nil, calls = %v, want %v", calls, want) } } // Final review F7 (3): a restart that answers ErrDeliveredOnWake acks the // revision as delivered on wake (Hibernating), the unit pass's marker, with // no restart claimed and no error, on both the write and the removal paths. func TestSyncExecAcksDeliveredOnWake(t *testing.T) { ctx, s, reg, dd, priv, _ := execKit(t) pk, err := ParsePrivateKey(priv, "test") if err != nil { t.Fatal(err) } if err := SetEnv(ctx, s, "bab", "eleven", map[string]string{"LOG_LEVEL": "debug"}); err != nil { t.Fatal(err) } host := "asleep.example.com" if err := reg.Upsert(registry.Route{Host: host, Kind: "instance", App: "eleven", Account: "bab", Status: "ready", Socket: "/run/carlos/asleep.sock", Hibernate: true}); err != nil { t.Fatal(err) } var acks []ConfigAck restarts := 0 cb := Callbacks{ Restart: func(string) error { restarts++; return ErrDeliveredOnWake }, Ack: func(a ConfigAck) { acks = append(acks, a) }, } if err := SyncExecWithCallbacks(ctx, reg, s, pk, dd, "", cb); err != nil { t.Fatal(err) } if restarts != 1 || len(acks) != 1 { t.Fatalf("restarts %d, acks %+v", restarts, acks) } if a := acks[0]; !a.Hibernating || a.Restarted || a.RestartError != "" { t.Fatalf("ack = %+v, want delivered on wake: Hibernating, not restarted, no error", a) } } // keyReadStore records every key and prefix a pass reads through it. type keyReadStore struct { release.Store mu sync.Mutex read []string } func (s *keyReadStore) note(k string) { s.mu.Lock(); s.read = append(s.read, k); s.mu.Unlock() } func (s *keyReadStore) Get(ctx context.Context, key string) ([]byte, string, error) { s.note(key) return s.Store.Get(ctx, key) } func (s *keyReadStore) List(ctx context.Context, prefix string) ([]string, error) { s.note(prefix) return s.Store.List(ctx, prefix) } func (s *keyReadStore) ListDirs(ctx context.Context, prefix string) ([]string, error) { s.note(prefix) return s.Store.ListDirs(ctx, prefix) } // TestSyncExecWithRouteFilter pins the scoped config pass the agent runs // when the change index says one app moved (S3 change-index design §4.5): // a route the filter leaves out is not in scope, so nothing is read for it, // its file is left exactly as it is (neither rewritten nor reaped), and the // routes the filter keeps converge as on a full pass. func TestSyncExecWithRouteFilter(t *testing.T) { t.Parallel() ctx, base, reg, dd, priv, _ := execKit(t) pk, err := ParsePrivateKey(priv, "test") if err != nil { t.Fatalf("ParsePrivateKey: %v", err) } for _, app := range []string{"shop", "cafe"} { if err := SetEnv(ctx, base, "bab", app, map[string]string{"APP": app}); err != nil { t.Fatalf("SetEnv %s: %v", app, err) } } if err := SetExecDelivery(ctx, base, "bab", "cafe", ExecDeliveryFile); err != nil { t.Fatalf("SetExecDelivery: %v", err) } for _, rt := range []registry.Route{ {Host: "shop.example.com", Kind: "instance", App: "shop", Account: "bab", Status: "ready", Socket: "/run/carlos/shop.sock"}, {Host: "cafe.example.com", Kind: "instance", App: "cafe", Account: "bab", Status: "ready", Socket: "/run/carlos/cafe.sock"}, // The addr shape: its opt-in is read before the resolution loop. {Host: "cafe-addr.example.com", Kind: "instance", App: "cafe", Account: "bab", Status: "ready", Addr: "127.0.0.1:8444"}, } { if err := reg.Upsert(rt); err != nil { t.Fatalf("Upsert %s: %v", rt.Host, err) } } stale := filepath.Join(dd, "cafe.example.com.config.env") if err := os.WriteFile(stale, []byte("OLD=1\n"), 0o600); err != nil { t.Fatal(err) } s := &keyReadStore{Store: base} keep := WithRouteFilter(func(rt registry.Route) bool { return rt.App != "cafe" }) var bounced []string if err := SyncExec(ctx, reg, s, pk, dd, func(h string) error { bounced = append(bounced, h); return nil }, "", nil, keep); err != nil { t.Fatalf("SyncExec: %v", err) } var shopRead bool for _, k := range s.read { if strings.HasPrefix(k, "apps/bab/cafe/") { t.Errorf("the filtered pass read %s, under an app the filter left out", k) } shopRead = shopRead || strings.HasPrefix(k, "apps/bab/shop/") } if !shopRead { t.Fatalf("the pass read nothing for shop (%v): the witness pinned nothing", s.read) } if got, err := os.ReadFile(filepath.Join(dd, "shop.example.com.config.env")); err != nil || string(got) != "APP=shop\n" { t.Fatalf("shop's file = %q, %v; want APP=shop", got, err) } if got, err := os.ReadFile(stale); err != nil || string(got) != "OLD=1\n" { t.Fatalf("cafe's file = %q, %v; a route the filter leaves out keeps its file as it is", got, err) } if _, err := os.Stat(filepath.Join(dd, "cafe-addr.example.com.config.env")); !os.IsNotExist(err) { t.Fatalf("cafe's addr route got a file from a pass that left it out: %v", err) } if !slices.Equal(bounced, []string{"shop.example.com"}) { t.Fatalf("bounced %v, want only shop.example.com", bounced) } }