package appconfig import ( "bytes" "context" "crypto/ecdh" "encoding/json" "errors" "os" "path/filepath" "strings" "sync" "testing" "github.com/carlosframework/platform/internal/registry" "github.com/carlosframework/platform/internal/release" ) // putInstanceRaw writes an instance-layer object without going through the // package's own writer, which does not exist until task 2. Tests in this // file that only READ the layer use it, so task 1 can be verified on its // own. func putInstanceRaw(t *testing.T, ctx context.Context, s release.Store, acct, app, host string, m *Manifest) string { t.Helper() body, err := json.MarshalIndent(m, "", " ") if err != nil { t.Fatalf("marshal instance manifest: %v", err) } etag, err := s.Put(ctx, InstanceConfigKey(acct, app, host), body, "") if err != nil { t.Fatalf("put instance manifest: %v", err) } return etag } // The layer sits under the app's own prefix, beside config.json and // environments/, so release.PurgeApp's prefix sweep reaches it and a purged // app leaves no instance overlay behind holding sealed secrets. func TestInstanceConfigKeysSitUnderTheAppsOwnPrefix(t *testing.T) { if got, want := InstancesConfigPrefix("bab", "titogo"), "apps/bab/titogo/instances-config/"; got != want { t.Fatalf("InstancesConfigPrefix = %q, want %q", got, want) } if got, want := InstanceConfigKey("bab", "titogo", "book.customer.example.com"), "apps/bab/titogo/instances-config/book.customer.example.com.json"; got != want { t.Fatalf("InstanceConfigKey = %q, want %q", got, want) } } // A host becomes a bucket key segment here, so the one gate every caller // passes through is where a separator, a traversal or an empty name is // refused. The shape is a bare lowercase hostname, the same thing a route // row's Host column holds and the same thing sync.go's own path guard // (rt.Host != filepath.Base(rt.Host)) already insists on before it derives // a file name from one. func TestValidInstanceHostRefusesAnythingThatIsNotABareHost(t *testing.T) { for _, bad := range []string{ "", ".", "..", "a/b", "../etc", "Book.example.com", "book..example.com", "-lead.example.com", "trail-.example.com", "book.example.com/", strings.Repeat("a", 254), } { if ValidInstanceHost(bad) { t.Errorf("ValidInstanceHost(%q) = true", bad) } } for _, good := range []string{"book.customer.example.com", "inst1", "a-b.c-d.example.com", "127-0-0-1.example.com"} { if !ValidInstanceHost(good) { t.Errorf("ValidInstanceHost(%q) = false", good) } } } // The reader answers three ways, and the caller needs all three apart: a // manifest and the etag it was read at, "no object" without an error (an // instance with no overlay is the normal case, not a fault), and an error // for anything else. The etag is what the delivery ack carries (design §6.5), // so a read that dropped it would leave the agent unable to say which // version of the layer it rendered. func TestLoadInstanceConfigReadsTheLayerAndItsETag(t *testing.T) { ctx := context.Background() s := release.NewDirStore(t.TempDir()) m, etag, found, err := LoadInstanceConfig(ctx, s, "bab", "titogo", "book.customer.example.com") if err != nil || found || m != nil || etag != "" { t.Fatalf("absent overlay = %+v %q %v %v, want (nil, \"\", false, nil)", m, etag, found, err) } want := putInstanceRaw(t, ctx, s, "bab", "titogo", "book.customer.example.com", &Manifest{Schema: 1, App: "titogo", Vars: map[string]Var{"TENANT_SMTP_USER": {Kind: "env", Value: "book"}}}) m, etag, found, err = LoadInstanceConfig(ctx, s, "bab", "titogo", "book.customer.example.com") if err != nil || !found || m == nil { t.Fatalf("present overlay = %+v %v %v", m, found, err) } if etag == "" || etag != want { t.Fatalf("etag = %q, want the object's own %q", etag, want) } if m.Vars["TENANT_SMTP_USER"].Value != "book" { t.Fatalf("overlay lost its var: %+v", m.Vars) } // The same newer-schema refusal every other bundle read makes: acting on // a manifest whose unknown fields json.Unmarshal has already dropped // would silently discard whatever they meant. putInstanceRaw(t, ctx, s, "bab", "titogo", "newer.example.com", &Manifest{Schema: 2, App: "titogo"}) if _, _, _, err := LoadInstanceConfig(ctx, s, "bab", "titogo", "newer.example.com"); err == nil { t.Fatal("a schema-2 overlay was accepted, want a refusal naming the upgrade") } // A host that is not a bare hostname never reaches the bucket at all. if _, _, _, err := LoadInstanceConfig(ctx, s, "bab", "titogo", "../config"); err == nil { t.Fatal("LoadInstanceConfig accepted a traversal host") } } // The write door. It returns the etag it wrote at, which is the whole // mechanism behind the delivery check (design §6.5): the caller keeps that // string and compares it with what an agent's ack says it rendered. func TestApplyInstanceWritesTheLayerAndReturnsItsETag(t *testing.T) { ctx := context.Background() s := release.NewDirStore(t.TempDir()) removed, etag, err := ApplyInstance(ctx, s, "bab", "titogo", "book.customer.example.com", map[string]Var{"TENANT_SMTP_USER": {Kind: "env", Value: "book"}}, nil) if err != nil { t.Fatalf("ApplyInstance: %v", err) } if removed != 0 { t.Fatalf("removed = %d on a create, want 0", removed) } if etag == "" { t.Fatal("ApplyInstance returned an empty etag, so nothing can check delivery against it") } m, readETag, found, err := LoadInstanceConfig(ctx, s, "bab", "titogo", "book.customer.example.com") if err != nil || !found { t.Fatalf("LoadInstanceConfig after the write = %+v %v %v", m, found, err) } if readETag != etag { t.Fatalf("the reader saw etag %q, the writer returned %q — the delivery check compares these two", readETag, etag) } if m.Schema != 1 || m.App != "titogo" || m.Vars["TENANT_SMTP_USER"].Value != "book" { t.Fatalf("the written overlay is not a schema-1 manifest of this app: %+v", m) } // A second write moves the etag, which is what makes a stale ack // detectable at all. _, etag2, err := ApplyInstance(ctx, s, "bab", "titogo", "book.customer.example.com", map[string]Var{"TENANT_SMTP_USER": {Kind: "env", Value: "book2"}}, nil) if err != nil { t.Fatalf("ApplyInstance (second): %v", err) } if etag2 == etag { t.Fatalf("etag did not move on a changed write: %q", etag2) } // Unset reports how many of the keys were actually there, the same // contract every other bundle's unset has. removed, _, err = ApplyInstance(ctx, s, "bab", "titogo", "book.customer.example.com", nil, []string{"TENANT_SMTP_USER", "NEVER_SET"}) if err != nil || removed != 1 { t.Fatalf("unset = %d %v, want 1 and no error", removed, err) } // The var-name and var-value rules are the bundle's, not this layer's: // applyAt refuses before anything reaches the bucket. if _, _, err := ApplyInstance(ctx, s, "bab", "titogo", "book.customer.example.com", map[string]Var{"not a name": {Kind: "env", Value: "x"}}, nil); err == nil { t.Fatal("an invalid var name was accepted into the instance layer") } if _, _, err := ApplyInstance(ctx, s, "bab", "titogo", "../config", nil, nil); err == nil { t.Fatal("ApplyInstance accepted a traversal host") } } // An unset against an overlay that does not exist is refused rather than // minting an empty one, the same refusal applyAt already makes for the // other two bundles and for the same reason: an empty manifest that EXISTS // means "configured, then fully unset" by this package's contract, which is // a state the sweeps converge a file to. func TestApplyInstanceRefusesUnsetOnlyAgainstAMissingOverlay(t *testing.T) { ctx := context.Background() s := release.NewDirStore(t.TempDir()) _, _, err := ApplyInstance(ctx, s, "bab", "titogo", "book.customer.example.com", nil, []string{"K"}) if err == nil { t.Fatal("unsetting from a nonexistent overlay was turned into a create") } if !strings.Contains(err.Error(), "book.customer.example.com") { t.Fatalf("the refusal does not name the instance: %v", err) } } // Writing the layer bumps the app's generation object, the way // SetExecDeliveryAt does. It has to: ConfigRouteFingerprints digests // registry columns only (routefingerprint.go:58) and cannot see a // bucket-side change, so without the bump an overlay would wait out the // agent's hourly floor with nothing to signal it. func TestWritingTheInstanceLayerBumpsTheGeneration(t *testing.T) { ctx := context.Background() s := release.NewDirStore(t.TempDir()) if err := SetEnv(ctx, s, "bab", "titogo", map[string]string{"LOG_LEVEL": "debug"}); err != nil { t.Fatalf("SetEnv: %v", err) } genETag := func() string { t.Helper() _, etag, err := s.Get(ctx, GenKey("bab", "titogo")) if err != nil { t.Fatalf("read generation: %v", err) } return etag } before := genETag() if _, _, err := ApplyInstance(ctx, s, "bab", "titogo", "book.customer.example.com", map[string]Var{"TENANT_SMTP_USER": {Kind: "env", Value: "book"}}, nil); err != nil { t.Fatalf("ApplyInstance: %v", err) } afterWrite := genETag() if afterWrite == before { t.Fatal("writing the instance layer did not bump the generation, so the fast loop would never see it") } if err := DeleteInstanceConfig(ctx, s, "bab", "titogo", "book.customer.example.com"); err != nil { t.Fatalf("DeleteInstanceConfig: %v", err) } if genETag() == afterWrite { t.Fatal("removing the instance layer did not bump the generation, so the route would keep its last merge") } if _, _, found, err := LoadInstanceConfig(ctx, s, "bab", "titogo", "book.customer.example.com"); err != nil || found { t.Fatalf("the overlay survived its delete: found=%v err=%v", found, err) } // Deleting again is a no-op success, so a retried teardown converges. if err := DeleteInstanceConfig(ctx, s, "bab", "titogo", "book.customer.example.com"); err != nil { t.Fatalf("second DeleteInstanceConfig: %v", err) } } // The precedence the design states: default < environment < instance // (§6.2). Three layers, each winning per key over the one below it, and the // instance layer's etag handed back so the caller can later check what was // delivered. func TestLoadMergedHostResolvesDefaultThenEnvironmentThenInstance(t *testing.T) { ctx := context.Background() s := release.NewDirStore(t.TempDir()) if err := SetEnv(ctx, s, "bab", "titogo", map[string]string{ "FROM_DEFAULT": "default", "FROM_ENV": "default", "FROM_INSTANCE": "default"}); err != nil { t.Fatalf("SetEnv: %v", err) } if _, err := ApplyEnvironment(ctx, s, "bab", "titogo", "prod", map[string]Var{ "FROM_ENV": {Kind: "env", Value: "environment"}, "FROM_INSTANCE": {Kind: "env", Value: "environment"}}, nil); err != nil { t.Fatalf("ApplyEnvironment: %v", err) } _, want, err := ApplyInstance(ctx, s, "bab", "titogo", "book.customer.example.com", map[string]Var{ "FROM_INSTANCE": {Kind: "env", Value: "instance"}}, nil) if err != nil { t.Fatalf("ApplyInstance: %v", err) } m, etag, err := LoadMergedHost(ctx, s, "bab", "titogo", "prod", "book.customer.example.com") if err != nil { t.Fatalf("LoadMergedHost: %v", err) } if etag != want { t.Fatalf("etag = %q, want the overlay's own %q", etag, want) } for k, wantVal := range map[string]string{ "FROM_DEFAULT": "default", "FROM_ENV": "environment", "FROM_INSTANCE": "instance"} { if got := m.Vars[k].Value; got != wantVal { t.Errorf("%s = %q, want %q", k, got, wantVal) } } // A sibling with no overlay of its own gets the two-layer merge and an // empty etag, which is what says "this host has no overlay" rather than // "this agent did not render one". m2, etag2, err := LoadMergedHost(ctx, s, "bab", "titogo", "prod", "other.customer.example.com") if err != nil { t.Fatalf("LoadMergedHost (sibling): %v", err) } if etag2 != "" { t.Fatalf("a host with no overlay reported etag %q", etag2) } if m2.Vars["FROM_INSTANCE"].Value != "environment" { t.Fatalf("the sibling saw another instance's value: %+v", m2.Vars) } } // With no overlay, LoadMergedHost is LoadMerged, to the field. That matters // for the unbound case in particular: LoadMerged returns the default bundle // OBJECT there, exec_delivery and all, and a merge that quietly rebuilt it // would strip a flag other code reads off the same manifest. func TestLoadMergedHostWithNoOverlayIsExactlyLoadMerged(t *testing.T) { ctx := context.Background() s := release.NewDirStore(t.TempDir()) if err := SetEnv(ctx, s, "bab", "titogo", map[string]string{"K": "v"}); err != nil { t.Fatalf("SetEnv: %v", err) } if err := SetExecDelivery(ctx, s, "bab", "titogo", ExecDeliveryFile); err != nil { t.Fatalf("SetExecDelivery: %v", err) } want, err := LoadMerged(ctx, s, "bab", "titogo", "") if err != nil { t.Fatalf("LoadMerged: %v", err) } got, etag, err := LoadMergedHost(ctx, s, "bab", "titogo", "", "book.customer.example.com") if err != nil || etag != "" { t.Fatalf("LoadMergedHost = %q %v", etag, err) } if got.ExecDelivery != want.ExecDelivery || got.Vars["K"] != want.Vars["K"] || len(got.Vars) != len(want.Vars) { t.Fatalf("LoadMergedHost diverged from LoadMerged with no overlay: %+v vs %+v", got, want) } } // exec_delivery is never read from this layer (the ruling at // appconfig.go:66-71). An overlay that carries the field does not put it in // the merged view, so nothing downstream can act on it. func TestInstanceLayerNeverCarriesExecDelivery(t *testing.T) { ctx := context.Background() s := release.NewDirStore(t.TempDir()) if err := SetEnv(ctx, s, "bab", "titogo", map[string]string{"K": "v"}); err != nil { t.Fatalf("SetEnv: %v", err) } putInstanceRaw(t, ctx, s, "bab", "titogo", "book.customer.example.com", &Manifest{ Schema: 1, App: "titogo", Vars: map[string]Var{"OWN": {Kind: "env", Value: "1"}}, ExecDelivery: ExecDeliveryFile}) m, _, err := LoadMergedHost(ctx, s, "bab", "titogo", "", "book.customer.example.com") if err != nil { t.Fatalf("LoadMergedHost: %v", err) } if m.ExecDelivery != "" { t.Fatalf("exec_delivery = %q: the instance layer must never decide delivery", m.ExecDelivery) } if m.Vars["OWN"].Value != "1" { t.Fatalf("the overlay's vars were dropped along with its flag: %+v", m.Vars) } } // A route bound to a bundle that does not exist is still an ERROR, overlay // or no overlay. The binding asserted the bundle exists; an overlay does not // make that assertion true, and answering with a partial merge would hand a // live route a file built from a binding nobody honoured. func TestLoadMergedHostStillRefusesAMissingBoundBundle(t *testing.T) { ctx := context.Background() s := release.NewDirStore(t.TempDir()) if err := SetEnv(ctx, s, "bab", "titogo", map[string]string{"K": "v"}); err != nil { t.Fatalf("SetEnv: %v", err) } if _, _, err := ApplyInstance(ctx, s, "bab", "titogo", "book.customer.example.com", map[string]Var{"OWN": {Kind: "env", Value: "1"}}, nil); err != nil { t.Fatalf("ApplyInstance: %v", err) } if _, _, err := LoadMergedHost(ctx, s, "bab", "titogo", "gone", "book.customer.example.com"); err == nil { t.Fatal("a missing bound bundle resolved anyway because the host had an overlay") } } // The index the sweeps use: one listing per app answers "which of this // app's instances have an overlay at all", so a pass costs a Get only for // the instances that do. func TestInstanceOverlaysListsOnlyThisAppsHosts(t *testing.T) { ctx := context.Background() s := release.NewDirStore(t.TempDir()) have, err := instanceOverlays(ctx, s, "bab", "titogo") if err != nil || len(have) != 0 { t.Fatalf("empty app = %v %v, want an empty set and no error", have, err) } for _, host := range []string{"book.customer.example.com", "cart.customer.example.com"} { if _, _, err := ApplyInstance(ctx, s, "bab", "titogo", host, map[string]Var{"OWN": {Kind: "env", Value: host}}, nil); err != nil { t.Fatalf("ApplyInstance %s: %v", host, err) } } if _, _, err := ApplyInstance(ctx, s, "bab", "eleven", "other.example.com", map[string]Var{"OWN": {Kind: "env", Value: "x"}}, nil); err != nil { t.Fatalf("ApplyInstance (other app): %v", err) } have, err = instanceOverlays(ctx, s, "bab", "titogo") if err != nil { t.Fatalf("instanceOverlays: %v", err) } if len(have) != 2 || !have["book.customer.example.com"] || !have["cart.customer.example.com"] { t.Fatalf("instanceOverlays = %v, want just this app's two hosts", have) } if have["other.example.com"] { t.Fatal("another app's overlay leaked into this app's index") } } // countingStore counts the reads and writes a pass makes, per key, so a // test can pin the COST of a feature and not just its output. The instance // layer's promise is one extra listing per app and one extra Get per // instance that actually has an overlay, which is a claim only a counter // can keep honest. type countingStore struct { release.Store mu sync.Mutex gets map[string]int lists map[string]int puts map[string]int } func newCountingStore(s release.Store) *countingStore { return &countingStore{Store: s, gets: map[string]int{}, lists: map[string]int{}, puts: map[string]int{}} } func (c *countingStore) Get(ctx context.Context, key string) ([]byte, string, error) { c.mu.Lock() c.gets[key]++ c.mu.Unlock() return c.Store.Get(ctx, key) } func (c *countingStore) List(ctx context.Context, prefix string) ([]string, error) { c.mu.Lock() c.lists[prefix]++ c.mu.Unlock() return c.Store.List(ctx, prefix) } func (c *countingStore) Put(ctx context.Context, key string, body []byte, ifMatch string) (string, error) { c.mu.Lock() c.puts[key]++ c.mu.Unlock() return c.Store.Put(ctx, key, body, ifMatch) } func (c *countingStore) n(m map[string]int, k string) int { c.mu.Lock() defer c.mu.Unlock() return m[k] } // total sums every count in m, so a test can assert "this pass wrote // nothing to the release store at all" without naming every key it might // have touched. func (c *countingStore) total(m map[string]int) int { c.mu.Lock() defer c.mu.Unlock() n := 0 for _, v := range m { n += v } return n } // syncInstanceKit is the unit-path fixture these tests share: a store, a // registry, an env dir and a parsed key pair. func syncInstanceKit(t *testing.T) (context.Context, release.Store, *registry.Registry, string, *ecdh.PrivateKey) { t.Helper() ctx := context.Background() s := release.NewDirStore(t.TempDir()) reg := testRegistry(t) priv, _, err := GenerateKey() if err != nil { t.Fatalf("GenerateKey: %v", err) } privKey, err := ParsePrivateKey(priv, "test") if err != nil { t.Fatalf("ParsePrivateKey: %v", err) } return ctx, s, reg, t.TempDir(), privKey } func unitRoute(t *testing.T, reg *registry.Registry, host, app, acct, env string) { t.Helper() if err := reg.Upsert(registry.Route{ Host: host, Kind: "instance", App: app, Account: acct, Status: "ready", Environment: env, Backing: registry.BackingUnit, }); err != nil { t.Fatalf("Upsert %s: %v", host, err) } } // The delivery this layer exists for: one instance of an app gets a value // its siblings never see. The sibling's file is the app merge, untouched. func TestSyncRendersTheInstanceOverlayForItsOwnRouteOnly(t *testing.T) { ctx, s, reg, envDir, priv := syncInstanceKit(t) if err := SetEnv(ctx, s, "bab", "titogo", map[string]string{"APP_WIDE": "yes", "TENANT": "none"}); err != nil { t.Fatalf("SetEnv: %v", err) } if _, _, err := ApplyInstance(ctx, s, "bab", "titogo", "book.example.com", map[string]Var{"TENANT": {Kind: "env", Value: "book"}}, nil); err != nil { t.Fatalf("ApplyInstance: %v", err) } unitRoute(t, reg, "book.example.com", "titogo", "bab", "") unitRoute(t, reg, "cart.example.com", "titogo", "bab", "") if err := Sync(ctx, reg, s, priv, envDir, func(string) error { return nil }, nil); err != nil { t.Fatalf("Sync: %v", err) } book, err := os.ReadFile(filepath.Join(envDir, "book.example.com.env")) if err != nil { t.Fatalf("read book env: %v", err) } if !bytes.Contains(book, []byte("TENANT=book\n")) || !bytes.Contains(book, []byte("APP_WIDE=yes\n")) { t.Fatalf("the overlaid route's file is wrong:\n%s", book) } cart, err := os.ReadFile(filepath.Join(envDir, "cart.example.com.env")) if err != nil { t.Fatalf("read cart env: %v", err) } if bytes.Contains(cart, []byte("book")) || !bytes.Contains(cart, []byte("TENANT=none\n")) { t.Fatalf("a sibling saw another instance's config:\n%s", cart) } // The sweep and the single-shot API must agree about what a host gets. // The sweeps resolve through the group cache and never call // LoadMergedHost, so this is the only thing holding the two together. want, _, err := LoadMergedHost(ctx, s, "bab", "titogo", "", "book.example.com") if err != nil { t.Fatalf("LoadMergedHost: %v", err) } rendered, err := RenderEnv(want, priv, Binding{Account: "bab", App: "titogo"}) if err != nil { t.Fatalf("RenderEnv: %v", err) } if !bytes.Contains(book, rendered) { t.Fatalf("the pass and LoadMergedHost disagree:\npass:\n%s\nLoadMergedHost:\n%s", book, rendered) } } // The cost. One listing per app, a Get for the instance that has an // overlay, and NOTHING for the instance that does not. func TestSyncCostsOneListingPerAppAndOneGetPerOverlay(t *testing.T) { ctx, base, reg, envDir, priv := syncInstanceKit(t) if err := SetEnv(ctx, base, "bab", "titogo", map[string]string{"APP_WIDE": "yes"}); err != nil { t.Fatalf("SetEnv: %v", err) } if _, _, err := ApplyInstance(ctx, base, "bab", "titogo", "book.example.com", map[string]Var{"TENANT": {Kind: "env", Value: "book"}}, nil); err != nil { t.Fatalf("ApplyInstance: %v", err) } unitRoute(t, reg, "book.example.com", "titogo", "bab", "") unitRoute(t, reg, "cart.example.com", "titogo", "bab", "") unitRoute(t, reg, "desk.example.com", "titogo", "bab", "") s := newCountingStore(base) if err := Sync(ctx, reg, s, priv, envDir, func(string) error { return nil }, nil); err != nil { t.Fatalf("Sync: %v", err) } if got := s.n(s.lists, InstancesConfigPrefix("bab", "titogo")); got != 1 { t.Fatalf("instance overlay listings = %d, want exactly 1 for three routes of one app", got) } if got := s.n(s.gets, InstanceConfigKey("bab", "titogo", "book.example.com")); got != 1 { t.Fatalf("overlay Gets for book = %d, want 1", got) } for _, host := range []string{"cart.example.com", "desk.example.com"} { if got := s.n(s.gets, InstanceConfigKey("bab", "titogo", host)); got != 0 { t.Fatalf("overlay Gets for %s = %d, want none: the listing already said it has no overlay", host, got) } } // A convergence pass reads the release store, it never writes to it — // every write this test triggers (SetEnv, ApplyInstance) went through // the unwrapped base store during setup, before s ever saw a call. if got := s.total(s.puts); got != 0 { t.Fatalf("Sync put %d object(s) to the release store, want 0: this pass only writes local env files", got) } } // The unit pass never delivers an exec-backed route's config at all // (TestSyncSkipsExecBackedRoutes) — SyncExec does, and reports its own // overlay failures on it correctly. Final review, plan 4: the unit pass // used to fetch the overlay for EVERY syncable route regardless of Backing // and only checked Backing 53 lines later, so it paid a Get for exactly the // exec-backed, overlay-bearing routes step 5 targets, and threw the result // away. This pins the fixed cost: zero Gets against that key from the unit // pass, whatever the route's overlay actually holds. func TestSyncPaysNoGetForAnExecBackedRoutesOverlay(t *testing.T) { ctx, base, reg, envDir, priv := syncInstanceKit(t) if err := SetEnv(ctx, base, "bab", "titogo", map[string]string{"APP_WIDE": "yes"}); err != nil { t.Fatalf("SetEnv: %v", err) } if _, _, err := ApplyInstance(ctx, base, "bab", "titogo", "book.example.com", map[string]Var{"TENANT": {Kind: "env", Value: "book"}}, nil); err != nil { t.Fatalf("ApplyInstance: %v", err) } // Exec-backed: Backing left at its zero value, the fleet default (same // shape as TestSyncSkipsExecBackedRoutes). if err := reg.Upsert(registry.Route{ Host: "book.example.com", Kind: "instance", App: "titogo", Account: "bab", Status: "ready", }); err != nil { t.Fatalf("Upsert: %v", err) } s := newCountingStore(base) if err := Sync(ctx, reg, s, priv, envDir, func(string) error { return nil }, nil); err != nil { t.Fatalf("Sync: %v", err) } if got := s.n(s.gets, InstanceConfigKey("bab", "titogo", "book.example.com")); got != 0 { t.Fatalf("unit pass Get'd an exec-backed route's overlay %d time(s), want 0: SyncExec owns delivering it, not Sync", got) } if _, err := os.Stat(filepath.Join(envDir, "book.example.com.env")); !os.IsNotExist(err) { t.Fatalf("expected no env file for an exec-backed route, stat err = %v", err) } } // A Get failure against an exec-backed route's overlay must be invisible to // the unit pass: gating the fetch on Backing == BackingUnit (rather than // merely discarding its result) means the failure never happens here at // all, so the pass cannot print the misleading "env file left as-is" claim // about a file it never writes for this route. SyncExec, exercised // separately, is the one that reports a failure on this exact key. func TestSyncPrintsNothingForAnExecBackedRoutesOverlayGetFailure(t *testing.T) { ctx, base, reg, envDir, priv := syncInstanceKit(t) if err := SetEnv(ctx, base, "bab", "titogo", map[string]string{"APP_WIDE": "yes"}); err != nil { t.Fatalf("SetEnv: %v", err) } if _, _, err := ApplyInstance(ctx, base, "bab", "titogo", "book.example.com", map[string]Var{"TENANT": {Kind: "env", Value: "book"}}, nil); err != nil { t.Fatalf("ApplyInstance: %v", err) } if err := reg.Upsert(registry.Route{ Host: "book.example.com", Kind: "instance", App: "titogo", Account: "bab", Status: "ready", }); err != nil { t.Fatalf("Upsert: %v", err) } failing := &failingKeyStore{Store: base, failKey: InstanceConfigKey("bab", "titogo", "book.example.com"), err: errors.New("s3: transient 503 SlowDown")} out := captureStdout(t, func() { if err := Sync(ctx, reg, failing, priv, envDir, func(string) error { return nil }, nil); err != nil { t.Fatalf("Sync: %v", err) } }) if strings.Contains(out, "unresolved") { t.Fatalf("the unit pass printed about an exec-backed route's overlay, which it never delivers or Gets:\n%s", out) } if !strings.Contains(out, "exec-backed") { t.Fatalf("expected the ordinary exec-backed notice (this route still has a default bundle), got:\n%s", out) } if _, err := os.Stat(filepath.Join(envDir, "book.example.com.env")); !os.IsNotExist(err) { t.Fatalf("expected no env file for an exec-backed route, stat err = %v", err) } } // Two independently corrupt overlays are two independent failures, not one. // The listing itself succeeds (both hosts are named), so the "once per app" // dedup for a failed LISTING must not swallow the second route's Get // failure — that would leave one of two stale credential files invisible, // which is exactly what this layer exists to surface. // // book converges with a valid overlay first, so the case below covers the // sharper claim too: an already-delivered route's file survives untouched // when its overlay later becomes unreadable, not just that a route that // never had a file stays fileless. func TestSyncReportsEachRoutesOverlayGetFailureSeparately(t *testing.T) { ctx, s, reg, envDir, priv := syncInstanceKit(t) if err := SetEnv(ctx, s, "bab", "titogo", map[string]string{"APP_WIDE": "yes"}); err != nil { t.Fatalf("SetEnv: %v", err) } // Only book is on the box for the baseline pass, so cart genuinely never // has a file at any point — registering it before the baseline would // let it converge on the shared APP_WIDE bundle too, which would blur // the two cases this test is pinning apart. unitRoute(t, reg, "book.example.com", "titogo", "bab", "") if _, _, err := ApplyInstance(ctx, s, "bab", "titogo", "book.example.com", map[string]Var{"TENANT": {Kind: "env", Value: "book"}}, nil); err != nil { t.Fatalf("ApplyInstance: %v", err) } if err := Sync(ctx, reg, s, priv, envDir, func(string) error { return nil }, nil); err != nil { t.Fatalf("Sync (baseline): %v", err) } delivered, err := os.ReadFile(filepath.Join(envDir, "book.example.com.env")) if err != nil { t.Fatalf("read book's file after the baseline pass: %v", err) } if !bytes.Contains(delivered, []byte("TENANT=book\n")) { t.Fatalf("the baseline overlay did not land: %q", delivered) } unitRoute(t, reg, "cart.example.com", "titogo", "bab", "") // Schema 2 is refused by getIfExistsAtETag ("newer than this binary // understands"), which is a Get-time failure specific to the object // read, not a listing failure — instanceOverlays only reads keys, never // their bodies, so the listing itself succeeds for both hosts. Deleting // book's valid overlay first, then putting the corrupt one raw (Put with // ifMatch "" is create-only), is what turns an already-delivered route's // overlay unreadable, mid-flight. if err := DeleteInstanceConfig(ctx, s, "bab", "titogo", "book.example.com"); err != nil { t.Fatalf("DeleteInstanceConfig: %v", err) } putInstanceRaw(t, ctx, s, "bab", "titogo", "book.example.com", &Manifest{Schema: 2, App: "titogo"}) putInstanceRaw(t, ctx, s, "bab", "titogo", "cart.example.com", &Manifest{Schema: 2, App: "titogo"}) out := captureStdout(t, func() { if err := Sync(ctx, reg, s, priv, envDir, func(string) error { return nil }, nil); err != nil { t.Fatalf("Sync: %v", err) } }) for _, host := range []string{"book.example.com", "cart.example.com"} { if !strings.Contains(out, host) { t.Fatalf("no line for %s's overlay failure, one route's stale file went unreported:\n%s", host, out) } } if n := strings.Count(out, "instance config unresolved"); n != 2 { t.Fatalf("instance config unresolved lines = %d, want 2 (one per corrupt overlay), got:\n%s", n, out) } // cart never had a file before this pass, and a Get failure must leave // that exactly as it is: no env file materializes from a resolved group // merge that an unresolved overlay was supposed to sit on top of. if _, err := os.Stat(filepath.Join(envDir, "cart.example.com.env")); !os.IsNotExist(err) { t.Fatalf("cart.example.com.env exists after an unresolved overlay: %v", err) } // book already had a delivered file, and its Get failure must leave that // file exactly as it was: an overlay going unreadable is not licence to // touch a route that already converged on it. cur, err := os.ReadFile(filepath.Join(envDir, "book.example.com.env")) if err != nil || !bytes.Equal(cur, delivered) { t.Fatalf("book's already-delivered file changed after its overlay became unreadable: %q, want %q (err %v)", cur, delivered, err) } } // A listing that fails is not a determination. The route keeps its file and // the pass says so, the same contract a bundle fetch failure already has. func TestSyncLeavesTheFileAloneWhenTheOverlayListingFails(t *testing.T) { ctx, base, reg, envDir, priv := syncInstanceKit(t) if err := SetEnv(ctx, base, "bab", "titogo", map[string]string{"APP_WIDE": "yes"}); err != nil { t.Fatalf("SetEnv: %v", err) } unitRoute(t, reg, "book.example.com", "titogo", "bab", "") path := filepath.Join(envDir, "book.example.com.env") if err := os.WriteFile(path, []byte("APP_WIDE=stale\n"), 0o600); err != nil { t.Fatalf("stage file: %v", err) } failing := &failingListStore{Store: base, failPrefix: InstancesConfigPrefix("bab", "titogo"), err: errors.New("s3: transient 503 SlowDown")} out := captureStdout(t, func() { if err := Sync(ctx, reg, failing, priv, envDir, func(string) error { return nil }, nil); err != nil { t.Fatalf("Sync: %v", err) } }) cur, err := os.ReadFile(path) if err != nil || string(cur) != "APP_WIDE=stale\n" { t.Fatalf("the file moved on a listing failure: %q %v", cur, err) } if !strings.Contains(out, "book.example.com") || !strings.Contains(out, "left as-is") { t.Fatalf("the pass was silent about an unresolved instance layer:\n%s", out) } } // The dedup sentinel itself, pinned across TWO routes of one app sharing // one failed LISTING (review finding, plan 4 task 7 fix round 1): every // other test naming a listing failure uses a single route, so a mutation // that stopped telling a listing failure apart from a per-host one — // errors.As(...) always false, or instanceOverlayListErr's wrapper dropped // entirely — would still leave every one of them green. This asserts the // dedup's actual effect: one line, not two, with one host anchor naming // whichever route the pass happened to reach first. func TestSyncDedupsTheOverlayListingFailureAcrossTwoRoutesOfOneApp(t *testing.T) { ctx, base, reg, envDir, priv := syncInstanceKit(t) if err := SetEnv(ctx, base, "bab", "titogo", map[string]string{"APP_WIDE": "yes"}); err != nil { t.Fatalf("SetEnv: %v", err) } unitRoute(t, reg, "book.example.com", "titogo", "bab", "") unitRoute(t, reg, "cart.example.com", "titogo", "bab", "") for _, host := range []string{"book.example.com", "cart.example.com"} { if err := os.WriteFile(filepath.Join(envDir, host+".env"), []byte("APP_WIDE=stale\n"), 0o600); err != nil { t.Fatalf("stage %s: %v", host, err) } } failing := &failingListStore{Store: base, failPrefix: InstancesConfigPrefix("bab", "titogo"), err: errors.New("s3: transient 503 SlowDown")} out := captureStdout(t, func() { if err := Sync(ctx, reg, failing, priv, envDir, func(string) error { return nil }, nil); err != nil { t.Fatalf("Sync: %v", err) } }) if n := strings.Count(out, "instance config unresolved"); n != 1 { t.Fatalf("instance config unresolved lines = %d, want exactly 1 for one shared LISTING failure across two routes, got:\n%s", n, out) } if n := strings.Count(out, "(seen at "); n != 1 { t.Fatalf("(seen at ...) host anchors = %d, want exactly 1, got:\n%s", n, out) } for _, host := range []string{"book.example.com", "cart.example.com"} { cur, err := os.ReadFile(filepath.Join(envDir, host+".env")) if err != nil || string(cur) != "APP_WIDE=stale\n" { t.Fatalf("%s moved on a listing failure: %q %v", host, cur, err) } } } // The exec mirror: SyncExec had NO listing-failure test at all before this // (its said-once-per-app contract, stated in the doc comment above the // overlayFor loop, was pinned nowhere). Same two-route shape as the Sync // test above, same three mutations it exists to catch. func TestSyncExecDedupsTheOverlayListingFailureAcrossTwoRoutesOfOneApp(t *testing.T) { ctx, base, reg, dd, privB64, _ := execKit(t) priv, err := ParsePrivateKey(privB64, "test") if err != nil { t.Fatalf("ParsePrivateKey: %v", err) } if err := SetEnv(ctx, base, "bab", "titogo", map[string]string{"APP_WIDE": "yes"}); err != nil { t.Fatalf("SetEnv: %v", err) } for _, host := range []string{"book.example.com", "cart.example.com"} { if err := reg.Upsert(registry.Route{ Host: host, Kind: "instance", App: "titogo", Account: "bab", Status: "ready", Socket: "/run/carlos/" + host + ".sock", }); err != nil { t.Fatalf("Upsert %s: %v", host, err) } if err := os.WriteFile(filepath.Join(dd, host+".config.env"), []byte("OLD=1\n"), 0o600); err != nil { t.Fatalf("stage %s: %v", host, err) } } failing := &failingListStore{Store: base, failPrefix: InstancesConfigPrefix("bab", "titogo"), err: errors.New("s3: transient 503 SlowDown")} out := captureStdout(t, func() { if err := SyncExec(ctx, reg, failing, priv, dd, func(string) error { return nil }, "", nil); err != nil { t.Fatalf("SyncExec: %v", err) } }) if n := strings.Count(out, "instance config unresolved"); n != 1 { t.Fatalf("instance config unresolved lines = %d, want exactly 1 for one shared LISTING failure across two routes, got:\n%s", n, out) } if n := strings.Count(out, "(seen at "); n != 1 { t.Fatalf("(seen at ...) host anchors = %d, want exactly 1, got:\n%s", n, out) } for _, host := range []string{"book.example.com", "cart.example.com"} { cur, err := os.ReadFile(filepath.Join(dd, host+".config.env")) if err != nil || string(cur) != "OLD=1\n" { t.Fatalf("%s moved on a listing failure: %q %v", host, cur, err) } } } // failingListStore makes List fail for exactly one prefix, the listing // shape of failingKeyStore (syncexec_test.go). type failingListStore struct { release.Store failPrefix string err error } func (f *failingListStore) List(ctx context.Context, prefix string) ([]string, error) { if prefix == f.failPrefix { return nil, f.err } return f.Store.List(ctx, prefix) } // The CRITICAL 1 guard's second door. An app with no config.json, a route // bound to nothing, and an overlay: the merged view is that overlay ALONE, // which is a wholesale replacement of whatever a hand-staged file holds. // Refuse it, loudly, every pass, and name the way out. func TestSyncRefusesToDropHandStagedKeysForAnOverlayAlone(t *testing.T) { ctx, s, reg, envDir, priv := syncInstanceKit(t) if _, _, err := ApplyInstance(ctx, s, "bab", "titogo", "book.example.com", map[string]Var{"TENANT": {Kind: "env", Value: "book"}}, nil); err != nil { t.Fatalf("ApplyInstance: %v", err) } unitRoute(t, reg, "book.example.com", "titogo", "bab", "") path := filepath.Join(envDir, "book.example.com.env") staged := "HAND_STAGED=1\nTENANT=old\n" if err := os.WriteFile(path, []byte(staged), 0o600); err != nil { t.Fatalf("stage file: %v", err) } var restarted []string out := captureStdout(t, func() { if err := Sync(ctx, reg, s, priv, envDir, func(h string) error { restarted = append(restarted, h) return nil }, nil); err != nil { t.Fatalf("Sync: %v", err) } }) cur, err := os.ReadFile(path) if err != nil || string(cur) != staged { t.Fatalf("a hand-staged file was replaced by an overlay-alone render: %q %v", cur, err) } if len(restarted) != 0 { t.Fatalf("restarted %v for a route this pass refused", restarted) } if !strings.Contains(out, "HAND_STAGED") || !strings.Contains(out, "book.example.com") { t.Fatalf("the refusal does not name the route and the key it would have dropped:\n%s", out) } } // The same shape, self-healing: nothing is lost, so the route converges. // An app with no default bundle and an instance overlay is a legitimate // state (step 5 creates exactly that for a fresh instance), and this guard // must not freeze it forever. func TestSyncConvergesAnOverlayAloneWhenNothingIsLost(t *testing.T) { ctx, s, reg, envDir, priv := syncInstanceKit(t) if _, _, err := ApplyInstance(ctx, s, "bab", "titogo", "book.example.com", map[string]Var{"TENANT": {Kind: "env", Value: "book"}}, nil); err != nil { t.Fatalf("ApplyInstance: %v", err) } unitRoute(t, reg, "book.example.com", "titogo", "bab", "") if err := Sync(ctx, reg, s, priv, envDir, func(string) error { return nil }, nil); err != nil { t.Fatalf("Sync: %v", err) } cur, err := os.ReadFile(filepath.Join(envDir, "book.example.com.env")) if err != nil { t.Fatalf("read env file: %v", err) } if !bytes.Contains(cur, []byte("TENANT=book\n")) { t.Fatalf("the overlay-alone render did not land:\n%s", cur) } } // And the case that must NOT change: no default bundle, no overlay, no // binding is still "never configured at all", and a never-configured app's // hand-staged file is not this pass's to touch. func TestSyncStillLeavesUnconfiguredAppsAlone(t *testing.T) { ctx, s, reg, envDir, priv := syncInstanceKit(t) unitRoute(t, reg, "book.example.com", "titogo", "bab", "") path := filepath.Join(envDir, "book.example.com.env") staged := "HAND_STAGED=1\n" if err := os.WriteFile(path, []byte(staged), 0o600); err != nil { t.Fatalf("stage file: %v", err) } if err := Sync(ctx, reg, s, priv, envDir, func(string) error { return nil }, nil); err != nil { t.Fatalf("Sync: %v", err) } cur, err := os.ReadFile(path) if err != nil || string(cur) != staged { t.Fatalf("an unconfigured app's file moved: %q %v", cur, err) } } // The exec half of the same delivery: one activator-spawned instance gets // its own value, its sibling does not, and the listing is paid for once. func TestSyncExecRendersTheInstanceOverlayForItsOwnRouteOnly(t *testing.T) { ctx, base, reg, dd, privB64, _ := execKit(t) priv, err := ParsePrivateKey(privB64, "test") if err != nil { t.Fatalf("ParsePrivateKey: %v", err) } if err := SetEnv(ctx, base, "bab", "titogo", map[string]string{"APP_WIDE": "yes", "TENANT": "none"}); err != nil { t.Fatalf("SetEnv: %v", err) } if _, _, err := ApplyInstance(ctx, base, "bab", "titogo", "book.example.com", map[string]Var{"TENANT": {Kind: "env", Value: "book"}}, nil); err != nil { t.Fatalf("ApplyInstance: %v", err) } for _, host := range []string{"book.example.com", "cart.example.com"} { if err := reg.Upsert(registry.Route{ Host: host, Kind: "instance", App: "titogo", Account: "bab", Status: "ready", Socket: "/run/carlos/" + host + ".sock", }); err != nil { t.Fatalf("Upsert %s: %v", host, err) } } s := newCountingStore(base) if err := SyncExec(ctx, reg, s, priv, dd, func(string) error { return nil }, "", nil); err != nil { t.Fatalf("SyncExec: %v", err) } book, err := os.ReadFile(filepath.Join(dd, "book.example.com.config.env")) if err != nil { t.Fatalf("read book config: %v", err) } if !bytes.Contains(book, []byte("TENANT=book\n")) || !bytes.Contains(book, []byte("APP_WIDE=yes\n")) { t.Fatalf("the overlaid route's config file is wrong:\n%s", book) } cart, err := os.ReadFile(filepath.Join(dd, "cart.example.com.config.env")) if err != nil { t.Fatalf("read cart config: %v", err) } if bytes.Contains(cart, []byte("book")) || !bytes.Contains(cart, []byte("TENANT=none\n")) { t.Fatalf("a sibling saw another instance's config:\n%s", cart) } if got := s.n(s.lists, InstancesConfigPrefix("bab", "titogo")); got != 1 { t.Fatalf("instance overlay listings = %d, want exactly 1 for two routes of one app", got) } if got := s.n(s.gets, InstanceConfigKey("bab", "titogo", "cart.example.com")); got != 0 { t.Fatalf("read an overlay for a host the listing did not name (%d times)", got) } } // exec_delivery is a BUNDLE-level flag and the instance layer is not one of // the bundles it is read from (the ruling at appconfig.go:66-71). A TCP-addr // route whose app has never opted in gets no file, whatever its overlay // says — a per-host delivery flag is exactly the granularity that ruling // refused, and honouring one here would be the config-that-lies hazard it // names. func TestSyncExecNeverReadsExecDeliveryFromTheInstanceLayer(t *testing.T) { ctx, s, reg, dd, privB64, _ := execKit(t) priv, err := ParsePrivateKey(privB64, "test") if err != nil { t.Fatalf("ParsePrivateKey: %v", err) } if err := SetEnv(ctx, s, "bab", "titogo", map[string]string{"APP_WIDE": "yes"}); err != nil { t.Fatalf("SetEnv: %v", err) } putInstanceRaw(t, ctx, s, "bab", "titogo", "router.example.com", &Manifest{ Schema: 1, App: "titogo", Vars: map[string]Var{"TENANT": {Kind: "env", Value: "book"}}, ExecDelivery: ExecDeliveryFile}) if err := reg.Upsert(registry.Route{ Host: "router.example.com", Kind: "instance", App: "titogo", Account: "bab", Status: "ready", Addr: "127.0.0.1:8080", }); err != nil { t.Fatalf("Upsert: %v", err) } if err := SyncExec(ctx, reg, s, priv, dd, func(string) error { return nil }, "", nil); err != nil { t.Fatalf("SyncExec: %v", err) } if _, err := os.Stat(filepath.Join(dd, "router.example.com.config.env")); !os.IsNotExist(err) { t.Fatalf("an instance overlay opted a route into file delivery: %v", err) } // Opt the APP in, and the same route gets the same overlay merged in the // ordinary way — the flag's door is the bundle, the overlay's payload is // vars. if err := SetExecDelivery(ctx, s, "bab", "titogo", ExecDeliveryFile); err != nil { t.Fatalf("SetExecDelivery: %v", err) } if err := SyncExec(ctx, reg, s, priv, dd, func(string) error { return nil }, "", nil); err != nil { t.Fatalf("SyncExec (opted in): %v", err) } cur, err := os.ReadFile(filepath.Join(dd, "router.example.com.config.env")) if err != nil { t.Fatalf("read router config: %v", err) } if !bytes.Contains(cur, []byte("TENANT=book\n")) { t.Fatalf("the overlay's vars did not land once the app opted in:\n%s", cur) } } // The exec mirror of TestSyncReportsEachRoutesOverlayGetFailureSeparately: // two independently corrupt overlays are two independent failures, not one. // The listing itself succeeds (both hosts are named), so the "once per app" // dedup for a failed LISTING must not swallow the second route's Get // failure — that would leave one of two stale config files invisible, // which is exactly what this layer exists to surface. func TestSyncExecReportsEachRoutesOverlayGetFailureSeparately(t *testing.T) { ctx, s, reg, dd, privB64, _ := execKit(t) priv, err := ParsePrivateKey(privB64, "test") if err != nil { t.Fatalf("ParsePrivateKey: %v", err) } if err := SetEnv(ctx, s, "bab", "titogo", map[string]string{"APP_WIDE": "yes"}); err != nil { t.Fatalf("SetEnv: %v", err) } // Schema 2 is refused by getIfExistsAtETag ("newer than this binary // understands"), which is a Get-time failure specific to the object // read, not a listing failure — instanceOverlays only reads keys, never // their bodies, so the listing itself succeeds for both hosts. putInstanceRaw(t, ctx, s, "bab", "titogo", "book.example.com", &Manifest{Schema: 2, App: "titogo"}) putInstanceRaw(t, ctx, s, "bab", "titogo", "cart.example.com", &Manifest{Schema: 2, App: "titogo"}) for _, host := range []string{"book.example.com", "cart.example.com"} { if err := reg.Upsert(registry.Route{ Host: host, Kind: "instance", App: "titogo", Account: "bab", Status: "ready", Socket: "/run/carlos/" + host + ".sock", }); err != nil { t.Fatalf("Upsert %s: %v", host, err) } } out := captureStdout(t, func() { if err := SyncExec(ctx, reg, s, priv, dd, func(string) error { return nil }, "", nil); err != nil { t.Fatalf("SyncExec: %v", err) } }) for _, host := range []string{"book.example.com", "cart.example.com"} { if !strings.Contains(out, host) { t.Fatalf("no line for %s's overlay failure, one route's stale file went unreported:\n%s", host, out) } } if n := strings.Count(out, "instance config unresolved"); n != 2 { t.Fatalf("instance config unresolved lines = %d, want 2 (one per corrupt overlay), got:\n%s", n, out) } // Neither route had a file before this pass, and a Get failure must // leave that exactly as it is: no config file materializes from a // resolved group merge that an unresolved overlay was supposed to sit // on top of. for _, host := range []string{"book.example.com", "cart.example.com"} { if _, err := os.Stat(filepath.Join(dd, host+".config.env")); !os.IsNotExist(err) { t.Fatalf("%s.config.env exists after an unresolved overlay: %v", host, err) } } } // An app with no default bundle at all, and one instance with an overlay. // The overlay is the whole configuration of that route, so its file is // written, not removed — while its sibling, which has nothing anywhere, is // reaped exactly as it is today. func TestSyncExecConvergesAnOverlayAloneInsteadOfRemoving(t *testing.T) { ctx, s, reg, dd, privB64, _ := execKit(t) priv, err := ParsePrivateKey(privB64, "test") if err != nil { t.Fatalf("ParsePrivateKey: %v", err) } if _, _, err := ApplyInstance(ctx, s, "bab", "titogo", "book.example.com", map[string]Var{"TENANT": {Kind: "env", Value: "book"}}, nil); err != nil { t.Fatalf("ApplyInstance: %v", err) } for _, host := range []string{"book.example.com", "cart.example.com"} { if err := reg.Upsert(registry.Route{ Host: host, Kind: "instance", App: "titogo", Account: "bab", Status: "ready", Socket: "/run/carlos/" + host + ".sock", }); err != nil { t.Fatalf("Upsert %s: %v", host, err) } if err := os.WriteFile(filepath.Join(dd, host+".config.env"), []byte("OLD=1\n"), 0o600); err != nil { t.Fatalf("stage %s: %v", host, err) } } if err := SyncExec(ctx, reg, s, priv, dd, func(string) error { return nil }, "", nil); err != nil { t.Fatalf("SyncExec: %v", err) } book, err := os.ReadFile(filepath.Join(dd, "book.example.com.config.env")) if err != nil { t.Fatalf("the overlaid route's file was removed: %v", err) } if !bytes.Contains(book, []byte("TENANT=book\n")) || bytes.Contains(book, []byte("OLD=1")) { t.Fatalf("the overlay-alone render did not converge:\n%s", book) } if _, err := os.Stat(filepath.Join(dd, "cart.example.com.config.env")); !os.IsNotExist(err) { t.Fatalf("a route with no config anywhere kept its file: %v", err) } } // The mass-removal guard, with an overlay in the middle of it. Two apps // whose manifests all read absent still freeze every removal on the box, // and the one route that has an overlay still converges: writing a file a // bundle asked for is not a removal, and the guard exists to stop // deletions, not delivery. func TestSyncExecMassRemovalGuardHoldsRemovalsButNotOverlays(t *testing.T) { ctx, s, reg, dd, privB64, _ := execKit(t) priv, err := ParsePrivateKey(privB64, "test") if err != nil { t.Fatalf("ParsePrivateKey: %v", err) } if _, _, err := ApplyInstance(ctx, s, "bab", "titogo", "book.example.com", map[string]Var{"TENANT": {Kind: "env", Value: "book"}}, nil); err != nil { t.Fatalf("ApplyInstance: %v", err) } routes := map[string]string{ "book.example.com": "titogo", "cart.example.com": "titogo", "desk.example.com": "eleven", } for host, app := range routes { if err := reg.Upsert(registry.Route{ Host: host, Kind: "instance", App: app, Account: "bab", Status: "ready", Socket: "/run/carlos/" + host + ".sock", }); err != nil { t.Fatalf("Upsert %s: %v", host, err) } if err := os.WriteFile(filepath.Join(dd, host+".config.env"), []byte("OLD=1\n"), 0o600); err != nil { t.Fatalf("stage %s: %v", host, err) } } out := captureStdout(t, func() { if err := SyncExec(ctx, reg, s, priv, dd, func(string) error { return nil }, "", nil); err != nil { t.Fatalf("SyncExec: %v", err) } }) if !strings.Contains(out, "refusing to remove any config files this pass") { t.Fatalf("the mass-removal guard did not fire:\n%s", out) } for _, host := range []string{"cart.example.com", "desk.example.com"} { cur, err := os.ReadFile(filepath.Join(dd, host+".config.env")) if err != nil || string(cur) != "OLD=1\n" { t.Fatalf("%s lost its file while the guard was up: %q %v", host, cur, err) } } book, err := os.ReadFile(filepath.Join(dd, "book.example.com.config.env")) if err != nil { t.Fatalf("the overlaid route was frozen by a guard about removals: %v", err) } if !bytes.Contains(book, []byte("TENANT=book\n")) { t.Fatalf("the overlay did not converge under the guard:\n%s", book) } } // The layer's only convergence signal is the generation object. The route // fingerprint cannot see a bucket write and must not start paying to try: // this pins both halves of that at once. func TestWritingAnOverlayMovesTheGenerationAndNotTheFingerprint(t *testing.T) { ctx := context.Background() s := release.NewDirStore(t.TempDir()) if err := SetEnv(ctx, s, "bab", "titogo", map[string]string{"APP_WIDE": "yes"}); err != nil { t.Fatalf("SetEnv: %v", err) } routes := []registry.Route{{ Host: "book.example.com", Kind: "instance", App: "titogo", Account: "bab", Status: "ready", Socket: "/run/carlos/book.sock", }} execBefore, unitBefore := ConfigRouteFingerprints(routes) _, genBefore, err := s.Get(ctx, GenKey("bab", "titogo")) if err != nil { t.Fatalf("read generation: %v", err) } if _, _, err := ApplyInstance(ctx, s, "bab", "titogo", "book.example.com", map[string]Var{"TENANT": {Kind: "env", Value: "book"}}, nil); err != nil { t.Fatalf("ApplyInstance: %v", err) } execAfter, unitAfter := ConfigRouteFingerprints(routes) if execAfter != execBefore || unitAfter != unitBefore { t.Fatal("the route fingerprint moved for a bucket-side write, which would cost a read per route per tick") } _, genAfter, err := s.Get(ctx, GenKey("bab", "titogo")) if err != nil { t.Fatalf("read generation: %v", err) } if genAfter == genBefore { t.Fatal("the generation did not move, so the fast loop has nothing to notice and delivery waits out the hourly floor") } } // One pass over a box holding every shape this layer comes in: an instance // with an overlay over its app's bundle, a sibling with none, and an // instance whose app has no bundle at all so its overlay stands alone. // Then the overlay is removed and the route converges back, with the ack // saying so. func TestOnePassOverEveryInstanceLayerShape(t *testing.T) { ctx, s, reg, dd, privB64, _ := execKit(t) priv, err := ParsePrivateKey(privB64, "test") if err != nil { t.Fatalf("ParsePrivateKey: %v", err) } if err := SetEnv(ctx, s, "bab", "titogo", map[string]string{"APP_WIDE": "yes", "TENANT": "none"}); err != nil { t.Fatalf("SetEnv: %v", err) } _, bookETag, err := ApplyInstance(ctx, s, "bab", "titogo", "book.example.com", map[string]Var{"TENANT": {Kind: "env", Value: "book"}}, nil) if err != nil { t.Fatalf("ApplyInstance (book): %v", err) } _, deskETag, err := ApplyInstance(ctx, s, "bab", "solo", "desk.example.com", map[string]Var{"TENANT": {Kind: "env", Value: "desk"}}, nil) if err != nil { t.Fatalf("ApplyInstance (desk): %v", err) } for host, app := range map[string]string{ "book.example.com": "titogo", "cart.example.com": "titogo", "desk.example.com": "solo"} { if err := reg.Upsert(registry.Route{ Host: host, Kind: "instance", App: app, Account: "bab", Status: "ready", Socket: "/run/carlos/" + host + ".sock", }); err != nil { t.Fatalf("Upsert %s: %v", host, err) } } acks := map[string]ConfigAck{} collect := func(a ConfigAck) { acks[a.Host] = a } if err := SyncExec(ctx, reg, s, priv, dd, func(string) error { return nil }, "", collect); err != nil { t.Fatalf("SyncExec: %v", err) } read := func(host string) string { t.Helper() body, err := os.ReadFile(filepath.Join(dd, host+".config.env")) if err != nil { t.Fatalf("read %s: %v", host, err) } return string(body) } if got := read("book.example.com"); !strings.Contains(got, "TENANT=book\n") || !strings.Contains(got, "APP_WIDE=yes\n") { t.Fatalf("book: %q", got) } if got := read("cart.example.com"); !strings.Contains(got, "TENANT=none\n") { t.Fatalf("cart saw another instance's config: %q", got) } if got := read("desk.example.com"); got != "TENANT=desk\n" { t.Fatalf("desk, whose app has no bundle, should render its overlay alone: %q", got) } for host, want := range map[string]string{ "book.example.com": bookETag, "cart.example.com": "", "desk.example.com": deskETag} { a, ok := acks[host] if !ok { t.Fatalf("no ack for %s", host) } if a.Schema != ConfigAckSchema || a.InstanceConfigETag != want { t.Fatalf("%s ack = %+v, want schema %d and etag %q", host, a, ConfigAckSchema, want) } if v := InstanceConfigDelivery(&a, want); v != InstanceConfigDelivered { t.Fatalf("%s delivery = %q, want %q", host, v, InstanceConfigDelivered) } } // Remove one overlay: that route converges back to its app's merge, and // its ack stops naming an overlay, which is what tells a reader the // removal actually reached the box. if err := DeleteInstanceConfig(ctx, s, "bab", "titogo", "book.example.com"); err != nil { t.Fatalf("DeleteInstanceConfig: %v", err) } acks = map[string]ConfigAck{} if err := SyncExec(ctx, reg, s, priv, dd, func(string) error { return nil }, "", collect); err != nil { t.Fatalf("SyncExec (after delete): %v", err) } if got := read("book.example.com"); !strings.Contains(got, "TENANT=none\n") { t.Fatalf("book did not converge back to its app's merge: %q", got) } if a := acks["book.example.com"]; a.InstanceConfigETag != "" || a.Schema != ConfigAckSchema { t.Fatalf("book ack after the delete = %+v, want the schema and no overlay", a) } if got := read("desk.example.com"); got != "TENANT=desk\n" { t.Fatalf("desk was disturbed by a sibling app's delete: %q", got) } } // The unit-path half of the removal claim. TestOnePassOverEveryInstanceLayerShape // pins the exec sweep's removal convergence; nothing else in the package // runs Sync (root's unit path) after a delete, so without this the claim // that a route "goes back to what the bundles alone say" once its overlay // is removed is only ever true for SyncExec. func TestSyncConvergesBackToTheBundleAfterAnOverlayIsDeleted(t *testing.T) { ctx, s, reg, envDir, priv := syncInstanceKit(t) if err := SetEnv(ctx, s, "bab", "titogo", map[string]string{"APP_WIDE": "yes", "TENANT": "none"}); err != nil { t.Fatalf("SetEnv: %v", err) } if _, _, err := ApplyInstance(ctx, s, "bab", "titogo", "book.example.com", map[string]Var{"TENANT": {Kind: "env", Value: "book"}}, nil); err != nil { t.Fatalf("ApplyInstance: %v", err) } unitRoute(t, reg, "book.example.com", "titogo", "bab", "") if err := Sync(ctx, reg, s, priv, envDir, func(string) error { return nil }, nil); err != nil { t.Fatalf("Sync: %v", err) } before, err := os.ReadFile(filepath.Join(envDir, "book.example.com.env")) if err != nil { t.Fatalf("read env file before the delete: %v", err) } if !bytes.Contains(before, []byte("TENANT=book\n")) { t.Fatalf("the overlay did not land before the delete: %q", before) } if err := DeleteInstanceConfig(ctx, s, "bab", "titogo", "book.example.com"); err != nil { t.Fatalf("DeleteInstanceConfig: %v", err) } if err := Sync(ctx, reg, s, priv, envDir, func(string) error { return nil }, nil); err != nil { t.Fatalf("Sync (after delete): %v", err) } after, err := os.ReadFile(filepath.Join(envDir, "book.example.com.env")) if err != nil { t.Fatalf("read env file after the delete: %v", err) } if bytes.Contains(after, []byte("TENANT=book\n")) || !bytes.Contains(after, []byte("TENANT=none\n")) { t.Fatalf("the route did not converge back to the bundle alone after the overlay was deleted: %q", after) } } // Every other test in this file passes a no-op restart callback, so a // regression that made resolving one route's overlay spuriously mark a // different route as changed would still pass the whole suite. This one // records which hosts restart is actually called for. The first pass // converges both routes from nothing and restarts both once, which is // ordinary first-convergence behaviour and not the claim under test; that // recording is discarded before the pass that matters, where only book's // overlay changes and cart's rendered bytes do not move at all. func TestSyncDoesNotRestartAnUnchangedSiblingWhileAnOverlayIsResolved(t *testing.T) { ctx, s, reg, envDir, priv := syncInstanceKit(t) if err := SetEnv(ctx, s, "bab", "titogo", map[string]string{"APP_WIDE": "yes", "TENANT": "none"}); err != nil { t.Fatalf("SetEnv: %v", err) } unitRoute(t, reg, "book.example.com", "titogo", "bab", "") unitRoute(t, reg, "cart.example.com", "titogo", "bab", "") var restarted []string record := func(host string) error { restarted = append(restarted, host) return nil } if err := Sync(ctx, reg, s, priv, envDir, record, nil); err != nil { t.Fatalf("Sync (baseline): %v", err) } restarted = nil if _, _, err := ApplyInstance(ctx, s, "bab", "titogo", "book.example.com", map[string]Var{"TENANT": {Kind: "env", Value: "book"}}, nil); err != nil { t.Fatalf("ApplyInstance: %v", err) } if err := Sync(ctx, reg, s, priv, envDir, record, nil); err != nil { t.Fatalf("Sync (overlay pass): %v", err) } if len(restarted) != 1 || restarted[0] != "book.example.com" { t.Fatalf("restarted = %v, want exactly [book.example.com]: cart has no overlay and its rendered file did not change, so it must not restart", restarted) } } // The exec mirror of the test above, over bounce rather than restart. func TestSyncExecDoesNotBounceAnUnchangedSiblingWhileAnOverlayIsResolved(t *testing.T) { ctx, s, reg, dd, privB64, _ := execKit(t) priv, err := ParsePrivateKey(privB64, "test") if err != nil { t.Fatalf("ParsePrivateKey: %v", err) } if err := SetEnv(ctx, s, "bab", "titogo", map[string]string{"APP_WIDE": "yes", "TENANT": "none"}); err != nil { t.Fatalf("SetEnv: %v", err) } for _, host := range []string{"book.example.com", "cart.example.com"} { if err := reg.Upsert(registry.Route{ Host: host, Kind: "instance", App: "titogo", Account: "bab", Status: "ready", Socket: "/run/carlos/" + host + ".sock", }); err != nil { t.Fatalf("Upsert %s: %v", host, err) } } var bounced []string record := func(host string) error { bounced = append(bounced, host) return nil } if err := SyncExec(ctx, reg, s, priv, dd, record, "", nil); err != nil { t.Fatalf("SyncExec (baseline): %v", err) } bounced = nil if _, _, err := ApplyInstance(ctx, s, "bab", "titogo", "book.example.com", map[string]Var{"TENANT": {Kind: "env", Value: "book"}}, nil); err != nil { t.Fatalf("ApplyInstance: %v", err) } if err := SyncExec(ctx, reg, s, priv, dd, record, "", nil); err != nil { t.Fatalf("SyncExec (overlay pass): %v", err) } if len(bounced) != 1 || bounced[0] != "book.example.com" { t.Fatalf("bounced = %v, want exactly [book.example.com]: cart has no overlay and its rendered file did not change, so it must not bounce", bounced) } }