package host import ( "bytes" "context" "crypto/ecdh" "crypto/rand" "encoding/base64" "encoding/json" "errors" "fmt" "log" "os" "strings" "sync" "testing" "time" "github.com/carlosframework/platform/internal/appconfig" "github.com/carlosframework/platform/internal/registry" "github.com/carlosframework/platform/internal/release" ) // Issue #213: the box-side guard. A box takes an instance's lease, or mints // its record, only if it can provide that instance's config. These tests // drive the guard through its Gate seam, the one function the activator and // the reconcile pass both call. type guardEnv struct { t *testing.T store *guardStore priv *ecdh.PrivateKey other *ecdh.PrivateKey clock *guardClock // dataDir holds leftover rendered files the tests plant to show the guard // ignores them. dataDir string } type guardClock struct { mu sync.Mutex t time.Time } func (c *guardClock) now() time.Time { c.mu.Lock(); defer c.mu.Unlock(); return c.t } func (c *guardClock) advance(d time.Duration) { c.mu.Lock() c.t = c.t.Add(d) c.mu.Unlock() } // guardStore counts Gets and can be made to fail every Get. type guardStore struct { release.Store mu sync.Mutex gets int failGet error } func (s *guardStore) Get(ctx context.Context, key string) ([]byte, string, error) { s.mu.Lock() s.gets++ err := s.failGet s.mu.Unlock() if err != nil { return nil, "", err } return s.Store.Get(ctx, key) } func (s *guardStore) getCount() int { s.mu.Lock(); defer s.mu.Unlock(); return s.gets } func (s *guardStore) setFail(err error) { s.mu.Lock() s.failGet = err s.mu.Unlock() } func newGuardEnv(t *testing.T) *guardEnv { t.Helper() priv, err := ecdh.P256().GenerateKey(rand.Reader) if err != nil { t.Fatal(err) } other, err := ecdh.P256().GenerateKey(rand.Reader) if err != nil { t.Fatal(err) } return &guardEnv{ t: t, store: &guardStore{Store: release.NewDirStore(t.TempDir())}, priv: priv, other: other, clock: &guardClock{t: time.Date(2026, 10, 5, 9, 0, 0, 0, time.UTC)}, dataDir: t.TempDir(), } } func (e *guardEnv) seal(plain string) appconfig.Var { e.t.Helper() env, err := appconfig.SealUnbound(e.priv.PublicKey(), []byte(plain)) if err != nil { e.t.Fatal(err) } return appconfig.Var{Kind: "secret", Envelope: env} } func (e *guardEnv) putConfig(acct, app string, vars map[string]appconfig.Var) { e.t.Helper() body, err := json.Marshal(appconfig.Manifest{Schema: 1, App: app, Vars: vars}) if err != nil { e.t.Fatal(err) } if _, err := e.store.Store.Put(context.Background(), appconfig.Key(acct, app), body, ""); err != nil { e.t.Fatal(err) } } // guard builds the guard with the given key in the environment ("" = unset) // and this env's store. func (e *guardEnv) guard(keyB64 string) *configGuard { g := newConfigGuard(func(k string) string { switch k { case "CARLOS_SECRETS_KEY": return keyB64 } return "" }, func(context.Context) (release.Store, error) { return e.store, nil }) g.now = e.clock.now return g } func keyB64(k *ecdh.PrivateKey) string { return base64.StdEncoding.EncodeToString(k.Bytes()) } func guardRoute(host string) registry.Route { return registry.Route{Host: host, Kind: "instance", App: "shop", Account: "bab", Socket: "/run/carlos/" + host + ".sock", Hibernate: true, Provisioned: true} } func reasonOfGate(t *testing.T, err error) string { t.Helper() var r *appconfig.ConfigRefusal if !errors.As(err, &r) { t.Fatalf("err = %v, want *appconfig.ConfigRefusal", err) } return r.Reason } func guardCaptureLog(t *testing.T) *bytes.Buffer { t.Helper() var buf bytes.Buffer old := log.Writer() log.SetOutput(&buf) t.Cleanup(func() { log.SetOutput(old) }) return &buf } func TestNeedsConfigIsTheShapeTheActivatorSpawns(t *testing.T) { cases := []struct { name string rt registry.Route want bool }{ {"socket instance", guardRoute("a.example.com"), true}, {"non-hibernating socket instance", registry.Route{Host: "a", Kind: "instance", App: "shop", Socket: "/s"}, true}, {"sidecar worker", registry.Route{Host: "a", Kind: "sidecar", App: "shop", Hibernate: true}, true}, {"unit-backed by column", registry.Route{Host: "a", Kind: "instance", App: "shop", Socket: "/s", Backing: registry.BackingUnit}, false}, {"unit-backed sidecar", registry.Route{Host: "a", Kind: "sidecar", App: "shop", Backing: registry.BackingUnit}, false}, // Dispatch is on Backing alone (SplitBackend.pick): a socket-backed // instance that merely NAMES a unit still reaches ExecBackend.Acquire. {"unit-named, socket-backed, empty Backing", registry.Route{Host: "a", Kind: "instance", App: "shop", Socket: "/s", Unit: "x.service", Hibernate: true}, true}, {"unit-named, addr-backed", registry.Route{Host: "a", Kind: "instance", App: "shop", Addr: "127.0.0.1:1", Unit: "x.service"}, false}, {"addr-backed", registry.Route{Host: "a", Kind: "instance", App: "shop", Addr: "127.0.0.1:9"}, false}, {"static", registry.Route{Host: "a", Kind: "static", App: "shop"}, false}, {"service", registry.Route{Host: "a", Kind: "service", App: "shop", Addr: "127.0.0.1:9"}, false}, {"platform", registry.Route{Host: "a", Kind: "instance", App: "console", Socket: "/s", Platform: true}, false}, {"no app", registry.Route{Host: "a", Kind: "instance", Socket: "/s"}, false}, } for _, c := range cases { if got := needsBoxConfig(c.rt); got != c.want { t.Errorf("%s: needsBoxConfig = %v, want %v", c.name, got, c.want) } } } // A deployment with no sealed config anywhere (dev) is unaffected: no key, no // manifest, no refusal, nothing published. func TestGuardNoSealedConfigAdmitsKeylessBox(t *testing.T) { e := newGuardEnv(t) g := e.guard("") if err := g.Gate(context.Background(), guardRoute("a.example.com")); err != nil { t.Fatalf("keyless box, nothing sealed: %v", err) } if list, total := g.Refusals(); len(list) != 0 || total != 0 { t.Errorf("refusals = %v (%d), want none", list, total) } } func TestGuardKeylessBoxRefusesSealedInstanceAndPublishesIt(t *testing.T) { e := newGuardEnv(t) e.putConfig("bab", "shop", map[string]appconfig.Var{"DB": e.seal("pw")}) logs := guardCaptureLog(t) g := e.guard("") err := g.Gate(context.Background(), guardRoute("kpmg.example.com")) if got := reasonOfGate(t, err); got != appconfig.ReasonNoKey { t.Fatalf("reason = %q, want no_key", got) } list, total := g.Refusals() if total != 1 || len(list) != 1 { t.Fatalf("refusals = %v (%d)", list, total) } r := list[0] if r.Host != "kpmg.example.com" || r.Reason != "no_key" || r.Count != 1 || !r.Since.Equal(e.clock.now()) { t.Errorf("refusal = %+v", r) } if n := strings.Count(logs.String(), "kpmg.example.com"); n != 1 { t.Errorf("log mentions the host %d times, want once:\n%s", n, logs.String()) } } func TestGuardKeyedBoxTakesIt(t *testing.T) { e := newGuardEnv(t) e.putConfig("bab", "shop", map[string]appconfig.Var{"DB": e.seal("pw")}) g := e.guard(keyB64(e.priv)) if err := g.Gate(context.Background(), guardRoute("a.example.com")); err != nil { t.Fatalf("keyed box: %v", err) } if _, total := g.Refusals(); total != 0 { t.Errorf("a keyed box publishes %d refusals", total) } } func TestGuardWrongAndBrokenKeysRefuse(t *testing.T) { e := newGuardEnv(t) e.putConfig("bab", "shop", map[string]appconfig.Var{"DB": e.seal("pw")}) for name, tc := range map[string]struct { key string want string }{ "wrong key": {keyB64(e.other), appconfig.ReasonCannotOpen}, "not base64": {"%%%not-a-key", appconfig.ReasonKeyUnparseable}, "short": {base64.StdEncoding.EncodeToString([]byte("abc")), appconfig.ReasonKeyUnparseable}, "blank": {" ", appconfig.ReasonNoKey}, } { t.Run(name, func(t *testing.T) { g := e.guard(tc.key) err := g.Gate(context.Background(), guardRoute("a.example.com")) if got := reasonOfGate(t, err); got != tc.want { t.Errorf("reason = %q, want %q", got, tc.want) } }) } } // Fail closed: a store the box cannot read is not "no sealed config". func TestGuardStoreErrorRefusesAndRetriesAfterTheTTL(t *testing.T) { e := newGuardEnv(t) e.putConfig("bab", "shop", map[string]appconfig.Var{"DB": e.seal("pw")}) g := e.guard(keyB64(e.priv)) ctx := context.Background() e.store.setFail(errors.New("s3: 503")) err := g.Gate(ctx, guardRoute("a.example.com")) if got := reasonOfGate(t, err); got != appconfig.ReasonStoreError { t.Fatalf("reason = %q, want store_error", got) } // Within the TTL a repeat is answered from the verdict, without another // bucket read, and still refuses even though the store has recovered. e.store.setFail(nil) before := e.store.getCount() if err := g.Gate(ctx, guardRoute("a.example.com")); err == nil { t.Fatal("refusal forgotten inside the TTL") } if got := e.store.getCount(); got != before { t.Errorf("a cached refusal read the store (%d -> %d)", before, got) } if list, _ := g.Refusals(); len(list) != 1 || list[0].Count != 2 { t.Errorf("refusals = %+v, want one with count 2", list) } // After the TTL it asks again and, the store being back, admits and // stops publishing the refusal. e.clock.advance(configGuardRefuseTTL + time.Second) if err := g.Gate(ctx, guardRoute("a.example.com")); err != nil { t.Fatalf("after the TTL with the store back: %v", err) } if _, total := g.Refusals(); total != 0 { t.Errorf("a recovered host still published (%d)", total) } } func TestGuardLogsOncePerHostAndAgainWhenTheReasonChanges(t *testing.T) { e := newGuardEnv(t) e.putConfig("bab", "shop", map[string]appconfig.Var{"DB": e.seal("pw")}) g := e.guard("") logs := guardCaptureLog(t) ctx := context.Background() for i := 0; i < 5; i++ { _ = g.Gate(ctx, guardRoute("a.example.com")) e.clock.advance(configGuardRefuseTTL + time.Second) // defeat the verdict cache } if n := strings.Count(logs.String(), "a.example.com"); n != 1 { t.Fatalf("5 refusals logged %d lines, want 1:\n%s", n, logs.String()) } _ = g.Gate(ctx, guardRoute("b.example.com")) if n := strings.Count(logs.String(), "b.example.com"); n != 1 { t.Errorf("a second host logged %d lines, want 1", n) } // The reason changes (a store outage now hides the answer): said again. e.store.setFail(errors.New("s3: down")) _ = g.Gate(ctx, guardRoute("a.example.com")) if n := strings.Count(logs.String(), "a.example.com"); n != 2 { t.Errorf("a changed reason logged %d lines for the host, want 2:\n%s", n, logs.String()) } } // Routes the box does not spawn never reach the store. func TestGuardIgnoresRoutesItDoesNotSpawn(t *testing.T) { e := newGuardEnv(t) e.putConfig("bab", "shop", map[string]appconfig.Var{"DB": e.seal("pw")}) g := e.guard("") for _, rt := range []registry.Route{ {Host: "s", Kind: "static", App: "shop"}, {Host: "u", Kind: "instance", App: "shop", Socket: "/s", Backing: registry.BackingUnit}, {Host: "p", Kind: "instance", App: "shop", Socket: "/s", Platform: true}, } { if err := g.Gate(context.Background(), rt); err != nil { t.Errorf("%s: %v", rt.Host, err) } } if e.store.getCount() != 0 { t.Errorf("read the store %d times for routes it does not spawn", e.store.getCount()) } } // Config lives in the deployment store, and CARLOS_HIBERNATE_BUCKET enables // leases without one. A box with no store configured therefore cannot know // that nothing is sealed, and refuses placed exec instances, unless the // operator has said explicitly that this deployment has no sealed config. func TestGuardNoDeployStoreFailsClosedUnlessOptedOut(t *testing.T) { e := newGuardEnv(t) build := func(env map[string]string) *configGuard { g := newConfigGuard(func(k string) string { return env[k] }, func(context.Context) (release.Store, error) { return nil, errNoDeployStore }) g.now = e.clock.now return g } g := build(nil) err := g.Gate(context.Background(), guardRoute("a.example.com")) if got := reasonOfGate(t, err); got != appconfig.ReasonNoSource { t.Fatalf("no store, no opt-out: reason = %q, want no_config_source", got) } if list, _ := g.Refusals(); len(list) != 1 || list[0].Reason != "no_config_source" { t.Errorf("refusals = %+v", list) } for _, v := range []string{"1", "true", " TRUE "} { g := build(map[string]string{"CARLOS_NO_SEALED_CONFIG": v}) if err := g.Gate(context.Background(), guardRoute("a.example.com")); err != nil { t.Errorf("CARLOS_NO_SEALED_CONFIG=%q: %v", v, err) } } for _, v := range []string{"", "0", "no", "yes please"} { g := build(map[string]string{"CARLOS_NO_SEALED_CONFIG": v}) if err := g.Gate(context.Background(), guardRoute("a.example.com")); err == nil { t.Errorf("CARLOS_NO_SEALED_CONFIG=%q opted out", v) } } // The opt-out is only about having NO store. With one, config is read. e.putConfig("bab", "shop", map[string]appconfig.Var{"DB": e.seal("pw")}) withStore := newConfigGuard(func(k string) string { if k == "CARLOS_NO_SEALED_CONFIG" { return "1" } return "" }, func(context.Context) (release.Store, error) { return e.store, nil }) withStore.now = e.clock.now if got := reasonOfGate(t, withStore.Gate(context.Background(), guardRoute("a.example.com"))); got != appconfig.ReasonNoKey { t.Errorf("opt-out with a real store and sealed config: reason = %q, want no_key", got) } // A store that is configured but will not open is "cannot tell". broken := newConfigGuard(func(string) string { return "" }, func(context.Context) (release.Store, error) { return nil, fmt.Errorf("aws: no credentials") }) broken.now = e.clock.now if got := reasonOfGate(t, broken.Gate(context.Background(), guardRoute("a.example.com"))); got != appconfig.ReasonStoreError { t.Errorf("unopenable store reason = %q, want store_error", got) } } // A route whose Unit is named but whose Backing is empty is exec-managed and // gated like any other. func TestGuardGatesAUnitNamedExecInstance(t *testing.T) { e := newGuardEnv(t) e.putConfig("bab", "shop", map[string]appconfig.Var{"DB": e.seal("pw")}) g := e.guard("") rt := guardRoute("a.example.com") rt.Unit = "x.service" if got := reasonOfGate(t, g.Gate(context.Background(), rt)); got != appconfig.ReasonNoKey { t.Errorf("reason = %q, want no_key", got) } rt.Backing = registry.BackingUnit if err := g.Gate(context.Background(), rt); err != nil { t.Errorf("a true unit-backed route was gated: %v", err) } } // There is no last-known-good fallback. A rendered .config.env left on // disk proves nothing about its values: names alone pass for "A=old", for empty // values, or for credentials rendered for another environment. A fresh start // needs the whole merged config to render now. func TestGuardIgnoresLeftoverRenderedFiles(t *testing.T) { foreign := func(e *guardEnv) string { v, err := appconfig.SealUnbound(e.other.PublicKey(), []byte("z")) if err != nil { t.Fatal(err) } return v } cases := map[string]struct { file string env string // the route's bound environment }{ "wrong values": {"A=old\nC=wrong\n", ""}, "empty values": {"A=\nC=\n", ""}, "every name present": {"# header\nA=x\nC=y\nP=1\n", ""}, "environment rebound": {"A=prod-secret\nC=prod-secret\n", "staging"}, } for name, tc := range cases { t.Run(name, func(t *testing.T) { e := newGuardEnv(t) e.putConfig("bab", "shop", map[string]appconfig.Var{ "A": {Kind: "secret", Envelope: foreign(e)}, // sealed for another key "C": e.seal("ok"), "P": {Kind: "env", Value: "1"}, }) if tc.env != "" { body, _ := json.Marshal(appconfig.Manifest{Schema: 1, App: "shop", Vars: map[string]appconfig.Var{"S": e.seal("staging")}}) if _, err := e.store.Store.Put(context.Background(), appconfig.EnvironmentKey("bab", "shop", tc.env), body, ""); err != nil { t.Fatal(err) } } g := e.guard(keyB64(e.priv)) rt := guardRoute("a.example.com") rt.Environment = tc.env if err := os.WriteFile(appconfig.ConfigEnvPath(e.dataDir, rt), []byte(tc.file), 0o600); err != nil { t.Fatal(err) } if got := reasonOfGate(t, g.Gate(context.Background(), rt)); got != appconfig.ReasonCannotOpen { t.Fatalf("reason = %q, want cannot_open: a leftover file must not admit a mixed-key config", got) } }) } } // The bypass is loud: logged at startup and once per admitted host, and // published in the private status block, and only while it is in effect. func TestGuardBypassIsLoud(t *testing.T) { e := newGuardEnv(t) noStore := func(context.Context) (release.Store, error) { return nil, errNoDeployStore } env := func(k string) string { if k == "CARLOS_NO_SEALED_CONFIG" { return "1" } return "" } g := newConfigGuard(env, noStore) g.now = e.clock.now logs := guardCaptureLog(t) if got := g.Status(); got != "bypassed (CARLOS_NO_SEALED_CONFIG)" { t.Errorf("status = %q", got) } g.Announce() if !strings.Contains(logs.String(), "BYPASSED") { t.Errorf("startup line missing:\n%s", logs.String()) } for i := 0; i < 3; i++ { if err := g.Gate(context.Background(), guardRoute("a.example.com")); err != nil { t.Fatal(err) } } _ = g.Gate(context.Background(), guardRoute("b.example.com")) if n := strings.Count(logs.String(), "BYPASS: admitting a.example.com"); n != 1 { t.Errorf("a.example.com logged %d times, want once:\n%s", n, logs.String()) } if n := strings.Count(logs.String(), "BYPASS: admitting b.example.com"); n != 1 { t.Errorf("b.example.com logged %d times, want once", n) } // The flag beside a real store is ignored, and says so, and is not // reported as a bypass. withStore := newConfigGuard(env, func(context.Context) (release.Store, error) { return e.store, nil }) logs2 := guardCaptureLog(t) if got := withStore.Status(); got != "" { t.Errorf("flag with a store reported %q", got) } withStore.Announce() if !strings.Contains(logs2.String(), "ignored") || strings.Contains(logs2.String(), "BYPASSED") { t.Errorf("flag-with-store line wrong:\n%s", logs2.String()) } // No flag: quiet, and nothing published. plain := newConfigGuard(func(string) string { return "" }, noStore) logs3 := guardCaptureLog(t) plain.Announce() if plain.Status() != "" || logs3.Len() != 0 { t.Errorf("a box without the flag said something: %q / %q", plain.Status(), logs3.String()) } } // The opt-out never covers a store that is configured and failing: the box // cannot tell, and refuses, flag or not. func TestGuardOptOutDoesNotCoverAConfiguredStoreError(t *testing.T) { e := newGuardEnv(t) env := func(k string) string { if k == "CARLOS_NO_SEALED_CONFIG" { return "1" } return "" } // Opening fails. broken := newConfigGuard(env, func(context.Context) (release.Store, error) { return nil, fmt.Errorf("aws: no credentials") }) broken.now = e.clock.now if got := reasonOfGate(t, broken.Gate(context.Background(), guardRoute("a.example.com"))); got != appconfig.ReasonStoreError { t.Errorf("open failure with the opt-out: reason = %q, want store_error", got) } if broken.Status() != "" { t.Errorf("an open failure was reported as a bypass: %q", broken.Status()) } // Opens, then reads fail. e.putConfig("bab", "shop", map[string]appconfig.Var{"DB": e.seal("pw")}) e.store.setFail(errors.New("s3: 503")) flaky := newConfigGuard(env, func(context.Context) (release.Store, error) { return e.store, nil }) flaky.now = e.clock.now if got := reasonOfGate(t, flaky.Gate(context.Background(), guardRoute("a.example.com"))); got != appconfig.ReasonStoreError { t.Errorf("read failure with the opt-out: reason = %q, want store_error", got) } } func TestGuardRefusalsAreSortedCappedAndPruned(t *testing.T) { e := newGuardEnv(t) e.putConfig("bab", "shop", map[string]appconfig.Var{"DB": e.seal("pw")}) g := e.guard("") ctx := context.Background() n := configGuardListCap + 5 for i := 0; i < n; i++ { _ = g.Gate(ctx, guardRoute(fmt.Sprintf("h%03d.example.com", i))) e.clock.advance(time.Second) } list, total := g.Refusals() if total != n || len(list) != configGuardListCap { t.Fatalf("list %d total %d, want %d and %d", len(list), total, configGuardListCap, n) } // Newest first, so the cap drops the oldest. if list[0].Host != fmt.Sprintf("h%03d.example.com", n-1) { t.Errorf("first = %s, want the newest", list[0].Host) } // A refusal nobody has asked about for a day is no longer current. e.clock.advance(configGuardRefusalMaxAge + time.Hour) if list, total := g.Refusals(); len(list) != 0 || total != 0 { t.Errorf("stale refusals survived: %d/%d", len(list), total) } } // Nothing secret in what is published: reason words and hosts only. func TestGuardPublishesNoSecretMaterial(t *testing.T) { e := newGuardEnv(t) v := e.seal("hunter2") e.putConfig("bab", "shop", map[string]appconfig.Var{"DB": v}) g := e.guard(keyB64(e.other)) logs := guardCaptureLog(t) _ = g.Gate(context.Background(), guardRoute("a.example.com")) list, _ := g.Refusals() raw, _ := json.Marshal(list) for _, s := range []string{string(raw), logs.String()} { for _, bad := range []string{v.Envelope, "hunter2", keyB64(e.other), keyB64(e.priv)} { if strings.Contains(s, bad) { t.Errorf("secret material %q in %s", bad, s) } } } } // The agent wires one armed guard into the reconcile config and the status // probe, and a box with none armed behaves exactly as before. func TestArmedGuardReachesReconcileConfigAndStatus(t *testing.T) { armConfigGuard(nil) t.Cleanup(func() { armConfigGuard(nil) }) if cfg := provisionTickConfig(func(string) string { return "" }); cfg.ConfigGate != nil { t.Error("no armed guard, but the reconcile config carries a gate") } if list, total := configRefusalProbe(); list != nil || total != 0 { t.Errorf("no armed guard, but the probe reports %v/%d", list, total) } e := newGuardEnv(t) e.putConfig("bab", "shop", map[string]appconfig.Var{"DB": e.seal("pw")}) g := e.guard("") armConfigGuard(g) cfg := provisionTickConfig(func(string) string { return "" }) if cfg.ConfigGate == nil { t.Fatal("armed guard not wired into the reconcile config") } if err := cfg.ConfigGate(context.Background(), guardRoute("a.example.com")); err == nil { t.Error("the reconcile gate admitted a sealed instance on a keyless box") } if list, total := configRefusalProbe(); len(list) != 1 || total != 1 { t.Errorf("probe = %v/%d, want the one refusal", list, total) } }