package activator import ( "context" "errors" "fmt" "os" "path/filepath" "strings" "testing" "time" "github.com/carlosframework/platform/internal/instancekey" "github.com/carlosframework/platform/internal/registry" ) // The app the replica tests launch is this test binary, running // TestInstanceKeyHelperProcess: rastrillo e3ea08de's carlos.InstanceKey // and its documented caller, so a test sees what an adopting app does, not // only which variables it was handed. const helperOutEnv = "GO_TEST_INSTANCE_KEY_HELPER_OUT" var ( errHelperNoInstanceKey = errors.New("ErrNoInstanceKey") errHelperNotOnCarlos = errors.New("ErrNotOnCarlos") ) // helperInstanceKey is rastrillo e3ea08de's InstanceKey(own) with Running(): // own, then $CARLOS_INSTANCE_KEY verbatim, then ErrNoInstanceKey when // $CARLOS_CONTROL_SOCKET is set, else ErrNotOnCarlos. func helperInstanceKey(own string) (string, error) { if own != "" { return own, nil } if k := os.Getenv(InstanceKeyVar); k != "" { return k, nil } if os.Getenv("CARLOS_CONTROL_SOCKET") != "" { return "", errHelperNoInstanceKey } return "", errHelperNotOnCarlos } // TestInstanceKeyHelperProcess is the app, not a test: with no output path // set it does nothing. It resolves its key as an adopting app with no key // of its own does, records the outcome, and exits: // - "key ": it sealed with the key it was handed; // - "refused": ErrNoInstanceKey, it did not start; // - "minted": ErrNotOnCarlos, and the documented caller minted its own. func TestInstanceKeyHelperProcess(t *testing.T) { out := os.Getenv(helperOutEnv) if out == "" { return } var outcome string switch key, err := helperInstanceKey(""); { case err == nil: outcome = "key " + instancekey.Digest(key) case errors.Is(err, errHelperNoInstanceKey): outcome = "refused" default: outcome = "minted" } _ = os.WriteFile(out+".tmp", []byte(outcome), 0o600) _ = os.Rename(out+".tmp", out) os.Exit(0) } // stageHelperApp makes this test binary host's live build, running only // TestInstanceKeyHelperProcess, and returns where its outcome lands. func stageHelperApp(t *testing.T, b *ExecBackend, host, ver string) string { t.Helper() self, err := os.Executable() if err != nil { t.Fatal(err) } out := filepath.Join(t.TempDir(), "outcome") t.Setenv(helperOutEnv, out) vdir := filepath.Join(b.OptDir, "versions", ver) if err := os.MkdirAll(vdir, 0o755); err != nil { t.Fatal(err) } bin := filepath.Join(vdir, "app") script := fmt.Sprintf("#!/bin/sh\nexec '%s' -test.run='^TestInstanceKeyHelperProcess$'\n", self) if err := os.WriteFile(bin, []byte(script), 0o755); err != nil { t.Fatal(err) } if err := os.MkdirAll(filepath.Join(b.OptDir, "live"), 0o755); err != nil { t.Fatal(err) } os.Remove(b.liveLink(host)) if err := os.Symlink(bin, b.liveLink(host)); err != nil { t.Fatal(err) } return out } // outcomeOf waits up to two seconds for the helper's outcome; "" is no app // ran at all: none recorded, and no child of this test binary is left // running that could (awaitChildFile). Call it once the launch returned. func outcomeOf(path string) string { return awaitChildFile(path, 2*time.Second) } // noControlSocket is a socket path too long to bind, so the launch's // control listener fails and CARLOS_CONTROL_SOCKET is withheld: the same // launch a claim box runs before O6, and a flat launch whose bind failed. func noControlSocket(t *testing.T) string { return filepath.Join(t.TempDir(), strings.Repeat("s", 120)+".sock") } type replicaCase struct { name string replica bool own, config string // env file bodies ("" = no config file) homeKey string // the home's effective key the box lists ("" = none listed) start string // "refused" (no app runs) or the helper outcome wanted wantKeyFromOwnOK bool // a home row: the outcome is the key minted into .env // bucket is the key the replica's configuration in the bucket renders // now (ConfiguredKey; "" renders none). bucketDown makes that read // fail; noBucket leaves ConfiguredKey nil. bucket string bucketDown, noBucket bool } // freshConfig is ExecBackend.ConfiguredKey for one case. func freshConfig(tc replicaCase) func(context.Context, registry.Route) (string, error) { if tc.noBucket { return nil } return func(context.Context, registry.Route) (string, error) { switch { case tc.bucketDown: return "", errors.New("bucket unreachable") case tc.bucket == "": return "", nil } return instancekey.Digest(tc.bucket), nil } } // Rulings 26 to 28 through a real launch and the helper (Astra rounds 4 and // 5): a replica runs only with exactly its home's key, listed by the // pre-roll check and, when configured, still the bucket's configured key; // or with none and a control socket its helper refuses on. Every launch // here has no control socket, which is where the helper would otherwise // mint. func TestReplicaLaunchWithoutAControlSocket(t *testing.T) { tok := "CARLOS_ADMIN_TOKEN=" + strings.Repeat("c", 64) + "\n" homeKey := strings.Repeat("H", 43) + "=" for _, tc := range []replicaCase{ {name: "a keyless replica is refused by the agent, not left to mint", replica: true, own: tok, start: "refused"}, {name: "a configured key that is the home's runs and seals with it", replica: true, own: tok, config: InstanceKeyVar + "=" + homeKey + "\n", homeKey: homeKey, bucket: homeKey, start: "key " + instancekey.Digest(homeKey)}, {name: "a pre-existing env key that is the home's runs", replica: true, own: tok + InstanceKeyVar + "=" + homeKey + "\n", homeKey: homeKey, start: "key " + instancekey.Digest(homeKey)}, {name: "a configured key that differs is refused", replica: true, own: tok, config: InstanceKeyVar + "=configured-B\n", homeKey: homeKey, bucket: "configured-B", start: "refused"}, {name: "a configured key with no verified home key is refused", replica: true, own: tok, config: InstanceKeyVar + "=" + homeKey + "\n", bucket: homeKey, start: "refused"}, // Ruling 27, Astra round 5 finding 1: the key changed in the bucket // to B and the home already holds it, while this box's sync is // behind and the replica's configuration and list entry still say A. {name: "the home updated, this replica's configuration delayed: refused", replica: true, own: tok, config: InstanceKeyVar + "=" + homeKey + "\n", homeKey: homeKey, bucket: "configured-B", start: "refused"}, {name: "the bucket cannot be read: refused", replica: true, own: tok, config: InstanceKeyVar + "=" + homeKey + "\n", homeKey: homeKey, bucketDown: true, start: "refused"}, {name: "no fresh reader: refused", replica: true, own: tok, config: InstanceKeyVar + "=" + homeKey + "\n", homeKey: homeKey, noBucket: true, start: "refused"}, // Ruling 27's residual, documented: a change the bucket does not // hold (the home's own .env edited by hand, or a per-host key // configured on the home only) is invisible here. The replica's // list entry and the bucket both still say A, so it runs with A. // Only the procedure, or O8's fresh evidence, closes this. {name: "residual: a change only on the home is not seen, and the replica runs with the old key", replica: true, own: tok, config: InstanceKeyVar + "=" + homeKey + "\n", homeKey: homeKey, bucket: homeKey, start: "key " + instancekey.Digest(homeKey)}, // Ruling 28: a home whose key exists only in configuration, with a // replica that is handed none (its own overlay blanks it). O8 never // delivers a configured key, so the replica stays keyless and is // refused here, where it has no control socket. {name: "a config-only home's keyless replica is refused", replica: true, own: tok, config: InstanceKeyVar + "=\n", homeKey: homeKey, bucket: "", start: "refused"}, {name: "a home row mints and its app seals with the minted key", own: tok, wantKeyFromOwnOK: true}, } { t.Run(tc.name, func(t *testing.T) { b := testExecBackend(t) rt := registry.Route{Host: "shop-au.example.com", Kind: "instance", Replica: tc.replica, Socket: noControlSocket(t)} own := filepath.Join(b.DataDir, rt.Host+".env") if err := os.WriteFile(own, []byte(tc.own), 0o600); err != nil { t.Fatal(err) } if tc.config != "" { if err := os.WriteFile(filepath.Join(b.DataDir, rt.Host+".config.env"), []byte(tc.config), 0o600); err != nil { t.Fatal(err) } } if tc.homeKey != "" { b.ReplicaHomeKeys = func(h string) (string, bool) { return instancekey.Digest(tc.homeKey), h == rt.Host } } b.ConfiguredKey = freshConfig(tc) out := stageHelperApp(t, b, rt.Host, "v1") ctx, cancel := recordedCtx(2*time.Second, out) defer cancel() proc, _, err := b.Start(ctx, rt) if proc != nil { defer proc.Stop(context.Background()) } got := outcomeOf(out) if tc.wantKeyFromOwnOK { env, _ := readEnvFile(own) if got != "key "+instancekey.Digest(env[InstanceKeyVar]) { t.Fatalf("home row: outcome %q, want the minted key", got) } return } if tc.start == "refused" { if !errors.Is(err, ErrReplicaKey) || got != "" { t.Fatalf("Start = %v, app outcome %q; want ErrReplicaKey and no app", err, got) } } else if got != tc.start { t.Fatalf("app outcome %q, want %q (Start: %v)", got, tc.start, err) } if raw, _ := os.ReadFile(own); string(raw) != tc.own { t.Fatal("a replica launch changed its own env file: a replica never mints") } }) } } // A replica's seed has no control socket: a keyless replica's seed is // refused before it runs; one handed its home's key runs. func TestReplicaSeedIsGated(t *testing.T) { tok := "CARLOS_ADMIN_TOKEN=" + strings.Repeat("c", 64) + "\n" homeKey := strings.Repeat("H", 43) + "=" for _, tc := range []struct { name, config string want string }{ {"keyless", "", ""}, {"the home's key", InstanceKeyVar + "=" + homeKey + "\n", "key " + instancekey.Digest(homeKey)}, } { t.Run(tc.name, func(t *testing.T) { b := testExecBackend(t) rt := registry.Route{Host: "shop-au.example.com", Kind: "instance", Replica: true, Seed: `["seed"]`} if err := os.WriteFile(filepath.Join(b.DataDir, rt.Host+".env"), []byte(tok), 0o600); err != nil { t.Fatal(err) } if tc.config != "" { if err := os.WriteFile(filepath.Join(b.DataDir, rt.Host+".config.env"), []byte(tc.config), 0o600); err != nil { t.Fatal(err) } } b.ReplicaHomeKeys = func(h string) (string, bool) { return instancekey.Digest(homeKey), true } b.ConfiguredKey = freshConfig(replicaCase{bucket: homeKey}) out := stageHelperApp(t, b, rt.Host, "v1") err := b.Seed(context.Background(), rt) got := outcomeOf(out) if tc.want == "" { if !errors.Is(err, ErrReplicaKey) || got != "" { t.Fatalf("Seed = %v, outcome %q; want ErrReplicaKey and no seed process", err, got) } return } if err != nil || got != tc.want { t.Fatalf("Seed = %v, outcome %q; want %q", err, got, tc.want) } }) } } // A worker (StartWorkerStaged) has no control socket, so a replica row's // worker is gated as its seed is (ruling 26), on the layer from the same // read: it runs only with its home's key, verified, and is refused with a // key that is not the home's or with none at all. A worker never mints // (ruling 12), and a worker on a row that is not a replica is unaffected. func TestReplicaWorkerIsGatedLikeASeed(t *testing.T) { tok := "CARLOS_ADMIN_TOKEN=" + strings.Repeat("c", 64) + "\n" homeKey := strings.Repeat("H", 43) + "=" for _, tc := range []struct { name string replica bool config string // .config.env ("" = none) bucket string // the key the bucket's configuration renders now want string // the helper's outcome; "" = refused, no worker ran // extraKey hands the home's key through extra as well as the env // files: a key whose origin the read cannot vouch for. extraKey bool }{ {"a replica worker with the home's key runs", true, InstanceKeyVar + "=" + homeKey + "\n", homeKey, "key " + instancekey.Digest(homeKey), false}, // The bucket renders configured-B too, so only the home-key digest // check can refuse this one. {"a replica worker with a different configured key is refused", true, InstanceKeyVar + "=configured-B\n", "configured-B", "", false}, {"a keyless replica worker is refused", true, "", "", "", false}, {"a worker on a row that is not a replica is unaffected", false, InstanceKeyVar + "=configured-B\n", "", "key " + instancekey.Digest("configured-B"), false}, {"a replica worker handed a key through extra is refused", true, InstanceKeyVar + "=" + homeKey + "\n", homeKey, "", true}, } { t.Run(tc.name, func(t *testing.T) { b := testExecBackend(t) rt := registry.Route{Host: "shop-au.example.com", Kind: "sidecar", Args: `["sidecar","run"]`, Replica: tc.replica} own := filepath.Join(b.DataDir, rt.Host+".env") if err := os.WriteFile(own, []byte(tok), 0o600); err != nil { t.Fatal(err) } if tc.config != "" { if err := os.WriteFile(filepath.Join(b.DataDir, rt.Host+".config.env"), []byte(tc.config), 0o600); err != nil { t.Fatal(err) } } if tc.replica { b.ReplicaHomeKeys = func(h string) (string, bool) { return instancekey.Digest(homeKey), h == rt.Host } b.ConfiguredKey = freshConfig(replicaCase{bucket: tc.bucket}) } out := stageHelperApp(t, b, rt.Host, "v1") extra := map[string]string{"CARLOS_SIDECAR_NAME": "bot"} if tc.extraKey { extra[InstanceKeyVar] = homeKey } proc, _, err := b.StartWorkerStaged(context.Background(), rt, extra, nil) if proc != nil { defer proc.Stop(context.Background()) } got := outcomeOf(out) if tc.want == "" { if !errors.Is(err, ErrReplicaKey) || proc != nil || got != "" { t.Fatalf("StartWorkerStaged = %v (proc %v), outcome %q; want ErrReplicaKey and no worker", err, proc != nil, got) } } else if err != nil || got != tc.want { t.Fatalf("StartWorkerStaged = %v, outcome %q; want %q", err, got, tc.want) } if raw, _ := os.ReadFile(own); string(raw) != tok { t.Fatal("a worker launch changed its own env file: a worker never mints") } }) } } // The legacy co-spawned sidecar (-sidecar.env, StartSidecar) runs the // app binary with no control socket, so on a replica host it is gated as // the worker is (ruling 26): the key in its own file, if any, must be the // home's verified key, and a keyless one is refused rather than left to // mint. A sidecar on a host that is not a replica is unaffected. func TestReplicaLegacySidecarIsGatedLikeTheWorker(t *testing.T) { homeKey := strings.Repeat("H", 43) + "=" for _, tc := range []struct { name string replica bool sidecar string // -sidecar.env want string // the helper's outcome; "" = refused, no sidecar ran }{ {"a replica's sidecar with the home's key runs", true, InstanceKeyVar + "=" + homeKey + "\n", "key " + instancekey.Digest(homeKey)}, {"a replica's sidecar with another key is refused", true, InstanceKeyVar + "=configured-B\n", ""}, {"a replica's keyless sidecar is refused", true, "OTHER=1\n", ""}, {"a sidecar on a host that is not a replica is unaffected", false, InstanceKeyVar + "=configured-B\n", "key " + instancekey.Digest("configured-B")}, } { t.Run(tc.name, func(t *testing.T) { b := testExecBackend(t) rt := registry.Route{Host: "shop-au.example.com", Kind: "instance", Replica: tc.replica} sidecarEnv := filepath.Join(b.DataDir, rt.Host+"-sidecar.env") if err := os.WriteFile(sidecarEnv, []byte(tc.sidecar), 0o600); err != nil { t.Fatal(err) } if tc.replica { b.ReplicaHomeKeys = func(h string) (string, bool) { return instancekey.Digest(homeKey), h == rt.Host } // The bucket agrees with whatever the case holds, so only the // home-key check decides. b.ConfiguredKey = freshConfig(replicaCase{bucket: "configured-B"}) } out := stageHelperApp(t, b, rt.Host, "v1") proc, err := b.StartSidecar(context.Background(), rt) if proc != nil { defer proc.Stop(context.Background()) } got := outcomeOf(out) if tc.want == "" { if !errors.Is(err, ErrReplicaKey) || proc != nil || got != "" { t.Fatalf("StartSidecar = %v (proc %v), outcome %q; want ErrReplicaKey and no sidecar", err, proc != nil, got) } } else if err != nil || got != tc.want { t.Fatalf("StartSidecar = %v, outcome %q; want %q", err, got, tc.want) } if raw, _ := os.ReadFile(sidecarEnv); string(raw) != tc.sidecar { t.Fatal("a sidecar launch changed its env file: nothing mints there") } }) } } // Two guards that a later check would otherwise back up: with no list entry // for the replica, or a bucket read that fails, the launch is refused for // that reason, named, and not merely because a digest compared against "" // failed. The refusal names the file an operator must fix. func TestReplicaGateNamesWhyItRefused(t *testing.T) { keyA := strings.Repeat("A", 43) + "=" rt := registry.Route{Host: "shop-au.example.com", Kind: "instance", Replica: true} env := map[string]string{InstanceKeyVar: keyA} listed := func(string) (string, bool) { return instancekey.Digest(keyA), true } bucketA := func(context.Context, registry.Route) (string, error) { return instancekey.Digest(keyA), nil } for _, tc := range []struct { name string homes func(string) (string, bool) bucket func(context.Context, registry.Route) (string, error) want string }{ {"no entry in the list", func(string) (string, bool) { return "", false }, bucketA, "no entry for it in /etc/carlos/replica-home-keys"}, {"no list reader at all", nil, bucketA, "no entry for it in /etc/carlos/replica-home-keys"}, {"the bucket read fails", listed, func(context.Context, registry.Route) (string, error) { return "", errors.New("bucket unreachable") }, "the bucket's configuration could not be read: bucket unreachable"}, } { t.Run(tc.name, func(t *testing.T) { b := &ExecBackend{DataDir: t.TempDir(), ReplicaHomeKeys: tc.homes, ConfiguredKey: tc.bucket} err := b.replicaKeyGate(rt, env, instancekey.LayerConfig, false) if !errors.Is(err, ErrReplicaKey) || !strings.Contains(err.Error(), tc.want) { t.Fatalf("gate = %v; want ErrReplicaKey naming %q", err, tc.want) } }) } } // Final review I5 and minor 5: the gate's refusal is logged once an hour // per host and reason, so a keyless replica's legacy sidecar, refused at // every retry of the sidecar sweep, does not fill the log; a new reason or // another host is still heard. A bucket error is logged apart from the // fixed reason. func TestReplicaGateLogsARefusalOncePerHostAndReason(t *testing.T) { buf := captureLog(t) b := &ExecBackend{DataDir: t.TempDir()} rt := registry.Route{Host: "ratelimit-au.example.com", Kind: "instance", Replica: true} for i := 0; i < 3; i++ { if err := b.replicaKeyGate(rt, map[string]string{}, instancekey.LayerNone, false); !errors.Is(err, ErrReplicaKey) { t.Fatalf("gate %d = %v", i, err) } } other := rt other.Host = "ratelimit-eu.example.com" if err := b.replicaKeyGate(other, map[string]string{}, instancekey.LayerNone, false); !errors.Is(err, ErrReplicaKey) { t.Fatal(err) } b.ReplicaHomeKeys = func(string) (string, bool) { return instancekey.Digest("home"), true } b.ConfiguredKey = func(context.Context, registry.Route) (string, error) { return "", errors.New("bucket unreachable") } env := map[string]string{instancekey.Var: "home"} for i := 0; i < 2; i++ { if err := b.replicaKeyGate(rt, env, instancekey.LayerConfig, false); !errors.Is(err, ErrReplicaKey) || !strings.Contains(err.Error(), "bucket unreachable") { t.Fatalf("gate with the bucket down = %v", err) } } lines := strings.Split(strings.TrimSpace(buf.String()), "\n") var au, eu, bucket int for _, l := range lines { switch { case strings.Contains(l, "replica ratelimit-au.example.com not started: the bucket's configuration could not be read: bucket unreachable"): bucket++ case strings.Contains(l, "replica ratelimit-au.example.com not started"): au++ case strings.Contains(l, "replica ratelimit-eu.example.com not started"): eu++ } } if au != 1 || eu != 1 || bucket != 1 { t.Fatalf("refusal lines: au %d, eu %d, bucket %d; want one each:\n%s", au, eu, bucket, buf.String()) } }