package console // The console half of the deploy signal (carlos-deploy spec §3): every bucket // write a box's fast loop consumes bumps control/serial, so a box notices in // seconds instead of waiting out its hourly floor. The properties pinned here, // in the order they'd hurt if lost: the write's own success never depends on // the bump landing, the bump DOES land on each door that moves something a box // reads, and the promote API says whether what it just moved is held by a bake // window — so the CLI can report the hold without a second round trip. import ( "context" "encoding/json" "net/http" "net/url" "strings" "testing" "time" "github.com/carlosframework/platform/internal/release" ) // serial reads the bucket's serial counter — 0 when the object was never // written, which is exactly what "nothing bumped" looks like. func serial(t *testing.T, fx *consoleFixture) int64 { t.Helper() doc, _, err := release.ReadSerial(context.Background(), fx.rel) if err != nil { t.Fatalf("read serial: %v", err) } return doc.Serial } // TestAPIPromoteBumpsSerial: the CLI's promote door moves a channel pointer — // the archetypal box-consumed write — so the serial goes from absent (0) to 1. func TestAPIPromoteBumpsSerial(t *testing.T) { fx := newConsoleFixture(t) if _, err := fx.st.CreateAppIn(OpsAccountID, "titogo", "paul@keymail.dev", time.Now()); err != nil { t.Fatal(err) } shipVersion(t, fx, "titogo", "abc1234") token := cliToken(t, fx, "paul@keymail.dev", "code-serial-promote") if got := serial(t, fx); got != 0 { t.Fatalf("serial before promote = %d, want 0", got) } status, out := postJSON(t, fx.ts.URL+"/api/cli/apps/bab/titogo/promote", map[string]string{"version": "abc1234", "channel": "canary/x"}, token) if status != http.StatusOK { t.Fatalf("promote = %d %v", status, out) } if got := serial(t, fx); got != 1 { t.Fatalf("serial after promote = %d, want 1", got) } } // TestAPIPromoteNoOpDoesNotBump: a promotion that moved nothing announces // nothing. The serial is "the bucket has news"; a no-op is not news, and a // bump for it would wake every box in the fleet for a pointer that did not // change. func TestAPIPromoteNoOpDoesNotBump(t *testing.T) { fx := newConsoleFixture(t) if _, err := fx.st.CreateAppIn(OpsAccountID, "titogo", "paul@keymail.dev", time.Now()); err != nil { t.Fatal(err) } shipVersion(t, fx, "titogo", "abc1234") seedChannel(t, fx, "titogo", "canary/x", "abc1234") token := cliToken(t, fx, "paul@keymail.dev", "code-serial-noop") status, out := postJSON(t, fx.ts.URL+"/api/cli/apps/bab/titogo/promote", map[string]string{"version": "abc1234", "channel": "canary/x"}, token) if status != http.StatusOK || str(t, out, "result") != "noop" { t.Fatalf("promote = %d %v, want 200 noop", status, out) } if got := serial(t, fx); got != 0 { t.Fatalf("serial after a no-op promote = %d, want 0", got) } } // TestAPIPromoteRefusedDoesNotBump: a refusal wrote nothing, so it announces // nothing — the bump sits strictly after the write, never before the gates. func TestAPIPromoteRefusedDoesNotBump(t *testing.T) { fx := newConsoleFixture(t) if _, err := fx.st.CreateAppIn(OpsAccountID, "titogo", "paul@keymail.dev", time.Now()); err != nil { t.Fatal(err) } shipVersion(t, fx, "titogo", "abc1234") token := cliToken(t, fx, "paul@keymail.dev", "code-serial-refused") status, _ := postRaw(t, fx.ts.URL+"/api/cli/apps/bab/titogo/promote", map[string]string{"version": "abc1234", "channel": "stable"}, token) if status != http.StatusForbidden { t.Fatalf("stable promote = %d, want 403", status) } if got := serial(t, fx); got != 0 { t.Fatalf("serial after a refused promote = %d, want 0", got) } } // held decodes only the field this task adds, which is also the shape the CLI // reads: an absent "held" and an explicit false are the same answer. func held(t *testing.T, body string) bool { t.Helper() var out struct { Held bool `json:"held"` } if err := json.Unmarshal([]byte(body), &out); err != nil { t.Fatalf("decode %q: %v", body, err) } return out.Held } // TestAPIPromoteHeldOnProductionBeta: beta is the one bakeable rung this door // can move, and a PRODUCTION app's beta head waits out the window before any // host adopts it. The response says so, so `carlos deploy` can report the hold // without a second call. func TestAPIPromoteHeldOnProductionBeta(t *testing.T) { fx := newConsoleFixture(t) if _, err := fx.st.CreateAppIn(OpsAccountID, "titogo", "paul@keymail.dev", time.Now()); err != nil { t.Fatal(err) } shipVersion(t, fx, "titogo", "abc1234") // The ladder: beta is entered from edge only. seedChannel(t, fx, "titogo", "edge", "abc1234") if err := release.SetProduction(context.Background(), fx.rel, fx.priv, release.OpsAccount, "titogo", true, "test"); err != nil { t.Fatal(err) } token := cliToken(t, fx, "paul@keymail.dev", "code-serial-held") status, body := postRaw(t, fx.ts.URL+"/api/cli/apps/bab/titogo/promote", map[string]string{"version": "abc1234", "channel": "beta"}, token) if status != http.StatusOK { t.Fatalf("beta promote = %d %s", status, body) } if !held(t, body) { t.Fatalf("beta promote on a production app = %s, want held true", body) } // The existing fields are untouched — `held` is additive. if !strings.Contains(body, `"result":"promoted"`) { t.Fatalf("promote body lost its result field: %s", body) } } // TestAPIPromoteNotHeldOffProduction: the same beta promotion on an app that // has NOT declared production adopts immediately — nothing bakes — so held is // false. Canary never bakes at all, in either state. func TestAPIPromoteNotHeldOffProduction(t *testing.T) { fx := newConsoleFixture(t) if _, err := fx.st.CreateAppIn(OpsAccountID, "titogo", "paul@keymail.dev", time.Now()); err != nil { t.Fatal(err) } shipVersion(t, fx, "titogo", "abc1234") seedChannel(t, fx, "titogo", "edge", "abc1234") token := cliToken(t, fx, "paul@keymail.dev", "code-serial-notheld") status, body := postRaw(t, fx.ts.URL+"/api/cli/apps/bab/titogo/promote", map[string]string{"version": "abc1234", "channel": "beta"}, token) if status != http.StatusOK { t.Fatalf("beta promote = %d %s", status, body) } if held(t, body) { t.Fatalf("beta promote on a non-production app = %s, want held false", body) } // Canary on the SAME app once production is on: the rung has no window, // so the flag cannot make it held. if err := release.SetProduction(context.Background(), fx.rel, fx.priv, release.OpsAccount, "titogo", true, "test"); err != nil { t.Fatal(err) } status, body = postRaw(t, fx.ts.URL+"/api/cli/apps/bab/titogo/promote", map[string]string{"version": "abc1234", "channel": "canary/x"}, token) if status != http.StatusOK { t.Fatalf("canary promote = %d %s", status, body) } if held(t, body) { t.Fatalf("canary promote on a production app = %s, want held false — canaries never bake", body) } } // TestAPIPromoteNoOpReportsHeld: the re-run case. `carlos deploy` twice with // the same version — a CI retry, a repeated command — is a NO-OP, and on a // production app whose beta head is mid-bake it is a no-op that is STILL HELD. // A no-op answer without the field decodes as held:false and sends the client // off to watch for an adopt a day away, which is the exact false failure the // field exists to prevent. func TestAPIPromoteNoOpReportsHeld(t *testing.T) { fx := newConsoleFixture(t) if _, err := fx.st.CreateAppIn(OpsAccountID, "titogo", "paul@keymail.dev", time.Now()); err != nil { t.Fatal(err) } shipVersion(t, fx, "titogo", "abc1234") seedChannel(t, fx, "titogo", "beta", "abc1234") seedChannel(t, fx, "titogo", "canary/x", "abc1234") if err := release.SetProduction(context.Background(), fx.rel, fx.priv, release.OpsAccount, "titogo", true, "test"); err != nil { t.Fatal(err) } token := cliToken(t, fx, "paul@keymail.dev", "code-serial-noopheld") // beta already carries this version: a no-op, and held. status, body := postRaw(t, fx.ts.URL+"/api/cli/apps/bab/titogo/promote", map[string]string{"version": "abc1234", "channel": "beta"}, token) if status != http.StatusOK || !strings.Contains(body, `"result":"noop"`) { t.Fatalf("repeat beta promote = %d %s, want 200 noop", status, body) } if !held(t, body) { t.Fatalf("no-op beta promote on a production app = %s, want held true", body) } // The same re-run on a rung with no window is a no-op that is NOT held — // so the field tracks the bake, not merely the no-op. status, body = postRaw(t, fx.ts.URL+"/api/cli/apps/bab/titogo/promote", map[string]string{"version": "abc1234", "channel": "canary/x"}, token) if status != http.StatusOK || !strings.Contains(body, `"result":"noop"`) { t.Fatalf("repeat canary promote = %d %s, want 200 noop", status, body) } if held(t, body) { t.Fatalf("no-op canary promote = %s, want held false — canaries never bake", body) } // Still no announcement: neither call moved a pointer. if got := serial(t, fx); got != 0 { t.Fatalf("serial after two no-op promotes = %d, want 0", got) } } // TestAPIRollbackBumpsSerial: a rollback moves the same pointer a promote // does, and is the one people are in the biggest hurry about. func TestAPIRollbackBumpsSerial(t *testing.T) { fx := newConsoleFixture(t) if _, err := fx.st.CreateAppIn(OpsAccountID, "titogo", "paul@keymail.dev", time.Now()); err != nil { t.Fatal(err) } shipVersion(t, fx, "titogo", "aaa1111") shipVersion(t, fx, "titogo", "bbb2222") seedChannel(t, fx, "titogo", "canary/x", "aaa1111") seedChannel(t, fx, "titogo", "canary/x", "bbb2222") token := cliToken(t, fx, "paul@keymail.dev", "code-serial-rollback") before := serial(t, fx) status, out := postJSON(t, fx.ts.URL+"/api/cli/apps/bab/titogo/rollback", map[string]string{"channel": "canary/x"}, token) if status != http.StatusOK { t.Fatalf("rollback = %d %v", status, out) } if got := serial(t, fx); got != before+1 { t.Fatalf("serial after rollback = %d, want %d", got, before+1) } } // TestAPITrashBumpsSerial: trash through the CLI/API door writes the marker // that stops instance minting and drops the app's routes (leg 1 of the fast // pass's reconcile) — the same reason restore bumps, mirrored on the way in. func TestAPITrashBumpsSerial(t *testing.T) { fx, token := trashFixture(t, "code-serial-trash") before := serial(t, fx) status, out := postBody(t, fx.ts.URL+"/api/cli/apps/bab/titogo/delete", token, "") if status != http.StatusOK { t.Fatalf("trash = %d %s", status, out) } if got := serial(t, fx); got != before+1 { t.Fatalf("serial after an API trash = %d, want %d", got, before+1) } } // TestAPIRestoreBumpsSerial: restore through the CLI/API door re-enables // instance minting for the app (removing the trash marker), which is leg 1 of // the fast pass's reconcile — the same reason the web door (TestWebRestoreBumpsSerial) // bumps. Without it, an app restored over the API stays invisible to the // fleet until the hourly floor comes round. func TestAPIRestoreBumpsSerial(t *testing.T) { fx, token := trashFixture(t, "code-serial-restore") if status, out := postBody(t, fx.ts.URL+"/api/cli/apps/bab/titogo/delete", token, ""); status != http.StatusOK { t.Fatalf("trash = %d %s", status, out) } before := serial(t, fx) status, out := postBody(t, fx.ts.URL+"/api/cli/apps/bab/titogo/restore", token, "") if status != http.StatusOK { t.Fatalf("restore = %d %s", status, out) } if got := serial(t, fx); got != before+1 { t.Fatalf("serial after an API restore = %d, want %d", got, before+1) } } // TestAPIConfigPutBumpsSerial: the config write the box's exec config-sync // leg consumes — the difference between an env var arriving in seconds and // arriving on the hour. func TestAPIConfigPutBumpsSerial(t *testing.T) { fx, token := configFixture(t, "code-serial-config") status, out := putJSON(t, configURL(fx, "titogo"), map[string]any{"set": map[string]any{"PORT": map[string]any{"kind": "env", "value": "8080"}}}, token) if status != http.StatusOK { t.Fatalf("config put = %d %v", status, out) } if got := serial(t, fx); got != 1 { t.Fatalf("serial after a config write = %d, want 1", got) } } // TestAPIConfigSyncBumpsSerial: `carlos env sync` exists to make the fleet // converge NOW; the generation touch without the announcement would leave it // waiting on the hourly loop it was written to skip. func TestAPIConfigSyncBumpsSerial(t *testing.T) { fx, token := configFixture(t, "code-serial-configsync") status, out := postJSON(t, configURL(fx, "titogo")+"/sync", map[string]any{}, token) if status != http.StatusOK { t.Fatalf("config sync = %d %v", status, out) } if got := serial(t, fx); got != 1 { t.Fatalf("serial after a config sync = %d, want 1", got) } } // TestAPIInstanceCreateBumpsSerial: an instance record is what the box's // reconcile leg mints a route from, so the record write is box-consumed. func TestAPIInstanceCreateBumpsSerial(t *testing.T) { fx, token := instanceFixture(t, "code-serial-instance") enableInstances(t, fx, "titogo", titoManifest()) status, out := postJSON(t, instancesURL(fx, "bab", "titogo"), map[string]any{"host": "acme.go.tito.io", "region": "ie"}, token) if status != http.StatusCreated { t.Fatalf("instance create = %d %v", status, out) } if got := serial(t, fx); got != 1 { t.Fatalf("serial after an instance create = %d, want 1", got) } // And the delete, which retires the route the same loop reconciles. req, err := http.NewRequest(http.MethodDelete, instancesURL(fx, "bab", "titogo")+"/acme.go.tito.io", nil) if err != nil { t.Fatal(err) } req.Header.Set("Authorization", "Bearer "+token) res, err := http.DefaultClient.Do(req) if err != nil { t.Fatal(err) } res.Body.Close() if res.StatusCode != http.StatusOK { t.Fatalf("instance delete = %d", res.StatusCode) } if got := serial(t, fx); got != 2 { t.Fatalf("serial after an instance delete = %d, want 2", got) } } // TestAPIInstanceEnableBumpsSerial: the manifest carries the channel every // minted route follows, so changing it changes what boxes converge onto. func TestAPIInstanceEnableBumpsSerial(t *testing.T) { fx, token := instanceFixture(t, "code-serial-enable") fx.srv.cfg.AppsDomain = "oncarlos.com" status, out := putJSON(t, enableURL(fx, "bab", "titogo"), map[string]any{"channel": "beta"}, token) if status != http.StatusOK { t.Fatalf("instance enable = %d %v", status, out) } if got := serial(t, fx); got != 1 { t.Fatalf("serial after instances enable = %d, want 1", got) } } // TestWebPromoteBumpsSerial: the browser door writes the same pointer as the // API one — a promotion from the console UI must reach boxes just as fast. func TestWebPromoteBumpsSerial(t *testing.T) { fx := newConsoleFixture(t) shipVersion(t, fx, "kass", "v9") c := signInPaul(t, fx) csrf := sessionToken(t, fx, c) status, _, body := postForm(t, c, fx.u("/apps/kass/promote"), url.Values{ "csrf": {csrf}, "channel": {"canary/x"}, "version": {"v9"}, "confirmed": {"1"}, }) if status != http.StatusSeeOther { t.Fatalf("web promote = %d, want 303:\n%s", status, body) } if got := serial(t, fx); got != 1 { t.Fatalf("serial after a web promote = %d, want 1", got) } } // TestWebProductionBumpsSerial: the production flag is read on every box's // adopt path (it decides whether a channel bakes at all), so flipping it is // box-consumed — and turning it OFF releases held versions, which is the // direction people wait on. func TestWebProductionBumpsSerial(t *testing.T) { fx := newConsoleFixture(t) c := signInPaul(t, fx) csrf := sessionToken(t, fx, c) status, _, body := postForm(t, c, fx.u("/apps/kass/production"), url.Values{"csrf": {csrf}, "production": {"on"}}) if status != http.StatusSeeOther { t.Fatalf("production on = %d, want 303:\n%s", status, body) } if got := serial(t, fx); got != 1 { t.Fatalf("serial after a production flip = %d, want 1", got) } } // TestWebRestoreBumpsSerial: restore is a box-consumed write like every other // door here — pulling the trash marker re-enables instance minting for the // app, and minting from records is leg 1 of the fast pass's reconcile. An // un-announced restore leaves the app un-mintable until the hourly floor, // which is exactly the wait the delete side already announces its way out of. func TestWebRestoreBumpsSerial(t *testing.T) { fx, c, csrf := trashPageFixture(t) before := serial(t, fx) status, _, body := postForm(t, c, fx.u("/trash/restore"), url.Values{"csrf": {csrf}, "app": {"kass"}}) if status != http.StatusSeeOther { t.Fatalf("restore = %d, want 303:\n%s", status, body) } if got := serial(t, fx); got != before+1 { t.Fatalf("serial after a restore = %d, want %d", got, before+1) } } // TestWebTrashBumpsSerial: the web delete door writes the same trash marker // the API door does, behind its own passkey ceremony — the announcement must // survive the step-up round trip, not just the plain-write path. func TestWebTrashBumpsSerial(t *testing.T) { fx := newConsoleFixture(t) c := signInPaul(t, fx) csrf := sessionToken(t, fx, c) before := serial(t, fx) status, loc, _ := postForm(t, c, fx.u("/apps/kass/delete"), url.Values{"csrf": {csrf}, "confirm": {"kass"}}) if status != http.StatusSeeOther || !strings.HasPrefix(loc, fx.fk.srv.URL) { t.Fatalf("un-stepped-up delete = %d -> %q, want a redirect into Keymail", status, loc) } if got := serial(t, fx); got != before { t.Fatalf("serial before the passkey confirmation = %d, want unchanged (%d)", got, before) } status, loc, body := completeStepUp(t, fx, c, loc) if status != http.StatusSeeOther || loc != fx.path("/") { t.Fatalf("resumed delete = %d -> %q, want 303 to the account home:\n%s", status, loc, body) } if got := serial(t, fx); got != before+1 { t.Fatalf("serial after a web trash = %d, want %d", got, before+1) } } // cancelAwareStore is a bucket that honours context cancellation, the way the // AWS SDK does and the DirStore (local disk, no ctx checks at all) does not — // without it, a canceled-context test here would pass for the wrong reason. type cancelAwareStore struct{ release.Store } func (c cancelAwareStore) Get(ctx context.Context, key string) ([]byte, string, error) { if err := ctx.Err(); err != nil { return nil, "", err } return c.Store.Get(ctx, key) } func (c cancelAwareStore) Put(ctx context.Context, key string, body []byte, ifMatch string) (string, error) { if err := ctx.Err(); err != nil { return "", err } return c.Store.Put(ctx, key, body, ifMatch) } // TestBumpSerialOutlivesTheRequest: the announcement is detached from the // request context (context.WithoutCancel). By the time a bump runs, the write // it announces has already landed and cannot be taken back — so a client that // hangs up in that window (a killed CI runner, a closed laptop, a proxy // timeout) must not cancel the announcement for a change that is permanently // in the bucket. Inheriting the cancellation loses it silently: the fleet // waits out the hourly floor for a deploy that already happened. func TestBumpSerialOutlivesTheRequest(t *testing.T) { fx := newConsoleFixture(t) fx.srv.cfg.Releases = cancelAwareStore{fx.rel} ctx, cancel := context.WithCancel(context.Background()) cancel() // the client is already gone fx.srv.bumpSerial(ctx) if got := serial(t, fx); got != 1 { t.Fatalf("serial after a bump on a canceled request context = %d, want 1", got) } } // TestBumpSerialSurvivesABrokenStore: the whole contract in one test — a bump // against a store that cannot be written returns normally and logs. Every call // site depends on this: a user-facing write must never fail, or stall, because // its announcement did. func TestBumpSerialSurvivesABrokenStore(t *testing.T) { fx := newConsoleFixture(t) fx.srv.cfg.Releases = brokenStore{fx.rel} done := make(chan struct{}) go func() { defer close(done) fx.srv.bumpSerial(context.Background()) }() select { case <-done: case <-time.After(10 * time.Second): t.Fatal("bumpSerial did not return — a lost bump must never stall a write") } }