package console // The four `carlos schedule` doors (spec §8). What is pinned here, in the // order it would hurt if lost: a recorded schedule actually lands in the // bucket AND is announced (a declaration nobody hears is a schedule that // fires an hour late at best); a rejected one lands NOTHING and announces // nothing; the list answers with `[]` rather than `null` for an app that has // declared nothing (the CLI ranges over both arrays); and a stranger — or a // member naming a schedule that does not exist — gets the same bytes back as // someone probing for an app that isn't there. import ( "bytes" "context" "encoding/json" "errors" "net/http" "os" "testing" "time" "github.com/carlosframework/platform/internal/release" "github.com/carlosframework/platform/internal/schedule" ) // scheduleFixture is restartFixture's twin: an app in the ops account and a // MEMBER token for it. Declaring recurring work is operating an app, not // publishing a release, so member-level is the whole gate. func scheduleFixture(t *testing.T, code string) (*consoleFixture, string) { t.Helper() fx := newConsoleFixture(t) now := time.Now() if _, err := fx.st.CreateAppIn(OpsAccountID, "titogo", "paul@keymail.dev", now); err != nil { t.Fatal(err) } if err := fx.st.AddMember(OpsAccountID, "alice@keymail.dev", RoleMember, now); err != nil { t.Fatal(err) } return fx, cliToken(t, fx, "alice@keymail.dev", code) } // declaredSchedules reads schedules.json straight from the bucket — what the // boxes will converge from, as opposed to what the door said it did. func declaredSchedules(t *testing.T, fx *consoleFixture, app string) (schedule.Declaration, string) { t.Helper() d, etag, err := schedule.ReadDeclaration(context.Background(), fx.rel, release.OpsAccount, app) if err != nil { t.Fatalf("read declaration: %v", err) } return d, etag } // runEtag is what every box watches for `run`: "" when nobody has ever asked. func runEtag(t *testing.T, fx *consoleFixture, app string) string { t.Helper() _, etag, err := fx.rel.Get(context.Background(), schedule.RunKey(release.OpsAccount, app)) if err != nil { if errors.Is(err, os.ErrNotExist) { return "" } t.Fatalf("read run signal: %v", err) } return etag } func decodeScheduleList(t *testing.T, out []byte) apiScheduleList { t.Helper() var res apiScheduleList if err := json.Unmarshal(out, &res); err != nil { t.Fatalf("schedule body %s: %v", out, err) } return res } // TestAPIScheduleSetRecordsAndAnnouncesIt is the write door whole: the object // a box converges from holds the schedule, the serial says "look now", and // the answer names the canonical account and the declaration as stored. func TestAPIScheduleSetRecordsAndAnnouncesIt(t *testing.T) { fx, token := scheduleFixture(t, "code-schedule-set") if _, etag := declaredSchedules(t, fx, "titogo"); etag != "" { t.Fatalf("schedules.json exists before any request: %q", etag) } body := `{"name":"sync","every":"6h","path":"/jobs/sync"}` status, out := postBody(t, fx.ts.URL+"/api/cli/apps/bab/titogo/schedules", token, body) if status != http.StatusOK { t.Fatalf("set = %d %s, want 200", status, out) } res := decodeScheduleList(t, out) if res.Account != "bab" || res.App != "titogo" { t.Fatalf("set body = %+v, want bab/titogo", res) } if len(res.Schedules) != 1 || res.Schedules[0].Name != "sync" || res.Schedules[0].Path != "/jobs/sync" { t.Fatalf("set body schedules = %+v, want the one just recorded", res.Schedules) } if res.UpdatedAt == "" { t.Error("set body carries no updated_at — the CLI has nothing to show for the write") } // The write door answers about the DECLARATION; instance reports belong // to the list door, and must be [] rather than null either way. if res.Instances == nil { t.Error("set body instances is null; the CLI ranges over it") } d, etag := declaredSchedules(t, fx, "titogo") if etag == "" { t.Fatal("no schedules.json in the bucket after a 200 — no box will ever converge it") } sc, ok := d.Find("sync") if !ok { t.Fatalf("bucket declaration = %+v, want sync in it", d.Schedules) } if sc.Path != "/jobs/sync" || sc.Every != "6h0m0s" { t.Errorf("stored schedule = %+v, want the normalised every/path", sc) } if got := serial(t, fx); got != 1 { t.Fatalf("serial after a set = %d, want 1 (otherwise boxes wait out the hourly floor)", got) } // A second set REPLACES by name rather than appending, and announces again. status, out = postBody(t, fx.ts.URL+"/api/cli/apps/bab/titogo/schedules", token, `{"name":"sync","cron":"0 8 * * *","path":"/jobs/sync2"}`) if status != http.StatusOK { t.Fatalf("second set = %d %s", status, out) } d, _ = declaredSchedules(t, fx, "titogo") if len(d.Schedules) != 1 || d.Schedules[0].Path != "/jobs/sync2" || d.Schedules[0].Every != "" { t.Fatalf("declaration after replace = %+v, want one schedule on cron", d.Schedules) } if got := serial(t, fx); got != 2 { t.Fatalf("serial after two sets = %d, want 2", got) } } // TestAPIScheduleSetRefusesAnImpossibleInterval: validation happens before // anything is written, so a typo costs a round trip and nothing else — no // object, no announcement, and the validator's own words back (a fixed // sentence would make the member guess which field was wrong). func TestAPIScheduleSetRefusesAnImpossibleInterval(t *testing.T) { fx, token := scheduleFixture(t, "code-schedule-set-bad") status, out := postBody(t, fx.ts.URL+"/api/cli/apps/bab/titogo/schedules", token, `{"name":"sync","every":"30s","path":"/jobs/sync"}`) if status != http.StatusBadRequest { t.Fatalf("set every=30s = %d %s, want 400", status, out) } if !bytes.Contains(out, []byte("shortest interval")) { t.Errorf("400 body = %s, want the validator's sentence", out) } if _, etag := declaredSchedules(t, fx, "titogo"); etag != "" { t.Error("a refused schedule was written to the bucket") } if got := serial(t, fx); got != 0 { t.Fatalf("serial after a refused set = %d, want 0 — the fleet was woken for nothing", got) } } // TestAPIScheduleSetRefusesPastTheCap: the per-app ceiling answers as the // member's problem (400, with the number), not as a bucket failure. func TestAPIScheduleSetRefusesPastTheCap(t *testing.T) { fx, token := scheduleFixture(t, "code-schedule-cap") ctx, now := context.Background(), time.Now() for i := 0; i < schedule.MaxSchedulesPerApp; i++ { sc := schedule.Schedule{Name: "job-" + string(rune('a'+i)), Every: "1h", Path: "/jobs/x"} if _, err := schedule.UpsertSchedule(ctx, fx.rel, release.OpsAccount, "titogo", sc, now); err != nil { t.Fatalf("seed %d: %v", i, err) } } status, out := postBody(t, fx.ts.URL+"/api/cli/apps/bab/titogo/schedules", token, `{"name":"onemore","every":"1h","path":"/jobs/x"}`) if status != http.StatusBadRequest { t.Fatalf("set past the cap = %d %s, want 400", status, out) } if !bytes.Contains(out, []byte("at most")) { t.Errorf("cap refusal = %s, want it to name the ceiling", out) } if got := serial(t, fx); got != 0 { t.Fatalf("serial after a refused set = %d, want 0", got) } } // TestAPIScheduleListJoinsDeclarationAndInstanceReports: the declaration says // what was ASKED; the status objects say what each instance will actually do // next. `carlos schedule ls` is only honest for a member with no box access // if this door carries both. func TestAPIScheduleListJoinsDeclarationAndInstanceReports(t *testing.T) { fx, token := scheduleFixture(t, "code-schedule-ls") // An app that has declared nothing answers with EMPTY ARRAYS, never null. status, raw := getRaw(t, fx.ts.URL+"/api/cli/apps/bab/titogo/schedules", token) if status != http.StatusOK { t.Fatalf("list before anything = %d %s, want 200", status, raw) } if !bytes.Contains([]byte(raw), []byte(`"schedules":[]`)) || !bytes.Contains([]byte(raw), []byte(`"instances":[]`)) { t.Fatalf("empty list body = %s, want [] for both arrays (the CLI ranges over them)", raw) } if status, out := postBody(t, fx.ts.URL+"/api/cli/apps/bab/titogo/schedules", token, `{"name":"sync","every":"6h","path":"/jobs/sync"}`); status != http.StatusOK { t.Fatalf("set = %d %s", status, out) } // A box's report, written the way a box writes it. st := schedule.Status{ Host: "x.bab.oncarlos.com", Box: "carlos-ie-1", ReportedAt: time.Now().UTC().Format(time.RFC3339), Timers: []schedule.TimerStatus{{ Name: "sync", Kind: "schedule", Spec: "every 6h0m0s", Path: "/jobs/sync", NextAt: 1787004000, LastAt: 1786982400, LastStatus: "200", LastMs: 3210, }}, } if err := schedule.WriteStatus(context.Background(), fx.rel, release.OpsAccount, "titogo", st); err != nil { t.Fatal(err) } status, raw = getRaw(t, fx.ts.URL+"/api/cli/apps/bab/titogo/schedules", token) if status != http.StatusOK { t.Fatalf("list = %d %s, want 200", status, raw) } res := decodeScheduleList(t, []byte(raw)) if len(res.Schedules) != 1 || res.Schedules[0].Name != "sync" { t.Fatalf("list schedules = %+v, want the declaration", res.Schedules) } if len(res.Instances) != 1 { t.Fatalf("list instances = %+v, want the one report a box wrote", res.Instances) } got := res.Instances[0] if got.Host != "x.bab.oncarlos.com" || got.Box != "carlos-ie-1" { t.Errorf("instance report = %+v, want the box's own", got) } if len(got.Timers) != 1 || got.Timers[0].NextAt != 1787004000 || got.Timers[0].LastStatus != "200" { t.Errorf("instance timers = %+v, want the box's next/last verbatim", got.Timers) } } // TestAPIScheduleRemoveDropsAndAnnounces, and the 404 for a name that was // never declared is the SAME BYTES as the 404 for an app that is not there: // a name is not an account, but a probe is a probe. func TestAPIScheduleRemoveDropsAndAnnounces(t *testing.T) { fx, token := scheduleFixture(t, "code-schedule-rm") if status, out := postBody(t, fx.ts.URL+"/api/cli/apps/bab/titogo/schedules", token, `{"name":"sync","every":"6h","path":"/jobs/sync"}`); status != http.StatusOK { t.Fatalf("set = %d %s", status, out) } status, out := postBody(t, fx.ts.URL+"/api/cli/apps/bab/titogo/schedules/sync/delete", token, "") if status != http.StatusOK { t.Fatalf("delete = %d %s, want 200", status, out) } res := decodeScheduleList(t, out) if len(res.Schedules) != 0 { t.Errorf("delete body schedules = %+v, want none left", res.Schedules) } if res.Schedules == nil { t.Error("delete body schedules is null; the CLI ranges over it") } if res.Instances == nil { t.Error("delete body instances is null; the CLI ranges over it") } d, _ := declaredSchedules(t, fx, "titogo") if _, ok := d.Find("sync"); ok { t.Error("sync is still in the bucket declaration after a 200") } if got := serial(t, fx); got != 2 { t.Fatalf("serial after set+delete = %d, want 2", got) } // An undeclared name, byte-for-byte against an app that is not there. unknownName, nameBody := postRaw(t, fx.ts.URL+"/api/cli/apps/bab/titogo/schedules/nosuch/delete", map[string]string{}, token) unknownApp, appBody := postRaw(t, fx.ts.URL+"/api/cli/apps/bab/nosuchapp/schedules/sync/delete", map[string]string{}, token) if unknownName != http.StatusNotFound || unknownApp != http.StatusNotFound { t.Fatalf("unknown name = %d, unknown app = %d, want 404 both", unknownName, unknownApp) } if !bytes.Equal([]byte(nameBody), []byte(appBody)) { t.Errorf("undeclared-name 404 body %q differs from unknown-app 404 body %q", nameBody, appBody) } // An invalid name is the same answer: nothing about it reaches the store. if code, body := postRaw(t, fx.ts.URL+"/api/cli/apps/bab/titogo/schedules/NOT_A_NAME/delete", map[string]string{}, token); code != http.StatusNotFound || body != appBody { t.Errorf("invalid name = %d %q, want the unknown-app 404", code, body) } if got := serial(t, fx); got != 2 { t.Fatalf("serial after three refusals = %d, want 2 — a refusal must not wake the fleet", got) } } // TestAPIScheduleRunTouchesTheSignal: the etag movement IS the signal, so a // second run must move it again, and a run for a name nobody declared must // not touch it at all. func TestAPIScheduleRunTouchesTheSignal(t *testing.T) { fx, token := scheduleFixture(t, "code-schedule-run") if status, out := postBody(t, fx.ts.URL+"/api/cli/apps/bab/titogo/schedules", token, `{"name":"sync","every":"6h","path":"/jobs/sync"}`); status != http.StatusOK { t.Fatalf("set = %d %s", status, out) } if got := runEtag(t, fx, "titogo"); got != "" { t.Fatalf("run signal exists before any run: %q", got) } status, out := postBody(t, fx.ts.URL+"/api/cli/apps/bab/titogo/schedules/sync/run", token, "") if status != http.StatusOK { t.Fatalf("run = %d %s, want 200", status, out) } var res struct { Requested bool `json:"requested"` Account string `json:"account"` App string `json:"app"` Name string `json:"name"` } if err := json.Unmarshal(out, &res); err != nil { t.Fatalf("run body %s: %v", out, err) } if !res.Requested || res.Account != "bab" || res.App != "titogo" || res.Name != "sync" { t.Fatalf("run body = %+v, want requested for bab/titogo sync", res) } first := runEtag(t, fx, "titogo") if first == "" { t.Fatal("no run signal in the bucket after a 200 — no box will ever fire it") } if got := serial(t, fx); got != 2 { t.Fatalf("serial after set+run = %d, want 2", got) } if status, out := postBody(t, fx.ts.URL+"/api/cli/apps/bab/titogo/schedules/sync/run", token, ""); status != http.StatusOK { t.Fatalf("second run = %d %s", status, out) } if second := runEtag(t, fx, "titogo"); second == first { t.Fatalf("run etag %q unchanged after a second run — change detection is by etag", second) } before := runEtag(t, fx, "titogo") // Undeclared: the byte-identical 404, and the signal stays where it was. code, body := postRaw(t, fx.ts.URL+"/api/cli/apps/bab/titogo/schedules/nosuch/run", map[string]string{}, token) _, appBody := postRaw(t, fx.ts.URL+"/api/cli/apps/bab/nosuchapp/schedules/sync/run", map[string]string{}, token) if code != http.StatusNotFound { t.Fatalf("run of an undeclared name = %d %s, want 404", code, body) } if !bytes.Equal([]byte(body), []byte(appBody)) { t.Errorf("undeclared-name 404 body %q differs from unknown-app 404 body %q", body, appBody) } if after := runEtag(t, fx, "titogo"); after != before { t.Errorf("a refused run touched the signal: %q -> %q", before, after) } if got := serial(t, fx); got != 3 { t.Fatalf("serial after two runs and a refusal = %d, want 3", got) } } // TestAPIScheduleStrangerLearnsNothing: a non-member's refusal is // byte-identical with an unknown app's on the read door, the same gate in the // same place as every other door here. func TestAPIScheduleStrangerLearnsNothing(t *testing.T) { fx, _ := scheduleFixture(t, "code-schedule-stranger") eve := cliToken(t, fx, "eve@keymail.dev", "code-schedule-eve") status, body := getRaw(t, fx.ts.URL+"/api/cli/apps/bab/titogo/schedules", eve) if status != http.StatusNotFound { t.Fatalf("list as non-member = %d, want 404", status) } _, probe := getRaw(t, fx.ts.URL+"/api/cli/apps/bab/nosuchapp/schedules", eve) if !bytes.Equal([]byte(body), []byte(probe)) { t.Errorf("non-member 404 body %q differs from unknown-app 404 body %q", body, probe) } if body != "404 page not found\n" { t.Errorf("list as non-member body = %q, want the stdlib http.NotFound output", body) } if status, _ := getRaw(t, fx.ts.URL+"/api/cli/apps/bab/titogo/schedules", ""); status != http.StatusUnauthorized { t.Errorf("list with no token = %d, want 401", status) } } // TestAPIScheduleWritesAreRateLimited: all three mutating doors spend the // shared per-person write budget, before they touch the bucket. `run` is the // one that most needs it — its whole effect is to make an app do work now. func TestAPIScheduleWritesAreRateLimited(t *testing.T) { fx, token := scheduleFixture(t, "code-schedule-limit") fx.srv.cli.write = newRateLimiter(1, time.Minute) // The one request the budget allows, which also leaves `sync` declared — // so the three refusals below are the BUDGET refusing, not a missing name. if status, out := postBody(t, fx.ts.URL+"/api/cli/apps/bab/titogo/schedules", token, `{"name":"sync","every":"6h","path":"/jobs/sync"}`); status != http.StatusOK { t.Fatalf("set = %d %s", status, out) } for _, u := range []string{ "/api/cli/apps/bab/titogo/schedules", "/api/cli/apps/bab/titogo/schedules/sync/delete", "/api/cli/apps/bab/titogo/schedules/sync/run", } { if status, out := postBody(t, fx.ts.URL+u, token, `{"name":"sync","every":"6h","path":"/jobs/sync"}`); status != http.StatusTooManyRequests { t.Errorf("%s on an exhausted budget = %d %s, want 429", u, status, out) } } // The READ door is unlimited, like every other read here. if status, _ := getRaw(t, fx.ts.URL+"/api/cli/apps/bab/titogo/schedules", token); status != http.StatusOK { t.Errorf("list on an exhausted write budget = %d, want 200", status) } } // TestAPIScheduleWithoutABucketIs503: the declaration IS a bucket object. A // console with nowhere to put it must say so rather than answer as if a box // would ever see it. func TestAPIScheduleWithoutABucketIs503(t *testing.T) { fx, token := scheduleFixture(t, "code-schedule-nobucket") fx.srv.cfg.Releases = nil for _, c := range []struct{ method, path string }{ {http.MethodGet, "/api/cli/apps/bab/titogo/schedules"}, {http.MethodPost, "/api/cli/apps/bab/titogo/schedules"}, {http.MethodPost, "/api/cli/apps/bab/titogo/schedules/sync/delete"}, {http.MethodPost, "/api/cli/apps/bab/titogo/schedules/sync/run"}, } { var status int if c.method == http.MethodGet { status, _ = getRaw(t, fx.ts.URL+c.path, token) } else { status, _ = postBody(t, fx.ts.URL+c.path, token, `{"name":"sync","every":"6h","path":"/jobs/sync"}`) } if status != http.StatusServiceUnavailable { t.Errorf("%s %s with no bucket = %d, want 503", c.method, c.path, status) } } } // scheduleListBrokenStore cannot LIST the status prefix at all: the shape a // bucket outage takes on the read door. type scheduleListBrokenStore struct{ release.Store } func (b scheduleListBrokenStore) List(context.Context, string) ([]string, error) { return nil, errors.New(brokenStoreErr) } // A total failure to read the instance reports is a 502 with a fixed // sentence, never a 200 that reads as "no instance has reported yet": a // member with no box access would take a broken console for a fleet that // has not caught up. The bucket's own error text stays in the log. func TestAPIScheduleListTotalStatusFailureIs502(t *testing.T) { fx, token := scheduleFixture(t, "code-schedule-ls-broken") fx.srv.cfg.Releases = scheduleListBrokenStore{fx.rel} status, raw := getRaw(t, fx.ts.URL+"/api/cli/apps/bab/titogo/schedules", token) if status != http.StatusBadGateway { t.Fatalf("list with an unlistable status prefix = %d %s, want 502", status, raw) } if !bytes.Contains([]byte(raw), []byte("could not read instance reports")) || bytes.Contains([]byte(raw), []byte("carlos-prod-secret-bucket")) { t.Fatalf("502 body = %s, want the fixed sentence and none of the store's text", raw) } } // One unreadable report among readable ones is the PARTIAL case: the door // answers 200 with what is knowable and logs the rest, so one box writing // rubbish never blanks the list for every other instance of the app. func TestAPIScheduleListPartialStatusFailureIs200(t *testing.T) { fx, token := scheduleFixture(t, "code-schedule-ls-partial") good := schedule.Status{Host: "good.bab.oncarlos.com", Box: "carlos-ie-1", ReportedAt: time.Now().UTC().Format(time.RFC3339)} if err := schedule.WriteStatus(context.Background(), fx.rel, release.OpsAccount, "titogo", good); err != nil { t.Fatal(err) } if _, err := fx.rel.Put(context.Background(), schedule.StatusKey(release.OpsAccount, "titogo", "bad.bab.oncarlos.com"), []byte("{not json"), ""); err != nil { t.Fatal(err) } status, raw := getRaw(t, fx.ts.URL+"/api/cli/apps/bab/titogo/schedules", token) if status != http.StatusOK { t.Fatalf("list with one bad report = %d %s, want 200", status, raw) } res := decodeScheduleList(t, []byte(raw)) if len(res.Instances) != 1 || res.Instances[0].Host != "good.bab.oncarlos.com" { t.Fatalf("instances = %+v, want the one readable report", res.Instances) } }