package console import ( "bytes" "context" "crypto/ed25519" "crypto/sha256" "encoding/base64" "encoding/hex" "encoding/json" "fmt" "mime/multipart" "net/http" "net/http/httptest" "os" "path/filepath" "reflect" "strings" "sync" "testing" "time" "github.com/carlosframework/platform/internal/publicrelease" "github.com/carlosframework/platform/internal/release" "github.com/carlosframework/platform/internal/status" ) // systemFeed serves a signed fake release from an httptest server and // points the fixture's console at it. func systemFeed(t *testing.T, fx *consoleFixture, assets map[string][]byte) { t.Helper() systemFeedHeld(t, fx, assets, nil) } // systemFeedHeld is systemFeed with a gate on /latest.txt: while hold is open // the feed never answers, so a promote that has to resolve "latest" stays in // flight for as long as the test needs it to. func systemFeedHeld(t *testing.T, fx *consoleFixture, assets map[string][]byte, hold <-chan struct{}) { t.Helper() pub, priv, _ := ed25519.GenerateKey(nil) var sums strings.Builder for name, body := range assets { s := sha256.Sum256(body) fmt.Fprintf(&sums, "%s %s\n", hex.EncodeToString(s[:]), name) } sig := base64.StdEncoding.EncodeToString(ed25519.Sign(priv, []byte(sums.String()))) mux := http.NewServeMux() mux.HandleFunc("/latest.txt", func(w http.ResponseWriter, r *http.Request) { if hold != nil { <-hold } fmt.Fprintln(w, "v0.18.0") }) mux.HandleFunc("/dl/v0.18.0/SHA256SUMS.txt", func(w http.ResponseWriter, r *http.Request) { fmt.Fprint(w, sums.String()) }) mux.HandleFunc("/dl/v0.18.0/SHA256SUMS.txt.sig", func(w http.ResponseWriter, r *http.Request) { fmt.Fprintln(w, sig) }) for name, body := range assets { body := body mux.HandleFunc("/dl/v0.18.0/"+name, func(w http.ResponseWriter, r *http.Request) { w.Write(body) }) } ts := httptest.NewServer(mux) t.Cleanup(ts.Close) fx.srv.cfg.SystemFeed = publicrelease.Feed{LatestURL: ts.URL + "/latest.txt", DownloadBase: ts.URL + "/dl", PubKey: base64.StdEncoding.EncodeToString(pub)} } func waitJob(t *testing.T, fx *consoleFixture, token, id string) map[string]any { t.Helper() for i := 0; i < 100; i++ { st, out := getJSON(t, fx.ts.URL+"/api/cli/system/jobs/"+id, token) if st != http.StatusOK { t.Fatalf("job = %d %v", st, out) } if out["done"] == true { return out } time.Sleep(20 * time.Millisecond) } t.Fatal("job never finished") return nil } func seedBox(t *testing.T, fx *consoleFixture, label, platform, channel, rel string) { t.Helper() b := status.CollectBox(nil, status.Probes{Release: rel, Platform: platform, SystemChannel: channel}, time.Now()) b.Label = label if err := status.WriteBox(context.Background(), fx.rel, b); err != nil { t.Fatal(err) } } func TestSystemDoorsAreForOpsOwnersOnly(t *testing.T) { fx := newConsoleFixture(t) if err := fx.st.AddMember(OpsAccountID, "member@keymail.dev", RoleMember, time.Now()); err != nil { t.Fatal(err) } member := cliToken(t, fx, "member@keymail.dev", "code-sys-member") // The refusal is the BARE 404 the operator doors give — raw, not // jsonError'd — so it is byte-identical to what a path that does not // exist answers. That identity is the point: a member must not be able // to learn that this deployment has a system surface at all, and the // nonexistent path below is the baseline every refusal is measured // against, which is why the assertions read the body and not just the // status. _, want := getRaw(t, fx.ts.URL+"/api/cli/system/no-such-door", member) for _, u := range []string{"/api/cli/system/status", "/api/cli/system/jobs/deadbeef"} { st, body := getRaw(t, fx.ts.URL+u, member) if st != http.StatusNotFound || body != want { t.Errorf("GET %s as a member = %d %q, want 404 %q", u, st, body, want) } } for _, u := range []string{"/api/cli/system/update", "/api/cli/system/rollback"} { st, body := postRaw(t, fx.ts.URL+u, map[string]string{"channel": "canary"}, member) if st != http.StatusNotFound || body != want { t.Errorf("POST %s as a member = %d %q, want 404 %q", u, st, body, want) } } if st, _ := getRaw(t, fx.ts.URL+"/api/cli/system/status", "not-a-token"); st == http.StatusOK { t.Error("unauthenticated status answered 200") } } // TestSystemDoorsWithoutABucketRefuse: Config.Releases == nil is a legal // deployment shape, and every door here reads or writes the bucket. An owner // on such a console must get a sentence, not the nil-interface panic // status.LoadBoxes and release.GetSystemPointer would otherwise give. func TestSystemDoorsWithoutABucketRefuse(t *testing.T) { fx := newConsoleFixture(t) owner := cliToken(t, fx, "paul@keymail.dev", "code-sys-nobucket") fx.srv.cfg.Releases = nil if st, out := getJSON(t, fx.ts.URL+"/api/cli/system/status", owner); st != http.StatusServiceUnavailable { t.Errorf("status without a bucket = %d %v, want 503", st, out) } if st, out := postJSON(t, fx.ts.URL+"/api/cli/system/update", map[string]any{"channel": "canary"}, owner); st != http.StatusServiceUnavailable { t.Errorf("update without a bucket = %d %v, want 503", st, out) } if st, out := postJSON(t, fx.ts.URL+"/api/cli/system/rollback", map[string]string{"channel": "canary"}, owner); st != http.StatusServiceUnavailable { t.Errorf("rollback without a bucket = %d %v, want 503", st, out) } } func TestSystemUpdateMirrorsSignsAndBumps(t *testing.T) { fx := newConsoleFixture(t) arm := []byte("ELF arm64") systemFeed(t, fx, map[string][]byte{"carlos-linux-arm64": arm, "carlos-linux-amd64": []byte("ELF amd64")}) seedBox(t, fx, "ie", "linux-arm64", "canary", "v0.17.0") owner := cliToken(t, fx, "paul@keymail.dev", "code-sys-owner") before, _, _ := release.ReadSerial(context.Background(), fx.rel) st, out := postJSON(t, fx.ts.URL+"/api/cli/system/update", map[string]any{"channel": "canary", "spread": "2m"}, owner) if st != http.StatusOK { t.Fatalf("update = %d %v", st, out) } job := waitJob(t, fx, owner, str(t, out, "job")) if job["error"] != "" { t.Fatalf("job failed: %v", job) } p, _, err := release.GetSystemPointer(context.Background(), fx.rel, fx.pub, "router", "canary") if err != nil { t.Fatal(err) } if p.Current.Version != "v0.18.0" || p.Spread != "2m0s" || p.Current.Source != "release:v0.18.0" { t.Fatalf("pointer = %+v", p) } a, ok := p.Current.Artifacts["linux-arm64"] if !ok { t.Fatalf("no arm64 artifact: %+v", p.Current.Artifacts) } if _, ok := p.Current.Artifacts["linux-amd64"]; ok { t.Fatal("mirrored a platform no box reports") } body, etag, err := fx.rel.Get(context.Background(), a.Key) if err != nil || !bytes.Equal(body, arm) { t.Fatalf("artifact not mirrored: %v", err) } s := sha256.Sum256(arm) if a.SHA256 != hex.EncodeToString(s[:]) { t.Fatalf("sha = %s", a.SHA256) } after, _, _ := release.ReadSerial(context.Background(), fx.rel) if after.Serial == before.Serial { t.Fatal("serial not bumped") } // A second promote of the same version to stable copies nothing and // refuses different bytes under the same version. st, out = postJSON(t, fx.ts.URL+"/api/cli/system/update", map[string]any{"channel": "stable", "version": "v0.18.0"}, owner) if st != http.StatusOK { t.Fatalf("second update = %d %v", st, out) } if job := waitJob(t, fx, owner, str(t, out, "job")); job["error"] != "" { t.Fatalf("second job: %v", job) } // Overwrite the object out from under the pointer — a CAS-guarded Put // with the etag we just read, because an empty ifMatch is create-only // and would refuse an existing key. A re-promote of that version must // then refuse: versions are immutable. if _, err := fx.rel.Put(context.Background(), a.Key, []byte("different"), etag); err != nil { t.Fatalf("overwrite the mirrored artifact: %v", err) } st, out = postJSON(t, fx.ts.URL+"/api/cli/system/update", map[string]any{"channel": "canary", "version": "v0.18.0"}, owner) if st != http.StatusOK { t.Fatalf("re-promote = %d %v", st, out) } job = waitJob(t, fx, owner, str(t, out, "job")) if e, _ := job["error"].(string); e == "" || !strings.Contains(e, "immutable") { t.Fatalf("re-promote over different bytes: %v", job) } } // TestSystemUpdateIsOnePromotePerChannel: a promote holds a whole platform // binary in the heap, and the door is reachable by a script. One at a time // per channel, and the second caller is told which job to watch rather than // being handed a parallel one. func TestSystemUpdateIsOnePromotePerChannel(t *testing.T) { fx := newConsoleFixture(t) hold := make(chan struct{}) var once sync.Once unhold := func() { once.Do(func() { close(hold) }) } defer unhold() // the feed server's Close waits on in-flight requests systemFeedHeld(t, fx, map[string][]byte{"carlos-linux-arm64": []byte("ELF arm64")}, hold) seedBox(t, fx, "ie", "linux-arm64", "canary", "v0.17.0") owner := cliToken(t, fx, "paul@keymail.dev", "code-sys-one") st, out := postJSON(t, fx.ts.URL+"/api/cli/system/update", map[string]any{"channel": "canary"}, owner) if st != http.StatusOK { t.Fatalf("first update = %d %v", st, out) } first := str(t, out, "job") // The slot is claimed by the handler, before the goroutine runs, so this // needs no sleep to be deterministic. st, out = postJSON(t, fx.ts.URL+"/api/cli/system/update", map[string]any{"channel": "canary"}, owner) if st != http.StatusConflict { t.Fatalf("second update on a busy channel = %d %v, want 409", st, out) } if str(t, out, "job") != first { t.Errorf("409 named job %q, want the running one %q", str(t, out, "job"), first) } // A DIFFERENT channel is not blocked by canary's promote. st, out = postJSON(t, fx.ts.URL+"/api/cli/system/update", map[string]any{"channel": "stable"}, owner) if st != http.StatusOK { t.Fatalf("update on the other channel = %d %v, want 200", st, out) } stable := str(t, out, "job") unhold() // Both jobs mirror the SAME artifact key concurrently — one wins the // create-only Put and the other must read it back and agree, not report // a conflict the operator did not cause. for _, id := range []string{first, stable} { if job := waitJob(t, fx, owner, id); job["error"] != "" { t.Fatalf("job %s: %v", id, job) } } // The slot is handed back before the job is marked done, so an operator // who watched it finish can promote again immediately. st, out = postJSON(t, fx.ts.URL+"/api/cli/system/update", map[string]any{"channel": "canary"}, owner) if st != http.StatusOK { t.Fatalf("update after the first finished = %d %v, want 200", st, out) } if job := waitJob(t, fx, owner, str(t, out, "job")); job["error"] != "" { t.Fatalf("re-promote after the slot was handed back: %v", job) } } // TestSystemUpdateRefusesWhenFleetsAreUnreadable: which platforms to mirror // is derived from what the fleet reports, so a fleet this console cannot // enumerate is not a fleet to quietly leave out — those boxes would never // find a binary to adopt. The promote refuses and says why. func TestSystemUpdateRefusesWhenFleetsAreUnreadable(t *testing.T) { fx := newConsoleFixture(t) systemFeed(t, fx, map[string][]byte{"carlos-linux-arm64": []byte("ELF arm64")}) seedBox(t, fx, "ie", "linux-arm64", "canary", "v0.17.0") owner := cliToken(t, fx, "paul@keymail.dev", "code-sys-fleeterr") // A registry that EXISTS and cannot be opened — a directory where the // database should be. Not a missing path: a path that is not there yet is // an ordinary console with no fleets, which the sibling test covers. broken := filepath.Join(t.TempDir(), "registry.db") if err := os.Mkdir(broken, 0o755); err != nil { t.Fatal(err) } fx.srv.cfg.Registry = broken st, out := postJSON(t, fx.ts.URL+"/api/cli/system/update", map[string]any{"channel": "canary"}, owner) if st != http.StatusOK { t.Fatalf("update = %d %v", st, out) } job := waitJob(t, fx, owner, str(t, out, "job")) e, _ := job["error"].(string) if !strings.Contains(e, "cannot list fleets") { t.Fatalf("job error = %q, want a refusal naming the fleet listing", e) } if _, _, err := release.GetSystemPointer(context.Background(), fx.rel, fx.pub, "router", "canary"); err == nil { t.Error("the pointer moved even though the platform set was unknown") } // The STATUS read degrades instead: the boxes that did answer, plus the // flag that says the list is short. st, out = getJSON(t, fx.ts.URL+"/api/cli/system/status", owner) if st != http.StatusOK { t.Fatalf("status = %d %v", st, out) } if out["partial"] != true { t.Errorf("status partial = %v, want true when a fleet could not be listed", out["partial"]) } if boxes, _ := out["boxes"].([]any); len(boxes) != 1 { t.Errorf("status boxes = %v, want the platform's own box still shown", out["boxes"]) } } // TestSystemUpdateWithNoRegistryHasNoFleets: a console with no route registry // — cfg.Registry unset, or the file not created yet — is an ordinary console // with no fleets, not one whose fleets are unreadable. It must be able to // promote, and its status must not claim to be short of anything. func TestSystemUpdateWithNoRegistryHasNoFleets(t *testing.T) { for _, c := range []struct{ name, registry string }{ {"unset", ""}, {"not created yet", filepath.Join(t.TempDir(), "never-made.db")}, } { t.Run(c.name, func(t *testing.T) { fx := newConsoleFixture(t) systemFeed(t, fx, map[string][]byte{"carlos-linux-arm64": []byte("ELF arm64")}) owner := cliToken(t, fx, "paul@keymail.dev", "code-sys-noreg-"+strings.ReplaceAll(c.name, " ", "-")) fx.srv.cfg.Registry = c.registry st, out := postJSON(t, fx.ts.URL+"/api/cli/system/update", map[string]any{"channel": "canary"}, owner) if st != http.StatusOK { t.Fatalf("update = %d %v", st, out) } if job := waitJob(t, fx, owner, str(t, out, "job")); job["error"] != "" { t.Fatalf("job: %v", job) } p, _, err := release.GetSystemPointer(context.Background(), fx.rel, fx.pub, "router", "canary") if err != nil { t.Fatal(err) } if _, ok := p.Current.Artifacts["linux-arm64"]; !ok { t.Fatalf("artifacts = %+v, want linux-arm64 mirrored", p.Current.Artifacts) } st, out = getJSON(t, fx.ts.URL+"/api/cli/system/status", owner) if st != http.StatusOK { t.Fatalf("status = %d %v", st, out) } if out["partial"] != false { t.Errorf("status partial = %v, want false — no registry is no fleets, not unreadable ones", out["partial"]) } }) } } // TestSystemStatusNamesAnUnverifiablePointer: null means "never promoted" // and nothing else. A pointer that exists but does not verify is the one // thing an operator most needs to see, so it must not render as the one // thing they would scroll past. func TestSystemStatusNamesAnUnverifiablePointer(t *testing.T) { fx := newConsoleFixture(t) owner := cliToken(t, fx, "paul@keymail.dev", "code-sys-tampered") key := release.SystemPointerKey("router", "canary") if _, err := fx.rel.Put(context.Background(), key, []byte(`{"payload":"bm90LWEtcG9pbnRlcg==","sig":"AAAA"}`), ""); err != nil { t.Fatal(err) } st, out := getJSON(t, fx.ts.URL+"/api/cli/system/status", owner) if st != http.StatusOK { t.Fatalf("status = %d %v", st, out) } p, _ := out["pointers"].(map[string]any) canary, ok := p["canary"].(map[string]any) if !ok { t.Fatalf("canary pointer = %v, want an object carrying the error (null would read as never promoted)", p["canary"]) } if e, _ := canary["error"].(string); e == "" { t.Errorf("canary pointer = %v, want an error naming the refusal", canary) } // stable really has never been promoted, and that IS null. if p["stable"] != nil { t.Errorf("stable pointer = %v, want null", p["stable"]) } } func TestSystemUpdateFromAnUpload(t *testing.T) { fx := newConsoleFixture(t) owner := cliToken(t, fx, "paul@keymail.dev", "code-sys-upload") var buf bytes.Buffer mw := multipart.NewWriter(&buf) meta, _ := mw.CreateFormField("meta") meta.Write([]byte(`{"version":"66587fa","platform":"linux-arm64"}`)) file, _ := mw.CreateFormFile("file", "carlos-linux-arm64") file.Write([]byte("ELF local")) mw.Close() req, _ := http.NewRequest(http.MethodPut, fx.ts.URL+"/api/cli/system/upload", &buf) req.Header.Set("Content-Type", mw.FormDataContentType()) req.Header.Set("Authorization", "Bearer "+owner) res, err := http.DefaultClient.Do(req) if err != nil { t.Fatal(err) } var up map[string]any if res.StatusCode != http.StatusOK { t.Fatalf("upload = %d", res.StatusCode) } if err := json.NewDecoder(res.Body).Decode(&up); err != nil { t.Fatal(err) } st, out := postJSON(t, fx.ts.URL+"/api/cli/system/update", map[string]any{"channel": "canary", "upload": up["upload"]}, owner) if st != http.StatusOK { t.Fatalf("update = %d %v", st, out) } if job := waitJob(t, fx, owner, str(t, out, "job")); job["error"] != "" { t.Fatalf("job: %v", job) } p, _, err := release.GetSystemPointer(context.Background(), fx.rel, fx.pub, "router", "canary") if err != nil || p.Current.Version != "66587fa" || p.Current.Source != "file" { t.Fatalf("pointer = %+v %v", p, err) } // Upload without a platform is refused at the door. var buf2 bytes.Buffer mw2 := multipart.NewWriter(&buf2) m2, _ := mw2.CreateFormField("meta") m2.Write([]byte(`{"version":"abc"}`)) f2, _ := mw2.CreateFormFile("file", "x") f2.Write([]byte("x")) mw2.Close() req2, _ := http.NewRequest(http.MethodPut, fx.ts.URL+"/api/cli/system/upload", &buf2) req2.Header.Set("Content-Type", mw2.FormDataContentType()) req2.Header.Set("Authorization", "Bearer "+owner) res2, _ := http.DefaultClient.Do(req2) if res2.StatusCode != http.StatusBadRequest { t.Fatalf("upload without platform = %d", res2.StatusCode) } } // Versions here are at or above release.LayoutFloor: since plan 5 the // rollback door judges the release it would move back to, and a version // that cannot be placed against the floor is refused while any box reports // no layout (layoutguard_test.go covers that side). func TestSystemStatusAndRollback(t *testing.T) { fx := newConsoleFixture(t) owner := cliToken(t, fx, "paul@keymail.dev", "code-sys-status") seedBox(t, fx, "ie", "linux-arm64", "canary", "v0.25.0") sg := release.RootSigner(fx.priv) rel := func(v string) release.SystemRelease { return release.SystemRelease{Version: v, Artifacts: map[string]release.SystemArtifact{"linux-arm64": {Key: release.SystemArtifactKey(v, "linux-arm64"), SHA256: strings.Repeat("a", 64)}}} } release.PromoteSystem(context.Background(), fx.rel, sg, "router", "canary", rel("v0.25.0"), release.SystemPromoteOptions{}) time.Sleep(1100 * time.Millisecond) release.PromoteSystem(context.Background(), fx.rel, sg, "router", "canary", rel("v0.26.0"), release.SystemPromoteOptions{}) st, out := getJSON(t, fx.ts.URL+"/api/cli/system/status", owner) if st != http.StatusOK { t.Fatalf("status = %d %v", st, out) } boxes := out["boxes"].([]any) if len(boxes) != 1 { t.Fatalf("boxes = %v", boxes) } b := boxes[0].(map[string]any) if b["label"] != "ie" || b["release"] != "v0.25.0" || b["target"] != "v0.26.0" || b["channel"] != "canary" { t.Fatalf("box row = %v", b) } time.Sleep(1100 * time.Millisecond) st, out = postJSON(t, fx.ts.URL+"/api/cli/system/rollback", map[string]string{"channel": "canary"}, owner) if st != http.StatusOK || str(t, out, "version") != "v0.25.0" { t.Fatalf("rollback = %d %v", st, out) } if st, out = postJSON(t, fx.ts.URL+"/api/cli/system/rollback", map[string]string{"channel": "canary"}, owner); st != http.StatusConflict { t.Fatalf("second rollback = %d %v (nothing left in history)", st, out) } // Rollback validates what it is asked to rewind, exactly as update // validates what it is asked to write — both arguments become part of an // object key. for _, body := range []map[string]string{{"channel": "edge"}, {"channel": ""}, {"channel": "canary", "component": "../etc"}} { if st, out := postJSON(t, fx.ts.URL+"/api/cli/system/rollback", body, owner); st != http.StatusBadRequest { t.Errorf("rollback %v = %d %v, want 400", body, st, out) } } } // TestSystemFeedIsAllThreeFieldsOrNone: a half-configured feed would fetch // one publisher's binaries and check them against another's key, which fails // as "the release is corrupt" rather than as "this console is misconfigured". func TestSystemFeedIsAllThreeFieldsOrNone(t *testing.T) { fx := newConsoleFixture(t) def := publicrelease.Default() full := publicrelease.Feed{LatestURL: "https://x/latest.txt", DownloadBase: "https://x/dl", PubKey: "k"} for _, c := range []struct { name string cfg publicrelease.Feed want publicrelease.Feed }{ {"unset", publicrelease.Feed{}, def}, {"all three", full, full}, {"base only", publicrelease.Feed{DownloadBase: full.DownloadBase}, def}, {"base and key, no latest", publicrelease.Feed{DownloadBase: full.DownloadBase, PubKey: full.PubKey}, def}, {"key only", publicrelease.Feed{PubKey: full.PubKey}, def}, } { fx.srv.cfg.SystemFeed = c.cfg if got := fx.srv.systemFeed(); got != c.want { t.Errorf("%s: systemFeed() = %+v, want %+v", c.name, got, c.want) } } } // TestSystemUpdateWritesAnUntaggedPointerBeforeTheFloor pins the channel // pointer `carlos system update --version ` writes while no host floor // is published (the claim trial's dev pointers on v0.28.3, untagged-pointer // ruling R2): the release as the door resolved it, {Version, Source // "release:", Artifacts}, and nothing else. No release tag, no moves // protocol and no console artifacts, because reserveHostUpdate tags a // release only once a floor is published. internal/host's install-channel // tests and the root-runner activation witness promote exactly this shape. func TestSystemUpdateWritesAnUntaggedPointerBeforeTheFloor(t *testing.T) { fx := newConsoleFixture(t) arm := []byte("ELF arm64") systemFeed(t, fx, map[string][]byte{"carlos-linux-arm64": arm}) seedBox(t, fx, "ie", "linux-arm64", "stable", "v0.17.0") owner := cliToken(t, fx, "paul@keymail.dev", "code-sys-untagged") st, out := postJSON(t, fx.ts.URL+"/api/cli/system/update", map[string]any{"channel": "stable", "version": "v0.18.0", "spread": "0s"}, owner) if st != http.StatusOK { t.Fatalf("update = %d %v", st, out) } if job := waitJob(t, fx, owner, str(t, out, "job")); job["error"] != "" { t.Fatalf("job failed: %v", job) } p, _, err := release.GetSystemPointer(context.Background(), fx.rel, fx.pub, "router", "stable") if err != nil { t.Fatal(err) } s := sha256.Sum256(arm) want := release.SystemRelease{Version: "v0.18.0", Source: "release:v0.18.0", ReleasedAt: p.Current.ReleasedAt, Artifacts: map[string]release.SystemArtifact{"linux-arm64": {Key: release.SystemArtifactKey("v0.18.0", "linux-arm64"), SHA256: hex.EncodeToString(s[:])}}} if !reflect.DeepEqual(p.Current, want) || p.Spread != "0s" { t.Fatalf("pointer current = %+v spread %s, want %+v", p.Current, p.Spread, want) } if _, err := release.VerifySystemIdentity(fx.pub, mustGetObject(t, fx, release.SystemPointerKey("router", "stable"))); err == nil { t.Fatal("an untagged pointer verified as a release identity") } } func mustGetObject(t *testing.T, fx *consoleFixture, key string) []byte { t.Helper() raw, _, err := fx.rel.Get(context.Background(), key) if err != nil { t.Fatal(err) } return raw }