package console import ( "context" "encoding/json" "fmt" "net/http" "strings" "testing" "time" "github.com/carlosframework/platform/internal/release" "github.com/carlosframework/platform/internal/status" ) // lastScoped reads the newest request in the app's host-retry log, where // a retry's grant rides (readiness round 2, Task 3), and checks the // restart-scope log took none. func lastScoped(t *testing.T, fx *consoleFixture, app string) release.RestartRequest { t.Helper() g, found, err := release.LoadHostRetries(context.Background(), fx.rel, fx.sqid, app) if err != nil || !found || len(g.Requests) == 0 { t.Fatalf("host retries = %+v, %v, %v", g, found, err) } if old, found, _ := release.LoadRestartScopes(context.Background(), fx.rel, fx.sqid, app); found && len(old.Requests) != 0 { t.Fatalf("a retry wrote the restart-scope log: %+v", old.Requests) } return g.Requests[len(g.Requests)-1] } // noRetryWritten checks neither log holds a request for app. func noRetryWritten(t *testing.T, fx *consoleFixture, app string) { t.Helper() ctx := context.Background() g, found, err := release.LoadHostRetries(ctx, fx.rel, fx.sqid, app) if err != nil { t.Fatalf("load the host retries: %v", err) } if found && len(g.Requests) != 0 { t.Fatalf("a retry was written: %+v", g.Requests) } g, found, err = release.LoadRestartScopes(ctx, fx.rel, fx.sqid, app) if err != nil { t.Fatalf("load the restart-scope log: %v", err) } if found && len(g.Requests) != 0 { t.Fatalf("a request was written to the restart-scope log: %+v", g.Requests) } } // heldEpoch is the restriction epoch writeHeldBox reports, and heldBook the // failure book it counts in (final review round 1, finding 3). const ( heldEpoch = 17 heldBook = "0123456789abcdef0123456789abcdef" ) func writeHeldBox(t *testing.T, fx *consoleFixture, host, app, build string) { t.Helper() box := status.CollectBox(nil, status.Probes{MachineID: "box1"}, time.Now()) box.Metrics = &status.BoxMetrics{Rollout: &status.RolloutReport{Hosts: map[string]status.HostRollout{ host: {Account: fx.sqid, App: app, Installed: build, Serving: "v1", Good: "v1", Awake: true, HeldBy: build, HeldEpoch: heldEpoch, HeldBook: heldBook, RetryV: status.HostRetryV, Reason: status.ReasonHealth}, }}} if err := status.WriteBox(context.Background(), fx.rel, box); err != nil { t.Fatal(err) } } func TestAPIInstanceRetryWritesARetryRequest(t *testing.T) { fx := newConsoleFixture(t) const host = "web.kass.carlosapp.dev" sibling(t, fx, "kass", host, "stable") token := cliToken(t, fx, "paul@keymail.dev", "code-host-retry") u := fx.ts.URL + "/api/cli/apps/" + fx.sqid + "/kass/instances/" + host + "/retry" if code, _, body := rawReq(t, http.MethodPost, u, token, `{}`); code != http.StatusConflict || !strings.Contains(body, fmt.Sprintf(apiRetryNotHeld, host)) { t.Fatalf("no report = %d %s, want 409 naming %s as not held", code, body, host) } writeHeldBox(t, fx, host, "kass", "v2") code, _, body := rawReq(t, http.MethodPost, u, token, `{}`) if code != http.StatusOK { t.Fatalf("retry = %d %s", code, body) } var out struct{ ID, Version, Host string } if err := json.Unmarshal([]byte(body), &out); err != nil || out.Version != "v2" || out.Host != host || out.ID == "" { t.Fatalf("answer = %s", body) } r := lastScoped(t, fx, "kass") if r.Scope.Kind != release.RestartScopeRetry || r.Scope.Value != host || r.ID != out.ID { t.Fatalf("stored = %+v", r) } // The entry carries the root's grant for the held build at the box's // epoch (spec §11, plan review finding 13). g, err := release.VerifyHostRetry(fx.pub, r.Scope.Signed, fx.sqid, "kass", host, time.Now()) if err != nil || g.Build != "v2" || g.Epoch != heldEpoch || g.Book != heldBook { t.Fatalf("signed grant = %+v, %v; want v2 at epoch %d of book %s", g, err, heldEpoch, heldBook) } if n := serial(t, fx); n != 1 { t.Fatalf("serial = %d", n) } code, _, body = rawReq(t, http.MethodPost, fx.ts.URL+"/api/cli/apps/"+fx.sqid+"/kass/instances/nope.kass.carlosapp.dev/retry", token, `{}`) if code != http.StatusNotFound || body != "404 page not found\n" { t.Fatalf("unknown host = %d %q", code, body) } } func TestAPIInstanceRetryIgnoresAnotherAppsReport(t *testing.T) { fx := newConsoleFixture(t) const host = "web.kass.carlosapp.dev" sibling(t, fx, "kass", host, "stable") writeHeldBox(t, fx, host, "otherapp", "z2") token := cliToken(t, fx, "paul@keymail.dev", "code-host-retry-x") u := fx.ts.URL + "/api/cli/apps/" + fx.sqid + "/kass/instances/" + host + "/retry" if code, _, body := rawReq(t, http.MethodPost, u, token, `{}`); code != http.StatusConflict || !strings.Contains(body, fmt.Sprintf(apiRetryNotHeld, host)) { t.Fatalf("another app's hold = %d %s, want 409 naming %s as not held", code, body, host) } } // A box that refuses every managed start reports no per-host state: its view // is built with Unavailable and no HeldBy, so the host is not held. This pins // that whole path to the 409. TestAPIInstanceRetryOfADeclinedBoxsHold covers // a view carrying both. func TestAPIInstanceRetryRefusesAnUnavailableBox(t *testing.T) { fx := newConsoleFixture(t) const host = "web.kass.carlosapp.dev" sibling(t, fx, "kass", host, "stable") box := status.CollectBox(nil, status.Probes{MachineID: "box1"}, time.Now()) box.Metrics = &status.BoxMetrics{ RouteVersions: map[string]string{host: "v2"}, Rollout: &status.RolloutReport{Unavailable: "no-deploy-store"}, } if err := status.WriteBox(context.Background(), fx.rel, box); err != nil { t.Fatal(err) } token := cliToken(t, fx, "paul@keymail.dev", "code-host-retry-u") u := fx.ts.URL + "/api/cli/apps/" + fx.sqid + "/kass/instances/" + host + "/retry" code, _, body := rawReq(t, http.MethodPost, u, token, `{}`) if code != http.StatusConflict || !strings.Contains(body, fmt.Sprintf(apiRetryNotHeld, host)) { t.Fatalf("unavailable box = %d %s, want 409 naming %s as not held", code, body, host) } } // Task 10 Correction 1 (plan review #5): a box that declined // CARLOS_SWAP=overlap but honours the failure book an armed run left // publishes its held hosts beside its decline code, and the door signs a // retry for one from that report. Any other Unavailable code is still // refused, whatever its report carries, and a declined box's host it does // not report as held is not held. func TestAPIInstanceRetryOfADeclinedBoxsHold(t *testing.T) { const host = "web.kass.carlosapp.dev" write := func(t *testing.T, fx *consoleFixture, code string, held bool) { t.Helper() ro := &status.RolloutReport{Unavailable: code, FailureBook: "honoured"} if held { ro.Hosts = map[string]status.HostRollout{host: {Account: fx.sqid, App: "kass", Installed: "v2", Good: "v1", Attempt: status.AttemptDown, AttemptBuild: "v2", HeldBy: "v2", HeldEpoch: heldEpoch, HeldBook: heldBook, RetryV: status.HostRetryV}} } box := status.CollectBox(nil, status.Probes{MachineID: "box1"}, time.Now()) box.Metrics = &status.BoxMetrics{RouteVersions: map[string]string{host: "v2"}, Rollout: ro} if err := status.WriteBox(context.Background(), fx.rel, box); err != nil { t.Fatal(err) } } for _, tc := range []struct { name string code string held bool ok bool }{ {"declined, held", "no-cgroup-delegation", true, true}, {"claims on, held", "host-claims-on", true, true}, {"declined, not held", "generations-off", false, false}, {"refusing, with hosts", "no-deploy-store", true, false}, {"unknown code, with hosts", "a-newer-code", true, false}, } { t.Run(tc.name, func(t *testing.T) { fx := newConsoleFixture(t) sibling(t, fx, "kass", host, "stable") write(t, fx, tc.code, tc.held) token := cliToken(t, fx, "paul@keymail.dev", "code-host-retry-d") u := fx.ts.URL + "/api/cli/apps/" + fx.sqid + "/kass/instances/" + host + "/retry" code, _, body := rawReq(t, http.MethodPost, u, token, `{}`) if !tc.ok { if code != http.StatusConflict || !strings.Contains(body, fmt.Sprintf(apiRetryNotHeld, host)) { t.Fatalf("retry = %d %s, want 409 naming %s as not held", code, body, host) } noRetryWritten(t, fx, "kass") return } if code != http.StatusOK { t.Fatalf("retry = %d %s, want the declined box's hold retried", code, body) } r := lastScoped(t, fx, "kass") g, err := release.VerifyHostRetry(fx.pub, r.Scope.Signed, fx.sqid, "kass", host, time.Now()) if err != nil || g.Build != "v2" || g.Epoch != heldEpoch || g.Book != heldBook { t.Fatalf("signed grant = %+v, %v; want v2 at epoch %d of book %s", g, err, heldEpoch, heldBook) } }) } // Another app's report under the host's name is not this app's hold. fx := newConsoleFixture(t) sibling(t, fx, "kass", host, "stable") box := status.CollectBox(nil, status.Probes{MachineID: "box1"}, time.Now()) box.Metrics = &status.BoxMetrics{RouteVersions: map[string]string{host: "v2"}, Rollout: &status.RolloutReport{ Unavailable: "no-cgroup-delegation", FailureBook: "honoured", Hosts: map[string]status.HostRollout{host: {Account: fx.sqid, App: "otherapp", Installed: "v2", HeldBy: "v2", HeldEpoch: heldEpoch, HeldBook: heldBook}}}} if err := status.WriteBox(context.Background(), fx.rel, box); err != nil { t.Fatal(err) } token := cliToken(t, fx, "paul@keymail.dev", "code-host-retry-o") u := fx.ts.URL + "/api/cli/apps/" + fx.sqid + "/kass/instances/" + host + "/retry" if code, _, body := rawReq(t, http.MethodPost, u, token, `{}`); code != http.StatusConflict { t.Fatalf("another app's declined hold = %d %s, want 409", code, body) } } // A box that reports the host and holds nothing back: there is nothing to // retry, and no request may be written. func TestAPIInstanceRetryRefusesAHostThatIsNotHeld(t *testing.T) { fx := newConsoleFixture(t) const host = "web.kass.carlosapp.dev" sibling(t, fx, "kass", host, "stable") box := status.CollectBox(nil, status.Probes{MachineID: "box1"}, time.Now()) box.Metrics = &status.BoxMetrics{Rollout: &status.RolloutReport{Hosts: map[string]status.HostRollout{ host: {Account: fx.sqid, App: "kass", Installed: "v2", Serving: "v2", Good: "v2", Awake: true}, }}} if err := status.WriteBox(context.Background(), fx.rel, box); err != nil { t.Fatal(err) } token := cliToken(t, fx, "paul@keymail.dev", "code-host-retry-n") u := fx.ts.URL + "/api/cli/apps/" + fx.sqid + "/kass/instances/" + host + "/retry" code, _, body := rawReq(t, http.MethodPost, u, token, `{}`) if code != http.StatusConflict || !strings.Contains(body, fmt.Sprintf(apiRetryNotHeld, host)) { t.Fatalf("a healthy host = %d %s, want 409 naming %s as not held", code, body, host) } noRetryWritten(t, fx, "kass") } // A console with no root key (read-only) cannot sign a retry: it answers // 503 and writes nothing, so no box is handed an unsigned entry it would // drop (spec §11). func TestAPIInstanceRetryWithNoKeyWritesNothing(t *testing.T) { fx := newConsoleFixture(t) const host = "web.kass.carlosapp.dev" sibling(t, fx, "kass", host, "stable") writeHeldBox(t, fx, host, "kass", "v2") fx.srv.cfg.Key = nil token := cliToken(t, fx, "paul@keymail.dev", "code-host-retry-k") u := fx.ts.URL + "/api/cli/apps/" + fx.sqid + "/kass/instances/" + host + "/retry" code, _, body := rawReq(t, http.MethodPost, u, token, `{}`) if code != http.StatusServiceUnavailable || !strings.Contains(body, apiRetryNoKey) { t.Fatalf("no key = %d %s, want 503", code, body) } noRetryWritten(t, fx, "kass") if n := serial(t, fx); n != 0 { t.Fatalf("serial = %d, want no announcement", n) } } // writeRetryReport publishes host held by v2 with the given retry fields, on // an armed box or, with code set, a declined one honouring its book. func writeRetryReport(t *testing.T, fx *consoleFixture, host, code string, retryV int, epoch uint64, book string) { t.Helper() hr := status.HostRollout{Account: fx.sqid, App: "kass", Installed: "v2", Good: "v1", HeldBy: "v2", Reason: status.ReasonHealth, HeldEpoch: epoch, HeldBook: book, RetryV: retryV} ro := &status.RolloutReport{Hosts: map[string]status.HostRollout{host: hr}} if code != "" { ro.Unavailable, ro.FailureBook = code, "honoured" hr.Attempt, hr.AttemptBuild, hr.Reason = status.AttemptDown, "v2", "" ro.Hosts[host] = hr } box := status.CollectBox(nil, status.Probes{MachineID: "box1"}, time.Now()) box.Metrics = &status.BoxMetrics{RouteVersions: map[string]string{host: "v2"}, Rollout: ro} if err := status.WriteBox(context.Background(), fx.rel, box); err != nil { t.Fatal(err) } } // Readiness round 2, Task 3 (plan 9's transitions): the door asks the // box's report before it writes a retry. A box with no retry_v (built // before the host-retry log) or no held_book cannot take one: 409, and // nothing is written or announced. A box whose failure book is not saved // publishes epoch 0, and no grant for it could clear anything: 409, try // again. A box with retry_v, an epoch and a book gets the signed grant in // host-retry.json, and nothing in restart-scope.json. Armed and declined // boxes alike. func TestAPIInstanceRetryAsksTheBoxsReportFirst(t *testing.T) { const host = "web.kass.carlosapp.dev" for _, box := range []string{"", "no-cgroup-delegation"} { for _, tc := range []struct { name string retryV int epoch uint64 book string want string // "" is the grant written }{ {"an older box", 0, heldEpoch, heldBook, apiRetryUnsupported}, {"an older box with no epoch", 0, 0, heldBook, apiRetryUnsupported}, {"no book", status.HostRetryV, heldEpoch, "", apiRetryUnsupported}, {"book not saved", status.HostRetryV, 0, heldBook, apiRetryUnsaved}, {"a box that takes it", status.HostRetryV, heldEpoch, heldBook, ""}, } { t.Run(box+" "+tc.name, func(t *testing.T) { fx := newConsoleFixture(t) sibling(t, fx, "kass", host, "stable") writeRetryReport(t, fx, host, box, tc.retryV, tc.epoch, tc.book) token := cliToken(t, fx, "paul@keymail.dev", "code-host-retry-v") u := fx.ts.URL + "/api/cli/apps/" + fx.sqid + "/kass/instances/" + host + "/retry" code, _, body := rawReq(t, http.MethodPost, u, token, `{}`) if tc.want != "" { if code != http.StatusConflict || !strings.Contains(body, tc.want) { t.Fatalf("retry = %d %s, want 409 %q", code, body, tc.want) } noRetryWritten(t, fx, "kass") if n := serial(t, fx); n != 0 { t.Fatalf("serial = %d, want no announcement", n) } return } if code != http.StatusOK { t.Fatalf("retry = %d %s", code, body) } r := lastScoped(t, fx, "kass") g, err := release.VerifyHostRetry(fx.pub, r.Scope.Signed, fx.sqid, "kass", host, time.Now()) if r.Scope.Kind != release.RestartScopeRetry || r.Scope.Value != host || err != nil || g.Build != "v2" || g.Epoch != heldEpoch || g.Book != heldBook { t.Fatalf("stored %+v, grant %+v, %v", r, g, err) } if n := serial(t, fx); n != 1 { t.Fatalf("serial = %d", n) } }) } } }