package console import ( "context" "errors" "fmt" "net/http" "net/url" "strings" "testing" "time" "github.com/carlosframework/platform/internal/provision" "github.com/carlosframework/platform/internal/release" ) // trashedTemp seeds tmp-a, expired, in the Trash with its 3-day window, the // state the expiry pass leaves. func trashedTemp(t *testing.T, fx *consoleFixture, app string, at time.Time) { t.Helper() tempApp(t, fx, app, at) if err := release.TrashApp(context.Background(), fx.rel, fx.sqid, app, "expiry", at); err != nil { t.Fatal(err) } if ok, err := fx.st.ExpireAppRow(OpsAccountID, app, at); err != nil || !ok { t.Fatal(ok, err) } } // Spec §6 and §9: rewrites expiry before deleting the tombstone, and clears // the deadline only after. The order is asserted at the moment of the delete. func TestRestoreRewritesExpiryBeforeTheMarkerGoes(t *testing.T) { fx := newConsoleFixture(t) now := time.Now().Truncate(time.Second) trashedTemp(t, fx, "tmp-a", now.Add(-time.Hour)) h := &hookStore{Store: fx.rel} marker := release.AppTombstoneKey(fx.sqid, "tmp-a") var atDelete *AppLife h.onDelete = func(key string) { if key == marker { atDelete = lifeOf(t, fx, "tmp-a") } } fx.srv.cfg.Releases = h if err := fx.srv.restoreApp(context.Background(), opsAccount(t, fx), "tmp-a", now); err != nil { t.Fatal(err) } if atDelete == nil { t.Fatal("the marker was never deleted") } if atDelete.ExpiresAt != now.Add(temporaryTTLDefault).Unix() || !atDelete.Restoring || atDelete.PurgeAfter == 0 { t.Fatalf("row at the marker delete = %+v, want expiry now+7d, restoring, deadline still set", atDelete) } if l := lifeOf(t, fx, "tmp-a"); l.Trashed() || l.Restoring || l.PurgeAfter != 0 || l.PurgeFor != 0 { t.Fatalf("row after = %+v, want live, no deadline", l) } } // Spec §9: a failed tombstone delete keeps the deadline and releases restoring. func TestRestoreWhoseMarkerDeleteFailsKeepsTheDeadline(t *testing.T) { fx := newConsoleFixture(t) now := time.Now().Truncate(time.Second) trashedTemp(t, fx, "tmp-a", now.Add(-time.Hour)) before := *lifeOf(t, fx, "tmp-a") fx.srv.cfg.Releases = &hookStore{Store: fx.rel, failDelete: map[string]int{release.AppTombstoneKey(fx.sqid, "tmp-a"): 1}} err := fx.srv.restoreApp(context.Background(), opsAccount(t, fx), "tmp-a", now) var rf *restoreFailure if !errors.As(err, &rf) || !rf.Marker { t.Fatalf("restoreApp = %v, want a marker-half restoreFailure", err) } l := lifeOf(t, fx, "tmp-a") if l.Restoring || !l.Trashed() || l.PurgeAfter != before.PurgeAfter || l.PurgeFor != before.PurgeFor { t.Fatalf("row = %+v, want trashed with the same deadline and restoring released", l) } } // Spec §9: a failed step 5 holds the cap slot until settled. func TestRestoreThatStopsBeforeStepFiveHoldsItsSlot(t *testing.T) { fx := newConsoleFixture(t) now := time.Now().Truncate(time.Second) for i := 0; i < temporaryAppCap-1; i++ { tempApp(t, fx, fmt.Sprintf("tmp-live%02d", i), now.Add(time.Hour)) } trashedTemp(t, fx, "tmp-a", now.Add(-time.Hour)) // Steps 3 and 4 done, step 5 never ran. fx.st.SetRestoring(OpsAccountID, "tmp-a", true) release.RestoreApp(context.Background(), fx.rel, fx.sqid, "tmp-a") if err := fx.srv.temporaryCapCheck(OpsAccountID); err == nil { t.Fatal("the half-restored app's slot was free for a new app to take") } if err := fx.srv.settleApp(context.Background(), OpsAccountID, fx.sqid, "tmp-a", now, always(true)); err != nil { t.Fatal(err) } if l := lifeOf(t, fx, "tmp-a"); l.Trashed() || l.Restoring { t.Fatalf("settled row = %+v, want live", l) } } // Spec §9: the cap is enforced on every console restore door. func TestRestoreDoorsEnforceTheCap(t *testing.T) { fx := newConsoleFixture(t) now := time.Now().Truncate(time.Second) for i := 0; i < temporaryAppCap; i++ { tempApp(t, fx, fmt.Sprintf("tmp-live%02d", i), now.Add(time.Hour)) } trashedTemp(t, fx, "tmp-a", now.Add(-time.Hour)) token := cliToken(t, fx, "paul@keymail.dev", "code-restore-cap") status, out := postJSON(t, fx.ts.URL+"/api/cli/apps/"+fx.sqid+"/tmp-a/restore", map[string]string{}, token) if status != http.StatusConflict || out["code"] != "cap" { t.Fatalf("API restore over the cap = %d %v, want 409 cap", status, out) } c, csrf := signInAs(t, fx, "paul@keymail.dev", "code-restore-cap-web") st, loc, _ := postForm(t, c, fx.u("/trash/restore"), url.Values{"csrf": {csrf}, "app": {"tmp-a"}}) if _, _, page := get(t, c, fx.ts.URL+loc); st != http.StatusSeeOther || !strings.Contains(page, "tmp-live00") { t.Fatalf("browser restore over the cap = %d, want the refusal listing the live apps", st) } if markerOf(t, fx, "tmp-a") == nil { t.Fatal("a refused restore removed the marker") } } // Spec §5 "Trash by hand": the doors set the window for a temporary row only. func TestTrashDoorsSetTheWindowForTemporaryRows(t *testing.T) { fx := newConsoleFixture(t) tempApp(t, fx, "tmp-a", time.Now().Add(30*24*time.Hour)) token := cliToken(t, fx, "paul@keymail.dev", "code-trash-window") for _, app := range []string{"tmp-a", "kass"} { if status, out := postJSON(t, fx.ts.URL+"/api/cli/apps/"+fx.sqid+"/"+app+"/delete", map[string]string{}, token); status != http.StatusOK { t.Fatalf("delete %s = %d %v", app, status, out) } } tmp, kass := lifeOf(t, fx, "tmp-a"), lifeOf(t, fx, "kass") if tmp.PurgeFor != tmp.DeletedAt || tmp.PurgeAfter != tmp.DeletedAt+72*3600 { t.Fatalf("temporary trash = %+v, want the 3-day window", tmp) } if kass.PurgeAfter != 0 || kass.PurgeFor != 0 { t.Fatalf("permanent trash = %+v, want none", kass) } } // waitForWaiter returns once a second holder is queued on key: the request // under test has passed every check it makes before the lock and is blocked // on it. Deterministic: it waits for the queue, not for a clock. func waitForWaiter(t *testing.T, l *keyedLocks, key string) { t.Helper() deadline := time.Now().Add(10 * time.Second) for time.Now().Before(deadline) { l.mu.Lock() n := 0 if e := l.m[key]; e != nil { n = e.refs } l.mu.Unlock() if n >= 2 { return } time.Sleep(time.Millisecond) } t.Fatalf("no request queued on %s", key) } // Astra plan R1 finding 1: a trash door wrote the marker and failed its row // update, so the row is live. The manual purge must settle first: without it // PurgeApp sweeps the objects and PurgeAppRow (guarded deleted_at != 0) // silently keeps a live row that nothing ever repairs. func TestManualPurgeAfterAFailedTrashRowUpdate(t *testing.T) { fx := newConsoleFixture(t) if err := release.TrashApp(context.Background(), fx.rel, fx.sqid, "kass", "paul@keymail.dev", time.Now()); err != nil { t.Fatal(err) } if l := lifeOf(t, fx, "kass"); l.Trashed() { t.Fatal("setup: the row must still be live") } c, csrf := signInAs(t, fx, "paul@keymail.dev", "code-purge-unsettled") postGated(t, fx, c, fx.u("/trash/delete"), url.Values{"csrf": {csrf}, "app": {"kass"}, "confirm": {"kass"}}) if keys := appKeys(t, fx, "kass"); len(keys) != 0 { t.Fatalf("purge left objects: %v", keys) } if l := lifeOf(t, fx, "kass"); l != nil { t.Fatalf("row = %+v after a purge, want it gone with the objects", l) } } // Astra plan round 4: restoring a provisioned temporary static app must not // send anyone to carlos add, which refuses every tmp- name. Its instance // record survives trash and restore, and with the marker gone provision's // reconcile mints the route again (box side: TestReconcileDeclinesTrashedApp, // run in Step 4). Both restore doors are checked. func TestRestoreOfAProvisionedTemporaryStaticApp(t *testing.T) { fx := newConsoleFixture(t) fx.srv.cfg.AppsDomain = "carlosapp.test" tempApp(t, fx, "tmp-site", time.Now().Add(24*time.Hour)) token := cliToken(t, fx, "paul@keymail.dev", "code-restore-provisioned") base := fx.ts.URL + "/api/cli/apps/" + fx.sqid + "/tmp-site" if st, out := putJSON(t, base+"/instances-manifest", map[string]any{}, token); st != http.StatusOK { t.Fatalf("enable = %d %v", st, out) } apex := "tmp-site." + fx.sqid + ".carlosapp.test" if st, _, out := rawReq(t, http.MethodPost, base+"/instances", token, `{"host":"`+apex+`","kind":"static","channel":"edge"}`); st != http.StatusCreated { t.Fatalf("instance = %d %s", st, out) } recordKept := func(when string) { t.Helper() rec, err := provision.GetRecord(context.Background(), fx.rel, fx.sqid, "tmp-site", apex) if err != nil || rec == nil || rec.Kind != "static" || rec.Host != apex { t.Fatalf("%s: instance record = %+v %v, want the static record at %s kept", when, rec, err, apex) } } // API: delete, then restore. if st, out := postJSON(t, base+"/delete", map[string]string{}, token); st != http.StatusOK { t.Fatalf("delete = %d %v", st, out) } recordKept("after trash") st, out := postJSON(t, base+"/restore", map[string]string{}, token) if st != http.StatusOK { t.Fatalf("restore = %d %v", st, out) } if note := str(t, out, "note"); strings.Contains(note, "carlos add") || !strings.Contains(note, "tmp-site") { t.Fatalf("restore note = %q, want the instance-record guidance, never carlos add", note) } recordKept("after API restore") if markerOf(t, fx, "tmp-site") != nil || lifeOf(t, fx, "tmp-site").Trashed() { t.Fatal("after restore the marker must be gone and the row live, so reconcile mints the route again") } // Browser: delete through the API again, restore from the Trash page. if st, out := postJSON(t, base+"/delete", map[string]string{}, token); st != http.StatusOK { t.Fatalf("second delete = %d %v", st, out) } c, csrf := signInAs(t, fx, "paul@keymail.dev", "code-restore-provisioned-web") st, loc, _ := postForm(t, c, fx.u("/trash/restore"), url.Values{"csrf": {csrf}, "app": {"tmp-site"}}) if st != http.StatusSeeOther { t.Fatalf("browser restore = %d", st) } if _, _, page := get(t, c, fx.ts.URL+loc); strings.Contains(page, "carlos add") || !strings.Contains(page, "tmp-site") { t.Fatal("the browser restore flash sends a temporary app to carlos add, or does not name it") } recordKept("after browser restore") } // Astra plan round 2: settling under the lock must not turn a trash retry into // a refusal. A marker already in the bucket (a headless trash) and a live row: // the API trash settles the row onto the marker's clock and then answers 200 // through its ordinary path, with the marker untouched, the existing deadline, // and a serial bump. func TestTrashAfterAHeadlessTrashIsIdempotent(t *testing.T) { fx := newConsoleFixture(t) ctx := context.Background() old := time.Now().Add(-5 * 24 * time.Hour).Truncate(time.Second) if err := release.TrashApp(ctx, fx.rel, fx.sqid, "kass", "headless", old); err != nil { t.Fatal(err) } serialBefore, _, _ := fx.rel.Get(ctx, release.SerialKey) token := cliToken(t, fx, "paul@keymail.dev", "code-trash-idempotent") status, out := postJSON(t, fx.ts.URL+"/api/cli/apps/"+fx.sqid+"/kass/delete", map[string]string{}, token) if status != http.StatusOK { t.Fatalf("delete after a headless trash = %d %v, want 200", status, out) } until, err := time.Parse(time.RFC3339, str(t, out, "restorable_until")) if err != nil || !until.Equal(old.Add(release.TrashCooloff)) { t.Fatalf("restorable_until = %v %v, want the headless marker's %v", until, err, old.Add(release.TrashCooloff)) } if tomb := markerOf(t, fx, "kass"); tomb == nil || !tomb.DeletedAt.Equal(old) || tomb.Actor != "headless" { t.Fatalf("marker = %+v, want the headless one, untouched", tomb) } if serialAfter, _, _ := fx.rel.Get(ctx, release.SerialKey); string(serialAfter) == string(serialBefore) { t.Fatal("the idempotent success skipped the serial bump") } } // Astra plan R1 finding 2 (spec §2: a check made before the lock is a hint). // Each case holds the app's lock, lets the destructive request queue on it, // changes a precondition, and releases. The door must refuse on what is true // once it holds the lock. func TestDestructiveDoorsRecheckAfterTheLock(t *testing.T) { run := func(t *testing.T, fx *consoleFixture, app string, request func() int, change func()) int { t.Helper() unlock := fx.srv.lockApp(fx.sqid, app) done := make(chan int, 1) go func() { done <- request() }() waitForWaiter(t, &fx.srv.appLocks, fx.sqid+"/"+app) change() unlock() return <-done } t.Run("API trash: marked control plane while waiting", func(t *testing.T) { fx := newConsoleFixture(t) token := cliToken(t, fx, "paul@keymail.dev", "code-recheck-api-cp") status := run(t, fx, "kass", func() int { st, _ := postRaw(t, fx.ts.URL+"/api/cli/apps/"+fx.sqid+"/kass/delete", map[string]string{}, token) return st }, func() { if err := fx.st.SetAppControlPlane(OpsAccountID, "kass", true); err != nil { t.Error(err) } }) if status != http.StatusForbidden || markerOf(t, fx, "kass") != nil || lifeOf(t, fx, "kass").Trashed() { t.Fatalf("status %d: the control plane was trashed by a request that checked the flag before waiting", status) } }) t.Run("API trash: row purged while waiting", func(t *testing.T) { fx := newConsoleFixture(t) token := cliToken(t, fx, "paul@keymail.dev", "code-recheck-api-gone") status := run(t, fx, "kass", func() int { st, _ := postRaw(t, fx.ts.URL+"/api/cli/apps/"+fx.sqid+"/kass/delete", map[string]string{}, token) return st }, func() { fx.st.TrashAppRow(OpsAccountID, "kass", time.Now()) fx.st.PurgeAppRow(OpsAccountID, "kass") }) if status != http.StatusNotFound || markerOf(t, fx, "kass") != nil { t.Fatalf("status %d: a delayed trash wrote a marker for a row that no longer exists", status) } }) t.Run("browser delete: marked control plane while waiting", func(t *testing.T) { fx := newConsoleFixture(t) c, csrf := signInAs(t, fx, "paul@keymail.dev", "code-recheck-web-cp") run(t, fx, "kass", func() int { st, _, _ := postGated(t, fx, c, fx.u("/apps/kass/delete"), url.Values{"csrf": {csrf}, "confirm": {"kass"}}) return st }, func() { if err := fx.st.SetAppControlPlane(OpsAccountID, "kass", true); err != nil { t.Error(err) } }) if markerOf(t, fx, "kass") != nil || lifeOf(t, fx, "kass").Trashed() { t.Fatal("the browser delete trashed the control plane after waiting on the lock") } }) t.Run("purge door: marked control plane while waiting", func(t *testing.T) { fx := newConsoleFixture(t) now := time.Now() if err := release.TrashApp(context.Background(), fx.rel, fx.sqid, "kass", "paul@keymail.dev", now); err != nil { t.Fatal(err) } fx.st.TrashAppRow(OpsAccountID, "kass", now) before := len(appKeys(t, fx, "kass")) c, csrf := signInAs(t, fx, "paul@keymail.dev", "code-recheck-purge-cp") run(t, fx, "kass", func() int { st, _, _ := postGated(t, fx, c, fx.u("/trash/delete"), url.Values{"csrf": {csrf}, "app": {"kass"}, "confirm": {"kass"}}) return st }, func() { if err := fx.st.SetAppControlPlane(OpsAccountID, "kass", true); err != nil { t.Error(err) } }) if got := len(appKeys(t, fx, "kass")); got != before || lifeOf(t, fx, "kass") == nil { t.Fatalf("the purge door destroyed the control plane after waiting on the lock (%d of %d keys left)", got, before) } }) } // Task 7 review, fix round 1: a restore whose step 5 failed has already // removed the marker. The retry must answer as a restore, with the audit row // and the serial bump, not "not in the trash". The failed attempt bumps the // serial too, because the bucket has already changed. Both doors. func TestRestoreRetryAfterAFailedRowUpdate(t *testing.T) { failStepFive := func(t *testing.T, fx *consoleFixture) func() { t.Helper() if _, err := fx.st.db.Exec(`CREATE TRIGGER fail_step_five BEFORE UPDATE ON apps WHEN OLD.deleted_at != 0 AND NEW.deleted_at = 0 BEGIN SELECT RAISE(ABORT, 'test: step 5 refused'); END`); err != nil { t.Fatal(err) } return func() { t.Helper() if _, err := fx.st.db.Exec(`DROP TRIGGER fail_step_five`); err != nil { t.Fatal(err) } } } serial := func(t *testing.T, fx *consoleFixture) string { t.Helper() b, _, _ := fx.rel.Get(context.Background(), release.SerialKey) return string(b) } t.Run("API", func(t *testing.T) { fx := newConsoleFixture(t) token := cliToken(t, fx, "paul@keymail.dev", "code-restore-retry-api") base := fx.ts.URL + "/api/cli/apps/" + fx.sqid + "/kass" if st, out := postJSON(t, base+"/delete", map[string]string{}, token); st != http.StatusOK { t.Fatalf("delete = %d %v", st, out) } restoreRow := failStepFive(t, fx) before := serial(t, fx) if st, out := postJSON(t, base+"/restore", map[string]string{}, token); st != http.StatusInternalServerError { t.Fatalf("restore with step 5 failing = %d %v, want 500", st, out) } if markerOf(t, fx, "kass") != nil || !lifeOf(t, fx, "kass").Trashed() { t.Fatal("setup: want the marker gone and the row still trashed") } if serial(t, fx) == before { t.Fatal("the failed step 5 skipped the serial bump, though the marker is gone") } restoreRow() before = serial(t, fx) if st, out := postJSON(t, base+"/restore", map[string]string{}, token); st != http.StatusOK || out["result"] != "restored" { t.Fatalf("retry = %d %v, want 200 restored", st, out) } if l := lifeOf(t, fx, "kass"); l.Trashed() || l.Restoring { t.Fatalf("row after the retry = %+v, want live", l) } auditedBy(t, fx, "kass", "restore", "paul@keymail.dev", "kass") if serial(t, fx) == before { t.Fatal("the retry skipped the serial bump") } }) t.Run("browser", func(t *testing.T) { fx := newConsoleFixture(t) token := cliToken(t, fx, "paul@keymail.dev", "code-restore-retry-web") if st, out := postJSON(t, fx.ts.URL+"/api/cli/apps/"+fx.sqid+"/kass/delete", map[string]string{}, token); st != http.StatusOK { t.Fatalf("delete = %d %v", st, out) } c, csrf := signInAs(t, fx, "paul@keymail.dev", "code-restore-retry-web-in") restoreRow := failStepFive(t, fx) before := serial(t, fx) postForm(t, c, fx.u("/trash/restore"), url.Values{"csrf": {csrf}, "app": {"kass"}}) if markerOf(t, fx, "kass") != nil || !lifeOf(t, fx, "kass").Trashed() { t.Fatal("setup: want the marker gone and the row still trashed") } if serial(t, fx) == before { t.Fatal("the failed step 5 skipped the serial bump, though the marker is gone") } restoreRow() before = serial(t, fx) st, loc, _ := postForm(t, c, fx.u("/trash/restore"), url.Values{"csrf": {csrf}, "app": {"kass"}}) if _, _, page := get(t, c, fx.ts.URL+loc); st != http.StatusSeeOther || strings.Contains(page, "nothing called") { t.Fatalf("retry = %d, want the restored flash, not the not-in-trash refusal", st) } if l := lifeOf(t, fx, "kass"); l.Trashed() || l.Restoring { t.Fatalf("row after the retry = %+v, want live", l) } auditedBy(t, fx, "kass", "restore", "paul@keymail.dev", "kass") if serial(t, fx) == before { t.Fatal("the retry skipped the serial bump") } }) }