package console import ( "context" "crypto/ed25519" "encoding/json" "errors" "net/url" "strconv" "strings" "testing" "time" "github.com/carlosframework/platform/internal/hostactors" "github.com/carlosframework/platform/internal/hostops" "github.com/carlosframework/platform/internal/lease/leasetest" "github.com/carlosframework/platform/internal/release" ) // Every console in these tests is built by the full production constructor // with the flag on from the start, exactly as cmd/console builds it: // OpenGated (triggers armed before NewServer), NewServer with HostClaim and // the RAW bucket, raw versioned store, signing key and console id in Config, // then ConfigureClaimRuntime, which reads the signed roster, builds the // control writer, the production data and versioned boundaries and the // authority, and replaces every raw handle in Config. Nothing is set on the // authority by hand and nothing is switched on after construction. Nothing // below writes SQL except through the console's own code or the test-only // seed scope, and the raw bucket is only written to stage the world // (roster, markers, the initial admission). const ( crAcctID = int64(OpsAccountID) // NewServer's backfill creates it, sqid "bab" crSqid = "bab" crApp = "kass" crAppID = "incarnation-a" crOwner = "paul@keymail.dev" crBucket = "deploy-bucket" crNS = "platform" crConsole = "console-a" crOther = "console-b" ) var crEnrollment = map[string]string{crConsole: strings.Repeat("a", 32), crOther: strings.Repeat("b", 32)} type claimRestoreWorld struct { ctx context.Context bucket *release.DirStore a, b *Server } // publishRestoreRoster signs the host-actor roster naming both consoles // into the raw bucket, as the operator's roster publication does. func publishRestoreRoster(t *testing.T, raw release.Store, key ed25519.PrivateKey) { t.Helper() consoles := map[string]hostactors.Console{} for id, enrollment := range crEnrollment { consoles[id] = hostactors.Console{Enrollment: enrollment, InstanceID: id, Namespaces: []string{crNS}, DeploymentBucket: crBucket} } roster := hostactors.Roster{V: 1, DeploymentBucket: crBucket, Revision: 1, Boxes: map[string]hostactors.Box{}, Consoles: consoles, Holds: map[string]hostactors.Reservation{}, Activations: map[string]hostactors.Reservation{}} envelope, err := release.SignRootEnvelope(release.RootSigner(key), roster) if err != nil { t.Fatal(err) } if _, err := raw.Put(context.Background(), hostactors.Key, envelope, ""); err != nil { t.Fatal(err) } } func newEnabledRestoreConsole(t *testing.T, raw release.Store, key ed25519.PrivateKey, name string) *Server { t.Helper() st := openGatedStore(t) if !st.claimGated() { t.Fatal("OpenGated returned an ungated store") } versions := leasetest.NewVersionedBucket() versions.AddBucket(crBucket) srv, err := NewServer(Config{Origin: "https://console.example", Keymail: "https://keymail.example", HostClaim: true, HostConsoleID: name, Bucket: crBucket, Releases: raw, DBVersions: versions, Key: key, ReleasePub: key.Public().(ed25519.PublicKey)}, st) if err != nil { t.Fatal(err) } if err := srv.ConfigureClaimRuntime(t.Context()); err != nil { t.Fatalf("ConfigureClaimRuntime: %v", err) } a := srv.cfg.ClaimAuthority if a == nil || a.Control == nil || a.ConsoleID != name || a.Enrollment != crEnrollment[name] || srv.cfg.Releases == raw { t.Fatalf("production constructor did not install the authority over the raw bucket: %+v", a) } if _, _, err := srv.claimRestoreRuntime(); err != nil { t.Fatalf("restore runtime: %v", err) } return srv } // seedClaimRow writes this console's row for the incarnation, under the // test-only seed scope: the world before the test, not a door under test. func seedClaimRow(t *testing.T, st *Store, trashedAt time.Time) { t.Helper() scope := []claimSQLScope{{AccountID: crAcctID, App: crApp, AppID: crAppID, Mode: claimModeEffect}} err := st.withClaimSQL(gateCtx(t), scope, allowAllClaimSQL, func(v *Store) error { if _, err := v.CreateAppIn(crAcctID, crApp, crOwner, trashedAt.Add(-time.Hour)); err != nil { return err } if err := v.BindClaimAppIdentity(crAcctID, crApp, crAppID); err != nil { return err } return v.TrashAppRow(crAcctID, crApp, trashedAt) }) if err != nil { t.Fatal(err) } } func newClaimRestoreWorld(t *testing.T) (*claimRestoreWorld, time.Time) { t.Helper() w := &claimRestoreWorld{ctx: t.Context(), bucket: release.NewDirStore(t.TempDir())} a := hostops.AppAdmission{V: 1, AppID: crAppID, Revision: 1, Epoch: 1, State: "open", Registrations: map[string]hostops.Registration{}, Effects: map[string]hostops.Effect{}} body, _ := json.Marshal(a) if _, err := w.bucket.Put(w.ctx, "apps/"+crSqid+"/"+crApp+"/admission.json", body, ""); err != nil { t.Fatal(err) } if _, err := w.bucket.Put(w.ctx, "apps/"+crSqid+"/"+crApp+"/releases/r1.json", []byte(`{}`), ""); err != nil { t.Fatal(err) } trashed := time.Now().Add(-time.Hour).UTC() if err := release.TrashApp(w.ctx, w.bucket, crSqid, crApp, crOwner, trashed); err != nil { t.Fatal(err) } _, key, err := ed25519.GenerateKey(nil) if err != nil { t.Fatal(err) } publishRestoreRoster(t, w.bucket, key) w.a = newEnabledRestoreConsole(t, w.bucket, key, crConsole) w.b = newEnabledRestoreConsole(t, w.bucket, key, crOther) seedClaimRow(t, w.a.store, trashed) seedClaimRow(t, w.b.store, trashed) return w, trashed } func (w *claimRestoreWorld) account(t *testing.T, s *Server) *Account { t.Helper() acct, err := s.store.AccountBySqid(crSqid) if err != nil || acct == nil { t.Fatalf("account: %+v %v", acct, err) } return acct } func (w *claimRestoreWorld) admission(t *testing.T) hostops.AdmissionObserved { t.Helper() a, err := hostops.ReadAdmission(w.ctx, w.bucket, crSqid, crApp, crAppID) if err != nil { t.Fatal(err) } return a } func (w *claimRestoreWorld) marker(t *testing.T) (*release.AppTombstone, string) { t.Helper() tomb, tag, err := readTombstoneVersion(w.ctx, w.bucket, crSqid, crApp) if err != nil { t.Fatal(err) } return tomb, tag } func requireLife(t *testing.T, s *Server, trashed, restoring bool) { t.Helper() life, err := s.store.AppLife(crAcctID, crApp) if err != nil || life == nil { t.Fatalf("life: %+v %v", life, err) } if life.Trashed() != trashed || life.Restoring != restoring { t.Fatalf("row trashed=%v restoring=%v, want %v %v", life.Trashed(), life.Restoring, trashed, restoring) } } func requireSettledReceipt(t *testing.T, a hostops.AdmissionObserved) hostops.RestoreMark { t.Helper() v := a.Value if v.State != "open" || v.Op != nil || v.LastRestore == nil || v.LastRestore.State != "complete" || v.LastRestore.AppID != crAppID || v.LastRestore.Epoch == 0 || v.LastRestore.Epoch > v.Epoch { t.Fatalf("restore not settled: %+v receipt %+v", v, v.LastRestore) } for id, e := range v.Effects { if e.State != "complete" { t.Fatalf("effect %s still %s", id, e.State) } } return *v.LastRestore } func (w *claimRestoreWorld) materialize(t *testing.T, s *Server) { t.Helper() if err := MaterializeApps(w.ctx, s.cfg.Releases, s.store, true); err != nil { t.Fatal(err) } } // Issuer crash points, in restoreClaimApp's order. const ( crashAfterReceipt = iota // PrepareRestore saved, nothing issued crashAfterIssue // effect issued, restoring slot set, marker still there crashAfterDelete // the exact-ETag delete ran, receipt still prepared ) // crashAt drives console A's own issuer steps up to a crash point, through // the same control writer, permit, grant and data boundary restoreClaimApp // uses, in its order (BeginExclusive, PrepareRestore, Issue, SQL, delete), // and stops without settling. func (w *claimRestoreWorld) crashAt(t *testing.T, point int) (op, etag string) { t.Helper() control, self, err := w.a.claimRestoreRuntime() if err != nil { t.Fatal(err) } tomb, etag := w.marker(t) op = "restore-crashed" owner := self owner.Controller = op now := time.Now() h, err := control.BeginExclusive(w.ctx, crSqid, crApp, crAppID, hostops.AppOperation{ID: op, Kind: hostops.OpAppMutation, AppID: crAppID, Phase: "issued", Intent: map[string]string{"action": "restore", hostops.RestoreActionStarted: strconv.FormatInt(now.Unix(), 10)}}, owner) if err != nil { t.Fatal(err) } if err := h.PrepareRestore(w.ctx, etag, tomb.DeletedAt.Unix()); err != nil { t.Fatal(err) } if point == crashAfterReceipt { return op, etag } pctx, _, err := h.Runner(time.Now).Issue(w.ctx, hostops.Effect{ID: op, Resource: hostops.OpAppMutation, Action: "restore"}) if err != nil { t.Fatal(err) } scope := claimSQLScope{AccountID: crAcctID, App: crApp, AppID: crAppID} err = w.a.store.WithAppMutation(w.ctx, []sqlGrant{permitGrant(pctx, w.a.cfg.Releases, crSqid, scope)}, func(v *Store) error { _, err := v.SetClaimRestoring(crAcctID, crApp, crAppID, true) return err }) if err != nil { t.Fatal(err) } if point == crashAfterDelete { if err := release.DeleteIfMatch(pctx, w.a.cfg.Releases, release.AppTombstoneKey(crSqid, crApp), etag); err != nil { t.Fatal(err) } } return op, etag } // closeAdmission stages a foreign exclusive operation (as a purge would // hold) straight into the bucket. func (w *claimRestoreWorld) closeAdmission(t *testing.T) hostops.AdmissionObserved { t.Helper() a := w.admission(t) v := a.Value v.State = "closed" v.Op = &hostops.AppOperation{ID: "purge", Kind: hostops.OpPurging, AppID: crAppID} v.Revision++ body, _ := json.Marshal(v) if _, err := w.bucket.Put(w.ctx, a.Key, body, a.ETag); err != nil { t.Fatal(err) } return w.admission(t) } func TestClaimRestoreGateIsArmedOnEnabledConsoles(t *testing.T) { w, _ := newClaimRestoreWorld(t) for _, s := range []*Server{w.a, w.b} { _, err := s.store.db.Exec(`UPDATE apps SET deleted_at = 0 WHERE app = ?`, crApp) requireDenied(t, "ungated restore write", err) requireLife(t, s, true, false) } if _, err := NewServer(Config{Origin: "https://console.example", Keymail: "https://keymail.example", Releases: w.bucket, HostClaim: true}, openTestStore(t)); err == nil { t.Fatal("NewServer accepted HostClaim on an ungated database") } ungated, err := NewServer(Config{Origin: "https://console.example", Keymail: "https://keymail.example", Releases: w.bucket}, openTestStore(t)) if err != nil { t.Fatal(err) } if ungated.store.claimGated() { t.Fatal("flag-off console was gated") } } func TestClaimRestoreAcrossTwoGatedConsoles(t *testing.T) { w, _ := newClaimRestoreWorld(t) _, oldTag := w.marker(t) if err := w.a.restoreApp(w.ctx, w.account(t, w.a), crApp, time.Now()); err != nil { t.Fatalf("restore: %v", err) } if tomb, _ := w.marker(t); tomb != nil { t.Fatal("marker survived the restore") } receipt := requireSettledReceipt(t, w.admission(t)) if receipt.TombstoneETag != oldTag { t.Fatalf("receipt names %q, want the deleted version %q", receipt.TombstoneETag, oldTag) } requireLife(t, w.a, false, false) // Console B consumes the same incarnation-bound receipt and writes no // canonical state. before := w.admission(t) requireLife(t, w.b, true, false) w.materialize(t, w.b) requireLife(t, w.b, false, false) if after := w.admission(t); after.Value.AppID != before.Value.AppID || after.Value.Op != nil || *after.Value.LastRestore != *before.Value.LastRestore { t.Fatal("projection changed the restore evidence or left admission closed") } // A later trash wins although the old receipt survives, on both consoles. if err := release.TrashApp(w.ctx, w.bucket, crSqid, crApp, crOwner, time.Now().Add(time.Minute)); err != nil { t.Fatal(err) } w.materialize(t, w.b) w.materialize(t, w.a) requireLife(t, w.b, true, false) requireLife(t, w.a, true, false) // The old receipt cannot be replayed against the new marker. scope := claimSQLScope{AccountID: crAcctID, App: crApp, AppID: crAppID} err := w.b.store.WithAppMutation(w.ctx, []sqlGrant{restoreReceiptGrant(w.ctx, w.bucket, crSqid, scope, receipt)}, func(v *Store) error { _, err := v.FinishClaimRestoreRow(crAcctID, crApp, crAppID, receipt.DeletedAt, time.Now()) return err }) if !errors.Is(err, ErrClaimSQLDenied) { t.Fatalf("old receipt under a new marker: %v", err) } requireLife(t, w.b, true, false) } // The receipt precedes Issue, so both crash windows before the delete // (no effect yet, and an issued effect) are finished from the receipt and // never abandoned. The no-effect window needs ControlWriter.RecoverMutation // to accept a prepared receipt with no effect (control-work adaptation). func TestClaimRestoreCrashAfterReceiptBeforeDelete(t *testing.T) { for _, c := range []struct { name string point int finisher string }{ {"receipt-only/sweep", crashAfterReceipt, "sweep"}, {"receipt-only/retry", crashAfterReceipt, "retry"}, {"issued/sweep", crashAfterIssue, "sweep"}, {"issued/retry", crashAfterIssue, "retry"}, } { finisher := c.finisher t.Run(c.name, func(t *testing.T) { w, _ := newClaimRestoreWorld(t) op, etag := w.crashAt(t, c.point) if _, tag := w.marker(t); tag != etag { t.Fatal("fixture is not the crash window") } requireLife(t, w.a, true, c.point == crashAfterIssue) // Projection never restores from a prepared receipt. w.materialize(t, w.b) requireLife(t, w.b, true, false) switch finisher { case "sweep": // the other console's admission sweep if err := w.b.reconcileClaimRestore(w.ctx, crSqid, crApp, crAppID); err != nil { t.Fatal(err) } w.materialize(t, w.a) case "retry": // the person presses Restore again on console A if err := w.a.restoreApp(w.ctx, w.account(t, w.a), crApp, time.Now()); err != nil { t.Fatalf("retry: %v", err) } } if tomb, _ := w.marker(t); tomb != nil { t.Fatal("receipt was not replayed") } if r := requireSettledReceipt(t, w.admission(t)); r.ID != op || r.TombstoneETag != etag { t.Fatalf("settled the wrong receipt: %+v", r) } w.materialize(t, w.b) requireLife(t, w.a, false, false) requireLife(t, w.b, false, false) }) } } func TestClaimRestoreCrashAfterDeleteBeforeCompletion(t *testing.T) { w, _ := newClaimRestoreWorld(t) op, etag := w.crashAt(t, crashAfterDelete) if tomb, _ := w.marker(t); tomb != nil { t.Fatal("fixture is not the crash window") } if a := w.admission(t).Value; a.Op == nil || a.Op.ID != op || a.LastRestore.State != "prepared" { t.Fatalf("fixture admission: %+v", a) } // While the operation is open, no console projects anything. w.materialize(t, w.b) requireLife(t, w.b, true, false) // B's own restore door finishes A's receipt first (step 0), then answers // it as a restore already done. if err := w.b.restoreApp(w.ctx, w.account(t, w.b), crApp, time.Now()); err != nil { t.Fatalf("B's restore should finish A's receipt: %v", err) } if r := requireSettledReceipt(t, w.admission(t)); r.ID != op || r.TombstoneETag != etag { t.Fatalf("settled the wrong receipt: %+v", r) } requireLife(t, w.b, false, false) w.materialize(t, w.a) requireLife(t, w.a, false, false) } func TestClaimRestoreSQLFailureIsLocalAndReplayable(t *testing.T) { w, _ := newClaimRestoreWorld(t) // A real SQL failure on console A only: a persistent trigger written on // A's trusted boot handle after arming. if _, err := w.a.store.bootDB().Exec(`CREATE TRIGGER fail_restore BEFORE UPDATE OF deleted_at ON apps WHEN NEW.deleted_at = 0 BEGIN SELECT RAISE(ABORT, 'test restore SQL failure'); END`); err != nil { t.Fatal(err) } err := w.a.restoreApp(w.ctx, w.account(t, w.a), crApp, time.Now()) var rf *restoreFailure if !errors.As(err, &rf) || rf.Marker || !strings.Contains(err.Error(), "test restore SQL failure") { t.Fatalf("restore = %v, want the row-only failure", err) } // The global effect settled; only A's projection is behind. receipt := requireSettledReceipt(t, w.admission(t)) if tomb, _ := w.marker(t); tomb != nil { t.Fatal("marker survived") } requireLife(t, w.a, true, true) // Console B replays the same incarnation-bound terminal receipt without // opening anything. before := w.admission(t) w.materialize(t, w.b) requireLife(t, w.b, false, false) after := w.admission(t) if after.Value.AppID != before.Value.AppID || after.Value.Op != nil || *after.Value.LastRestore != *before.Value.LastRestore { t.Fatal("replay changed the restore evidence or left admission closed") } // A replays it too once its SQL works again. w.materialize(t, w.a) requireLife(t, w.a, true, true) // still failing: nothing reopened if _, err := w.a.store.bootDB().Exec(`DROP TRIGGER fail_restore`); err != nil { t.Fatal(err) } w.materialize(t, w.a) requireLife(t, w.a, false, false) if r := requireSettledReceipt(t, w.admission(t)); r != receipt { t.Fatal("receipt changed during replay") } } func TestClaimRestoreReplayNeverReopensClosedAdmission(t *testing.T) { w, _ := newClaimRestoreWorld(t) if err := w.a.restoreApp(w.ctx, w.account(t, w.a), crApp, time.Now()); err != nil { t.Fatal(err) } receipt := requireSettledReceipt(t, w.admission(t)) // Another operation now owns the app (as a purge would), with no marker // present. B still has its stale trashed row. closed := w.closeAdmission(t) w.materialize(t, w.b) requireLife(t, w.b, true, false) scope := claimSQLScope{AccountID: crAcctID, App: crApp, AppID: crAppID} err := w.b.store.WithAppMutation(w.ctx, []sqlGrant{restoreReceiptGrant(w.ctx, w.bucket, crSqid, scope, receipt)}, func(v *Store) error { _, err := v.FinishClaimRestoreRow(crAcctID, crApp, crAppID, receipt.DeletedAt, time.Now()) return err }) if !errors.Is(err, ErrClaimSQLDenied) { t.Fatalf("receipt replayed under a foreign operation: %v", err) } // A restore door on B neither starts an operation nor reopens this one. var rf *restoreFailure if err := w.b.restoreApp(w.ctx, w.account(t, w.b), crApp, time.Now()); !errors.As(err, &rf) || !rf.Marker { t.Fatalf("restore under a foreign operation = %v", err) } if now := w.admission(t); now.ETag != closed.ETag { t.Fatal("canonical admission changed") } requireLife(t, w.b, true, false) } func TestClaimRestoreSuccessorIncarnationIsRefused(t *testing.T) { w, _ := newClaimRestoreWorld(t) if err := w.a.restoreApp(w.ctx, w.account(t, w.a), crApp, time.Now()); err != nil { t.Fatal(err) } receipt := requireSettledReceipt(t, w.admission(t)) // A grant for a different incarnation cannot touch this row. other := claimSQLScope{AccountID: crAcctID, App: crApp, AppID: "incarnation-b"} r := receipt r.AppID = "incarnation-b" err := w.b.store.WithAppMutation(w.ctx, []sqlGrant{restoreReceiptGrant(w.ctx, w.bucket, crSqid, other, r)}, func(v *Store) error { _, err := v.FinishClaimRestoreRow(crAcctID, crApp, crAppID, r.DeletedAt, time.Now()) return err }) if err == nil { t.Fatal("successor grant reached the old incarnation") } requireLife(t, w.b, true, false) } // A restore's receipt carries the closed epoch its operation ran in. Later // legitimate operations reopen admission and raise the epoch; a console that // missed the restore still catches it up, through materialization and // through its own restore door, because the receipt binds the same immutable // app_id with receipt.Epoch <= admission.Epoch, no marker and no purge. func TestClaimRestoreReplaysAfterLaterEpochBumps(t *testing.T) { for _, via := range []string{"materialize", "door"} { t.Run(via, func(t *testing.T) { w, _ := newClaimRestoreWorld(t) if err := w.a.restoreApp(w.ctx, w.account(t, w.a), crApp, time.Now()); err != nil { t.Fatal(err) } receipt := requireSettledReceipt(t, w.admission(t)) control, self, err := w.a.claimRestoreRuntime() if err != nil { t.Fatal(err) } for i := 0; i < 2; i++ { op := "later-mutation-" + strconv.Itoa(i) owner := self owner.Controller = op h, err := control.BeginExclusive(w.ctx, crSqid, crApp, crAppID, hostops.AppOperation{ID: op, Kind: hostops.OpAppMutation, AppID: crAppID, Phase: "issued", Intent: map[string]string{"action": "settings"}}, owner) if err != nil { t.Fatal(err) } if err := h.FinishMutation(w.ctx); err != nil { t.Fatal(err) } } now := w.admission(t).Value if now.Epoch < receipt.Epoch+3 || now.LastRestore == nil || *now.LastRestore != receipt { t.Fatalf("fixture: epoch %d, receipt %+v", now.Epoch, now.LastRestore) } requireLife(t, w.b, true, false) switch via { case "materialize": w.materialize(t, w.b) case "door": if err := w.b.restoreApp(w.ctx, w.account(t, w.b), crApp, time.Now()); err != nil { t.Fatalf("B's restore door on a lagging row: %v", err) } } requireLife(t, w.b, false, false) if after := w.admission(t); after.Value.Epoch <= now.Epoch || after.Value.Op != nil || *after.Value.LastRestore != receipt { t.Fatal("catching up failed to release its projection operation or changed the receipt") } }) } } // The terminal replay grants bind terminal permits: a callback holding a // valid restore receipt or trash marker grant can still only write that // projection's columns of that incarnation's apps row. INSERT, a column the // projection does not own, DELETE and unbinding app_id are refused by SQLite. func TestClaimRestoreTerminalGrantsOnlyWriteTheirProjection(t *testing.T) { w, _ := newClaimRestoreWorld(t) if err := w.a.restoreApp(w.ctx, w.account(t, w.a), crApp, time.Now()); err != nil { t.Fatal(err) } receipt := requireSettledReceipt(t, w.admission(t)) scope := claimSQLScope{AccountID: crAcctID, App: crApp, AppID: crAppID} attempt := func(v *Store) { t.Helper() _, err := v.db.Exec(`INSERT INTO audit (app, address, action, created_at, account_id) VALUES (?, 'p', 'forged', 1, ?)`, crApp, crAcctID) requireDenied(t, "insert", err) _, err = v.db.Exec(`INSERT INTO apps (app, account_id, created_by, created_at) VALUES ('kass-twin', ?, 'x', 1)`, crAcctID) requireDenied(t, "insert apps (unbound adoption)", err) _, err = v.db.Exec(`UPDATE apps SET placement = 'evil' WHERE account_id = ? AND app = ?`, crAcctID, crApp) requireDenied(t, "foreign column", err) _, err = v.db.Exec(`UPDATE apps SET app_id = '' WHERE account_id = ? AND app = ?`, crAcctID, crApp) requireDenied(t, "unbinding app_id", err) _, err = v.db.Exec(`DELETE FROM apps WHERE account_id = ? AND app = ?`, crAcctID, crApp) requireDenied(t, "delete", err) } err := w.b.claimProjection(w.ctx, w.account(t, w.b), crApp, func(pctx context.Context) error { return w.b.store.WithAppMutation(pctx, []sqlGrant{restoreReceiptGrant(pctx, w.bucket, crSqid, scope, receipt)}, func(v *Store) error { attempt(v) changed, err := v.FinishClaimRestoreRow(crAcctID, crApp, crAppID, receipt.DeletedAt, time.Now()) if err == nil && !changed { err = errors.New("the projection itself did not land") } return err }) }) if err != nil { t.Fatal(err) } requireLife(t, w.b, false, false) // A later trash: the trash-marker grant hides the row and nothing else. if err := release.TrashApp(w.ctx, w.bucket, crSqid, crApp, crOwner, time.Now().Add(time.Minute)); err != nil { t.Fatal(err) } tomb, tag := w.marker(t) err = w.b.claimProjection(w.ctx, w.account(t, w.b), crApp, func(pctx context.Context) error { return w.b.store.WithAppMutation(pctx, []sqlGrant{tombstoneGrant(pctx, w.bucket, crSqid, scope, tag)}, func(v *Store) error { attempt(v) _, err := v.TrashClaimAppRow(crAcctID, crApp, crAppID, tomb.DeletedAt) return err }) }) if err != nil { t.Fatal(err) } requireLife(t, w.b, true, false) if n := countWhere(t, w.b.store, "apps", "app = ? AND placement = '' AND app_id = ?", crApp, crAppID); n != 1 { t.Fatal("a terminal grant wrote outside its projection") } // While any operation owns the app, the marker grant refuses outright: a // materializer cannot clear a live restore's restoring slot. w.closeAdmission(t) err = w.b.store.WithAppMutation(w.ctx, []sqlGrant{tombstoneGrant(w.ctx, w.bucket, crSqid, scope, tag)}, func(v *Store) error { _, err := v.SetClaimRestoring(crAcctID, crApp, crAppID, false) return err }) if !errors.Is(err, ErrClaimSQLDenied) { t.Fatalf("trash-marker grant under an operation: %v", err) } } // The disabled branch is the base restoreApp, byte for byte: its separate // SQL failure response is unchanged. func TestClaimDisabledRestorePreservesSQLFailureResponse(t *testing.T) { fx, c, csrf := trashPageFixture(t) if _, err := fx.st.db.Exec(`CREATE TRIGGER fail_restore BEFORE UPDATE OF deleted_at ON apps WHEN NEW.deleted_at=0 BEGIN SELECT RAISE(ABORT,'test restore SQL failure'); END`); err != nil { t.Fatal(err) } _, location, _ := postForm(t, c, fx.u("/trash/restore"), url.Values{"csrf": {csrf}, "app": {"kass"}}) _, _, body := get(t, c, fx.ts.URL+location) if !strings.Contains(body, "The trash marker was removed") || strings.Contains(body, "Could not remove the trash marker") { t.Fatalf("wrong disabled SQL failure response: %s", body) } if tomb, err := release.ReadAppTombstone(t.Context(), fx.rel, fx.sqid, "kass"); err != nil || tomb != nil { t.Fatalf("marker: %+v %v", tomb, err) } }