package console import ( "context" "errors" "os" "path/filepath" "strings" "sync" "testing" "time" "github.com/carlosframework/platform/internal/hostops" "github.com/carlosframework/platform/internal/release" ) // I2-2: a console database restored onto a replacement instance opens under // a new database ID. Effects bound to the original ID can never settle here. func TestPlan9Fix2RestoreOntoReplacementInstanceAdoptsNewIdentity(t *testing.T) { path := filepath.Join(t.TempDir(), "console.db") openAs := func(instance string) (*Store, string) { t.Helper() restore := UseSQLInstanceIdentityForTest(instance) defer restore() s, err := OpenGated(path) if err != nil { t.Fatalf("open on %s: %v", instance, err) } id, err := s.sqlDatabaseID() if err != nil { s.Close() t.Fatal(err) } return s, id } s, original := openAs("i-original") s.Close() replacement, adopted := openAs("i-replacement") if adopted == original || !strings.Contains(adopted, ":i-replacement:") { replacement.Close() t.Fatalf("replacement did not adopt a new identity: original %q, adopted %q", original, adopted) } nonce := func(id string) string { return strings.SplitN(id, ":", 2)[0] } if nonce(adopted) == nonce(original) { replacement.Close() t.Fatalf("adoption kept the original nonce %q", nonce(original)) } // The original's effects stay unsettled: a fence for them is refused. sc := hostops.SQLReceiptScope{Effect: "e1", Operation: "op1", AppID: "app", Epoch: 1, Database: original} if _, err := replacement.FenceSQLOutcome(t.Context(), sc); !errors.Is(err, hostops.ErrIdentity) { replacement.Close() t.Fatalf("replacement fenced an effect bound to the original database: %v", err) } replacement.Close() // The adopted identity is stable across restarts on the replacement. again, id := openAs("i-replacement") again.Close() if id != adopted { t.Fatalf("adopted identity changed on reopen: %q then %q", adopted, id) } // Moving the copy back to the original instance is another adoption, not // a return to the original identity. back, id := openAs("i-original") back.Close() if id == original || id == adopted { t.Fatalf("copy moved back reused an earlier identity: %q", id) } // m-3: the same file on the same instance keeps its ID (a block-level // restore looks like this). The runbook's break-glass step copies it to // a new file and moves that into place, which gives it a new ID. same, sameID := openAs("i-original") same.Close() if sameID != id { t.Fatalf("same file on the same instance changed identity: %q then %q", id, sameID) } b, err := os.ReadFile(path) if err != nil { t.Fatal(err) } if err = os.WriteFile(path+".copy", b, 0o600); err != nil { t.Fatal(err) } if err = os.Rename(path+".copy", path); err != nil { t.Fatal(err) } copied, copiedID := openAs("i-original") copied.Close() if copiedID == id { t.Fatalf("a file copy moved into place kept the identity %q", id) } } // replacedFixture is a gated console whose sweep sees one SQL effect past // sql_recover_after bound to database, with two operators of whom only // ops-a has a keymail grant. func replacedFixture(t *testing.T, st *Store, database string) (*Server, *[]string) { t.Helper() raw := release.NewDirStore(t.TempDir()) w := claimTestWriter(t, raw) reg := hostops.Registration{Kind: hostops.RegistrationIngest, Owner: claimTestOwner, AppID: "app", Epoch: 1, Expires: time.Now().Add(time.Hour)} past := time.Now().Add(-time.Minute).UTC().Format(time.RFC3339Nano) seedClaimAdmission(t, raw, "bab", "demo", hostops.AppAdmission{V: 1, AppID: "app", Revision: 1, Epoch: 1, State: "open", Registrations: map[string]hostops.Registration{"old": reg}, Effects: map[string]hostops.Effect{"old": {ID: "old", OperationID: "old", AppID: "app", Resource: "app-sql:old", Action: "sql-ingest", Owner: claimTestOwner, State: "uncertain", Inputs: map[string]string{"sql_database": database, "sql_recover_after": past}}}}) for _, addr := range []string{"ops-a@keymail.dev", "ops-b@keymail.dev"} { if err := st.SetOperator(addr, true); err != nil { t.Fatal(err) } } if err := st.SetGrantToken("ops-a@keymail.dev", "grant-a", time.Now()); err != nil { t.Fatal(err) } var mu sync.Mutex sent := &[]string{} s := &Server{store: st, cfg: Config{HostClaimOps: &hostops.Controller{Store: raw, Control: w, Now: time.Now}}} s.sendApp = func(_ context.Context, _, to, _, _ string) (string, error) { mu.Lock() defer mu.Unlock() *sent = append(*sent, to) return "msg", nil } return s, sent } // m-4: the alert send is bounded by the sweep's own context, not a fresh // background deadline per recipient. func TestPlan9Fix2ReplacedAlertSendHonoursSweepContext(t *testing.T) { st := openGatedStore(t) s, _ := replacedFixture(t, st, "restored-away") for _, addr := range []string{"ops-b@keymail.dev", "ops-c@keymail.dev"} { if err := st.SetOperator(addr, true); err != nil { t.Fatal(err) } if err := st.SetGrantToken(addr, "grant-"+addr, time.Now()); err != nil { t.Fatal(err) } } s.sendApp = func(ctx context.Context, _, _, _, _ string) (string, error) { <-ctx.Done() return "", ctx.Err() } ctx, cancel := context.WithTimeout(t.Context(), 200*time.Millisecond) defer cancel() start := time.Now() err := s.sweepOneClaimAdmission(ctx, hostops.App{Account: "bab", Name: "demo"}) if !errors.Is(err, ErrSQLDatabaseReplaced) { t.Fatalf("sweep: %v", err) } if took := time.Since(start); took > 5*time.Second { t.Fatalf("a stalled keymail send held the serial sweep for %v", took) } } // m-4: a transient failure to read the recipients is retried on the next // sweep instead of being recorded as "nobody to tell". func TestPlan9Fix2ReplacedAlertRetriesAfterRecipientFailure(t *testing.T) { st := openGatedStore(t) s, sent := replacedFixture(t, st, "restored-away") if _, err := st.rawBoot.Exec(`ALTER TABLE users RENAME TO users_away`); err != nil { t.Fatal(err) } app := hostops.App{Account: "bab", Name: "demo"} if err := s.sweepOneClaimAdmission(t.Context(), app); !errors.Is(err, ErrSQLDatabaseReplaced) { t.Fatalf("sweep: %v", err) } if _, known, _ := st.AlertSeen(sqlReceiptAlertBox, sqlReplacedRule("bab", "demo", "old"), noMailboxAddress); known { t.Fatal("a transient recipient failure was recorded as no mailbox") } if _, err := st.rawBoot.Exec(`ALTER TABLE users_away RENAME TO users`); err != nil { t.Fatal(err) } if err := s.sweepOneClaimAdmission(t.Context(), app); !errors.Is(err, ErrSQLDatabaseReplaced) { t.Fatalf("sweep: %v", err) } if len(*sent) != 1 || (*sent)[0] != "ops-a@keymail.dev" { t.Fatalf("alert not delivered after the recipients recovered: %v", *sent) } } // m-4: the missing-grant note is logged when an alert is delivered, not on // every sweep that finds nothing new to say. func TestPlan9Fix2ReplacedAlertMissingGrantLoggedOnce(t *testing.T) { st := openGatedStore(t) s, sent := replacedFixture(t, st, "restored-away") buf := captureLog(t) app := hostops.App{Account: "bab", Name: "demo"} for range 3 { if err := s.sweepOneClaimAdmission(t.Context(), app); !errors.Is(err, ErrSQLDatabaseReplaced) { t.Fatalf("sweep: %v", err) } } if len(*sent) != 1 { t.Fatalf("sent %v, want one delivery", *sent) } if n := strings.Count(buf.String(), "no keymail grant for ops-b@keymail.dev"); n != 1 { t.Fatalf("missing-grant note logged %d times over three sweeps:\n%s", n, buf.String()) } } // m-4: a database replaced under the running console alerts like a restored // one, rather than surfacing only as an identity error. func TestPlan9Fix2DatabaseReplacedUnderRunningConsoleAlerts(t *testing.T) { st := openGatedStore(t) id, err := st.sqlDatabaseID() if err != nil { t.Fatal(err) } s, sent := replacedFixture(t, st, id) // Replace the file under the running process: same path, new inode. path := st.gate.path b, err := os.ReadFile(path) if err != nil { t.Fatal(err) } if err = os.WriteFile(path+".restored", b, 0o600); err != nil { t.Fatal(err) } if err = os.Rename(path+".restored", path); err != nil { t.Fatal(err) } err = s.sweepOneClaimAdmission(t.Context(), hostops.App{Account: "bab", Name: "demo"}) if !errors.Is(err, ErrSQLDatabaseReplaced) { t.Fatalf("live replacement surfaced only as %v", err) } if len(*sent) != 1 { t.Fatalf("no operator alert for a live replacement: %v", *sent) } } // m-5: a callback that calls back into the outer, captured Store (rather // than the view it was given) is refused at once instead of waiting on the // connection the view holds, and no remote check runs first. func TestPlan9Fix2OuterStoreInsideViewRefused(t *testing.T) { s := openGatedStore(t) seedGateApp(t, s) t.Run("nested mutation through the outer store", func(t *testing.T) { calls := 0 g := sqlGrant{scope: gateScope, kind: grantEffect, check: func(context.Context) error { calls++; return nil }} ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) defer cancel() err := s.withClaimSQL(ctx, []claimSQLScope{gateScope}, allowAllClaimSQL, func(*Store) error { return s.WithAppMutation(ctx, []sqlGrant{g}, func(*Store) error { return nil }) }) if !errors.Is(err, ErrClaimSQLScope) { t.Fatalf("nested mutation through the outer store: %v", err) } if calls != 0 { t.Fatalf("ran %d remote checks before refusing", calls) } }) t.Run("plain outer query", func(t *testing.T) { ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) defer cancel() var qerr error err := s.withClaimSQL(ctx, []claimSQLScope{gateScope}, allowAllClaimSQL, func(*Store) error { _, qerr = s.Operators() return nil }) if err != nil { t.Fatal(err) } if !errors.Is(qerr, ErrClaimSQLScope) { t.Fatalf("outer query inside the view: %v", qerr) } }) // Other goroutines still wait their turn rather than being refused. t.Run("other goroutine waits", func(t *testing.T) { release := make(chan struct{}) held := make(chan struct{}) done := make(chan error, 1) go func() { done <- s.withClaimSQL(t.Context(), []claimSQLScope{gateScope}, allowAllClaimSQL, func(*Store) error { close(held) <-release return nil }) }() <-held got := make(chan error, 1) go func() { _, err := s.Operators(); got <- err }() time.Sleep(50 * time.Millisecond) close(release) if err := <-done; err != nil { t.Fatal(err) } if err := <-got; err != nil { t.Fatalf("concurrent outer call refused: %v", err) } }) }