package console import ( "context" "encoding/json" "errors" "fmt" "net/http" "strings" "sync" "testing" "time" "github.com/carlosframework/platform/internal/objstore" "github.com/carlosframework/platform/internal/release" "github.com/carlosframework/platform/internal/svcrole" ) func storeProvisionURL(fx *consoleFixture, app string) string { return storeURL(fx, app) + "/provision" } func approveStoreAccount(t *testing.T, fx *consoleFixture, by string) { t.Helper() putEntitlement(t, fx, fx.sqid, &objstore.Entitlement{ Applied: &objstore.Applied{By: "paul@keymail.dev", At: time.Now().Add(-time.Minute)}, Decision: &objstore.Decision{Approved: true, By: by, At: time.Now()}, }) } type pausedBucketProvisioner struct { objstore.Provisioner entered chan struct{} release chan struct{} once sync.Once } func (p *pausedBucketProvisioner) EnsureBucket(ctx context.Context, name string) (string, error) { p.once.Do(func() { close(p.entered) select { case <-p.release: case <-ctx.Done(): } }) return p.Provisioner.EnsureBucket(ctx, name) } type provisionResult struct { status int out map[string]any } func provisionAsync(t *testing.T, fx *consoleFixture, token string) <-chan provisionResult { t.Helper() ch := make(chan provisionResult, 1) go func() { defer close(ch) status, raw := postRaw(t, storeProvisionURL(fx, "titogo"), map[string]any{}, token) out := map[string]any{} if err := json.Unmarshal([]byte(raw), &out); err != nil { out["raw"] = raw } ch <- provisionResult{status: status, out: out} }() return ch } func waitPaused(t *testing.T, p *pausedBucketProvisioner) { t.Helper() select { case <-p.entered: case <-time.After(3 * time.Second): t.Fatal("provision did not reach the paused backend") } } func TestAPIStoreProvisionNeedsAnApprovedEntitlement(t *testing.T) { fx, fake, token, _ := storeArmed(t, "code-store-provision-unapproved") declareStore(t, fx, token) status, _ := postJSON(t, storeProvisionURL(fx, "titogo"), map[string]any{}, token) if status < 400 { t.Fatalf("provision without approval = %d, want a refusal", status) } untouched(t, fake, "an unapproved provision") if rec := storedRecord(t, fx, "titogo"); rec.Grant != nil { t.Fatalf("an unapproved provision wrote a grant: %+v", rec.Grant) } } func TestAPIStoreProvisionWithoutABackendDoesNotGrant(t *testing.T) { fx, fake, token, _ := storeArmed(t, "code-store-provision-nobackend") declareStore(t, fx, token) approveStoreAccount(t, fx, "operator@keymail.dev") fx.srv.cfg.StoreProvisioner = nil status, _ := postJSON(t, storeProvisionURL(fx, "titogo"), map[string]any{}, token) if status != http.StatusServiceUnavailable { t.Fatalf("provision without a backend = %d, want 503", status) } untouched(t, fake, "a backend-less provision") if rec := storedRecord(t, fx, "titogo"); rec.Grant != nil { t.Fatalf("a backend-less provision wrote a grant: %+v", rec.Grant) } } func TestAPIStoreProvisionAttributesAccountApproval(t *testing.T) { fx, fake, token, priv := storeArmed(t, "code-store-provision-happy") declareStore(t, fx, token) approveStoreAccount(t, fx, "operator@keymail.dev") fx.srv.cfg.StoreOperators = nil status, out := postJSON(t, storeProvisionURL(fx, "titogo"), map[string]any{}, token) if status != http.StatusOK { t.Fatalf("provision = %d %v, want 200", status, out) } rec := storedRecord(t, fx, "titogo") if rec.Provisioned == nil || rec.Grant == nil { t.Fatalf("provisioned record = %+v", rec) } if rec.Grant.Source != objstore.GrantSourceAccountApproval || rec.Grant.By != "operator@keymail.dev" { t.Fatalf("grant = %+v, want the account approval source and deciding operator", rec.Grant) } u := fake.Users[rec.Provisioned.IAMUser] secret, ok := u.Keys[rec.Provisioned.KeyID] if !ok { t.Fatalf("recorded key %q is absent from backend keys %v", rec.Provisioned.KeyID, u.Keys) } if got := openSecret(t, priv, appVars(t, fx, "titogo")["CARLOS_STORE_SECRET_KEY"]); got != secret { t.Fatalf("delivered secret does not match the recorded backend key") } if entitlementBlock(t, out)["decision"] == nil { t.Fatalf("provision reply omitted the entitlement: %v", out) } } func TestAPIStoreProvisionCompletedStoreIsANoOp(t *testing.T) { fx, fake, token, _ := storeArmed(t, "code-store-provision-noop") declareStore(t, fx, token) approveStoreAccount(t, fx, "operator@keymail.dev") if status, out := postJSON(t, storeProvisionURL(fx, "titogo"), map[string]any{}, token); status != http.StatusOK { t.Fatalf("first provision = %d %v", status, out) } before := storedRecord(t, fx, "titogo").Provisioned.KeyID if status, out := postJSON(t, storeProvisionURL(fx, "titogo"), map[string]any{}, token); status != http.StatusOK { t.Fatalf("second provision = %d %v", status, out) } after := storedRecord(t, fx, "titogo").Provisioned.KeyID if after != before || len(fake.Users["carlos-store-bab-titogo"].Keys) != 1 { t.Fatalf("no-op changed key %q to %q or minted extras: %v", before, after, fake.Users) } } func TestAPIStoreProvisionRetriesAFailedGrantWithoutChangingBucket(t *testing.T) { fx, fake, token, _ := storeArmed(t, "code-store-provision-retry") declareStore(t, fx, token) approveStoreAccount(t, fx, "operator@keymail.dev") fake.FailNext["EnsureBucket"] = errors.New("backend detail must stay private") status, out := postJSON(t, storeProvisionURL(fx, "titogo"), map[string]any{}, token) if status != http.StatusBadGateway { t.Fatalf("failed provision = %d %v, want 502", status, out) } if strings.Contains(str(t, out, "error"), "backend detail") { t.Fatalf("provision response leaked the backend failure: %v", out) } failed := storedRecord(t, fx, "titogo") if failed.Grant == nil || failed.Provisioned != nil { t.Fatalf("failed provision record = %+v, want grant-only", failed) } bucket := failed.Grant.Bucket if status, out := postJSON(t, storeProvisionURL(fx, "titogo"), map[string]any{}, token); status != http.StatusOK { t.Fatalf("retry provision = %d %v, want 200", status, out) } got := storedRecord(t, fx, "titogo") if got.Provisioned == nil || got.Provisioned.Bucket != bucket || got.Grant.Bucket != bucket { t.Fatalf("retry changed or failed to provision bucket %q: %+v", bucket, got) } } func TestAPIStoreProvisionSerializesAndRereadsEntitlement(t *testing.T) { fx, fake, token, _ := storeArmed(t, "code-store-provision-revoke") declareStore(t, fx, token) approveStoreAccount(t, fx, "operator@keymail.dev") paused := &pausedBucketProvisioner{Provisioner: fake, entered: make(chan struct{}), release: make(chan struct{})} fx.srv.cfg.StoreProvisioner = paused defer func() { select { case <-paused.release: default: close(paused.release) } }() first := provisionAsync(t, fx, token) waitPaused(t, paused) second := provisionAsync(t, fx, token) waitStoreProvisionQueued(t, fx) e, etag, err := objstore.GetEntitlement(context.Background(), fx.rel, fx.sqid) if err != nil { t.Fatal(err) } e.Decision = &objstore.Decision{Approved: false, By: "operator@keymail.dev", At: time.Now(), Reason: "capacity review"} if _, err := objstore.PutEntitlement(context.Background(), fx.rel, fx.sqid, e, etag); err != nil { t.Fatal(err) } close(paused.release) if got := <-first; got.status != http.StatusOK { t.Fatalf("first provision = %d %v, want 200", got.status, got.out) } if got := <-second; got.status < 400 { t.Fatalf("queued provision after revocation = %d %v, want a refusal", got.status, got.out) } status, out := getJSON(t, storeURL(fx, "titogo"), token) if status != http.StatusOK || block(t, entitlementBlock(t, out), "decision")["approved"] != false { t.Fatalf("declined status = %d %v", status, out) } rec := storedRecord(t, fx, "titogo") if rec.Provisioned == nil || len(fake.Users[rec.Provisioned.IAMUser].Keys) != 1 { t.Fatalf("serialized provisions left the wrong estate: record=%+v users=%v", rec, fake.Users) } } func TestAPIStoreDeclareSamePrefixDuringProvisionIsNoOp(t *testing.T) { fx, fake, token, _ := storeArmed(t, "code-store-provision-declare-race") declareStore(t, fx, token) approveStoreAccount(t, fx, "operator@keymail.dev") paused := &pausedBucketProvisioner{Provisioner: fake, entered: make(chan struct{}), release: make(chan struct{})} fx.srv.cfg.StoreProvisioner = paused defer func() { select { case <-paused.release: default: close(paused.release) } }() result := provisionAsync(t, fx, token) waitPaused(t, paused) before, beforeETag, err := objstore.Get(context.Background(), fx.rel, release.OpsAccount, "titogo") if err != nil { t.Fatal(err) } status, out := postJSON(t, storeURL(fx, "titogo"), map[string]any{}, token) if status != http.StatusOK { t.Fatalf("same-prefix declare = %d %v, want 200", status, out) } after, afterETag, err := objstore.Get(context.Background(), fx.rel, release.OpsAccount, "titogo") if err != nil { t.Fatal(err) } if afterETag != beforeETag || after.Declared.By != before.Declared.By || !after.Declared.At.Equal(before.Declared.At) { t.Fatalf("same-prefix declare changed record: etag %q -> %q, declared %+v -> %+v", beforeETag, afterETag, before.Declared, after.Declared) } close(paused.release) if got := <-result; got.status != http.StatusOK { t.Fatalf("provision after no-op declare = %d %v, want 200", got.status, got.out) } } func TestAPIStoreAdminServiceDeclarationKeepsServiceAttribution(t *testing.T) { fx, _, _, _ := storeArmed(t, "code-store-provision-service") admin := serviceToken(t, fx.st, OpsAccountID, "storebot", "", svcrole.Admin) status, out := postJSON(t, storeURL(fx, "titogo"), map[string]any{}, admin) if status != http.StatusOK { t.Fatalf("admin service declare = %d %v, want 200", status, out) } if got := storedRecord(t, fx, "titogo").Declared.By; got != "service:bab/storebot" { t.Fatalf("declared.by = %q, want service attribution", got) } } func waitStoreProvisionQueued(t *testing.T, fx *consoleFixture) { t.Helper() deadline := time.Now().Add(3 * time.Second) for { fx.srv.storeWalks.mu.Lock() e := fx.srv.storeWalks.m[storeWalkKey(fx.sqid, "titogo", "")] refs := 0 if e != nil { refs = e.refs } fx.srv.storeWalks.mu.Unlock() if refs == 2 { return } if time.Now().After(deadline) { t.Fatalf("second provision never queued: refs=%d", refs) } time.Sleep(time.Millisecond) } } func TestAPIStoreConcurrentProvisionsMintOneDeliveredKey(t *testing.T) { fx, fake, token, priv := storeArmed(t, "code-store-provision-two") declareStore(t, fx, token) approveStoreAccount(t, fx, "operator@keymail.dev") paused := &pausedBucketProvisioner{Provisioner: fake, entered: make(chan struct{}), release: make(chan struct{})} fx.srv.cfg.StoreProvisioner = paused defer func() { select { case <-paused.release: default: close(paused.release) } }() first := provisionAsync(t, fx, token) waitPaused(t, paused) second := provisionAsync(t, fx, token) waitStoreProvisionQueued(t, fx) close(paused.release) for _, result := range []<-chan provisionResult{first, second} { select { case got := <-result: if got.status != http.StatusOK { t.Fatalf("provision = %d %v", got.status, got.out) } case <-time.After(3 * time.Second): t.Fatal("provision did not finish") } } rec := storedRecord(t, fx, "titogo") if rec.Provisioned == nil { t.Fatal("store was not provisioned") } keys := fake.Users[rec.Provisioned.IAMUser].Keys if rec.Provisioned.KeyID != "MEMKEY1" || len(keys) != 1 { t.Fatalf("wanted exactly one mint: record=%+v keys=%v", rec.Provisioned, keys) } vars := appVars(t, fx, "titogo") if openSecret(t, priv, vars["CARLOS_STORE_ACCESS_KEY"]) != rec.Provisioned.KeyID { t.Fatal("delivered key differs from record") } if openSecret(t, priv, vars["CARLOS_STORE_SECRET_KEY"]) != keys[rec.Provisioned.KeyID] { t.Fatal("delivered secret differs from backend") } } func TestAPIStoreDeclareReportsEntitlementWithoutProvisioning(t *testing.T) { for _, approved := range []bool{false, true} { for _, backend := range []bool{false, true} { t.Run(fmt.Sprintf("approved=%t/backend=%t", approved, backend), func(t *testing.T) { fx, fake, token, _ := storeArmed(t, "code-store-declare-entitlement") if approved { approveStoreAccount(t, fx, "operator@keymail.dev") } if !backend { fx.srv.cfg.StoreProvisioner = nil fx.srv.cfg.StoreDeployment = objstore.Deployment{} } status, out := postJSON(t, storeURL(fx, "titogo"), map[string]any{}, token) if status != http.StatusOK { t.Fatalf("declare = %d %v", status, out) } if _, present := out["entitlement"]; !present { t.Fatal("reply omitted entitlement") } if approved { if !block(t, entitlementBlock(t, out), "decision")["approved"].(bool) { t.Fatal("approval missing") } } else if out["entitlement"] != nil { t.Fatalf("unexpected entitlement: %v", out) } if (out["backend"] != "") != backend { t.Fatalf("backend mismatch: %v", out) } if out["grant"] != nil || out["provisioned"] != nil { t.Fatalf("declare minted estate: %v", out) } untouched(t, fake, "declare") }) } } } func TestAPIStoreProvisionMemberAndServiceGates(t *testing.T) { fx, fake, token, _ := storeArmed(t, "code-store-provision-gates") declareStore(t, fx, token) approveStoreAccount(t, fx, "operator@keymail.dev") stranger := cliToken(t, fx, "stranger@keymail.dev", "code-store-provision-stranger") status, body := postRaw(t, storeProvisionURL(fx, "titogo"), map[string]any{}, stranger) if status != http.StatusNotFound || body != "404 page not found\n" { t.Fatalf("stranger = %d %q", status, body) } for _, role := range []svcrole.Role{svcrole.Publish, svcrole.Instance, svcrole.Operate} { svc := serviceToken(t, fx.st, OpsAccountID, "denied-"+string(role), "", role) status, _ := postRaw(t, storeProvisionURL(fx, "titogo"), map[string]any{}, svc) if status != http.StatusForbidden { t.Fatalf("service %s = %d", role, status) } } untouched(t, fake, "refused provision") admin := serviceToken(t, fx.st, OpsAccountID, "store-admin", "titogo", svcrole.Admin) status, out := postJSON(t, storeProvisionURL(fx, "titogo"), map[string]any{}, admin) if status != http.StatusOK { t.Fatalf("admin provision = %d %v", status, out) } if storedRecord(t, fx, "titogo").Grant.By != "operator@keymail.dev" { t.Fatal("service replaced approving actor") } auditedBy(t, fx, "titogo", "store-provision", "service:bab/store-admin", "provisioned") } func TestAPIStoreProvisionRequiresDeclarationAndSealKey(t *testing.T) { fx, fake, token, _ := storeArmed(t, "code-store-provision-preconditions") approveStoreAccount(t, fx, "operator@keymail.dev") status, out := postJSON(t, storeProvisionURL(fx, "titogo"), map[string]any{}, token) if status != http.StatusConflict { t.Fatalf("undeclared = %d %v", status, out) } declareStore(t, fx, token) fx.srv.cfg.SecretsPub = "" status, out = postJSON(t, storeProvisionURL(fx, "titogo"), map[string]any{}, token) if status != http.StatusServiceUnavailable { t.Fatalf("unsealable = %d %v", status, out) } untouched(t, fake, "unsealable provision") } func TestAPIStoreProvisionCannotAdoptABucket(t *testing.T) { fx, fake, token, _ := storeArmed(t, "code-store-provision-no-adoption") declareStore(t, fx, token) approveStoreAccount(t, fx, "operator@keymail.dev") status, out := postJSON(t, storeProvisionURL(fx, "titogo"), map[string]any{"bucket": "someone-elses-data"}, token) if status != http.StatusOK && status != http.StatusBadRequest { t.Fatalf("provision with adoption input = %d %v", status, out) } if _, exists := fake.Buckets["someone-elses-data"]; exists { t.Fatal("member provision adopted caller bucket") } rec := storedRecord(t, fx, "titogo") if rec.Grant != nil && rec.Grant.Bucket != objstore.MintedBucket(fx.srv.cfg.StoreDeployment, fx.sqid, "titogo") { t.Fatalf("grant chose caller bucket: %+v", rec.Grant) } } func TestAPIStoreProvisionFailsClosedOnUnreadableApproval(t *testing.T) { fx, fake, token, _ := storeArmed(t, "code-store-provision-unreadable") declareStore(t, fx, token) if _, err := fx.rel.Put(context.Background(), objstore.EntitlementKey(fx.sqid), []byte("private backend details"), ""); err != nil { t.Fatal(err) } status, out := postJSON(t, storeProvisionURL(fx, "titogo"), map[string]any{}, token) if status != http.StatusBadGateway { t.Fatalf("unreadable approval = %d %v", status, out) } if strings.Contains(str(t, out, "error"), "private") { t.Fatal("approval error leaked backend details") } untouched(t, fake, "unreadable approval") }