package console import ( "bytes" "context" "errors" "net/http" "net/http/httptest" "os" "testing" "time" "github.com/carlosframework/platform/internal/objstore" "github.com/carlosframework/platform/internal/release" ) func TestSeedOwnsStore(t *testing.T) { id := "sd_aaaaaaaaaaaaaaaaaaaaaaaaaa" for _, c := range []struct { rec *objstore.Record want bool }{ {nil, true}, // no store: this seed may make one {&objstore.Record{Declared: &objstore.Declared{}}, true}, // declared only: no estate yet {&objstore.Record{Declared: &objstore.Declared{Seed: id}, Grant: &objstore.GrantBlock{}}, true}, {&objstore.Record{Declared: &objstore.Declared{Seed: "sd_bbbbbbbbbbbbbbbbbbbbbbbbbb"}, Grant: &objstore.GrantBlock{}}, false}, {&objstore.Record{Declared: &objstore.Declared{}, Provisioned: &objstore.Provisioned{}}, false}, {&objstore.Record{Grant: &objstore.GrantBlock{}}, false}, // estate nobody declared: not this seed's } { if got := seedOwnsStore(c.rec, id); got != c.want { t.Errorf("seedOwnsStore(%+v) = %v, want %v", c.rec, got, c.want) } } } const ( seedA = "sd_aaaaaaaaaaaaaaaaaaaaaaaaaa" seedB = "sd_bbbbbbbbbbbbbbbbbbbbbbbbbb" ) // seededStoreFixture is privateStorageFixture with a neutral app, shop, // whose canary environment the seeded tests provision. func seededStoreFixture(t *testing.T, code string) (*consoleFixture, *objstore.MemProvisioner, *Account) { t.Helper() fx, fake, _, _ := privateStorageFixture(t, code) if _, err := fx.st.CreateAppIn(OpsAccountID, "shop", "paul@keymail.dev", time.Now()); err != nil { t.Fatal(err) } acct, err := fx.st.AccountByID(OpsAccountID) if err != nil { t.Fatal(err) } return fx, fake, acct } func putCanaryStore(t *testing.T, fx *consoleFixture, rec *objstore.Record) { t.Helper() if _, err := objstore.PutIn(context.Background(), fx.rel, release.OpsAccount, "shop", "canary", rec, ""); err != nil { t.Fatal(err) } } func canaryStore(t *testing.T, fx *consoleFixture) *objstore.Record { t.Helper() rec, _, err := objstore.GetIn(context.Background(), fx.rel, release.OpsAccount, "shop", "canary") if err != nil { t.Fatal(err) } return rec } // A store that already holds estate and is not this seed's is refused // under the store lock, before any declaration or grant: a concurrent // `carlos store create --environment canary`, or another seed, must never // hand a seeded instance a store that is not new and empty. Nothing is // answered on w; the caller speaks the refusal. func TestInstanceStoreProvisionForRefusesAStoreThatIsNotThisSeeds(t *testing.T) { at := time.Now().UTC().Add(-time.Hour) for name, planted := range map[string]*objstore.Record{ "another seed's grant": { Declared: &objstore.Declared{EnvPrefix: "CARLOS_STORE", At: at, By: "alice@keymail.dev", Seed: seedB}, Grant: &objstore.GrantBlock{Backend: "mem", Bucket: "carlos-bab-shop-canary", Source: objstore.GrantSourceAccountApproval, At: at, By: "operator@keymail.dev"}, }, "an ordinary store's grant": { Declared: &objstore.Declared{EnvPrefix: "CARLOS_STORE", At: at, By: "alice@keymail.dev"}, Grant: &objstore.GrantBlock{Backend: "mem", Bucket: "carlos-bab-shop-canary", Source: objstore.GrantSourceAccountApproval, At: at, By: "operator@keymail.dev"}, }, "an ordinary provisioned store": { Declared: &objstore.Declared{EnvPrefix: "CARLOS_STORE", At: at, By: "alice@keymail.dev"}, Provisioned: &objstore.Provisioned{Bucket: "carlos-bab-shop-canary", Region: "eu-west-1", IAMUser: "carlos-store-bab-shop-canary", At: at}, }, } { t.Run(name, func(t *testing.T) { fx, fake, acct := seededStoreFixture(t, "code-seed-taken") putCanaryStore(t, fx, planted) w := httptest.NewRecorder() got, ok, err := fx.srv.instanceStoreProvisionFor(context.Background(), w, acct, "shop", "canary", "alice@keymail.dev", seedA) if ok || !errors.Is(err, errSeedStoreTaken) { t.Fatalf("provision = %v, %v; want errSeedStoreTaken", ok, err) } if got == nil || storeBucketName(got) != "carlos-bab-shop-canary" { t.Fatalf("returned record = %+v, want the store that is there, for the caller to name", got) } if w.Body.Len() != 0 { t.Fatalf("answered %d %q, want nothing (the caller speaks this refusal)", w.Code, w.Body.String()) } if len(fake.Buckets) != 0 || len(fake.Users) != 0 { t.Fatalf("backend buckets %v, users %v; want no grant walk", fake.Buckets, fake.Users) } after := canaryStore(t, fx) if after.Declared == nil || after.Declared.Seed != planted.Declared.Seed || !after.Declared.At.Equal(at) { t.Fatalf("declaration = %+v, want the planted one untouched", after.Declared) } }) } } // With no store, the seed declares one marked with its id and provisions // it; a retry of the same seed then resumes that store. func TestInstanceStoreProvisionForMarksANewStoreWithTheSeed(t *testing.T) { fx, fake, acct := seededStoreFixture(t, "code-seed-new") for range 2 { w := httptest.NewRecorder() out, ok, err := fx.srv.instanceStoreProvisionFor(context.Background(), w, acct, "shop", "canary", "alice@keymail.dev", seedA) if !ok || err != nil { t.Fatalf("provision = %v, %v (%d %s)", ok, err, w.Code, w.Body.String()) } if out.Provisioned == nil || out.Declared == nil || out.Declared.Seed != seedA { t.Fatalf("store = %+v, want a provisioned store declared by %s", out, seedA) } } if _, ok := fake.Buckets["carlos-bab-shop-canary"]; !ok || len(fake.Buckets) != 1 { t.Fatalf("backend buckets = %v, want the one canary bucket", fake.Buckets) } } // A declared-only record holds no estate, so a seed takes it over: the // declaration is rewritten for this seed (keeping no other seed's mark) // and the store is provisioned. func TestInstanceStoreProvisionForRedeclaresADeclaredOnlyRecord(t *testing.T) { for name, decl := range map[string]*objstore.Declared{ "another seed's": {EnvPrefix: "CARLOS_STORE", At: time.Now().UTC().Add(-time.Hour), By: "alice@keymail.dev", Seed: seedB}, "an ordinary": {EnvPrefix: "CARLOS_STORE", At: time.Now().UTC().Add(-time.Hour), By: "alice@keymail.dev"}, } { t.Run(name, func(t *testing.T) { fx, _, acct := seededStoreFixture(t, "code-seed-redeclare") putCanaryStore(t, fx, &objstore.Record{Declared: decl}) w := httptest.NewRecorder() out, ok, err := fx.srv.instanceStoreProvisionFor(context.Background(), w, acct, "shop", "canary", "bob@keymail.dev", seedA) if !ok || err != nil { t.Fatalf("provision = %v, %v (%d %s)", ok, err, w.Code, w.Body.String()) } if out.Declared == nil || out.Declared.Seed != seedA || out.Declared.By != "bob@keymail.dev" || out.Provisioned == nil { t.Fatalf("store = %+v / %+v, want provisioned and declared by %s", out, out.Declared, seedA) } }) } } // takeCanaryBucketName plants another store of shop that names canary's // bucket, so canary's first grant fails with ErrStoreNameTaken before it // writes a grant block. func takeCanaryBucketName(t *testing.T, fx *consoleFixture) { t.Helper() at := time.Now().UTC() rec := &objstore.Record{ Declared: &objstore.Declared{EnvPrefix: "CARLOS_STORE", At: at, By: "alice@keymail.dev"}, Grant: &objstore.GrantBlock{Backend: "mem", Bucket: "carlos-bab-shop-canary", Source: objstore.GrantSourceAccountApproval, At: at, By: "operator@keymail.dev"}, } if _, err := objstore.PutIn(context.Background(), fx.rel, release.OpsAccount, "shop", "staging", rec, ""); err != nil { t.Fatal(err) } } func canaryRaw(t *testing.T, fx *consoleFixture) []byte { t.Helper() raw, _, err := fx.rel.Get(context.Background(), objstore.KeyIn(release.OpsAccount, "shop", "canary")) if errors.Is(err, os.ErrNotExist) { return nil } if err != nil { t.Fatal(err) } return raw } // A seed whose grant fails before writing its block leaves no record // marked with its id. Another path could otherwise grant that record and // keep the mark, and the seed's finish would tear down a store it never // made. Here the seed created the record, so the record is gone. func TestInstanceStoreProvisionForRollsBackAFreshDeclaration(t *testing.T) { fx, fake, acct := seededStoreFixture(t, "code-seed-rollback-fresh") takeCanaryBucketName(t, fx) w := httptest.NewRecorder() _, ok, err := fx.srv.instanceStoreProvisionFor(context.Background(), w, acct, "shop", "canary", "alice@keymail.dev", seedA) if ok || err != nil || w.Code != http.StatusConflict { t.Fatalf("provision = %v, %v (%d %s); want the name-taken refusal answered", ok, err, w.Code, w.Body.String()) } if raw := canaryRaw(t, fx); raw != nil { t.Fatalf("canary record = %s, want none: the seed's declaration must not outlive its failed grant", raw) } if len(fake.Buckets) != 0 { t.Fatalf("backend buckets = %v, want none", fake.Buckets) } } // Here the seed re-declared a record that was there before it, so the // record goes back to exactly the bytes it had. func TestInstanceStoreProvisionForRestoresARedeclaredRecord(t *testing.T) { for name, decl := range map[string]*objstore.Declared{ "another seed's": {EnvPrefix: "SHOP_FILES", At: time.Now().UTC().Add(-time.Hour), By: "alice@keymail.dev", Seed: seedB}, "an ordinary": {EnvPrefix: "SHOP_FILES", At: time.Now().UTC().Add(-time.Hour), By: "alice@keymail.dev"}, } { t.Run(name, func(t *testing.T) { fx, _, acct := seededStoreFixture(t, "code-seed-rollback-redeclare") putCanaryStore(t, fx, &objstore.Record{Declared: decl}) takeCanaryBucketName(t, fx) before := canaryRaw(t, fx) w := httptest.NewRecorder() _, ok, err := fx.srv.instanceStoreProvisionFor(context.Background(), w, acct, "shop", "canary", "bob@keymail.dev", seedA) if ok || err != nil || w.Code != http.StatusConflict { t.Fatalf("provision = %v, %v (%d %s); want the name-taken refusal answered", ok, err, w.Code, w.Body.String()) } if after := canaryRaw(t, fx); !bytes.Equal(after, before) { t.Fatalf("canary record after the failed seed:\n%s\nwant byte-identical to before:\n%s", after, before) } }) } } // A grant that wrote its block before failing has made estate for this // seed: the record keeps the block and the mark, for the seed's finish. func TestInstanceStoreProvisionForKeepsAGrantItWrote(t *testing.T) { fx, fake, acct := seededStoreFixture(t, "code-seed-keep-grant") fake.FailNext = map[string]error{"EnsureUser": errors.New("iam is down")} w := httptest.NewRecorder() _, ok, err := fx.srv.instanceStoreProvisionFor(context.Background(), w, acct, "shop", "canary", "alice@keymail.dev", seedA) if ok || err != nil || w.Code != http.StatusBadGateway { t.Fatalf("provision = %v, %v (%d %s); want the walk failure answered", ok, err, w.Code, w.Body.String()) } rec := canaryStore(t, fx) if rec == nil || rec.Grant == nil || rec.Declared == nil || rec.Declared.Seed != seedA { t.Fatalf("canary record = %+v, want the grant block and the seed's mark kept", rec) } } // A seed that wrote its grant block and then aborted leaves a store marked // with its id. An ordinary create sharing that environment adopts the // store, and the mark goes in the same write, so the seed's finish cannot // tear down a store another host now uses. func TestInstanceStoreProvisionAdoptingASeedsStoreClearsTheMark(t *testing.T) { fx, fake, acct := seededStoreFixture(t, "code-seed-adopted") fake.FailNext = map[string]error{"EnsureUser": errors.New("iam is down")} w := httptest.NewRecorder() if _, ok, _ := fx.srv.instanceStoreProvisionFor(context.Background(), w, acct, "shop", "canary", "alice@keymail.dev", seedA); ok { t.Fatal("the seeded provision succeeded, want its walk to fail after the grant block") } if rec := canaryStore(t, fx); rec == nil || rec.Grant == nil || rec.Declared.Seed != seedA { t.Fatalf("canary record = %+v, want the seed's grant block and mark", rec) } w = httptest.NewRecorder() out, ok := fx.srv.instanceStoreProvision(context.Background(), w, acct, "shop", "canary", "bob@keymail.dev") if !ok { t.Fatalf("ordinary provision refused: %d %s", w.Code, w.Body.String()) } for _, rec := range []*objstore.Record{out, canaryStore(t, fx)} { if rec.Declared == nil || rec.Declared.Seed != "" || rec.Provisioned == nil { t.Fatalf("store after adoption = %+v / %+v, want provisioned with no seed mark", rec, rec.Declared) } } } // Without a seed nothing is rolled back: a failed grant leaves the // declaration standing, as it always has, for a retry to resume. func TestInstanceStoreProvisionLeavesItsDeclarationOnAFailedGrant(t *testing.T) { fx, _, acct := seededStoreFixture(t, "code-unseeded-failed-grant") takeCanaryBucketName(t, fx) w := httptest.NewRecorder() if _, ok := fx.srv.instanceStoreProvision(context.Background(), w, acct, "shop", "canary", "alice@keymail.dev"); ok || w.Code != http.StatusConflict { t.Fatalf("provision = %v (%d %s); want the name-taken refusal", ok, w.Code, w.Body.String()) } rec := canaryStore(t, fx) if rec == nil || rec.Declared == nil || rec.Declared.Seed != "" || rec.Declared.By != "alice@keymail.dev" || rec.Grant != nil { t.Fatalf("canary record = %+v, want the unmarked declaration left standing", rec) } } // Without a seed, a declared-only record keeps its author and timestamp, // exactly as before: only a seeded create re-declares. A seed's mark on it // does not survive the grant: the store is this create's, not the seed's, // and the seed's finish must never tear it down. func TestInstanceStoreProvisionKeepsAnExistingDeclaration(t *testing.T) { fx, _, acct := seededStoreFixture(t, "code-unseeded-keep") at := time.Now().UTC().Add(-time.Hour).Truncate(time.Second) putCanaryStore(t, fx, &objstore.Record{Declared: &objstore.Declared{EnvPrefix: "CARLOS_STORE", At: at, By: "alice@keymail.dev", Seed: seedB}}) w := httptest.NewRecorder() out, ok := fx.srv.instanceStoreProvision(context.Background(), w, acct, "shop", "canary", "bob@keymail.dev") if !ok { t.Fatalf("provision refused: %d %s", w.Code, w.Body.String()) } for _, rec := range []*objstore.Record{out, canaryStore(t, fx)} { if rec.Declared == nil || rec.Declared.By != "alice@keymail.dev" || !rec.Declared.At.Equal(at) || rec.Declared.Seed != "" || rec.Grant == nil { t.Fatalf("declaration = %+v, want the existing one with the seed's mark cleared by the grant", rec.Declared) } } }