package console import ( "context" "crypto/ecdsa" "crypto/elliptic" "crypto/rand" "encoding/json" "errors" "io" "net/http" "net/http/httptest" "os" "strings" "testing" "time" "github.com/carlosframework/platform/internal/dbdoor" "github.com/carlosframework/platform/internal/hostclaim" "github.com/carlosframework/platform/internal/lease" "github.com/carlosframework/platform/internal/lease/leasetest" "github.com/carlosframework/platform/internal/provision" "github.com/carlosframework/platform/internal/release" ) type doorVersionHook struct { lease.VersionedStore after func() } func (v doorVersionHook) ListVersions(ctx context.Context, b, p string) ([]lease.ObjectVersion, error) { out, err := v.VersionedStore.ListVersions(ctx, b, p) if v.after != nil { v.after() } return out, err } type doorReadFault struct { release.Store err error } func (s *doorReadFault) Get(ctx context.Context, key string) ([]byte, string, error) { if s.err != nil { return nil, "", s.err } return s.Store.Get(ctx, key) } func doorPlanFixture(t *testing.T, legacy bool) (*Server, dbRequest, provision.Record, *leasetest.VersionedBucket) { t.Helper() rec := provision.Record{Host: "shop.example", Account: "bab", App: "jam"} inc, err := hostclaim.Incarnation(rec.Account, rec.App, rec.Host, rec.CreatedAt) if err != nil { t.Fatal(err) } loc := "instances/" + rec.Host if !legacy { loc += "/" + inc } claim := hostclaim.Claim{V: 2, Host: rec.Host, Gen: 1, Holder: &hostclaim.Holder{Family: "instance", Key: provision.RecordKey(rec.Account, rec.App, rec.Host), Account: rec.Account, App: rec.App, Incarnation: inc, Location: loc}, Op: hostclaim.Op{Kind: hostclaim.KindServing}} store := release.NewDirStore(t.TempDir()) data, _ := json.Marshal(claim) key, _ := hostclaim.Key(rec.Host) if _, err := store.Put(t.Context(), key, data, ""); err != nil { t.Fatal(err) } model := leasetest.NewVersionedBucket() model.AddBucket("data") s := &Server{cfg: Config{HomeRegion: "ie", HostClaim: true, Releases: store, HibernateBucket: "data", DBVersions: model}, dbHandleKey: []byte(strings.Repeat("s", 32))} g := dbRequest{acct: &Account{Sqid: "bab"}, app: "jam", host: rec.Host} g.claim, err = s.dbClaim(t.Context(), g, rec) if err != nil { t.Fatal(err) } return s, g, rec, model } func writeDoorClaim(t *testing.T, s *Server, g dbRequest, c *hostclaim.Claim) { t.Helper() key, _ := hostclaim.Key(g.host) _, etag, err := s.cfg.Releases.Get(t.Context(), key) if err != nil { t.Fatal(err) } b, err := json.Marshal(c) if err != nil { t.Fatal(err) } if _, err = s.cfg.Releases.Put(t.Context(), key, b, etag); err != nil { t.Fatal(err) } } func cloneDoorClaim(c *hostclaim.Claim) *hostclaim.Claim { b, _ := json.Marshal(c) var out hostclaim.Claim _ = json.Unmarshal(b, &out) return &out } func TestDoorPlanHandles(t *testing.T) { key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) if err != nil { t.Fatal(err) } secret := dbHandleSecret(key) h := dbFileHandle{V: 1, Host: "shop.example", Location: "instances/shop.example/inc", Key: "instances/shop.example/inc/0000/a", VersionID: "v+/?", Incarnation: "inc", Expiry: 200, Account: "bab", App: "jam", Size: 6} token, err := signDBHandle(secret, h) if err != nil { t.Fatal(err) } got, err := verifyDBHandle(dbHandleSecret(key), token, time.Unix(199, 0)) if err != nil || got != h { t.Fatal(got, err) } for _, bad := range []string{"", token + "x", "x" + token, strings.Repeat("x", 8193)} { if _, err := verifyDBHandle(secret, bad, time.Unix(199, 0)); !errors.Is(err, errDBHandle) { t.Fatal(bad, err) } } if _, err := verifyDBHandle([]byte(strings.Repeat("x", 32)), token, time.Unix(199, 0)); !errors.Is(err, errDBHandle) { t.Fatal(err) } if _, err := verifyDBHandle(secret, token, time.Unix(200, 0)); !errors.Is(err, errDBHandleExpired) { t.Fatal(err) } } func TestDoorPlanListingAndPinnedOpen(t *testing.T) { for _, legacy := range []bool{false, true} { t.Run(map[bool]string{true: "legacy", false: "scoped"}[legacy], func(t *testing.T) { s, g, _, m := doorPlanFixture(t, legacy) loc := g.claim.Claim.Holder.Location key := dbVersionKey(loc, 0, 1, 1) m.Put("data", key, []byte("older")) id := m.Put("data", key, append(doorLTXHeader(t, 1, 1), []byte("original")...)) deleted := dbVersionKey(loc, 0, 2, 2) m.Put("data", deleted, []byte("hidden")) m.BareDelete("data", deleted) m.Put("data", loc+"/lock.json", []byte("secret")) m.Put("data", dbVersionKey(loc+"/other-incarnation", 0, 3, 3), []byte("other")) m.Put("data", dbVersionKey(loc+"-sibling", 0, 4, 4), []byte("sibling")) m.Put("data", dbVersionKey(loc+"/g-old", 0, 5, 5), []byte("generation")) m.PageSize = 1 out, err := s.listDBVersions(t.Context(), g, 0) if err != nil || out.Protocol != 2 || len(out.Files) != 1 { t.Fatalf("%+v %v", out, err) } token := out.Files[0].Handle h, err := verifyDBHandle(s.dbHandleKey, token, time.Now()) if err != nil || h.VersionID != id || h.Location != loc { t.Fatal(h, err) } req := httptest.NewRequest(http.MethodGet, "/", nil) req.Header.Set(dbdoor.HandleHeader, token) rd, err := s.pinnedDBReader(req, g, 0, 1, 1) if err != nil { t.Fatal(err) } // Stall after admission, then overwrite the identical legacy key and // advance the holder to a new incarnation before the physical open. m.Put("data", key, []byte("replacement")) next := cloneDoorClaim(g.claim.Claim) next.Gen++ next.Holder.CreatedAt = time.Unix(1, 0) next.Holder.Incarnation, _ = hostclaim.Incarnation("bab", "jam", g.host, next.Holder.CreatedAt) next.Holder.Location = "instances/" + g.host + "/" + next.Holder.Incarnation writeDoorClaim(t, s, g, next) rc, err := rd.OpenLTXFile(t.Context(), 0, 1, 1, 102, 3) if err != nil { t.Fatal(err) } body, err := io.ReadAll(rc) rc.Close() if err != nil || string(body) != "igi" { t.Fatalf("%q %v", body, err) } g2 := g g2.claim.Claim = next if _, err := s.pinnedDBReader(req, g2, 0, 1, 1); err == nil { t.Fatal("new incarnation accepted old handle") } bad := g bad.host = "other.example" if _, err := s.pinnedDBReader(req, bad, 0, 1, 1); err == nil { t.Fatal("cross host") } if _, err := s.pinnedDBReader(req, g, 0, 2, 2); err == nil { t.Fatal("cross coordinate") } if err := m.DeleteVersion(t.Context(), "data", key, id); err != nil { t.Fatal(err) } if _, err := rd.OpenLTXFile(t.Context(), 0, 1, 1, 0, 0); !errors.Is(err, os.ErrNotExist) { t.Fatal(err) } }) } } func TestDoorPlanBracket(t *testing.T) { for _, change := range []string{"holder", "op", "same-gen-op", "chore", "absent", "malformed", "read-error"} { t.Run(change, func(t *testing.T) { s, g, _, m := doorPlanFixture(t, false) m.Put("data", dbVersionKey(g.claim.Claim.Holder.Location, 0, 1, 1), doorLTXHeader(t, 1, 1)) fault := &doorReadFault{Store: s.cfg.Releases} s.cfg.Releases = fault s.cfg.DBVersions = doorVersionHook{VersionedStore: m, after: func() { next := cloneDoorClaim(g.claim.Claim) next.Gen++ switch change { case "holder": next.Holder.Stamp = next.Gen case "op": next.Op = hostclaim.Op{Kind: hostclaim.KindDeleting, ID: "delete"} case "same-gen-op": next.Gen-- next.Op.Exec = "another-box" case "chore": next.Touched.Complete = true case "absent": fault.err = os.ErrNotExist return case "read-error": fault.err = errors.New("offline") return case "malformed": next.V = 999 } writeDoorClaim(t, s, g, next) }} out, err := s.listDBVersions(t.Context(), g, 0) if change == "chore" { if err != nil || len(out.Files) != 1 { t.Fatal(out, err) } } else if err == nil || len(out.Files) != 0 { t.Fatalf("partial/success: %+v %v", out, err) } }) } } func TestDoorPlanClaimRefusals(t *testing.T) { for _, change := range []string{"absent", "free", "deleting", "parking", "purging", "owner", "incarnation", "location", "malformed", "read-error"} { t.Run(change, func(t *testing.T) { s, g, rec, m := doorPlanFixture(t, true) m.FailOn = func(op, key string) error { t.Fatalf("refusal reached data bucket: %s %s", op, key); return nil } c := cloneDoorClaim(g.claim.Claim) want := errDBClaimClosed switch change { case "absent": s.cfg.Releases = release.NewDirStore(t.TempDir()) want = errDBNotAdopted case "free": c.Holder = nil c.Op = hostclaim.Op{} case "deleting", "parking", "purging": c.Op = hostclaim.Op{Kind: change, ID: "x"} case "owner": g.app = "other" want = os.ErrNotExist case "incarnation": rec.CreatedAt = time.Unix(1, 0) want = os.ErrNotExist case "location": c.Holder.Location = "" want = hostclaim.ErrMalformed case "malformed": c.V = 99 want = hostclaim.ErrMalformed case "read-error": want = errors.New("offline") s.cfg.Releases = &doorReadFault{Store: s.cfg.Releases, err: want} } if change != "absent" && change != "read-error" { writeDoorClaim(t, s, dbRequest{host: rec.Host}, c) } if _, err := s.dbClaim(t.Context(), g, rec); !errors.Is(err, want) { t.Fatalf("%v want %v", err, want) } }) } } func TestDoorPlanListingFailureAndRanges(t *testing.T) { s, g, _, m := doorPlanFixture(t, true) for _, lo := range []uint64{1, 2, 3} { m.Put("data", dbVersionKey(g.claim.Claim.Holder.Location, 0, lo, lo), []byte("sixsix")) } m.PageSize = 1 m.FailListAfterPages = 1 if out, err := s.listDBVersions(t.Context(), g, 0); err == nil || len(out.Files) != 0 { t.Fatal(out, err) } for _, tc := range []struct { offset, size, want int64 ok bool }{{0, 0, 6, true}, {2, 0, 4, true}, {2, 2, 2, true}, {2, 100, 4, true}, {6, 0, 0, false}, {-1, 0, 0, false}, {0, -1, 0, false}} { n, err := dbVersionRange(6, tc.offset, tc.size) if (err == nil) != tc.ok || n != tc.want { t.Fatal(tc, n, err) } } w := httptest.NewRecorder() r := httptest.NewRequest("GET", "/", nil) r.Header.Set("Range", "bytes=2-3") r.Header.Set(dbdoor.HandleHeader, "signed") if err := dbVersionHeaders(w, r, 6, 2, 2); err != nil { t.Fatal(err) } if w.Header().Get("Content-Length") != "2" || w.Header().Get("Content-Range") != "bytes 2-3/6" || w.Header().Get(dbdoor.HandleHeader) != "signed" { t.Fatal(w.Header()) } } type doorOpenHook struct { lease.VersionedStore open func(context.Context, string, string, string) (io.ReadCloser, error) } func (v doorOpenHook) OpenVersion(ctx context.Context, b, k, id string) (io.ReadCloser, error) { return v.open(ctx, b, k, id) } type doorTrackedReader struct { io.Reader closed bool } func (r *doorTrackedReader) Close() error { r.closed = true; return nil } type doorCancelledReader struct{ ctx context.Context } func (r doorCancelledReader) Read([]byte) (int, error) { return 0, r.ctx.Err() } func TestDoorPlanRangeReaderCloses(t *testing.T) { for _, mode := range []string{"complete", "short-skip", "cancelled-skip"} { t.Run(mode, func(t *testing.T) { s, g, _, m := doorPlanFixture(t, true) ctx, cancel := context.WithCancel(t.Context()) defer cancel() tracked := &doorTrackedReader{Reader: strings.NewReader("abcdef")} if mode == "short-skip" { tracked.Reader = strings.NewReader("a") } if mode == "cancelled-skip" { cancel() tracked.Reader = doorCancelledReader{ctx: ctx} } key := dbVersionKey(g.claim.Claim.Holder.Location, 0, 1, 1) s.cfg.DBVersions = doorOpenHook{VersionedStore: m, open: func(ctx context.Context, b, k, id string) (io.ReadCloser, error) { if b != "data" || k != key || id != "selected" { t.Fatal(b, k, id) } return tracked, nil }} rd := &dbVersionReader{store: s.cfg.DBVersions, bucket: "data", handle: dbFileHandle{Location: g.claim.Claim.Holder.Location, Key: key, VersionID: "selected", Size: 6}} rc, err := rd.OpenLTXFile(ctx, 0, 1, 1, 2, 2) if mode != "complete" { if err == nil || !tracked.closed { t.Fatal(err, tracked.closed) } if mode == "cancelled-skip" && !errors.Is(err, context.Canceled) { t.Fatal(err) } return } if err != nil { t.Fatal(err) } b, err := io.ReadAll(rc) if err != nil || string(b) != "cd" { t.Fatal(string(b), err) } if tracked.closed { t.Fatal("closed before caller finished") } rc.Close() if !tracked.closed { t.Fatal("reader leaked") } }) } } // TestDBDoorRefusesDuringMoves (moves spec ยง19.1, Review Focus 4, K14): the // db door refuses on every move kind, in every phase, including // landing/committed and the restoring and reverting opened phases; it // reopens at the close, when the claim is serving again. The refusal is a // 409 with Retry-After: 1, and the data bucket is never read. func TestDBDoorRefusesDuringMoves(t *testing.T) { const move = "mv_abcdefghijklmnopqrstuvwxyz" reloc := func(phase, exec string) hostclaim.Op { return hostclaim.Op{Kind: hostclaim.KindRelocating, ID: move, Phase: phase, Exec: exec, Intent: map[string]string{"region": "uk", "source_box": "ie-box", "target_box": "uk-box"}} } cases := []hostclaim.Op{ {Kind: hostclaim.KindMoving, ID: move, Phase: hostclaim.PhaseMoving, Exec: "ie-box"}, {Kind: hostclaim.KindMoving, ID: move, Phase: hostclaim.PhaseSealed}, {Kind: hostclaim.KindMoving, ID: move, Phase: hostclaim.PhaseAborting, Exec: "ie-box"}, {Kind: hostclaim.KindLanding, ID: move, Phase: "checking", Exec: "uk-box"}, {Kind: hostclaim.KindLanding, ID: move, Phase: hostclaim.PhaseLandingProbing, Exec: "uk-box"}, {Kind: hostclaim.KindLanding, ID: move, Phase: hostclaim.PhaseLandingProbed, Exec: "uk-box"}, {Kind: hostclaim.KindLanding, ID: move, Phase: hostclaim.PhaseLandingCommitted, Exec: "uk-box"}, {Kind: hostclaim.KindReverting, ID: move, Exec: "ie-box", Journal: map[string]string{hostclaim.JournalRevPhase: hostclaim.RevPhaseEvidence}}, {Kind: hostclaim.KindReverting, ID: move, Exec: "ie-box", Journal: map[string]string{hostclaim.JournalRevPhase: hostclaim.RevPhaseVerify}}, {Kind: hostclaim.KindReverting, ID: move, Phase: hostclaim.PhaseProbing, Exec: "ie-box"}, {Kind: hostclaim.KindReverting, ID: move, Phase: hostclaim.PhaseOpened, Exec: "ie-box"}, {Kind: hostclaim.KindRestoring, ID: move, Phase: hostclaim.PhaseReleasing, Exec: "uk-box"}, {Kind: hostclaim.KindRestoring, ID: move, Phase: hostclaim.PhaseReleased}, {Kind: hostclaim.KindRestoring, ID: move, Phase: hostclaim.PhaseClaimed, Exec: "ie-box"}, {Kind: hostclaim.KindRestoring, ID: move, Phase: hostclaim.PhaseProbing, Exec: "ie-box"}, {Kind: hostclaim.KindRestoring, ID: move, Phase: hostclaim.PhaseProbed, Exec: "ie-box"}, {Kind: hostclaim.KindRestoring, ID: move, Phase: hostclaim.PhaseOpened, Exec: "ie-box"}, reloc("starting", "ie-box"), reloc("releasing", "none"), reloc("pinning", "none"), reloc("installing", "uk-box"), reloc("installed", "uk-box"), reloc("probing", "uk-box"), reloc("probe-failed", "uk-box"), reloc("serving", "uk-box"), } kinds := map[string]bool{} for _, op := range cases { kinds[op.Kind] = true name := op.Kind + "/" + op.Phase if rp := op.Journal[hostclaim.JournalRevPhase]; rp != "" { name += "rev_phase=" + rp } t.Run(name, func(t *testing.T) { s, g, rec, m := doorPlanFixture(t, false) m.FailOn = func(op, key string) error { t.Fatalf("a refusal reached the data bucket: %s %s", op, key); return nil } c := cloneDoorClaim(g.claim.Claim) c.Op = op writeDoorClaim(t, s, dbRequest{host: rec.Host}, c) _, err := s.dbClaim(t.Context(), g, rec) if !errors.Is(err, errDBClaimClosed) { t.Fatalf("%v, want the door closed", err) } w := httptest.NewRecorder() s.dbClaimFailure(w, httptest.NewRequest(http.MethodGet, "/", nil), g, err) if w.Code != http.StatusConflict || w.Header().Get("Retry-After") != "1" { t.Fatalf("%d Retry-After %q, want 409 and 1", w.Code, w.Header().Get("Retry-After")) } }) } for _, k := range []string{hostclaim.KindMoving, hostclaim.KindLanding, hostclaim.KindReverting, hostclaim.KindRestoring, hostclaim.KindRelocating} { if !kinds[k] { t.Errorf("the table has no %s row", k) } } // The close: serving again, the door reopens. t.Run("serving", func(t *testing.T) { s, g, rec, _ := doorPlanFixture(t, false) c := cloneDoorClaim(g.claim.Claim) c.Op = hostclaim.Op{Kind: hostclaim.KindServing} writeDoorClaim(t, s, dbRequest{host: rec.Host}, c) if _, err := s.dbClaim(t.Context(), g, rec); err != nil { t.Fatalf("serving: %v", err) } }) }