package lease import ( "context" "errors" "fmt" "net/http" "net/http/httptest" "sync" "testing" "time" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/credentials" awss3 "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/benbjohnson/litestream" lss3 "github.com/benbjohnson/litestream/s3" ) // deleteStub answers the lease's DELETE with each status in turn and // records the If-Match header it was sent. type deleteStub struct { mu sync.Mutex statuses []int ifMatch []string } func (s *deleteStub) ServeHTTP(w http.ResponseWriter, r *http.Request) { if r.Method != http.MethodDelete { http.Error(w, "unexpected "+r.Method, http.StatusMethodNotAllowed) return } s.mu.Lock() s.ifMatch = append(s.ifMatch, r.Header.Get("If-Match")) status := http.StatusNoContent if len(s.statuses) > 0 { status, s.statuses = s.statuses[0], s.statuses[1:] } s.mu.Unlock() switch status { case http.StatusNoContent: w.WriteHeader(status) case http.StatusPreconditionFailed: w.Header().Set("Content-Type", "application/xml") w.WriteHeader(status) _, _ = w.Write([]byte("PreconditionFailedno")) default: w.Header().Set("Content-Type", "application/xml") w.WriteHeader(status) _, _ = w.Write([]byte("AccessDeniedno")) } } func (s *deleteStub) sent() []string { s.mu.Lock() defer s.mu.Unlock() return append([]string(nil), s.ifMatch...) } // heldAgainst builds a Held whose lease record has ETag etag, with its // leaser pointed at srv. The TTL is long, so the renewer never renews // inside the test. func heldAgainst(t *testing.T, srv *httptest.Server, etag string) *Held { t.Helper() client := awss3.New(awss3.Options{ Region: "us-east-1", BaseEndpoint: aws.String(srv.URL), UsePathStyle: true, Credentials: credentials.NewStaticCredentialsProvider("k", "s", ""), RetryMaxAttempts: 1, }) leaser := lss3.NewLeaser() leaser.SetClient(client) leaser.Bucket = "b" leaser.Path = "p" leaser.TTL = time.Hour rec := &litestream.Lease{Generation: 1, ETag: etag, Owner: "test", ExpiresAt: time.Now().Add(time.Hour)} return &Held{leaser: leaser, rn: startRenewer(leaser, rec, time.Now(), marginForTTL(time.Hour))} } // A failed release leaves a conditional obligation that RetryRelease // discharges with the same ETag (#292): Release spends the handle, so // without it the lease stays held for its whole TTL. func TestRetryReleaseDischargesAFailedRelease(t *testing.T) { stub := &deleteStub{statuses: []int{http.StatusForbidden, http.StatusNoContent}} srv := httptest.NewServer(stub) defer srv.Close() h := heldAgainst(t, srv, `"e1"`) ctx := context.Background() if err := h.Release(ctx); err == nil { t.Fatal("Release succeeded against a refusing store") } if !h.ReleaseOwed() { t.Fatal("a failed release left no obligation") } if err := h.RetryRelease(ctx); err != nil { t.Fatalf("RetryRelease: %v", err) } if h.ReleaseOwed() { t.Fatal("the obligation outlived a successful retry") } if err := h.RetryRelease(ctx); err != nil { t.Fatalf("RetryRelease with nothing owed: %v", err) } got := stub.sent() if len(got) != 2 || got[0] != `"e1"` || got[1] != `"e1"` { t.Fatalf("If-Match sent = %q, want the lease's ETag on both deletes, and no third", got) } } // A retry whose condition fails (the lease expired and someone else holds // it now) has nothing of ours left to release: the obligation is resolved, // and nothing else's lease is deleted. func TestRetryReleaseResolvesWhenTheLeaseIsNoLongerOurs(t *testing.T) { stub := &deleteStub{statuses: []int{http.StatusForbidden, http.StatusPreconditionFailed}} srv := httptest.NewServer(stub) defer srv.Close() h := heldAgainst(t, srv, `"e1"`) ctx := context.Background() _ = h.Release(ctx) if err := h.RetryRelease(ctx); err != nil { t.Fatalf("RetryRelease after a failed condition: %v", err) } if h.ReleaseOwed() { t.Fatal("a lease that is no longer ours is still owed") } } // A successful release owes nothing. func TestASuccessfulReleaseOwesNothing(t *testing.T) { stub := &deleteStub{} srv := httptest.NewServer(stub) defer srv.Close() h := heldAgainst(t, srv, `"e1"`) if err := h.Release(context.Background()); err != nil { t.Fatalf("Release: %v", err) } if h.ReleaseOwed() { t.Fatal("a successful release left an obligation") } } // Round 2 P2: under a persistent DELETE failure with acquisition still // allowed, obligations must not pile up forever. Each failed release stays // owed until its own record's expiry; past it, a retry that still fails // retires the obligation (ErrReleaseLapsed) rather than retrying forever. // Two acquisitions in a row, both failing to release, across that expiry. func TestAnOwedReleaseIsRetiredOnceItsRecordHasExpired(t *testing.T) { stub := &deleteStub{statuses: []int{ http.StatusForbidden, http.StatusForbidden, http.StatusForbidden, http.StatusForbidden, http.StatusForbidden, http.StatusForbidden, http.StatusForbidden, http.StatusForbidden, }} srv := httptest.NewServer(stub) defer srv.Close() ctx := context.Background() expires := time.Now().Add(300 * time.Millisecond) var held []*Held for i := 0; i < 2; i++ { h := heldExpiring(t, srv, fmt.Sprintf(`"e%d"`, i), expires) if err := h.Release(ctx); err == nil { t.Fatal("Release succeeded against a refusing store") } held = append(held, h) } for _, h := range held { if err := h.RetryRelease(ctx); err == nil || errors.Is(err, ErrReleaseLapsed) { t.Fatalf("a retry before the record's expiry = %v, want the store's refusal", err) } if !h.ReleaseOwed() { t.Fatal("an unexpired record stopped being owed") } } time.Sleep(time.Until(expires) + 50*time.Millisecond) for _, h := range held { if err := h.RetryRelease(ctx); !errors.Is(err, ErrReleaseLapsed) { t.Fatalf("a failed retry past the record's expiry = %v, want ErrReleaseLapsed", err) } if h.ReleaseOwed() { t.Fatal("an expired record is still owed") } } } // heldExpiring is heldAgainst with the record's own expiry set. func heldExpiring(t *testing.T, srv *httptest.Server, etag string, expires time.Time) *Held { t.Helper() h := heldAgainst(t, srv, etag) h.rn.mu.Lock() h.rn.lease.ExpiresAt = expires h.rn.mu.Unlock() return h }