package lease
import (
"context"
"errors"
"fmt"
"net/http"
"net/http/httptest"
"sync"
"testing"
"time"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/credentials"
awss3 "github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/benbjohnson/litestream"
lss3 "github.com/benbjohnson/litestream/s3"
)
// deleteStub answers the lease's DELETE with each status in turn and
// records the If-Match header it was sent.
type deleteStub struct {
mu sync.Mutex
statuses []int
ifMatch []string
}
func (s *deleteStub) ServeHTTP(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodDelete {
http.Error(w, "unexpected "+r.Method, http.StatusMethodNotAllowed)
return
}
s.mu.Lock()
s.ifMatch = append(s.ifMatch, r.Header.Get("If-Match"))
status := http.StatusNoContent
if len(s.statuses) > 0 {
status, s.statuses = s.statuses[0], s.statuses[1:]
}
s.mu.Unlock()
switch status {
case http.StatusNoContent:
w.WriteHeader(status)
case http.StatusPreconditionFailed:
w.Header().Set("Content-Type", "application/xml")
w.WriteHeader(status)
_, _ = w.Write([]byte("PreconditionFailedno"))
default:
w.Header().Set("Content-Type", "application/xml")
w.WriteHeader(status)
_, _ = w.Write([]byte("AccessDeniedno"))
}
}
func (s *deleteStub) sent() []string {
s.mu.Lock()
defer s.mu.Unlock()
return append([]string(nil), s.ifMatch...)
}
// heldAgainst builds a Held whose lease record has ETag etag, with its
// leaser pointed at srv. The TTL is long, so the renewer never renews
// inside the test.
func heldAgainst(t *testing.T, srv *httptest.Server, etag string) *Held {
t.Helper()
client := awss3.New(awss3.Options{
Region: "us-east-1",
BaseEndpoint: aws.String(srv.URL),
UsePathStyle: true,
Credentials: credentials.NewStaticCredentialsProvider("k", "s", ""),
RetryMaxAttempts: 1,
})
leaser := lss3.NewLeaser()
leaser.SetClient(client)
leaser.Bucket = "b"
leaser.Path = "p"
leaser.TTL = time.Hour
rec := &litestream.Lease{Generation: 1, ETag: etag, Owner: "test", ExpiresAt: time.Now().Add(time.Hour)}
return &Held{leaser: leaser, rn: startRenewer(leaser, rec, time.Now(), marginForTTL(time.Hour))}
}
// A failed release leaves a conditional obligation that RetryRelease
// discharges with the same ETag (#292): Release spends the handle, so
// without it the lease stays held for its whole TTL.
func TestRetryReleaseDischargesAFailedRelease(t *testing.T) {
stub := &deleteStub{statuses: []int{http.StatusForbidden, http.StatusNoContent}}
srv := httptest.NewServer(stub)
defer srv.Close()
h := heldAgainst(t, srv, `"e1"`)
ctx := context.Background()
if err := h.Release(ctx); err == nil {
t.Fatal("Release succeeded against a refusing store")
}
if !h.ReleaseOwed() {
t.Fatal("a failed release left no obligation")
}
if err := h.RetryRelease(ctx); err != nil {
t.Fatalf("RetryRelease: %v", err)
}
if h.ReleaseOwed() {
t.Fatal("the obligation outlived a successful retry")
}
if err := h.RetryRelease(ctx); err != nil {
t.Fatalf("RetryRelease with nothing owed: %v", err)
}
got := stub.sent()
if len(got) != 2 || got[0] != `"e1"` || got[1] != `"e1"` {
t.Fatalf("If-Match sent = %q, want the lease's ETag on both deletes, and no third", got)
}
}
// A retry whose condition fails (the lease expired and someone else holds
// it now) has nothing of ours left to release: the obligation is resolved,
// and nothing else's lease is deleted.
func TestRetryReleaseResolvesWhenTheLeaseIsNoLongerOurs(t *testing.T) {
stub := &deleteStub{statuses: []int{http.StatusForbidden, http.StatusPreconditionFailed}}
srv := httptest.NewServer(stub)
defer srv.Close()
h := heldAgainst(t, srv, `"e1"`)
ctx := context.Background()
_ = h.Release(ctx)
if err := h.RetryRelease(ctx); err != nil {
t.Fatalf("RetryRelease after a failed condition: %v", err)
}
if h.ReleaseOwed() {
t.Fatal("a lease that is no longer ours is still owed")
}
}
// A successful release owes nothing.
func TestASuccessfulReleaseOwesNothing(t *testing.T) {
stub := &deleteStub{}
srv := httptest.NewServer(stub)
defer srv.Close()
h := heldAgainst(t, srv, `"e1"`)
if err := h.Release(context.Background()); err != nil {
t.Fatalf("Release: %v", err)
}
if h.ReleaseOwed() {
t.Fatal("a successful release left an obligation")
}
}
// Round 2 P2: under a persistent DELETE failure with acquisition still
// allowed, obligations must not pile up forever. Each failed release stays
// owed until its own record's expiry; past it, a retry that still fails
// retires the obligation (ErrReleaseLapsed) rather than retrying forever.
// Two acquisitions in a row, both failing to release, across that expiry.
func TestAnOwedReleaseIsRetiredOnceItsRecordHasExpired(t *testing.T) {
stub := &deleteStub{statuses: []int{
http.StatusForbidden, http.StatusForbidden, http.StatusForbidden, http.StatusForbidden,
http.StatusForbidden, http.StatusForbidden, http.StatusForbidden, http.StatusForbidden,
}}
srv := httptest.NewServer(stub)
defer srv.Close()
ctx := context.Background()
expires := time.Now().Add(300 * time.Millisecond)
var held []*Held
for i := 0; i < 2; i++ {
h := heldExpiring(t, srv, fmt.Sprintf(`"e%d"`, i), expires)
if err := h.Release(ctx); err == nil {
t.Fatal("Release succeeded against a refusing store")
}
held = append(held, h)
}
for _, h := range held {
if err := h.RetryRelease(ctx); err == nil || errors.Is(err, ErrReleaseLapsed) {
t.Fatalf("a retry before the record's expiry = %v, want the store's refusal", err)
}
if !h.ReleaseOwed() {
t.Fatal("an unexpired record stopped being owed")
}
}
time.Sleep(time.Until(expires) + 50*time.Millisecond)
for _, h := range held {
if err := h.RetryRelease(ctx); !errors.Is(err, ErrReleaseLapsed) {
t.Fatalf("a failed retry past the record's expiry = %v, want ErrReleaseLapsed", err)
}
if h.ReleaseOwed() {
t.Fatal("an expired record is still owed")
}
}
}
// heldExpiring is heldAgainst with the record's own expiry set.
func heldExpiring(t *testing.T, srv *httptest.Server, etag string, expires time.Time) *Held {
t.Helper()
h := heldAgainst(t, srv, etag)
h.rn.mu.Lock()
h.rn.lease.ExpiresAt = expires
h.rn.mu.Unlock()
return h
}