package leasetest
import (
"context"
"crypto/sha256"
"encoding/hex"
"encoding/xml"
"errors"
"fmt"
"io"
"net/http"
"net/http/httptest"
"net/url"
"strings"
"testing"
"github.com/carlosframework/platform/internal/lease"
)
// failingReader yields some bytes and then an error, like a dropped upload.
type failingReader struct{ r io.Reader }
func (f *failingReader) Read(p []byte) (int, error) {
n, err := f.r.Read(p)
if err == io.EOF {
return n, errors.New("connection reset")
}
return n, err
}
func TestS3ServerFailedUploadStoresNothing(t *testing.T) {
m := NewVersionedBucket()
m.AddBucket("data")
h, _ := NewS3Server(m)
before := m.Snapshot()
req := httptest.NewRequest(http.MethodPut, "/data/k", &failingReader{strings.NewReader("partial")})
req.Header.Set("If-None-Match", "*")
rec := httptest.NewRecorder()
h.ServeHTTP(rec, req)
if rec.Code < 400 {
t.Fatalf("failed upload answered %d", rec.Code)
}
if m.Snapshot() != before {
t.Fatalf("failed upload changed the bucket:\n%s", m.Snapshot())
}
}
func do(t *testing.T, h http.Handler, method, path string, body string, hdr map[string]string) *httptest.ResponseRecorder {
t.Helper()
req := httptest.NewRequest(method, path, strings.NewReader(body))
for k, v := range hdr {
req.Header.Set(k, v)
}
rec := httptest.NewRecorder()
h.ServeHTTP(rec, req)
return rec
}
// A plain PUT (no If-None-Match) stores a new version beside the existing
// one, as S3 with versioning on does; the create-only PUT keeps refusing
// an existing current object. A plain DELETE adds a delete marker.
func TestS3ServerPlainPutCreatesAVersion(t *testing.T) {
m := NewVersionedBucket()
m.AddBucket("data")
h, _ := NewS3Server(m)
first := do(t, h, http.MethodPut, "/data/k", "one", nil)
second := do(t, h, http.MethodPut, "/data/k", "two", nil)
if first.Code != 200 || second.Code != 200 {
t.Fatalf("plain puts: %d %d", first.Code, second.Code)
}
v1, v2 := first.Header().Get("x-amz-version-id"), second.Header().Get("x-amz-version-id")
if v1 == "" || v2 == "" || v1 == v2 {
t.Fatalf("version ids %q %q", v1, v2)
}
if got, _ := m.Current("data", "k"); string(got) != "two" {
t.Fatalf("current %q", got)
}
if old, err := m.GetVersion(context.Background(), "data", "k", v1); err != nil || string(old) != "one" {
t.Fatalf("the first version is gone: %q %v", old, err)
}
// a GET naming no version reads the current one
if rec := do(t, h, http.MethodGet, "/data/k", "", nil); rec.Code != 200 || rec.Body.String() != "two" {
t.Fatalf("plain get: %d %q", rec.Code, rec.Body.String())
}
if rec := do(t, h, http.MethodGet, "/data/absent", "", nil); rec.Code != http.StatusNotFound {
t.Fatalf("plain get of an absent key: %d", rec.Code)
}
// the create-only path is unchanged
if rec := do(t, h, http.MethodPut, "/data/k", "three", map[string]string{"If-None-Match": "*"}); rec.Code != http.StatusPreconditionFailed {
t.Fatalf("create-only over a current object: %d", rec.Code)
}
if rec := do(t, h, http.MethodPut, "/data/fresh", "x", map[string]string{"If-None-Match": "*"}); rec.Code != 200 || rec.Header().Get("x-amz-version-id") == "" {
t.Fatalf("create-only on a fresh key: %d", rec.Code)
}
// a bare DELETE leaves a delete marker, the versions stay
if rec := do(t, h, http.MethodDelete, "/data/k", "", nil); rec.Code != http.StatusNoContent {
t.Fatalf("bare delete: %d", rec.Code)
}
if _, ok := m.Current("data", "k"); ok {
t.Fatal("the key still reads after a bare delete")
}
if old, err := m.GetVersion(context.Background(), "data", "k", v2); err != nil || string(old) != "two" {
t.Fatalf("a bare delete removed a version: %q %v", old, err)
}
}
// A copy that names no source version copies the source's current one, and
// honors x-amz-copy-source-if-match against its ETag (S3's behavior; the
// staging mirror copies this way).
func TestS3ServerCopyOfCurrentVersion(t *testing.T) {
m := NewVersionedBucket()
m.AddBucket("src")
m.AddBucket("dst")
m.Put("src", "a/b c+d", []byte("body"))
h, _ := NewS3Server(m)
src := "/src/a/b%20c%2Bd"
rec := do(t, h, http.MethodPut, "/dst/out", "", map[string]string{"x-amz-copy-source": src, "x-amz-copy-source-if-match": `"` + md5hex([]byte("body")) + `"`})
if rec.Code != 200 || rec.Header().Get("x-amz-version-id") == "" || !strings.Contains(rec.Body.String(), md5hex([]byte("body"))) {
t.Fatalf("copy: %d %q %v", rec.Code, rec.Body.String(), rec.Header())
}
rec = do(t, h, http.MethodPut, "/dst/out2", "", map[string]string{"x-amz-copy-source": src, "x-amz-copy-source-if-match": `"deadbeef"`})
if rec.Code != http.StatusPreconditionFailed || !strings.Contains(rec.Body.String(), "PreconditionFailed") {
t.Fatalf("stale if-match: %d %q", rec.Code, rec.Body.String())
}
if _, ok := m.Current("dst", "out2"); ok {
t.Fatal("a failed precondition copied")
}
}
// Real S3 rejects a PUT whose body is not the length the request declares,
// or whose sha256 is not the one the request signed, and stores nothing. The
// fake must do the same, or a client that sends a short or altered body looks
// like it succeeded (a transport can hand the server a partial body before it
// fails).
func TestS3ServerValidatesPutBody(t *testing.T) {
sum := sha256.Sum256([]byte("abcdefghij"))
good := hex.EncodeToString(sum[:])
wrong := hex.EncodeToString(make([]byte, 32))
put := func(h http.Handler, key, body string, declared int64, sha string, extra map[string]string) *httptest.ResponseRecorder {
req := httptest.NewRequest(http.MethodPut, "/data/"+key, strings.NewReader(body))
req.ContentLength = declared
if sha != "" {
req.Header.Set("x-amz-content-sha256", sha)
}
for k, v := range extra {
req.Header.Set(k, v)
}
rec := httptest.NewRecorder()
h.ServeHTTP(rec, req)
return rec
}
for _, c := range []struct {
name string
body string
declared int64
sha string
code string // empty: accepted
}{
{"exact", "abcdefghij", 10, good, ""},
{"unsigned payload", "abcdefghij", 10, "UNSIGNED-PAYLOAD", ""},
{"no sha header", "abcdefghij", 10, "", ""},
{"unknown length", "abcdefghij", -1, good, ""},
{"short body", "abc", 10, good, "IncompleteBody"},
{"short body unsigned", "abc", 10, "UNSIGNED-PAYLOAD", "IncompleteBody"},
{"long body", "abcdefghijkl", 10, "", "IncompleteBody"},
{"wrong sha", "abcdefghij", 10, wrong, "XAmzContentSHA256Mismatch"},
{"altered body", "abcdefghiX", 10, good, "XAmzContentSHA256Mismatch"},
} {
for _, createOnly := range []bool{false, true} {
m := NewVersionedBucket()
m.AddBucket("data")
h, _ := NewS3Server(m)
extra := map[string]string{}
if createOnly {
extra["If-None-Match"] = "*"
}
rec := put(h, "k", c.body, c.declared, c.sha, extra)
_, stored := m.Current("data", "k")
if c.code == "" {
if rec.Code != 200 || !stored {
t.Errorf("%s (create-only %v): %d, stored %v", c.name, createOnly, rec.Code, stored)
}
continue
}
if rec.Code != http.StatusBadRequest || !strings.Contains(rec.Body.String(), ""+c.code+"") {
t.Errorf("%s (create-only %v): %d %q, want 400 %s", c.name, createOnly, rec.Code, rec.Body.String(), c.code)
}
if stored {
t.Errorf("%s (create-only %v): a rejected PUT stored the body", c.name, createOnly)
}
}
}
}
// A copy carries no body: its Content-Length must be what arrives and its
// signed sha256 (when a digest) the hash of nothing. A copy that fails the
// check stores nothing.
func TestS3ServerValidatesCopyBody(t *testing.T) {
empty := sha256.Sum256(nil)
m := NewVersionedBucket()
m.AddBucket("src")
m.AddBucket("dst")
m.Put("src", "a", []byte("body"))
h, _ := NewS3Server(m)
copyWith := func(dst, body string, declared int64, sha string) *httptest.ResponseRecorder {
req := httptest.NewRequest(http.MethodPut, "/dst/"+dst, strings.NewReader(body))
req.ContentLength = declared
req.Header.Set("x-amz-copy-source", "/src/a")
if sha != "" {
req.Header.Set("x-amz-content-sha256", sha)
}
rec := httptest.NewRecorder()
h.ServeHTTP(rec, req)
return rec
}
if rec := copyWith("ok", "", 0, hex.EncodeToString(empty[:])); rec.Code != 200 {
t.Fatalf("a well-formed copy: %d %q", rec.Code, rec.Body.String())
}
if rec := copyWith("ok2", "", 0, "UNSIGNED-PAYLOAD"); rec.Code != 200 {
t.Fatalf("an unsigned-payload copy: %d %q", rec.Code, rec.Body.String())
}
if rec := copyWith("long", "xyz", 0, ""); rec.Code != http.StatusBadRequest || !strings.Contains(rec.Body.String(), "IncompleteBody") {
t.Fatalf("a copy with a body longer than declared: %d %q", rec.Code, rec.Body.String())
}
if rec := copyWith("sha", "", 0, hex.EncodeToString(make([]byte, 32))); rec.Code != http.StatusBadRequest || !strings.Contains(rec.Body.String(), "XAmzContentSHA256Mismatch") {
t.Fatalf("a copy with the wrong sha256: %d %q", rec.Code, rec.Body.String())
}
for _, k := range []string{"long", "sha"} {
if _, ok := m.Current("dst", k); ok {
t.Fatalf("a rejected copy stored %q", k)
}
}
}
func TestS3ServerConditionalReplaceUsesReadETag(t *testing.T) {
m := NewVersionedBucket()
m.AddBucket("data")
h, _ := NewS3Server(m)
first := do(t, h, http.MethodPut, "/data/lock.json", "expired", map[string]string{"If-None-Match": "*"})
read := do(t, h, http.MethodGet, "/data/lock.json", "", nil)
etag := read.Header().Get("ETag")
if etag == "" || etag != first.Header().Get("ETag") {
t.Fatal("GET did not return current ETag", etag)
}
if got := do(t, h, http.MethodPut, "/data/lock.json", "new-owner", map[string]string{"If-Match": etag}); got.Code != 200 {
t.Fatal("takeover", got.Code)
}
before := m.Snapshot()
if got := do(t, h, http.MethodPut, "/data/lock.json", "stale-owner", map[string]string{"If-Match": etag}); got.Code != http.StatusPreconditionFailed {
t.Fatal("stale CAS accepted", got.Code)
}
if m.Snapshot() != before {
t.Fatal("stale CAS mutated versions")
}
if got := do(t, h, http.MethodPut, "/data/missing", "new", map[string]string{"If-Match": etag}); got.Code != http.StatusPreconditionFailed {
t.Fatal("CAS on missing accepted", got.Code)
}
do(t, h, http.MethodDelete, "/data/lock.json", "", nil)
if got := do(t, h, http.MethodPut, "/data/lock.json", "new", map[string]string{"If-Match": etag}); got.Code != http.StatusPreconditionFailed {
t.Fatal("CAS over delete marker accepted", got.Code)
}
}
// DeleteObject honours If-Match as S3 does: a stale ETag is a 412 and removes
// nothing, a missing key is a 404, the current ETag deletes, and a plain
// delete (no header) is unchanged.
func TestS3ServerConditionalDeleteUsesReadETag(t *testing.T) {
m := NewVersionedBucket()
m.AddBucket("data")
h, _ := NewS3Server(m)
do(t, h, http.MethodPut, "/data/lock.json", "one", map[string]string{"If-None-Match": "*"})
stale := do(t, h, http.MethodGet, "/data/lock.json", "", nil).Header().Get("ETag")
do(t, h, http.MethodPut, "/data/lock.json", "two", map[string]string{"If-Match": stale})
current := do(t, h, http.MethodGet, "/data/lock.json", "", nil).Header().Get("ETag")
before := m.Snapshot()
if got := do(t, h, http.MethodDelete, "/data/lock.json", "", map[string]string{"If-Match": stale}); got.Code != http.StatusPreconditionFailed {
t.Fatal("stale conditional delete accepted", got.Code)
}
if m.Snapshot() != before {
t.Fatal("stale conditional delete mutated versions")
}
if got := do(t, h, http.MethodDelete, "/data/lock.json", "", map[string]string{"If-Match": current}); got.Code != http.StatusNoContent {
t.Fatal("current conditional delete refused", got.Code)
}
if got := do(t, h, http.MethodDelete, "/data/lock.json", "", map[string]string{"If-Match": current}); got.Code != http.StatusNotFound {
t.Fatal("conditional delete of a deleted key", got.Code)
}
if got := do(t, h, http.MethodDelete, "/data/never", "", map[string]string{"If-Match": current}); got.Code != http.StatusNotFound {
t.Fatal("conditional delete of a missing key", got.Code)
}
if got := do(t, h, http.MethodDelete, "/data/never", "", nil); got.Code != http.StatusNoContent {
t.Fatal("plain delete changed", got.Code)
}
}
// listV2 decodes one ListObjectsV2 page from the raw wire.
func listV2(t *testing.T, h http.Handler, query string) (listBucketResult, int) {
t.Helper()
rec := do(t, h, http.MethodGet, "/data?list-type=2"+query, "", nil)
var res listBucketResult
if rec.Code == http.StatusOK {
if err := xml.Unmarshal(rec.Body.Bytes(), &res); err != nil {
t.Fatalf("decode %q: %v", rec.Body.String(), err)
}
}
return res, rec.Code
}
// ListObjectsV2 on the wire: the current objects under a prefix in key
// order, each with its size and quoted ETag; a key whose newest version is
// a delete marker, an older version and a sibling prefix are not listed;
// max-keys pages, and the continuation token resumes after the page;
// start-after skips; a bad token and a negative max-keys are 400s; a
// delimiter rolls keys into common prefixes, which page like keys.
func TestS3ServerListObjectsV2(t *testing.T) {
m := NewVersionedBucket()
m.AddBucket("data")
h, _ := NewS3Server(m)
for _, k := range []string{"p/c", "p/a", "p/b/x", "p/d", "q/a", "p2/a"} {
m.Put("data", k, []byte("body of "+k))
}
m.Put("data", "p/a", []byte("newer"))
m.BareDelete("data", "p/d")
res, code := listV2(t, h, "&prefix=p/")
if code != http.StatusOK {
t.Fatal(code)
}
var keys []string
for _, o := range res.Contents {
keys = append(keys, o.Key)
}
if strings.Join(keys, ",") != "p/a,p/b/x,p/c" || res.IsTruncated || res.KeyCount != 3 || res.NextContinuationToken != "" {
t.Fatalf("listing %+v", res)
}
if o := res.Contents[0]; o.Size != int64(len("newer")) || o.ETag != `"`+md5hex([]byte("newer"))+`"` {
t.Fatalf("p/a listed as %+v", o)
}
first, _ := listV2(t, h, "&prefix=p/&max-keys=2")
if len(first.Contents) != 2 || !first.IsTruncated || first.NextContinuationToken == "" || first.Contents[1].Key != "p/b/x" {
t.Fatalf("first page %+v", first)
}
second, _ := listV2(t, h, "&prefix=p/&max-keys=2&continuation-token="+url.QueryEscape(first.NextContinuationToken))
if len(second.Contents) != 1 || second.Contents[0].Key != "p/c" || second.IsTruncated || second.ContinuationToken != first.NextContinuationToken {
t.Fatalf("second page %+v", second)
}
after, _ := listV2(t, h, "&prefix=p/&start-after=p/a")
if len(after.Contents) != 2 || after.Contents[0].Key != "p/b/x" {
t.Fatalf("start-after %+v", after)
}
if _, code := listV2(t, h, "&continuation-token=%21%21"); code != http.StatusBadRequest {
t.Fatal("bad token", code)
}
if _, code := listV2(t, h, "&max-keys=-1"); code != http.StatusBadRequest {
t.Fatal("negative max-keys", code)
}
// A delimiter rolls p/b/x into the common prefix p/b/, which counts
// against max-keys and can end a page.
m.Put("data", "p/b/y", []byte("y"))
d1, _ := listV2(t, h, "&prefix=p/&delimiter=/&max-keys=2")
if len(d1.Contents) != 1 || d1.Contents[0].Key != "p/a" || len(d1.CommonPrefixes) != 1 || d1.CommonPrefixes[0].Prefix != "p/b/" ||
d1.KeyCount != 2 || !d1.IsTruncated || d1.Delimiter != "/" {
t.Fatalf("delimited first page %+v", d1)
}
d2, _ := listV2(t, h, "&prefix=p/&delimiter=/&max-keys=2&continuation-token="+url.QueryEscape(d1.NextContinuationToken))
if len(d2.Contents) != 1 || d2.Contents[0].Key != "p/c" || len(d2.CommonPrefixes) != 0 || d2.IsTruncated {
t.Fatalf("delimited second page %+v", d2)
}
top, _ := listV2(t, h, "&delimiter=/")
var tops []string
for _, p := range top.CommonPrefixes {
tops = append(tops, p.Prefix)
}
if len(top.Contents) != 0 || strings.Join(tops, ",") != "p/,p2/,q/" {
t.Fatalf("top level %+v", top)
}
}
// The SDK's own paginator, through lease.NewS3Mirror, reads every page:
// 2,500 keys at the default 1,000 a page, and again at a PageSize of 7.
func TestS3ServerListObjectsV2PaginatesTheSDK(t *testing.T) {
m := NewVersionedBucket()
m.AddBucket("data")
h, _ := NewS3Server(m)
srv := httptest.NewServer(h)
defer srv.Close()
t.Setenv("AWS_ACCESS_KEY_ID", "test-key")
t.Setenv("AWS_SECRET_ACCESS_KEY", "test-secret")
t.Setenv("AWS_SESSION_TOKEN", "")
t.Setenv("AWS_EC2_METADATA_DISABLED", "true")
t.Setenv("AWS_CONFIG_FILE", t.TempDir()+"/absent")
t.Setenv("AWS_SHARED_CREDENTIALS_FILE", t.TempDir()+"/absent")
t.Setenv("CARLOS_S3_ENDPOINT", srv.URL)
want := map[string]bool{}
for i := range 2500 {
k := fmt.Sprintf("big/%05d", i)
m.Put("data", k, []byte(k))
want[k] = true
}
m.Put("data", "bigger/x", []byte("x")) // a sibling prefix, never listed
ctx := context.Background()
mirror, err := lease.NewS3Mirror(ctx, "eu-west-1", "eu-west-1")
if err != nil {
t.Fatal(err)
}
for _, page := range []int{1000, 7} {
m.PageSize = page
got, err := mirror.List(ctx, "data", "big/")
if err != nil {
t.Fatalf("page size %d: %v", page, err)
}
if len(got) != len(want) {
t.Fatalf("page size %d: listed %d keys, want %d", page, len(got), len(want))
}
for k, o := range got {
if !want[k] || o.Size != int64(len(k)) || o.ETag != md5hex([]byte(k)) {
t.Fatalf("page size %d: %s listed as %+v", page, k, o)
}
}
}
}