package release
import (
"context"
"errors"
"fmt"
"io"
"net/http"
"net/http/httptest"
"strings"
"testing"
)
// rgwFake is a fake S3 that reproduces the one behaviour measured against
// Hetzner Object Storage (Ceph RGW) on 2026-08-07: If-Match on PUT is compared
// against the UNQUOTED etag, so the quoted entity-tag form S3 itself returns
// never matches and every compare-and-swap replace 412s.
//
// It is deliberately strict in both directions — a wrong etag is rejected in
// either form — so a fix that merely stopped sending the condition would fail
// this test rather than pass it.
type rgwFake struct {
etag string // current etag, unquoted
body string
}
func newRGWFake(t *testing.T) (*rgwFake, string) {
t.Helper()
f := &rgwFake{}
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
precondFailed := func() {
w.Header().Set("Content-Type", "application/xml")
w.WriteHeader(http.StatusPreconditionFailed)
_, _ = w.Write([]byte(`` +
`PreconditionFailedetag mismatch`))
}
switch r.Method {
case http.MethodGet:
if f.etag == "" {
w.Header().Set("Content-Type", "application/xml")
w.WriteHeader(http.StatusNotFound)
_, _ = w.Write([]byte(`` +
`NoSuchKeyabsent`))
return
}
w.Header().Set("ETag", `"`+f.etag+`"`) // quoted, exactly as S3 does
_, _ = w.Write([]byte(f.body))
case http.MethodPut:
raw, _ := io.ReadAll(r.Body)
switch {
case r.Header.Get("If-None-Match") == "*":
if f.etag != "" {
precondFailed() // create-only against an existing object
return
}
case r.Header.Get("If-Match") != "":
got := r.Header.Get("If-Match")
// THE QUIRK: RGW compares against the unquoted etag, so a
// correctly-quoted If-Match never matches.
if got != f.etag {
precondFailed()
return
}
}
f.body = string(raw)
f.etag = fmt.Sprintf("%08x", len(f.body)*2654435761)
w.Header().Set("ETag", `"`+f.etag+`"`)
w.WriteHeader(http.StatusOK)
default:
w.WriteHeader(http.StatusMethodNotAllowed)
}
}))
t.Cleanup(srv.Close)
return f, srv.URL
}
func rgwEnv(t *testing.T, url string) {
t.Helper()
t.Setenv("AWS_ACCESS_KEY_ID", "k")
t.Setenv("AWS_SECRET_ACCESS_KEY", "s")
t.Setenv("AWS_SESSION_TOKEN", "")
t.Setenv("AWS_REGION", "nbg1")
t.Setenv("AWS_EC2_METADATA_DISABLED", "true")
t.Setenv("CARLOS_S3_ENDPOINT", url)
}
// TestS3Store_CASReplaceAgainstRGW is the regression test for the
// compare-and-swap failure that made the private CARLOS box report itself
// UNREACHABLE: a single writer lost CAS 20/20 times against Hetzner because
// every replace sent a quoted If-Match.
//
// The subtests are a matched pair — the quirk off case must FAIL, or the fix
// is not what is making the on case pass.
func TestS3Store_CASReplaceAgainstRGW(t *testing.T) {
ctx := context.Background()
replaceOnce := func(t *testing.T) error {
t.Helper()
_, url := newRGWFake(t)
rgwEnv(t, url)
s, err := NewS3Store(ctx, "b")
if err != nil {
t.Fatalf("NewS3Store: %v", err)
}
// Create-only first: this path uses If-None-Match:* and works on RGW,
// which is exactly why a fresh deployment looks healthy.
if _, err := s.Put(ctx, "status/boxes/x.json", []byte(`{"n":0}`), ""); err != nil {
t.Fatalf("create-only Put should succeed even on RGW: %v", err)
}
_, etag, err := s.Get(ctx, "status/boxes/x.json")
if err != nil {
t.Fatalf("Get: %v", err)
}
if !strings.HasPrefix(etag, `"`) {
t.Fatalf("fake should hand back a quoted etag like S3 does, got %q", etag)
}
// The replace — the operation that fails in production.
_, err = s.Put(ctx, "status/boxes/x.json", []byte(`{"n":1}`), etag)
return err
}
t.Run("quirk off: reproduces the production failure", func(t *testing.T) {
t.Setenv("CARLOS_S3_QUIRKS", "")
err := replaceOnce(t)
if !errors.Is(err, ErrCASConflict) {
t.Fatalf("want ErrCASConflict without the quirk (this is the bug), got %v", err)
}
})
t.Run("quirk on: replace succeeds", func(t *testing.T) {
// The fake's host is 127.0.0.1, so match-by-hostname cannot fire;
// force by name, the same way an operator would on a new store.
t.Setenv("CARLOS_S3_QUIRKS", "unquoted-if-match")
if err := replaceOnce(t); err != nil {
t.Fatalf("replace should succeed with the unquoted-if-match quirk: %v", err)
}
})
}
// TestS3Store_IfMatchOnDeletedObjectIsCASConflict: a replace-only write whose
// object was DELETED under it comes back 404/NoSuchKey rather than 412 —
// there is no etag left to fail the precondition against. It is still a lost
// conditional write (the caller decided this body from bytes that no longer
// exist), and classifying it as anything else surfaces as a 502 "could not
// update, try again" that will never succeed, on a door that already knows how
// to answer a conflict. Found reviewing the instance-record PATCH, 2026-08-12.
//
// The create-only half is the guard on the guard: on an If-None-Match:* write
// a 404 is not an expected answer at all, and masking it into a conflict would
// hide a broken store behind a retry-shaped error.
func TestS3Store_IfMatchOnDeletedObjectIsCASConflict(t *testing.T) {
ctx := context.Background()
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPut {
w.WriteHeader(http.StatusMethodNotAllowed)
return
}
w.Header().Set("Content-Type", "application/xml")
w.WriteHeader(http.StatusNotFound)
_, _ = w.Write([]byte(`` +
`NoSuchKeythe object was deleted`))
}))
t.Cleanup(srv.Close)
rgwEnv(t, srv.URL)
s, err := NewS3Store(ctx, "b")
if err != nil {
t.Fatalf("NewS3Store: %v", err)
}
_, err = s.Put(ctx, "apps/bab/titogo/instances/ha.go.tito.io.json", []byte(`{"schema":1}`), `"abc123"`)
if !errors.Is(err, ErrCASConflict) {
t.Fatalf("If-Match Put against a deleted object = %v, want ErrCASConflict", err)
}
if _, err := s.Put(ctx, "apps/bab/titogo/instances/ha.go.tito.io.json", []byte(`{"schema":1}`), ""); errors.Is(err, ErrCASConflict) {
t.Fatalf("create-only Put reported a 404 as a CAS conflict: %v", err)
} else if err == nil {
t.Fatal("create-only Put against a 404-ing store returned no error at all")
}
}
// TestS3Store_CASStillRejectsAStaleEtag is the safety half. Stripping quotes
// must FIX compare-and-swap, not remove it: a genuinely stale etag has to keep
// losing, or every CAS in CARLOS silently becomes a blind overwrite.
func TestS3Store_CASStillRejectsAStaleEtag(t *testing.T) {
ctx := context.Background()
_, url := newRGWFake(t)
rgwEnv(t, url)
t.Setenv("CARLOS_S3_QUIRKS", "unquoted-if-match")
s, err := NewS3Store(ctx, "b")
if err != nil {
t.Fatalf("NewS3Store: %v", err)
}
key := "status/boxes/y.json"
if _, err := s.Put(ctx, key, []byte(`{"n":0}`), ""); err != nil {
t.Fatalf("seed Put: %v", err)
}
_, stale, err := s.Get(ctx, key)
if err != nil {
t.Fatalf("Get: %v", err)
}
// Someone else moves the object on.
if _, err := s.Put(ctx, key, []byte(`{"n":1,"other":true}`), stale); err != nil {
t.Fatalf("first replace: %v", err)
}
// Our now-stale etag must lose.
if _, err := s.Put(ctx, key, []byte(`{"n":2}`), stale); !errors.Is(err, ErrCASConflict) {
t.Fatalf("a stale etag must still lose CAS, got %v — the quirk has disabled compare-and-swap", err)
}
}