package release import ( "context" "errors" "fmt" "io" "net/http" "net/http/httptest" "strings" "testing" ) // rgwFake is a fake S3 that reproduces the one behaviour measured against // Hetzner Object Storage (Ceph RGW) on 2026-08-07: If-Match on PUT is compared // against the UNQUOTED etag, so the quoted entity-tag form S3 itself returns // never matches and every compare-and-swap replace 412s. // // It is deliberately strict in both directions — a wrong etag is rejected in // either form — so a fix that merely stopped sending the condition would fail // this test rather than pass it. type rgwFake struct { etag string // current etag, unquoted body string } func newRGWFake(t *testing.T) (*rgwFake, string) { t.Helper() f := &rgwFake{} srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { precondFailed := func() { w.Header().Set("Content-Type", "application/xml") w.WriteHeader(http.StatusPreconditionFailed) _, _ = w.Write([]byte(`` + `PreconditionFailedetag mismatch`)) } switch r.Method { case http.MethodGet: if f.etag == "" { w.Header().Set("Content-Type", "application/xml") w.WriteHeader(http.StatusNotFound) _, _ = w.Write([]byte(`` + `NoSuchKeyabsent`)) return } w.Header().Set("ETag", `"`+f.etag+`"`) // quoted, exactly as S3 does _, _ = w.Write([]byte(f.body)) case http.MethodPut: raw, _ := io.ReadAll(r.Body) switch { case r.Header.Get("If-None-Match") == "*": if f.etag != "" { precondFailed() // create-only against an existing object return } case r.Header.Get("If-Match") != "": got := r.Header.Get("If-Match") // THE QUIRK: RGW compares against the unquoted etag, so a // correctly-quoted If-Match never matches. if got != f.etag { precondFailed() return } } f.body = string(raw) f.etag = fmt.Sprintf("%08x", len(f.body)*2654435761) w.Header().Set("ETag", `"`+f.etag+`"`) w.WriteHeader(http.StatusOK) default: w.WriteHeader(http.StatusMethodNotAllowed) } })) t.Cleanup(srv.Close) return f, srv.URL } func rgwEnv(t *testing.T, url string) { t.Helper() t.Setenv("AWS_ACCESS_KEY_ID", "k") t.Setenv("AWS_SECRET_ACCESS_KEY", "s") t.Setenv("AWS_SESSION_TOKEN", "") t.Setenv("AWS_REGION", "nbg1") t.Setenv("AWS_EC2_METADATA_DISABLED", "true") t.Setenv("CARLOS_S3_ENDPOINT", url) } // TestS3Store_CASReplaceAgainstRGW is the regression test for the // compare-and-swap failure that made the private CARLOS box report itself // UNREACHABLE: a single writer lost CAS 20/20 times against Hetzner because // every replace sent a quoted If-Match. // // The subtests are a matched pair — the quirk off case must FAIL, or the fix // is not what is making the on case pass. func TestS3Store_CASReplaceAgainstRGW(t *testing.T) { ctx := context.Background() replaceOnce := func(t *testing.T) error { t.Helper() _, url := newRGWFake(t) rgwEnv(t, url) s, err := NewS3Store(ctx, "b") if err != nil { t.Fatalf("NewS3Store: %v", err) } // Create-only first: this path uses If-None-Match:* and works on RGW, // which is exactly why a fresh deployment looks healthy. if _, err := s.Put(ctx, "status/boxes/x.json", []byte(`{"n":0}`), ""); err != nil { t.Fatalf("create-only Put should succeed even on RGW: %v", err) } _, etag, err := s.Get(ctx, "status/boxes/x.json") if err != nil { t.Fatalf("Get: %v", err) } if !strings.HasPrefix(etag, `"`) { t.Fatalf("fake should hand back a quoted etag like S3 does, got %q", etag) } // The replace — the operation that fails in production. _, err = s.Put(ctx, "status/boxes/x.json", []byte(`{"n":1}`), etag) return err } t.Run("quirk off: reproduces the production failure", func(t *testing.T) { t.Setenv("CARLOS_S3_QUIRKS", "") err := replaceOnce(t) if !errors.Is(err, ErrCASConflict) { t.Fatalf("want ErrCASConflict without the quirk (this is the bug), got %v", err) } }) t.Run("quirk on: replace succeeds", func(t *testing.T) { // The fake's host is 127.0.0.1, so match-by-hostname cannot fire; // force by name, the same way an operator would on a new store. t.Setenv("CARLOS_S3_QUIRKS", "unquoted-if-match") if err := replaceOnce(t); err != nil { t.Fatalf("replace should succeed with the unquoted-if-match quirk: %v", err) } }) } // TestS3Store_IfMatchOnDeletedObjectIsCASConflict: a replace-only write whose // object was DELETED under it comes back 404/NoSuchKey rather than 412 — // there is no etag left to fail the precondition against. It is still a lost // conditional write (the caller decided this body from bytes that no longer // exist), and classifying it as anything else surfaces as a 502 "could not // update, try again" that will never succeed, on a door that already knows how // to answer a conflict. Found reviewing the instance-record PATCH, 2026-08-12. // // The create-only half is the guard on the guard: on an If-None-Match:* write // a 404 is not an expected answer at all, and masking it into a conflict would // hide a broken store behind a retry-shaped error. func TestS3Store_IfMatchOnDeletedObjectIsCASConflict(t *testing.T) { ctx := context.Background() srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.Method != http.MethodPut { w.WriteHeader(http.StatusMethodNotAllowed) return } w.Header().Set("Content-Type", "application/xml") w.WriteHeader(http.StatusNotFound) _, _ = w.Write([]byte(`` + `NoSuchKeythe object was deleted`)) })) t.Cleanup(srv.Close) rgwEnv(t, srv.URL) s, err := NewS3Store(ctx, "b") if err != nil { t.Fatalf("NewS3Store: %v", err) } _, err = s.Put(ctx, "apps/bab/titogo/instances/ha.go.tito.io.json", []byte(`{"schema":1}`), `"abc123"`) if !errors.Is(err, ErrCASConflict) { t.Fatalf("If-Match Put against a deleted object = %v, want ErrCASConflict", err) } if _, err := s.Put(ctx, "apps/bab/titogo/instances/ha.go.tito.io.json", []byte(`{"schema":1}`), ""); errors.Is(err, ErrCASConflict) { t.Fatalf("create-only Put reported a 404 as a CAS conflict: %v", err) } else if err == nil { t.Fatal("create-only Put against a 404-ing store returned no error at all") } } // TestS3Store_CASStillRejectsAStaleEtag is the safety half. Stripping quotes // must FIX compare-and-swap, not remove it: a genuinely stale etag has to keep // losing, or every CAS in CARLOS silently becomes a blind overwrite. func TestS3Store_CASStillRejectsAStaleEtag(t *testing.T) { ctx := context.Background() _, url := newRGWFake(t) rgwEnv(t, url) t.Setenv("CARLOS_S3_QUIRKS", "unquoted-if-match") s, err := NewS3Store(ctx, "b") if err != nil { t.Fatalf("NewS3Store: %v", err) } key := "status/boxes/y.json" if _, err := s.Put(ctx, key, []byte(`{"n":0}`), ""); err != nil { t.Fatalf("seed Put: %v", err) } _, stale, err := s.Get(ctx, key) if err != nil { t.Fatalf("Get: %v", err) } // Someone else moves the object on. if _, err := s.Put(ctx, key, []byte(`{"n":1,"other":true}`), stale); err != nil { t.Fatalf("first replace: %v", err) } // Our now-stale etag must lose. if _, err := s.Put(ctx, key, []byte(`{"n":2}`), stale); !errors.Is(err, ErrCASConflict) { t.Fatalf("a stale etag must still lose CAS, got %v — the quirk has disabled compare-and-swap", err) } }