// Package s3compat is the one place CARLOS records how a given S3-compatible // store differs from AWS S3, and the only place that knows CARLOS_S3_ENDPOINT. // // Two kinds of adjustment live here. The first is universal — anything that is // not AWS needs a base endpoint, path-style addressing and when-required // checksums (default-on flexible checksums wrap bodies in aws-chunked framing // some compatibles reject). Those were already duplicated in // release.NewS3Store and internal/lease; Options folds them into one. // // The second is per-provider QUIRKS: places a store answers differently enough // that correct S3 code breaks against it. Those go in the quirks table below, // each entry naming the provider it matches, what it changes, and — the part // that matters six months later — the measurement that justifies it. Adding a // provider is one entry; nothing else in the codebase changes. // // Deliberately NOT covered: litestream's replica client, which builds its own // S3 client we cannot decorate. It needs none of the quirks here — it writes // LTX files with unconditional PUTs — and its own provider auto-detection // already handles the universal settings when an Endpoint is set. The leaser // is different: it takes a ready-made client (lss3.Leaser.SetClient), which is // exactly how the If-Match quirk below reaches it. package s3compat import ( "context" "net/url" "os" "strings" "github.com/aws/aws-sdk-go-v2/aws" awsmiddleware "github.com/aws/aws-sdk-go-v2/aws/middleware" awss3 "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/aws/smithy-go/middleware" smithyhttp "github.com/aws/smithy-go/transport/http" ) // Endpoint reports the CARLOS_S3_ENDPOINT override, or "" for stock AWS. Read // per call rather than cached at init so a test — and a process that re-reads // its config — sees the value it just set. func Endpoint() string { return strings.TrimSpace(os.Getenv("CARLOS_S3_ENDPOINT")) } // EndpointHost is the bare host of the override ("" for stock AWS), which is // what quirk matchers key on. A bare host with no scheme still parses: the // value is normalised to a URL first, so both "https://x.example" and // "x.example" match the same way. func EndpointHost() string { ep := Endpoint() if ep == "" { return "" } if !strings.Contains(ep, "://") { ep = "https://" + ep } u, err := url.Parse(ep) if err != nil { return "" } return u.Hostname() } // Options returns the S3 client option every CARLOS S3 client should apply. // With no override it is a NO-OP — the flagship's pure-AWS path must stay // byte-identical, so nothing here may touch a client that talks to real S3. func Options() func(*awss3.Options) { return func(o *awss3.Options) { ep := Endpoint() if ep == "" { return } o.BaseEndpoint = &ep o.UsePathStyle = true o.RequestChecksumCalculation = aws.RequestChecksumCalculationWhenRequired o.ResponseChecksumValidation = aws.ResponseChecksumValidationWhenRequired for _, q := range Quirks { if q.enabled(EndpointHost()) { q.Apply(o) } } } } // Quirk is one provider's documented deviation from S3 and the fix for it. type Quirk struct { // Name is the stable identifier, also the token CARLOS_S3_QUIRKS uses to // force the quirk on for a store whose hostname Match does not know. Name string // Reason is why this exists — ideally the measurement, not a guess. Reason string // Match reports whether the quirk applies to an endpoint host. A nil Match // means "every non-AWS store", which should be rare: prefer naming the // providers actually measured, so an unmeasured store fails loudly in its // own way rather than silently inheriting someone else's workaround. Match func(host string) bool // Apply adjusts the client. Wire-level fixes register SDK middleware here. Apply func(*awss3.Options) } func (q Quirk) enabled(host string) bool { for _, forced := range strings.Split(os.Getenv("CARLOS_S3_QUIRKS"), ",") { if strings.TrimSpace(forced) == q.Name { return true } } if q.Match == nil { return true } return q.Match(host) } // Quirks is the table. Add a provider by adding an entry. var Quirks = []Quirk{unquotedIfMatch} // isHetzner matches Hetzner Object Storage (Ceph RGW), e.g. // nbg1.your-objectstorage.com. Same suffix litestream's own // IsHetznerEndpoint uses. func isHetzner(host string) bool { return strings.HasSuffix(host, ".your-objectstorage.com") } // unquotedIfMatch is the fix for the compare-and-swap failure measured against // Hetzner Object Storage on 2026-08-07. // // RGW compares If-Match against its UNQUOTED internal etag, so the quoted // entity-tag form that S3 returns — and that the AWS SDK faithfully round-trips // — never matches. Measured against the carlos bucket: // // PUT If-Match: "" -> 412 (quoted, as AWS returns it) // PUT If-Match: -> OK (same etag, unquoted) // PUT If-Match: -> 412 (still correctly rejected) // PUT If-None-Match: * (exists) -> 412 (correctly rejected) // // The third line is what makes this safe: the condition IS evaluated, only the // quoting form differs, so stripping the quotes fixes compare-and-swap rather // than disabling it. RFC 9110 entity-tags are quoted, so AWS is right and RGW // is wrong — but CARLOS still has to interoperate. // // Scoped to PutObject on purpose. Conditional DELETE accepts the quoted form // on the same store (it is how the lease releases today, and it works), and a // working path is not worth touching on the strength of an untested guess. // // What it unblocks: every CAS replace in CARLOS — release pointers and // manifests, the ledger head, appconfig generations, provision records, status // snapshots, metrics rollups — plus litestream's lease heartbeat, which renews // with PUT If-Match every TTL/3 and so lost the lease at the first renewal, // standing a woken instance back down five minutes later, every time. var unquotedIfMatch = Quirk{ Name: "unquoted-if-match", Reason: "Ceph RGW (Hetzner) compares If-Match against the unquoted etag; the quoted form always 412s", Match: isHetzner, Apply: func(o *awss3.Options) { o.APIOptions = append(o.APIOptions, unquoteIfMatchMiddleware) }, } // unquoteIfMatchMiddleware strips the surrounding quotes from a PutObject's // If-Match header. // // It runs in the BUILD step, which is after serialization has written the // header and before the Finalize step signs the request — the only window // where the change is both possible and covered by the signature. Rewriting // after signing would produce a SignatureDoesNotMatch instead of a fix. func unquoteIfMatchMiddleware(stack *middleware.Stack) error { return stack.Build.Add(middleware.BuildMiddlewareFunc("carlosUnquoteIfMatch", func(ctx context.Context, in middleware.BuildInput, next middleware.BuildHandler) ( middleware.BuildOutput, middleware.Metadata, error, ) { if awsmiddleware.GetOperationName(ctx) == "PutObject" { if req, ok := in.Request.(*smithyhttp.Request); ok { if v := req.Header.Get("If-Match"); v != "" { if u := strings.Trim(v, `"`); u != v { req.Header.Set("If-Match", u) } } } } return next.HandleBuild(ctx, in) }, ), middleware.After) }