package lease
// CARLOS_S3_ENDPOINT tests. Unlike lease_test.go these need no AWS and no
// CARLOS_TEST_S3_BUCKET: the point of the override is that S3 traffic can be
// pointed anywhere, so the test points it at an httptest server and asserts
// the request actually arrives. A regression here — the leaser silently going
// back to real AWS — is exactly what cost this deployment hibernation and the
// whole `carlos instances create` provisioning path.
import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"sync"
"testing"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/benbjohnson/litestream"
lss3 "github.com/benbjohnson/litestream/s3"
"github.com/carlosframework/platform/internal/rootaws"
)
// fakeS3 is just enough of S3 for the leaser's three verbs (GET/PUT/DELETE on
// one key) to complete a full acquire-then-release, recording every request so
// a test can assert on where the traffic went.
type fakeS3 struct {
mu sync.Mutex
reqs []string // "METHOD /path"
body []byte // last PUT body
}
func (f *fakeS3) record(method, path string) {
f.mu.Lock()
defer f.mu.Unlock()
f.reqs = append(f.reqs, method+" "+path)
}
func (f *fakeS3) requests() []string {
f.mu.Lock()
defer f.mu.Unlock()
return append([]string(nil), f.reqs...)
}
func (f *fakeS3) putBody() []byte {
f.mu.Lock()
defer f.mu.Unlock()
return append([]byte(nil), f.body...)
}
func newFakeS3(t *testing.T) (*fakeS3, string) {
t.Helper()
f := &fakeS3{}
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
f.record(r.Method, r.URL.Path)
switch r.Method {
case http.MethodGet:
// No lease on the key yet. The XML body matters: litestream
// classifies "absent" by the S3 error CODE, and a bare 404 with no
// body would read as a hard failure instead.
w.Header().Set("Content-Type", "application/xml")
w.WriteHeader(http.StatusNotFound)
_, _ = w.Write([]byte(`` +
`NoSuchKeyno such key`))
case http.MethodPut:
b, _ := io.ReadAll(r.Body)
f.mu.Lock()
f.body = b
f.mu.Unlock()
w.Header().Set("ETag", `"lease-etag"`)
w.WriteHeader(http.StatusOK)
case http.MethodDelete:
w.WriteHeader(http.StatusNoContent)
default:
w.WriteHeader(http.StatusMethodNotAllowed)
}
}))
t.Cleanup(srv.Close)
return f, srv.URL
}
// fakeCreds plants credentials the AWS default chain will resolve without
// touching the operator's real profile or IMDS — the same isolation instinct
// as carlosTestEnv, applied to the SDK instead of the carlos binary.
func fakeCreds(t *testing.T) {
t.Helper()
t.Setenv("AWS_ACCESS_KEY_ID", "test-key")
t.Setenv("AWS_SECRET_ACCESS_KEY", "test-secret")
t.Setenv("AWS_SESSION_TOKEN", "")
// Cleared, not left ambient: region resolution is under test, so a region
// inherited from whatever machine runs the suite would make these results
// depend on the box. Tests that care set it back explicitly.
t.Setenv("AWS_REGION", "")
t.Setenv("AWS_DEFAULT_REGION", "")
t.Setenv("AWS_EC2_METADATA_DISABLED", "true")
}
// TestAcquire_ReachesEndpointOverride is the regression test for the bug this
// file exists for: with CARLOS_S3_ENDPOINT set, the lease's lock.json must be
// written through the override — not to real AWS. It asserts on the recorded
// requests, so it fails loudly (rather than hanging on a live AWS dial) if the
// leaser ever stops honouring the endpoint.
func TestAcquire_ReachesEndpointOverride(t *testing.T) {
f, url := newFakeS3(t)
fakeCreds(t)
t.Setenv("CARLOS_S3_ENDPOINT", url)
// Region deliberately empty, and fakeCreds leaves the chain empty too, so
// this also exercises the last resort: a client signing for no region at
// all would fail every request against the fake.
held, err := Acquire(context.Background(), "test-bucket", "inst/example.com", "")
if err != nil {
t.Fatalf("Acquire against %s: %v", url, err)
}
got := f.requests()
// Path-style addressing: bucket in the path, not the hostname — a
// compatible store has no per-bucket virtual host.
wantKey := "/test-bucket/inst/example.com/lock.json"
var sawGet, sawPut bool
for _, r := range got {
switch r {
case "GET " + wantKey:
sawGet = true
case "PUT " + wantKey:
sawPut = true
}
}
// An absent lock is one create-only PUT (createLease); a GET comes
// only when a lock is present. Either way the PUT must arrive here.
_ = sawGet
if !sawPut {
t.Fatalf("lease traffic did not reach the override: got %v, want a PUT of %s", got, wantKey)
}
// The PUT body is the lock itself — proof the acquire really happened
// here rather than the server merely being probed.
var lease litestream.Lease
if err := json.Unmarshal(f.putBody(), &lease); err != nil {
t.Fatalf("decode written lock.json (%q): %v", f.putBody(), err)
}
if lease.Owner == "" {
t.Errorf("written lease carries no owner: %+v", lease)
}
if err := held.Release(context.Background()); err != nil {
t.Fatalf("Release: %v", err)
}
var sawDelete bool
for _, r := range f.requests() {
if r == "DELETE "+wantKey {
sawDelete = true
}
}
if !sawDelete {
t.Errorf("release did not reach the override: got %v", f.requests())
}
}
// TestNewLeaserClient_EndpointConfig pins the client options either side of the
// override, because the no-override case is the flagship's production path and
// must stay byte-identical to what it was before endpoint.go existed.
func TestNewLeaserClient_EndpointConfig(t *testing.T) {
fakeCreds(t)
t.Run("override sets a path-style base endpoint", func(t *testing.T) {
t.Setenv("CARLOS_S3_ENDPOINT", "https://fsn1.your-objectstorage.com")
c, _, err := newLeaserClient(context.Background(), "eu-central-1")
if err != nil {
t.Fatalf("newLeaserClient: %v", err)
}
o := c.Options()
if o.BaseEndpoint == nil || *o.BaseEndpoint != "https://fsn1.your-objectstorage.com" {
t.Errorf("BaseEndpoint = %v, want the override", o.BaseEndpoint)
}
if !o.UsePathStyle {
t.Error("UsePathStyle = false, want true under an endpoint override")
}
})
t.Run("no override leaves stock AWS addressing", func(t *testing.T) {
t.Setenv("CARLOS_S3_ENDPOINT", "")
c, _, err := newLeaserClient(context.Background(), "eu-west-1")
if err != nil {
t.Fatalf("newLeaserClient: %v", err)
}
o := c.Options()
if o.BaseEndpoint != nil {
t.Errorf("BaseEndpoint = %q, want nil (stock AWS)", *o.BaseEndpoint)
}
if o.UsePathStyle {
t.Error("UsePathStyle = true, want false without an override")
}
})
}
// TestNewReplicaClient_EndpointConfig covers the other half of a lease: the
// LTX files beside lock.json. A replica client that ignored the override while
// the leaser honoured it would split one lease across two stores.
func TestNewReplicaClient_EndpointConfig(t *testing.T) {
fakeCreds(t)
t.Run("override", func(t *testing.T) {
t.Setenv("CARLOS_S3_ENDPOINT", "https://fsn1.your-objectstorage.com")
c, err := newReplicaClient("b", "p", "eu-central-1")
if err != nil {
t.Fatal(err)
}
if c.Endpoint != "https://fsn1.your-objectstorage.com" {
t.Errorf("Endpoint = %q, want the override", c.Endpoint)
}
if !c.ForcePathStyle {
t.Error("ForcePathStyle = false, want true under an endpoint override")
}
if c.Bucket != "b" || c.Path != "p" || c.Region != "eu-central-1" {
t.Errorf("bucket/path/region = %q/%q/%q, want b/p/eu-central-1", c.Bucket, c.Path, c.Region)
}
})
t.Run("no override", func(t *testing.T) {
t.Setenv("CARLOS_S3_ENDPOINT", "")
c, err := newReplicaClient("b", "p", "eu-west-1")
if err != nil {
t.Fatal(err)
}
if c.Endpoint != "" || c.ForcePathStyle {
t.Errorf("Endpoint/ForcePathStyle = %q/%v, want empty/false without an override", c.Endpoint, c.ForcePathStyle)
}
})
}
// TestRegionFor pins the order of authority. The case that matters most is
// "chain wins over the non-AWS default": a compatible-store deployment sets
// AWS_REGION to its own location (carlos.exciting.io uses nbg1) and that is
// what release.NewS3Store already signs for against the same bucket —
// substituting a constant would make the lease sign differently from every
// other S3 client in the process.
func TestRegionFor(t *testing.T) {
tests := []struct {
name, endpoint, explicit, chain, want string
}{
{"explicit wins over everything", "https://s3.example.com", "fsn1", "nbg1", "fsn1"},
{"chain wins over the non-AWS default", "https://s3.example.com", "", "nbg1", "nbg1"},
{"non-AWS default only fills a total blank", "https://s3.example.com", "", "", lss3.DefaultRegion},
{"no override leaves a blank blank", "", "", "", ""},
{"no override keeps the chain", "", "", "eu-west-1", "eu-west-1"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
t.Setenv("CARLOS_S3_ENDPOINT", tt.endpoint)
if got := regionFor(tt.explicit, tt.chain); got != tt.want {
t.Errorf("regionFor(%q, %q) with endpoint %q = %q, want %q",
tt.explicit, tt.chain, tt.endpoint, got, tt.want)
}
})
}
}
// TestResolveRegion_PrefersAWSRegionEnv is the end-to-end version of the case
// above, through the real AWS chain: with AWS_REGION set and no explicit
// region, the lease must sign for AWS_REGION — not for litestream's non-AWS
// constant. This is the exact shape of the private CARLOS box's host.env.
func TestResolveRegion_PrefersAWSRegionEnv(t *testing.T) {
fakeCreds(t)
t.Setenv("CARLOS_S3_ENDPOINT", "https://nbg1.your-objectstorage.com")
t.Setenv("AWS_REGION", "nbg1")
got, err := resolveRegion(context.Background(), "")
if err != nil {
t.Fatalf("resolveRegion: %v", err)
}
if got != "nbg1" {
t.Errorf("resolveRegion = %q, want %q (AWS_REGION must outrank the non-AWS default)", got, "nbg1")
}
}
// TestRestoreCreatesTheOutputPathsParentDirectory pins the review fix: a
// nested row's instance directory does not exist before its first wake (it
// is created by the spawn path, which runs after Restore), so Restore must
// create OutputPath's parent itself rather than fail with ENOENT before the
// spawn path ever gets a chance to.
func TestRestoreCreatesTheOutputPathsParentDirectory(t *testing.T) {
fakeCreds(t)
// Every listing this exercises, the v0.3.x/LTX format comparison and
// the LTX restore plan itself, is a ListObjectsV2 call; answering all
// of them with an empty bucket makes Restore take the "no replica"
// path, the one every fresh nested row actually takes on its first wake.
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/xml")
fmt.Fprint(w, `
b0false`)
}))
defer srv.Close()
t.Setenv("CARLOS_S3_ENDPOINT", srv.URL)
root := t.TempDir()
dbPath := filepath.Join(root, "instances", "h.example.com", "h.example.com.db")
if _, err := os.Stat(filepath.Dir(dbPath)); !os.IsNotExist(err) {
t.Fatalf("test setup: parent directory already exists")
}
restored, err := Restore(context.Background(), dbPath, "test-bucket", "inst/h.example.com", "")
if err != nil {
t.Fatalf("Restore into a path whose parent is absent: %v", err)
}
if restored {
t.Fatal("restored = true against an empty bucket, want false (nothing to restore)")
}
if fi, err := os.Stat(filepath.Dir(dbPath)); err != nil || !fi.IsDir() {
t.Fatalf("Restore did not create the output path's parent directory: %v", err)
}
}
// ReadReplicaClient is the db door's reader (spec 2026-09-02 §6). It must
// carry the coordinates it was given and sign for the region lease.Restore
// would: an explicit region wins, and an empty one falls to the same chain
// resolution the writer uses — the endpoint override's non-AWS default when
// nothing else answers.
func TestReadReplicaClientCarriesBucketPrefixRegion(t *testing.T) {
fakeCreds(t)
c, err := ReadReplicaClient(context.Background(), "data-bucket", "instances/a.example.com", "eu-west-1")
if err != nil {
t.Fatal(err)
}
if c.Bucket != "data-bucket" {
t.Errorf("Bucket = %q, want data-bucket", c.Bucket)
}
if c.Path != "instances/a.example.com" {
t.Errorf("Path = %q, want instances/a.example.com", c.Path)
}
if c.Region != "eu-west-1" {
t.Errorf("Region = %q, want eu-west-1", c.Region)
}
}
func TestReadReplicaClientResolvesAnEmptyRegionLikeRestore(t *testing.T) {
fakeCreds(t)
t.Setenv("CARLOS_S3_ENDPOINT", "http://127.0.0.1:1")
c, err := ReadReplicaClient(context.Background(), "data-bucket", "instances/a.example.com", "")
if err != nil {
t.Fatal(err)
}
if c.Region != lss3.DefaultRegion {
t.Errorf("Region = %q, want litestream's non-AWS default %q (the writer's resolution)", c.Region, lss3.DefaultRegion)
}
if c.Endpoint != "http://127.0.0.1:1" || !c.ForcePathStyle {
t.Errorf("Endpoint = %q ForcePathStyle = %v, want the override applied", c.Endpoint, c.ForcePathStyle)
}
}
func TestRootReplicaClientEnvironmentKeepsDefaultChain(t *testing.T) {
for _, region := range []string{"eu-west-1", ""} {
t.Run("region="+region, func(t *testing.T) {
fakeCreds(t)
t.Setenv("AWS_SESSION_TOKEN", "test-session-token")
t.Setenv("AWS_PROFILE", "")
t.Setenv("AWS_DEFAULT_PROFILE", "")
dir := t.TempDir()
marker := filepath.Join(dir, "executed")
config := filepath.Join(dir, "config")
if err := os.WriteFile(config, []byte("[default]\ncredential_process = touch "+marker+"\n"), 0600); err != nil {
t.Fatal(err)
}
t.Setenv("AWS_CONFIG_FILE", config)
t.Setenv("AWS_SHARED_CREDENTIALS_FILE", config)
var mu sync.Mutex
var locations, listings int
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if !strings.Contains(r.Header.Get("Authorization"), "Credential=test-key/") || r.Header.Get("X-Amz-Security-Token") != "test-session-token" {
t.Error("request did not carry the explicit keys and session token")
}
w.Header().Set("Content-Type", "application/xml")
mu.Lock()
defer mu.Unlock()
if r.URL.Query().Has("location") {
locations++
fmt.Fprint(w, `eu-west-1`)
} else {
listings++
fmt.Fprint(w, `b0false`)
}
}))
defer srv.Close()
// Keep Endpoint empty to exercise Litestream's bucket-region lookup.
t.Setenv("CARLOS_S3_ENDPOINT", "")
t.Setenv("AWS_ENDPOINT_URL_S3", srv.URL)
c, err := newReplicaClientForUID("b", "p", region, 0, rootaws.OffcloudCredentials)
if err != nil {
t.Fatal(err)
}
c.ForcePathStyle = true
if c.AccessKeyID != "" || c.SecretAccessKey != "" {
t.Fatal("environment credentials must use the upstream default chain")
}
if _, err := c.GenerationsV3(context.Background()); err != nil {
t.Fatal(err)
}
mu.Lock()
defer mu.Unlock()
if listings == 0 || (region == "" && locations != 1) || (region != "" && locations != 0) {
t.Fatalf("locations=%d listings=%d for region %q", locations, listings, region)
}
if _, err := os.Stat(marker); !os.IsNotExist(err) {
t.Fatal("credential_process executed")
}
})
}
}
func TestNonRootReplicaClientKeepsDefaultProvider(t *testing.T) {
c, err := newReplicaClientForUID("b", "p", "eu-west-1", 1001, func() (aws.Credentials, error) {
t.Fatal("non-root read root credentials")
return aws.Credentials{}, nil
})
if err != nil {
t.Fatal(err)
}
if c.AccessKeyID != "" || c.SecretAccessKey != "" {
t.Fatal("non-root credential selection changed")
}
}
func TestRootReplicaClientOffcloudCredentials(t *testing.T) {
for _, token := range []string{"", "FILE_TOKEN"} {
t.Run("token="+token, func(t *testing.T) {
body := "[default]\naws_access_key_id=FILE_ACCESS\naws_secret_access_key=FILE_SECRET\n"
if token != "" {
body += "aws_session_token=" + token + "\n"
}
c, err := newReplicaClientForUID("b", "p", "eu-west-1", 0, func() (aws.Credentials, error) {
return rootaws.ParseStaticCredentials([]byte(body))
})
if token != "" {
if !errors.Is(err, errRootReplicaSessionToken) || c != nil {
t.Fatalf("token-bearing file: client=%v err=%v", c, err)
}
return
}
if err != nil {
t.Fatal(err)
}
if c.AccessKeyID != "FILE_ACCESS" || c.SecretAccessKey != "FILE_SECRET" {
t.Fatal("off-cloud file keys were not set on the upstream client")
}
})
}
}
func TestRootReplicaClientWithoutOffcloudKeepsDefaultChain(t *testing.T) {
c, err := newReplicaClientForUID("b", "p", "eu-west-1", 0, func() (aws.Credentials, error) {
return aws.Credentials{}, nil
})
if err != nil {
t.Fatal(err)
}
if c.AccessKeyID != "" || c.SecretAccessKey != "" {
t.Fatal("instance-profile credentials must use the upstream default chain")
}
}
func TestRootReplicaClientRefusesUnsafeOffcloudCredentials(t *testing.T) {
refused := errors.New("unsafe file")
c, err := newReplicaClientForUID("b", "p", "eu-west-1", 0, func() (aws.Credentials, error) {
return aws.Credentials{}, refused
})
if !errors.Is(err, refused) || c != nil {
t.Fatalf("unsafe file: client=%v err=%v", c, err)
}
}