package edge
import (
"bufio"
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"slices"
"strings"
"sync/atomic"
"syscall"
"testing"
"time"
"github.com/carlosframework/platform/internal/provision"
"github.com/carlosframework/platform/internal/registry"
"github.com/carlosframework/platform/internal/release"
)
func openReg(t *testing.T) *registry.Registry {
t.Helper()
reg, err := registry.Open(filepath.Join(t.TempDir(), "registry.db"))
if err != nil {
t.Fatalf("open registry: %v", err)
}
return reg
}
// hostEchoBackend is an upstream that records the Host header it saw and
// answers a fixed body — enough to prove both routing and Host preservation.
func hostEchoBackend(t *testing.T, body string) (*httptest.Server, *string) {
t.Helper()
seenHost := new(string)
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
*seenHost = r.Host
io.WriteString(w, body)
}))
t.Cleanup(ts.Close)
return ts, seenHost
}
// getHost GETs the edge server with an explicit Host header, the way a real
// client reaches a routed hostname.
func getHost(t *testing.T, edgeURL, host string) (int, string) {
t.Helper()
req, err := http.NewRequest("GET", edgeURL+"/", nil)
if err != nil {
t.Fatalf("new request: %v", err)
}
req.Host = host
resp, err := http.DefaultClient.Do(req)
if err != nil {
t.Fatalf("GET %s (host %s): %v", edgeURL, host, err)
}
defer resp.Body.Close()
b, err := io.ReadAll(resp.Body)
if err != nil {
t.Fatalf("read body: %v", err)
}
return resp.StatusCode, string(b)
}
func TestRoutesHostToTCPUpstream(t *testing.T) {
backend, seenHost := hostEchoBackend(t, "hello from backend")
addr := backend.Listener.Addr().String()
reg := openReg(t)
if err := reg.Upsert(registry.Route{Host: "app.example.com", Kind: "service", Addr: addr, Status: "ready"}); err != nil {
t.Fatalf("upsert: %v", err)
}
e, err := New(reg, nil)
if err != nil {
t.Fatalf("New: %v", err)
}
srv := httptest.NewServer(e)
defer srv.Close()
code, body := getHost(t, srv.URL, "app.example.com")
if code != http.StatusOK {
t.Fatalf("status = %d, want 200", code)
}
if body != "hello from backend" {
t.Fatalf("body = %q, want backend body", body)
}
if *seenHost != "app.example.com" {
t.Fatalf("backend saw Host %q, want app.example.com (inbound Host must be preserved)", *seenHost)
}
if !e.Allowed("app.example.com") {
t.Fatalf("Allowed(app.example.com) = false, want true")
}
}
func TestSkipsNotReadyRoutes(t *testing.T) {
backend, _ := hostEchoBackend(t, "should never be reached")
addr := backend.Listener.Addr().String()
reg := openReg(t)
if err := reg.Upsert(registry.Route{Host: "pending.example.com", Kind: "service", Addr: addr, Status: "pending"}); err != nil {
t.Fatalf("upsert: %v", err)
}
e, err := New(reg, nil)
if err != nil {
t.Fatalf("New: %v", err)
}
srv := httptest.NewServer(e)
defer srv.Close()
code, _ := getHost(t, srv.URL, "pending.example.com")
if code != http.StatusNotFound {
t.Fatalf("status = %d, want 404 for a not-ready route", code)
}
// Allowed feeds the autocert HostPolicy: a not-ready route must not mint
// a certificate either.
if e.Allowed("pending.example.com") {
t.Fatalf("Allowed(pending.example.com) = true, want false")
}
}
func TestReloadPicksUpNewRoute(t *testing.T) {
backend, _ := hostEchoBackend(t, "late arrival")
addr := backend.Listener.Addr().String()
reg := openReg(t)
e, err := New(reg, nil)
if err != nil {
t.Fatalf("New: %v", err)
}
srv := httptest.NewServer(e)
defer srv.Close()
if code, _ := getHost(t, srv.URL, "new.example.com"); code != http.StatusNotFound {
t.Fatalf("pre-reload status = %d, want 404", code)
}
// Write via a SECOND handle on the same file — the cross-process shape
// (`carlos add` while the edge runs) that exposed the stale-WAL quirk.
reg2, err := registry.Open(reg.Path())
if err != nil {
t.Fatalf("open second handle: %v", err)
}
if err := reg2.Upsert(registry.Route{Host: "new.example.com", Kind: "service", Addr: addr, Status: "ready"}); err != nil {
t.Fatalf("upsert via second handle: %v", err)
}
if err := e.Reload(); err != nil {
t.Fatalf("Reload: %v", err)
}
code, body := getHost(t, srv.URL, "new.example.com")
if code != http.StatusOK || body != "late arrival" {
t.Fatalf("post-reload = %d %q, want 200 %q", code, body, "late arrival")
}
}
func TestUnixSocketUpstream(t *testing.T) {
sock := filepath.Join(t.TempDir(), "u.sock")
ln, err := net.Listen("unix", sock)
if err != nil {
// Sandboxed environments block unix-socket listeners (EPERM); that is
// an environment limit, not an edge bug.
if errors.Is(err, syscall.EPERM) || strings.Contains(err.Error(), "operation not permitted") {
t.Skipf("unix-socket listeners not permitted in this environment: %v", err)
}
t.Fatalf("listen unix: %v", err)
}
seenHost := new(string)
backend := &httptest.Server{
Listener: ln,
Config: &http.Server{Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
*seenHost = r.Host
io.WriteString(w, "over the socket")
})},
}
backend.Start()
defer backend.Close()
reg := openReg(t)
if err := reg.Upsert(registry.Route{Host: "sock.example.com", Kind: "instance", Socket: sock, Status: "ready"}); err != nil {
t.Fatalf("upsert: %v", err)
}
e, err := New(reg, nil)
if err != nil {
t.Fatalf("New: %v", err)
}
srv := httptest.NewServer(e)
defer srv.Close()
code, body := getHost(t, srv.URL, "sock.example.com")
if code != http.StatusOK {
t.Fatalf("status = %d, want 200", code)
}
if body != "over the socket" {
t.Fatalf("body = %q, want socket backend body", body)
}
if *seenHost != "sock.example.com" {
t.Fatalf("backend saw Host %q, want sock.example.com", *seenHost)
}
}
func TestFirstFingerprintObservationReloads(t *testing.T) {
backend, _ := hostEchoBackend(t, "seen on first poll")
addr := backend.Listener.Addr().String()
// Snapshot BEFORE the route exists: New's startup reload sees zero routes.
reg := openReg(t)
e, err := New(reg, nil)
if err != nil {
t.Fatalf("New: %v", err)
}
srv := httptest.NewServer(e)
defer srv.Close()
// The route lands after the startup reload but before the watcher starts —
// exactly the write a baseline-only first tick would silently absorb.
reg2, err := registry.Open(reg.Path())
if err != nil {
t.Fatalf("open second handle: %v", err)
}
if err := reg2.Upsert(registry.Route{Host: "first.example.com", Kind: "service", Addr: addr, Status: "ready"}); err != nil {
t.Fatalf("upsert: %v", err)
}
stop := make(chan struct{})
defer close(stop)
go e.Watch(stop)
// No second write: the FIRST observed fingerprint alone must reload.
deadline := time.Now().Add(6 * time.Second)
for time.Now().Before(deadline) {
if code, body := getHost(t, srv.URL, "first.example.com"); code == http.StatusOK {
if body != "seen on first poll" {
t.Fatalf("body = %q, want backend body", body)
}
return
}
time.Sleep(100 * time.Millisecond)
}
t.Fatalf("route never served: first fingerprint observation was baselined away instead of reloading")
}
// fakeActivator records the hibernation hook's calls.
type fakeActivator struct {
served int
retired []map[string]bool
}
func (f *fakeActivator) ServeHTTP(w http.ResponseWriter, r *http.Request, rt registry.Route, proxy http.Handler) bool {
f.served++
proxy.ServeHTTP(w, r) // "wake" is instant; proxy straight through
return true
}
func (f *fakeActivator) RetireExcept(keep map[string]bool) {
f.retired = append(f.retired, keep)
}
// testEdgeWithStore builds an Edge with one route already loaded and static
// serving (the plumbing kind=status shares — see EnableStatic's doc comment)
// wired to a fresh, empty DirStore. It returns the store (so a test can Put
// bucket objects straight into it) and the Edge itself as an http.Handler:
// Edge.ServeHTTP dispatches on the request's Host header, so tests drive it
// with httptest.NewRecorder + httptest.NewRequest — no real listener, hence
// no unix-socket/EPERM sandbox concerns (contrast TestUnixSocketUpstream).
func testEdgeWithStore(t *testing.T, rt registry.Route) (release.Store, http.Handler) {
t.Helper()
reg := openReg(t)
if err := reg.Upsert(rt); err != nil {
t.Fatalf("upsert: %v", err)
}
e, err := New(reg, nil)
if err != nil {
t.Fatalf("New: %v", err)
}
store := release.NewDirStore(filepath.Join(t.TempDir(), "store"))
// The status class doesn't verify signatures (unlike static), but
// EnableStatic's signature still asks for a pubkey — any well-formed one
// works since it's never used on this path.
_, pubB64, err := release.GenerateKey()
if err != nil {
t.Fatalf("generate key: %v", err)
}
pub, err := release.ParsePublicKey(pubB64, "test")
if err != nil {
t.Fatalf("parse public key: %v", err)
}
e.EnableStatic(store, pub, filepath.Join(t.TempDir(), "cache"))
if err := e.Reload(); err != nil {
t.Fatalf("Reload: %v", err)
}
return store, e
}
func TestStatusRouteServesBucketBacked(t *testing.T) {
s, h := testEdgeWithStore(t, registry.Route{Host: "status.test", Kind: "status", Status: "ready"})
s.Put(context.Background(), "status/fleet.json", []byte(`{"generated_at":"x"}`), "")
s.Put(context.Background(), "status/index.html", []byte("
Fleet status"), "")
for path, want := range map[string]string{"/": "Fleet status", "/fleet.json": "generated_at"} {
rec := httptest.NewRecorder()
h.ServeHTTP(rec, httptest.NewRequest("GET", "http://status.test"+path, nil))
if rec.Code != 200 || !strings.Contains(rec.Body.String(), want) {
t.Errorf("%s: %d %q", path, rec.Code, rec.Body.String())
}
}
rec := httptest.NewRecorder()
h.ServeHTTP(rec, httptest.NewRequest("GET", "http://status.test/boxes/anything", nil))
if rec.Code != 404 {
t.Errorf("non-allowlisted path must 404, got %d", rec.Code)
}
}
// TestStatusRouteContentTypes pins the three content types the brief names:
// a client (or the console iframe embedding the page) depends on these to
// render the HTML instead of downloading it, parse the JSON, and apply the
// stylesheet.
func TestStatusRouteContentTypes(t *testing.T) {
s, h := testEdgeWithStore(t, registry.Route{Host: "status.test", Kind: "status", Status: "ready"})
s.Put(context.Background(), "status/index.html", []byte(""), "")
s.Put(context.Background(), "status/fleet.json", []byte(`{}`), "")
s.Put(context.Background(), "status/status.css", []byte("body{}"), "")
for path, wantPrefix := range map[string]string{
"/": "text/html",
"/fleet.json": "application/json",
"/status.css": "text/css",
} {
rec := httptest.NewRecorder()
h.ServeHTTP(rec, httptest.NewRequest("GET", "http://status.test"+path, nil))
if ct := rec.Header().Get("Content-Type"); !strings.HasPrefix(ct, wantPrefix) {
t.Errorf("%s: Content-Type = %q, want prefix %q", path, ct, wantPrefix)
}
}
}
// TestStatusRouteCacheServesStaleOnStoreError proves the two halves of the
// per-key cache's contract: inside the TTL, a mutated store object is NOT
// picked up (the cache, not the bucket, is the source of truth); after the
// TTL, a store error serves the last-fetched copy rather than failing the
// request — the status page outliving a bucket blip is the point (see the
// statusHandler doc comment in edge.go).
func TestStatusRouteCacheServesStaleOnStoreError(t *testing.T) {
rt := registry.Route{Host: "status.test", Kind: "status", Status: "ready"}
reg := openReg(t)
if err := reg.Upsert(rt); err != nil {
t.Fatalf("upsert: %v", err)
}
e, err := New(reg, nil)
if err != nil {
t.Fatalf("New: %v", err)
}
storeDir := filepath.Join(t.TempDir(), "store")
store := release.NewDirStore(storeDir)
_, pubB64, err := release.GenerateKey()
if err != nil {
t.Fatalf("generate key: %v", err)
}
pub, err := release.ParsePublicKey(pubB64, "test")
if err != nil {
t.Fatalf("parse public key: %v", err)
}
e.EnableStatic(store, pub, filepath.Join(t.TempDir(), "cache"))
if err := e.Reload(); err != nil {
t.Fatalf("Reload: %v", err)
}
if _, err := store.Put(context.Background(), "status/fleet.json", []byte("first"), ""); err != nil {
t.Fatalf("seed put: %v", err)
}
sh := e.statusHandlerFor(rt).(*statusHandler)
sh.ttl = 20 * time.Millisecond
fakeNow := time.Now()
sh.now = func() time.Time { return fakeNow }
rec := httptest.NewRecorder()
sh.ServeHTTP(rec, httptest.NewRequest("GET", "http://status.test/fleet.json", nil))
if rec.Code != 200 || rec.Body.String() != "first" {
t.Fatalf("first fetch = %d %q, want 200 %q", rec.Code, rec.Body.String(), "first")
}
// Mutate the store underneath the cache, but stay inside the TTL: the
// cached copy must still be served, proving the cache isn't a pass-through.
if _, err := store.Put(context.Background(), "status/fleet.json", []byte("second"), etagOfFile(t, storeDir, "status/fleet.json")); err != nil {
t.Fatalf("mutate put: %v", err)
}
rec = httptest.NewRecorder()
sh.ServeHTTP(rec, httptest.NewRequest("GET", "http://status.test/fleet.json", nil))
if rec.Code != 200 || rec.Body.String() != "first" {
t.Fatalf("inside-TTL fetch = %d %q, want cached %q", rec.Code, rec.Body.String(), "first")
}
// Advance the clock seam past the TTL: the handler must re-read the store
// and pick up the mutation.
fakeNow = fakeNow.Add(21 * time.Millisecond)
rec = httptest.NewRecorder()
sh.ServeHTTP(rec, httptest.NewRequest("GET", "http://status.test/fleet.json", nil))
if rec.Code != 200 || rec.Body.String() != "second" {
t.Fatalf("post-TTL fetch = %d %q, want refreshed %q", rec.Code, rec.Body.String(), "second")
}
// Now break the store (remove the underlying file, simulating a bucket
// error) and advance past the TTL again: with a stale copy in hand, the
// handler must keep serving it rather than 503ing.
fakeNow = fakeNow.Add(21 * time.Millisecond)
if err := os.Remove(filepath.Join(storeDir, "status", "fleet.json")); err != nil {
t.Fatalf("remove backing file: %v", err)
}
rec = httptest.NewRecorder()
sh.ServeHTTP(rec, httptest.NewRequest("GET", "http://status.test/fleet.json", nil))
if rec.Code != 200 || rec.Body.String() != "second" {
t.Fatalf("stale-on-error fetch = %d %q, want stale %q", rec.Code, rec.Body.String(), "second")
}
}
// TestStatusRouteEmptyStoreServes503 covers the honest-failure half: a key
// that has NEVER been successfully fetched, hit with a store error (here:
// the object was never published), must 503 rather than fabricate a body.
func TestStatusRouteEmptyStoreServes503(t *testing.T) {
_, h := testEdgeWithStore(t, registry.Route{Host: "status.test", Kind: "status", Status: "ready"})
rec := httptest.NewRecorder()
h.ServeHTTP(rec, httptest.NewRequest("GET", "http://status.test/fleet.json", nil))
if rec.Code != http.StatusServiceUnavailable {
t.Fatalf("empty-store fetch = %d, want 503", rec.Code)
}
}
// etagOfFile reads a DirStore-backed key straight off disk to recover its
// current etag for a replace-only Put — the same read-etag-then-Put dance
// internal/status/fleet.go's putOverwrite does against the real interface;
// done directly here since the test wants to mutate underneath an existing
// cache, not through it.
func etagOfFile(t *testing.T, root, key string) string {
t.Helper()
_, etag, err := release.NewDirStore(root).Get(context.Background(), key)
if err != nil {
t.Fatalf("read etag for %s: %v", key, err)
}
return etag
}
func TestHibernatingRouteGoesThroughActivator(t *testing.T) {
backend, _ := hostEchoBackend(t, "woken")
addr := backend.Listener.Addr().String()
reg := openReg(t)
if err := reg.Upsert(registry.Route{Host: "hib.example.com", Kind: "instance", Addr: addr, Status: "ready", Hibernate: true}); err != nil {
t.Fatalf("upsert: %v", err)
}
act := &fakeActivator{}
e, err := New(reg, act)
if err != nil {
t.Fatalf("New: %v", err)
}
srv := httptest.NewServer(e)
defer srv.Close()
code, body := getHost(t, srv.URL, "hib.example.com")
if code != http.StatusOK || body != "woken" {
t.Fatalf("got %d %q, want 200 woken", code, body)
}
if act.served != 1 {
t.Fatalf("activator ServeHTTP called %d times, want 1", act.served)
}
// Reload reconciles the activator with the new route set.
if len(act.retired) == 0 {
t.Fatalf("RetireExcept never called on reload")
}
last := act.retired[len(act.retired)-1]
if !last["hib.example.com"] {
t.Fatalf("RetireExcept keep-set %v missing hib.example.com", last)
}
}
// refusingActivator fails the test immediately if ever invoked — used to
// prove a cache hit never reaches the activator at all.
type refusingActivator struct{ t *testing.T }
func (r refusingActivator) ServeHTTP(w http.ResponseWriter, req *http.Request, rt registry.Route, proxy http.Handler) bool {
r.t.Fatal("activator invoked on what should have been a cache hit — this defeats the whole point of caching")
return true
}
func (r refusingActivator) RetireExcept(keep map[string]bool) {}
func TestServeHTTPCacheHitNeverInvokesActivator(t *testing.T) {
reg := openReg(t)
// Addr is deliberately a bogus, never-dialed address: if the cache
// check is ever bypassed, proxying to it fails loudly instead of
// silently succeeding, which would mask the bug.
if err := reg.Upsert(registry.Route{Host: "cached.example.com", Kind: "instance", Addr: "127.0.0.1:1", Status: "ready", Hibernate: true}); err != nil {
t.Fatalf("upsert: %v", err)
}
e, err := New(reg, refusingActivator{t: t})
if err != nil {
t.Fatalf("New: %v", err)
}
e.EnableCache(NewCache(DefaultCacheMaxEntries, DefaultCacheMaxBodyBytes))
key := CacheKey{Host: "cached.example.com", Method: "GET", Path: "/event"}
e.cache.Put(key, "", &CachedResponse{
StatusCode: 200,
Header: http.Header{"Content-Type": {"text/html"}},
Body: []byte("cached page"),
StoredAt: time.Now(),
TTL: time.Minute,
})
rec := httptest.NewRecorder()
e.ServeHTTP(rec, httptest.NewRequest("GET", "http://cached.example.com/event", nil))
if rec.Code != 200 {
t.Fatalf("status = %d, want 200", rec.Code)
}
if rec.Body.String() != "cached page" {
t.Fatalf("body = %q, want %q", rec.Body.String(), "cached page")
}
}
func TestServeHTTPCacheMissPopulatesCacheFromResponse(t *testing.T) {
backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Cache-Control", "max-age=60")
w.WriteHeader(200)
w.Write([]byte("fresh page"))
}))
t.Cleanup(backend.Close)
reg := openReg(t)
if err := reg.Upsert(registry.Route{Host: "miss.example.com", Kind: "instance", Addr: backend.Listener.Addr().String(), Status: "ready"}); err != nil {
t.Fatalf("upsert: %v", err)
}
e, err := New(reg, nil)
if err != nil {
t.Fatalf("New: %v", err)
}
e.EnableCache(NewCache(DefaultCacheMaxEntries, DefaultCacheMaxBodyBytes))
rec1 := httptest.NewRecorder()
e.ServeHTTP(rec1, httptest.NewRequest("GET", "http://miss.example.com/event", nil))
if rec1.Body.String() != "fresh page" {
t.Fatalf("first response body = %q, want %q", rec1.Body.String(), "fresh page")
}
// Close the backend so a second real request would fail to dial —
// proves the second request is answered from cache, not the backend.
backend.Close()
rec2 := httptest.NewRecorder()
e.ServeHTTP(rec2, httptest.NewRequest("GET", "http://miss.example.com/event", nil))
if rec2.Body.String() != "fresh page" {
t.Fatalf("second response body = %q, want %q (should be served from cache)", rec2.Body.String(), "fresh page")
}
}
func TestServeHTTPNeverCachesNon2xx(t *testing.T) {
backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Cache-Control", "max-age=60")
w.WriteHeader(500)
w.Write([]byte("boom"))
}))
t.Cleanup(backend.Close)
reg := openReg(t)
if err := reg.Upsert(registry.Route{Host: "err.example.com", Kind: "instance", Addr: backend.Listener.Addr().String(), Status: "ready"}); err != nil {
t.Fatalf("upsert: %v", err)
}
e, err := New(reg, nil)
if err != nil {
t.Fatalf("New: %v", err)
}
e.EnableCache(NewCache(DefaultCacheMaxEntries, DefaultCacheMaxBodyBytes))
e.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest("GET", "http://err.example.com/event", nil))
if _, ok := e.cache.get(CacheKey{Host: "err.example.com", Method: "GET", Path: "/event"}); ok {
t.Fatal("a 500 response must never be cached")
}
}
// varyBackendEdge wires an Edge in front of a backend that echoes the request's
// Accept-Language and counts its own invocations — the counter is what proves a
// second request was answered from cache rather than re-proxied, which asserting
// on response bodies alone cannot do (a miss echoes the same body a hit would).
func varyBackendEdge(t *testing.T, host string, hits *atomic.Int64) *Edge {
t.Helper()
backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
hits.Add(1)
w.Header().Set("Cache-Control", "max-age=60")
w.Header().Set("Vary", "Accept-Language")
w.WriteHeader(200)
w.Write([]byte("lang=" + r.Header.Get("Accept-Language")))
}))
t.Cleanup(backend.Close)
reg := openReg(t)
if err := reg.Upsert(registry.Route{Host: host, Kind: "instance", Addr: backend.Listener.Addr().String(), Status: "ready"}); err != nil {
t.Fatalf("upsert: %v", err)
}
e, err := New(reg, nil)
if err != nil {
t.Fatalf("New: %v", err)
}
e.EnableCache(NewCache(DefaultCacheMaxEntries, DefaultCacheMaxBodyBytes))
return e
}
// getLang issues a GET carrying Accept-Language: lang (omitted entirely when
// lang is "") and returns the body.
func getLang(t *testing.T, e *Edge, url, lang string) string {
t.Helper()
req := httptest.NewRequest("GET", url, nil)
if lang != "" {
req.Header.Set("Accept-Language", lang)
}
rec := httptest.NewRecorder()
e.ServeHTTP(rec, req)
return rec.Body.String()
}
// A Vary-carrying response must be both separated per variant AND actually
// re-served from cache on a repeat request. The backend-invocation count is
// the load-bearing assertion: pre-fix, ServeHTTP looked entries up with an
// empty VaryValue while maybeStore stored them under varyValue(request, vary),
// so every request was a miss — 4 backend hits, not 2 — while the bodies still
// differed per language, which is why the old body-comparison assertion passed
// with Vary support entirely broken.
func TestServeHTTPVaryAcceptLanguageProducesSeparateEntries(t *testing.T) {
var hits atomic.Int64
e := varyBackendEdge(t, "i18n.example.com", &hits)
const url = "http://i18n.example.com/event"
for _, want := range []struct{ lang, body string }{
{"en", "lang=en"}, {"en", "lang=en"}, {"fr", "lang=fr"}, {"fr", "lang=fr"},
} {
if got := getLang(t, e, url, want.lang); got != want.body {
t.Fatalf("Accept-Language %q: body = %q, want %q", want.lang, got, want.body)
}
}
if n := hits.Load(); n != 2 {
t.Fatalf("backend invoked %d times across 4 requests (en, en, fr, fr), want 2 — the repeat of each language must be served from cache", n)
}
}
// The core regression: a repeat of an identical request against a
// Vary-carrying response must hit. Here no Accept-Language is sent at all, so
// both requests resolve to the same (empty-valued) variant.
func TestServeHTTPVaryRepeatIdenticalRequestHitsCache(t *testing.T) {
var hits atomic.Int64
e := varyBackendEdge(t, "repeat.example.com", &hits)
const url = "http://repeat.example.com/event"
if got := getLang(t, e, url, ""); got != "lang=" {
t.Fatalf("first body = %q, want %q", got, "lang=")
}
if got := getLang(t, e, url, ""); got != "lang=" {
t.Fatalf("second body = %q, want %q", got, "lang=")
}
if n := hits.Load(); n != 1 {
t.Fatalf("backend invoked %d times across 2 identical requests, want 1 — the second must be a cache hit", n)
}
}
// A crawler pumping distinct Vary values at one URL must not be able to fill
// the cache with unreachable variants: the per-URL cap bounds how many a
// single URL can occupy, evicting the oldest as new ones arrive.
func TestServeHTTPVaryVariantCapBoundsEntriesPerURL(t *testing.T) {
var hits atomic.Int64
e := varyBackendEdge(t, "flood.example.com", &hits)
const url = "http://flood.example.com/event"
langs := make([]string, 4*maxVaryVariantsPerURL)
for i := range langs {
langs[i] = fmt.Sprintf("xx-%d", i)
getLang(t, e, url, langs[i])
}
uk := CacheKey{Host: "flood.example.com", Method: "GET", Path: "/event"}.urlKey()
e.cache.mu.Lock()
n := len(e.cache.byURL[uk])
e.cache.mu.Unlock()
if n > maxVaryVariantsPerURL {
t.Fatalf("one URL holds %d cached variants, want at most %d", n, maxVaryVariantsPerURL)
}
// The newest variant survives; the oldest was evicted to make room.
before := hits.Load()
getLang(t, e, url, langs[len(langs)-1])
if hits.Load() != before {
t.Fatal("the most recent Vary variant should still be cached")
}
getLang(t, e, url, langs[0])
if hits.Load() != before+1 {
t.Fatal("the oldest Vary variant should have been evicted by the per-URL cap")
}
}
// cacheProbeEdge wires an Edge whose backend answers with headers chosen per
// path by hdr — the shape every "must not be cached" rule test needs.
func cacheProbeEdge(t *testing.T, host string, hdr func(path string, h http.Header)) *Edge {
t.Helper()
backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
hdr(r.URL.Path, w.Header())
w.WriteHeader(200)
w.Write([]byte("body for " + r.URL.Path))
}))
t.Cleanup(backend.Close)
reg := openReg(t)
if err := reg.Upsert(registry.Route{Host: host, Kind: "instance", Addr: backend.Listener.Addr().String(), Status: "ready"}); err != nil {
t.Fatalf("upsert: %v", err)
}
e, err := New(reg, nil)
if err != nil {
t.Fatalf("New: %v", err)
}
e.EnableCache(NewCache(DefaultCacheMaxEntries, DefaultCacheMaxBodyBytes))
return e
}
func cachedAt(e *Edge, host, path string) bool {
_, ok := e.cache.get(CacheKey{Host: host, Method: "GET", Path: path})
return ok
}
// Cache-Control: private means "for one visitor only" — caching it at the
// shared edge and replaying it to everyone else is a cross-visitor leak.
func TestServeHTTPRefusesCacheControlPrivate(t *testing.T) {
e := cacheProbeEdge(t, "priv.example.com", func(path string, h http.Header) {
if path == "/private" {
h.Set("Cache-Control", "private, max-age=600")
return
}
h.Set("Cache-Control", "max-age=600")
})
e.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest("GET", "http://priv.example.com/private", nil))
e.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest("GET", "http://priv.example.com/shared", nil))
if cachedAt(e, "priv.example.com", "/private") {
t.Fatal("a Cache-Control: private response must never be cached at the shared edge")
}
if !cachedAt(e, "priv.example.com", "/shared") {
t.Fatal("an ordinary max-age response must still be cached")
}
}
// A Set-Cookie response carries per-visitor state (a session, a CSRF token);
// caching it would hand that cookie to every later visitor.
func TestServeHTTPRefusesResponseWithSetCookie(t *testing.T) {
e := cacheProbeEdge(t, "cookie2.example.com", func(path string, h http.Header) {
h.Set("Cache-Control", "max-age=600")
if path == "/session" {
h.Set("Set-Cookie", "sid=abc123; Path=/; HttpOnly")
}
})
e.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest("GET", "http://cookie2.example.com/session", nil))
e.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest("GET", "http://cookie2.example.com/plain", nil))
if cachedAt(e, "cookie2.example.com", "/session") {
t.Fatal("a response carrying Set-Cookie must never be cached — the cookie would be replayed to other visitors")
}
if !cachedAt(e, "cookie2.example.com", "/plain") {
t.Fatal("a response without Set-Cookie must still be cached")
}
}
// A response to an Authorization-bearing request is presumed private (RFC 9111
// §3.5) unless the app explicitly marks it public.
func TestServeHTTPRefusesAuthorizedRequestUnlessPublic(t *testing.T) {
e := cacheProbeEdge(t, "auth.example.com", func(path string, h http.Header) {
if path == "/public" {
h.Set("Cache-Control", "public, max-age=600")
return
}
h.Set("Cache-Control", "max-age=600")
})
authGet := func(path string) {
req := httptest.NewRequest("GET", "http://auth.example.com"+path, nil)
req.Header.Set("Authorization", "Bearer secret-token")
e.ServeHTTP(httptest.NewRecorder(), req)
}
authGet("/account")
authGet("/public")
e.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest("GET", "http://auth.example.com/anon", nil))
if cachedAt(e, "auth.example.com", "/account") {
t.Fatal("a response to an Authorization-bearing request must not be cached without an explicit Cache-Control: public")
}
if !cachedAt(e, "auth.example.com", "/public") {
t.Fatal("Cache-Control: public overrides the Authorization concern (RFC 9111 §3.5) — this response should be cached")
}
if !cachedAt(e, "auth.example.com", "/anon") {
t.Fatal("an unauthenticated request's ordinary max-age response must still be cached")
}
}
// An app asking for a year of freshness would make a single lost purge
// effectively permanent; the edge clamps every TTL to its own ceiling so a
// missed purge always self-heals within a bounded window (spec decision 4).
func TestServeHTTPClampsMaxAgeToCeiling(t *testing.T) {
e := cacheProbeEdge(t, "ttl.example.com", func(path string, h http.Header) {
h.Set("Cache-Control", "max-age=31536000")
})
e.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest("GET", "http://ttl.example.com/event", nil))
resp, ok := e.cache.get(CacheKey{Host: "ttl.example.com", Method: "GET", Path: "/event"})
if !ok {
t.Fatal("expected the response to be cached")
}
if resp.TTL != DefaultCacheMaxTTL {
t.Fatalf("cached TTL = %s, want it clamped to %s", resp.TTL, DefaultCacheMaxTTL)
}
}
// RFC 9111 §5.2.2.10: s-maxage overrides max-age for a SHARED cache, and a
// private cache ignores it. That pair is the only way an origin can say "hold
// this at the edge for two minutes, and have every browser revalidate" — which
// is what makes a purge effective end to end rather than leaving a client-side
// tail the purge channel cannot reach. Reading max-age alone turned exactly
// that header ("s-maxage=120, max-age=0") into "don't cache at all", so the
// origins trying hardest to stay purgeable were the ones that silently lost
// edge caching (issue #403; tito PRO-4758/PRO-4759).
func TestServeHTTPPrefersSMaxageOverMaxAge(t *testing.T) {
cases := []struct {
path string
cc []string // Cache-Control, one entry per header line
ttl time.Duration // 0 = must not be cached at all
}{
{"/pair", []string{"s-maxage=120, max-age=0"}, 120 * time.Second},
{"/pair-reversed", []string{"max-age=0, s-maxage=120"}, 120 * time.Second},
{"/pair-split", []string{"s-maxage=120", "max-age=0"}, 120 * time.Second},
{"/pair-split-reversed", []string{"max-age=0", "s-maxage=120"}, 120 * time.Second},
{"/shared-only", []string{"s-maxage=120"}, 120 * time.Second},
{"/mixed-case", []string{"S-MaxAge=120, Max-Age=0"}, 120 * time.Second},
{"/spaced", []string{"s-maxage = 120 , max-age=0"}, 120 * time.Second},
{"/shared-shorter", []string{"s-maxage=30, max-age=600"}, 30 * time.Second},
{"/max-age-only", []string{"max-age=60"}, 60 * time.Second},
{"/clamped", []string{"s-maxage=31536000, max-age=0"}, DefaultCacheMaxTTL},
{"/shared-zero", []string{"s-maxage=0, max-age=600"}, 0},
{"/private", []string{"private, s-maxage=120"}, 0},
{"/no-store", []string{"no-store, s-maxage=120"}, 0},
{"/no-cache", []string{"s-maxage=120, no-cache"}, 0},
{"/malformed-shared", []string{"s-maxage=soon, max-age=600"}, 0},
{"/malformed-max-age", []string{"max-age=soon, s-maxage=120"}, 0},
}
byPath := map[string][]string{}
for _, c := range cases {
byPath[c.path] = c.cc
}
e := cacheProbeEdge(t, "smaxage.example.com", func(path string, h http.Header) {
for _, line := range byPath[path] {
h.Add("Cache-Control", line)
}
})
for _, c := range cases {
t.Run(c.path, func(t *testing.T) {
e.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest("GET", "http://smaxage.example.com"+c.path, nil))
resp, ok := e.cache.get(CacheKey{Host: "smaxage.example.com", Method: "GET", Path: c.path})
if c.ttl == 0 {
if ok {
t.Fatalf("Cache-Control %q was cached; it must be refused", c.cc)
}
return
}
if !ok {
t.Fatalf("Cache-Control %q was not cached; the edge should hold it for %s", c.cc, c.ttl)
}
if resp.TTL != c.ttl {
t.Fatalf("Cache-Control %q cached with TTL %s, want %s", c.cc, resp.TTL, c.ttl)
}
})
}
}
// The stored response is replayed to later visitors verbatim, so the browser
// half of the pair has to survive the edge: an origin that said max-age=0
// still gets a revalidating browser on a cache HIT, not the edge's own idea of
// the client lifetime. Nothing rewrites Cache-Control on the way downstream —
// which is the point of reading s-maxage rather than inventing a header.
func TestServeHTTPCacheHitReplaysOriginCacheControl(t *testing.T) {
e := cacheProbeEdge(t, "replay.example.com", func(path string, h http.Header) {
h.Set("Cache-Control", "s-maxage=120, max-age=0")
})
get := func(label string) {
t.Helper()
rec := httptest.NewRecorder()
e.ServeHTTP(rec, httptest.NewRequest("GET", "http://replay.example.com/page", nil))
if got := rec.Header().Get("Cache-Control"); got != "s-maxage=120, max-age=0" {
t.Fatalf("%s: Cache-Control = %q, want the origin's own header", label, got)
}
}
get("miss")
if !cachedAt(e, "replay.example.com", "/page") {
t.Fatal("s-maxage=120 should have been stored at the edge")
}
get("hit")
}
func TestServeHTTPRefusesVaryCookie(t *testing.T) {
backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Cache-Control", "max-age=60")
w.Header().Set("Vary", "Cookie")
w.WriteHeader(200)
w.Write([]byte("should not be cached"))
}))
t.Cleanup(backend.Close)
reg := openReg(t)
if err := reg.Upsert(registry.Route{Host: "cookie.example.com", Kind: "instance", Addr: backend.Listener.Addr().String(), Status: "ready"}); err != nil {
t.Fatalf("upsert: %v", err)
}
e, err := New(reg, nil)
if err != nil {
t.Fatalf("New: %v", err)
}
e.EnableCache(NewCache(DefaultCacheMaxEntries, DefaultCacheMaxBodyBytes))
e.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest("GET", "http://cookie.example.com/event", nil))
if _, ok := e.cache.get(CacheKey{Host: "cookie.example.com", Method: "GET", Path: "/event"}); ok {
t.Fatal("a response Vary-ing on Cookie must never be cached — it would defeat per-visitor correctness")
}
}
// cachedVariants counts every cached variant of one URL, whatever VaryValue
// each was stored under. A "must not be cached" assertion has to ask this
// question rather than Get with an empty VaryValue: a response that varies is
// stored under a computed variant key, so the empty-key Get misses and the
// assertion passes whether or not the refusal actually fired.
func cachedVariants(e *Edge, host, path string) int {
uk := CacheKey{Host: host, Method: "GET", Path: path}.urlKey()
e.cache.mu.Lock()
defer e.cache.mu.Unlock()
return len(e.cache.byURL[uk])
}
// getWith issues a GET carrying one extra header and returns the recorder.
func getWith(t *testing.T, e *Edge, url, name, value string) *httptest.ResponseRecorder {
t.Helper()
req := httptest.NewRequest("GET", url, nil)
if name != "" {
req.Header.Set(name, value)
}
rec := httptest.NewRecorder()
e.ServeHTTP(rec, req)
return rec
}
// RFC 9110 §5.3: one header field may arrive as several lines, and that is
// semantically identical to a single comma-joined line — which is exactly what
// http.Header.Add produces and what layered middleware emits in practice
// (compression adds Vary: Accept-Encoding, the session layer separately adds
// Vary: Cookie). http.Header.Get returns only the FIRST line, so reading Vary
// that way made everything after it invisible: the Vary: Cookie refusal never
// fired and the entry was keyed on Accept-Encoding alone, so one visitor's
// cookie-scoped response was replayed to the next visitor.
func TestServeHTTPRefusesVaryCookieSplitAcrossHeaderLines(t *testing.T) {
backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Cache-Control", "max-age=60")
// Two separate Add calls, the standard shape for a list-valued header.
w.Header().Add("Vary", "Accept-Encoding")
w.Header().Add("Vary", "Cookie")
w.WriteHeader(200)
w.Write([]byte("dashboard for " + r.Header.Get("Cookie")))
}))
t.Cleanup(backend.Close)
reg := openReg(t)
if err := reg.Upsert(registry.Route{Host: "split.example.com", Kind: "instance", Addr: backend.Listener.Addr().String(), Status: "ready"}); err != nil {
t.Fatalf("upsert: %v", err)
}
e, err := New(reg, nil)
if err != nil {
t.Fatalf("New: %v", err)
}
e.EnableCache(NewCache(DefaultCacheMaxEntries, DefaultCacheMaxBodyBytes))
const url = "http://split.example.com/dashboard"
alice := getWith(t, e, url, "Cookie", "sid=alice").Body.String()
if alice != "dashboard for sid=alice" {
t.Fatalf("alice body = %q, want %q", alice, "dashboard for sid=alice")
}
if n := cachedVariants(e, "split.example.com", "/dashboard"); n != 0 {
// Errorf, not Fatalf: the replay assertion below is the live
// consequence and is worth reporting in the same run.
t.Errorf("%d variants cached for a response that varies on Cookie, want 0 — a multi-line Vary is still a Vary", n)
}
bob := getWith(t, e, url, "Cookie", "sid=bob").Body.String()
if bob != "dashboard for sid=bob" {
t.Fatalf("bob was served %q — one visitor's cookie-scoped response was replayed to another", bob)
}
}
// The Vary refusals read the raw field value and compare case-insensitively,
// so they fire on every spelling a response can use — including the ones the
// canonicalization downstream would have tidied up anyway. Pinned separately
// because canonicalization is the kind of change that invites moving the
// refusals behind it, where a name the canonicalizer does not recognise (or
// declines to rewrite) would slip past.
func TestServeHTTPRefusesVaryCookieAndStarInAnySpelling(t *testing.T) {
e := cacheProbeEdge(t, "spell.example.com", func(path string, h http.Header) {
h.Set("Cache-Control", "max-age=60")
switch path {
case "/lower":
h.Set("Vary", "accept-encoding, cookie")
case "/upper":
h.Add("Vary", "Accept-Language")
h.Add("Vary", "COOKIE")
case "/star":
h.Set("Vary", "Accept-Encoding , * ")
case "/ok":
h.Set("Vary", "accept-encoding")
}
})
for _, path := range []string{"/lower", "/upper", "/star", "/ok"} {
e.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest("GET", "http://spell.example.com"+path, nil))
}
for _, path := range []string{"/lower", "/upper", "/star"} {
if n := cachedVariants(e, "spell.example.com", path); n != 0 {
t.Errorf("%s: %d variants cached, want 0 — Vary: Cookie and Vary: * must be refused whatever their spelling or line", path, n)
}
}
if n := cachedVariants(e, "spell.example.com", "/ok"); n != 1 {
t.Errorf("/ok: %d variants cached, want 1 — an ordinary Vary in lower case must still be cached", n)
}
}
// The same root cause on Cache-Control: with max-age=600 on one line and
// private on the next, Header.Get saw only max-age and the private response
// was cached at the shared edge — the cross-visitor leak the private rule
// exists to prevent.
func TestServeHTTPRefusesCacheControlPrivateSplitAcrossHeaderLines(t *testing.T) {
e := cacheProbeEdge(t, "splitcc.example.com", func(path string, h http.Header) {
if path == "/private" {
h.Add("Cache-Control", "max-age=600")
h.Add("Cache-Control", "private")
return
}
h.Set("Cache-Control", "max-age=600")
})
e.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest("GET", "http://splitcc.example.com/private", nil))
e.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest("GET", "http://splitcc.example.com/shared", nil))
if n := cachedVariants(e, "splitcc.example.com", "/private"); n != 0 {
t.Fatalf("%d entries cached for a response whose second Cache-Control line says private, want 0", n)
}
if !cachedAt(e, "splitcc.example.com", "/shared") {
t.Fatal("an ordinary max-age response must still be cached")
}
}
// The same root cause once more, on the Authorization rule: a response to an
// Authorization-bearing request is private per RFC 9111 §3.5 unless
// Cache-Control says public — and "public" on a second header line counts.
func TestServeHTTPAuthorizedPublicSplitAcrossHeaderLines(t *testing.T) {
e := cacheProbeEdge(t, "splitauth.example.com", func(path string, h http.Header) {
h.Add("Cache-Control", "max-age=600")
h.Add("Cache-Control", "public")
})
req := httptest.NewRequest("GET", "http://splitauth.example.com/public", nil)
req.Header.Set("Authorization", "Bearer secret-token")
e.ServeHTTP(httptest.NewRecorder(), req)
if !cachedAt(e, "splitauth.example.com", "/public") {
t.Fatal("Cache-Control: public on a second header line still overrides the Authorization concern")
}
}
// An app that emits Vary only for some requests — CORS middleware adding
// Vary: Origin only when the request carries Origin, content negotiation
// adding Vary: Accept only when it must — used to poison its own URL
// permanently: the first Vary-carrying response recorded the Vary names for
// that URL and nothing ever cleared them, so later non-Vary responses were
// stored under the empty variant while every lookup kept resolving against the
// remembered names. Store and lookup drifted apart, every ordinary request
// missed, and on a hibernating route every miss is a wake — a remote,
// unauthenticated wake-storm from one request.
func TestServeHTTPConditionalVaryDoesNotPoisonURL(t *testing.T) {
var hits atomic.Int64
backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
hits.Add(1)
w.Header().Set("Cache-Control", "max-age=60")
if origin := r.Header.Get("Origin"); origin != "" {
w.Header().Set("Vary", "Origin")
w.Header().Set("Access-Control-Allow-Origin", origin)
}
w.WriteHeader(200)
w.Write([]byte("page"))
}))
t.Cleanup(backend.Close)
reg := openReg(t)
if err := reg.Upsert(registry.Route{Host: "cors.example.com", Kind: "instance", Addr: backend.Listener.Addr().String(), Status: "ready"}); err != nil {
t.Fatalf("upsert: %v", err)
}
e, err := New(reg, nil)
if err != nil {
t.Fatalf("New: %v", err)
}
e.EnableCache(NewCache(DefaultCacheMaxEntries, DefaultCacheMaxBodyBytes))
const url = "http://cors.example.com/page"
// One unauthenticated request on a cold URL, eliciting the Vary variant.
getWith(t, e, url, "Origin", "https://crawler.example")
// Then ordinary traffic: the first is a legitimate miss, every one after
// it must be served from cache.
var last *httptest.ResponseRecorder
for i := 0; i < 10; i++ {
last = getWith(t, e, url, "", "")
if got := last.Body.String(); got != "page" {
t.Fatalf("ordinary request %d body = %q, want %q", i, got, "page")
}
}
if n := hits.Load(); n != 2 {
t.Fatalf("backend invoked %d times for 1 Origin request + 10 ordinary ones, want 2 — after the first ordinary miss every ordinary request must hit the cache", n)
}
if acao := last.Header().Get("Access-Control-Allow-Origin"); acao != "" {
t.Fatalf("ordinary request served the Origin variant (Access-Control-Allow-Origin: %s)", acao)
}
}
// The same root cause on the Authorization rule, but in the read of the
// REQUEST'S Authorization header rather than the response's Cache-Control:
// http.Header.Get returns only the first line, so a request that splits
// Authorization across two lines (an empty first line, the real credential on
// the second — Go's HTTP server accepts exactly this from a client) made Get
// see "" and the request looked unauthenticated. The RFC 9111 §3.5 refusal
// never fired and a credentialed response got cached at the shared edge for
// everyone.
func TestServeHTTPRefusesAuthorizationSplitAcrossHeaderLines(t *testing.T) {
e := cacheProbeEdge(t, "splitauthreq.example.com", func(path string, h http.Header) {
if path == "/public" {
h.Set("Cache-Control", "max-age=60, public")
return
}
h.Set("Cache-Control", "max-age=60")
})
req := httptest.NewRequest("GET", "http://splitauthreq.example.com/private", nil)
req.Header.Add("Authorization", "")
req.Header.Add("Authorization", "Bearer secret-token")
e.ServeHTTP(httptest.NewRecorder(), req)
if n := cachedVariants(e, "splitauthreq.example.com", "/private"); n != 0 {
t.Fatalf("%d variants cached for a request whose Authorization arrived on a second header line, want 0 — a multi-line Authorization is still an Authorization", n)
}
// Positive control: the same split-Authorization request, but the
// response says public — proving the refusal isn't just breaking caching
// wholesale.
pubReq := httptest.NewRequest("GET", "http://splitauthreq.example.com/public", nil)
pubReq.Header.Add("Authorization", "")
pubReq.Header.Add("Authorization", "Bearer secret-token")
e.ServeHTTP(httptest.NewRecorder(), pubReq)
if !cachedAt(e, "splitauthreq.example.com", "/public") {
t.Fatal("Cache-Control: public must still be cached even with a split-line Authorization present")
}
}
// RFC 9111 §4.1 / RFC 9110 §12.5.5: Vary: * means the response is NEVER
// reusable from a shared cache — it signals that the response depends on
// something not captured by request headers at all. The token-scanning loop
// treated "*" as if it were an ordinary header name: varyValue computed the
// constant string "*=;" for every request regardless of who they were, so one
// variant was stored and replayed to everyone.
func TestServeHTTPRefusesVaryStar(t *testing.T) {
backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Cache-Control", "max-age=60")
w.Header().Set("Vary", "*")
w.WriteHeader(200)
w.Write([]byte("dashboard for " + r.Header.Get("Cookie")))
}))
t.Cleanup(backend.Close)
reg := openReg(t)
if err := reg.Upsert(registry.Route{Host: "varystar.example.com", Kind: "instance", Addr: backend.Listener.Addr().String(), Status: "ready"}); err != nil {
t.Fatalf("upsert: %v", err)
}
e, err := New(reg, nil)
if err != nil {
t.Fatalf("New: %v", err)
}
e.EnableCache(NewCache(DefaultCacheMaxEntries, DefaultCacheMaxBodyBytes))
const url = "http://varystar.example.com/dashboard"
alice := getWith(t, e, url, "Cookie", "sid=alice").Body.String()
if alice != "dashboard for sid=alice" {
t.Fatalf("alice body = %q, want %q", alice, "dashboard for sid=alice")
}
if n := cachedVariants(e, "varystar.example.com", "/dashboard"); n != 0 {
// Errorf, not Fatalf: the replay assertion below is the live
// consequence and is worth reporting in the same run.
t.Errorf("%d variants cached for a response carrying Vary: *, want 0 — Vary: * means never reusable from a shared cache", n)
}
bob := getWith(t, e, url, "Cookie", "sid=bob").Body.String()
if bob != "dashboard for sid=bob" {
t.Fatalf("bob was served %q — one visitor's response was replayed to another despite Vary: *", bob)
}
}
// An app that emits Vary conditionally — Vary: Accept-Language when the
// request negotiates a language, no Vary at all when it doesn't — used to
// thrash its own cache entries. One Vary scheme was recorded per URL, so every
// flip between the two response shapes rewrote that record and swept every
// variant stored under the previous scheme. Worse than a miss: with the record
// cleared, a language-negotiating request resolved to the EMPTY variant and was
// served the no-Vary body, so the language the app negotiated silently
// disappeared for everyone.
//
// With the Vary names stored per entry, the two schemes coexist: each request
// shape matches the entry its own shape produced, both stay live, and only the
// first of each is a miss.
func TestServeHTTPConditionalVaryKeepsBothVariantsLive(t *testing.T) {
var hits atomic.Int64
backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
hits.Add(1)
w.Header().Set("Cache-Control", "max-age=60")
lang := r.Header.Get("Accept-Language")
if lang != "" {
w.Header().Set("Vary", "Accept-Language")
}
w.WriteHeader(200)
w.Write([]byte("lang=" + lang))
}))
t.Cleanup(backend.Close)
reg := openReg(t)
if err := reg.Upsert(registry.Route{Host: "mixedvary.example.com", Kind: "instance", Addr: backend.Listener.Addr().String(), Status: "ready"}); err != nil {
t.Fatalf("upsert: %v", err)
}
e, err := New(reg, nil)
if err != nil {
t.Fatalf("New: %v", err)
}
e.EnableCache(NewCache(DefaultCacheMaxEntries, DefaultCacheMaxBodyBytes))
const url = "http://mixedvary.example.com/page"
// Alternate the two shapes. The first of each is a legitimate miss; every
// repeat after that must be served from cache, with its OWN body.
for i := 0; i < 6; i++ {
if got := getLang(t, e, url, "en"); got != "lang=en" {
t.Fatalf("round %d: language-negotiating request served %q, want %q", i, got, "lang=en")
}
if got := getLang(t, e, url, ""); got != "lang=" {
t.Fatalf("round %d: plain request served %q, want %q", i, got, "lang=")
}
}
if n := hits.Load(); n != 2 {
t.Fatalf("backend invoked %d times across 12 requests alternating two Vary schemes, want 2 — after one miss each, both variants must stay live and hit", n)
}
if n := cachedVariants(e, "mixedvary.example.com", "/page"); n != 2 {
t.Fatalf("URL holds %d cached variants, want 2 — the varying and the non-varying response must coexist, not evict each other", n)
}
}
// hijackableRecorder embeds httptest.ResponseRecorder (which deliberately
// does NOT implement http.Hijacker) and adds a no-op Hijack, standing in for
// a real connection's ResponseWriter — the minimum shape needed to prove
// ServeHTTP passes an Upgrade request's ResponseWriter through untouched
// instead of wrapping it in recordingResponseWriter (which pre-fix had no
// Hijack, breaking the stdlib reverse proxy's WebSocket-upgrade path).
type hijackableRecorder struct {
*httptest.ResponseRecorder
}
func (h *hijackableRecorder) Hijack() (net.Conn, *bufio.ReadWriter, error) {
return nil, nil, nil
}
// hijackCheckingActivator fails the test if the ResponseWriter it's handed
// doesn't satisfy http.Hijacker — proving ServeHTTP special-cased the
// Upgrade request instead of wrapping it in recordingResponseWriter.
type hijackCheckingActivator struct{ t *testing.T }
func (h hijackCheckingActivator) ServeHTTP(w http.ResponseWriter, r *http.Request, rt registry.Route, proxy http.Handler) bool {
if _, ok := w.(http.Hijacker); !ok {
h.t.Fatal("ResponseWriter passed to activator for an Upgrade request does not implement http.Hijacker — recordingResponseWriter must not wrap upgrade requests")
}
return true
}
func (h hijackCheckingActivator) RetireExcept(keep map[string]bool) {}
// TestServeHTTPUpgradeRequestBypassesRecording proves a Connection: Upgrade
// request (a WebSocket handshake) to a hibernating route reaches the
// activator with a still-Hijacker-capable ResponseWriter — i.e. ServeHTTP
// never wraps it in recordingResponseWriter, which would break the stdlib
// reverse proxy's hijack-based upgrade path and, with it, the open WebSocket
// the activator relies on to keep the instance's idle clock from firing.
func TestServeHTTPUpgradeRequestBypassesRecording(t *testing.T) {
reg := openReg(t)
// Addr is a bogus, never-dialed address — this test only needs to reach
// the activator, never the real backend.
if err := reg.Upsert(registry.Route{Host: "ws.example.com", Kind: "instance", Addr: "127.0.0.1:1", Status: "ready", Hibernate: true}); err != nil {
t.Fatalf("upsert: %v", err)
}
e, err := New(reg, hijackCheckingActivator{t: t})
if err != nil {
t.Fatalf("New: %v", err)
}
e.EnableCache(NewCache(DefaultCacheMaxEntries, DefaultCacheMaxBodyBytes))
req := httptest.NewRequest("GET", "http://ws.example.com/socket", nil)
req.Header.Set("Connection", "Upgrade")
req.Header.Set("Upgrade", "websocket")
rec := &hijackableRecorder{ResponseRecorder: httptest.NewRecorder()}
e.ServeHTTP(rec, req)
}
// TestIsUpgradeRequestDetectsConnectionSplitAcrossHeaderLines is the last of
// the four Header.Get-on-a-list-valued-header bugs found on this branch
// (see joinHeader's doc comment): Connection is legitimately list-valued
// (RFC 9110 §5.3), and a client that sends "Connection: keep-alive" and
// "Connection: Upgrade" as two separate header lines is just as much an
// upgrade request as one that sends them comma-joined on one line — but
// http.Header.Get only ever returns the first line, so isUpgradeRequest
// would silently treat the split form as an ordinary request and let it
// fall through to the cache/recordingResponseWriter path instead of the
// untouched-passthrough branch.
func TestIsUpgradeRequestDetectsConnectionSplitAcrossHeaderLines(t *testing.T) {
req := httptest.NewRequest("GET", "http://ws.example.com/socket", nil)
req.Header.Add("Connection", "keep-alive")
req.Header.Add("Connection", "Upgrade")
if !isUpgradeRequest(req) {
t.Fatal("Connection split across two header lines must still be detected as an upgrade request")
}
}
// activatorFunc adapts a plain function to the Activator interface, for
// tests that want custom wake-counting behavior without a full fake type.
type activatorFunc func(w http.ResponseWriter, r *http.Request, rt registry.Route, proxy http.Handler) bool
func (f activatorFunc) ServeHTTP(w http.ResponseWriter, r *http.Request, rt registry.Route, proxy http.Handler) bool {
return f(w, r, rt, proxy)
}
func (f activatorFunc) RetireExcept(keep map[string]bool) {
// No-op for tests that don't need retirement tracking.
}
// TestCacheBeforeWakeFullCycleOverPurgeSocket is the full cycle with the REAL
// purge channel in it: the same cold-miss → cached-repeat → purge → fresh
// sequence as TestCacheBeforeWakeFullCycle, except the purge arrives the way a
// spawned instance actually sends it — a JSON POST /purge over the unix
// control socket ListenPurge binds (the socket whose path the activator hands
// the child as CARLOS_CONTROL_SOCKET). The sibling test below drives the
// in-process cache API, which cannot catch a broken socket, encoding, or
// host-scoping; this one proves the whole channel from "instance calls purge"
// to "edge cache entry is gone."
func TestCacheBeforeWakeFullCycleOverPurgeSocket(t *testing.T) {
body := "version 1"
backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Cache-Control", "max-age=60")
w.WriteHeader(200)
w.Write([]byte(body))
}))
t.Cleanup(backend.Close)
wakes := 0
act := activatorFunc(func(w http.ResponseWriter, r *http.Request, rt registry.Route, proxy http.Handler) bool {
wakes++
proxy.ServeHTTP(w, r)
return true
})
reg := openReg(t)
if err := reg.Upsert(registry.Route{Host: "sock-cycle.example.com", Kind: "instance", Addr: backend.Listener.Addr().String(), Status: "ready", Hibernate: true}); err != nil {
t.Fatalf("upsert: %v", err)
}
e, err := New(reg, act)
if err != nil {
t.Fatalf("New: %v", err)
}
cache := NewCache(DefaultCacheMaxEntries, DefaultCacheMaxBodyBytes)
e.EnableCache(cache)
// The control socket the activator would create for this instance and
// hand it as CARLOS_CONTROL_SOCKET.
controlSocket := filepath.Join(t.TempDir(), "instance.sock.control")
pl, err := ListenPurge(controlSocket, "sock-cycle.example.com", cache)
if err != nil {
if errors.Is(err, syscall.EPERM) || strings.Contains(err.Error(), "operation not permitted") {
t.Skipf("unix-socket listeners not permitted in this environment: %v", err)
}
t.Fatalf("ListenPurge: %v", err)
}
defer pl.Close()
get := func() string {
rec := httptest.NewRecorder()
e.ServeHTTP(rec, httptest.NewRequest("GET", "http://sock-cycle.example.com/event", nil))
return rec.Body.String()
}
if got := get(); got != "version 1" {
t.Fatalf("first request body = %q, want %q", got, "version 1")
}
if wakes != 1 {
t.Fatalf("expected exactly 1 wake for the cold request, got %d", wakes)
}
if got := get(); got != "version 1" || wakes != 1 {
t.Fatalf("second request body = %q wakes = %d, want %q and no second wake", got, wakes, "version 1")
}
// The write happens, and the instance purges its own page over the control
// socket — exactly the request a real instance makes.
body = "version 2"
payload, _ := json.Marshal(purgeRequest{Path: "/event", Query: ""})
resp, err := dialPurgeSocket(t, controlSocket).Post("http://unix/purge", "application/json", bytes.NewReader(payload))
if err != nil {
t.Fatalf("purge POST over control socket: %v", err)
}
resp.Body.Close()
if resp.StatusCode != http.StatusOK {
t.Fatalf("purge status = %d, want 200", resp.StatusCode)
}
if got := get(); got != "version 2" {
t.Fatalf("post-purge request body = %q, want %q — the purge never reached the cache through the socket", got, "version 2")
}
if wakes != 2 {
t.Fatalf("expected the post-purge request to miss and wake once more, wakes = %d", wakes)
}
}
// TestCacheBeforeWakeFullCycle exercises the sequence the whole feature
// exists for: a cold miss wakes the instance and populates the cache; a
// repeat request hits the cache and does NOT wake the instance again; a
// write's purge call makes the next request miss and see fresh content.
func TestCacheBeforeWakeFullCycle(t *testing.T) {
body := "version 1"
backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Cache-Control", "max-age=60")
w.WriteHeader(200)
w.Write([]byte(body))
}))
t.Cleanup(backend.Close)
wakes := 0
act := activatorFunc(func(w http.ResponseWriter, r *http.Request, rt registry.Route, proxy http.Handler) bool {
wakes++
proxy.ServeHTTP(w, r)
return true
})
reg := openReg(t)
if err := reg.Upsert(registry.Route{Host: "cycle.example.com", Kind: "instance", Addr: backend.Listener.Addr().String(), Status: "ready", Hibernate: true}); err != nil {
t.Fatalf("upsert: %v", err)
}
e, err := New(reg, act)
if err != nil {
t.Fatalf("New: %v", err)
}
cache := NewCache(DefaultCacheMaxEntries, DefaultCacheMaxBodyBytes)
e.EnableCache(cache)
get := func() string {
rec := httptest.NewRecorder()
e.ServeHTTP(rec, httptest.NewRequest("GET", "http://cycle.example.com/event", nil))
return rec.Body.String()
}
if got := get(); got != "version 1" {
t.Fatalf("first request body = %q, want %q", got, "version 1")
}
if wakes != 1 {
t.Fatalf("expected exactly 1 wake for the cold request, got %d", wakes)
}
if got := get(); got != "version 1" {
t.Fatalf("second request body = %q, want %q", got, "version 1")
}
if wakes != 1 {
t.Fatalf("expected the second request to hit cache without waking again, wakes = %d", wakes)
}
// Simulate the write + purge an instance does after a real change.
body = "version 2"
cache.PurgeURL("cycle.example.com", "/event", "")
if got := get(); got != "version 2" {
t.Fatalf("post-purge request body = %q, want %q", got, "version 2")
}
if wakes != 2 {
t.Fatalf("expected the post-purge request to miss and wake once more, wakes = %d", wakes)
}
}
type mockResponseWriterFor1xx struct {
header http.Header
codes []int
}
func (m *mockResponseWriterFor1xx) Header() http.Header {
return m.header
}
func (m *mockResponseWriterFor1xx) WriteHeader(status int) {
// Stand-in for a real ResponseWriter. httptest.ResponseRecorder rejects
// writes after a 1xx status (RFC 9110 forbids bodies on 1xx), but a real
// ResponseWriter accepts the sequence 1xx then 2xx (103 Early Hints
// followed by the final response status) and forwards both. This mock
// accepts and records both.
m.codes = append(m.codes, status)
}
func (m *mockResponseWriterFor1xx) Write(p []byte) (int, error) {
return len(p), nil
}
func TestRecordingWriterIgnoresInformational(t *testing.T) {
mock := &mockResponseWriterFor1xx{header: make(http.Header)}
rec := &recordingResponseWriter{
ResponseWriter: mock,
request: httptest.NewRequest("GET", "http://app.example/x", nil),
cacheable: true,
maxBody: 1 << 20,
}
rec.WriteHeader(http.StatusEarlyHints)
rec.Header().Set("Cache-Control", "max-age=60")
rec.WriteHeader(http.StatusOK)
if _, err := rec.Write([]byte("final body")); err != nil {
t.Fatal(err)
}
if rec.status != http.StatusOK {
t.Fatalf("recorded status %d, want %d — informational must not latch", rec.status, http.StatusOK)
}
if !slices.Equal(mock.codes, []int{http.StatusEarlyHints, http.StatusOK}) {
t.Fatalf("underlying writer saw %v, want [103 200] — the 103 must be forwarded, not swallowed", mock.codes)
}
e := &Edge{cache: NewCache(DefaultCacheMaxEntries, DefaultCacheMaxBodyBytes)}
key := CacheKey{Host: "app.example", Method: "GET", Path: "/x"}
e.maybeStore(key, rec)
if _, ok := e.cache.get(key); !ok {
t.Fatal("response written after a 103 was not cached")
}
}
// getHostHeader is getHost's sibling for tests that need to inspect a
// response header rather than the body — the deploy watch's proof
// (X-Carlos-Version) lives there, not in the body.
func getHostHeader(t *testing.T, edgeURL, host string) *http.Response {
t.Helper()
req, err := http.NewRequest("GET", edgeURL+"/", nil)
if err != nil {
t.Fatalf("new request: %v", err)
}
req.Host = host
resp, err := http.DefaultClient.Do(req)
if err != nil {
t.Fatalf("GET %s (host %s): %v", edgeURL, host, err)
}
defer resp.Body.Close()
io.Copy(io.Discard, resp.Body)
return resp
}
func TestVersionHeaderOnVersionedRoute(t *testing.T) {
backend, _ := hostEchoBackend(t, "hello from backend")
addr := backend.Listener.Addr().String()
reg := openReg(t)
if err := reg.Upsert(registry.Route{Host: "app.example.com", Kind: "service", Addr: addr, Status: "ready", Version: "abc1234"}); err != nil {
t.Fatalf("upsert: %v", err)
}
e, err := New(reg, nil)
if err != nil {
t.Fatalf("New: %v", err)
}
srv := httptest.NewServer(e)
defer srv.Close()
resp := getHostHeader(t, srv.URL, "app.example.com")
if got := resp.Header.Get("X-Carlos-Version"); got != "abc1234" {
t.Fatalf("X-Carlos-Version = %q, want %q", got, "abc1234")
}
}
func TestNoVersionHeaderOnUnversionedRoute(t *testing.T) {
backend, _ := hostEchoBackend(t, "hello from backend")
addr := backend.Listener.Addr().String()
reg := openReg(t)
if err := reg.Upsert(registry.Route{Host: "app.example.com", Kind: "service", Addr: addr, Status: "ready"}); err != nil {
t.Fatalf("upsert: %v", err)
}
e, err := New(reg, nil)
if err != nil {
t.Fatalf("New: %v", err)
}
srv := httptest.NewServer(e)
defer srv.Close()
resp := getHostHeader(t, srv.URL, "app.example.com")
if got, ok := resp.Header["X-Carlos-Version"]; ok {
t.Fatalf("X-Carlos-Version present (%v), want absent for an unversioned route", got)
}
}
// ---------------------------------------------------------------------------
// Multi-upstream addr routes (HA design Part II, issue #152)
// ---------------------------------------------------------------------------
// testProbePath is the health path every multi-upstream test configures. It is
// deliberately NOT "/": countingBackend ignores it, so a counter can never
// confuse the prober's own traffic for proxied requests — a test that counted
// probes would report "both upstreams were used" on a proxy that balanced
// nothing at all.
const testProbePath = "/healthz"
// countingBackend is an upstream that counts the PROXIED requests it served
// and answers a fixed body. Probe requests (testProbePath) are answered 200
// but never counted.
func countingBackend(t *testing.T, body string) (string, *atomic.Int64) {
t.Helper()
hits := new(atomic.Int64)
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path == testProbePath {
w.WriteHeader(http.StatusOK)
return
}
hits.Add(1)
io.WriteString(w, body)
}))
t.Cleanup(ts.Close)
return ts.Listener.Addr().String(), hits
}
// deadAddr reserves a real loopback port and closes the listener immediately,
// yielding an address that is well-formed and guaranteed to REFUSE
// connections. That refusal is instant, so a test asserting the dial-failure
// path never waits out a timeout — and, unlike a made-up port, it cannot
// collide with something else the machine is running.
func deadAddr(t *testing.T) string {
t.Helper()
ln, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatalf("listen: %v", err)
}
addr := ln.Addr().String()
if err := ln.Close(); err != nil {
t.Fatalf("close reserved listener: %v", err)
}
return addr
}
// healthJSON marshals a probe config the way provision does when it mints a
// route, so the tests exercise the same column bytes Reload parses in
// production.
func healthJSON(t *testing.T, h provision.Health) string {
t.Helper()
b, err := json.Marshal(h)
if err != nil {
t.Fatalf("marshal health: %v", err)
}
return string(b)
}
// frozenHealth is a probe config whose cadence cannot fire within a test: one
// tick runs immediately on Sync (hostProber.run), then nothing for a minute,
// and fail_after is far beyond what a single tick could reach. Any live-set
// change a test observes under it therefore came from the REQUEST path's
// dial-failure fast path (proberSet.Eject), never from a probe.
func frozenHealth() provision.Health {
return provision.Health{Path: testProbePath, Status: 200, IntervalMS: 60000, FailAfter: 1000, RiseAfter: 1000}
}
func TestMultiUpstreamBalancesAcrossLiveBackends(t *testing.T) {
addrA, hitsA := countingBackend(t, "A")
addrB, hitsB := countingBackend(t, "B")
reg := openReg(t)
if err := reg.Upsert(registry.Route{
Host: "ha.example.com", Kind: "service", Status: "ready",
Addrs: addrA + "," + addrB,
Health: healthJSON(t, frozenHealth()),
}); err != nil {
t.Fatalf("upsert: %v", err)
}
e, err := New(reg, nil)
if err != nil {
t.Fatalf("New: %v", err)
}
defer e.probers.Stop()
srv := httptest.NewServer(e)
defer srv.Close()
for i := 0; i < 20; i++ {
code, body := getHost(t, srv.URL, "ha.example.com")
if code != http.StatusOK {
t.Fatalf("request %d: status = %d, want 200", i, code)
}
if body != "A" && body != "B" {
t.Fatalf("request %d: body = %q, want one of the two upstreams", i, body)
}
}
// Exactly even, not merely "both were used". The cursor is per route and
// starts at zero, keep-alives are off so every request dials, and both
// upstreams stay live throughout — so 10/10 is the only correct answer
// here, and a loose "> 0" assertion would sit quietly through a 19/1 split
// that means the rotation has broken.
if hitsA.Load() != 10 || hitsB.Load() != 10 {
t.Fatalf("upstream hits = A:%d B:%d, want 10/10 (strict round-robin over the live set)",
hitsA.Load(), hitsB.Load())
}
}
func TestDialFailureRetriesOtherUpstreamNoUserError(t *testing.T) {
dead := deadAddr(t)
addrA, hitsA := countingBackend(t, "A")
reg := openReg(t)
// Dead upstream FIRST in the list, so the very first request must retry
// past it inside the same request rather than answering an error.
if err := reg.Upsert(registry.Route{
Host: "ha.example.com", Kind: "service", Status: "ready",
Addrs: dead + "," + addrA,
Health: healthJSON(t, frozenHealth()),
}); err != nil {
t.Fatalf("upsert: %v", err)
}
e, err := New(reg, nil)
if err != nil {
t.Fatalf("New: %v", err)
}
defer e.probers.Stop()
srv := httptest.NewServer(e)
defer srv.Close()
for i := 0; i < 10; i++ {
code, body := getHost(t, srv.URL, "ha.example.com")
if code != http.StatusOK {
t.Fatalf("request %d: status = %d, want 200 — a dead upstream must cost a retry, not a user-visible error", i, code)
}
if body != "A" {
t.Fatalf("request %d: body = %q, want %q (the live upstream)", i, body, "A")
}
}
if got := hitsA.Load(); got != 10 {
t.Fatalf("live upstream served %d requests, want 10", got)
}
// The fast path: frozenHealth's cadence cannot have ejected anything, so
// this can only be the in-request dial failure.
if live := e.probers.Live("ha.example.com"); !slices.Equal(live, []string{addrA}) {
t.Fatalf("live set = %v, want [%s] — a failed dial must eject %s immediately", live, addrA, dead)
}
}
func TestAllUpstreamsDownAnswers503WithRetryAfter(t *testing.T) {
dead1, dead2 := deadAddr(t), deadAddr(t)
reg := openReg(t)
if err := reg.Upsert(registry.Route{
Host: "ha.example.com", Kind: "service", Status: "ready",
Addrs: dead1 + "," + dead2,
Health: healthJSON(t, frozenHealth()),
}); err != nil {
t.Fatalf("upsert: %v", err)
}
e, err := New(reg, nil)
if err != nil {
t.Fatalf("New: %v", err)
}
defer e.probers.Stop()
srv := httptest.NewServer(e)
defer srv.Close()
resp := getHostHeader(t, srv.URL, "ha.example.com")
if resp.StatusCode != http.StatusServiceUnavailable {
t.Fatalf("status = %d, want 503 (a dark upstream set is 'come back shortly', not the stdlib's bare 502)", resp.StatusCode)
}
if got := resp.Header.Get("Retry-After"); got != "2" {
t.Fatalf("Retry-After = %q, want %q", got, "2")
}
// A second request must answer the same way rather than hanging or
// escalating: every upstream is ejected by now, so the live set is empty
// and the transport never dials at all.
resp2 := getHostHeader(t, srv.URL, "ha.example.com")
if resp2.StatusCode != http.StatusServiceUnavailable || resp2.Header.Get("Retry-After") != "2" {
t.Fatalf("second request = %d Retry-After %q, want 503/2 with an empty live set",
resp2.StatusCode, resp2.Header.Get("Retry-After"))
}
if live := e.probers.Live("ha.example.com"); len(live) != 0 {
t.Fatalf("live set = %v, want empty after both upstreams refused", live)
}
}
func TestSingleAddrWithHealthGets503Semantics(t *testing.T) {
dead := deadAddr(t)
reg := openReg(t)
// One upstream, carried in the LEGACY Addr column, but with health
// configured: probing is what buys the 503 semantics, not the count.
if err := reg.Upsert(registry.Route{
Host: "solo.example.com", Kind: "service", Status: "ready",
Addr: dead,
Health: healthJSON(t, frozenHealth()),
}); err != nil {
t.Fatalf("upsert: %v", err)
}
e, err := New(reg, nil)
if err != nil {
t.Fatalf("New: %v", err)
}
defer e.probers.Stop()
srv := httptest.NewServer(e)
defer srv.Close()
resp := getHostHeader(t, srv.URL, "solo.example.com")
if resp.StatusCode != http.StatusServiceUnavailable {
t.Fatalf("status = %d, want 503 for a probed single upstream", resp.StatusCode)
}
if got := resp.Header.Get("Retry-After"); got != "2" {
t.Fatalf("Retry-After = %q, want %q", got, "2")
}
// A probed single-addr route is on the dashboard like any other.
if st, ok := e.probers.State()["solo.example.com"]; !ok || len(st) != 1 || st[0].Addr != dead {
t.Fatalf("prober state = %v, want one entry for %s", st, dead)
}
}
// TestUpstreamStatesExportsProbeTable pins Task 6's accessor: the box status
// snapshot's only door onto the probe table is Edge.UpstreamStates, which must
// report exactly what the unexported proberSet holds — same hosts, same addrs,
// same order — and must be safe to call on an edge with nothing probed (a
// pure-proxy `carlos edge`, which is most of them), where it reports empty
// rather than nil-panicking or inventing a host.
func TestUpstreamStatesExportsProbeTable(t *testing.T) {
addrA, _ := countingBackend(t, "A")
dead := deadAddr(t)
reg := openReg(t)
if err := reg.Upsert(registry.Route{
Host: "ha.example.com", Kind: "service", Status: "ready",
Addrs: addrA + "," + dead,
Health: healthJSON(t, frozenHealth()),
}); err != nil {
t.Fatalf("upsert: %v", err)
}
// A second, UNPROBED route: no health config, so it holds no prober and
// must not appear in the exported table at all.
if err := reg.Upsert(registry.Route{
Host: "plain.example.com", Kind: "service", Status: "ready", Addr: addrA,
}); err != nil {
t.Fatalf("upsert: %v", err)
}
e, err := New(reg, nil)
if err != nil {
t.Fatalf("New: %v", err)
}
defer e.probers.Stop()
got := e.UpstreamStates()
if len(got) != 1 {
t.Fatalf("UpstreamStates() = %+v, want exactly the one probed host", got)
}
states := got["ha.example.com"]
if len(states) != 2 || states[0].Addr != addrA || states[1].Addr != dead {
t.Fatalf("states = %+v, want [%s %s] in the route's addr order", states, addrA, dead)
}
// Optimistic start (prober.go rule 1) is what the snapshot reports for a
// freshly synced host — frozenHealth cannot have ticked a transition in.
for _, st := range states {
if !st.Up || st.Since.IsZero() {
t.Errorf("state %+v: a freshly synced upstream must be up with a Since stamp", st)
}
}
// The exported view is a COPY: mutating it must not reach the live table
// the request path picks from.
states[0].Up = false
if again := e.UpstreamStates()["ha.example.com"]; !again[0].Up {
t.Error("UpstreamStates() handed out a view of live state — the snapshot must never be able to eject an upstream")
}
// An edge with nothing probed answers empty, not nil-panic.
plainReg := openReg(t)
if err := plainReg.Upsert(registry.Route{Host: "plain.example.com", Kind: "service", Status: "ready", Addr: addrA}); err != nil {
t.Fatalf("upsert: %v", err)
}
plain, err := New(plainReg, nil)
if err != nil {
t.Fatalf("New (unprobed): %v", err)
}
defer plain.probers.Stop()
if st := plain.UpstreamStates(); len(st) != 0 {
t.Errorf("UpstreamStates() on an unprobed edge = %+v, want empty", st)
}
}
func TestLegacySingleAddrRouteUnchanged(t *testing.T) {
dead := deadAddr(t)
reg := openReg(t)
if err := reg.Upsert(registry.Route{
Host: "legacy.example.com", Kind: "service", Status: "ready", Addr: dead,
}); err != nil {
t.Fatalf("upsert: %v", err)
}
e, err := New(reg, nil)
if err != nil {
t.Fatalf("New: %v", err)
}
defer e.probers.Stop()
srv := httptest.NewServer(e)
defer srv.Close()
resp := getHostHeader(t, srv.URL, "legacy.example.com")
if resp.StatusCode != http.StatusBadGateway {
t.Fatalf("status = %d, want 502 — an UNPROBED route must keep its pre-multi-upstream behavior byte for byte", resp.StatusCode)
}
if got := resp.Header.Get("Retry-After"); got != "" {
t.Fatalf("Retry-After = %q, want absent on the legacy path", got)
}
if st := e.probers.State(); len(st) != 0 {
t.Fatalf("prober state = %v, want empty — a route with no health config is not probed", st)
}
}
func TestReloadSwapsUpstreamSet(t *testing.T) {
addrA, _ := countingBackend(t, "A")
addrB, _ := countingBackend(t, "B")
health := healthJSON(t, frozenHealth())
reg := openReg(t)
if err := reg.Upsert(registry.Route{
Host: "ha.example.com", Kind: "service", Status: "ready",
Addrs: addrA, Health: health,
}); err != nil {
t.Fatalf("upsert: %v", err)
}
e, err := New(reg, nil)
if err != nil {
t.Fatalf("New: %v", err)
}
defer e.probers.Stop()
srv := httptest.NewServer(e)
defer srv.Close()
if code, body := getHost(t, srv.URL, "ha.example.com"); code != http.StatusOK || body != "A" {
t.Fatalf("pre-reload = %d %q, want 200 %q", code, body, "A")
}
// Write via a SECOND handle, the cross-process shape a reconcile pass has.
reg2, err := registry.Open(reg.Path())
if err != nil {
t.Fatalf("open second handle: %v", err)
}
if err := reg2.SetUpstreams("ha.example.com", "", addrB, health); err != nil {
t.Fatalf("SetUpstreams: %v", err)
}
if err := e.Reload(); err != nil {
t.Fatalf("Reload: %v", err)
}
if code, body := getHost(t, srv.URL, "ha.example.com"); code != http.StatusOK || body != "B" {
t.Fatalf("post-reload = %d %q, want 200 %q (the new upstream set must be served)", code, body, "B")
}
st := e.probers.State()["ha.example.com"]
if len(st) != 1 || st[0].Addr != addrB {
t.Fatalf("prober state = %v, want exactly one entry for %s — the probe specs must follow the route", st, addrB)
}
}
// upgradeEchoBackend answers a protocol-switch handshake the way a WebSocket
// server does — 101, then raw bytes on the hijacked connection — prefixing
// whatever it reads with tag so a test can tell WHICH upstream served it.
func upgradeEchoBackend(t *testing.T, tag string) string {
t.Helper()
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path == testProbePath {
w.WriteHeader(http.StatusOK)
return
}
hj, ok := w.(http.Hijacker)
if !ok {
http.Error(w, "not hijackable", http.StatusInternalServerError)
return
}
conn, buf, err := hj.Hijack()
if err != nil {
return
}
defer conn.Close()
buf.WriteString("HTTP/1.1 101 Switching Protocols\r\nUpgrade: websocket\r\nConnection: Upgrade\r\n\r\n")
if err := buf.Flush(); err != nil {
return
}
line, err := buf.ReadString('\n')
if err != nil {
return
}
buf.WriteString(tag + ":" + line)
buf.Flush()
}))
t.Cleanup(ts.Close)
return ts.Listener.Addr().String()
}
// TestMultiUpstreamServesWebSocketUpgrade guards the one thing multiProxy's
// upstream keep-alive suppression could plausibly have broken. A transport
// with DisableKeepAlives normally adds `Connection: close` to its outbound
// request, which would poison a protocol-switch handshake — the stdlib
// exempts upgrade requests from that (transport.go's isProtocolSwitchHeader
// check), and this test is what keeps that exemption load-bearing rather than
// assumed.
func TestMultiUpstreamServesWebSocketUpgrade(t *testing.T) {
addr := upgradeEchoBackend(t, "ws")
reg := openReg(t)
if err := reg.Upsert(registry.Route{
Host: "ws.example.com", Kind: "service", Status: "ready",
Addrs: addr, Health: healthJSON(t, frozenHealth()),
}); err != nil {
t.Fatalf("upsert: %v", err)
}
e, err := New(reg, nil)
if err != nil {
t.Fatalf("New: %v", err)
}
defer e.probers.Stop()
srv := httptest.NewServer(e)
defer srv.Close()
// Pin that this route really took the multiProxy arm: only a probed route
// has a prober, so without this the test would still pass if the switch
// had quietly built a plain tcpProxy — and it is tcpProxy's transport,
// not multiProxy's keep-alive-free one, that the upgrade path is being
// checked against.
if live := e.probers.Live("ws.example.com"); len(live) != 1 {
t.Fatalf("live set = %v, want one probed upstream — this test must exercise multiProxy, not tcpProxy", live)
}
c, err := net.Dial("tcp", strings.TrimPrefix(srv.URL, "http://"))
if err != nil {
t.Fatalf("dial edge: %v", err)
}
defer c.Close()
if err := c.SetDeadline(time.Now().Add(10 * time.Second)); err != nil {
t.Fatalf("set deadline: %v", err)
}
if _, err := fmt.Fprint(c, "GET /socket HTTP/1.1\r\nHost: ws.example.com\r\nConnection: Upgrade\r\nUpgrade: websocket\r\n\r\n"); err != nil {
t.Fatalf("write handshake: %v", err)
}
br := bufio.NewReader(c)
resp, err := http.ReadResponse(br, nil)
if err != nil {
t.Fatalf("read handshake response: %v", err)
}
if resp.StatusCode != http.StatusSwitchingProtocols {
t.Fatalf("handshake status = %d, want 101 — an upgrade over a multi-upstream route must switch protocols", resp.StatusCode)
}
if _, err := fmt.Fprint(c, "ping\n"); err != nil {
t.Fatalf("write post-upgrade frame: %v", err)
}
line, err := br.ReadString('\n')
if err != nil {
t.Fatalf("read post-upgrade frame: %v", err)
}
if line != "ws:ping\n" {
t.Fatalf("post-upgrade echo = %q, want %q — bytes must flow both ways after the switch", line, "ws:ping\n")
}
}
// TestUnparseableHealthServesUnprobed pins the parse-failure contract: a
// health column the edge cannot read costs the route its PROBING, never its
// SERVICE. The alternative — skipping the route, or building a proxy over an
// empty live set — turns one bad character in a config column into a dead
// host, which is the one outcome this branch exists to prevent.
func TestUnparseableHealthServesUnprobed(t *testing.T) {
addr, hits := countingBackend(t, "served anyway")
reg := openReg(t)
if err := reg.Upsert(registry.Route{
Host: "broken.example.com", Kind: "service", Status: "ready",
Addrs: addr, Health: "{not json",
}); err != nil {
t.Fatalf("upsert: %v", err)
}
e, err := New(reg, nil)
if err != nil {
t.Fatalf("New: %v", err)
}
defer e.probers.Stop()
srv := httptest.NewServer(e)
defer srv.Close()
code, body := getHost(t, srv.URL, "broken.example.com")
if code != http.StatusOK || body != "served anyway" {
t.Fatalf("= %d %q, want 200 %q — an unreadable health config must not take the host off the air",
code, body, "served anyway")
}
if got := hits.Load(); got != 1 {
t.Fatalf("backend served %d requests, want 1", got)
}
if st := e.probers.State(); len(st) != 0 {
t.Fatalf("prober state = %v, want empty — a health config that did not parse cannot become a probing job", st)
}
}
// TestMultiAddrWithoutHealthServesFirstUnprobed covers the row shape nothing
// mints but the switch must still answer for: several addrs, no health
// config. Before multi-upstream routing such a row was served by NOTHING (the
// old arm gated on the legacy Addr column, which this shape leaves empty), so
// the behavior asserted here is new by construction and needs pinning:
// upstream ONE is served, unprobed, with the legacy 502 semantics. Balancing
// is deliberately not offered — without a probe there is no live set, and
// rotating onto upstreams nothing is checking would be worse than using one.
func TestMultiAddrWithoutHealthServesFirstUnprobed(t *testing.T) {
first, firstHits := countingBackend(t, "first")
second, secondHits := countingBackend(t, "second")
dead := deadAddr(t)
reg := openReg(t)
if err := reg.Upsert(registry.Route{
Host: "unprobed.example.com", Kind: "service", Status: "ready",
Addrs: first + "," + second,
}); err != nil {
t.Fatalf("upsert: %v", err)
}
// A second route of the same shape whose first upstream is dead, so the
// failure semantics can be asserted alongside the success ones.
if err := reg.Upsert(registry.Route{
Host: "unprobed-dead.example.com", Kind: "service", Status: "ready",
Addrs: dead + "," + second,
}); err != nil {
t.Fatalf("upsert: %v", err)
}
e, err := New(reg, nil)
if err != nil {
t.Fatalf("New: %v", err)
}
defer e.probers.Stop()
srv := httptest.NewServer(e)
defer srv.Close()
for i := 0; i < 6; i++ {
code, body := getHost(t, srv.URL, "unprobed.example.com")
if code != http.StatusOK || body != "first" {
t.Fatalf("request %d = %d %q, want 200 %q (addrs[0], unprobed)", i, code, body, "first")
}
}
if got := firstHits.Load(); got != 6 {
t.Fatalf("first upstream served %d requests, want all 6", got)
}
if got := secondHits.Load(); got != 0 {
t.Fatalf("second upstream served %d requests, want 0 — an unprobed route must not balance onto upstreams nothing is checking", got)
}
// Dead first upstream: the LEGACY failure shape, not the probed one. No
// live set exists to fail away from, so there is nothing to promise a
// client about coming back.
resp := getHostHeader(t, srv.URL, "unprobed-dead.example.com")
if resp.StatusCode != http.StatusBadGateway {
t.Fatalf("dead first upstream = %d, want 502 (the unprobed path keeps the stdlib behavior)", resp.StatusCode)
}
if got := resp.Header.Get("Retry-After"); got != "" {
t.Fatalf("Retry-After = %q, want absent — 503 semantics are bought by probing, not by the addr count", got)
}
if got := secondHits.Load(); got != 0 {
t.Fatalf("second upstream served %d requests, want 0 — the unprobed path must not retry onto a sibling upstream", got)
}
if st := e.probers.State(); len(st) != 0 {
t.Fatalf("prober state = %v, want empty — neither route carries a health config", st)
}
}
// TestServeHTTPMetersRequestsAndBytesExcludingStatus drives a full Edge over
// HTTP (mirroring TestRoutesHostToTCPUpstream) with EnableUsage armed, and
// proves the single wrap site in ServeHTTP counts every plain-path request
// while a Kind=="status" route (the platform's own status page — not
// billable, per the placement rules) never shows up in Drain().
func TestServeHTTPMetersRequestsAndBytesExcludingStatus(t *testing.T) {
body := "metered response body"
backend, _ := hostEchoBackend(t, body)
addr := backend.Listener.Addr().String()
reg := openReg(t)
if err := reg.Upsert(registry.Route{Host: "metered.example.com", Kind: "service", Addr: addr, Account: "bab", App: "hello", Status: "ready"}); err != nil {
t.Fatalf("upsert metered route: %v", err)
}
if err := reg.Upsert(registry.Route{Host: "status.example.com", Kind: "status", Status: "ready"}); err != nil {
t.Fatalf("upsert status route: %v", err)
}
e, err := New(reg, nil)
if err != nil {
t.Fatalf("New: %v", err)
}
// Static/status serving needs a store wired in for the status route to
// answer at all (see testEdgeWithStore); an empty pubkey is fine since
// the status class never verifies signatures.
store := release.NewDirStore(filepath.Join(t.TempDir(), "store"))
_, pubB64, err := release.GenerateKey()
if err != nil {
t.Fatalf("generate key: %v", err)
}
pub, err := release.ParsePublicKey(pubB64, "test")
if err != nil {
t.Fatalf("parse public key: %v", err)
}
e.EnableStatic(store, pub, filepath.Join(t.TempDir(), "cache"))
store.Put(context.Background(), "status/index.html", []byte("status"), "")
if err := e.Reload(); err != nil {
t.Fatalf("Reload: %v", err)
}
meter := NewUsageMeter()
e.EnableUsage(meter)
srv := httptest.NewServer(e)
defer srv.Close()
for i := 0; i < 2; i++ {
code, got := getHost(t, srv.URL, "metered.example.com")
if code != http.StatusOK || got != body {
t.Fatalf("request %d: %d %q, want 200 %q", i, code, got, body)
}
}
if code, _ := getHost(t, srv.URL, "status.example.com"); code != http.StatusOK {
t.Fatalf("status route: %d, want 200", code)
}
rows := meter.Drain()
row := rows["metered.example.com"]
wantBytes := int64(2 * len(body))
if row.Requests != 2 || row.BytesOut != wantBytes {
t.Fatalf("metered.example.com row = %+v, want Requests=2 BytesOut=%d", row, wantBytes)
}
if row.Account != "bab" || row.App != "hello" {
t.Fatalf("metered.example.com row = %+v, want Account=bab App=hello", row)
}
if _, ok := rows["status.example.com"]; ok {
t.Fatalf("status route must not be metered, got %+v", rows["status.example.com"])
}
}
// TestSidecarRouteIsNeverServable is the structural guarantee behind the
// sidecar member kind: a sidecar is a process that serves nothing and that
// nothing may dial, and the edge must refuse it on KIND rather than on the
// accident of it carrying no backend columns.
//
// The row below is adversarial on purpose. It carries a Socket AND an Addr
// AND a Tunnel AND a fleet placement (Region + FleetBox), on an edge with a
// tunnel hub armed and the fleet registered — every column and every
// precondition that any arm of Reload's dispatch switch looks at, all at
// once. Eleven's sidecars are fleet-placed, so the fleet arm (which checks no
// kind at all) is reached by such a row on shape alone. If this test ever
// fails, a sidecar is reachable from the internet.
func TestSidecarRouteIsNeverServable(t *testing.T) {
const host = "bot.eleven.bdm.sidecar.internal"
reg := openReg(t)
if err := reg.UpsertFleet(registry.Fleet{Name: "eleven", Account: "amadan", Bucket: "eleven-data"}); err != nil {
t.Fatalf("UpsertFleet: %v", err)
}
if err := reg.AddFleetBox(registry.FleetBox{Fleet: "eleven", Label: "box1", TokenHash: sha256Hex("tok"), Status: "active"}); err != nil {
t.Fatalf("AddFleetBox: %v", err)
}
if err := reg.Upsert(registry.Route{
Host: host,
Kind: "sidecar",
Socket: filepath.Join(t.TempDir(), "whatever.sock"),
Addr: "127.0.0.1:9",
Tunnel: "eleven/box1",
Region: "fleet/eleven",
FleetBox: "box1",
Status: "ready",
App: "eleven",
Account: "bdm",
Platform: true,
}); err != nil {
t.Fatalf("upsert sidecar route: %v", err)
}
e, err := New(reg, nil)
if err != nil {
t.Fatalf("New: %v", err)
}
// Arm the hub, so the Tunnel and fleet arms of the dispatch switch are
// live rather than skipped by their e.tunnels == nil guard.
e.EnableTunnels(NewTunnelHub("tunnel.example.com", nil))
if err := e.Reload(); err != nil {
t.Fatalf("Reload: %v", err)
}
if _, ok := e.current()[host]; ok {
t.Fatalf("a sidecar row entered the routing map — it is reachable from the internet")
}
// Allowed is the autocert HostPolicy: a sidecar host must never cost an
// ACME issuance either.
if e.Allowed(host) {
t.Fatalf("Allowed(%s) = true — a certificate must never be requested for a sidecar host", host)
}
srv := httptest.NewServer(e)
defer srv.Close()
if code, _ := getHost(t, srv.URL, host); code != http.StatusNotFound {
t.Fatalf("GET %s = %d, want 404 — a sidecar must be unreachable", host, code)
}
}