package edge import ( "bufio" "bytes" "context" "encoding/json" "errors" "fmt" "io" "net" "net/http" "net/http/httptest" "os" "path/filepath" "slices" "strings" "sync/atomic" "syscall" "testing" "time" "github.com/carlosframework/platform/internal/provision" "github.com/carlosframework/platform/internal/registry" "github.com/carlosframework/platform/internal/release" ) func openReg(t *testing.T) *registry.Registry { t.Helper() reg, err := registry.Open(filepath.Join(t.TempDir(), "registry.db")) if err != nil { t.Fatalf("open registry: %v", err) } return reg } // hostEchoBackend is an upstream that records the Host header it saw and // answers a fixed body — enough to prove both routing and Host preservation. func hostEchoBackend(t *testing.T, body string) (*httptest.Server, *string) { t.Helper() seenHost := new(string) ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { *seenHost = r.Host io.WriteString(w, body) })) t.Cleanup(ts.Close) return ts, seenHost } // getHost GETs the edge server with an explicit Host header, the way a real // client reaches a routed hostname. func getHost(t *testing.T, edgeURL, host string) (int, string) { t.Helper() req, err := http.NewRequest("GET", edgeURL+"/", nil) if err != nil { t.Fatalf("new request: %v", err) } req.Host = host resp, err := http.DefaultClient.Do(req) if err != nil { t.Fatalf("GET %s (host %s): %v", edgeURL, host, err) } defer resp.Body.Close() b, err := io.ReadAll(resp.Body) if err != nil { t.Fatalf("read body: %v", err) } return resp.StatusCode, string(b) } func TestRoutesHostToTCPUpstream(t *testing.T) { backend, seenHost := hostEchoBackend(t, "hello from backend") addr := backend.Listener.Addr().String() reg := openReg(t) if err := reg.Upsert(registry.Route{Host: "app.example.com", Kind: "service", Addr: addr, Status: "ready"}); err != nil { t.Fatalf("upsert: %v", err) } e, err := New(reg, nil) if err != nil { t.Fatalf("New: %v", err) } srv := httptest.NewServer(e) defer srv.Close() code, body := getHost(t, srv.URL, "app.example.com") if code != http.StatusOK { t.Fatalf("status = %d, want 200", code) } if body != "hello from backend" { t.Fatalf("body = %q, want backend body", body) } if *seenHost != "app.example.com" { t.Fatalf("backend saw Host %q, want app.example.com (inbound Host must be preserved)", *seenHost) } if !e.Allowed("app.example.com") { t.Fatalf("Allowed(app.example.com) = false, want true") } } func TestSkipsNotReadyRoutes(t *testing.T) { backend, _ := hostEchoBackend(t, "should never be reached") addr := backend.Listener.Addr().String() reg := openReg(t) if err := reg.Upsert(registry.Route{Host: "pending.example.com", Kind: "service", Addr: addr, Status: "pending"}); err != nil { t.Fatalf("upsert: %v", err) } e, err := New(reg, nil) if err != nil { t.Fatalf("New: %v", err) } srv := httptest.NewServer(e) defer srv.Close() code, _ := getHost(t, srv.URL, "pending.example.com") if code != http.StatusNotFound { t.Fatalf("status = %d, want 404 for a not-ready route", code) } // Allowed feeds the autocert HostPolicy: a not-ready route must not mint // a certificate either. if e.Allowed("pending.example.com") { t.Fatalf("Allowed(pending.example.com) = true, want false") } } func TestReloadPicksUpNewRoute(t *testing.T) { backend, _ := hostEchoBackend(t, "late arrival") addr := backend.Listener.Addr().String() reg := openReg(t) e, err := New(reg, nil) if err != nil { t.Fatalf("New: %v", err) } srv := httptest.NewServer(e) defer srv.Close() if code, _ := getHost(t, srv.URL, "new.example.com"); code != http.StatusNotFound { t.Fatalf("pre-reload status = %d, want 404", code) } // Write via a SECOND handle on the same file — the cross-process shape // (`carlos add` while the edge runs) that exposed the stale-WAL quirk. reg2, err := registry.Open(reg.Path()) if err != nil { t.Fatalf("open second handle: %v", err) } if err := reg2.Upsert(registry.Route{Host: "new.example.com", Kind: "service", Addr: addr, Status: "ready"}); err != nil { t.Fatalf("upsert via second handle: %v", err) } if err := e.Reload(); err != nil { t.Fatalf("Reload: %v", err) } code, body := getHost(t, srv.URL, "new.example.com") if code != http.StatusOK || body != "late arrival" { t.Fatalf("post-reload = %d %q, want 200 %q", code, body, "late arrival") } } func TestUnixSocketUpstream(t *testing.T) { sock := filepath.Join(t.TempDir(), "u.sock") ln, err := net.Listen("unix", sock) if err != nil { // Sandboxed environments block unix-socket listeners (EPERM); that is // an environment limit, not an edge bug. if errors.Is(err, syscall.EPERM) || strings.Contains(err.Error(), "operation not permitted") { t.Skipf("unix-socket listeners not permitted in this environment: %v", err) } t.Fatalf("listen unix: %v", err) } seenHost := new(string) backend := &httptest.Server{ Listener: ln, Config: &http.Server{Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { *seenHost = r.Host io.WriteString(w, "over the socket") })}, } backend.Start() defer backend.Close() reg := openReg(t) if err := reg.Upsert(registry.Route{Host: "sock.example.com", Kind: "instance", Socket: sock, Status: "ready"}); err != nil { t.Fatalf("upsert: %v", err) } e, err := New(reg, nil) if err != nil { t.Fatalf("New: %v", err) } srv := httptest.NewServer(e) defer srv.Close() code, body := getHost(t, srv.URL, "sock.example.com") if code != http.StatusOK { t.Fatalf("status = %d, want 200", code) } if body != "over the socket" { t.Fatalf("body = %q, want socket backend body", body) } if *seenHost != "sock.example.com" { t.Fatalf("backend saw Host %q, want sock.example.com", *seenHost) } } func TestFirstFingerprintObservationReloads(t *testing.T) { backend, _ := hostEchoBackend(t, "seen on first poll") addr := backend.Listener.Addr().String() // Snapshot BEFORE the route exists: New's startup reload sees zero routes. reg := openReg(t) e, err := New(reg, nil) if err != nil { t.Fatalf("New: %v", err) } srv := httptest.NewServer(e) defer srv.Close() // The route lands after the startup reload but before the watcher starts — // exactly the write a baseline-only first tick would silently absorb. reg2, err := registry.Open(reg.Path()) if err != nil { t.Fatalf("open second handle: %v", err) } if err := reg2.Upsert(registry.Route{Host: "first.example.com", Kind: "service", Addr: addr, Status: "ready"}); err != nil { t.Fatalf("upsert: %v", err) } stop := make(chan struct{}) defer close(stop) go e.Watch(stop) // No second write: the FIRST observed fingerprint alone must reload. deadline := time.Now().Add(6 * time.Second) for time.Now().Before(deadline) { if code, body := getHost(t, srv.URL, "first.example.com"); code == http.StatusOK { if body != "seen on first poll" { t.Fatalf("body = %q, want backend body", body) } return } time.Sleep(100 * time.Millisecond) } t.Fatalf("route never served: first fingerprint observation was baselined away instead of reloading") } // fakeActivator records the hibernation hook's calls. type fakeActivator struct { served int retired []map[string]bool } func (f *fakeActivator) ServeHTTP(w http.ResponseWriter, r *http.Request, rt registry.Route, proxy http.Handler) bool { f.served++ proxy.ServeHTTP(w, r) // "wake" is instant; proxy straight through return true } func (f *fakeActivator) RetireExcept(keep map[string]bool) { f.retired = append(f.retired, keep) } // testEdgeWithStore builds an Edge with one route already loaded and static // serving (the plumbing kind=status shares — see EnableStatic's doc comment) // wired to a fresh, empty DirStore. It returns the store (so a test can Put // bucket objects straight into it) and the Edge itself as an http.Handler: // Edge.ServeHTTP dispatches on the request's Host header, so tests drive it // with httptest.NewRecorder + httptest.NewRequest — no real listener, hence // no unix-socket/EPERM sandbox concerns (contrast TestUnixSocketUpstream). func testEdgeWithStore(t *testing.T, rt registry.Route) (release.Store, http.Handler) { t.Helper() reg := openReg(t) if err := reg.Upsert(rt); err != nil { t.Fatalf("upsert: %v", err) } e, err := New(reg, nil) if err != nil { t.Fatalf("New: %v", err) } store := release.NewDirStore(filepath.Join(t.TempDir(), "store")) // The status class doesn't verify signatures (unlike static), but // EnableStatic's signature still asks for a pubkey — any well-formed one // works since it's never used on this path. _, pubB64, err := release.GenerateKey() if err != nil { t.Fatalf("generate key: %v", err) } pub, err := release.ParsePublicKey(pubB64, "test") if err != nil { t.Fatalf("parse public key: %v", err) } e.EnableStatic(store, pub, filepath.Join(t.TempDir(), "cache")) if err := e.Reload(); err != nil { t.Fatalf("Reload: %v", err) } return store, e } func TestStatusRouteServesBucketBacked(t *testing.T) { s, h := testEdgeWithStore(t, registry.Route{Host: "status.test", Kind: "status", Status: "ready"}) s.Put(context.Background(), "status/fleet.json", []byte(`{"generated_at":"x"}`), "") s.Put(context.Background(), "status/index.html", []byte("Fleet status"), "") for path, want := range map[string]string{"/": "Fleet status", "/fleet.json": "generated_at"} { rec := httptest.NewRecorder() h.ServeHTTP(rec, httptest.NewRequest("GET", "http://status.test"+path, nil)) if rec.Code != 200 || !strings.Contains(rec.Body.String(), want) { t.Errorf("%s: %d %q", path, rec.Code, rec.Body.String()) } } rec := httptest.NewRecorder() h.ServeHTTP(rec, httptest.NewRequest("GET", "http://status.test/boxes/anything", nil)) if rec.Code != 404 { t.Errorf("non-allowlisted path must 404, got %d", rec.Code) } } // TestStatusRouteContentTypes pins the three content types the brief names: // a client (or the console iframe embedding the page) depends on these to // render the HTML instead of downloading it, parse the JSON, and apply the // stylesheet. func TestStatusRouteContentTypes(t *testing.T) { s, h := testEdgeWithStore(t, registry.Route{Host: "status.test", Kind: "status", Status: "ready"}) s.Put(context.Background(), "status/index.html", []byte(""), "") s.Put(context.Background(), "status/fleet.json", []byte(`{}`), "") s.Put(context.Background(), "status/status.css", []byte("body{}"), "") for path, wantPrefix := range map[string]string{ "/": "text/html", "/fleet.json": "application/json", "/status.css": "text/css", } { rec := httptest.NewRecorder() h.ServeHTTP(rec, httptest.NewRequest("GET", "http://status.test"+path, nil)) if ct := rec.Header().Get("Content-Type"); !strings.HasPrefix(ct, wantPrefix) { t.Errorf("%s: Content-Type = %q, want prefix %q", path, ct, wantPrefix) } } } // TestStatusRouteCacheServesStaleOnStoreError proves the two halves of the // per-key cache's contract: inside the TTL, a mutated store object is NOT // picked up (the cache, not the bucket, is the source of truth); after the // TTL, a store error serves the last-fetched copy rather than failing the // request — the status page outliving a bucket blip is the point (see the // statusHandler doc comment in edge.go). func TestStatusRouteCacheServesStaleOnStoreError(t *testing.T) { rt := registry.Route{Host: "status.test", Kind: "status", Status: "ready"} reg := openReg(t) if err := reg.Upsert(rt); err != nil { t.Fatalf("upsert: %v", err) } e, err := New(reg, nil) if err != nil { t.Fatalf("New: %v", err) } storeDir := filepath.Join(t.TempDir(), "store") store := release.NewDirStore(storeDir) _, pubB64, err := release.GenerateKey() if err != nil { t.Fatalf("generate key: %v", err) } pub, err := release.ParsePublicKey(pubB64, "test") if err != nil { t.Fatalf("parse public key: %v", err) } e.EnableStatic(store, pub, filepath.Join(t.TempDir(), "cache")) if err := e.Reload(); err != nil { t.Fatalf("Reload: %v", err) } if _, err := store.Put(context.Background(), "status/fleet.json", []byte("first"), ""); err != nil { t.Fatalf("seed put: %v", err) } sh := e.statusHandlerFor(rt).(*statusHandler) sh.ttl = 20 * time.Millisecond fakeNow := time.Now() sh.now = func() time.Time { return fakeNow } rec := httptest.NewRecorder() sh.ServeHTTP(rec, httptest.NewRequest("GET", "http://status.test/fleet.json", nil)) if rec.Code != 200 || rec.Body.String() != "first" { t.Fatalf("first fetch = %d %q, want 200 %q", rec.Code, rec.Body.String(), "first") } // Mutate the store underneath the cache, but stay inside the TTL: the // cached copy must still be served, proving the cache isn't a pass-through. if _, err := store.Put(context.Background(), "status/fleet.json", []byte("second"), etagOfFile(t, storeDir, "status/fleet.json")); err != nil { t.Fatalf("mutate put: %v", err) } rec = httptest.NewRecorder() sh.ServeHTTP(rec, httptest.NewRequest("GET", "http://status.test/fleet.json", nil)) if rec.Code != 200 || rec.Body.String() != "first" { t.Fatalf("inside-TTL fetch = %d %q, want cached %q", rec.Code, rec.Body.String(), "first") } // Advance the clock seam past the TTL: the handler must re-read the store // and pick up the mutation. fakeNow = fakeNow.Add(21 * time.Millisecond) rec = httptest.NewRecorder() sh.ServeHTTP(rec, httptest.NewRequest("GET", "http://status.test/fleet.json", nil)) if rec.Code != 200 || rec.Body.String() != "second" { t.Fatalf("post-TTL fetch = %d %q, want refreshed %q", rec.Code, rec.Body.String(), "second") } // Now break the store (remove the underlying file, simulating a bucket // error) and advance past the TTL again: with a stale copy in hand, the // handler must keep serving it rather than 503ing. fakeNow = fakeNow.Add(21 * time.Millisecond) if err := os.Remove(filepath.Join(storeDir, "status", "fleet.json")); err != nil { t.Fatalf("remove backing file: %v", err) } rec = httptest.NewRecorder() sh.ServeHTTP(rec, httptest.NewRequest("GET", "http://status.test/fleet.json", nil)) if rec.Code != 200 || rec.Body.String() != "second" { t.Fatalf("stale-on-error fetch = %d %q, want stale %q", rec.Code, rec.Body.String(), "second") } } // TestStatusRouteEmptyStoreServes503 covers the honest-failure half: a key // that has NEVER been successfully fetched, hit with a store error (here: // the object was never published), must 503 rather than fabricate a body. func TestStatusRouteEmptyStoreServes503(t *testing.T) { _, h := testEdgeWithStore(t, registry.Route{Host: "status.test", Kind: "status", Status: "ready"}) rec := httptest.NewRecorder() h.ServeHTTP(rec, httptest.NewRequest("GET", "http://status.test/fleet.json", nil)) if rec.Code != http.StatusServiceUnavailable { t.Fatalf("empty-store fetch = %d, want 503", rec.Code) } } // etagOfFile reads a DirStore-backed key straight off disk to recover its // current etag for a replace-only Put — the same read-etag-then-Put dance // internal/status/fleet.go's putOverwrite does against the real interface; // done directly here since the test wants to mutate underneath an existing // cache, not through it. func etagOfFile(t *testing.T, root, key string) string { t.Helper() _, etag, err := release.NewDirStore(root).Get(context.Background(), key) if err != nil { t.Fatalf("read etag for %s: %v", key, err) } return etag } func TestHibernatingRouteGoesThroughActivator(t *testing.T) { backend, _ := hostEchoBackend(t, "woken") addr := backend.Listener.Addr().String() reg := openReg(t) if err := reg.Upsert(registry.Route{Host: "hib.example.com", Kind: "instance", Addr: addr, Status: "ready", Hibernate: true}); err != nil { t.Fatalf("upsert: %v", err) } act := &fakeActivator{} e, err := New(reg, act) if err != nil { t.Fatalf("New: %v", err) } srv := httptest.NewServer(e) defer srv.Close() code, body := getHost(t, srv.URL, "hib.example.com") if code != http.StatusOK || body != "woken" { t.Fatalf("got %d %q, want 200 woken", code, body) } if act.served != 1 { t.Fatalf("activator ServeHTTP called %d times, want 1", act.served) } // Reload reconciles the activator with the new route set. if len(act.retired) == 0 { t.Fatalf("RetireExcept never called on reload") } last := act.retired[len(act.retired)-1] if !last["hib.example.com"] { t.Fatalf("RetireExcept keep-set %v missing hib.example.com", last) } } // refusingActivator fails the test immediately if ever invoked — used to // prove a cache hit never reaches the activator at all. type refusingActivator struct{ t *testing.T } func (r refusingActivator) ServeHTTP(w http.ResponseWriter, req *http.Request, rt registry.Route, proxy http.Handler) bool { r.t.Fatal("activator invoked on what should have been a cache hit — this defeats the whole point of caching") return true } func (r refusingActivator) RetireExcept(keep map[string]bool) {} func TestServeHTTPCacheHitNeverInvokesActivator(t *testing.T) { reg := openReg(t) // Addr is deliberately a bogus, never-dialed address: if the cache // check is ever bypassed, proxying to it fails loudly instead of // silently succeeding, which would mask the bug. if err := reg.Upsert(registry.Route{Host: "cached.example.com", Kind: "instance", Addr: "127.0.0.1:1", Status: "ready", Hibernate: true}); err != nil { t.Fatalf("upsert: %v", err) } e, err := New(reg, refusingActivator{t: t}) if err != nil { t.Fatalf("New: %v", err) } e.EnableCache(NewCache(DefaultCacheMaxEntries, DefaultCacheMaxBodyBytes)) key := CacheKey{Host: "cached.example.com", Method: "GET", Path: "/event"} e.cache.Put(key, "", &CachedResponse{ StatusCode: 200, Header: http.Header{"Content-Type": {"text/html"}}, Body: []byte("cached page"), StoredAt: time.Now(), TTL: time.Minute, }) rec := httptest.NewRecorder() e.ServeHTTP(rec, httptest.NewRequest("GET", "http://cached.example.com/event", nil)) if rec.Code != 200 { t.Fatalf("status = %d, want 200", rec.Code) } if rec.Body.String() != "cached page" { t.Fatalf("body = %q, want %q", rec.Body.String(), "cached page") } } func TestServeHTTPCacheMissPopulatesCacheFromResponse(t *testing.T) { backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Cache-Control", "max-age=60") w.WriteHeader(200) w.Write([]byte("fresh page")) })) t.Cleanup(backend.Close) reg := openReg(t) if err := reg.Upsert(registry.Route{Host: "miss.example.com", Kind: "instance", Addr: backend.Listener.Addr().String(), Status: "ready"}); err != nil { t.Fatalf("upsert: %v", err) } e, err := New(reg, nil) if err != nil { t.Fatalf("New: %v", err) } e.EnableCache(NewCache(DefaultCacheMaxEntries, DefaultCacheMaxBodyBytes)) rec1 := httptest.NewRecorder() e.ServeHTTP(rec1, httptest.NewRequest("GET", "http://miss.example.com/event", nil)) if rec1.Body.String() != "fresh page" { t.Fatalf("first response body = %q, want %q", rec1.Body.String(), "fresh page") } // Close the backend so a second real request would fail to dial — // proves the second request is answered from cache, not the backend. backend.Close() rec2 := httptest.NewRecorder() e.ServeHTTP(rec2, httptest.NewRequest("GET", "http://miss.example.com/event", nil)) if rec2.Body.String() != "fresh page" { t.Fatalf("second response body = %q, want %q (should be served from cache)", rec2.Body.String(), "fresh page") } } func TestServeHTTPNeverCachesNon2xx(t *testing.T) { backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Cache-Control", "max-age=60") w.WriteHeader(500) w.Write([]byte("boom")) })) t.Cleanup(backend.Close) reg := openReg(t) if err := reg.Upsert(registry.Route{Host: "err.example.com", Kind: "instance", Addr: backend.Listener.Addr().String(), Status: "ready"}); err != nil { t.Fatalf("upsert: %v", err) } e, err := New(reg, nil) if err != nil { t.Fatalf("New: %v", err) } e.EnableCache(NewCache(DefaultCacheMaxEntries, DefaultCacheMaxBodyBytes)) e.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest("GET", "http://err.example.com/event", nil)) if _, ok := e.cache.get(CacheKey{Host: "err.example.com", Method: "GET", Path: "/event"}); ok { t.Fatal("a 500 response must never be cached") } } // varyBackendEdge wires an Edge in front of a backend that echoes the request's // Accept-Language and counts its own invocations — the counter is what proves a // second request was answered from cache rather than re-proxied, which asserting // on response bodies alone cannot do (a miss echoes the same body a hit would). func varyBackendEdge(t *testing.T, host string, hits *atomic.Int64) *Edge { t.Helper() backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { hits.Add(1) w.Header().Set("Cache-Control", "max-age=60") w.Header().Set("Vary", "Accept-Language") w.WriteHeader(200) w.Write([]byte("lang=" + r.Header.Get("Accept-Language"))) })) t.Cleanup(backend.Close) reg := openReg(t) if err := reg.Upsert(registry.Route{Host: host, Kind: "instance", Addr: backend.Listener.Addr().String(), Status: "ready"}); err != nil { t.Fatalf("upsert: %v", err) } e, err := New(reg, nil) if err != nil { t.Fatalf("New: %v", err) } e.EnableCache(NewCache(DefaultCacheMaxEntries, DefaultCacheMaxBodyBytes)) return e } // getLang issues a GET carrying Accept-Language: lang (omitted entirely when // lang is "") and returns the body. func getLang(t *testing.T, e *Edge, url, lang string) string { t.Helper() req := httptest.NewRequest("GET", url, nil) if lang != "" { req.Header.Set("Accept-Language", lang) } rec := httptest.NewRecorder() e.ServeHTTP(rec, req) return rec.Body.String() } // A Vary-carrying response must be both separated per variant AND actually // re-served from cache on a repeat request. The backend-invocation count is // the load-bearing assertion: pre-fix, ServeHTTP looked entries up with an // empty VaryValue while maybeStore stored them under varyValue(request, vary), // so every request was a miss — 4 backend hits, not 2 — while the bodies still // differed per language, which is why the old body-comparison assertion passed // with Vary support entirely broken. func TestServeHTTPVaryAcceptLanguageProducesSeparateEntries(t *testing.T) { var hits atomic.Int64 e := varyBackendEdge(t, "i18n.example.com", &hits) const url = "http://i18n.example.com/event" for _, want := range []struct{ lang, body string }{ {"en", "lang=en"}, {"en", "lang=en"}, {"fr", "lang=fr"}, {"fr", "lang=fr"}, } { if got := getLang(t, e, url, want.lang); got != want.body { t.Fatalf("Accept-Language %q: body = %q, want %q", want.lang, got, want.body) } } if n := hits.Load(); n != 2 { t.Fatalf("backend invoked %d times across 4 requests (en, en, fr, fr), want 2 — the repeat of each language must be served from cache", n) } } // The core regression: a repeat of an identical request against a // Vary-carrying response must hit. Here no Accept-Language is sent at all, so // both requests resolve to the same (empty-valued) variant. func TestServeHTTPVaryRepeatIdenticalRequestHitsCache(t *testing.T) { var hits atomic.Int64 e := varyBackendEdge(t, "repeat.example.com", &hits) const url = "http://repeat.example.com/event" if got := getLang(t, e, url, ""); got != "lang=" { t.Fatalf("first body = %q, want %q", got, "lang=") } if got := getLang(t, e, url, ""); got != "lang=" { t.Fatalf("second body = %q, want %q", got, "lang=") } if n := hits.Load(); n != 1 { t.Fatalf("backend invoked %d times across 2 identical requests, want 1 — the second must be a cache hit", n) } } // A crawler pumping distinct Vary values at one URL must not be able to fill // the cache with unreachable variants: the per-URL cap bounds how many a // single URL can occupy, evicting the oldest as new ones arrive. func TestServeHTTPVaryVariantCapBoundsEntriesPerURL(t *testing.T) { var hits atomic.Int64 e := varyBackendEdge(t, "flood.example.com", &hits) const url = "http://flood.example.com/event" langs := make([]string, 4*maxVaryVariantsPerURL) for i := range langs { langs[i] = fmt.Sprintf("xx-%d", i) getLang(t, e, url, langs[i]) } uk := CacheKey{Host: "flood.example.com", Method: "GET", Path: "/event"}.urlKey() e.cache.mu.Lock() n := len(e.cache.byURL[uk]) e.cache.mu.Unlock() if n > maxVaryVariantsPerURL { t.Fatalf("one URL holds %d cached variants, want at most %d", n, maxVaryVariantsPerURL) } // The newest variant survives; the oldest was evicted to make room. before := hits.Load() getLang(t, e, url, langs[len(langs)-1]) if hits.Load() != before { t.Fatal("the most recent Vary variant should still be cached") } getLang(t, e, url, langs[0]) if hits.Load() != before+1 { t.Fatal("the oldest Vary variant should have been evicted by the per-URL cap") } } // cacheProbeEdge wires an Edge whose backend answers with headers chosen per // path by hdr — the shape every "must not be cached" rule test needs. func cacheProbeEdge(t *testing.T, host string, hdr func(path string, h http.Header)) *Edge { t.Helper() backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { hdr(r.URL.Path, w.Header()) w.WriteHeader(200) w.Write([]byte("body for " + r.URL.Path)) })) t.Cleanup(backend.Close) reg := openReg(t) if err := reg.Upsert(registry.Route{Host: host, Kind: "instance", Addr: backend.Listener.Addr().String(), Status: "ready"}); err != nil { t.Fatalf("upsert: %v", err) } e, err := New(reg, nil) if err != nil { t.Fatalf("New: %v", err) } e.EnableCache(NewCache(DefaultCacheMaxEntries, DefaultCacheMaxBodyBytes)) return e } func cachedAt(e *Edge, host, path string) bool { _, ok := e.cache.get(CacheKey{Host: host, Method: "GET", Path: path}) return ok } // Cache-Control: private means "for one visitor only" — caching it at the // shared edge and replaying it to everyone else is a cross-visitor leak. func TestServeHTTPRefusesCacheControlPrivate(t *testing.T) { e := cacheProbeEdge(t, "priv.example.com", func(path string, h http.Header) { if path == "/private" { h.Set("Cache-Control", "private, max-age=600") return } h.Set("Cache-Control", "max-age=600") }) e.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest("GET", "http://priv.example.com/private", nil)) e.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest("GET", "http://priv.example.com/shared", nil)) if cachedAt(e, "priv.example.com", "/private") { t.Fatal("a Cache-Control: private response must never be cached at the shared edge") } if !cachedAt(e, "priv.example.com", "/shared") { t.Fatal("an ordinary max-age response must still be cached") } } // A Set-Cookie response carries per-visitor state (a session, a CSRF token); // caching it would hand that cookie to every later visitor. func TestServeHTTPRefusesResponseWithSetCookie(t *testing.T) { e := cacheProbeEdge(t, "cookie2.example.com", func(path string, h http.Header) { h.Set("Cache-Control", "max-age=600") if path == "/session" { h.Set("Set-Cookie", "sid=abc123; Path=/; HttpOnly") } }) e.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest("GET", "http://cookie2.example.com/session", nil)) e.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest("GET", "http://cookie2.example.com/plain", nil)) if cachedAt(e, "cookie2.example.com", "/session") { t.Fatal("a response carrying Set-Cookie must never be cached — the cookie would be replayed to other visitors") } if !cachedAt(e, "cookie2.example.com", "/plain") { t.Fatal("a response without Set-Cookie must still be cached") } } // A response to an Authorization-bearing request is presumed private (RFC 9111 // §3.5) unless the app explicitly marks it public. func TestServeHTTPRefusesAuthorizedRequestUnlessPublic(t *testing.T) { e := cacheProbeEdge(t, "auth.example.com", func(path string, h http.Header) { if path == "/public" { h.Set("Cache-Control", "public, max-age=600") return } h.Set("Cache-Control", "max-age=600") }) authGet := func(path string) { req := httptest.NewRequest("GET", "http://auth.example.com"+path, nil) req.Header.Set("Authorization", "Bearer secret-token") e.ServeHTTP(httptest.NewRecorder(), req) } authGet("/account") authGet("/public") e.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest("GET", "http://auth.example.com/anon", nil)) if cachedAt(e, "auth.example.com", "/account") { t.Fatal("a response to an Authorization-bearing request must not be cached without an explicit Cache-Control: public") } if !cachedAt(e, "auth.example.com", "/public") { t.Fatal("Cache-Control: public overrides the Authorization concern (RFC 9111 §3.5) — this response should be cached") } if !cachedAt(e, "auth.example.com", "/anon") { t.Fatal("an unauthenticated request's ordinary max-age response must still be cached") } } // An app asking for a year of freshness would make a single lost purge // effectively permanent; the edge clamps every TTL to its own ceiling so a // missed purge always self-heals within a bounded window (spec decision 4). func TestServeHTTPClampsMaxAgeToCeiling(t *testing.T) { e := cacheProbeEdge(t, "ttl.example.com", func(path string, h http.Header) { h.Set("Cache-Control", "max-age=31536000") }) e.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest("GET", "http://ttl.example.com/event", nil)) resp, ok := e.cache.get(CacheKey{Host: "ttl.example.com", Method: "GET", Path: "/event"}) if !ok { t.Fatal("expected the response to be cached") } if resp.TTL != DefaultCacheMaxTTL { t.Fatalf("cached TTL = %s, want it clamped to %s", resp.TTL, DefaultCacheMaxTTL) } } // RFC 9111 §5.2.2.10: s-maxage overrides max-age for a SHARED cache, and a // private cache ignores it. That pair is the only way an origin can say "hold // this at the edge for two minutes, and have every browser revalidate" — which // is what makes a purge effective end to end rather than leaving a client-side // tail the purge channel cannot reach. Reading max-age alone turned exactly // that header ("s-maxage=120, max-age=0") into "don't cache at all", so the // origins trying hardest to stay purgeable were the ones that silently lost // edge caching (issue #403; tito PRO-4758/PRO-4759). func TestServeHTTPPrefersSMaxageOverMaxAge(t *testing.T) { cases := []struct { path string cc []string // Cache-Control, one entry per header line ttl time.Duration // 0 = must not be cached at all }{ {"/pair", []string{"s-maxage=120, max-age=0"}, 120 * time.Second}, {"/pair-reversed", []string{"max-age=0, s-maxage=120"}, 120 * time.Second}, {"/pair-split", []string{"s-maxage=120", "max-age=0"}, 120 * time.Second}, {"/pair-split-reversed", []string{"max-age=0", "s-maxage=120"}, 120 * time.Second}, {"/shared-only", []string{"s-maxage=120"}, 120 * time.Second}, {"/mixed-case", []string{"S-MaxAge=120, Max-Age=0"}, 120 * time.Second}, {"/spaced", []string{"s-maxage = 120 , max-age=0"}, 120 * time.Second}, {"/shared-shorter", []string{"s-maxage=30, max-age=600"}, 30 * time.Second}, {"/max-age-only", []string{"max-age=60"}, 60 * time.Second}, {"/clamped", []string{"s-maxage=31536000, max-age=0"}, DefaultCacheMaxTTL}, {"/shared-zero", []string{"s-maxage=0, max-age=600"}, 0}, {"/private", []string{"private, s-maxage=120"}, 0}, {"/no-store", []string{"no-store, s-maxage=120"}, 0}, {"/no-cache", []string{"s-maxage=120, no-cache"}, 0}, {"/malformed-shared", []string{"s-maxage=soon, max-age=600"}, 0}, {"/malformed-max-age", []string{"max-age=soon, s-maxage=120"}, 0}, } byPath := map[string][]string{} for _, c := range cases { byPath[c.path] = c.cc } e := cacheProbeEdge(t, "smaxage.example.com", func(path string, h http.Header) { for _, line := range byPath[path] { h.Add("Cache-Control", line) } }) for _, c := range cases { t.Run(c.path, func(t *testing.T) { e.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest("GET", "http://smaxage.example.com"+c.path, nil)) resp, ok := e.cache.get(CacheKey{Host: "smaxage.example.com", Method: "GET", Path: c.path}) if c.ttl == 0 { if ok { t.Fatalf("Cache-Control %q was cached; it must be refused", c.cc) } return } if !ok { t.Fatalf("Cache-Control %q was not cached; the edge should hold it for %s", c.cc, c.ttl) } if resp.TTL != c.ttl { t.Fatalf("Cache-Control %q cached with TTL %s, want %s", c.cc, resp.TTL, c.ttl) } }) } } // The stored response is replayed to later visitors verbatim, so the browser // half of the pair has to survive the edge: an origin that said max-age=0 // still gets a revalidating browser on a cache HIT, not the edge's own idea of // the client lifetime. Nothing rewrites Cache-Control on the way downstream — // which is the point of reading s-maxage rather than inventing a header. func TestServeHTTPCacheHitReplaysOriginCacheControl(t *testing.T) { e := cacheProbeEdge(t, "replay.example.com", func(path string, h http.Header) { h.Set("Cache-Control", "s-maxage=120, max-age=0") }) get := func(label string) { t.Helper() rec := httptest.NewRecorder() e.ServeHTTP(rec, httptest.NewRequest("GET", "http://replay.example.com/page", nil)) if got := rec.Header().Get("Cache-Control"); got != "s-maxage=120, max-age=0" { t.Fatalf("%s: Cache-Control = %q, want the origin's own header", label, got) } } get("miss") if !cachedAt(e, "replay.example.com", "/page") { t.Fatal("s-maxage=120 should have been stored at the edge") } get("hit") } func TestServeHTTPRefusesVaryCookie(t *testing.T) { backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Cache-Control", "max-age=60") w.Header().Set("Vary", "Cookie") w.WriteHeader(200) w.Write([]byte("should not be cached")) })) t.Cleanup(backend.Close) reg := openReg(t) if err := reg.Upsert(registry.Route{Host: "cookie.example.com", Kind: "instance", Addr: backend.Listener.Addr().String(), Status: "ready"}); err != nil { t.Fatalf("upsert: %v", err) } e, err := New(reg, nil) if err != nil { t.Fatalf("New: %v", err) } e.EnableCache(NewCache(DefaultCacheMaxEntries, DefaultCacheMaxBodyBytes)) e.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest("GET", "http://cookie.example.com/event", nil)) if _, ok := e.cache.get(CacheKey{Host: "cookie.example.com", Method: "GET", Path: "/event"}); ok { t.Fatal("a response Vary-ing on Cookie must never be cached — it would defeat per-visitor correctness") } } // cachedVariants counts every cached variant of one URL, whatever VaryValue // each was stored under. A "must not be cached" assertion has to ask this // question rather than Get with an empty VaryValue: a response that varies is // stored under a computed variant key, so the empty-key Get misses and the // assertion passes whether or not the refusal actually fired. func cachedVariants(e *Edge, host, path string) int { uk := CacheKey{Host: host, Method: "GET", Path: path}.urlKey() e.cache.mu.Lock() defer e.cache.mu.Unlock() return len(e.cache.byURL[uk]) } // getWith issues a GET carrying one extra header and returns the recorder. func getWith(t *testing.T, e *Edge, url, name, value string) *httptest.ResponseRecorder { t.Helper() req := httptest.NewRequest("GET", url, nil) if name != "" { req.Header.Set(name, value) } rec := httptest.NewRecorder() e.ServeHTTP(rec, req) return rec } // RFC 9110 §5.3: one header field may arrive as several lines, and that is // semantically identical to a single comma-joined line — which is exactly what // http.Header.Add produces and what layered middleware emits in practice // (compression adds Vary: Accept-Encoding, the session layer separately adds // Vary: Cookie). http.Header.Get returns only the FIRST line, so reading Vary // that way made everything after it invisible: the Vary: Cookie refusal never // fired and the entry was keyed on Accept-Encoding alone, so one visitor's // cookie-scoped response was replayed to the next visitor. func TestServeHTTPRefusesVaryCookieSplitAcrossHeaderLines(t *testing.T) { backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Cache-Control", "max-age=60") // Two separate Add calls, the standard shape for a list-valued header. w.Header().Add("Vary", "Accept-Encoding") w.Header().Add("Vary", "Cookie") w.WriteHeader(200) w.Write([]byte("dashboard for " + r.Header.Get("Cookie"))) })) t.Cleanup(backend.Close) reg := openReg(t) if err := reg.Upsert(registry.Route{Host: "split.example.com", Kind: "instance", Addr: backend.Listener.Addr().String(), Status: "ready"}); err != nil { t.Fatalf("upsert: %v", err) } e, err := New(reg, nil) if err != nil { t.Fatalf("New: %v", err) } e.EnableCache(NewCache(DefaultCacheMaxEntries, DefaultCacheMaxBodyBytes)) const url = "http://split.example.com/dashboard" alice := getWith(t, e, url, "Cookie", "sid=alice").Body.String() if alice != "dashboard for sid=alice" { t.Fatalf("alice body = %q, want %q", alice, "dashboard for sid=alice") } if n := cachedVariants(e, "split.example.com", "/dashboard"); n != 0 { // Errorf, not Fatalf: the replay assertion below is the live // consequence and is worth reporting in the same run. t.Errorf("%d variants cached for a response that varies on Cookie, want 0 — a multi-line Vary is still a Vary", n) } bob := getWith(t, e, url, "Cookie", "sid=bob").Body.String() if bob != "dashboard for sid=bob" { t.Fatalf("bob was served %q — one visitor's cookie-scoped response was replayed to another", bob) } } // The Vary refusals read the raw field value and compare case-insensitively, // so they fire on every spelling a response can use — including the ones the // canonicalization downstream would have tidied up anyway. Pinned separately // because canonicalization is the kind of change that invites moving the // refusals behind it, where a name the canonicalizer does not recognise (or // declines to rewrite) would slip past. func TestServeHTTPRefusesVaryCookieAndStarInAnySpelling(t *testing.T) { e := cacheProbeEdge(t, "spell.example.com", func(path string, h http.Header) { h.Set("Cache-Control", "max-age=60") switch path { case "/lower": h.Set("Vary", "accept-encoding, cookie") case "/upper": h.Add("Vary", "Accept-Language") h.Add("Vary", "COOKIE") case "/star": h.Set("Vary", "Accept-Encoding , * ") case "/ok": h.Set("Vary", "accept-encoding") } }) for _, path := range []string{"/lower", "/upper", "/star", "/ok"} { e.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest("GET", "http://spell.example.com"+path, nil)) } for _, path := range []string{"/lower", "/upper", "/star"} { if n := cachedVariants(e, "spell.example.com", path); n != 0 { t.Errorf("%s: %d variants cached, want 0 — Vary: Cookie and Vary: * must be refused whatever their spelling or line", path, n) } } if n := cachedVariants(e, "spell.example.com", "/ok"); n != 1 { t.Errorf("/ok: %d variants cached, want 1 — an ordinary Vary in lower case must still be cached", n) } } // The same root cause on Cache-Control: with max-age=600 on one line and // private on the next, Header.Get saw only max-age and the private response // was cached at the shared edge — the cross-visitor leak the private rule // exists to prevent. func TestServeHTTPRefusesCacheControlPrivateSplitAcrossHeaderLines(t *testing.T) { e := cacheProbeEdge(t, "splitcc.example.com", func(path string, h http.Header) { if path == "/private" { h.Add("Cache-Control", "max-age=600") h.Add("Cache-Control", "private") return } h.Set("Cache-Control", "max-age=600") }) e.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest("GET", "http://splitcc.example.com/private", nil)) e.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest("GET", "http://splitcc.example.com/shared", nil)) if n := cachedVariants(e, "splitcc.example.com", "/private"); n != 0 { t.Fatalf("%d entries cached for a response whose second Cache-Control line says private, want 0", n) } if !cachedAt(e, "splitcc.example.com", "/shared") { t.Fatal("an ordinary max-age response must still be cached") } } // The same root cause once more, on the Authorization rule: a response to an // Authorization-bearing request is private per RFC 9111 §3.5 unless // Cache-Control says public — and "public" on a second header line counts. func TestServeHTTPAuthorizedPublicSplitAcrossHeaderLines(t *testing.T) { e := cacheProbeEdge(t, "splitauth.example.com", func(path string, h http.Header) { h.Add("Cache-Control", "max-age=600") h.Add("Cache-Control", "public") }) req := httptest.NewRequest("GET", "http://splitauth.example.com/public", nil) req.Header.Set("Authorization", "Bearer secret-token") e.ServeHTTP(httptest.NewRecorder(), req) if !cachedAt(e, "splitauth.example.com", "/public") { t.Fatal("Cache-Control: public on a second header line still overrides the Authorization concern") } } // An app that emits Vary only for some requests — CORS middleware adding // Vary: Origin only when the request carries Origin, content negotiation // adding Vary: Accept only when it must — used to poison its own URL // permanently: the first Vary-carrying response recorded the Vary names for // that URL and nothing ever cleared them, so later non-Vary responses were // stored under the empty variant while every lookup kept resolving against the // remembered names. Store and lookup drifted apart, every ordinary request // missed, and on a hibernating route every miss is a wake — a remote, // unauthenticated wake-storm from one request. func TestServeHTTPConditionalVaryDoesNotPoisonURL(t *testing.T) { var hits atomic.Int64 backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { hits.Add(1) w.Header().Set("Cache-Control", "max-age=60") if origin := r.Header.Get("Origin"); origin != "" { w.Header().Set("Vary", "Origin") w.Header().Set("Access-Control-Allow-Origin", origin) } w.WriteHeader(200) w.Write([]byte("page")) })) t.Cleanup(backend.Close) reg := openReg(t) if err := reg.Upsert(registry.Route{Host: "cors.example.com", Kind: "instance", Addr: backend.Listener.Addr().String(), Status: "ready"}); err != nil { t.Fatalf("upsert: %v", err) } e, err := New(reg, nil) if err != nil { t.Fatalf("New: %v", err) } e.EnableCache(NewCache(DefaultCacheMaxEntries, DefaultCacheMaxBodyBytes)) const url = "http://cors.example.com/page" // One unauthenticated request on a cold URL, eliciting the Vary variant. getWith(t, e, url, "Origin", "https://crawler.example") // Then ordinary traffic: the first is a legitimate miss, every one after // it must be served from cache. var last *httptest.ResponseRecorder for i := 0; i < 10; i++ { last = getWith(t, e, url, "", "") if got := last.Body.String(); got != "page" { t.Fatalf("ordinary request %d body = %q, want %q", i, got, "page") } } if n := hits.Load(); n != 2 { t.Fatalf("backend invoked %d times for 1 Origin request + 10 ordinary ones, want 2 — after the first ordinary miss every ordinary request must hit the cache", n) } if acao := last.Header().Get("Access-Control-Allow-Origin"); acao != "" { t.Fatalf("ordinary request served the Origin variant (Access-Control-Allow-Origin: %s)", acao) } } // The same root cause on the Authorization rule, but in the read of the // REQUEST'S Authorization header rather than the response's Cache-Control: // http.Header.Get returns only the first line, so a request that splits // Authorization across two lines (an empty first line, the real credential on // the second — Go's HTTP server accepts exactly this from a client) made Get // see "" and the request looked unauthenticated. The RFC 9111 §3.5 refusal // never fired and a credentialed response got cached at the shared edge for // everyone. func TestServeHTTPRefusesAuthorizationSplitAcrossHeaderLines(t *testing.T) { e := cacheProbeEdge(t, "splitauthreq.example.com", func(path string, h http.Header) { if path == "/public" { h.Set("Cache-Control", "max-age=60, public") return } h.Set("Cache-Control", "max-age=60") }) req := httptest.NewRequest("GET", "http://splitauthreq.example.com/private", nil) req.Header.Add("Authorization", "") req.Header.Add("Authorization", "Bearer secret-token") e.ServeHTTP(httptest.NewRecorder(), req) if n := cachedVariants(e, "splitauthreq.example.com", "/private"); n != 0 { t.Fatalf("%d variants cached for a request whose Authorization arrived on a second header line, want 0 — a multi-line Authorization is still an Authorization", n) } // Positive control: the same split-Authorization request, but the // response says public — proving the refusal isn't just breaking caching // wholesale. pubReq := httptest.NewRequest("GET", "http://splitauthreq.example.com/public", nil) pubReq.Header.Add("Authorization", "") pubReq.Header.Add("Authorization", "Bearer secret-token") e.ServeHTTP(httptest.NewRecorder(), pubReq) if !cachedAt(e, "splitauthreq.example.com", "/public") { t.Fatal("Cache-Control: public must still be cached even with a split-line Authorization present") } } // RFC 9111 §4.1 / RFC 9110 §12.5.5: Vary: * means the response is NEVER // reusable from a shared cache — it signals that the response depends on // something not captured by request headers at all. The token-scanning loop // treated "*" as if it were an ordinary header name: varyValue computed the // constant string "*=;" for every request regardless of who they were, so one // variant was stored and replayed to everyone. func TestServeHTTPRefusesVaryStar(t *testing.T) { backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Cache-Control", "max-age=60") w.Header().Set("Vary", "*") w.WriteHeader(200) w.Write([]byte("dashboard for " + r.Header.Get("Cookie"))) })) t.Cleanup(backend.Close) reg := openReg(t) if err := reg.Upsert(registry.Route{Host: "varystar.example.com", Kind: "instance", Addr: backend.Listener.Addr().String(), Status: "ready"}); err != nil { t.Fatalf("upsert: %v", err) } e, err := New(reg, nil) if err != nil { t.Fatalf("New: %v", err) } e.EnableCache(NewCache(DefaultCacheMaxEntries, DefaultCacheMaxBodyBytes)) const url = "http://varystar.example.com/dashboard" alice := getWith(t, e, url, "Cookie", "sid=alice").Body.String() if alice != "dashboard for sid=alice" { t.Fatalf("alice body = %q, want %q", alice, "dashboard for sid=alice") } if n := cachedVariants(e, "varystar.example.com", "/dashboard"); n != 0 { // Errorf, not Fatalf: the replay assertion below is the live // consequence and is worth reporting in the same run. t.Errorf("%d variants cached for a response carrying Vary: *, want 0 — Vary: * means never reusable from a shared cache", n) } bob := getWith(t, e, url, "Cookie", "sid=bob").Body.String() if bob != "dashboard for sid=bob" { t.Fatalf("bob was served %q — one visitor's response was replayed to another despite Vary: *", bob) } } // An app that emits Vary conditionally — Vary: Accept-Language when the // request negotiates a language, no Vary at all when it doesn't — used to // thrash its own cache entries. One Vary scheme was recorded per URL, so every // flip between the two response shapes rewrote that record and swept every // variant stored under the previous scheme. Worse than a miss: with the record // cleared, a language-negotiating request resolved to the EMPTY variant and was // served the no-Vary body, so the language the app negotiated silently // disappeared for everyone. // // With the Vary names stored per entry, the two schemes coexist: each request // shape matches the entry its own shape produced, both stay live, and only the // first of each is a miss. func TestServeHTTPConditionalVaryKeepsBothVariantsLive(t *testing.T) { var hits atomic.Int64 backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { hits.Add(1) w.Header().Set("Cache-Control", "max-age=60") lang := r.Header.Get("Accept-Language") if lang != "" { w.Header().Set("Vary", "Accept-Language") } w.WriteHeader(200) w.Write([]byte("lang=" + lang)) })) t.Cleanup(backend.Close) reg := openReg(t) if err := reg.Upsert(registry.Route{Host: "mixedvary.example.com", Kind: "instance", Addr: backend.Listener.Addr().String(), Status: "ready"}); err != nil { t.Fatalf("upsert: %v", err) } e, err := New(reg, nil) if err != nil { t.Fatalf("New: %v", err) } e.EnableCache(NewCache(DefaultCacheMaxEntries, DefaultCacheMaxBodyBytes)) const url = "http://mixedvary.example.com/page" // Alternate the two shapes. The first of each is a legitimate miss; every // repeat after that must be served from cache, with its OWN body. for i := 0; i < 6; i++ { if got := getLang(t, e, url, "en"); got != "lang=en" { t.Fatalf("round %d: language-negotiating request served %q, want %q", i, got, "lang=en") } if got := getLang(t, e, url, ""); got != "lang=" { t.Fatalf("round %d: plain request served %q, want %q", i, got, "lang=") } } if n := hits.Load(); n != 2 { t.Fatalf("backend invoked %d times across 12 requests alternating two Vary schemes, want 2 — after one miss each, both variants must stay live and hit", n) } if n := cachedVariants(e, "mixedvary.example.com", "/page"); n != 2 { t.Fatalf("URL holds %d cached variants, want 2 — the varying and the non-varying response must coexist, not evict each other", n) } } // hijackableRecorder embeds httptest.ResponseRecorder (which deliberately // does NOT implement http.Hijacker) and adds a no-op Hijack, standing in for // a real connection's ResponseWriter — the minimum shape needed to prove // ServeHTTP passes an Upgrade request's ResponseWriter through untouched // instead of wrapping it in recordingResponseWriter (which pre-fix had no // Hijack, breaking the stdlib reverse proxy's WebSocket-upgrade path). type hijackableRecorder struct { *httptest.ResponseRecorder } func (h *hijackableRecorder) Hijack() (net.Conn, *bufio.ReadWriter, error) { return nil, nil, nil } // hijackCheckingActivator fails the test if the ResponseWriter it's handed // doesn't satisfy http.Hijacker — proving ServeHTTP special-cased the // Upgrade request instead of wrapping it in recordingResponseWriter. type hijackCheckingActivator struct{ t *testing.T } func (h hijackCheckingActivator) ServeHTTP(w http.ResponseWriter, r *http.Request, rt registry.Route, proxy http.Handler) bool { if _, ok := w.(http.Hijacker); !ok { h.t.Fatal("ResponseWriter passed to activator for an Upgrade request does not implement http.Hijacker — recordingResponseWriter must not wrap upgrade requests") } return true } func (h hijackCheckingActivator) RetireExcept(keep map[string]bool) {} // TestServeHTTPUpgradeRequestBypassesRecording proves a Connection: Upgrade // request (a WebSocket handshake) to a hibernating route reaches the // activator with a still-Hijacker-capable ResponseWriter — i.e. ServeHTTP // never wraps it in recordingResponseWriter, which would break the stdlib // reverse proxy's hijack-based upgrade path and, with it, the open WebSocket // the activator relies on to keep the instance's idle clock from firing. func TestServeHTTPUpgradeRequestBypassesRecording(t *testing.T) { reg := openReg(t) // Addr is a bogus, never-dialed address — this test only needs to reach // the activator, never the real backend. if err := reg.Upsert(registry.Route{Host: "ws.example.com", Kind: "instance", Addr: "127.0.0.1:1", Status: "ready", Hibernate: true}); err != nil { t.Fatalf("upsert: %v", err) } e, err := New(reg, hijackCheckingActivator{t: t}) if err != nil { t.Fatalf("New: %v", err) } e.EnableCache(NewCache(DefaultCacheMaxEntries, DefaultCacheMaxBodyBytes)) req := httptest.NewRequest("GET", "http://ws.example.com/socket", nil) req.Header.Set("Connection", "Upgrade") req.Header.Set("Upgrade", "websocket") rec := &hijackableRecorder{ResponseRecorder: httptest.NewRecorder()} e.ServeHTTP(rec, req) } // TestIsUpgradeRequestDetectsConnectionSplitAcrossHeaderLines is the last of // the four Header.Get-on-a-list-valued-header bugs found on this branch // (see joinHeader's doc comment): Connection is legitimately list-valued // (RFC 9110 §5.3), and a client that sends "Connection: keep-alive" and // "Connection: Upgrade" as two separate header lines is just as much an // upgrade request as one that sends them comma-joined on one line — but // http.Header.Get only ever returns the first line, so isUpgradeRequest // would silently treat the split form as an ordinary request and let it // fall through to the cache/recordingResponseWriter path instead of the // untouched-passthrough branch. func TestIsUpgradeRequestDetectsConnectionSplitAcrossHeaderLines(t *testing.T) { req := httptest.NewRequest("GET", "http://ws.example.com/socket", nil) req.Header.Add("Connection", "keep-alive") req.Header.Add("Connection", "Upgrade") if !isUpgradeRequest(req) { t.Fatal("Connection split across two header lines must still be detected as an upgrade request") } } // activatorFunc adapts a plain function to the Activator interface, for // tests that want custom wake-counting behavior without a full fake type. type activatorFunc func(w http.ResponseWriter, r *http.Request, rt registry.Route, proxy http.Handler) bool func (f activatorFunc) ServeHTTP(w http.ResponseWriter, r *http.Request, rt registry.Route, proxy http.Handler) bool { return f(w, r, rt, proxy) } func (f activatorFunc) RetireExcept(keep map[string]bool) { // No-op for tests that don't need retirement tracking. } // TestCacheBeforeWakeFullCycleOverPurgeSocket is the full cycle with the REAL // purge channel in it: the same cold-miss → cached-repeat → purge → fresh // sequence as TestCacheBeforeWakeFullCycle, except the purge arrives the way a // spawned instance actually sends it — a JSON POST /purge over the unix // control socket ListenPurge binds (the socket whose path the activator hands // the child as CARLOS_CONTROL_SOCKET). The sibling test below drives the // in-process cache API, which cannot catch a broken socket, encoding, or // host-scoping; this one proves the whole channel from "instance calls purge" // to "edge cache entry is gone." func TestCacheBeforeWakeFullCycleOverPurgeSocket(t *testing.T) { body := "version 1" backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Cache-Control", "max-age=60") w.WriteHeader(200) w.Write([]byte(body)) })) t.Cleanup(backend.Close) wakes := 0 act := activatorFunc(func(w http.ResponseWriter, r *http.Request, rt registry.Route, proxy http.Handler) bool { wakes++ proxy.ServeHTTP(w, r) return true }) reg := openReg(t) if err := reg.Upsert(registry.Route{Host: "sock-cycle.example.com", Kind: "instance", Addr: backend.Listener.Addr().String(), Status: "ready", Hibernate: true}); err != nil { t.Fatalf("upsert: %v", err) } e, err := New(reg, act) if err != nil { t.Fatalf("New: %v", err) } cache := NewCache(DefaultCacheMaxEntries, DefaultCacheMaxBodyBytes) e.EnableCache(cache) // The control socket the activator would create for this instance and // hand it as CARLOS_CONTROL_SOCKET. controlSocket := filepath.Join(t.TempDir(), "instance.sock.control") pl, err := ListenPurge(controlSocket, "sock-cycle.example.com", cache) if err != nil { if errors.Is(err, syscall.EPERM) || strings.Contains(err.Error(), "operation not permitted") { t.Skipf("unix-socket listeners not permitted in this environment: %v", err) } t.Fatalf("ListenPurge: %v", err) } defer pl.Close() get := func() string { rec := httptest.NewRecorder() e.ServeHTTP(rec, httptest.NewRequest("GET", "http://sock-cycle.example.com/event", nil)) return rec.Body.String() } if got := get(); got != "version 1" { t.Fatalf("first request body = %q, want %q", got, "version 1") } if wakes != 1 { t.Fatalf("expected exactly 1 wake for the cold request, got %d", wakes) } if got := get(); got != "version 1" || wakes != 1 { t.Fatalf("second request body = %q wakes = %d, want %q and no second wake", got, wakes, "version 1") } // The write happens, and the instance purges its own page over the control // socket — exactly the request a real instance makes. body = "version 2" payload, _ := json.Marshal(purgeRequest{Path: "/event", Query: ""}) resp, err := dialPurgeSocket(t, controlSocket).Post("http://unix/purge", "application/json", bytes.NewReader(payload)) if err != nil { t.Fatalf("purge POST over control socket: %v", err) } resp.Body.Close() if resp.StatusCode != http.StatusOK { t.Fatalf("purge status = %d, want 200", resp.StatusCode) } if got := get(); got != "version 2" { t.Fatalf("post-purge request body = %q, want %q — the purge never reached the cache through the socket", got, "version 2") } if wakes != 2 { t.Fatalf("expected the post-purge request to miss and wake once more, wakes = %d", wakes) } } // TestCacheBeforeWakeFullCycle exercises the sequence the whole feature // exists for: a cold miss wakes the instance and populates the cache; a // repeat request hits the cache and does NOT wake the instance again; a // write's purge call makes the next request miss and see fresh content. func TestCacheBeforeWakeFullCycle(t *testing.T) { body := "version 1" backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Cache-Control", "max-age=60") w.WriteHeader(200) w.Write([]byte(body)) })) t.Cleanup(backend.Close) wakes := 0 act := activatorFunc(func(w http.ResponseWriter, r *http.Request, rt registry.Route, proxy http.Handler) bool { wakes++ proxy.ServeHTTP(w, r) return true }) reg := openReg(t) if err := reg.Upsert(registry.Route{Host: "cycle.example.com", Kind: "instance", Addr: backend.Listener.Addr().String(), Status: "ready", Hibernate: true}); err != nil { t.Fatalf("upsert: %v", err) } e, err := New(reg, act) if err != nil { t.Fatalf("New: %v", err) } cache := NewCache(DefaultCacheMaxEntries, DefaultCacheMaxBodyBytes) e.EnableCache(cache) get := func() string { rec := httptest.NewRecorder() e.ServeHTTP(rec, httptest.NewRequest("GET", "http://cycle.example.com/event", nil)) return rec.Body.String() } if got := get(); got != "version 1" { t.Fatalf("first request body = %q, want %q", got, "version 1") } if wakes != 1 { t.Fatalf("expected exactly 1 wake for the cold request, got %d", wakes) } if got := get(); got != "version 1" { t.Fatalf("second request body = %q, want %q", got, "version 1") } if wakes != 1 { t.Fatalf("expected the second request to hit cache without waking again, wakes = %d", wakes) } // Simulate the write + purge an instance does after a real change. body = "version 2" cache.PurgeURL("cycle.example.com", "/event", "") if got := get(); got != "version 2" { t.Fatalf("post-purge request body = %q, want %q", got, "version 2") } if wakes != 2 { t.Fatalf("expected the post-purge request to miss and wake once more, wakes = %d", wakes) } } type mockResponseWriterFor1xx struct { header http.Header codes []int } func (m *mockResponseWriterFor1xx) Header() http.Header { return m.header } func (m *mockResponseWriterFor1xx) WriteHeader(status int) { // Stand-in for a real ResponseWriter. httptest.ResponseRecorder rejects // writes after a 1xx status (RFC 9110 forbids bodies on 1xx), but a real // ResponseWriter accepts the sequence 1xx then 2xx (103 Early Hints // followed by the final response status) and forwards both. This mock // accepts and records both. m.codes = append(m.codes, status) } func (m *mockResponseWriterFor1xx) Write(p []byte) (int, error) { return len(p), nil } func TestRecordingWriterIgnoresInformational(t *testing.T) { mock := &mockResponseWriterFor1xx{header: make(http.Header)} rec := &recordingResponseWriter{ ResponseWriter: mock, request: httptest.NewRequest("GET", "http://app.example/x", nil), cacheable: true, maxBody: 1 << 20, } rec.WriteHeader(http.StatusEarlyHints) rec.Header().Set("Cache-Control", "max-age=60") rec.WriteHeader(http.StatusOK) if _, err := rec.Write([]byte("final body")); err != nil { t.Fatal(err) } if rec.status != http.StatusOK { t.Fatalf("recorded status %d, want %d — informational must not latch", rec.status, http.StatusOK) } if !slices.Equal(mock.codes, []int{http.StatusEarlyHints, http.StatusOK}) { t.Fatalf("underlying writer saw %v, want [103 200] — the 103 must be forwarded, not swallowed", mock.codes) } e := &Edge{cache: NewCache(DefaultCacheMaxEntries, DefaultCacheMaxBodyBytes)} key := CacheKey{Host: "app.example", Method: "GET", Path: "/x"} e.maybeStore(key, rec) if _, ok := e.cache.get(key); !ok { t.Fatal("response written after a 103 was not cached") } } // getHostHeader is getHost's sibling for tests that need to inspect a // response header rather than the body — the deploy watch's proof // (X-Carlos-Version) lives there, not in the body. func getHostHeader(t *testing.T, edgeURL, host string) *http.Response { t.Helper() req, err := http.NewRequest("GET", edgeURL+"/", nil) if err != nil { t.Fatalf("new request: %v", err) } req.Host = host resp, err := http.DefaultClient.Do(req) if err != nil { t.Fatalf("GET %s (host %s): %v", edgeURL, host, err) } defer resp.Body.Close() io.Copy(io.Discard, resp.Body) return resp } func TestVersionHeaderOnVersionedRoute(t *testing.T) { backend, _ := hostEchoBackend(t, "hello from backend") addr := backend.Listener.Addr().String() reg := openReg(t) if err := reg.Upsert(registry.Route{Host: "app.example.com", Kind: "service", Addr: addr, Status: "ready", Version: "abc1234"}); err != nil { t.Fatalf("upsert: %v", err) } e, err := New(reg, nil) if err != nil { t.Fatalf("New: %v", err) } srv := httptest.NewServer(e) defer srv.Close() resp := getHostHeader(t, srv.URL, "app.example.com") if got := resp.Header.Get("X-Carlos-Version"); got != "abc1234" { t.Fatalf("X-Carlos-Version = %q, want %q", got, "abc1234") } } func TestNoVersionHeaderOnUnversionedRoute(t *testing.T) { backend, _ := hostEchoBackend(t, "hello from backend") addr := backend.Listener.Addr().String() reg := openReg(t) if err := reg.Upsert(registry.Route{Host: "app.example.com", Kind: "service", Addr: addr, Status: "ready"}); err != nil { t.Fatalf("upsert: %v", err) } e, err := New(reg, nil) if err != nil { t.Fatalf("New: %v", err) } srv := httptest.NewServer(e) defer srv.Close() resp := getHostHeader(t, srv.URL, "app.example.com") if got, ok := resp.Header["X-Carlos-Version"]; ok { t.Fatalf("X-Carlos-Version present (%v), want absent for an unversioned route", got) } } // --------------------------------------------------------------------------- // Multi-upstream addr routes (HA design Part II, issue #152) // --------------------------------------------------------------------------- // testProbePath is the health path every multi-upstream test configures. It is // deliberately NOT "/": countingBackend ignores it, so a counter can never // confuse the prober's own traffic for proxied requests — a test that counted // probes would report "both upstreams were used" on a proxy that balanced // nothing at all. const testProbePath = "/healthz" // countingBackend is an upstream that counts the PROXIED requests it served // and answers a fixed body. Probe requests (testProbePath) are answered 200 // but never counted. func countingBackend(t *testing.T, body string) (string, *atomic.Int64) { t.Helper() hits := new(atomic.Int64) ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.URL.Path == testProbePath { w.WriteHeader(http.StatusOK) return } hits.Add(1) io.WriteString(w, body) })) t.Cleanup(ts.Close) return ts.Listener.Addr().String(), hits } // deadAddr reserves a real loopback port and closes the listener immediately, // yielding an address that is well-formed and guaranteed to REFUSE // connections. That refusal is instant, so a test asserting the dial-failure // path never waits out a timeout — and, unlike a made-up port, it cannot // collide with something else the machine is running. func deadAddr(t *testing.T) string { t.Helper() ln, err := net.Listen("tcp", "127.0.0.1:0") if err != nil { t.Fatalf("listen: %v", err) } addr := ln.Addr().String() if err := ln.Close(); err != nil { t.Fatalf("close reserved listener: %v", err) } return addr } // healthJSON marshals a probe config the way provision does when it mints a // route, so the tests exercise the same column bytes Reload parses in // production. func healthJSON(t *testing.T, h provision.Health) string { t.Helper() b, err := json.Marshal(h) if err != nil { t.Fatalf("marshal health: %v", err) } return string(b) } // frozenHealth is a probe config whose cadence cannot fire within a test: one // tick runs immediately on Sync (hostProber.run), then nothing for a minute, // and fail_after is far beyond what a single tick could reach. Any live-set // change a test observes under it therefore came from the REQUEST path's // dial-failure fast path (proberSet.Eject), never from a probe. func frozenHealth() provision.Health { return provision.Health{Path: testProbePath, Status: 200, IntervalMS: 60000, FailAfter: 1000, RiseAfter: 1000} } func TestMultiUpstreamBalancesAcrossLiveBackends(t *testing.T) { addrA, hitsA := countingBackend(t, "A") addrB, hitsB := countingBackend(t, "B") reg := openReg(t) if err := reg.Upsert(registry.Route{ Host: "ha.example.com", Kind: "service", Status: "ready", Addrs: addrA + "," + addrB, Health: healthJSON(t, frozenHealth()), }); err != nil { t.Fatalf("upsert: %v", err) } e, err := New(reg, nil) if err != nil { t.Fatalf("New: %v", err) } defer e.probers.Stop() srv := httptest.NewServer(e) defer srv.Close() for i := 0; i < 20; i++ { code, body := getHost(t, srv.URL, "ha.example.com") if code != http.StatusOK { t.Fatalf("request %d: status = %d, want 200", i, code) } if body != "A" && body != "B" { t.Fatalf("request %d: body = %q, want one of the two upstreams", i, body) } } // Exactly even, not merely "both were used". The cursor is per route and // starts at zero, keep-alives are off so every request dials, and both // upstreams stay live throughout — so 10/10 is the only correct answer // here, and a loose "> 0" assertion would sit quietly through a 19/1 split // that means the rotation has broken. if hitsA.Load() != 10 || hitsB.Load() != 10 { t.Fatalf("upstream hits = A:%d B:%d, want 10/10 (strict round-robin over the live set)", hitsA.Load(), hitsB.Load()) } } func TestDialFailureRetriesOtherUpstreamNoUserError(t *testing.T) { dead := deadAddr(t) addrA, hitsA := countingBackend(t, "A") reg := openReg(t) // Dead upstream FIRST in the list, so the very first request must retry // past it inside the same request rather than answering an error. if err := reg.Upsert(registry.Route{ Host: "ha.example.com", Kind: "service", Status: "ready", Addrs: dead + "," + addrA, Health: healthJSON(t, frozenHealth()), }); err != nil { t.Fatalf("upsert: %v", err) } e, err := New(reg, nil) if err != nil { t.Fatalf("New: %v", err) } defer e.probers.Stop() srv := httptest.NewServer(e) defer srv.Close() for i := 0; i < 10; i++ { code, body := getHost(t, srv.URL, "ha.example.com") if code != http.StatusOK { t.Fatalf("request %d: status = %d, want 200 — a dead upstream must cost a retry, not a user-visible error", i, code) } if body != "A" { t.Fatalf("request %d: body = %q, want %q (the live upstream)", i, body, "A") } } if got := hitsA.Load(); got != 10 { t.Fatalf("live upstream served %d requests, want 10", got) } // The fast path: frozenHealth's cadence cannot have ejected anything, so // this can only be the in-request dial failure. if live := e.probers.Live("ha.example.com"); !slices.Equal(live, []string{addrA}) { t.Fatalf("live set = %v, want [%s] — a failed dial must eject %s immediately", live, addrA, dead) } } func TestAllUpstreamsDownAnswers503WithRetryAfter(t *testing.T) { dead1, dead2 := deadAddr(t), deadAddr(t) reg := openReg(t) if err := reg.Upsert(registry.Route{ Host: "ha.example.com", Kind: "service", Status: "ready", Addrs: dead1 + "," + dead2, Health: healthJSON(t, frozenHealth()), }); err != nil { t.Fatalf("upsert: %v", err) } e, err := New(reg, nil) if err != nil { t.Fatalf("New: %v", err) } defer e.probers.Stop() srv := httptest.NewServer(e) defer srv.Close() resp := getHostHeader(t, srv.URL, "ha.example.com") if resp.StatusCode != http.StatusServiceUnavailable { t.Fatalf("status = %d, want 503 (a dark upstream set is 'come back shortly', not the stdlib's bare 502)", resp.StatusCode) } if got := resp.Header.Get("Retry-After"); got != "2" { t.Fatalf("Retry-After = %q, want %q", got, "2") } // A second request must answer the same way rather than hanging or // escalating: every upstream is ejected by now, so the live set is empty // and the transport never dials at all. resp2 := getHostHeader(t, srv.URL, "ha.example.com") if resp2.StatusCode != http.StatusServiceUnavailable || resp2.Header.Get("Retry-After") != "2" { t.Fatalf("second request = %d Retry-After %q, want 503/2 with an empty live set", resp2.StatusCode, resp2.Header.Get("Retry-After")) } if live := e.probers.Live("ha.example.com"); len(live) != 0 { t.Fatalf("live set = %v, want empty after both upstreams refused", live) } } func TestSingleAddrWithHealthGets503Semantics(t *testing.T) { dead := deadAddr(t) reg := openReg(t) // One upstream, carried in the LEGACY Addr column, but with health // configured: probing is what buys the 503 semantics, not the count. if err := reg.Upsert(registry.Route{ Host: "solo.example.com", Kind: "service", Status: "ready", Addr: dead, Health: healthJSON(t, frozenHealth()), }); err != nil { t.Fatalf("upsert: %v", err) } e, err := New(reg, nil) if err != nil { t.Fatalf("New: %v", err) } defer e.probers.Stop() srv := httptest.NewServer(e) defer srv.Close() resp := getHostHeader(t, srv.URL, "solo.example.com") if resp.StatusCode != http.StatusServiceUnavailable { t.Fatalf("status = %d, want 503 for a probed single upstream", resp.StatusCode) } if got := resp.Header.Get("Retry-After"); got != "2" { t.Fatalf("Retry-After = %q, want %q", got, "2") } // A probed single-addr route is on the dashboard like any other. if st, ok := e.probers.State()["solo.example.com"]; !ok || len(st) != 1 || st[0].Addr != dead { t.Fatalf("prober state = %v, want one entry for %s", st, dead) } } // TestUpstreamStatesExportsProbeTable pins Task 6's accessor: the box status // snapshot's only door onto the probe table is Edge.UpstreamStates, which must // report exactly what the unexported proberSet holds — same hosts, same addrs, // same order — and must be safe to call on an edge with nothing probed (a // pure-proxy `carlos edge`, which is most of them), where it reports empty // rather than nil-panicking or inventing a host. func TestUpstreamStatesExportsProbeTable(t *testing.T) { addrA, _ := countingBackend(t, "A") dead := deadAddr(t) reg := openReg(t) if err := reg.Upsert(registry.Route{ Host: "ha.example.com", Kind: "service", Status: "ready", Addrs: addrA + "," + dead, Health: healthJSON(t, frozenHealth()), }); err != nil { t.Fatalf("upsert: %v", err) } // A second, UNPROBED route: no health config, so it holds no prober and // must not appear in the exported table at all. if err := reg.Upsert(registry.Route{ Host: "plain.example.com", Kind: "service", Status: "ready", Addr: addrA, }); err != nil { t.Fatalf("upsert: %v", err) } e, err := New(reg, nil) if err != nil { t.Fatalf("New: %v", err) } defer e.probers.Stop() got := e.UpstreamStates() if len(got) != 1 { t.Fatalf("UpstreamStates() = %+v, want exactly the one probed host", got) } states := got["ha.example.com"] if len(states) != 2 || states[0].Addr != addrA || states[1].Addr != dead { t.Fatalf("states = %+v, want [%s %s] in the route's addr order", states, addrA, dead) } // Optimistic start (prober.go rule 1) is what the snapshot reports for a // freshly synced host — frozenHealth cannot have ticked a transition in. for _, st := range states { if !st.Up || st.Since.IsZero() { t.Errorf("state %+v: a freshly synced upstream must be up with a Since stamp", st) } } // The exported view is a COPY: mutating it must not reach the live table // the request path picks from. states[0].Up = false if again := e.UpstreamStates()["ha.example.com"]; !again[0].Up { t.Error("UpstreamStates() handed out a view of live state — the snapshot must never be able to eject an upstream") } // An edge with nothing probed answers empty, not nil-panic. plainReg := openReg(t) if err := plainReg.Upsert(registry.Route{Host: "plain.example.com", Kind: "service", Status: "ready", Addr: addrA}); err != nil { t.Fatalf("upsert: %v", err) } plain, err := New(plainReg, nil) if err != nil { t.Fatalf("New (unprobed): %v", err) } defer plain.probers.Stop() if st := plain.UpstreamStates(); len(st) != 0 { t.Errorf("UpstreamStates() on an unprobed edge = %+v, want empty", st) } } func TestLegacySingleAddrRouteUnchanged(t *testing.T) { dead := deadAddr(t) reg := openReg(t) if err := reg.Upsert(registry.Route{ Host: "legacy.example.com", Kind: "service", Status: "ready", Addr: dead, }); err != nil { t.Fatalf("upsert: %v", err) } e, err := New(reg, nil) if err != nil { t.Fatalf("New: %v", err) } defer e.probers.Stop() srv := httptest.NewServer(e) defer srv.Close() resp := getHostHeader(t, srv.URL, "legacy.example.com") if resp.StatusCode != http.StatusBadGateway { t.Fatalf("status = %d, want 502 — an UNPROBED route must keep its pre-multi-upstream behavior byte for byte", resp.StatusCode) } if got := resp.Header.Get("Retry-After"); got != "" { t.Fatalf("Retry-After = %q, want absent on the legacy path", got) } if st := e.probers.State(); len(st) != 0 { t.Fatalf("prober state = %v, want empty — a route with no health config is not probed", st) } } func TestReloadSwapsUpstreamSet(t *testing.T) { addrA, _ := countingBackend(t, "A") addrB, _ := countingBackend(t, "B") health := healthJSON(t, frozenHealth()) reg := openReg(t) if err := reg.Upsert(registry.Route{ Host: "ha.example.com", Kind: "service", Status: "ready", Addrs: addrA, Health: health, }); err != nil { t.Fatalf("upsert: %v", err) } e, err := New(reg, nil) if err != nil { t.Fatalf("New: %v", err) } defer e.probers.Stop() srv := httptest.NewServer(e) defer srv.Close() if code, body := getHost(t, srv.URL, "ha.example.com"); code != http.StatusOK || body != "A" { t.Fatalf("pre-reload = %d %q, want 200 %q", code, body, "A") } // Write via a SECOND handle, the cross-process shape a reconcile pass has. reg2, err := registry.Open(reg.Path()) if err != nil { t.Fatalf("open second handle: %v", err) } if err := reg2.SetUpstreams("ha.example.com", "", addrB, health); err != nil { t.Fatalf("SetUpstreams: %v", err) } if err := e.Reload(); err != nil { t.Fatalf("Reload: %v", err) } if code, body := getHost(t, srv.URL, "ha.example.com"); code != http.StatusOK || body != "B" { t.Fatalf("post-reload = %d %q, want 200 %q (the new upstream set must be served)", code, body, "B") } st := e.probers.State()["ha.example.com"] if len(st) != 1 || st[0].Addr != addrB { t.Fatalf("prober state = %v, want exactly one entry for %s — the probe specs must follow the route", st, addrB) } } // upgradeEchoBackend answers a protocol-switch handshake the way a WebSocket // server does — 101, then raw bytes on the hijacked connection — prefixing // whatever it reads with tag so a test can tell WHICH upstream served it. func upgradeEchoBackend(t *testing.T, tag string) string { t.Helper() ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.URL.Path == testProbePath { w.WriteHeader(http.StatusOK) return } hj, ok := w.(http.Hijacker) if !ok { http.Error(w, "not hijackable", http.StatusInternalServerError) return } conn, buf, err := hj.Hijack() if err != nil { return } defer conn.Close() buf.WriteString("HTTP/1.1 101 Switching Protocols\r\nUpgrade: websocket\r\nConnection: Upgrade\r\n\r\n") if err := buf.Flush(); err != nil { return } line, err := buf.ReadString('\n') if err != nil { return } buf.WriteString(tag + ":" + line) buf.Flush() })) t.Cleanup(ts.Close) return ts.Listener.Addr().String() } // TestMultiUpstreamServesWebSocketUpgrade guards the one thing multiProxy's // upstream keep-alive suppression could plausibly have broken. A transport // with DisableKeepAlives normally adds `Connection: close` to its outbound // request, which would poison a protocol-switch handshake — the stdlib // exempts upgrade requests from that (transport.go's isProtocolSwitchHeader // check), and this test is what keeps that exemption load-bearing rather than // assumed. func TestMultiUpstreamServesWebSocketUpgrade(t *testing.T) { addr := upgradeEchoBackend(t, "ws") reg := openReg(t) if err := reg.Upsert(registry.Route{ Host: "ws.example.com", Kind: "service", Status: "ready", Addrs: addr, Health: healthJSON(t, frozenHealth()), }); err != nil { t.Fatalf("upsert: %v", err) } e, err := New(reg, nil) if err != nil { t.Fatalf("New: %v", err) } defer e.probers.Stop() srv := httptest.NewServer(e) defer srv.Close() // Pin that this route really took the multiProxy arm: only a probed route // has a prober, so without this the test would still pass if the switch // had quietly built a plain tcpProxy — and it is tcpProxy's transport, // not multiProxy's keep-alive-free one, that the upgrade path is being // checked against. if live := e.probers.Live("ws.example.com"); len(live) != 1 { t.Fatalf("live set = %v, want one probed upstream — this test must exercise multiProxy, not tcpProxy", live) } c, err := net.Dial("tcp", strings.TrimPrefix(srv.URL, "http://")) if err != nil { t.Fatalf("dial edge: %v", err) } defer c.Close() if err := c.SetDeadline(time.Now().Add(10 * time.Second)); err != nil { t.Fatalf("set deadline: %v", err) } if _, err := fmt.Fprint(c, "GET /socket HTTP/1.1\r\nHost: ws.example.com\r\nConnection: Upgrade\r\nUpgrade: websocket\r\n\r\n"); err != nil { t.Fatalf("write handshake: %v", err) } br := bufio.NewReader(c) resp, err := http.ReadResponse(br, nil) if err != nil { t.Fatalf("read handshake response: %v", err) } if resp.StatusCode != http.StatusSwitchingProtocols { t.Fatalf("handshake status = %d, want 101 — an upgrade over a multi-upstream route must switch protocols", resp.StatusCode) } if _, err := fmt.Fprint(c, "ping\n"); err != nil { t.Fatalf("write post-upgrade frame: %v", err) } line, err := br.ReadString('\n') if err != nil { t.Fatalf("read post-upgrade frame: %v", err) } if line != "ws:ping\n" { t.Fatalf("post-upgrade echo = %q, want %q — bytes must flow both ways after the switch", line, "ws:ping\n") } } // TestUnparseableHealthServesUnprobed pins the parse-failure contract: a // health column the edge cannot read costs the route its PROBING, never its // SERVICE. The alternative — skipping the route, or building a proxy over an // empty live set — turns one bad character in a config column into a dead // host, which is the one outcome this branch exists to prevent. func TestUnparseableHealthServesUnprobed(t *testing.T) { addr, hits := countingBackend(t, "served anyway") reg := openReg(t) if err := reg.Upsert(registry.Route{ Host: "broken.example.com", Kind: "service", Status: "ready", Addrs: addr, Health: "{not json", }); err != nil { t.Fatalf("upsert: %v", err) } e, err := New(reg, nil) if err != nil { t.Fatalf("New: %v", err) } defer e.probers.Stop() srv := httptest.NewServer(e) defer srv.Close() code, body := getHost(t, srv.URL, "broken.example.com") if code != http.StatusOK || body != "served anyway" { t.Fatalf("= %d %q, want 200 %q — an unreadable health config must not take the host off the air", code, body, "served anyway") } if got := hits.Load(); got != 1 { t.Fatalf("backend served %d requests, want 1", got) } if st := e.probers.State(); len(st) != 0 { t.Fatalf("prober state = %v, want empty — a health config that did not parse cannot become a probing job", st) } } // TestMultiAddrWithoutHealthServesFirstUnprobed covers the row shape nothing // mints but the switch must still answer for: several addrs, no health // config. Before multi-upstream routing such a row was served by NOTHING (the // old arm gated on the legacy Addr column, which this shape leaves empty), so // the behavior asserted here is new by construction and needs pinning: // upstream ONE is served, unprobed, with the legacy 502 semantics. Balancing // is deliberately not offered — without a probe there is no live set, and // rotating onto upstreams nothing is checking would be worse than using one. func TestMultiAddrWithoutHealthServesFirstUnprobed(t *testing.T) { first, firstHits := countingBackend(t, "first") second, secondHits := countingBackend(t, "second") dead := deadAddr(t) reg := openReg(t) if err := reg.Upsert(registry.Route{ Host: "unprobed.example.com", Kind: "service", Status: "ready", Addrs: first + "," + second, }); err != nil { t.Fatalf("upsert: %v", err) } // A second route of the same shape whose first upstream is dead, so the // failure semantics can be asserted alongside the success ones. if err := reg.Upsert(registry.Route{ Host: "unprobed-dead.example.com", Kind: "service", Status: "ready", Addrs: dead + "," + second, }); err != nil { t.Fatalf("upsert: %v", err) } e, err := New(reg, nil) if err != nil { t.Fatalf("New: %v", err) } defer e.probers.Stop() srv := httptest.NewServer(e) defer srv.Close() for i := 0; i < 6; i++ { code, body := getHost(t, srv.URL, "unprobed.example.com") if code != http.StatusOK || body != "first" { t.Fatalf("request %d = %d %q, want 200 %q (addrs[0], unprobed)", i, code, body, "first") } } if got := firstHits.Load(); got != 6 { t.Fatalf("first upstream served %d requests, want all 6", got) } if got := secondHits.Load(); got != 0 { t.Fatalf("second upstream served %d requests, want 0 — an unprobed route must not balance onto upstreams nothing is checking", got) } // Dead first upstream: the LEGACY failure shape, not the probed one. No // live set exists to fail away from, so there is nothing to promise a // client about coming back. resp := getHostHeader(t, srv.URL, "unprobed-dead.example.com") if resp.StatusCode != http.StatusBadGateway { t.Fatalf("dead first upstream = %d, want 502 (the unprobed path keeps the stdlib behavior)", resp.StatusCode) } if got := resp.Header.Get("Retry-After"); got != "" { t.Fatalf("Retry-After = %q, want absent — 503 semantics are bought by probing, not by the addr count", got) } if got := secondHits.Load(); got != 0 { t.Fatalf("second upstream served %d requests, want 0 — the unprobed path must not retry onto a sibling upstream", got) } if st := e.probers.State(); len(st) != 0 { t.Fatalf("prober state = %v, want empty — neither route carries a health config", st) } } // TestServeHTTPMetersRequestsAndBytesExcludingStatus drives a full Edge over // HTTP (mirroring TestRoutesHostToTCPUpstream) with EnableUsage armed, and // proves the single wrap site in ServeHTTP counts every plain-path request // while a Kind=="status" route (the platform's own status page — not // billable, per the placement rules) never shows up in Drain(). func TestServeHTTPMetersRequestsAndBytesExcludingStatus(t *testing.T) { body := "metered response body" backend, _ := hostEchoBackend(t, body) addr := backend.Listener.Addr().String() reg := openReg(t) if err := reg.Upsert(registry.Route{Host: "metered.example.com", Kind: "service", Addr: addr, Account: "bab", App: "hello", Status: "ready"}); err != nil { t.Fatalf("upsert metered route: %v", err) } if err := reg.Upsert(registry.Route{Host: "status.example.com", Kind: "status", Status: "ready"}); err != nil { t.Fatalf("upsert status route: %v", err) } e, err := New(reg, nil) if err != nil { t.Fatalf("New: %v", err) } // Static/status serving needs a store wired in for the status route to // answer at all (see testEdgeWithStore); an empty pubkey is fine since // the status class never verifies signatures. store := release.NewDirStore(filepath.Join(t.TempDir(), "store")) _, pubB64, err := release.GenerateKey() if err != nil { t.Fatalf("generate key: %v", err) } pub, err := release.ParsePublicKey(pubB64, "test") if err != nil { t.Fatalf("parse public key: %v", err) } e.EnableStatic(store, pub, filepath.Join(t.TempDir(), "cache")) store.Put(context.Background(), "status/index.html", []byte("status"), "") if err := e.Reload(); err != nil { t.Fatalf("Reload: %v", err) } meter := NewUsageMeter() e.EnableUsage(meter) srv := httptest.NewServer(e) defer srv.Close() for i := 0; i < 2; i++ { code, got := getHost(t, srv.URL, "metered.example.com") if code != http.StatusOK || got != body { t.Fatalf("request %d: %d %q, want 200 %q", i, code, got, body) } } if code, _ := getHost(t, srv.URL, "status.example.com"); code != http.StatusOK { t.Fatalf("status route: %d, want 200", code) } rows := meter.Drain() row := rows["metered.example.com"] wantBytes := int64(2 * len(body)) if row.Requests != 2 || row.BytesOut != wantBytes { t.Fatalf("metered.example.com row = %+v, want Requests=2 BytesOut=%d", row, wantBytes) } if row.Account != "bab" || row.App != "hello" { t.Fatalf("metered.example.com row = %+v, want Account=bab App=hello", row) } if _, ok := rows["status.example.com"]; ok { t.Fatalf("status route must not be metered, got %+v", rows["status.example.com"]) } } // TestSidecarRouteIsNeverServable is the structural guarantee behind the // sidecar member kind: a sidecar is a process that serves nothing and that // nothing may dial, and the edge must refuse it on KIND rather than on the // accident of it carrying no backend columns. // // The row below is adversarial on purpose. It carries a Socket AND an Addr // AND a Tunnel AND a fleet placement (Region + FleetBox), on an edge with a // tunnel hub armed and the fleet registered — every column and every // precondition that any arm of Reload's dispatch switch looks at, all at // once. Eleven's sidecars are fleet-placed, so the fleet arm (which checks no // kind at all) is reached by such a row on shape alone. If this test ever // fails, a sidecar is reachable from the internet. func TestSidecarRouteIsNeverServable(t *testing.T) { const host = "bot.eleven.bdm.sidecar.internal" reg := openReg(t) if err := reg.UpsertFleet(registry.Fleet{Name: "eleven", Account: "amadan", Bucket: "eleven-data"}); err != nil { t.Fatalf("UpsertFleet: %v", err) } if err := reg.AddFleetBox(registry.FleetBox{Fleet: "eleven", Label: "box1", TokenHash: sha256Hex("tok"), Status: "active"}); err != nil { t.Fatalf("AddFleetBox: %v", err) } if err := reg.Upsert(registry.Route{ Host: host, Kind: "sidecar", Socket: filepath.Join(t.TempDir(), "whatever.sock"), Addr: "127.0.0.1:9", Tunnel: "eleven/box1", Region: "fleet/eleven", FleetBox: "box1", Status: "ready", App: "eleven", Account: "bdm", Platform: true, }); err != nil { t.Fatalf("upsert sidecar route: %v", err) } e, err := New(reg, nil) if err != nil { t.Fatalf("New: %v", err) } // Arm the hub, so the Tunnel and fleet arms of the dispatch switch are // live rather than skipped by their e.tunnels == nil guard. e.EnableTunnels(NewTunnelHub("tunnel.example.com", nil)) if err := e.Reload(); err != nil { t.Fatalf("Reload: %v", err) } if _, ok := e.current()[host]; ok { t.Fatalf("a sidecar row entered the routing map — it is reachable from the internet") } // Allowed is the autocert HostPolicy: a sidecar host must never cost an // ACME issuance either. if e.Allowed(host) { t.Fatalf("Allowed(%s) = true — a certificate must never be requested for a sidecar host", host) } srv := httptest.NewServer(e) defer srv.Close() if code, _ := getHost(t, srv.URL, host); code != http.StatusNotFound { t.Fatalf("GET %s = %d, want 404 — a sidecar must be unreachable", host, code) } }