package edge import ( "context" "errors" "io" "net" "net/http" "net/http/httptest" "strings" "sync" "syscall" "testing" "time" "github.com/carlosframework/platform/internal/apperr" "github.com/carlosframework/platform/internal/registry" "github.com/carlosframework/platform/internal/release" ) type memErrSink struct { mu sync.Mutex recs []apperr.Record apps []string } func (m *memErrSink) Add(acct, app string, r apperr.Record) { m.mu.Lock() defer m.mu.Unlock() m.recs = append(m.recs, r) m.apps = append(m.apps, acct+"/"+app) } // snap copies the sink's records under the lock — every assertion reads // through this rather than the fields directly, so a test can never // observe a torn append from a concurrent Add. func (m *memErrSink) snap() []apperr.Record { m.mu.Lock() defer m.mu.Unlock() out := make([]apperr.Record, len(m.recs)) copy(out, m.recs) return out } func TestEdgeEmitsErrorOnUpstream5xx(t *testing.T) { backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch r.URL.Path { case "/boom": http.Error(w, "kaboom", http.StatusInternalServerError) case "/missing": http.NotFound(w, r) default: w.Header().Set("Content-Type", "text/html") w.Write([]byte("

ok

")) } })) t.Cleanup(backend.Close) reg := openReg(t) rt := registry.Route{Host: "app.example.com", Kind: "service", Addr: backend.Listener.Addr().String(), Status: "ready", App: "web", Version: "v7"} if err := reg.Upsert(rt); err != nil { t.Fatal(err) } e, err := New(reg, nil) if err != nil { t.Fatal(err) } sink := &memErrSink{} e.EnableErrors(sink) srv := httptest.NewServer(e) t.Cleanup(srv.Close) do := func(method, path string) int { req, _ := http.NewRequest(method, srv.URL+path, nil) req.Host = "app.example.com" resp, err := http.DefaultClient.Do(req) if err != nil { t.Fatal(err) } resp.Body.Close() return resp.StatusCode } if do("POST", "/boom?token=abc") != 500 || do("GET", "/missing") != 404 || do("GET", "/") != 200 { t.Fatal("upstream statuses must pass through unchanged") } recs := sink.snap() if len(recs) != 1 { t.Fatalf("errors emitted = %d, want 1 (the 500 only)", len(recs)) } r := recs[0] if r.Source != apperr.SourceEdge || r.Kind != apperr.KindHTTP || r.Status != 500 || r.Host != "app.example.com" || r.Release != "v7" { t.Fatalf("record = %+v", r) } if r.Name != "500 POST /boom" || r.Path != "/boom" || strings.Contains(r.Name, "token") { t.Fatalf("name/path = %q %q", r.Name, r.Path) } // The route carries no Account, so RouteAccount falls back to // OpsAccount: attribution must land on the ops account, not "". if sink.apps[0] != release.OpsAccount+"/web" { t.Fatalf("attribution = %q, want %q", sink.apps[0], release.OpsAccount+"/web") } } // answering503 is an Activator that answers every request itself with the // wake-path 503 the real one writes, and marks it as the activator's own — // the same MarkAnswered call internal/activator's serve makes before each of // its four wake-path http.Error(…, 503) writes. type answering503 struct{} func (answering503) ServeHTTP(w http.ResponseWriter, r *http.Request, rt registry.Route, proxy http.Handler) bool { MarkAnswered(r) http.Error(w, "instance failed to wake", http.StatusServiceUnavailable) return true } func (answering503) RetireExcept(map[string]bool) {} func TestEdgeSkipsActivatorAnswered503(t *testing.T) { reg := openReg(t) rt := registry.Route{Host: "sleepy.example.com", Kind: "instance", Socket: t.TempDir() + "/x.sock", Status: "ready", Hibernate: true, App: "web"} if err := reg.Upsert(rt); err != nil { t.Fatal(err) } e, err := New(reg, answering503{}) if err != nil { t.Fatal(err) } sink := &memErrSink{} e.EnableErrors(sink) rec := httptest.NewRecorder() req := httptest.NewRequest("GET", "http://sleepy.example.com/", nil) e.ServeHTTP(rec, req) if rec.Code != 503 { t.Fatalf("status = %d", rec.Code) } if recs := sink.snap(); len(recs) != 0 { t.Fatalf("an activator-answered 503 is already a platform log error; got %d edge errors", len(recs)) } } // plain503 is answering503's negative twin: it writes the same wake-path // 503 but never calls MarkAnswered, so the edge must not mistake it for the // activator's own already-logged error. type plain503 struct{} func (plain503) ServeHTTP(w http.ResponseWriter, r *http.Request, rt registry.Route, proxy http.Handler) bool { http.Error(w, "instance failed to wake", http.StatusServiceUnavailable) return true } func (plain503) RetireExcept(map[string]bool) {} // TestEdgeEmitsUnmarked503OnHibernatingRoute is the positive control for // TestEdgeSkipsActivatorAnswered503: with the MarkAnswered call gone, the // same hibernating route's 503 must be filed as exactly one edge error. func TestEdgeEmitsUnmarked503OnHibernatingRoute(t *testing.T) { reg := openReg(t) rt := registry.Route{Host: "sleepy.example.com", Kind: "instance", Socket: t.TempDir() + "/x.sock", Status: "ready", Hibernate: true, App: "web"} if err := reg.Upsert(rt); err != nil { t.Fatal(err) } e, err := New(reg, plain503{}) if err != nil { t.Fatal(err) } sink := &memErrSink{} e.EnableErrors(sink) rec := httptest.NewRecorder() req := httptest.NewRequest("GET", "http://sleepy.example.com/", nil) e.ServeHTTP(rec, req) if rec.Code != 503 { t.Fatalf("status = %d", rec.Code) } recs := sink.snap() if len(recs) != 1 { t.Fatalf("errors emitted = %d, want 1 (the unmarked 503)", len(recs)) } if recs[0].Status != 503 || recs[0].Source != apperr.SourceEdge { t.Fatalf("record = %+v", recs[0]) } } // TestEdgeFilesUnavailableByCause: a 502/503/504 goes under Availability, // named by status and cause rather than path, so one dead instance probed // on two paths is one fingerprint — and a 503 the app wrote itself says so. func TestEdgeFilesUnavailableByCause(t *testing.T) { backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { http.Error(w, "busy", http.StatusServiceUnavailable) })) t.Cleanup(backend.Close) dead := httptest.NewServer(http.NotFoundHandler()) deadAddr := dead.Listener.Addr().String() dead.Close() // nothing listens here now: every dial is refused reg := openReg(t) for _, rt := range []registry.Route{ {Host: "busy.example.com", Kind: "service", Addr: backend.Listener.Addr().String(), Status: "ready", App: "web", Version: "v7"}, {Host: "dead.example.com", Kind: "service", Addr: deadAddr, Status: "ready", App: "web", Version: "v7"}, } { if err := reg.Upsert(rt); err != nil { t.Fatal(err) } } e, err := New(reg, nil) if err != nil { t.Fatal(err) } sink := &memErrSink{} e.EnableErrors(sink) srv := httptest.NewServer(e) t.Cleanup(srv.Close) do := func(host, path string) int { req, _ := http.NewRequest("GET", srv.URL+path, nil) req.Host = host resp, err := http.DefaultClient.Do(req) if err != nil { t.Fatal(err) } resp.Body.Close() return resp.StatusCode } if got := do("busy.example.com", "/sandbox"); got != 503 { t.Fatalf("busy status = %d", got) } if do("dead.example.com", "/.env") != 502 || do("dead.example.com", "/phpinfo.php") != 502 { t.Fatal("a refused dial must answer 502") } recs := sink.snap() if len(recs) != 3 { t.Fatalf("records = %d, want 3", len(recs)) } busy := recs[0] if busy.Kind != apperr.KindUnavailable || busy.Cause != apperr.CauseUpstream || busy.Name != "503 · the app answered it itself" || busy.Path != "/sandbox" { t.Fatalf("busy record = %+v", busy) } for _, r := range recs[1:] { if r.Kind != apperr.KindUnavailable || r.Cause != apperr.CauseRefused || r.Status != 502 || strings.Contains(r.Name, "/") { t.Fatalf("dead record = %+v", r) } } if recs[1].Name != recs[2].Name || recs[1].Path == recs[2].Path { t.Fatalf("two paths of one refused instance must share a name and keep their own paths: %+v", recs[1:]) } if !apperr.IsAvailability(busy) { t.Fatal("IsAvailability(busy) = false") } } // TestProxyCauseClientGone: a proxy failure after the client left is the // client's doing, whatever the error says, and is never filed. func TestProxyCauseClientGone(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) cancel() r := httptest.NewRequest("GET", "http://x/", nil).WithContext(ctx) if got := proxyCause(r, syscall.ECONNRESET); got != causeClientGone { t.Fatalf("proxyCause = %q, want %q", got, causeClientGone) } r, out := withOutcome(httptest.NewRequest("GET", "http://x/", nil)) for _, c := range []struct { err error want string }{ {&net.OpError{Op: "dial", Err: syscall.ECONNREFUSED}, apperr.CauseRefused}, {io.ErrUnexpectedEOF, apperr.CauseReset}, {context.DeadlineExceeded, apperr.CauseTimeout}, {errors.New("malformed"), apperr.CauseProxy}, } { if got := proxyCause(r, c.err); got != c.want { t.Errorf("proxyCause(%v) = %q, want %q", c.err, got, c.want) } } // The same leaving after a long wait is the instance's doing: a dead // socket redialled for 13s, or a wedged instance nothing times out. gone, o := withOutcome(httptest.NewRequest("GET", "http://x/", nil).WithContext(ctx)) o.start = time.Now().Add(-clientGoneGrace - time.Second) if got := proxyCause(gone, syscall.ECONNREFUSED); got != apperr.CauseAbandoned { t.Fatalf("long wait then gone: proxyCause = %q, want %q", got, apperr.CauseAbandoned) } if got := tunnelCause(gone, syscall.ECONNREFUSED); got != apperr.CauseAbandoned { t.Fatalf("long wait then gone via tunnel: %q", got) } noteCause(r, apperr.CauseReset) if out.cause != apperr.CauseReset { t.Fatalf("noteCause did not reach the outcome: %+v", out) } sink := &memErrSink{} e := &Edge{} e.EnableErrors(sink) e.emitEdgeError(registry.Route{Host: "h", App: "web"}, false, r, 502, causeClientGone) if n := len(sink.snap()); n != 0 { t.Fatalf("a client-gone 502 was filed (%d records)", n) } } // TestEdgeFilesOnlyItsOwnFailuresThroughATunnel: when this box proxies over // a reverse channel, the upstream is another box's edge, which files what // it saw serving the app. This box files the hop failing, never a status // that merely passed through it. func TestEdgeFilesOnlyItsOwnFailuresThroughATunnel(t *testing.T) { sink := &memErrSink{} e := &Edge{} e.EnableErrors(sink) r, _ := withOutcome(httptest.NewRequest("GET", "http://x/admin", nil)) rt := registry.Route{Host: "t.example.com", App: "web", Tunnel: "box1"} e.emitEdgeError(rt, true, r, 503, "") // the serving box's own 503, passed through e.emitEdgeError(rt, true, r, 500, "") // the app's 500, filed on the serving box if n := len(sink.snap()); n != 0 { t.Fatalf("pass-through statuses were filed on the front box: %+v", sink.snap()) } e.emitEdgeError(rt, true, r, 502, apperr.CauseTunnel) e.emitEdgeError(rt, true, r, 503, apperr.CauseFleetDark) if recs := sink.snap(); len(recs) != 2 || recs[0].Cause != apperr.CauseTunnel || recs[1].Cause != apperr.CauseFleetDark { t.Fatalf("the hop's own failures must be filed: %+v", recs) } } // TestEdgeServingFleetBoxFilesItsOwn is the other end of the same route: // the fleet box that SERVES a fleet-placed app carries the placement on its // row too, but has no hub and serves from the app's own address. Everything // the app answers there must be filed — it is the only box that can. func TestEdgeServingFleetBoxFilesItsOwn(t *testing.T) { backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.URL.Path == "/busy" { http.Error(w, "busy", http.StatusServiceUnavailable) return } http.Error(w, "boom", http.StatusInternalServerError) })) t.Cleanup(backend.Close) reg := openReg(t) rt := registry.Route{Host: "f.example.com", Kind: "service", Addr: backend.Listener.Addr().String(), Status: "ready", App: "web", Region: registry.PlacementTarget("eleven"), Tunnel: "box1"} if err := reg.Upsert(rt); err != nil { t.Fatal(err) } e, err := New(reg, nil) // no hub: this box is the far end, not the front if err != nil { t.Fatal(err) } sink := &memErrSink{} e.EnableErrors(sink) for _, path := range []string{"/boom", "/busy"} { rec := httptest.NewRecorder() e.ServeHTTP(rec, httptest.NewRequest("GET", "http://f.example.com"+path, nil)) } recs := sink.snap() if len(recs) != 2 || recs[0].Status != 500 || recs[1].Cause != apperr.CauseUpstream { t.Fatalf("the serving box must file its app's 500 and 503: %+v", recs) } } // TestEdgeFilesAbandonedWaitEndToEnd drives a real request at an instance // that never answers, through ServeHTTP and the proxy, with a client that // gives up after the grace: the wait is measured from where the edge // attaches the outcome, and the outage is filed rather than hidden. func TestEdgeFilesAbandonedWaitEndToEnd(t *testing.T) { old := clientGoneGrace clientGoneGrace = 50 * time.Millisecond t.Cleanup(func() { clientGoneGrace = old }) release := make(chan struct{}) backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { select { case <-release: case <-r.Context().Done(): } })) t.Cleanup(func() { close(release); backend.Close() }) reg := openReg(t) if err := reg.Upsert(registry.Route{Host: "wedged.example.com", Kind: "service", Addr: backend.Listener.Addr().String(), Status: "ready", App: "web"}); err != nil { t.Fatal(err) } e, err := New(reg, nil) if err != nil { t.Fatal(err) } sink := &memErrSink{} e.EnableErrors(sink) ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) defer cancel() req := httptest.NewRequest("GET", "http://wedged.example.com/slow", nil).WithContext(ctx) e.ServeHTTP(httptest.NewRecorder(), req) recs := sink.snap() if len(recs) != 1 || recs[0].Cause != apperr.CauseAbandoned || recs[0].Status != 502 { t.Fatalf("records = %+v, want one abandoned 502", recs) } } // TestEdgeFilesHopFailuresThroughServeHTTP pins the other half of the // tunnel rule end to end: the front box files nothing a proxy did not note // a cause for, so each hop failure it produces itself MUST note one, or it // is filed nowhere. A hub with no connected channel answers a -tunnel route // with 502 (tunnel) and a fleet route with 503 (fleet_dark). func TestEdgeFilesHopFailuresThroughServeHTTP(t *testing.T) { reg := openReg(t) if err := reg.UpsertFleet(registry.Fleet{Name: "acme", Account: "amadan", Bucket: "acme-data"}); err != nil { t.Fatal(err) } if err := reg.AddFleetBox(registry.FleetBox{Fleet: "acme", Label: "pi-1", TokenHash: sha256Hex("tok1"), Status: "active"}); err != nil { t.Fatal(err) } for _, rt := range []registry.Route{ {Host: "fleet.example", Kind: "instance", Status: "ready", App: "web", Region: registry.PlacementTarget("acme"), FleetBox: "pi-1"}, {Host: "tun.example", Kind: "service", Status: "ready", App: "web", Addr: "127.0.0.1:1", Tunnel: "pi-box"}, } { if err := reg.Upsert(rt); err != nil { t.Fatal(err) } } if err := reg.SetTunnel("tun.example", "pi-box"); err != nil { t.Fatal(err) } e, err := New(reg, nil) if err != nil { t.Fatal(err) } e.EnableTunnels(NewTunnelHub("tunnel.test", map[string]string{"pi-box": "sekrit"})) if err := e.Reload(); err != nil { t.Fatal(err) } sink := &memErrSink{} e.EnableErrors(sink) for host, want := range map[string]int{"fleet.example": 503, "tun.example": 502} { rec := httptest.NewRecorder() e.ServeHTTP(rec, httptest.NewRequest("GET", "http://"+host+"/", nil)) if rec.Code != want { t.Fatalf("%s = %d, want %d", host, rec.Code, want) } } causes := map[string]string{} for _, r := range sink.snap() { causes[r.Host] = r.Cause } if len(causes) != 2 || causes["fleet.example"] != apperr.CauseFleetDark || causes["tun.example"] != apperr.CauseTunnel { t.Fatalf("hop failures filed = %+v, want fleet_dark and tunnel, one each", sink.snap()) } }