package edge // The edge-local door (#175 P7, Tasks 5 to 7). Every request here is driven // through serveDoor, the door's own handler, exactly as the unix listener // hands it one, except where a test says it drives the real socket. import ( "bytes" "context" "fmt" "io" "net" "net/http" "net/http/httptest" "net/url" "os" "path/filepath" "strings" "sync" "testing" "time" "github.com/carlosframework/platform/internal/doorkey" "github.com/carlosframework/platform/internal/gate" "github.com/carlosframework/platform/internal/registry" "github.com/carlosframework/platform/internal/release" "github.com/carlosframework/platform/internal/seeds" ) // doorTestAddr is the box's public address in these tests. It is TEST-NET-3 // (RFC 5737): global unicast and not private by Go's rules, so openDoor // accepts it. const doorTestAddr = "203.0.113.10" // ---- the upstream ----------------------------------------------------------- // doorUpstream is a tenant app that records everything the edge handed it. type doorUpstream struct { mu sync.Mutex n int hdr http.Header peer string } func (u *doorUpstream) seen() (int, http.Header, string) { u.mu.Lock() defer u.mu.Unlock() return u.n, u.hdr.Clone(), u.peer } func newDoorUpstream(t *testing.T, body string) (*httptest.Server, *doorUpstream) { t.Helper() u := &doorUpstream{} ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { u.mu.Lock() u.n++ u.hdr = r.Header.Clone() u.peer = r.RemoteAddr u.mu.Unlock() io.WriteString(w, body) })) t.Cleanup(ts.Close) return ts, u } // ---- keys --------------------------------------------------------------------- // mint returns a raw door key for account/app and the entry a record lists // for it. func (fx *doorKeyFixture) mint(t *testing.T, account, app string) (string, doorkey.Entry) { t.Helper() raw, entry, err := doorkey.NewToken(account, app) if err != nil { t.Fatal(err) } return raw, entry } // record signs a record for account/app listing exactly entries. func (fx *doorKeyFixture) record(t *testing.T, account, app string, rev uint64, entries ...doorkey.Entry) []byte { t.Helper() now := fx.clk.now() wire, err := doorkey.Sign(fx.priv, doorkey.Record{V: 1, Account: account, App: app, Revision: rev, Keys: entries, SignedAt: now, ExpiresAt: now.Add(doorkey.Lifetime)}) if err != nil { t.Fatal(err) } return wire } // ---- the scene ----------------------------------------------------------------- const ( doorOwnHost = "own.test" // bae/titogo, the caller's app doorOtherHost = "other.test" // bae/other doorSplitHost = "split.test" // bae/titogo; /a/ to titogo, /b/ to other doorGatedHost = "gated.test" // bae/titogo, behind the password gate doorWildParent = "apps.test" // *.apps.test is an alias of own.test doorTunnelHost = "tunnel.apps.test" doorSteerHost = "edge.apps.test" doorPassword = "correct horse" doorOwnBody = "the caller's own app" doorOtherBody = "another app" doorSplitABody = "split a" doorSplitBBody = "split b" doorGatedBody = "gated page" doorFrontBody = "split front" doorHomeEdgeKey = "betty" ) type doorScene struct { e *Edge fx *doorKeyFixture store release.Store // key is a valid door key for bae/titogo; otherKey for bae/other. key, otherKey string entry doorkey.Entry own, other *doorUpstream splitA *doorUpstream splitB *doorUpstream gated *doorUpstream } // newDoorScene is an edge serving two apps' routes locally, with a door-key // mirror over its own store and the box address set. It is built the way the // package's other tests build an Edge (openReg, New), with the mirror and the // address set the way armDoor sets them. func newDoorScene(t *testing.T) *doorScene { t.Helper() s := &doorScene{fx: newDoorKeyFixture(t), store: meshTestStore(t)} ownSrv, own := newDoorUpstream(t, doorOwnBody) otherSrv, other := newDoorUpstream(t, doorOtherBody) aSrv, a := newDoorUpstream(t, doorSplitABody) bSrv, b := newDoorUpstream(t, doorSplitBBody) frontSrv, _ := newDoorUpstream(t, doorFrontBody) gatedSrv, gated := newDoorUpstream(t, doorGatedBody) s.own, s.other, s.splitA, s.splitB, s.gated = own, other, a, b, gated hash, err := gate.Hash(doorPassword) if err != nil { t.Fatal(err) } reg := openReg(t) for _, rt := range []registry.Route{ {Host: doorOwnHost, Addr: ownSrv.Listener.Addr().String(), Account: "bae", App: "titogo"}, {Host: doorOtherHost, Addr: otherSrv.Listener.Addr().String(), Account: "bae", App: "other"}, {Host: doorSplitHost, Addr: frontSrv.Listener.Addr().String(), Account: "bae", App: "titogo"}, {Host: "a.target.test", Addr: aSrv.Listener.Addr().String(), Account: "bae", App: "titogo"}, // The other app's prefix is gated, so a login POST naming it is // the one that would mint a cookie if the door let it through. {Host: "b.target.test", Addr: bSrv.Listener.Addr().String(), Account: "bae", App: "other", Gate: hash}, {Host: doorGatedHost, Addr: gatedSrv.Listener.Addr().String(), Account: "bae", App: "titogo", Gate: hash}, } { rt.Kind, rt.Status = "service", "ready" if err := reg.Upsert(rt); err != nil { t.Fatalf("upsert %s: %v", rt.Host, err) } } upsertPath(t, reg, doorSplitHost, "a", "a.target.test") upsertPath(t, reg, doorSplitHost, "b", "b.target.test") // A wildcard owned by the caller that would answer the reserved names // if the door let them through to the lookup. upsertAlias(t, reg, "*."+doorWildParent, doorOwnHost) e, err := New(reg, nil) if err != nil { t.Fatal(err) } t.Cleanup(e.probers.Stop) e.EnableTunnels(NewTunnelHub(doorTunnelHost, map[string]string{"pi-box": "sekrit"})) e.steeringHost = doorSteerHost var otherEntry doorkey.Entry s.key, s.entry = s.fx.mint(t, "bae", "titogo") s.otherKey, otherEntry = s.fx.mint(t, "bae", "other") putJSON(t, s.store, doorkey.RecordKey("bae", "titogo"), string(s.fx.record(t, "bae", "titogo", 1, s.entry))) putJSON(t, s.store, doorkey.RecordKey("bae", "other"), string(s.fx.record(t, "bae", "other", 1, otherEntry))) e.doorKeys = s.fx.mirror(s.store) refreshDoorKeys(t, e.doorKeys) e.doorAddr = doorTestAddr s.e = e return s } // do drives one request through the door with key (none if ""). func (s *doorScene) do(t *testing.T, method, host, target, key string, hdr map[string]string, body io.Reader) *httptest.ResponseRecorder { t.Helper() req := httptest.NewRequest(method, "http://"+host+target, body) req.Host = host // What a unix listener reports as the peer. req.RemoteAddr = "@" if key != "" { req.Header.Set(doorkey.Header, key) } for k, v := range hdr { req.Header.Set(k, v) } rec := httptest.NewRecorder() s.e.serveDoor(rec, req) return rec } func wantUnknownHost(t *testing.T, rec *httptest.ResponseRecorder, what string) { t.Helper() if rec.Code != http.StatusNotFound || !strings.Contains(rec.Body.String(), "unknown host") { t.Fatalf("%s: %d %q, want the unknown-host answer", what, rec.Code, rec.Body.String()) } } func wantServed(t *testing.T, rec *httptest.ResponseRecorder, body, what string) { t.Helper() if rec.Code != http.StatusOK || rec.Body.String() != body { t.Fatalf("%s: %d %q, want 200 %q", what, rec.Code, rec.Body.String(), body) } } func upstreamCalls(u *doorUpstream) int { n, _, _ := u.seen() return n } // ---- Task 5: enforcement in ServeHTTP ------------------------------------------ // 1. A door caller reaches its own app's local route. func TestDoorServesItsOwnAppsLocalRoute(t *testing.T) { s := newDoorScene(t) wantServed(t, s.do(t, http.MethodGet, doorOwnHost, "/", s.key, nil, nil), doorOwnBody, "own route") if upstreamCalls(s.own) != 1 { t.Fatal("the caller's own upstream was not reached") } } // 2. Another app's local route answers unknown host, and its upstream never // hears of it. func TestDoorRefusesAnotherAppsLocalRoute(t *testing.T) { s := newDoorScene(t) wantUnknownHost(t, s.do(t, http.MethodGet, doorOtherHost, "/", s.key, nil, nil), "another app's route") if n := upstreamCalls(s.other); n != 0 { t.Fatalf("the other app's upstream saw %d request(s)", n) } // The other app's own key reaches it, so the refusal is the scope and // not the route. wantServed(t, s.do(t, http.MethodGet, doorOtherHost, "/", s.otherKey, nil, nil), doorOtherBody, "other app with its own key") } // 3. On a path-prefix host the check is against the route the prefix picks. func TestDoorPathPrefixHostChecksTheFinalRoute(t *testing.T) { s := newDoorScene(t) wantServed(t, s.do(t, http.MethodGet, doorSplitHost, "/a/x", s.key, nil, nil), doorSplitABody, "/a/x") wantUnknownHost(t, s.do(t, http.MethodGet, doorSplitHost, "/b/x", s.key, nil, nil), "/b/x") if n := upstreamCalls(s.splitB); n != 0 { t.Fatalf("the other app's prefix upstream saw %d request(s)", n) } } // 4. A gate login POST whose next names another app's prefix is refused // before the gate judges the password: the password is right, and still no // cookie comes back. func TestDoorGateLoginPostForAnotherAppsPrefixIsUnknownHost(t *testing.T) { s := newDoorScene(t) for _, next := range []string{"/b/x", "/%62/x", "/b?tab=1"} { form := url.Values{"password": {doorPassword}, "next": {next}} rec := s.do(t, http.MethodPost, doorSplitHost, gate.LoginPath, s.key, map[string]string{"Content-Type": "application/x-www-form-urlencoded"}, strings.NewReader(form.Encode())) wantUnknownHost(t, rec, "login POST next="+next) if c := rec.Result().Cookies(); len(c) != 0 { t.Fatalf("next=%s: the gate judged the password and set %v", next, c) } } // The control: the same POST from the other app's own key does get a // cookie, so the refusal above is the door's and not a broken form. form := url.Values{"password": {doorPassword}, "next": {"/b/x"}} rec := s.do(t, http.MethodPost, doorSplitHost, gate.LoginPath, s.otherKey, map[string]string{"Content-Type": "application/x-www-form-urlencoded"}, strings.NewReader(form.Encode())) if rec.Code != http.StatusSeeOther || len(rec.Result().Cookies()) == 0 { t.Fatalf("control: the other app's own login = %d, cookies %v", rec.Code, rec.Result().Cookies()) } } // 5. The tunnel and steering-probe names answer unknown host to a door // caller in any spelling, although the caller's own wildcard would match // them, and neither the hub nor the probe runs. func TestDoorRefusesReservedHostnames(t *testing.T) { s := newDoorScene(t) // Preconditions: the public door answers both names itself. pub := httptest.NewServer(s.e) t.Cleanup(pub.Close) if code, body := getHost(t, pub.URL, doorSteerHost); code != http.StatusOK || !strings.Contains(body, steeringProbeBody) { t.Fatalf("precondition: the public steering probe answered %d %q", code, body) } for _, host := range []string{ doorTunnelHost, strings.ToUpper(doorTunnelHost), doorTunnelHost + ".", "Tunnel.Apps.Test.", doorSteerHost, strings.ToUpper(doorSteerHost), doorSteerHost + ".", "Edge.Apps.Test.", } { for _, path := range []string{"/", tunnelRegisterPath, storeDoorPrefix + "x"} { rec := s.do(t, http.MethodGet, host, path, s.key, map[string]string{ "Carlos-Tunnel-Name": "pi-box", "Authorization": "Bearer sekrit"}, nil) wantUnknownHost(t, rec, host+path) if strings.Contains(rec.Body.String(), steeringProbeBody) { t.Fatalf("%s%s: the steering probe answered a door caller", host, path) } } } if n := upstreamCalls(s.own); n != 0 { t.Fatalf("the wildcard's upstream saw %d request(s) for a reserved name", n) } // And an ordinary name under the same wildcard is the caller's. wantServed(t, s.do(t, http.MethodGet, "anything."+doorWildParent, "/", s.key, nil, nil), doorOwnBody, "wildcard name") } // 6. A host served on another box: the door checks the owner that box // published, and dials the box Owner named. func TestDoorRemoteHostChecksThePublishedOwner(t *testing.T) { s := newDoorScene(t) if err := mintMeshKey(context.Background(), s.store); err != nil { t.Fatal(err) } rec, eip := fakeHomeEdge(t, nil) putJSON(t, s.store, "control/edges/"+doorHomeEdgeKey+".json", fmt.Sprintf(`{"v":1,"box":%q,"eip":%q,"hosts":{`+ `"example.com":{"role":"home","account":"bae","app":"titogo"},`+ `"foreign.example":{"role":"home","account":"bae","app":"other"},`+ `"mixed.example":{"role":"home"}}}`, doorHomeEdgeKey, eip)) m := newHomeMap(s.store, "annie") refreshHomeMap(t, m) s.e.homes.Store(m) if acct, app, gotEIP, box, ok := m.Owner("example.com"); !ok || acct != "bae" || app != "titogo" || gotEIP != eip || box != doorHomeEdgeKey { t.Fatalf("precondition: Owner = %q %q %q %q %v", acct, app, gotEIP, box, ok) } r := s.do(t, http.MethodGet, "example.com", "/", s.key, nil, nil) if r.Code != http.StatusOK || r.Body.String() != homeEdgeBody { t.Fatalf("owned remote host = %d %q, want the home edge's answer", r.Code, r.Body.String()) } seen := rec.get() if seen.n != 1 || seen.host != "example.com" || seen.hop != "1" || seen.auth == "" { t.Fatalf("home edge saw %+v, want one signed hop for example.com", seen) } if _, ok := s.e.backhaulInflight.Load(doorHomeEdgeKey); !ok { t.Fatalf("the hop was not counted against box %q, the box Owner named", doorHomeEdgeKey) } for _, host := range []string{"foreign.example", "mixed.example", "nobody.example"} { wantUnknownHost(t, s.do(t, http.MethodGet, host, "/", s.key, nil, nil), host) } if got := rec.get().n; got != 1 { t.Fatalf("the home edge saw %d request(s), want only the owned one", got) } } // 7. The mark comes only from serveDoor: a public request carrying the door // key and any other X-Carlos-* header is an ordinary client, neither scoped // nor given the door's scheme and address. func TestDoorMarkNeverComesFromHeaders(t *testing.T) { s := newDoorScene(t) pub := httptest.NewServer(s.e) t.Cleanup(pub.Close) req, _ := http.NewRequest(http.MethodGet, pub.URL+"/", nil) req.Host = doorOtherHost for _, h := range []string{"X-Carlos-Door", "X-Carlos-Door-Caller", "X-Carlos-Account", "X-Carlos-App"} { req.Header.Set(h, "bae/titogo") } req.Header.Set(doorkey.Header, s.key) resp, err := http.DefaultClient.Do(req) if err != nil { t.Fatal(err) } body := readBody(t, resp) resp.Body.Close() if resp.StatusCode != http.StatusOK || body != doorOtherBody { t.Fatalf("public request = %d %q, want the route served as to any client", resp.StatusCode, body) } _, hdr, _ := s.other.seen() if got := hdr.Get("X-Forwarded-Proto"); got != "http" { t.Fatalf("X-Forwarded-Proto = %q, want http: the door's scheme was applied to a public request", got) } if got := hdr.Get("X-Forwarded-For"); got == doorTestAddr { t.Fatal("a public request was given the door's address") } // The public steering probe still answers for a request carrying a key. req, _ = http.NewRequest(http.MethodGet, pub.URL+"/", nil) req.Host = doorSteerHost req.Header.Set(doorkey.Header, s.key) resp, err = http.DefaultClient.Do(req) if err != nil { t.Fatal(err) } if body := readBody(t, resp); resp.StatusCode != http.StatusOK || !strings.Contains(body, steeringProbeBody) { t.Fatalf("public probe with a door key = %d %q", resp.StatusCode, body) } resp.Body.Close() } // ---- Task 6: the handler -------------------------------------------------------- // 1. No key, a malformed key, a key for an app with no record, and a revoked // key are 401 with no-store, and nothing upstream runs. func TestDoorRefusesMissingMalformedAndRevokedKeys(t *testing.T) { s := newDoorScene(t) stranger, _ := s.fx.mint(t, "bae", "norecord") forged := s.key[:len(s.key)-1] + "A" if forged == s.key { forged = s.key[:len(s.key)-1] + "B" } check := func(what, key string) { t.Helper() rec := s.do(t, http.MethodGet, doorOwnHost, "/", key, nil, nil) if rec.Code != http.StatusUnauthorized || strings.TrimSpace(rec.Body.String()) != "door key refused" { t.Fatalf("%s: %d %q, want 401 door key refused", what, rec.Code, rec.Body.String()) } if cc := rec.Header().Get("Cache-Control"); cc != "no-store" { t.Fatalf("%s: Cache-Control = %q, want no-store", what, cc) } } check("no key", "") check("malformed key", "cdk1.bae.titogo.nope") check("unknown app", stranger) check("wrong secret", forged) wantServed(t, s.do(t, http.MethodGet, doorOwnHost, "/", s.key, nil, nil), doorOwnBody, "before the revoke") putJSON(t, s.store, doorkey.RecordKey("bae", "titogo"), string(s.fx.record(t, "bae", "titogo", 2))) refreshDoorKeys(t, s.e.doorKeys) check("revoked key", s.key) if n := upstreamCalls(s.own); n != 1 { t.Fatalf("upstream saw %d request(s), want only the one before the revoke", n) } } // 2. The key never reaches the app; the app's own headers do. func TestDoorStripsItsKeyAndPassesAppHeaders(t *testing.T) { s := newDoorScene(t) wantServed(t, s.do(t, http.MethodGet, doorOwnHost, "/", s.key, map[string]string{"X-Tito-Auth-User": "u_123", "X-Tito-Auth-Sig": "sig"}, nil), doorOwnBody, "valid key") _, hdr, _ := s.own.seen() if v := hdr.Get(doorkey.Header); v != "" { t.Fatalf("the door key reached the app: %q", v) } if hdr.Get("X-Tito-Auth-User") != "u_123" || hdr.Get("X-Tito-Auth-Sig") != "sig" { t.Fatalf("the app's own headers were touched: %v", hdr) } } // 3. Hop vocabulary from a door caller goes nowhere: not to the app, and a // correctly signed hop for a host not served here still meets the door's // owner check (unknown host), never the stale-hop 503. func TestDoorDropsHopVocabulary(t *testing.T) { s := newDoorScene(t) if err := mintMeshKey(context.Background(), s.store); err != nil { t.Fatal(err) } m := newHomeMap(s.store, "annie") refreshHomeMap(t, m) s.e.homes.Store(m) key, _ := m.meshKeys() wantServed(t, s.do(t, http.MethodGet, doorOwnHost, "/", s.key, map[string]string{ hopHeader: "1", hopAuthHeader: signHop(key, doorOwnHost, time.Now())}, nil), doorOwnBody, "local route") _, hdr, _ := s.own.seen() if hdr.Get(hopHeader) != "" || hdr.Get(hopAuthHeader) != "" { t.Fatalf("hop headers reached the app: %v", hdr) } rec := s.do(t, http.MethodGet, "elsewhere.test", "/", s.key, map[string]string{ hopHeader: "1", hopAuthHeader: signHop(key, "elsewhere.test", time.Now())}, nil) if rec.Code == http.StatusServiceUnavailable { t.Fatal("a door caller's signed hop earned the stale-hop refusal") } wantUnknownHost(t, rec, "signed hop for an unserved host") } // 4. Every claimed address goes; the app sees the box's public address and // https, whether or not the caller sent any; the visitor hash and the gate's // limiter key read the same address. func TestDoorPresentsTheBoxAddress(t *testing.T) { s := newDoorScene(t) for _, sent := range []map[string]string{ {"X-Forwarded-For": "198.51.100.7", "Forwarded": "for=198.51.100.9", "X-Forwarded-Proto": "http", "X-Forwarded-Host": "evil.test"}, nil, } { wantServed(t, s.do(t, http.MethodGet, doorOwnHost, "/", s.key, sent, nil), doorOwnBody, "door request") _, hdr, _ := s.own.seen() if got := hdr.Values("X-Forwarded-For"); len(got) != 1 || got[0] != doorTestAddr { t.Fatalf("sent %v: X-Forwarded-For = %q, want exactly %s", sent, got, doorTestAddr) } if got := hdr.Values("X-Forwarded-Proto"); len(got) != 1 || got[0] != "https" { t.Fatalf("sent %v: X-Forwarded-Proto = %q, want https", sent, got) } if got := hdr.Get("Forwarded"); got != "" { t.Fatalf("sent %v: Forwarded = %q reached the app", sent, got) } if got := hdr.Get("X-Forwarded-Host"); got != doorOwnHost { t.Fatalf("sent %v: X-Forwarded-Host = %q, want %s", sent, got, doorOwnHost) } } req := httptest.NewRequest(http.MethodGet, "http://"+doorOwnHost+"/", nil) req.RemoteAddr = "@" req.Header.Set(doorkey.Header, s.key) req.Header.Set("X-Forwarded-For", "198.51.100.7") in, ok := s.e.doorRequest(req) if !ok { t.Fatal("doorRequest refused a valid key") } if got := clientIP(in); got != doorTestAddr { t.Fatalf("clientIP = %q, want %s", got, doorTestAddr) } // The gate's limiter key is RemoteAddr without its port (gate.clientKey). if host, _, err := net.SplitHostPort(in.RemoteAddr); err != nil || host != doorTestAddr { t.Fatalf("RemoteAddr = %q, want %s:", in.RemoteAddr, doorTestAddr) } if c, ok := doorCallerFrom(in.Context()); !ok || c.Account != "bae" || c.App != "titogo" { t.Fatalf("door mark = %+v %v", c, ok) } } // allAlternates sets every name in doorAddressHeaders, plus the four // stripForwardedHeaders names, to addr. func allAlternates(addr string) map[string]string { h := map[string]string{"Forwarded": "for=" + addr, "X-Forwarded-For": addr, "X-Forwarded-Host": "evil.test", "X-Forwarded-Proto": "http"} for _, n := range doorAddressHeaders { h[n] = addr } return h } // 4b. None of the alternate address headers reaches the app. func TestDoorStripsAlternateAddressHeaders(t *testing.T) { s := newDoorScene(t) for _, want := range []string{"X-Forwarded", "Forwarded-For", "Client-IP", "X-Real-IP", "True-Client-IP", "CF-Connecting-IP", "Fastly-Client-IP", "X-Client-IP", "X-Cluster-Client-IP", "X-Originating-IP"} { found := false for _, n := range doorAddressHeaders { found = found || n == want } if !found { t.Fatalf("doorAddressHeaders lacks %s", want) } } wantServed(t, s.do(t, http.MethodGet, doorOwnHost, "/", s.key, allAlternates("8.8.8.8"), nil), doorOwnBody, "door request") _, hdr, _ := s.own.seen() for _, n := range doorAddressHeaders { if v := hdr.Values(n); len(v) != 0 { t.Fatalf("%s = %q reached the app", n, v) } } if got := hdr.Get("Forwarded"); got != "" { t.Fatalf("Forwarded = %q reached the app", got) } } // ---- the two resolver rule sets (4c) ---------------------------------------------- func mustPrefixes(cidrs ...string) []*net.IPNet { out := make([]*net.IPNet, 0, len(cidrs)) for _, c := range cidrs { _, n, err := net.ParseCIDR(c) if err != nil { panic(err) } out = append(out, n) } return out } func inAny(ip net.IP, nets []*net.IPNet) bool { for _, n := range nets { if n.Contains(ip) { return true } } return false } // railsTrusted is ActionDispatch::RemoteIp::TRUSTED_PROXIES (Rails 8.0.2). var railsTrusted = mustPrefixes("127.0.0.0/8", "::1/128", "fc00::/7", "10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16", "169.254.0.0/16", "fe80::/10") // railsIPs is RemoteIp#ips_from: a comma list, non-IP entries dropped. func railsIPs(values []string) []string { var out []string for _, v := range values { for _, p := range strings.Split(v, ",") { p = strings.TrimSpace(p) if net.ParseIP(p) != nil { out = append(out, p) } } } return out } func reversed(s []string) []string { out := make([]string, len(s)) for i, v := range s { out[len(s)-1-i] = v } return out } // railsRemoteIP is Rails 8.0.2's RemoteIp::GetIp#calculate_ip, with the // spoofing check off (its default outcome when the headers agree): // // remote_addr = ips_from(REMOTE_ADDR).last // client_ips = ips_from(Client-IP).reverse // forwarded_ips = ips_from(X-Forwarded-For).reverse // ips = forwarded_ips + client_ips // filter_proxies([remote_addr] + ips).first || ips.last || remote_addr func railsRemoteIP(h http.Header, peer string) string { ips := append(reversed(railsIPs(h.Values("X-Forwarded-For"))), reversed(railsIPs(h.Values("Client-IP")))...) for _, ip := range append([]string{peer}, ips...) { if !inAny(net.ParseIP(ip), railsTrusted) { return ip } } if len(ips) > 0 { return ips[len(ips)-1] } return peer } // pythonPrivate is ipaddress.IPv4Address/IPv6Address.is_private's range // list (CPython 3.12). 203.0.113.0/24 and 198.51.100.0/24 are on it. var pythonPrivate = mustPrefixes("0.0.0.0/8", "10.0.0.0/8", "127.0.0.0/8", "169.254.0.0/16", "172.16.0.0/12", "192.0.0.0/29", "192.0.0.170/31", "192.0.2.0/24", "192.168.0.0/16", "198.18.0.0/15", "198.51.100.0/24", "203.0.113.0/24", "240.0.0.0/4", "255.255.255.255/32", "::1/128", "::/128", "::ffff:0:0/96", "100::/64", "2001::/23", "2001:2::/48", "2001:db8::/32", "2001:10::/28", "fc00::/7", "fe80::/10") var pythonShared = mustPrefixes("100.64.0.0/10") // ipwareHeaders is python-ipware 3.0's default precedence, as HTTP header // names, REMOTE_ADDR last. var ipwareHeaders = []string{"X-Forwarded-For", "Client-IP", "X-Real-IP", "X-Forwarded", "X-Cluster-Client-IP", "Forwarded-For", "Forwarded", "CF-Connecting-IP", "X-Client-IP", "True-Client-IP", "Fastly-Client-IP"} // ipwareClientIP is python-ipware 3.0's IpWare.get_client_ip(meta) with no // proxy count or trusted list: headers in precedence order, the left-most // address of each; the first global one wins at once, otherwise the first // private one, otherwise the first loopback one. func ipwareClientIP(h http.Header, peer string) (ip string, routable bool) { var private, loopback []string consider := func(v string) (string, bool) { first := strings.TrimSpace(strings.Split(v, ",")[0]) p := net.ParseIP(first) if p == nil { return "", false } switch { case p.IsLoopback(): loopback = append(loopback, first) case inAny(p, pythonPrivate) || inAny(p, pythonShared): private = append(private, first) default: return first, true } return "", false } for _, name := range ipwareHeaders { if v := strings.Join(h.Values(name), ", "); strings.TrimSpace(v) != "" { if ip, ok := consider(v); ok { return ip, true } } } if ip, ok := consider(peer); ok { return ip, true } if len(private) > 0 { return private[0], false } if len(loopback) > 0 { return loopback[0], false } return "", false } // resolveBoth feeds the upstream's headers and peer to both rule sets. func resolveBoth(t *testing.T, what string, hdr http.Header, peer, want string) { t.Helper() host, _, err := net.SplitHostPort(peer) if err != nil { host = peer } if !net.ParseIP(host).IsLoopback() { t.Fatalf("%s: test bug: the upstream's peer %q is not loopback", what, host) } if got := railsRemoteIP(hdr, host); got != want { t.Fatalf("%s: Rails RemoteIp resolves %q, want %s", what, got, want) } got, routable := ipwareClientIP(hdr, host) if got != want { t.Fatalf("%s: python-ipware resolves %q, want %s", what, got, want) } // ipware's routable flag is Python's is_global. TEST-NET-3 is on // Python's private list, so the plan's documentation address is not // routable there; a real EIP is. if wantRoutable := !inAny(net.ParseIP(want), pythonPrivate); routable != wantRoutable { t.Fatalf("%s: python-ipware routable = %v for %s, want %v", what, routable, want, wantRoutable) } } // 4c. With every alternate address header set to a global address, both // resolvers still land on the box address, never loopback, a private address // or the caller's claim. The same inbound headers, unstripped, resolve to // the claim under both, which is what makes the rule sets a real test. func TestDoorResolvesToTheBoxAddressUnderRailsAndIpware(t *testing.T) { inbound := http.Header{} for k, v := range allAlternates("8.8.8.8") { inbound.Set(k, v) } if got := railsRemoteIP(inbound, "127.0.0.1"); got != "8.8.8.8" { t.Fatalf("control: Rails resolves the raw inbound headers to %q, want the claim", got) } inbound.Del("X-Forwarded-For") if got := railsRemoteIP(inbound, "127.0.0.1"); got != "8.8.8.8" { t.Fatalf("control: Rails falls back to Client-IP as %q, want the claim", got) } if got, _ := ipwareClientIP(inbound, "127.0.0.1"); got != "8.8.8.8" { t.Fatalf("control: ipware resolves the raw inbound headers to %q, want the claim", got) } // The plan's documentation address, and one shaped like a real EIP. for _, addr := range []string{doorTestAddr, "34.248.10.10"} { s := newDoorScene(t) s.e.doorAddr = addr wantServed(t, s.do(t, http.MethodGet, doorOwnHost, "/", s.key, allAlternates("8.8.8.8"), nil), doorOwnBody, "door request") _, hdr, peer := s.own.seen() resolveBoth(t, "served locally as "+addr, hdr, peer, addr) } } // 5. The door is not a bypass: a gated route answers with the gate's page. func TestDoorGatedRouteStillAnswersWithTheGatePage(t *testing.T) { s := newDoorScene(t) rec := s.do(t, http.MethodGet, doorGatedHost, "/", s.key, nil, nil) if rec.Code != http.StatusUnauthorized || !strings.Contains(rec.Body.String(), "Password required") { t.Fatalf("gated route through the door = %d %q, want the gate's 401 page", rec.Code, rec.Body.String()) } if n := upstreamCalls(s.gated); n != 0 { t.Fatalf("the gated upstream saw %d request(s)", n) } } // 5a. The door is not a bypass of the import gate either (seeded-instances // spec ยง5.4): the caller's own route, swapped for an uncommitted import, // passes the door's scope check and then meets the gate. Without the verify // token the answer is the gate's 503, not unknown host; with it, a GET // reaches the import and the app never sees the header. Another app's key // is still refused at the scope check, before the gate says anything. func TestDoorMeetsTheImportGate(t *testing.T) { s := newDoorScene(t) importSrv, imported := newGateBackend(t, "imported", "") token, hash, err := seeds.NewVerifyToken() if err != nil { t.Fatal(err) } gateForImport(t, s.e.reg, doorOwnHost, importSrv.Listener.Addr().String(), hash, false) if err := s.e.Reload(); err != nil { t.Fatal(err) } wantGateRefusal(t, s.do(t, http.MethodGet, doorOwnHost, "/", s.key, nil, nil), token, "the door with no token") wantGateRefusal(t, s.do(t, http.MethodPost, doorOwnHost, "/", s.key, map[string]string{ImportVerifyHeader: token}, nil), token, "the door with the token on a POST") if n, _ := imported.seen(); n != 0 || upstreamCalls(s.own) != 0 { t.Fatalf("the door reached a gated row: import %d, legacy %d", n, upstreamCalls(s.own)) } wantUnknownHost(t, s.do(t, http.MethodGet, doorOwnHost, "/", s.otherKey, map[string]string{ImportVerifyHeader: token}, nil), "another app's key on a gated host") rec := s.do(t, http.MethodGet, doorOwnHost, "/", s.key, map[string]string{ImportVerifyHeader: token}, nil) if rec.Code != http.StatusOK || rec.Body.String() != "imported" { t.Fatalf("verify through the door = %d %q, want the import", rec.Code, rec.Body.String()) } if n, saw := imported.seen(); n != 1 || len(saw) != 0 { t.Fatalf("verify through the door: import reached %d times, header at the app %q", n, saw) } } // ---- 5b: a backhauled door request ----------------------------------------------- // 5b. Two edges joined as the backhaul e2e joins them: the door request lands // on betty, which does not serve the host, and is backhauled to annie. The // far app sees the box address and https, and so do both resolvers, when // annie verifies the hop and again when it cannot. Unverified, annie uses the // TCP peer, which in production is the sending box's own public address; // loopback cannot carry that, so annie's front door reports it (the harness's // peerAs). func TestDoorBackhaulPresentsTheBoxAddress(t *testing.T) { e2eQuietLogs(t) bucket := t.TempDir() seed := release.NewDirStore(bucket) ca := newE2ECA(t) if err := MintMeshKey(context.Background(), seed); err != nil { t.Fatal(err) } for _, host := range []string{e2eHomedHost, e2eSenderHost} { ca.publish(t, seed, host) } appSrv, app := newDoorUpstream(t, e2eHomeMarker) senderSrv, _ := newDoorUpstream(t, e2eSenderMarker) a := newBackhaulEdge(t, bucket, "annie", ca, nil, map[string]string{e2eHomedHost: appSrv.Listener.Addr().String()}) b := newBackhaulEdge(t, bucket, "betty", ca, nil, map[string]string{e2eSenderHost: senderSrv.Listener.Addr().String()}) aEIP, port, err := net.SplitHostPort(a.tlsAddr) if err != nil { t.Fatal(err) } bEIP, _, _ := net.SplitHostPort(b.tlsAddr) oldPort, oldCAs := backhaulPort, backhaulRootCAs backhaulPort, backhaulRootCAs = port, ca.pool t.Cleanup(func() { backhaulPort, backhaulRootCAs = oldPort, oldCAs }) putJSON(t, seed, "control/edges/annie.json", fmt.Sprintf( `{"v":1,"box":"annie","eip":%q,"hosts":{%q:{"role":"home","account":"bae","app":"titogo"}}}`, aEIP, e2eHomedHost)) putJSON(t, seed, "control/edges/betty.json", fmt.Sprintf( `{"v":1,"box":"betty","eip":%q,"hosts":{%q:{"role":"home","account":"bae","app":"other"}}}`, bEIP, e2eSenderHost)) for _, be := range []*backhaulEdge{a, b} { if err := be.mirror.Refresh(context.Background()); err != nil { t.Fatal(err) } refreshHomeMap(t, be.homes) } fx := newDoorKeyFixture(t) raw, entry := fx.mint(t, "bae", "titogo") putJSON(t, seed, doorkey.RecordKey("bae", "titogo"), string(fx.record(t, "bae", "titogo", 1, entry))) b.edge.doorKeys = fx.mirror(b.store) refreshDoorKeys(t, b.edge.doorKeys) b.edge.doorAddr = doorTestAddr // farXFF sends one door request and returns the far app's chain. farXFF := func() []string { t.Helper() req := httptest.NewRequest(http.MethodGet, "http://"+e2eHomedHost+"/", nil) req.Host = e2eHomedHost req.RemoteAddr = "@" req.Header.Set(doorkey.Header, raw) rec := httptest.NewRecorder() b.edge.serveDoor(rec, req) if rec.Code != http.StatusOK { t.Fatalf("door request = %d %q", rec.Code, rec.Body.String()) } _, hdr, _ := app.seen() return hdr.Values("X-Forwarded-For") } door := func(what string) { t.Helper() before := upstreamCalls(app) req := httptest.NewRequest(http.MethodGet, "http://"+e2eHomedHost+"/deep", nil) req.Host = e2eHomedHost req.RemoteAddr = "@" req.Header.Set(doorkey.Header, raw) for k, v := range allAlternates("8.8.8.8") { req.Header.Set(k, v) } rec := httptest.NewRecorder() b.edge.serveDoor(rec, req) if rec.Code != http.StatusOK || rec.Body.String() != e2eHomeMarker { t.Fatalf("%s: %d %q, want the home edge's app", what, rec.Code, rec.Body.String()) } n, hdr, peer := app.seen() if n != before+1 { t.Fatalf("%s: the far app saw %d new request(s), want one", what, n-before) } if got := hdr.Values("X-Forwarded-For"); len(got) != 1 || got[0] != doorTestAddr { t.Fatalf("%s: X-Forwarded-For = %q, want exactly %s", what, got, doorTestAddr) } if got := hdr.Values("X-Forwarded-Proto"); len(got) != 1 || got[0] != "https" { t.Fatalf("%s: X-Forwarded-Proto = %q, want https", what, got) } for _, n := range append([]string{"Forwarded", doorkey.Header, hopHeader, hopAuthHeader}, doorAddressHeaders...) { if v := hdr.Values(n); len(v) != 0 { t.Fatalf("%s: %s = %q reached the far app", what, n, v) } } resolveBoth(t, what, hdr, peer, doorTestAddr) } // Verified: annie believes betty's chain, not the peer. A peer that is // neither address proves which one the app was given. const elsewhere = "192.0.2.77" a.peerAs.Store(elsewhere) door("verified hop") // annie can no longer verify betty's hops: its mirrored key is not the // one betty signs with. The chain is dropped and the peer is used, which // the elsewhere peer shows; with the peer at the box's own address, as // in production, the app sees the same as above. a.homes.keyMu.Lock() a.homes.keyCurrent, a.homes.keyNext = bytes.Repeat([]byte{7}, 32), nil a.homes.keyMu.Unlock() if got := farXFF(); len(got) != 1 || got[0] != elsewhere { t.Fatalf("unverifiable hop: X-Forwarded-For = %q, want the peer %s (the chain must not be believed)", got, elsewhere) } a.peerAs.Store(doorTestAddr) door("unverifiable hop") } // ---- Task 6: the listener ------------------------------------------------------ // shortSockDir is a socket directory short enough for sun_path. func shortSockDir(t *testing.T) string { t.Helper() dir, err := os.MkdirTemp("", "door") if err != nil { t.Fatal(err) } t.Cleanup(func() { os.RemoveAll(dir) }) return dir } func unixClient(path string) *http.Client { return &http.Client{Timeout: 10 * time.Second, Transport: &http.Transport{ DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) { var d net.Dialer return d.DialContext(ctx, "unix", path) }, }} } func doorGet(t *testing.T, path, host, key string) (int, string) { t.Helper() req, _ := http.NewRequest(http.MethodGet, "http://door/", nil) req.Host = host if key != "" { req.Header.Set(doorkey.Header, key) } resp, err := unixClient(path).Do(req) if err != nil { t.Fatalf("door request: %v", err) } defer resp.Body.Close() return resp.StatusCode, readBody(t, resp) } // 6. openDoor makes a 0660 socket that serves the door, closeDoor removes // it, and a door with no mirror or no public address opens nothing. func TestOpenDoorServesOnA0660Socket(t *testing.T) { s := newDoorScene(t) path := filepath.Join(shortSockDir(t), "edge-local.sock") // A stale socket from a previous run is replaced. stale, err := net.Listen("unix", path) if err != nil { t.Fatal(err) } stale.(*net.UnixListener).SetUnlinkOnClose(false) stale.Close() closeDoor, err := s.e.openDoor(path) if err != nil { t.Fatalf("openDoor: %v", err) } fi, err := os.Stat(path) if err != nil { t.Fatal(err) } if fi.Mode()&os.ModeSocket == 0 || fi.Mode().Perm() != 0o660 { t.Fatalf("socket mode = %v, want a 0660 socket", fi.Mode()) } if code, body := doorGet(t, path, doorOwnHost, s.key); code != http.StatusOK || body != doorOwnBody { t.Fatalf("over the socket = %d %q", code, body) } if code, _ := doorGet(t, path, doorOwnHost, ""); code != http.StatusUnauthorized { t.Fatalf("no key over the socket = %d, want 401", code) } closeDoor() if _, err := os.Stat(path); !os.IsNotExist(err) { t.Fatalf("socket still there after closeDoor: %v", err) } closeDoor() // idempotent for _, tc := range []struct { what string arm func(e *Edge) }{ {"nil mirror", func(e *Edge) { e.doorKeys = nil }}, {"no address", func(e *Edge) { e.doorAddr = "" }}, {"loopback", func(e *Edge) { e.doorAddr = "127.0.0.1" }}, {"private", func(e *Edge) { e.doorAddr = "10.0.0.5" }}, {"not an address", func(e *Edge) { e.doorAddr = "unknown" }}, } { s := newDoorScene(t) tc.arm(s.e) p := filepath.Join(shortSockDir(t), "d.sock") if c, err := s.e.openDoor(p); err == nil { c() t.Fatalf("%s: openDoor opened a door", tc.what) } if _, err := os.Stat(p); !os.IsNotExist(err) { t.Fatalf("%s: openDoor left something at the path: %v", tc.what, err) } } // A path that is not a socket is not removed. p := filepath.Join(shortSockDir(t), "file") if err := os.WriteFile(p, []byte("keep"), 0o600); err != nil { t.Fatal(err) } if c, err := newDoorScene(t).e.openDoor(p); err == nil { c() t.Fatal("openDoor replaced a regular file") } if b, err := os.ReadFile(p); err != nil || string(b) != "keep" { t.Fatalf("the regular file was touched: %q %v", b, err) } } // ---- Task 7: lifecycle ---------------------------------------------------------- func TestWithEdgeDoorRecordsPathAndAddress(t *testing.T) { e := &Edge{} WithEdgeDoor("/run/carlos/edge-local.sock", doorTestAddr)(e) if e.doorPath != "/run/carlos/edge-local.sock" || e.doorAddr != doorTestAddr { t.Fatalf("doorPath %q doorAddr %q", e.doorPath, e.doorAddr) } } // armDoor over a DirStore: before any record exists a key is refused (and // nothing panics); once a signed record lands and the mirror polls, the // same key is served over the real socket; closeDoor removes the socket. func TestArmDoorServesOverTheSocketAndClosesIt(t *testing.T) { buf := captureLog(t) fx := newDoorKeyFixture(t) oldInterval := doorKeyInterval doorKeyInterval = 20 * time.Millisecond t.Cleanup(func() { doorKeyInterval = oldInterval }) backend, _ := newDoorUpstream(t, doorOwnBody) reg := openReg(t) if err := reg.Upsert(registry.Route{Host: doorOwnHost, Kind: "service", Status: "ready", Addr: backend.Listener.Addr().String(), Account: "bae", App: "titogo"}); err != nil { t.Fatal(err) } e, err := New(reg, nil) if err != nil { t.Fatal(err) } t.Cleanup(e.probers.Stop) store := release.NewDirStore(t.TempDir()) e.staticStore, e.staticPub = store, fx.pub path := filepath.Join(shortSockDir(t), "edge-local.sock") WithEdgeDoor(path, doorTestAddr)(e) if err := e.armDoor(context.Background()); err != nil { t.Fatalf("armDoor: %v", err) } t.Cleanup(e.closeDoor) raw, entry := fx.mint(t, "bae", "titogo") if code, body := doorGet(t, path, doorOwnHost, raw); code != http.StatusUnauthorized { t.Fatalf("before any record = %d %q, want 401", code, body) } putJSON(t, store, doorkey.RecordKey("bae", "titogo"), string(fx.record(t, "bae", "titogo", 1, entry))) deadline := time.Now().Add(5 * time.Second) for { code, body := doorGet(t, path, doorOwnHost, raw) if code == http.StatusOK && body == doorOwnBody { break } if time.Now().After(deadline) { t.Fatalf("the mirror's loop never admitted the key: %d %q", code, body) } time.Sleep(20 * time.Millisecond) } if !strings.Contains(buf.String(), "edge-local door open on "+path) { t.Fatalf("no open line in the log:\n%s", buf.String()) } e.closeDoor() if _, err := os.Stat(path); !os.IsNotExist(err) { t.Fatalf("socket still there after closeDoor: %v", err) } } // With no store, no public key, or no usable address, armDoor opens nothing // and says why. func TestArmDoorStaysClosedAndSaysWhy(t *testing.T) { fx := newDoorKeyFixture(t) for _, tc := range []struct { what, addr, want string store bool pub bool }{ {"no store", doorTestAddr, "no deployment store", false, true}, {"no public key", doorTestAddr, "no release public key", true, false}, {"no address", "", "no usable public address", true, true}, {"private address", "10.1.2.3", "no usable public address", true, true}, } { buf := captureLog(t) e := &Edge{} if tc.store { e.staticStore = release.NewDirStore(t.TempDir()) } if tc.pub { e.staticPub = fx.pub } path := filepath.Join(shortSockDir(t), "d.sock") WithEdgeDoor(path, tc.addr)(e) if err := e.armDoor(context.Background()); err == nil { t.Fatalf("%s: armDoor succeeded", tc.what) } if e.closeDoor != nil || e.doorKeys != nil { t.Fatalf("%s: armDoor left a door or a mirror behind", tc.what) } if _, err := os.Stat(path); !os.IsNotExist(err) { t.Fatalf("%s: something exists at the socket path: %v", tc.what, err) } if got := buf.String(); !strings.Contains(got, "edge-local door off") || !strings.Contains(got, tc.want) { t.Fatalf("%s: log = %q, want the door off and %q", tc.what, got, tc.want) } } // No door path: nothing at all, not even a line. buf := captureLog(t) e := &Edge{} if err := e.armDoor(context.Background()); err != nil { t.Fatalf("armDoor with no path: %v", err) } if buf.Len() != 0 { t.Fatalf("armDoor with no path logged %q", buf.String()) } }