// `lchat fetch run` — the public-web fetch service (docs/fetch.md). A shared, // identity-free service that fronts *public web content* for clients: OpenGraph // unfurls and an HMAC-signed image proxy for the cards' images. Like links // (linkscmd.go) it's a subcommand of this binary, not a separate module — // it deliberately reuses the instance's own unfurl machinery (unfurl.go: // fetchOG, the SSRF dial guard, the sealed convergent cache, previewFlight). // // Trust model, in one breath: the fetch service sees URLs (the documented unfurl // exception) attached to NOTHING — no token, no instance, no member. Behind // the router it doesn't even see client IPs (nothing forwards them — that's a // feature, with the honest cost that per-IP rate buckets collapse to global // ones there; hashcash is the planned real gate, see docs/fetch.md). It must // never grow an endpoint that touches member content: member photos/files are // encrypted client-side and are categorically out of scope here. // // The image proxy is camo-shaped (GitHub's trick): /img serves only URLs the // unfurler itself signed when it built a card, so it can't be driven as an // open proxy, with no identity needed. Signing happens at response time (the // cache stores the raw og:image URL), so the key could rotate without // invalidating cached cards. package main import ( "bytes" "crypto/hmac" "crypto/rand" "crypto/sha256" "database/sql" "encoding/base64" "encoding/hex" "encoding/json" "flag" "fmt" "image" "image/jpeg" "image/png" "io" "log" "net" "net/http" "net/url" "os" "strconv" "strings" "sync" "time" xdraw "golang.org/x/image/draw" ) const ( fetchImgMaxBytes = 8 * 1024 * 1024 // largest card image we'll proxy fetchImgTimeout = 15 * time.Second fetchImgCache = "public, max-age=86400" // Cards render at most a few hundred CSS px wide; og:images routinely // arrive at several thousand. Anything wider than Over re-encodes at To. fetchImgResizeOver = 1600 fetchImgResizeTo = 1200 ) // fetchImgClient fetches card images. Same SSRF posture as ogClient — every // dial (and every redirect hop) re-validates the resolved IP. var fetchImgClient = &http.Client{ Timeout: fetchImgTimeout, Transport: &http.Transport{ DialContext: (&net.Dialer{ Timeout: fetchImgTimeout, Control: guardDialControl, }).DialContext, TLSHandshakeTimeout: fetchImgTimeout, ResponseHeaderTimeout: fetchImgTimeout, DisableKeepAlives: true, }, CheckRedirect: func(req *http.Request, via []*http.Request) error { if len(via) >= 5 { return fmt.Errorf("too many redirects") } if req.URL.Scheme != "http" && req.URL.Scheme != "https" { return fmt.Errorf("non-http redirect") } return nil }, } type fetchServer struct { db *sql.DB salt string // service-wide preview salt (convergent-encryption pepper) signKey []byte // HMAC key for /img URLs; minted once, lives only in the DB // inflight coalesces concurrent unfurls exactly like the instance's // (main.go): one fetch per URL+language, everyone shares the result. inflight sync.Map // Ceilings, not budgets — and behind the router every caller shares one // bucket (no client IPs are forwarded, on purpose). See the package doc. unfurlLimiter *modLimiter imgLimiter *modLimiter gifLimiter *modLimiter // The platform's Giphy API key (LCHAT_GIPHY_KEY). Empty disables /api/gif // (404), and clients fall back to the space's bring-your-own-key path. giphyKey string // Hashcash admission (fetchpow.go); nil = off (the default). Its seal key // is minted into the DB alongside the salt so restarts keep tickets valid. pow *fetchPowGate powSealKey []byte } func newFetchServer(dbPath string) (*fetchServer, error) { d, err := sql.Open("sqlite", dbPath) if err != nil { return nil, err } d.SetMaxOpenConns(1) for _, p := range []string{ "PRAGMA journal_mode=WAL", "PRAGMA busy_timeout=5000", "PRAGMA synchronous=NORMAL", } { if _, err := d.Exec(p); err != nil { return nil, err } } for _, ddl := range []string{ `CREATE TABLE IF NOT EXISTS config(key TEXT PRIMARY KEY, value TEXT NOT NULL)`, // Same sealed shape as the instance's link_previews_sealed: the row key // is a salted hash and the blob opens only with a key derived from the // language+URL, so the shared cache at rest reveals nothing a reader // doesn't already know the URL for. `CREATE TABLE IF NOT EXISTS fetch_previews( url_hash TEXT PRIMARY KEY, blob TEXT NOT NULL, fetched_at INTEGER NOT NULL)`, } { if _, err := d.Exec(ddl); err != nil { return nil, err } } g := &fetchServer{ db: d, unfurlLimiter: newModLimiter(30, 30.0/60), gifLimiter: newModLimiter(60, 60.0/60), imgLimiter: newModLimiter(240, 120.0/60), } if g.salt, err = fetchConfigSecret(d, "preview_salt"); err != nil { return nil, err } keyHex, err := fetchConfigSecret(d, "img_sign_key") if err != nil { return nil, err } if g.signKey, err = hex.DecodeString(keyHex); err != nil { return nil, err } powHex, err := fetchConfigSecret(d, "pow_seal_key") if err != nil { return nil, err } if g.powSealKey, err = hex.DecodeString(powHex); err != nil { return nil, err } return g, nil } // armPow turns the hashcash gate on at the given difficulty (0 = off). func (g *fetchServer) armPow(bits int) error { gate, err := newFetchPowGate(bits, g.powSealKey) if err != nil { return err } g.pow = gate return nil } // fetchConfigSecret returns the named config value, minting a random one on // first boot — the same insert-if-absent pattern as the instance's // previewSalt, so restarts and concurrent starts converge on one value. func fetchConfigSecret(d *sql.DB, key string) (string, error) { buf := make([]byte, 32) if _, err := rand.Read(buf); err != nil { return "", err } if _, err := d.Exec( "INSERT OR IGNORE INTO config(key, value) VALUES(?, ?)", key, hex.EncodeToString(buf)); err != nil { return "", err } var v string err := d.QueryRow("SELECT value FROM config WHERE key = ?", key).Scan(&v) return v, err } func (g *fetchServer) handler() http.Handler { mux := http.NewServeMux() // The two open endpoints sit behind the (optional) hashcash gate; /img // needs none — its HMAC signature is already the admission. mux.HandleFunc("GET /api/unfurl", g.powGated(g.serveUnfurl)) mux.HandleFunc("GET /img", g.serveImage) mux.HandleFunc("GET /api/gif", g.powGated(g.serveGifSearch)) mux.HandleFunc("GET /healthz", func(w http.ResponseWriter, r *http.Request) { fmt.Fprintln(w, "ok") }) mux.HandleFunc("GET /api/version", func(w http.ResponseWriter, r *http.Request) { writeJSON(w, http.StatusOK, map[string]any{"service": "fetch", "build": buildVersion}) }) // The fetch service has no pages of its own; a stray visit goes to the real site. mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { http.Redirect(w, r, "https://elevenmessenger.com", http.StatusFound) }) // Every caller is a browser on some OTHER origin (each space is its own // origin; the service serves them all), so without CORS headers fetch() // can never read a response and every client silently falls back to its // instance path. `*` is the honest scope: the service is anonymous by // design — no cookies, no credentials, nothing origin-scoped to protect. return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Access-Control-Allow-Origin", "*") // The pow gate hands fresh tickets back on a response header; without // this, cross-origin JS can see the response but not that header. w.Header().Set("Access-Control-Expose-Headers", "X-Fetch-Ticket") mux.ServeHTTP(w, r) }) } // powGated wraps an open endpoint in the hashcash gate (nil gate = pass). // Admission rides query params, not headers — a custom request header would // cost every cross-origin caller a CORS preflight round trip. // // - ?ticket= a sealed, unexpired ticket → in, rate-bucketed by the ticket // - ?pow=&nonce= a solved challenge → consumed, request served, and a // fresh ticket returned in X-Fetch-Ticket for the requests that follow // - neither → 401 {"pow":{"challenge","bits"}} — the mint rides the // rejection, so admission costs one round trip, not two func (g *fetchServer) powGated(next http.HandlerFunc) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { if g.pow == nil { next(w, r) return } q := r.URL.Query() if t := q.Get("ticket"); t != "" && g.pow.ticketOK(t) { next(w, r) return } if c := q.Get("pow"); c != "" && g.pow.verifyAndConsume(c, q.Get("nonce")) == nil { if t, err := g.pow.mintTicket(); err == nil { w.Header().Set("X-Fetch-Ticket", t) } next(w, r) return } ch, err := g.pow.mintChallenge() if err != nil { httpError(w, http.StatusInternalServerError, "pow mint failed") return } writeJSON(w, http.StatusUnauthorized, map[string]any{ "pow": map[string]any{"challenge": ch, "bits": g.pow.bits}, }) } } // rateKey buckets a gated request by its ticket (each solved client gets its // own bucket); ungated requests keep the caller-IP key, with its documented // behind-the-router collapse. func (g *fetchServer) rateKey(r *http.Request) string { if g.pow != nil { if t := r.URL.Query().Get("ticket"); t != "" { return "t:" + t } } return fetchCallerKey(r) } // fetchCallerKey is the rate-limit bucket key. Mirrors linkscmd.go — behind // the router (unix socket, no forwarded IPs) this degrades to one shared // bucket, which is the accepted posture for now (package doc). func fetchCallerKey(r *http.Request) string { ip, _, err := net.SplitHostPort(r.RemoteAddr) if err != nil { return r.RemoteAddr } return ip } // serveUnfurl is the instance handler's anonymous twin: no token, same URL // validation, same language handling, same stale-serve semantics — but the // card's image URL is rewritten to the signed /img proxy before it leaves. func (g *fetchServer) serveUnfurl(w http.ResponseWriter, r *http.Request) { if !g.unfurlLimiter.allow(g.rateKey(r)) { httpError(w, http.StatusTooManyRequests, "slow down") return } raw := strings.TrimSpace(r.URL.Query().Get("url")) u, err := url.Parse(raw) if err != nil || u.Host == "" || (u.Scheme != "http" && u.Scheme != "https") || len(raw) > 2048 { httpError(w, http.StatusBadRequest, "bad url") return } norm := u.String() lang := previewLang(r.Header.Get("Accept-Language")) if pv, ok := g.getPreview(norm, lang); ok { writeJSON(w, http.StatusOK, g.signedCard(r, pv)) ttl := int64(previewFreshTTL) if !pv.OK { ttl = previewFailTTL } if time.Now().Unix()-pv.FetchedAt > ttl { go g.fetchAndCache(norm, lang) } return } writeJSON(w, http.StatusOK, g.signedCard(r, g.fetchAndCache(norm, lang))) } // fetchAndCache mirrors the instance's fetchAndCachePreview (main.go), // including the mid-fetch waiters fix — against the fetch service's own store. func (g *fetchServer) fetchAndCache(norm, lang string) LinkPreview { flightKey := previewCacheKey(lang, norm) f := &previewFlight{done: make(chan struct{})} if cur, busy := g.inflight.LoadOrStore(flightKey, f); busy { flight := cur.(*previewFlight) <-flight.done return flight.pv } defer func() { g.inflight.Delete(flightKey) close(f.done) }() pv, err := fetchOG(norm, lang) if err != nil { pv = LinkPreview{URL: norm, OK: false} } pv.URL = norm pv.FetchedAt = time.Now().Unix() g.putPreview(pv, lang) f.pv = pv return pv } func (g *fetchServer) getPreview(u, lang string) (LinkPreview, bool) { key := previewCacheKey(lang, u) var blob string var fetchedAt int64 err := g.db.QueryRow( "SELECT blob, fetched_at FROM fetch_previews WHERE url_hash = ?", previewURLHash(g.salt, key), ).Scan(&blob, &fetchedAt) if err != nil { return LinkPreview{}, false } pv, ok := openPreview(g.salt, key, blob) if !ok { return LinkPreview{}, false } pv.URL = u pv.FetchedAt = fetchedAt return pv, true } func (g *fetchServer) putPreview(pv LinkPreview, lang string) { key := previewCacheKey(lang, pv.URL) blob, err := sealPreview(g.salt, key, pv) if err != nil { return } // Same put-time TTL sweep as the instance cache: prunes with activity. g.db.Exec("DELETE FROM fetch_previews WHERE fetched_at < ?", time.Now().Unix()-previewMaxAge) g.db.Exec( `INSERT INTO fetch_previews(url_hash, blob, fetched_at) VALUES(?, ?, ?) ON CONFLICT(url_hash) DO UPDATE SET blob=excluded.blob, fetched_at=excluded.fetched_at`, previewURLHash(g.salt, key), blob, pv.FetchedAt) } // signedCard swaps a card's raw og:image URL for the signed /img proxy URL, // so the viewer's browser talks to the fetch service, not the image's host (no member // IP reaches it, and http-only images stop dying as mixed content). func (g *fetchServer) signedCard(r *http.Request, pv LinkPreview) LinkPreview { if pv.Image != "" { pv.Image = g.proxyImageURL(r, pv.Image) } return pv } func (g *fetchServer) proxyImageURL(r *http.Request, img string) string { mac := hmac.New(sha256.New, g.signKey) mac.Write([]byte(img)) scheme := "https" if r.TLS == nil && fetchLoopbackHost(r.Host) { scheme = "http" // tests and bare local runs; behind the router TLS is the router's } return scheme + "://" + r.Host + "/img?u=" + base64.RawURLEncoding.EncodeToString([]byte(img)) + "&s=" + hex.EncodeToString(mac.Sum(nil)) } func fetchLoopbackHost(hostport string) bool { host, _, err := net.SplitHostPort(hostport) if err != nil { host = hostport } if host == "localhost" { return true } ip := net.ParseIP(host) return ip != nil && ip.IsLoopback() } // serveImage is the camo half: verify the signature (only URLs this service // itself put in a card are proxyable — never an open proxy), then fetch and // re-serve the image over our origin. Buffered rather than streamed so an // oversize image is rejected cleanly instead of truncated mid-body; the size // cap bounds the memory. func (g *fetchServer) serveImage(w http.ResponseWriter, r *http.Request) { if !g.imgLimiter.allow(fetchCallerKey(r)) { httpError(w, http.StatusTooManyRequests, "slow down") return } rawImg, err := base64.RawURLEncoding.DecodeString(r.URL.Query().Get("u")) if err != nil { httpError(w, http.StatusBadRequest, "bad url") return } sig, err := hex.DecodeString(r.URL.Query().Get("s")) if err != nil { httpError(w, http.StatusBadRequest, "bad signature") return } mac := hmac.New(sha256.New, g.signKey) mac.Write(rawImg) if !hmac.Equal(sig, mac.Sum(nil)) { httpError(w, http.StatusForbidden, "bad signature") return } imgURL, err := url.Parse(string(rawImg)) if err != nil || (imgURL.Scheme != "http" && imgURL.Scheme != "https") { httpError(w, http.StatusBadRequest, "bad url") return } req, err := http.NewRequestWithContext(r.Context(), "GET", imgURL.String(), nil) if err != nil { httpError(w, http.StatusBadGateway, "fetch failed") return } req.Header.Set("User-Agent", "lchat-fetch/1.0 (+https://elevenmessenger.com)") req.Header.Set("Accept", "image/*") resp, err := fetchImgClient.Do(req) if err != nil { httpError(w, http.StatusBadGateway, "fetch failed") return } defer resp.Body.Close() ct := strings.ToLower(resp.Header.Get("Content-Type")) if resp.StatusCode != http.StatusOK || !strings.HasPrefix(ct, "image/") { httpError(w, http.StatusBadGateway, "not an image") return } body, err := io.ReadAll(io.LimitReader(resp.Body, fetchImgMaxBytes+1)) if err != nil || len(body) > fetchImgMaxBytes { httpError(w, http.StatusBadGateway, "image too large") return } body = shrinkCardImage(ct, body) w.Header().Set("Content-Type", resp.Header.Get("Content-Type")) w.Header().Set("Cache-Control", fetchImgCache) w.Header().Set("X-Content-Type-Options", "nosniff") w.Write(body) } // shrinkCardImage re-encodes an oversized JPEG/PNG down to card width — an // og:image is often thousands of px wide for a card that renders at a few // hundred, and every viewer pays that transfer. Only those two formats: a // resized GIF would freeze its animation, and nothing else here has a stdlib // encoder. Every failure path returns the original bytes — polish must never // break a card — and a "shrink" that came out bigger keeps the original too. func shrinkCardImage(ct string, body []byte) []byte { if !strings.HasPrefix(ct, "image/jpeg") && !strings.HasPrefix(ct, "image/png") { return body } cfg, kind, err := image.DecodeConfig(bytes.NewReader(body)) if err != nil || cfg.Width <= fetchImgResizeOver { return body } // Decompression bomb: a tiny file can claim enormous pixel dims. Don't // decode those ourselves — pass the bytes through and let the browser's // own limits deal with it. if cfg.Height <= 0 || cfg.Width*cfg.Height > 40<<20 { return body } src, _, err := image.Decode(bytes.NewReader(body)) if err != nil { return body } h := cfg.Height * fetchImgResizeTo / cfg.Width if h < 1 { h = 1 } dst := image.NewRGBA(image.Rect(0, 0, fetchImgResizeTo, h)) xdraw.CatmullRom.Scale(dst, dst.Bounds(), src, src.Bounds(), xdraw.Over, nil) var out bytes.Buffer switch kind { case "png": err = png.Encode(&out, dst) default: err = jpeg.Encode(&out, dst, &jpeg.Options{Quality: 80}) } if err != nil || out.Len() >= len(body) { return body } return out.Bytes() } // giphyAPIBase is a var so tests can stand a fake upstream in. var giphyAPIBase = "https://api.giphy.com/v1/gifs/" // serveGifSearch fronts Giphy search/trending with the PLATFORM's API key // (docs/fetch.md § Giphy), so a space needs no key of its own. The response is // Giphy's own page shape, passed through untouched except that every giphy- // hosted media URL is rewritten to this service's signed /img proxy — the // browser then never talks to Giphy at all: search terms reach it from this // service's IP (identity-free), and GIF bytes come back over our origin. // No key configured → 404, and the client falls back to the space's // bring-your-own-key path (hostBridge.gif in shell.js). func (g *fetchServer) serveGifSearch(w http.ResponseWriter, r *http.Request) { if g.giphyKey == "" { httpError(w, http.StatusNotFound, "no gif upstream") return } if !g.gifLimiter.allow(g.rateKey(r)) { httpError(w, http.StatusTooManyRequests, "slow down") return } q := strings.TrimSpace(r.URL.Query().Get("q")) if runes := []rune(q); len(runes) > 100 { q = string(runes[:100]) } limit := 24 if n, err := strconv.Atoi(r.URL.Query().Get("limit")); err == nil && n >= 1 && n <= 50 { limit = n } up := giphyAPIBase + "trending?api_key=" + url.QueryEscape(g.giphyKey) + "&limit=" + strconv.Itoa(limit) + "&rating=pg-13" if q != "" { up = giphyAPIBase + "search?api_key=" + url.QueryEscape(g.giphyKey) + "&q=" + url.QueryEscape(q) + "&limit=" + strconv.Itoa(limit) + "&rating=pg-13" } req, err := http.NewRequestWithContext(r.Context(), "GET", up, nil) if err != nil { httpError(w, http.StatusBadGateway, "gif search failed") return } resp, err := fetchImgClient.Do(req) if err != nil { httpError(w, http.StatusBadGateway, "gif search failed") return } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { httpError(w, http.StatusBadGateway, "gif search failed") return } var page any if json.NewDecoder(io.LimitReader(resp.Body, 4*1024*1024)).Decode(&page) != nil { httpError(w, http.StatusBadGateway, "gif search failed") return } // Results are transient search pages — never cache them (the media URLs // inside are signed against this service's key anyway). w.Header().Set("Cache-Control", "no-store") writeJSON(w, http.StatusOK, g.signGiphyMedia(r, page)) } // signGiphyMedia walks a decoded Giphy page and swaps every giphy-hosted URL // string for this service's signed /img URL, wherever it sits — Giphy nests // renditions several levels deep and adds new ones over time, so a generic // walk beats a field list. Only giphy hosts are signed: other strings (ids, // titles, third-party "source" pages we'd never want proxyable) pass through. func (g *fetchServer) signGiphyMedia(r *http.Request, v any) any { switch t := v.(type) { case map[string]any: for k, vv := range t { t[k] = g.signGiphyMedia(r, vv) } return t case []any: for i, vv := range t { t[i] = g.signGiphyMedia(r, vv) } return t case string: ok := func(s string) bool { return s == "https" || (s == "http" && allowPrivatePreviewHosts) } if u, err := url.Parse(t); err == nil && ok(u.Scheme) && giphyMediaHost(u.Hostname()) { return g.proxyImageURL(r, t) } return t default: return t } } func giphyMediaHost(h string) bool { if allowPrivatePreviewHosts { return true // tests: the fake upstream's media lives on loopback } h = strings.ToLower(h) return h == "giphy.com" || strings.HasSuffix(h, ".giphy.com") } // runFetchCmd dispatches `lchat fetch `, mirroring runLinksCmd. func runFetchCmd(args []string) { if len(args) < 1 { fmt.Fprintln(os.Stderr, `usage: lchat fetch commands: run serve the public-web fetch service (unfurl + signed image proxy; docs/fetch.md)`) os.Exit(2) } switch args[0] { case "run": runFetchRun(args[1:]) default: die("fetch: unknown command %q", args[0]) } } // runFetchRun is `lchat fetch run`. func runFetchRun(args []string) { fs := flag.NewFlagSet("fetch", flag.ExitOnError) addr := fs.String("addr", envOr("ADDR", ":1749"), "listen address (env: ADDR)") socket := fs.String("socket", envOr("LCHAT_SOCKET", ""), "unix socket to serve on instead of -addr (env: LCHAT_SOCKET) — how the router fronts this service") dbPath := fs.String("db", envOr("LCHAT_FETCH_DB", "fetch.db"), "sqlite path for the sealed preview cache + service secrets (env: LCHAT_FETCH_DB)") _ = fs.Parse(args) g, err := newFetchServer(*dbPath) if err != nil { die("fetch: %v", err) } // The one env secret: the platform Giphy key (unit env file /data/fetch.env). // Absent, /api/gif 404s and clients use the space's own key instead. g.giphyKey = strings.TrimSpace(os.Getenv("LCHAT_GIPHY_KEY")) // Hashcash difficulty (LCHAT_FETCH_POW_BITS, 0/unset = off): arm before // pointing the hosted fleet here; leave off for dev and self-hosters. if bits, _ := strconv.Atoi(strings.TrimSpace(os.Getenv("LCHAT_FETCH_POW_BITS"))); bits > 0 { if err := g.armPow(bits); err != nil { die("fetch: pow: %v", err) } log.Printf("fetch: hashcash admission armed (%d bits)", bits) } ln, addrDesc, err := listen(*addr, *socket) if err != nil { die("fetch: %v", err) } // Deliberately the only startup log line — the fetch service never logs URLs. log.Printf("fetch: serving on %s", addrDesc) srv := &http.Server{Handler: g.handler(), ReadHeaderTimeout: 10 * time.Second} if err := srv.Serve(ln); err != nil { log.Fatal(err) } }