package main // Route replication across the activator mesh (mesh.go). registerInstance's // reg.upsert only ever writes ONE box's registry — but a host under // LCHAT_MESH_DOMAINS needs a routes row on every live mesh member before HRW // can honestly place it there (mesh.go's inScope doc comment: a box with no // row for a host 404s it, never wakes it). This file is the fan-out: // // - An admin RPC, POST /api/mesh/route (meshRouteHandler, wired by runRouter // only when the mesh is on), inserts or updates one route. // - registerInstance calls fanOutMeshRoute right after its local upsert, // best-effort against whichever peers are live right now. // - `lchat mesh-seed [--region ] ` (runMeshSeed) bulk-replicates // every existing route the joining box owns once — the catch-up a freshly // joined box needs before it can serve anything, since it joins with an // empty registry. // // Which peers get a route is the UNION of two sets (docs/multi-edge.md, Phase 1): // the boxes covering the host by DOMAIN (its LCHAT_MESH_DOMAINS — today's // fleet-wide fan-out for an in-domain host) and the boxes owning it by REGION // (those advertising the route's home_region). The region arm is the multi-edge // fix: a space placed by home_region onto a box whose per-country domain does NOT // cover the host (mesh.go candidatesFor) still needs a registry row THERE to // serve when the DNS-target box proxies to it — domain scope alone would strand // it. With no LCHAT_MESH_REGION and no home_region anywhere the region arm is // always empty, so this stays byte-identical to the pre-region, domain-only // fan-out. (Completing TLS to a region-owner box for a host outside ITS domain // is a separate, still-unbuilt slice — see meshReplicateRoute.) // // Both paths send only the fields a route's CONTENT actually needs (host, // kind, hibernate flag, per-instance config) — never Socket/DBPath, which are // box-local filesystem layout the receiving box must derive itself exactly as // registerInstance does for a route created locally, and never // Status/NextDue/CreatedAt, which are receiver-local lifecycle state. // // Deletions replicate too (#1346): both delete paths (controld's // deleteInstance and `lchat remove`) fan a tombstone // (fanOutMeshRouteDelete → DELETE /api/mesh/route/{host}), and // `mesh-seed --prune` reconciles whatever a best-effort fan-out missed — // without this, a destroyed space lived on (and WOKE, from a stale replica) // on any peer that held its row, violating the destroy promise. Live case: // six zombies swept off kepler 2026-07-25. Deletion fan-out and --prune are // still DOMAIN-scoped only: a region-owned, out-of-domain row's tombstone is // the next multi-edge slice (alongside out-of-domain TLS, below) — inert today // because nothing region-places out-of-domain hosts until that TLS lands. // // Honest gap, narrowed: a HIBERNATE FLIP still doesn't replicate — and must // not naively: a peer holding Hibernate=false would proxy to a local socket // that doesn't exist instead of forwarding. A flipped-to-systemd host needs // a "lives elsewhere" route shape peers don't have yet. Renames don't exist // (host is the key). import ( "bytes" "context" "crypto/subtle" "crypto/tls" "encoding/json" "flag" "fmt" "io" "log" "net" "net/http" "net/url" "os" "path/filepath" "strings" "time" ) // meshRoutePayload is the wire shape for one replicated route — deliberately // narrower than Route (see file doc comment). The Env* fields carry the // per-instance EnvironmentFile values that are NOT route fields (admin token, // entitlement, seeded name): a hibernating route's first wake on the receiving // box reads /data/.env exactly as a local wake would (activator.go // s3Backend.start), so without them a peer-woken instance would boot with no // admin token and the creating box's setup link could never authenticate. // They ride the same verified-TLS, mesh-token-authenticated hop the // PushRelayToken already does. type meshRoutePayload struct { Host string Kind string Hibernate bool Replication string AppStoreURL string HomeOrigin string PushRelayURL string PushRelayToken string HomeRegion string Provider string Version string EnvAdminToken string EnvEntitlement string EnvGroupName string } func routeToPayload(r Route) meshRoutePayload { return meshRoutePayload{ Host: r.Host, Kind: r.Kind, Hibernate: r.Hibernate, Replication: r.Replication, AppStoreURL: r.AppStoreURL, HomeOrigin: r.HomeOrigin, PushRelayURL: r.PushRelayURL, PushRelayToken: r.PushRelayToken, HomeRegion: r.HomeRegion, Provider: r.Provider, Version: r.Version, } } // payloadFromRouteEnv is routeToPayload plus the instance env values the // receiving box must reproduce (see the meshRoutePayload doc). env is the // same map materializeInstance writes to /data/.env (or a readEnvFile // of it, for the seed). func payloadFromRouteEnv(r Route, env map[string]string) meshRoutePayload { p := routeToPayload(r) p.EnvAdminToken = env["LCHAT_ADMIN_TOKEN"] p.EnvEntitlement = env["LCHAT_ENTITLEMENT"] p.EnvGroupName = env["LCHAT_GROUP_NAME"] return p } // toLocalRoute rebuilds a full Route from a peer's payload, deriving the // box-local Socket/DBPath itself — never trusting a peer's filesystem layout // — exactly as registerInstance does for a route created locally. func (p meshRoutePayload) toLocalRoute() (Route, error) { host := strings.ToLower(strings.TrimSpace(p.Host)) if !validHost(host) { return Route{}, fmt.Errorf("%w: %q", errInvalidHost, host) } socket := filepath.Join(instanceSocketDir, host+".sock") if err := checkSocketLen(socket); err != nil { return Route{}, err } return Route{ Host: host, Kind: p.Kind, Socket: socket, DBPath: filepath.Join(instanceDataDir, host+".db"), AppStoreURL: p.AppStoreURL, HomeOrigin: p.HomeOrigin, PushRelayURL: p.PushRelayURL, PushRelayToken: p.PushRelayToken, HomeRegion: p.HomeRegion, Provider: p.Provider, Hibernate: p.Hibernate, Replication: p.Replication, Version: p.Version, }, nil } // meshReplicateRoute POSTs one route to peer's /api/mesh/route, authenticated // with the mesh's shared token. Used both by fanOutMeshRoute and by // runMeshSeed's bulk seed. // The peer's own base URL (LCHAT_MESH_SELF, typically a bare address like // https://10.0.0.5) was never issued a certificate of its own — only // registered, wildcard-covered instance hostnames get one. So — exactly like // mesh.go's own newPeerProxy authenticates a proxied hop by the INSTANCE // hostname, never the peer's bare identity — this dials the peer's raw // address but performs TLS (and sends Host) as route.Host: a wildcard-covered // hostname the receiving box's wildcard manager hands back a valid cert for // regardless of whether that host is registered THERE yet (the "unknown // subdomain still completes TLS" property wildcard.go relies on). For a // DOMAIN-covered target that cert is the box's own wildcard, so TLS completes // exactly as before. Region replication (fanOutMeshRoute's union with the // route's home_region owners) can now also target a box whose per-country // LCHAT_MESH_DOMAINS does NOT cover payload.Host — an out-of-domain host that // box may hold no wildcard for, so this handshake can fail there. Making a box // present a valid cert for an out-of-domain host it now owns by region is the // NEXT multi-edge slice (docs/multi-edge.md), deliberately not built here; a // same-domain fleet (production today) is unaffected. func meshReplicateRoute(ctx context.Context, peer, token string, payload meshRoutePayload) error { body, err := json.Marshal(payload) if err != nil { return err } resp, err := meshPeerRequest(ctx, peer, token, http.MethodPost, "/api/mesh/route", payload.Host, body) if err != nil { return err } defer resp.Body.Close() if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusNoContent { return fmt.Errorf("peer %s: %s", peer, resp.Status) } return nil } // replicationTargets is the set of OTHER live boxes that must hold route's // registry row: the boxes covering host by DOMAIN (this box's // LCHAT_MESH_DOMAINS — today's fleet-wide fan-out for an in-domain host) UNION // the boxes owning host by REGION (those advertising route.HomeRegion). The // union is the multi-edge fix (docs/multi-edge.md): a space placed by home_region // onto a box whose per-country domain does NOT cover the host still needs a row // there to serve when the DNS-target box proxies to it — domain scope alone would // strand it. Self is excluded (registerInstance already upserted the local row), // and a box qualifying by BOTH domain and region is sent to once. // // Region targets are non-empty only on a region-aware box (one that fetched peer // regions; m.regions is empty otherwise) and only for a route carrying a // home_region, so with no LCHAT_MESH_REGION and no home_region anywhere this is // exactly the domain-scoped set — byte-identical to pre-region replication. func (m *meshSvc) replicationTargets(route Route) []string { seen := map[string]bool{m.self: true} // never replicate to ourselves var targets []string add := func(boxes []string) { for _, b := range boxes { if !seen[b] { seen[b] = true targets = append(targets, b) } } } if m.inScope(route.Host) { add(m.liveBoxes()) // domain-scoped: the whole live fleet, as before regions } add(m.boxesInRegion(route.HomeRegion)) // region-owned: the owner box(es) return targets } // boxesInRegion returns the live boxes whose advertised region matches region — // the mesh members that own a route with this home_region. It is candidatesFor's // region filter keyed on a region string (rather than a host's registry row), // reusing slice 1's peer-region view (m.regions, self via m.region — never a // second advertisement mechanism). An empty region, or a region-agnostic box // (m.regions empty because it never fetched peer regions), yields no matches, // which is what keeps region replication inert until the fleet opts into regions. func (m *meshSvc) boxesInRegion(region string) []string { if region == "" { return nil } live := m.liveBoxes() m.mu.Lock() regions := m.regions m.mu.Unlock() var matched []string for _, b := range live { r := regions[b] if b == m.self { r = m.region // our own region is known locally, never via S3 } if r == region { matched = append(matched, b) } } return matched } // fanOutMeshRoute replicates route to every OTHER box that must hold its // registry row — the DOMAIN ∪ REGION union replicationTargets computes // (docs/multi-edge.md) — best-effort: a peer that's down or errors just can't // yet serve this host (see the file doc comment's honest gap on mutations — // creation retries only via a later add-instance or an operator's mesh-seed // rerun). A no-op — byte-identical to today — whenever the mesh is off, or the // host is out of LCHAT_MESH_DOMAINS scope AND this box can't region-place it // (no LCHAT_MESH_REGION, or the route carries no home_region), so self-hosters, // non-mesh boxes, and every region-agnostic box are untouched. Called right // after registerInstance's local reg.upsert. func fanOutMeshRoute(hc *hibernateConfig, route Route, env map[string]string) { mesh, err := meshFromEnv(hc) if err != nil || mesh == nil { return } // Pre-region early-out, kept byte-identical: an out-of-domain host on a box // that can't region-place it (region unset, or the route has no home_region) // replicates nowhere, and — crucially — doesn't even beat. An agnostic box // always lands here for an out-of-scope host, exactly as before regions. if !mesh.inScope(route.Host) && (mesh.region == "" || route.HomeRegion == "") { return } payload := payloadFromRouteEnv(route, env) mesh.beat() // one synchronous PUT+LIST(+region refresh) for a fresh view for _, peer := range mesh.replicationTargets(route) { ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) err := meshReplicateRoute(ctx, peer, mesh.token, payload) cancel() if err != nil { log.Printf("mesh: replicate %s to %s failed: %v — it can't serve this host until a later replication succeeds", route.Host, peer, err) } else { log.Printf("mesh: replicated %s to %s", route.Host, peer) } } } // meshPeerRequest performs one authenticated request against a mesh peer's // raw address, TLS'd (and Host'd) as sniHost — the wildcard-covered instance // hostname trick meshReplicateRoute documents above. Shared by the route // upsert, the tombstone, and the reconciliation listing. func meshPeerRequest(ctx context.Context, peer, token, method, path, sniHost string, body []byte) (*http.Response, error) { peerURL, err := url.Parse(peer) if err != nil || peerURL.Host == "" { return nil, fmt.Errorf("bad peer URL %q: %w", peer, err) } reqURL := url.URL{Scheme: peerURL.Scheme, Host: sniHost, Path: path} var rd io.Reader if body != nil { rd = bytes.NewReader(body) } req, err := http.NewRequestWithContext(ctx, method, reqURL.String(), rd) if err != nil { return nil, err } req.Header.Set("Content-Type", "application/json") req.Header.Set("Authorization", "Bearer "+token) dialAddr := peerDialAddr(peerURL) transport := &http.Transport{ DialContext: func(ctx context.Context, network, _ string) (net.Conn, error) { return (&net.Dialer{Timeout: 10 * time.Second}).DialContext(ctx, network, dialAddr) }, } if peerURL.Scheme == "https" { transport.DialTLSContext = func(ctx context.Context, network, _ string) (net.Conn, error) { conn, err := (&net.Dialer{Timeout: 10 * time.Second}).DialContext(ctx, network, dialAddr) if err != nil { return nil, err } tlsConn := tls.Client(conn, &tls.Config{ServerName: sniHost}) if err := tlsConn.HandshakeContext(ctx); err != nil { conn.Close() return nil, err } return tlsConn, nil } } return (&http.Client{Transport: transport}).Do(req) } // meshDeleteRoute sends host's tombstone to one peer — the deletion half of // replication (#1346). Idempotent on the receiving side, so re-sends and // reconciliation sweeps are always safe. func meshDeleteRoute(ctx context.Context, peer, token, host string) error { resp, err := meshPeerRequest(ctx, peer, token, http.MethodDelete, "/api/mesh/route/"+host, host, nil) if err != nil { return err } defer resp.Body.Close() if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusNoContent { return fmt.Errorf("peer %s: %s", peer, resp.Status) } return nil } // meshListRoutes reads a peer's in-scope hibernating route hosts — what // mesh-seed --prune diffs against the source registry. The SNI rides any // wildcard-covered host the caller knows (its own first route works); the // peer answers from its registry either way. func meshListRoutes(ctx context.Context, peer, token, sniHost string) ([]string, error) { resp, err := meshPeerRequest(ctx, peer, token, http.MethodGet, "/api/mesh/routes", sniHost, nil) if err != nil { return nil, err } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { return nil, fmt.Errorf("peer %s: %s", peer, resp.Status) } var hosts []string if err := json.NewDecoder(io.LimitReader(resp.Body, 1<<20)).Decode(&hosts); err != nil { return nil, err } return hosts, nil } // fanOutMeshRouteDelete replicates a deletion to every live mesh peer — the // tombstone counterpart of fanOutMeshRoute, called from both delete paths // (controld's deleteInstance and `lchat remove`) after the local row is gone. // Best-effort exactly like creation fan-out: a peer that's down keeps its // stale row until a `mesh-seed --prune` reconciles it. func fanOutMeshRouteDelete(hc *hibernateConfig, host string) { mesh, err := meshFromEnv(hc) if err != nil || mesh == nil || !mesh.inScope(host) { return } mesh.beat() for _, peer := range mesh.liveBoxes() { if peer == mesh.self { continue } ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) err := meshDeleteRoute(ctx, peer, mesh.token, host) cancel() if err != nil { log.Printf("mesh: delete %s on %s failed: %v — its stale row stays until a mesh-seed --prune", host, peer, err) } else { log.Printf("mesh: deleted %s on %s", host, peer) } } } // meshRouteHandler is the receiving side: POST /api/mesh/route, wired by // runRouter only when the mesh is on. Bearer-authenticated with the same // shared token every mesh member is configured with — without this, any // caller could hand this box arbitrary routes. func meshRouteHandler(reg *Registry, rt *router, token string) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { const prefix = "Bearer " auth := r.Header.Get("Authorization") if !strings.HasPrefix(auth, prefix) || subtle.ConstantTimeCompare([]byte(strings.TrimPrefix(auth, prefix)), []byte(token)) != 1 { http.Error(w, "unauthorized", http.StatusUnauthorized) return } var payload meshRoutePayload if err := json.NewDecoder(io.LimitReader(r.Body, 1<<16)).Decode(&payload); err != nil { http.Error(w, "bad request", http.StatusBadRequest) return } route, err := payload.toLocalRoute() if err != nil { http.Error(w, err.Error(), http.StatusBadRequest) return } if err := reg.upsert(route); err != nil { log.Printf("mesh: route replication upsert %s failed: %v", route.Host, err) http.Error(w, "internal error", http.StatusInternalServerError) return } // The instance env travels with the route (meshRoutePayload doc): a // peer-woken instance must boot with the SAME admin token its creator // minted, or the setup link dies and the admin console is orphaned. // The sender is canonical, so a re-replication may overwrite — but a // payload WITHOUT a token (an older peer) never clobbers a file that // exists. if payload.EnvAdminToken != "" { env := instanceEnvMap(payload.EnvAdminToken, payload.AppStoreURL, payload.HomeOrigin, payload.PushRelayURL, payload.PushRelayToken, payload.EnvEntitlement, payload.EnvGroupName) if err := writeEnvFile(filepath.Join(instanceDataDir, route.Host+".env"), env); err != nil { log.Printf("mesh: route replication env write %s failed: %v — its first wake here will boot token-less", route.Host, err) } } if err := rt.reload(); err != nil { log.Printf("mesh: route replication reload after %s failed: %v", route.Host, err) } w.WriteHeader(http.StatusNoContent) } } // meshAuthed wraps a mesh endpoint behind the shared bearer token — the same // constant-time gate meshRouteHandler carries inline. func meshAuthed(token string, next http.HandlerFunc) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { const prefix = "Bearer " auth := r.Header.Get("Authorization") if !strings.HasPrefix(auth, prefix) || subtle.ConstantTimeCompare([]byte(strings.TrimPrefix(auth, prefix)), []byte(token)) != 1 { http.Error(w, "unauthorized", http.StatusUnauthorized) return } next(w, r) } } // meshRouteDeleteHandler: DELETE /api/mesh/route/{host} — a peer says this // space was destroyed. Idempotent (an absent row still answers 204, so // re-sends and prune sweeps never error). Deletes the row, the instance's // webhook-ingest row (in. dies with the space, mirroring lchat remove), // and the REPLICATED env file — it holds a destroyed space's admin token and // arrived via replication, so it's ours to take back; the db/blob files are // member-data-shaped and stay, exactly like every other delete path (the // data-safety rule). The reload's retire pass stops any live activator child. func meshRouteDeleteHandler(reg *Registry, rt *router, token string) http.HandlerFunc { return meshAuthed(token, func(w http.ResponseWriter, r *http.Request) { host := strings.ToLower(strings.TrimSpace(r.PathValue("host"))) if !validHost(host) { http.Error(w, "bad host", http.StatusBadRequest) return } if err := reg.delete(host); err != nil { log.Printf("mesh: tombstone delete %s failed: %v", host, err) http.Error(w, "internal error", http.StatusInternalServerError) return } if err := reg.delete("in." + host); err != nil { log.Printf("mesh: tombstone delete in.%s failed: %v", host, err) } if err := os.Remove(filepath.Join(instanceDataDir, host+".env")); err != nil && !os.IsNotExist(err) { log.Printf("mesh: tombstone env removal for %s failed: %v", host, err) } if err := rt.reload(); err != nil { log.Printf("mesh: reload after tombstone %s failed: %v", host, err) } log.Printf("mesh: tombstoned %s (peer says it was destroyed)", host) w.WriteHeader(http.StatusNoContent) }) } // meshRouteListHandler: GET /api/mesh/routes — this box's in-scope hibernating // instance hosts, the set a peer's mesh-seed --prune may reconcile. Routing // metadata only (hostnames), gated by the same shared token as everything // else on this surface. func meshRouteListHandler(reg *Registry, rt *router, token string, domains []string) http.HandlerFunc { return meshAuthed(token, func(w http.ResponseWriter, r *http.Request) { routes, err := reg.list() if err != nil { http.Error(w, "internal error", http.StatusInternalServerError) return } hosts := []string{} for _, t := range routes { if t.Kind == "instance" && t.Hibernate && hostInDomains(t.Host, domains) { hosts = append(hosts, t.Host) } } w.Header().Set("Content-Type", "application/json") json.NewEncoder(w).Encode(hosts) }) } // pruneCandidates picks the peer hosts a reconciling seed may tombstone: // in-scope peer rows whose host the SOURCE registry doesn't know at all (any // kind, any flag — existing anywhere on the source means alive). The zero // guard is load-bearing: a source that sees none of its own in-scope routes // (wrong registry path, empty box) must never be allowed to conclude the // peer's whole set is dead. func pruneCandidates(sourceHosts, peerHosts, domains []string, sourceInScope int) []string { if sourceInScope == 0 { return nil } known := make(map[string]bool, len(sourceHosts)) for _, h := range sourceHosts { known[h] = true } var out []string for _, h := range peerHosts { if !known[h] && hostInDomains(h, domains) { out = append(out, h) } } return out } // runMeshSeed is `lchat mesh-seed [--region ] `: // bulk-replicates the hibernating routes one peer must hold — the one-time // catch-up a freshly joined box needs (it starts with an empty registry) before // HRW can honestly place any host there. It seeds by the same DOMAIN ∪ REGION // union fanOutMeshRoute uses (docs/multi-edge.md): a route whose host is under // this seed's LCHAT_MESH_DOMAINS, PLUS — when --region names the joining box's // placement region (its LCHAT_MESH_REGION) — a route whose home_region matches // it, even if out of domain. Without --region the filter is exactly domain scope, // so an existing same-domain fleet seeds byte-identically to before. // // Reads LCHAT_MESH_DOMAINS/LCHAT_MESH_TOKEN directly rather than through // meshFromEnv: the CALLER (the already-running box doing the seeding) need not // have LCHAT_MESH_SELF set yet — seeding is an operator step that runs before // the mesh itself is flipped on (see docs/instance-router.md). The joining box's // region is passed explicitly rather than looked up, so the seed needs no // bucket/heartbeat wiring — the operator who just set LCHAT_MESH_REGION on the // new box supplies the same value here. func runMeshSeed(args []string) { fs := flag.NewFlagSet("mesh-seed", flag.ExitOnError) registryPath := fs.String("registry", envOr("LCHAT_REGISTRY", "/data/registry.db"), "router registry SQLite path") envFile := fs.String("env-file", "", "load LCHAT_MESH_* (and anything else) from this KEY=value file first — lets a sudo'd operator run borrow the router's env (e.g. /data/router.env) without a root shell") prune := fs.Bool("prune", false, "after seeding, tombstone the peer's in-scope hibernating rows this registry doesn't know — reconciles deletions a best-effort fan-out missed (#1346); refuses on an empty source view") peerRegion := fs.String("region", "", "the JOINING box's placement region (its LCHAT_MESH_REGION); when set, routes whose home_region matches are seeded too, even if out of LCHAT_MESH_DOMAINS") _ = fs.Parse(reorderFlagsFirst(fs, args)) if *envFile != "" { kv, err := readEnvFile(*envFile) if err != nil { die("mesh-seed: read --env-file: %v", err) } for k, v := range kv { if os.Getenv(k) == "" { // real env always wins over the file os.Setenv(k, v) } } } peer := fs.Arg(0) if peer == "" { die("mesh-seed: usage: lchat mesh-seed [--env-file /data/router.env] [--region ] [--prune] ") } domains := splitDomains(os.Getenv("LCHAT_MESH_DOMAINS")) if len(domains) == 0 { die("mesh-seed: LCHAT_MESH_DOMAINS must be set to the domains this seed covers") } token := strings.TrimSpace(os.Getenv("LCHAT_MESH_TOKEN")) if token == "" { die("mesh-seed: LCHAT_MESH_TOKEN must be set") } region := strings.TrimSpace(*peerRegion) reg, err := openRegistry(*registryPath) if err != nil { die("mesh-seed: open registry: %v", err) } routes, err := reg.list() if err != nil { die("mesh-seed: list routes: %v", err) } seeded := 0 for _, route := range routes { if !route.Hibernate { continue } // DOMAIN ∪ REGION, mirroring fanOutMeshRoute/replicationTargets: seed a // route the joining box covers by domain, or (when --region is given) one // it owns by home_region even though it's out of domain. domainCovered := hostInDomains(route.Host, domains) regionOwned := region != "" && route.HomeRegion == region if !domainCovered && !regionOwned { continue } // The instance env rides along (meshRoutePayload doc) — without it the // peer wakes this host token-less. Missing file (never materialized // here either) just seeds the bare route, exactly as before. env, envErr := readEnvFile(filepath.Join(instanceDataDir, route.Host+".env")) if envErr != nil && !os.IsNotExist(envErr) { fmt.Printf(" %s: env unreadable (%v) — seeding the bare route\n", route.Host, envErr) } ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) err := meshReplicateRoute(ctx, peer, token, payloadFromRouteEnv(route, env)) cancel() if err != nil { fmt.Printf(" %s: FAILED (%v)\n", route.Host, err) continue } fmt.Printf(" %s: seeded\n", route.Host) seeded++ } fmt.Printf("seeded %d/%d route(s) the joining box owns to %s\n", seeded, len(routes), peer) if *prune { // Reconcile: anything the peer serves in-scope that this registry has // no row for AT ALL (any kind, any flag) was deleted here — tombstone // it there. The zero guard in pruneCandidates keeps an empty or // misread source from ever mass-deleting a peer. var all []string inScope := 0 var sni string for _, route := range routes { all = append(all, route.Host) if route.Hibernate && hostInDomains(route.Host, domains) { inScope++ if sni == "" { sni = route.Host } } } if sni == "" { die("mesh-seed: --prune refused — this registry has no in-scope hibernating routes to speak for") } ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) peerHosts, err := meshListRoutes(ctx, peer, token, sni) cancel() if err != nil { die("mesh-seed: --prune: list peer routes: %v", err) } pruned := 0 for _, h := range pruneCandidates(all, peerHosts, domains, inScope) { ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) err := meshDeleteRoute(ctx, peer, token, h) cancel() if err != nil { fmt.Printf(" %s: prune FAILED (%v)\n", h, err) continue } fmt.Printf(" %s: pruned (deleted here, tombstoned there)\n", h) pruned++ } fmt.Printf("pruned %d stale route(s) on %s\n", pruned, peer) } }