package host import ( "bytes" "context" "encoding/json" "errors" "fmt" "log" "math/rand/v2" "os" "slices" "strings" "sync" "time" "github.com/carlosframework/platform/internal/appconfig" "github.com/carlosframework/platform/internal/hostclaim" "github.com/carlosframework/platform/internal/registry" "github.com/carlosframework/platform/internal/release" "github.com/carlosframework/platform/internal/domains" ) // The multi-region ingress spec (2026-08-20, Part 3, closes the design half // of #238): every platform box publishes what it homes so any edge can fold // the fleet's per-box files into a deployment-wide host→home map. v1 writes // only `role: "home"` — the anticipated read-local role values are Task 3's // concern, not this file's. // // edgesFilePrefix is the flat prefix those files live under. It is named // rather than spelled twice because there is now a READER on this side too — // steering_loop.go lists it to fold the fleet's files (internal/edge's // homeMap has its own copy, on the far side of an import boundary this // package does not cross). const edgesFilePrefix = "control/edges/" func edgesFileKey(box string) string { return edgesFilePrefix + box + ".json" } // edgeHost is the per-host entry in edgesFile.Hosts. An object, not a bare // string, is deliberately load-bearing: it is read-local anticipation — // Task 3's mirror and later roles (e.g. a "read" role) need somewhere to // grow into without a wire-shape break. type edgeHost struct { Role string `json:"role"` // Platform marks a row the platform minted (registry Platform or // Provisioned) — the class edge.currentHosts admits to wildcard-parent // derivation. Additive; always emitted (never omitempty), so a file with // no "platform" key at all is definitely a pre-mark publisher, not a box // homing nothing platform-class. A reader seeing no flag treats the host // as unclassed and derives no parent from it (#300). Platform bool `json:"platform"` // Alias marks an attached custom domain (route_aliases, #287) as opposed // to a registry route. Additive (omitempty). The reader's fold lets a // route beat an alias on a name collision across boxes: a tenant // attaching an operator's hostname must never capture it fleet-wide. Alias bool `json:"alias,omitempty"` // Replica marks the account's checkout replica for this host (#162): // this box serves the account from a one-way mirror of it — same app, // same account, role=replica record — instead of holding the // authoritative data. Additive (omitempty), so every file written // before the field existed parses byte-identically, and a reader that // has never heard of it ignores it. Published from the HOME route's // Replicas column (registry.Route.Replicas): a replica placement is // where the account is SERVED, and the home's row is the one that // knows. Read by the steering converger (a replica placement earns a // latency record) and available to every edges-file reader; the edge's // own dispatch never acts on it — a replica box serves its row exactly // as any home serves its own, and a replica never hairpins. Replica bool `json:"replica,omitempty"` // Wildcard marks a delegated customer wildcard parent (wildcard_domains, // #305): the key is "*." and every edge should load the parent's // mirrored wildcard PEM. With role "home" the box also serves the // catch-all (dial target); with role "wildcard" it is the cert half // alone. Additive (omitempty); never platform. Wildcard bool `json:"wildcard,omitempty"` // Account and App name the one app that owns everything this box serves // under the hostname (#175 P7). The edge-local door on another box checks // a door key against them before it backhauls here. Published only when // every route behind the name agrees: the row itself, every path-prefix // row sharing it, and for an alias or a served catch-all, its primary's. // A hostname split between apps publishes neither, and the door refuses // it remotely. Additive (omitempty). Account string `json:"account,omitempty"` App string `json:"app,omitempty"` // Stamp is the claim's holder.stamp for the installation this box is // serving (region HA spec 1 HA4). 0 = unstamped: no claim, or a writer // that predates it. Two files claiming one host are settled by the // higher stamp on the edge (homemap.go foldHomes). Stamp int64 `json:"stamp,omitempty"` } // hostOwner folds the owners of every route served under one hostname. Any // disagreement, or any route with no account or app, leaves no owner. type hostOwner struct { account, app string set, mixed bool } func (o *hostOwner) add(account, app string) { switch { case account == "" || app == "": o.mixed = true case !o.set: o.account, o.app, o.set = account, app, true case o.account != account || o.app != app: o.mixed = true } } func (o hostOwner) owner() (account, app string) { if !o.set || o.mixed { return "", "" } return o.account, o.app } // servedOwners answers, for each served row's hostname, the owner the edge's // own dispatch would reach under it: the row's route, and the route each // path-prefix row on it targets. A target that names a route in the registry // counts whatever that route's status. When it is not ready the edge falls // back to the host's own route today, but it counts anyway, so the owner never // outlives the target turning ready. A target naming no route here can only // fall back to the host's own route, which is already counted. func servedOwners(routes []registry.Route, served map[string]edgeHost, paths []registry.RoutePath) map[string]*hostOwner { byHost := make(map[string]registry.Route, len(routes)) for _, r := range routes { byHost[r.Host] = r } owners := make(map[string]*hostOwner, len(served)) for host := range served { r := byHost[host] o := &hostOwner{} o.add(release.RouteAccount(r), r.App) owners[host] = o } for _, p := range paths { o, ok := owners[p.Host] if !ok { continue } if t, ok := byHost[p.Target]; ok { o.add(release.RouteAccount(t), t.App) } } return owners } // edgeHeadroom is this box's spare capacity at the time the file was // written. Zero values mean "could not read", never "none left": a chooser // ranks an unreadable box last, it does not exclude it. type edgeHeadroom struct { MemFreeMiB int64 `json:"mem_free_mib"` DiskFreeMiB int64 `json:"disk_free_mib"` Awake int `json:"awake"` // MemTotalMiB and DiskTotalMiB are what the free figures are shares // of (region HA spec 2 §7). Additive and omitted when 0, a reading // that failed: a reader takes absent and 0 alike, as unknown. MemTotalMiB int64 `json:"mem_total_mib,omitempty"` DiskTotalMiB int64 `json:"disk_total_mib,omitempty"` } const capRegionHA1 = "region-ha-1" // edgesFileCapabilities publishes region-ha-1 only when CARLOS_REGION_HA is // set to something other than "" or "0". offline-move-1 is added per pass // by withOfflineCapability. func edgesFileCapabilities(getenv func(string) string) []string { if !regionHAEnabled(getenv) { return nil } return []string{capRegionHA1} } // withOfflineCapability is caps with offline-move-1 and offline-stage-1 when // capable (a box that can take part in an offline move can stage one). The loop // passes armedOfflineCapable(): the armed moves worker's own answer, the one // the reconcile hooks act on, so a box never publishes the capability // without the hold that makes it safe (a target that published it with no // armed worker would mint a landing row ready). caps is not modified. func withOfflineCapability(caps []string, capable bool) []string { if !capable { return caps } return append(slices.Clone(caps), capOfflineMove1, capOfflineStage1) } // headroomCapabilities is what headroomFor is asked about: region HA's // capability alone. offline-move-1 publishes no headroom, so a box that // gains only it publishes the file it always has, plus the one string. func headroomCapabilities(caps []string) []string { if slices.Contains(caps, capRegionHA1) { return caps } return nil } // awakeCount is how many routes this box has running now. An error reads // as zero: headroom is advisory. func awakeCount(reg *registry.Registry) int { routes, err := reg.List() if err != nil { return 0 } n := 0 for _, r := range routes { if r.Status == "ready" && !r.Hibernate { n++ } } return n } // edgesFile is the wire shape at edgesFileKey(box), schema v1 (spec Part 3): // // {"v":1,"box":"quiet-harbour","eip":"203.0.113.10","region":"eu-west-1","generation":"2026-08-22T09:00:00Z","hosts":{"app.example.com":{"role":"home"}}} // // Region is additive (Part 3, Task 4's steering converger needs it for // latency records): omitempty, absent entirely when it could not be // resolved (see edgesFileRegion) — a pre-region body written by an older // build still parses, and this build's own body for an unresolved-region // box is byte-identical to what it always published. // // The value is the AWS region (e.g. "eu-west-1"), NOT CARLOS_REGION — that // env var is the CARLOS placement code ("ie", "fleet/"), a DIFFERENT // vocabulary that Task 5's ChangeResourceRecordSets calls would reject // batch-wide if it ever reached here (plan review catch, 2026-08-23). // edgesFileRegion resolves the real thing. type edgesFile struct { V int `json:"v"` Box string `json:"box"` EIP string `json:"eip"` Region string `json:"region,omitempty"` // PrivateIP is the address an edge in this box's region dials (region // HA spec 2 §6), and PrivateCIDR the region network that holds it, // both validated by edgesfile.PrivateAddress before they are written. // Routing data only. Additive and omitempty: a reader that does not know // them, or finds them absent, dials EIP, as before. PrivateIP string `json:"private_ip,omitempty"` PrivateCIDR string `json:"private_cidr,omitempty"` // Placement is this box's CARLOS placement code — CARLOS_REGION verbatim // ("ie", "uk"), the OTHER region vocabulary (see Region above, issue // #329). Additive and omitempty like Region: a pre-placement body parses // unchanged, and a box with no CARLOS_REGION set (single-box // deployments, the legal unset state) publishes no key at all. The // console's create door folds every file's Placement into the armed set // a requested instance region is validated against — dormant until at // least one box publishes the field, so roll order does not matter. Placement string `json:"placement,omitempty"` // DataBucket/DataBucketRegion are this box's hibernate/data bucket // coordinates (CARLOS_HIBERNATE_BUCKET/_REGION verbatim) — published so // a move SOURCE can aim its staging mirror at the target box's bucket // from the same file set everything else reads (instance-move spec). // Additive + omitempty like Placement; a box with no hibernate bucket // (activator disabled) publishes neither key. DataBucket string `json:"data_bucket,omitempty"` DataBucketRegion string `json:"data_bucket_region,omitempty"` // HibernationConfigured is this box's own answer to "is my hibernation // activator wired" — true exactly when CARLOS_HIBERNATE_BUCKET is set, // which is the same env, read in the same process, that decides whether // activator.ConfigFromEnv builds a Config at all (internal/host/agent.go // arms the activator iff that Config is non-nil, and log.Fatalfs rather // than continuing if the env is set but malformed — so a live agent // process cannot publish this field disagreeing with its own activator). // // Deliberately NOT omitempty, and this is the whole point of the field. // A reader needs to tell three states apart, and only an explicitly // published `false` can carry the third: // // absent → the writer predates this field; its capability is UNKNOWN // true → the activator is configured // false → the activator is not configured // // Before it, the only signal was data_bucket, and its absence was // ambiguous in exactly the way that matters: an old writer and a // bucket-less new box produce byte-identical files. A console reading // that had to guess, and the guess it made was to infer one box's // silence from ANOTHER box's speech — so the first box in a fleet to // publish a bucket withdrew every region that had not yet rolled, all // of them healthy. One writer's upgrade cannot establish another // writer's capabilities; this field is how each box answers for itself. // // It reports CONFIGURATION, not verified hosting readiness: a box whose // IAM role cannot reach the bucket it names, or whose // CARLOS_HIBERNATE_REGION is wrong, publishes true here and still fails // every wake. Naming it for what it actually knows is the point. HibernationConfigured bool `json:"hibernation_configured"` // ServesSharedPool and AcceptsInbound are this box's own answers to // "would my reconcile pass mint an ORDINARY instance record" — one in // the shared "" pool, needing inbound service (nearest-region spec // §6.1). Deliberately NOT omitempty, HibernationConfigured's precedent: // absent is a writer that predates the field (unknown), false is a box // that said no. The console needs both positively true before it makes // an AUTOMATIC choice of this box's region; explicit requests ignore // them. Computed from the same ReconcileConfig the reconcile pass uses // (edgesFileEligibility), never a second reading of the environment. ServesSharedPool bool `json:"serves_shared_pool"` AcceptsInbound bool `json:"accepts_inbound"` // SecretsKeyFP is the fingerprint of this box's CARLOS_SECRETS_KEY's // public half (appconfig.KeyFingerprint), NEVER the key. Additive and // omitempty, the move-any-app spec's shape: absent means this box holds // no key, or one that does not parse — so it renders no exec config, // and an instance placed here would run with none (nearest-region spec // R9). The console requires it to equal the fingerprint of its own // CARLOS_SECRETS_PUBKEY before it places anything here automatically. SecretsKeyFP string `json:"secrets_key_fp,omitempty"` // BoxID is the name claims use for this box (region HA spec 1 §2). // Today it is the box's label: every box predates region HA and keeps // it. Plan 1b adds instance ids for Auto Scaling boxes, from a positive // signal rather than from being on EC2. It is NOT evidence of identity: // that is the box record (internal/boxid). A reader compares the two and // treats a mismatch as an ineligible box. BoxID string `json:"box_id,omitempty"` // Capabilities gates multi-box regions (spec §8): "region-ha-1" once // CARLOS_REGION_HA is on. Capabilities []string `json:"capabilities,omitempty"` // Draining is set while the placer empties this box (spec 4). Draining bool `json:"draining,omitempty"` // Headroom is what placement chooses by (spec 2). It is refreshed when // the file is written for another reason and on the loop's own tick; a // headroom change alone never forces a write (see edgesFileWrite). Headroom *edgeHeadroom `json:"headroom,omitempty"` // The moves protocol floor (moves spec §11; withMovesFields). The move // door admits a move only when every file of both regions shows the // published floor armed here and carlos.prev judged against it. // // MovesProtocol is the protocol this build speaks. NOT omitempty: absent // is a writer that predates moves, which the door reads as below any // floor. MovesProtocol int `json:"moves_protocol"` // MovesFloorMinRelease is the min_release of the floor armed on this box // (/.moves/floor.min-release); absent: none armed. MovesFloorMinRelease string `json:"moves_floor_min_release,omitempty"` MovesFloorState string `json:"moves_floor_state,omitempty"` MovesFloorGeneration uint64 `json:"moves_floor_generation,omitempty"` // PrevRelease is carlos.prev's release (carlos.prev.version); "none" // when there is no carlos.prev; absent when it is unknown. PrevRelease string `json:"prev_release,omitempty"` // PrevAllowedFor is the verdict (carlos.prev.allowed): the min_release // carlos.prev was judged at or above. Absent: no verdict. PrevAllowedFor string `json:"prev_allowed_for,omitempty"` // The seeds protocol (seeded-instances spec §3.3), read by the console's // seedRegionReady. SeedsProtocol is not omitempty: 0 is a real answer // (a build that cannot place a seed). SeedsFloorMinRelease is the // armed seeds floor's min_release (absent: not armed), and // PrevSeedsAllowedFor the floor carlos.prev was judged at or above. SeedsFloorState string `json:"seeds_floor_state,omitempty"` SeedsFloorGeneration uint64 `json:"seeds_floor_generation,omitempty"` SeedsProtocol int `json:"seeds_protocol"` SeedsFloorMinRelease string `json:"seeds_floor_min_release,omitempty"` PrevSeedsAllowedFor string `json:"prev_seeds_allowed_for,omitempty"` Generation string `json:"generation"` Hosts map[string]edgeHost `json:"hosts"` } // edgesFilePass builds this box's edges file from the routes currently in // reg (every SERVED row on this box, every SERVING kind — the split between // those belongs to the reader, Task 3, not to this writer; the split between // serving and NOT, which no reader of this file can make, belongs here — see // the sidecar guard in the loop) and publishes it at edgesFileKey(box) when // it changed. // // The change gate is the marshaled body WITH Generation cleared: Generation // is a fresh RFC3339 stamp every pass, so comparing the full body would // write on every tick even when the box's homed hosts and EIP are // identical — a timestamp-only refresh is not a change worth a bucket // write. *last holds the previous pass's gate body (also Generation- // cleared) and is updated only after a successful write, mirroring // registrySnapshotPass's *lastFP contract. // // eip may be "" (readPublicIP's "unknown" case, e.g. not on EC2, IMDS // disabled, or a transient metadata-endpoint failure). A blank reading is // NOT taken at face value: this box's public IP does not change tick to // tick, so "" almost always means the probe hiccuped, not that the address // vanished — publishing it verbatim would blank a known-good EIP for // every remote edge's dial address for up to one loop cadence. When eip is // "" and *last already holds a published file, this pass reuses that // file's EIP instead. "" only ever reaches the wire for a box that has // NEVER had a discoverable IP — the DR runbook's hosts list still gets // published in that case, just without a dial address. // // region is the AWS region resolved once at loop arm by edgesFileRegion // (the domains loop's read-once precedent — see edgesFileLoop) and is part // of the change gate like every other field except Generation: a region // relabel is a real change and must write, not be swallowed as a // timestamp-only refresh would be. // withheldAliasSaid throttles the not-published line in edgesFilePass to // once per alias per process (cleared when the alias publishes). var withheldAliasSaid sync.Map // edgesFilePass publishes one file. ident is this box's IDENTITY half of // the file — Box, Region, Placement, DataBucket, DataBucketRegion — built // once by the loop; eip is passed per pass because it is re-probed every // tick and held across blanks (below). Hosts are derived here. func edgesFilePass(ctx context.Context, s release.Store, reg *registry.Registry, ident edgesFile, eip string, last *[]byte) error { return edgesFileWrite(ctx, s, reg, ident, eip, last, false) } // edgesFileWrite is edgesFilePass with one more input. refresh is true on // the loop's own timed pass: the file is then written even if nothing but // its headroom reading changed, so headroom is never more than one timer // period stale. On every other pass a headroom change alone is not a // reason to write. func edgesFileWrite(ctx context.Context, s release.Store, reg *registry.Registry, ident edgesFile, eip string, last *[]byte, refresh bool) error { return edgesFileWriteClaims(ctx, s, reg, ident, eip, last, refresh, nil) } // errBoxAddressesUnknown fails an edges file pass whose address source could // not tell what this box holds (region HA on). Nothing has been written. var errBoxAddressesUnknown = errors.New("edges file: the box's addresses could not be read") // edgesFileWriteClaims is edgesFileWrite with the claim reader region HA // adds (edgesclaims.go). claims is nil unless CARLOS_REGION_HA is on, and // nil publishes exactly what edgesFileWrite always has. func edgesFileWriteClaims(ctx context.Context, s release.Store, reg *registry.Registry, ident edgesFile, eip string, last *[]byte, refresh bool, claims *edgesClaims) error { box := ident.Box routes, err := reg.List() if err != nil { return fmt.Errorf("edges: list: %w", err) } hosts := make(map[string]edgeHost, len(routes)) for _, r := range routes { // edge.Reload (internal/edge/edge.go) is the deployment's own // definition of "serves": it skips any row whose Status != "ready" // because a pending/creating/failed row has no working backend and // is never dialed by any edge, home or remote. Mirroring that gate // here means this file never claims role:"home" for a host nothing // actually serves. Hibernating routes stay Status "ready" (that is // what makes wake-on-request possible), so nothing reachable is // excluded by this filter. if r.Status != "ready" { continue } // A sidecar is NEVER advertised as served — by this box or any // other. This is publishResidentFleetBindings' guard on the file // that superseded it, and it is load-bearing rather than tidy, // because the edge's own Kind refusal cannot reach this path: that // refusal lives in Reload, which builds THIS box's routing map from // rows it holds. A remote edge never sees the row at all. It sees // this file, misses on its own upstream lookup, and takes the // backhaul arm (edge.go's e.backhaul → homeMap) — dialing the home // box with the sidecar's name as SNI, which is a live proxy to the // one member the platform guarantees nothing can dial. The same // entry also passes deploymentHost, so the name becomes orderable // by the cert machinery deployment-wide. // // So the writer's "ALL kinds, the serving-class split belongs to // the reader" rule above holds only while every kind is servable. // It stops holding at the first row that is unroutable BY DESIGN, // and a reader that cannot tell (a remote edge, reading JSON) is // not a reader that can make the split. if r.IsSidecar() { continue } // Keys are registry row Host values verbatim, INCLUDING *.parent // catch-all wildcard rows (Task 3's reconcile pass derives these // from a claim's catch-all row, same as edge.go's upstream map) — // this writer performs no expansion. A reader that needs exact-host // answers must apply edge.go's lookupUpstream one-label wildcard // rule itself: an exact match wins, and on a miss "*." // (the leading dot of the first label only) is tried next. // // The replica mark rides the ROW (#162): a route minted from a // role=replica record publishes with replica:true wherever its own // host appears, so every reader of the file can tell the account's // homes from its checkout replicas. See the replica loop below for // the account apex that rides along. hosts[r.Host] = edgeHost{Role: "home", Platform: r.Platform || r.Provisioned, Replica: r.Replica} } // Region HA (spec 1 §5.4): a relocatable host is home only on the box // its claim names, stamped with the installation. Before owners and // aliases, so an alias of a withheld host is withheld with it. applyEdgesClaims(ctx, claims, routes, hosts) // Owners (#175 P7): computed over the served rows alone, before any alias // or catch-all joins the map, so an alias can only ever inherit a route's. paths, err := reg.ListPaths() if err != nil { return fmt.Errorf("edges: list paths: %w", err) } owners := servedOwners(routes, hosts, paths) for host, h := range hosts { h.Account, h.App = owners[host].owner() hosts[host] = h } // Attached custom domains (#151) live in route_aliases, not routes, and // were absent from this file (#287): a non-home edge neither backhauled // them (Home missed) nor answered their validations or let the issuer // order for them (KnownHost missed). Publish each alias whose primary is // a served row above, under the alias's own name — the reader wants // exact names, and every reader already does the right thing with one: // Home dials the home box with the alias as SNI, and the home box's own // upstream map serves it. Bounds, each doing work: // // - only an alias whose DNS the sweep has stamped "pointing" (at one of // PLATFORM_EDGE_IPS). Attach needs no DNS proof, so the alias set is // tenant-chosen and unbounded; publishing every one would hand a // tenant an unbounded set of names the shared issuer will order for // (#277's amplification envelope covers routes, which the platform // provisions). A name that actually resolves to this deployment is // the name whose validations must be answerable everywhere; // - never platform-class: a customer's name derives no wildcard parent; // - marked alias: the reader lets a real route on another box beat it; // - no "*.parent" catch-all alias: a remote edge could dial it but has // no cert for it (delegated wildcard parents are box-local), so it // stays exactly as before — served on its home box only; // - an alias shadowed by a real route here is the route's, as // edge.Reload rules. aliases, err := reg.ListAliases() if err != nil { return fmt.Errorf("edges: list aliases: %w", err) } for _, a := range aliases { if a.Alias == "" || strings.HasPrefix(a.Alias, "*.") { continue } if _, served := hosts[a.Host]; !served { continue } if a.DNSStatus != "pointing" { // Withheld, and said so ONCE per name per process: an alias // whose DNS resolves to an address outside PLATFORM_EDGE_IPS // (a region EIP missing from that list is the usual cause) is // otherwise served on its home box only, silently. if _, said := withheldAliasSaid.LoadOrStore(a.Alias, true); !said { log.Printf("edges: alias %s not published — dns_status %q, want \"pointing\" (does PLATFORM_EDGE_IPS list every armed edge?)", a.Alias, a.DNSStatus) } continue } withheldAliasSaid.Delete(a.Alias) if _, taken := hosts[a.Alias]; taken { continue } // The edge serves an alias with its primary's own entry, prefix map // included, so the alias's owner is the primary's, never the alias // row's own columns. acct, app := owners[a.Host].owner() // The alias is served by its primary's installation, so it carries // the primary's stamp (region HA HA4): an unstamped alias would tie // with a dead predecessor's and the edge would keep the first box // by name. hosts[a.Alias] = edgeHost{Role: "home", Alias: true, Account: acct, App: app, Stamp: hosts[a.Host].Stamp} } // Replica placements (#162): a row minted from a role=replica record — // this box holds it BECAUSE its placement is this box's (the reconciler's // scope test is the only mint gate) — publishes with the replica mark // set, which the main loop above already stamps onto its own host (the // mark rides the ROW). Nothing beyond that host is claimed: the home's // hostname and the platform apex live on the home box's rows, and a name // this box cannot serve from a local row must not be claimed (a // replica-claimed apex would let foldHomes' first-sorted-box rule hand // the apex DIAL target to a replica box fleet-wide). Custom domains // reach the edges by their owner's CNAME to edge. anyway // (the steering fold's own documented path). // Delegated customer wildcards (#305): the wildcard_domains rows are the // parents this box obtains "*." for (a claim with the Wildcard // capability, primary homed here). Published as the catch-all key with // its own class so a remote edge both dials one-label hosts under it // (Home's rule) and loads the parent's mirrored wildcard PEM — the two // halves #302 could not deliver from an alias row alone. Never platform. wds, err := reg.ListWildcardDomains() if err != nil { return fmt.Errorf("edges: list wildcard domains: %w", err) } // Two halves, published separately: the CERT half (every edge loads the // parent's PEM) always; the DIAL half (role home — remote edges backhaul // one-label hosts under the parent) only when this box actually serves // the catch-all, i.e. the claim has Catchall and its "*." alias // row points at a served primary. A Wildcard claim without Catchall // serves only its exact aliases (published above, each on its own // terms); advertising a dial target for it would have region edges // terminate and backhaul any invented name under the parent to a box // that answers 503. // catchallServed maps each served parent to the primary its "*." // alias row points at, the route the edge serves the catch-all with. catchallServed := map[string]string{} for _, a := range aliases { if parent, ok := strings.CutPrefix(a.Alias, "*."); ok { if _, served := owners[a.Host]; served { catchallServed[parent] = a.Host } } } for _, wd := range wds { if wd.Host == "" || strings.HasPrefix(wd.Host, "*.") { continue } key := "*." + wd.Host if prev, taken := hosts[key]; taken { // A real route owns the key (its catch-all row, published above): // the route keeps its role and class, and the parent's class is // ADDED — the reader's fold merges classes the same way, and a // parent whose PEM must load everywhere is a fact about the name, // not about which row got there first. prev.Wildcard = true hosts[key] = prev continue } // Marked alias whenever the DIAL half is on, and the mark is // load-bearing rather than cosmetic. The catch-all key never reaches // the alias loop above (it skips "*." rows on purpose), so this is // the only place its class can be set — and edgesfile.DeploymentRoutes // reads exactly `Role == RoleHome && !Alias` as "some live box serves // this hostname as a real route". Published unmarked, a served // catch-all is therefore evidence against itself: the next // domains.ReconcileAliases pass reads our own file back, refuses the // claim that produced the row, and drops the alias — which turns the // dial half off, unmarks nothing, and lets the pass after that // recreate it. That loop is not theoretical; it ran on *.woodstar.app // every 5-6 minutes, taking the name down about half of every hour. // A catch-all we serve IS a materialized claim, which is what the // mark means, so say so. role, alias := "wildcard", false var acct, app string var stamp int64 if primary, served := catchallServed[wd.Host]; served { role, alias = "home", true stamp = hosts[primary].Stamp // the dial half is the primary's installation // The owner is the wildcard claim's, and only when the primary // the catch-all is served by (with its prefixes) agrees with it: // every route behind the name has to have one owner. o := *owners[primary] o.add(wd.Account, wd.App) acct, app = o.owner() } hosts[key] = edgeHost{Role: role, Wildcard: true, Alias: alias, Account: acct, App: app, Stamp: stamp} } var private BoxAddresses if claims != nil && claims.addrs != nil { // Region HA spec 2 §6 (A1-12): publish only what the box's address // source reports now. An authoritative blank publishes no address, // never the last one: that may be the region's EIP, which the load // balancer has since taken. An unknown read (a metadata hiccup) // writes nothing: the previous file stands and the loop retries, // so one failed request does not hide a healthy box from its // readers. read, ok := claims.addrs.BoxAddresses() if !ok { return errBoxAddressesUnknown } private = read eip = private.Public } else if eip == "" && *last != nil { var prior edgesFile if err := json.Unmarshal(*last, &prior); err == nil { eip = prior.EIP } } f := ident f.V, f.EIP, f.Hosts, f.Generation = 1, eip, hosts, "" f.PrivateIP, f.PrivateCIDR = private.Private, private.PrivateCIDR gateCopy := f gateCopy.Headroom = nil // a changing reading is not, by itself, a reason to write gate, err := json.Marshal(gateCopy) if err != nil { return fmt.Errorf("edges: marshal: %w", err) } if bytes.Equal(gate, *last) && !(refresh && f.Headroom != nil) { return nil } f.Generation = time.Now().UTC().Format(time.RFC3339) body, err := json.Marshal(f) if err != nil { return fmt.Errorf("edges: marshal: %w", err) } if err := putReplacing(ctx, s, edgesFileKey(box), body); err != nil { return err } *last = gate return nil } // seedEdgesFileGate reads this box's last-published edges file at loop arm // and returns the change-gate body (Generation cleared) to prime *last, // mirroring seedSnapshotFingerprint's contract exactly (snapshot_loop.go): // without this, a freshly started (or crash-looping) process's in-memory // *last starts empty, so edgesFilePass's own change gate — bytes.Equal(gate, // *last) — can never match on the first tick and the process republishes an // UNCHANGED file on every restart, burning an ETag/ifMatch round trip for // nothing (plan-2 final review, Important 1, first half). // // The same read also recovers the EIP hold for free: edgesFilePass's blank- // probe fallback reads *last's EIP when this tick's probe is "", and the // seeded gate still carries the prior file's EIP (only Generation is // cleared). Without this seed, a box that restarts and hits a transient // IMDS blip on its very first tick has a nil *last, the hold's `*last != // nil` guard never fires, and the box would publish "" — blanking a known- // good EIP for every remote edge's dial address until the next successful // probe (the same review's Important 1, second half; the restart-blip // case). One seed read closes both. // // Any read or parse failure — no prior file yet, a corrupt object, a store // that is down — returns nil: the loop then behaves exactly as it always // has on a box's first-ever tick. This must never fail the loop. func seedEdgesFileGate(ctx context.Context, s release.Store, box string) []byte { body, _, err := s.Get(ctx, edgesFileKey(box)) if err != nil { return nil } var prior edgesFile if err := json.Unmarshal(body, &prior); err != nil { return nil } prior.Generation = "" prior.Headroom = nil // the gate never includes headroom: see edgesFileWrite gate, err := json.Marshal(prior) if err != nil { return nil } return gate } // edgesFileRegion resolves this box's AWS region, pure and getenv/probe- // injectable — domainsTickConfig's own shape (agent.go): the network probe // (readPlacementRegion, an IMDS call) is read ONCE by the caller and handed // in as probedRegion, so this function itself never touches the network and // its override/unset/trim contract is unit-pinned without a live IMDS // server in the test. // // CARLOS_AWS_REGION, when set, wins outright — the escape hatch for testing // and for any box this build ever runs on OFF EC2. Otherwise the IMDS // probe's answer stands, trimmed. Neither set means "" — a box that is // off EC2 with no override is correctly UNARMED for steering (Task 4's // converger gates on this field parsing as an AWS region, never just // non-empty), not an error: DNS health checks target EIPs, which never // depended on this. func edgesFileRegion(getenv func(string) string, probedRegion string) string { if override := strings.TrimSpace(getenv("CARLOS_AWS_REGION")); override != "" { return override } return strings.TrimSpace(probedRegion) } // edgesFileEligibility derives the two published eligibility answers from // the reconcile config this box actually runs with — provisionTickConfig, // the same parse the provision tick uses — so a spelling the reconciler // reads one way can never be published another. func edgesFileEligibility(getenv func(string) string) (sharedPool, inbound bool) { return provisionTickConfig(getenv).SharedInstanceEligibility() } // edgesFileSecretsFP fingerprints the box's secrets key, read the same way // the config loops read it (agent.go configSyncLoop). The key itself never // leaves this function. func edgesFileSecretsFP(getenv func(string) string) string { raw := strings.TrimSpace(getenv("CARLOS_SECRETS_KEY")) if raw == "" { return "" } priv, err := appconfig.ParsePrivateKey(raw, "CARLOS_SECRETS_KEY") if err != nil { log.Printf("agent: CARLOS_SECRETS_KEY does not parse, so no secrets_key_fp is published: %v", err) return "" } return appconfig.KeyFingerprint(priv.PublicKey()) } // edgesFileLoop publishes this box's edges file every 5 minutes + jitter — // the same cadence and arming site as registrySnapshotLoop — AND within one // watch interval of any change to this box's registry (#174). // // The change path is what a new host needs. Every other edge learns that this // box homes a host from this file (internal/edge's homeMap mirror, a 30s // poll), and until it has, a visitor steered to one of them gets that edge's // unknown-host answer. On the timer alone that window was the whole 5min + // jitter cycle plus the poll: a titogo sandbox host served here at +51s was a // dead end on the uk edge for about a minute more, and could have been for // six. Watching the registry fingerprint (the same 2s poll edge.Watch // reloads on) closes the writer's half to seconds; what is left is the // reader's 30s poll. // // It stays cheap because edgesFilePass is: local registry reads, and a // bucket write only when the file's body (Generation aside) changed. A // deploy or a health flip moves the fingerprint without moving the hosts, // so it costs a pass and no write. // // The trigger covers what a new host needs, not every input to the file. // Fingerprint hashes a route's row and status (so a route turning ready // moves it), aliases' names and wildcard_domains; it does not hash a route's // platform, provisioned or kind columns, nor an alias's dns_status. A change // to only those still waits for the timer, and so does a newly attached // custom domain: its pass runs at once but withholds it until the DNS sweep // stamps it "pointing". // // The timer is kept for what the fingerprint deliberately does not see: // route_aliases' dns_status (a cadence-stamped column Fingerprint excludes) // decides which aliases are published, and a failed write must be retried // even if nothing else changes. // // Platform-box job, retentionLoop's rules: skipped on fleet boxes, disabled // with a log line when there is no deployment bucket. Single-box deployments: // the file is published and nothing reads it yet. func edgesFileLoop(reg *registry.Registry) { ctx := context.Background() if shouldSkipPublisherElection(os.Getenv) { log.Printf("agent: edges file loop is a platform-box job — skipped on a fleet box") return } store, err := openDeployStore(ctx) if err != nil { log.Printf("agent: edges file loop disabled: %v", err) return } box := metricsBoxLabel(os.Getenv, readMachineID()) // Eligibility is read once here at arm, from the reconcile config's own // parse, never a second reading of CARLOS_POOLS or CARLOS_INBOUND. sharedPool, inbound := edgesFileEligibility(os.Getenv) // Region resolved once here at arm — the domains loop's read-once // precedent (domainsLoop reads its env once before the loop and threads // the result through every tick via cfg, never re-reading or re-probing // per pass). readPlacementRegion is the one IMDS call; edgesFileRegion // applies the CARLOS_AWS_REGION override on top, pure. region := edgesFileRegion(os.Getenv, readPlacementRegion()) // The placement code alongside it: CARLOS_REGION, the same env the // reconcile pass scopes by — the two vocabularies travel in one file so // a reader can join them without a second source. ident := edgesFile{ Box: box, Region: region, Placement: strings.TrimSpace(os.Getenv("CARLOS_REGION")), DataBucket: strings.TrimSpace(os.Getenv("CARLOS_HIBERNATE_BUCKET")), DataBucketRegion: strings.TrimSpace(os.Getenv("CARLOS_HIBERNATE_REGION")), // Same env, same trim, one line apart, deliberately: this boolean // must never disagree with the bucket beside it, and the way to // guarantee that is to derive both from the one read rather than to // ask two questions that could drift. HibernationConfigured: strings.TrimSpace(os.Getenv("CARLOS_HIBERNATE_BUCKET")) != "", ServesSharedPool: sharedPool, AcceptsInbound: inbound, SecretsKeyFP: edgesFileSecretsFP(os.Getenv), BoxID: box, Capabilities: edgesFileCapabilities(os.Getenv), } // The flag-derived capabilities; offline-move-1 joins them on each pass // once the moves worker is armed and capable, and db-read-1 joins them // on each pass while the armed db-reads loop's last readiness probe // passed (db region forward spec §7.1). The db-reads agent's draining // check reads the draining flag this file publishes, stored on each // pass, so the box and the console's evidence rule read one value. baseCaps := ident.Capabilities dataDir := provisionTickConfig(os.Getenv).DataDir optDir := release.OptDir() // The local floor files this file publishes are refreshed hourly by // their own loop, on this loop's store handle (moves spec §11). go movesFloorLoop(ctx, store, dataDir) // Region HA reads each relocatable host's claim (spec 1 §5.4); off, the // file is exactly what it always was. var claims *edgesClaims if regionHAEnabled(os.Getenv) { claims = newEdgesClaims(box, func(ctx context.Context, host string) (hostclaim.Observed, error) { return hostclaim.Read(ctx, store, host) }) } log.Printf("agent: edges file loop armed (5min + jitter, and %s after a registry change, %s)", edgesFileWatchInterval, edgesFileKey(box)) last := seedEdgesFileGate(ctx, store, box) sched := newEdgesFileSchedule(time.Now(), func() time.Duration { return 5*time.Minute + rand.N(time.Minute) }) state := &edgesLoopState{sched: sched, claims: claims} tick := time.NewTicker(edgesFileWatchInterval) defer tick.Stop() fpErrs, passErrs := 0, 0 // The EIP cannot change under a running box, so it is read once here and // re-read only on the timer's passes, never on every change pass: off // EC2 each read can block for the probe's whole timeout. eip := readPublicIP() for now := range tick.C { fp, err := reg.Fingerprint() if err != nil { // Not fatal: the timer still publishes. Counted and throttled the // way edge.Watch counts its own poll errors. fpErrs++ if fpErrs == 1 || fpErrs%300 == 0 { log.Printf("agent: edges file registry poll error (#%d): %v", fpErrs, err) } } else { fpErrs = 0 } if !state.due(now, fp, err == nil) { continue } if state.timed() { eip = readPublicIP() } ident.Capabilities = withDBReadCapability(withOfflineCapability(baseCaps, armedOfflineCapable()), dbReadsCapable()) dbReadsDraining.Store(ident.Draining) ident.Headroom = headroomFor(headroomCapabilities(ident.Capabilities), dataDir, reg) ident = withMovesFields(ident, dataDir, optDir) ident = withSeedsFields(ident, dataDir, optDir) err = edgesFileWriteClaims(ctx, store, reg, ident, eip, &last, state.timed(), claims) if err != nil { // Throttled: a bucket that refuses every write (an IAM Deny, an // outage) is retried on a backoff, and each retry must not be a // journal line. passErrs++ if passErrs == 1 || passErrs%30 == 0 { log.Printf("agent: edges file pass failed (#%d): %v", passErrs, err) } } else { passErrs = 0 } state.done(now, err == nil) } } // edgesFileWatchInterval is how often edgesFileLoop polls the registry // fingerprint: edge.Watch's own 2s, so this file follows a route change on // the same beat the local edge starts serving it. const edgesFileWatchInterval = 2 * time.Second // edgesFileSchedule decides, tick by tick, whether edgesFileLoop runs a // pass. Pulled out of the loop so the decision is testable without a // bucket, a registry or a clock. // // A pass is due when the timer is up, or when the fingerprint moved since // the last SUCCESSFUL pass. The first fingerprint seen only baselines: the // first timed pass stays a minute after arm, as it always was, rather than // publishing in the loop's first two seconds. // // The baseline advances only when a pass succeeds, so a change whose pass // failed is retried rather than lost until the timer — the rule edge.Watch // keeps for a failed Reload. Unlike a Reload, a pass is a bucket write, so // the retries back off: 2s doubling to the timer's own cadence, reset by the // next success. A bucket that refuses every write is therefore asked about // as often as it was before the change trigger existed, not every tick. type edgesFileSchedule struct { next time.Time // when the timer is next due jitter func() time.Duration // the cadence after each timed pass seen string // the fingerprint on the tick being decided baseline string // the fingerprint the last good pass covered armed bool // a baseline exists timed bool // the pass being run is the timer's retryAt time.Time // no change pass before this, after a failure backoff time.Duration // the wait after the next failure } // Bounds on the change-pass retry backoff. const ( edgesFileRetryMin = 2 * time.Second edgesFileRetryMax = 5 * time.Minute ) func newEdgesFileSchedule(now time.Time, jitter func() time.Duration) *edgesFileSchedule { return &edgesFileSchedule{next: now.Add(time.Minute), jitter: jitter} } // due reports whether to run a pass on this tick. fpOK=false means the // fingerprint could not be read, which never triggers a change pass. func (s *edgesFileSchedule) due(now time.Time, fp string, fpOK bool) bool { s.timed = !now.Before(s.next) s.seen = "" if fpOK { s.seen = fp if !s.armed { s.baseline, s.armed = fp, true } } changed := fpOK && fp != s.baseline && !now.Before(s.retryAt) return s.timed || changed } // done records the outcome of the pass due just asked for. func (s *edgesFileSchedule) done(now time.Time, ok bool) { if s.timed { // The timer moves on whatever happened: a failed timed pass is // retried by the next timed one, exactly as before this schedule. s.next = now.Add(s.jitter()) } if ok { if s.seen != "" { s.baseline = s.seen } s.retryAt, s.backoff = time.Time{}, 0 return } if s.backoff == 0 { s.backoff = edgesFileRetryMin } s.retryAt = now.Add(s.backoff) s.backoff = min(2*s.backoff, edgesFileRetryMax) } // aliasClaimCache is the claim fold both alias legs share (the fast-converge // leg in fullPass and the five-minute instances pass). Process-lifetime and // package-level for the same reason withheldAliasSaid above is: there is one // agent per process, and both legs are asking about the same deployment's // claims, so two caches would double the cold reads and halve the hit rate // for nothing. // // The fast leg is why this exists at all. It runs on every serial bump — a // seconds cadence — and ListClaims is one List plus a GET per claim in the // WHOLE deployment. At the hundreds of custom domains titogo is heading for, // that is a steady drip of bucket reads on every box, sitting ahead of the // adopt leg inside a two-second convergence promise. See domains.ClaimCache. var aliasClaimCache domains.ClaimCache