package host // steering_loop.go — the DNS steering converger's housekeeper loop // (multi-region ingress spec, Part 4; plan 2026-08-23-multi-region-steering // Task 6). The converger itself lives in internal/steering; this file is the // wiring: what it reads, when it runs, and what it refuses to run on. import ( "context" "fmt" "log" "math/rand/v2" "os" "sort" "strings" "time" "github.com/carlosframework/platform/internal/provision" "github.com/carlosframework/platform/internal/registry" "github.com/carlosframework/platform/internal/release" "github.com/carlosframework/platform/internal/steering" ) // loadEdgesFileBodies reads every box's edges file under control/edges/ and // returns the raw bodies in listing order (sorted by key, so the fold // downstream is stable tick to tick). // // FAIL-CLOSED, and this is the whole point of the function: a listing that // errors, or ANY object that lists but will not Get, returns an ERROR and no // bodies. The alternative — carrying on with what came back — is the SHORT // READ that Task 5's shrink guard was added to survive: the fold would be // missing a live box's edge, every record belonging to that box would look // managed-but-unwanted, and the pass would delete them. NXDOMAIN for a live // steered host, caused by a bucket blip rather than by anything an operator // did. // // This is deliberately STRICTER than internal/edge's homeMap, which reads the // same prefix and tolerates a single unreadable object by carrying that box's // previous parse forward. homeMap can do that because it HOLDS a last-good // snapshot; this pass holds nothing between ticks, so "tolerate" here would // mean "silently converge on less". Even a raced delete (list saw the key, // Get says it is gone — a box retired mid-pass) fails the pass: the cost is // one skipped tick, five minutes later the listing no longer names it, and // the withdrawal happens on complete information instead of a race. // // A key that is not .json directly under the prefix is not read at all — // the same key-shape filter homeMap applies. A stray object under the prefix // is not this deployment's edges file, and must not be able to fail the pass // forever. func loadEdgesFileBodies(ctx context.Context, s release.Store) ([][]byte, error) { keys, err := s.List(ctx, edgesFilePrefix) if err != nil { return nil, fmt.Errorf("steering: list %s: %w", edgesFilePrefix, err) } sort.Strings(keys) out := make([][]byte, 0, len(keys)) for _, key := range keys { name, ok := strings.CutPrefix(key, edgesFilePrefix) if !ok || !strings.HasSuffix(name, ".json") || strings.Contains(name, "/") { continue } body, _, err := s.Get(ctx, key) if err != nil { return nil, fmt.Errorf("steering: read %s: %w", key, err) } out = append(out, body) } return out, nil } // steeredRoutesFromRecords reads the deployment's steered hosts from the // BUCKET RECORDS — provision.ListAllRecords — and never from this box's // registry. // // The registry is the wrong source and the bug is not theoretical (review // finding, 2026-08-23). Routes are NOT uniformly provisioned: an // everywhereClass record (exec/unit/socket kinds) is homed to ONE box, so // that host is absent from every other box's registry. This duty runs on // whichever box holds the housekeeper lease — so a steered, home-boxed host // read from the local registry would be present in the desired set while the // lease sat on its home box and ABSENT the moment the lease moved. Absent // means managed-but-unwanted, which means DELETED: a live host goes NXDOMAIN // until the lease comes back, and the shrink guard does not fire at 2-of-8. // DNS bound to housekeeper elections is not a system anyone can operate. // The records are the deployment's truth, identical from every box. // // The mapping is deliberately MINIMAL: desiredState (internal/steering) // reads exactly three fields off a registry.Route — Steering (the "latency" // opt-in), Host (the DNS name, and the wildcard check) and Pool (steering // requires the shared pool) — verified field by field against that fold, so // carrying more would be inventing agreement the fold does not need. // // Records with Steering == "" are dropped HERE rather than passed down: the // fold skips them SILENTLY anyway (steeredHosts' `case "": continue`), so // nothing an operator would see is lost. Every other value — including // invalid ones, wildcard hosts and pool-scoped rows — is passed through // untouched so the fold's own loud, documented refusals still fire. This // loader must not become a second, quieter policy gate. func steeredRoutesFromRecords(ctx context.Context, s release.Store) ([]registry.Route, error) { recs, err := provision.ListAllRecords(ctx, s) if err != nil { return nil, err } out := make([]registry.Route, 0, len(recs)) for _, rec := range recs { if rec.Steering == "" { continue } out = append(out, registry.Route{Host: rec.Host, Steering: rec.Steering, Pool: rec.Pool}) } return out, nil } // steeringPass is one converge attempt: read both inputs from the STORE, // then converge. The registry is not consulted at all — see // steeredRoutesFromRecords. // // Both reads fail closed (see loadEdgesFileBodies and ListAllRecords) and // BOTH must succeed before converge is called even once — a records read // that failed is not "no steered hosts" any more than a failed edges listing // is "no edges". converge is injected rather than called directly so this // contract is testable without an AWS client: the real one is the closure // steeringLoop builds around steering.ConvergeDeployment. func steeringPass(ctx context.Context, s release.Store, converge func(context.Context, []registry.Route, [][]byte) error) error { edges, err := loadEdgesFileBodies(ctx, s) if err != nil { return err } // The records through the record view when the change index is armed // and the view can vouch; the bucket otherwise (viewStore). rs, _, _ := viewStore(ctx, s) routes, err := steeredRoutesFromRecords(ctx, rs) if err != nil { return err } return converge(ctx, routes, edges) } // steeringDuty wraps one pass as a housekeeper duty. // // DORMANCY (zoneID == "") is enforced here, at the top, and it is a // structural claim rather than a promise: pass is the closure holding the // store handle and the AWS client — every read and every API call this duty // can make — so a dormant duty that never calls it does neither, by // construction. Every box in production today is in exactly this state. // // DUTY-STATUS HONESTY — read this before trusting a green tick. A nil return // from this duty does NOT mean the zone converged. internal/steering's two // safety refusals (the empty-desired-set refusal and the proportional shrink // guard) both log loudly and return nil, deliberately: the duty cannot fix // either, and marking it failed every five minutes would bury the one line // that says what actually happened. So a refused pass is indistinguishable // from a clean one at THIS layer, by design, and the operator surface for // steering is the JOURNAL — `journalctl -u carlos -g "^steering:"` — not the // housekeeper's error log. // // The case with no escape hatch, recorded for the rollout notes: a GENUINE // withdrawal of more than half the managed records (a real multi-box // retirement) is refused on every tick, forever, with no flag to override it. // Recovery is an operator deleting the records by hand, or withdrawing the // boxes in batches under the limit. Threading a status back up through the // duty was considered and left out of this task on purpose — it changes the // housekeeperDuty contract for every duty, and the journal already carries // the truth. // The pass == nil half of the guard is one token of totality on a path that // writes production DNS: steeringLoop leaves pass nil in exactly the branch // that leaves zoneID empty, so the two conditions are the same condition // today — but "today" is doing load-bearing work in that sentence, and the // failure mode if it ever stopped being true is a nil call that panics the // agent's goroutine rather than a duty that declines to run. func steeringDuty(zoneID string, pass func(context.Context) error) housekeeperDuty { return housekeeperDuty{"steering converge", func(ctx context.Context) error { if zoneID == "" || pass == nil { return nil } return pass(ctx) }} } // steeringLoop converges the deployment's steering zone every 5 minutes + // jitter, on the one box holding the housekeeper lease. // // Its shape is mintMeshKeyLoop's, deliberately and for the same reason // (housekeeper.go's own comment): its OWN loop, gated on nothing but // deployment-store presence, never a duty folded into retentionLoop's tick — // that loop never arms at all on a box missing CARLOS_RELEASE_PUBKEY, and // such a box still WINS the housekeeper election, so a steering duty living // there would silently never converge, fleet-wide, with no error anywhere. // // Env, read ONCE at arm (domainsLoop's read-once precedent): // // - CARLOS_STEERING_ZONE_ID — the zone. Unset, blank, or malformed and the // loop still ARMS but its duty is dormant: one log line here, and nothing // else, ever. Malformed is logged loudly by SteeringZoneID itself and is // dormancy, never a crash loop — a typo must not take a box's agent down. // - PLATFORM_APPS_DOMAIN — the domain the synthetic edge. // steering hostname is built from. Empty is not fatal: the fold logs it // and mints no steering hostname (steered per-host records are // unaffected). // // FIRST REAL BEHAVIOUR, when a deployment does set the zone id while still // running ONE box: the steering hostname and every steered host get a // single-edge latency record — the same answers today's static DNS gives — // plus one Route53 health check per EIP, which is what adds rung-2 // withdrawal for steered hosts even single-box. Nothing about it needs a // second region to be correct. func steeringLoop() { ctx := context.Background() if shouldSkipPublisherElection(os.Getenv) { log.Printf("agent: steering loop is a platform-box job — skipped on a fleet box") return } store, err := openDeployStore(ctx) if err != nil { log.Printf("agent: steering loop disabled: %v", err) return } zoneID := steering.SteeringZoneID() var pass func(context.Context) error if zoneID == "" { log.Printf("agent: steering: zone unset — dormant (CARLOS_STEERING_ZONE_ID is unset or unusable; the duty reads nothing and calls no AWS API)") log.Printf("agent: steering loop armed (5min + jitter) — dormant") } else { // Constructed only on a deployment that actually steers: the default // credential chain is the box instance profile, and a dormant box has // no business resolving credentials at all. client, err := steering.NewClient(ctx) if err != nil { log.Printf("agent: steering loop disabled: %v", err) return } appsDomain := strings.TrimSpace(os.Getenv("PLATFORM_APPS_DOMAIN")) pass = func(ctx context.Context) error { return steeringPass(ctx, store, func(ctx context.Context, routes []registry.Route, edges [][]byte) error { return steering.ConvergeDeployment(ctx, client, zoneID, routes, edges, appsDomain) }) } log.Printf("agent: steering loop armed (5min + jitter), zone %s, apps domain %q — converges only while this box holds the housekeeper lease", zoneID, appsDomain) } duty := steeringDuty(zoneID, pass) delay := time.Minute for { time.Sleep(delay) delay = 5*time.Minute + rand.N(time.Minute) runHousekeeperDuties(housekeeperGate, duty) } }