package domains import ( "context" "crypto/ed25519" "crypto/x509" "encoding/json" "encoding/pem" "errors" "fmt" "log" "net" "os" "path/filepath" "slices" "strings" "time" "github.com/carlosframework/platform/internal/registry" "github.com/carlosframework/platform/internal/release" ) // Config is everything one reconcile pass reads its world through. Every // field is a seam: the tests inject fixtures, the agent injects production // values (internal/host wires it). type Config struct { HostClaim bool // AppsDomain is the deployment's platform apps domain (flagship: // "oncarlos.com"). "" makes the WHOLE pass inert — no minting, no // manifests read, zero registry writes — the litestream convention: // unconfigured means the feature does not exist. AppsDomain string // EdgeIPs answers this deployment's current edge addresses; a host // resolving to any of them is "pointing". A FUNC, not a slice: the set // moves at runtime as edges arm and retire, while the agent's config is // built once at loop arm — a boot-frozen list is exactly the bug this // replaced (a correctly-pointed domain steered to another region's edge // read as pointing elsewhere). nil, or a func returning nothing, leaves // DNS stamping standing down for that pass — see section 4. EdgeIPs func() []string // Resolve is the DNS seam (net resolver in production, a fake in // tests); nil skips DNS stamping entirely. Resolve func(host string) ([]string, error) // AcmeDir is the edge's autocert DirCache directory; "" skips TLS // stamping entirely. AcmeDir string // Now is the clock seam; nil = time.Now. Now func() time.Time // ReleasePub verifies the production flag: before a trash marker is // allowed to drop an app's routes (§1c) and again before the auto-purge // pass destroys anything (§1d). nil disables purging entirely — a box // that cannot verify must not destroy — and leaves the route-drop // unguarded, which §1c explains and accepts. ReleasePub ed25519.PublicKey } // Reconcile runs the spec §3 steps in order, idempotently: bucket account // manifests → registry accounts + route linkage, platform-domain minting, // canary-host minting/removal, then DNS and TLS health stamps. Per-item // failures are logged and skipped — one bad manifest or one unresolvable // host must never take the pass down; only setup-level failures (listing the // bucket or the registry) return an error. // // Removing a route happens on exactly two grounds. INFERRED: a route with // Platform set whose canary channel pointer is gone from the bucket — and // inference stops there, because custom domains are owner property. // INSTRUCTED: every route of an app carrying a valid trash marker, custom // domains included. The marker is not something this pass inferred; it is the // owner saying stop serving this app, and a custom domain still serving a // "deleted" app would be the worse betrayal (§1c) — unless the app is flagged // production, which outranks the marker on any box that can verify the flag. func Reconcile(ctx context.Context, s release.Store, reg *registry.Registry, cfg Config) error { if cfg.AppsDomain == "" { return nil // inert until configured } if cfg.Now == nil { cfg.Now = time.Now } // One bucket listing feeds the app→account key-path scan and the // canary-pointer scan below; accounts/ is listed separately because it // must be read FIRST — the account set is what makes a key path // unambiguous. acctKeys, err := s.List(ctx, "accounts/") if err != nil { return fmt.Errorf("domains: list accounts: %w", err) } keys, err := s.List(ctx, "apps/") if err != nil { return fmt.Errorf("domains: list bucket: %w", err) } routes, err := reg.List() if err != nil { return fmt.Errorf("domains: list routes: %w", err) } // --- 1. account materialization ------------------------------------------ known := materializeAccounts(ctx, s, acctKeys, reg) // --- 1c. trashed apps ---------------------------------------------------- // A valid trash marker (apps///deleted.json) is the owner's // explicit instruction to stop serving the app. This extends the pass's // destructive-move charter deliberately: inference never destroys owner // property, but a marker is not inference — so ALL the app's routes go, // custom domains included. A custom domain that kept serving a "deleted" // app would be the worse betrayal. A marker that fails validation // (ReadAppTombstone) changes nothing: an instruction we cannot fully // believe must not destroy routes, so the app stays live and the log // says why, every pass, until someone fixes or removes the object. // // And a valid marker does NOT outrank the production flag. The marker is // unsigned by design — anything holding PutObject on the bucket can write // one — so without this check a writer with no signing key could park a // marker on a production app and take its routes down fleet-wide, on // every box, at the next tick. Dropping every route of a production app // is an outage, which is precisely what the flag exists to prevent, so // the same contradiction the purge path refuses (§1d) is refused here: // flag ON, or flag UNREADABLE, and the marker is not acted on at all. // // Collected before the linkage below purely so the set is complete when // the removal pass (right after 1b's stamping) reads it. The linkage // itself does NOT skip trashed apps — see the stamping loop for why. trashed := map[appKey]*release.AppTombstone{} for _, key := range keys { acct, app, prefix, ok := appKeyOwner(key, known) if !ok || key != prefix+"deleted.json" { continue } tomb, err := release.ReadAppTombstone(ctx, s, acct, app) if err != nil { log.Printf("domains: ignoring trash marker for %s/%s: %v", acct, app, err) continue } if tomb == nil { continue } // A KEYLESS box acts on the marker, deliberately. It cannot verify a // flag, and the alternative — hold every trash forever wherever there // is no pubkey — would break hiding entirely on legitimately keyless // deployments, whose owners have no way to make the guard pass. The // guard is where it matters: every SERVING box carries the pubkey // already (adoption needs it), so production traffic is covered, and // the destructive half (§1d) refuses outright without a key. if cfg.ReleasePub != nil { flag, err := release.GetProduction(ctx, s, cfg.ReleasePub, acct, app) if err != nil { log.Printf("domains: cannot read the production flag for %s/%s — refusing to act on the trash marker: %v", acct, app, err) continue } if flag != nil && flag.Production { log.Printf("domains: trash marker on production app %s/%s — refusing to act on it; turn production off first", acct, app) continue } } trashed[appKey{acct, app}] = tomb } // --- 1b. app→account linkage, from the KEY PATH -------------------------- // apps///… says the app belongs to that account. A legacy // unqualified apps//… key says nothing at all since the read-fallback // was retired — the runtime cannot read those objects, so they must not // mint a route or claim an app (appKeyOwner). The shapes are told apart by // whether the first segment is a KNOWN account sqid, which is why the // account set had to be read first. // owners is app name -> the accounts that hold objects for it. A LIST, not // one account: the same app name legitimately exists in several accounts, // and collapsing them to one would re-home every tenant's app onto // whichever account the bucket listing happened to reach last. owners := map[string][]string{} for _, key := range keys { acct, app, _, ok := appKeyOwner(key, known) if !ok { continue } if !slices.Contains(owners[app], acct) { owners[app] = append(owners[app], acct) } } // Trashed apps are stamped here like any other, deliberately: the stamp is // what makes the removal lookup below match. A route with an empty account // column resolves to ops until 1b writes the account the BUCKET asserts, // so skipping trashed apps here would leave such a row reading "ops" while // its marker sits under the tenant's sqid — and the app would go on // serving from its custom domain forever. Stamping a row that is about to // be deleted costs one write and buys "ALL of them". for i := range routes { if routes[i].App == "" { continue // an app-less route (status page, bare proxy) has no account } acct := resolveRouteAccount(routes[i], owners[routes[i].App], cfg.AppsDomain) if acct == "" || routes[i].Account == acct { continue } if err := reg.SetAccount(routes[i].Host, acct); err != nil { log.Printf("domains: stamp account on %s: %v", routes[i].Host, err) continue } routes[i].Account = acct } // Trashed apps stop routing NOW: every route of a trashed (account, app) // is removed, and the shrunken slice is what minting sees, so nothing // below re-mints an apex or canary host for an app in the trash. // // Except a host in the import journal (seeded-instances spec §5.7): its // row is an uncommitted import's, and the journal owns it until the // rollback puts the original route back and retires the journal row. // Deleting it here would lose the import's gate and leave the rollback // no row to restore over. The provision pass skips these hosts for the // same reason. The row goes on a later pass, once the journal lets go. // A journal that cannot be read could hide any host, so then no trashed // route is removed this pass. // // The list is from the top of the pass, and this loop runs alongside // the provision pass. A rollback can put another owner's original row // back at a listed host and retire its journal row in between, so the // delete is conditional on the row still being the one listed // (DeleteIfUnchanged); a row that changed is left for the next pass. if len(trashed) > 0 { journaled, jerr := importJournalHosts(reg) if jerr != nil { log.Printf("domains: the import journal could not be read: %v (no route of a trashed app is removed this pass)", jerr) } keptLive := routes[:0] for _, rt := range routes { if rt.App != "" && trashed[appKey{release.RouteAccount(rt), rt.App}] != nil { if jerr != nil || journaled[rt.Host] { if jerr == nil { log.Printf("domains: not removing route %s of trashed app %s yet: an import on it is still journaled", rt.Host, rt.App) } keptLive = append(keptLive, rt) continue } deleted, err := reg.DeleteIfUnchanged(rt) if err != nil { log.Printf("domains: remove route %s of trashed app %s: %v", rt.Host, rt.App, err) keptLive = append(keptLive, rt) // still there; try next pass continue } if !deleted { log.Printf("domains: not removing route %s of trashed app %s: the row changed since this pass listed it", rt.Host, rt.App) keptLive = append(keptLive, rt) // judged again next pass, from a fresh list continue } log.Printf("domains: removed route %s — app %s is in the trash", rt.Host, rt.App) continue } keptLive = append(keptLive, rt) } routes = keptLive } // --- 1d. auto-purge ------------------------------------------------------ // Trashed longer than TrashCooloff → destroy. The marker's own deleted_at // is the clock (bucket-side; restore deletes the marker, so there is // nothing to replay). PurgeApp RE-READS the marker immediately before // destroying — this listing is only a hint — and re-checks the production // flag; both guards fail toward keeping data. Idempotent and race-proof: // concurrent boxes converge because every step tolerates already-gone and // the marker goes last. for k, tomb := range trashed { if cfg.Now().Sub(tomb.DeletedAt) < release.TrashCooloff { continue } if cfg.HostClaim { if err := release.RequestClaimAutoPurge(ctx, s, k.acct, k.app, cfg.Now()); err != nil { log.Printf("domains: request auto-purge %s/%s: %v", k.acct, k.app, err) } continue } if err := release.PurgeApp(ctx, s, cfg.ReleasePub, k.acct, k.app, cfg.Now()); err != nil { log.Printf("domains: auto-purge %s/%s: %v", k.acct, k.app, err) continue } log.Printf("domains: purged %s/%s — trashed %s, past the %s cooloff", k.acct, k.app, tomb.DeletedAt.Format(time.RFC3339), release.TrashCooloff) } // --- 2. platform-domain minting ------------------------------------------ byHost := map[string]bool{} // hasApex tracks the (ACCOUNT, app) pairs that already own a platform // apex. The account is part of the key because app names are unique per // account, never globally: two tenants can both run a "blog", and each // deserves its own apex. // // Within one account the guard is deliberately "already has one" rather // than "already has the account-correct one": minting is additive (nothing // here ever removes an apex), so keying on the expected host would mint a // SECOND apex the moment an app's account changed, leaving both live // forever with no rule for which is canonical. One app per account, one // apex; moving an app between accounts is a deliberate operator migration, // not a silent side effect. hasApex := map[appKey]bool{} for _, rt := range routes { byHost[rt.Host] = true if rt.Platform && !rt.Canary { hasApex[appKey{release.RouteAccount(rt), rt.App}] = true } } for _, k := range candidateApps(routes) { if hasApex[k] { continue } host := fmt.Sprintf("%s.%s.%s", k.app, k.acct, cfg.AppsDomain) if byHost[host] { continue // already minted — idempotent } src := firstReadyUpstream(routes, k) if src == nil { continue // nothing serving to copy an upstream from; next pass } nr := mintedRoute(host, k.app, k.acct, *src) if err := reg.Upsert(nr); err != nil { log.Printf("domains: mint %s: %v", host, err) continue } log.Printf("domains: minted platform domain %s (upstream %s)", host, src.Host) routes = append(routes, nr) byHost[host] = true hasApex[k] = true } // --- 3. canary hosts: mint for hostless pointers, remove for retired ---- canaryChannels := map[appKey]map[string]bool{} // (account, app) -> live canary/ channels for _, key := range keys { acctOfKey, app, _, ok := appKeyOwner(key, known) if !ok { continue } app, name, ok := canaryPointer(key, known) if !ok { continue } k := appKey{acctOfKey, app} if trashed[k] != nil { continue // no canary host for an app in the trash } if canaryChannels[k] == nil { canaryChannels[k] = map[string]bool{} } channel := "canary/" + name canaryChannels[k][channel] = true followed := false for _, rt := range routes { if rt.App == app && release.RouteAccount(rt) == k.acct && rt.Channel == channel { followed = true break } } if followed { continue } acct := k.acct src := firstReadyUpstream(routes, k) if src == nil { continue // app has nothing serving yet; the pointer waits } // Static apps ONLY. A minted route can only copy an upstream that // already exists, and for an instance/service app the only one // available is the app's stable process — so a "canary" host here // would proxy straight to stable while the console labeled it with // the canary's version: a URL that lies about what it serves. // Spawning a real per-host canary instance is the not-yet-built // `carlos canary`'s job, not something reconciliation can fake. For // static apps the copied Channel genuinely IS the backend (the edge // resolves the pointer to find that channel's tar), so those mint // honestly. if src.Kind != "static" { log.Printf("domains: canary channel %s for app %s has no host: minting one needs a per-host canary instance (carlos canary), not an alias to stable — skipping", channel, app) continue } host := fmt.Sprintf("%s.%s.%s.%s", name, app, acct, cfg.AppsDomain) nr := mintedRoute(host, app, acct, *src) nr.Channel = channel nr.Canary = true if err := reg.Upsert(nr); err != nil { log.Printf("domains: mint canary %s: %v", host, err) continue } log.Printf("domains: minted canary domain %s (channel %s)", host, channel) routes = append(routes, nr) byHost[host] = true } // Removal: ONLY canary hosts this pass minted, whose pointer is now gone // from the bucket. Three conditions, each load-bearing: Platform (a // custom route on the same channel is owner property — the flag, not the // channel, grants removal authority), Canary (a STATIC app's apex mint // legitimately carries a canary channel too, since that's how it finds // its tar — removing it on retirement would delete the app's platform // domain and re-mint it next pass, forever), and the channel prefix. kept := routes[:0] for _, rt := range routes { if rt.Platform && rt.Canary && strings.HasPrefix(rt.Channel, "canary/") && !canaryChannels[appKey{release.RouteAccount(rt), rt.App}][rt.Channel] { if err := reg.Delete(rt.Host); err != nil { log.Printf("domains: remove retired canary route %s: %v", rt.Host, err) kept = append(kept, rt) // still there; stamp it honestly below continue } log.Printf("domains: removed retired canary route %s (channel %s)", rt.Host, rt.Channel) continue } kept = append(kept, rt) } routes = kept // Sections 4 and 5 — the per-box health stamps — live in StampHealth // (#303): they write THIS box's registry and must run on every box, not // only on the housekeeper this pass is gated to. The agent runs it once // per tick after this pass, on every box; this pass does not, so the // housekeeper stamps once like everyone else. return nil } // StampHealth is the per-box half of the domains pass: DNS health (section // 4, "pointing"/"elsewhere"/"" on routes and aliases) and TLS health // (section 5, cert_expires_at). Both stamp the LOCAL registry from a local // view — this box's resolver, this box's autocert cache — so they are // every box's job on every tick, with no lease involved: gating them to // the housekeeper (#289) froze every other box's stamps (#303), and the // edges-file publisher's "pointing" bound (#287) needs the HOME box's own // row stamped. // // Housekeeper or not, the stamps are what the console shows: a home box's // status snapshot carries them, and a non-housekeeper home box would show // every attached hostname as un-pointing forever (#303) — and the edges-file // publisher's "pointing" bound (#287) reads the home box's own row. // // routes may be nil, in which case the registry is listed here. Inert with // no AppsDomain, per Config's contract. func StampHealth(ctx context.Context, reg *registry.Registry, cfg Config, routes []registry.Route) error { if cfg.AppsDomain == "" { return nil } if cfg.Now == nil { cfg.Now = time.Now } if routes == nil { var err error if routes, err = reg.List(); err != nil { return fmt.Errorf("domains: list routes: %w", err) } } // aliases are the extra hostnames route_aliases holds (Task 2/#151), // health-stamped by sections 4 and 5 below exactly like the routes they // point at. Listed ONCE, above both sections, and guarded the same way // each section already guards itself — a box with neither Resolve nor // AcmeDir configured pays no aliases query at all. var aliases []registry.RouteAlias if cfg.Resolve != nil || cfg.AcmeDir != "" { var err error aliases, err = reg.ListAliases() if err != nil { return fmt.Errorf("domains: list aliases: %w", err) } } // --- 4. DNS health -------------------------------------------------------- // The edge set is asked for ONCE PER PASS, not once per box start: under // latency steering a customer's correctly-pointed domain resolves to // whichever region's edge the resolver is nearest, so the comparison set // has to be the deployment's live one. if cfg.Resolve != nil && cfg.EdgeIPs != nil { edge := map[string]bool{} for _, ip := range cfg.EdgeIPs() { edge[ip] = true } // Load-bearing: stamping against an empty set would mark every host // in the deployment `elsewhere` at once — a fleet-wide lying red // raised by a bucket read that failed, not by anyone's DNS. Unknown // is not a verdict this pass is allowed to guess at, so it stands // down and leaves the previous stamps alone until it knows again. // // This is the SET-level case of the same rule the per-host loop // below applies: what we do not know, we do not stamp. if len(edge) == 0 { log.Printf("domains: no edge addresses known this pass — DNS stamping stands down rather than stamping every host elsewhere") } else { now := cfg.Now().Unix() // OUR failure never overwrites a verdict. "unresolved" means the // lookup itself failed (SERVFAIL, timeout, no resolver) — one bad // DNS tick must not flip a working domain, because the edges-file // publisher gates on "pointing" (#287), so that flip would // unpublish every alias fleet-wide for a cadence. The stamp time // is left alone too, so staleness stays visible (dns_checked_at // stops advancing) rather than being laundered into a fresh // unknown. // // An authoritative NXDOMAIN ("") is NOT that case and DOES stamp: // the resolver answered, and the answer is that the record is // gone. A customer who deletes their record must stop reading // "pointing" — refusing that news would be the same lie in the // opposite direction. for _, rt := range routes { status := dnsStatus(cfg.Resolve, rt.Host, edge) if status == unresolvedDNS && rt.DNSStatus != "" { continue } if err := reg.SetDNSStatus(rt.Host, status, now); err != nil { log.Printf("domains: stamp dns on %s: %v", rt.Host, err) } } for _, a := range aliases { status := dnsStatus(cfg.Resolve, a.Alias, edge) if status == unresolvedDNS && a.DNSStatus != "" { continue } if err := reg.SetAliasDNSStatus(a.Alias, status, now); err != nil { log.Printf("domains: stamp dns on alias %s: %v", a.Alias, err) } } } } // --- 5. TLS health -------------------------------------------------------- if cfg.AcmeDir != "" { for _, rt := range routes { at := hostCertExpiry(cfg.AcmeDir, rt.Host, cfg.AppsDomain) if err := reg.SetCertExpiresAt(rt.Host, at); err != nil { log.Printf("domains: stamp cert expiry on %s: %v", rt.Host, err) } } for _, a := range aliases { at := hostCertExpiry(cfg.AcmeDir, a.Alias, cfg.AppsDomain) if err := reg.SetAliasCertExpiresAt(a.Alias, at); err != nil { log.Printf("domains: stamp cert expiry on alias %s: %v", a.Alias, err) } } } return nil } // unresolvedDNS is the stamp for "we could not look this up" as distinct // from "the resolver says there is no such record". The distinction is the // difference between blaming the customer and owning the outage. const unresolvedDNS = "unresolved" // dnsStatus resolves host and classifies the answer against edge. Three // distinct unknowns, deliberately: // // - "" — the resolver answered authoritatively that the name does not // exist. The customer has not created the record yet; guidance is right. // - unresolvedDNS — the lookup FAILED (SERVFAIL, timeout, no resolver). // That is OUR problem, and telling the customer their record is missing // would blame them for our outage. Section 4 never lets it overwrite an // existing verdict. // - "elsewhere"/"pointing" — a real answer, judged against the set. // // Shared by the route and alias loops in section 4 so the two verdicts can // never drift apart. func dnsStatus(resolve func(host string) ([]string, error), host string, edge map[string]bool) string { ips, err := resolve(host) if err != nil { var dnsErr *net.DNSError if errors.As(err, &dnsErr) && dnsErr.IsNotFound { return "" } return unresolvedDNS } for _, ip := range ips { if edge[ip] { return "pointing" } } return "elsewhere" } // OpsSqid is the built-in ops account's frozen sqid (registry id 1). Every // deployment has it, console or not. var OpsSqid = Sqid(1) // AccountManifestSqid reports whether key is an account manifest // (accounts//account.json, exactly one path level deep) and which sqid // its PATH claims. // // Exported because the console materializes accounts from the SAME objects // this reconcile pass does. One parser, not two: if the two ever disagreed // about what counts as a manifest, the console and the registry would hold // different account sets on the same box — and the account set is what tells a // qualified bucket key from a legacy one. func AccountManifestSqid(key string) (string, bool) { rest, ok := strings.CutPrefix(key, "accounts/") if !ok { return "", false } sqid, ok := strings.CutSuffix(rest, "/account.json") if !ok || sqid == "" || strings.Contains(sqid, "/") { return "", false } return sqid, true } // materializeAccounts turns the deployment's account manifests into this // box's registry account rows, and returns the set of sqids that exist. // // The return value is the point: known is what makes an apps///… // key path unambiguous (appKeyOwner), so it drives the trash-marker scan, // the app->account linkage and the canary-pointer scan — every part of the // pass that reads a bucket key. It is a function rather than an inline loop // so that invariant is reachable by a test; there is one vantage, and reg is // never nil. func materializeAccounts(ctx context.Context, s release.Store, acctKeys []string, reg *registry.Registry) map[string]bool { // The console originates account truth as accounts//account.json in // the bucket; this is the only place it becomes registry state, and the id // in the manifest is TAKEN AS GIVEN — never assigned here. That is the // whole durability property: a rebuilt registry, or a second box, ends up // with byte-identical account identity, so platform domains never re-home. // A deployment with no console has no manifests: everything stays under // the built-in ops account. known := map[string]bool{OpsSqid: true} // sqid -> materialized for _, key := range acctKeys { sqid, ok := AccountManifestSqid(key) if !ok { continue } body, _, err := s.Get(ctx, key) if err != nil { log.Printf("domains: skipping %s: read failed: %v", key, err) continue } var m AccountManifest if err := json.Unmarshal(body, &m); err != nil || m.ID <= 0 || m.Sqid == "" { log.Printf("domains: skipping %s: malformed account manifest: %v", key, err) continue } // Three things must agree: the key path, the body's sqid, and the // frozen encoding of the body's id. A manifest that disagrees with // itself is a lie about identity — materializing it would put an // account at a domain that stops resolving the moment anything // recomputes the sqid — so it is skipped, loudly, never guessed at. if m.Sqid != sqid || Sqid(m.ID) != sqid { log.Printf("domains: skipping %s: sqid %q/%q is not Sqid(%d)=%q", key, sqid, m.Sqid, m.ID, Sqid(m.ID)) continue } // A tombstoned manifest is materialized nowhere and, if this registry // still carries the account from before it was retired, un-materialized // here. Leaving the row would keep the account listed by `carlos // accounts list` on every box forever — a deletion that only ever took // effect in the console that performed it. if m.Deleted() { if err := reg.DeleteAccount(m.Sqid); err != nil { log.Printf("domains: retire account %s: %v", sqid, err) } continue } // KNOWN FIRST, then the write. The manifest is the authority on // which accounts EXIST; the registry row is only this box's local // copy of that fact, and the two answer different questions. known // answers the first — it is what tells a qualified bucket key // (apps///…) from a legacy one, and nothing else. // // So a failed row-write must not un-know the account. If it did, // every one of that account's keys would read as legacy for the rest // of the pass (appKeyOwner returns !ok), and this box would silently // skip the account's trash markers, route linkage and canary // pointers — a transient SQLite error turning into "this tenant's // apps stop reconciling" until some later pass happens to write the // row. Staying known costs only a stale `carlos accounts list` row on // this one box, which the next pass fixes. known[sqid] = true if err := reg.PutAccount(m.ID, m.Sqid, m.Name); err != nil { log.Printf("domains: materialize account %s: %v", sqid, err) continue } } return known } // appKeyOwner reads the app→account linkage straight out of a bucket key: // apps///… belongs to that account. known is the set of // materialized account sqids, which is what makes a key path unambiguous — // hence accounts being read first. // // A LEGACY unqualified apps//… key is not linkage. It used to be (it // meant "ops, until this app is migrated"), but the read-fallback's // retirement on 2026-08-03 made those objects unreadable to the runtime, and // bytes nobody can serve say nothing about who owns an app. Acting on them // here would be worse than useless: reconciliation would mint routes in front // of builds no box can install, and a leftover copy under a migrated app // would claim it back for ops. Nothing removes legacy objects, so this stays // true for as long as they sit there. // // Keys with nothing after the app segment (apps//) are not linkage // either: an object sitting directly at the app prefix says nothing about // ownership. It also returns the app's key prefix, so callers can parse what // comes after it without rebuilding it. // // Requiring a KNOWN sqid also closes a latent bug the old legacy branch hid: // a genuinely qualified key whose account had not materialized yet fell // through to it and came back as (ops, "") — the account's own sqid // read as an app NAME, on any box whose account manifests reconcile had not // reached. That minted apps out of account ids. Now it returns nothing and // the next pass, with the account materialized, reads the key correctly. func appKeyOwner(key string, known map[string]bool) (acct, app, prefix string, ok bool) { rest, ok := strings.CutPrefix(key, "apps/") if !ok { return "", "", "", false } acct, rest, ok = strings.Cut(rest, "/") if !ok || !known[acct] { // legacy apps//…, or an account we know nothing about return "", "", "", false } app, _, hasThird := strings.Cut(rest, "/") if !hasThird || app == "" { return "", "", "", false } // The one place the two namespaces can genuinely collide: an app whose // NAME happens to equal a materialized account sqid — a legacy key for an // app called "bab" is apps/bab/channels/…, which parses as ops's app // "channels". A qualified key's third segment is the app name; a legacy // key's second is always one of the release plane's own words, so those // two words are never an app here. if app == "channels" || app == "releases" { return "", "", "", false } return acct, app, "apps/" + acct + "/" + app + "/", true } // canaryPointer reports whether key is an account-qualified canary channel // pointer (apps///channels/canary/.json), and its app + // canary name. A legacy unqualified pointer is not one: see appKeyOwner. func canaryPointer(key string, known map[string]bool) (app, name string, ok bool) { _, app, prefix, ok := appKeyOwner(key, known) if !ok { return "", "", false } rest, ok := strings.CutPrefix(key, prefix+"channels/canary/") if !ok { return "", "", false } name, ok = strings.CutSuffix(rest, ".json") if !ok || name == "" || strings.Contains(name, "/") { return "", "", false } return app, name, true } // appKey is an app's identity: the account it belongs to plus its name. App // names are unique WITHIN an account, never globally, so nothing about an app // may be keyed on the name alone. type appKey struct{ acct, app string } // resolveRouteAccount decides which account a route's app belongs to, given // the accounts the BUCKET says hold objects under that app name. "" means // leave the row alone. // // The rules, in order: // - the row's own account already holds objects: it is right, keep it; // - exactly one account holds them: adopt it (this is what makes a // migration land); // - several do, and the row names none of them: leave it alone. Picking one // would re-home somebody's app onto another tenant's bytes, which is far // worse than a row that reconciliation declines to touch. func resolveRouteAccount(rt registry.Route, owners []string, appsDomain string) string { if len(owners) == 0 { if rt.Account == "" { return OpsSqid // nothing shipped yet: ops, like a console-less box } return "" // the row says something; the bucket says nothing. Believe the row. } // A row that names one of the owners is right; keep it. if release.ValidAccount(rt.Account) && slices.Contains(owners, rt.Account) { return rt.Account } // The row names nothing (or something the bucket doesn't hold). Before // any fallback: a platform host literally names its owner in its own // labels — ... #344 was the ops fallback // winning this tie (a stray pipeline.json under ops vs the tenant's // whole app), and the config sync then deleting the tenant's delivered // config for the mis-stamped account. if sq := release.HostAccountLabel(rt.Host, appsDomain); sq != "" && slices.Contains(owners, sq) { return sq } if len(owners) == 1 { return owners[0] } // Several owners and nothing that decides between them. NOT ops-if- // present: an unstamped row already reads as ops at runtime, so // stamping buys nothing — and a wrong stamp is what #344 cost. log.Printf("domains: %s (app %q) names account %q, but objects exist under %v — leaving it alone", rt.Host, rt.App, rt.Account, owners) return "" } // candidateApps returns the (account, app) pairs eligible for a platform // domain, in first-seen (host) order: every app — static sites included, a // shipped static tree deserves an addressable URL as much as an instance // does — outside the platform-reserved svc/* namespace, with at least one // route that is NOT itself platform-minted. An app whose only routes are // minted ones has been removed by its operator, and re-minting from a mint // would keep a ghost alive forever. // // The account is the one already stamped on the route by the linkage step // above, so an app whose objects have moved mints under the account that now // owns them. func candidateApps(routes []registry.Route) []appKey { seen := map[appKey]bool{} real := map[appKey]bool{} // has a non-platform route var order []appKey for _, rt := range routes { if rt.App == "" || strings.HasPrefix(rt.App, "svc/") { continue } k := appKey{release.RouteAccount(rt), rt.App} if !seen[k] { seen[k] = true order = append(order, k) } if !rt.Platform { real[k] = true } } var out []appKey for _, k := range order { if real[k] { out = append(out, k) } } return out } // firstReadyUpstream picks the deterministic upstream a minted domain // serves from: the app's first ready route in host order (reg.List's order) // that has something to copy and is not itself a minted route — minting // from a mint would be self-reference. "Something to copy" is a proxy // upstream (Socket or Addr) for instances and services, or — for static // routes — the release channel, since a static route's backend IS its // channel pointer, not a socket. func firstReadyUpstream(routes []registry.Route, k appKey) *registry.Route { for i := range routes { rt := &routes[i] if rt.App != k.app || release.RouteAccount(*rt) != k.acct || rt.Platform || rt.Status != "ready" { continue } if rt.Socket != "" || rt.Addr != "" || (rt.Kind == "static" && rt.Channel != "") { return rt } } return nil } // mintedRoute is the platform route shape: same upstream trio // (Socket/Addr/Backing) and Kind as the source, ready (the upstream is — // that's how it was chosen), platform-flagged, account-stamped. // // Channel is copied for static routes and NOT for anything else, because // the two serving classes reach their backend by different mechanisms. A // proxied route (instance/service) reaches its backend through Socket/Addr, // already copied above — the channel would be decoration, and a copied // canary channel would make the apex route look like a canary follower, so // retiring that canary would delete the app's platform domain. A static // route has no socket at all: the edge's static handler resolves the app's // CHANNEL POINTER to find the shipped tar, so a channel-less static mint // would serve nothing. Hence the asymmetry — same field, opposite verdicts, // for reasons that live one layer down in internal/edge. // // Environment is never set here, by design (2026-08-02-plan-environments.md // Task 2): reconcile does not derive an environment binding for a minted // route, so `carlos route -environment` is allowed on a plain Platform apex. // It is NOT allowed on a Platform+Canary route (platformCanaryOwnedRow, // cli.go) — not because this function derives Environment, but because if // such a route's `followed` check (below) goes false again, the Upsert(nr) // that follows rebuilds the whole row from this function's values. That is // the same zero-value-on-re-mint caveat runRoute documents for // Channel/Version/Status/DNS/Cert. Since the 2026-08-02 review, // registry.Upsert preserves a non-empty existing environment through exactly // this path (IMPORTANT 3), so a binding is no longer silently lost here — // but a row this function rebuilds is still not one the route verb hands an // operator, which is why the CLI refusal exists on top of the preservation. // // Region is copied from src (customer-fleets follow-up): a fleet-placed // source's minted apex must carry the same "fleet/" Region so the // platform edge dispatches it through the fleet's channel rather than // treating it as unconstrained. When src is fleet-placed // (registry.FleetPlacement(src.Region)), Socket is deliberately left empty // even though it was just copied above the src.Kind == "static" carve-out // would otherwise leave it — a fleet-placed source's Socket is a path on // the FLEET BOX, not this platform process, so copying it verbatim onto the // minted apex would give it a dead local socket instead of letting it ride // the fleet dispatch Region now enables. func mintedRoute(host, app, account string, src registry.Route) registry.Route { rt := registry.Route{ Host: host, Kind: src.Kind, Socket: src.Socket, Addr: src.Addr, Backing: src.Backing, App: app, Status: "ready", Platform: true, Account: account, Region: src.Region, } if src.Kind == "static" { rt.Channel = src.Channel } if _, ok := registry.FleetPlacement(src.Region); ok { rt.Socket = "" } return rt } // hostCertExpiry resolves host's cert expiry from the acme cache dir as the // LATER of the per-host autocert entry and the covering wildcard's PEM // (wildcard-.pem). Two reasons a bare per-host lookup isn't enough: a // wildcard-covered host never gets a per-host entry at all (without the // wildcard fallback status would call it certless forever), and a host // GRANDFATHERED onto a per-host cert from before its wildcard existed keeps // that file on disk forever — autocert stops renewing it once the wildcard // starts serving the host, so the stale file's expiry would shadow the // wildcard's and status would cry "expiring" over a cert the edge no longer // serves. The edge always serves whichever cert is actually valid, so the max // of the two is the only truthful answer. 0 = unknown, never a guess. // // The wildcard parent is host's generic one-label strip, not an // appsDomain-shaped derivation (appsDomain is otherwise unused here): one // label folds into "*.", so any host's covering wildcard is its one-label // parent's, whether that parent is an . derivation, a // static domain (go.tito.io), or a delegated customer domain (woodstar.app). // Trying the read is cheaper than knowing which — a miss is a miss. An apex // host's own strip lands one label short of the parent that names ITS // covering wildcard file, so the apex never matches its own wildcard. func hostCertExpiry(acmeDir, host, appsDomain string) int64 { perHost := certExpiry(filepath.Join(acmeDir, host)) rsa := certExpiry(filepath.Join(acmeDir, host+"+rsa")) if rsa > perHost { perHost = rsa } wildcard := int64(0) if i := strings.IndexByte(host, '.'); i > 0 { parent := strings.ToLower(host[i+1:]) wildcard = certExpiry(filepath.Join(acmeDir, "wildcard-"+parent+".pem")) } if wildcard > perHost { return wildcard } return perHost } // certExpiry reads the autocert DirCache entry at path (the PEM bundle // cachePut writes: private key block first, then the cert chain) and returns // the leaf's NotAfter as unix seconds. Anything short of a parseable // certificate — no file, no CERTIFICATE block, a corrupt one — is 0: // unknown, never a guess. func certExpiry(path string) int64 { body, err := os.ReadFile(path) if err != nil { return 0 } for len(body) > 0 { var block *pem.Block block, body = pem.Decode(body) if block == nil { return 0 } if block.Type != "CERTIFICATE" { continue // the private-key block(s) come first in the bundle } cert, err := x509.ParseCertificate(block.Bytes) if err != nil { return 0 } return cert.NotAfter.Unix() } return 0 } // importJournalHosts is every host the registry's import journal holds. func importJournalHosts(reg *registry.Registry) (map[string]bool, error) { js, err := reg.ImportJournals() if err != nil { return nil, err } hosts := make(map[string]bool, len(js)) for _, j := range js { hosts[j.Host] = true } return hosts, nil }