package edge // Wildcard TLS (ACME DNS-01) for instant app hostnames — the port of // Eleven's wildcard.go this file's TLS wiring always reserved space for // (spec: docs/superpowers/specs/2026-08-05-wildcard-certs-design.md). // // A TLS wildcard matches exactly one label, and app hosts are two labels // under the apps domain (..), so there is no single // *. cert. Instead one wildcard per PARENT: // // - dynamic parents ., derived from the live route // table under PLATFORM_APPS_DOMAIN — a new account's first route starts // issuance immediately, so every later app under it has working TLS the // moment its route exists (and CT logs see the account, not app names); // - static parents from CARLOS_WILDCARD_DOMAINS (tito: go.tito.io, whose // instance hosts are one label deep). // // Canary hosts (...…, one label too deep) and custom // domains stay per-host HTTP-01 — except a custom domain approved as a // delegated wildcard parent (spec §4), which gets its own *. // wildcard via a challenge TXT published in our own zone. The wildcard // only changes how a handshake GETS a certificate: routing, HostPolicy and // the :80 handler are // untouched, and every failure (DNS provider outage, ACME outage, // unreadable cache) degrades to exactly the per-host behaviour — never a // dead edge. import ( "bytes" "context" "crypto" "crypto/ecdsa" "crypto/elliptic" "crypto/rand" "crypto/tls" "crypto/x509" "crypto/x509/pkix" "encoding/pem" "errors" "fmt" "log" "net" "os" "sort" "strings" "sync" "sync/atomic" "time" "golang.org/x/crypto/acme" "golang.org/x/crypto/acme/autocert" "github.com/carlosframework/platform/internal/domains" "github.com/carlosframework/platform/internal/provision" ) const letsEncryptDirectory = "https://acme-v02.api.letsencrypt.org/directory" // wildcardRenewBefore is how close to expiry we re-issue. LE certs live 90 // days; renewing with 30 left tolerates a month of provider outage. const wildcardRenewBefore = 30 * 24 * time.Hour type wildcardManager struct { // cache is the shared cert cache (certcache.go) — the same backend // autocert uses for per-host certs, so a future pooled-edge S3 backend // shares wildcards too. Keys: wildcardPEMKey / wildcardAccountKeyName. cache autocert.Cache dns dnsProvider directory string staticDomains []string appsDomain string hosts func() []string // the deployment's live hosts (Edge.deploymentHosts) // delegated lists customer domains approved for delegated-challenge // wildcard issuance (spec §4: claim-derived; nil = none). Their // challenge TXTs publish into delegationZone — our own zone — at // _acme-challenge.., which the customer has // CNAMEd _acme-challenge. to. The platform never touches, // or knows, the customer's DNS provider. delegated func() []string delegationZone string // delegatedSet is refresh()'s snapshot of delegated(), read by // challengeFQDN under mu (issuance runs on run()'s goroutine). delegatedSet map[string]bool // refreshMu serializes refresh() (see there); mu guards the fields. refreshMu sync.Mutex mu sync.Mutex parents []string // last computed parent set (sorted) certs map[string]*tls.Certificate // nextAttempt holds a per-parent backoff floor: run() skips a parent // until this time has passed. Set on obtain failure, cleared on success — // keeps a wake (which fires on ANY parent lacking a usable cert) from // retrying a persistently failing parent more than hourly (see run()). nextAttempt map[string]time.Time // wake nudges run() out of its 12h sleep when refresh() finds a parent // with no usable cert (a new account's first route). Capacity 1: a wake // is a level, not a count. wake chan struct{} // issuer gates ORDERING (certsync.go): in a multi-edge deployment only // the housekeeper talks to the CA, and its context ends the moment the // holding does, so a stand-down mid-order is observable inside the order. // nil = ungated, which is every single-box deployment and `carlos edge` // without the agent. Loading and serving a cert some other box obtained // is never gated — that is the whole point of the shared cache. issuer IssuerGate // lastNotIssuerLog rate-limits the one line a non-issuer prints when it // stands down from a pass. Read/written under mu. lastNotIssuerLog time.Time } func newWildcardManager(staticDomains []string, appsDomain string, hosts func() []string, delegated func() []string, delegationZone string, cache autocert.Cache, dns dnsProvider) *wildcardManager { return &wildcardManager{ cache: cache, dns: dns, directory: envOr("CARLOS_ACME_DIRECTORY", letsEncryptDirectory), staticDomains: staticDomains, appsDomain: strings.ToLower(strings.TrimSpace(appsDomain)), hosts: hosts, delegated: delegated, delegationZone: strings.ToLower(strings.TrimSpace(delegationZone)), certs: map[string]*tls.Certificate{}, nextAttempt: map[string]time.Time{}, delegatedSet: map[string]bool{}, wake: make(chan struct{}, 1), } } // maxDelegatedParents bounds the delegated set this box will carry — // local rows plus every parent other boxes publish (#305). The per-box // cap on wildcard_domains (internal/domains, 32) exists because every // delegated parent shares one ACME account's new-order budget; a fleet's // union must not multiply that by the box count, or one broken delegation // zone would have the issuer retrying 32×N orders an hour. Local rows come // first; the overflow is logged, once per pass, never silently dropped. const maxDelegatedParents = 32 // unionDelegated is the delegated source's merge: local rows, then remote // parents, capped. A remote parent on a box with NO delegation zone is a // rollout mistake worth a line — this box could win the issuer lease and // would then fail every order for it against the customer's own zone. func unionDelegated(local, remote []string, zone string) []string { seen := make(map[string]bool, len(local)+len(remote)) out := make([]string, 0, len(local)+len(remote)) add := func(d string) { d = strings.ToLower(strings.TrimSpace(d)) if d == "" || seen[d] { return } seen[d] = true out = append(out, d) } for _, d := range local { add(d) } nLocal := len(out) for _, d := range remote { add(d) } if len(out) > nLocal && zone == "" { delegationZoneUnsetOnce.Do(func() { log.Printf("edge: %d delegated wildcard parent(s) published by other boxes but CARLOS_ACME_DELEGATION_ZONE is unset here — set it fleet-wide, or this box cannot issue for them if it becomes the issuer", len(out)-nLocal) }) } if len(out) > maxDelegatedParents { if n := int32(len(out)); delegatedCapLogged.Swap(n) != n { log.Printf("edge: delegated wildcard parents capped at %d (%d distinct: %d local, %d remote) — the overflow gets no wildcard", maxDelegatedParents, len(out), nLocal, len(out)-nLocal) } out = out[:maxDelegatedParents] } return out } // Standing conditions are said once: an unset zone stays unset, and the cap // line repeats only when the count it reports changes. var ( delegationZoneUnsetOnce sync.Once delegatedCapLogged atomic.Int32 ) // parentsFor computes the wildcard parent set: staticAndDelegated (the // static CARLOS_WILDCARD_DOMAINS plus any delegated parents refresh() has // already vetted — both arrive pre-merged through this one parameter) union // . for every route host exactly two labels under // appsDomain (..). Canary hosts (three labels), the // apex, and foreign domains contribute nothing. Sorted and deduped so the // set is deterministic for logs and tests. // // must be shaped like an account sqid (domains.ValidSqid): every // account parent the platform mints is ., and a label that // could never be a sqid names no account. Without that bound a route someone // placed at x.. would order a wildcard certificate for // a parent nobody owns, one per invented label, against the deployment's // shared ACME budget (#215). The console refuses such first routes for // sqid-shaped labels that are not the attaching account's own; this is the // edge's half, for the labels no account could hold. func parentsFor(staticAndDelegated []string, appsDomain string, hosts []string) []string { seen := map[string]bool{} var out []string add := func(p string) { if !seen[p] { seen[p] = true out = append(out, p) } } for _, d := range staticAndDelegated { add(strings.ToLower(d)) } if appsDomain != "" { suffix := "." + strings.ToLower(appsDomain) for _, h := range hosts { rest, ok := strings.CutSuffix(strings.ToLower(h), suffix) if !ok { continue } if labels := strings.Split(rest, "."); len(labels) == 2 && domains.ValidSqid(labels[1]) { add(labels[1] + suffix) } } } sort.Strings(out) return out } // refresh recomputes the parent set from the live route table (called from // the reload hook on every route swap, and once at startup), folding in // delegated parents (spec §4) alongside the static and per-account ones. // Platform parents always win: a delegated entry that collides with a // static domain, or falls under the apps domain, is dropped here — before // it reaches parentsFor's union or delegatedSet — because letting it // through would redirect a PLATFORM parent's challenge to a delegation // target where no CNAME exists, silently degrading its wildcard renewal to // per-host. If any parent lacks a usable cert it wakes run() — that is what // makes a NEW account's wildcard start issuing the moment its first route // lands, instead of on the next 12h tick. A wake cannot bypass a failing // parent's backoff: run() still skips any parent whose nextAttempt hasn't // passed, so a wake only advances parents that aren't already backing off. func (w *wildcardManager) refresh() { // One derivation at a time: hosts() and delegated() are read outside // w.mu, and three callers (the reload hook, the cert mirror's wake, the // home map's wake — two of them 30s pollers) could otherwise interleave // so that an older read lands last and drops a parent until the next // wake. Serialized, the last writer is the last reader. w.refreshMu.Lock() defer w.refreshMu.Unlock() var delegated []string if w.delegated != nil { static := make(map[string]bool, len(w.staticDomains)) for _, d := range w.staticDomains { static[strings.ToLower(strings.TrimSpace(d))] = true } appsSuffix := "" if w.appsDomain != "" { appsSuffix = "." + w.appsDomain } for _, d := range w.delegated() { if d = strings.ToLower(strings.TrimSpace(d)); d == "" { continue } if static[d] || (appsSuffix != "" && strings.HasSuffix(d, appsSuffix)) { continue } if w.delegationZone != "" && len("_acme-challenge."+d+"."+w.delegationZone) > 253 { log.Printf("edge: delegated wildcard %s dropped: composed challenge name exceeds 253 bytes", d) continue } delegated = append(delegated, d) } } parents := parentsFor(append(append([]string(nil), w.staticDomains...), delegated...), w.appsDomain, w.hosts()) now := time.Now() w.mu.Lock() w.parents = parents set := make(map[string]bool, len(delegated)) for _, d := range delegated { set[d] = true } w.delegatedSet = set need := false for _, p := range parents { c := w.certs[p] if c == nil || now.Add(wildcardRenewBefore).After(c.Leaf.NotAfter) { need = true break } } w.mu.Unlock() if need { select { case w.wake <- struct{}{}: default: } } } // currentParents is the last computed parent set — the mirror's bound on // which wildcard PEMs the issuer's backfill may publish (certsync.go). func (w *wildcardManager) currentParents() []string { w.mu.Lock() defer w.mu.Unlock() return append([]string(nil), w.parents...) } // certFor returns the wildcard cert for serverName, or nil when the name // isn't exactly one label under a current parent (a wildcard doesn't cover // the apex or deeper levels) or no usable cert is loaded — nil sends the // handshake to the per-host autocert path. func (w *wildcardManager) certFor(serverName string) *tls.Certificate { sn := strings.ToLower(strings.TrimSuffix(serverName, ".")) w.mu.Lock() defer w.mu.Unlock() for _, p := range w.parents { sub, ok := strings.CutSuffix(sn, "."+p) if !ok || sub == "" || strings.Contains(sub, ".") { continue } cert := w.certs[p] if cert == nil || time.Now().After(cert.Leaf.NotAfter) { // Never serve a dead cert: a wildcard whose renewal has been // broken past its actual expiry falls back to per-host issuance // (the pre-wildcard behaviour) instead of hard-failing every // handshake under the parent. return nil } return cert } return nil } func (w *wildcardManager) needsObtain(parent string, now time.Time) bool { w.mu.Lock() cert := w.certs[parent] w.mu.Unlock() return cert == nil || now.Add(wildcardRenewBefore).After(cert.Leaf.NotAfter) } // backingOff reports whether parent is still inside its post-failure backoff // window — run() skips it until nextAttempt passes, so a wake (which fires // on ANY parent lacking a usable cert) can't retry a persistently failing // parent more than hourly. func (w *wildcardManager) backingOff(parent string, now time.Time) bool { w.mu.Lock() defer w.mu.Unlock() next, ok := w.nextAttempt[parent] return ok && now.Before(next) } // loadCached pulls parent's wildcard from the cert cache into memory, // reporting whether a parseable cert was loaded. Called by run() before any // obtain attempt, which covers both startup (certs an earlier process // obtained) and a pooled-edge future (certs a sibling box obtained). func (w *wildcardManager) loadCached(ctx context.Context, parent string) bool { raw, err := w.cache.Get(ctx, wildcardPEMKey(parent)) if errors.Is(err, autocert.ErrCacheMiss) { return false } if err == nil { var cert *tls.Certificate if cert, err = parseWildcardPEM(raw); err == nil { if !coversParent(cert, parent) { err = errors.New("cached cert does not cover parent or key mismatch") } else { w.mu.Lock() w.certs[parent] = cert w.mu.Unlock() log.Printf("edge: wildcard *.%s loaded from cache (valid until %s)", parent, cert.Leaf.NotAfter.Format(time.RFC3339)) return true } } } log.Printf("edge: wildcard *.%s: cached cert unreadable (%v) — will re-obtain", parent, err) return false } // coversParent reports whether cert is usable as *.: the leaf must // name a one-label host under parent and the private key must match the // leaf. A cached entry failing this is treated as unreadable — served // certs must never be worse than no cert (per-host issuance covers). func coversParent(cert *tls.Certificate, parent string) bool { if cert.Leaf == nil || cert.Leaf.VerifyHostname("acme-probe."+parent) != nil { return false } priv, ok := cert.PrivateKey.(interface{ Public() crypto.PublicKey }) if !ok { return false } pub, ok := priv.Public().(interface{ Equal(crypto.PublicKey) bool }) return ok && pub.Equal(cert.Leaf.PublicKey) } // challengeFQDN returns where parent's DNS-01 TXT is published: the // delegation target inside our own zone for delegated parents, or // directly at _acme-challenge. otherwise. With no delegation // zone configured the direct form is used for everyone — the // misconfigured state degrades to the historical behavior and per-host // issuance keeps covering (obtain will fail on the CA's lookup, log, // and back off — never a dead edge). func (w *wildcardManager) challengeFQDN(parent string) string { if w.delegationZone != "" && w.isDelegated(parent) { return "_acme-challenge." + parent + "." + w.delegationZone } return "_acme-challenge." + parent } // isDelegated reports whether parent is in the delegated set. func (w *wildcardManager) isDelegated(parent string) bool { w.mu.Lock() defer w.mu.Unlock() return w.delegatedSet[parent] } // wildcardPEMKey is the cache key for a parent's wildcard cert+key PEM — // Eleven's file name, so a dir cache written by that lineage reads with no // migration, and the keys sit additively beside autocert's per-host entries. func wildcardPEMKey(domain string) string { return "wildcard-" + domain + ".pem" } // ---- PEM encode/parse (key + chain in one blob; the cert cache owns // placement, permissions, and write atomicity) ------------------------------- func encodeWildcardPEM(cert *tls.Certificate, key *ecdsa.PrivateKey) ([]byte, error) { var buf bytes.Buffer der, err := x509.MarshalECPrivateKey(key) if err != nil { return nil, err } if err := pem.Encode(&buf, &pem.Block{Type: "EC PRIVATE KEY", Bytes: der}); err != nil { return nil, err } for _, c := range cert.Certificate { if err := pem.Encode(&buf, &pem.Block{Type: "CERTIFICATE", Bytes: c}); err != nil { return nil, err } } return buf.Bytes(), nil } func parseWildcardPEM(raw []byte) (*tls.Certificate, error) { cert := &tls.Certificate{} for { var block *pem.Block block, raw = pem.Decode(raw) if block == nil { break } switch block.Type { case "EC PRIVATE KEY": key, err := x509.ParseECPrivateKey(block.Bytes) if err != nil { return nil, err } cert.PrivateKey = key case "CERTIFICATE": cert.Certificate = append(cert.Certificate, block.Bytes) } } if cert.PrivateKey == nil || len(cert.Certificate) == 0 { return nil, errors.New("missing key or certificate in PEM") } leaf, err := x509.ParseCertificate(cert.Certificate[0]) if err != nil { return nil, err } cert.Leaf = leaf return cert, nil } // run keeps the wildcard certs current: an immediate pass at startup (first // issuance / cache load), then every 12h, dropping to hourly retries after a // failure — and refresh() wakes it early when a route swap surfaces a parent // with no usable cert (a new account). func (w *wildcardManager) run() { for { delay := w.pass() select { case <-time.After(delay): case <-w.wake: } } } // pass is one sweep over the current parent set — load what the cache // already has, order what is still missing — and returns how long run() // should wait before the next one. Split out of run() so the gating and // backoff decisions are testable without spinning the loop. func (w *wildcardManager) pass() time.Duration { delay := 12 * time.Hour w.mu.Lock() parents := append([]string(nil), w.parents...) w.mu.Unlock() standDowns := 0 for _, p := range parents { if !w.needsObtain(p, time.Now()) { continue } if w.backingOff(p, time.Now()) { // A wake (fired because some OTHER parent needs a cert) does // not shorten this parent's own hourly retry. continue } // Another process (or pool sibling) may have obtained it already. // Ungated on purpose: reading a cert someone else issued — including // one this box's mirror of control/certs/ has just landed — is // SERVING, and every edge serves. ctx, cancel := context.WithTimeout(context.Background(), time.Minute) w.loadCached(ctx, p) cancel() if !w.needsObtain(p, time.Now()) { continue } // The order gate (certsync.go): one elected issuer talks to the CA. // A non-issuer stands down HERE, after the cache load and before the // CA — so it still picks up the issuer's wildcard, and never arms the // per-parent backoff (standing down is not a failure; the moment this // box wins the lease it must be able to order immediately). held, ok := w.issuerGate() if !ok { standDowns++ continue } // The order runs under the HOLDING's context, not a bare background // one: an ACME order takes seconds to minutes, and work started as // the issuer must not outlive the authority it started under. ctx, cancel = context.WithTimeout(held, 10*time.Minute) err := w.obtain(ctx, p) cancel() if err != nil { // A missing zone is configuration, not a blip: asking hourly would // only repeat the line. It is still re-checked, twice a day, so a // zone added to the provider later is picked up without a restart. retry := time.Hour if errors.Is(err, errNoZone) { retry = noZoneRetry } log.Printf("edge: wildcard *.%s: %v — per-host issuance keeps covering; retrying in %s", p, err, retry) if retry < delay { delay = retry } w.mu.Lock() w.nextAttempt[p] = time.Now().Add(retry) w.mu.Unlock() continue } w.mu.Lock() delete(w.nextAttempt, p) cert := w.certs[p] w.mu.Unlock() log.Printf("edge: wildcard *.%s obtained (valid until %s)", p, cert.Leaf.NotAfter.Format(time.RFC3339)) } if standDowns > 0 { w.logStandDown(standDowns) // Come back in an hour, not twelve. A non-issuer's next pass is what // LOADS the wildcard the issuer has meanwhile minted into this box's // cache — certFor serves from the in-memory map loadCached fills — and // refresh() only wakes the loop on a route swap, which a quiet box may // not see for a day. No ACME traffic is involved in the retry. if delay > time.Hour { delay = time.Hour } } return delay } // issuerGate is this manager's read of the deployment's issuer gate, with // the ungated default applied (see IssuerGate, certsync.go). func (w *wildcardManager) issuerGate() (context.Context, bool) { if w.issuer == nil { return context.Background(), true } return w.issuer() } // logStandDown says once an hour, at most, that this box wanted wildcards it // is not allowed to order. Not silent — a box with no wildcard and no reason // in the journal is the hard kind of production puzzle — and not per-pass // either: refresh() can wake a pass on every route swap, and on a // non-issuer every one of those would repeat the same line. func (w *wildcardManager) logStandDown(parents int) { now := time.Now() w.mu.Lock() quiet := now.Sub(w.lastNotIssuerLog) < time.Hour if !quiet { w.lastNotIssuerLog = now } w.mu.Unlock() if quiet { return } log.Printf("edge: wildcard: %d parent(s) need issuance but this box is not the deployment's certificate issuer — serving what the issuer publishes", parents) } // noZoneRetry is how long a parent whose challenge zone the DNS provider // does not hold waits before it is looked at again (see pass). const noZoneRetry = 12 * time.Hour // obtain runs one full ACME DNS-01 issuance for *.parent and swaps the // result in. func (w *wildcardManager) obtain(ctx context.Context, parent string) error { // Before the CA hears anything: can the provider publish the challenge // at all? A parent whose zone lives elsewhere (per-account platform // parents on a deployment whose apps domain is not in the provider's // account, the case that found this) would otherwise place an order // every hour, fail at present, and deactivate it again — no harm to the // rate limits, since the authorization is deactivated and never // validated, but a pointless round trip to the CA and a journal line per // parent per hour. The lookup is the one present would make first. if zc, ok := w.dns.(zoneChecker); ok { if err := zc.checkZone(ctx, w.challengeFQDN(parent)); err != nil { return fmt.Errorf("cannot publish the DNS-01 challenge: %w", err) } } acctKey, err := w.accountKey(ctx) if err != nil { return fmt.Errorf("account key: %v", err) } client := &acme.Client{Key: acctKey, DirectoryURL: w.directory} if _, err := client.Register(ctx, &acme.Account{}, acme.AcceptTOS); err != nil && err != acme.ErrAccountAlreadyExists { return fmt.Errorf("acme register: %v", err) } order, err := client.AuthorizeOrder(ctx, []acme.AuthzID{{Type: "dns", Value: "*." + parent}}) if err != nil { return fmt.Errorf("authorize order: %v", err) } for _, zurl := range order.AuthzURLs { if err := w.solveDNS01(ctx, client, parent, zurl); err != nil { return err } } certKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) if err != nil { return err } csr, err := x509.CreateCertificateRequest(rand.Reader, &x509.CertificateRequest{ Subject: pkix.Name{CommonName: "*." + parent}, DNSNames: []string{"*." + parent}, }, certKey) if err != nil { return fmt.Errorf("csr: %v", err) } ders, _, err := client.CreateOrderCert(ctx, order.FinalizeURL, csr, true) if err != nil { return fmt.Errorf("finalize: %v", err) } leaf, err := x509.ParseCertificate(ders[0]) if err != nil { return fmt.Errorf("parse issued cert: %v", err) } cert := &tls.Certificate{Certificate: ders, PrivateKey: certKey, Leaf: leaf} // Paranoia: never store what certFor would refuse to serve. if !coversParent(cert, parent) { return fmt.Errorf("issued cert does not cover *.%s", parent) } raw, err := encodeWildcardPEM(cert, certKey) if err != nil { return fmt.Errorf("encode: %v", err) } if err := w.cache.Put(ctx, wildcardPEMKey(parent), raw); err != nil { return fmt.Errorf("persist: %v", err) } w.mu.Lock() w.certs[parent] = cert w.mu.Unlock() return nil } // solveDNS01 answers one authorization: publish the challenge TXT record, // wait for the CA to see it, and always clean the record up afterwards. func (w *wildcardManager) solveDNS01(ctx context.Context, client *acme.Client, parent, authzURL string) (retErr error) { authz, err := client.GetAuthorization(ctx, authzURL) if err != nil { return fmt.Errorf("get authorization: %v", err) } if authz.Status == acme.StatusValid { return nil // a recent order already proved this name } // On failure, deactivate the pending authorization rather than abandoning // it — pending authzs count against LE's per-account cap (autocert does // the same "to reduce rate limit quotas"). Best-effort on a fresh context. defer func() { if retErr == nil || authz.Status != acme.StatusPending { return } dctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() if err := client.RevokeAuthorization(dctx, authz.URI); err != nil { log.Printf("edge: wildcard *.%s: deactivate abandoned authorization: %v", parent, err) } }() var chal *acme.Challenge for _, c := range authz.Challenges { if c.Type == "dns-01" { chal = c break } } if chal == nil { return errors.New("CA offered no dns-01 challenge") } txt, err := client.DNS01ChallengeRecord(chal.Token) if err != nil { return err } challengeFQDN := w.challengeFQDN(parent) cleanup, err := w.dns.present(ctx, challengeFQDN, txt) if err != nil { return fmt.Errorf("publish challenge TXT: %v", err) } // Clean up with a fresh context: the record must not outlive the attempt // even when ctx is already dead (stale challenge TXTs break the NEXT run). defer func() { cctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() if err := cleanup(cctx); err != nil { log.Printf("edge: wildcard *.%s: cleanup of challenge TXT failed: %v", parent, err) } }() // Best-effort: give the record a moment to reach the authoritative NS // before telling the CA to look. Proceed on timeout — the CA does its own // retries, and a hard failure just comes back through the retry loop. waitForTXT(ctx, challengeFQDN, txt, time.Minute) if _, err := client.Accept(ctx, chal); err != nil { return fmt.Errorf("accept challenge: %v", err) } if _, err := client.WaitAuthorization(ctx, authz.URI); err != nil { return fmt.Errorf("authorization: %v", err) } return nil } // wildcardAccountKeyName is the ACME account key's cache key — Eleven's file // name, for the same no-migration reason as wildcardPEMKey. const wildcardAccountKeyName = "wildcard-account.key" // accountKey loads (or mints) the ACME account key. Separate from autocert's // account on purpose: this flow manages its own registration lifecycle. func (w *wildcardManager) accountKey(ctx context.Context) (*ecdsa.PrivateKey, error) { raw, err := w.cache.Get(ctx, wildcardAccountKeyName) if err == nil { block, _ := pem.Decode(raw) if block == nil { return nil, fmt.Errorf("%s: not PEM", wildcardAccountKeyName) } return x509.ParseECPrivateKey(block.Bytes) } if !errors.Is(err, autocert.ErrCacheMiss) { // Only a genuine miss mints a key: treating a backend fault as "no // key yet" would register a fresh ACME account on every hiccup. Fail // and let obtain()'s hourly retry come back around. return nil, err } key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) if err != nil { return nil, err } der, err := x509.MarshalECPrivateKey(key) if err != nil { return nil, err } // Crash-safety (temp + rename, 0600) is the cache backend's contract — // the dir backend inherits it from autocert.DirCache. raw = pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: der}) if err := w.cache.Put(ctx, wildcardAccountKeyName, raw); err != nil { return nil, err } return key, nil } // newWildcardFromEnv builds the manager from the environment, or returns nil // when wildcards are off: no CARLOS_WILDCARD_DOMAINS and no // PLATFORM_APPS_DOMAIN (nothing to cover — silent, the historical default), // or no DNS provider configured (logged: half a config deserves a reason in // the journal). nil means RunWithActivator wires nothing and the edge // behaves exactly as before this feature existed. func newWildcardFromEnv(e *Edge, cache autocert.Cache) *wildcardManager { static := splitDomains(os.Getenv("CARLOS_WILDCARD_DOMAINS")) appsDomain := strings.ToLower(strings.TrimSpace(os.Getenv("PLATFORM_APPS_DOMAIN"))) if len(static) == 0 && appsDomain == "" { return nil } // Steering (spec Part 4, steeringprobe.go): the deployment's Route53 // health checks handshake with SNI edge., which is one label // under the apps domain — one label SHALLOWER than the per-account // parents this manager derives, so nothing here covered it and every // check failed at the handshake, withdrawing every record it was meant to // protect. Adding the apps domain itself as a parent is the whole fix, // and it is exactly what a static parent IS (CARLOS_WILDCARD_DOMAINS // carries go.tito.io for the same reason: a domain whose one-label-deep // hosts need certs). // // DNS-01 through this manager, rather than a narrow per-host exception // for the one name, for three reasons — and NOT because a per-host order // could not complete (an earlier draft of this comment claimed that and // was wrong: autocert orders under context.Background() with a 5-minute // budget, autocert.go:274, so a health checker hanging up on the // handshake does not cancel the order it triggered): // // 1. the exception would have to breach the Allowed rule (PR #180) in // TWO places, not one: issuerHostPolicy's registry check, and the // mirror's m.allowed bound on the live HTTP-01 read-through // (certsync.go's challengeResponse). Allowed is deliberately // exact-registry-only, and it is what keeps an attacker-chosen name // from turning into an order or a bucket read; // 2. once a deployment steers, the name resolves to EVERY armed edge, so // the CA reaches an arbitrary one and per-host issuance would hang on // the cross-box challenge mirror for a name that is deployment // infrastructure rather than a member's route; // 3. every order attempt is paid for in live failed health checks (the // probe fails until the cert exists, and the records it gates are // withdrawn meanwhile) and in the shared ACME account's // failed-validation budget. The wildcard path spends neither: it // answers in our own zone, on its own schedule, and needs no A record // to exist first — which matters, because the record only appears // once the converger runs. // // Gated on the steering zone, not on the apps domain: both production // deployments set an apps domain today and neither steers, and minting an // unasked-for *. on them (a real ACME order, a real CT log // entry, and a change to which cert their existing one-label hosts are // served from) is not something a dormant feature gets to do. // // One posture change to state outright, because it is what this parent // COSTS: a stranger's one-label unknown host under the apps domain // (anything.) now COMPLETES the TLS handshake — certFor // serves the wildcard before HostPolicy is ever consulted — and reaches // the ordinary unknown-host answer: since #174 the pending page's 503 // (pending.go), which gives up to the not-found text or the -fallback // URL after its retry window; before that a 404 or a -fallback 302. Before this // parent existed it failed at the handshake instead, because autocert's // policy refuses a name that is not a registry route. Nothing is // disclosed that a routed host does not already disclose, and every // per-ACCOUNT parent has had exactly this property since wildcards // landed — but it is a real change on a steering deployment and belongs // here beside the CT-log entry, not only in a review thread. // // The parent is derived from e.steeringHost, NOT from this function's own // appsDomain read: New() normalized the environment once (edge.go, // steeringProbeHostFromEnv — which trims a trailing dot, and the Getenv // above does not). Two reads meant PLATFORM_APPS_DOMAIN "oncarlos.com." // answered probes on edge.oncarlos.com while arming a parent // "oncarlos.com." that certFor's CutSuffix could never match: no cert, // every health check failing, on the one name the box promises to serve. // One value, one normalization, both halves. if e.steeringHost != "" { static = append(static, strings.TrimPrefix(e.steeringHost, "edge.")) } dns, reason := dnsProviderFromEnv() if dns == nil { log.Printf("edge: wildcard certs off (%s) — per-host issuance continues", reason) return nil } // Delegated parents (spec §4): the SOURCE is claim-derived and lands // with the claim-derivation pass (Plan C); until then the zone is // configured but the set is empty — inert by construction. rawZone := os.Getenv("CARLOS_ACME_DELEGATION_ZONE") zone := strings.ToLower(strings.TrimSpace(rawZone)) if zone != "" && !provision.ValidHostShape(zone) { // Degrade, don't die: the edge must still boot on a bad zone value. // Delegation goes off; per-host and direct (own-zone) DNS-01 keep // covering everyone else. log.Printf("edge: CARLOS_ACME_DELEGATION_ZONE %q invalid — delegation off, per-host and direct DNS-01 continue", rawZone) zone = "" } // Delegated source: the claim-derivation pass (Task 3, internal/domains) // writes wildcard_domains; refresh() reads it fresh on every call, same // as e.deploymentHosts above. A read error returns the LAST successful // list, not nil/empty: refresh() runs on startup, on OnReload, and on the // cert mirror's and home map's wakes (run()'s 12h ticker never // re-invokes the delegated source itself), so // on a quiet box a transient error would otherwise empty the delegated // set until some unrelated registry write next moves the fingerprint — // unbounded, and it would also stop parentsFor from including a parent // whose cert is already cached and valid (certFor iterates w.parents). // Caching here is what makes the log line's "unchanged this pass" // literally true. var ( mu sync.Mutex lastGood []string ) delegated := func() []string { wds, err := e.reg.ListWildcardDomains() if err != nil { mu.Lock() out := append([]string(nil), lastGood...) mu.Unlock() log.Printf("edge: wildcard domains read failed: %v — delegated set unchanged this pass", err) return unionDelegated(out, e.homes.Load().WildcardParents(), zone) } out := make([]string, 0, len(wds)) for _, w := range wds { out = append(out, w.Host) } mu.Lock() lastGood = out mu.Unlock() // Plus every delegated parent another box publishes (#305): the // issuer minted its wildcard, the mirror put the PEM on this disk, // and refresh's static/apps-domain precedence applies to these // exactly as to local rows. Read-through, never folded into // lastGood: the home map is its own last-good. return unionDelegated(out, e.homes.Load().WildcardParents(), zone) } // Hosts are the DEPLOYMENT's (this box's routes plus every name another // box homes, #300), so a region edge derives the account parents of apps // it only ever backhauls to — and loads their mirrored wildcards. w := newWildcardManager(static, appsDomain, e.deploymentHosts, delegated, zone, cache, dns) // DNS-01 orders are gated on the same election per-host HTTP-01 orders // are (certsync.go): one issuer per deployment, whichever protocol the // name needs. The cache handle is already the publishing one, so the // wildcard PEM and its ACME account key distribute with everything else. w.issuer = e.issuerGate return w } // wrapGetCertificate serves the wildcard for covered names and hands // everything else to inner (autocert's per-host path) — split out so the // fall-through contract is testable without a TLS listener. func wrapGetCertificate(w *wildcardManager, inner func(*tls.ClientHelloInfo) (*tls.Certificate, error)) func(*tls.ClientHelloInfo) (*tls.Certificate, error) { return func(hello *tls.ClientHelloInfo) (*tls.Certificate, error) { // A tls-alpn-01 validation handshake must reach autocert's challenge // cert, never the wildcard — the CA is probing the per-host order the // inner path holds. if len(hello.SupportedProtos) == 1 && hello.SupportedProtos[0] == acme.ALPNProto { return inner(hello) } if c := w.certFor(hello.ServerName); c != nil { return c, nil } return inner(hello) } } // waitForTXT polls DNS until name resolves to a TXT containing value, or the // deadline passes (best-effort — see caller). func waitForTXT(ctx context.Context, name, value string, timeout time.Duration) { deadline := time.Now().Add(timeout) for time.Now().Before(deadline) && ctx.Err() == nil { if txts, err := net.DefaultResolver.LookupTXT(ctx, name); err == nil { for _, t := range txts { if t == value { return } } } time.Sleep(3 * time.Second) } }