// Cert warm (#299 stage 3): the platform orders and RENEWS certificates for // the custom domains it serves, instead of waiting for a visitor's handshake // to trigger the first issuance. // // Why this exists at all. Per-host certs are minted lazily by autocert on the // first TLS handshake for a name. That is fine for a name people visit from // the box that serves it, and wrong for a custom domain on a steering // deployment, for two compounding reasons: // // - The FIRST visitor pays a multi-second ACME order, and if the customer's // DNS only just landed, they are also the one who discovers it failed. // - RENEWAL is worse. autocert renews through per-host `domainRenewal` // timers that live in process memory. They die on every fleet roll and // re-arm only when a handshake reaches m.cert again. Under latency // steering a custom domain's visitors may never land on the issuer, so // after a roll NOTHING re-arms the timer and the cert quietly expires at // day 90 — on a host that was serving perfectly the whole time. // // So the warm calls GetCertificate for every host in its set on EVERY pass, // unconditionally. That is not wasteful and it is not a retry loop: for a // healthy cert it is one cache read plus the startRenew that re-arms the // timer, and the re-arm is the entire point. A skip-if-fresh check would // defeat it — autocert's m.cert returns any unexpired cached cert, so a cert // 29 days from expiry is a cache hit and no renewal is ever scheduled. package edge import ( "context" "crypto/tls" "errors" "log" "net" "sort" "strings" "sync" "time" "golang.org/x/crypto/acme" "github.com/carlosframework/platform/internal/certwarm" "github.com/carlosframework/platform/internal/release" ) const ( // warmInterval is the quiet-path cadence. Renewal is the job with the // real deadline (autocert renews at ~30 days remaining), so hourly is // far more often than strictly needed — the cost of a pass over a // healthy set is a cache read per host, and being early is free. warmInterval = time.Hour // warmBackoff is the per-host floor after a failed order. Matches the // wildcard manager's hourly retry. warmBackoff = time.Hour // warmRateLimitBackoff is the floor after the CA says rateLimited. // Deliberately much harder than warmBackoff: hourly retries INTO a rate // limit are what turn a transient limit into a sustained one, and every // host in the deployment shares the one ACME account's budget. warmRateLimitBackoff = 12 * time.Hour // warmOrdersPerPass bounds NEW orders per pass. Renewal re-arms are not // orders and are never capped — they are the reason the pass exists. // When the cap bites, pass() returns a short delay so a backlog drains // in minutes rather than trickling out over days (the wildcard pass() // precedent). warmOrdersPerPass = 10 // warmCappedDelay is how soon to come back with capped work remaining. warmCappedDelay = time.Minute ) // certWarm proactively orders and renews per-host certificates. // // It is a sibling of wildcardManager, with three deliberate departures from // it, each of which is a bug if copied across: // // - No holding-context claim. The wildcard manager runs its own ACME // client, so it can hand the holding's context to the order. The warm's // trigger is autocert's GetCertificate, which takes NO context and builds // its own background one with a 5-minute timeout. So the gate is // re-checked BETWEEN orders instead, and a ≤5-minute overhang past a lost // holding is accepted — inside the double-holder envelope issuerHostPolicy // already documents. If the holding ends mid-order, the finishing // cachePut re-checks the gate and the cert lands local-only; the new // issuer re-orders it. Bounded waste, not a hazard. // - No cache-load-before-gate step. The wildcard manager loads cached certs // before standing down because it SERVES from its own map. The warm // serves nothing — per-host serving is the mirror plus autocert — so a // non-issuer scanning caches would be pure waste. It stands down at the // gate. // - The unconditional GetCertificate described in the package comment. type certWarm struct { // hosts is the warm set: exact hostnames this deployment serves that are // not under the apps domain and not covered by a wildcard. See // Edge.warmSet. hosts func() []string // getCert is autocert's per-host path (NOT the wildcard-wrapped one — a // wrapped call would be answered by the wildcard and order nothing). getCert func(*tls.ClientHelloInfo) (*tls.Certificate, error) // pointsHere reports whether host's DNS resolves to this deployment. // Ordering for a half-pointed name burns the shared account's // failed-validation budget; see dnsGate. pointsHere func(host string) bool // issuer gates ORDERING, exactly as it gates the wildcard manager's. issuer IssuerGate // store is the deployment bucket, for the durable outcome records // (#299 §3e, internal/certwarm). Nil disables recording entirely and // changes nothing else: warming, backoff and logging are unaffected, so // an agent-less `carlos edge` and every test that does not care keep // working exactly as before. store release.Store mu sync.Mutex // nextAttempt is the per-host backoff floor. nextAttempt map[string]time.Time // warmed is the set of hosts a cert has been obtained for at least once, // so a first order can be told from a renewal re-arm for the order cap // and for logging. warmed map[string]bool lastStandDownLog time.Time // lastSweep and lastExpirySweep pace the cert sweep (certsweep.go). // Deliberately NOT guarded by mu, unlike everything else on this struct: // they are touched only from sweep(), which is only ever reached from // pass(), which only ever runs on run()'s single goroutine. A mutex here // would suggest a concurrency that does not exist. lastSweep time.Time lastExpirySweep time.Time } func newCertWarm(hosts func() []string, getCert func(*tls.ClientHelloInfo) (*tls.Certificate, error), pointsHere func(string) bool, issuer IssuerGate, store release.Store) *certWarm { return &certWarm{ hosts: hosts, getCert: getCert, pointsHere: pointsHere, issuer: issuer, store: store, nextAttempt: map[string]time.Time{}, warmed: map[string]bool{}, } } // issuerGate is this warm's read of the deployment's gate, ungated default // applied (see IssuerGate, certsync.go). func (c *certWarm) issuerGate() (context.Context, bool) { if c.issuer == nil { return context.Background(), true } return c.issuer() } func (c *certWarm) run() { for { time.Sleep(c.pass()) } } // pass warms every host in the set once and returns how long to wait. func (c *certWarm) pass() time.Duration { hosts := c.hosts() // Stand down before ANY work: unlike the wildcard manager there is no // cache to load first, so a non-issuer has nothing useful to do here. // The gate comes BEFORE the empty-set check, not after, because the // sweep at the bottom of this pass has work to do in a deployment with // no custom domains at all — spent challenge tokens and long-expired // certs are not the warm set's business. ctx, ok := c.issuerGate() if !ok { c.logStandDown(len(hosts)) return warmInterval } if len(hosts) == 0 { c.sweep(ctx, time.Now()) return warmInterval } // The durable outcome records (#299 §3e), read ONCE per pass. Two things // come out of this read and neither is available in memory: the attempt // count a previous issuer accumulated (so a failover does not restart the // clock on a domain that has been broken for days), and the set of hosts // that currently HAVE a record, which is what makes both the // delete-on-success and the prune below decidable without a Get per host. records := c.loadRecords(ctx) inSet := make(map[string]bool, len(hosts)) for _, h := range hosts { inSet[h] = true } orders, capped := 0, false for _, h := range hosts { now := time.Now() c.mu.Lock() until, backing := c.nextAttempt[h] seen := c.warmed[h] c.mu.Unlock() if backing && now.Before(until) { continue } // The gate is re-checked BETWEEN orders, not once per pass: a pass // over a large set can outlive a holding, and every order after the // holding ends is an order this box has no authority to place. gctx, ok := c.issuerGate() if !ok { c.logStandDown(len(hosts) - orders) return warmInterval } // A host with no cert yet is a NEW order and is both capped and // DNS-gated. A host already warmed is a renewal re-arm: uncapped and // ungated, because it is a cache read that costs the CA nothing and // re-arms the timer that renewal depends on. Gating the re-arm on DNS // would strand exactly the domain whose resolver is briefly sick. if !seen { if orders >= warmOrdersPerPass { capped = true continue } if !c.pointsHere(h) { continue } orders++ } if err := c.warmOne(h); err != nil { c.fail(gctx, h, err, records[h]) continue } c.mu.Lock() delete(c.nextAttempt, h) first := !c.warmed[h] c.warmed[h] = true c.mu.Unlock() if first { log.Printf("edge: cert warmed for %s", h) } // A record exists only while a host is FAILING, so success removes // it. That is what makes "no record" the healthy case and keeps the // console's fold one object per broken host rather than one per // domain (internal/certwarm's package comment). if _, had := records[h]; had { c.clearRecord(gctx, h, "recovered") delete(records, h) } } c.prune(ctx, records, inSet) // Housekeeping last: it is the least urgent thing this pass does, and // running it after the orders means a slow bucket delays litter removal // rather than certificate issuance (certsweep.go). c.sweep(ctx, time.Now()) if capped { return warmCappedDelay } return warmInterval } // warmOne asks autocert for host's certificate, which mints it when absent // and re-arms its renewal timer when present. func (c *certWarm) warmOne(host string) error { _, err := c.getCert(warmHello(host)) return err } // fail arms the per-host backoff, harder when the CA is rate-limiting, and // records the outcome durably. // // prior is this host's record as the pass read it (the zero Record when // there is none). The attempt count and the "failing since" stamp are // carried FORWARD from it rather than recomputed from memory — that is the // whole reason the record is in the bucket rather than on the heap. func (c *certWarm) fail(ctx context.Context, host string, err error, prior certwarm.Record) { backoff := warmBackoff class := certwarm.ClassOther var ae *acme.Error if errors.As(err, &ae) && ae.ProblemType == "urn:ietf:params:acme:error:rateLimited" { backoff = warmRateLimitBackoff class = certwarm.ClassRateLimited log.Printf("edge: cert warm for %s rate-limited by the CA — backing off %s (the whole deployment shares one ACME account)", host, backoff) } else { log.Printf("edge: cert warm for %s: %v — retrying in %s", host, err, backoff) } now := time.Now() next := now.Add(backoff) c.mu.Lock() c.nextAttempt[host] = next c.mu.Unlock() first := prior.FirstFailedAt if first == 0 { first = now.Unix() } c.writeRecord(ctx, certwarm.Record{ Host: host, Attempts: prior.Attempts + 1, FirstFailedAt: first, LastFailedAt: now.Unix(), NextAttemptAt: next.Unix(), Class: class, Error: err.Error(), }, now) } func (c *certWarm) logStandDown(hosts int) { c.mu.Lock() defer c.mu.Unlock() if time.Since(c.lastStandDownLog) < time.Hour { return } c.lastStandDownLog = time.Now() log.Printf("edge: cert warm standing down for %d host(s) — another box holds the issuer lease", hosts) } // warmHello builds the synthetic ClientHelloInfo the warm hands autocert. // // The CipherSuites entry is LOAD-BEARING and must not be dropped as // decoration. autocert picks the key type from the hello via supportsECDSA // (ported here as helloSupportsECDSA): with a hello carrying no signature // schemes, no curves and no cipher suites, all three checks fall through to // false, autocert chooses certKey{isRSA: true}, and the warm mints an RSA // cert under "+rsa" — while every real browser asks for "" and // gets nothing. The order would be spent, the name still uncovered, and the // first visitor would trigger a SECOND issuance. // // One ECDSA suite is enough: helloSupportsECDSA's first two checks skip on // nil, and its cipher-suite loop returns true on this suite. func warmHello(host string) *tls.ClientHelloInfo { return &tls.ClientHelloInfo{ ServerName: host, CipherSuites: []uint16{tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256}, } } // warmSet is the set of hostnames the cert warm orders and renews for: // EXACT names this deployment serves that are not under the apps domain and // not covered by a wildcard. // // The predicate is deliberately simpler than "aliases" or "custom domains". // A custom domain is a foreign domain by definition — the attach door refuses // apps-domain hosts outright — so "not under the apps domain" covers attached // aliases and first-route claims alike, with no need to reason about which // registry table a name arrived through. That matters: first-route claims // (`carlos deploy -kind static -host `) mint a real ROUTE, not an // alias, so an alias-only rule would silently never warm the shipped headline // flow. // // What it excludes, and why each one: // // - "*." rows — covered by a wildcard PEM, and autocert refuses the // wildcard label anyway. // - anything under the apps domain — canary hosts and platform hosts. This // is the exclusion that keeps a first run from being a burst: canaries are // three labels under the apps domain, no wildcard parent covers them, and // a naive "everything we serve" set would order a cert AND a CT-log entry // for every one of them. // - anything one label under a CONFIGURED wildcard parent, tested BY SHAPE // against currentParents() rather than by asking certFor. certFor returns // nil for a parent whose cert has not been obtained yet, so a shape-blind // test would warm every app of a new account per-host during exactly the // window before their wildcard lands. func (e *Edge) warmSet(appsDomain string, parents func() []string) []string { appsDomain = strings.ToLower(strings.TrimSpace(appsDomain)) covered := wildcardCoverer(parents) seen := map[string]bool{} var out []string consider := func(h string) { h = strings.ToLower(strings.TrimSpace(h)) if h == "" || seen[h] || strings.HasPrefix(h, "*.") { return } if appsDomain != "" && (h == appsDomain || strings.HasSuffix(h, "."+appsDomain)) { return } if covered(h) { return } seen[h] = true out = append(out, h) } for h := range e.current() { consider(h) } for _, h := range e.homes.Load().ServedHosts() { consider(h) } sort.Strings(out) return out } // wildcardCoverer reports whether a host is exactly one label under a // wildcard parent this box has configured or derived — by SHAPE, never by // whether the parent's cert exists yet (see warmSet). A nil parents func // covers nothing, which is the correct answer for a deployment with no // wildcard manager at all. func wildcardCoverer(parents func() []string) func(string) bool { return func(host string) bool { if parents == nil { return false } for _, p := range parents() { rest, ok := strings.CutSuffix(host, "."+p) if ok && rest != "" && !strings.Contains(rest, ".") { return true } } return false } } // dnsGate builds the "does this name point at us?" test the warm applies // before placing a NEW order. // // Let's Encrypt caps failed validations at 5 per account per hostname per // hour, and the ACME account is shared by every name this issuer mints. So a // half-configured customer domain that we order for anyway does not just fail // its own order — it spends budget every other tenant needs. The gate is what // keeps one customer's unfinished DNS from being everyone's problem. // // ALL resolved answers must be in the edge set, not merely one. A name in // mid-migration resolves to both the old host and ours; an any-match test // passes it, the order goes out, and validation fails from the CA's vantage // anyway — burning exactly the budget this gate exists to protect. Requiring // all of them means the gate only opens once the customer has actually // finished. // // An empty edge set fails CLOSED (no host points anywhere), matching the // domains pass's stand-down: a set we could not read is not evidence about // anybody's DNS, and the cost of waiting a pass is one hour of delay against // the cost of burning a shared rate limit. func dnsGate(resolve func(string) ([]string, error), edgeIPs func() []string) func(string) bool { return func(host string) bool { set := map[string]bool{} for _, ip := range edgeIPs() { set[ip] = true } if len(set) == 0 { return false } ips, err := resolve(host) if err != nil || len(ips) == 0 { return false } for _, ip := range ips { if !set[ip] { return false } } return true } } // warmResolver is the DNS gate's lookup: the system resolver, bounded so a // hung one costs this host its turn in the pass and not the pass itself. func warmResolver(host string) ([]string, error) { ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() return net.DefaultResolver.LookupHost(ctx, host) } // ---- the durable outcome records (#299 §3e) --------------------------------- // // Every one of these degrades to a no-op rather than to a failure. A bucket // blip must never stop the warm from warming: the records are how the // console explains a failure, not how the fix happens, and a pass that // cannot write one has still ordered the certificate. // warmStoreTimeout budgets the record I/O for one pass. Generous relative to // the work (a listing plus a handful of small objects) and tiny relative to // the hourly cadence, so a slow bucket delays a pass rather than wedging it. const warmStoreTimeout = 30 * time.Second // loadRecords reads every outcome record in the deployment. // // The cost is one object per BROKEN host, not one per domain — see the // package comment on internal/certwarm — so this stays a listing plus a // small handful of Gets in any deployment that is mostly working, and grows // only in one that is mostly not. // // A read failure returns what it has. Partial knowledge is the right answer // here in both directions: a host whose record failed to load is treated as // having none, so its attempt count restarts at one (an undercount, which // only ever delays the console reaching "failed") and its record is left // alone by the prune (never a delete on the strength of a read we could not // complete). func (c *certWarm) loadRecords(ctx context.Context) map[string]certwarm.Record { out := map[string]certwarm.Record{} if c.store == nil { return out } ctx, cancel := context.WithTimeout(ctx, warmStoreTimeout) defer cancel() keys, err := c.store.List(ctx, certwarm.Prefix) if err != nil { log.Printf("edge: cert warm: could not list %s: %v — attempt counts fall back to this process's own", certwarm.Prefix, err) return out } for _, key := range keys { host := certwarm.HostFromKey(key) if host == "" { continue } body, _, err := c.store.Get(ctx, key) if err != nil { log.Printf("edge: cert warm: could not read %s: %v", key, err) continue } rec, err := certwarm.Parse(body) if err != nil { // A record this build cannot read is left in place, not deleted: // it may be a NEWER issuer's, and a rolling fleet has both. log.Printf("edge: cert warm: could not parse %s: %v", key, err) continue } out[host] = rec } return out } // writeRecord publishes one host's outcome, CAS so a transient double-holder // cannot half-overwrite the other's record. func (c *certWarm) writeRecord(ctx context.Context, rec certwarm.Record, now time.Time) { if c.store == nil { return } body, err := rec.Encode(now) if err != nil { log.Printf("edge: cert warm: could not encode the record for %s: %v", rec.Host, err) return } ctx, cancel := context.WithTimeout(ctx, warmStoreTimeout) defer cancel() if err := putCAS(ctx, c.store, certwarm.Key(rec.Host), body); err != nil { log.Printf("edge: cert warm: could not record the failure for %s: %v", rec.Host, err) } } // clearRecord removes one host's record. why names the occasion in the log — // "recovered" or "detached" — because those are opposite news and a single // deletion line would hide which one happened. func (c *certWarm) clearRecord(ctx context.Context, host, why string) { if c.store == nil { return } ctx, cancel := context.WithTimeout(ctx, warmStoreTimeout) defer cancel() if err := c.store.Delete(ctx, certwarm.Key(host)); err != nil { log.Printf("edge: cert warm: could not clear the %s record for %s: %v", why, host, err) return } log.Printf("edge: cert warm: cleared the outcome record for %s (%s)", host, why) } // prune drops records for hosts that are no longer in the warm set — a // detached custom domain, or one whose route went away. // // This deliberately DOES delete on the strength of an inference ("not in the // set"), which §3f forbids for certificates. The distinction is ownership, // not caution: a certificate is the member's property and a wrong deletion // takes their site down, whereas a warm record is the platform's own note // about its own attempts and a wrong deletion costs an attempt count that // the next failure rebuilds. The prune is also fenced by pass()'s early // return on an empty set, so the one case that would delete EVERYTHING — // a warm set that briefly folds to nothing — never reaches here. func (c *certWarm) prune(ctx context.Context, records map[string]certwarm.Record, inSet map[string]bool) { if c.store == nil || len(records) == 0 { return } for host := range records { if inSet[host] { continue } c.clearRecord(ctx, host, "detached") } }