// Cert sweep (#299 §3f, #243): the issuer removes cert-prefix entries that // are provably dead — and nothing else. // // Two kinds of litter accumulate under control/certs/, for two different // reasons, and they need two different predicates: // // - SPENT CHALLENGE TOKENS (#243). The issuer writes an HTTP-01 response // or a tls-alpn-01 token cert, validates, and deletes it — but the // delete is housekeeper-gated, so a stand-down or a crash mid-order // leaves the token behind permanently. A validation's whole life is // seconds, so AGE is a complete predicate here. // - CERTS FOR HOSTS NOBODY SERVES ANY MORE. A detached custom domain's // cert is never republished (ownsCertName refuses) and never deleted, // so the object lingers, the mirror serves it to matching SNI until it // expires, and the expired object then lingers forever. // // THE PREDICATE FOR THE SECOND IS EXPIRY, NEVER DETACHMENT, and the reason // is worth stating at length because the tempting version of this sweep is // catastrophic. A sweep keyed "no claim exists for this host" would delete // every non-claim-backed cert in the deployment — every platform host, every // wildcard PEM, the account keys — and worse, a registry lag or a stale // edges read would take a LIVE host's certs down fleet-wide through the // mirror. This repo's standing rule is that inference never destroys owner // property (internal/domains/reconcile.go §1c), and a certificate is the // member's property. // // So: an entry is deleted only when its own leaf expired long ago. That // dissolves the "the mirror serves it until expiry" concern by // construction, and it makes a wrong deletion harmless — the entry was // already dead, and anything still wanted is re-minted by the warm. // // Account keys are never touched, by name and again by shape: they are the // issuer's identity at the CA, and deleting one costs the deployment its // ACME registration. package edge import ( "context" "log" "strings" "time" "github.com/carlosframework/platform/internal/release" ) const ( // tokenLitterTTL is how long a spent challenge token may linger before // the sweep takes it (#243). A validation completes in seconds and // autocert's own order context is five minutes, so an hour is generous // by two orders of magnitude — deliberately, because deleting a token // out from under a live validation burns the order. tokenLitterTTL = time.Hour // certSweepParseAge is the pre-filter: an object younger than this is // never even read, let alone deleted. // // It is what keeps the sweep from being the per-object N+1 it would // otherwise be. autocert renews at ~30 days remaining on a 90-day // certificate, so a HEALTHY entry is rewritten roughly every 60 days and // its object stamp never gets this old. An entry that does is one // nothing is renewing — which is exactly the population worth reading. // Steady-state cost: one listing, zero reads. certSweepParseAge = 60 * 24 * time.Hour // certSweepGrace is how long a leaf must have been EXPIRED before its // entry is removed. A month, because there is no hurry whatsoever: the // cost of keeping a dead certificate is bytes, and the cost of removing // a live one is an outage. certSweepGrace = 30 * 24 * time.Hour // certSweepInterval is how often the expiry half runs — the half that // costs reads. A certificate dead for a month can wait a day. certSweepInterval = 24 * time.Hour // certSweepMinInterval is the floor for the whole sweep, token half // included. It matches the warm's own quiet-path cadence, so an ordinary // pass sweeps and a capped-drain pass (which comes back every minute // until a backlog clears) does not. certSweepMinInterval = warmInterval // certSweepDeletesPerPass bounds the damage any single pass can do, in // either direction: a bug here deletes at most this many objects before // somebody sees the log, and a genuine backlog drains over days rather // than in one burst against the bucket. certSweepDeletesPerPass = 50 ) // sweep removes provably dead entries under the cert prefix. Issuer-gated by // its caller; see certWarm.pass. // // Never returns an error. Every failure is logged and skipped: a sweep is // housekeeping, and housekeeping that can fail a pass would take the warm // down with it. func (c *certWarm) sweep(ctx context.Context, now time.Time) { if c.store == nil { return } // Not on a capped-drain pass. When a first-time backlog trips // warmOrdersPerPass, pass() comes back every minute until it clears, and // housekeeping does not need doing sixty times an hour — the token TTL // is an hour and the expiry half is daily. Without this the drain would // spend a listing of the whole cert prefix per minute for nothing. if !c.lastSweep.IsZero() && now.Sub(c.lastSweep) < certSweepMinInterval { return } lister, ok := c.store.(release.ModLister) if !ok { // No stamps, no age, no sweep. Guessing at an object's age from its // contents is exactly the inference this file refuses to make. return } ctx, cancel := context.WithTimeout(ctx, certSweepTimeout) defer cancel() entries, err := lister.ListMod(ctx, certPrefix) if err != nil { log.Printf("edge: cert sweep: could not list %s: %v", certPrefix, err) return } c.lastSweep = now expiryDue := now.Sub(c.lastExpirySweep) >= certSweepInterval deletes := 0 for _, e := range entries { if deletes >= certSweepDeletesPerPass { log.Printf("edge: cert sweep: stopping at %d deletions this pass — the rest go next pass", deletes) break } name, ok := strings.CutPrefix(e.Key, certPrefix) if !ok || name == "" { continue } // Account material, first and unconditionally. Never by age, never // by expiry, never at all. if accountKeyName(name) { continue } switch { case isChallengeToken(name): if now.Sub(e.Mod) < tokenLitterTTL { continue } if c.deleteEntry(ctx, e.Key, "spent challenge token") { deletes++ } case expiryDue && now.Sub(e.Mod) >= certSweepParseAge: if c.sweepExpired(ctx, e.Key, name, now) { deletes++ } } } if expiryDue { c.lastExpirySweep = now } } // certSweepTimeout budgets one sweep. It runs on the warm's own goroutine, // so a hung bucket must not hold the next warm pass hostage. const certSweepTimeout = 2 * time.Minute // isChallengeToken reports whether name is validation scratch rather than a // serving artifact — an HTTP-01 response body or a tls-alpn-01 token cert. func isChallengeToken(name string) bool { return strings.HasSuffix(name, httpTokenSuffix) || strings.HasSuffix(name, alpnTokenSuffix) } // sweepExpired reads one long-untouched entry and removes it only if its own // leaf says it is long dead. // // Everything that is not a confident yes is a no. An entry that will not // read, will not parse, or carries no leaf is LEFT ALONE — it may be a shape // a newer build publishes, and a sweep that deletes what it cannot // understand is a sweep that deletes tomorrow's format. func (c *certWarm) sweepExpired(ctx context.Context, key, name string, now time.Time) bool { raw, _, err := c.store.Get(ctx, key) if err != nil { log.Printf("edge: cert sweep: could not read %s: %v", key, err) return false } cert, err := parseCertEntry(raw) if err != nil || cert.Leaf == nil { log.Printf("edge: cert sweep: %s does not parse as a certificate, leaving it alone", key) return false } if now.Sub(cert.Leaf.NotAfter) < certSweepGrace { return false } return c.deleteEntry(ctx, key, "certificate expired "+cert.Leaf.NotAfter.UTC().Format("2006-01-02")+" for "+name) } // deleteEntry removes one object and says so. Every deletion under this // prefix is logged with its reason — there is no such thing as a quiet // delete of cert material. func (c *certWarm) deleteEntry(ctx context.Context, key, why string) bool { if err := c.store.Delete(ctx, key); err != nil { log.Printf("edge: cert sweep: could not delete %s: %v", key, err) return false } log.Printf("edge: cert sweep: removed %s — %s", key, why) return true }