package edge import ( "context" "crypto/tls" "errors" "os" "strings" "sync" "testing" "time" "golang.org/x/crypto/acme" "github.com/carlosframework/platform/internal/certwarm" "github.com/carlosframework/platform/internal/release" ) // THE trap this whole file exists to avoid. autocert picks the key type from // the hello. A hello with no signature schemes, no curves and no cipher // suites reads as "no ECDSA", so autocert mints an RSA cert under // "+rsa" while every browser asks for "" — the order spent, the // name still uncovered, and the first real visitor triggering a second // issuance. // // This pins the warm's hello against the repo's own port of autocert's // chooser, so the two cannot drift. func TestWarmHelloAsksForAnECDSACert(t *testing.T) { if !helloSupportsECDSA(warmHello("app.example.com")) { t.Fatal("the warm's hello reads as RSA-only — autocert would mint +rsa and no browser would ever be served it") } // The bare hello is the bug, kept here so the test documents WHY the // cipher-suite entry above is load-bearing rather than decorative. if helloSupportsECDSA(&tls.ClientHelloInfo{ServerName: "app.example.com"}) { t.Fatal("an empty hello now reads as ECDSA-capable — warmHello's cipher suite may no longer be needed, re-derive the trap before deleting it") } } func TestWarmHelloCarriesTheServerName(t *testing.T) { if got := warmHello("App.Example.com").ServerName; got != "App.Example.com" { t.Fatalf("ServerName = %q", got) } } // The warm set is "exact hosts we serve that are not under the apps domain // and not wildcard-covered". Each exclusion below is a separate bug if it // regresses. func TestWarmSetSelectsCustomDomainsOnly(t *testing.T) { e := &Edge{} e.upstreams.Store(&map[string]upstream{ "www.customer.test": {}, "shop.customer.test": {}, "app.acct.oncarlos.test": {}, // platform host — apps domain "c1.app.acct.oncarlos.test": {}, // canary — apps domain, 3 labels "*.catchall.test": {}, // catch-all — wildcard covers it "one.go.tito.test": {}, // one label under a configured parent "deep.one.go.tito.test": {}, // TWO labels under it: NOT covered }) parents := func() []string { return []string{"go.tito.test"} } got := e.warmSet("oncarlos.test", parents) want := []string{"deep.one.go.tito.test", "shop.customer.test", "www.customer.test"} if strings.Join(got, ",") != strings.Join(want, ",") { t.Fatalf("warm set = %v, want %v", got, want) } } // A parent whose cert has not been obtained yet must STILL exclude its // children. Testing coverage by asking certFor (which returns nil until the // wildcard lands) would warm every app of a new account per-host during // exactly the window before their wildcard arrives — the burst the shape // test exists to prevent. func TestWarmSetExcludesChildrenOfAnUnobtainedParent(t *testing.T) { e := &Edge{} e.upstreams.Store(&map[string]upstream{"new.acct.test": {}}) // currentParents lists the parent; no cert exists for it anywhere. if got := e.warmSet("oncarlos.test", func() []string { return []string{"acct.test"} }); len(got) != 0 { t.Fatalf("warm set = %v; a configured parent covers its children whether or not its cert exists yet", got) } } func TestWarmSetWithNoWildcardManagerCoversNothing(t *testing.T) { e := &Edge{} e.upstreams.Store(&map[string]upstream{"www.customer.test": {}}) if got := e.warmSet("oncarlos.test", nil); len(got) != 1 || got[0] != "www.customer.test" { t.Fatalf("warm set = %v, want the custom domain", got) } } // ALL answers must be in the edge set. A name mid-migration resolves to the // old host AND ours; an any-match gate passes it, the order goes out, and // validation fails from the CA's vantage anyway — burning the shared // account's failed-validation budget, which is exactly what the gate is for. func TestDNSGateRequiresEveryAnswerToBeOurs(t *testing.T) { edges := func() []string { return []string{"203.0.113.7", "198.51.100.9"} } for name, tc := range map[string]struct { ips []string err error want bool }{ "all ours": {ips: []string{"203.0.113.7", "198.51.100.9"}, want: true}, "one of ours": {ips: []string{"203.0.113.7"}, want: true}, "partly elsewhere": {ips: []string{"203.0.113.7", "192.0.2.1"}, want: false}, "wholly elsewhere": {ips: []string{"192.0.2.1"}, want: false}, "no answer": {ips: nil, want: false}, "resolver failed": {err: errors.New("SERVFAIL"), want: false}, } { gate := dnsGate(func(string) ([]string, error) { return tc.ips, tc.err }, edges) if got := gate("www.customer.test"); got != tc.want { t.Errorf("%s: gate = %v, want %v", name, got, tc.want) } } } // An unreadable edge set is not evidence about anybody's DNS. Failing closed // costs one pass of delay; failing open spends a shared rate limit. func TestDNSGateFailsClosedOnAnEmptyEdgeSet(t *testing.T) { gate := dnsGate( func(string) ([]string, error) { return []string{"203.0.113.7"}, nil }, func() []string { return nil }, ) if gate("www.customer.test") { t.Fatal("an empty edge set opened the gate") } } // The heart of the design: a host that already has a cert is asked for AGAIN // every pass. autocert renews through in-memory timers that die on every // fleet roll and re-arm only via a GetCertificate that reaches m.cert — so a // skip-if-fresh check would silently stop renewing every custom domain whose // visitors never land on the issuer. func TestWarmReAsksForAlreadyWarmHostsEveryPass(t *testing.T) { var mu sync.Mutex calls := map[string]int{} w := newCertWarm( func() []string { return []string{"www.customer.test"} }, func(h *tls.ClientHelloInfo) (*tls.Certificate, error) { mu.Lock() defer mu.Unlock() calls[h.ServerName]++ return &tls.Certificate{}, nil }, func(string) bool { return true }, nil, nil, ) for i := 0; i < 3; i++ { w.pass() } if calls["www.customer.test"] != 3 { t.Fatalf("GetCertificate calls = %d, want one per pass — the renewal re-arm depends on it", calls["www.customer.test"]) } } // A renewal re-arm is NOT gated on DNS. It is a cache read that costs the CA // nothing, and gating it would strand exactly the domain whose resolver is // briefly sick — dropping its renewal on the floor for as long as the // resolver stays unhappy. func TestWarmRenewalReArmIgnoresTheDNSGate(t *testing.T) { calls := 0 pointing := true w := newCertWarm( func() []string { return []string{"www.customer.test"} }, func(*tls.ClientHelloInfo) (*tls.Certificate, error) { calls++; return &tls.Certificate{}, nil }, func(string) bool { return pointing }, nil, nil, ) w.pass() // first order: gate open, host becomes warmed pointing = false w.pass() // DNS now fails; the re-arm must still happen if calls != 2 { t.Fatalf("calls = %d, want 2 — a warmed host's re-arm is not DNS-gated", calls) } } // A never-warmed host IS DNS-gated: ordering for a half-pointed name spends // the shared account's failed-validation budget. func TestWarmDoesNotOrderForAHostThatDoesNotPointHere(t *testing.T) { calls := 0 w := newCertWarm( func() []string { return []string{"www.customer.test"} }, func(*tls.ClientHelloInfo) (*tls.Certificate, error) { calls++; return &tls.Certificate{}, nil }, func(string) bool { return false }, nil, nil, ) w.pass() if calls != 0 { t.Fatalf("calls = %d, want 0 — the DNS gate must hold the first order", calls) } } // A non-issuer does no work at all: unlike the wildcard manager there is no // cache to load first, because the warm serves nothing. func TestWarmStandsDownWhenNotTheIssuer(t *testing.T) { calls := 0 w := newCertWarm( func() []string { return []string{"www.customer.test"} }, func(*tls.ClientHelloInfo) (*tls.Certificate, error) { calls++; return &tls.Certificate{}, nil }, func(string) bool { return true }, func() (context.Context, bool) { return nil, false }, nil, ) if d := w.pass(); d != warmInterval { t.Fatalf("delay = %s", d) } if calls != 0 { t.Fatalf("a non-issuer placed %d call(s)", calls) } } // A rate limit must back off much harder than an ordinary failure: hourly // retries INTO a rate limit are what turn a transient limit into a sustained // one, and every host shares the one ACME account's budget. func TestWarmBacksOffHarderOnARateLimit(t *testing.T) { rateLimited := &acme.Error{ProblemType: "urn:ietf:params:acme:error:rateLimited"} w := newCertWarm(nil, nil, nil, nil, nil) w.fail(context.Background(), "a.test", rateLimited, certwarm.Record{}) w.fail(context.Background(), "b.test", errors.New("connection refused"), certwarm.Record{}) w.mu.Lock() defer w.mu.Unlock() limited, ordinary := time.Until(w.nextAttempt["a.test"]), time.Until(w.nextAttempt["b.test"]) if limited <= ordinary { t.Fatalf("rate-limited backoff %s is not harder than the ordinary %s", limited, ordinary) } } // A failing host is not retried until its backoff expires — and a healthy // host in the same set is not held up by it. func TestWarmHonoursPerHostBackoff(t *testing.T) { calls := map[string]int{} w := newCertWarm( func() []string { return []string{"bad.test", "good.test"} }, func(h *tls.ClientHelloInfo) (*tls.Certificate, error) { calls[h.ServerName]++ if h.ServerName == "bad.test" { return nil, errors.New("order failed") } return &tls.Certificate{}, nil }, func(string) bool { return true }, nil, nil, ) w.pass() w.pass() if calls["bad.test"] != 1 { t.Errorf("bad.test tried %d times, want 1 — backoff not honoured", calls["bad.test"]) } if calls["good.test"] != 2 { t.Errorf("good.test tried %d times, want 2 — a failing sibling must not stall it", calls["good.test"]) } } // New orders are capped per pass, and a capped pass comes back SOON so a // backlog drains in minutes rather than trickling out over days. Renewal // re-arms are never capped. func TestWarmCapsNewOrdersAndReturnsSoon(t *testing.T) { var hosts []string for i := 0; i < warmOrdersPerPass+5; i++ { hosts = append(hosts, string(rune('a'+i))+".customer.test") } calls := 0 w := newCertWarm( func() []string { return hosts }, func(*tls.ClientHelloInfo) (*tls.Certificate, error) { calls++; return &tls.Certificate{}, nil }, func(string) bool { return true }, nil, nil, ) d := w.pass() if calls != warmOrdersPerPass { t.Fatalf("placed %d orders, want the cap of %d", calls, warmOrdersPerPass) } if d != warmCappedDelay { t.Fatalf("capped pass returned %s, want the short %s so the backlog drains", d, warmCappedDelay) } } // ---- the durable outcome records (#299 §3e) --------------------------------- func warmRecordIn(t *testing.T, s release.Store, host string) (certwarm.Record, bool) { t.Helper() body, _, err := s.Get(context.Background(), certwarm.Key(host)) if errors.Is(err, os.ErrNotExist) { return certwarm.Record{}, false } if err != nil { t.Fatalf("reading the record for %s: %v", host, err) } rec, err := certwarm.Parse(body) if err != nil { t.Fatalf("parsing the record for %s: %v", host, err) } return rec, true } // THE point of §3e. Attempt counts used to live in process memory, so every // fleet roll and every failover restarted the clock on a domain that had // been broken for days — and the console could never say more than // "issuing-cert". The count and the "failing since" stamp must be carried // forward from the bucket, not from the heap. func TestWarmCarriesAttemptCountsAcrossAFailover(t *testing.T) { s := release.NewDirStore(t.TempDir()) newIssuer := func() *certWarm { return newCertWarm( func() []string { return []string{"bad.customer.test"} }, func(*tls.ClientHelloInfo) (*tls.Certificate, error) { return nil, errors.New("order failed") }, func(string) bool { return true }, nil, s, ) } first := newIssuer() first.pass() rec, ok := warmRecordIn(t, s, "bad.customer.test") if !ok || rec.Attempts != 1 { t.Fatalf("after one pass: record %+v ok=%v; want attempts 1", rec, ok) } failingSince := rec.FirstFailedAt // A brand-new process, with no memory of the first one: exactly what a // roll or a lease handover produces. for i := 2; i <= 4; i++ { w := newIssuer() w.pass() rec, ok = warmRecordIn(t, s, "bad.customer.test") if !ok || rec.Attempts != i { t.Fatalf("failover %d: record %+v ok=%v; want attempts %d — the count reset", i, rec, ok, i) } } if rec.FirstFailedAt != failingSince { t.Errorf("first_failed_at moved from %d to %d — \"failing since\" must survive a failover", failingSince, rec.FirstFailedAt) } } // A rate limit is carried as its own class, and with the twelve-hour retry // time, because those are the two facts that make it explicable rather than // just another failure. func TestWarmRecordsTheRateLimitClassAndItsNextAttempt(t *testing.T) { s := release.NewDirStore(t.TempDir()) w := newCertWarm( func() []string { return []string{"limited.customer.test"} }, func(*tls.ClientHelloInfo) (*tls.Certificate, error) { return nil, &acme.Error{ProblemType: "urn:ietf:params:acme:error:rateLimited"} }, func(string) bool { return true }, nil, s, ) w.pass() rec, ok := warmRecordIn(t, s, "limited.customer.test") if !ok { t.Fatal("no record written for a rate-limited host") } if rec.Class != certwarm.ClassRateLimited { t.Errorf("class = %q, want %q", rec.Class, certwarm.ClassRateLimited) } if until := time.Until(time.Unix(rec.NextAttemptAt, 0)); until < warmRateLimitBackoff-time.Minute { t.Errorf("next attempt in %s, want about %s", until, warmRateLimitBackoff) } } // A record exists only while a host is FAILING. Success removes it, which is // what makes absence the healthy case and keeps the console's fold one // object per broken host rather than one per domain. func TestWarmClearsTheRecordOnRecovery(t *testing.T) { s := release.NewDirStore(t.TempDir()) broken := true w := newCertWarm( func() []string { return []string{"flaky.customer.test"} }, func(*tls.ClientHelloInfo) (*tls.Certificate, error) { if broken { return nil, errors.New("order failed") } return &tls.Certificate{}, nil }, func(string) bool { return true }, nil, s, ) w.pass() if _, ok := warmRecordIn(t, s, "flaky.customer.test"); !ok { t.Fatal("no record after the failure") } broken = false // The per-host backoff is in memory, so a fresh issuer is the honest way // to reach the next attempt without sleeping an hour. recovered := newCertWarm(w.hosts, w.getCert, w.pointsHere, nil, s) recovered.pass() if rec, ok := warmRecordIn(t, s, "flaky.customer.test"); ok { t.Errorf("record %+v survived recovery — absence must be the healthy case", rec) } } // A detached domain leaves the warm set, and its record goes with it. // Deleting on an inference is forbidden for CERTIFICATES (§3f) because those // are the member's property; a record of our own attempts is ours to drop. func TestWarmPrunesRecordsForHostsThatLeftTheSet(t *testing.T) { s := release.NewDirStore(t.TempDir()) hosts := []string{"gone.customer.test", "stays.customer.test"} w := newCertWarm( func() []string { return hosts }, func(*tls.ClientHelloInfo) (*tls.Certificate, error) { return nil, errors.New("order failed") }, func(string) bool { return true }, nil, s, ) w.pass() for _, h := range hosts { if _, ok := warmRecordIn(t, s, h); !ok { t.Fatalf("no record for %s after the failure", h) } } hosts = []string{"stays.customer.test"} w.pass() if rec, ok := warmRecordIn(t, s, "gone.customer.test"); ok { t.Errorf("record %+v for a detached host survived the prune", rec) } if _, ok := warmRecordIn(t, s, "stays.customer.test"); !ok { t.Error("the prune took a record for a host still in the set") } } // The one prune case that would delete EVERY record is a warm set that // briefly folds to nothing — an edges-file read that came back empty, say. // pass() returns before any of this on an empty set, so it can never happen; // this pins that, because the guard is one `if` away from being refactored out. func TestWarmPruneNeverRunsOnAnEmptySet(t *testing.T) { s := release.NewDirStore(t.TempDir()) hosts := []string{"bad.customer.test"} w := newCertWarm( func() []string { return hosts }, func(*tls.ClientHelloInfo) (*tls.Certificate, error) { return nil, errors.New("order failed") }, func(string) bool { return true }, nil, s, ) w.pass() hosts = nil w.pass() if _, ok := warmRecordIn(t, s, "bad.customer.test"); !ok { t.Error("an empty warm set pruned every record — a transient fold to nothing must never delete") } } // A non-issuer writes nothing at all. Two boxes both recording their own // view of a host would be exactly the whole-file overwrite §3e's per-host // CAS exists to avoid, with the added wrinkle that the non-issuer's view is // not even real — it places no orders. func TestWarmStandDownWritesNoRecords(t *testing.T) { s := release.NewDirStore(t.TempDir()) w := newCertWarm( func() []string { return []string{"bad.customer.test"} }, func(*tls.ClientHelloInfo) (*tls.Certificate, error) { return nil, errors.New("order failed") }, func(string) bool { return true }, func() (context.Context, bool) { return nil, false }, s, ) w.pass() if _, ok := warmRecordIn(t, s, "bad.customer.test"); ok { t.Error("a stood-down box wrote an outcome record") } } // The records are observability, never a precondition. A bucket that cannot // be listed or written must not stop the warm from warming: the certificate // is the job, the record only explains it. func TestWarmKeepsWarmingWhenTheRecordStoreIsBroken(t *testing.T) { s := &failingStore{Store: release.NewDirStore(t.TempDir()), putErr: errors.New("bucket down")} calls := 0 w := newCertWarm( func() []string { return []string{"www.customer.test"} }, func(*tls.ClientHelloInfo) (*tls.Certificate, error) { calls++; return nil, errors.New("order failed") }, func(string) bool { return true }, nil, s, ) if d := w.pass(); d != warmInterval { t.Fatalf("delay = %s, want the ordinary %s", d, warmInterval) } if calls != 1 { t.Fatalf("GetCertificate calls = %d, want 1 — a broken record store stopped the warm", calls) } }