package edge import ( "context" "errors" "os" "testing" "time" "github.com/carlosframework/platform/internal/release" ) func sweepWarm(t *testing.T, s release.Store) *certWarm { t.Helper() return newCertWarm(func() []string { return nil }, nil, nil, nil, s) } // seedCertEntry writes one cert-prefix object and backdates its stamp, which // is how an object that has been sitting in the bucket for months looks. func seedCertEntry(t *testing.T, dir string, s release.Store, name string, body []byte, age time.Duration) { t.Helper() if _, err := s.Put(context.Background(), certKey(name), body, ""); err != nil { t.Fatal(err) } when := time.Now().Add(-age) if err := os.Chtimes(dir+"/"+certKey(name), when, when); err != nil { t.Fatal(err) } } func certGone(t *testing.T, s release.Store, name string) bool { t.Helper() _, _, err := s.Get(context.Background(), certKey(name)) return errors.Is(err, os.ErrNotExist) } func expiredPEM(t *testing.T, host string, expiredAgo time.Duration) []byte { t.Helper() cert, key := selfSignedLeaf(t, host, time.Now().Add(-expiredAgo)) pem, err := encodeWildcardPEM(cert, key) if err != nil { t.Fatal(err) } return pem } // #243: a spent token whose gated delete never ran. Age is a complete // predicate — a validation's whole life is seconds. func TestSweepTakesSpentChallengeTokens(t *testing.T) { dir := t.TempDir() s := release.NewDirStore(dir) w := sweepWarm(t, s) seedCertEntry(t, dir, s, "abc123"+httpTokenSuffix, []byte("key-authorization"), 2*time.Hour) seedCertEntry(t, dir, s, "www.customer.test"+alpnTokenSuffix, []byte("whatever"), 2*time.Hour) // A token from a validation that may still be running must survive. seedCertEntry(t, dir, s, "fresh"+httpTokenSuffix, []byte("key-authorization"), time.Minute) w.sweep(context.Background(), time.Now()) if !certGone(t, s, "abc123"+httpTokenSuffix) { t.Error("a spent HTTP-01 token survived the sweep") } if !certGone(t, s, "www.customer.test"+alpnTokenSuffix) { t.Error("a spent tls-alpn-01 token cert survived the sweep") } if certGone(t, s, "fresh"+httpTokenSuffix) { t.Error("a token from a possibly-live validation was deleted — that burns the order") } } // §3f's whole ruling. The predicate is the leaf's own expiry and nothing // else: a cert for a host nobody serves any more goes ONLY once it is long // dead, and a current cert is untouchable however old its object is. func TestSweepDeletesOnlyLongExpiredCertificates(t *testing.T) { dir := t.TempDir() s := release.NewDirStore(dir) w := sweepWarm(t, s) old := certSweepParseAge + 24*time.Hour seedCertEntry(t, dir, s, "dead.customer.test", expiredPEM(t, "dead.customer.test", certSweepGrace+24*time.Hour), old) // Expired, but not long enough ago. Patience costs bytes; haste costs // an outage. seedCertEntry(t, dir, s, "recent.customer.test", expiredPEM(t, "recent.customer.test", time.Hour), old) // A CURRENT cert whose object happens to be ancient — a CA with long // certs, or a renewal that has simply not been needed. cert, key := selfSignedLeaf(t, "live.customer.test", time.Now().Add(365*24*time.Hour)) livePEM, err := encodeWildcardPEM(cert, key) if err != nil { t.Fatal(err) } seedCertEntry(t, dir, s, "live.customer.test", livePEM, old) w.sweep(context.Background(), time.Now()) if !certGone(t, s, "dead.customer.test") { t.Error("a certificate that expired a month ago survived") } if certGone(t, s, "recent.customer.test") { t.Error("a certificate that expired an hour ago was deleted — the grace period is not being applied") } if certGone(t, s, "live.customer.test") { t.Fatal("A CURRENT CERTIFICATE WAS DELETED. The predicate is expiry, never object age.") } } // The account keys are the issuer's identity at the CA. They are written // once and never rewritten, so they are ALWAYS older than the parse floor — // which is exactly why they are excluded by name before anything else is // considered. Deleting one costs the deployment its ACME registration. func TestSweepNeverTouchesAccountKeys(t *testing.T) { dir := t.TempDir() s := release.NewDirStore(dir) w := sweepWarm(t, s) ancient := 5 * 365 * 24 * time.Hour seedCertEntry(t, dir, s, autocertAccountKeyName, []byte("account key material"), ancient) seedCertEntry(t, dir, s, wildcardAccountKeyName, []byte("account key material"), ancient) w.sweep(context.Background(), time.Now()) for _, name := range []string{autocertAccountKeyName, wildcardAccountKeyName} { if certGone(t, s, name) { t.Fatalf("ACCOUNT KEY %s WAS DELETED — the deployment has lost its ACME registration", name) } } } // A sweep that deletes what it cannot understand is a sweep that deletes // tomorrow's format. Anything that will not read, will not parse, or has no // leaf is left exactly where it is. func TestSweepLeavesWhatItCannotUnderstand(t *testing.T) { dir := t.TempDir() s := release.NewDirStore(dir) w := sweepWarm(t, s) old := certSweepParseAge + 24*time.Hour seedCertEntry(t, dir, s, "future-shape.customer.test", []byte("a format this build has never seen"), old) w.sweep(context.Background(), time.Now()) if certGone(t, s, "future-shape.customer.test") { t.Error("an entry this build could not parse was deleted") } } // The pre-filter is what keeps this from being an N+1: nothing younger than // the parse floor is even read. A healthy deployment renews every ~60 days, // so the steady-state cost is one listing and zero reads. func TestSweepReadsNothingYoungerThanTheParseFloor(t *testing.T) { dir := t.TempDir() s := &bucketGetCounter{Store: release.NewDirStore(dir)} w := sweepWarm(t, s) for _, host := range []string{"a.customer.test", "b.customer.test", "c.customer.test"} { seedCertEntry(t, dir, s, host, expiredPEM(t, host, time.Hour), 10*24*time.Hour) } before := s.gets() w.sweep(context.Background(), time.Now()) if n := s.gets() - before; n != 0 { t.Errorf("sweep spent %d get(s) on entries younger than the parse floor, want 0", n) } } // The expiry half runs daily, not hourly: it is the half that costs reads, // and a certificate that has been dead for a month can wait a day. The token // half runs every pass, because it needs no reads at all. func TestSweepRunsTheExpiryHalfDailyAndTheTokenHalfEveryPass(t *testing.T) { dir := t.TempDir() s := release.NewDirStore(dir) w := sweepWarm(t, s) old := certSweepParseAge + 24*time.Hour now := time.Now() seedCertEntry(t, dir, s, "dead-one.customer.test", expiredPEM(t, "dead-one.customer.test", certSweepGrace+24*time.Hour), old) w.sweep(context.Background(), now) if !certGone(t, s, "dead-one.customer.test") { t.Fatal("the first sweep did not run the expiry half") } // An hour later, a second dead cert appears and a token goes stale. seedCertEntry(t, dir, s, "dead-two.customer.test", expiredPEM(t, "dead-two.customer.test", certSweepGrace+24*time.Hour), old) seedCertEntry(t, dir, s, "tok"+httpTokenSuffix, []byte("key-authorization"), 2*time.Hour) w.sweep(context.Background(), now.Add(time.Hour)) if certGone(t, s, "dead-two.customer.test") { t.Error("the expiry half ran again an hour later — it is meant to be daily") } if !certGone(t, s, "tok"+httpTokenSuffix) { t.Error("the token half did not run on the second pass — it is meant to run every pass") } // A day later it comes back round. w.sweep(context.Background(), now.Add(certSweepInterval+time.Minute)) if !certGone(t, s, "dead-two.customer.test") { t.Error("the expiry half did not run again after a day") } } // A store with no last-modified stamps gives the sweep no age to judge by, // and guessing an object's age from its contents is exactly the inference // this file refuses to make. It does nothing at all rather than something // approximate. func TestSweepStandsDownWithoutStamps(t *testing.T) { dir := t.TempDir() inner := release.NewDirStore(dir) w := sweepWarm(t, stamplessEdgeStore{inner}) seedCertEntry(t, dir, inner, "tok"+httpTokenSuffix, []byte("key-authorization"), 30*24*time.Hour) w.sweep(context.Background(), time.Now()) if certGone(t, inner, "tok"+httpTokenSuffix) { t.Error("a store with no stamps still had entries deleted") } } type stamplessEdgeStore struct{ inner release.Store } func (s stamplessEdgeStore) Get(ctx context.Context, key string) ([]byte, string, error) { return s.inner.Get(ctx, key) } func (s stamplessEdgeStore) Put(ctx context.Context, key string, body []byte, ifMatch string) (string, error) { return s.inner.Put(ctx, key, body, ifMatch) } func (s stamplessEdgeStore) List(ctx context.Context, prefix string) ([]string, error) { return s.inner.List(ctx, prefix) } func (s stamplessEdgeStore) ListDirs(ctx context.Context, prefix string) ([]string, error) { return s.inner.ListDirs(ctx, prefix) } func (s stamplessEdgeStore) Delete(ctx context.Context, key string) error { return s.inner.Delete(ctx, key) } // A bug in the predicate deletes at most this many objects before somebody // reads the log, and a genuine backlog drains over days rather than in one // burst against the bucket. func TestSweepBoundsHowMuchOnePassCanDelete(t *testing.T) { dir := t.TempDir() s := release.NewDirStore(dir) w := sweepWarm(t, s) total := certSweepDeletesPerPass + 5 for i := 0; i < total; i++ { seedCertEntry(t, dir, s, string(rune('a'+i%26))+string(rune('a'+i/26))+".tok"+httpTokenSuffix, []byte("key-authorization"), 2*time.Hour) } w.sweep(context.Background(), time.Now()) left, err := s.List(context.Background(), certPrefix) if err != nil { t.Fatal(err) } if len(left) != total-certSweepDeletesPerPass { t.Errorf("%d entries left after one pass, want %d — the per-pass bound is not holding", len(left), total-certSweepDeletesPerPass) } } // The sweep's work exists whether or not this deployment has custom // domains: spent challenge tokens and long-expired certificates are not the // warm SET's business. A pass over an empty set still sweeps — the gate check // deliberately comes before the empty-set return in pass(). func TestPassWithNoWarmSetStillSweeps(t *testing.T) { dir := t.TempDir() s := release.NewDirStore(dir) w := sweepWarm(t, s) seedCertEntry(t, dir, s, "orphan"+httpTokenSuffix, []byte("key-authorization"), 2*time.Hour) if d := w.pass(); d != warmInterval { t.Fatalf("pass over an empty set returned %s", d) } if !certGone(t, s, "orphan"+httpTokenSuffix) { t.Error("a deployment with no custom domains never sweeps its own token litter") } } // A non-issuer sweeps nothing. Deleting cert material is the sharpest thing // this package does, and a box that is not the issuer has no authority to // do it. func TestSweepNeverRunsOnANonIssuer(t *testing.T) { dir := t.TempDir() s := release.NewDirStore(dir) w := newCertWarm(func() []string { return nil }, nil, nil, func() (context.Context, bool) { return nil, false }, s) seedCertEntry(t, dir, s, "orphan"+httpTokenSuffix, []byte("key-authorization"), 2*time.Hour) w.pass() if certGone(t, s, "orphan"+httpTokenSuffix) { t.Error("a stood-down box deleted cert material") } } // A first-time backlog trips warmOrdersPerPass and pass() comes back every // minute until it clears. Housekeeping must not ride along sixty times an // hour: the token TTL is an hour and the expiry half is daily, so a listing // of the whole cert prefix per minute would be spent on nothing. func TestSweepDoesNotRunOnEveryCappedDrainPass(t *testing.T) { dir := t.TempDir() s := &listCounter{Store: release.NewDirStore(dir)} w := sweepWarm(t, s) now := time.Now() w.sweep(context.Background(), now) if s.lists != 1 { t.Fatalf("first sweep listed %d time(s), want 1", s.lists) } w.sweep(context.Background(), now.Add(time.Minute)) w.sweep(context.Background(), now.Add(2*time.Minute)) if s.lists != 1 { t.Errorf("sweep listed %d times across three passes a minute apart, want 1", s.lists) } w.sweep(context.Background(), now.Add(certSweepMinInterval+time.Second)) if s.lists != 2 { t.Errorf("sweep listed %d times, want 2 — an ordinary pass an hour later must sweep", s.lists) } } type listCounter struct { release.Store lists int } func (l *listCounter) ListMod(ctx context.Context, prefix string) ([]release.KeyMod, error) { l.lists++ return l.Store.(release.ModLister).ListMod(ctx, prefix) }