package edge import ( "bytes" "context" "errors" "io" "log" "os" "path/filepath" "slices" "strings" "sync" "testing" "time" "golang.org/x/crypto/acme/autocert" "github.com/carlosframework/platform/internal/release" ) // issuerMirror is a mirror over local+bucket whose box holds the issuer gate // when held is true. Refresh has run, so the snapshot reflects the bucket. func issuerMirror(t *testing.T, s release.Store, local autocert.Cache, held bool) *certMirror { t.Helper() m := newCertMirror(s, local) m.mayBackfill = func() bool { return true } // an agent-run box m.issuing = func() bool { return held } // The registry: every *.example.com name except "gone". m.allowed = func(host string) bool { return strings.HasSuffix(host, ".example.com") && !strings.HasPrefix(host, "gone.") } // The wildcard manager's parent set: acct only. m.wildcardParents = func() []string { return []string{"acct.example.com"} } if err := m.Refresh(context.Background()); err != nil { t.Fatal(err) } return m } func putLocal(t *testing.T, c autocert.Cache, name string, body []byte) { t.Helper() if err := c.Put(context.Background(), name, body); err != nil { t.Fatal(err) } } func bucketHas(t *testing.T, s release.Store, name string) ([]byte, bool) { t.Helper() b, err := certStoreGet(t, s, certKey(name)) if err != nil { return nil, false } return b, true } // The flagship roll's shape: a cache full of certs minted before publishing // existed, a bucket holding almost nothing. One pass and the bucket has // every live cert — and nothing that should not be there. func TestBackfillPublishesPreExistingCertsOnce(t *testing.T) { log.SetOutput(io.Discard) t.Cleanup(func() { log.SetOutput(os.Stderr) }) s := release.NewDirStore(t.TempDir()) local := autocert.DirCache(t.TempDir()) later := time.Now().Add(60 * 24 * time.Hour) live := hostCertPEM(t, "a.example.com", later) putLocal(t, local, "a.example.com", live) putLocal(t, local, "a.example.com+rsa", hostCertPEM(t, "a.example.com", later)) putLocal(t, local, "wildcard-acct.example.com.pem", hostCertPEM(t, "*.acct.example.com", later)) putLocal(t, local, "expired.example.com", hostCertPEM(t, "expired.example.com", time.Now().Add(-time.Hour))) putLocal(t, local, "garbage.example.com", []byte("not a cert")) putLocal(t, local, "a.example.com+http-01", []byte("token-body")) putLocal(t, local, "a.example.com+token", hostCertPEM(t, "a.example.com", later)) // Disk residue that is NOT the deployment's to replicate: a route // detached long ago, a DirCache temp file a kill left behind, a // wildcard PEM named for a parent its leaf does not cover. putLocal(t, local, "gone.example.com", hostCertPEM(t, "gone.example.com", later)) putLocal(t, local, "a.example.com1762384410", live) putLocal(t, local, "wildcard-other.example.com.pem", hostCertPEM(t, "*.acct.example.com", later)) putLocal(t, local, "wildcard-left.example.com.pem", hostCertPEM(t, "*.left.example.com", later)) // parent no longer current putLocal(t, local, "acme_account+key", []byte("LOCAL-ACCOUNT")) putLocal(t, local, wildcardAccountKeyName, []byte("LOCAL-WILDCARD-ACCOUNT")) m := issuerMirror(t, s, local, true) m.backfill(context.Background()) for _, name := range []string{"a.example.com", "a.example.com+rsa", "wildcard-acct.example.com.pem"} { if _, ok := bucketHas(t, s, name); !ok { t.Errorf("%s not backfilled", name) } } if got, _ := bucketHas(t, s, "a.example.com"); string(got) != string(live) { t.Errorf("a.example.com body differs from the local cache") } for _, name := range []string{"expired.example.com", "garbage.example.com", "a.example.com+http-01", "a.example.com+token", "gone.example.com", "a.example.com1762384410", "wildcard-other.example.com.pem", "wildcard-left.example.com.pem"} { if _, ok := bucketHas(t, s, name); ok { t.Errorf("%s must not be published", name) } } // Account keys: the bucket had none, so this box's become the // deployment's. for name, want := range map[string]string{"acme_account+key": "LOCAL-ACCOUNT", wildcardAccountKeyName: "LOCAL-WILDCARD-ACCOUNT"} { if got, ok := bucketHas(t, s, name); !ok || string(got) != want { t.Errorf("%s: got %q, %v; want seeded", name, got, ok) } } // Steady state: a second pass after a poll writes nothing. A fresh // store wrapper counts. if err := m.Refresh(context.Background()); err != nil { t.Fatal(err) } c := &putCountingStore{Store: s} m.s = c m.backfill(context.Background()) if c.puts != 0 { t.Fatalf("second pass published %d entries; want 0", c.puts) } } type putCountingStore struct { release.Store puts int } func (c *putCountingStore) Put(ctx context.Context, key string, body []byte, ifMatch string) (string, error) { c.puts++ return c.Store.Put(ctx, key, body, ifMatch) } // #318: certs publish from ANY agent-run box, lease or no lease — the // rung-1 rehearsal found the issuer-only rule left every cert minted on a // box that never held the lease sitting on that box's disk alone, exactly // the bytes a failover loses. Only the agent-run predicate bars a mirror: // a hand-run edge, and a mirror built without an Edge, still never publish. func TestBackfillRunsOnANonIssuer(t *testing.T) { log.SetOutput(io.Discard) t.Cleanup(func() { log.SetOutput(os.Stderr) }) s := release.NewDirStore(t.TempDir()) local := autocert.DirCache(t.TempDir()) putLocal(t, local, "a.example.com", hostCertPEM(t, "a.example.com", time.Now().Add(time.Hour))) m2 := newCertMirror(s, local) // mayBackfill nil: no Edge, never publishes m2.backfill(context.Background()) if _, ok := bucketHas(t, s, "a.example.com"); ok { t.Fatal("a gate-less mirror published") } m := issuerMirror(t, s, local, false) // agent-run, NOT holding the lease m.backfill(context.Background()) if _, ok := bucketHas(t, s, "a.example.com"); !ok { t.Fatal("an agent-run non-issuer did not publish its own cert") } // The Edge wiring: eligibility is issuer-gate present AND contending // in the election; a store-carrying edge with no agent (hand-run) may // never backfill, and neither may an agent-run box whose housekeeper // mode is "never" (the grandfathered fleet box). e := &Edge{staticStore: s} m3 := e.newCertMirror(local) if m3.mayBackfill() { t.Fatal("a hand-run edge may backfill") } if m3.issuing() { t.Fatal("an ungated edge counts as the issuer") } e.issuer = func() (context.Context, bool) { return context.Background(), false } if m3.mayBackfill() { t.Fatal("an agent-run box that does not contend may backfill (fleet box)") } e.contends = func() bool { return false } if m3.mayBackfill() { t.Fatal("a contends()==false box may backfill (fleet box)") } e.contends = func() bool { return true } if !m3.mayBackfill() { t.Fatal("an agent-run contender that does not hold the lease may not backfill") } if m3.issuing() { t.Fatal("a non-holding edge counts as the issuer") } e.issuer = func() (context.Context, bool) { return context.Background(), true } if !m3.issuing() { t.Fatal("a holding edge is not the issuer") } // And end-to-end through backfill itself: the hand-run shape (gate // present through serving-side wiring is irrelevant — no contender // predicate) walks nothing and publishes nothing. e2 := &Edge{staticStore: s} putLocal(t, local, "handrun.example.com", hostCertPEM(t, "handrun.example.com", time.Now().Add(time.Hour))) m4 := e2.newCertMirror(local) m4.allowed = func(string) bool { return true } m4.backfill(context.Background()) if _, ok := bucketHas(t, s, "handrun.example.com"); ok { t.Fatal("a hand-run edge published through backfill") } } // Seeding the deployment's ACME identity stays single-writer: a non-issuer // never offers its own key to an empty bucket — and the unseeded key is NOT // settled, so the same box publishes it on the first pass where it holds. func TestBackfillAccountKeySeedsOnlyFromTheIssuer(t *testing.T) { log.SetOutput(io.Discard) t.Cleanup(func() { log.SetOutput(os.Stderr) }) s := release.NewDirStore(t.TempDir()) local := autocert.DirCache(t.TempDir()) putLocal(t, local, "acme_account+key", []byte("MINE")) held := false m := issuerMirror(t, s, local, false) m.issuing = func() bool { return held } m.backfill(context.Background()) if _, ok := bucketHas(t, s, "acme_account+key"); ok { t.Fatal("a non-issuer seeded the account key") } held = true // the same process later wins the lease: same stamps, same memo m.backfill(context.Background()) if got, _ := bucketHas(t, s, "acme_account+key"); string(got) != "MINE" { t.Fatalf("the new issuer did not seed the account key it holds: %q", got) } } // #318's repair loop: another box clobbers the bucket with an OLDER leaf // after this box settled against the newer one. The stamp carries the // bucket copy's identity, so the settled box un-settles on its next poll // and republishes the newer cert — the clobber is transient, not durable. func TestBackfillRepairsAnOlderLeafClobber(t *testing.T) { var buf strings.Builder log.SetOutput(&buf) t.Cleanup(func() { log.SetOutput(os.Stderr) }) s := release.NewDirStore(t.TempDir()) local := autocert.DirCache(t.TempDir()) newer := hostCertPEM(t, "a.example.com", time.Now().Add(48*time.Hour)) older := hostCertPEM(t, "a.example.com", time.Now().Add(1*time.Hour)) putLocal(t, local, "a.example.com", newer) m := issuerMirror(t, s, local, false) m.backfill(context.Background()) if got, _ := bucketHas(t, s, "a.example.com"); !bytes.Equal(got, newer) { t.Fatal("setup: newer cert not published") } if err := m.Refresh(context.Background()); err != nil { t.Fatal(err) } m.backfill(context.Background()) // settles: disk == bucket == newer if !strings.Contains(buf.String(), "backfilled 1 cache entry") { t.Fatalf("the publishing pass did not report itself:\n%s", buf.String()) } // Another box lands the older leaf (its view was stale). putBucket(t, s, "a.example.com", older) if err := m.Refresh(context.Background()); err != nil { t.Fatal(err) } m.backfill(context.Background()) if got, _ := bucketHas(t, s, "a.example.com"); !bytes.Equal(got, newer) { t.Fatal("the settled box did not repair an older-leaf clobber") } } // A wildcard PEM on a non-issuer's disk publishes like any owned cert — // bounded by the manager's parent set exactly as on the issuer. func TestBackfillWildcardFromANonIssuer(t *testing.T) { log.SetOutput(io.Discard) t.Cleanup(func() { log.SetOutput(os.Stderr) }) s := release.NewDirStore(t.TempDir()) local := autocert.DirCache(t.TempDir()) putLocal(t, local, "wildcard-acct.example.com.pem", hostCertPEM(t, "*.acct.example.com", time.Now().Add(time.Hour))) putLocal(t, local, "wildcard-other.example.com.pem", hostCertPEM(t, "*.other.example.com", time.Now().Add(time.Hour))) m := issuerMirror(t, s, local, false) m.backfill(context.Background()) if _, ok := bucketHas(t, s, "wildcard-acct.example.com.pem"); !ok { t.Fatal("a non-issuer did not publish the wildcard PEM for a current parent") } if _, ok := bucketHas(t, s, "wildcard-other.example.com.pem"); ok { t.Fatal("a wildcard PEM outside the parent set published") } } // The other cert shapes ride the same freshness path as plain hosts. func TestBackfillWildcardAndRSAThroughTheFreshnessPath(t *testing.T) { log.SetOutput(io.Discard) t.Cleanup(func() { log.SetOutput(os.Stderr) }) s := release.NewDirStore(t.TempDir()) local := autocert.DirCache(t.TempDir()) soon, later := time.Now().Add(24*time.Hour), time.Now().Add(48*time.Hour) putBucket(t, s, "wildcard-acct.example.com.pem", hostCertPEM(t, "*.acct.example.com", soon)) newWild := hostCertPEM(t, "*.acct.example.com", later) putLocal(t, local, "wildcard-acct.example.com.pem", newWild) putBucket(t, s, "a.example.com+rsa", hostCertPEM(t, "a.example.com", soon)) newRSA := hostCertPEM(t, "a.example.com", later) putLocal(t, local, "a.example.com+rsa", newRSA) // A bucket object that does not parse is replaced by a live local one. putBucket(t, s, "b.example.com", []byte("corrupt")) goodB := hostCertPEM(t, "b.example.com", later) putLocal(t, local, "b.example.com", goodB) m := issuerMirror(t, s, local, true) m.backfill(context.Background()) for name, want := range map[string][]byte{"wildcard-acct.example.com.pem": newWild, "a.example.com+rsa": newRSA, "b.example.com": goodB} { if got, _ := bucketHas(t, s, name); string(got) != string(want) { t.Errorf("%s: bucket copy not brought forward", name) } } } type failingPutStore struct { release.Store mu sync.Mutex puts int } func (f *failingPutStore) Put(context.Context, string, []byte, string) (string, error) { f.mu.Lock() defer f.mu.Unlock() f.puts++ return "", errors.New("AccessDenied") } // A write-denied bucket retries every pass and says so on the file's // first-and-every-tenth cadence, never every 30 seconds. func TestBackfillFailureIsRetriedAndThrottled(t *testing.T) { var buf strings.Builder log.SetOutput(&buf) t.Cleanup(func() { log.SetOutput(os.Stderr) }) inner := release.NewDirStore(t.TempDir()) local := autocert.DirCache(t.TempDir()) putLocal(t, local, "a.example.com", hostCertPEM(t, "a.example.com", time.Now().Add(time.Hour))) m := issuerMirror(t, inner, local, true) f := &failingPutStore{Store: inner} m.s = f for i := 0; i < 12; i++ { m.backfill(context.Background()) } if f.puts != 12 { t.Fatalf("puts = %d; want one retry per pass", f.puts) } if n := strings.Count(buf.String(), "not published"); n != 2 { t.Fatalf("logged %d times over 12 failing passes; want 2 (#1, #10):\n%s", n, buf.String()) } } // Steady state does not re-read the disk: an entry whose stamp has not // moved since it matched the bucket is skipped on the listing alone. func TestBackfillSteadyStateDoesNotReread(t *testing.T) { log.SetOutput(io.Discard) t.Cleanup(func() { log.SetOutput(os.Stderr) }) s := release.NewDirStore(t.TempDir()) local := autocert.DirCache(t.TempDir()) putLocal(t, local, "a.example.com", hostCertPEM(t, "a.example.com", time.Now().Add(time.Hour))) m := issuerMirror(t, s, local, true) m.backfill(context.Background()) if err := m.Refresh(context.Background()); err != nil { t.Fatal(err) } // Residue the pass settles as not-owned must not be re-read either. putLocal(t, local, "gone.example.com", hostCertPEM(t, "gone.example.com", time.Now().Add(time.Hour))) m.backfill(context.Background()) if os.Getuid() == 0 { t.Skip("root ignores mode bits") } // Listing and stat still work; any read now fails loudly. for _, name := range []string{"a.example.com", "gone.example.com"} { if err := os.Chmod(filepath.Join(string(local), name), 0o000); err != nil { t.Fatal(err) } } var buf strings.Builder log.SetOutput(&buf) c := &putCountingStore{Store: s} m.s = c m.backfill(context.Background()) if c.puts != 0 { t.Fatalf("steady state published %d", c.puts) } if strings.Contains(buf.String(), "not published") { t.Fatalf("steady state re-read the disk:\n%s", buf.String()) } // And a moved stamp IS re-read: touch one and the read fails visibly. now := time.Now().Add(time.Second) if err := os.Chtimes(filepath.Join(string(local), "a.example.com"), now, now); err != nil { t.Fatal(err) } m.backfill(context.Background()) if !strings.Contains(buf.String(), "not published") { t.Fatal("a changed stamp was not re-read") } } // autocert writes the cache while the backfill walks it. Whatever the // interleaving, nothing but complete, owned entries reach the bucket. func TestBackfillRacesAConcurrentIssuance(t *testing.T) { log.SetOutput(io.Discard) t.Cleanup(func() { log.SetOutput(os.Stderr) }) s := release.NewDirStore(t.TempDir()) local := autocert.DirCache(t.TempDir()) later := time.Now().Add(time.Hour) pems := map[string][]byte{} for i := 0; i < 8; i++ { h := string(rune('a'+i)) + ".example.com" pems[h] = hostCertPEM(t, h, later) } m := issuerMirror(t, s, local, true) var wg sync.WaitGroup wg.Add(2) go func() { defer wg.Done() for h, pem := range pems { putLocal(t, local, h, pem) } }() go func() { defer wg.Done() for i := 0; i < 20; i++ { m.backfill(context.Background()) } }() wg.Wait() m.backfill(context.Background()) keys, _ := s.List(context.Background(), certPrefix) for _, k := range keys { name := strings.TrimPrefix(k, certPrefix) want, ok := pems[name] if !ok { t.Errorf("stray bucket key %s", k) continue } if got, _ := bucketHas(t, s, name); string(got) != string(want) { t.Errorf("%s: bucket body is not the cache's", name) } } if len(keys) != len(pems) { t.Fatalf("bucket has %d of %d", len(keys), len(pems)) } } // The renewed-before-the-gate case: the bucket's copy is older than the // disk's, so the disk wins; and the reverse — the bucket is fresher — leaves // the bucket alone (the mirror will bring the disk up instead). func TestBackfillReplacesOnlyAnOlderBucketCert(t *testing.T) { log.SetOutput(io.Discard) t.Cleanup(func() { log.SetOutput(os.Stderr) }) s := release.NewDirStore(t.TempDir()) local := autocert.DirCache(t.TempDir()) soon := time.Now().Add(24 * time.Hour) later := time.Now().Add(48 * time.Hour) oldPEM := hostCertPEM(t, "old.example.com", soon) newPEM := hostCertPEM(t, "old.example.com", later) putBucket(t, s, "old.example.com", oldPEM) putLocal(t, local, "old.example.com", newPEM) freshBucket := hostCertPEM(t, "fresh.example.com", later) putBucket(t, s, "fresh.example.com", freshBucket) putLocal(t, local, "fresh.example.com", hostCertPEM(t, "fresh.example.com", soon)) m := issuerMirror(t, s, local, true) m.backfill(context.Background()) if got, _ := bucketHas(t, s, "old.example.com"); string(got) != string(newPEM) { t.Error("an older bucket cert was not replaced by the disk's newer one") } if got, _ := bucketHas(t, s, "fresh.example.com"); string(got) != string(freshBucket) { t.Error("a fresher bucket cert was clobbered by an older local one") } } // The deployment's ACME identity is whichever key the bucket already // holds — a takeover issuer's own key must never displace it. func TestBackfillNeverReplacesABucketAccountKey(t *testing.T) { s := release.NewDirStore(t.TempDir()) local := autocert.DirCache(t.TempDir()) putBucket(t, s, "acme_account+key", []byte("DEPLOYMENT")) putBucket(t, s, wildcardAccountKeyName, []byte("DEPLOYMENT-WILDCARD")) // Refresh seeds the bucket keys locally only when absent; make the local // ones differ first. putLocal(t, local, "acme_account+key", []byte("MINE")) putLocal(t, local, wildcardAccountKeyName, []byte("MINE-WILDCARD")) m := issuerMirror(t, s, local, true) m.backfill(context.Background()) if got, _ := bucketHas(t, s, "acme_account+key"); string(got) != "DEPLOYMENT" { t.Fatalf("account key replaced: %q", got) } if got, _ := bucketHas(t, s, wildcardAccountKeyName); string(got) != "DEPLOYMENT-WILDCARD" { t.Fatalf("wildcard account key replaced: %q", got) } } // Only an on-disk cache can be enumerated; anything else is a quiet no-op, // never a panic in the poll loop. func TestBackfillNeedsADirCache(t *testing.T) { s := release.NewDirStore(t.TempDir()) m := issuerMirror(t, s, nil, true) m.backfill(context.Background()) m2 := issuerMirror(t, s, newCountingCache(t), true) m2.backfill(context.Background()) m3 := issuerMirror(t, s, autocert.DirCache(t.TempDir()+"/missing"), true) m3.backfill(context.Background()) if keys, _ := s.List(context.Background(), certPrefix); len(keys) != 0 { t.Fatalf("published %v", keys) } } // The poll loop runs the backfill after each successful refresh: an issuer // that boots with a full cache and an empty bucket fills it without waiting // for a renewal. func TestCertMirrorLoopBackfillsOnTheIssuer(t *testing.T) { prev := certMirrorInterval certMirrorInterval = 5 * time.Millisecond t.Cleanup(func() { certMirrorInterval = prev }) log.SetOutput(io.Discard) t.Cleanup(func() { log.SetOutput(os.Stderr) }) s := release.NewDirStore(t.TempDir()) local := autocert.DirCache(t.TempDir()) putLocal(t, local, "a.example.com", hostCertPEM(t, "a.example.com", time.Now().Add(time.Hour))) m := newCertMirror(s, local) m.mayBackfill = func() bool { return true } m.issuing = func() bool { return true } m.allowed = func(string) bool { return true } stop := make(chan struct{}) done := make(chan struct{}) go func() { m.run(stop); close(done) }() deadline := time.Now().Add(2 * time.Second) for { if _, ok := bucketHas(t, s, "a.example.com"); ok { break } if time.Now().After(deadline) { t.Fatal("the loop never backfilled") } time.Sleep(2 * time.Millisecond) } // And the mirror then serves what it published. deadline = time.Now().Add(2 * time.Second) for { if c, ok := m.Cert("a.example.com"); ok && c != nil { break } if time.Now().After(deadline) { t.Fatal("the mirror never served the backfilled cert") } time.Sleep(2 * time.Millisecond) } close(stop) <-done } // The bucket side is part of the stamp: an entry settled as in sync that // later leaves the bucket (a prune, a mis-delete) republishes on the next // pass even though the file has not moved — otherwise #291's failure mode // returns by a different door. func TestBackfillRepublishesAnEntryThatLeftTheBucket(t *testing.T) { log.SetOutput(io.Discard) t.Cleanup(func() { log.SetOutput(os.Stderr) }) s := release.NewDirStore(t.TempDir()) local := autocert.DirCache(t.TempDir()) putLocal(t, local, "a.example.com", hostCertPEM(t, "a.example.com", time.Now().Add(time.Hour))) ctx := context.Background() m := issuerMirror(t, s, local, true) m.backfill(ctx) if err := m.Refresh(ctx); err != nil { t.Fatal(err) } m.backfill(ctx) // settles in sync if err := s.Delete(ctx, certKey("a.example.com")); err != nil { t.Fatal(err) } if err := m.Refresh(ctx); err != nil { t.Fatal(err) } m.backfill(ctx) if _, ok := bucketHas(t, s, "a.example.com"); !ok { t.Fatal("an entry that left the bucket was never republished") } } // The production wiring: the wildcard manager's parent set reaches the // mirror, or every wildcard PEM would silently never backfill. func TestAttachWildcardToMirrorWiresParentsAndWake(t *testing.T) { w := &wildcardManager{parents: []string{"acct.example.com"}} m := newCertMirror(release.NewDirStore(t.TempDir()), nil) attachWildcardToMirror(m, w) if m.wildcardParents == nil || !slices.Equal(m.wildcardParents(), []string{"acct.example.com"}) { t.Fatal("wildcardParents not wired to the manager's current set") } if m.wake == nil { t.Fatal("wake not wired") } attachWildcardToMirror(nil, w) // no mirror: no panic attachWildcardToMirror(m, nil) // no manager: leaves the wiring alone if m.wildcardParents == nil { t.Fatal("a nil manager cleared the wiring") } }