package edge // Task 7 of the multi-region ingress plan: the acceptance test the whole of // certsync.go exists for — ONE bucket, TWO edges, one certificate. // // Edge A holds the housekeeper gate and is the only box allowed to talk to a // CA. Edge B holds nothing, may never order, and yet has to (a) answer the // HTTP-01 challenge for an order A is running and (b) serve the certificate A // minted. Everything between the two is the control bucket's control/certs/ // prefix and nothing else: no RPC, no gossip, no second writer. // // The CA here is a fake one (fakeACME below) rather than a mock of the ACME // client, because the property under test is not "autocert was called" but // "an independent validator, reaching a DIFFERENT box than the one that // ordered, got the right key authorization and issued a certificate that a // real TLS client will accept". A mock cannot fail that; a validator can. // It is deliberately pointed at edge B's :80 handler and NEVER at edge A's — // A's handler is served too, purely so the test can assert it received // nothing. // // Conventions borrowed from the neighbouring harnesses (tunnel_e2e_test.go, // fleet_e2e_test.go): real loopback listeners, in-process components wired // the way edge.go's RunWithActivatorReady wires them, and bounds asserted // rather than narrated. Helpers already in this package (testIssuerEdge, // ecdsaHello, offlineManager, hostCertPEM, putBucket, localCert) are reused // rather than re-cut. import ( "bytes" "context" "crypto/ecdsa" "crypto/elliptic" "crypto/rand" "crypto/tls" "crypto/x509" "crypto/x509/pkix" "encoding/base64" "encoding/json" "encoding/pem" "errors" "fmt" "io" "log" "math/big" "net" "net/http" "net/http/httptest" "os" "slices" "strings" "sync" "sync/atomic" "testing" "time" "golang.org/x/crypto/acme" "golang.org/x/crypto/acme/autocert" "github.com/carlosframework/platform/internal/release" ) const ( // e2eHost is the name edge A mints for and edge B serves. A .test TLD, so // nothing here can resolve or be reached for real. e2eHost = "app.bab.example.test" // e2eUnmintedHost is in both edges' registries but is never issued: it is // what the gated twin is asked to mint, so the refusal is the GATE's and // not "that host isn't mine". e2eUnmintedHost = "other.bab.example.test" // e2eLocalOnlyHost exists only in edge B's own on-disk cache — yesterday's // material, from before the election moved. Nothing publishes it. e2eLocalOnlyHost = "old.bab.example.test" ) // ---- the fake ACME CA -------------------------------------------------------- // fakeACME is a minimal RFC 8555 CA: directory, nonce, account, order, one // http-01 authorization, finalize, certificate. It is NOT a security model — // it never verifies a JWS signature and never checks a nonce, because neither // is under test. What it DOES check is the only thing that matters here: that // the HTTP-01 response body served back to it equals token + "." + // JWK-thumbprint of the account key that placed the order. That check is what // makes "edge B answered edge A's challenge" a proof rather than a narration. // // It offers ONLY an http-01 challenge. autocert prefers tls-alpn-01 and falls // back to http-01 after a whole failed order (autocert.go's // AuthorizeOrderLoop); offering one type keeps the run to a single order, so // the order counter below is an exact count of "who talked to a CA". type fakeACME struct { srv *httptest.Server rootKey *ecdsa.PrivateKey root *x509.Certificate mu sync.Mutex // validateVia is the base URL the CA fetches HTTP-01 challenges from — // set to edge B's :80 server, never edge A's. validateVia string // thumb is the account key's JWK thumbprint, learned from the JWS // protected header of the new-account request. thumb string // accounts and orders are the counters the test asserts on: "B started // zero ACME orders" is this number not moving. accounts int orders int seq int byID map[string]*fakeACMEOrder // validations records every challenge fetch the CA made, in order. validations []fakeACMEValidation // faults collects anything the CA found wrong. Never t.Fatalf: these // handlers run on the server's goroutines and, for deactivatePendingAuthz, // can outlive the test. faults []string } type fakeACMEOrder struct { domain string token string authz string // pending | valid | invalid status string // pending | ready | valid chain []byte // PEM, once finalized } // fakeACMEValidation is one HTTP-01 fetch: where it went, what Host it // carried, the order's token, and whether the body was the right key // authorization. type fakeACMEValidation struct { url string host string token string ok bool } func newFakeACME(t *testing.T) *fakeACME { t.Helper() key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) if err != nil { t.Fatal(err) } tmpl := &x509.Certificate{ SerialNumber: big.NewInt(1), Subject: pkix.Name{CommonName: "CARLOS test CA"}, NotBefore: time.Now().Add(-time.Hour), NotAfter: time.Now().Add(365 * 24 * time.Hour), KeyUsage: x509.KeyUsageCertSign, BasicConstraintsValid: true, IsCA: true, } der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key) if err != nil { t.Fatal(err) } root, err := x509.ParseCertificate(der) if err != nil { t.Fatal(err) } ca := &fakeACME{rootKey: key, root: root, byID: map[string]*fakeACMEOrder{}} mux := http.NewServeMux() mux.HandleFunc("/directory", ca.serveDirectory) mux.HandleFunc("/nonce", func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) }) mux.HandleFunc("/new-account", ca.serveNewAccount) mux.HandleFunc("/new-order", ca.serveNewOrder) mux.HandleFunc("/authz/", ca.serveAuthorization) mux.HandleFunc("/chal/", ca.serveChallengeAccept) mux.HandleFunc("/order/", ca.serveOrder) mux.HandleFunc("/finalize/", ca.serveFinalize) mux.HandleFunc("/cert/", ca.serveCert) // Every ACME response carries a fresh Replay-Nonce; the client will not // make a second POST without one. ca.srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Replay-Nonce", ca.nextNonce()) mux.ServeHTTP(w, r) })) t.Cleanup(ca.srv.Close) return ca } func (ca *fakeACME) directoryURL() string { return ca.srv.URL + "/directory" } func (ca *fakeACME) url(path string) string { return ca.srv.URL + path } // validateAgainst points the CA's HTTP-01 validator at one edge's :80 server. func (ca *fakeACME) validateAgainst(base string) { ca.mu.Lock() defer ca.mu.Unlock() ca.validateVia = base } func (ca *fakeACME) nextNonce() string { ca.mu.Lock() defer ca.mu.Unlock() ca.seq++ return fmt.Sprintf("nonce-%d", ca.seq) } func (ca *fakeACME) counts() (accounts, orders int) { ca.mu.Lock() defer ca.mu.Unlock() return ca.accounts, ca.orders } func (ca *fakeACME) lastValidation() (fakeACMEValidation, bool) { ca.mu.Lock() defer ca.mu.Unlock() if len(ca.validations) == 0 { return fakeACMEValidation{}, false } return ca.validations[len(ca.validations)-1], true } func (ca *fakeACME) recordFault(format string, args ...any) { ca.mu.Lock() defer ca.mu.Unlock() ca.faults = append(ca.faults, fmt.Sprintf(format, args...)) } // checkFaults fails the test with anything the CA rejected along the way — a // malformed JWS, a CSR for the wrong name, an unknown order. Called at the end // of each phase so a subtle protocol mistake surfaces as itself rather than as // a timeout. func (ca *fakeACME) checkFaults(t *testing.T) { t.Helper() ca.mu.Lock() faults := append([]string(nil), ca.faults...) ca.mu.Unlock() for _, f := range faults { t.Errorf("fake CA: %s", f) } } func (ca *fakeACME) rootPool() *x509.CertPool { pool := x509.NewCertPool() pool.AddCert(ca.root) return pool } func (ca *fakeACME) serveDirectory(w http.ResponseWriter, _ *http.Request) { writeJSON(w, http.StatusOK, map[string]string{ "newNonce": ca.url("/nonce"), "newAccount": ca.url("/new-account"), "newOrder": ca.url("/new-order"), "revokeCert": ca.url("/revoke"), "keyChange": ca.url("/key-change"), }) } func (ca *fakeACME) serveNewAccount(w http.ResponseWriter, r *http.Request) { protected, _, err := decodeJWS(r) if err != nil { ca.recordFault("new-account: %v", err) http.Error(w, "bad jws", http.StatusBadRequest) return } raw, ok := protected["jwk"] if !ok { ca.recordFault("new-account: no jwk in the protected header (a KID-form registration?)") http.Error(w, "no jwk", http.StatusBadRequest) return } thumb, err := jwkThumbprintFromJSON(raw) if err != nil { ca.recordFault("new-account: %v", err) http.Error(w, "bad jwk", http.StatusBadRequest) return } ca.mu.Lock() ca.accounts++ n := ca.accounts ca.thumb = thumb ca.mu.Unlock() w.Header().Set("Location", ca.url(fmt.Sprintf("/acct/%d", n))) writeJSON(w, http.StatusCreated, map[string]any{"status": "valid"}) } func (ca *fakeACME) serveNewOrder(w http.ResponseWriter, r *http.Request) { _, payload, err := decodeJWS(r) if err != nil { ca.recordFault("new-order: %v", err) http.Error(w, "bad jws", http.StatusBadRequest) return } var req struct { Identifiers []struct{ Type, Value string } `json:"identifiers"` } if err := json.Unmarshal(payload, &req); err != nil || len(req.Identifiers) != 1 { ca.recordFault("new-order: want exactly one identifier, got %q (%v)", payload, err) http.Error(w, "bad order", http.StatusBadRequest) return } tokenBytes := make([]byte, 32) if _, err := rand.Read(tokenBytes); err != nil { ca.recordFault("new-order: %v", err) http.Error(w, "no entropy", http.StatusInternalServerError) return } ca.mu.Lock() ca.orders++ id := fmt.Sprintf("%d", ca.orders) ca.byID[id] = &fakeACMEOrder{ domain: req.Identifiers[0].Value, // base64url of 32 bytes: 43 chars, which is what // validChallengeToken's entropy floor is calibrated against. token: base64.RawURLEncoding.EncodeToString(tokenBytes), authz: "pending", status: "pending", } ca.mu.Unlock() w.Header().Set("Location", ca.url("/order/"+id)) writeJSON(w, http.StatusCreated, ca.orderJSON(id)) } // orderJSON renders an order in the shape acme.responseOrder parses. Caller // must not hold ca.mu. func (ca *fakeACME) orderJSON(id string) map[string]any { ca.mu.Lock() defer ca.mu.Unlock() o := ca.byID[id] if o == nil { return map[string]any{"status": "invalid"} } body := map[string]any{ "status": o.status, "identifiers": []map[string]string{{"type": "dns", "value": o.domain}}, "authorizations": []string{ca.url("/authz/" + id)}, "finalize": ca.url("/finalize/" + id), } if o.status == "valid" { body["certificate"] = ca.url("/cert/" + id) } return body } func writeJSON(w http.ResponseWriter, code int, body any) { w.Header().Set("Content-Type", "application/json") w.WriteHeader(code) _ = json.NewEncoder(w).Encode(body) } // ---- JWS / JWK ----------------------------------------------------------- // decodeJWS pulls the protected header and the payload out of a flattened JWS. // The signature is deliberately not verified — see fakeACME's doc. func decodeJWS(r *http.Request) (map[string]json.RawMessage, []byte, error) { raw, err := io.ReadAll(io.LimitReader(r.Body, 1<<20)) if err != nil { return nil, nil, fmt.Errorf("read body: %w", err) } var msg struct { Protected string `json:"protected"` Payload string `json:"payload"` } if err := json.Unmarshal(raw, &msg); err != nil { return nil, nil, fmt.Errorf("not a JWS: %w", err) } phead, err := base64.RawURLEncoding.DecodeString(msg.Protected) if err != nil { return nil, nil, fmt.Errorf("protected header: %w", err) } var protected map[string]json.RawMessage if err := json.Unmarshal(phead, &protected); err != nil { return nil, nil, fmt.Errorf("protected header JSON: %w", err) } // POST-as-GET carries a zero-length payload. payload, err := base64.RawURLEncoding.DecodeString(msg.Payload) if err != nil { return nil, nil, fmt.Errorf("payload: %w", err) } return protected, payload, nil } // jwkThumbprintFromJSON reconstructs the P-256 public key from a JWK and hands // it to acme.JWKThumbprint — the library's OWN canonicalization, so the // expected key authorization is computed the way a real CA computes it rather // than by hashing whatever bytes happened to be on the wire. func jwkThumbprintFromJSON(raw json.RawMessage) (string, error) { var jwk struct{ Crv, Kty, X, Y string } if err := json.Unmarshal(raw, &jwk); err != nil { return "", fmt.Errorf("jwk JSON: %w", err) } if jwk.Kty != "EC" || jwk.Crv != "P-256" { return "", fmt.Errorf("jwk kty/crv = %q/%q, want EC/P-256 (autocert's account key)", jwk.Kty, jwk.Crv) } x, err := base64.RawURLEncoding.DecodeString(jwk.X) if err != nil { return "", fmt.Errorf("jwk x: %w", err) } y, err := base64.RawURLEncoding.DecodeString(jwk.Y) if err != nil { return "", fmt.Errorf("jwk y: %w", err) } return acme.JWKThumbprint(&ecdsa.PublicKey{ Curve: elliptic.P256(), X: new(big.Int).SetBytes(x), Y: new(big.Int).SetBytes(y), }) } // ---- two edges ----------------------------------------------------------- // e2eEdge is one box, wired exactly as edge.go's RunWithActivatorReady wires // it: the raw local cache kept aside, the publishing wrapper over it, the // mirror built from the RAW handle, autocert's manager over the wrapper, and // the two chains (handshake and :80) composed in the same order. type e2eEdge struct { edge *Edge store *e2eRecordingStore local autocert.Cache mirror *certMirror mgr *autocert.Manager // getCert is the full handshake chain: mirror in front of autocert, with // the direct local-cache last resort behind it. getCert func(*tls.ClientHelloInfo) (*tls.Certificate, error) // httpHandler is the full :80 ACME chain: the mirror's challenge handler // in front of autocert's HTTPHandler. httpHandler http.Handler tlsConfig *tls.Config // hits counts requests that reached this box's :80 chain. hits atomic.Int64 } // newE2EEdge builds one edge over its OWN handle on the shared bucket // directory — two boxes, two clients, one bucket, which is what makes the // disjointness assertion in the P6 subtest meaningful. func newE2EEdge(t *testing.T, bucketDir string, holds bool, directoryURL string, hosts ...string) *e2eEdge { t.Helper() b := &e2eEdge{store: &e2eRecordingStore{Store: release.NewDirStore(bucketDir)}} b.edge = testIssuerEdge(hosts...) b.edge.staticStore = b.store b.edge.issuer = func() (context.Context, bool) { if !holds { return nil, false } return context.Background(), true } // The RAW cache, kept before the publishing wrapper goes on (edge.go). b.local = autocert.DirCache(t.TempDir()) cache := b.edge.armCertIssuance(b.local) if _, wrapped := cache.(*publishingCache); !wrapped { t.Fatal("a store-backed edge must publish every issuance artifact") } b.mirror = b.edge.newCertMirror(b.local) if b.mirror == nil { t.Fatal("a store-backed edge must have a mirror to serve from") } b.mgr = &autocert.Manager{ Prompt: autocert.AcceptTOS, Cache: cache, HostPolicy: b.edge.issuerHostPolicy(), Client: &acme.Client{DirectoryURL: directoryURL}, } b.tlsConfig = b.mgr.TLSConfig() b.getCert = b.mirror.wrapGetCertificate(b.tlsConfig.GetCertificate) b.tlsConfig.GetCertificate = b.getCert // HTTPHandler is what sets autocert's tryHTTP01, so it has to be built // before any order starts — exactly as edge.go builds it before binding. fallback := http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusTeapot) }) chain := b.mirror.challengeHandler(b.mgr.HTTPHandler(fallback)) b.httpHandler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { b.hits.Add(1) chain.ServeHTTP(w, r) }) return b } // serveHTTP puts this edge's :80 chain on a loopback listener and returns its // base URL — what the CA is (or, for edge A, deliberately is not) pointed at. func (b *e2eEdge) serveHTTP(t *testing.T) string { t.Helper() srv := httptest.NewServer(b.httpHandler) t.Cleanup(srv.Close) return srv.URL } // serveTLS puts this edge's handshake chain on a loopback listener. Raw // tls.Server rather than an HTTP server: the assertion is about the // certificate the handshake produces, and nothing above it. func (b *e2eEdge) serveTLS(t *testing.T) net.Listener { t.Helper() ln, err := net.Listen("tcp", "127.0.0.1:0") if err != nil { t.Fatal(err) } t.Cleanup(func() { _ = ln.Close() }) cfg := b.tlsConfig go func() { for { conn, err := ln.Accept() if err != nil { return } go func() { tc := tls.Server(conn, cfg) _ = tc.HandshakeContext(context.Background()) _ = tc.Close() }() } }() return ln } // e2eDial completes one handshake against ln with SNI name, verifying the // chain against the fake CA's root — a real client's verification, not a // peek at the certificate the server chose. func e2eDial(ln net.Listener, name string, roots *x509.CertPool) (*tls.ConnectionState, error) { conn, err := tls.Dial("tcp", ln.Addr().String(), &tls.Config{ ServerName: name, RootCAs: roots, MinVersion: tls.VersionTLS12, }) if err != nil { return nil, err } defer conn.Close() state := conn.ConnectionState() return &state, nil } // e2eRecordingStore is one box's handle on the bucket, remembering every key // it read and every key it wrote. Reads bound what the mirror touches (P6); // writes are how "a gated box publishes nothing" is asserted without trusting // the gate's own bookkeeping. type e2eRecordingStore struct { release.Store mu sync.Mutex reads []string wrote []string } func (s *e2eRecordingStore) Get(ctx context.Context, key string) ([]byte, string, error) { s.mu.Lock() s.reads = append(s.reads, key) s.mu.Unlock() return s.Store.Get(ctx, key) } func (s *e2eRecordingStore) List(ctx context.Context, prefix string) ([]string, error) { s.mu.Lock() s.reads = append(s.reads, prefix) s.mu.Unlock() return s.Store.List(ctx, prefix) } func (s *e2eRecordingStore) Put(ctx context.Context, key string, body []byte, ifMatch string) (string, error) { s.mu.Lock() s.wrote = append(s.wrote, key) s.mu.Unlock() return s.Store.Put(ctx, key, body, ifMatch) } func (s *e2eRecordingStore) Delete(ctx context.Context, key string) error { s.mu.Lock() s.wrote = append(s.wrote, key) s.mu.Unlock() return s.Store.Delete(ctx, key) } func (s *e2eRecordingStore) snapshot() (reads, wrote []string) { s.mu.Lock() defer s.mu.Unlock() return append([]string(nil), s.reads...), append([]string(nil), s.wrote...) } // e2eQuietLogs sends the edge's own log lines to the void. The chain under // test is deliberately chatty (publishing armed, mirror armed, serving from // the local cache); the test's output is the assertions. func e2eQuietLogs(t *testing.T) { t.Helper() log.SetOutput(io.Discard) t.Cleanup(func() { log.SetOutput(os.Stderr) }) } // ---- P1/P2/P3/P6: the two-edge scenario -------------------------------------- func TestCertSyncE2EMintOnOneServeOnTheOther(t *testing.T) { e2eQuietLogs(t) bucket := t.TempDir() ca := newFakeACME(t) // Edge B first: the CA has to be pointed at it before edge A places an // order, because B is the box that will answer the challenge. edgeB := newE2EEdge(t, bucket, false, ca.directoryURL(), e2eHost, e2eUnmintedHost, e2eLocalOnlyHost) ca.validateAgainst(edgeB.serveHTTP(t)) // Edge A: the housekeeper. Its :80 chain is served too, so "the CA never // reached the issuer" is an assertion and not an assumption. edgeA := newE2EEdge(t, bucket, true, ca.directoryURL(), e2eHost, e2eUnmintedHost) edgeA.serveHTTP(t) // B's own on-disk material from before the election moved: nothing // publishes this, so only the direct local-cache last resort can serve it. if err := edgeB.local.Put(context.Background(), e2eLocalOnlyHost, hostCertPEM(t, e2eLocalOnlyHost, time.Now().Add(60*24*time.Hour))); err != nil { t.Fatal(err) } lnB := edgeB.serveTLS(t) // Before anything is minted, B has nothing for e2eHost anywhere: empty // mirror, empty local cache, and autocert gated shut. The handshake must // fail — otherwise the post-mint success below would prove nothing. if _, err := e2eDial(lnB, e2eHost, ca.rootPool()); err == nil { t.Fatal("edge B answered a handshake for a certificate that does not exist yet") } // ---- the mint, on edge A only ---- minted, err := edgeA.getCert(ecdsaHello(e2eHost)) if err != nil { t.Fatalf("the issuer failed to mint %s: %v", e2eHost, err) } ca.checkFaults(t) if minted.Leaf == nil || minted.Leaf.VerifyHostname(e2eHost) != nil { t.Fatalf("minted cert does not cover %s", e2eHost) } if err := minted.Leaf.CheckSignatureFrom(ca.root); err != nil { t.Fatalf("minted cert was not issued by the CA under test: %v", err) } t.Run("P1 the challenge was answered by the other edge", func(t *testing.T) { v, ok := ca.lastValidation() if !ok { t.Fatal("the CA never validated anything — the order did not go through HTTP-01") } if !v.ok { t.Fatalf("validation failed: the body served for %s was not the key authorization", v.host) } if v.host != e2eHost { t.Fatalf("validated Host = %q, want %q", v.host, e2eHost) } if n := edgeB.hits.Load(); n == 0 { t.Fatal("edge B's :80 chain was never reached — the challenge was not answered off-issuer") } if n := edgeA.hits.Load(); n != 0 { t.Fatalf("edge A's :80 chain served %d request(s) — the CA must never have reached the issuer", n) } // The answer could only have come out of the bucket: B never ran an // order, so it holds no token in memory, and its own autocert // HTTPHandler 403s every challenge while gated. if err := edgeB.edge.issuerHostPolicy()(context.Background(), e2eHost); !errors.Is(err, errNotIssuer) { t.Fatalf("edge B's HostPolicy = %v, want errNotIssuer — its own handler must be shut", err) } // And it was a LIVE read, not a poll: B has never refreshed, so its // snapshot is still empty — challengeResponse's read-through is the // only path that could have produced the body, and the exact key it // read is the order's own token entry. if n := len(edgeB.mirror.snapshot().entries); n != 0 { t.Fatalf("edge B's mirror holds %d entry/entries already — the read-through is not what was tested", n) } want := certKey(v.token + httpTokenSuffix) reads, _ := edgeB.store.snapshot() if !slices.Contains(reads, want) { t.Fatalf("edge B read %v, want a live read of %q", reads, want) } }) accountsAfterMint, ordersAfterMint := ca.counts() if accountsAfterMint != 1 || ordersAfterMint != 1 { t.Fatalf("CA saw %d account(s) and %d order(s) after one mint, want 1 and 1", accountsAfterMint, ordersAfterMint) } t.Run("P2 the other edge serves the issuer's cert after one refresh", func(t *testing.T) { if err := edgeB.mirror.Refresh(context.Background()); err != nil { t.Fatalf("one refresh must be enough: %v", err) } // The in-memory mirror map holds it — the half that serves a box whose // own disk is empty. if c := edgeB.mirror.certificate(e2eHost, false); c == nil { t.Fatal("the mirror map has no certificate for the host the issuer minted") } else if !bytes.Equal(c.Leaf.Raw, minted.Leaf.Raw) { t.Fatal("the mirror holds a DIFFERENT certificate than the one the issuer minted") } state, err := e2eDial(lnB, e2eHost, ca.rootPool()) if err != nil { t.Fatalf("edge B must serve the issuer's certificate: %v", err) } if len(state.PeerCertificates) == 0 { t.Fatal("no peer certificate") } if !bytes.Equal(state.PeerCertificates[0].Raw, minted.Leaf.Raw) { t.Fatal("edge B served a certificate that is not the one edge A minted") } if got := state.ServerName; got != e2eHost { t.Fatalf("SNI = %q, want %q", got, e2eHost) } accounts, orders := ca.counts() if accounts != accountsAfterMint || orders != ordersAfterMint { t.Fatalf("edge B talked to the CA: accounts %d→%d, orders %d→%d — it must start zero", accountsAfterMint, accounts, ordersAfterMint, orders) } _, wrote := edgeB.store.snapshot() if len(wrote) != 0 { t.Fatalf("a gated edge wrote %v to the control bucket — only the issuer may", wrote) } }) t.Run("P3 the gated twin refuses to mint and still serves", func(t *testing.T) { // Edge B is edge A's twin in every respect but the gate: same // construction, same store, same CA client, HTTP-01 armed. Asked for a // host nobody has minted, it must refuse as the ISSUER gate and start // no order at all. _, ordersBefore := ca.counts() _, err := edgeB.getCert(ecdsaHello(e2eUnmintedHost)) if !errors.Is(err, errNotIssuer) { t.Fatalf("gated twin minting %s = %v, want errNotIssuer", e2eUnmintedHost, err) } if _, orders := ca.counts(); orders != ordersBefore { t.Fatalf("the gated twin started %d order(s)", orders-ordersBefore) } // ...while still serving what is on its OWN disk. autocert refuses the // name before it ever looks there (HostPolicy precedes cacheGet), so // this can only be the chain's direct local-cache last resort. got, err := edgeB.getCert(ecdsaHello(e2eLocalOnlyHost)) if err != nil { t.Fatalf("a gated box must keep serving its own cached material: %v", err) } if got.Leaf == nil || got.Leaf.VerifyHostname(e2eLocalOnlyHost) != nil { t.Fatalf("served cert does not cover %s", e2eLocalOnlyHost) } if _, ok := edgeB.mirror.Cert(e2eLocalOnlyHost); ok { t.Fatal("test bug: that host was supposed to exist only on B's disk") } }) t.Run("P6 the mirror reads its own handle, never the fleet door", func(t *testing.T) { // Structural: the mirror holds the edge's own deployment-store handle. if edgeB.mirror.s != release.Store(edgeB.store) { t.Fatal("the mirror is not reading through this box's own store handle") } if edgeB.store == edgeA.store { t.Fatal("test bug: the two edges must hold separate handles on the one bucket") } // And the door could not have served those reads even if something // tried: the fleet ACL (storedoor.go) refuses every key under // control/certs/ for every tenant and fleet, so a fleet box reaching // for cert material over the door gets the byte-identical 404. for _, key := range []string{certKey(e2eHost), certKey("acme_account+key"), certPrefix} { if aclRead("bdk", "pi-fleet", key) { t.Fatalf("the fleet door ACL allows %q — cert material must never be reachable through it", key) } } // Every read this box's mirror made stayed inside the one prefix. reads, _ := edgeB.store.snapshot() for _, key := range reads { if !strings.HasPrefix(key, certPrefix) { t.Fatalf("the mirror read %q, outside %s", key, certPrefix) } } }) ca.checkFaults(t) } // ---- the CA's remaining handlers --------------------------------------------- // // Split out below the scenario they serve, so the test reads first. func (ca *fakeACME) orderID(r *http.Request) (string, *fakeACMEOrder, bool) { idx := strings.LastIndex(r.URL.Path, "/") id := r.URL.Path[idx+1:] ca.mu.Lock() defer ca.mu.Unlock() o, ok := ca.byID[id] return id, o, ok } // serveAuthorization answers POST-as-GET on an authorization, offering the one // http-01 challenge (see fakeACME's doc for why only one type). func (ca *fakeACME) serveAuthorization(w http.ResponseWriter, r *http.Request) { id, o, ok := ca.orderID(r) if !ok { ca.recordFault("authz %s: unknown", id) http.Error(w, "no such authz", http.StatusNotFound) return } ca.mu.Lock() body := map[string]any{ "identifier": map[string]string{"type": "dns", "value": o.domain}, "status": o.authz, "challenges": []map[string]string{{ "type": "http-01", "url": ca.url("/chal/" + id), "token": o.token, "status": o.authz, }}, } ca.mu.Unlock() writeJSON(w, http.StatusOK, body) } // serveChallengeAccept is the CA's validation. It runs synchronously: the // whole point of the read-through in challengeResponse is that a validation's // life is measured in seconds, and a CA that answered on the next poll would // be testing the wrong thing. func (ca *fakeACME) serveChallengeAccept(w http.ResponseWriter, r *http.Request) { id, o, ok := ca.orderID(r) if !ok { ca.recordFault("challenge %s: unknown", id) http.Error(w, "no such challenge", http.StatusNotFound) return } ca.mu.Lock() base, thumb, domain, token := ca.validateVia, ca.thumb, o.domain, o.token ca.mu.Unlock() url := base + acmeChallengePrefix + token got, ferr := ca.fetchChallenge(url, domain) want := token + "." + thumb valid := ferr == nil && got == want ca.mu.Lock() ca.validations = append(ca.validations, fakeACMEValidation{url: url, host: domain, token: token, ok: valid}) if valid { o.authz, o.status = "valid", "ready" } else { o.authz, o.status = "invalid", "invalid" ca.faults = append(ca.faults, fmt.Sprintf( "http-01 for %s at %s: body %q, want %q (fetch err %v)", domain, url, got, want, ferr)) } ca.mu.Unlock() writeJSON(w, http.StatusOK, map[string]string{ "type": "http-01", "url": ca.url("/chal/" + id), "token": token, "status": "valid", }) } // fetchChallenge is the validator: an ordinary HTTP GET carrying the Host // header of the name under validation, exactly as a CA sends it (and exactly // what challengeResponse's host bound is calibrated against). func (ca *fakeACME) fetchChallenge(url, host string) (string, error) { req, err := http.NewRequest("GET", url, nil) if err != nil { return "", err } req.Host = host client := &http.Client{Timeout: 10 * time.Second} res, err := client.Do(req) if err != nil { return "", err } defer res.Body.Close() body, err := io.ReadAll(io.LimitReader(res.Body, 1<<16)) if err != nil { return "", err } if res.StatusCode != http.StatusOK { return "", fmt.Errorf("challenge fetch: %s", res.Status) } return string(body), nil } func (ca *fakeACME) serveOrder(w http.ResponseWriter, r *http.Request) { id, _, ok := ca.orderID(r) if !ok { ca.recordFault("order %s: unknown", id) http.Error(w, "no such order", http.StatusNotFound) return } w.Header().Set("Location", ca.url("/order/"+id)) writeJSON(w, http.StatusOK, ca.orderJSON(id)) } func (ca *fakeACME) serveFinalize(w http.ResponseWriter, r *http.Request) { id, o, ok := ca.orderID(r) if !ok { ca.recordFault("finalize %s: unknown", id) http.Error(w, "no such order", http.StatusNotFound) return } _, payload, err := decodeJWS(r) if err != nil { ca.recordFault("finalize %s: %v", id, err) http.Error(w, "bad jws", http.StatusBadRequest) return } var req struct { CSR string `json:"csr"` } if err := json.Unmarshal(payload, &req); err != nil { ca.recordFault("finalize %s: %v", id, err) http.Error(w, "bad finalize", http.StatusBadRequest) return } der, err := base64.RawURLEncoding.DecodeString(req.CSR) if err != nil { ca.recordFault("finalize %s: csr: %v", id, err) http.Error(w, "bad csr", http.StatusBadRequest) return } ca.mu.Lock() domain, authz := o.domain, o.authz ca.mu.Unlock() if authz != "valid" { ca.recordFault("finalize %s: authorization is %q — a CA must not issue on an unvalidated order", id, authz) http.Error(w, "not authorized", http.StatusForbidden) return } chain, err := ca.issue(der, domain) if err != nil { ca.recordFault("finalize %s: %v", id, err) http.Error(w, "cannot issue", http.StatusBadRequest) return } ca.mu.Lock() o.chain, o.status = chain, "valid" ca.mu.Unlock() w.Header().Set("Location", ca.url("/order/"+id)) writeJSON(w, http.StatusOK, ca.orderJSON(id)) } func (ca *fakeACME) serveCert(w http.ResponseWriter, r *http.Request) { id, o, ok := ca.orderID(r) if !ok { ca.recordFault("cert %s: unknown", id) http.Error(w, "no such cert", http.StatusNotFound) return } ca.mu.Lock() chain := o.chain ca.mu.Unlock() w.Header().Set("Content-Type", "application/pem-certificate-chain") w.WriteHeader(http.StatusOK) _, _ = w.Write(chain) } // issue signs the CSR's public key for domain under the fake root. The CSR's // own signature and name are checked: a CA that issued for whatever it was // asked would let a chain-of-custody bug through unnoticed. func (ca *fakeACME) issue(csrDER []byte, domain string) ([]byte, error) { csr, err := x509.ParseCertificateRequest(csrDER) if err != nil { return nil, fmt.Errorf("parse csr: %w", err) } if err := csr.CheckSignature(); err != nil { return nil, fmt.Errorf("csr signature: %w", err) } if len(csr.DNSNames) != 1 || csr.DNSNames[0] != domain { return nil, fmt.Errorf("csr names %v, want [%s]", csr.DNSNames, domain) } serial, err := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 96)) if err != nil { return nil, err } tmpl := &x509.Certificate{ SerialNumber: serial, Subject: pkix.Name{CommonName: domain}, DNSNames: []string{domain}, NotBefore: time.Now().Add(-time.Hour), // 90 days, like the real thing: autocert's renewal timer is then // ~60 days out and nothing fires during the test. NotAfter: time.Now().Add(90 * 24 * time.Hour), KeyUsage: x509.KeyUsageDigitalSignature, ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, } der, err := x509.CreateCertificate(rand.Reader, tmpl, ca.root, csr.PublicKey, ca.rootKey) if err != nil { return nil, err } var buf bytes.Buffer if err := pem.Encode(&buf, &pem.Block{Type: "CERTIFICATE", Bytes: der}); err != nil { return nil, err } if err := pem.Encode(&buf, &pem.Block{Type: "CERTIFICATE", Bytes: ca.root.Raw}); err != nil { return nil, err } return buf.Bytes(), nil } // ---- P4: the renewal timer's cache-first check ------------------------------- // R3: autocert's renewal timers never consult HostPolicy (see // issuerHostPolicy), so a box that has stood down still has armed timers. What // stops one ordering is renewal.go's do(), which reads the CACHE first and // returns without touching a CA when the cached cert is far enough from // expiry. The mirror's syncLocal is what puts a fresher cert there. // // The boundary, stated rather than papered over: autocert's real timer is not // fast-forwarded here — time.AfterFunc's deadline is unreachable from a test // and faking it would test the fake. What IS proven is every input the timer's // decision reads: the local cache holds the fresher bytes after one Refresh, // autocert's own cacheGet (the exact call do() makes, reached here through // GetCertificate) returns the fresher certificate, and the renewal threshold // re-derived from renewal.go's next() flips from "due" to "not due" across the // refresh. func TestCertSyncE2EFresherBucketCertReachesAutocertsCache(t *testing.T) { e2eQuietLogs(t) ctx := context.Background() store := release.NewDirStore(t.TempDir()) local := autocert.DirCache(t.TempDir()) // Stale but live: a 90-day cert with 20 days left, which is inside // autocert's 30-day renewal threshold — this box's timer is due to order. stale := e2eDatedCertPEM(t, e2eHost, time.Now().Add(-70*24*time.Hour), time.Now().Add(20*24*time.Hour)) if err := local.Put(ctx, e2eHost, stale); err != nil { t.Fatal(err) } if due := e2eRenewalDue(t, stale); !due { t.Fatal("test bug: the seeded cert must be inside the renewal threshold, or there is nothing to rescue") } // The issuer has already renewed and published: 90 days, all of them left. fresh := e2eDatedCertPEM(t, e2eHost, time.Now().Add(-time.Hour), time.Now().Add(90*24*time.Hour)) putBucket(t, store, e2eHost, fresh) m := newCertMirror(store, local) if err := m.Refresh(ctx); err != nil { t.Fatal(err) } got, err := localCert(t, local, e2eHost) if err != nil { t.Fatalf("the mirror must write fresh material into this box's own cache: %v", err) } if !bytes.Equal(got, fresh) { t.Fatal("the local cache still holds the stale cert — the renewal timer would go to the CA") } if due := e2eRenewalDue(t, got); due { t.Fatal("the cached cert is still inside the renewal threshold: do() would fall through and order") } // The cache-first read do() makes, made through the public door onto it. // The manager is UNGATED (this box may have won the election back) and // points at a dead directory, so a cache miss fails offline and fast // rather than reaching a CA. e := testIssuerEdge(e2eHost) served, err := offlineManager(t, local, e.issuerHostPolicy()).GetCertificate(ecdsaHello(e2eHost)) if err != nil { t.Fatalf("autocert's own cache read must find the mirrored cert: %v", err) } freshLeaf := e2eLeafOf(t, fresh) staleLeaf := e2eLeafOf(t, stale) if !freshLeaf.NotAfter.After(staleLeaf.NotAfter) { t.Fatal("test bug: the two certs are not ordered by expiry") } if !served.Leaf.NotAfter.Equal(freshLeaf.NotAfter) { t.Fatalf("autocert served a cert expiring %s, want the mirrored one at %s", served.Leaf.NotAfter, freshLeaf.NotAfter) } } // e2eDatedCertPEM is hostCertPEM with a caller-chosen NotBefore, so a test can // build a cert whose LIFETIME (and therefore autocert's renewal threshold) is // realistic rather than an artefact of the helper. func e2eDatedCertPEM(t *testing.T, host string, notBefore, notAfter time.Time) []byte { t.Helper() key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) if err != nil { t.Fatal(err) } tmpl := &x509.Certificate{ SerialNumber: big.NewInt(2), Subject: pkix.Name{CommonName: host}, DNSNames: []string{host}, NotBefore: notBefore, NotAfter: notAfter, } der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key) if err != nil { t.Fatal(err) } leaf, err := x509.ParseCertificate(der) if err != nil { t.Fatal(err) } raw, err := encodeWildcardPEM(&tls.Certificate{Certificate: [][]byte{der}, PrivateKey: key, Leaf: leaf}, key) if err != nil { t.Fatal(err) } return raw } func e2eLeafOf(t *testing.T, entry []byte) *x509.Certificate { t.Helper() cert, err := parseCertEntry(entry) if err != nil { t.Fatal(err) } return cert.Leaf } // e2eRenewalDue re-derives autocert's own renewal decision (renewal.go's // next(), with no RenewBefore set): threshold = min(lifetime/3, 30 days), and // the timer's cache-first branch falls through to an ORDER when the renewal // moment has already passed. Jitter is ignored — it is at most an hour and // both fixtures here are days from the boundary. func e2eRenewalDue(t *testing.T, entry []byte) bool { t.Helper() leaf := e2eLeafOf(t, entry) threshold := min(leaf.NotAfter.Sub(leaf.NotBefore)/3, 30*24*time.Hour) return !leaf.NotAfter.Add(-threshold).After(time.Now()) } // ---- P5: Refresh under concurrency ------------------------------------------- // The mirror has no refreshMu: Refresh is written for the single poll loop // that calls it (run(), plus the one synchronous boot pass edge.go runs BEFORE // that goroutine starts). This test runs it from several goroutines at once // anyway, because "only one caller today" is a property of the callers and not // of the code, and the -race gate is where a wrong answer to that question is // cheap to learn. // // What it asserts is memory safety and a coherent OUTCOME: whichever // interleaving wins, the published snapshot and the local cache both end up // holding every entry the bucket holds. It deliberately does NOT assert // linearizability — two overlapping polls can publish out of order, and the // next poll converges. See the task report for that finding. func TestCertSyncE2EConcurrentRefreshIsRaceFree(t *testing.T) { e2eQuietLogs(t) ctx := context.Background() store := release.NewDirStore(t.TempDir()) hosts := []string{"a." + e2eHost, "b." + e2eHost, "c." + e2eHost} want := map[string][]byte{} for _, h := range hosts { body := hostCertPEM(t, h, time.Now().Add(80*24*time.Hour)) putBucket(t, store, h, body) want[h] = body } putBucket(t, store, "acme_account+key", []byte("ACCOUNT KEY")) local := autocert.DirCache(t.TempDir()) m := newCertMirror(store, local) var wg sync.WaitGroup for range 4 { wg.Add(1) go func() { defer wg.Done() for range 5 { if err := m.Refresh(ctx); err != nil { t.Errorf("concurrent refresh: %v", err) return } } }() } wg.Wait() snap := m.snapshot() for h, body := range want { if got, ok := snap.entries[h]; !ok || !bytes.Equal(got, body) { t.Fatalf("snapshot entry %s = %q (%v), want the published body", h, got, ok) } if snap.certs[h] == nil { t.Fatalf("snapshot has no parsed certificate for %s", h) } got, err := localCert(t, local, h) if err != nil || !bytes.Equal(got, body) { t.Fatalf("local cache %s = %q, %v — every entry must land exactly once", h, got, err) } } if got, err := localCert(t, local, "acme_account+key"); err != nil || string(got) != "ACCOUNT KEY" { t.Fatalf("account key seeding raced: %q, %v", got, err) } } // ---- #277: the non-home edge answers, and the home-boxed name mints --------- // e2eRemoteHomedHost is homed on edge A and absent from edge B's registry — // the shape the original scenario deliberately did not have (there, B carried // every host in its own route table). const e2eRemoteHomedHost = "remote.bab.example.test" // The automated shadow of the dev proof that failed as #277: DNS for an // exactly-named registry ROUTE points at edge B, which homes nothing for that // name. B backhauls to A, A (the issuer) sees the hello and orders, and the CA // then validates against B — the box whose registry has never heard of the // host. Before the widening, B refused the challenge, every authorization // failed, and the name's Let's Encrypt budget went in minutes. After it, B // answers out of the bucket, A mints, and the certificate reaches B's mirror. // // A route, deliberately, and not an attached custom domain: an alias attach // lands in route_aliases (internal/domains/aliases.go), which edgesFilePass // never publishes — so no alias is in any home map, on any box. The spec // amendment records that as an open follow-up rather than a claim this test // could make. // // The backhaul leg itself is not re-proved here (homemap_test.go owns it); // what this pins is that everything CERT-side works with the host in ONE // box's registry and the other box's home map. func TestNonHomeEdgeAnswersValidationForRemoteHomedHost(t *testing.T) { e2eQuietLogs(t) bucket := t.TempDir() ca := newFakeACME(t) // Edge B: no routes at all for the name — this is the box DNS points at. edgeB := newE2EEdge(t, bucket, false, ca.directoryURL()) ca.validateAgainst(edgeB.serveHTTP(t)) if edgeB.edge.Allowed(e2eRemoteHomedHost) { t.Fatal("test bug: edge B must not carry the host in its own registry") } // ...but it mirrors the deployment's edges files, in which edge A homes // the name exactly. That, and only that, is what admits the validation. homes := newHomeMap(release.NewDirStore(bucket), "edge-b") homedOn(t, homes, "edge-a", "203.0.113.10", e2eRemoteHomedHost, "*.other.example.test") edgeB.edge.homes.Store(homes) // Edge A: the housekeeper, and the box that homes the route. Its own :80 // chain is served purely so "the CA never reached the issuer" is an // assertion rather than an assumption. edgeA := newE2EEdge(t, bucket, true, ca.directoryURL(), e2eRemoteHomedHost) edgeA.serveHTTP(t) lnB := edgeB.serveTLS(t) if _, err := e2eDial(lnB, e2eRemoteHomedHost, ca.rootPool()); err == nil { t.Fatal("edge B answered a handshake for a certificate that does not exist yet") } // The mint, on edge A — exactly what a backhauled hello triggers there. minted, err := edgeA.getCert(ecdsaHello(e2eRemoteHomedHost)) if err != nil { t.Fatalf("the issuer failed to mint a host it homes: %v", err) } ca.checkFaults(t) if minted.Leaf == nil || minted.Leaf.VerifyHostname(e2eRemoteHomedHost) != nil { t.Fatalf("minted cert does not cover %s", e2eRemoteHomedHost) } t.Run("the validation was answered by the box that homes nothing", func(t *testing.T) { v, ok := ca.lastValidation() if !ok { t.Fatal("the CA never validated anything") } if !v.ok || v.host != e2eRemoteHomedHost { t.Fatalf("validation of %q ok=%v — the body B served was not the key authorization", v.host, v.ok) } if n := edgeB.hits.Load(); n == 0 { t.Fatal("edge B's :80 chain was never reached") } if n := edgeA.hits.Load(); n != 0 { t.Fatalf("edge A's :80 chain served %d request(s) — the CA must never have reached the issuer", n) } // It came out of the bucket, live: B never polled, and its own // autocert handler is shut for this name by the issuer gate. if n := len(edgeB.mirror.snapshot().entries); n != 0 { t.Fatalf("edge B's mirror holds %d entry/entries — the read-through is not what was tested", n) } if err := edgeB.edge.issuerHostPolicy()(context.Background(), e2eRemoteHomedHost); !errors.Is(err, errNotIssuer) { t.Fatalf("edge B's HostPolicy = %v, want errNotIssuer — a non-issuer orders nothing, however it is named", err) } want := certKey(v.token + httpTokenSuffix) reads, _ := edgeB.store.snapshot() if !slices.Contains(reads, want) { t.Fatalf("edge B read %v, want a live read of %q", reads, want) } }) t.Run("the tls-alpn-01 twin is admitted for the same name", func(t *testing.T) { // The fake CA offers http-01 only (its doc says why), so the ALPN half // is proved at the same seam the unit tests use: a token cert in the // bucket, an acme-tls/1 hello at edge B for a name only edge A homes. // Written through a THIRD handle on the bucket, standing in for the // issuer's publishingCache: edge B's own handle must stay write-free, // which the last subtest asserts. issuerHandle := release.NewDirStore(bucket) putBucket(t, issuerHandle, e2eRemoteHomedHost+alpnTokenSuffix, hostCertPEM(t, e2eRemoteHomedHost, time.Now().Add(24*time.Hour))) if c := edgeB.mirror.alpnTokenCert(context.Background(), e2eRemoteHomedHost); c == nil { t.Fatal("edge B refused a tls-alpn-01 validation for a host edge A homes") } // And the label under the mirrored catch-all is still refused (#180). // The token cert for it is PUT FIRST and the label is asserted // dialable: without both, a nil answer here would prove only that the // bucket is empty, and the refusal would pass even if KnownHost leaked // through Home()'s one-label rule. const invented = "invented.other.example.test" putBucket(t, issuerHandle, invented+alpnTokenSuffix, hostCertPEM(t, invented, time.Now().Add(24*time.Hour))) if _, _, ok := edgeB.edge.homes.Load().Home(invented); !ok { t.Fatal("test bug: the invented label is supposed to be DIALABLE through the one-label rule") } if c := edgeB.mirror.alpnTokenCert(context.Background(), invented); c != nil { t.Fatal("a label under a mirrored catch-all was served a token cert") } }) t.Run("the minted cert reaches the non-home edge", func(t *testing.T) { if err := edgeB.mirror.Refresh(context.Background()); err != nil { t.Fatalf("one refresh must be enough: %v", err) } state, err := e2eDial(lnB, e2eRemoteHomedHost, ca.rootPool()) if err != nil { t.Fatalf("edge B must serve the certificate edge A minted: %v", err) } if len(state.PeerCertificates) == 0 || !bytes.Equal(state.PeerCertificates[0].Raw, minted.Leaf.Raw) { t.Fatal("edge B served a certificate that is not the one edge A minted") } _, wrote := edgeB.store.snapshot() if len(wrote) != 0 { t.Fatalf("a gated edge wrote %v to the control bucket — only the issuer may", wrote) } }) ca.checkFaults(t) }