package edge import ( "context" "errors" "net/http" "net/http/httptest" "os" "path/filepath" "strings" "testing" "time" "golang.org/x/crypto/acme/autocert" "github.com/carlosframework/platform/internal/lbhealth" "github.com/carlosframework/platform/internal/registry" "github.com/carlosframework/platform/internal/release" ) // sweeper is an Activator whose only working method is the sweep beat. type sweeper struct { Activator at time.Time } func (s sweeper) LastSweep() time.Time { return s.at } func lbEnv(v string) func(string) string { return func(k string) string { if k == "CARLOS_REGION_HA" { return v } return "" } } func TestLBHealthStateFollowsRegionHA(t *testing.T) { if s := lbHealthFromEnv(lbEnv(""), nil); s != nil { t.Fatal("region HA unset armed the health answer") } if s := lbHealthFromEnv(lbEnv("0"), nil); s != nil { t.Fatal("region HA off armed the health answer") } at := time.Now() s := lbHealthFromEnv(lbEnv("1"), nil) if s != lbhealth.Default { t.Fatal("region HA on: not the process's state") } t.Cleanup(func() { lbhealth.Default.SetActivator(nil) }) lbHealthFromEnv(lbEnv("true"), sweeper{at: at}) if got := lbhealth.Default.Failing(at); strings.Contains(strings.Join(got, ","), lbhealth.CondActivator) { t.Fatalf("the activator's sweep was not wired: %v", got) } } // fallbackGet drives the :80 handler with one request. func fallbackGet(e *Edge, method, path, host string) *httptest.ResponseRecorder { w := httptest.NewRecorder() r := httptest.NewRequest(method, path, nil) r.Host = host e.httpFallbackHandler().ServeHTTP(w, r) return w } // healthyState meets every condition now. func healthyState() *lbhealth.State { s := &lbhealth.State{} now := time.Now() s.SetHosting(true) s.MapLoaded() s.KeyLoaded() s.CertsLoaded() s.EdgeBeat(now) s.RefreshAttempted(now) s.SetActivator(func() time.Time { return now }) return s } // The answer is on :80, for any host (the load balancer sends the // target's address), GET or HEAD, ahead of the redirect and unknown-host // branches. With region HA off the path is ordinary :80 traffic. func TestLBHealthAnswersOnPort80ForAnyHost(t *testing.T) { e, err := New(openReg(t), nil) if err != nil { t.Fatal(err) } t.Cleanup(e.probers.Stop) e.lbHealth = healthyState() for _, host := range []string{"10.0.1.7", "10.0.1.7:80", "unknown.example.com"} { w := fallbackGet(e, http.MethodGet, lbhealth.Path, host) if w.Code != http.StatusOK || w.Body.String() != "ok\n" { t.Fatalf("host %s: %d %q", host, w.Code, w.Body.String()) } } if w := fallbackGet(e, http.MethodHead, lbhealth.Path, "10.0.1.7"); w.Code != http.StatusOK { t.Fatalf("HEAD: %d", w.Code) } joining := healthyState() joining.SetHosting(false) e.lbHealth = joining if w := fallbackGet(e, http.MethodGet, lbhealth.Path, "10.0.1.7"); w.Code != http.StatusServiceUnavailable || !strings.Contains(w.Body.String(), lbhealth.CondHosting) { t.Fatalf("joining: %d %q", w.Code, w.Body.String()) } // A POST, or another path, is ordinary :80 traffic. if w := fallbackGet(e, http.MethodPost, lbhealth.Path, "10.0.1.7"); w.Code == http.StatusOK || w.Code == http.StatusServiceUnavailable { t.Fatalf("POST answered by the health check: %d", w.Code) } e.lbHealth = nil if w := fallbackGet(e, http.MethodGet, lbhealth.Path, "10.0.1.7"); w.Code == http.StatusOK || strings.Contains(w.Body.String(), "unavailable:") { t.Fatalf("region HA off answered the health check: %d %q", w.Code, w.Body.String()) } } // blockingStore makes every List wait until release is closed, and fail // while down is set: a refresh stuck inside one attempt, and a bucket // outage. type blockingStore struct { release.Store block chan struct{} down bool entered chan struct{} } func (s *blockingStore) List(ctx context.Context, prefix string) ([]string, error) { if s.block != nil { s.entered <- struct{}{} <-s.block } if s.down { return nil, errors.New("bucket unreachable") } return s.Store.List(ctx, prefix) } func (s *blockingStore) Get(ctx context.Context, key string) ([]byte, string, error) { if s.down { return nil, "", errors.New("bucket unreachable") } return s.Store.Get(ctx, key) } // The home map latches its first loaded map and key, stamps every attempt // it finishes, failed or not, and stamps nothing while one is stuck. func TestHomeMapFeedsTheHealthState(t *testing.T) { inner := meshTestStore(t) if err := mintMeshKey(context.Background(), inner); err != nil { t.Fatal(err) } putJSON(t, inner, "control/edges/betty.json", `{"v":1,"box":"betty","eip":"2.2.2.2","hosts":{"b.test":{"role":"home"}}}`) s := &blockingStore{Store: inner} h := &lbhealth.State{} m := newHomeMap(s, "annie") m.health = h refreshHomeMap(t, m) now := time.Now() failing := strings.Join(h.Failing(now), ",") if strings.Contains(failing, lbhealth.CondForwarding) || strings.Contains(failing, lbhealth.CondRefresh) { t.Fatalf("after a good refresh: %s", failing) } // A bucket outage: the attempt fails but finishes, and the latches hold. s.down = true before := time.Now() if err := m.Refresh(context.Background()); err == nil { t.Fatal("refresh against a down bucket succeeded") } if got := strings.Join(h.Failing(before.Add(lbhealth.RefreshWindow)), ","); strings.Contains(got, lbhealth.CondForwarding) || strings.Contains(got, lbhealth.CondRefresh) { t.Fatalf("a failed attempt dropped forwarding or went unstamped: %s", got) } // A stuck attempt stamps nothing, at entry or otherwise: the last stamp // is the failed attempt above, taken more than a second before t0, so // a minute and a bit short of the window after t0 it is already stale. // A stamp taken at entry would be fresh here. s.down = false s.block, s.entered = make(chan struct{}), make(chan struct{}, 1) time.Sleep(1200 * time.Millisecond) t0 := time.Now() done := make(chan struct{}) go func() { _ = m.Refresh(context.Background()); close(done) }() <-s.entered if got := strings.Join(h.Failing(t0.Add(lbhealth.RefreshWindow-time.Second)), ","); !strings.Contains(got, lbhealth.CondRefresh) { t.Fatalf("a refresh stuck inside its attempt was stamped at entry: %s", got) } stuckAt := time.Now() if got := strings.Join(h.Failing(stuckAt.Add(lbhealth.RefreshWindow+time.Second)), ","); !strings.Contains(got, lbhealth.CondRefresh) { t.Fatalf("a refresh stuck for longer than the window still counts: %s", got) } close(s.block) <-done } // A key that never loads leaves forwarding unarmed, whatever the map did. func TestHomeMapWithNoMeshKeyIsNotArmed(t *testing.T) { s := meshTestStore(t) putJSON(t, s, "control/edges/betty.json", `{"v":1,"box":"betty","eip":"2.2.2.2","hosts":{}}`) h := &lbhealth.State{} m := newHomeMap(s, "annie") m.health = h refreshHomeMap(t, m) if got := strings.Join(h.Failing(time.Now()), ","); !strings.Contains(got, lbhealth.CondForwarding) { t.Fatalf("no mesh key, yet forwarding armed: %s", got) } } func TestCertMirrorLatchesItsFirstLoad(t *testing.T) { s := meshTestStore(t) h := &lbhealth.State{} m := newCertMirror(s, autocert.DirCache(t.TempDir())) m.health = h if !strings.Contains(strings.Join(h.Failing(time.Now()), ","), lbhealth.CondCerts) { t.Fatal("certificates counted before any load") } if err := m.Refresh(context.Background()); err != nil { t.Fatal(err) } if strings.Contains(strings.Join(h.Failing(time.Now()), ","), lbhealth.CondCerts) { t.Fatal("a loaded mirror did not latch the certificate condition") } } // The edge's registry watch beats on every poll it completes. func TestRegistryWatchBeats(t *testing.T) { e, err := New(openReg(t), nil) if err != nil { t.Fatal(err) } t.Cleanup(e.probers.Stop) h := &lbhealth.State{} e.lbHealth = h stop := make(chan struct{}) t.Cleanup(func() { close(stop) }) go e.Watch(stop) deadline := time.Now().Add(5 * time.Second) for strings.Contains(strings.Join(h.Failing(time.Now()), ","), lbhealth.CondEdge) { if time.Now().After(deadline) { t.Fatal("the registry watch never beat") } time.Sleep(50 * time.Millisecond) } } // A certificate mirror whose first list fails has loaded nothing: the // certificate condition stays unlatched, and a later good refresh latches it. func TestCertMirrorFailedRefreshLeavesCertsUnlatched(t *testing.T) { inner := meshTestStore(t) s := &blockingStore{Store: inner, down: true} h := &lbhealth.State{} m := newCertMirror(s, autocert.DirCache(t.TempDir())) m.health = h if err := m.Refresh(context.Background()); err == nil { t.Fatal("refresh against a down bucket succeeded") } if !strings.Contains(strings.Join(h.Failing(time.Now()), ","), lbhealth.CondCerts) { t.Fatal("a failed refresh latched the certificate condition") } s.down = false if err := m.Refresh(context.Background()); err != nil { t.Fatal(err) } if strings.Contains(strings.Join(h.Failing(time.Now()), ","), lbhealth.CondCerts) { t.Fatal("the next good refresh did not latch") } } // failingFingerprint is a registry whose file stops being a database after // it opened (a directory stands where it was), so Fingerprint errors on every call. func failingFingerprint(t *testing.T) *registry.Registry { t.Helper() path := filepath.Join(t.TempDir(), "registry.db") reg, err := registry.Open(path) if err != nil { t.Fatal(err) } for _, f := range []string{path, path + "-wal", path + "-shm"} { _ = os.Remove(f) } if err := os.Mkdir(path, 0o700); err != nil { t.Fatal(err) } if _, err := reg.Fingerprint(); err == nil { t.Fatal("setup: Fingerprint still works") } return reg } // A registry poll that fails does not beat. func TestRegistryWatchDoesNotBeatOnFingerprintError(t *testing.T) { e, err := New(openReg(t), nil) if err != nil { t.Fatal(err) } t.Cleanup(e.probers.Stop) e.reg = failingFingerprint(t) h := &lbhealth.State{} e.lbHealth = h stop := make(chan struct{}) t.Cleanup(func() { close(stop) }) go e.Watch(stop) time.Sleep(5 * time.Second) if !strings.Contains(strings.Join(h.Failing(time.Now()), ","), lbhealth.CondEdge) { t.Fatal("a failing Fingerprint beat") } } // wedgedProbe runs an edge over a state that is healthy at setup (every beat // taken then), so only the edge's beat can still move, and reports what is // failing a little before the window ends. func wedgedProbe(t *testing.T, wedge bool) (failing string) { t.Helper() old := servingProbeTimeout servingProbeTimeout = 100 * time.Millisecond t.Cleanup(func() { servingProbeTimeout = old }) e, err := New(openReg(t), nil) if err != nil { t.Fatal(err) } t.Cleanup(e.probers.Stop) e.EnableCache(NewCache(10, 1<<20)) h := healthyState() h.SetActivator(time.Now) // the activator is not what this test wedges e.lbHealth = h if wedge { // A request path deadlocked inside the cache: every cacheable GET // takes this lock. Released at cleanup so the probe goroutines end. e.cache.mu.Lock() t.Cleanup(e.cache.mu.Unlock) } stop := make(chan struct{}) t.Cleanup(func() { close(stop) }) go e.Watch(stop) time.Sleep(5 * time.Second) // Set the clock 57s ahead: a beat from the last tick or two is still // inside the 60s window, one from before the wedge (setup, 5s+ ago) is not. now := time.Now().Add(lbhealth.HeartbeatWindow - 3*time.Second) got := h.Failing(now) w := httptest.NewRecorder() lbhealth.Serve(w, httptest.NewRequest(http.MethodGet, lbhealth.Path, nil), h, now) if wedge != (w.Code == http.StatusServiceUnavailable) { t.Fatalf("wedge=%v: the health answer was %d %q", wedge, w.Code, w.Body.String()) } return strings.Join(got, ",") } // A deadlocked request path keeps the registry poll running, but the edge // stops beating, and the health answer turns 503 naming the edge once the // last beat ages out. func TestEdgeBeatStopsWhenTheServingPathIsWedged(t *testing.T) { if got := wedgedProbe(t, true); !strings.Contains(got, lbhealth.CondEdge) { t.Fatalf("a wedged serving path kept beating: %q", got) } } func TestEdgeBeatContinuesWhenTheServingPathIsFree(t *testing.T) { if got := wedgedProbe(t, false); strings.Contains(got, lbhealth.CondEdge) { t.Fatalf("a free serving path stopped beating: %q", got) } }