// Package lbhealth is a box's own liveness, as the load balancer's health // check reads it (region HA spec 2 §6). Every fact here is this box's own: // a flag or a timestamp that one of its own loops sets. Nothing here reads // a disk, a bucket or the network, so an outage of something the boxes // share (the deployment bucket) never turns a working box unhealthy and // never gets it terminated. // // The edge answers GET or HEAD /.carlos/lb-health on port 80 from State. // The loops that feed it are the agent's: the claim agent sets hosting, // the home map latches forwarding and stamps its refresh attempts, the // certificate mirror latches its first load, and the edge's registry watch // and the activator's sweeper each beat. package lbhealth import ( "io" "net/http" "strings" "sync" "sync/atomic" "time" ) // Path is the health check's path, matched exactly. const Path = "/.carlos/lb-health" // HeartbeatWindow is how recent the edge's and the activator's beats must // be. Both loops beat every 30 seconds or faster. const HeartbeatWindow = 60 * time.Second // RefreshWindow is how recent the home map's last completed refresh // attempt must be: its 30-second interval plus its two-minute request // timeout, with room to spare. A failed attempt counts, so a bucket outage // where every attempt fails fast keeps this satisfied; a worker stuck // inside one attempt does not. const RefreshWindow = 5 * time.Minute // The condition names a 503 lists, in this order. const ( CondHosting = "hosting" CondForwarding = "forwarding" CondCerts = "certificates" CondEdge = "edge" CondActivator = "activator" CondRefresh = "refresh" ) // State is one box's liveness facts. The zero value is a box that has // proven nothing, and answers 503. A nil *State ignores every update. type State struct { hosting atomic.Bool mapLoaded atomic.Bool keyLoaded atomic.Bool certs atomic.Bool edgeBeat atomic.Int64 // unix nanoseconds; 0 is never refreshed atomic.Int64 // unix nanoseconds; 0 is never mu sync.RWMutex activator func() time.Time } // Default is the process's state. The agent runs the edge and the // activator in one process, so the claim wiring, the edge and the // activator share it. var Default = &State{} // SetHosting records whether the agent is in hosting mode: enrolled, with // its guard armed (§5). Region HA spec 2's plan 2b adds joining mode, whose // mode selection is what sets this false; until then the claim agent sets // it true once its claim wiring is running (ClaimWiring.startRegional). func (s *State) SetHosting(on bool) { if s != nil { s.hosting.Store(on) } } // Hosting reports what SetHosting last recorded. func (s *State) Hosting() bool { return s != nil && s.hosting.Load() } // MapLoaded latches the first successful load of the forwarding map. func (s *State) MapLoaded() { if s != nil { s.mapLoaded.Store(true) } } // KeyLoaded latches the first successful load of a mesh key. func (s *State) KeyLoaded() { if s != nil { s.keyLoaded.Store(true) } } // CertsLoaded latches the first successful load of the certificate mirror // into the local cache the TLS handshake reads. func (s *State) CertsLoaded() { if s != nil { s.certs.Store(true) } } // EdgeBeat records that the edge's registry watch completed a poll at t. func (s *State) EdgeBeat(t time.Time) { if s != nil { s.edgeBeat.Store(t.UnixNano()) } } // RefreshAttempted records that the home map's refresh finished an // attempt at t, whether it succeeded or failed. func (s *State) RefreshAttempted(t time.Time) { if s != nil { s.refreshed.Store(t.UnixNano()) } } // SetActivator gives the state the activator's own beat: the time its // sweeper last finished a sweep. nil means there is no activator, which // fails the activator condition. func (s *State) SetActivator(last func() time.Time) { if s == nil { return } s.mu.Lock() s.activator = last s.mu.Unlock() } func (s *State) activatorBeat() time.Time { s.mu.RLock() last := s.activator s.mu.RUnlock() if last == nil { return time.Time{} } return last() } // within reports whether at is set and no older than window at now. func within(at, now time.Time, window time.Duration) bool { return !at.IsZero() && now.Sub(at) <= window } func stamp(n int64) time.Time { if n == 0 { return time.Time{} } return time.Unix(0, n) } // Failing lists the conditions that do not hold at now, in the order of // the Cond names. An empty list is a healthy box. A nil *State fails // every condition. func (s *State) Failing(now time.Time) []string { if s == nil { return []string{CondHosting, CondForwarding, CondCerts, CondEdge, CondActivator, CondRefresh} } var out []string if !s.hosting.Load() { out = append(out, CondHosting) } if !s.mapLoaded.Load() || !s.keyLoaded.Load() { out = append(out, CondForwarding) } if !s.certs.Load() { out = append(out, CondCerts) } if !within(stamp(s.edgeBeat.Load()), now, HeartbeatWindow) { out = append(out, CondEdge) } if !within(s.activatorBeat(), now, HeartbeatWindow) { out = append(out, CondActivator) } if !within(stamp(s.refreshed.Load()), now, RefreshWindow) { out = append(out, CondRefresh) } return out } // Serve answers the health check when r is one, and reports whether it // did. It matches as narrowly as the steering probe: the exact path, GET // or HEAD, any host. It answers 200 "ok" when every condition holds and // 503 naming the failing conditions otherwise, never cached. It only reads // s. func Serve(w http.ResponseWriter, r *http.Request, s *State, now time.Time) bool { if r.URL.Path != Path || (r.Method != http.MethodGet && r.Method != http.MethodHead) { return false } body, code := "ok\n", http.StatusOK if failing := s.Failing(now); len(failing) > 0 { body, code = "unavailable: "+strings.Join(failing, ",")+"\n", http.StatusServiceUnavailable } w.Header().Set("Content-Type", "text/plain; charset=utf-8") w.Header().Set("Cache-Control", "no-store") w.WriteHeader(code) if r.Method == http.MethodGet { _, _ = io.WriteString(w, body) } return true }