package metrics import ( "strconv" "time" ) // Rules is the thresholds and sustain windows Evaluate checks each tick // against. Frac fields are fractions (0..1), matching HostSample; LagSecs is // the one exception — keymail relay lag arrives in seconds, not a fraction. type Rules struct { DiskFrac float64 // fire >, default 0.90, immediate MemFrac float64 // default 0.90 MemSustain time.Duration // default 10 * time.Minute CPUFrac float64 // default 0.95 CPUSustain time.Duration // default 15 * time.Minute LagSecs int64 // default 900, immediate; inert while readings are nil ClearBy float64 // hysteresis, default 0.05 (fractions); lag clears at LagSecs-300 // GeoIPSustain is how long a geoip reason must hold before the rule // fires (geoip spec §10.4); it clears on the first reading of 0. GeoIPSustain time.Duration // default 30 * time.Minute } // DefaultRules are the thresholds shipped with the platform. func DefaultRules() Rules { return Rules{ DiskFrac: 0.90, MemFrac: 0.90, MemSustain: 10 * time.Minute, CPUFrac: 0.95, CPUSustain: 15 * time.Minute, LagSecs: 900, ClearBy: 0.05, // GeoIPSustain: geoip spec §10.4. GeoIPSustain: 30 * time.Minute, } } // RulesFromEnv applies percent-integer overrides on top of DefaultRules: // CARLOS_ALERT_DISK_PCT, CARLOS_ALERT_MEM_PCT, CARLOS_ALERT_CPU_PCT. An // unset, unparsable, or out-of-range (0..100) value keeps the default — // deliberately pure (no logging); a caller that wants to log a rejected // override asks PctFrac the same question this does, since this package // touches neither stderr nor the OS. func RulesFromEnv(getenv func(string) string) Rules { r := DefaultRules() if v, ok := PctFrac(getenv("CARLOS_ALERT_DISK_PCT")); ok { r.DiskFrac = v } if v, ok := PctFrac(getenv("CARLOS_ALERT_MEM_PCT")); ok { r.MemFrac = v } if v, ok := PctFrac(getenv("CARLOS_ALERT_CPU_PCT")); ok { r.CPUFrac = v } return r } // PctFrac converts one percent-integer override string ("95") to the // fraction Rules holds (0.95), reporting ok=false for empty, unparsable, or // out-of-range (0..100) input. Exported so the agent can LOG an override it // set but this rejected without either duplicating the rule or dragging // stderr into this package. func PctFrac(s string) (float64, bool) { if s == "" { return 0, false } n, err := strconv.Atoi(s) if err != nil || n < 0 || n > 100 { return 0, false } return float64(n) / 100.0, true } // Observation is one tick's readings for a box. A nil field means "no // data this tick" and is handled identically everywhere in Evaluate: the // corresponding rule's state is left untouched (a firing alert stays // firing, a quiet rule stays quiet, nothing is ever emitted for it). type Observation struct { At time.Time CPUFrac *float64 MemFrac *float64 DiskFrac *float64 LagSecs *int64 // GeoIP is the box's geoip reason code (geoip spec §10.4): 0 none, // 1 stale, 2 expiring, 3 expired, 4 not loaded, 5 feed failing. nil on // a box whose edge has not published one. GeoIP *float64 } // RuleState is one rule's sustain/hysteresis bookkeeping. Since and FiredAt // carry `omitempty`, but encoding/json's omitempty never treats a struct // (time.Time is a struct) as empty regardless of its zero value — so a // never-triggered RuleState does NOT drop these keys. It marshals as // {"firing":false,"since":"0001-01-01T00:00:00Z","fired_at":"0001-01-01T00:00:00Z"}, // both fields always present. The round-trip is still lossless (the zero // time reproduces exactly on Unmarshal), so this is a documentation note, // not a bug: any consumer of this wire shape (the agent loop's persistence, // the console watcher) must treat the zero time value itself — not key // absence — as "never set". See TestRuleStateJSONRoundTrip. type RuleState struct { Firing bool `json:"firing"` Since time.Time `json:"since,omitempty"` // continuously-above-threshold since (sustain tracking) FiredAt time.Time `json:"fired_at,omitempty"` // set on fire, cleared on clear Value float64 `json:"value,omitempty"` // reading at last transition } // AlertState is the JSON persisted to disk by the agent loop and published // to alerts/boxes/.json (see AlertKey), and read back by the console // watcher that diffs Rules[...].Firing across polls. type AlertState struct { V int `json:"v"` // 1 Box string `json:"box"` UpdatedAt time.Time `json:"updated_at"` Rules map[string]RuleState `json:"rules"` // "disk","mem","cpu","lag" } // Event is emitted on a firing or clearing transition — never on every // tick, and never for a nil reading. type Event struct { Rule string `json:"rule"` Firing bool `json:"firing"` // true=fired, false=cleared Value float64 `json:"value"` Box string `json:"box"` } // AlertKey is the bucket key an AlertState for box is stored/published // under. func AlertKey(box string) string { return "alerts/boxes/" + box + ".json" } // Evaluate advances AlertState by one Observation and returns the new // state plus any fire/clear transitions as Events. // // Evaluate never mutates st: it builds a fresh Rules map (copying each // RuleState by value — RuleState holds no pointers or slices, so the copy // is a true deep copy) and returns that in the new AlertState. Callers // comparing old vs new state across the call — the console watcher this // feeds diffs Rules[...].Firing on exactly that assumption — see two // independent maps, never the same one mutated in place. // // Semantics, per rule ("disk", "mem", "cpu", "lag"): // - nil reading: that rule's state is left exactly as-is. No data never // fires and never clears — clearing needs evidence. // - Not firing, reading > threshold: Since is set if it was zero; the // rule fires (emitting an Event) once obs.At.Sub(Since) >= sustain // (immediate rules use sustain 0, so they fire the same tick Since is // set). A reading back at-or-below threshold before sustain elapses // resets Since to zero — the sustain window must be continuous. // - Firing, reading < threshold-ClearBy: the rule clears (emitting an // Event) and Since/FiredAt are zeroed. // - Firing, reading in [threshold-ClearBy, threshold] (or still above // threshold): stays firing — this is the hysteresis band, and it is // deliberately wider than just "below threshold" so a reading // oscillating right at the line doesn't flap. func Evaluate(r Rules, st AlertState, obs Observation) (AlertState, []Event) { out := AlertState{ V: st.V, Box: st.Box, UpdatedAt: obs.At, Rules: make(map[string]RuleState, len(st.Rules)), } for name, rs := range st.Rules { out.Rules[name] = rs } var events []Event emit := func(name string, firing bool, value float64) { events = append(events, Event{Rule: name, Firing: firing, Value: value, Box: st.Box}) } evalFrac := func(name string, reading *float64, threshold float64, sustain time.Duration) { if reading == nil { return } val := *reading cur := out.Rules[name] clearLine := threshold - r.ClearBy if cur.Firing { if val < clearLine { cur.Firing = false cur.Since = time.Time{} cur.FiredAt = time.Time{} cur.Value = val out.Rules[name] = cur emit(name, false, val) } return } if val > threshold { if cur.Since.IsZero() { cur.Since = obs.At } if obs.At.Sub(cur.Since) >= sustain { cur.Firing = true cur.FiredAt = obs.At cur.Value = val out.Rules[name] = cur emit(name, true, val) return } out.Rules[name] = cur return } if !cur.Since.IsZero() { cur.Since = time.Time{} out.Rules[name] = cur } } evalLagSecs := func(name string, reading *int64, threshold int64, clearBy int64, sustain time.Duration) { if reading == nil { return } val := float64(*reading) cur := out.Rules[name] clearLine := float64(threshold - clearBy) thresholdF := float64(threshold) if cur.Firing { if val < clearLine { cur.Firing = false cur.Since = time.Time{} cur.FiredAt = time.Time{} cur.Value = val out.Rules[name] = cur emit(name, false, val) } return } if val > thresholdF { if cur.Since.IsZero() { cur.Since = obs.At } if obs.At.Sub(cur.Since) >= sustain { cur.Firing = true cur.FiredAt = obs.At cur.Value = val out.Rules[name] = cur emit(name, true, val) return } out.Rules[name] = cur return } if !cur.Since.IsZero() { cur.Since = time.Time{} out.Rules[name] = cur } } evalFrac("disk", obs.DiskFrac, r.DiskFrac, 0) evalFrac("mem", obs.MemFrac, r.MemFrac, r.MemSustain) evalFrac("cpu", obs.CPUFrac, r.CPUFrac, r.CPUSustain) evalLagSecs("lag", obs.LagSecs, r.LagSecs, 300, 0) // geoip: a reason code against 0.5, so any reason fires after the // sustain and a reading of 0 clears (geoip spec §10.4). evalFrac("geoip", obs.GeoIP, 0.5, r.GeoIPSustain) // A reason that changes WHILE firing (expiring becomes expired) is a // new firing event with the new code: the alert stays up, its sustain // is not restarted, and the operator is told what it now means (code // review R2-5). if obs.GeoIP != nil { if cur := out.Rules["geoip"]; cur.Firing && *obs.GeoIP > 0.5 && cur.Value != *obs.GeoIP { cur.Value = *obs.GeoIP out.Rules["geoip"] = cur emit("geoip", true, *obs.GeoIP) } } return out, events }