package console import ( "context" "encoding/json" "fmt" "log" "sort" "strconv" "strings" "time" "github.com/carlosframework/platform/internal/metrics" ) // The keymail half of alert delivery lives here rather than on the box, // because the only keymail send path that exists is the console's per-user // OAuth grant (Keymail.SendApp) and the agent has no keymail identity. The // box judges itself and publishes alerts/boxes/.json; this watcher // polls that prefix, diffs each rule's firing state against what it last // told each operator (the alert_seen table), and mails the difference — // each operator with that operator's OWN grant token, so nobody is sent // mail on a credential they did not personally issue. // // The diff is per RECIPIENT, not per rule, because delivery fails per // recipient: a success records that operator's row and a failure leaves // only theirs unrecorded, so the next pass retries exactly the operator // who missed out. One operator's revoked grant can no longer re-page every // healthy operator on the fleet, tick after tick. // // The failure posture throughout is "never go silent": an unreadable or // unparsable alert object is logged and skipped, an undelivered transition // stays pending for that operator until it lands, and no path here can // panic a console that is otherwise serving fine. // alertWatchInterval is the poll cadence from the spec: a minute is fast // enough for a disk filling up and slow enough to be free. const alertWatchInterval = time.Minute // alertPrefix is the bucket prefix the agent publishes AlertState under // (metrics.AlertKey). const alertPrefix = "alerts/boxes/" // StartAlertWatcher runs alertPass immediately and then every minute until // ctx is done. Started from cmd/console's serve(), deliberately NOT from // NewServer: tests construct servers by the dozen and none of them should // grow a background goroutine or a keymail send. func (s *Server) StartAlertWatcher(ctx context.Context) { t := time.NewTicker(alertWatchInterval) defer t.Stop() for { if err := s.alertPass(ctx); err != nil { log.Printf("console: alert watch: %v", err) } select { case <-ctx.Done(): return case <-t.C: } } } // operatorGrant is one recipient and the token the console sends AS them // with. The token is never logged. type operatorGrant struct { address string token string } // alertRun is one pass's working state: who to mail, and the operators with // no grant to mail through — reported at most once per pass, and only if // the pass had something to say, so a fleet at rest logs nothing. type alertRun struct { to []operatorGrant missing []string noted bool } // noteMissing names the operators the console cannot write as, at most once // per pass, called only when the pass actually has something to deliver. func (r *alertRun) noteMissing() { if r.noted || len(r.missing) == 0 { return } r.noted = true log.Printf("console: alert watch: no keymail grant for %s — not notified (they must sign in to the console once)", strings.Join(r.missing, ", ")) } // noMailboxAddress is the synthetic alert_seen recipient standing for "no // operator was mailable when this transition happened". Recording it stops // the transition replaying — and being re-logged — every minute forever, // at the honest cost that the alert is simply lost: an operator who gains a // grant later is not caught up on it (see alertSeenFor). const noMailboxAddress = "" // alertPass reads every alerts/boxes/*.json and, for each rule, diffs its // firing state against what alert_seen says each operator was last told, // mailing and recording per recipient. It returns an error only for a failure // that makes the whole pass meaningless (listing the bucket, or reading the // operator list); per-box and per-rule trouble is logged and stepped over. func (s *Server) alertPass(ctx context.Context) error { if s.cfg.Releases == nil || s.sendApp == nil { return nil } keys, err := s.cfg.Releases.List(ctx, alertPrefix) if err != nil { return fmt.Errorf("list %s: %w", alertPrefix, err) } if len(keys) == 0 { return nil // no box has ever published an alert: nothing to do } run, err := s.alertRecipients() if err != nil { return err } for _, key := range keys { if !strings.HasSuffix(key, ".json") { continue } body, _, err := s.cfg.Releases.Get(ctx, key) if err != nil { log.Printf("console: alert watch: read %s: %v", key, err) continue } var st metrics.AlertState if err := json.Unmarshal(body, &st); err != nil { log.Printf("console: alert watch: unparsable %s: %v", key, err) continue } box := st.Box if box == "" { // Trust the key over an empty body field: the key is what the // publisher's own AlertKey produced. box = strings.TrimSuffix(strings.TrimPrefix(key, alertPrefix), ".json") } names := make([]string, 0, len(st.Rules)) for name := range st.Rules { names = append(names, name) } sort.Strings(names) for _, name := range names { s.alertRule(ctx, run, box, name, st.Rules[name], st.UpdatedAt) } } return nil } // alertRecipients resolves the operator list into recipients, splitting off // the operators the console cannot write as. func (s *Server) alertRecipients() (*alertRun, error) { addrs, err := s.store.Operators() if err != nil { return nil, fmt.Errorf("operators: %w", err) } return s.grantRecipients(addrs) } // grantRecipients resolves canonical addresses to their own Keymail grants. // Alert delivery and one-shot console notifications share this lookup so both // keep the same missing-grant and per-recipient-token behaviour. func (s *Server) grantRecipients(addrs []string) (*alertRun, error) { run := &alertRun{} seen := make(map[string]bool) for _, addr := range addrs { addr = strings.ToLower(strings.TrimSpace(addr)) if addr == "" || seen[addr] { continue } seen[addr] = true tok, err := s.store.GrantToken(addr) if err != nil { return nil, fmt.Errorf("grant token for %s: %w", addr, err) } if tok == "" { run.missing = append(run.missing, addr) continue } run.to = append(run.to, operatorGrant{address: addr, token: tok}) } return run, nil } // alertRule handles one rule of one box: every mailable operator whose // recorded state differs from the published one gets a keymail, and each // send is recorded on its own. func (s *Server) alertRule(ctx context.Context, run *alertRun, box, rule string, rs metrics.RuleState, updated time.Time) { if len(run.to) == 0 { // Nobody the console can write as. Record the transition against the // synthetic recipient so it stops replaying, and say so out loud — // this is the one path where an alert is genuinely dropped. if s.alertDueFor(box, rule, noMailboxAddress, rs) { run.noteMissing() log.Printf("console: alert watch: %s/%s transition dropped — no operator holds a keymail grant", box, rule) s.recordAlertSeenFor(box, rule, noMailboxAddress, rs) } return } var due []operatorGrant for _, op := range run.to { if s.alertDueFor(box, rule, op.address, rs) { due = append(due, op) } } if len(due) == 0 { return } run.noteMissing() subject, body := alertMessage(box, rule, rs, updated) for _, op := range due { if _, err := s.sendApp(ctx, op.token, op.address, subject, body); err != nil { // The address is safe to log; the token never is. Leaving this // operator's row unrecorded is the retry: the next pass mails // them and nobody else. log.Printf("console: alert watch: keymail %s/%s to %s: %v", box, rule, op.address, err) continue } s.recordAlertSeenFor(box, rule, op.address, rs) } } // geoipReasonRule is the dedup key for one geoip reason (geoip spec // §10.4). The geoip rule's VALUE is a reason code, and a reason that // changes while firing (expiring becomes expired) is news the operator // must get, though firing never went false (code review R4-3). The plain // "geoip" row still carries firing/cleared; one extra row per reason marks // which reasons this address has been told about. func geoipReasonRule(code float64) string { return fmt.Sprintf("geoip@%d", int(code)) } // alertDueFor is alertDue plus the geoip reason rule. func (s *Server) alertDueFor(box, rule, address string, rs metrics.RuleState) bool { due := s.alertDue(box, rule, address, rs.Firing) if rule == "geoip" && rs.Firing && !due { due = s.alertDue(box, geoipReasonRule(rs.Value), address, true) } return due } // recordAlertSeenFor records the transition and, for geoip, which reason // this address was LAST told about: exactly one reason row is true while // firing and none after a clear, so a reason seen before (2, 3, back to 2, // or a later episode) is news again (code review R5 on R4-3). func (s *Server) recordAlertSeenFor(box, rule, address string, rs metrics.RuleState) { s.recordAlertSeen(box, rule, address, rs.Firing) if rule != "geoip" { return } for code := 1; code <= 5; code++ { s.recordAlertSeen(box, geoipReasonRule(float64(code)), address, rs.Firing && int(rs.Value) == code) } } // alertDue reports whether address must be told that (box, rule) is now in // this firing state, recording a quiet first sighting as it goes — nobody // wants mail saying the disk is fine. // // An address the console has never written to about this rule falls back to // the synthetic no-mailbox row: if the transition was already dropped for // want of any grant at all, an operator who gains a grant afterwards is not // retroactively paged about it. func (s *Server) alertDue(box, rule, address string, firing bool) bool { seen, known, err := s.store.AlertSeen(box, rule, address) if err != nil { log.Printf("console: alert watch: alert_seen %s/%s for %s: %v", box, rule, address, err) return false } if !known && address != noMailboxAddress { seen, known, err = s.store.AlertSeen(box, rule, noMailboxAddress) if err != nil { log.Printf("console: alert watch: alert_seen %s/%s: %v", box, rule, err) return false } } if known && seen == firing { return false } if !known && !firing { // First sighting of a quiet rule. Record it so a later fire is a // transition; say nothing. s.recordAlertSeen(box, rule, address, false) return false } return true } func (s *Server) recordAlertSeen(box, rule, address string, firing bool) { if err := s.store.SetAlertSeen(box, rule, address, firing); err != nil { log.Printf("console: alert watch: record %s/%s for %s: %v", box, rule, address, err) } } // alertValueText renders one rule's transition value in that rule's own // unit. Every rule internal/metrics evaluates carries a fraction EXCEPT lag, // which is keymail relay seconds (see metrics.Rules.LagSecs) — so a single // percent formatter for all of them turns the first lag alert to fire into // "92000%". Used by both operator-facing surfaces: the dashboard's alerts // table and the alert mail below. // // An unknown rule name (a future rule this console has not been taught) is // rendered as a bare number rather than being given someone else's unit: a // unitless figure is honest, a wrong unit is not. func alertValueText(rule string, value float64) string { switch rule { case "disk", "mem", "cpu": return fmt.Sprintf("%.0f%%", value*100) case "lag": return fmt.Sprintf("%.0fs", value) case "geoip": // A reason code, not a measurement (geoip spec §10.4). return geoipReasonText(int(value)) default: return strconv.FormatFloat(value, 'g', -1, 64) } } // alertMessage is the operator-facing copy. Plain sentences, and no needle // beyond the value that triggered the transition — this mail lands in an // inbox, not on the operator dashboard. func alertMessage(box, rule string, rs metrics.RuleState, updated time.Time) (subject, body string) { at := updated if rs.Firing && !rs.FiredAt.IsZero() { at = rs.FiredAt } stamp := at.UTC().Format(time.RFC3339) if rs.Firing { subject = fmt.Sprintf("[carlos] %s: %s alert", box, rule) body = fmt.Sprintf("The %s alert is firing on %s.\n\nValue at the transition: %s\nTime (UTC): %s\n\n"+ "You are getting this because you hold the CARLOS fleet-operator role.\n", rule, box, alertValueText(rule, rs.Value), stamp) return subject, body } subject = fmt.Sprintf("[carlos] %s: %s cleared", box, rule) body = fmt.Sprintf("The %s alert has cleared on %s.\n\nValue at the transition: %s\nTime (UTC): %s\n\n"+ "You are getting this because you hold the CARLOS fleet-operator role.\n", rule, box, alertValueText(rule, rs.Value), stamp) return subject, body } // geoipReasonText names a geoip rule reason code (geoip spec §10.4). func geoipReasonText(code int) string { switch code { case 1: return "database over 45 days old" case 2: return "GeoLite2 database expires within 9 days" case 3: return "database expired and removed" case 4: return "published database not loaded for over an hour" case 5: return "feed failing" } return "geoip reason " + strconv.Itoa(code) }