package console import ( "context" "encoding/json" "strings" "testing" "time" "github.com/carlosframework/platform/internal/metrics" "github.com/carlosframework/platform/internal/release" ) // sentMail is one captured sendApp call. The token is captured so a test can // assert each operator is mailed with THEIR OWN grant, never a shared one. type sentMail struct { token, to, subject, body string } // alertFixture is a console fixture with a captured sendApp. err makes // sends fail; failFor narrows the failure to one recipient, which is how // the per-recipient retry is pinned. type alertFixture struct { *consoleFixture sent []sentMail err error failFor string } // newCapturedFixture wires the capture and nothing else — no operators, so // a test can build its own recipient shape. func newCapturedFixture(t *testing.T) *alertFixture { t.Helper() fx := newConsoleFixture(t) af := &alertFixture{consoleFixture: fx} fx.srv.sendApp = func(_ context.Context, token, to, subject, body string) (string, error) { af.sent = append(af.sent, sentMail{token, to, subject, body}) if af.err != nil && (af.failFor == "" || af.failFor == to) { return "", af.err } return "msg-1", nil } return af } // newAlertFixture is the common shape: two operators, only one of whom ever // granted the console send access. func newAlertFixture(t *testing.T) *alertFixture { t.Helper() af := newCapturedFixture(t) for _, addr := range []string{"ops-a@keymail.dev", "ops-b@keymail.dev"} { if err := af.st.SetOperator(addr, true); err != nil { t.Fatal(err) } } if err := af.st.SetGrantToken("ops-a@keymail.dev", "grant-a", time.Now()); err != nil { t.Fatal(err) } return af } func (af *alertFixture) writeAlert(t *testing.T, st metrics.AlertState) { t.Helper() b, err := json.Marshal(st) if err != nil { t.Fatal(err) } // Publishing is a CAS write (create-only on ""), so re-publishing a box's // state — which is what a fire→clear transition is — needs the etag. key := metrics.AlertKey(st.Box) etag := "" if _, cur, err := af.rel.Get(context.Background(), key); err == nil { etag = cur } if _, err := af.rel.Put(context.Background(), key, b, etag); err != nil { t.Fatal(err) } } func (af *alertFixture) pass(t *testing.T) { t.Helper() af.sent = nil if err := af.srv.alertPass(context.Background()); err != nil { t.Fatal(err) } } func firingDisk(firing bool) metrics.AlertState { rs := metrics.RuleState{Firing: firing, Value: 0.95} if firing { rs.FiredAt = time.Date(2026, 8, 2, 10, 0, 0, 0, time.UTC) rs.Since = rs.FiredAt } else { rs.Value = 0.80 } return metrics.AlertState{ V: 1, Box: "box-a", UpdatedAt: time.Date(2026, 8, 2, 10, 5, 0, 0, time.UTC), Rules: map[string]metrics.RuleState{"disk": rs}, } } // TestAlertPassFiresOnceThenGoesQuiet is the spec's core promise: a keymail // on the TRANSITION, to the operators who hold a grant, and silence while // the alert stays in the state the console already reported. func TestAlertPassFiresOnceThenGoesQuiet(t *testing.T) { af := newAlertFixture(t) af.writeAlert(t, firingDisk(true)) af.pass(t) if len(af.sent) != 1 { t.Fatalf("first pass sent %d keymails, want 1: %+v", len(af.sent), af.sent) } got := af.sent[0] if got.to != "ops-a@keymail.dev" || got.token != "grant-a" { t.Fatalf("sent to %q with token %q, want ops-a@keymail.dev/grant-a", got.to, got.token) } if got.subject != "[carlos] box-a: disk alert" { t.Fatalf("subject = %q", got.subject) } // "95%", not "0.95": the mail carries the rule's own unit, the same as // the operator dashboard's alerts table (see alertValueText). for _, want := range []string{"disk", "box-a", "95%", "2026-08-02T10:00:00Z"} { if !strings.Contains(got.body, want) { t.Fatalf("body is missing %q:\n%s", want, got.body) } } if strings.Contains(got.body, "grant-a") { t.Fatal("body leaked the grant token") } // Recorded: the same state on the next pass says nothing. firing, known, err := af.st.AlertSeen("box-a", "disk", "ops-a@keymail.dev") if err != nil || !known || !firing { t.Fatalf("AlertSeen = %v,%v,%v — want true,true,nil", firing, known, err) } af.pass(t) if len(af.sent) != 0 { t.Fatalf("second pass sent %d keymails, want 0: %+v", len(af.sent), af.sent) } } // TestAlertPassClears covers the other half of the transition: the alert // going away is news too. func TestAlertPassClears(t *testing.T) { af := newAlertFixture(t) af.writeAlert(t, firingDisk(true)) af.pass(t) af.writeAlert(t, firingDisk(false)) af.pass(t) if len(af.sent) != 1 { t.Fatalf("clear pass sent %d keymails, want 1: %+v", len(af.sent), af.sent) } if af.sent[0].subject != "[carlos] box-a: disk cleared" { t.Fatalf("subject = %q", af.sent[0].subject) } af.pass(t) if len(af.sent) != 0 { t.Fatalf("pass after clear sent %d keymails, want 0", len(af.sent)) } } // TestAlertPassRetriesAfterSendFailure: a failed send leaves the transition // unrecorded, so the next tick delivers it. A missed page is worse than a // duplicate one. func TestAlertPassRetriesAfterSendFailure(t *testing.T) { af := newAlertFixture(t) af.writeAlert(t, firingDisk(true)) af.err = context.DeadlineExceeded af.pass(t) if len(af.sent) != 1 { t.Fatalf("failing pass attempted %d sends, want 1", len(af.sent)) } if _, known, err := af.st.AlertSeen("box-a", "disk", "ops-a@keymail.dev"); err != nil || known { t.Fatalf("AlertSeen recorded despite a send failure (known=%v, err=%v)", known, err) } af.err = nil af.pass(t) if len(af.sent) != 1 { t.Fatalf("retry pass sent %d keymails, want 1", len(af.sent)) } if firing, known, err := af.st.AlertSeen("box-a", "disk", "ops-a@keymail.dev"); err != nil || !known || !firing { t.Fatalf("AlertSeen = %v,%v,%v after a successful retry", firing, known, err) } } // TestAlertPassRetriesOnlyTheFailedOperator is why alert_seen is keyed per // recipient: one operator's dead grant must not re-page every healthy // operator on the fleet, tick after tick. func TestAlertPassRetriesOnlyTheFailedOperator(t *testing.T) { af := newAlertFixture(t) if err := af.st.SetGrantToken("ops-b@keymail.dev", "grant-b", time.Now()); err != nil { t.Fatal(err) } af.writeAlert(t, firingDisk(true)) af.err = context.DeadlineExceeded af.failFor = "ops-b@keymail.dev" af.pass(t) if len(af.sent) != 2 { t.Fatalf("first pass attempted %d sends, want 2: %+v", len(af.sent), af.sent) } if firing, known, err := af.st.AlertSeen("box-a", "disk", "ops-a@keymail.dev"); err != nil || !known || !firing { t.Fatalf("ops-a AlertSeen = %v %v %v — a healthy delivery was not recorded", firing, known, err) } if _, known, err := af.st.AlertSeen("box-a", "disk", "ops-b@keymail.dev"); err != nil || known { t.Fatalf("ops-b AlertSeen recorded despite a failed send (known=%v, err=%v)", known, err) } af.err = nil af.pass(t) if len(af.sent) != 1 || af.sent[0].to != "ops-b@keymail.dev" { t.Fatalf("retry pass sent %+v, want exactly one mail to ops-b", af.sent) } if af.sent[0].token != "grant-b" { t.Fatalf("retry used token %q, want ops-b's own grant", af.sent[0].token) } af.pass(t) if len(af.sent) != 0 { t.Fatalf("pass after the retry sent %d keymails, want 0", len(af.sent)) } } // TestAlertPassNoMailableOperator pins the one path where an alert is // genuinely dropped: nobody holds a grant, so there is nothing to send and // the transition is recorded against the synthetic recipient so it does not // replay every minute forever. The honest cost — documented, and asserted // here rather than left to be discovered — is that an operator who gains a // grant afterwards is never told about that transition. func TestAlertPassNoMailableOperator(t *testing.T) { af := newCapturedFixture(t) if err := af.st.SetOperator("ops-b@keymail.dev", true); err != nil { t.Fatal(err) } af.writeAlert(t, firingDisk(true)) af.pass(t) if len(af.sent) != 0 { t.Fatalf("sent %d keymails with no grant anywhere: %+v", len(af.sent), af.sent) } if firing, known, err := af.st.AlertSeen("box-a", "disk", ""); err != nil || !known || !firing { t.Fatalf("synthetic AlertSeen = %v %v %v, want true true nil — the drop was not recorded", firing, known, err) } // The grant arrives after the fact. The alert is still firing, but the // console has already recorded the transition as handled: no catch-up. if err := af.st.SetGrantToken("ops-b@keymail.dev", "grant-b", time.Now()); err != nil { t.Fatal(err) } af.pass(t) if len(af.sent) != 0 { t.Fatalf("a newly-granted operator was retroactively paged: %+v", af.sent) } // But the NEXT transition reaches them normally. af.writeAlert(t, firingDisk(false)) af.pass(t) if len(af.sent) != 1 || af.sent[0].to != "ops-b@keymail.dev" { t.Fatalf("clear after the grant arrived sent %+v, want one mail to ops-b", af.sent) } } // TestAlertPassQuietFirstSighting: a rule the console has never seen that is // NOT firing is recorded silently. Nobody wants a keymail saying the disk is // fine. func TestAlertPassQuietFirstSighting(t *testing.T) { af := newAlertFixture(t) af.writeAlert(t, firingDisk(false)) af.pass(t) if len(af.sent) != 0 { t.Fatalf("quiet first sighting sent %d keymails, want 0: %+v", len(af.sent), af.sent) } if firing, known, err := af.st.AlertSeen("box-a", "disk", "ops-a@keymail.dev"); err != nil || !known || firing { t.Fatalf("AlertSeen = %v,%v,%v — want false,true,nil", firing, known, err) } } // TestAlertPassSurvivesJunk: an unparsable object under alerts/ is skipped, // and the boxes either side of it are still processed. A watcher that dies // on one bad file is a watcher that stops paging. func TestAlertPassSurvivesJunk(t *testing.T) { af := newAlertFixture(t) if _, err := af.rel.Put(context.Background(), "alerts/boxes/box-junk.json", []byte("{not json"), ""); err != nil { t.Fatal(err) } af.writeAlert(t, firingDisk(true)) af.pass(t) if len(af.sent) != 1 { t.Fatalf("pass sent %d keymails, want 1: %+v", len(af.sent), af.sent) } } // TestAlertPassEmptyBucket: no alerts/ prefix at all is a no-op, not an // error — a fleet that has never published an alert is the normal case. func TestAlertPassEmptyBucket(t *testing.T) { fx := newConsoleFixture(t) fx.srv.sendApp = func(context.Context, string, string, string, string) (string, error) { t.Fatal("sent a keymail with no alerts published") return "", nil } if err := fx.srv.alertPass(context.Background()); err != nil { t.Fatal(err) } } // TestNewServerDefaultsSendApp: production wiring goes through the console's // own keymail client; nothing else in NewServer starts a goroutine. func TestNewServerDefaultsSendApp(t *testing.T) { fx := newConsoleFixture(t) if fx.srv.sendApp == nil { t.Fatal("NewServer left sendApp nil — the watcher would never deliver") } var _ release.Store = fx.rel } func geoipAlert(code float64, firing bool) metrics.AlertState { rs := metrics.RuleState{Firing: firing, Value: code, FiredAt: time.Date(2026, 10, 6, 10, 0, 0, 0, time.UTC)} return metrics.AlertState{V: 1, Box: "box-a", UpdatedAt: time.Date(2026, 10, 6, 10, 5, 0, 0, time.UTC), Rules: map[string]metrics.RuleState{"geoip": rs}} } // Code review R4-3: a geoip reason that changes while firing is mailed. func TestAlertPassMailsAGeoIPReasonChange(t *testing.T) { af := newAlertFixture(t) step := func(st metrics.AlertState, wantMails int, wantBody string) { t.Helper() af.writeAlert(t, st) af.pass(t) // resets af.sent got := af.sent if len(got) != wantMails { t.Fatalf("mails = %d, want %d: %+v", len(got), wantMails, got) } if wantBody != "" && !strings.Contains(got[0].body, wantBody) { t.Fatalf("body lacks %q: %s", wantBody, got[0].body) } } step(geoipAlert(2, true), 1, "expires within 9 days") step(geoipAlert(2, true), 0, "") step(geoipAlert(3, true), 1, "expired and removed") step(geoipAlert(0, false), 1, "") // A later episode, and a return to an earlier reason within one: each // is news (code review R5 on R4-3). step(geoipAlert(2, true), 1, "expires within 9 days") step(geoipAlert(3, true), 1, "expired and removed") step(geoipAlert(2, true), 1, "expires within 9 days") step(geoipAlert(2, true), 0, "") }