package host import ( "context" "errors" "os" "path/filepath" "strings" "testing" "time" "github.com/aws/aws-sdk-go-v2/service/sns" "github.com/carlosframework/platform/internal/metrics" "github.com/carlosframework/platform/internal/release" ) // fakeSNS records every Publish and can be told to fail. It is the only SNS // the tests in this file ever talk to. type fakeSNS struct { in []*sns.PublishInput err error } func (f *fakeSNS) Publish(_ context.Context, in *sns.PublishInput, _ ...func(*sns.Options)) (*sns.PublishOutput, error) { f.in = append(f.in, in) if f.err != nil { return nil, f.err } return &sns.PublishOutput{}, nil } const testTopicARN = "arn:aws:sns:eu-west-1:271376211898:carlos-flagship-ie-alerts" var sinkTestAt = time.Date(2026, 10, 5, 12, 0, 0, 0, time.UTC) func TestFormatAlertFireAndClear(t *testing.T) { rules := metrics.DefaultRules() cases := []struct { name string ev metrics.Event subjectHas []string bodyHas []string subjectNot []string bodyNotHave []string }{ { name: "disk fires", ev: metrics.Event{Rule: "disk", Firing: true, Value: 0.917, Box: "ada-stargazing"}, subjectHas: []string{"carlos alert", "disk", "92%", "ada-stargazing"}, bodyHas: []string{"disk", "92%", "ada-stargazing", "90%", "2026-10-05 12:00 UTC", "clears"}, }, { name: "disk clears", ev: metrics.Event{Rule: "disk", Firing: false, Value: 0.30, Box: "ada-stargazing"}, subjectHas: []string{"carlos alert cleared", "disk", "30%", "ada-stargazing"}, bodyHas: []string{"back", "30%", "ada-stargazing"}, }, { name: "mem fires with its sustain window", ev: metrics.Event{Rule: "mem", Firing: true, Value: 0.94, Box: "box-a"}, subjectHas: []string{"memory", "94%", "box-a"}, bodyHas: []string{"10 minutes", "90%"}, }, { name: "cpu fires with its sustain window", ev: metrics.Event{Rule: "cpu", Firing: true, Value: 0.97, Box: "box-a"}, subjectHas: []string{"cpu", "97%", "box-a"}, bodyHas: []string{"15 minutes", "95%"}, }, { name: "lag is seconds not a percentage", ev: metrics.Event{Rule: "lag", Firing: true, Value: 1200, Box: "box-a"}, subjectHas: []string{"lag", "1200 seconds"}, subjectNot: []string{"%"}, bodyHas: []string{"900 seconds"}, }, { name: "an unknown rule still reads", ev: metrics.Event{Rule: "newrule", Firing: true, Value: 0.5, Box: "box-a"}, subjectHas: []string{"newrule", "box-a"}, }, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { subject, body := formatAlert(rules, sinkTestAt, c.ev) for _, w := range c.subjectHas { if !strings.Contains(subject, w) { t.Errorf("subject %q missing %q", subject, w) } } for _, w := range c.subjectNot { if strings.Contains(subject, w) { t.Errorf("subject %q should not contain %q", subject, w) } } for _, w := range c.bodyHas { if !strings.Contains(body, w) { t.Errorf("body missing %q:\n%s", w, body) } } // SNS rejects a subject over 100 chars, or with a newline or // any non-printable-ASCII byte; the mail would never be sent. if len(subject) == 0 || len(subject) > 100 { t.Errorf("subject length %d, want 1..100: %q", len(subject), subject) } for _, r := range subject { if r < 0x20 || r > 0x7e { t.Errorf("subject has a character SNS refuses: %q", subject) break } } for _, s := range []string{subject, body} { if strings.ContainsRune(s, '\u2014') || strings.ContainsRune(s, '\u2013') { t.Errorf("em or en dash in copy: %q", s) } } }) } } // TestFormatAlertCarriesNoSecrets: the message is the event, the rule's // level and a time. No URL, ARN, credential, or env value can reach it, // because nothing but those three inputs is passed in. func TestFormatAlertCarriesNoSecrets(t *testing.T) { subject, body := formatAlert(metrics.DefaultRules(), sinkTestAt, metrics.Event{Rule: "disk", Firing: true, Value: 0.95, Box: "box-a"}) for _, banned := range []string{"http", "arn:", "secret", "token", "key", "password"} { if strings.Contains(strings.ToLower(subject+body), banned) { t.Errorf("message contains %q:\n%s\n%s", banned, subject, body) } } } func TestAlertSinksDeliverSNSOnly(t *testing.T) { pub := &fakeSNS{} counts := &alertCounters{} s := newAlertSinks("", nil, testTopicARN, pub, counts) s.deliver(context.Background(), metrics.DefaultRules(), sinkTestAt, metrics.Event{Rule: "disk", Firing: true, Value: 0.95, Box: "box-a"}) if len(pub.in) != 1 { t.Fatalf("publishes = %d, want 1", len(pub.in)) } in := pub.in[0] if got := deref(in.TopicArn); got != testTopicARN { t.Errorf("TopicArn = %q", got) } if !strings.Contains(deref(in.Subject), "disk") || !strings.Contains(deref(in.Message), "box-a") { t.Errorf("subject/message not readable: %q / %q", deref(in.Subject), deref(in.Message)) } if counts.sns.Load() != 0 || counts.webhook.Load() != 0 { t.Errorf("no failures expected, got sns=%d webhook=%d", counts.sns.Load(), counts.webhook.Load()) } } func TestAlertSinksDeliverWebhookOnlyIsUnchanged(t *testing.T) { var posts [][]byte var urls []string post := func(url string, body []byte) error { urls = append(urls, url) posts = append(posts, body) return nil } s := newAlertSinks("http://example.test/hook", post, "", nil, &alertCounters{}) s.deliver(context.Background(), metrics.DefaultRules(), sinkTestAt, metrics.Event{Rule: "disk", Firing: true, Value: 0.95, Box: "box-a"}) if len(posts) != 1 || urls[0] != "http://example.test/hook" { t.Fatalf("posts = %d urls = %v", len(posts), urls) } if string(posts[0]) != `{"rule":"disk","firing":true,"value":0.95,"box":"box-a"}` { t.Errorf("webhook payload changed: %s", posts[0]) } } func TestAlertSinksBothAreUsed(t *testing.T) { pub := &fakeSNS{} posts := 0 post := func(string, []byte) error { posts++; return nil } s := newAlertSinks("http://example.test/hook", post, testTopicARN, pub, &alertCounters{}) s.deliver(context.Background(), metrics.DefaultRules(), sinkTestAt, metrics.Event{Rule: "mem", Firing: true, Value: 0.95, Box: "box-a"}) if posts != 1 || len(pub.in) != 1 { t.Errorf("webhook posts = %d, sns publishes = %d, want 1 and 1", posts, len(pub.in)) } } func TestAlertSinksNoneConfiguredDoesNothing(t *testing.T) { s := newAlertSinks("", nil, "", nil, &alertCounters{}) s.deliver(context.Background(), metrics.DefaultRules(), sinkTestAt, metrics.Event{Rule: "disk", Firing: true, Value: 0.95, Box: "box-a"}) var nilSinks *alertSinks nilSinks.deliver(context.Background(), metrics.DefaultRules(), sinkTestAt, metrics.Event{Rule: "disk", Firing: true, Value: 0.95, Box: "box-a"}) } // TestAlertSinksSNSFailureIsCountedAndNeverBlocksTheWebhook: a failing sink // is logged and counted, the other sink still runs, and nothing panics or // returns an error that could fail the tick. func TestAlertSinksSNSFailureIsCountedAndNeverBlocksTheWebhook(t *testing.T) { pub := &fakeSNS{err: errors.New("AccessDenied: not authorized to perform sns:Publish")} posts := 0 post := func(string, []byte) error { posts++; return nil } counts := &alertCounters{} s := newAlertSinks("http://example.test/hook", post, testTopicARN, pub, counts) s.deliver(context.Background(), metrics.DefaultRules(), sinkTestAt, metrics.Event{Rule: "disk", Firing: true, Value: 0.95, Box: "box-a"}) if posts != 1 { t.Errorf("webhook posts = %d, want 1 even though sns failed", posts) } if counts.sns.Load() != 1 { t.Errorf("sns failures = %d, want 1", counts.sns.Load()) } if counts.webhook.Load() != 0 { t.Errorf("webhook failures = %d, want 0", counts.webhook.Load()) } } func TestAlertSinksWebhookFailureIsCounted(t *testing.T) { pub := &fakeSNS{} post := func(string, []byte) error { return errors.New("boom") } counts := &alertCounters{} s := newAlertSinks("http://example.test/hook", post, testTopicARN, pub, counts) s.deliver(context.Background(), metrics.DefaultRules(), sinkTestAt, metrics.Event{Rule: "disk", Firing: true, Value: 0.95, Box: "box-a"}) if counts.webhook.Load() != 1 || len(pub.in) != 1 { t.Errorf("webhook failures = %d (want 1), sns publishes = %d (want 1)", counts.webhook.Load(), len(pub.in)) } } // TestAlertTickDeliversThroughSinks: alertTick hands each transition to the // sinks exactly once, and a steady state sends nothing more. func TestAlertTickDeliversThroughSinks(t *testing.T) { ctx := context.Background() store := release.NewDirStore(t.TempDir()) statePath := filepath.Join(t.TempDir(), "monitor", "alerts.json") box := "box-a" rules := metrics.DefaultRules() st := &metrics.AlertState{V: 1, Box: box, Rules: map[string]metrics.RuleState{}} pub := &fakeSNS{} sinks := newAlertSinks("", nil, testTopicARN, pub, &alertCounters{}) disk := 0.95 latest := metrics.Sample{At: sinkTestAt, Host: metrics.HostSample{DiskFrac: &disk}} if err := alertTick(ctx, store, box, rules, st, latest, statePath, sinks); err != nil { t.Fatal(err) } if err := alertTick(ctx, store, box, rules, st, latest, statePath, sinks); err != nil { t.Fatal(err) } if len(pub.in) != 1 { t.Fatalf("publishes = %d, want 1 (second tick has no transition)", len(pub.in)) } if !strings.Contains(deref(pub.in[0].Message), box) { t.Errorf("message does not name the box: %q", deref(pub.in[0].Message)) } // Clearing is a transition too. low := 0.30 cleared := metrics.Sample{At: sinkTestAt.Add(time.Minute), Host: metrics.HostSample{DiskFrac: &low}} if err := alertTick(ctx, store, box, rules, st, cleared, statePath, sinks); err != nil { t.Fatal(err) } if len(pub.in) != 2 || !strings.Contains(deref(pub.in[1].Subject), "cleared") { t.Errorf("clear not published: %d publishes", len(pub.in)) } if _, err := os.Stat(statePath); err != nil { t.Errorf("state not persisted: %v", err) } } func TestParseAlertTopicARN(t *testing.T) { cases := []struct { in string wantRegion string wantErr bool }{ {testTopicARN, "eu-west-1", false}, {"arn:aws:sns:ap-southeast-2:271376211898:carlos-flagship-au-alerts", "ap-southeast-2", false}, {" " + testTopicARN + "\n", "eu-west-1", false}, {"", "", true}, {"not-an-arn", "", true}, {"arn:aws:s3:::some-bucket", "", true}, {"arn:aws:sqs:eu-west-1:271376211898:queue", "", true}, {"arn:aws:sns:::topic", "", true}, {"arn:aws:sns:eu-west-1:271376211898:", "", true}, // A subscription ARN is topic ARN plus ":". It is // an sns ARN with all the right fields, and publishing to it fails. {testTopicARN + ":8f4c0e7a-1b2c-4d3e-9f50-0123456789ab", "", true}, {"arn:aws:sns:eu-west-1:271376211898:topic:subscription-id", "", true}, {"arn:aws:sns:eu-west-1:271376211898:a:b", "", true}, {"arn:aws:sns:eu-west-1:271376211898:name.fifo", "", true}, {"arn:aws:sns:eu-west-1:271376211898:bad name", "", true}, {"arn:aws:sns:eu-west-1:27137621189:carlos-alerts", "", true}, {"arn:aws:sns:eu-west-1:abcdefghijkl:carlos-alerts", "", true}, {"arn:aws:sns:eu-west-1:271376211898:carlos-alerts\nx", "", true}, {"arn:aws:sns:eu-west-1:271376211898:*", "", true}, } for _, c := range cases { region, err := parseAlertTopicARN(c.in) if (err != nil) != c.wantErr || region != c.wantRegion { t.Errorf("parseAlertTopicARN(%q) = %q, %v; want %q, err=%v", c.in, region, err, c.wantRegion, c.wantErr) } } } func TestAlertDeliveryProbe(t *testing.T) { env := func(m map[string]string) func(string) string { return func(k string) string { return m[k] } } counts := &alertCounters{} got := alertDeliveryProbe(env(nil), counts) if got.Webhook || got.SNS || got.Delivered() { t.Errorf("nothing set: %+v, want nothing delivered", got) } got = alertDeliveryProbe(env(map[string]string{"CARLOS_ALERT_WEBHOOK": "http://x"}), counts) if !got.Webhook || got.SNS || !got.Delivered() { t.Errorf("webhook only: %+v", got) } // A valid ARN alone is not a sink until buildAlertSinks has built the // client: before that (a stalled build, or a metrics loop that exited // first) the box must not claim sns. got = alertDeliveryProbe(env(map[string]string{"CARLOS_ALERT_SNS_TOPIC_ARN": testTopicARN}), counts) if got.SNS || got.SNSUnavailable || got.Delivered() { t.Errorf("sink not yet built must not read as sns: %+v", got) } counts.snsInit.Store(snsInitOK) got = alertDeliveryProbe(env(map[string]string{"CARLOS_ALERT_SNS_TOPIC_ARN": testTopicARN}), counts) if got.Webhook || !got.SNS || !got.Delivered() { t.Errorf("sns only: %+v", got) } // A malformed ARN is not a sink: the box has nowhere to deliver, and // the snapshot must say so rather than claim an sns sink it cannot use. got = alertDeliveryProbe(env(map[string]string{"CARLOS_ALERT_SNS_TOPIC_ARN": "nonsense"}), counts) if got.SNS || got.Delivered() { t.Errorf("bad arn counted as a sink: %+v", got) } counts.sns.Add(3) counts.webhook.Add(2) got = alertDeliveryProbe(env(map[string]string{"CARLOS_ALERT_SNS_TOPIC_ARN": testTopicARN}), counts) if got.SNSFailures != 3 || got.WebhookFailures != 2 { t.Errorf("failure counts not carried: %+v", got) } } // fakeClientFactory is the buildAlertSinks seam: it records the region it // was asked for and either hands back a fake or fails. type fakeClientFactory struct { regions []string err error pub *fakeSNS } func (f *fakeClientFactory) build(_ context.Context, region string) (snsPublisher, error) { f.regions = append(f.regions, region) if f.err != nil { return nil, f.err } return f.pub, nil } func TestBuildAlertSinksInitialisedSNS(t *testing.T) { env := func(m map[string]string) func(string) string { return func(k string) string { return m[k] } } counts := &alertCounters{} f := &fakeClientFactory{pub: &fakeSNS{}} sinks := buildAlertSinks(context.Background(), env(map[string]string{"CARLOS_ALERT_SNS_TOPIC_ARN": testTopicARN}), counts, f.build) if len(f.regions) != 1 || f.regions[0] != "eu-west-1" { t.Fatalf("client built for %v, want the ARN's region eu-west-1", f.regions) } sinks.deliver(context.Background(), metrics.DefaultRules(), sinkTestAt, metrics.Event{Rule: "disk", Firing: true, Value: 0.95, Box: "box-a"}) if len(f.pub.in) != 1 { t.Fatalf("publishes = %d, want 1", len(f.pub.in)) } got := alertDeliveryProbe(env(map[string]string{"CARLOS_ALERT_SNS_TOPIC_ARN": testTopicARN}), counts) if !got.SNS || got.SNSUnavailable || got.SNSFailures != 0 { t.Errorf("a working sink must read as sns: %+v", got) } } // TestBuildAlertSinksClientFailureIsVisible is the review's finding: with a // valid ARN in the env but a client that cannot be built (no region, bad // profile), the sink is off. The status probe must say so, not report the // env as if it were a working sink with zero failures. func TestBuildAlertSinksClientFailureIsVisible(t *testing.T) { envMap := map[string]string{"CARLOS_ALERT_SNS_TOPIC_ARN": testTopicARN} env := func(k string) string { return envMap[k] } counts := &alertCounters{} f := &fakeClientFactory{err: errors.New("failed to get shared config profile")} sinks := buildAlertSinks(context.Background(), env, counts, f.build) // Delivering must neither panic nor publish. sinks.deliver(context.Background(), metrics.DefaultRules(), sinkTestAt, metrics.Event{Rule: "disk", Firing: true, Value: 0.95, Box: "box-a"}) got := alertDeliveryProbe(env, counts) if got.SNS || !got.SNSUnavailable || got.Delivered() { t.Errorf("failed init must read as unavailable and not delivered: %+v", got) } if got.SNSFailures < 1 { t.Errorf("failed init must be counted, got %d", got.SNSFailures) } // With a webhook as well, the box still delivers, and says what is wrong. envMap["CARLOS_ALERT_WEBHOOK"] = "http://example.test/hook" got = alertDeliveryProbe(env, counts) if !got.Webhook || got.SNS || !got.SNSUnavailable || !got.Delivered() { t.Errorf("webhook alone must still deliver: %+v", got) } } func TestBuildAlertSinksBadARNIsNotASinkAndBuildsNoClient(t *testing.T) { counts := &alertCounters{} f := &fakeClientFactory{pub: &fakeSNS{}} env := func(k string) string { if k == "CARLOS_ALERT_SNS_TOPIC_ARN" { return testTopicARN + ":subscription-id" } return "" } buildAlertSinks(context.Background(), env, counts, f.build) if len(f.regions) != 0 { t.Errorf("a subscription ARN must not build a client: %v", f.regions) } got := alertDeliveryProbe(env, counts) if got.SNS || got.Delivered() { t.Errorf("subscription ARN counted as a sink: %+v", got) } } func TestBuildAlertSinksNothingConfigured(t *testing.T) { counts := &alertCounters{} f := &fakeClientFactory{pub: &fakeSNS{}} buildAlertSinks(context.Background(), func(string) string { return "" }, counts, f.build) if len(f.regions) != 0 { t.Errorf("no ARN set, yet a client was built: %v", f.regions) } got := alertDeliveryProbe(func(string) string { return "" }, counts) if got.Delivered() || got.SNSUnavailable { t.Errorf("nothing configured: %+v", got) } } func deref(s *string) string { if s == nil { return "" } return *s }