// Per-tenant send, bounce and complaint counts, and the limits they enforce // (tenant-email spec §9, §10). // // ENFORCEMENT IS REACTIVE. SES has no per-identity rate limit, so a cap can // only be applied by counting what has already been sent and then stopping // further sends. There are minutes of latency between the send and the // event, which means a cap of 50 is a cap of "about 50". That is stated // here rather than implied, because a caller who believes it is exact will // eventually be surprised by a bill or a complaint. package console import ( "context" "fmt" "log" "time" "github.com/carlosframework/platform/internal/mail" ) // MailCounts is one app's activity for one UTC day. type MailCounts struct { Sends int Bounces int Complaints int Rejects int // Deliveries is the denominator that makes a bounce count readable — // "12 bounces" means something different against 50 delivered than // against 50 sent and 12 delivered. Display-only, deliberately: see // BounceRate below and the migration's own note. Deliveries int } // BounceRate and ComplaintRate are zero on a day with no sends — not NaN. // A NaN compares false against every threshold, so a divide-by-zero here // would present as a limiter that silently never trips. func (c MailCounts) BounceRate() float64 { if c.Sends == 0 { return 0 } return float64(c.Bounces) / float64(c.Sends) } func (c MailCounts) ComplaintRate() float64 { if c.Sends == 0 { return 0 } return float64(c.Complaints) / float64(c.Sends) } // The thresholds SES reviews an account on. Applied per tenant so one // tenant's stale list cannot put every tenant under review. const ( BounceThreshold = 0.05 ComplaintThreshold = 0.001 // MinSendsForRates stops a single bounce on a two-message day reading // as a 50% bounce rate and pausing a healthy tenant. MinSendsForRates = 50 ) // TierCap is the daily send cap for a tier. // // An untiered or unrecognised account gets the FREE cap, never an unlimited // one. That default is what lets this ship before the billing work lands, // and the direction matters: falling back to unlimited would hand every // unknown account an uncapped estate. func TierCap(tier string) int { switch tier { case "pro": return 10000 case "starter": return 1000 default: return 50 } } // mailCountColumn maps an mail.Event* type to the column RecordMailEvent // increments. false means "not one of the four counted types" — the same // tolerance mail.ParseEvent extends to a type it does not recognise: SES // adds event types, and refusing one here would need to fail the whole // ingest rather than just skip counting it. func mailCountColumn(evType string) (string, bool) { switch evType { case mail.EventSend: return "sends", true case mail.EventBounce: return "bounces", true case mail.EventComplaint: return "complaints", true case mail.EventReject: return "rejects", true case mail.EventDelivery: return "deliveries", true default: return "", false } } // RecordMailEvent increments the one counter evType names for (acct, app, // sender, day). An event type this function does not know is silently // ignored — mirrored from mail.ParseEvent's own tolerance of unknown types, // for the same reason: SES adding a type must never turn into a 500 that // makes SNS retry forever. // // Writes mail_sender_counts alone. mail_counts stays on disk, unwritten, // until DropLegacyMailCounts removes it a release later. func (s *Store) RecordMailEvent(ctx context.Context, acct, app, sender, day, evType string) error { col, ok := mailCountColumn(evType) if !ok { return nil } if sender == "" { sender = mail.SenderAppID } q := fmt.Sprintf(`INSERT INTO mail_sender_counts (account, app, sender, day, %s) VALUES (?, ?, ?, ?, 1) ON CONFLICT(account, app, sender, day) DO UPDATE SET %s = %s + 1`, col, col, col) // A write of acct/app's row alone, under that app's permit // (claim_mail_sql.go). return s.withMailAppSQL(ctx, acct, app, "sql.RecordMailEvent", func(v *Store) error { _, err := v.db.ExecContext(ctx, q, acct, app, sender, day) return err }) } // MailCounts returns one sender's counts for one UTC day. An EMPTY sender // reads every sender the app has, which is what the status reply's // unchanged top-level figures mean and what a roll-up of an app in // per-instance mode needs. A day with no rows at all is a legitimate zero. func (s *Store) MailCounts(ctx context.Context, acct, app, sender, day string) (MailCounts, error) { q := `SELECT COALESCE(SUM(sends), 0), COALESCE(SUM(bounces), 0), COALESCE(SUM(complaints), 0), COALESCE(SUM(rejects), 0), COALESCE(SUM(deliveries), 0) FROM mail_sender_counts WHERE account = ? AND app = ? AND day = ?` args := []any{acct, app, day} if sender != "" { q += ` AND sender = ?` args = append(args, sender) } var c MailCounts // An aggregate always returns exactly one row, so there is no ErrNoRows // branch to get wrong and a quiet day reads as zeroes by construction. if err := s.db.QueryRowContext(ctx, q, args...).Scan( &c.Sends, &c.Bounces, &c.Complaints, &c.Rejects, &c.Deliveries); err != nil { return MailCounts{}, err } return c, nil } // resumeStaleCapPause is the ONE place the day-roll resume check lives // (R2, review follow-up). enforceMailLimits' only production caller is the // SES event path (recordAndEnforceMail): a paused app's IAM key is // deactivated, so it sends nothing, so no events ever arrive to run that // check, so an app alone on its own domain saw its cap pause as permanent // as before the cap/rate split — while the CLI now promises it clears // automatically. The email-status READ path (handleAPIEmailStatus) is the // other caller: a member running `carlos email status` is exactly the // moment a rolled-over daily budget should clear, and it is a moment that // happens without SES traffic. // // Both callers share this one function so they can never drift. // // Fail-safe by construction, matching "RATE AND MANUAL PAUSES NEVER // AUTO-RESUME" below: resumes ONLY when sn is currently paused (Active == // false), the pause is a CAP pause on a sender that is not PerInstance // (PauseKind == mail.PauseKindCap — a RATE pause, a PER-INSTANCE pause or a // kind-less manual/pre-existing pause, PauseKind == "", all fall through // untouched), and the pause was applied on a UTC day STRICTLY EARLIER than // now — same-day resumes would make the cap meaningless. // // Returns the freshly re-read sender when it resumes (sn's SetSenderPaused // write moves Active, PauseKind and At, so callers must not patch the // in-memory copy by hand) and whether a resume happened; sn is passed // straight back, and resumed is false, in every other case, including // sn == nil or sn.Provisioned == nil. func (s *Server) resumeStaleCapPause(ctx context.Context, acct, app, sender string, sn *mail.Sender, now time.Time) (*mail.Sender, bool, error) { if sn == nil || sn.Provisioned == nil || sn.Provisioned.Active { return sn, false, nil } // A RATE pause, a MANUAL pause and a PER-INSTANCE pause all fall through // untouched. The first two never auto-resume by design; the third must // not, because resuming reactivates every key including the standalones // and the app credential every instance still holds. if sn.Provisioned.PauseKind != mail.PauseKindCap || sn.PerInstance { return sn, false, nil } pausedDay := sn.Provisioned.At.UTC().Format("2006-01-02") today := now.UTC().Format("2006-01-02") if pausedDay >= today { // Paused earlier TODAY (or a clock oddity puts the pause in the // future) — not a day roll yet. return sn, false, nil } if _, err := mail.SetSenderPaused(ctx, s.claimMailStore(), s.cfg.MailProvisioner, acct, app, sender, false, "", now); err != nil { return nil, false, err } if a, err := s.store.AccountBySqid(acct); err != nil { log.Printf("console: mail resume audit: resolving account %s: %v", acct, err) } else if a != nil { s.audit(a, app, "ses", "mail.resumed", fmt.Sprintf("%s: daily cap pause cleared on the UTC day roll", mailAuditSubject(app, sender))) } fresh, _, err := mail.GetSender(ctx, s.cfg.Releases, acct, app, sender) if err != nil { return nil, false, err } return fresh, true, nil } // enforceMailLimits pauses a sender that has exceeded its daily cap or is // past either SES threshold, and reports why. The cap and the pause are the // sender's own, so one sender's breach leaves its siblings sending. // // The rate thresholds are applied only once MinSendsForRates messages have // gone out that day: without that floor, one bounce on a two-message morning // is a 50% bounce rate and a healthy tenant is paused before it has sent // anything real. // // A pause MUST be visible. It is written to the account's audit log with the // counts that caused it and surfaced on the app page — a silent pause // presents to the member as "email stopped working", which is the failure // this whole mechanism is supposed to prevent someone else's tenant causing. func (s *Server) enforceMailLimits(ctx context.Context, acct, app, sender string, now time.Time) (bool, string, error) { sn, _, err := mail.GetSender(ctx, s.cfg.Releases, acct, app, sender) if err != nil { return false, "", err } if sn == nil || sn.Provisioned == nil { return false, "", nil } if !sn.Provisioned.Active { fresh, resumed, err := s.resumeStaleCapPause(ctx, acct, app, sender, sn, now) if err != nil { return false, "", err } if !resumed { return false, "", nil } // Re-read via resumeStaleCapPause: SetSenderPaused just wrote a // fresh sender (Active, PauseKind and At all moved), and everything // below reads sn fresh rather than patching the in-memory copy by // hand. sn = fresh if sn == nil || sn.Provisioned == nil || !sn.Provisioned.Active { return false, "", nil } } day := now.UTC().Format("2006-01-02") // The real sender, not "": MailCounts treats "" as every sender the app // has, and this cap belongs to ONE sender — summing every sibling's // traffic into it would let one sender's cap trip on another sender's // mail, which is exactly what keying the cap to the sender is meant to // stop. c, err := s.store.MailCounts(ctx, acct, app, sender, day) if err != nil { return false, "", err } cap := TierCap("") if sn.Grant != nil && sn.Grant.DailyCap > 0 { cap = sn.Grant.DailyCap } var reason, kind string switch { case c.Sends >= cap: reason = fmt.Sprintf("daily cap %d reached (%d sent)", cap, c.Sends) kind = mail.PauseKindCap // The send floor is what stops one bounce on a two-message morning // reading as a 50% bounce rate and pausing a healthy tenant. case c.Sends >= MinSendsForRates && c.BounceRate() > BounceThreshold: reason = fmt.Sprintf("bounce rate %.1f%% over %.0f%% (%d of %d)", c.BounceRate()*100, BounceThreshold*100, c.Bounces, c.Sends) kind = mail.PauseKindRate case c.Sends >= MinSendsForRates && c.ComplaintRate() > ComplaintThreshold: reason = fmt.Sprintf("complaint rate %.2f%% over %.1f%% (%d of %d)", c.ComplaintRate()*100, ComplaintThreshold*100, c.Complaints, c.Sends) kind = mail.PauseKindRate default: return false, "", nil } if _, err := mail.SetSenderPaused(ctx, s.claimMailStore(), s.cfg.MailProvisioner, acct, app, sender, true, kind, now); err != nil { return false, "", err } // Audit AFTER the write it announces, and never before — a pause the // member cannot see is the failure §9 exists to prevent someone else's // tenant causing. if a, err := s.store.AccountBySqid(acct); err != nil { log.Printf("console: mail pause audit: resolving account %s: %v", acct, err) } else if a != nil { s.audit(a, app, "ses", "mail.paused", fmt.Sprintf("%s: %s", mailAuditSubject(app, sender), reason)) } return true, reason, nil } // mailAuditSubject names what a pause or resume audit line is about. These // lines render on the app overview page, so they are member-facing copy // that never went through this repo's copy-review gate for the "/sender" // form. Every app has exactly one sender today (the app sender, mail. // SenderAppID), so naming it on that line would add "/app" for zero // information a member could use, and would be an unreviewed copy change. // Only a sender OTHER than the app sender earns the "/sender" suffix, which // keeps every live app's audit line byte-identical to what it already // prints. func mailAuditSubject(app, sender string) string { if sender == mail.SenderAppID { return app } return app + "/" + sender } // RATE AND MANUAL PAUSES NEVER AUTO-RESUME. A RATE pause that lifted itself // at midnight would let an abusive tenant breach SES's account-review // thresholds again every day forever, which is not a limit — it is a // schedule. A MANUAL (operator) pause is a human's deliberate decision, // which only a human — through the door — gets to undo. Resume for either // is a decision, taken through the door. // // A CAP pause is different (I3, review follow-up, amending the rule this // comment used to state without exception): it is computed from PER-DAY // counts, so "never auto-resumes" turned an ordinary tier limit into a // permanent one — see the day-roll check above.