// SES event parsing (spec §9). Events are the abuse signal AND the only // source of a per-tenant send count — SES has no per-identity rate limit to // read, so a cap can only be enforced by counting what already went out. package mail import ( "encoding/json" "fmt" "strings" "time" "unicode" "unicode/utf8" ) const ( EventSend = "Send" EventBounce = "Bounce" EventComplaint = "Complaint" EventReject = "Reject" EventDelivery = "Delivery" EventDeliveryDelay = "DeliveryDelay" ) // Event is the part of an SES notification this package acts on. type Event struct { Type string Domain string From string MessageID string // At is when what the event reports happened: the bounce, the delivery, // the complaint — falling back to the send, which is all a Send or a // Reject has. SentAt is always the send. At time.Time SentAt time.Time // Caller is the IAM identity SES says sent the message — the automatic // "ses:caller-identity" tag. CARLOS mints one per app (UserName), so // where the sending domain can only name every app that shares it, this // names the one app that actually sent. Empty when SES did not say; // AppOfCaller reads it back. Caller string // Destination is who the message was addressed to (mail.destination). Destination []string // Outcomes is what this event says about each recipient it concerns: // the bounced, complained-about, delivered or delayed addresses, each // with what the far end said. A Send or a Reject is about the whole // message and carries none. Outcomes []Outcome // Detail is the event-wide explanation, empty where the type has none. Detail EventDetail // Incomplete says why part of what only the log uses could not be read — // a field in a shape this code did not expect. The event is still good // for the counts; the caller logs this. Incomplete string } // Outcome is one recipient's part in an event. type Outcome struct { Recipient string // Status is the enhanced status code the receiving server gave, e.g. // "5.1.1". Empty on a delivery and a complaint. Status string // Diagnostic is the receiving server's own words: the bounce or delay // diagnostic, or the SMTP response it accepted a delivery with. Diagnostic string } // EventDetail is what an event says about the message as a whole. type EventDetail struct { BounceType string // Permanent, Transient, Undetermined BounceSubType string // General, NoEmail, Suppressed, MailboxFull, ... ComplaintType string // abuse, fraud, not-spam, ... when the provider said RejectReason string // e.g. "Bad content" DelayType string // e.g. MailboxFull, TransientCommunicationFailure } // The longest a field from an event is kept at. An address is at most 254 // octets by RFC 5321; a diagnostic is the far end's free text and has no // limit of its own, so it gets one here — enough for any real explanation, // not enough for a receiving server to fill a table with. const ( maxAddressLen = 254 maxDiagnosticLen = 1000 maxDetailLen = 64 ) // cleanField drops control characters and bounds the length. Every string // that comes out of an event and is kept goes through it: the event is // signed by SNS, but the text in it was written by whichever mail server // answered, and it ends up in a table and on a terminal. func cleanField(s string, max int) string { s = strings.Map(func(r rune) rune { // C0, DEL and C1 (U+009B is a CSI on some terminals), and the // invisible formatting class — bidi overrides among it — which can // make a line on a terminal say something the bytes do not. if unicode.IsControl(r) || unicode.Is(unicode.Cf, r) { return ' ' } return r }, strings.TrimSpace(s)) if len(s) > max { s = s[:max] for len(s) > 0 && !utf8.ValidString(s) { s = s[:len(s)-1] } } return s } // DomainOf extracts the lowercase domain from an address, tolerating a // display-name wrapper. Empty when there is no domain to find. Strips one // trailing dot and any stray angle bracket. func DomainOf(address string) string { s := strings.TrimSpace(address) if i := strings.LastIndex(s, "<"); i >= 0 { if j := strings.LastIndex(s, ">"); j > i { s = s[i+1 : j] } } at := strings.LastIndex(s, "@") if at < 0 || at == len(s)-1 { return "" } dom := strings.TrimSpace(s[at+1:]) // Strip stray trailing > from unbalanced brackets if len(dom) > 0 && dom[len(dom)-1] == '>' { dom = strings.TrimRight(dom, ">") if dom == "" { return "" } } // Strip exactly one trailing dot (FQDN normalisation) if len(dom) > 1 && dom[len(dom)-1] == '.' { dom = dom[:len(dom)-1] } return strings.ToLower(dom) } // ParseEvent reads one SES notification. // // An UNKNOWN event type parses successfully and is carried through as-is. // SES adds event types, and refusing one would make the ingest endpoint // answer 500, which makes SNS retry it forever — a self-inflicted flood // over a field nothing acts on. // // A missing source IS refused: an event that can be attributed to no tenant // is worse than no event, because it would be silently dropped from the // counts that enforce a cap. func ParseEvent(body []byte) (Event, error) { // Two reads of the same body, deliberately. This one is everything the // counts and the cap depend on, typed exactly as it was before the log // existed, and it is the only one allowed to refuse the event: a refused // event is a count silently dropped. var raw struct { EventType string `json:"eventType"` Mail struct { Timestamp string `json:"timestamp"` MessageID string `json:"messageId"` Source string `json:"source"` } `json:"mail"` } if err := json.Unmarshal(body, &raw); err != nil { return Event{}, fmt.Errorf("mail: parse ses event: %w", err) } if raw.Mail.Source == "" { return Event{}, fmt.Errorf("mail: ses event has no mail.source — it can be attributed to no tenant") } // Refuse control characters on the raw source: an injected newline anywhere // in the field indicates malformation and must not be trusted. for _, c := range raw.Mail.Source { if c < 32 || c == 127 { return Event{}, fmt.Errorf("mail: ses event source contains control character (0x%02x)", c) } } // Extract the address part (stripping display-name wrapper if present) and // test it for ambiguity. A single envelope MAIL FROM must extract to exactly // one @ and no commas. extractedAddr := raw.Mail.Source s := strings.TrimSpace(raw.Mail.Source) if i := strings.LastIndex(s, "<"); i >= 0 { if j := strings.LastIndex(s, ">"); j > i { // Has a bracketed span; refuse if there are multiple bracket pairs // (indicated by another < before the final one). if strings.Contains(s[:i], "<") { return Event{}, fmt.Errorf("mail: ses event source %q is ambiguous — multiple bracketed address spans", raw.Mail.Source) } extractedAddr = strings.TrimSpace(s[i+1 : j]) } } // Check the extracted address for ambiguity: commas or multiple @ symbols. // Note: "a@b"@example.com (a quoted local part containing @) is refused here. // This is deliberate: SES mail.source is a bare envelope MAIL FROM with no // display name, and quoted local parts with @ essentially never appear there. // Quote-aware parsing is not worth the machinery for a case that does not occur. if strings.Contains(extractedAddr, ",") || strings.Count(extractedAddr, "@") > 1 { return Event{}, fmt.Errorf("mail: ses event source %q is ambiguous — cannot reliably attribute to one tenant", raw.Mail.Source) } dom := DomainOf(raw.Mail.Source) if dom == "" { return Event{}, fmt.Errorf("mail: ses event source %q has no domain", raw.Mail.Source) } ev := Event{ Type: raw.EventType, Domain: dom, From: raw.Mail.Source, MessageID: raw.Mail.MessageID, } if raw.Mail.Timestamp != "" { if at, err := time.Parse(time.RFC3339, raw.Mail.Timestamp); err == nil { ev.SentAt = at ev.At = at } } ev.MessageID = cleanField(ev.MessageID, maxDiagnosticLen) readLogFields(body, raw.EventType, &ev) return ev, nil } // readLogFields is the second read: what only the per-message log uses — // who the message went to, who sent it, and what the receiving server said. // A field here in a shape this code does not expect is not a reason to lose // the event, so nothing here refuses it: the part that does not read is left // empty, the reason is kept in ev.Incomplete for the caller to log, and the // count goes ahead. The envelope and the event's own block are read // separately, so a malformed bounce block does not also cost the sender. func readLogFields(body []byte, eventType string, ev *Event) { type recipient struct { EmailAddress string `json:"emailAddress"` Status string `json:"status"` DiagnosticCode string `json:"diagnosticCode"` } var problems []string var envelope struct { Mail struct { Destination []string `json:"destination"` Tags map[string][]string `json:"tags"` } `json:"mail"` } if err := json.Unmarshal(body, &envelope); err != nil { problems = append(problems, "mail: "+err.Error()) } if ids := envelope.Mail.Tags["ses:caller-identity"]; len(ids) > 0 { ev.Caller = cleanField(ids[0], maxDiagnosticLen) } for _, to := range envelope.Mail.Destination { if to = cleanField(to, maxAddressLen); to != "" { ev.Destination = append(ev.Destination, to) } } // Each type carries its own block; the others are simply empty. Read // only the one this event is, so a malformed message that carries two // cannot have one type's recipients reported under another's name. var raw struct { Bounce struct { Timestamp string `json:"timestamp"` BounceType string `json:"bounceType"` BounceSubType string `json:"bounceSubType"` BouncedRecipients []recipient `json:"bouncedRecipients"` } `json:"bounce"` Complaint struct { Timestamp string `json:"timestamp"` ComplaintFeedbackType string `json:"complaintFeedbackType"` ComplainedRecipients []recipient `json:"complainedRecipients"` } `json:"complaint"` Delivery struct { Timestamp string `json:"timestamp"` Recipients []string `json:"recipients"` SMTPResponse string `json:"smtpResponse"` } `json:"delivery"` Reject struct { Reason string `json:"reason"` } `json:"reject"` DeliveryDelay struct { Timestamp string `json:"timestamp"` DelayType string `json:"delayType"` DelayedRecipients []recipient `json:"delayedRecipients"` } `json:"deliveryDelay"` } block := map[string]string{ EventBounce: "bounce", EventComplaint: "complaint", EventDelivery: "delivery", EventReject: "reject", EventDeliveryDelay: "deliveryDelay", }[eventType] if block != "" { var blocks map[string]json.RawMessage if err := json.Unmarshal(body, &blocks); err == nil { // Wrap the one block back under its own key and read only that. if one, ok := blocks[block]; ok { if err := json.Unmarshal([]byte(`{"`+block+`":`+string(one)+`}`), &raw); err != nil { problems = append(problems, block+": "+err.Error()) } } } } if len(problems) > 0 { ev.Incomplete = cleanField(strings.Join(problems, "; "), maxDiagnosticLen) } // When the thing this event reports happened, where SES says: a bounce // or a delay can arrive a day after the send, and an event dated by the // send is both wrong on a screen and invisible to "since yesterday". eventTime := map[string]string{ EventBounce: raw.Bounce.Timestamp, EventComplaint: raw.Complaint.Timestamp, EventDelivery: raw.Delivery.Timestamp, EventDeliveryDelay: raw.DeliveryDelay.Timestamp, }[eventType] if eventTime != "" { if at, err := time.Parse(time.RFC3339, eventTime); err == nil { ev.At = at } } outcomes := func(rs []recipient) { for _, r := range rs { to := cleanField(r.EmailAddress, maxAddressLen) if to == "" { continue } ev.Outcomes = append(ev.Outcomes, Outcome{Recipient: to, Status: cleanField(r.Status, maxDetailLen), Diagnostic: cleanField(r.DiagnosticCode, maxDiagnosticLen)}) } } switch eventType { case EventBounce: ev.Detail.BounceType = cleanField(raw.Bounce.BounceType, maxDetailLen) ev.Detail.BounceSubType = cleanField(raw.Bounce.BounceSubType, maxDetailLen) outcomes(raw.Bounce.BouncedRecipients) case EventComplaint: ev.Detail.ComplaintType = cleanField(raw.Complaint.ComplaintFeedbackType, maxDetailLen) outcomes(raw.Complaint.ComplainedRecipients) case EventDelivery: said := cleanField(raw.Delivery.SMTPResponse, maxDiagnosticLen) for _, to := range raw.Delivery.Recipients { if to = cleanField(to, maxAddressLen); to != "" { ev.Outcomes = append(ev.Outcomes, Outcome{Recipient: to, Diagnostic: said}) } } case EventReject: ev.Detail.RejectReason = cleanField(raw.Reject.Reason, maxDiagnosticLen) case EventDeliveryDelay: ev.Detail.DelayType = cleanField(raw.DeliveryDelay.DelayType, maxDetailLen) outcomes(raw.DeliveryDelay.DelayedRecipients) } } // Recipients is who this event is about, one entry per address: the // recipients it reports an outcome for, or — for an event about the whole // message, a Send or a Reject — everyone it was addressed to. A per-recipient // event (a bounce, a delivery, a complaint, a delay) that names nobody is NOT // widened to every addressee: that would report a bounce for addresses that // did not bounce. It, like any event that names nobody at all, yields a // single empty entry, so it is still recorded once rather than not at all. func (ev Event) Recipients() []Outcome { if len(ev.Outcomes) > 0 { return ev.Outcomes } wholeMessage := ev.Type == EventSend || ev.Type == EventReject if len(ev.Destination) == 0 || !wholeMessage { return []Outcome{{}} } out := make([]Outcome, 0, len(ev.Destination)) for _, to := range ev.Destination { out = append(out, Outcome{Recipient: to}) } return out } // Summary is Detail in a few words, for a table: "Permanent/NoEmail", // "abuse", "Bad content". Empty when the event has nothing to add. func (d EventDetail) Summary() string { switch { case d.BounceType != "" && d.BounceSubType != "": return d.BounceType + "/" + d.BounceSubType case d.BounceType != "": return d.BounceType case d.ComplaintType != "": return d.ComplaintType case d.RejectReason != "": return d.RejectReason } return d.DelayType }