package analytics import ( "crypto/sha256" "encoding/binary" "log" "regexp" "strings" "sync" "time" "github.com/carlosframework/platform/internal/apperr" "github.com/carlosframework/platform/internal/applog" "github.com/carlosframework/platform/internal/usage" ) // Obs is one served response as the edge saw it (spec §3.3). The edge // fills it from the observer defer; nothing here touches the request. type Obs struct { Acct, App, Host string Method string Status int ContentType string Path string // raw; sanitised here RefererHost string // host only, "" = direct UA string ClientIP string Bytes int64 Prefetch bool // Sec-Purpose/Purpose: prefetch FetchDest string // Sec-Fetch-Dest, "" when absent When time.Time // GeoIP (geoip spec §9): the request's one answer, as the ingress edge // computed it or as this edge adopted it. GeoState is hit | miss | off, // "" from an edge that predates GeoIP (counted like off). Country string GeoSource string GeoState string } // AppKey is the (account, app) the meter buckets by. type AppKey = applog.AppKey // IsPageview applies §3.3's rule: GET, 200, text/html, not prefetch, dest // document or absent, not under /__carlos/. A bot UA is reported // separately and never a pageview. func IsPageview(o Obs) (pageview, bot bool) { if o.Method != "GET" || o.Status != 200 { return false, false } mt := strings.ToLower(strings.TrimSpace(strings.SplitN(o.ContentType, ";", 2)[0])) if mt != "text/html" { return false, false } if o.Prefetch || (o.FetchDest != "" && o.FetchDest != "document") { return false, false } if strings.HasPrefix(o.Path, "/__carlos/") { return false, false } if IsBot(o.UA) { return false, true } return true, false } // VisitorHash is sha256(salt ‖ host ‖ ip ‖ ua) folded to 64 bits for the // sketch; the platform never stores the inputs. func VisitorHash(salt []byte, host, ip, ua string) uint64 { h := sha256.New() h.Write(salt) h.Write([]byte{0}) h.Write([]byte(host)) h.Write([]byte{0}) h.Write([]byte(ip)) h.Write([]byte{0}) h.Write([]byte(ua)) return binary.BigEndian.Uint64(h.Sum(nil)[:8]) } var eventName = regexp.MustCompile(`^[a-z0-9_.-]{1,64}$`) // EventName normalises a named event: lowercased, ≤64 bytes of [a-z0-9_.-]. func EventName(s string) (string, bool) { s = strings.ToLower(strings.TrimSpace(s)) if !eventName.MatchString(s) { return "", false } return s, true } // saltRetryDelay bounds how often a failing DaySalt is retried. Visitors // go uncounted while it fails, but Observe must never do that I/O (or log) // on every request — one attempt, and one log line, per window. const saltRetryDelay = 30 * time.Second // Meter accumulates per-app deltas between flushes. One per edge process; // Observe sits on the serving path (one mutex, no I/O except the first // salt read of a day). type Meter struct { box string saltDir string now func() time.Time mu sync.Mutex cur map[AppKey]map[string]*Delta // app → date → delta being built pending map[AppKey][]Delta // re-queued after a failed flush, each marked once dropPending map[AppKey]int64 // deltas dropped by a Requeue that found an already-marked one, awaiting the next delta to land on seq int64 saltDay string salt []byte saltFailDay string // date DaySalt last failed for, "" once it next succeeds saltFailAt time.Time // when it failed; not retried until saltRetryDelay has passed started string // date the process started; "" once the wall clock has moved past it (see Drain) restartedFlagged map[AppKey]bool // apps whose start-day delta has already carried Restarted } // NewMeter returns a meter whose salts live in saltDir (the box data dir). func NewMeter(saltDir, box string, now func() time.Time) *Meter { return &Meter{box: box, saltDir: saltDir, now: now, cur: map[AppKey]map[string]*Delta{}, pending: map[AppKey][]Delta{}, dropPending: map[AppKey]int64{}, seq: now().UnixMilli(), started: usage.DayOf(now()), restartedFlagged: map[AppKey]bool{}} } func (m *Meter) saltFor(date string) []byte { if m.saltDay == date && m.salt != nil { return m.salt } if m.saltFailDay == date && m.now().Sub(m.saltFailAt) < saltRetryDelay { return nil // still within the backoff window: don't retry, don't log again } s, err := DaySalt(m.saltDir, date) if err != nil { m.saltFailDay, m.saltFailAt = date, m.now() log.Printf("analytics: day salt: %v (visitors not counted until it works)", err) return nil } m.saltFailDay = "" m.saltDay, m.salt = date, s return s } // delta returns the app's delta being built for date, creating it (and // setting Restarted) on first touch. Restarted is per (app, day), not per // process: every app gets its own first-delta-of-the-start-day flag, since // each app's counts since the last flush may be missing regardless of // which app happened to be observed first. restartedFlagged stops a later // drain's fresh delta for the same app-day from being flagged again. func (m *Meter) delta(k AppKey, date string) *Delta { days := m.cur[k] if days == nil { days = map[string]*Delta{} m.cur[k] = days } d := days[date] if d == nil { restarted := date == m.started && !m.restartedFlagged[k] d = &Delta{Date: date, Box: m.box, Hours: map[string]HourRow{}, Visitors: &HLL{}, Pages: NewTopK(), Referers: NewTopK(), Events: NewTopK(), Restarted: restarted, Countries: map[string]int64{}, GeoSources: map[string]int64{}} days[date] = d if restarted { m.restartedFlagged[k] = true // this app's start-day note is spent; a later drain's fresh delta for the same day must not repeat it } } return d } // Observe counts one response. Bytes count for every response; pageview, // bot, visitor, page and referer only when the rule says so. func (m *Meter) Observe(o Obs) { pv, bot := IsPageview(o) date, hour := usage.DayOf(o.When), usage.HourLabel(o.When) path := apperr.SanitizePath(o.Path) m.mu.Lock() defer m.mu.Unlock() d := m.delta(AppKey{Acct: o.Acct, App: o.App}, date) row := d.Hours[hour] row.Bytes += o.Bytes if bot { row.Bots++ } if pv { row.PV++ if salt := m.saltFor(date); salt != nil { d.Visitors.Add(VisitorHash(salt, o.Host, o.ClientIP, o.UA)) } d.Pages.Add(path, 1) ref := o.RefererHost if ref == "" { ref = "direct" } d.Referers.Add(strings.ToLower(ref), 1) countGeo(d, o) } d.Hours[hour] = row } // Track counts one named event (the doors call it; nothing does in step 2). // An invalid name is not silently discarded: it counts against the app's // current-day Dropped.A.Malformed so the honesty block sees it. func (m *Meter) Track(acct, app, name string, when time.Time) { m.mu.Lock() defer m.mu.Unlock() d := m.delta(AppKey{Acct: acct, App: app}, usage.DayOf(when)) n, ok := EventName(name) if !ok { d.Dropped.A.Malformed++ return } d.Events.Add(n, 1) } // Drain returns everything buffered — re-queued deltas first, then the // current ones, each stamped with a fresh box-monotonic seq — and resets. // A window an earlier Requeue had to drop (because it was already marked // re-queued when Requeue saw it again) is not silently lost: its count is // added to Dropped.Flush on the first delta this drain emits for that app. func (m *Meter) Drain() map[AppKey][]Delta { m.mu.Lock() defer m.mu.Unlock() out := map[AppKey][]Delta{} for k, ds := range m.pending { out[k] = append(out[k], ds...) } m.pending = map[AppKey][]Delta{} for k, days := range m.cur { for _, d := range days { m.seq++ d.Seq = m.seq out[k] = append(out[k], *d) } } m.cur = map[AppKey]map[string]*Delta{} for k, ds := range out { if len(ds) == 0 { continue } if dropped := m.dropPending[k]; dropped > 0 { ds[0].Dropped.Flush += dropped delete(m.dropPending, k) } } if m.started != "" && usage.DayOf(m.now()) != m.started { m.started = "" // the restart note is only for the day it happened m.restartedFlagged = map[AppKey]bool{} // free the bookkeeping once no future delta can consult it } return out } // Requeue holds a failed flush for the next drain. Each delta may survive // one failed flush: Requeue marks it (an unexported field on Delta, never // serialised) and holds it in pending. A delta Requeue is handed that is // already marked — the flush carrying it forward also failed, with no // successful Drain landing it in between — is dropped instead, and the // drop is counted per app in dropPending rather than kept, so pending // never grows past one window's worth of history no matter how many // flushes in a row fail. Seq is left untouched — a re-queued delta keeps // the seq it was drained with, so a replay that actually landed is a // no-op at the day file. func (m *Meter) Requeue(failed map[AppKey][]Delta) { m.mu.Lock() defer m.mu.Unlock() for k, ds := range failed { for _, d := range ds { if d.requeued { m.dropPending[k]++ continue } d.requeued = true m.pending[k] = append(m.pending[k], d) } } }