#!/usr/bin/env bash # πŸ€– Public status publish (issue #550 Parts 11/12): every 5 minutes, build two # small world-readable JSONs β€” fleet.json (fleet state) and books.json (the # books: Stripe income aggregates + itemized infrastructure expenses) β€” and # upload them plus the static viewer pages to the public status bucket # (infra/prod/status.tf). docs/status.md is the operator guide. # # ============================ PRIVACY β€” NON-NEGOTIABLE ======================== # Everything this script uploads is PUBLIC. The rules (decision #19 in #550: # income publishes AGGREGATED, never per-payer; invariant 12: no PII in our # stores, so none can leak here β€” but we hold the line at the aggregation # boundary anyway): # # fleet.json β€” AGGREGATES ONLY. Totals, counts, version distribution, # service up/down booleans, replication health. NEVER a hostname (member # spaces obviously, but also test-* canaries β€” the simple rule is NO # hostnames at all), never per-space member/message counts, never anything # keyed by space. Hostnames are read internally (to match sockets) and # must never reach the JSON. # # Capacity + hosts (Paul's call: bucketed words, not needles): the box's # room-for-new-spaces and per-box load publish as ABSTRACT BAND WORDS only # (plenty / getting-full / at-capacity; comfortable / busy / critical). # The caps, ratios, free GB, and percentages are read internally to pick # the word and MUST NEVER reach the JSON β€” no remaining-slot counts, no GB, # no percentages, no cap values. Host labels are whimsical name-action # pairs hashed from the machine id ("ada-stargazing") β€” stable per box, # never peer URLs, IPs, hostnames, or the machine id itself. # # A box's `region` is COARSE BY DESIGN: a continent + country pair # ("EU Β· Finland") β€” never a city, datacenter, IP, or hostname. Resolved # from the box's own cloud metadata (see box_region below); the # STATUS_BOX_REGION env overrides; unknown β†’ the field is omitted # entirely. Detected, mapped, or absent β€” never guessed. # # books.json β€” AGGREGATED INCOME ONLY. Active-subscription count (by tier β€” # tier names are the public price book), MRR total in EUR, gross collected # this month, and the Connect aggregates (spaces' gross sales volume this # month + our commission on it β€” Paul's call: if we know the GMV, it's # public; totals only, like everything else here). NEVER customer ids, # emails, subscription ids, connected-account ids, per-payer lines, or # timestamps of individual payments. The Stripe key is read for the API # calls and must never be echoed, logged, or uploaded. # # books.json expenses β€” the deliberate mirror image (decision #19's other # half: income aggregates, EXPENSES PUBLISH ITEMIZED β€” the amounts are # public on purpose). One line per provider: provider word + amount + its # own billing currency (never FX-converted β€” no rate, no misstatement). # NEVER account ids, invoice ids, workspace ids, resource-level lines, or # usage breakdowns. The AWS creds and the Anthropic Admin key are read for # the API calls and must never be echoed, logged, or uploaded. # # carbon.json β€” two halves. `built` is COMMITTED data (the AI-build carbon # tally, web/status/carbon.json, accrued by tools/carbon-tally.mjs), staged # with the pages and republished as-is β€” aggregate token counts + kg only, # never per-session data. `run` is a LIVE operational estimate this script # INJECTS at upload (build_carbon): AGGREGATES ONLY β€” total vCPUs, watts, # grid intensity, banded kg, and the public instance count as the per-space # denominator (NEVER per-member, never a hostname/IP). Absent inputs β†’ run # stays absent and the page shows "pending", never a fake zero. # # contributors.json β€” also COMMITTED data (tools/contributors.mjs), # republished as-is. Public GitHub identities + commit counts only; # contributors own their row (bio, hidden) in the committed JSON. # # security.json β€” also COMMITTED data (deploy/security-scan.sh, run at # BUILD time where the Go toolchain lives β€” this box never computes a # scan), republished as-is. It is REDACTED BY CONSTRUCTION: a status word, # an advisory COUNT, a module count, the build sha, a timestamp β€” never a # CVE/GHSA/OSV id, package, or code location of anything unpatched (that # would be an attacker roadmap; see the scan script's disclosure header). # Absent file β†’ the page shows "scan pending", never a fake clean. # # Adding a field? It must be an aggregate over the whole fleet / all payers, # or it doesn't ship. # ============================================================================== # # Privilege: runs as the lchat user, not root β€” least privilege that works: # - /data/registry.db and /data/cloud.env are lchat-owned on the hardened # box (all units run as lchat since PR #186; /data is owned by it). # - `systemctl is-active` and `ss -xlH` are unprivileged read-only queries. # - the journal read (replication lag) needs the systemd-journal group, # granted via SupplementaryGroups in the unit below. # Only `install` (writing units, enabling the timer) needs root. # # Usage: # status-publish.sh # build + upload (what the timer runs) # status-publish.sh install # root: install unit + timer, enable # status-publish.sh --dry-run DIR # build both JSONs into DIR from # # self-constructed fake registry/Stripe # # fixtures; no box state touched, no # # upload β€” the committed contract test # # Prerequisites on the box: # /data/status.env β€” AWS creds for the status-publisher IAM user # (infra/prod/status.tf; Put/Get/List on the status # bucket, no Delete, plus ce:GetCostAndUsage for the # expenses section). chown lchat, chmod 0600. Optional # expense vars ride here too: STATUS_HETZNER_TOKEN (a # read-only Hetzner Cloud API token β€” computes the # monthly server cost; never echoed) and/or # STATUS_HETZNER_MONTHLY_CENTS (operator constant β€” # overrides the token when both are set), # STATUS_CLAUDE_TO_DATE_CENTS, STATUS_APPLE_YEARLY_CENTS, # STATUS_ANTHROPIC_ADMIN_KEY (an sk-ant-admin… key with # read-only usage scope β€” NOT the sidecars' completion # key), and STATUS_DNSIMPLE_TOKEN + STATUS_DNSIMPLE_ACCOUNT # (an account API token β€” read-only use here, never # echoed β€” plus the numeric account id). # /opt/lchat/status-pages/ β€” the viewer pages, copied from web/status/ # (status.css, index.html, books.html, economics.html, # economics.json, carbon.html, carbon.json, # contributors.html, contributors.json, security.html, # security.json, prompts.html, icon-192.png). status.css # is the shared stylesheet every page links; it uploads # before the pages (see publish()). # aws CLI + sqlite3 (apt install awscli sqlite3). set -euo pipefail BUCKET=${STATUS_BUCKET:-eleven-status-427880590669} REGION=${STATUS_REGION:-eu-west-1} REGISTRY_DB=${STATUS_REGISTRY_DB:-/data/registry.db} CLOUD_ENV=${STATUS_CLOUD_ENV:-/data/cloud.env} # The control daemon's env (capacity caps: LCHAT_MAX_INSTANCES etc.). May be # root-owned/unreadable to the lchat user β€” then the code defaults below apply, # which are exactly what controld itself defaults to when the vars are unset. CONTROL_ENV=${STATUS_CONTROL_ENV:-/data/control.env} # The router's own env β€” read (same lchat user, same file the router loads) # only for the mesh's bucket/prefix, so this box can find its live peers' # self-published stats (mesh_peer_boxes below). Absent LCHAT_HIBERNATE_BUCKET/ # _PREFIX here just means the mesh isn't configured on this box β€” hosts stays # a single-box array, byte-identical to before multi-box existed. ROUTER_ENV=${STATUS_ROUTER_ENV:-/data/router.env} # Staleness cutoff for a peer's self-published stats object β€” three missed # beats, mirroring mesh.go's own meshStaleAfter convention, at the STATS # cadence (meshstats.go's meshStatsInterval, 60s) rather than the membership # heartbeat's faster one. MESH_STATS_STALE_SECONDS=${STATUS_MESH_STATS_STALE_SECONDS:-180} # Box-load + disk inputs (STATUS_ overrides exist for testability; the dry-run # uses fixture twins instead and never reads live box state). MEMINFO=${STATUS_MEMINFO:-/proc/meminfo} DATA_DIR=${STATUS_DATA_DIR:-/data} # Metered-expense cache (see cached_cost_cents): AWS Cost Explorer and the # Anthropic cost report both bill/limit per request while this script runs # every 5 minutes, so their results persist here between runs (one JSON object # per provider: {fetched_at, month, month_to_date_cents}). Lives in /data so # it survives reboots; the dry-run repoints it into its fixture dir. COSTS_CACHE=${STATUS_COSTS_CACHE:-$DATA_DIR/status-costs-cache.json} # Coarse public region of this box β€” continent + country ONLY ("EU Β· Germany"); # see PRIVACY above. Optional: unset omits the field from fleet.json. Per-box # regions will come from mesh heartbeats when multi-box lands. Resolution: # STATUS_BOX_REGION env (explicit override wins) β†’ the box's own cloud # metadata service (Hetzner, then AWS IMDSv2), mapped zoneβ†’coarse region β†’ # omitted entirely when nothing answers. Hand-set values drift (the box was # labelled Germany for a day; it lives in Helsinki) β€” asking the machine # where it is can't. The map stays coarse by design: continent + country. box_region() { if [ -n "${STATUS_BOX_REGION:-}" ]; then printf '%s' "$STATUS_BOX_REGION"; return; fi [ -n "$DRY_RUN" ] && return # dry-run never probes live box state local zone zone=$(curl -sm2 http://169.254.169.254/hetzner/v1/metadata/availability-zone 2>/dev/null || true) case "$zone" in hel*) printf 'EU Β· Finland'; return ;; fsn*|nbg*) printf 'EU Β· Germany'; return ;; ash*|hil*) printf 'NA Β· USA'; return ;; sin*) printf 'AS Β· Singapore'; return ;; esac # AWS: IMDSv2 (token required; both calls fail closed in ~2s off-cloud) local tok az tok=$(curl -sm2 -X PUT http://169.254.169.254/latest/api/token -H "X-aws-ec2-metadata-token-ttl-seconds: 60" 2>/dev/null || true) [ -n "$tok" ] && az=$(curl -sm2 -H "X-aws-ec2-metadata-token: $tok" http://169.254.169.254/latest/meta-data/placement/region 2>/dev/null || true) case "$az" in eu-west-1) printf 'EU Β· Ireland' ;; eu-west-2) printf 'EU Β· UK' ;; eu-central-1) printf 'EU Β· Germany' ;; eu-north-1) printf 'EU Β· Sweden' ;; us-east-*|us-west-*) printf 'NA Β· USA' ;; *) : ;; # unknown zone: omit, never guess esac } STATUS_ENV=${STATUS_ENV_FILE:-/data/status.env} PAGES_DIR=${STATUS_PAGES_DIR:-/opt/lchat/status-pages} STRIPE_API=${STATUS_STRIPE_API:-https://api.stripe.com} # Carbon "run" block coefficients (see build_carbon + docs/comparison-carbon.md # Β§ The run methodology; snapshot July 2026, all BANDED low/central/high). The # fleet hibernates and runs at a fraction of the box, so the attributable draw # is a low-single-digit-to-~15W slice of a lightly loaded server β€” modelled here # as per-vCPU average watts (marginal β†’ whole-box-idle-inclusive): CARBON_W_LO=${STATUS_CARBON_W_LO:-1.5} # W/vCPU, marginal (load-proportional) CARBON_W_CE=${STATUS_CARBON_W_CE:-3.5} # W/vCPU, central CARBON_W_HI=${STATUS_CARBON_W_HI:-7.5} # W/vCPU, whole-box idle-inclusive share CARBON_PUE=${STATUS_CARBON_PUE:-1.15} # AWS-published fleet-PUE ballpark # AWS storage+requests: near-zero but NOT zero β€” a small explicit banded term # (~a gram a day: ~0.1 TB of our ciphertext Γ— ~1 Wh/TB-h Γ— 24h Γ— a grid). CARBON_AWS_LO=${STATUS_CARBON_AWS_LO:-0.0002} CARBON_AWS_CE=${STATUS_CARBON_AWS_CE:-0.001} CARBON_AWS_HI=${STATUS_CARBON_AWS_HI:-0.003} # Bot inference (the sidecars' ongoing Anthropic API use): small today, bounded # wide β€” the high end allows for busy days. Precise token counts exist in the # provider console; we publish a bounded estimate rather than couple to it. CARBON_INF_LO=${STATUS_CARBON_INF_LO:-0.0005} CARBON_INF_CE=${STATUS_CARBON_INF_CE:-0.005} CARBON_INF_HI=${STATUS_CARBON_INF_HI:-0.05} # The fleet's start-of-life (drives the cumulative-to-date figure). The dry-run # pins STATUS_CARBON_DAYS instead so its cumulative is deterministic. CARBON_SINCE=${STATUS_CARBON_SINCE:-2026-07-05} SELF=/opt/lchat/status-publish.sh UNIT=/etc/systemd/system/lchat-status.service TIMER=/etc/systemd/system/lchat-status.timer DRY_RUN="" FIXDIR="" TMP=$(mktemp -d) trap 'rm -rf "$TMP"' EXIT # sq runs a query against an on-disk DB with a busy timeout (the router writes # the registry concurrently); sqj evaluates JSON expressions in :memory:. All # JSON this script emits is BUILT BY SQLITE's json functions, never by string # interpolation in bash β€” DB/Stripe-sourced strings (versions, tier keys) get # escaped correctly and can't break or inject into the output. sq() { sqlite3 -cmd '.timeout 3000' "$REGISTRY_DB" "$1"; } sqj() { sqlite3 :memory: "$1"; } now_iso() { date -u +%Y-%m-%dT%H:%M:%SZ; } # ---- box probes (each has a dry-run twin so --dry-run exercises the same # ---- aggregation code with no live systemctl/ss/journalctl) ------------------ # svc_active β€” unprivileged read-only state query. svc_active() { if [ -n "$DRY_RUN" ]; then grep -qxF "$1" "$FIXDIR/active-units.txt"; return; fi systemctl is-active --quiet "$1" } # music_sockets β€” the kernel's list of LISTENING unix sockets. This is the # awake test for hibernating hosts, and the choice matters: # - Socket-file EXISTENCE is NOT sound: an instance unlinks a stale socket # file on start (main.go listen()) but never unlinks on exit, so a slept # instance leaves the file behind. Existence would count every host that # has ever woken as awake forever. # - Dialing the socket (curl --unix-socket) IS sound but touches the # instance β€” a probe every 5 minutes must not risk interacting with the # activator's idle accounting or show up in instance logs. # - `ss -xlH` reads the socket table: an entry exists iff a process has the # path BOUND AND LISTENING right now (the activator's children bind the # same /run/lchat/.sock path systemd units use). Zero side effects. music_sockets() { if [ -n "$DRY_RUN" ]; then cat "$FIXDIR/music.txt"; return; fi if command -v ss >/dev/null 2>&1; then ss -xlH 2>/dev/null || true; fi } # replication_lag_json β€” seconds since the fleet replicator (lchat-litestream, # deploy/fleet-backup.sh) last logged anything: an approximation of replication # lag (litestream logs each sync; a healthy replicator is chatty, a wedged one # goes silent). "null" when the journal is unreadable or empty β€” published as # null, never faked. replication_lag_json() { if [ -n "$DRY_RUN" ]; then cat "$FIXDIR/replication-lag.txt"; return; fi local line ts now line=$(journalctl -u lchat-litestream -n1 -o json 2>/dev/null || true) if [ -z "$line" ]; then echo "null"; return; fi printf '%s' "$line" > "$TMP/journal.json" ts=$(sqj "SELECT CAST(json_extract(readfile('$TMP/journal.json'), '\$.__REALTIME_TIMESTAMP') AS INTEGER) / 1000000" || echo "") if [ -z "$ts" ] || [ "$ts" = "0" ]; then echo "null"; return; fi now=$(date -u +%s) echo $((now - ts)) } # ---- capacity + hosts probes (ABSTRACT BANDS ONLY β€” see PRIVACY above) ------- # env_int β€” one integer from an env file (same grep/cut # shape as the Stripe key read). Missing file/var or a non-numeric value falls # back to the default β€” for the capacity caps the defaults mirror controld.go's # code defaults (#663), so an absent/unreadable control.env yields the same # numbers controld itself enforces. env_int() { local v v=$(grep -E "^$2=" "$1" 2>/dev/null | head -n1 | cut -d= -f2- || true) case "$v" in ''|*[!0-9]*) echo "$3" ;; *) echo "$v" ;; esac } # env_str β€” like env_int but for string values (bucket names, # prefixes): no numeric coercion, empty when the file or var is absent. env_str() { grep -E "^$2=" "$1" 2>/dev/null | head -n1 | cut -d= -f2- || true } # b64url_decode reverses encodeHeartbeatKey (mesh.go): url-safe alphabet, # padding stripped β€” re-add it (len mod 4 tells you how much) before handing # off to the standard decoder. Used by mesh_peer_boxes to recognize (and # skip) THIS box's own entry in the stats prefix. b64url_decode() { local s s=$(printf '%s' "$1" | tr -- '-_' '+/') case $(( ${#s} % 4 )) in 2) s="${s}==" ;; 3) s="${s}=" ;; esac printf '%s' "$s" | base64 -d 2>/dev/null } # disk_free_gb β€” whole GB free on the data disk (`df -Pk`: POSIX, and avail # matches controld's Statfs Bavail). Empty on failure β€” an unknown contributes # nothing to the band, never a guess. disk_free_gb() { if [ -n "$DRY_RUN" ]; then cat "$FIXDIR/disk-free-gb.txt"; return; fi df -Pk "$DATA_DIR" 2>/dev/null | awk 'NR==2 && $4 ~ /^[0-9]+$/ { printf "%d", $4 / 1048576 }' || true } # meminfo_kb β€” one kB figure from /proc/meminfo (MemTotal / # MemAvailable drive the box-load band). Empty when unreadable (e.g. a # non-Linux dev machine) β€” box load then publishes null, never a guess. meminfo_kb() { local src="$MEMINFO" if [ -n "$DRY_RUN" ]; then src="$FIXDIR/meminfo.txt"; fi awk -v f="$1:" '$1 == f { print $2; exit }' "$src" 2>/dev/null || true } # pct_band β€” the three public bands as an index: 0 below 70%, 1 at # 70–90%, 2 above 90%. These thresholds ARE the abstraction: only the band # word ships, the percentage dies here. pct_band() { if [ "$1" -lt 70 ]; then echo 0; elif [ "$1" -le 90 ]; then echo 1; else echo 2; fi; } # The two public vocabularies, one per surface: capacity (room for new spaces) # and per-box load. Same bands, different words. band_word() { case "$1" in 0) echo plenty ;; 1) echo getting-full ;; *) echo at-capacity ;; esac; } load_word() { case "$1" in 0) echo comfortable ;; 1) echo busy ;; *) echo critical ;; esac; } # ---- fleet.json -------------------------------------------------------------- # box_label derives a stable, whimsical, anonymous name for this box β€” # "ada-stargazing" β€” from a hash of the machine id. Notable names, real and # fictional (history, literature, myth, science, pop culture), paired with an # action. The hash direction only (id -> label) is published, so the label # leaks nothing; the action list deliberately avoids the dashboard's own # status vocabulary (no sleeping/waking/resting β€” a box is not an instance). box_label() { local names=(ada alan grace marie nikola leonardo cleopatra homer sappho darwin einstein curie turing hopper tesla gandalf frodo sherlock watson ahab ishmael gatsby scout atticus hermione matilda mowgli merlin arthur morgana beowulf odysseus penelope achilles athena apollo freya thor loki amelia boudicca cyrano quixote sancho ophelia viola rosalind puck oberon titania darcy pip scrooge nemo phileas zorro heidi galileo kepler newton archimedes hypatia confucius rumi frida vincent wolfgang ludwig clara) local actions=(stargazing juggling whistling painting sailing gardening baking skating humming doodling fishing knitting strolling yodeling beekeeping fencing waltzing sketching brewing climbing cartwheeling birdwatching pondering inventing exploring composing conducting spelunking kayaking orienteering puppeteering tinkering) local idfile=${STATUS_MACHINE_ID_FILE:-/etc/machine-id} local seed seed=$( { cat "$idfile" 2>/dev/null || hostname; } | sha256sum | cut -c1-16) # two independent hex slices -> indices; bash arithmetic handles hex via 16# local n=$(( 16#${seed:0:8} % ${#names[@]} )) local a=$(( 16#${seed:8:8} % ${#actions[@]} )) printf '%s-%s' "${names[$n]}" "${actions[$a]}" } # mesh_peer_boxes prints a JSON array of every OTHER live mesh box's # self-published stats object β€” {label, awake, asleep, load} β€” read from the # hibernation bucket's stats side-channel (internal/activator/heartbeat.go's # PutStats, a sibling object next to the membership heartbeat every mesh # member already writes; meshstats.go computes each box's own snapshot on its # own slower cadence). Deliberately NOT a new box-to-box port: this rides the # same S3 substrate the mesh already uses for membership, exactly the seam # mesh.go's own doc comment anticipates ("a future AWS pool" that "may not # even share a network"). Empty array β€” not an error β€” whenever this box # isn't mesh-configured, has no live peers, or any AWS call fails; a missing # peer here just means the Fleet page undercounts by one box, never a # publish failure. # # As a side effect (one listing, not two), also writes THIS box's own # self-published stats body to $TMP/mesh-self-stats-result.json (removed if # unpublished/stale/mesh off) β€” computeBoxStats (meshstats.go) already # ownership-partitions this box's own hib count the same way it does for a # peer, so build_fleet can prefer this over its own local `ss` sweep instead # of re-deriving an unfiltered view that double-counts against a peer's row. # A file, not a global var: the caller captures this function's stdout with # `$(...)`, which runs it in a subshell β€” a global assignment made in there # never survives back to the parent shell, but a file write does. mesh_peer_boxes() { local self_out="$TMP/mesh-self-stats-result.json" rm -f "$self_out" if [ -n "$DRY_RUN" ]; then if [ -f "$FIXDIR/mesh-self-stats.json" ]; then cp "$FIXDIR/mesh-self-stats.json" "$self_out"; fi if [ -f "$FIXDIR/mesh-peers.json" ]; then cat "$FIXDIR/mesh-peers.json"; else echo '[]'; fi return fi local bucket region prefix mesh_prefix self bucket=$(env_str "$ROUTER_ENV" LCHAT_HIBERNATE_BUCKET) prefix=$(env_str "$ROUTER_ENV" LCHAT_HIBERNATE_PREFIX) if [ -z "$bucket" ] || [ -z "$prefix" ]; then echo '[]'; return; fi region=$(env_str "$ROUTER_ENV" LCHAT_HIBERNATE_REGION) region=${region:-eu-west-1} # This box's own mesh identity, if its OWN mesh is on (LCHAT_MESH_SELF set) # β€” skipped below. Without this, a box whose mesh is on double-counts # itself the moment it starts self-publishing to this same stats channel: # once as "the local box" (this script's own OS reads, elsewhere in # build_fleet) and again as a "peer" read back from S3. Found live # 2026-07-13, box A's flip: the Fleet page briefly showed hosts.count 3 # (one real duplicate) instead of 2. self=$(env_str "$ROUTER_ENV" LCHAT_MESH_SELF) # Must match hc.meshPrefix() (activator.go) exactly β€” the membership # heartbeat lives at "/mesh/members/", and statsKey # (internal/activator/heartbeat.go) appends "-stats" to that WHOLE path, # not to the bare hibernation prefix: "/mesh/members-stats/". # (Caught 2026-07-13 before the flip: an earlier version of this function # listed "-stats/" instead β€” empty either way today since nothing # has ever published there, so it silently proved nothing.) mesh_prefix="${prefix}/mesh/members" # `aws s3 ls` (the high-level command), never `aws s3api list-objects*` β€” # the latter crashes on this fleet's aws-cli/Python combination (a Python # 3.13+ argparse regression in botocore's generated help text: ValueError # "badly formed help string", thrown while building the argument parser, # before any request is even sent β€” found live on box A, 2026-07-13, # confirmed against both list-objects-v2 and the older list-objects; `aws # s3 ls`/`aws s3 cp` are unaffected, and are what the rest of this script # already relies on). `s3 ls` hits the same ListObjectsV2 API underneath; # its plain-text output already has everything needed (date, time, key). local listing listing=$(aws s3 ls "s3://$bucket/${mesh_prefix}-stats/" --region "$region" 2>/dev/null) || true [ -n "$listing" ] || { echo '[]'; return; } # A plain string compare against the cutoff is exact as long as both sides # use the same "YYYY-MM-DD HH:MM:SS" shape (space-, not T-separated β€” # `aws s3 ls`'s own format) β€” no date-library parsing needed, and a box # whose clock is off only ever misjudges its OWN staleness, same tradeoff # mesh.go's own reader-vs-S3-clock note accepts for membership. local cutoff peers="[]" line date_part time_part key tmp body cutoff=$(date -u -d "-${MESH_STATS_STALE_SECONDS} seconds" '+%Y-%m-%d %H:%M:%S') while IFS= read -r line; do [ -n "$line" ] || continue date_part=$(printf '%s' "$line" | awk '{print $1}') time_part=$(printf '%s' "$line" | awk '{print $2}') key=$(printf '%s' "$line" | awk '{print $4}') [ -n "$key" ] || continue [ "$date_part $time_part" '>' "$cutoff" ] || continue # stale β€” treat like a dead mesh member tmp="$TMP/mesh-stat-$(printf '%s' "$key" | tr '/' '_').json" aws s3 cp "s3://$bucket/${mesh_prefix}-stats/$key" "$tmp" --region "$region" >/dev/null 2>&1 || continue body=$(cat "$tmp" 2>/dev/null) || continue [ -n "$body" ] || continue if [ -n "$self" ] && [ "$(b64url_decode "$key")" = "$self" ]; then printf '%s' "$body" > "$self_out" continue fi peers=$(sqj "SELECT json_insert( json('$(printf '%s' "$peers" | sed "s/'/''/g")'), '\$[#]', json('$(printf '%s' "$body" | sed "s/'/''/g")'))") done <<< "$listing" printf '%s' "$peers" } build_fleet() { local out=$1 now total systemd_n hib awake_json asleep_json versions now=$(now_iso) total=$(sq "SELECT COUNT(*) FROM routes WHERE kind='instance'") hib=$(sq "SELECT COUNT(*) FROM routes WHERE kind='instance' AND hibernate=1") systemd_n=$((total - hib)) # Awake = hibernating hosts whose socket is currently bound+music (see # music_sockets above). Hostnames stay inside this loop β€” only the # counts leave it. If `ss` is unavailable the split is published as null # (unknown), never guessed. if [ -n "$DRY_RUN" ] || command -v ss >/dev/null 2>&1; then local music awake=0 sock music=$(music_sockets) while IFS= read -r sock; do [ -z "$sock" ] && continue # -F fixed-string match: every socket path starts /run/lchat/, so one # path can never be a substring of another host's line. if printf '%s\n' "$music" | grep -qF "$sock"; then awake=$((awake+1)); fi done < <(sq "SELECT socket FROM routes WHERE kind='instance' AND hibernate=1") awake_json=$awake asleep_json=$((hib - awake)) else awake_json=null asleep_json=null fi # Version distribution: version strings and counts only (git SHAs / # release versions β€” already public via the signed release feed). No # hostnames, ever. versions=$(sq "SELECT COALESCE(json_group_array(json_object('version', v, 'count', c)), json('[]')) FROM (SELECT CASE WHEN version='' THEN 'unknown' ELSE version END AS v, COUNT(*) AS c FROM routes WHERE kind='instance' GROUP BY v ORDER BY c DESC, v)") local home_up=false push_up=false cloud_up=false connect_up=false rep_ok=false svc_active lchat-home && home_up=true svc_active lchat-push && push_up=true svc_active lchat-cloud && cloud_up=true svc_active lchat-connect && connect_up=true svc_active lchat-litestream && rep_ok=true local lag lag=$(replication_lag_json) # ---- capacity: the abstract room-for-new-spaces band ------------------------ # Worst-of over every input controld's atCapacity() enforces (controld.go # #663) plus memory pressure, each folded through pct_band's 70/90 lines. # Only the resulting WORD is published; every number below dies in this # block. Caps come from the control daemon's env, defaulting to controld's # own code defaults: 40 instances, 5 GB disk floor, registered ceiling # 0 β†’ 10x instances. local max_inst min_free max_reg hib_creates band=0 b max_inst=$(env_int "$CONTROL_ENV" LCHAT_MAX_INSTANCES 40) min_free=$(env_int "$CONTROL_ENV" LCHAT_MIN_FREE_GB 5) max_reg=$(env_int "$CONTROL_ENV" LCHAT_MAX_REGISTERED 0) hib_creates=$(env_int "$CONTROL_ENV" LCHAT_CONTROL_HIBERNATE_CREATES 0) # A zero cap would be a config typo, and we must not divide by it. [ "$max_inst" -gt 0 ] || max_inst=40 if [ "$hib_creates" = 1 ]; then # hibernate-creates on: the instance cap guards what's MATERIALIZED # (systemd-managed rows β€” hibernating rows hold no process); total # registrations get the higher sanity ceiling. Mirrors atCapacity(). b=$(pct_band $((systemd_n * 100 / max_inst))); if [ "$b" -gt "$band" ]; then band=$b; fi [ "$max_reg" -gt 0 ] || max_reg=$((10 * max_inst)) b=$(pct_band $((total * 100 / max_reg))); if [ "$b" -gt "$band" ]; then band=$b; fi else b=$(pct_band $((total * 100 / max_inst))); if [ "$b" -gt "$band" ]; then band=$b; fi fi # Disk: the floor is controld's hard refusal line, so at/below it is # at-capacity outright. Above it, band the PROXIMITY to the floor # (floor/free): free at 10x the floor reads 10% (plenty), at 1.25x it reads # 80% (getting-full). local free_gb free_gb=$(disk_free_gb) if [ -n "$free_gb" ]; then if [ "$free_gb" -le "$min_free" ]; then band=2 else b=$(pct_band $((min_free * 100 / free_gb))); if [ "$b" -gt "$band" ]; then band=$b; fi fi fi # Memory: utilization (MemTotal - MemAvailable)/MemTotal feeds both the # capacity band and this box's load word. Unreadable meminfo β†’ load is # null and memory contributes nothing (unknowns never guess). local mem_total mem_avail mem_pct load_json=null mem_total=$(meminfo_kb MemTotal) mem_avail=$(meminfo_kb MemAvailable) if [ -n "$mem_total" ] && [ -n "$mem_avail" ] && [ "$mem_total" -gt 0 ]; then mem_pct=$(((mem_total - mem_avail) * 100 / mem_total)) b=$(pct_band "$mem_pct"); if [ "$b" -gt "$band" ]; then band=$b; fi load_json="\"$(load_word "$b")\"" fi local capacity_level capacity_level=$(band_word "$band") # ---- hosts: this box, plus every other live mesh member -------------------- # Labels are whimsical name-action pairs derived from a HASH of the machine # id ("ada-stargazing") β€” stable per box, and never peer URLs, IPs, # hostnames, or the machine id itself. Each box's awake/asleep entry here is # its OWN locally-verified split. mesh_peer_boxes must run FIRST: besides # every peer's own snapshot, it also writes THIS box's own self-published # one to $TMP/mesh-self-stats-result.json β€” computeBoxStats (meshstats.go) # already filters that to routes this box currently HRW-owns, so preferring # it here # (over the raw local `ss` sweep above) keeps this box's row from # double-reporting a mesh-replicated route a peer now owns. Falls back to # the local sweep when mesh is off, unpublished yet, or stale β€” same # "unknowns degrade gracefully" rule as the rest of this file. (Below, the # top-level `instances.awake`/`asleep` fields are summed across every box in # this array β€” a per-box figure here is this box's own ownership-scoped # view, not the fleet total.) The optional region is the operator-set coarse # continent + country word pair (see PRIVACY) β€” absent env, absent key, # never guessed; escaped like the version strings so a quote can't break out # of the SQL literal. A peer's region (if its own LCHAT_BOX_REGION is set β€” # meshstats.go's boxRegion) rides through automatically: mesh_peer_boxes # inserts each peer's whole self-published object verbatim, so an extra key # needs no bash change here. `always_on` stays a genuinely local-box-only # concept, though β€” no peer publishes one. local boxes peer_boxes region_kv="" peer_boxes=$(mesh_peer_boxes) local self_stats_file="$TMP/mesh-self-stats-result.json" if [ -s "$self_stats_file" ]; then local self_esc self_awake self_esc=$(cat "$self_stats_file" | sed "s/'/''/g") self_awake=$(sqj "SELECT json_extract('$self_esc', '\$.awake')") if [ "$self_awake" != null ] && [ -n "$self_awake" ]; then awake_json=$self_awake asleep_json=$(sqj "SELECT json_extract('$self_esc', '\$.asleep')") fi fi # Resolved here, not at file top: DRY_RUN/argv must be parsed first so the # dry-run's no-live-probes guard actually guards (and the dry-run fixture's # direct BOX_REGION= assignment still wins over a re-resolve). [ -z "${BOX_REGION:-}" ] && BOX_REGION=$(box_region) if [ -n "$BOX_REGION" ]; then region_kv=", 'region', '$(printf '%s' "$BOX_REGION" | sed "s/'/''/g")'" fi boxes=$(sqj "SELECT json_array(json_object( 'label', '$(box_label)', 'always_on', $systemd_n, 'awake', json('$awake_json'), 'asleep', json('$asleep_json'), 'load', json('$load_json')$region_kv))") boxes=$(sqj "SELECT json_group_array(json(value)) FROM ( SELECT value FROM json_each('$(printf '%s' "$boxes" | sed "s/'/''/g")') UNION ALL SELECT value FROM json_each('$(printf '%s' "$peer_boxes" | sed "s/'/''/g")'))") local host_count host_count=$(sqj "SELECT json_array_length('$(printf '%s' "$boxes" | sed "s/'/''/g")')") # ---- fleet-wide awake/asleep: sum every box's own verified count ---------- # `hib` above is already fleet-wide (this box's registry is the replicated # superset β€” mesh-seed/fan-out only ever ADD rows to a peer, never remove # them here), but `awake_json`/`asleep_json` up to this point are still only # this box's own `ss` check β€” a host actually running on a peer has no local # socket, so it silently read as "asleep" here even while genuinely awake. # `boxes` now holds every live box's own verified split, so summing across # it turns the local view into the real fleet-wide one. If any box's awake # is unknown (null β€” e.g. `ss` missing there), the fleet split stays null # too, rather than guessing: same "unknowns never guess" rule as the rest of # this file. local boxes_esc unknown_awake boxes_esc=$(printf '%s' "$boxes" | sed "s/'/''/g") unknown_awake=$(sqj "SELECT COUNT(*) FROM json_each('$boxes_esc') WHERE json_extract(value, '\$.awake') IS NULL") if [ "$unknown_awake" = 0 ]; then awake_json=$(sqj "SELECT SUM(json_extract(value, '\$.awake')) FROM json_each('$boxes_esc')") asleep_json=$((hib - awake_json)) else awake_json=null asleep_json=null fi # Assembled by sqlite json_object: the only interpolated values are # script-controlled numerals/booleans/band-words and the sqlite-built arrays. sqj "SELECT json_object( 'generated_at', '$now', 'instances', json_object( 'total', $total, 'systemd', $systemd_n, 'hibernating', $hib, 'awake', json('$awake_json'), 'asleep', json('$asleep_json')), 'hosts', json_object( 'count', $host_count, 'boxes', json('$boxes')), 'capacity', json_object( 'level', '$capacity_level'), 'versions', json('$(printf '%s' "$versions" | sed "s/'/''/g")'), 'services', json_object( 'home', json('$home_up'), 'push', json('$push_up'), 'cloud', json('$cloud_up'), 'connect', json('$connect_up')), 'replication', json_object( 'healthy', json('$rep_ok'), 'lag_seconds', json('$lag')))" > "$out" } # ---- books.json -------------------------------------------------------------- # stripe_get β€” one authenticated GET. The key rides # only in the header; -sS keeps curl from ever printing it. Dry-run reads # numbered fixture files instead (same downstream aggregation code). STRIPE_FIXTURE_N=0 stripe_get() { if [ -n "$DRY_RUN" ]; then STRIPE_FIXTURE_N=$((STRIPE_FIXTURE_N + 1)) local kind=subs case "$1" in /v1/balance_transactions*) kind=bal ;; /v1/application_fees*) kind=fees ;; esac cp "$FIXDIR/stripe-$kind-$STRIPE_FIXTURE_N.json" "$2" return fi curl -sSg --fail --max-time 30 -H "Authorization: Bearer $STRIPE_KEY" \ "$STRIPE_API$1" > "$2" } # fetch_pages β€” follow Stripe's cursor pagination # (has_more + starting_after) into $TMP/-N.json. Hard cap of 50 pages # (5000 objects) keeps a pathological account from wedging the run. Every # failure is an explicit `return 1` β€” callers run this inside an `if !`, where # set -e is suspended, and a half-fetched month must degrade to "unavailable", # never aggregate to zeros that read as "no income". fetch_pages() { local path=$1 query=$2 prefix=$3 after="" n=0 f has_more while :; do n=$((n + 1)) f="$TMP/$prefix-$n.json" stripe_get "$path?limit=100${query:+&$query}${after:+&starting_after=$after}" "$f" || return 1 # A page that isn't a valid list envelope is a failed/garbled fetch. [ "$(sqj "SELECT json_valid(readfile('$f')) AND json_type(readfile('$f'), '\$.data')='array'")" = "1" ] || return 1 has_more=$(sqj "SELECT json_extract(readfile('$f'), '\$.has_more')") [ "$has_more" = "1" ] && [ "$n" -lt 50 ] || break after=$(sqj "SELECT json_extract(readfile('$f'), '\$.data[#-1].id')") [ -n "$after" ] || return 1 done # Reset the fixture counter so the next endpoint's dry-run pages start at 1. STRIPE_FIXTURE_N=0 } books_unavailable() { # $1=out $2=reason (fixed, script-controlled) $3=expenses json ("" = omit) local exp_kv="" [ -n "${3:-}" ] && exp_kv=", 'expenses', json('$3')" sqj "SELECT json_object('generated_at', '$(now_iso)', 'available', json('false'), 'reason', '$2'$exp_kv)" > "$1" } # ---- expenses (infrastructure spend) ----------------------------------------- # Decision #19's other half: income aggregates, EXPENSES PUBLISH ITEMIZED β€” # one line per provider, real amount, that provider's own billing currency # (never FX-converted: publishing USD as USD and EUR as EUR means there is no # exchange rate to misstate). A provider that can't be honestly resolved is # OMITTED β€” never zero, never a guess β€” and with no providers at all the whole # expenses object is omitted, matching books_unavailable's degrade discipline. # env_cents β€” validate an operator-set integer-cents constant; echoes # nothing for anything that isn't a plain non-negative integer. env_cents() { case "${1:-}" in ''|*[!0-9]*) ;; *) printf '%s' "$1" ;; esac; } # fetch_aws_cents β€” one live Cost Explorer query: month start β†’ # tomorrow (so today's partial day is included), MONTHLY granularity, # UnblendedCost. Cost Explorer is a GLOBAL endpoint homed in us-east-1 β€” the # --region below is the CE API's home, unrelated to STATUS_REGION (the status # bucket's region). Integrity before publishing: the Amount must be a plain # decimal and the Unit USD, or the fetch counts as failed β€” a garbled or # negative figure (credits) is omitted, never misstated. Dry-run twin reads a # fixture instead of calling AWS. fetch_aws_cents() { local month=$1 f="$TMP/aws-ce.json" end amount unit if [ -n "$DRY_RUN" ]; then [ -f "$FIXDIR/aws-ce.json" ] || return 1 cp "$FIXDIR/aws-ce.json" "$f" else command -v aws >/dev/null 2>&1 || return 1 end=$(sqj "SELECT date('now','+1 day')") aws ce get-cost-and-usage --region us-east-1 \ --time-period "Start=$month-01,End=$end" \ --granularity MONTHLY --metrics UnblendedCost \ --cli-connect-timeout 10 --cli-read-timeout 30 \ --output json > "$f" 2>/dev/null || return 1 fi [ "$(sqj "SELECT json_valid(readfile('$f'))")" = "1" ] || return 1 amount=$(sqj "SELECT json_extract(readfile('$f'), '\$.ResultsByTime[0].Total.UnblendedCost.Amount')") unit=$(sqj "SELECT json_extract(readfile('$f'), '\$.ResultsByTime[0].Total.UnblendedCost.Unit')") [ "$unit" = "USD" ] || return 1 case "$amount" in ''|.|*[!0-9.]*|*.*.*) return 1 ;; esac sqj "SELECT CAST(ROUND($amount * 100) AS INTEGER)" } # hetzner_get_page β€” one authenticated GET of the Hetzner # Cloud servers list (Bearer auth; the token rides only in the header β€” -sS # keeps curl from ever printing it). Dry-run reads numbered fixture pages # instead, so the caller's pagination loop is exercised by the twin. hetzner_get_page() { if [ -n "$DRY_RUN" ]; then [ -f "$FIXDIR/hetzner-servers-$1.json" ] || return 1 cp "$FIXDIR/hetzner-servers-$1.json" "$2" return fi [ -n "${STATUS_HETZNER_TOKEN:-}" ] || return 1 curl -sS --fail --max-time 30 \ -H "Authorization: Bearer $STATUS_HETZNER_TOKEN" \ "https://api.hetzner.cloud/v1/servers?page=$1&per_page=50" > "$2" } # fetch_hetzner_cents β€” the account's monthly server cost, computed # from the Cloud API: GET /v1/servers, pages followed via # meta.pagination.next_page (hard cap 10 pages / 500 servers β€” a partial sum # must never publish). Each server embeds its type's per-location price list # (server_type.prices[]); the entry whose location matches the server's own # location.name prices it β€” price_monthly.gross, a decimal string, VAT # included. EXACTLY one entry may match per server or the fetch counts as # failed: no match means the server can't be honestly priced, and a duplicate # would double-count. The prices carry no currency field, so the published # `eur` is the documented constant (Hetzner bills in EUR), not parsed. The # figure is SERVER COMPUTE ONLY, deliberately: primary IPs, backups, volumes, # and traffic overages price via the separate /pricing endpoint, not # per-server β€” revisit if that delta ever matters. The arg is unused # (a flat price list, not month-to-date); it exists for cached_cost_cents' # shared interface. STATUS_HETZNER_TOKEN is a READ-ONLY Cloud API token and # must never be echoed or logged; the operator constant # STATUS_HETZNER_MONTHLY_CENTS overrides this fetch entirely (expenses_object). fetch_hetzner_cents() { local page=1 n=0 f next amt load="CREATE TABLE srv(j TEXT);" local match="json_extract(p.value,'\$.location') = json_extract(s.j,'\$.location.name')" while :; do n=$((n + 1)) f="$TMP/hetzner-servers-$n.json" hetzner_get_page "$page" "$f" || return 1 [ "$(sqj "SELECT json_valid(readfile('$f')) AND json_type(readfile('$f'), '\$.servers')='array' AND json_type(readfile('$f'), '\$.meta.pagination')='object'")" = "1" ] || return 1 load="$load INSERT INTO srv SELECT value FROM json_each(readfile('$f'), '\$.servers');" next=$(sqj "SELECT COALESCE(json_extract(readfile('$f'), '\$.meta.pagination.next_page'), '')") [ -n "$next" ] || break case "$next" in *[!0-9]*) return 1 ;; esac [ "$n" -lt 10 ] || return 1 page=$next done # Exactly one matching price row per server (see above), and every matched # amount a plain decimal β€” or the figure is omitted, never misstated. [ "$(sqj "$load SELECT COUNT(*) FROM srv s WHERE (SELECT COUNT(*) FROM json_each(s.j, '\$.server_type.prices') p WHERE $match) <> 1")" = "0" ] || return 1 while IFS= read -r amt; do case "$amt" in ''|.|*[!0-9.]*|*.*.*) return 1 ;; esac done < <(sqj "$load SELECT json_extract(p.value, '\$.price_monthly.gross') FROM srv s, json_each(s.j, '\$.server_type.prices') p WHERE $match") sqj "$load SELECT CAST(ROUND(COALESCE(SUM(json_extract(p.value, '\$.price_monthly.gross')), 0) * 100) AS INTEGER) FROM srv s, json_each(s.j, '\$.server_type.prices') p WHERE $match" } # fetch_anthropic_cents β€” one live Admin API cost-report call: # GET /v1/organizations/cost_report, daily buckets from the month start # (limit=31 covers a full month in one page; no ending_at β†’ runs to now, so # today's partial day is included). Amounts come back as decimal strings of # CENTS ("123.45" = $1.23), currency documented as always USD β€” a non-USD row # or a has_more page we didn't fetch counts as failure (never publish a # partial or unverified figure). STATUS_ANTHROPIC_ADMIN_KEY is an ADMIN key # (sk-ant-admin…) with read-only usage scope β€” NOT the sidecars' completion # key: it can read org cost/usage metadata and nothing else. It rides only in # the x-api-key header (-sS keeps curl from printing it) and must never be # echoed or logged. Dry-run twin reads a fixture instead. fetch_anthropic_cents() { local month=$1 f="$TMP/anthropic-cost.json" if [ -n "$DRY_RUN" ]; then [ -f "$FIXDIR/anthropic-cost.json" ] || return 1 cp "$FIXDIR/anthropic-cost.json" "$f" else [ -n "${STATUS_ANTHROPIC_ADMIN_KEY:-}" ] || return 1 curl -sS --fail --max-time 30 \ -H "x-api-key: $STATUS_ANTHROPIC_ADMIN_KEY" \ -H "anthropic-version: 2023-06-01" \ "https://api.anthropic.com/v1/organizations/cost_report?starting_at=${month}-01T00:00:00Z&bucket_width=1d&limit=31" \ > "$f" || return 1 fi [ "$(sqj "SELECT json_valid(readfile('$f')) AND json_type(readfile('$f'), '\$.data')='array'")" = "1" ] || return 1 [ "$(sqj "SELECT COALESCE(json_extract(readfile('$f'), '\$.has_more'), 0)")" != "1" ] || return 1 [ "$(sqj "SELECT COUNT(*) FROM json_each(readfile('$f'), '\$.data') d, json_each(d.value, '\$.results') r WHERE json_extract(r.value, '\$.currency') <> 'USD'")" = "0" ] || return 1 sqj "SELECT CAST(ROUND(COALESCE(SUM(json_extract(r.value, '\$.amount')), 0)) AS INTEGER) FROM json_each(readfile('$f'), '\$.data') d, json_each(d.value, '\$.results') r" } # fetch_dnsimple_cents β€” one billing-charges page from DNSimple's API # (the product domains live there): GET /v2/{account}/billing/charges with # start_date=-01 / end_date= (ISO dates; tomorrow so today's # charges are included), per_page=100. A month of domain charges is a handful β€” # pagination.total_pages > 1 counts as failure (a partial sum must never # publish). Amounts are decimal strings of DOLLARS ("14.50"); the response # carries NO currency field, so the published currency is the documented # constant (DNSimple's price book is USD), not parsed. Only `collected` # charges sum β€” a refunded charge is not spend β€” and every summed amount must # be a plain decimal or the fetch counts as failed. STATUS_DNSIMPLE_TOKEN is # an account API token, read-only use here: it rides only in the # Authorization header (-sS keeps curl from printing it) and must never be # echoed or logged. Dry-run twin reads a fixture instead. fetch_dnsimple_cents() { local month=$1 f="$TMP/dnsimple-charges.json" end amt if [ -n "$DRY_RUN" ]; then [ -f "$FIXDIR/dnsimple-charges.json" ] || return 1 cp "$FIXDIR/dnsimple-charges.json" "$f" else [ -n "${STATUS_DNSIMPLE_TOKEN:-}" ] && [ -n "${STATUS_DNSIMPLE_ACCOUNT:-}" ] || return 1 end=$(sqj "SELECT date('now','+1 day')") curl -sS --fail --max-time 30 \ -H "Authorization: Bearer $STATUS_DNSIMPLE_TOKEN" \ "https://api.dnsimple.com/v2/$STATUS_DNSIMPLE_ACCOUNT/billing/charges?start_date=${month}-01&end_date=$end&per_page=100" \ > "$f" || return 1 fi [ "$(sqj "SELECT json_valid(readfile('$f')) AND json_type(readfile('$f'), '\$.data')='array'")" = "1" ] || return 1 [ "$(sqj "SELECT COALESCE(json_extract(readfile('$f'), '\$.pagination.total_pages'), 1) <= 1")" = "1" ] || return 1 while IFS= read -r amt; do case "$amt" in ''|.|*[!0-9.]*|*.*.*) return 1 ;; esac done < <(sqj "SELECT json_extract(c.value, '\$.total_amount') FROM json_each(readfile('$f'), '\$.data') c WHERE json_extract(c.value, '\$.state')='collected'") sqj "SELECT CAST(ROUND(COALESCE(SUM(json_extract(c.value, '\$.total_amount')), 0) * 100) AS INTEGER) FROM json_each(readfile('$f'), '\$.data') c WHERE json_extract(c.value, '\$.state')='collected'" } # cached_cost_cents [default-age-hours] β€” the shared # fetch-through-cache for the API-computed providers (aws, hetzner, anthropic, # dnsimple β€” fetch__cents above). These APIs charge or rate-limit # per request (Cost Explorer bills ~$0.01 per query) and this script runs # every 5 minutes β€” an uncached publisher would spend real money asking how # much it spends. So each provider's result is cached in $COSTS_CACHE and # refetched only when its entry is older than STATUS_COSTS_MAX_AGE_HOURS # (default: the caller's default-age-hours arg, else 6 β€” hetzner passes 24, # its flat price list changes ~never) or from a previous month (last month's # total is not this month's spend β€” a cross-month entry never publishes; for # hetzner that just means one refetch at the month turn). A failed refetch # falls back to the valid-but-stale same-month entry (stale beats absent); # no usable entry + failure β†’ echoes nothing and the provider is omitted. # The cache field name month_to_date_cents is generic storage β€” for hetzner # it holds the flat monthly figure. cached_cost_cents() { local provider=$1 month=$2 now have_cache="" cache_ts=0 cache_month="" cache_cents="" max_age_h cents now=$(date -u +%s) max_age_h=${STATUS_COSTS_MAX_AGE_HOURS:-${3:-6}} case "$max_age_h" in ''|*[!0-9]*) max_age_h=6 ;; esac if [ -f "$COSTS_CACHE" ] && [ "$(sqj "SELECT json_valid(readfile('$COSTS_CACHE'))" 2>/dev/null)" = "1" ]; then have_cache=1 cache_ts=$(sqj "SELECT COALESCE(json_extract(readfile('$COSTS_CACHE'), '\$.$provider.fetched_at'), 0)") cache_month=$(sqj "SELECT COALESCE(json_extract(readfile('$COSTS_CACHE'), '\$.$provider.month'), '')") cache_cents=$(sqj "SELECT COALESCE(json_extract(readfile('$COSTS_CACHE'), '\$.$provider.month_to_date_cents'), '')") fi case "$cache_ts" in ''|*[!0-9]*) cache_ts=0 ;; esac case "$cache_cents" in ''|*[!0-9]*) cache_cents="" ;; esac [ "$cache_month" = "$month" ] || cache_cents="" # cross-month cache never publishes if [ -n "$cache_cents" ] && [ $((now - cache_ts)) -lt $((max_age_h * 3600)) ]; then echo "$cache_cents" return 0 fi cents=$("fetch_${provider}_cents" "$month") || cents="" case "$cents" in ''|*[!0-9]*) cents="" ;; esac if [ -n "$cents" ]; then # Merge this provider's entry into the cache (other providers' entries survive). local base="'{}'" [ -n "$have_cache" ] && base="readfile('$COSTS_CACHE')" sqj "SELECT json_set($base, '\$.$provider', json_object('fetched_at', $now, 'month', '$month', 'month_to_date_cents', $cents))" \ > "$COSTS_CACHE.tmp" && mv "$COSTS_CACHE.tmp" "$COSTS_CACHE" echo "$cents" return 0 fi [ -n "$cache_cents" ] && echo "$cache_cents" return 0 } # expenses_object β€” the expenses JSON object, or nothing when no provider # resolves. Providers, in publish order: # aws β€” Cost Explorer month-to-date (metered; cached above), USD. # hetzner β€” the flat-rate Hetzner Cloud servers. No billing API exists, # but the servers API prices each server (its type's embedded # per-location price list), so STATUS_HETZNER_TOKEN computes # the account's monthly server cost (fetch_hetzner_cents β€” # server compute only; cached, 24h default freshness). The # operator-set constant STATUS_HETZNER_MONTHLY_CENTS overrides # the API when set β€” operator truth wins. EUR either way # (Hetzner bills in EUR). Neither set β†’ omitted. # anthropic β€” Admin API cost report month-to-date (metered; cached), USD β€” # the bot sidecars' API spend. # claude.ai β€” the claude.ai subscriptions: no API for them, and the honest # figure Paul can attest is LIFETIME spend, not a monthly β€” so # the operator-set constant STATUS_CLAUDE_TO_DATE_CENTS (EUR) # publishes as to_date_cents, the cumulative shape. It's an # operator-attested estimate by nature, which is why the page # labels it "(approx.)". # dnsimple β€” the product domains (elevenmessenger.com, 11ch.at, # 11factor.org): billing-charges month-to-date (metered; # cached), USD. # apple β€” the Apple Developer Program membership: fixed YEARLY, no # billing API, so the operator-set constant # STATUS_APPLE_YEARLY_CENTS (EUR). Published as billed β€” # yearly_cents, never divided into a fake monthly. expenses_object() { local month providers="" cents month=$(sqj "SELECT strftime('%Y-%m','now')") cents=$(cached_cost_cents aws "$month") [ -n "$cents" ] && providers=$(sqj "SELECT json_object('provider','aws','month_to_date_cents',$cents,'currency','usd')") # Operator constant first β€” when set, the API is never called (or cached). cents=$(env_cents "${STATUS_HETZNER_MONTHLY_CENTS:-}") [ -n "$cents" ] || cents=$(cached_cost_cents hetzner "$month" 24) [ -n "$cents" ] && providers="${providers:+$providers,}$(sqj "SELECT json_object('provider','hetzner','monthly_cents',$cents,'currency','eur')")" cents=$(cached_cost_cents anthropic "$month") [ -n "$cents" ] && providers="${providers:+$providers,}$(sqj "SELECT json_object('provider','anthropic','month_to_date_cents',$cents,'currency','usd')")" # Cumulative, not monthly: the operator attests lifetime spend to date (an # approximation by nature β€” the page carries the "(approx.)" marker). cents=$(env_cents "${STATUS_CLAUDE_TO_DATE_CENTS:-}") [ -n "$cents" ] && providers="${providers:+$providers,}$(sqj "SELECT json_object('provider','claude.ai','to_date_cents',$cents,'currency','eur')")" cents=$(cached_cost_cents dnsimple "$month") [ -n "$cents" ] && providers="${providers:+$providers,}$(sqj "SELECT json_object('provider','dnsimple','month_to_date_cents',$cents,'currency','usd')")" cents=$(env_cents "${STATUS_APPLE_YEARLY_CENTS:-}") [ -n "$cents" ] && providers="${providers:+$providers,}$(sqj "SELECT json_object('provider','apple','yearly_cents',$cents,'currency','eur')")" [ -n "$providers" ] || return 0 sqj "SELECT json_object('month', '$month', 'providers', json('[$providers]'))" } build_books() { local out=$1 now month month_start expenses_json expenses_kv="" now=$(now_iso) # Expenses resolve independently of Stripe β€” income being unavailable must # not hide infrastructure spend we can honestly state, so the expenses # object rides the degraded shape too (and is simply absent when no # provider resolves). expenses_json=$(expenses_object) [ -n "$expenses_json" ] && expenses_kv=", 'expenses', json('$expenses_json')" # The Stripe key comes from the cloud provisioner's env (billing.go reads the # same var). Absent β†’ billing isn't configured here; publish an honest # "unavailable" rather than zeros that would read as "no income". STRIPE_KEY=$(grep -E '^LCHAT_STRIPE_SECRET_KEY=' "$CLOUD_ENV" 2>/dev/null | head -n1 | cut -d= -f2- || true) if [ -z "${STRIPE_KEY:-}" ]; then books_unavailable "$out" "billing not configured" "$expenses_json" return fi # Month boundary via sqlite (portable β€” GNU/BSD date flags differ). month=$(sqj "SELECT strftime('%Y-%m','now')") month_start=$(sqj "SELECT strftime('%s', datetime('now','start of month'))") # The three aggregation sources (decision #19 β€” these are the ONLY Stripe # reads, all fleet-wide): # GET /v1/subscriptions?status=active&limit=100 (+ cursor pages) # β†’ active count, per-tier counts (metadata.tier = public price-book # key), MRR from each item's price amount normalized to monthly. # GET /v1/balance_transactions?created[gte]=&limit=100 # β†’ gross collected this month (positive charge/payment amounts). # GET /v1/application_fees?created[gte]=&expand[]=data.charge # β†’ spaces selling to their own members via Stripe Connect # (subscriptions.go's stripeConnect: direct charges on the space # admin's connected account, an application fee to the platform). # Those charges live on the CONNECTED accounts, so the two reads # above never see them; the fee list is the platform's one honest # window: each fee's amount is our commission, and its expanded # charge carries the gross sale amount β†’ GMV. # Degrade to "unavailable" on any fetch failure β€” a Stripe outage must not # publish half-numbers. if ! fetch_pages /v1/subscriptions "status=active" subs || ! fetch_pages /v1/balance_transactions "created[gte]=$month_start" bal || ! fetch_pages /v1/application_fees "created[gte]=$month_start&expand[]=data.charge" fees; then books_unavailable "$out" "stripe unreachable" "$expenses_json" return fi # Aggregate with sqlite json_each β€” never per-object output, only SUM/COUNT/ # GROUP BY. Per-payer fields (ids, customer, created) die inside this query. { echo "CREATE TABLE subs(j TEXT);" echo "CREATE TABLE bal(j TEXT);" echo "CREATE TABLE fees(j TEXT);" local f for f in "$TMP"/subs-*.json; do echo "INSERT INTO subs SELECT value FROM json_each(readfile('$f'), '\$.data');" done for f in "$TMP"/bal-*.json; do echo "INSERT INTO bal SELECT value FROM json_each(readfile('$f'), '\$.data');" done for f in "$TMP"/fees-*.json; do echo "INSERT INTO fees SELECT value FROM json_each(readfile('$f'), '\$.data');" done # MRR: unit_amount Γ— quantity per item, normalized to a month. EUR items # only (the price book is EUR; mixing currencies without FX would lie). # Unknown intervals contribute 0 to MRR but still count as subscribers. # Connect: commission = the fee amounts, GMV = each fee's expanded charge # amount β€” same gross semantics as collected_month_cents (positive EUR # amounts; refunds don't net). GMV is the spaces' money, not ours: our # income from these sales is exactly the commission line, which is why it # ships as its own field and never folds into collected_month_cents. cat < 0), 0) AS INTEGER), 'connect', json_object( 'gmv_month_cents', CAST(COALESCE(( SELECT SUM(json_extract(f.j,'\$.charge.amount')) FROM fees f WHERE json_extract(f.j,'\$.charge.currency')='eur' AND json_extract(f.j,'\$.charge.amount') > 0), 0) AS INTEGER), 'commission_month_cents', CAST(COALESCE(( SELECT SUM(json_extract(f.j,'\$.amount')) FROM fees f WHERE json_extract(f.j,'\$.currency')='eur' AND json_extract(f.j,'\$.amount') > 0), 0) AS INTEGER))$expenses_kv); SQL } | sqlite3 :memory: > "$out" } # ---- carbon.json: republish the committed built ledger + inject a live run block # grid_band β€” the box region's grid carbon intensity as " " # g COβ‚‚e/kWh. low is always ~0 (a market-based renewable contract β€” Hetzner's # EU sites run on renewable electricity); central+high are the LOCATION grid, # banded. Region resolves from box_region() (coarse continent+country; the prod # box is Helsinki β†’ Finland's low-carbon grid). Operator overrides: # STATUS_CARBON_GRID_CENTRAL / _HIGH. Cited in docs/comparison-carbon.md. grid_band() { local gce ghi region region=${BOX_REGION:-$(box_region)} case "$region" in "EU Β· Finland") gce=100; ghi=130 ;; # Finnish grid ~80–130 (Fingrid/ElectricityMaps 2026) "EU Β· Sweden") gce=40; ghi=60 ;; "EU Β· Germany") gce=350; ghi=400 ;; "EU Β· Ireland") gce=300; ghi=380 ;; "EU Β· UK") gce=210; ghi=280 ;; "NA Β· USA") gce=370; ghi=480 ;; *) gce=350; ghi=480 ;; # unknown: IEA-ish global band esac [ -n "${STATUS_CARBON_GRID_CENTRAL:-}" ] && gce=$STATUS_CARBON_GRID_CENTRAL [ -n "${STATUS_CARBON_GRID_HIGH:-}" ] && ghi=$STATUS_CARBON_GRID_HIGH printf '0 %s %s' "$gce" "$ghi" } # hetzner_vcpus β€” total vCPUs across every server in the Hetzner account (the # hardware the whole fleet runs on; sums so it scales as boxes are added). Same # paginated servers-API read as fetch_hetzner_cents (dry-run twin included); # each server_type.cores must be an integer or we can't honestly total. Prints # the sum, or returns 1 (no token / bad page) so the run block stays absent. hetzner_vcpus() { local page=1 n=0 f next load="CREATE TABLE srv(j TEXT);" while :; do n=$((n + 1)) f="$TMP/hetzner-vcpus-$n.json" hetzner_get_page "$page" "$f" || return 1 [ "$(sqj "SELECT json_valid(readfile('$f')) AND json_type(readfile('$f'), '\$.servers')='array' AND json_type(readfile('$f'), '\$.meta.pagination')='object'")" = "1" ] || return 1 load="$load INSERT INTO srv SELECT value FROM json_each(readfile('$f'), '\$.servers');" next=$(sqj "SELECT COALESCE(json_extract(readfile('$f'), '\$.meta.pagination.next_page'), '')") [ -n "$next" ] || break case "$next" in *[!0-9]*) return 1 ;; esac [ "$n" -lt 10 ] || return 1 page=$next done [ "$(sqj "$load SELECT COUNT(*) FROM srv WHERE json_type(j, '\$.server_type.cores') <> 'integer'")" = "0" ] || return 1 sqj "$load SELECT COALESCE(SUM(json_extract(j, '\$.server_type.cores')), 0) FROM srv" } # carbon_run_object β€” the live operational estimate as a JSON object, or nothing # printed when the router term can't be resolved (no Hetzner token / fetch fail) # so the published `run` stays absent and the page shows "pending", never a fake # zero. Math (all BANDED): daily kg = vCPUs Γ— W/vCPU Γ— 24h Γ— grid(g/kWh) Γ— PUE # / 1e6, plus small explicit AWS-storage + bot-inference terms. Reported as a # daily rate, a cumulative-to-date (Γ— days since CARBON_SINCE, or a pinned # STATUS_CARBON_DAYS), and normalised per active space (denominator = the public # instance count from the registry β€” NEVER per-member). Aggregates only: vCPU # total, watts, grid, kg, space count β€” no hostnames/IPs/per-space needles. carbon_run_object() { local vcpus glo gce ghi spaces days since now vcpus=$(hetzner_vcpus) || vcpus="" case "$vcpus" in ''|*[!0-9]*|0) return 0 ;; esac # no router term β†’ run absent set -- $(grid_band); glo=$1 gce=$2 ghi=$3 spaces=$(sq "SELECT COUNT(*) FROM routes WHERE kind='instance'" 2>/dev/null || echo "") case "$spaces" in ''|*[!0-9]*|0) spaces=1 ;; esac since=$CARBON_SINCE days=${STATUS_CARBON_DAYS:-} [ -n "$days" ] || days=$(sqj "SELECT MAX(1, CAST(julianday('now') - julianday('$since') AS INTEGER))") case "$days" in ''|*[!0-9]*) days=1 ;; esac now=$(now_iso) # Daily bands live in the CTE d(lo,ce,hi); cumulative/per-space derive from the # UNROUNDED daily so rounding never compounds. Watts round to whole; kg to 6dp. sqj "WITH d(lo, ce, hi) AS (VALUES( $vcpus*$CARBON_W_LO*24*$glo*$CARBON_PUE/1e6 + $CARBON_AWS_LO + $CARBON_INF_LO, $vcpus*$CARBON_W_CE*24*$gce*$CARBON_PUE/1e6 + $CARBON_AWS_CE + $CARBON_INF_CE, $vcpus*$CARBON_W_HI*24*$ghi*$CARBON_PUE/1e6 + $CARBON_AWS_HI + $CARBON_INF_HI)) SELECT json_object( 'generated_at', '$now', 'since', '$since', 'spaces', $spaces, 'pue', $CARBON_PUE, 'vcpus', $vcpus, 'watts', json_object('low', CAST(ROUND($vcpus*$CARBON_W_LO) AS INTEGER), 'central', CAST(ROUND($vcpus*$CARBON_W_CE) AS INTEGER), 'high', CAST(ROUND($vcpus*$CARBON_W_HI) AS INTEGER)), 'grid_g_per_kwh', json_object('low', $glo, 'central', $gce, 'high', $ghi), 'daily_kg', json_object('low', ROUND(lo,6), 'central', ROUND(ce,6), 'high', ROUND(hi,6)), 'cumulative_kg', json_object('low', ROUND(lo*$days,6), 'central', ROUND(ce*$days,6), 'high', ROUND(hi*$days,6)), 'per_space_daily_kg', json_object('low', ROUND(lo/$spaces,6), 'central', ROUND(ce/$spaces,6), 'high', ROUND(hi/$spaces,6)), 'services', json_array( json_object('name', 'router + instances', 'daily_kg_central', ROUND($vcpus*$CARBON_W_CE*24*$gce*$CARBON_PUE/1e6, 6)), json_object('name', 'object storage (AWS)', 'daily_kg_central', $CARBON_AWS_CE), json_object('name', 'bot inference', 'daily_kg_central', $CARBON_INF_CE))) FROM d" } # build_carbon β€” republish the committed carbon.json (the built # ledger, tools/carbon-tally.mjs) and inject a live `run` block. We nest HERE # rather than in the tool: the committed file stays exactly the built ledger # ({ built:{…} }), and the only transform is this one merge β€” so "carbon is # committed data" holds and the run block never touches git. A malformed source # or no live inputs β†’ republished verbatim (run stays absent β†’ page = pending). build_carbon() { local out=$1 src=$2 if [ "$(sqj "SELECT json_valid(readfile('$src')) AND json_type(readfile('$src'), '\$.built')='object'")" != "1" ]; then cp "$src" "$out"; return fi carbon_run_object > "$TMP/carbon-run.json" || true if [ -s "$TMP/carbon-run.json" ]; then sqj "SELECT json_set(readfile('$src'), '\$.run', json(readfile('$TMP/carbon-run.json')))" > "$out" else cp "$src" "$out" fi } # ---- upload ------------------------------------------------------------------ upload() { # file key content-type cache-control aws s3 cp "$1" "s3://$BUCKET/$2" --region "$REGION" \ --content-type "$3" --cache-control "$4" --no-progress >/dev/null } publish() { command -v aws >/dev/null 2>&1 || { echo "aws CLI missing (apt install awscli)" >&2; exit 1; } # Creds for the scoped status-publisher IAM user (never in tofu state; mint # by hand per infra/prod/status.tf). The unit injects them via # EnvironmentFile; a manual run sources the same file. if [ -z "${AWS_ACCESS_KEY_ID:-}" ] && [ -f "$STATUS_ENV" ]; then set -a; . "$STATUS_ENV"; set +a fi build_fleet "$TMP/fleet.json" build_books "$TMP/books.json" # JSONs: short cache so "as of" stays honest at the 5-minute cadence. upload "$TMP/fleet.json" fleet.json "application/json" "max-age=60" upload "$TMP/books.json" books.json "application/json" "max-age=60" # The viewer pages + brand mark (from web/status/, staged on the box). # Re-uploading tiny idempotent files every run self-heals a missed copy. if [ -d "$PAGES_DIR" ]; then # The shared stylesheet goes up FIRST, before any page that s it, so a # freshly-uploaded page is never served ahead of its styles (a page fetched # in the gap would render unstyled until it revalidates). Same-origin in the # status bucket β€” no third-party request, so the privacy line holds. [ -f "$PAGES_DIR/status.css" ] && upload "$PAGES_DIR/status.css" status.css "text/css; charset=utf-8" "max-age=300" # The shared masthead+nav builder rides up with the stylesheet, before any # page that mounts it, for the same never-served-ahead reason. [ -f "$PAGES_DIR/status-nav.js" ] && upload "$PAGES_DIR/status-nav.js" status-nav.js "text/javascript; charset=utf-8" "max-age=300" [ -f "$PAGES_DIR/index.html" ] && upload "$PAGES_DIR/index.html" index.html "text/html; charset=utf-8" "max-age=300" [ -f "$PAGES_DIR/books.html" ] && upload "$PAGES_DIR/books.html" books.html "text/html; charset=utf-8" "max-age=300" # Tier economics: page + committed snapshot JSON (web/status/economics.json, # re-derived by hand when prices or input rates change β€” docs/status.md # Β§ Keeping the tier-economics page current). Republished as-is. [ -f "$PAGES_DIR/economics.html" ] && upload "$PAGES_DIR/economics.html" economics.html "text/html; charset=utf-8" "max-age=300" [ -f "$PAGES_DIR/economics.json" ] && upload "$PAGES_DIR/economics.json" economics.json "application/json" "max-age=300" # Investing: page + committed funding ledger + the waterfall reference # implementation (docs/funding.md). waterfall.js is the NORMATIVE # algorithm module β€” the page imports and runs the same file it serves, # so it must publish with an explicit JS content-type (module imports # fail on a guessed one). Republished as-is, like economics. [ -f "$PAGES_DIR/funding.html" ] && upload "$PAGES_DIR/funding.html" funding.html "text/html; charset=utf-8" "max-age=300" [ -f "$PAGES_DIR/funding.json" ] && upload "$PAGES_DIR/funding.json" funding.json "application/json" "max-age=300" [ -f "$PAGES_DIR/waterfall.js" ] && upload "$PAGES_DIR/waterfall.js" waterfall.js "text/javascript; charset=utf-8" "max-age=300" [ -f "$PAGES_DIR/carbon.html" ] && upload "$PAGES_DIR/carbon.html" carbon.html "text/html; charset=utf-8" "max-age=300" # The carbon tally is committed data (the built ledger) β€” republished as-is # EXCEPT for a live `run` block injected here (build_carbon). Built untouched, # run recomputed each publish; absent inputs β†’ run stays absent (page pending). if [ -f "$PAGES_DIR/carbon.json" ]; then build_carbon "$TMP/carbon.json" "$PAGES_DIR/carbon.json" upload "$TMP/carbon.json" carbon.json "application/json" "max-age=300" fi [ -f "$PAGES_DIR/contributors.html" ] && upload "$PAGES_DIR/contributors.html" contributors.html "text/html; charset=utf-8" "max-age=300" [ -f "$PAGES_DIR/contributors.json" ] && upload "$PAGES_DIR/contributors.json" contributors.json "application/json" "max-age=300" # The security ledger: page + committed, build-time-scanned, REDACTED JSON # (deploy/security-scan.sh β€” see the header). A missing security.json is # honest: the page renders "scan pending" rather than a fake clean, so the # page uploads even when the JSON hasn't been staged. [ -f "$PAGES_DIR/security.html" ] && upload "$PAGES_DIR/security.html" security.html "text/html; charset=utf-8" "max-age=300" [ -f "$PAGES_DIR/security.json" ] && upload "$PAGES_DIR/security.json" security.json "application/json" "max-age=300" # The prompt registry viewer. Its corpus (prompts/…) is NOT uploaded here: # a contributor accrues + syncs it straight to the bucket β€” far too large # for a 5-minute republish (docs/status.md Β§ The prompt registry). [ -f "$PAGES_DIR/prompts.html" ] && upload "$PAGES_DIR/prompts.html" prompts.html "text/html; charset=utf-8" "max-age=300" # The music ledger: page + committed index + month shards (accrued per # session by tools/music-ledger.mjs β€” art rides inside the shards as # data URIs, so the page stays external-request-free). Republished as-is, # like economics; the glob picks up new months automatically. [ -f "$PAGES_DIR/music.html" ] && upload "$PAGES_DIR/music.html" music.html "text/html; charset=utf-8" "max-age=300" for f in "$PAGES_DIR"/music*.json; do [ -f "$f" ] && upload "$f" "$(basename "$f")" "application/json" "max-age=300" done # Milestones: reconciled from the box's running copy 2026-07-18 β€” the line # was hand-added there (pre-dating this file's funding additions) and the # repo owed it back. [ -f "$PAGES_DIR/milestones.html" ] && upload "$PAGES_DIR/milestones.html" milestones.html "text/html; charset=utf-8" "max-age=300" # Roadmap: the forward companion to milestones β€” the plan for the road to V1. # Static like milestones (no JSON feed, no contract-test coverage). [ -f "$PAGES_DIR/roadmap.html" ] && upload "$PAGES_DIR/roadmap.html" roadmap.html "text/html; charset=utf-8" "max-age=300" [ -f "$PAGES_DIR/icon-192.png" ] && upload "$PAGES_DIR/icon-192.png" icon-192.png "image/png" "max-age=86400" else echo "warning: $PAGES_DIR missing β€” JSONs published, pages skipped (copy web/status/* there)" >&2 fi echo "published to s3://$BUCKET (fleet + books + pages, $(now_iso))" } # ---- install (root) ---------------------------------------------------------- install_units() { [ "$(id -u)" = 0 ] || { echo "install needs root (writes systemd units)" >&2; exit 1; } [ -f "$STATUS_ENV" ] || { echo "missing $STATUS_ENV (AWS creds for status-publisher) β€” mint per infra/prod/status.tf" >&2; exit 1; } if [ "$(readlink -f "$0" 2>/dev/null || echo "$0")" != "$SELF" ]; then cp "$0" "$SELF" && chmod 0755 "$SELF" fi mkdir -p "$PAGES_DIR" # Same hardening posture as the litestream unit. ReadWritePaths=/data is # required even though we only SELECT: WAL readers take shared-memory locks # (writes to the -shm file), so a read-only /data breaks registry reads. cat > "$UNIT" <<'EOF' [Unit] Description=Eleven public status publish (fleet + books aggregates) Wants=network-online.target After=network-online.target [Service] Type=oneshot User=lchat Group=lchat SupplementaryGroups=systemd-journal EnvironmentFile=/data/status.env ExecStart=/opt/lchat/status-publish.sh NoNewPrivileges=true ProtectSystem=strict ReadWritePaths=/data ProtectHome=true PrivateTmp=true MemoryMax=256M EOF # Persistent=false on purpose: a missed run's output is superseded by the # next one five minutes later β€” catch-up publishes are pure noise. cat > "$TIMER" <<'EOF' [Unit] Description=Eleven public status publish β€” every 5 minutes [Timer] OnBootSec=2min OnUnitActiveSec=5min RandomizedDelaySec=20 Persistent=false [Install] WantedBy=timers.target EOF systemctl daemon-reload systemctl enable --now lchat-status.timer echo "installed β€” verify with: systemctl list-timers lchat-status.timer; journalctl -u lchat-status -n 20" echo "remember: copy web/status/* (pages, committed JSONs, icon-192.png) to $PAGES_DIR" } # ---- dry-run: self-constructed fixtures, real aggregation code ---------------- dry_run() { local outdir=$1 mkdir -p "$outdir" DRY_RUN=1 FIXDIR="$TMP/fixtures" mkdir -p "$FIXDIR" REGISTRY_DB="$FIXDIR/registry.db" CLOUD_ENV="$FIXDIR/cloud.env" CONTROL_ENV="$FIXDIR/control.env" # A fake registry: 6 instances (2 systemd-managed, 4 hibernating of which 2 # are awake), 2 service rows (must not count as instances), 3 versions. sqlite3 "$REGISTRY_DB" <<'SQL' CREATE TABLE routes ( host TEXT PRIMARY KEY, kind TEXT NOT NULL, socket TEXT NOT NULL, version TEXT NOT NULL DEFAULT '', hibernate INTEGER NOT NULL DEFAULT 0 ); INSERT INTO routes VALUES ('alpha.example', 'instance', '/run/lchat/alpha.example.sock', 'v100', 0), ('bravo.example', 'instance', '/run/lchat/bravo.example.sock', 'v100', 0), ('charlie.example', 'instance', '/run/lchat/charlie.example.sock', 'v100', 1), ('delta.example', 'instance', '/run/lchat/delta.example.sock', 'v101', 1), ('echo.example', 'instance', '/run/lchat/echo.example.sock', 'v101', 1), ('foxtrot.example', 'instance', '/run/lchat/foxtrot.example.sock', '', 1), ('home.example', 'service', '/run/lchat/home.sock', '', 0), ('push.example', 'service', '/run/lchat/push.sock', '', 0); SQL # ss -xlH shape: charlie + delta music (awake), echo + foxtrot not. cat > "$FIXDIR/music.txt" <<'EOF' u_str LISTEN 0 4096 /run/lchat/alpha.example.sock 12001 * 0 u_str LISTEN 0 4096 /run/lchat/bravo.example.sock 12002 * 0 u_str LISTEN 0 4096 /run/lchat/charlie.example.sock 12003 * 0 u_str LISTEN 0 4096 /run/lchat/delta.example.sock 12004 * 0 EOF printf '%s\n' lchat-home lchat-push lchat-cloud lchat-litestream > "$FIXDIR/active-units.txt" # connect down on purpose # Fixed machine-id fixture so the dry-run label is deterministic everywhere # (a dev Mac has no /etc/machine-id; hostname fallback would vary per machine). echo dry-run-fixture-machine-id > "$FIXDIR/machine-id" export STATUS_MACHINE_ID_FILE="$FIXDIR/machine-id" # Fixed region fixture: exercises the STATUS_BOX_REGION seam end-to-end (the # contract test pins both the value and its coarse continent+country shape). BOX_REGION="EU Β· Germany" echo 42 > "$FIXDIR/replication-lag.txt" # Multi-host fixture (STATUS_DRY_MESH_PEER=1): one other live mesh box's # self-published stats snapshot, exactly the shape internal/activator/ # heartbeat.go's PutStats writes and mesh_peer_boxes reads back. Default # (unset) keeps hosts.count at 1, byte-identical to the single-box shape β€” # the mesh's own "absence changes nothing" convention, mirrored here. This # also seeds THIS box's own self-published stats fixture # (mesh-self-stats.json), gated the same way: when mesh is off, this box # never self-publishes either, so the single-box shape must stay untouched. if [ "${STATUS_DRY_MESH_PEER:-}" = 1 ]; then # awake=1 here, not some arbitrary number: this peer only ever runs # instances mesh-replicated FROM this box's own registry, so its awake # count must fit within this box's own currently-"asleep" locally instances # (echo + foxtrot, 2 of them) β€” otherwise the fleet-wide awake/asleep sum # below goes negative, which would mean the fixture models an impossible # state, not a real one. # region present here (unlike an earlier version of this fixture) β€” # exercises a peer that HAS set its own LCHAT_BOX_REGION, proving the # merge's raw passthrough carries an extra key with no bash-side change. cat > "$FIXDIR/mesh-peers.json" <<'EOF' [{"label":"turing-orienteering","awake":1,"asleep":1,"load":"busy","region":"UK Β· London"}] EOF # awake=1/asleep=2 (3 total) here, one fewer than the local `ss` sweep's # own awake=2/asleep=2 (4 total) above β€” proves build_fleet actually # prefers this ownership-filtered snapshot over its own unfiltered local # sweep, not just reads it and ignores it. cat > "$FIXDIR/mesh-self-stats.json" <<'EOF' {"label":"hypatia-sketching","awake":1,"asleep":2,"load":"comfortable","region":"EU Β· Germany"} EOF fi # Capacity fixtures (control caps + disk + meminfo): one set per public band # (default plenty; STATUS_DRY_CAPACITY=getting-full|at-capacity), so the # abstract derivation is testable end-to-end at every level without touching # box state. hibernate-creates is on in all sets (the live posture the bands # mirror); the caps are otherwise left unset so the default set exercises # controld.go's code defaults (40 / 5 GB floor / 10x registered). echo "LCHAT_CONTROL_HIBERNATE_CREATES=1" > "$FIXDIR/control.env" case "${STATUS_DRY_CAPACITY:-}" in getting-full) # Registered ratio 6/8 = 75% β†’ getting-full, while disk (100 GB) and # memory (25% used) stay roomy β€” capacity and box load are independent # signals, so load reads comfortable here. echo "LCHAT_MAX_REGISTERED=8" >> "$FIXDIR/control.env" echo 100 > "$FIXDIR/disk-free-gb.txt" printf 'MemTotal: 4014980 kB\nMemAvailable: 3011235 kB\n' > "$FIXDIR/meminfo.txt" ;; at-capacity) # Disk below the 5 GB floor β†’ at-capacity outright; memory at 95% used # β†’ load critical. echo 3 > "$FIXDIR/disk-free-gb.txt" printf 'MemTotal: 4014980 kB\nMemAvailable: 200749 kB\n' > "$FIXDIR/meminfo.txt" ;; *) # Everything roomy: ratios ~5%/1.5%, floor proximity 5%, memory 25% # used β†’ plenty / comfortable. echo 100 > "$FIXDIR/disk-free-gb.txt" printf 'MemTotal: 4014980 kB\nMemAvailable: 3011235 kB\n' > "$FIXDIR/meminfo.txt" ;; esac # STATUS_DRY_BOOKS=unavailable exercises the degraded books path (no key). if [ "${STATUS_DRY_BOOKS:-}" = unavailable ]; then : > "$FIXDIR/cloud.env" else echo "LCHAT_STRIPE_SECRET_KEY=sk_test_dryrun_not_a_real_key" > "$FIXDIR/cloud.env" fi # Expense fixtures (STATUS_DRY_EXPENSES=omit|stale-cache|hetzner-constant; # default = all six providers resolve, hetzner from its API fixture). The # cost cache is repointed into FIXDIR β€” the dry run never touches $DATA_DIR. # The API fixtures carry fixture-only ids (request id, workspace id, charge # references/domains, server names): only the parsed integers may survive # into books.json β€” the leak grep below is the proof. COSTS_CACHE="$FIXDIR/costs-cache.json" case "${STATUS_DRY_EXPENSES:-}" in omit) # No API fixtures, no cache, no operator constants β†’ every provider, # and then the whole expenses object, is omitted. STATUS_HETZNER_MONTHLY_CENTS="" STATUS_CLAUDE_TO_DATE_CENTS="" STATUS_APPLE_YEARLY_CENTS="" ;; stale-cache) # No API fixtures (all four "fetches" fail) but valid same-month cache # entries well past the max age β†’ the stale figures serve rather than # vanishing (stale beats absent for month-to-date). STATUS_HETZNER_MONTHLY_CENTS="" STATUS_CLAUDE_TO_DATE_CENTS="" STATUS_APPLE_YEARLY_CENTS="" sqj "SELECT json_object( 'aws', json_object('fetched_at', strftime('%s','now') - 999999, 'month', strftime('%Y-%m','now'), 'month_to_date_cents', 999), 'hetzner', json_object('fetched_at', strftime('%s','now') - 999999, 'month', strftime('%Y-%m','now'), 'month_to_date_cents', 1399), 'anthropic', json_object('fetched_at', strftime('%s','now') - 999999, 'month', strftime('%Y-%m','now'), 'month_to_date_cents', 888), 'dnsimple', json_object('fetched_at', strftime('%s','now') - 999999, 'month', strftime('%Y-%m','now'), 'month_to_date_cents', 777))" \ > "$COSTS_CACHE" ;; *) # Default: hetzner's constant is cleared so its API fixture drives the # figure (and an ambient operator env can't leak in); # STATUS_DRY_EXPENSES=hetzner-constant sets it WITH the fixture still # present, asserting the constant wins over a resolvable API path. STATUS_HETZNER_MONTHLY_CENTS="" [ "${STATUS_DRY_EXPENSES:-}" = hetzner-constant ] && STATUS_HETZNER_MONTHLY_CENTS=1400 STATUS_CLAUDE_TO_DATE_CENTS=229000 STATUS_APPLE_YEARLY_CENTS=9900 # Cost Explorer shape; Amount 12.3399999 USD exercises the cents rounding (β†’ 1234). cat > "$FIXDIR/aws-ce.json" <<'EOF' {"ce_fixture_request_id":"req_fixture_1","ResultsByTime":[{"TimePeriod":{"Start":"2026-07-01","End":"2026-07-12"},"Total":{"UnblendedCost":{"Amount":"12.3399999","Unit":"USD"}},"Groups":[],"Estimated":true}]} EOF # Hetzner servers shape, TWO pages (exercises meta.pagination.next_page): # each server carries its type's per-location price list, and only the # entry matching the server's own location.name may price it β€” the # decoy entries for the other location differ on purpose. Gross monthly # EUR strings: 16.4288 (cpx31 @ hel1; NOT the fsn1 16.1840) + 4.5100 # (cx22 @ fsn1; NOT the hel1 4.6900) = 20.9388 β†’ rounds to 2094 cents. # Server names are fixture markers that must not survive aggregation. cat > "$FIXDIR/hetzner-servers-1.json" <<'EOF' {"servers":[ {"id":101,"name":"hzfix-alpha","status":"running", "location":{"id":3,"name":"hel1","description":"Helsinki DC Park 1","country":"FI","city":"Helsinki","network_zone":"eu-central"}, "server_type":{"id":104,"name":"cpx31","cores":4,"memory":8,"disk":160,"deprecated":false, "prices":[ {"location":"fsn1","price_hourly":{"net":"0.0225","gross":"0.0268"},"price_monthly":{"net":"13.6000","gross":"16.1840"}}, {"location":"hel1","price_hourly":{"net":"0.0229","gross":"0.0273"},"price_monthly":{"net":"13.8057","gross":"16.4288"}} ]}} ],"meta":{"pagination":{"page":1,"per_page":1,"previous_page":null,"next_page":2,"last_page":2,"total_entries":2}}} EOF cat > "$FIXDIR/hetzner-servers-2.json" <<'EOF' {"servers":[ {"id":102,"name":"hzfix-bravo","status":"running", "location":{"id":1,"name":"fsn1","description":"Falkenstein DC Park 1","country":"DE","city":"Falkenstein","network_zone":"eu-central"}, "server_type":{"id":22,"name":"cx22","cores":2,"memory":4,"disk":40,"deprecated":false, "prices":[ {"location":"fsn1","price_hourly":{"net":"0.0063","gross":"0.0075"},"price_monthly":{"net":"3.7900","gross":"4.5100"}}, {"location":"hel1","price_hourly":{"net":"0.0066","gross":"0.0079"},"price_monthly":{"net":"3.9400","gross":"4.6900"}} ]}} ],"meta":{"pagination":{"page":2,"per_page":1,"previous_page":1,"next_page":null,"last_page":2,"total_entries":2}}} EOF # Anthropic cost-report shape: two daily buckets, amounts are decimal # strings of CENTS (150.25 + 99.5 β†’ rounds to 250). The workspace id is # a fixture marker that must not survive aggregation. cat > "$FIXDIR/anthropic-cost.json" <<'EOF' {"data":[ {"starting_at":"2026-07-01T00:00:00Z","ending_at":"2026-07-02T00:00:00Z", "results":[{"amount":"150.25","currency":"USD","workspace_id":"wrkspc_fixture_1"}]}, {"starting_at":"2026-07-02T00:00:00Z","ending_at":"2026-07-03T00:00:00Z", "results":[{"amount":"99.5","currency":"USD","workspace_id":"wrkspc_fixture_1"}]} ],"has_more":false,"next_page":null} EOF # DNSimple billing-charges shape (amounts = decimal DOLLAR strings, no # currency field, pagination envelope): two collected charges # (14.50 + 10.00 β†’ 2450 cents) plus a refunded one that must NOT count β€” # the state filter is part of the contract. References and domains are # fixture markers that must not survive aggregation. cat > "$FIXDIR/dnsimple-charges.json" <<'EOF' {"data":[ {"invoiced_at":"2026-07-02T09:00:00Z","total_amount":"14.50","balance_amount":"0.00","reference":"dnsfix-1001","state":"collected", "items":[{"description":"Renewal dnsfix-one.example","amount":"14.50","product_id":11,"product_type":"domain-renewal","product_reference":"dnsfix-one.example"}]}, {"invoiced_at":"2026-07-03T09:00:00Z","total_amount":"10.00","balance_amount":"0.00","reference":"dnsfix-1002","state":"collected", "items":[{"description":"Renewal dnsfix-two.example","amount":"10.00","product_id":12,"product_type":"domain-renewal","product_reference":"dnsfix-two.example"}]}, {"invoiced_at":"2026-07-04T09:00:00Z","total_amount":"14.50","balance_amount":"0.00","reference":"dnsfix-1003","state":"refunded", "items":[{"description":"Register dnsfix-three.example","amount":"14.50","product_id":13,"product_type":"domain-registration","product_reference":"dnsfix-three.example"}]} ],"pagination":{"current_page":1,"per_page":100,"total_entries":3,"total_pages":1}} EOF ;; esac # Two subscription pages (exercises pagination) + one balance page. Shapes # follow Stripe's list envelope; per-payer fields present in the fixture # (ids, customer) must NOT survive into books.json β€” that's the test. cat > "$FIXDIR/stripe-subs-1.json" <<'EOF' {"object":"list","has_more":true,"data":[ {"id":"sub_fixture_a","customer":"cus_fixture_a","status":"active","metadata":{"tier":"friends","space_id":"7"}, "items":{"data":[{"quantity":1,"price":{"unit_amount":400,"currency":"eur","recurring":{"interval":"month","interval_count":1}}}]}}, {"id":"sub_fixture_b","customer":"cus_fixture_b","status":"active","metadata":{"tier":"family","space_id":"9"}, "items":{"data":[{"quantity":1,"price":{"unit_amount":800,"currency":"eur","recurring":{"interval":"month","interval_count":1}}}]}} ]} EOF cat > "$FIXDIR/stripe-subs-2.json" <<'EOF' {"object":"list","has_more":false,"data":[ {"id":"sub_fixture_c","customer":"cus_fixture_c","status":"active","metadata":{"tier":"friends","space_id":"12"}, "items":{"data":[{"quantity":1,"price":{"unit_amount":4800,"currency":"eur","recurring":{"interval":"year","interval_count":1}}}]}} ]} EOF cat > "$FIXDIR/stripe-bal-1.json" <<'EOF' {"object":"list","has_more":false,"data":[ {"id":"txn_fixture_1","type":"charge","amount":400,"currency":"eur","created":1751328000}, {"id":"txn_fixture_2","type":"charge","amount":800,"currency":"eur","created":1751414400}, {"id":"txn_fixture_3","type":"refund","amount":-400,"currency":"eur","created":1751500800}, {"id":"txn_fixture_4","type":"payout","amount":-1200,"currency":"eur","created":1751587200} ]} EOF # Application fees with the charge expanded (Connect: spaces selling to # their members). Two EUR sales β€” a €9.00 PWYW sale with its 90c commission # (the real first-sale story) plus a €5.50 one β€” and a USD decoy that must # not count on either side of the currency filter. GMV 900+550=1450, # commission 90+55=145. The fee/charge/account ids are fixture markers that # must not survive aggregation. cat > "$FIXDIR/stripe-fees-1.json" <<'EOF' {"object":"list","has_more":false,"data":[ {"id":"fee_fixture_1","account":"acct_fixture_a","amount":90,"currency":"eur","refunded":false, "charge":{"id":"ch_fixture_1","object":"charge","amount":900,"currency":"eur","paid":true}}, {"id":"fee_fixture_2","account":"acct_fixture_a","amount":55,"currency":"eur","refunded":false, "charge":{"id":"ch_fixture_2","object":"charge","amount":550,"currency":"eur","paid":true}}, {"id":"fee_fixture_3","account":"acct_fixture_b","amount":120,"currency":"usd","refunded":false, "charge":{"id":"ch_fixture_3","object":"charge","amount":1200,"currency":"usd","paid":true}} ]} EOF build_fleet "$outdir/fleet.json" build_books "$outdir/books.json" # Self-checks: valid JSON, and no per-payer fixture value leaked through. sqj "SELECT CASE WHEN json_valid(readfile('$outdir/fleet.json')) AND json_valid(readfile('$outdir/books.json')) THEN 'ok' ELSE 'INVALID' END" \ | grep -qx ok || { echo "dry-run: output is not valid JSON" >&2; exit 1; } if grep -qE 'sub_fixture|cus_fixture|txn_fixture|fee_fixture|ch_fixture|acct_fixture|req_fixture|ce_fixture|wrkspc_fixture|dnsfix|hzfix|\.example' "$outdir/fleet.json" "$outdir/books.json"; then echo "dry-run: PRIVACY LEAK β€” a per-payer/provider-account id or hostname reached the output" >&2 exit 1 fi # Connect aggregates: GMV from the expanded charges, commission from the # fee amounts, both EUR-only (the USD decoy proves the filter). Skipped in # the degraded-books mode, where no income fields exist at all. if [ "${STATUS_DRY_BOOKS:-}" != unavailable ]; then [ "$(sqj "SELECT json_extract(readfile('$outdir/books.json'), '\$.connect.gmv_month_cents')")" = "1450" ] || { echo "dry-run: connect GMV must sum the expanded charges, EUR only (want 1450)" >&2; exit 1; } [ "$(sqj "SELECT json_extract(readfile('$outdir/books.json'), '\$.connect.commission_month_cents')")" = "145" ] || { echo "dry-run: connect commission must sum the fee amounts, EUR only (want 145)" >&2; exit 1; } fi # Capacity must publish as an abstract band word β€” a known word, and not a # single digit anywhere in the capacity subtree (no slots, GB, percentages). local lvl cap lvl=$(sqj "SELECT json_extract(readfile('$outdir/fleet.json'), '\$.capacity.level')") case "$lvl" in plenty|getting-full|at-capacity) ;; *) echo "dry-run: capacity.level '$lvl' is not one of the three band words" >&2 exit 1 ;; esac cap=$(sqj "SELECT json_extract(readfile('$outdir/fleet.json'), '\$.capacity')") if printf '%s' "$cap" | grep -q '[0-9]'; then echo "dry-run: PRIVACY β€” capacity subtree carries a number: $cap" >&2 exit 1 fi # Expenses: itemized by design, but the shape is still pinned β€” and the # cache seams must behave (cached_cost_cents). Default: all six providers # from fixtures/env (hetzner computed from its servers-API fixture), with # the API results freshly cached; hetzner-constant: same fixtures but the # operator constant wins; omit: no expenses key at all; stale-cache: all # four API fetches failed and the seeded same-month cache served # (aws 999 / hetzner 1399 / anthropic 888 / dnsimple 777). local aws_c hz_c an_c dn_c aws_c=$(sqj "SELECT json_extract(readfile('$outdir/books.json'), '\$.expenses.providers[0].month_to_date_cents')") case "${STATUS_DRY_EXPENSES:-}" in omit) [ "$(sqj "SELECT json_type(readfile('$outdir/books.json'), '\$.expenses') IS NULL")" = "1" ] || { echo "dry-run: expenses must be omitted entirely when no provider resolves" >&2; exit 1; } ;; stale-cache) hz_c=$(sqj "SELECT json_extract(readfile('$outdir/books.json'), '\$.expenses.providers[1].monthly_cents')") an_c=$(sqj "SELECT json_extract(readfile('$outdir/books.json'), '\$.expenses.providers[2].month_to_date_cents')") dn_c=$(sqj "SELECT json_extract(readfile('$outdir/books.json'), '\$.expenses.providers[3].month_to_date_cents')") [ "$aws_c" = "999" ] && [ "$hz_c" = "1399" ] && [ "$an_c" = "888" ] && [ "$dn_c" = "777" ] || { echo "dry-run: stale same-month cache must serve on fetch failure (got aws='$aws_c' hetzner='$hz_c' anthropic='$an_c' dnsimple='$dn_c')" >&2; exit 1; } ;; *) an_c=$(sqj "SELECT json_extract(readfile('$outdir/books.json'), '\$.expenses.providers[2].month_to_date_cents')") dn_c=$(sqj "SELECT json_extract(readfile('$outdir/books.json'), '\$.expenses.providers[4].month_to_date_cents')") [ "$aws_c" = "1234" ] || { echo "dry-run: CE fixture must yield 1234 cents (got '$aws_c')" >&2; exit 1; } [ "$an_c" = "250" ] || { echo "dry-run: anthropic fixture must yield 250 cents (got '$an_c')" >&2; exit 1; } [ "$dn_c" = "2450" ] || { echo "dry-run: dnsimple fixture must yield 2450 cents β€” collected only (got '$dn_c')" >&2; exit 1; } hz_c=$(sqj "SELECT json_extract(readfile('$outdir/books.json'), '\$.expenses.providers[1].monthly_cents')") if [ "${STATUS_DRY_EXPENSES:-}" = hetzner-constant ]; then [ "$hz_c" = "1400" ] || { echo "dry-run: hetzner constant must override the API fixture β€” 1400 cents (got '$hz_c')" >&2; exit 1; } # The constant must short-circuit the API entirely: nothing cached. [ "$(sqj "SELECT json_extract(readfile('$COSTS_CACHE'), '\$.hetzner') IS NULL")" = "1" ] || { echo "dry-run: hetzner constant must skip the API (a hetzner cache entry exists)" >&2; exit 1; } else [ "$hz_c" = "2094" ] || { echo "dry-run: hetzner servers fixture must yield 2094 cents β€” location-matched gross, rounded (got '$hz_c')" >&2; exit 1; } [ "$(sqj "SELECT json_extract(readfile('$COSTS_CACHE'), '\$.hetzner.month_to_date_cents')")" = "2094" ] || { echo "dry-run: hetzner API result must be cached" >&2; exit 1; } fi [ "$(sqj "SELECT json_extract(readfile('$outdir/books.json'), '\$.expenses.providers[3].to_date_cents')")" = "229000" ] || { echo "dry-run: claude.ai constant must yield 229000 cents, cumulative (to_date_cents)" >&2; exit 1; } [ "$(sqj "SELECT json_extract(readfile('$outdir/books.json'), '\$.expenses.providers[5].yearly_cents')")" = "9900" ] || { echo "dry-run: apple constant must yield 9900 cents, as billed (yearly)" >&2; exit 1; } [ -f "$COSTS_CACHE" ] && [ "$(sqj "SELECT json_extract(readfile('$COSTS_CACHE'), '\$.aws.month_to_date_cents')")" = "1234" ] && [ "$(sqj "SELECT json_extract(readfile('$COSTS_CACHE'), '\$.anthropic.month_to_date_cents')")" = "250" ] && [ "$(sqj "SELECT json_extract(readfile('$COSTS_CACHE'), '\$.dnsimple.month_to_date_cents')")" = "2450" ] || { echo "dry-run: metered results must be cached per provider" >&2; exit 1; } ;; esac # ---- carbon.json: the committed BUILT ledger (a fixed fixture here, so the # run fixture stays stable across real accruals) + a live-computed RUN block. # The run block folds from the Hetzner servers fixture (cpx31=4 + cx22=2 β†’ 6 # vCPUs), the box region grid (Germany β†’ 0/350/400 g/kWh), PUE 1.15, a pinned # 10-day span and the fixture registry's 6 instances. Only the expense modes # that stage the Hetzner servers fixture (default + hetzner-constant) give the # run its vCPU input; omit/stale-cache legitimately leave run absent (pending). cat > "$FIXDIR/carbon-built.json" <<'EOF' { "updated_at": "2026-07-11T00:00:00.000Z", "methodology": "https://github.com/elevenmessenger/messenger/blob/main/docs/comparison-carbon.md", "built": { "unit": "kg CO2e", "factors_mg_co2e_per_output_token": { "low": 0.1, "central": 0.55, "high": 3 }, "not_counted": "cache reads, cloud-side agent runs, and contributor machines not yet accrued", "entries": [ { "period_start": "2026-07-01T00:00:00.000Z", "period_end": "2026-07-11T00:00:00.000Z", "source": "ada-sketching", "basis": "measured", "sessions": 10, "output_tokens": 1000000, "fresh_input_tokens": 5000000, "cache_read_tokens": 90000000, "kg_low": 0.1, "kg_central": 0.6, "kg_high": 3, "note": "dry-run fixture" } ], "totals": { "output_tokens": 1000000, "kg_low": 0.1, "kg_central": 0.6, "kg_high": 3 } } } EOF local cvc cwt cgr cda ccu cps CARBON_SINCE=2026-07-02 STATUS_CARBON_DAYS=10 build_carbon "$outdir/carbon.json" "$FIXDIR/carbon-built.json" [ "$(sqj "SELECT json_valid(readfile('$outdir/carbon.json'))")" = "1" ] || { echo "dry-run: carbon.json is not valid JSON" >&2; exit 1; } [ "$(sqj "SELECT json_type(readfile('$outdir/carbon.json'), '\$.built')='object'")" = "1" ] || { echo "dry-run: carbon built ledger missing (built must be republished verbatim)" >&2; exit 1; } case "${STATUS_DRY_EXPENSES:-}" in omit|stale-cache) [ "$(sqj "SELECT json_type(readfile('$outdir/carbon.json'), '\$.run') IS NULL")" = "1" ] || { echo "dry-run: carbon run must be absent when no server inputs resolve (pending, never a fake zero)" >&2; exit 1; } ;; *) cvc=$(sqj "SELECT json_extract(readfile('$outdir/carbon.json'), '\$.run.vcpus')") cwt=$(sqj "SELECT json_extract(readfile('$outdir/carbon.json'), '\$.run.watts.central')") cgr=$(sqj "SELECT json_extract(readfile('$outdir/carbon.json'), '\$.run.grid_g_per_kwh.central')") cda=$(sqj "SELECT json_extract(readfile('$outdir/carbon.json'), '\$.run.daily_kg.central')") ccu=$(sqj "SELECT json_extract(readfile('$outdir/carbon.json'), '\$.run.cumulative_kg.central')") cps=$(sqj "SELECT json_extract(readfile('$outdir/carbon.json'), '\$.run.per_space_daily_kg.central')") # daily central = 6 Γ— 3.5 W Γ— 24h Γ— 350 g/kWh Γ— 1.15 / 1e6 + 0.001 + 0.005 # = 0.20886 kg; cumulative = Γ—10 days = 2.0886; per space = Γ·6. [ "$cvc" = "6" ] && [ "$cwt" = "21" ] && [ "$cgr" = "350" ] && [ "$cda" = "0.20886" ] && [ "$ccu" = "2.0886" ] && [ "$cps" = "0.03481" ] || { echo "dry-run: carbon run math (got vcpus=$cvc watts=$cwt grid=$cgr daily=$cda cumulative=$ccu per_space=$cps)" >&2; exit 1; } ;; esac # Privacy: no fixture marker, hostname, socket path, or IP may reach carbon.json. if grep -qE 'hzfix|ada-sketching@|\.example|\.sock|[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+' "$outdir/carbon.json"; then echo "dry-run: PRIVACY LEAK β€” a hostname/IP reached carbon.json" >&2 exit 1 fi echo "--- fleet.json ---"; cat "$outdir/fleet.json" echo "--- books.json ---"; cat "$outdir/books.json" echo "--- carbon.json ---"; cat "$outdir/carbon.json" echo "dry-run ok: valid JSON, no hostnames, no per-payer fields, capacity abstract ($lvl), expenses ${STATUS_DRY_EXPENSES:-itemized}, carbon run ${cda:-absent}" } case "${1:-}" in install) install_units ;; --dry-run) [ -n "${2:-}" ] || { echo "usage: $0 --dry-run " >&2; exit 1; } dry_run "$2" ;; "") publish ;; *) echo "usage: $0 [install | --dry-run ]" >&2; exit 1 ;; esac