#!/usr/bin/env sh # ๐Ÿค– ops canary โ€” per-session test instances (CLAUDE.md ยง Per-session canaries), # with the rules that used to live only in prose encoded as behaviour: # # ./deploy/ops canary create # test-.11ch.at, flagged --canary # ./deploy/ops canary deploy # build THIS branch -> stage -> deploy --pin # ./deploy/ops canary reset # wipe back to first-run (iterate onboarding) # ./deploy/ops canary destroy # reset (archive DB) + remove --stop # ./deploy/ops canary list # every canary + pinned row on the box # # The rules, and where each is enforced: # - Canaries are test- on the wildcard domain โ€” no DNS step, no cert # wait, visually distinct from real member spaces (this script only ever # forms test-. hostnames). # - Creation passes --canary: the registry flag is a DELIBERATE signal, never # inferred from the hostname โ€” it is what lets `lchat reset` wipe the DB at # destroy time while refusing to touch any real instance. # - Deploys always --pin. An unpinned canary is moved onto the channel build # by the hourly auto-updater โ€” out from under the session, silently. # - Destroy wipes the data. `lchat remove` deliberately leaves /data behind # (right for real instances, wrong for a canary), and the hardened sudoers # allowlist has no `rm` โ€” so the encoded path is `lchat reset` (archives # the DB to /data/trash, canary-gated) then `remove --stop` (which also # disables any bot/hooks sidecar units). # - Admin tokens for canaries are stored NOWHERE (not 1Password) on purpose: # everything about a canary dies with its session. # - Fleet-wide operations skip AUTO=pinned rows โ€” each is someone's live # canary. `list` shows them so a sweep can check first. set -eu here="$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)" root="$(CDPATH= cd -- "$here/.." && pwd)" # The box canaries live on โ€” an ssh alias; user/hostname come from # ~/.ssh/config (box specifics in CLAUDE.local.md). Canaries live on the DEV # MESH by default (Paul, 2026-07-22): throwaway test instances don't belong on # the box real members live on, and they'd eat its instance cap. Helsinki is # the dev default because build-linux.sh emits amd64 (Dublin is arm64). Alias # to add once per machine: # Host elevendev-fi # HostName 157.180.123.183 # User root # IdentityFile ~/.ssh/eleven-dev-box # Prod (LCHAT_CANARY_BOX=elevenbox LCHAT_CANARY_DOMAIN=11ch.at) is the # deliberate exception, not the default. # NB dev has no wildcard cert BY DESIGN (the DNSimple token never goes on a # dev box), so each dev canary host burns one per-host HTTP-01 cert from the # shared 50/week 11ch.at LE budget โ€” fine at canary cadence, not for bursts. BOX="${LCHAT_CANARY_BOX:-elevendev-fi}" DOMAIN="${LCHAT_CANARY_DOMAIN:-fi.dev.11ch.at}" # The home server a canary's instance points at (LCHAT_HOME_ORIGIN โ€” the # sign-in/enrol origin baked into its pages). Dev canaries must ride the DEV # home: without this, add-instance leaves the env empty and the binary falls # back to the compiled PRODUCTION home โ€” a dev canary whose sign-in round # trips hit prod (old code, real accounts). Keyed off the domain so the # deliberate prod exception (LCHAT_CANARY_DOMAIN=11ch.at) keeps the compiled # default; LCHAT_CANARY_HOME_ORIGIN overrides either way. case "$DOMAIN" in *.dev.*) HOME_ORIGIN="${LCHAT_CANARY_HOME_ORIGIN:-https://home.dev.elevenmessenger.com}" ;; *) HOME_ORIGIN="${LCHAT_CANARY_HOME_ORIGIN:-}" ;; esac LCHAT=/opt/lchat/lchat usage() { echo "usage: ops canary create|deploy|reset|destroy | ops canary list" >&2 echo " slug: lowercase letters/digits/hyphens โ€” your branch or feature name" >&2 exit 2 } # Canary links (the admin link's token, the setup/onboarding link's one-time # secret) must never land in a session transcript โ€” transcripts are public. So # every verb that mints one writes the full output to ~/.link ON THE BOX # and this filter redacts the URL fragments/tokens from what echoes locally. # Fetch the real links from your own terminal: ssh cat .link redact_secrets() { sed -E 's/#[^[:space:]]+/#/g'; } cmd="${1:-}"; slug="${2:-}" host_for() { # Everything below derives the hostname from here, so this script is # structurally incapable of touching a non-test- host. case "$1" in *[!a-z0-9-]*|-*|"") echo "ops canary: bad slug '$1' (lowercase letters/digits/hyphens)" >&2; exit 2 ;; esac echo "test-$1.$DOMAIN" } case "$cmd" in create) host="$(host_for "$slug")" echo "ops canary: creating $host on $BOX (flagged --canary)" # add-instance prints the admin link + the one-time setup link โ€” both carry # secrets, so the full output is staged box-side and echoed here redacted. homeflag="" [ -n "$HOME_ORIGIN" ] && homeflag="--home-origin '$HOME_ORIGIN'" out="$(ssh "$BOX" "sudo -n $LCHAT add-instance --canary $homeflag '$host' > \$HOME/$host.link 2>&1; st=\$?; cat \$HOME/$host.link; exit \$st")" || { printf '%s\n' "$out" | redact_secrets >&2 exit 1 } printf '%s\n' "$out" | redact_secrets cat <&2 fi echo "ops canary: building $ver for linux/amd64" out="$(mktemp -t lchat-canary.XXXXXX)" trap 'rm -f "$out"' EXIT cd "$root" # build-linux.sh compiles the package at the current directory OUT="$out" "$here/build-linux.sh" "$ver" # A per-slug upload name so parallel sessions' uploads never trample each # other in the shared incoming dir (a fixed name is a race). up="/opt/lchat/incoming/lchat-upload-$slug" ssh "$BOX" 'mkdir -p /opt/lchat/incoming' scp -q "$out" "$BOX:$up" staged="$(ssh "$BOX" "sudo -n $LCHAT stage -file '$up' && rm -f '$up'")" echo " $staged" staged_ver="$(echo "$staged" | awk '{print $2}')" [ "$staged_ver" = "$ver" ] || { echo "ops canary: staged '$staged_ver' != built '$ver' โ€” aborting" >&2; exit 1; } # --pin is not optional: without it the hourly auto-updater moves the canary # back onto its channel build, out from under the session. deploy itself # health-checks /api/version and auto-rolls back a bad build. ssh "$BOX" "sudo -n $LCHAT deploy '$host' --version '$ver' --pin" cat < \$HOME/$host.link 2>&1; st=\$?; cat \$HOME/$host.link; exit \$st")" || { printf '%s\n' "$out" | redact_secrets >&2 echo "ops canary: reset refused โ€” if $host is genuinely a throwaway, flag it:" >&2 echo " ssh $BOX sudo -n $LCHAT canary $host --on # then re-run reset" >&2 exit 1 } printf '%s\n' "$out" | redact_secrets cat <&2 echo "ops canary: reset refused โ€” if $host is genuinely a throwaway, flag it:" >&2 echo " ssh $BOX sudo -n $LCHAT canary $host --on # then re-run destroy" >&2 exit 1 } printf '%s\n' "$out" | redact_secrets # remove --stop: drops the route (and in.), stops + disables the # instance unit and any bot/hooks sidecar units. The staged link file goes # with it โ€” its secrets are dead, and a stale file would mislead the next # session to reuse this slug. ssh "$BOX" "sudo -n $LCHAT remove '$host' --stop; rm -f \$HOME/$host.link" cat <