#!/usr/bin/env sh
# 🤖 ops dev — the two-node dev mesh (Dublin + Helsinki; docs/multi-edge.md,
# runbook docs/multi-edge-dev-cutover.md).
#
#   ./deploy/ops dev plan                  # tofu plan, with the AMI-replacement guard
#   ./deploy/ops dev deploy [ie|fi]        # build -> scp -> swap -> restart (both by default)
#   ./deploy/ops dev deploy --stage-only   # stop after scp (pair the swap with env edits)
#   ./deploy/ops dev deploy --home         # ALSO swap the home service binary (lchat-home)
#   ./deploy/ops dev provision [ie|fi]     # refresh systemd units + route the short-link apex
#   ./deploy/ops dev rebuild --yes         # destroy + recreate the Dublin box (new keys)
#   ./deploy/ops dev rebuild --yes --adopt # provision a Dublin box something ELSE already
#                                          # replaced (no tofu touch) — same bootstrap
#
# What this encodes that prose used to carry:
#   - The dev mesh is NOT a release channel. Nothing here touches the signed
#     feed; deploying is scp + stage + swap, by hand, on purpose (the mesh runs
#     unmerged code the fleet updater must never see).
#   - The verb sorts its own credentials out (the release-server.sh shape):
#     values already in the environment win; otherwise Bitwarden fills the
#     gaps — AWS_* from the per-operator dev agent item (docs/secrets.md
#     § AWS), and DNSIMPLE_* from that item too when it carries them, else
#     from the standalone DNSimple token item the migration split out.
#     Unlocks interactively when the vault is locked; no tty -> instructions,
#     never a hang; values are exported for this process only, never printed.
#   - Dublin is arm64 (t4g), Helsinki amd64 — build both, ship the right one.
#   - The binary is staged BESIDE the live one and swapped in a separate step,
#     so router+cloud restart exactly once, and the previous binary is parked
#     as a .bak for one-command rollback.
#   - Dublin runs router + cloud + control (restart all three); Helsinki runs
#     only the router.
#   - `deploy` swaps the BINARY; `provision` refreshes the UNITS. They are
#     separate because deploy restarts only units that already exist, so a
#     newly-added unit never appears on a dev box however often you deploy —
#     only `lchat bootstrap` writes unit files. provision runs it with
#     --no-update-timer and then asserts the timer really is off, because a
#     plain bootstrap would enable the hourly updater and quietly replace the
#     unmerged code under test with the current stable build.
#   - compute.tf's aws_ami is `most_recent = true` with no lifecycle guard: the
#     day Canonical publishes a newer image, a plain `tofu apply` DESTROYS AND
#     RECREATES the Dublin box (its /data dies with it; only the EIP survives).
#     `plan` refuses to look green when that is pending; `rebuild` is the
#     deliberate version of the same event.
#   - The dev security group admits SSH only from the per-operator /32
#     allowlist (compute.tf ssh_ingress_cidr), so being outside it looks
#     exactly like a dead box (`ops doctor dev` names which it is). Adding
#     yourself is a PR + `ops dev apply` (an in-place SG update); removing an
#     entry revokes that operator — only with their say-so.
#   - EC2 injects the SSH keypair ONLY at first boot — which is why `rebuild`
#     exists: swapping the key pair in compute.tf does nothing to a running
#     box, and replacing the box is the honest way to rotate a key you don't
#     hold. Helsinki is hand-provisioned Hetzner and cannot be rebuilt from
#     here (only its scoped S3 IAM user lives in the tofu).
set -eu

here="$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)"
root="$(CDPATH= cd -- "$here/.." && pwd)"
# The Bitwarden item names + value locations this script reads — a contract,
# not a search (deploy/vault-items.sh).
# shellcheck source=deploy/vault-items.sh
. "$here/vault-items.sh"
# shellcheck source=deploy/bw-lib.sh
. "$here/bw-lib.sh"

# The boxes, as committed in docs/multi-edge-dev-cutover.md. The Dublin key is
# the SHARED box key (comment `eleven-dev-box` — box-named, not personal),
# never vaulted: copies live only on operator machines. ssh falls back to
# ~/.ssh/config if it's absent.
BOX_IE="${LCHAT_DEV_BOX_IE:-ubuntu@54.154.95.205}"
BOX_FI="${LCHAT_DEV_BOX_FI:-root@157.180.123.183}"
KEY_IE="$HOME/.ssh/eleven-dev-box"
AWS_PROFILE="${LCHAT_DEV_AWS_PROFILE:-eleven-dev}"
# The dev mesh's short-link apex — the dev-side stand-in for 11ch.at. It is the
# APEX of the dev subtree, not a host under it, because that is what production
# is: the wildcard cert that covers every space does not cover this name, so
# rehearsing short links here exercises the same apex DNS + certificate path
# that production needs. Its A record is aws_route53_record.dev_apex.
DEV_LINKS_HOST="${LCHAT_DEV_LINKS_HOST:-dev.11ch.at}"

ssh_ie() { if [ -f "$KEY_IE" ]; then ssh -i "$KEY_IE" -o ConnectTimeout=10 "$@"; else ssh -o ConnectTimeout=10 "$@"; fi; }
scp_ie() { if [ -f "$KEY_IE" ]; then scp -i "$KEY_IE" "$@"; else scp "$@"; fi; }

usage() {
  echo "usage: ops dev plan [tofu args] | ops dev apply [tofu args] |" >&2
  echo "       ops dev deploy [ie|fi] [--stage-only] [--home] | ops dev provision [ie|fi] |" >&2
  echo "       ops dev rebuild [--yes]" >&2
  exit 2
}

# plan_guard <planfile|-> [tofu args...]: run a plan, show it, and hard-stop
# when it would DESTROY anything. Shared by `plan` (whose whole job this is)
# and `apply` (so an interactive "yes" can't sleepwalk into the AMI rebuild).
# Extra args pass through, so a targeted apply guards exactly the graph it
# will touch. With a planfile (not "-"), the plan is also SAVED so apply can
# execute exactly what was inspected — nothing can change between the guarded
# plan and the applied one.
#
# The stop condition is the plan summary's own destroy COUNT, not resource-
# header wording: the original grep for "must be replaced" demonstrably missed
# a live AMI replacement on 2026-07-23 (the operator got a bare approve prompt
# and the Dublin box was destroyed), and a count can't be missed by phrasing
# or formatting drift. The wording grep stays as a second net.
plan_guard() {
  pf="$1"; shift
  planout="$(mktemp -t lchat-dev-plan.XXXXXX)"
  outflag=""
  [ "$pf" = - ] || outflag="-out=$pf"
  # -detailed-exitcode: 0 no changes, 2 changes present, 1 error. Changes are
  # fine; what we refuse to shrug at is anything pending DESTRUCTION.
  # -no-color so the detection greps see the same bytes a human does.
  # (Captured to a file, not piped through tee — a pipeline would report tee's
  # exit status and silently lose tofu's.)
  rc=0
  # shellcheck disable=SC2086 # outflag is deliberately word-split (empty or one flag)
  run_tofu plan -input=false -no-color -detailed-exitcode $outflag "$@" >"$planout" 2>&1 || rc=$?
  cat "$planout"
  if [ "$rc" -eq 1 ]; then rm -f "$planout"; exit 1; fi
  destroys="$(sed -n 's/^Plan: .* \([0-9][0-9]*\) to destroy.*/\1/p' "$planout" | tail -1)"
  if [ "${destroys:-0}" -gt 0 ] || grep -Eq "must be replaced|forces replacement" "$planout"; then
    rm -f "$planout"
    cat >&2 <<'EOF'

  ██ STOP — this plan DESTROYS or REPLACES something (see the plan above).
  If it is aws_instance.ie_node, the cause is usually a newer Canonical AMI —
  a plain apply would DESTROY the Dublin box; its /data dies with it, only
  the EIP (and DNS) survive. Your options:
    - apply just the safe resource:  ops dev apply -target=<addr>
    - make the rebuild deliberate:   ops dev rebuild   (new keys, guided)
    - a genuinely wanted destroy: run tofu by hand in infra/dev-mesh,
      eyes open — this verb will not do it for you.
EOF
    exit 1
  fi
  rm -f "$planout"
}

# --- credentials ----------------------------------------------------------
# Only these five ever leave the Bitwarden item for the environment — an
# allowlist, so a stray line in the item can never export something else.
DEV_CRED_VARS='AWS_ACCESS_KEY_ID|AWS_SECRET_ACCESS_KEY|AWS_SESSION_TOKEN|DNSIMPLE_TOKEN|DNSIMPLE_ACCOUNT'

have_aws_auth() {
  [ -n "${AWS_ACCESS_KEY_ID:-}" ] && return 0
  aws configure list-profiles 2>/dev/null | grep -qx "$AWS_PROFILE"
}

# Run tofu with whichever auth we ended up with. Explicit env credentials win;
# AWS_PROFILE is only set when that profile really exists (setting it to an
# absent profile would itself be the error).
run_tofu() {
  if [ -n "${AWS_ACCESS_KEY_ID:-}" ]; then tofu "$@"; else AWS_PROFILE="$AWS_PROFILE" tofu "$@"; fi
}

# find_item <search> <pinned-name> <hint> — resolve exactly one vault item,
# echoing its name. bw reads a LOCAL CACHE of the vault: an item added,
# removed, or deduplicated in another client is invisible here until
# `bw sync` — so a surprising result (none, or more than one) gets one sync
# and retry before it is believed. Two items SHARING a name is its own error:
# `bw get item <name>` can't disambiguate, so the vault needs a rename/delete.
find_item() {
  search="$1"; pinned="$2"; hint="$3"
  if [ -n "$pinned" ]; then printf '%s\n' "$pinned"; return 0; fi
  synced=no
  while :; do
    names="$(bw_run 30 list items --search "$search" 2>/dev/null | jq -r '.[].name' || true)"
    n="$(printf '%s\n' "$names" | grep -c . || true)"
    if [ "$n" = 1 ]; then printf '%s\n' "$names"; return 0; fi
    if [ "$synced" = no ]; then
      echo "ops dev: $n items match '$search' — syncing bw's local cache and retrying" >&2
      bw_run 60 sync >/dev/null 2>&1 || true
      synced=yes
      continue
    fi
    if [ "$n" = 0 ]; then
      echo "ops dev: no item matching '$search' in your vault ($hint)" >&2
    elif [ "$(printf '%s\n' "$names" | sort -u | grep -c .)" -lt "$n" ]; then
      echo "ops dev: DUPLICATE items named alike match '$search' — bw can't fetch by an" >&2
      echo "  ambiguous name; delete or rename one in the vault, then re-run:" >&2
      printf '%s\n' "$names" | sed 's/^/    /' >&2
    else
      echo "ops dev: several items match '$search' — pin one ($hint):" >&2
      printf '%s\n' "$names" | sed 's/^/    /' >&2
    fi
    return 1
  done
}

load_dev_creds() {
  command -v bw >/dev/null 2>&1 || {
    echo "ops dev: bw is not installed, and the credentials aren't in the environment" >&2
    return 1; }
  command -v jq >/dev/null 2>&1 || { echo "ops dev: jq is required" >&2; return 1; }
  st="$(bw_run 20 status 2>/dev/null | jq -r '.status' 2>/dev/null || true)"
  case "$st" in
    unlocked) ;;
    locked)
      if [ -t 0 ]; then
        echo "ops dev: unlocking bitwarden to read the dev agent creds" >&2
        # Our prompt, never bw's; watchdogged (bw-lib.sh) — bw's own prompt
        # inside $(...) is invisible and reads as a hang.
        BW_SESSION="$(bw_unlock_session)" || true
        [ -n "${BW_SESSION:-}" ] || { echo "ops dev: unlock failed" >&2; return 1; }
        export BW_SESSION
      else
        echo "ops dev: bitwarden is locked and there is no tty to prompt on" >&2
        echo "  bw unlock   # then paste the 'export BW_SESSION=...' line it prints, and re-run" >&2
        return 1
      fi ;;
    *)
      echo "ops dev: bw is not logged in (bw login — EU vault: https://vault.bitwarden.eu)" >&2
      return 1 ;;
  esac

  # The dev agent item is the one contract entry that is a naming CONVENTION
  # rather than an exact name — it is per-operator (each has a revocable
  # aws-dev-agent-<name> IAM identity, docs/secrets.md § AWS), so resolve
  # "exactly one item with the contract prefix"; LCHAT_DEV_CREDS_ITEM pins one.
  item="$(find_item "$BW_ITEM_DEV_AGENT_PREFIX" "${LCHAT_DEV_CREDS_ITEM:-}" \
    "each operator gets their own dev agent creds, docs/secrets.md § AWS; LCHAT_DEV_CREDS_ITEM pins one")" \
    || return 1

  # The item is the sealed env file: KEY=VALUE lines in its secure note (or
  # custom fields). Pull only the allowlisted names; never echo a value.
  lines="$(bw_run 30 get item "$item" 2>/dev/null | jq -r --arg re "^($DEV_CRED_VARS)=" '
      ((.notes // "" | split("\n")) + [.fields[]? | "\(.name)=\(.value)"])[]
      | select(test($re))' || true)"
  if [ -n "$lines" ]; then
    while IFS= read -r line; do
      k="${line%%=*}"
      # Environment always wins — the item only fills gaps. ($k is allowlisted
      # by the select() above, so the eval can't be steered by item content.)
      eval "cur=\${$k:-}"
      # shellcheck disable=SC2163 # the K=V-operand form of export is intended
      [ -n "$cur" ] || export "$line"
    done <<EOF
$lines
EOF
    echo "ops dev: credentials loaded from bitwarden item '$item'" >&2
  else
    echo "ops dev: note: '$item' carried no AWS_*/DNSIMPLE_* lines in its note or fields" >&2
  fi

  # The Bitwarden migration split the DNSimple token out of the sops-era
  # bundle into its own item, so when the agent item didn't carry it, fetch it
  # from there — by the contract's EXACT name and slot (the login password),
  # no searching, no guessing. A failed fetch gets one cache sync + retry (a
  # freshly shared item is invisible to bw's local cache until then), then a
  # crisp contract error. The account id is public (committed in main.tf) —
  # only the token is a secret.
  if [ -z "${DNSIMPLE_TOKEN:-}" ]; then
    ditem="${LCHAT_DNSIMPLE_ITEM:-$BW_ITEM_DNSIMPLE}"
    json="$(bw_run 30 get item "$ditem" 2>/dev/null || true)"
    if [ -z "$json" ]; then
      echo "ops dev: item '$ditem' not in bw's local cache — syncing and retrying" >&2
      bw_run 60 sync >/dev/null 2>&1 || true
      json="$(bw_run 30 get item "$ditem" 2>/dev/null || true)"
    fi
    if [ -z "$json" ]; then
      echo "ops dev: no bitwarden item named exactly '$ditem'" >&2
      echo "  — the vault contract (deploy/vault-items.sh); is it shared to your collection," >&2
      echo "  or was it renamed without a matching PR?" >&2
      return 1
    fi
    tok="$(printf '%s' "$json" | jq -r '.login.password // empty' 2>/dev/null || true)"
    [ -n "$tok" ] || {
      echo "ops dev: the vault contract expects the DNSimple token in the PASSWORD slot" >&2
      echo "  of '$ditem' (deploy/vault-items.sh) — fix the item, or export DNSIMPLE_TOKEN" >&2
      return 1; }
    export DNSIMPLE_TOKEN="$tok"
    echo "ops dev: DNSIMPLE_TOKEN loaded from bitwarden item '$ditem'" >&2
  fi
}

# ensure_creds: make DNSIMPLE_TOKEN + AWS auth real, from env or Bitwarden.
ensure_creds() {
  # The account id is public (176940, committed in main.tf) — default it here
  # rather than in the bw path, so a hand-exported DNSIMPLE_TOKEN (which skips
  # Bitwarden entirely) still gets a working provider config.
  [ -n "${DNSIMPLE_ACCOUNT:-}" ] || export DNSIMPLE_ACCOUNT=176940
  if [ -z "${DNSIMPLE_TOKEN:-}" ] || ! have_aws_auth; then
    load_dev_creds || {
      echo "ops dev: still missing credentials — export DNSIMPLE_TOKEN/DNSIMPLE_ACCOUNT and" >&2
      echo "  AWS_ACCESS_KEY_ID/AWS_SECRET_ACCESS_KEY (or configure the '$AWS_PROFILE' aws" >&2
      echo "  profile), or make your dev agent item reachable in bw and re-run" >&2
      exit 1
    }
  fi
  [ -n "${DNSIMPLE_TOKEN:-}" ] || {
    echo "ops dev: DNSIMPLE_TOKEN still unset (the item didn't carry it?)" >&2; exit 1; }
  have_aws_auth || {
    echo "ops dev: no AWS credentials (env or '$AWS_PROFILE' profile)" >&2; exit 1; }
}

# swap <ie|fi> <ver>: back up the live binary, move the staged one in, restart
# the units that box runs, and confirm the running stamp matches the build.
swap_box() {
  which_box="$1"; ver="$2"
  if [ "$which_box" = ie ]; then box="$BOX_IE"; units="lchat-router lchat-cloud lchat-control"; run() { ssh_ie "$box" "$@"; }
  else box="$BOX_FI"; units="lchat-router"; run() { ssh -o ConnectTimeout=10 "$box" "$@"; }
  fi
  run "sudo cp /opt/lchat/lchat /opt/lchat/lchat.bak-pre-$ver \
       && sudo mv /opt/lchat/incoming/lchat-new /opt/lchat/lchat \
       && sudo chmod +x /opt/lchat/lchat \
       && sudo systemctl restart $units"
  running="$(run '/opt/lchat/lchat version' 2>/dev/null || true)"
  if [ "$running" = "$ver" ] && run 'systemctl is-active --quiet lchat-router'; then
    echo "  $which_box: running $running"
  else
    echo "ops dev: $which_box is NOT healthy after the swap (version '$running', wanted '$ver')" >&2
    echo "  inspect:  ssh $box journalctl -u lchat-router -n 60 --no-pager" >&2
    echo "  rollback: ssh $box 'sudo mv /opt/lchat/lchat.bak-pre-$ver /opt/lchat/lchat && sudo systemctl restart $units'" >&2
    exit 1
  fi
}

# swap_home <ie|fi> <ver>: the home service rides its own binary (releases
# ship it as lchat-home-linux-* via updaterservices.go; the dev mesh swaps it
# directly, same as the control binary). A box that doesn't RUN home (Helsinki
# today — home runs on the dev apex box only) is skipped, never failed.
#
# Enabled, not merely present: `lchat bootstrap` writes the unit file on every
# box it touches, so "the unit exists" says nothing about whether this box is
# a home server. Helsinki has the file, disabled, with no /data/home.env — and
# the old presence test walked straight into restarting it, which fails on the
# missing env ("Failed to load environment files") and took the whole deploy
# down with it after Dublin had already swapped fine.
swap_home() {
  which_box="$1"; ver="$2"
  if [ "$which_box" = ie ]; then box="$BOX_IE"; run() { ssh_ie "$box" "$@"; }
  else box="$BOX_FI"; run() { ssh -o ConnectTimeout=10 "$box" "$@"; }
  fi
  case "$(run 'systemctl is-enabled lchat-home.service 2>/dev/null || true')" in
    enabled|enabled-runtime|static) ;;
    *)
      echo "  $which_box: lchat-home not enabled here — home skipped"
      run 'rm -f /opt/lchat/incoming/home-new' || true
      return 0 ;;
  esac
  run "sudo cp /opt/lchat/home /opt/lchat/home.prev-$ver 2>/dev/null || true; \
       sudo mv /opt/lchat/incoming/home-new /opt/lchat/home \
       && sudo chmod +x /opt/lchat/home \
       && sudo systemctl restart lchat-home"
  if run 'systemctl is-active --quiet lchat-home'; then
    echo "  $which_box: lchat-home running ($ver)"
  else
    echo "ops dev: $which_box lchat-home is NOT active after the swap" >&2
    echo "  inspect:  ssh $box journalctl -u lchat-home -n 60 --no-pager" >&2
    echo "  rollback: ssh $box 'sudo mv /opt/lchat/home.prev-$ver /opt/lchat/home && sudo systemctl restart lchat-home'" >&2
    exit 1
  fi
}

cmd="${1:-}"
[ $# -gt 0 ] && shift
case "$cmd" in

plan)
  cd "$root/infra/dev-mesh"
  ensure_creds
  [ -d .terraform ] || run_tofu init -input=false
  plan_guard - "$@"
  ;;

apply)
  # The self-serve apply: same credential fetch, same destroy guard. What
  # applies is the SAVED plan the guard inspected — never a fresh one — so a
  # refresh between guard and apply can't smuggle a replacement in. The verb
  # keeps its own explicit confirm (a saved-plan apply skips tofu's). Extra
  # args pass through (e.g. `ops dev apply
  # -target=aws_security_group.ie_node` to land a security-group change
  # while something destructive is pending).
  cd "$root/infra/dev-mesh"
  ensure_creds
  [ -d .terraform ] || run_tofu init -input=false
  planfile="$(mktemp -t lchat-dev-planfile.XXXXXX)"
  trap 'rm -f "$planfile"' EXIT
  plan_guard "$planfile" "$@"
  printf 'apply exactly this plan? (type yes): '
  read -r answer || answer=""   # EOF (no tty) refuses below instead of dying silently
  [ "$answer" = yes ] || { echo "ops dev: not applied"; exit 1; }
  run_tofu apply -input=false "$planfile"
  ;;

deploy)
  targets="ie fi"; stage_only=no; with_home=no
  for a in "$@"; do
    case "$a" in
      ie|fi) targets="$a" ;;
      --stage-only) stage_only=yes ;;
      --home) with_home=yes ;;
      *) usage ;;
    esac
  done

  # The invariant `provision` asserts, checked again HERE — before a build, a
  # copy and a restart go to waste. A dev box with the auto-updater enabled
  # takes your unmerged code back to the stable channel within the hour, and
  # the deploy that put it there reports success on the way past: seen
  # 2026-08-28, when an update pass landed 55 seconds after a swap and quietly
  # restored the previous home binary. Cheap SSH, loud failure, named fix.
  for t in $targets; do
    if [ "$t" = ie ]; then probe_box="$BOX_IE"; timer="$(ssh_ie "$probe_box" 'systemctl is-enabled lchat-update.timer 2>/dev/null || true')"
    else probe_box="$BOX_FI"; timer="$(ssh -o ConnectTimeout=10 "$probe_box" 'systemctl is-enabled lchat-update.timer 2>/dev/null || true')"
    fi
    if [ "$timer" = enabled ]; then
      echo "ops dev: ABORT — $t's auto-update timer is ENABLED." >&2
      echo "  It would replace whatever you deploy with the current stable build," >&2
      echo "  within the hour, silently. Turn it off first:" >&2
      echo "    ./deploy/ops dev provision $t" >&2
      exit 1
    fi
  done

  ver="$(git -C "$root" rev-parse --short HEAD)"
  if ! git -C "$root" diff --quiet || ! git -C "$root" diff --cached --quiet; then
    ver="${ver}-dirty"
    echo "ops dev: working tree has uncommitted changes — building anyway ($ver)" >&2
  fi

  tmp="$(mktemp -d -t lchat-dev.XXXXXX)"
  trap 'rm -rf "$tmp"' EXIT
  cd "$root"   # build-linux.sh compiles the package at the current directory
  for t in $targets; do
    case "$t" in
      ie) arch=arm64 ;;   # Dublin is a Graviton box
      fi) arch=amd64 ;;
    esac
    echo "ops dev: building $ver for linux/$arch ($t)"
    OUT="$tmp/lchat-$t" ARCH="$arch" "$here/build-linux.sh" "$ver"
    if [ "$with_home" = yes ]; then
      # The home service is its own package/binary (like release-server.sh's
      # service builds) — no version stamp, the unit check gates the swap.
      echo "ops dev: building home for linux/$arch ($t)"
      GOOS=linux GOARCH="$arch" CGO_ENABLED=0 go build -o "$tmp/home-$t" ./home
    fi
  done

  for t in $targets; do
    if [ "$t" = ie ]; then box="$BOX_IE"; else box="$BOX_FI"; fi
    echo "ops dev: staging on $t ($box)"
    # Stage beside the live binary, never over it — the swap is its own step.
    # An SSH hang here usually means you are outside the security group's
    # /32 allowlist, not that the box is down: run `ops doctor dev`.
    if [ "$t" = ie ]; then
      # The hardening pass roots /opt/lchat — hand the staging dir back to
      # the ssh user so plain scp can write it (the swap is sudo either way).
      ssh_ie "$box" 'sudo mkdir -p /opt/lchat/incoming && sudo chown "$(id -un)" /opt/lchat/incoming'
      scp_ie -q "$tmp/lchat-$t" "$box:/opt/lchat/incoming/lchat-new"
      [ "$with_home" = yes ] && scp_ie -q "$tmp/home-$t" "$box:/opt/lchat/incoming/home-new"
    else
      ssh -o ConnectTimeout=10 "$box" 'sudo mkdir -p /opt/lchat/incoming && sudo chown "$(id -un)" /opt/lchat/incoming'
      scp -q "$tmp/lchat-$t" "$box:/opt/lchat/incoming/lchat-new"
      [ "$with_home" = yes ] && scp -q "$tmp/home-$t" "$box:/opt/lchat/incoming/home-new"
    fi
  done

  if [ "$stage_only" = yes ]; then
    echo "ops dev: staged at /opt/lchat/incoming/lchat-new on: $targets (not swapped)"
    [ "$with_home" = yes ] && echo "  (home staged beside it as incoming/home-new)"
    echo "  swap by hand when ready (pairs with env edits — see the cutover runbook)"
    exit 0
  fi

  for t in $targets; do
    echo "ops dev: swapping + restarting $t"
    swap_box "$t" "$ver"
    [ "$with_home" = yes ] && swap_home "$t" "$ver"
  done
  echo
  echo "note: this opened dev-box SSH sessions — if this session hasn't written its"
  echo "      access-ledger why-entry yet: ./deploy/ops ledger \"...\""
  ;;

provision)
  # Refresh the dev boxes' systemd units from the binary they are already
  # running, and route the short-link redirector at the dev apex.
  #
  # Why this is a verb and not a documented ssh sequence: `deploy` only swaps
  # the binary and restarts the units that already exist, so a NEW unit (like
  # lchat-links) never appears on a dev box no matter how many times you
  # deploy. Only `lchat bootstrap` writes units — and running plain bootstrap
  # here would be a trap, because it enables the hourly auto-updater, which
  # would replace this box's unmerged code with the current stable build. The
  # whole point of the dev mesh is that it is NOT a release channel. So this
  # passes --no-update-timer (which disables the timer outright rather than
  # merely skipping it) and then VERIFIES the timer really is off before
  # touching anything else.
  targets="ie fi"
  for a in "$@"; do
    case "$a" in
      ie|fi) targets="$a" ;;
      *) usage ;;
    esac
  done

  for t in $targets; do
    if [ "$t" = ie ]; then box="$BOX_IE"; run() { ssh_ie "$box" "$@"; }
    else box="$BOX_FI"; run() { ssh -o ConnectTimeout=10 "$box" "$@"; }
    fi
    # Dublin holds the EIP that dev.11ch.at points at, so it is the dev apex
    # and bootstraps AS it (one step: starts the redirector and registers the
    # route). Helsinki is not the apex, so its bootstrap leaves the redirector
    # disabled — which is the behaviour worth exercising here too.
    if [ "$t" = ie ]; then apexflag="--links-apex $DEV_LINKS_HOST"; else apexflag=""; fi

    # Ordering guard: provision drives flags that only exist in a NEW binary,
    # and `deploy` is what puts one on the box. Running these in the wrong
    # order otherwise fails deep inside bootstrap's flag parsing with
    # "flag provided but not defined", which reads like a bug in this script
    # rather than a missing step.
    if ! run 'sudo /opt/lchat/lchat bootstrap -h 2>&1' | grep -q 'links-apex'; then
      echo "ops dev: ABORT — the binary on $t predates --links-apex." >&2
      echo "  provision configures the box; deploy is what ships the code. Run:" >&2
      echo "    ./deploy/ops dev deploy $t" >&2
      exit 1
    fi

    echo "ops dev: bootstrapping units on $t ($box)${apexflag:+ (as short-link apex $DEV_LINKS_HOST)}"
    run "sudo /opt/lchat/lchat bootstrap --no-update-timer $apexflag"

    # The invariant, asserted rather than assumed: if bootstrap ever regains a
    # path that leaves the updater on, this box starts silently overwriting the
    # unmerged code under test and the mesh stops meaning anything.
    timer="$(run 'systemctl is-enabled lchat-update.timer 2>/dev/null || true')"
    case "$timer" in
      enabled)
        echo "ops dev: ABORT — $t's auto-update timer is ENABLED after bootstrap." >&2
        echo "  The dev mesh runs unmerged code; an updater pass would replace it with" >&2
        echo "  the current stable build. Disable it before using this box:" >&2
        echo "    ssh $box 'sudo systemctl disable --now lchat-update.timer'" >&2
        exit 1 ;;
      *) echo "  $t: auto-update timer is '$timer' (not enabled — correct for dev)" ;;
    esac

    links="$(run 'systemctl is-active lchat-links 2>/dev/null || true')"
    if [ -n "$apexflag" ]; then
      echo "  $t: lchat-links is $links (apex — expected active)"
      [ "$links" = active ] || {
        echo "ops dev: $t is the apex but its links unit did not start — inspect:" >&2
        echo "    ssh $box journalctl -u lchat-links -n 40 --no-pager" >&2
        exit 1
      }
    else
      echo "  $t: lchat-links is $links (not the apex — expected inactive)"
      [ "$links" != active ] || echo "ops dev: note — $t runs the redirector but isn't the apex" >&2
    fi
  done

  # bootstrap --links-apex registered the route above, so there is no separate
  # add-service step: prove the result instead.
  case "$targets" in *ie*)
    # Prove it end to end. The certificate is issued lazily on first handshake,
    # so a cold apex can take a few seconds — and a TLS failure here is the
    # exact symptom of the production bug this rehearses (an unrouted host gets
    # no certificate at all, so it fails the handshake rather than 404ing).
    echo "ops dev: checking https://$DEV_LINKS_HOST/ ..."
    apex=""
    for _ in 1 2 3 4 5 6; do
      apex="$(curl -sS -o /dev/null -m 15 -w '%{http_code} -> %{redirect_url}' "https://$DEV_LINKS_HOST/" 2>&1 || true)"
      case "$apex" in 30*) break ;; esac
      sleep 5
    done
    echo "  apex:  $apex"
    echo "  (expect a 30x to elevenmessenger.com — that is the redirector answering)"
    case "$apex" in
      30*) ;;
      *) echo "ops dev: the apex did not redirect. If this is a TLS error, check the" >&2
         echo "  A record exists (ops dev apply lands aws_route53_record.dev_apex) —" >&2
         echo "  a *.dev.11ch.at wildcard does NOT cover the bare apex." >&2
         exit 1 ;;
    esac

    # /l/<host> only redirects to a host that verifiably runs lchat, so point it
    # at a real dev space rather than a made-up name.
    probe="${LCHAT_DEV_LINK_PROBE:-}"
    if [ -n "$probe" ]; then
      echo "ops dev: checking https://$DEV_LINKS_HOST/l/$probe ..."
      echo "  /l/:   $(curl -sS -o /dev/null -m 20 -w '%{http_code} -> %{redirect_url}' "https://$DEV_LINKS_HOST/l/$probe" 2>&1 || true)"
      echo "  (expect a 30x to https://$probe — a 404 means the target failed verification)"
    else
      echo "ops dev: set LCHAT_DEV_LINK_PROBE=<a live dev space host> to also test /l/<host>"
    fi
  ;; esac

  echo
  echo "note: this opened dev-box SSH sessions — if this session hasn't written its"
  echo "      access-ledger why-entry yet: ./deploy/ops ledger \"...\""
  ;;

rebuild)
  # Destroy + recreate the Dublin box — the deliberate key rotation. EC2
  # injects the keypair only at first boot, so this is how you take over a box
  # whose key you don't hold: commit YOUR public key + /32 to compute.tf,
  # replace the instance, and it boots trusting only you. The EIP, S3 bucket
  # (hibernated spaces, mesh membership, the id ledger), IAM, and the Route53
  # zone are separate resources and survive; the box's /data does not.
  confirm=no; adopt=no
  for a in "$@"; do case "$a" in --yes) confirm=yes ;; --adopt) adopt=yes ;; *) usage ;; esac; done
  ct="$root/infra/dev-mesh/compute.tf"

  # A fresh box trusts exactly one key — make sure it will be one you hold.
  # Generating is safe (refuses to overwrite); committing it is the PR's job.
  if [ ! -f "$KEY_IE" ]; then
    echo "ops dev: no $KEY_IE — generating a fresh box key (never vaulted; the"
    echo "  rebuilt box trusts it alone, so share it with the other operators)"
    ssh-keygen -q -t ed25519 -N '' -C eleven-dev-box -f "$KEY_IE"
  fi
  mypub="$(ssh-keygen -y -f "$KEY_IE" | awk '{print $1" "$2}')"
  tfpub="$(awk '/resource "aws_key_pair" "ie_node"/,/^}/' "$ct" \
    | sed -n 's/.*public_key *= *"\([^"]*\)".*/\1/p' | awk '{print $1" "$2}')"
  if [ "$mypub" != "$tfpub" ]; then
    echo "ops dev: compute.tf's key pair is NOT your key — tofu applies what the file says," >&2
    echo "  so the rebuilt box would trust someone else. Put your public key in" >&2
    echo "  aws_key_pair.ie_node (a reviewed PR — the public key is committed on purpose):" >&2
    echo "    public_key = \"$(ssh-keygen -y -f "$KEY_IE")\"" >&2
    exit 1
  fi

  # Same rule for the SSH allowlist: outside it, the post-rebuild SSH wait
  # would hang forever and read as a dead box. TF_VAR_ssh_ingress_cidr (a
  # JSON list) overrides the committed default, same as for tofu itself;
  # either way, every a.b.c.d/nn token counts.
  cidrs="$(printf '%s' "${TF_VAR_ssh_ingress_cidr:-}" \
    | grep -oE '[0-9]+(\.[0-9]+){3}/[0-9]+' || true)"
  [ -n "$cidrs" ] || cidrs="$(awk '/variable "ssh_ingress_cidr"/,/^}/' "$ct" \
    | grep -oE '[0-9]+(\.[0-9]+){3}/[0-9]+' || true)"
  mine="$(curl -s -m5 https://api.ipify.org 2>/dev/null || true)"
  if [ -z "$mine" ] || ! printf '%s\n' "$cidrs" | sed 's|/.*||' | grep -qxF "$mine"; then
    echo "ops dev: the security group would admit [$(printf '%s' "$cidrs" | tr '\n' ' ')] but your IP" >&2
    echo "  is ${mine:-unknown} — the rebuilt box would be unreachable from here. Add your /32" >&2
    echo "  to ssh_ingress_cidr in compute.tf (removing an entry revokes that operator)." >&2
    exit 1
  fi

  if [ "$confirm" != yes ]; then
    cat <<EOF
ops dev rebuild — this DESTROYS and recreates the Dublin box.
  dies:     everything on its disk — /data (non-hibernating dev spaces, the
            router registry, router/cloud/home env files and their tokens)
  survives: the EIP (DNS keeps pointing right), the S3 bucket (hibernated
            spaces, mesh membership, the id ledger), IAM, the Route53 zone
  needs:    LCHAT_MESH_TOKEN re-paired with Helsinki afterwards, and the env
            files rebuilt (the runbook's Step 3 blocks)
With --adopt: no destroy — the box was ALREADY replaced (an AMI-drift apply,
a console rebuild); this just runs the same provisioning against it.
Coordinate first — this tears down the live half of the dev mesh until
re-provisioned. Re-run with --yes to proceed (tofu will still show its plan
and ask for approval).
EOF
    exit 1
  fi

  if [ "$adopt" = yes ]; then
    # Adopt: the instance was already replaced outside this verb, so there is
    # nothing for tofu to do — which also means no AWS/DNSimple credentials
    # are needed. The facts tofu would have output come from elsewhere: the
    # EIP survives replacement (it IS the committed BOX_IE address), and the
    # bucket is shared with Helsinki, which survived and knows it.
    ip="${BOX_IE#*@}"
    bucket="$(ssh -o ConnectTimeout=10 "$BOX_FI" \
      "sudo grep '^LCHAT_HIBERNATE_BUCKET=' /data/router.env | cut -d= -f2-" 2>/dev/null || true)"
    [ -n "$bucket" ] || { echo "ops dev: couldn't read the mesh bucket from Helsinki's router.env — is $BOX_FI up?" >&2; exit 1; }
  else
    cd "$root/infra/dev-mesh"
    ensure_creds
    [ -d .terraform ] || run_tofu init -input=false
    # tofu's own interactive approve is the second gate — read the plan: it
    # should replace aws_instance.ie_node (and the key pair), nothing else
    # destructive. (-replace overrides the instance's ignore_changes=[ami],
    # so pinning the AMI does not break deliberate rebuilds.)
    run_tofu apply -replace=aws_instance.ie_node

    ip="$(run_tofu output -raw ie_node_ip)"
    bucket="$(run_tofu output -raw mesh_bucket)"
  fi
  # Same EIP, new host key — the stale known_hosts entry would make every ssh
  # fail with "REMOTE HOST IDENTIFICATION HAS CHANGED", so drop it first.
  ssh-keygen -R "$ip" >/dev/null 2>&1 || true
  echo "ops dev: waiting for the new box to accept ssh ($ip)"
  i=0
  until ssh -i "$KEY_IE" -o ConnectTimeout=5 -o StrictHostKeyChecking=accept-new \
        "ubuntu@$ip" true 2>/dev/null; do
    i=$((i + 1))
    [ "$i" -lt 60 ] || { echo "ops dev: box not reachable after 5m — check the AWS console" >&2; exit 1; }
    sleep 5
  done

  # First provision: fresh arm64 binary + `lchat bootstrap` (writes the
  # router/control/cloud units and starts the router). After this, routine
  # updates go through `ops dev deploy ie`.
  ver="$(git -C "$root" rev-parse --short HEAD)"
  tmp="$(mktemp -d -t lchat-rebuild.XXXXXX)"
  trap 'rm -rf "$tmp"' EXIT
  cd "$root"   # we were in infra/dev-mesh; build-linux.sh compiles the cwd package
  echo "ops dev: building $ver for linux/arm64 and bootstrapping"
  OUT="$tmp/lchat" ARCH=arm64 "$here/build-linux.sh" "$ver"
  scp_ie -q "$tmp/lchat" "ubuntu@$ip:/tmp/lchat"
  ssh_ie "ubuntu@$ip" 'sudo mkdir -p /opt/lchat && sudo mv /tmp/lchat /opt/lchat/lchat \
    && sudo chmod +x /opt/lchat/lchat && sudo /opt/lchat/lchat bootstrap'

  # The env files are derived facts plus one secret — encoded here so no
  # runbook is needed. The mesh token is READ from the surviving Helsinki box
  # (the pair must match) and never printed; the cloud admin token is freshly
  # minted, because a rebuild is a new cloud.
  echo "ops dev: writing /data/router.env + /data/cloud.env"
  mesh_token="$(ssh -o ConnectTimeout=10 "$BOX_FI" \
    "sudo grep '^LCHAT_MESH_TOKEN=' /data/router.env | cut -d= -f2-" 2>/dev/null || true)"
  [ -n "$mesh_token" ] || echo "ops dev: WARNING: couldn't read Helsinki's mesh token —" \
    "router.env is written without LCHAT_MESH_TOKEN; the mesh stays unpaired until it is set on both boxes" >&2
  admin_token="$(head -c 24 /dev/urandom | od -An -tx1 | tr -d ' \n')"
  {
    printf 'LCHAT_MESH_DOMAINS=dev.11ch.at\n'
    printf 'LCHAT_DNS_PROVIDER=route53\n'
    printf 'LCHAT_WILDCARD_DOMAINS=dev.11ch.at\n'
    printf 'AWS_REGION=eu-west-1\n'
    printf 'LCHAT_HIBERNATE_BUCKET=%s\n' "$bucket"
    printf 'LCHAT_MESH_REGION=ie\n'
    printf 'LCHAT_MESH_SELF=https://one.ie.dev.11ch.at\n'
    [ -z "$mesh_token" ] || printf 'LCHAT_MESH_TOKEN=%s\n' "$mesh_token"
  } | ssh_ie "ubuntu@$ip" 'sudo tee /data/router.env >/dev/null && sudo chmod 600 /data/router.env'
  {
    printf 'LCHAT_CLOUD_BASE_DOMAIN=dev.11ch.at\n'
    printf 'LCHAT_CLOUD_ID_LEDGER=1\n'
    printf 'LCHAT_HIBERNATE_BUCKET=%s\n' "$bucket"
    printf 'LCHAT_HIBERNATE_REGION=eu-west-1\n'
    printf 'LCHAT_CLOUD_REGION=ie\n'
    printf 'LCHAT_ADMIN_TOKEN=%s\n' "$admin_token"
  } | ssh_ie "ubuntu@$ip" 'sudo tee /data/cloud.env >/dev/null && sudo chmod 600 /data/cloud.env'
  ssh_ie "ubuntu@$ip" 'sudo systemctl restart lchat-router lchat-cloud lchat-control'

  # A fresh registry knows no routes: the surviving peer bulk-seeds every
  # in-scope route into it (meshreplicate.go — mesh-seed exists exactly for a
  # freshly joined box). sudo strips env, so the peer's mesh config is
  # sourced explicitly (the old runbook's hardest-won gotcha).
  if [ -n "$mesh_token" ]; then
    echo "ops dev: seeding routes from Helsinki"
    ssh -o ConnectTimeout=10 "$BOX_FI" \
      "sudo sh -c 'set -a; . /data/router.env; set +a; /opt/lchat/lchat mesh-seed https://one.ie.dev.11ch.at'" \
      || echo "ops dev: WARNING: mesh-seed failed — rerun it on Helsinki when the router is up" >&2
  fi

  # controld extras: without these a rebuilt Dublin silently provisions
  # non-hibernating, uncapped spaces — the opposite of the dev mesh's point.
  echo "ops dev: writing /data/control.env"
  {
    printf 'LCHAT_CONTROL_HIBERNATE_CREATES=1\n'
    printf 'LCHAT_MAX_INSTANCES=40\n'
    printf 'LCHAT_HIBERNATE_BUCKET=%s\n' "$bucket"
    printf 'LCHAT_HIBERNATE_REGION=eu-west-1\n'
  } | ssh_ie "ubuntu@$ip" 'sudo tee /data/control.env >/dev/null && sudo chmod 600 /data/control.env && sudo systemctl restart lchat-control'

  # The dev home server (encoded here per the old completion note). bootstrap
  # already wrote its unit; home is its own binary. The env carries the
  # WebAuthn RP origin — it MUST stay home.dev.elevenmessenger.com or every
  # previously-enrolled dev passkey orphans — and one caller token per region,
  # minted from each cloud's admin API over its local socket. Token values
  # move through pipes and shell variables only; nothing prints them.
  echo "ops dev: building + standing up the dev home server"
  GOOS=linux GOARCH=arm64 CGO_ENABLED=0 go build -o "$tmp/home" ./home
  scp_ie -q "$tmp/home" "ubuntu@$ip:/tmp/home"
  ssh_ie "ubuntu@$ip" 'sudo mv /tmp/home /opt/lchat/home && sudo chmod +x /opt/lchat/home && sudo chown root:root /opt/lchat/home'

  tok_of() { sed -n 's/.*"token":"\([^"]*\)".*/\1/p'; }
  ie_caller="$(ssh_ie "ubuntu@$ip" 'sudo sh' <<EOS | tok_of
curl -s --unix-socket /run/lchat/cloud.sock -X POST http://cloud/api/admin/tokens \
  -H "Authorization: Bearer $admin_token" -d '{"label":"home.dev"}'
EOS
)"
  fi_caller="$(ssh -o ConnectTimeout=10 "$BOX_FI" 'sudo sh' <<'EOS' | tok_of
ADM="$(grep '^LCHAT_ADMIN_TOKEN=' /data/cloud.env | cut -d= -f2-)"
curl -s --unix-socket /run/lchat/cloud.sock -X POST http://cloud/api/admin/tokens \
  -H "Authorization: Bearer $ADM" -d '{"label":"home.dev"}'
EOS
)"
  [ -n "$ie_caller" ] || echo "ops dev: WARNING: no ie caller token minted — home's ie region will 502 until one is set in /data/home.env" >&2
  [ -n "$fi_caller" ] || echo "ops dev: WARNING: no fi caller token minted — home's fi region will 502 until one is set in /data/home.env" >&2
  {
    printf 'LCHAT_HOME_ORIGIN=https://home.dev.elevenmessenger.com\n'
    printf 'LCHAT_CLOUD_REGIONS=[{"code":"ie","name":"Ireland","url":"https://cloud.ie.dev.11ch.at","token":"%s"},{"code":"fi","name":"Finland","url":"https://cloud.fi.dev.11ch.at","token":"%s"}]\n' "$ie_caller" "$fi_caller"
    printf 'LCHAT_CLOUD_DEFAULT_REGION=ie\n'
    # Sign in with Apple: the audiences are committed facts (web Services ID +
    # iOS bundle id); the account-keying pepper is minted fresh — fine on a
    # rebuilt (empty) anchor DB, where there are no siwa anchors to orphan.
    # Without these the /siwa/return trampoline answers 404 and the Apple door
    # dies silently (2026-07-23). Google/Microsoft doors need OAuth secrets and
    # stay dark until set by hand.
    printf 'LCHAT_HOME_SIWA_CLIENT_IDS=com.elevenmessenger.signin,com.elevenmessenger.messenger\n'
    printf 'LCHAT_HOME_SIWA_PEPPER=%s\n' "$(head -c 24 /dev/urandom | od -An -tx1 | tr -d ' \n')"
  } | ssh_ie "ubuntu@$ip" 'sudo tee /data/home.env >/dev/null && sudo chmod 600 /data/home.env'
  ssh_ie "ubuntu@$ip" 'sudo systemctl enable --now lchat-home'
  # Service routes are local registry rows mesh-seed doesn't carry; re-adding
  # an existing row fails harmlessly on a re-run.
  ssh_ie "ubuntu@$ip" 'sudo /opt/lchat/lchat add-service home.dev.elevenmessenger.com --socket /run/lchat/home.sock' \
    || echo "ops dev: (home route already present)"
  ssh_ie "ubuntu@$ip" 'sudo /opt/lchat/lchat add-service cloud.ie.dev.11ch.at --socket /run/lchat/cloud.sock' \
    || echo "ops dev: (cloud route already present)"

  cat <<EOF

Dublin rebuilt on $ip (same EIP — DNS untouched), running $ver, trusting only
your key: env files written (router/cloud/control/home), units restarted,
routes seeded from Helsinki, home + cloud service routes added.
Still not encoded here — EXTEND THIS VERB when next needed, don't reach for a
runbook: the dev connect broker (lchat-connect-dev needs the vault's sandbox
Stripe pair) and the cloud.env demo/pool/billing extras.
  verify: ssh ubuntu@$ip journalctl -u lchat-router -n 40 --no-pager | grep -i wildcard
          curl -s https://home.dev.elevenmessenger.com/api/features
  then:   land the compute.tf PR if it changed for this rebuild, and
          ./deploy/ops ledger "rebuilt the dev-mesh Dublin box"
EOF
  ;;

*) usage ;;
esac