#!/usr/bin/env sh # ๐Ÿค– doctor โ€” one read-only preflight for every credential and access path an # operator needs. Answers "can I actually deploy right now?" in one command, # instead of six documents. Verb one of the ops surface (`./deploy/ops`) โ€” as # verbs land there, their prerequisites get a check here, so the preflight # stays honest. # # ./deploy/ops doctor # everything (doctor.sh runs standalone too) # ./deploy/ops doctor release # one section: tools|release|fleet|dev # ./deploy/ops doctor --no-ssh # skip the box checks (see the note below) # ./deploy/ops doctor --no-unlock # never prompt (it prompts to unlock by # # default, so the key check is real) # # Why this exists: the knowledge for this was spread across CLAUDE.md, # CLAUDE.local.md, docs/server-releases.md, docs/multi-edge-dev-cutover.md, the # vault's RUNBOOK.md, and raw tofu source โ€” and drifted from reality in four # places at once. Prose drifts silently; a script that runs drifts loudly. Every # check below is a fact about *this machine right now*, not a claim in a doc. # # Two rules this script holds to, because it touches real credentials: # 1. It NEVER prints a secret. Checks assert reachability/permission and # discard output โ€” a fetched credential is tested for existence, never # echoed. The one interactive moment (the Bitwarden unlock) captures the # session token into a variable and never displays it. # 2. It NEVER mutates anything โ€” no deploys, no writes, no unlocks. Every # remote command is a read (`lchat instances`, `id`, `groups`). # # NOTE: the `fleet` section opens real operator SSH sessions, which incur a # why-entry in the public access ledger (web/status/ssh/, RUNBOOK.md ยง Box # access). Use --no-ssh when you only want the local picture. set -u section="${1:-all}" no_ssh=no unlock=yes for a in "$@"; do [ "$a" = "--no-ssh" ] && no_ssh=yes [ "$a" = "--no-unlock" ] && unlock=no done case "$section" in --*) section=all ;; esac root="$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)" # The Bitwarden item names checked below โ€” the vault contract. # shellcheck source=deploy/vault-items.sh . "$root/deploy/vault-items.sh" # shellcheck source=deploy/bw-lib.sh . "$root/deploy/bw-lib.sh" # Only used to point at gen-fleet-conf.sh when a tunnel is down. VAULT="${LCHAT_VAULT_DIR:-$HOME/elevenmessenger/secrets-vault}" # Box aliases from ~/.ssh/config; `ssh -G` resolves user+hostname, so the ssh # config stays the single source of truth for how to reach a box. BOXES="${LCHAT_BOXES:-elevenbox kepler}" RELEASES_REPO="${LCHAT_RELEASES_REPO:-elevenmessenger/releases}" fails=0 warns=0 # --- output helpers ------------------------------------------------------- # Two columns: a fixed-width label, then a verdict. Fix hints are indented # under the failing line so the next action is always adjacent to the problem. ok() { printf ' \033[32mok\033[0m %s\n' "$1"; } bad() { printf ' \033[31mFAIL\033[0m %s\n' "$1"; fails=$((fails + 1)); } warn() { printf ' \033[33mwarn\033[0m %s\n' "$1"; warns=$((warns + 1)); } fix() { printf ' โ†’ %s\n' "$1"; } head_() { printf '\n\033[1m%s\033[0m\n' "$1"; } have() { command -v "$1" >/dev/null 2>&1; } want() { [ "$section" = all ] || [ "$section" = "$1" ]; } # --- tools ---------------------------------------------------------------- check_tools() { head_ "Local tooling" # Each tool is annotated with what breaks without it, so a missing one is # self-explanatory rather than a bare name. for pair in \ "bw:release signing key (Bitwarden)" \ "gh:publishing to the releases repo" \ "jq:parsing manifests + bw/gh output" \ "go:building the release binaries" \ "tofu:the dev-mesh infrastructure" \ "aws:the dev-mesh + release S3 mirror" \ "ssh:reaching every box" do t="${pair%%:*}"; why="${pair#*:}" if have "$t"; then ok "$t"; else bad "$t โ€” needed for $why" fix "brew install $t" fi done } # --- release publishing --------------------------------------------------- # Confirm the item is reachable WITHOUT printing it: we only look at whether # the fetch succeeded, never at the value. check_signing_item() { if bw_run 30 get item "$BW_ITEM_SIGNING_KEY" \ >/dev/null 2>&1; then ok "signing key item is present" else bad "signing key item not found in this vault" fix "check it is shared to your org collection (BITWARDEN-MIGRATION.md)" fi } check_release() { head_ "Publishing a release" if ! have bw; then bad "bw not installed โ€” cannot reach the signing key" fix "brew install bitwarden-cli" else st="$(bw_run 20 status 2>/dev/null | tr ',' '\n' | sed -n 's/.*"status":"\([a-z]*\)".*/\1/p')" case "$st" in unlocked) ok "bitwarden unlocked" check_signing_item ;; locked) # Unlock by default: "locked, so I can't tell you" is a non-answer, and # the whole point is to find out whether the key is actually reachable. # The session is this process's only โ€” a child cannot export BW_SESSION # back to the calling shell โ€” so this deepens the check and nothing # more. Without a tty there is nobody to prompt, so degrade instead of # hanging; --no-unlock forces that path deliberately. if [ "$unlock" = yes ] && [ -t 0 ]; then printf ' unlocking bitwarden to verify the signing key\n' # Our prompt, never bw's; watchdogged (bw-lib.sh) โ€” bw's own # prompt inside $(...) is invisible and reads as a hang. BW_SESSION="$(bw_unlock_session)" || BW_SESSION="" export BW_SESSION if [ -n "${BW_SESSION:-}" ]; then ok "bitwarden unlocked (this run only)" check_signing_item else bad "unlock failed โ€” signing key not verified" fix "wrong master password, or run 'bw unlock' by hand to see why" fi else warn "bitwarden is locked โ€” signing key not verified" fix "bw unlock # then paste the 'export BW_SESSION=...' line it prints, and re-run" fi ;; *) bad "bitwarden not logged in" fix "bw login # EU vault: https://vault.bitwarden.eu" ;; esac fi if have gh && gh auth status >/dev/null 2>&1; then ok "gh authenticated" push="$(gh api "repos/$RELEASES_REPO" --jq .permissions.push 2>/dev/null)" if [ "$push" = "true" ]; then ok "write access to $RELEASES_REPO"; else bad "no write access to $RELEASES_REPO" fix "ask for push rights on the releases repo" fi else bad "gh not authenticated" fix "gh auth login" fi # release-server.sh refuses to build from a dirty tree; surface that here # rather than 20 minutes into a release. if [ -n "$(git -C "$root" status --porcelain 2>/dev/null)" ]; then warn "working tree is dirty โ€” release-server.sh will refuse to publish" fix "commit or stash, and publish from a clean main" else ok "working tree is clean" fi } # --- fleet boxes ---------------------------------------------------------- check_fleet() { head_ "Fleet boxes (WireGuard + SSH + sudo)" if [ "$no_ssh" = yes ]; then warn "skipped (--no-ssh)" return fi for box in $BOXES; do # ssh -G resolves the config the way ssh itself will, so we test what will # actually happen rather than re-parsing ~/.ssh/config by hand. hostname="$(ssh -G "$box" 2>/dev/null | sed -n 's/^hostname //p' | head -1)" user="$(ssh -G "$box" 2>/dev/null | sed -n 's/^user //p' | head -1)" # One connection attempt tells us both things. A tunnel that is down and a # key that is refused look identical from a bare exit code, so classify the # error text instead โ€” and skip ping, whose -W flag means milliseconds on # macOS and seconds on Linux. err="$(ssh -o ConnectTimeout=8 -o BatchMode=yes "$box" true 2>&1)" if [ -z "$err" ]; then ok "$box โ€” tunnel up, ssh as '$user' ($hostname)" else case "$err" in *"Permission denied"*) bad "$box โ€” reachable, but ssh as '$user' was refused" fix "confirm which key the box authorises; a key not at a default" fix "path needs IdentityFile + IdentitiesOnly in ~/.ssh/config" ;; *"Host key verification failed"*) # The box answered โ€” BatchMode just can't accept a first-seen host # key. Distinct from unreachable/refused, and self-serve fixable. bad "$box โ€” reachable, but its host key isn't in known_hosts yet" fix "one-off interactive: ssh $box # accept the host key, then re-run" ;; *"Could not resolve hostname"*) # ssh -G echoes an unknown alias back rather than failing, so an # absent Host block surfaces here as a resolve error. bad "$box โ€” no Host block in ~/.ssh/config (or DNS failure)" fix "add a Host block for $box, or set LCHAT_BOXES" ;; *"timed out"*|*"No route"*|*"Network is unreachable"*) bad "$box โ€” $hostname unreachable (WireGuard down?)" fix "bring the tunnel up, then re-run; regenerate it with" fix "cd $VAULT && ./gen-fleet-conf.sh > ~/Downloads/fleet.conf" ;; *) bad "$box โ€” $(printf '%s' "$err" | head -1)" ;; esac continue fi # lchat-operators is what the sudoers allowlist keys off; without it the # ssh login succeeds but every privileged command fails. if ssh -o BatchMode=yes "$box" 'id -nG' 2>/dev/null \ | tr ' ' '\n' | grep -qx lchat-operators; then ok "$box โ€” in lchat-operators" elif [ "$user" = root ]; then ok "$box โ€” root (allowlist not applicable)" else bad "$box โ€” not in lchat-operators" fix "ask an operator to add you to the group" fi if ssh -o BatchMode=yes "$box" \ 'sudo -n /opt/lchat/lchat instances' >/dev/null 2>&1; then ok "$box โ€” sudo allowlist works (can deploy)" else bad "$box โ€” 'sudo -n lchat' denied" fix "check /etc/sudoers.d/lchat-operators (deploy/sudoers-operators)" fi done } # --- dev mesh ------------------------------------------------------------- check_dev() { head_ "Dev mesh (Dublin + Helsinki)" # The Dublin box authorises exactly one key, committed in compute.tf: the # SHARED box key (its comment is the box's name, not a person's) โ€” created # at provisioning, never vaulted, so copies exist only on operator machines. keyfile="$HOME/.ssh/eleven-dev-box" if [ -f "$keyfile" ]; then ok "eleven-dev-box key present" else bad "no $keyfile" fix "the shared dev-box key โ€” not in any vault; ask an operator who holds it" fi # What `ops dev deploy` actually needs: a working SSH session to each box. # Same one-attempt classification as the fleet section โ€” and here a timeout # is USUALLY the single-/32 security group (named by the CIDR check below), # not a dead box. if [ "$no_ssh" = yes ]; then warn "dev box ssh checks skipped (--no-ssh)" else for spec in "ie:${LCHAT_DEV_BOX_IE:-ubuntu@54.154.95.205}" \ "fi:${LCHAT_DEV_BOX_FI:-root@157.180.123.183}"; do node="${spec%%:*}"; dest="${spec#*:}" if [ "$node" = ie ] && [ -f "$keyfile" ]; then err="$(ssh -i "$keyfile" -o ConnectTimeout=8 -o BatchMode=yes "$dest" true 2>&1)" else err="$(ssh -o ConnectTimeout=8 -o BatchMode=yes "$dest" true 2>&1)" fi if [ -z "$err" ]; then ok "dev $node โ€” ssh works ($dest)" else case "$err" in *"Permission denied"*) bad "dev $node โ€” reachable, but ssh as '$dest' was refused" fix "ie wants ~/.ssh/eleven-dev-box; fi wants the 1Password hetzner key" ;; *"Host key verification failed"*) bad "dev $node โ€” reachable, but its host key isn't in known_hosts yet" fix "one-off interactive: ssh $dest # accept the host key, then re-run" ;; *"timed out"*|*"No route"*|*"Network is unreachable"*) bad "dev $node โ€” unreachable; if the IP check below fails, that is why" fix "your IP must be in the security group's allowlist (ssh_ingress_cidr)" ;; *) bad "dev $node โ€” $(printf '%s' "$err" | head -1)" ;; esac fi done fi # ops dev pulls AWS_*/DNSIMPLE_* from the per-operator dev agent item in # Bitwarden itself, so a standing profile is a convenience, not a # prerequisite โ€” only "no route to any credential" is a failure. if [ -n "${AWS_ACCESS_KEY_ID:-}" ]; then ok "AWS credentials in the environment" elif aws configure list-profiles 2>/dev/null | grep -qx eleven-dev; then ok "aws profile 'eleven-dev' configured" elif have bw; then warn "no standing AWS auth โ€” ops dev will pull the dev agent creds from Bitwarden per run" else bad "no AWS auth at all (no env creds, no 'eleven-dev' profile, no bw)" fix "install bw, or seed the profile from your dev agent item (docs/secrets.md ยง AWS)" fi # SSH ingress is a short per-operator /32 allowlist (compute.tf, # ssh_ingress_cidr), so being outside it looks exactly like the box being # down. Say which it is. Anchored to the variable block โ€” every a.b.c.d/nn # inside it counts, so keep literal IPs out of its description text. cidrs="$(awk '/variable "ssh_ingress_cidr"/,/^}/' \ "$root/infra/dev-mesh/compute.tf" 2>/dev/null \ | grep -oE '[0-9]+(\.[0-9]+){3}/[0-9]+')" if [ -n "$cidrs" ]; then mine="$(curl -s -m5 https://api.ipify.org 2>/dev/null)" if [ -n "$mine" ] && printf '%s\n' "$cidrs" | sed 's|/.*||' | grep -qxF "$mine"; then ok "your IP ($mine) is in the dev security group's allowlist" else bad "your IP (${mine:-unknown}) is not in [$(printf '%s' "$cidrs" | tr '\n' ' ')] โ€” SSH will hang, not refuse" fix "add your /32 to var.ssh_ingress_cidr (a PR), then: ops dev apply" fix "removing someone else's /32 revokes their access โ€” only with their say-so" fi fi } # --- run ------------------------------------------------------------------ printf '\033[1mlchat operator preflight\033[0m โ€” read-only; no secrets printed\n' want tools && check_tools want release && check_release want fleet && check_fleet want dev && check_dev printf '\n' if [ "$fails" -eq 0 ] && [ "$warns" -eq 0 ]; then printf '\033[32mAll checks passed.\033[0m\n' elif [ "$fails" -eq 0 ]; then printf '\033[33m%s warning(s), no failures.\033[0m\n' "$warns" else printf '\033[31m%s failure(s), %s warning(s).\033[0m\n' "$fails" "$warns" fi # Non-zero on hard failures only, so this is usable as a gate in other scripts # while warnings (a locked vault, a dirty tree) stay informational. [ "$fails" -eq 0 ]