#!/usr/bin/env bash
# 🤖 Fleet DB backup via Litestream (issue #550: continuous replication, not
# snapshots — closes the "instance DBs have no off-box backup" gap).
#
# Runs ON the router box, as root, idempotent — re-run any time (e.g. after
# add-instance) to pick up new databases; it regenerates the config and
# restarts the replicator. Everything replicated is instance ciphertext plus
# routing metadata the box already holds, so an off-box copy is
# threat-model-free (the E2EE rule).
#
# Prerequisites:
#   /data/litestream.env — AWS creds for the scoped litestream-fleet IAM user
#     (AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY; object-rw on the fleet
#     bucket only — minted per infra/prod/backup.tf, key NOT in tofu state).
#
# Usage:
#   fleet-backup.sh                    # replicate every /data/*.db
#   fleet-backup.sh test.luan.chat.db  # canary: just the named DB file(s)
#
# Verify after first run:
#   /opt/lchat/litestream databases -config /etc/litestream.yml
#   journalctl -u lchat-litestream -n 20
# Restore drill (to a scratch path, never over the live file):
#   AWS creds in env, then:
#   /opt/lchat/litestream restore -config /etc/litestream.yml -o /tmp/drill.db /data/<host>.db
set -euo pipefail

LITESTREAM_VERSION="0.5.14"
BUCKET="eleven-prod-fleet-backup-427880590669"
REGION="eu-west-1"
BIN=/opt/lchat/litestream
CONF=/etc/litestream.yml
ENVFILE=/data/litestream.env
UNIT=/etc/systemd/system/lchat-litestream.service

[ -f "$ENVFILE" ] || { echo "missing $ENVFILE (AWS creds for litestream-fleet) — mint per infra/prod/backup.tf" >&2; exit 1; }

# 1. The binary (skip when the right version is already installed).
if ! "$BIN" version 2>/dev/null | grep -q "$LITESTREAM_VERSION"; then
  echo "installing litestream v$LITESTREAM_VERSION"
  curl -fsSL "https://github.com/benbjohnson/litestream/releases/download/v${LITESTREAM_VERSION}/litestream-${LITESTREAM_VERSION}-linux-x86_64.tar.gz" \
    | tar -xzO litestream > "$BIN.tmp"
  chmod 0755 "$BIN.tmp" && mv "$BIN.tmp" "$BIN"
fi

# 2. The config, regenerated from what's actually in /data. One replica prefix
# per database file; trash and sidecar session dirs are not databases.
shopt -s nullglob
# Hibernating instances are excluded: the activator's embedded Litestream
# owns their replication (to the prod-instances bucket, under the lease) —
# a second replicator on the same WAL would fight it for checkpointing.
hibernating=""
if [ -f /data/registry.db ] && command -v sqlite3 >/dev/null; then
  hibernating=$(sqlite3 /data/registry.db \
    "SELECT host FROM routes WHERE hibernate = 1" 2>/dev/null || true)
fi
skip_hibernating() {
  local base; base=$(basename "$1" .db)
  for h in $hibernating; do [ "$h" = "$base" ] && return 0; done
  return 1
}
dbs=()
if [ $# -gt 0 ]; then
  for name in "$@"; do dbs+=("/data/$name"); done
else
  for f in /data/*.db; do
    skip_hibernating "$f" && { echo "skipping $f (hibernating — activator owns its replication)"; continue; }
    dbs+=("$f")
  done
fi
[ ${#dbs[@]} -gt 0 ] || { echo "no databases matched" >&2; exit 1; }

{
  echo "# generated by deploy/fleet-backup.sh — re-run the script, don't edit"
  # Cadence: this replicator is a BACKUP, not failover (the activator owns
  # failover for hibernating hosts), so every timer below trades recency for
  # S3 request volume. Litestream's defaults (sync 1s, L1 compaction 30s, L0
  # retention LISTs every 15s) burned ~113k Tier-1 requests in 14h across the
  # fleet with near-zero writes — the retention/compaction monitors LIST per
  # database per tick whether or not anything changed.
  cat <<'YAML'
# RPO: at most ~60s of a space's ciphertext is lost if the box dies — for a
# backup of family-chat instances that is plenty; idle databases upload
# nothing regardless (an idle sync tick makes no S3 calls).
sync-interval: 60s
# Compaction ladder, ~10x the 30s/5m/1h defaults with the same geometric
# shape (restore read-amplification stays bounded). L1's tick is the request
# driver — each level's monitor costs up to two LISTs per database per tick.
levels:
  - interval: 5m
  - interval: 30m
  - interval: 3h
# Snapshots stay daily — the gentle direction here is LESS often, not the
# hourly one might reach for: each snapshot uploads a full database image.
# Retention 72h keeps ~3 daily snapshots so one bad newest snapshot never
# strands the restore chain. v0.5 enforces snapshot retention on the snapshot
# tick itself (there is no separate retention-check knob), so retention
# sweeps are daily too.
snapshot:
  interval: 24h
  retention: 72h
# The L0 retention monitor is the worst idle offender: every tick LISTs L1
# (and usually L0) per database even with zero writes, and its 15s default
# exists for litestream's VFS read replicas, which we don't run. L0 files
# only become reclaimable after an L1 compaction, so hourly sweeps delete
# exactly the same files — just 240x fewer LISTs. l0-retention is the minimum
# age before a compacted L0 file may go; 15m costs pennies of storage and
# keeps recent increments around for point-in-time restores.
l0-retention: 15m
l0-retention-check-interval: 1h
YAML
  echo "dbs:"
  for f in "${dbs[@]}"; do
    [ -f "$f" ] || { echo "warning: $f missing, skipped" >&2; continue; }
    printf '  - path: %s\n    replica:\n      type: s3\n      bucket: %s\n      path: fleet/%s\n      region: %s\n' \
      "$f" "$BUCKET" "$(basename "$f")" "$REGION"
  done
} > "$CONF"
chmod 0644 "$CONF"
echo "wrote $CONF (${#dbs[@]} databases)"

# 3. The unit. Runs as the lchat user (owns /data), same hardening posture as
# the other units; litestream is a pure outbound client.
cat > "$UNIT" <<'EOF'
[Unit]
Description=Litestream fleet DB replication (eleven)
After=network-online.target
Wants=network-online.target

[Service]
User=lchat
Group=lchat
EnvironmentFile=/data/litestream.env
ExecStart=/opt/lchat/litestream replicate -config /etc/litestream.yml
Restart=always
RestartSec=5
NoNewPrivileges=true
ProtectSystem=strict
ReadWritePaths=/data
ProtectHome=true
PrivateTmp=true
MemoryMax=512M

[Install]
WantedBy=multi-user.target
EOF

systemctl daemon-reload
systemctl enable lchat-litestream
systemctl restart lchat-litestream
systemctl --no-pager --lines=0 status lchat-litestream || true
echo "done — verify with: journalctl -u lchat-litestream -n 20"