# Proves the S3 replicas actually restore: pulls every database named in
# /etc/litestream.yml back from S3 into a private tmp, opens it, and runs an
# integrity check. An un-restored backup is not a backup. Runs daily via
# carlos-restore-verify.timer; a failure leaves this unit in state 'failed',
# where fleet sweeps see it loudly. Ported from titogo's
# deploy/systemd/titogo-restore-verify.service.
[Unit]
Description=carlos restore-verify (restore every S3 replica, integrity-check it)
After=network-online.target carlos-litestream.service
Wants=network-online.target

[Service]
Type=oneshot
ExecStart=/opt/carlos/carlos ops restore-verify -config /etc/litestream.yml
EnvironmentFile=/data/litestream.env
# Restore into a DISK-backed directory, not the PrivateTmp tmpfs. Both are
# charged to this cgroup — page cache is accounted just like tmpfs, so the
# reported peak barely moves (flagship: 485.6M on tmpfs, 494.4M on disk, for a
# 412M tenant). What changes is what the ceiling DOES when the restore does not
# fit, and that is the whole point:
#
#   tmpfs pages are unevictable  -> hitting MemoryMax KILLS the drill
#   page cache is reclaimable    -> hitting MemoryMax THROTTLES it
#
# Measured on the flagship 2026-08-16, same 412M database, MemoryMax=128M:
# on tmpfs, oom-killed after 8.3s; on disk, still running after 10 minutes
# under reclaim (stopped by hand). So the restored file no longer has to FIT
# in the cgroup's memory — a tenant outgrowing the ceiling degrades instead of
# failing.
#
# It does NOT make size irrelevant: a badly undersized ceiling makes the drill
# crawl. Keep MemoryMax comfortably above the largest tenant. systemd creates
# and cleans %C/carlos-verify and binds it writable despite
# ProtectSystem=strict.
CacheDirectory=carlos-verify
Environment=TMPDIR=%C/carlos-verify
User=carlos
Group=carlos
CapabilityBoundingSet=
NoNewPrivileges=yes
ProtectSystem=strict
ProtectHome=yes
PrivateTmp=yes
PrivateDevices=yes
RestrictAddressFamilies=AF_INET AF_INET6
MemoryDenyWriteExecute=yes
# Sized to sit ABOVE the largest single database, not to bound the process:
# restored bytes are charged here either way (see the TMPDIR note). Measured
# after the move to disk: a fleet of ~30 small replicas peaks around 150M,
# a single ~400M tenant database around 500M. Two earlier lessons are
# load-bearing and kept: the service was OOM-killed at a ~256M peak when
# restored copies ACCUMULATED (RestoreVerify now deletes each once checked
# — see its comment), and tmpfs is why unfitting files were fatal rather
# than slow.
#
# WATCH: a single database approaching this ceiling will throttle, not
# fail, when it crosses — raise this then.
MemoryMax=512M