package main

import (
	"bytes"
	"context"
	"crypto/sha256"
	"database/sql"
	"encoding/hex"
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"log"
	"net/http"
	"os"
	"path/filepath"
	"runtime"
	"strings"
	"syscall"
	"time"
)

// The box-side auto-updater: `lchat update run`, fired hourly by
// lchat-update.timer (root — it stages binaries and restarts units, like
// lchat-control). One pass reads the signed channel feed (release.go), asks
// each instance what its admin chose (GET /api/update-prefs over the unix
// socket — instances are sandboxed and can't update themselves), then
// downloads/stages/applies per those prefs, reusing the exact deploy dance
// (deployHostVersion: repoint → restart → health-check → auto-rollback).
// It shares the deploy flock with `deploy`/`rollback`, so a timer firing
// mid-operator-deploy waits its turn (fails fast, next tick retries).
//
// Trust + anti-replay: the manifest is Ed25519-verified, binaries are hashed
// against the signed sha256, and the last accepted generated_at is persisted —
// a replayed stale feed is refused, so nobody can downgrade the fleet by
// re-serving an old (validly signed) manifest. See docs/server-releases.md.

func incomingDir() string { return filepath.Join(optDir, "incoming") }

// manifestStampPath persists the newest generated_at ever accepted.
func manifestStampPath() string { return filepath.Join(optDir, "manifest-stamp") }

func readManifestStamp() time.Time {
	raw, err := os.ReadFile(manifestStampPath())
	if err != nil {
		return time.Time{}
	}
	t, err := time.Parse(time.RFC3339, strings.TrimSpace(string(raw)))
	if err != nil {
		return time.Time{}
	}
	return t
}

func writeManifestStamp(t time.Time) error {
	return os.WriteFile(manifestStampPath(), []byte(t.Format(time.RFC3339)+"\n"), 0o644)
}

// fetchReleaseManifestFrom downloads and signature-verifies the channel feed
// from one URL.
func fetchReleaseManifestFrom(url string) (*releaseManifest, error) {
	pub, err := releasePubKey()
	if err != nil {
		return nil, err
	}
	client := &http.Client{Timeout: 2 * time.Minute}
	resp, err := client.Get(url)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		return nil, fmt.Errorf("fetch manifest: %s", resp.Status)
	}
	raw, err := io.ReadAll(io.LimitReader(resp.Body, 1<<20))
	if err != nil {
		return nil, err
	}
	return verifyReleaseManifest(pub, raw)
}

// fetchReleaseManifest reads the channel feed, falling back to the S3 mirror
// (LCHAT_RELEASE_MIRROR_URL) when the primary fetch fails — or when it
// verifies but is STALE: GitHub's CDN has served a pre-publish manifest for
// minutes after a clobber, which errors nowhere. Stale here means a
// generated_at strictly older than the newest feed this box ever accepted
// (the anti-replay stamp); a feed equal to the stamp is indistinguishable
// from "no new release", so it never consults the mirror — the hourly pass
// stays a single request in the steady state. Signature verification is
// identical on both paths (the mirror adds availability, never trust), and
// every fallback logs loudly. Shared with the instance-side availability
// checker (updatecheck.go), where no stamp file exists and so only outright
// primary failure falls back.
func fetchReleaseManifest(url string) (*releaseManifest, error) {
	m, err := fetchReleaseManifestFrom(url)
	base := releaseMirrorURL
	if base == "" {
		return m, err
	}
	stamp := readManifestStamp()
	stale := err == nil && m.GeneratedAt.Before(stamp)
	if err == nil && !stale {
		return m, nil
	}
	murl := mirrorManifestURL(base)
	mm, merr := fetchReleaseManifestFrom(murl)
	switch {
	case err != nil && merr != nil:
		log.Printf("update: primary feed failed (%v); mirror %s failed too (%v)", err, murl, merr)
		return nil, err
	case err != nil:
		log.Printf("update: primary feed failed (%v) — using mirror %s (signature verified)", err, murl)
		return mm, nil
	case merr != nil:
		log.Printf("update: primary feed is stale (generated_at %s < last accepted %s) and mirror %s failed (%v) — keeping the primary result",
			m.GeneratedAt.Format(time.RFC3339), stamp.Format(time.RFC3339), murl, merr)
		return m, nil
	case mm.GeneratedAt.After(m.GeneratedAt):
		log.Printf("update: primary feed is stale (generated_at %s < last accepted %s) — using fresher mirror %s (generated_at %s)",
			m.GeneratedAt.Format(time.RFC3339), stamp.Format(time.RFC3339), murl, mm.GeneratedAt.Format(time.RFC3339))
		return mm, nil
	}
	// The mirror is no fresher: the stale primary stands, and updatePass's
	// replay check refuses it — the mirror never masks a replayed feed.
	return m, nil
}

// updatePrefs is what an instance's admin chose (GET /api/update-prefs).
type updatePrefs struct {
	AutoUpdate   bool   `json:"auto_update"`
	AutoDownload bool   `json:"auto_download"`
	Channel      string `json:"channel"`
	// Pin, when set, freezes the instance on that exact version — the updater
	// skips it entirely (pin beats channel). How a canary declares itself.
	Pin string `json:"pin"`
	// Major, when set, keeps the instance on that major version: minors and
	// patches follow the channel, a head whose semver crosses the major is
	// held (majorHold, release.go).
	Major string `json:"major"`
}

// fetchUpdatePrefs reads an instance's prefs over its unix socket. A binary
// too old to have the endpoint 404s — that means the defaults (on/on/stable),
// which is exactly how a pre-feature fleet adopts auto-updating.
func fetchUpdatePrefs(socket string) (updatePrefs, error) {
	defaults := updatePrefs{AutoUpdate: true, AutoDownload: true, Channel: "stable"}
	client := unixHTTPClient(socket)
	resp, err := client.Get("http://lchat/api/update-prefs")
	if err != nil {
		return defaults, err
	}
	defer resp.Body.Close()
	if resp.StatusCode == http.StatusNotFound {
		return defaults, nil
	}
	if resp.StatusCode != http.StatusOK {
		return defaults, fmt.Errorf("update-prefs: %s", resp.Status)
	}
	var p updatePrefs
	if err := json.NewDecoder(io.LimitReader(resp.Body, 1<<16)).Decode(&p); err != nil {
		return defaults, err
	}
	if !validChannel(p.Channel) {
		p.Channel = "stable"
	}
	if !validVersion(p.Pin) {
		p.Pin = "" // a garbled pin must not freeze anything silently
	}
	if !validMajor(p.Major) {
		p.Major = "" // same rule as the pin
	}
	return p, nil
}

// hibernatingPrefs reads an instance's update prefs straight from its parked
// DB — an activator-managed host can't answer the socket fetch without being
// woken, and waking the fleet hourly would defeat hibernation. Read-only, so
// it can't disturb an awake child's WAL. FAIL-CLOSED: an absent DB (never
// woken here; state parked only in S3) or any read trouble returns an error
// and the caller skips the host — the admin's pin or updates-off may live
// only in the unreadable copy, and "the one host that must not move, moves"
// is the failure that matters. A skipped host converges on the pass after
// its first wake leaves a warm local DB.
func hibernatingPrefs(host string) (updatePrefs, error) {
	p := updatePrefs{AutoUpdate: true, AutoDownload: true, Channel: "stable"}
	path := filepath.Join(instanceDataDir, host+".db")
	if _, err := os.Stat(path); err != nil {
		return p, err
	}
	// Even a mode=ro open CREATES <db>-wal/-shm when they're absent (a cleanly
	// slept instance has neither) — and this runs as root, so without repair
	// they'd be root-owned and the instance (service user) would EACCES on its
	// next wake: space down until a manual chown. Fix ownership on the way out.
	defer repairDBSidecarOwnership(path)
	conn, err := sql.Open("sqlite", "file:"+path+"?mode=ro&_pragma=busy_timeout(3000)")
	if err != nil {
		return p, err
	}
	defer conn.Close()
	conn.SetMaxOpenConns(1)
	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
	defer cancel()
	get := func(key string) (string, bool, error) {
		var v string
		err := conn.QueryRowContext(ctx, "SELECT value FROM config WHERE key = ?", key).Scan(&v)
		if err == sql.ErrNoRows {
			return "", false, nil
		}
		if err != nil {
			return "", false, err
		}
		return v, true, nil
	}
	// Same semantics as the instance's own readers: booleans default ON when
	// absent, an unrecognised channel falls back to stable, a garbled pin
	// must not freeze anything silently.
	if v, ok, err := get("update_auto"); err != nil {
		return p, err
	} else if ok {
		p.AutoUpdate = v == "1"
	}
	if v, ok, err := get("update_download"); err != nil {
		return p, err
	} else if ok {
		p.AutoDownload = v == "1"
	}
	if v, ok, err := get("update_channel"); err != nil {
		return p, err
	} else if ok && validChannel(v) {
		p.Channel = v
	}
	if v, ok, err := get("update_pin"); err != nil {
		return p, err
	} else if ok && validVersion(v) {
		p.Pin = v
	}
	if v, ok, err := get("update_major"); err != nil {
		return p, err
	} else if ok && validMajor(v) {
		p.Major = v
	}
	return p, nil
}

// repairDBSidecarOwnership chowns <db>-wal/-shm to the DB file's own uid/gid.
// A root-run read-only open of a checkpointed WAL DB creates those two files
// root-owned (mode=ro guards the DB's content, not the WAL control files),
// and the instance — running as the service user — then can't open its own
// DB: the add-bot root-owned-dir incident as a library. Best-effort: non-root
// callers can't chown and can't create root-owned files either.
func repairDBSidecarOwnership(dbPath string) {
	st, err := os.Stat(dbPath)
	if err != nil {
		return
	}
	sys, ok := st.Sys().(*syscall.Stat_t)
	if !ok {
		return
	}
	for _, p := range []string{dbPath + "-wal", dbPath + "-shm"} {
		if _, err := os.Stat(p); err == nil {
			_ = os.Chown(p, int(sys.Uid), int(sys.Gid))
		}
	}
}

// updaterArch is the architecture this box runs — a var so tests can exercise
// the arm64 resolution without faking runtime.GOARCH.
var updaterArch = runtime.GOARCH

// downloadBinaryToTemp streams one URL into a temp file in the incoming dir
// and refuses bytes whose sha256 doesn't match the signed manifest entry.
func downloadBinaryToTemp(url, sha, ver string) (string, error) {
	if err := os.MkdirAll(incomingDir(), 0o755); err != nil {
		return "", err
	}
	f, err := os.CreateTemp(incomingDir(), "dl-")
	if err != nil {
		return "", err
	}
	defer f.Close()
	fail := func(err error) (string, error) {
		os.Remove(f.Name())
		return "", err
	}
	client := &http.Client{Timeout: 10 * time.Minute}
	resp, err := client.Get(url)
	if err != nil {
		return fail(err)
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		return fail(fmt.Errorf("download %s: %s", ver, resp.Status))
	}
	h := sha256.New()
	if _, err := io.Copy(io.MultiWriter(f, h), resp.Body); err != nil {
		return fail(err)
	}
	if got := hex.EncodeToString(h.Sum(nil)); got != sha {
		return fail(fmt.Errorf("download %s: sha256 mismatch (refusing to stage)", ver))
	}
	return f.Name(), nil
}

// downloadReleaseBinary fetches this box's arch of a release into the incoming
// dir and refuses anything whose sha256 doesn't match the signed manifest
// entry. A failed primary download (GitHub 404s brand-new assets for minutes
// after a publish) falls back to the S3 mirror, loudly; the sha256 gate is
// identical on both paths, so a mirror serving the wrong bytes only ever
// causes a refusal. Returns the temp path; the caller stages it and removes
// the temp.
func downloadReleaseBinary(e releaseEntry) (string, error) {
	url, sha, ok := e.forArch(updaterArch)
	if !ok {
		// An arm64 box facing a pre-arm64 release: skip, never install the
		// wrong arch. The next release publishes both and the box catches up.
		return "", fmt.Errorf("release %s has no %s build — skipping", e.Version, updaterArch)
	}
	path, err := downloadBinaryToTemp(url, sha, e.Version)
	if err == nil {
		return path, nil
	}
	murl := ""
	if releaseMirrorURL != "" {
		murl = mirrorBinaryURL(releaseMirrorURL, url)
	}
	if murl == "" {
		return "", err
	}
	log.Printf("update: primary download of %s failed (%v) — trying mirror %s", e.Version, err, murl)
	path, merr := downloadBinaryToTemp(murl, sha, e.Version)
	if merr != nil {
		return "", fmt.Errorf("download %s: primary failed (%v); mirror failed too (%v)", e.Version, err, merr)
	}
	return path, nil
}

// ensureVersionStaged makes a manifest entry's binary available in the version
// store, downloading it if needed. Idempotent across hosts in one pass.
func ensureVersionStaged(e releaseEntry) error {
	if _, err := os.Stat(versionBinPath(e.Version)); err == nil {
		return nil
	}
	tmp, err := downloadReleaseBinary(e)
	if err != nil {
		return err
	}
	defer os.Remove(tmp)
	return stageVersion(tmp, e.Version)
}

// updatePass is one full check-and-apply sweep, split from runUpdateRun so
// tests drive it directly (die()-free: per-host problems log and continue —
// an unreachable instance must not strand the rest of the fleet unchecked).
// ignoreBake is the local, root-only override of the supply-chain bake window
// (bake.go) — deliberately never a publisher-side flag.
func updatePass(reg *Registry, unitTmpl string, dryRun, ignoreBake bool) error {
	m, err := fetchReleaseManifest(releaseManifestURL)
	if err != nil {
		// No feed is a normal state (the releases repo may not exist yet, or
		// the box is offline) — never an error exit the timer would flap on.
		fmt.Printf("update: no usable manifest (%v) — nothing to do\n", err)
		return nil
	}
	if last := readManifestStamp(); m.GeneratedAt.Before(last) {
		return fmt.Errorf("manifest generated_at %s is older than the last accepted %s — refusing a replayed feed",
			m.GeneratedAt.Format(time.RFC3339), last.Format(time.RFC3339))
	}
	if !dryRun {
		if err := writeManifestStamp(m.GeneratedAt); err != nil {
			return fmt.Errorf("record manifest stamp: %v", err)
		}
	}
	// Tick the bake clock: record every channel head this box hasn't seen at
	// that head before, so the supply-chain window is measured from the box's
	// own first sighting, never a feed timestamp (bake.go). Dry runs compute
	// eligibility but persist nothing.
	bake := loadFirstSeen()
	if bake.observe(m, time.Now()) && !dryRun {
		if err := bake.save(); err != nil {
			return fmt.Errorf("record bake clock: %v", err)
		}
	}

	routes, err := reg.list()
	if err != nil {
		return err
	}
	// A version that failed its health check on one host isn't tried on the
	// next this pass — one bad build shouldn't bounce every instance in turn.
	poisoned := map[string]bool{}
	// Versions instances are pinned to must survive GC even when no channel or
	// registry row mentions them (a canary's branch build).
	var pinnedVers []string
	// Hibernating hosts, deferred to a second phase (see the branch below).
	var hibernating []Route
	for _, rt := range routes {
		if rt.Kind != "instance" {
			continue
		}
		if rt.Status != "ready" {
			fmt.Printf("  %s: skipped (status %s)\n", rt.Host, rt.Status)
			continue
		}
		if rt.Hibernate {
			// Processed in a second phase below, AFTER every unit-managed host:
			// repoints have no health check, so let an awake host discover a
			// boot-broken build (and poison it) before any sleeper is moved.
			hibernating = append(hibernating, rt)
			continue
		}
		prefs, err := fetchUpdatePrefs(rt.Socket)
		if err != nil {
			fmt.Printf("  %s: unreachable (%v) — skipped\n", rt.Host, err)
			continue
		}
		if prefs.Pin != "" {
			// Pinned: the instance declared its own version — leave it alone
			// entirely (this is what keeps a canary from being stomped back to
			// the channel head an hour after it's deployed).
			pinnedVers = append(pinnedVers, prefs.Pin)
			fmt.Printf("  %s: pinned to %s — left alone\n", rt.Host, prefs.Pin)
			continue
		}
		e, ok := m.Channels[prefs.Channel]
		if !ok {
			fmt.Printf("  %s: channel %s has no release yet\n", rt.Host, prefs.Channel)
			continue
		}
		cur, _ := liveVersion(rt.Host)
		if cur == e.Version {
			continue // up to date — stay quiet on the hourly tick
		}
		if hold := majorHold(prefs.Major, e); hold != "" {
			fmt.Printf("  %s: %s\n", rt.Host, hold)
			continue
		}
		if hold := bakeHold(bake, prefs.Channel, e.Version, ignoreBake, time.Now()); hold != "" {
			fmt.Printf("  %s: %s (%s) is %s — apply now: lchat deploy %s --version %s\n",
				rt.Host, e.Version, prefs.Channel, hold, rt.Host, e.Version)
			continue
		}
		if !prefs.AutoDownload && !prefs.AutoUpdate {
			fmt.Printf("  %s: %s available on %s (downloads off — admin's call)\n", rt.Host, e.Version, prefs.Channel)
			continue
		}
		if dryRun {
			fmt.Printf("  %s: would update %s -> %s (%s)\n", rt.Host, cur, e.Version, prefs.Channel)
			continue
		}
		if err := ensureVersionStaged(e); err != nil {
			fmt.Printf("  %s: %v\n", rt.Host, err)
			continue
		}
		if !prefs.AutoUpdate {
			fmt.Printf("  %s: %s downloaded and staged (auto-update off — apply with `lchat deploy %s --version %s`)\n",
				rt.Host, e.Version, rt.Host, e.Version)
			continue
		}
		if poisoned[e.Version] {
			fmt.Printf("  %s: skipping %s (failed health check on another instance this pass)\n", rt.Host, e.Version)
			continue
		}
		rtCopy := rt
		changed, err := deployHostVersion(reg, &rtCopy, unitTmpl+rt.Host, e.Version)
		if err != nil {
			poisoned[e.Version] = true
			fmt.Printf("  %s: update to %s failed: %v\n", rt.Host, e.Version, err)
			continue
		}
		if changed {
			fmt.Printf("  %s: updated %s -> %s (%s)\n", rt.Host, cur, e.Version, prefs.Channel)
		}
	}

	// Second phase: hibernating hosts. An activator-managed host has no unit
	// to restart and must not be woken just to update (an hourly wake would
	// defeat hibernation). Instead: repoint its live symlink — the NEXT wake
	// execs the new binary, since the router's spawn resolves
	// /opt/lchat/live/<host> at exec time. Prefs come from the parked DB
	// (read-only, fail-closed). Two stated trades vs the unit path, both in
	// docs/server-releases.md: no health check (nothing runs to probe — a bad
	// build surfaces at wake; recover with `lchat rollback <host>`, which
	// repoints back), and a currently-AWAKE child keeps its old binary until
	// it next sleeps (a running process doesn't follow the symlink).
	for _, rt := range hibernating {
		prefs, err := hibernatingPrefs(rt.Host)
		if err != nil {
			// The admin's pin or opt-out may live only in the copy we couldn't
			// read — never move a host whose wishes are unknown. It converges
			// on the pass after its first wake leaves a warm local DB.
			fmt.Printf("  %s: hibernating, parked prefs unreadable (%v) — skipped\n", rt.Host, err)
			continue
		}
		if prefs.Pin != "" {
			pinnedVers = append(pinnedVers, prefs.Pin)
			fmt.Printf("  %s: pinned to %s — left alone\n", rt.Host, prefs.Pin)
			continue
		}
		e, ok := m.Channels[prefs.Channel]
		if !ok {
			fmt.Printf("  %s: channel %s has no release yet\n", rt.Host, prefs.Channel)
			continue
		}
		cur, _ := liveVersion(rt.Host)
		if cur == e.Version {
			continue // up to date — stay quiet on the hourly tick
		}
		if hold := majorHold(prefs.Major, e); hold != "" {
			fmt.Printf("  %s: %s\n", rt.Host, hold)
			continue
		}
		if hold := bakeHold(bake, prefs.Channel, e.Version, ignoreBake, time.Now()); hold != "" {
			fmt.Printf("  %s: %s (%s) is %s — apply now: lchat deploy %s --version %s\n",
				rt.Host, e.Version, prefs.Channel, hold, rt.Host, e.Version)
			continue
		}
		if !prefs.AutoDownload && !prefs.AutoUpdate {
			fmt.Printf("  %s: %s available on %s (downloads off — admin's call)\n", rt.Host, e.Version, prefs.Channel)
			continue
		}
		if dryRun {
			fmt.Printf("  %s: would repoint %s -> %s (%s; hibernating)\n", rt.Host, cur, e.Version, prefs.Channel)
			continue
		}
		if err := ensureVersionStaged(e); err != nil {
			fmt.Printf("  %s: %v\n", rt.Host, err)
			continue
		}
		if !prefs.AutoUpdate {
			fmt.Printf("  %s: %s downloaded and staged (auto-update off — apply with `lchat deploy %s --version %s`)\n",
				rt.Host, e.Version, rt.Host, e.Version)
			continue
		}
		if poisoned[e.Version] {
			fmt.Printf("  %s: skipping %s (failed health check on another instance this pass)\n", rt.Host, e.Version)
			continue
		}
		if err := pointLive(rt.Host, e.Version); err != nil {
			fmt.Printf("  %s: repoint failed: %v\n", rt.Host, err)
			continue
		}
		if err := reg.setVersions(rt.Host, e.Version, cur); err != nil {
			fmt.Printf("  %s: repointed but recording the version failed: %v\n", rt.Host, err)
			continue
		}
		fmt.Printf("  %s: repointed %s -> %s (%s; hibernating — runs it on next wake)\n", rt.Host, cur, e.Version, prefs.Channel)
	}

	if !dryRun {
		gcVersions(reg, m, pinnedVers)
	}
	// Last, the box's own trust plane: the control binary rides the same
	// signed feed (see controlSelfUpdate) — after instances, so a build that
	// poisons a health check this pass never also becomes the control plane.
	// A self-update failure is already logged loudly inside (it rolls itself
	// back); surface the summary here but never fail the whole pass on it —
	// the instances above already converged. Passing `poisoned` makes the
	// "a build that poisons a health check this pass never also becomes the
	// control plane" promise real: a head that just failed on an instance is
	// skipped for the control binary too.
	if err := controlSelfUpdate(m, bake, poisoned, dryRun, ignoreBake); err != nil {
		fmt.Printf("update: control self-update: %v\n", err)
	}
	// The sibling services (connect/push/home) converge from the same signed
	// entry, after the control binary — see updaterservices.go.
	serviceBinariesUpdate(m, bake, poisoned, dryRun, ignoreBake)
	return nil
}

// controlSelfUpdate keeps the CONTROL binary — /opt/lchat/lchat, the path
// every privileged operation execs through (router, controld, cloud, this
// very updater) — on its own release channel, from the same signed feed the
// instance pass just verified. Before the operator-allowlist hardening this
// was a manual root `cp`; with root SSH gone (docs/instance-router.md § Box
// access), the already-root update pass is the one place the swap can live —
// and it adds no trust the box didn't have: same signature, same sha, same
// bake window as every instance binary.
//
// LCHAT_CONTROL_CHANNEL (in the unit's /data/update.env) picks the channel;
// "stable" is the default — the trust plane should be the LAST thing on a
// box to take a new build — and "off" disables self-update entirely. The
// swap is an atomic same-filesystem rename over the existing path (running
// services keep their old inode until restarted; sudo never sees a missing
// path), with the outgoing build kept beside it as .prev-<build>.
//
// The swap is NOT committed blind: after bouncing lchat-control/lchat-cloud
// onto the new binary we confirm the control plane reports the new build. The
// strong gate is the control daemon's live /healthz (waitControlHealthy); when
// lchat-control isn't running to answer it, we fall back to the weaker but
// controld-independent `<self> version` integrity check (controlBinaryReports)
// rather than roll back a binary the probe simply couldn't reach. If neither
// gate passes we restore .prev and confirm the box recovered — a boot-broken
// control plane rolling itself back is the whole point of this function, since
// with root SSH gone an operator can't reach in to fix a bricked box by hand.
// lchat-control and lchat-cloud are the only units bounced (backend sockets,
// invisible to members); the ROUTER is deliberately NOT restarted here — its
// front door should never blip on a timer's schedule, and it has no /healthz to
// probe, so it runs the old build until an operator restarts it, noted below.
//
// A rolled-back head is QUARANTINED (writeFailedControlVersion, persisted under
// optDir) so the hourly pass doesn't reinstall + roll it back forever; the
// quarantine clears when a good build commits or the channel moves past it. A
// head that poisoned an instance's health check THIS pass is skipped up front.
//
// Returns an error only when something actually went wrong (a pre-swap failure,
// or a swap that was rolled back); every no-op path returns nil. updatePass
// logs the summary but never fails the pass on it — the instances already
// converged before this runs.
//
// RESIDUAL RISK (deliberately not addressed here): the swap → health-check
// window is not crash-atomic. A reboot or kill in that window can leave the
// loose /opt/lchat/lchat as the unproven NEW build with no process left to run
// the rollback — so nothing auto-recovers until a human renames .prev back.
// Closing it means a boot-time recovery (an ExecStartPre on lchat-router /
// lchat-control that verifies the loose binary and reverts to .prev on doubt);
// that's deferred because it touches the ingress start path, which we don't
// want a timer-adjacent change to destabilise.
func controlSelfUpdate(m *releaseManifest, bake *firstSeenState, poisoned map[string]bool, dryRun, ignoreBake bool) error {
	if buildVersion == "dev" {
		// An unstamped binary (local build, test binary) has no place in the
		// channel ordering — self-updating it would clobber whatever the
		// operator is hand-running. Only release-stamped builds converge.
		return nil
	}
	channel := envOr("LCHAT_CONTROL_CHANNEL", "stable")
	if channel == "off" {
		return nil
	}
	e, ok := m.Channels[channel]
	if !ok || e.Version == buildVersion {
		return nil
	}
	// The channel moved past a previously-quarantined head — forget the
	// quarantine and let the new head be tried.
	if failed := readFailedControlVersion(); failed != "" && failed != e.Version {
		clearFailedControlVersion()
	}
	// A head we already swapped in, health-checked, and rolled back stays
	// quarantined until a human pins/patches the channel or it moves on — never
	// reinstalled + rolled back every hour.
	if e.Version == readFailedControlVersion() {
		fmt.Printf("update: control build %s is quarantined (failed its health check; pin or patch the channel to move)\n", e.Version)
		return nil
	}
	// A build that failed a health check on an instance THIS pass must never
	// then become the control plane (updatePass runs us after the instances).
	if poisoned[e.Version] {
		fmt.Printf("update: skipping control update to %s — it failed a health check on an instance this pass\n", e.Version)
		return nil
	}
	if hold := bakeHold(bake, channel, e.Version, ignoreBake, time.Now()); hold != "" {
		fmt.Printf("update: control binary %s -> %s: %s\n", buildVersion, e.Version, hold)
		return nil
	}
	if dryRun {
		fmt.Printf("update: control binary would update %s -> %s (%s; dry run)\n", buildVersion, e.Version, channel)
		return nil
	}
	self, err := selfPath()
	if err != nil {
		return err
	}
	tmp, err := downloadReleaseBinary(e)
	if err != nil {
		return err
	}
	defer os.Remove(tmp)
	if err := os.Chmod(tmp, 0o755); err != nil {
		return fmt.Errorf("chmod: %v", err)
	}
	// Stage the verified build into the version store BEFORE swapping it live.
	// gcVersions keeps every channel head, so this protects the good target from
	// the pruner and makes it reachable to `lchat rollback`/re-deploy — the swap
	// alone would leave the new build only as the loose /opt/lchat/lchat, a file
	// the version store has never heard of.
	if err := stageVersion(tmp, e.Version); err != nil {
		return fmt.Errorf("stage %s: %v", e.Version, err)
	}
	// One .prev at a time: the previous previous build is already two behind
	// and reachable from the version store anyway.
	if old, _ := filepath.Glob(self + ".prev-*"); len(old) > 0 {
		for _, p := range old {
			os.Remove(p)
		}
	}
	// .prev is written HERE, from the still-running (known-good) binary, and only
	// ever just before a swap — so it always points at the last build that passed
	// its health check. A rollback below restores it and returns immediately,
	// never advancing .prev past an unproven build; the invariant is that the
	// loose /opt/lchat/lchat is always either health-confirmed or being actively
	// rolled back to one.
	prev := self + ".prev-" + buildVersion
	if err := copyFileAtomic(self, prev, 0o755); err != nil {
		return fmt.Errorf("keep rollback copy: %v", err)
	}
	// downloadReleaseBinary lands in incomingDir() — same filesystem as the
	// control binary, so this rename atomically replaces the path.
	if err := os.Rename(tmp, self); err != nil {
		return fmt.Errorf("install: %v", err)
	}
	// The binary is swapped; prove it comes up before walking away. Bounce the
	// backend units onto it, then confirm the control plane reports the new build.
	restartControlUnits()
	if herr := waitControlHealthy(e.Version, deployHealthTimeout); herr != nil {
		// The live probe failed — but it's only AUTHORITATIVE when lchat-control
		// is actually running to answer it. try-restart is a no-op on a box where
		// the unit is inactive (never enabled, or intentionally stopped), so a
		// failed probe there means "nobody answered", not "the binary is bad".
		if isActive(controlUnit) {
			return rollbackControl(self, prev, e.Version, herr) // authoritative failure
		}
		// controld isn't up to probe. Fall back to the weaker, controld-independent
		// gate: run `<self> version` and require it names the target build. That
		// proves the new binary loads and self-reports the right build/arch (a
		// corrupt or wrong-arch binary fails), NOT that the service boots — hence
		// "weaker", used only because the strong gate is unavailable. controld
		// starts fresh from this binary when next needed.
		if verr := controlBinaryReports(self, e.Version); verr != nil {
			return rollbackControl(self, prev, e.Version, fmt.Errorf("lchat-control inactive and the new binary failed its integrity check: %v", verr))
		}
		log.Printf("update: lchat-control inactive — committing control update to %s on the `version` integrity check (weaker gate; live probe unavailable)", e.Version)
		// fall through to commit
	}
	// Committed. Any stale quarantine no longer applies.
	clearFailedControlVersion()
	// F5: cloud rode the same swap+restart but isn't the health gate (the router
	// and chat are unaffected by it) — so don't roll back on it, just flag it.
	if !isActive(cloudUnit) {
		log.Printf("update: WARN lchat-cloud not active after control update to %s — check the provisioner", e.Version)
	}
	fmt.Printf("update: control binary %s -> %s (%s; healthy; rollback at %s)\n", buildVersion, e.Version, channel, prev)
	fmt.Printf("update: lchat-router still runs %s — restart it when convenient to converge\n", buildVersion)
	return nil
}

// rollbackControl restores the last-known-good control binary (.prev) after a
// swapped-in build failed its health check, confirms the box came back, and
// quarantines the bad build so the hourly pass doesn't reinstall + roll it back
// forever. It returns a non-nil error describing the rollback — or a screaming,
// non-looping CONTROL PLANE BRICKED error if recovery itself fails.
func rollbackControl(self, prev, newVer string, healthErr error) error {
	log.Printf("update: control binary %s -> %s is UNHEALTHY (%v) — rolling back to %s", buildVersion, newVer, healthErr, buildVersion)
	if rberr := os.Rename(prev, self); rberr != nil {
		return fmt.Errorf("CONTROL PLANE BRICKED: %s failed its health check AND restoring %s failed: %v (health error: %v)", newVer, prev, rberr, healthErr)
	}
	restartControlUnits()
	// Confirm the restored binary with the same strong/weak gate: the live probe
	// when controld runs, else the `version` integrity check.
	var recerr error
	if isActive(controlUnit) {
		recerr = waitControlHealthy(buildVersion, deployHealthTimeout)
	} else {
		recerr = controlBinaryReports(self, buildVersion)
	}
	if recerr != nil {
		// Genuine brick: neither the new build nor the restored old one is
		// healthy. Scream, but do NOT loop — a human has to look now.
		log.Printf("update: CONTROL PLANE BRICKED — rolled back to %s but IT is unhealthy too (%v); box needs a human NOW", buildVersion, recerr)
		return fmt.Errorf("CONTROL PLANE BRICKED: %s unhealthy, rolled back to %s but it is unhealthy too (%v) — box needs a human NOW (original health error: %v)", newVer, buildVersion, recerr, healthErr)
	}
	// Persist the quarantine so the next hourly pass short-circuits this head
	// instead of reinstalling + rolling it back again.
	if err := writeFailedControlVersion(newVer); err != nil {
		log.Printf("update: warning: couldn't persist control quarantine for %s: %v", newVer, err)
	}
	log.Printf("update: rolled back to %s and it is healthy again — %s is quarantined until a good build ships or you pin/patch the channel", buildVersion, newVer)
	return fmt.Errorf("control binary %s failed its health check and was rolled back to %s: %v", newVer, buildVersion, healthErr)
}

// controlUnit / cloudUnit / linksUnit are the systemd units that exec the
// control binary (/opt/lchat/lchat) directly, rather than a per-instance
// versioned live symlink — so they keep running the OLD inode until something
// restarts them.
const (
	controlUnit = "lchat-control.service"
	cloudUnit   = "lchat-cloud.service"
	linksUnit   = "lchat-links.service"
)

// controlBinaryUnits is the one list of them, so the updater and the tests that
// assert "one restart each, no rollback round" can't disagree about how many
// that is. Adding a unit here is all it takes to put it on the update path.
var controlBinaryUnits = []string{controlUnit, cloudUnit, linksUnit}

// restartControlUnits bounces the backend units that exec the control binary —
// lchat-control, lchat-cloud, and lchat-links — onto whatever /opt/lchat/lchat
// now is. The ROUTER is deliberately excluded: it's the fleet's only ingress,
// has no /healthz to probe, and must never blip on the updater's schedule (it
// keeps its old inode until an operator restarts it). Restart failures only
// log — the health probe that follows is the real gate on whether the new
// build is up.
//
// links belongs here for the same reason cloud does, and is the cheapest of
// the three to bounce: it holds no database and no connections, just an
// in-memory verification cache that re-populates lazily. Without it the
// redirector would be the one service that never picks up a new build at all —
// try-restart is a no-op on a box where it isn't running, so boxes that route
// no short links are unaffected.
func restartControlUnits() {
	for _, unit := range controlBinaryUnits {
		if err := systemctl("try-restart", unit); err != nil {
			fmt.Printf("update: %s restart: %v (new build applies on its next start)\n", unit, err)
		}
	}
}

// isActive reports whether a systemd unit is currently active (`systemctl
// is-active --quiet` exits 0 only when active). A var so tests can fake the
// box's unit state — it decides whether the control /healthz probe is even
// answerable, so a failed probe on a box that simply isn't running controld
// doesn't roll back a perfectly good binary.
var isActive = func(unit string) bool {
	return systemctlQuery("is-active", "--quiet", unit) == nil
}

// controlBinaryReports is the controld-independent integrity gate: run
// `<path> version` and require its output names wantBuild. It proves the binary
// execs and self-reports the expected build/arch, which is weaker than a live
// /healthz (it does NOT prove the service boots) but is all we have when
// lchat-control isn't running to answer a probe. A var so tests can fake the
// exec without a real binary on disk.
var controlBinaryReports = func(path, wantBuild string) error {
	got, err := readBinaryVersion(path)
	if err != nil {
		return fmt.Errorf("run %s version: %v", path, err)
	}
	if !strings.Contains(got, wantBuild) {
		return fmt.Errorf("%s reported build %q, want %q", path, got, wantBuild)
	}
	return nil
}

// controlFailedVersionPath persists the last control head that failed its health
// check and was rolled back (mirrors manifestStampPath). It lives under optDir
// so it survives the updater process and is read on the next hourly pass.
func controlFailedVersionPath() string { return filepath.Join(optDir, ".control-failed-version") }

// readFailedControlVersion returns the quarantined control build, or "" if none.
func readFailedControlVersion() string {
	raw, err := os.ReadFile(controlFailedVersionPath())
	if err != nil {
		return ""
	}
	return strings.TrimSpace(string(raw))
}

// writeFailedControlVersion records ver as quarantined.
func writeFailedControlVersion(ver string) error {
	return os.WriteFile(controlFailedVersionPath(), []byte(ver+"\n"), 0o644)
}

// clearFailedControlVersion lifts the quarantine (a good build committed, or the
// channel moved on). Absent file is fine — clearing "nothing" is a no-op.
func clearFailedControlVersion() {
	if err := os.Remove(controlFailedVersionPath()); err != nil && !os.IsNotExist(err) {
		log.Printf("update: warning: couldn't clear control quarantine file: %v", err)
	}
}

// waitControlHealthy polls the control daemon's /healthz until it reports
// wantBuild, or timeout elapses — the gate on the box's self-update. After the
// control binary is swapped and lchat-control restarted, this confirms the NEW
// build came up (wantBuild = the new version); on the rollback path it confirms
// the OLD build came back (wantBuild = buildVersion, this updater process's own
// still-running id).
//
// lchat-control runs Restart=always (RestartSec=1), so a crash-looping build
// flaps between activating and active: the socket comes and goes and dials fail
// with "connection refused" in between. A transient refusal is therefore
// neither success nor failure — keep retrying across it until either the build
// matches (healthy, return early) or the whole timeout expires (unhealthy). It
// mirrors waitHealthyVersion, but against the control socket and /healthz.
func waitControlHealthy(wantBuild string, timeout time.Duration) error {
	deadline := time.Now().Add(timeout)
	client := unixHTTPClient(controlSocketPath)
	var last error
	for time.Now().Before(deadline) {
		resp, err := client.Get("http://lchat/healthz")
		if err == nil {
			var body struct {
				Build string `json:"build"`
			}
			derr := json.NewDecoder(io.LimitReader(resp.Body, 1<<16)).Decode(&body)
			resp.Body.Close()
			if derr == nil && body.Build == wantBuild {
				return nil
			}
			if derr != nil {
				last = fmt.Errorf("read /healthz: %v", derr)
			} else {
				last = fmt.Errorf("control build %q != expected %q", body.Build, wantBuild)
			}
		} else {
			last = err // often "connection refused" mid-restart — retried, not fatal
		}
		time.Sleep(150 * time.Millisecond)
	}
	return fmt.Errorf("control daemon didn't report build %q within %s: %v", wantBuild, timeout, last)
}

// gcVersions prunes version-store dirs nothing references anymore: kept are
// every route's current + previous version, every channel head, the control
// binary's own build, every pinned version (keepExtra), and whatever any live
// symlink points at (belt and braces — the symlinks are what units actually
// exec). Without this, hourly edge updates would grow /opt/lchat/versions
// without bound on a box that enforces a free-disk floor. Leftover crashed
// downloads are swept too.
func gcVersions(reg *Registry, m *releaseManifest, keepExtra []string) {
	keep := map[string]bool{buildVersion: true}
	for _, v := range keepExtra {
		keep[v] = true
	}
	routes, err := reg.list()
	if err != nil {
		return
	}
	for _, rt := range routes {
		keep[rt.Version], keep[rt.PrevVersion] = true, true
	}
	if m != nil {
		for _, e := range m.Channels {
			keep[e.Version] = true
		}
	}
	if links, err := os.ReadDir(liveDir()); err == nil {
		for _, l := range links {
			if v, err := liveVersion(l.Name()); err == nil {
				keep[v] = true
			}
		}
	}
	entries, err := os.ReadDir(versionsDir())
	if err != nil {
		return
	}
	for _, en := range entries {
		if !en.IsDir() || keep[en.Name()] {
			continue
		}
		if err := os.RemoveAll(filepath.Join(versionsDir(), en.Name())); err == nil {
			fmt.Printf("  pruned unused version %s\n", en.Name())
		}
	}
	// Our downloads are temp-then-removed; a surviving dl-* file is a crashed
	// pass. (Operator-scp'd files in incoming/ are left alone.)
	if files, err := os.ReadDir(incomingDir()); err == nil {
		for _, f := range files {
			if strings.HasPrefix(f.Name(), "dl-") {
				_ = os.Remove(filepath.Join(incomingDir(), f.Name()))
			}
		}
	}
}

// runUpdate: `lchat update run|status|pin|unpin|major` — the auto-updater CLI.
func runUpdate(args []string) {
	if len(args) == 0 {
		die("update: subcommand required: run | status | pin | unpin | major")
	}
	switch args[0] {
	case "run":
		runUpdateRun(args[1:])
	case "status":
		runUpdateStatus(args[1:])
	case "pin":
		runUpdatePin(args[1:], true)
	case "unpin":
		runUpdatePin(args[1:], false)
	case "major":
		runUpdateMajor(args[1:])
	default:
		die("update: unknown subcommand %q (run | status | pin | unpin | major)", args[0])
	}
}

// pinHost writes update_pin into the instance's own settings via its admin API
// over the unix socket — the pin is instance state (the same value the admin
// panel shows and /api/update-prefs reports), not a registry side-channel. The
// admin token comes from the instance's EnvironmentFile, like `admin-link`.
// The write is read back before reporting success: a pin the operator
// believes is set but isn't is exactly how the next hourly pass stomps a
// canary.
func pinHost(rt *Route, pin string) error {
	if err := writeHostSetting(rt, "update_pin", pin); err != nil {
		return err
	}
	return confirmHostSetting(rt, "pin", pin, func(p updatePrefs) string { return p.Pin })
}

// confirmHostSetting re-reads /api/update-prefs after a settings write and
// errors when the value didn't stick. An instance binary that predates a
// setting ACCEPTS the POST and silently ignores the unknown field — found
// live: `update major` against a pre-0.11.0 build reported success while
// storing nothing.
func confirmHostSetting(rt *Route, what, want string, read func(updatePrefs) string) error {
	p, err := fetchUpdatePrefs(rt.Socket)
	if err != nil {
		return fmt.Errorf("wrote the %s but couldn't read it back (%v) — verify with `lchat update status`", what, err)
	}
	if got := read(p); got != want {
		return fmt.Errorf("%s accepted the %s write but reports %q — its binary likely predates the setting; update the instance first, then retry", rt.Host, what, got)
	}
	return nil
}

// writeHostSetting posts one settings key into the instance's own admin API —
// the shared plumbing behind `update pin`/`update major`.
func writeHostSetting(rt *Route, key, value string) error {
	env, err := readEnvFile(filepath.Join(instanceDataDir, rt.Host+".env"))
	if err != nil {
		return err
	}
	tok := env["LCHAT_ADMIN_TOKEN"]
	if tok == "" {
		return fmt.Errorf("no LCHAT_ADMIN_TOKEN in %s/%s.env", instanceDataDir, rt.Host)
	}
	body, err := json.Marshal(map[string]string{key: value})
	if err != nil {
		return err
	}
	req, err := http.NewRequest("POST", "http://lchat/api/admin/settings", bytes.NewReader(body))
	if err != nil {
		return err
	}
	req.Header.Set("Authorization", "Bearer "+tok)
	resp, err := unixHTTPClient(rt.Socket).Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		return fmt.Errorf("settings write: %s", resp.Status)
	}
	return nil
}

// runUpdatePin: `lchat update pin <host> [<ver>]` freezes an instance on a
// version (default: whatever it's running now); `unpin` resumes its channel.
func runUpdatePin(args []string, pin bool) {
	cmd := "unpin"
	if pin {
		cmd = "pin"
	}
	fs := flag.NewFlagSet("update "+cmd, flag.ExitOnError)
	registryPath := fs.String("registry", envOr("LCHAT_REGISTRY", "/data/registry.db"), "router registry SQLite path")
	_ = fs.Parse(reorderFlagsFirst(fs, args))

	host := strings.ToLower(strings.TrimSpace(fs.Arg(0)))
	if !validHost(host) {
		die("update %s: invalid host %q", cmd, host)
	}
	reg, err := openRegistry(*registryPath)
	if err != nil {
		die("update %s: open registry: %v", cmd, err)
	}
	rt, err := reg.get(host)
	if err != nil || rt == nil || rt.Kind != "instance" {
		die("update %s: %s is not a managed instance", cmd, host)
	}

	ver := ""
	if pin {
		ver = strings.TrimSpace(fs.Arg(1))
		if ver == "" {
			// Default to what the instance is running — "freeze me here".
			if ver, _ = liveVersion(host); ver == "" {
				die("update pin: %s has no live version — pass one explicitly", host)
			}
		}
		if !validVersion(ver) {
			die("update pin: invalid version %q", ver)
		}
	}
	if err := pinHost(rt, ver); err != nil {
		die("update %s %s: %v", cmd, host, err)
	}
	if pin {
		fmt.Printf("pinned %s to %s (auto-updates paused; `lchat update unpin %s` or pick a channel in its admin panel to resume)\n", host, ver, host)
	} else {
		fmt.Printf("unpinned %s (back on its channel from the next update pass)\n", host)
	}
}

// runUpdateMajor: `lchat update major <host> <n>|clear` keeps an instance on
// one major version — minors/patches keep flowing, a major bump holds until
// the admin raises or clears it. Unlike `pin` there is no default: staying on
// a major is a statement about the future, so the operator says which one.
func runUpdateMajor(args []string) {
	fs := flag.NewFlagSet("update major", flag.ExitOnError)
	registryPath := fs.String("registry", envOr("LCHAT_REGISTRY", "/data/registry.db"), "router registry SQLite path")
	_ = fs.Parse(reorderFlagsFirst(fs, args))

	host := strings.ToLower(strings.TrimSpace(fs.Arg(0)))
	if !validHost(host) {
		die("update major: invalid host %q", host)
	}
	major := strings.TrimSpace(fs.Arg(1))
	if major == "clear" {
		major = ""
	} else if major == "" || !validMajor(major) {
		die("update major: pass a major version number, or `clear` to follow everything")
	}
	reg, err := openRegistry(*registryPath)
	if err != nil {
		die("update major: open registry: %v", err)
	}
	rt, err := reg.get(host)
	if err != nil || rt == nil || rt.Kind != "instance" {
		die("update major: %s is not a managed instance", host)
	}
	if err := writeHostSetting(rt, "update_major", major); err != nil {
		die("update major %s: %v", host, err)
	}
	if err := confirmHostSetting(rt, "major pin", major, func(p updatePrefs) string { return p.Major }); err != nil {
		die("update major: %v", err)
	}
	if major == "" {
		fmt.Printf("cleared %s's major pin (follows its channel fully from the next update pass)\n", host)
	} else {
		fmt.Printf("%s stays on major %s (minors and patches keep flowing; a release crossing major %s is held until this is raised or cleared)\n", host, major, major)
	}
}

func runUpdateRun(args []string) {
	fs := flag.NewFlagSet("update run", flag.ExitOnError)
	registryPath := fs.String("registry", envOr("LCHAT_REGISTRY", "/data/registry.db"), "router registry SQLite path")
	unitTmpl := fs.String("unit", envOr("LCHAT_UNIT", "lchat@"), "systemd instance unit template prefix")
	dryRun := fs.Bool("dry-run", false, "report what would change without staging or restarting anything")
	ignoreBake := fs.Bool("ignore-bake", false, "install channel heads still inside their supply-chain bake window (the deliberate local override)")
	_ = fs.Parse(reorderFlagsFirst(fs, args))

	unlock, err := acquireDeployLock()
	if err != nil {
		die("update: %v", err)
	}
	defer unlock()

	reg, err := openRegistry(*registryPath)
	if err != nil {
		die("update: open registry: %v", err)
	}
	if err := updatePass(reg, *unitTmpl, *dryRun, *ignoreBake); err != nil {
		die("update: %v", err)
	}
}

// statusPrefs resolves the prefs a status row shows: the live socket when the
// host answers; a hibernating host's parked DB (the same fail-closed read the
// update pass trusts) when it doesn't. The error keeps the caller's honest
// "?" for a host unreadable either way — status must never guess what an
// admin chose.
func statusPrefs(rt Route) (updatePrefs, error) {
	p, err := fetchUpdatePrefs(rt.Socket)
	if err != nil && rt.Hibernate {
		return hibernatingPrefs(rt.Host)
	}
	return p, err
}

// runUpdateStatus prints where every instance stands relative to its channel.
func runUpdateStatus(args []string) {
	fs := flag.NewFlagSet("update status", flag.ExitOnError)
	registryPath := fs.String("registry", envOr("LCHAT_REGISTRY", "/data/registry.db"), "router registry SQLite path")
	probe := fs.Bool("probe", false, "also read each instance's socket /api/version — the build it's ACTUALLY serving, which an awake hibernating instance can lag behind its symlink (docs/awake-instance-convergence.md)")
	_ = fs.Parse(reorderFlagsFirst(fs, args))

	reg, err := openRegistry(*registryPath)
	if err != nil {
		die("update: open registry: %v", err)
	}
	routes, err := reg.list()
	if err != nil {
		die("update: %v", err)
	}
	m, merr := fetchReleaseManifest(releaseManifestURL)
	bake := loadFirstSeen()

	if *probe {
		fmt.Printf("%-34s %-8s %-8s %-12s %-12s %s\n", "HOST", "CHANNEL", "AUTO", "SYMLINK", "SERVING", "AVAILABLE")
	} else {
		fmt.Printf("%-34s %-8s %-8s %-12s %s\n", "HOST", "CHANNEL", "AUTO", "CURRENT", "AVAILABLE")
	}
	staleServing := false
	for _, rt := range routes {
		if rt.Kind != "instance" {
			continue
		}
		channel, auto, major := "?", "?", ""
		if prefs, err := statusPrefs(rt); err == nil {
			channel, major = prefs.Channel, prefs.Major
			switch {
			case prefs.Pin != "":
				auto = "pinned" // pin beats channel; the updater skips this host
			case prefs.AutoDownload && prefs.AutoUpdate:
				auto = "dl+upd"
			case prefs.AutoDownload:
				auto = "dl"
			case prefs.AutoUpdate:
				auto = "upd"
			default:
				auto = "off"
			}
		}
		cur, _ := liveVersion(rt.Host)
		if cur == "" {
			cur = "-"
		}
		avail := "-"
		if m != nil {
			if e, ok := m.Channels[channel]; ok {
				avail = e.Version
				switch {
				case e.Version == cur:
					avail += " (current)"
				case majorHold(major, e) != "":
					avail += " (held by major pin " + major + ")"
				case bakeDelays[channel] > 0:
					// Status reads the bake clock without ticking it — only an
					// update pass observes. An unticked head shows as held too.
					if at, ok := bake.bakeEligibleAt(channel, e.Version); !ok {
						avail += " (held — bake starts on the next update pass)"
					} else if time.Now().Before(at) {
						avail += " (held until " + at.UTC().Format("2006-01-02 15:04 MST") + ")"
					}
				}
			}
		}
		if *probe {
			// The build the process is actually serving (vs. the symlink `cur`).
			serving, ok := probeVersion(rt.Socket)
			if !ok {
				serving = "asleep" // socket didn't answer — hibernating, converges on wake
			} else if serving != cur {
				serving += " ⚠" // awake but stale: symlink moved, process didn't
				staleServing = true
			}
			fmt.Printf("%-34s %-8s %-8s %-12s %-12s %s\n", rt.Host, channel, auto, cur, serving, avail)
		} else {
			fmt.Printf("%-34s %-8s %-8s %-12s %s\n", rt.Host, channel, auto, cur, avail)
		}
	}
	if staleServing {
		fmt.Println("\n⚠ = awake but serving an older build than its symlink names (a running process doesn't follow the")
		fmt.Println("    symlink). Converges on next sleep/wake, or cycle now with: lchat hibernate <host> --off && … --on")
		fmt.Println("    (docs/awake-instance-convergence.md).")
	}
	if merr != nil {
		fmt.Printf("(no manifest: %v)\n", merr)
	}
}