package main
import (
"crypto/ed25519"
"crypto/rand"
"crypto/sha256"
"encoding/base64"
"encoding/hex"
"encoding/json"
"net"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"sync"
"testing"
"time"
)
// startFakeInstance serves /api/update-prefs + /api/version on a real unix
// socket. Its reported build follows the host's live symlink — so pointLive +
// a stubbed systemctl restart looks exactly like a real swap to
// waitHealthyVersion. prefsCode 404 simulates a pre-feature binary; brokenVer
// (if set) makes that version report a wrong build, forcing the rollback path.
func startFakeInstance(t *testing.T, host string, prefsCode int, prefs updatePrefs, brokenVer string) string {
t.Helper()
// Short base dir: unix socket paths have a ~104-char limit on darwin.
dir, err := os.MkdirTemp("", "lch")
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { os.RemoveAll(dir) })
socket := filepath.Join(dir, host+".sock")
mux := http.NewServeMux()
mux.HandleFunc("/api/update-prefs", func(w http.ResponseWriter, r *http.Request) {
if prefsCode != http.StatusOK {
http.Error(w, "no such endpoint", prefsCode)
return
}
writeJSON(w, http.StatusOK, map[string]any{
"auto_update": prefs.AutoUpdate, "auto_download": prefs.AutoDownload, "channel": prefs.Channel, "pin": prefs.Pin, "major": prefs.Major,
})
})
mux.HandleFunc("/api/version", func(w http.ResponseWriter, r *http.Request) {
v, _ := liveVersion(host)
if brokenVer != "" && v == brokenVer {
v = "borked"
}
writeJSON(w, http.StatusOK, map[string]any{"build": v})
})
ln, err := net.Listen("unix", socket)
if err != nil {
t.Fatal(err)
}
srv := &http.Server{Handler: mux}
go srv.Serve(ln)
t.Cleanup(func() { srv.Close() })
return socket
}
// feedSemvers, when a test seeds it, stamps a semver onto matching versions
// in every manifest startFeed signs (tests run serially, so a package-level
// map is safe; entries are cleared via t.Cleanup in the tests that set them).
var feedSemvers = map[string]string{}
// startFeed signs entries into a manifest and serves it (plus each version's
// binary bytes) from a loopback server, swapping in a test trust root.
func startFeed(t *testing.T, binaries map[string]string) (*httptest.Server, func(channels map[string]string)) {
t.Helper()
pub, priv, err := ed25519.GenerateKey(rand.Reader)
if err != nil {
t.Fatal(err)
}
var mu sync.Mutex
var wrapper []byte
mux := http.NewServeMux()
mux.HandleFunc("/manifest.json", func(w http.ResponseWriter, r *http.Request) {
mu.Lock()
defer mu.Unlock()
w.Write(wrapper)
})
mux.HandleFunc("/bin/", func(w http.ResponseWriter, r *http.Request) {
ver := strings.TrimPrefix(r.URL.Path, "/bin/")
w.Write([]byte(binaries[ver]))
})
// Each version's arm64 build is distinct bytes, so a test can prove the
// right arch was staged.
mux.HandleFunc("/arm/", func(w http.ResponseWriter, r *http.Request) {
ver := strings.TrimPrefix(r.URL.Path, "/arm/")
w.Write([]byte(binaries[ver] + "@arm64"))
})
ts := httptest.NewServer(mux)
t.Cleanup(ts.Close)
oldKey, oldURL := releasePubKeyB64, releaseManifestURL
releasePubKeyB64 = base64.StdEncoding.EncodeToString(pub)
releaseManifestURL = ts.URL + "/manifest.json"
t.Cleanup(func() { releasePubKeyB64, releaseManifestURL = oldKey, oldURL })
// Pin the box arch so feed tests are deterministic on any dev machine
// (an arm64 Mac would otherwise resolve the arm64 variant everywhere);
// arch-specific tests override AFTER calling startFeed.
prevArch := updaterArch
updaterArch = "amd64"
t.Cleanup(func() { updaterArch = prevArch })
setChannels := func(channels map[string]string) {
m := &releaseManifest{Channels: map[string]releaseEntry{}}
for ch, ver := range channels {
sum := sha256.Sum256([]byte(binaries[ver]))
armSum := sha256.Sum256([]byte(binaries[ver] + "@arm64"))
m.Channels[ch] = releaseEntry{
Version: ver,
URL: ts.URL + "/bin/" + ver,
SHA256: hex.EncodeToString(sum[:]),
ARM64URL: ts.URL + "/arm/" + ver,
ARM64SHA256: hex.EncodeToString(armSum[:]),
Semver: feedSemvers[ver],
}
}
w, err := signReleaseManifest(priv, m)
if err != nil {
t.Fatal(err)
}
mu.Lock()
wrapper = w
mu.Unlock()
}
return ts, setChannels
}
// signManifestAt builds a signed wrapper with a caller-controlled
// generated_at — signReleaseManifest always stamps now, but the mirror tests
// need feeds whose relative ages are exact.
func signManifestAt(t *testing.T, priv ed25519.PrivateKey, m *releaseManifest, at time.Time) []byte {
t.Helper()
m.Schema = 1
m.GeneratedAt = at.UTC().Truncate(time.Second)
payload, err := json.Marshal(m)
if err != nil {
t.Fatal(err)
}
b64 := base64.StdEncoding.EncodeToString(payload)
sig := ed25519.Sign(priv, []byte(b64))
w, err := json.Marshal(signedManifest{Payload: b64, Sig: base64.StdEncoding.EncodeToString(sig)})
if err != nil {
t.Fatal(err)
}
return w
}
// swapFeedGlobals points the trust root, primary URL, and mirror base at test
// servers, restoring them after.
func swapFeedGlobals(t *testing.T, pubB64, primaryURL, mirrorBase string) {
t.Helper()
oldKey, oldURL, oldMirror := releasePubKeyB64, releaseManifestURL, releaseMirrorURL
releasePubKeyB64, releaseManifestURL, releaseMirrorURL = pubB64, primaryURL, mirrorBase
t.Cleanup(func() { releasePubKeyB64, releaseManifestURL, releaseMirrorURL = oldKey, oldURL, oldMirror })
}
// stubSystemctl records unit restarts and leaves the "restart" to the fake
// instance's symlink-following /api/version.
func stubSystemctl(t *testing.T) *[]string {
t.Helper()
var calls []string
prev := systemctl
systemctl = func(args ...string) error {
calls = append(calls, strings.Join(args, " "))
return nil
}
t.Cleanup(func() { systemctl = prev })
return &calls
}
func seedInstance(t *testing.T, reg *Registry, host, socket, ver string) {
t.Helper()
src := filepath.Join(t.TempDir(), "old")
writeFakeBinary(t, src, "old-binary-"+ver)
if err := stageVersion(src, ver); err != nil {
t.Fatal(err)
}
if err := pointLive(host, ver); err != nil {
t.Fatal(err)
}
if err := reg.upsert(Route{Host: host, Kind: "instance", Status: "ready", Socket: socket, Version: ver}); err != nil {
t.Fatal(err)
}
}
// zeroBake disables the supply-chain bake window (bake.go) for tests that
// exercise the rest of the pipeline; bake behaviour has its own tests below.
func zeroBake(t *testing.T) {
t.Helper()
prev := bakeDelays
bakeDelays = map[string]time.Duration{"edge": 0, "beta": 0, "stable": 0}
t.Cleanup(func() { bakeDelays = prev })
}
func TestUpdatePassAppliesPerPrefs(t *testing.T) {
withTempOpt(t)
zeroBake(t)
calls := stubSystemctl(t)
shrink := deployHealthTimeout
deployHealthTimeout = 2 * time.Second
t.Cleanup(func() { deployHealthTimeout = shrink })
_, setChannels := startFeed(t, map[string]string{
"new1234": "new-binary-content",
"beta5555": "beta-binary-content",
})
setChannels(map[string]string{"stable": "new1234", "beta": "beta5555"})
reg, err := openRegistry(filepath.Join(t.TempDir(), "reg.db"))
if err != nil {
t.Fatal(err)
}
// h1: defaults-by-404 (old binary) — updates to stable.
// h2: auto-update off — downloaded + staged, never restarted.
// h3: everything off, beta channel — beta's binary must not even download.
// h4: explicit on/on/stable — updates.
s1 := startFakeInstance(t, "h1.example.com", http.StatusNotFound, updatePrefs{}, "")
s2 := startFakeInstance(t, "h2.example.com", http.StatusOK, updatePrefs{AutoUpdate: false, AutoDownload: true, Channel: "stable"}, "")
s3 := startFakeInstance(t, "h3.example.com", http.StatusOK, updatePrefs{AutoUpdate: false, AutoDownload: false, Channel: "beta"}, "")
s4 := startFakeInstance(t, "h4.example.com", http.StatusOK, updatePrefs{AutoUpdate: true, AutoDownload: true, Channel: "stable"}, "")
seedInstance(t, reg, "h1.example.com", s1, "old1111")
seedInstance(t, reg, "h2.example.com", s2, "old1111")
seedInstance(t, reg, "h3.example.com", s3, "old1111")
seedInstance(t, reg, "h4.example.com", s4, "old1111")
if err := updatePass(reg, "lchat@", false, false); err != nil {
t.Fatalf("updatePass: %v", err)
}
assertLive := func(host, want string) {
t.Helper()
if v, _ := liveVersion(host); v != want {
t.Errorf("%s live version = %q, want %q", host, v, want)
}
}
assertLive("h1.example.com", "new1234")
assertLive("h2.example.com", "old1111") // staged only
assertLive("h3.example.com", "old1111")
assertLive("h4.example.com", "new1234")
// h2's chosen build is staged and byte-exact; h3's channel never downloaded.
staged, err := os.ReadFile(versionBinPath("new1234"))
if err != nil || string(staged) != "new-binary-content" {
t.Fatalf("staged binary = %q, %v", staged, err)
}
if _, err := os.Stat(versionBinPath("beta5555")); !os.IsNotExist(err) {
t.Fatal("beta5555 was downloaded despite downloads off")
}
// Registry recorded the moves (and only the moves).
rt, _ := reg.get("h1.example.com")
if rt.Version != "new1234" || rt.PrevVersion != "old1111" {
t.Fatalf("h1 registry = %s/%s", rt.Version, rt.PrevVersion)
}
rt, _ = reg.get("h2.example.com")
if rt.Version != "old1111" {
t.Fatalf("h2 registry = %s", rt.Version)
}
// Two instances moved (h1, h4); each restarts the instance and then its
// sidecars so both track the new version (restartBotSidecar +
// restartHooksSidecar). Six calls, no more — h2 (download-only) and h3
// (downloads off) never restart.
got := strings.Join(*calls, "; ")
for _, want := range []string{
"restart lchat@h1.example.com", "try-restart lchat-bot@h1.example.com", "try-restart lchat-hooks@h1.example.com",
"restart lchat@h4.example.com", "try-restart lchat-bot@h4.example.com", "try-restart lchat-hooks@h4.example.com",
} {
if !strings.Contains(got, want) {
t.Fatalf("missing %q in systemctl calls = %v", want, *calls)
}
}
if len(*calls) != 6 {
t.Fatalf("systemctl calls = %v", *calls)
}
// The pass recorded the manifest stamp for anti-replay.
if readManifestStamp().IsZero() {
t.Fatal("manifest stamp not written")
}
}
func TestUpdatePassRollsBackAndPoisons(t *testing.T) {
withTempOpt(t)
zeroBake(t)
calls := stubSystemctl(t)
shrink := deployHealthTimeout
deployHealthTimeout = 500 * time.Millisecond
t.Cleanup(func() { deployHealthTimeout = shrink })
_, setChannels := startFeed(t, map[string]string{"bad9999": "does-not-boot"})
setChannels(map[string]string{"stable": "bad9999"})
reg, err := openRegistry(filepath.Join(t.TempDir(), "reg.db"))
if err != nil {
t.Fatal(err)
}
// Both instances report "borked" while on bad9999 → health check fails.
s1 := startFakeInstance(t, "h1.example.com", http.StatusNotFound, updatePrefs{}, "bad9999")
s2 := startFakeInstance(t, "h2.example.com", http.StatusNotFound, updatePrefs{}, "bad9999")
seedInstance(t, reg, "h1.example.com", s1, "old1111")
seedInstance(t, reg, "h2.example.com", s2, "old1111")
if err := updatePass(reg, "lchat@", false, false); err != nil {
t.Fatalf("updatePass: %v", err)
}
// h1 tried, failed, rolled back; h2 was spared (the version is poisoned).
for _, host := range []string{"h1.example.com", "h2.example.com"} {
if v, _ := liveVersion(host); v != "old1111" {
t.Errorf("%s live version = %q, want rollback to old1111", host, v)
}
rt, _ := reg.get(host)
if rt.Version != "old1111" {
t.Errorf("%s registry version = %q", host, rt.Version)
}
}
// h1: restart onto bad + rollback restart = 2 calls; h2: none.
if len(*calls) != 2 || strings.Contains(strings.Join(*calls, ";"), "h2.example.com") {
t.Fatalf("systemctl calls = %v", *calls)
}
}
func TestUpdatePassRefusesReplayedManifest(t *testing.T) {
withTempOpt(t)
stubSystemctl(t)
_, setChannels := startFeed(t, map[string]string{"new1234": "x"})
setChannels(map[string]string{"stable": "new1234"})
// Pretend we've already accepted a manifest from the future.
if err := writeManifestStamp(time.Now().Add(time.Hour)); err != nil {
t.Fatal(err)
}
reg, err := openRegistry(filepath.Join(t.TempDir(), "reg.db"))
if err != nil {
t.Fatal(err)
}
if err := updatePass(reg, "lchat@", false, false); err == nil || !strings.Contains(err.Error(), "replayed") {
t.Fatalf("want replay refusal, got %v", err)
}
}
func TestUpdatePassDryRunTouchesNothing(t *testing.T) {
withTempOpt(t)
zeroBake(t)
calls := stubSystemctl(t)
_, setChannels := startFeed(t, map[string]string{"new1234": "content"})
setChannels(map[string]string{"stable": "new1234"})
reg, err := openRegistry(filepath.Join(t.TempDir(), "reg.db"))
if err != nil {
t.Fatal(err)
}
s1 := startFakeInstance(t, "h1.example.com", http.StatusNotFound, updatePrefs{}, "")
seedInstance(t, reg, "h1.example.com", s1, "old1111")
if err := updatePass(reg, "lchat@", true, false); err != nil {
t.Fatal(err)
}
if v, _ := liveVersion("h1.example.com"); v != "old1111" {
t.Fatalf("dry-run moved the symlink to %q", v)
}
if _, err := os.Stat(versionBinPath("new1234")); !os.IsNotExist(err) {
t.Fatal("dry-run downloaded/staged a binary")
}
if len(*calls) != 0 {
t.Fatalf("dry-run ran systemctl: %v", *calls)
}
if !readManifestStamp().IsZero() {
t.Fatal("dry-run wrote the manifest stamp")
}
}
func TestUpdatePassStagesTheRightArch(t *testing.T) {
withTempOpt(t)
zeroBake(t)
stubSystemctl(t)
_, setChannels := startFeed(t, map[string]string{"new1234": "binary-content"})
setChannels(map[string]string{"stable": "new1234"})
// After startFeed (which pins amd64): this box is arm64.
updaterArch = "arm64"
reg, err := openRegistry(filepath.Join(t.TempDir(), "reg.db"))
if err != nil {
t.Fatal(err)
}
s1 := startFakeInstance(t, "h1.example.com", http.StatusNotFound, updatePrefs{}, "")
seedInstance(t, reg, "h1.example.com", s1, "old1111")
if err := updatePass(reg, "lchat@", false, false); err != nil {
t.Fatal(err)
}
staged, err := os.ReadFile(versionBinPath("new1234"))
if err != nil || string(staged) != "binary-content@arm64" {
t.Fatalf("arm64 box staged %q, %v — must be the arm64 build", staged, err)
}
if v, _ := liveVersion("h1.example.com"); v != "new1234" {
t.Fatalf("arm64 apply didn't land: live = %q", v)
}
}
func TestUpdatePassSkipsArchlessRelease(t *testing.T) {
withTempOpt(t)
zeroBake(t)
calls := stubSystemctl(t)
prevArch := updaterArch
updaterArch = "arm64"
t.Cleanup(func() { updaterArch = prevArch })
// A pre-arm64 manifest: flat amd64 fields only. An arm64 box must skip it
// rather than install the wrong arch.
pub, priv, _ := ed25519.GenerateKey(rand.Reader)
feed := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
wrapper, _ := signReleaseManifest(priv, &releaseManifest{Channels: map[string]releaseEntry{
"stable": {Version: "new1234", URL: "https://example.com/lchat", SHA256: strings.Repeat("ab", 32)},
}})
w.Write(wrapper)
}))
defer feed.Close()
oldKey, oldURL := releasePubKeyB64, releaseManifestURL
releasePubKeyB64 = base64.StdEncoding.EncodeToString(pub)
releaseManifestURL = feed.URL
t.Cleanup(func() { releasePubKeyB64, releaseManifestURL = oldKey, oldURL })
reg, err := openRegistry(filepath.Join(t.TempDir(), "reg.db"))
if err != nil {
t.Fatal(err)
}
s1 := startFakeInstance(t, "h1.example.com", http.StatusNotFound, updatePrefs{}, "")
seedInstance(t, reg, "h1.example.com", s1, "old1111")
if err := updatePass(reg, "lchat@", false, false); err != nil {
t.Fatal(err)
}
if v, _ := liveVersion("h1.example.com"); v != "old1111" {
t.Fatalf("arm64 box moved to an amd64-only release: %q", v)
}
if _, err := os.Stat(versionBinPath("new1234")); !os.IsNotExist(err) {
t.Fatal("amd64 binary staged on an arm64 box")
}
if len(*calls) != 0 {
t.Fatalf("restarts on a skipped release: %v", *calls)
}
}
func TestUpdatePassSkipsPinnedInstance(t *testing.T) {
withTempOpt(t)
calls := stubSystemctl(t)
_, setChannels := startFeed(t, map[string]string{"new1234": "channel-head"})
setChannels(map[string]string{"stable": "new1234"})
reg, err := openRegistry(filepath.Join(t.TempDir(), "reg.db"))
if err != nil {
t.Fatal(err)
}
// Pinned to a branch build no channel or registry row mentions (the canary
// case) — while the stable channel points somewhere newer.
s1 := startFakeInstance(t, "h1.example.com", http.StatusOK,
updatePrefs{AutoUpdate: true, AutoDownload: true, Channel: "stable", Pin: "pin7777"}, "")
seedInstance(t, reg, "h1.example.com", s1, "old1111")
bin := filepath.Join(t.TempDir(), "b")
writeFakeBinary(t, bin, "canary-build")
if err := stageVersion(bin, "pin7777"); err != nil {
t.Fatal(err)
}
if err := updatePass(reg, "lchat@", false, false); err != nil {
t.Fatal(err)
}
// Left alone entirely: no restart, no symlink move, no channel download.
if len(*calls) != 0 {
t.Fatalf("pinned instance was touched: %v", *calls)
}
if v, _ := liveVersion("h1.example.com"); v != "old1111" {
t.Fatalf("pinned instance moved to %q", v)
}
if _, err := os.Stat(versionBinPath("new1234")); !os.IsNotExist(err) {
t.Fatal("channel head downloaded for a pinned instance")
}
// And its pinned build survived the pass's GC despite being orphaned
// from the registry/manifest point of view.
if _, err := os.Stat(versionBinPath("pin7777")); err != nil {
t.Fatal("GC pruned the pinned version")
}
}
func TestGCVersionsKeepsEverythingReferenced(t *testing.T) {
withTempOpt(t)
reg, err := openRegistry(filepath.Join(t.TempDir(), "reg.db"))
if err != nil {
t.Fatal(err)
}
bin := filepath.Join(t.TempDir(), "b")
writeFakeBinary(t, bin, "x")
for _, v := range []string{"cur1", "prev1", "head1", "orphan1", "orphan2", "linked1", "pinned9"} {
if err := stageVersion(bin, v); err != nil {
t.Fatal(err)
}
}
reg.upsert(Route{Host: "a.example.com", Kind: "instance", Status: "ready", Version: "cur1", PrevVersion: "prev1"})
// A live symlink nothing in the registry mentions must still survive.
if err := pointLive("b.example.com", "linked1"); err != nil {
t.Fatal(err)
}
m := &releaseManifest{Channels: map[string]releaseEntry{"stable": {Version: "head1"}}}
// pinned9 is referenced only by an instance's pin (keepExtra) — a canary's
// branch build that no registry row or channel mentions.
gcVersions(reg, m, []string{"pinned9"})
for _, keep := range []string{"cur1", "prev1", "head1", "linked1", "pinned9"} {
if _, err := os.Stat(versionBinPath(keep)); err != nil {
t.Errorf("gc pruned %s (should be kept)", keep)
}
}
for _, gone := range []string{"orphan1", "orphan2"} {
if _, err := os.Stat(versionBinPath(gone)); !os.IsNotExist(err) {
t.Errorf("gc kept %s (should be pruned)", gone)
}
}
// buildVersion ("dev" under test) is always kept — the control binary's own.
// (No dir staged for it here; just assert gc didn't error on its absence.)
}
func TestPinHostWritesViaAdminAPI(t *testing.T) {
dir, err := os.MkdirTemp("", "lch")
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { os.RemoveAll(dir) })
// pinHost reads the admin token from the instance's EnvironmentFile.
prevDataDir := instanceDataDir
instanceDataDir = dir
t.Cleanup(func() { instanceDataDir = prevDataDir })
if err := os.WriteFile(filepath.Join(dir, "h1.example.com.env"), []byte("LCHAT_ADMIN_TOKEN=tok123\n"), 0o600); err != nil {
t.Fatal(err)
}
var gotAuth, gotPin string
mux := http.NewServeMux()
mux.HandleFunc("POST /api/admin/settings", func(w http.ResponseWriter, r *http.Request) {
gotAuth = r.Header.Get("Authorization")
var body struct {
UpdatePin *string `json:"update_pin"`
}
json.NewDecoder(r.Body).Decode(&body)
if body.UpdatePin != nil {
gotPin = *body.UpdatePin
}
writeJSON(w, http.StatusOK, map[string]any{"ok": true})
})
// A current binary reports the stored pin back — pinHost verifies against
// this before claiming success.
mux.HandleFunc("/api/update-prefs", func(w http.ResponseWriter, r *http.Request) {
writeJSON(w, http.StatusOK, map[string]any{
"auto_update": true, "auto_download": true, "channel": "stable", "pin": gotPin,
})
})
socket := filepath.Join(dir, "h1.sock")
ln, err := net.Listen("unix", socket)
if err != nil {
t.Fatal(err)
}
srv := &http.Server{Handler: mux}
go srv.Serve(ln)
t.Cleanup(func() { srv.Close() })
rt := &Route{Host: "h1.example.com", Kind: "instance", Socket: socket}
if err := pinHost(rt, "pin7777"); err != nil {
t.Fatal(err)
}
if gotAuth != "Bearer tok123" || gotPin != "pin7777" {
t.Fatalf("auth %q pin %q", gotAuth, gotPin)
}
if err := pinHost(rt, ""); err != nil {
t.Fatal(err)
}
if gotPin != "" {
t.Fatalf("unpin sent %q", gotPin)
}
}
// An instance binary that predates a setting accepts the settings POST and
// silently ignores the unknown field. The read-back turns that into a loud
// error instead of a false success (a pin that didn't stick is how the next
// hourly pass stomps a canary; found live with `update major` vs a pre-0.11
// build).
func TestPinHostRefusesSilentlyIgnoredWrite(t *testing.T) {
dir, err := os.MkdirTemp("", "lch")
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { os.RemoveAll(dir) })
prevDataDir := instanceDataDir
instanceDataDir = dir
t.Cleanup(func() { instanceDataDir = prevDataDir })
if err := os.WriteFile(filepath.Join(dir, "h1.example.com.env"), []byte("LCHAT_ADMIN_TOKEN=tok123\n"), 0o600); err != nil {
t.Fatal(err)
}
mux := http.NewServeMux()
// The old binary: 200 on the write, but the field never appears in prefs.
mux.HandleFunc("POST /api/admin/settings", func(w http.ResponseWriter, r *http.Request) {
writeJSON(w, http.StatusOK, map[string]any{"ok": true})
})
mux.HandleFunc("/api/update-prefs", func(w http.ResponseWriter, r *http.Request) {
writeJSON(w, http.StatusOK, map[string]any{
"auto_update": true, "auto_download": true, "channel": "stable", "pin": "",
})
})
socket := filepath.Join(dir, "h1.sock")
ln, err := net.Listen("unix", socket)
if err != nil {
t.Fatal(err)
}
srv := &http.Server{Handler: mux}
go srv.Serve(ln)
t.Cleanup(func() { srv.Close() })
rt := &Route{Host: "h1.example.com", Kind: "instance", Socket: socket}
if err := pinHost(rt, "pin7777"); err == nil || !strings.Contains(err.Error(), "predates") {
t.Fatalf("silently ignored pin write reported success (err=%v)", err)
}
// Same helper, the major-pin shape: a prefs response with no "major" field
// decodes as "", so a wanted value must fail and a clear must pass.
if err := confirmHostSetting(rt, "major pin", "1", func(p updatePrefs) string { return p.Major }); err == nil {
t.Fatal("missing major field confirmed as set")
}
if err := confirmHostSetting(rt, "major pin", "", func(p updatePrefs) string { return p.Major }); err != nil {
t.Fatalf("clear should confirm against an old binary: %v", err)
}
}
// entryFor is a syntactically valid manifest entry (validateManifest passes)
// whose binary URL nothing in the manifest tests ever downloads.
func entryFor(ver string) releaseEntry {
return releaseEntry{
Version: ver,
URL: "https://example.com/server-" + ver + "/lchat-linux-amd64",
SHA256: strings.Repeat("ab", 32),
}
}
func TestFetchManifestFallsBackToMirrorOnFailure(t *testing.T) {
withTempOpt(t) // the staleness check reads the stamp from optDir
pub, priv, err := ed25519.GenerateKey(rand.Reader)
if err != nil {
t.Fatal(err)
}
good := signManifestAt(t, priv,
&releaseManifest{Channels: map[string]releaseEntry{"stable": entryFor("new1234")}}, time.Now())
primary := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
http.Error(w, "gone", http.StatusNotFound) // GitHub's brand-new-asset 404 window
}))
defer primary.Close()
mirror := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
// The mirror keeps the feed at /manifest.json.
if r.URL.Path != "/releases/manifest.json" {
http.NotFound(w, r)
return
}
w.Write(good)
}))
defer mirror.Close()
swapFeedGlobals(t, base64.StdEncoding.EncodeToString(pub),
primary.URL+"/manifest.json", mirror.URL+"/releases")
m, err := fetchReleaseManifest(releaseManifestURL)
if err != nil {
t.Fatalf("mirror fallback failed: %v", err)
}
if m.Channels["stable"].Version != "new1234" {
t.Fatalf("mirror manifest channels = %+v", m.Channels)
}
}
func TestFetchManifestRejectsBadMirrorSignature(t *testing.T) {
withTempOpt(t)
pub, _, err := ed25519.GenerateKey(rand.Reader) // the trusted root...
if err != nil {
t.Fatal(err)
}
_, evilPriv, err := ed25519.GenerateKey(rand.Reader) // ...and an untrusted signer
if err != nil {
t.Fatal(err)
}
forged := signManifestAt(t, evilPriv,
&releaseManifest{Channels: map[string]releaseEntry{"stable": entryFor("evil999")}}, time.Now())
primary := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
http.Error(w, "down", http.StatusServiceUnavailable)
}))
defer primary.Close()
mirror := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Write(forged)
}))
defer mirror.Close()
swapFeedGlobals(t, base64.StdEncoding.EncodeToString(pub),
primary.URL+"/manifest.json", mirror.URL+"/releases")
if _, err := fetchReleaseManifest(releaseManifestURL); err == nil {
t.Fatal("a mirror manifest signed by the wrong key was accepted")
}
}
func TestFetchManifestPrefersFresherMirrorWhenPrimaryStale(t *testing.T) {
withTempOpt(t)
pub, priv, err := ed25519.GenerateKey(rand.Reader)
if err != nil {
t.Fatal(err)
}
now := time.Now()
staleFeed := signManifestAt(t, priv,
&releaseManifest{Channels: map[string]releaseEntry{"stable": entryFor("old1111")}}, now.Add(-2*time.Hour))
freshFeed := signManifestAt(t, priv,
&releaseManifest{Channels: map[string]releaseEntry{"stable": entryFor("new1234")}}, now)
// The box last accepted a feed newer than what the primary now serves —
// exactly what GitHub's stale CDN looks like right after a publish.
if err := writeManifestStamp(now.Add(-time.Hour)); err != nil {
t.Fatal(err)
}
primary := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Write(staleFeed)
}))
defer primary.Close()
mirror := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Write(freshFeed)
}))
defer mirror.Close()
swapFeedGlobals(t, base64.StdEncoding.EncodeToString(pub),
primary.URL+"/manifest.json", mirror.URL+"/releases")
m, err := fetchReleaseManifest(releaseManifestURL)
if err != nil {
t.Fatal(err)
}
if got := m.Channels["stable"].Version; got != "new1234" {
t.Fatalf("stale primary won over a fresher mirror: stable = %q", got)
}
}
func TestFetchManifestKeepsStalePrimaryWhenMirrorNoFresher(t *testing.T) {
withTempOpt(t)
pub, priv, err := ed25519.GenerateKey(rand.Reader)
if err != nil {
t.Fatal(err)
}
now := time.Now()
staleFeed := signManifestAt(t, priv,
&releaseManifest{Channels: map[string]releaseEntry{"stable": entryFor("old1111")}}, now.Add(-2*time.Hour))
if err := writeManifestStamp(now); err != nil {
t.Fatal(err)
}
serveStale := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Write(staleFeed) })
primary := httptest.NewServer(serveStale)
defer primary.Close()
mirror := httptest.NewServer(serveStale)
defer mirror.Close()
swapFeedGlobals(t, base64.StdEncoding.EncodeToString(pub),
primary.URL+"/manifest.json", mirror.URL+"/releases")
// Both stale: the primary result stands, and updatePass's replay check is
// what refuses it — the mirror must never mask a genuinely replayed feed.
m, err := fetchReleaseManifest(releaseManifestURL)
if err != nil {
t.Fatal(err)
}
if !m.GeneratedAt.Before(readManifestStamp()) {
t.Fatal("expected the stale primary manifest back (for the replay check to refuse)")
}
reg, err := openRegistry(filepath.Join(t.TempDir(), "reg.db"))
if err != nil {
t.Fatal(err)
}
if err := updatePass(reg, "lchat@", false, false); err == nil || !strings.Contains(err.Error(), "replayed") {
t.Fatalf("want replay refusal, got %v", err)
}
}
// mirrorBinaryFeed wires a primary feed whose manifest is fine but whose
// binary URL 404s, plus a mirror holding mirrorContent under the derived
// server-/ key — the GitHub new-asset-404 shape.
func mirrorBinaryFeed(t *testing.T, binaryContent, mirrorContent string) {
t.Helper()
pub, priv, err := ed25519.GenerateKey(rand.Reader)
if err != nil {
t.Fatal(err)
}
sum := sha256.Sum256([]byte(binaryContent))
primaryMux := http.NewServeMux() // serves only the manifest; the binary path 404s
primary := httptest.NewServer(primaryMux)
t.Cleanup(primary.Close)
entry := releaseEntry{
Version: "new1234",
URL: primary.URL + "/server-new1234/lchat-linux-amd64",
SHA256: hex.EncodeToString(sum[:]),
}
wrapper := signManifestAt(t, priv,
&releaseManifest{Channels: map[string]releaseEntry{"stable": entry}}, time.Now())
primaryMux.HandleFunc("/manifest.json", func(w http.ResponseWriter, r *http.Request) { w.Write(wrapper) })
mirror := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
// The mirror stores binaries under the primary URL's last two segments.
if r.URL.Path != "/releases/server-new1234/lchat-linux-amd64" {
http.NotFound(w, r)
return
}
w.Write([]byte(mirrorContent))
}))
t.Cleanup(mirror.Close)
swapFeedGlobals(t, base64.StdEncoding.EncodeToString(pub),
primary.URL+"/manifest.json", mirror.URL+"/releases")
prevArch := updaterArch
updaterArch = "amd64"
t.Cleanup(func() { updaterArch = prevArch })
}
func TestUpdatePassFallsBackToMirrorBinary(t *testing.T) {
withTempOpt(t)
zeroBake(t)
stubSystemctl(t)
shrink := deployHealthTimeout
deployHealthTimeout = 2 * time.Second
t.Cleanup(func() { deployHealthTimeout = shrink })
content := "mirrored-binary-content"
mirrorBinaryFeed(t, content, content)
reg, err := openRegistry(filepath.Join(t.TempDir(), "reg.db"))
if err != nil {
t.Fatal(err)
}
s1 := startFakeInstance(t, "h1.example.com", http.StatusNotFound, updatePrefs{}, "")
seedInstance(t, reg, "h1.example.com", s1, "old1111")
if err := updatePass(reg, "lchat@", false, false); err != nil {
t.Fatal(err)
}
if v, _ := liveVersion("h1.example.com"); v != "new1234" {
t.Fatalf("mirror binary fallback didn't land: live = %q", v)
}
staged, err := os.ReadFile(versionBinPath("new1234"))
if err != nil || string(staged) != content {
t.Fatalf("staged binary = %q, %v — must be the mirror's bytes", staged, err)
}
}
func TestUpdatePassRejectsBadMirrorBinary(t *testing.T) {
withTempOpt(t)
zeroBake(t)
calls := stubSystemctl(t)
// The mirror serves bytes that don't hash to the signed sha256 — a
// tampered (or stale) mirror must cause a refusal, never an install.
mirrorBinaryFeed(t, "the-real-binary", "tampered-bytes")
reg, err := openRegistry(filepath.Join(t.TempDir(), "reg.db"))
if err != nil {
t.Fatal(err)
}
s1 := startFakeInstance(t, "h1.example.com", http.StatusNotFound, updatePrefs{}, "")
seedInstance(t, reg, "h1.example.com", s1, "old1111")
if err := updatePass(reg, "lchat@", false, false); err != nil {
t.Fatal(err)
}
if v, _ := liveVersion("h1.example.com"); v != "old1111" {
t.Fatalf("tampered mirror binary was installed: live = %q", v)
}
if _, err := os.Stat(versionBinPath("new1234")); !os.IsNotExist(err) {
t.Fatal("tampered mirror binary was staged")
}
if len(*calls) != 0 {
t.Fatalf("restarts despite a refused download: %v", *calls)
}
}
func TestAcquireDeployLockExcludes(t *testing.T) {
withTempOpt(t)
unlock, err := acquireDeployLock()
if err != nil {
t.Fatal(err)
}
if _, err := acquireDeployLock(); err == nil {
t.Fatal("second acquire succeeded — deploy and update could interleave")
}
unlock()
unlock2, err := acquireDeployLock()
if err != nil {
t.Fatalf("re-acquire after release: %v", err)
}
unlock2()
}
// Hibernating hosts (#550): no unit to restart and never woken — the pass
// repoints their live symlink instead, so the NEXT wake execs the new binary.
// Prefs come from the parked DB; no DB reads as the defaults; pins hold.
func TestUpdatePassRepointsHibernating(t *testing.T) {
withTempOpt(t)
zeroBake(t)
calls := stubSystemctl(t)
oldData := instanceDataDir
instanceDataDir = t.TempDir()
t.Cleanup(func() { instanceDataDir = oldData })
_, setChannels := startFeed(t, map[string]string{
"new1234": "new-binary-content",
"edge777": "edge-binary-content",
})
setChannels(map[string]string{"stable": "new1234", "edge": "edge777"})
reg, err := openRegistry(filepath.Join(t.TempDir(), "reg.db"))
if err != nil {
t.Fatal(err)
}
seed := func(host string) {
t.Helper()
src := filepath.Join(t.TempDir(), "old")
writeFakeBinary(t, src, "old-binary")
if err := stageVersion(src, "old1111"); err != nil {
t.Fatal(err)
}
if err := pointLive(host, "old1111"); err != nil {
t.Fatal(err)
}
if err := reg.upsert(Route{Host: host, Kind: "instance", Status: "ready",
Socket: filepath.Join(t.TempDir(), "no.sock"), Hibernate: true, Version: "old1111"}); err != nil {
t.Fatal(err)
}
}
seedDB := func(host string, kv map[string]string) {
t.Helper()
db, err := openDB(filepath.Join(instanceDataDir, host+".db"))
if err != nil {
t.Fatal(err)
}
for k, v := range kv {
if err := db.setConfig(k, v); err != nil {
t.Fatal(err)
}
}
db.sql.Close()
}
// h1: parked DB with no update keys — defaults (stable/on/on) → repointed.
// h2: parked DB says edge → repointed to the edge head.
// h3: pinned in the parked DB → untouched.
// h4: downloads+updates off → untouched.
// h5: NO parked DB — prefs unknowable, fail-closed → untouched.
for _, h := range []string{"h1.hib", "h2.hib", "h3.hib", "h4.hib", "h5.hib"} {
seed(h)
}
seedDB("h1.hib", map[string]string{})
seedDB("h2.hib", map[string]string{"update_channel": "edge"})
seedDB("h3.hib", map[string]string{"update_pin": "old1111"})
seedDB("h4.hib", map[string]string{"update_auto": "0", "update_download": "0"})
if err := updatePass(reg, "lchat@", false, false); err != nil {
t.Fatalf("updatePass: %v", err)
}
assertLive := func(host, want string) {
t.Helper()
if v, _ := liveVersion(host); v != want {
t.Errorf("%s live version = %q, want %q", host, v, want)
}
}
assertLive("h1.hib", "new1234")
assertLive("h2.hib", "edge777")
assertLive("h3.hib", "old1111")
assertLive("h4.hib", "old1111")
assertLive("h5.hib", "old1111")
// The registry records the move (and the previous version, for rollback).
rt, _ := reg.get("h1.hib")
if rt.Version != "new1234" || rt.PrevVersion != "old1111" {
t.Errorf("h1 registry = %q prev %q", rt.Version, rt.PrevVersion)
}
// The whole point: no unit was restarted, nothing was woken.
if len(*calls) != 0 {
t.Fatalf("systemctl touched for hibernating hosts: %v", *calls)
}
// A second pass is quiet + idempotent.
if err := updatePass(reg, "lchat@", false, false); err != nil {
t.Fatalf("second pass: %v", err)
}
assertLive("h1.hib", "new1234")
}
func TestStatusPrefsReadsParkedDB(t *testing.T) {
oldData := instanceDataDir
instanceDataDir = t.TempDir()
t.Cleanup(func() { instanceDataDir = oldData })
deadSocket := filepath.Join(t.TempDir(), "no.sock")
// A sleeping pinned canary: nothing listens on its socket, its prefs live
// only in the parked DB — status must show them, not "?" (an invisible pin
// is how a fleet operation stomps a canary).
db, err := openDB(filepath.Join(instanceDataDir, "h1.hib.db"))
if err != nil {
t.Fatal(err)
}
for k, v := range map[string]string{"update_channel": "edge", "update_pin": "pin1234"} {
if err := db.setConfig(k, v); err != nil {
t.Fatal(err)
}
}
db.sql.Close()
p, err := statusPrefs(Route{Host: "h1.hib", Hibernate: true, Socket: deadSocket})
if err != nil {
t.Fatalf("statusPrefs: %v", err)
}
if p.Channel != "edge" || p.Pin != "pin1234" {
t.Fatalf("prefs = %+v, want edge/pin1234", p)
}
// Fail-closed both ways: a sleeping host with no local DB stays an honest
// "?", and a unit-managed host's dead socket never falls back to a DB read.
if _, err := statusPrefs(Route{Host: "h2.hib", Hibernate: true, Socket: deadSocket}); err == nil {
t.Fatal("expected an error for an absent parked DB")
}
if _, err := statusPrefs(Route{Host: "h1.hib", Hibernate: false, Socket: deadSocket}); err == nil {
t.Fatal("expected an error for a dead socket on a unit-managed host")
}
}