package main import ( "crypto/ed25519" "crypto/rand" "crypto/sha256" "encoding/base64" "encoding/hex" "encoding/json" "net" "net/http" "net/http/httptest" "os" "path/filepath" "strings" "sync" "testing" "time" ) // startFakeInstance serves /api/update-prefs + /api/version on a real unix // socket. Its reported build follows the host's live symlink — so pointLive + // a stubbed systemctl restart looks exactly like a real swap to // waitHealthyVersion. prefsCode 404 simulates a pre-feature binary; brokenVer // (if set) makes that version report a wrong build, forcing the rollback path. func startFakeInstance(t *testing.T, host string, prefsCode int, prefs updatePrefs, brokenVer string) string { t.Helper() // Short base dir: unix socket paths have a ~104-char limit on darwin. dir, err := os.MkdirTemp("", "lch") if err != nil { t.Fatal(err) } t.Cleanup(func() { os.RemoveAll(dir) }) socket := filepath.Join(dir, host+".sock") mux := http.NewServeMux() mux.HandleFunc("/api/update-prefs", func(w http.ResponseWriter, r *http.Request) { if prefsCode != http.StatusOK { http.Error(w, "no such endpoint", prefsCode) return } writeJSON(w, http.StatusOK, map[string]any{ "auto_update": prefs.AutoUpdate, "auto_download": prefs.AutoDownload, "channel": prefs.Channel, "pin": prefs.Pin, "major": prefs.Major, }) }) mux.HandleFunc("/api/version", func(w http.ResponseWriter, r *http.Request) { v, _ := liveVersion(host) if brokenVer != "" && v == brokenVer { v = "borked" } writeJSON(w, http.StatusOK, map[string]any{"build": v}) }) ln, err := net.Listen("unix", socket) if err != nil { t.Fatal(err) } srv := &http.Server{Handler: mux} go srv.Serve(ln) t.Cleanup(func() { srv.Close() }) return socket } // feedSemvers, when a test seeds it, stamps a semver onto matching versions // in every manifest startFeed signs (tests run serially, so a package-level // map is safe; entries are cleared via t.Cleanup in the tests that set them). var feedSemvers = map[string]string{} // startFeed signs entries into a manifest and serves it (plus each version's // binary bytes) from a loopback server, swapping in a test trust root. func startFeed(t *testing.T, binaries map[string]string) (*httptest.Server, func(channels map[string]string)) { t.Helper() pub, priv, err := ed25519.GenerateKey(rand.Reader) if err != nil { t.Fatal(err) } var mu sync.Mutex var wrapper []byte mux := http.NewServeMux() mux.HandleFunc("/manifest.json", func(w http.ResponseWriter, r *http.Request) { mu.Lock() defer mu.Unlock() w.Write(wrapper) }) mux.HandleFunc("/bin/", func(w http.ResponseWriter, r *http.Request) { ver := strings.TrimPrefix(r.URL.Path, "/bin/") w.Write([]byte(binaries[ver])) }) // Each version's arm64 build is distinct bytes, so a test can prove the // right arch was staged. mux.HandleFunc("/arm/", func(w http.ResponseWriter, r *http.Request) { ver := strings.TrimPrefix(r.URL.Path, "/arm/") w.Write([]byte(binaries[ver] + "@arm64")) }) ts := httptest.NewServer(mux) t.Cleanup(ts.Close) oldKey, oldURL := releasePubKeyB64, releaseManifestURL releasePubKeyB64 = base64.StdEncoding.EncodeToString(pub) releaseManifestURL = ts.URL + "/manifest.json" t.Cleanup(func() { releasePubKeyB64, releaseManifestURL = oldKey, oldURL }) // Pin the box arch so feed tests are deterministic on any dev machine // (an arm64 Mac would otherwise resolve the arm64 variant everywhere); // arch-specific tests override AFTER calling startFeed. prevArch := updaterArch updaterArch = "amd64" t.Cleanup(func() { updaterArch = prevArch }) setChannels := func(channels map[string]string) { m := &releaseManifest{Channels: map[string]releaseEntry{}} for ch, ver := range channels { sum := sha256.Sum256([]byte(binaries[ver])) armSum := sha256.Sum256([]byte(binaries[ver] + "@arm64")) m.Channels[ch] = releaseEntry{ Version: ver, URL: ts.URL + "/bin/" + ver, SHA256: hex.EncodeToString(sum[:]), ARM64URL: ts.URL + "/arm/" + ver, ARM64SHA256: hex.EncodeToString(armSum[:]), Semver: feedSemvers[ver], } } w, err := signReleaseManifest(priv, m) if err != nil { t.Fatal(err) } mu.Lock() wrapper = w mu.Unlock() } return ts, setChannels } // signManifestAt builds a signed wrapper with a caller-controlled // generated_at — signReleaseManifest always stamps now, but the mirror tests // need feeds whose relative ages are exact. func signManifestAt(t *testing.T, priv ed25519.PrivateKey, m *releaseManifest, at time.Time) []byte { t.Helper() m.Schema = 1 m.GeneratedAt = at.UTC().Truncate(time.Second) payload, err := json.Marshal(m) if err != nil { t.Fatal(err) } b64 := base64.StdEncoding.EncodeToString(payload) sig := ed25519.Sign(priv, []byte(b64)) w, err := json.Marshal(signedManifest{Payload: b64, Sig: base64.StdEncoding.EncodeToString(sig)}) if err != nil { t.Fatal(err) } return w } // swapFeedGlobals points the trust root, primary URL, and mirror base at test // servers, restoring them after. func swapFeedGlobals(t *testing.T, pubB64, primaryURL, mirrorBase string) { t.Helper() oldKey, oldURL, oldMirror := releasePubKeyB64, releaseManifestURL, releaseMirrorURL releasePubKeyB64, releaseManifestURL, releaseMirrorURL = pubB64, primaryURL, mirrorBase t.Cleanup(func() { releasePubKeyB64, releaseManifestURL, releaseMirrorURL = oldKey, oldURL, oldMirror }) } // stubSystemctl records unit restarts and leaves the "restart" to the fake // instance's symlink-following /api/version. func stubSystemctl(t *testing.T) *[]string { t.Helper() var calls []string prev := systemctl systemctl = func(args ...string) error { calls = append(calls, strings.Join(args, " ")) return nil } t.Cleanup(func() { systemctl = prev }) return &calls } func seedInstance(t *testing.T, reg *Registry, host, socket, ver string) { t.Helper() src := filepath.Join(t.TempDir(), "old") writeFakeBinary(t, src, "old-binary-"+ver) if err := stageVersion(src, ver); err != nil { t.Fatal(err) } if err := pointLive(host, ver); err != nil { t.Fatal(err) } if err := reg.upsert(Route{Host: host, Kind: "instance", Status: "ready", Socket: socket, Version: ver}); err != nil { t.Fatal(err) } } // zeroBake disables the supply-chain bake window (bake.go) for tests that // exercise the rest of the pipeline; bake behaviour has its own tests below. func zeroBake(t *testing.T) { t.Helper() prev := bakeDelays bakeDelays = map[string]time.Duration{"edge": 0, "beta": 0, "stable": 0} t.Cleanup(func() { bakeDelays = prev }) } func TestUpdatePassAppliesPerPrefs(t *testing.T) { withTempOpt(t) zeroBake(t) calls := stubSystemctl(t) shrink := deployHealthTimeout deployHealthTimeout = 2 * time.Second t.Cleanup(func() { deployHealthTimeout = shrink }) _, setChannels := startFeed(t, map[string]string{ "new1234": "new-binary-content", "beta5555": "beta-binary-content", }) setChannels(map[string]string{"stable": "new1234", "beta": "beta5555"}) reg, err := openRegistry(filepath.Join(t.TempDir(), "reg.db")) if err != nil { t.Fatal(err) } // h1: defaults-by-404 (old binary) — updates to stable. // h2: auto-update off — downloaded + staged, never restarted. // h3: everything off, beta channel — beta's binary must not even download. // h4: explicit on/on/stable — updates. s1 := startFakeInstance(t, "h1.example.com", http.StatusNotFound, updatePrefs{}, "") s2 := startFakeInstance(t, "h2.example.com", http.StatusOK, updatePrefs{AutoUpdate: false, AutoDownload: true, Channel: "stable"}, "") s3 := startFakeInstance(t, "h3.example.com", http.StatusOK, updatePrefs{AutoUpdate: false, AutoDownload: false, Channel: "beta"}, "") s4 := startFakeInstance(t, "h4.example.com", http.StatusOK, updatePrefs{AutoUpdate: true, AutoDownload: true, Channel: "stable"}, "") seedInstance(t, reg, "h1.example.com", s1, "old1111") seedInstance(t, reg, "h2.example.com", s2, "old1111") seedInstance(t, reg, "h3.example.com", s3, "old1111") seedInstance(t, reg, "h4.example.com", s4, "old1111") if err := updatePass(reg, "lchat@", false, false); err != nil { t.Fatalf("updatePass: %v", err) } assertLive := func(host, want string) { t.Helper() if v, _ := liveVersion(host); v != want { t.Errorf("%s live version = %q, want %q", host, v, want) } } assertLive("h1.example.com", "new1234") assertLive("h2.example.com", "old1111") // staged only assertLive("h3.example.com", "old1111") assertLive("h4.example.com", "new1234") // h2's chosen build is staged and byte-exact; h3's channel never downloaded. staged, err := os.ReadFile(versionBinPath("new1234")) if err != nil || string(staged) != "new-binary-content" { t.Fatalf("staged binary = %q, %v", staged, err) } if _, err := os.Stat(versionBinPath("beta5555")); !os.IsNotExist(err) { t.Fatal("beta5555 was downloaded despite downloads off") } // Registry recorded the moves (and only the moves). rt, _ := reg.get("h1.example.com") if rt.Version != "new1234" || rt.PrevVersion != "old1111" { t.Fatalf("h1 registry = %s/%s", rt.Version, rt.PrevVersion) } rt, _ = reg.get("h2.example.com") if rt.Version != "old1111" { t.Fatalf("h2 registry = %s", rt.Version) } // Two instances moved (h1, h4); each restarts the instance and then its // sidecars so both track the new version (restartBotSidecar + // restartHooksSidecar). Six calls, no more — h2 (download-only) and h3 // (downloads off) never restart. got := strings.Join(*calls, "; ") for _, want := range []string{ "restart lchat@h1.example.com", "try-restart lchat-bot@h1.example.com", "try-restart lchat-hooks@h1.example.com", "restart lchat@h4.example.com", "try-restart lchat-bot@h4.example.com", "try-restart lchat-hooks@h4.example.com", } { if !strings.Contains(got, want) { t.Fatalf("missing %q in systemctl calls = %v", want, *calls) } } if len(*calls) != 6 { t.Fatalf("systemctl calls = %v", *calls) } // The pass recorded the manifest stamp for anti-replay. if readManifestStamp().IsZero() { t.Fatal("manifest stamp not written") } } func TestUpdatePassRollsBackAndPoisons(t *testing.T) { withTempOpt(t) zeroBake(t) calls := stubSystemctl(t) shrink := deployHealthTimeout deployHealthTimeout = 500 * time.Millisecond t.Cleanup(func() { deployHealthTimeout = shrink }) _, setChannels := startFeed(t, map[string]string{"bad9999": "does-not-boot"}) setChannels(map[string]string{"stable": "bad9999"}) reg, err := openRegistry(filepath.Join(t.TempDir(), "reg.db")) if err != nil { t.Fatal(err) } // Both instances report "borked" while on bad9999 → health check fails. s1 := startFakeInstance(t, "h1.example.com", http.StatusNotFound, updatePrefs{}, "bad9999") s2 := startFakeInstance(t, "h2.example.com", http.StatusNotFound, updatePrefs{}, "bad9999") seedInstance(t, reg, "h1.example.com", s1, "old1111") seedInstance(t, reg, "h2.example.com", s2, "old1111") if err := updatePass(reg, "lchat@", false, false); err != nil { t.Fatalf("updatePass: %v", err) } // h1 tried, failed, rolled back; h2 was spared (the version is poisoned). for _, host := range []string{"h1.example.com", "h2.example.com"} { if v, _ := liveVersion(host); v != "old1111" { t.Errorf("%s live version = %q, want rollback to old1111", host, v) } rt, _ := reg.get(host) if rt.Version != "old1111" { t.Errorf("%s registry version = %q", host, rt.Version) } } // h1: restart onto bad + rollback restart = 2 calls; h2: none. if len(*calls) != 2 || strings.Contains(strings.Join(*calls, ";"), "h2.example.com") { t.Fatalf("systemctl calls = %v", *calls) } } func TestUpdatePassRefusesReplayedManifest(t *testing.T) { withTempOpt(t) stubSystemctl(t) _, setChannels := startFeed(t, map[string]string{"new1234": "x"}) setChannels(map[string]string{"stable": "new1234"}) // Pretend we've already accepted a manifest from the future. if err := writeManifestStamp(time.Now().Add(time.Hour)); err != nil { t.Fatal(err) } reg, err := openRegistry(filepath.Join(t.TempDir(), "reg.db")) if err != nil { t.Fatal(err) } if err := updatePass(reg, "lchat@", false, false); err == nil || !strings.Contains(err.Error(), "replayed") { t.Fatalf("want replay refusal, got %v", err) } } func TestUpdatePassDryRunTouchesNothing(t *testing.T) { withTempOpt(t) zeroBake(t) calls := stubSystemctl(t) _, setChannels := startFeed(t, map[string]string{"new1234": "content"}) setChannels(map[string]string{"stable": "new1234"}) reg, err := openRegistry(filepath.Join(t.TempDir(), "reg.db")) if err != nil { t.Fatal(err) } s1 := startFakeInstance(t, "h1.example.com", http.StatusNotFound, updatePrefs{}, "") seedInstance(t, reg, "h1.example.com", s1, "old1111") if err := updatePass(reg, "lchat@", true, false); err != nil { t.Fatal(err) } if v, _ := liveVersion("h1.example.com"); v != "old1111" { t.Fatalf("dry-run moved the symlink to %q", v) } if _, err := os.Stat(versionBinPath("new1234")); !os.IsNotExist(err) { t.Fatal("dry-run downloaded/staged a binary") } if len(*calls) != 0 { t.Fatalf("dry-run ran systemctl: %v", *calls) } if !readManifestStamp().IsZero() { t.Fatal("dry-run wrote the manifest stamp") } } func TestUpdatePassStagesTheRightArch(t *testing.T) { withTempOpt(t) zeroBake(t) stubSystemctl(t) _, setChannels := startFeed(t, map[string]string{"new1234": "binary-content"}) setChannels(map[string]string{"stable": "new1234"}) // After startFeed (which pins amd64): this box is arm64. updaterArch = "arm64" reg, err := openRegistry(filepath.Join(t.TempDir(), "reg.db")) if err != nil { t.Fatal(err) } s1 := startFakeInstance(t, "h1.example.com", http.StatusNotFound, updatePrefs{}, "") seedInstance(t, reg, "h1.example.com", s1, "old1111") if err := updatePass(reg, "lchat@", false, false); err != nil { t.Fatal(err) } staged, err := os.ReadFile(versionBinPath("new1234")) if err != nil || string(staged) != "binary-content@arm64" { t.Fatalf("arm64 box staged %q, %v — must be the arm64 build", staged, err) } if v, _ := liveVersion("h1.example.com"); v != "new1234" { t.Fatalf("arm64 apply didn't land: live = %q", v) } } func TestUpdatePassSkipsArchlessRelease(t *testing.T) { withTempOpt(t) zeroBake(t) calls := stubSystemctl(t) prevArch := updaterArch updaterArch = "arm64" t.Cleanup(func() { updaterArch = prevArch }) // A pre-arm64 manifest: flat amd64 fields only. An arm64 box must skip it // rather than install the wrong arch. pub, priv, _ := ed25519.GenerateKey(rand.Reader) feed := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { wrapper, _ := signReleaseManifest(priv, &releaseManifest{Channels: map[string]releaseEntry{ "stable": {Version: "new1234", URL: "https://example.com/lchat", SHA256: strings.Repeat("ab", 32)}, }}) w.Write(wrapper) })) defer feed.Close() oldKey, oldURL := releasePubKeyB64, releaseManifestURL releasePubKeyB64 = base64.StdEncoding.EncodeToString(pub) releaseManifestURL = feed.URL t.Cleanup(func() { releasePubKeyB64, releaseManifestURL = oldKey, oldURL }) reg, err := openRegistry(filepath.Join(t.TempDir(), "reg.db")) if err != nil { t.Fatal(err) } s1 := startFakeInstance(t, "h1.example.com", http.StatusNotFound, updatePrefs{}, "") seedInstance(t, reg, "h1.example.com", s1, "old1111") if err := updatePass(reg, "lchat@", false, false); err != nil { t.Fatal(err) } if v, _ := liveVersion("h1.example.com"); v != "old1111" { t.Fatalf("arm64 box moved to an amd64-only release: %q", v) } if _, err := os.Stat(versionBinPath("new1234")); !os.IsNotExist(err) { t.Fatal("amd64 binary staged on an arm64 box") } if len(*calls) != 0 { t.Fatalf("restarts on a skipped release: %v", *calls) } } func TestUpdatePassSkipsPinnedInstance(t *testing.T) { withTempOpt(t) calls := stubSystemctl(t) _, setChannels := startFeed(t, map[string]string{"new1234": "channel-head"}) setChannels(map[string]string{"stable": "new1234"}) reg, err := openRegistry(filepath.Join(t.TempDir(), "reg.db")) if err != nil { t.Fatal(err) } // Pinned to a branch build no channel or registry row mentions (the canary // case) — while the stable channel points somewhere newer. s1 := startFakeInstance(t, "h1.example.com", http.StatusOK, updatePrefs{AutoUpdate: true, AutoDownload: true, Channel: "stable", Pin: "pin7777"}, "") seedInstance(t, reg, "h1.example.com", s1, "old1111") bin := filepath.Join(t.TempDir(), "b") writeFakeBinary(t, bin, "canary-build") if err := stageVersion(bin, "pin7777"); err != nil { t.Fatal(err) } if err := updatePass(reg, "lchat@", false, false); err != nil { t.Fatal(err) } // Left alone entirely: no restart, no symlink move, no channel download. if len(*calls) != 0 { t.Fatalf("pinned instance was touched: %v", *calls) } if v, _ := liveVersion("h1.example.com"); v != "old1111" { t.Fatalf("pinned instance moved to %q", v) } if _, err := os.Stat(versionBinPath("new1234")); !os.IsNotExist(err) { t.Fatal("channel head downloaded for a pinned instance") } // And its pinned build survived the pass's GC despite being orphaned // from the registry/manifest point of view. if _, err := os.Stat(versionBinPath("pin7777")); err != nil { t.Fatal("GC pruned the pinned version") } } func TestGCVersionsKeepsEverythingReferenced(t *testing.T) { withTempOpt(t) reg, err := openRegistry(filepath.Join(t.TempDir(), "reg.db")) if err != nil { t.Fatal(err) } bin := filepath.Join(t.TempDir(), "b") writeFakeBinary(t, bin, "x") for _, v := range []string{"cur1", "prev1", "head1", "orphan1", "orphan2", "linked1", "pinned9"} { if err := stageVersion(bin, v); err != nil { t.Fatal(err) } } reg.upsert(Route{Host: "a.example.com", Kind: "instance", Status: "ready", Version: "cur1", PrevVersion: "prev1"}) // A live symlink nothing in the registry mentions must still survive. if err := pointLive("b.example.com", "linked1"); err != nil { t.Fatal(err) } m := &releaseManifest{Channels: map[string]releaseEntry{"stable": {Version: "head1"}}} // pinned9 is referenced only by an instance's pin (keepExtra) — a canary's // branch build that no registry row or channel mentions. gcVersions(reg, m, []string{"pinned9"}) for _, keep := range []string{"cur1", "prev1", "head1", "linked1", "pinned9"} { if _, err := os.Stat(versionBinPath(keep)); err != nil { t.Errorf("gc pruned %s (should be kept)", keep) } } for _, gone := range []string{"orphan1", "orphan2"} { if _, err := os.Stat(versionBinPath(gone)); !os.IsNotExist(err) { t.Errorf("gc kept %s (should be pruned)", gone) } } // buildVersion ("dev" under test) is always kept — the control binary's own. // (No dir staged for it here; just assert gc didn't error on its absence.) } func TestPinHostWritesViaAdminAPI(t *testing.T) { dir, err := os.MkdirTemp("", "lch") if err != nil { t.Fatal(err) } t.Cleanup(func() { os.RemoveAll(dir) }) // pinHost reads the admin token from the instance's EnvironmentFile. prevDataDir := instanceDataDir instanceDataDir = dir t.Cleanup(func() { instanceDataDir = prevDataDir }) if err := os.WriteFile(filepath.Join(dir, "h1.example.com.env"), []byte("LCHAT_ADMIN_TOKEN=tok123\n"), 0o600); err != nil { t.Fatal(err) } var gotAuth, gotPin string mux := http.NewServeMux() mux.HandleFunc("POST /api/admin/settings", func(w http.ResponseWriter, r *http.Request) { gotAuth = r.Header.Get("Authorization") var body struct { UpdatePin *string `json:"update_pin"` } json.NewDecoder(r.Body).Decode(&body) if body.UpdatePin != nil { gotPin = *body.UpdatePin } writeJSON(w, http.StatusOK, map[string]any{"ok": true}) }) // A current binary reports the stored pin back — pinHost verifies against // this before claiming success. mux.HandleFunc("/api/update-prefs", func(w http.ResponseWriter, r *http.Request) { writeJSON(w, http.StatusOK, map[string]any{ "auto_update": true, "auto_download": true, "channel": "stable", "pin": gotPin, }) }) socket := filepath.Join(dir, "h1.sock") ln, err := net.Listen("unix", socket) if err != nil { t.Fatal(err) } srv := &http.Server{Handler: mux} go srv.Serve(ln) t.Cleanup(func() { srv.Close() }) rt := &Route{Host: "h1.example.com", Kind: "instance", Socket: socket} if err := pinHost(rt, "pin7777"); err != nil { t.Fatal(err) } if gotAuth != "Bearer tok123" || gotPin != "pin7777" { t.Fatalf("auth %q pin %q", gotAuth, gotPin) } if err := pinHost(rt, ""); err != nil { t.Fatal(err) } if gotPin != "" { t.Fatalf("unpin sent %q", gotPin) } } // An instance binary that predates a setting accepts the settings POST and // silently ignores the unknown field. The read-back turns that into a loud // error instead of a false success (a pin that didn't stick is how the next // hourly pass stomps a canary; found live with `update major` vs a pre-0.11 // build). func TestPinHostRefusesSilentlyIgnoredWrite(t *testing.T) { dir, err := os.MkdirTemp("", "lch") if err != nil { t.Fatal(err) } t.Cleanup(func() { os.RemoveAll(dir) }) prevDataDir := instanceDataDir instanceDataDir = dir t.Cleanup(func() { instanceDataDir = prevDataDir }) if err := os.WriteFile(filepath.Join(dir, "h1.example.com.env"), []byte("LCHAT_ADMIN_TOKEN=tok123\n"), 0o600); err != nil { t.Fatal(err) } mux := http.NewServeMux() // The old binary: 200 on the write, but the field never appears in prefs. mux.HandleFunc("POST /api/admin/settings", func(w http.ResponseWriter, r *http.Request) { writeJSON(w, http.StatusOK, map[string]any{"ok": true}) }) mux.HandleFunc("/api/update-prefs", func(w http.ResponseWriter, r *http.Request) { writeJSON(w, http.StatusOK, map[string]any{ "auto_update": true, "auto_download": true, "channel": "stable", "pin": "", }) }) socket := filepath.Join(dir, "h1.sock") ln, err := net.Listen("unix", socket) if err != nil { t.Fatal(err) } srv := &http.Server{Handler: mux} go srv.Serve(ln) t.Cleanup(func() { srv.Close() }) rt := &Route{Host: "h1.example.com", Kind: "instance", Socket: socket} if err := pinHost(rt, "pin7777"); err == nil || !strings.Contains(err.Error(), "predates") { t.Fatalf("silently ignored pin write reported success (err=%v)", err) } // Same helper, the major-pin shape: a prefs response with no "major" field // decodes as "", so a wanted value must fail and a clear must pass. if err := confirmHostSetting(rt, "major pin", "1", func(p updatePrefs) string { return p.Major }); err == nil { t.Fatal("missing major field confirmed as set") } if err := confirmHostSetting(rt, "major pin", "", func(p updatePrefs) string { return p.Major }); err != nil { t.Fatalf("clear should confirm against an old binary: %v", err) } } // entryFor is a syntactically valid manifest entry (validateManifest passes) // whose binary URL nothing in the manifest tests ever downloads. func entryFor(ver string) releaseEntry { return releaseEntry{ Version: ver, URL: "https://example.com/server-" + ver + "/lchat-linux-amd64", SHA256: strings.Repeat("ab", 32), } } func TestFetchManifestFallsBackToMirrorOnFailure(t *testing.T) { withTempOpt(t) // the staleness check reads the stamp from optDir pub, priv, err := ed25519.GenerateKey(rand.Reader) if err != nil { t.Fatal(err) } good := signManifestAt(t, priv, &releaseManifest{Channels: map[string]releaseEntry{"stable": entryFor("new1234")}}, time.Now()) primary := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { http.Error(w, "gone", http.StatusNotFound) // GitHub's brand-new-asset 404 window })) defer primary.Close() mirror := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { // The mirror keeps the feed at /manifest.json. if r.URL.Path != "/releases/manifest.json" { http.NotFound(w, r) return } w.Write(good) })) defer mirror.Close() swapFeedGlobals(t, base64.StdEncoding.EncodeToString(pub), primary.URL+"/manifest.json", mirror.URL+"/releases") m, err := fetchReleaseManifest(releaseManifestURL) if err != nil { t.Fatalf("mirror fallback failed: %v", err) } if m.Channels["stable"].Version != "new1234" { t.Fatalf("mirror manifest channels = %+v", m.Channels) } } func TestFetchManifestRejectsBadMirrorSignature(t *testing.T) { withTempOpt(t) pub, _, err := ed25519.GenerateKey(rand.Reader) // the trusted root... if err != nil { t.Fatal(err) } _, evilPriv, err := ed25519.GenerateKey(rand.Reader) // ...and an untrusted signer if err != nil { t.Fatal(err) } forged := signManifestAt(t, evilPriv, &releaseManifest{Channels: map[string]releaseEntry{"stable": entryFor("evil999")}}, time.Now()) primary := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { http.Error(w, "down", http.StatusServiceUnavailable) })) defer primary.Close() mirror := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Write(forged) })) defer mirror.Close() swapFeedGlobals(t, base64.StdEncoding.EncodeToString(pub), primary.URL+"/manifest.json", mirror.URL+"/releases") if _, err := fetchReleaseManifest(releaseManifestURL); err == nil { t.Fatal("a mirror manifest signed by the wrong key was accepted") } } func TestFetchManifestPrefersFresherMirrorWhenPrimaryStale(t *testing.T) { withTempOpt(t) pub, priv, err := ed25519.GenerateKey(rand.Reader) if err != nil { t.Fatal(err) } now := time.Now() staleFeed := signManifestAt(t, priv, &releaseManifest{Channels: map[string]releaseEntry{"stable": entryFor("old1111")}}, now.Add(-2*time.Hour)) freshFeed := signManifestAt(t, priv, &releaseManifest{Channels: map[string]releaseEntry{"stable": entryFor("new1234")}}, now) // The box last accepted a feed newer than what the primary now serves — // exactly what GitHub's stale CDN looks like right after a publish. if err := writeManifestStamp(now.Add(-time.Hour)); err != nil { t.Fatal(err) } primary := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Write(staleFeed) })) defer primary.Close() mirror := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Write(freshFeed) })) defer mirror.Close() swapFeedGlobals(t, base64.StdEncoding.EncodeToString(pub), primary.URL+"/manifest.json", mirror.URL+"/releases") m, err := fetchReleaseManifest(releaseManifestURL) if err != nil { t.Fatal(err) } if got := m.Channels["stable"].Version; got != "new1234" { t.Fatalf("stale primary won over a fresher mirror: stable = %q", got) } } func TestFetchManifestKeepsStalePrimaryWhenMirrorNoFresher(t *testing.T) { withTempOpt(t) pub, priv, err := ed25519.GenerateKey(rand.Reader) if err != nil { t.Fatal(err) } now := time.Now() staleFeed := signManifestAt(t, priv, &releaseManifest{Channels: map[string]releaseEntry{"stable": entryFor("old1111")}}, now.Add(-2*time.Hour)) if err := writeManifestStamp(now); err != nil { t.Fatal(err) } serveStale := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Write(staleFeed) }) primary := httptest.NewServer(serveStale) defer primary.Close() mirror := httptest.NewServer(serveStale) defer mirror.Close() swapFeedGlobals(t, base64.StdEncoding.EncodeToString(pub), primary.URL+"/manifest.json", mirror.URL+"/releases") // Both stale: the primary result stands, and updatePass's replay check is // what refuses it — the mirror must never mask a genuinely replayed feed. m, err := fetchReleaseManifest(releaseManifestURL) if err != nil { t.Fatal(err) } if !m.GeneratedAt.Before(readManifestStamp()) { t.Fatal("expected the stale primary manifest back (for the replay check to refuse)") } reg, err := openRegistry(filepath.Join(t.TempDir(), "reg.db")) if err != nil { t.Fatal(err) } if err := updatePass(reg, "lchat@", false, false); err == nil || !strings.Contains(err.Error(), "replayed") { t.Fatalf("want replay refusal, got %v", err) } } // mirrorBinaryFeed wires a primary feed whose manifest is fine but whose // binary URL 404s, plus a mirror holding mirrorContent under the derived // server-/ key — the GitHub new-asset-404 shape. func mirrorBinaryFeed(t *testing.T, binaryContent, mirrorContent string) { t.Helper() pub, priv, err := ed25519.GenerateKey(rand.Reader) if err != nil { t.Fatal(err) } sum := sha256.Sum256([]byte(binaryContent)) primaryMux := http.NewServeMux() // serves only the manifest; the binary path 404s primary := httptest.NewServer(primaryMux) t.Cleanup(primary.Close) entry := releaseEntry{ Version: "new1234", URL: primary.URL + "/server-new1234/lchat-linux-amd64", SHA256: hex.EncodeToString(sum[:]), } wrapper := signManifestAt(t, priv, &releaseManifest{Channels: map[string]releaseEntry{"stable": entry}}, time.Now()) primaryMux.HandleFunc("/manifest.json", func(w http.ResponseWriter, r *http.Request) { w.Write(wrapper) }) mirror := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { // The mirror stores binaries under the primary URL's last two segments. if r.URL.Path != "/releases/server-new1234/lchat-linux-amd64" { http.NotFound(w, r) return } w.Write([]byte(mirrorContent)) })) t.Cleanup(mirror.Close) swapFeedGlobals(t, base64.StdEncoding.EncodeToString(pub), primary.URL+"/manifest.json", mirror.URL+"/releases") prevArch := updaterArch updaterArch = "amd64" t.Cleanup(func() { updaterArch = prevArch }) } func TestUpdatePassFallsBackToMirrorBinary(t *testing.T) { withTempOpt(t) zeroBake(t) stubSystemctl(t) shrink := deployHealthTimeout deployHealthTimeout = 2 * time.Second t.Cleanup(func() { deployHealthTimeout = shrink }) content := "mirrored-binary-content" mirrorBinaryFeed(t, content, content) reg, err := openRegistry(filepath.Join(t.TempDir(), "reg.db")) if err != nil { t.Fatal(err) } s1 := startFakeInstance(t, "h1.example.com", http.StatusNotFound, updatePrefs{}, "") seedInstance(t, reg, "h1.example.com", s1, "old1111") if err := updatePass(reg, "lchat@", false, false); err != nil { t.Fatal(err) } if v, _ := liveVersion("h1.example.com"); v != "new1234" { t.Fatalf("mirror binary fallback didn't land: live = %q", v) } staged, err := os.ReadFile(versionBinPath("new1234")) if err != nil || string(staged) != content { t.Fatalf("staged binary = %q, %v — must be the mirror's bytes", staged, err) } } func TestUpdatePassRejectsBadMirrorBinary(t *testing.T) { withTempOpt(t) zeroBake(t) calls := stubSystemctl(t) // The mirror serves bytes that don't hash to the signed sha256 — a // tampered (or stale) mirror must cause a refusal, never an install. mirrorBinaryFeed(t, "the-real-binary", "tampered-bytes") reg, err := openRegistry(filepath.Join(t.TempDir(), "reg.db")) if err != nil { t.Fatal(err) } s1 := startFakeInstance(t, "h1.example.com", http.StatusNotFound, updatePrefs{}, "") seedInstance(t, reg, "h1.example.com", s1, "old1111") if err := updatePass(reg, "lchat@", false, false); err != nil { t.Fatal(err) } if v, _ := liveVersion("h1.example.com"); v != "old1111" { t.Fatalf("tampered mirror binary was installed: live = %q", v) } if _, err := os.Stat(versionBinPath("new1234")); !os.IsNotExist(err) { t.Fatal("tampered mirror binary was staged") } if len(*calls) != 0 { t.Fatalf("restarts despite a refused download: %v", *calls) } } func TestAcquireDeployLockExcludes(t *testing.T) { withTempOpt(t) unlock, err := acquireDeployLock() if err != nil { t.Fatal(err) } if _, err := acquireDeployLock(); err == nil { t.Fatal("second acquire succeeded — deploy and update could interleave") } unlock() unlock2, err := acquireDeployLock() if err != nil { t.Fatalf("re-acquire after release: %v", err) } unlock2() } // Hibernating hosts (#550): no unit to restart and never woken — the pass // repoints their live symlink instead, so the NEXT wake execs the new binary. // Prefs come from the parked DB; no DB reads as the defaults; pins hold. func TestUpdatePassRepointsHibernating(t *testing.T) { withTempOpt(t) zeroBake(t) calls := stubSystemctl(t) oldData := instanceDataDir instanceDataDir = t.TempDir() t.Cleanup(func() { instanceDataDir = oldData }) _, setChannels := startFeed(t, map[string]string{ "new1234": "new-binary-content", "edge777": "edge-binary-content", }) setChannels(map[string]string{"stable": "new1234", "edge": "edge777"}) reg, err := openRegistry(filepath.Join(t.TempDir(), "reg.db")) if err != nil { t.Fatal(err) } seed := func(host string) { t.Helper() src := filepath.Join(t.TempDir(), "old") writeFakeBinary(t, src, "old-binary") if err := stageVersion(src, "old1111"); err != nil { t.Fatal(err) } if err := pointLive(host, "old1111"); err != nil { t.Fatal(err) } if err := reg.upsert(Route{Host: host, Kind: "instance", Status: "ready", Socket: filepath.Join(t.TempDir(), "no.sock"), Hibernate: true, Version: "old1111"}); err != nil { t.Fatal(err) } } seedDB := func(host string, kv map[string]string) { t.Helper() db, err := openDB(filepath.Join(instanceDataDir, host+".db")) if err != nil { t.Fatal(err) } for k, v := range kv { if err := db.setConfig(k, v); err != nil { t.Fatal(err) } } db.sql.Close() } // h1: parked DB with no update keys — defaults (stable/on/on) → repointed. // h2: parked DB says edge → repointed to the edge head. // h3: pinned in the parked DB → untouched. // h4: downloads+updates off → untouched. // h5: NO parked DB — prefs unknowable, fail-closed → untouched. for _, h := range []string{"h1.hib", "h2.hib", "h3.hib", "h4.hib", "h5.hib"} { seed(h) } seedDB("h1.hib", map[string]string{}) seedDB("h2.hib", map[string]string{"update_channel": "edge"}) seedDB("h3.hib", map[string]string{"update_pin": "old1111"}) seedDB("h4.hib", map[string]string{"update_auto": "0", "update_download": "0"}) if err := updatePass(reg, "lchat@", false, false); err != nil { t.Fatalf("updatePass: %v", err) } assertLive := func(host, want string) { t.Helper() if v, _ := liveVersion(host); v != want { t.Errorf("%s live version = %q, want %q", host, v, want) } } assertLive("h1.hib", "new1234") assertLive("h2.hib", "edge777") assertLive("h3.hib", "old1111") assertLive("h4.hib", "old1111") assertLive("h5.hib", "old1111") // The registry records the move (and the previous version, for rollback). rt, _ := reg.get("h1.hib") if rt.Version != "new1234" || rt.PrevVersion != "old1111" { t.Errorf("h1 registry = %q prev %q", rt.Version, rt.PrevVersion) } // The whole point: no unit was restarted, nothing was woken. if len(*calls) != 0 { t.Fatalf("systemctl touched for hibernating hosts: %v", *calls) } // A second pass is quiet + idempotent. if err := updatePass(reg, "lchat@", false, false); err != nil { t.Fatalf("second pass: %v", err) } assertLive("h1.hib", "new1234") } func TestStatusPrefsReadsParkedDB(t *testing.T) { oldData := instanceDataDir instanceDataDir = t.TempDir() t.Cleanup(func() { instanceDataDir = oldData }) deadSocket := filepath.Join(t.TempDir(), "no.sock") // A sleeping pinned canary: nothing listens on its socket, its prefs live // only in the parked DB — status must show them, not "?" (an invisible pin // is how a fleet operation stomps a canary). db, err := openDB(filepath.Join(instanceDataDir, "h1.hib.db")) if err != nil { t.Fatal(err) } for k, v := range map[string]string{"update_channel": "edge", "update_pin": "pin1234"} { if err := db.setConfig(k, v); err != nil { t.Fatal(err) } } db.sql.Close() p, err := statusPrefs(Route{Host: "h1.hib", Hibernate: true, Socket: deadSocket}) if err != nil { t.Fatalf("statusPrefs: %v", err) } if p.Channel != "edge" || p.Pin != "pin1234" { t.Fatalf("prefs = %+v, want edge/pin1234", p) } // Fail-closed both ways: a sleeping host with no local DB stays an honest // "?", and a unit-managed host's dead socket never falls back to a DB read. if _, err := statusPrefs(Route{Host: "h2.hib", Hibernate: true, Socket: deadSocket}); err == nil { t.Fatal("expected an error for an absent parked DB") } if _, err := statusPrefs(Route{Host: "h1.hib", Hibernate: false, Socket: deadSocket}); err == nil { t.Fatal("expected an error for a dead socket on a unit-managed host") } }