package main import ( "crypto/sha256" "encoding/hex" "fmt" "net" "net/http" "net/http/httptest" "os" "path/filepath" "strings" "sync" "testing" "time" ) // selfUpdateEnv points every seam at temp doubles: a fake control binary on // the same temp "filesystem" the download lands on, a fresh bake state, a // recording systemctl, and the health probe aimed at a temp socket path (which // a test binds via fakeControlSocket, or leaves unbound to model a refused // dial). deployHealthTimeout is shrunk so an unhealthy path fails in ~a beat. // By default isActive reports controld UP (so the live /healthz probe is the // authoritative gate, the pre-F2 behaviour); tests exercising the // inactive-controld fallback override isActive + controlBinaryReports. func selfUpdateEnv(t *testing.T, newBinary []byte) (self string, entry releaseEntry, calls *[]string) { t.Helper() oldOpt, oldSelf, oldSysctl, oldChannel := optDir, selfPath, systemctl, os.Getenv("LCHAT_CONTROL_CHANNEL") oldBuild, oldSock, oldTimeout := buildVersion, controlSocketPath, deployHealthTimeout oldActive, oldBinReports := isActive, controlBinaryReports buildVersion = "testold1" // the dev-build guard must not fire in these tests t.Cleanup(func() { optDir, selfPath, systemctl, buildVersion = oldOpt, oldSelf, oldSysctl, oldBuild controlSocketPath, deployHealthTimeout = oldSock, oldTimeout isActive, controlBinaryReports = oldActive, oldBinReports os.Setenv("LCHAT_CONTROL_CHANNEL", oldChannel) }) optDir = t.TempDir() self = filepath.Join(optDir, "lchat") if err := os.WriteFile(self, []byte("OLD-CONTROL-BINARY"), 0o755); err != nil { t.Fatal(err) } selfPath = func() (string, error) { return self, nil } var c []string calls = &c systemctl = func(args ...string) error { c = append(c, strings.Join(args, " ")) return nil } os.Setenv("LCHAT_CONTROL_CHANNEL", "stable") // Default seams: controld is up (the /healthz probe is the gate), and the // `version` integrity check is a poisoned stub — a test that expects the // inactive-controld fallback must stub both, or this loud error trips. isActive = func(string) bool { return true } controlBinaryReports = func(string, string) error { return fmt.Errorf("controlBinaryReports not stubbed for this test") } // The health probe dials a unix socket, whose sun_path is short-limited — a // dedicated short temp dir, not optDir (which can be a long macOS path). sockDir, err := os.MkdirTemp("", "lch") if err != nil { t.Fatal(err) } t.Cleanup(func() { os.RemoveAll(sockDir) }) controlSocketPath = filepath.Join(sockDir, "c.sock") deployHealthTimeout = 400 * time.Millisecond srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Write(newBinary) })) t.Cleanup(srv.Close) sum := sha256.Sum256(newBinary) entry = releaseEntry{Version: "abc1234", URL: srv.URL + "/lchat", SHA256: hex.EncodeToString(sum[:]), Size: int64(len(newBinary)), ARM64URL: srv.URL + "/lchat-arm64", ARM64SHA256: hex.EncodeToString(sum[:]), ARM64Size: int64(len(newBinary))} return self, entry, calls } // fakeControlSocket binds a unix socket at controlSocketPath that answers // GET /healthz with a build — the same shape controld's real /healthz serves — // so waitControlHealthy sees a healthy new build or a stale old one. It returns // a setter so a test can flip the reported build mid-run (e.g. old→new after a // channel moves). A test that wants a REFUSED dial (a crash-looping binary) // simply doesn't call this, leaving the path unbound. func fakeControlSocket(t *testing.T, build string) (set func(string)) { t.Helper() var mu sync.Mutex cur := build set = func(b string) { mu.Lock(); cur = b; mu.Unlock() } mux := http.NewServeMux() mux.HandleFunc("GET /healthz", func(w http.ResponseWriter, _ *http.Request) { mu.Lock() b := cur mu.Unlock() writeJSON(w, http.StatusOK, map[string]any{"build": b}) }) ln, err := net.Listen("unix", controlSocketPath) if err != nil { t.Fatal(err) } srv := &http.Server{Handler: mux} go srv.Serve(ln) t.Cleanup(func() { srv.Close() }) return set } // The swap end to end: verified download, staged into the version store, atomic // install over the running path, a .prev rollback beside it, and the backend // services bounced — router deliberately untouched. The control socket reports // the new build, so the health probe passes and the swap commits. func TestControlSelfUpdateSwaps(t *testing.T) { newBin := []byte("NEW-CONTROL-BINARY") self, entry, calls := selfUpdateEnv(t, newBin) fakeControlSocket(t, entry.Version) // healthy on the new build m := &releaseManifest{Schema: 1, GeneratedAt: time.Now(), Channels: map[string]releaseEntry{"stable": entry}} // ignoreBake=true is the deliberate override path; the bake-hold path is // pinned separately below. if err := controlSelfUpdate(m, loadFirstSeen(), nil, false, true); err != nil { t.Fatalf("healthy swap returned error: %v", err) } got, err := os.ReadFile(self) if err != nil || string(got) != string(newBin) { t.Fatalf("control binary not swapped: %q %v", got, err) } // Staged into the store too, so gcVersions protects it and rollback can reach it. if staged, err := os.ReadFile(versionBinPath(entry.Version)); err != nil || string(staged) != string(newBin) { t.Fatalf("new build not staged into the version store: %q %v", staged, err) } prev, err := os.ReadFile(self + ".prev-" + buildVersion) if err != nil || string(prev) != "OLD-CONTROL-BINARY" { t.Fatalf("rollback copy missing/wrong: %q %v", prev, err) } joined := strings.Join(*calls, ";") if !strings.Contains(joined, "try-restart lchat-control.service") || !strings.Contains(joined, "try-restart lchat-cloud.service") { t.Fatalf("backend services not bounced: %v", *calls) } if strings.Contains(joined, "lchat-router") { t.Fatalf("the router must never restart on the timer's schedule: %v", *calls) } } // The happy path in isolation: a healthy new build commits, does NOT roll back, // bounces the backends exactly once (two calls, not four), and returns nil. func TestControlSelfUpdateHealthyCommits(t *testing.T) { newBin := []byte("NEW-CONTROL-BINARY") self, entry, calls := selfUpdateEnv(t, newBin) fakeControlSocket(t, entry.Version) m := &releaseManifest{Schema: 1, GeneratedAt: time.Now(), Channels: map[string]releaseEntry{"stable": entry}} if err := controlSelfUpdate(m, loadFirstSeen(), nil, false, true); err != nil { t.Fatalf("healthy commit returned error: %v", err) } if got, _ := os.ReadFile(self); string(got) != string(newBin) { t.Fatalf("healthy build should stay swapped in: %q", got) } // One restart of each backend, never a second (rollback) round. Counted // against controlBinaryUnits rather than a literal, so adding a unit to the // update path doesn't read as a spurious extra restart here. if n, want := strings.Count(strings.Join(*calls, ";"), "try-restart"), len(controlBinaryUnits); n != want { t.Fatalf("expected exactly %d restarts (no rollback), got %d: %v", want, n, *calls) } } // The whole point: a swapped-in build that never reports its version is rolled // back. The control socket reports the OLD build (a restart that didn't take), // so the new-build probe times out; the binary on disk is restored to .prev's // content, the backends are restarted a SECOND time, and the function errors. func TestControlSelfUpdateRollsBackOnUnhealthy(t *testing.T) { newBin := []byte("NEW-CONTROL-BINARY") self, entry, calls := selfUpdateEnv(t, newBin) fakeControlSocket(t, buildVersion) // stale: reports the OLD build after the swap m := &releaseManifest{Schema: 1, GeneratedAt: time.Now(), Channels: map[string]releaseEntry{"stable": entry}} err := controlSelfUpdate(m, loadFirstSeen(), nil, false, true) if err == nil { t.Fatal("an unhealthy swap must return an error") } if got, _ := os.ReadFile(self); string(got) != "OLD-CONTROL-BINARY" { t.Fatalf("binary not restored to the last-known-good: %q", got) } joined := strings.Join(*calls, ";") if n := strings.Count(joined, "try-restart lchat-control.service"); n != 2 { t.Fatalf("expected control restarted twice (swap + rollback), got %d: %v", n, *calls) } if strings.Contains(joined, "lchat-router") { t.Fatalf("the router must never restart, even on rollback: %v", *calls) } } // The genuine brick: the new build is unhealthy AND the rolled-back old build is // unhealthy too (nothing ever answers /healthz). The swap is still physically // reverted on disk, but a distinct, screaming error is surfaced — and it does // not loop. func TestControlSelfUpdateRollbackAlsoFailsScreams(t *testing.T) { newBin := []byte("NEW-CONTROL-BINARY") self, entry, _ := selfUpdateEnv(t, newBin) // No fakeControlSocket at all → every /healthz dial is refused → both the // post-swap and the post-rollback probes time out. m := &releaseManifest{Schema: 1, GeneratedAt: time.Now(), Channels: map[string]releaseEntry{"stable": entry}} err := controlSelfUpdate(m, loadFirstSeen(), nil, false, true) if err == nil || !strings.Contains(err.Error(), "CONTROL PLANE BRICKED") { t.Fatalf("expected a distinct BRICKED error, got: %v", err) } // The rollback rename is a pure file op, so it still restores the good binary // even when the box won't come back up on it. if got, _ := os.ReadFile(self); string(got) != "OLD-CONTROL-BINARY" { t.Fatalf("rollback rename should still restore the old binary: %q", got) } } // F2: when lchat-control isn't running, a failed /healthz probe is NOT // authoritative — try-restart was a no-op and nothing could answer. A // genuinely good binary (it execs and self-reports the target build via the // `version` integrity check) must COMMIT, not be falsely rolled back, and the // BRICKED path must not be taken. No socket is bound (the probe refuses). func TestControlSelfUpdateInactiveControldCommits(t *testing.T) { newBin := []byte("NEW-CONTROL-BINARY") self, entry, calls := selfUpdateEnv(t, newBin) isActive = func(string) bool { return false } // controld not running controlBinaryReports = func(_, want string) error { if want == entry.Version { return nil // the new binary loads + self-reports the target build } return fmt.Errorf("unexpected integrity target %q", want) } m := &releaseManifest{Schema: 1, GeneratedAt: time.Now(), Channels: map[string]releaseEntry{"stable": entry}} if err := controlSelfUpdate(m, loadFirstSeen(), nil, false, true); err != nil { t.Fatalf("inactive controld + good binary must commit, got error: %v", err) } if got, _ := os.ReadFile(self); string(got) != string(newBin) { t.Fatalf("good binary must stay committed, not rolled back: %q", got) } joined := strings.Join(*calls, ";") if n, want := strings.Count(joined, "try-restart"), len(controlBinaryUnits); n != want { t.Fatalf("expected exactly %d restarts (commit, no rollback), got %d: %v", want, n, *calls) } if strings.Contains(joined, "lchat-router") { t.Fatalf("router must never restart: %v", *calls) } if got := readFailedControlVersion(); got != "" { t.Fatalf("a committed build must not be quarantined: %q", got) } } // F3: a head that fails its health check is quarantined (persisted under // optDir), so the next hourly pass short-circuits it instead of reinstalling + // rolling it back forever — and the quarantine lifts when the channel moves to // a good build. func TestControlSelfUpdateQuarantinesFailedHead(t *testing.T) { newBin := []byte("NEW-CONTROL-BINARY") self, entry, calls := selfUpdateEnv(t, newBin) setBuild := fakeControlSocket(t, buildVersion) // stale: reports OLD after the swap m := &releaseManifest{Schema: 1, GeneratedAt: time.Now(), Channels: map[string]releaseEntry{"stable": entry}} // First pass: the head fails its health check, is rolled back and quarantined. if err := controlSelfUpdate(m, loadFirstSeen(), nil, false, true); err == nil { t.Fatal("an unhealthy head must return an error") } if got := readFailedControlVersion(); got != entry.Version { t.Fatalf("failed head not quarantined: %q", got) } if got, _ := os.ReadFile(self); string(got) != "OLD-CONTROL-BINARY" { t.Fatalf("binary not rolled back: %q", got) } // Second pass on the SAME head: the quarantine short-circuits it — no // download, no swap, no restarts. before := len(*calls) if err := controlSelfUpdate(m, loadFirstSeen(), nil, false, true); err != nil { t.Fatalf("quarantined head should be a silent no-op, got: %v", err) } if len(*calls) != before { t.Fatalf("quarantined head must not restart anything: %v", (*calls)[before:]) } if got, _ := os.ReadFile(self); string(got) != "OLD-CONTROL-BINARY" { t.Fatalf("quarantined head must not swap: %q", got) } // The channel moves to a DIFFERENT, healthy head: the quarantine clears and // the new build commits. e2 := entry e2.Version = "def5678" // same download bytes + sha, new version id setBuild(e2.Version) // socket now reports the new build → healthy m2 := &releaseManifest{Schema: 1, GeneratedAt: time.Now(), Channels: map[string]releaseEntry{"stable": e2}} if err := controlSelfUpdate(m2, loadFirstSeen(), nil, false, true); err != nil { t.Fatalf("healthy new head should commit, got: %v", err) } if got := readFailedControlVersion(); got != "" { t.Fatalf("quarantine should be cleared after the channel moved on, still %q", got) } if got, _ := os.ReadFile(self); string(got) != string(newBin) { t.Fatalf("new head not committed: %q", got) } } // F4: a head that failed a health check on an INSTANCE this pass (in the // poisoned set) must never then become the control plane — no download, no // swap, no quarantine write. func TestControlSelfUpdateSkipsPoisonedHead(t *testing.T) { newBin := []byte("NEW-CONTROL-BINARY") self, entry, calls := selfUpdateEnv(t, newBin) m := &releaseManifest{Schema: 1, GeneratedAt: time.Now(), Channels: map[string]releaseEntry{"stable": entry}} poisoned := map[string]bool{entry.Version: true} if err := controlSelfUpdate(m, loadFirstSeen(), poisoned, false, true); err != nil { t.Fatalf("a poisoned head should be a silent no-op, got: %v", err) } if got, _ := os.ReadFile(self); string(got) != "OLD-CONTROL-BINARY" { t.Fatalf("poisoned head must not swap: %q", got) } if len(*calls) != 0 { t.Fatalf("poisoned head must not restart anything: %v", *calls) } if got := readFailedControlVersion(); got != "" { t.Fatalf("skipping a poisoned head must not write a quarantine: %q", got) } } // A head still inside its bake window is held — the trust plane is the LAST // thing on a box that should take a new build. func TestControlSelfUpdateHonorsBake(t *testing.T) { self, entry, calls := selfUpdateEnv(t, []byte("NEW")) m := &releaseManifest{Schema: 1, GeneratedAt: time.Now(), Channels: map[string]releaseEntry{"stable": entry}} bake := loadFirstSeen() bake.observe(m, time.Now()) // first sighting is now → inside the window controlSelfUpdate(m, bake, nil, false, false) if got, _ := os.ReadFile(self); string(got) != "OLD-CONTROL-BINARY" { t.Fatalf("baking head must not install: %q", got) } if len(*calls) != 0 { t.Fatalf("no restarts while held: %v", *calls) } } // "off" and already-at-head are silent no-ops; a sha mismatch leaves the // binary untouched (the verified-download contract, pinned here for the // control path specifically). func TestControlSelfUpdateNoOpsAndBadSha(t *testing.T) { self, entry, _ := selfUpdateEnv(t, []byte("NEW")) os.Setenv("LCHAT_CONTROL_CHANNEL", "off") controlSelfUpdate(&releaseManifest{Channels: map[string]releaseEntry{"stable": entry}}, loadFirstSeen(), nil, false, true) if got, _ := os.ReadFile(self); string(got) != "OLD-CONTROL-BINARY" { t.Fatal("off must be a no-op") } os.Setenv("LCHAT_CONTROL_CHANNEL", "stable") same := entry same.Version = buildVersion controlSelfUpdate(&releaseManifest{Channels: map[string]releaseEntry{"stable": same}}, loadFirstSeen(), nil, false, true) if got, _ := os.ReadFile(self); string(got) != "OLD-CONTROL-BINARY" { t.Fatal("at-head must be a no-op") } bad := entry bad.SHA256 = strings.Repeat("00", 32) bad.ARM64SHA256 = strings.Repeat("00", 32) // whichever arch runs the test controlSelfUpdate(&releaseManifest{Channels: map[string]releaseEntry{"stable": bad}}, loadFirstSeen(), nil, false, true) if got, _ := os.ReadFile(self); string(got) != "OLD-CONTROL-BINARY" { t.Fatal("a sha mismatch must never install") } }