package main // The sibling services — connect broker, push relay, home server — run from // loose /opt/lchat/ binaries that root used to scp onto the box, a path // the operator-allowlist hardening removed (docs/instance-router.md § Box // access). They now converge from the SAME signed manifest entry the control // binary follows (controlSelfUpdate): publish uploads lchat--linux- // beside the instance binaries and pins their sha256s INSIDE the signed // payload, and the already-root hourly pass swaps any binary whose on-disk // sha doesn't match — same signature, same bake window, same rollback shape. // // Convergence is sha-driven, not version-driven: the services carry no build // stamp, and comparing hashes also heals a tampered or half-written binary. // A box that doesn't run a service simply has no /opt/lchat/ file and // is skipped — nothing is ever installed onto a box that didn't opt in. import ( "crypto/sha256" "encoding/hex" "fmt" "io" "log" "os" "path/filepath" "strings" "time" ) // serviceBinaries maps a manifest `services` key to the loose binary path // component under optDir and the unit that execs it. Order is cosmetic. // lchat-links is deliberately absent: it's a subcommand of the control binary, // so it rides controlSelfUpdate. var serviceBinaries = []struct { Name string // manifest key AND /opt/lchat/ Unit string }{ {"connect", "lchat-connect"}, {"push", "lchat-push"}, {"home", "lchat-home"}, } // serviceHealthSettle is how long a bounced unit gets to crash before we call // the swap good — Restart=always units flap fast, so a binary that can't boot // shows as inactive/activating within a beat. A var so tests shrink it. var serviceHealthSettle = 3 * time.Second func serviceFailedPath(name string) string { return filepath.Join(optDir, ".service-failed-"+name) } func readServiceFailed(name string) string { b, err := os.ReadFile(serviceFailedPath(name)) if err != nil { return "" } return strings.TrimSpace(string(b)) } // fileSHA256 hex-hashes a file; os.IsNotExist(err) is the caller's "this box // doesn't run the service" signal. func fileSHA256(path string) (string, error) { f, err := os.Open(path) if err != nil { return "", err } defer f.Close() h := sha256.New() if _, err := io.Copy(h, f); err != nil { return "", err } return hex.EncodeToString(h.Sum(nil)), nil } // serviceBinariesUpdate converges the sibling-service binaries onto the // control channel's head. Runs from updatePass AFTER controlSelfUpdate — on // the pass that swaps the control binary this (still-old) process runs the // converge with old code, which is fine: the logic is data-driven from the // manifest, and the next pass runs it from the new build anyway. Like the // control swap, failures log and never fail the pass — the instances above // already converged. func serviceBinariesUpdate(m *releaseManifest, bake *firstSeenState, poisoned map[string]bool, dryRun, ignoreBake bool) { channel := envOr("LCHAT_CONTROL_CHANNEL", "stable") if channel == "off" { return } e, ok := m.Channels[channel] if !ok || len(e.Services) == 0 { // Pre-feature releases (and promote's rebuilt-from-assets entries) // carry no services block — nothing to converge on. return } if poisoned[e.Version] { fmt.Printf("update: skipping service binaries for %s — it failed a health check on an instance this pass\n", e.Version) return } if hold := bakeHold(bake, channel, e.Version, ignoreBake, time.Now()); hold != "" { fmt.Printf("update: service binaries -> %s: %s\n", e.Version, hold) return } for _, svc := range serviceBinaries { sb, ok := e.Services[svc.Name] if !ok { continue } path := filepath.Join(optDir, svc.Name) cur, err := fileSHA256(path) if err != nil { // No file = this box doesn't run the service; anything else is // worth a line but not a retry loop. if !os.IsNotExist(err) { fmt.Printf("update: service %s: %v\n", svc.Name, err) } continue } url, want, ok := sb.forArch(updaterArch) if !ok { continue // no build for this arch — same skip rule as instances } if cur == want { continue } if failed := readServiceFailed(svc.Name); failed != "" { if failed == e.Version { fmt.Printf("update: service %s build %s is quarantined (failed its restart; pin or patch the channel to move)\n", svc.Name, e.Version) continue } os.Remove(serviceFailedPath(svc.Name)) // channel moved on — retry } if dryRun { fmt.Printf("update: service %s would update to %s (%s; dry run)\n", svc.Name, e.Version, channel) continue } if err := swapServiceBinary(svc.Name, svc.Unit, path, url, want, e.Version); err != nil { fmt.Printf("update: service %s: %v\n", svc.Name, err) } } } // swapServiceBinary downloads (sha-gated, mirror fallback), atomically swaps // with a .prev kept beside it, and — only when the unit is actually running — // bounces it and confirms it stays up, rolling back and quarantining on // failure. An inactive unit gets the new file silently: the box staged the // binary but hasn't enabled the service, so there is nothing to health-check. func swapServiceBinary(name, unit, path, url, wantSha, ver string) error { tmp, err := downloadBinaryToTemp(url, wantSha, ver) if err != nil { if releaseMirrorURL == "" { return err } murl := mirrorBinaryURL(releaseMirrorURL, url) log.Printf("update: service %s primary download failed (%v) — trying mirror %s", name, err, murl) if tmp, err = downloadBinaryToTemp(murl, wantSha, ver); err != nil { return fmt.Errorf("download: primary and mirror failed (%v)", err) } } defer os.Remove(tmp) if err := os.Chmod(tmp, 0o755); err != nil { return fmt.Errorf("chmod: %v", err) } prev := path + ".prev" if err := copyFileAtomic(path, prev, 0o755); err != nil { return fmt.Errorf("keep rollback copy: %v", err) } // downloadBinaryToTemp lands in incomingDir() — same filesystem, atomic. if err := os.Rename(tmp, path); err != nil { return fmt.Errorf("install: %v", err) } if !isActive(unit) { fmt.Printf("update: service %s -> %s (unit %s inactive — applies when it starts; rollback at %s)\n", name, ver, unit, prev) return nil } if err := systemctl("try-restart", unit); err != nil { fmt.Printf("update: service %s restart: %v (new build applies on its next start)\n", name, err) return nil } time.Sleep(serviceHealthSettle) if isActive(unit) { os.Remove(serviceFailedPath(name)) fmt.Printf("update: service %s -> %s (%s healthy; rollback at %s)\n", name, ver, unit, prev) return nil } // The bounced unit didn't stay up: restore the known-good binary, bounce it // back, and quarantine the head so the hourly pass doesn't flap forever. log.Printf("update: service %s -> %s is UNHEALTHY — rolling back", name, ver) if rberr := os.Rename(prev, path); rberr != nil { return fmt.Errorf("SERVICE %s BRICKED: new build unhealthy AND restoring %s failed: %v", name, prev, rberr) } if err := systemctl("try-restart", unit); err != nil { log.Printf("update: service %s rollback restart: %v", name, err) } if werr := os.WriteFile(serviceFailedPath(name), []byte(ver+"\n"), 0o644); werr != nil { log.Printf("update: service %s: persist quarantine: %v", name, werr) } return fmt.Errorf("build %s failed its restart — rolled back and quarantined", ver) }