// The settle check — D1 of the console release defences (spec // 2026-10-06-console-release-defences-design.md §2.2-§2.4). // // `systemctl try-restart` of a Type=simple unit returns 0 as soon as the start // job is queued, so it says nothing about whether the new build runs. On // 2026-10-06 the console's new process printed the CLI's usage and exited 2 // after 55 ms while try-restart reported success, and adoptHost's rollback // never fired. settleUnit watches the unit after the restart and returns // *release.UnitNotUpError when the new build does not come up, which drives // that rollback. package host import ( "context" "encoding/json" "errors" "fmt" "io" "log" "net" "net/http" "os" "os/exec" "regexp" "strconv" "strings" "time" "github.com/carlosframework/platform/internal/provision" "github.com/carlosframework/platform/internal/registry" "github.com/carlosframework/platform/internal/release" ) type unitProps struct { Name string FragmentPath string DropInPaths []string ActiveState string SubState string // "dead" with ActiveState inactive is a stop, not a crash InvocationID string MainPID int NRestarts int ExecPath string // ExecStart's executable path ActiveEnterTimestamp time.Time // when the unit last entered active; zero when unknown } type unitSystem interface { Show(unit string) (unitProps, error) TryRestart(unit string) error Restart(unit string) error ResetFailed(unit string) error JournalTail(unit string, n int) string Exe(pid int) (string, error) StartTime(pid int) (string, error) } type systemdUnits struct{} func (systemdUnits) Show(unit string) (unitProps, error) { out, err := exec.Command("systemctl", "show", unit, "-p", "Id", "-p", "FragmentPath", "-p", "DropInPaths", "-p", "ActiveState", "-p", "SubState", "-p", "InvocationID", "-p", "MainPID", "-p", "NRestarts", "-p", "ExecStart", "-p", "ActiveEnterTimestamp").Output() if err != nil { return unitProps{}, fmt.Errorf("systemctl show %s: %w", unit, err) } return parseShow(string(out)), nil } func (systemdUnits) TryRestart(unit string) error { return systemctlTryRestartUnit(unit) } func (systemdUnits) Restart(unit string) error { return systemctlVerb("restart", unit) } func (systemdUnits) ResetFailed(unit string) error { return systemctlVerb("reset-failed", unit) } func (systemdUnits) JournalTail(unit string, n int) string { out, _ := exec.Command("journalctl", "-u", unit, "-o", "cat", "-n", strconv.Itoa(n), "--no-pager").Output() return strings.TrimSpace(string(out)) } func (systemdUnits) Exe(pid int) (string, error) { return os.Readlink(fmt.Sprintf("/proc/%d/exe", pid)) } func (systemdUnits) StartTime(pid int) (string, error) { if pid <= 0 { return "", os.ErrPermission } body, err := os.ReadFile(fmt.Sprintf("/proc/%d/stat", pid)) if err != nil { return "", err } i := strings.LastIndexByte(string(body), ')') if i < 0 { return "", os.ErrPermission } fields := strings.Fields(string(body[i+1:])) if len(fields) <= 19 { return "", os.ErrPermission } if n, err := strconv.ParseUint(fields[19], 10, 64); err != nil || n == 0 { return "", os.ErrPermission } return fields[19], nil } func systemctlVerb(verb, unit string) error { out, err := exec.Command("systemctl", verb, unit).CombinedOutput() if err != nil { return fmt.Errorf("systemctl %s %s: %w: %s", verb, unit, err, strings.TrimSpace(string(out))) } return nil } var execPathRE = regexp.MustCompile(`path=([^ ;]+)`) func parseShow(out string) unitProps { var p unitProps for _, line := range strings.Split(out, "\n") { k, v, ok := strings.Cut(strings.TrimSpace(line), "=") if !ok { continue } switch k { case "Id": p.Name = v case "FragmentPath": p.FragmentPath = v case "DropInPaths": p.DropInPaths = strings.Fields(v) case "ActiveState": p.ActiveState = v case "SubState": p.SubState = v case "InvocationID": p.InvocationID = v case "MainPID": p.MainPID, _ = strconv.Atoi(v) case "NRestarts": p.NRestarts, _ = strconv.Atoi(v) case "ExecStart": if m := execPathRE.FindStringSubmatch(v); m != nil { p.ExecPath = m[1] } case "ActiveEnterTimestamp": p.ActiveEnterTimestamp = parseSystemdTime(v) } } return p } // parseSystemdTime reads systemctl show's default timestamp form, // "Mon 2026-10-06 09:30:30 UTC". Boxes run in UTC. "" and "n/a" are zero, // which every caller reads as "unknown", never as "long ago". func parseSystemdTime(v string) time.Time { t, err := time.Parse("Mon 2006-01-02 15:04:05 MST", strings.TrimSpace(v)) if err != nil { return time.Time{} } return t } type settleOpts struct { RequireExe bool Timeout time.Duration // the whole window (CARLOS_UNIT_SETTLE) Poll time.Duration // 500ms Hold time.Duration // no-probe readiness: active this long in one invocation WantDir string // the new version's directory; MainPID's exe must be under it Ready func() error // nil = no upstream to probe; Hold applies Now func() time.Time Sleep func(time.Duration) } // errUnitStopped is settleUnit's answer for a unit that was stopped while it // was being watched (inactive, SubState dead): an operator's stop, not a // crash. It is neither a healthy build nor a failed one, so the caller judges // nothing and restarts nothing. var errUnitStopped = errors.New("unit was stopped during settle") // settleUnit watches unit after a try-restart. beforeInvocation is the // InvocationID before the restart; "" means the unit was not running, and an // inactive unit is then "nothing started", not a failure. Otherwise a unit // that goes inactive/dead returns errUnitStopped. func settleUnit(sys unitSystem, unit, beforeInvocation string, o settleOpts) error { deadline := o.Now().Add(o.Timeout) var inv string // the new invocation, once seen active var activeSince time.Time var lastReady error var lastExeErr error // wrongExe is the executable MainPID ran on this poll when it was not // under WantDir. Not a failure on its own: for Type=simple systemd reports // the new invocation active as soon as it has forked, BEFORE it execs the // ExecStart binary, so /proc//exe can briefly name systemd's own // executor (systemd-executor, or systemd itself on older releases). It is // treated like not-ready, and only a deadline that still sees it fails. var wrongExe string var binCheckSkipped bool // permission error on Exe; skip binary check for rest of settle notUp := func(reason string) error { return &release.UnitNotUpError{Unit: unit, Reason: reason, Journal: sys.JournalTail(unit, 20)} } for { p, err := sys.Show(unit) if err != nil { // One failed `systemctl show` says nothing about the build (a // busy bus, a timed-out D-Bus call). Retried until the deadline, // and only a deadline that still cannot observe the unit fails. if !o.Now().Before(deadline) { return notUp(fmt.Sprintf("cannot observe %s: %v", unit, err)) } o.Sleep(o.Poll) continue } // Inactive with SubState dead and nothing pending is a STOP: a crash // reads failed, and a crash systemd will restart reads activating // (auto-restart). try-restart blocks until its start job is done, so // this is not the gap between the stop and start halves of the // restart either. Somebody stopped the unit on purpose, before or // during this check; the caller must neither judge nor restart it. if beforeInvocation != "" && p.ActiveState == "inactive" && p.SubState == "dead" { return errUnitStopped } if beforeInvocation == "" && p.ActiveState == "inactive" { return nil } // NRestarts check on fresh invocation, before the switch if p.InvocationID != beforeInvocation && p.InvocationID != "" && p.NRestarts > 0 { return notUp(fmt.Sprintf("systemd restarted it %d time(s)", p.NRestarts)) } switch { case p.ActiveState == "failed": return notUp("unit entered failed state") case inv != "" && p.InvocationID != inv: return notUp("systemd restarted it (new invocation)") case inv != "" && p.NRestarts > 0: return notUp(fmt.Sprintf("systemd restarted it %d time(s)", p.NRestarts)) case inv == "" && p.ActiveState == "active" && p.InvocationID != beforeInvocation: inv, activeSince = p.InvocationID, o.Now() } wrongExe = "" if inv != "" && p.ActiveState == "active" { // Check the binary only if we haven't hit a permission error if !binCheckSkipped { exe, err := sys.Exe(p.MainPID) if err != nil { if errors.Is(err, os.ErrPermission) && !o.RequireExe { // Permission error; log once and skip binary check for rest log.Printf("adopt: %s: cannot read /proc/%d/exe (%v); binary check skipped", unit, p.MainPID, err) binCheckSkipped = true } else { // Other error; remember it and skip Ready/Hold on this poll lastExeErr = err // Continue to next poll without calling Ready/Hold } } else if o.WantDir != "" && !strings.HasPrefix(exe, strings.TrimSuffix(o.WantDir, "/")+"/") { wrongExe = exe // not yet the new build: not ready, poll again } } // Only call Ready/Hold once this poll saw the new build running if lastExeErr == nil && wrongExe == "" { if o.Ready != nil { if lastReady = o.Ready(); lastReady == nil { return nil } } else if o.Now().Sub(activeSince) >= o.Hold { return nil } } } if !o.Now().Before(deadline) { if wrongExe != "" { return notUp(fmt.Sprintf("running %s, not the new build in %s", wrongExe, o.WantDir)) } if lastExeErr != nil { return notUp(fmt.Sprintf("exe unreadable: %v", lastExeErr)) } if lastReady != nil { return notUp(fmt.Sprintf("not ready after %s: %v", o.Timeout, lastReady)) } return notUp(fmt.Sprintf("not ready after %s", o.Timeout)) } lastExeErr = nil // reset exe error for next poll o.Sleep(o.Poll) } } func unitSettleTimeout(getenv func(string) string) time.Duration { d, err := time.ParseDuration(strings.TrimSpace(getenv("CARLOS_UNIT_SETTLE"))) switch { case err != nil || d <= 0: return 30 * time.Second case d < 5*time.Second: return 5 * time.Second } return d } // routeReady is the readiness check for rt's effective upstream (spec §2.4): // Socket wins over Addr. With a configured probe it is an HTTP GET; without // one, a connect. nil when the route has no upstream: settle then requires // the unit to stay active for its Hold. func routeReady(rt registry.Route, timeout time.Duration) func() error { network, address := "unix", rt.Socket if address == "" { network, address = "tcp", rt.Addr } if address == "" { return nil } if rt.Health == "" { return func() error { c, err := net.DialTimeout(network, address, timeout) if err != nil { return fmt.Errorf("connect %s: %w", address, err) } return c.Close() } } var h provision.Health if err := json.Unmarshal([]byte(rt.Health), &h); err != nil { return func() error { return fmt.Errorf("route health probe does not parse: %w", err) } } h = h.WithDefaults() hostHdr := h.HostHeader if hostHdr == "" { hostHdr = rt.Host } client := &http.Client{Timeout: timeout, CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }, Transport: &http.Transport{ DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) { var d net.Dialer return d.DialContext(ctx, network, address) }, }} return func() error { req, err := http.NewRequest(http.MethodGet, "http://upstream"+h.Path, nil) if err != nil { return err } req.Host = hostHdr resp, err := client.Do(req) if err != nil { return fmt.Errorf("dial: %v", err) } io.Copy(io.Discard, io.LimitReader(resp.Body, 4<<10)) resp.Body.Close() if resp.StatusCode != h.Status { return fmt.Errorf("status %d (want %d)", resp.StatusCode, h.Status) } return nil } }