package activator import ( "context" "errors" "fmt" "time" "github.com/carlosframework/platform/internal/registry" "github.com/carlosframework/platform/internal/release" ) // The held wake check's refusals (settings spec §4.3 rule 3). None of them // ran anything of a build, so none is a failure of one. Members read them // through memberSafeWakeReason (ruling R-WAKEWORDS), never as this text. var ( errBuildUnknown = errors.New("build unknown: use Update now or set a pin") errStaleLink = errors.New("the installed build is not this instance's build, and root has not installed it yet") errTargetFailed = errors.New("this instance's build failed on this box, so it is not started again") ) // targetFailedError is errTargetFailed naming the build, so the member's // sentence can name it too. type targetFailedError struct{ host, build string } func (e *targetFailedError) Error() string { return fmt.Sprintf("%v: %s on %s", errTargetFailed, e.build, e.host) } func (e *targetFailedError) Unwrap() error { return errTargetFailed } // HeldWake is the agent's answer for a held instance's wake (settings spec // §4.3 rule 3). type HeldWake struct { Target string // the build the instance must run Held bool // false: root follows the channel for it; today's wake Unknown bool // rule 3's "build unknown" Failed bool // root reports Target failed on this box (its decision, or a refusal record) // AskRoot: root has made no decision for the instance's current // settings (ruling R-HELD-PURE), so whether it is held is not known // here. The wake holds: it runs Target only if the live link already // names it, and otherwise asks root for a pass and waits for its // decision. Target is the build state's build ("" when there is none). AskRoot bool // Unread: the build state could not be read for this wake (its required // refresh failed). Refused: never decided from a stale copy (Astra round // 1 finding 1). Unread bool // Staged is a retained copy of Target on this box whose bytes match its // staged manifest; nil when there is none. Staged *release.StagedIdentity } // heldPoll is how often a held wake re-reads the live link while it waits // for root to install its target. A var so tests can shrink it. var heldPoll = 250 * time.Millisecond // wakeCheck is settings spec §4.3 rule 3's wake half, with its own admission // rule so it holds with CARLOS_SWAP off and no planner (Astra I3 #2, #5): // // - not held: "" and nothing to do, today's wake; // - unread (the build state could not be read for this wake): // errBuildStateUnwritten, nothing asked or run (Astra round 1 finding 1); // - root has not decided for the current settings (AskRoot, ruling // R-HELD-PURE): the build state's build runs if the live link already // names it; otherwise root is asked for a pass (RequestStage naming no // build) and the wake waits for its decision, within the same bound, // then goes on below with that answer. It never runs a live link it // does not know it may run; // - unknown: errBuildUnknown, and root is not asked (it installs nothing); // - a target root reports failed: refused (errTargetFailed) with no // planner, even while it is still the live link; with a planner the // wake is not held, and zdd-5's failure-aware plan chooses (recovery // onto the good build, or a refusal); // - the target installed: the target, and the caller checks the identity // it captures against it (heldIdentityOK); // - with no planner and direct set (the caller can exec a named file), a // retained verified copy of the target, which the caller runs as its // identity, recorded first; root is asked to install the target too, // without waiting (R-RETAINED-NUDGE); // - otherwise root is asked to install the target (RequestStage, the // stage relay) and the wake waits for the live link to name it, within // its own deadline and never past wakeWaitBound, the bound its waiters // have. A wait that ends first is errStaleLink. The answer is asked // again every heldReask polls, and a changed answer ends the wait at // once with its own refusal (unknown, failed) or errStaleLink. // // It never sets e.pin (Astra I2 #4): with a planner, what boots after root's // install is the plan's. The string is the target ("" when the wake is not // held). func (a *Service) wakeCheck(ctx context.Context, host string, direct bool) (string, *release.StagedIdentity, error) { if a.HeldTarget == nil { return "", nil, nil } h := a.HeldTarget(host) if !h.Held { return "", nil, nil } if h.Unread { return "", nil, fmt.Errorf("%w: the build state of %s could not be read for its wake", errBuildStateUnwritten, host) } wait, cancel := context.WithTimeout(ctx, wakeWaitBound) defer cancel() if h.AskRoot { var err error if h, err = a.awaitRootDecision(wait, host, h); err != nil { return "", nil, err } switch { case !h.Held: return "", nil, nil case h.Unread: return "", nil, fmt.Errorf("%w: the build state of %s could not be read for its wake", errBuildStateUnwritten, host) } } if h.Unknown || h.Target == "" { return "", nil, errBuildUnknown } if h.Failed { if a.Planner != nil { return "", nil, nil // zdd-5's decideWake: recovery onto the good build, or a refusal } return "", nil, &targetFailedError{host: host, build: h.Target} } installed := a.liveVersion(host) if installed == h.Target { return h.Target, nil, nil } if direct && a.Planner == nil && h.Staged != nil && h.Staged.Version == h.Target { // Run the retained copy; step 2b records it first. Root is still // asked to install the target, without waiting (ruling // R-RETAINED-NUDGE): until it repoints, the live link names another // build than the one running, and the sweep's convergence would // cycle the instance after ConvergeGrace, waking it onto the same // copy each time, until root's own pass. if a.RequestStage != nil { a.RequestStage(host, h.Target) } return h.Target, h.Staged, nil } if a.RequestStage != nil { a.RequestStage(host, h.Target) } tick := time.NewTicker(heldPoll) defer tick.Stop() for n := 1; ; n++ { if a.liveVersion(host) == h.Target { return h.Target, nil, nil } if n%heldReask == 0 && wait.Err() == nil { // The answer is asked again while the wake waits: a member's // change (a pin, Update now, auto) or root's verdict (failed, // unknown) ends this wait now, rather than at its bound onto a // target that is no longer the instance's. The next wake checks // afresh. The re-ask is bounded by the wait (Task 12a): it may // read the bucket for up to its own budget, and an answer that // comes back after the wait has ended is not waited for. next, ok := a.reaskHeld(wait, host) if !ok { return "", nil, fmt.Errorf("%w: %s is on %q, wants %s", errStaleLink, host, installed, h.Target) } if !sameHeld(next, h) { switch { case next.Held && (next.Unknown || next.Target == ""): return "", nil, errBuildUnknown case next.Held && next.Failed && a.Planner == nil: return "", nil, &targetFailedError{host: host, build: next.Target} } return "", nil, fmt.Errorf("%w: %s's target changed from %s while it waited", errStaleLink, host, h.Target) } } select { case <-wait.Done(): return "", nil, fmt.Errorf("%w: %s is on %q, wants %s", errStaleLink, host, installed, h.Target) case <-tick.C: } } } // heldReask is how many polls a held wake's wait makes between asking // HeldTarget again (which may read the bucket). const heldReask = 8 // awaitRootDecision is wakeCheck's wait for root when root has made no // decision for the instance's current settings (h.AskRoot; ruling // R-HELD-PURE). It returns h itself when the wake may go on now: its target // failed (the failure rule decides, below) or the live link already names // it, the build the data last ran. Otherwise it asks root for a pass, // naming no build (it does not know which root will install), and asks // HeldTarget again every heldReask polls until root's answer is no longer // AskRoot, which it returns; or the live link names the target (root decided // to hold it and installed it). A wait that ends first is errStaleLink. func (a *Service) awaitRootDecision(wait context.Context, host string, h HeldWake) (HeldWake, error) { if h.Failed || (h.Target != "" && a.liveVersion(host) == h.Target) { return h, nil } if a.RequestStage != nil { a.RequestStage(host, "") } tick := time.NewTicker(heldPoll) defer tick.Stop() for n := 1; ; n++ { if h.Target != "" && a.liveVersion(host) == h.Target { return h, nil } if n%heldReask == 0 && wait.Err() == nil { next, ok := a.reaskHeld(wait, host) if !ok { break } if !next.AskRoot { return next, nil } h = next } select { case <-wait.Done(): return HeldWake{}, fmt.Errorf("%w: root has not decided for %s's current update settings", errStaleLink, host) case <-tick.C: } } return HeldWake{}, fmt.Errorf("%w: root has not decided for %s's current update settings", errStaleLink, host) } // reaskHeld is HeldTarget(host) bounded by wait: false when wait ends first. // The call itself is not cancelled (HeldTarget takes no context); it finishes // within its own budget into a buffered channel nobody reads. func (a *Service) reaskHeld(wait context.Context, host string) (HeldWake, bool) { got := make(chan HeldWake, 1) go func() { got <- a.HeldTarget(host) }() select { case h := <-got: return h, true case <-wait.Done(): return HeldWake{}, false } } // sameHeld reports whether two answers send the wake to the same place. func sameHeld(x, y HeldWake) bool { return x.Held == y.Held && x.Target == y.Target && x.Unknown == y.Unknown && x.Failed == y.Failed && x.AskRoot == y.AskRoot && x.Unread == y.Unread } // liveVersion is what live/ names now, "" when it cannot be read. func (a *Service) liveVersion(host string) string { if a.liveVer == nil { return "" } v, _ := a.liveVer(host) return v } // heldIdentityOK reports whether a held wake may run id: its target, or the // good build the planner chose as recovery because the failure book blocks // the target (zero-downtime failure records outrank every rule, §4.3). An // identity with no version (a live link outside the version store) is never // the target (ruling R-EXEC-NOVERSION-2). func heldIdentityOK(target string, id release.StagedIdentity, plan WakePlan, rt registry.Route) bool { if target == "" || id.Version == target { return true } return plan.planned() && !plan.FirstWake && id.Version != "" && id.Version == rt.GoodVersion }