// wakeplan.go is the activator's half of the rollout scheduler (zero-downtime
// deploys spec §6.4, §8.2): which build a wake boots, the first-wake budget,
// the fallback to the good build, and the 10 s observation. All of it is
// reached through Service.Planner, which only CARLOS_SWAP=overlap sets. With
// Planner nil (the flag off) nothing here runs.
package activator

import (
	"context"
	"errors"
	"fmt"
	"log"
	"sync"
	"time"

	"github.com/carlosframework/platform/internal/registry"
	"github.com/carlosframework/platform/internal/release"
	"github.com/carlosframework/platform/internal/rolloutwords"
)

// WakePlan is the scheduler's answer to "which build does this wake boot"
// (spec §8.2). The zero plan keeps today's wake.
type WakePlan struct {
	// Boot is the artifact to exec, by its immutable staged path.
	Boot release.StagedIdentity
	Mode SwapMode
	// FirstWake: Boot has never served on this instance. The wake is a canary
	// for it, with the 55 s budget and the 10 s observation.
	FirstWake bool
	// Fallback is the good build, booted when a first wake in overlap mode
	// fails. Nil when there is none.
	Fallback *release.StagedIdentity
	// Refuse, when set, fails the wake before the lease with this reason.
	Refuse string
	// FailedUnder is the PromotedAt of the pointer Boot was promoted under,
	// read by the planner before the wake runs, so a failed first wake is
	// published with no blocking read (spec §8.1), as dispatch.FailedUnder
	// is for an attempt. Zero: the failure's own time stands in.
	FailedUnder time.Time
	// Check is the planner's selection gate (spec §7.4, §8.1–8.4): nil when
	// build may still boot. Planning runs before the lease and the restore,
	// so a sibling's failure recorded since can hold the build it picked.
	// The activator asks it at every planned exec, after the restore; the
	// good build passes unless a hold build has run on the instance since it
	// last passed (the scheduler's hold-ran marker), where nothing falls
	// back. Nil: nothing to re-check. It is called without a.mu held.
	Check func(build string) error
	// Token identifies this planned wake (final review F4). The planner
	// assigns it, unique per plan it makes (a replan, checkPlan's, is a new
	// plan with a new token), before it registers anything for the wake.
	// The activator never makes or changes one: it copies the token of the
	// plan the wake ran, unchanged, into every GenEvent of that wake,
	// afterWake's and the first-wake observer's, so the scheduler can act
	// only on the wake it registered. Zero: the planner matches nothing by
	// token.
	Token uint64
	// ConfigRev is the config revision the planner snapshotted for this
	// wake: the delivered revision to activate once the wake serves. The
	// activator does not know which revision a wake started with (it reads
	// the instance's env file at the spawn), so it carries this one,
	// unchanged, into every GenEvent of the wake, beside Token.
	ConfigRev string
}

func (p WakePlan) planned() bool { return p.Boot.StagedPath != "" }

// errPlanStale: the planned build is no longer the one to boot. A
// supersession (gateSuperseded), so it records nothing and falls back to
// nothing (classifyStep's stepStale).
var errPlanStale = fmt.Errorf("the planned build is no longer the one to boot: %w", errSwapSuperseded)

// checkPlan is a planned wake's selection gate before its spawn (spec §7.4,
// §8.1–8.4), after the lease and the restore: a refused plan is planned
// again once, on the state already restored, and a second refusal abandons
// the wake (errPlanStale). It returns the plan to boot. The exec itself asks
// the returned plan's Check once more (planGate), with no second replan.
// The planner and Check are called without a.mu held.
func (a *Service) checkPlan(ctx context.Context, e *actEntry, rt registry.Route, plan WakePlan) (WakePlan, error) {
	if plan.Check == nil {
		return plan, nil
	}
	err := plan.Check(plan.Boot.Version)
	if err == nil {
		return plan, nil
	}
	if a.Planner == nil {
		return plan, fmt.Errorf("%w: %v", errPlanStale, err)
	}
	next := a.Planner(ctx, rt)
	if next.Refuse != "" || !next.planned() {
		return plan, fmt.Errorf("%w: %v; replanned: %s", errPlanStale, err, next.Refuse)
	}
	if next.Check != nil {
		if nerr := next.Check(next.Boot.Version); nerr != nil {
			// The replan never becomes the wake's plan, so afterWake will
			// report only the first plan: the replan's own terminal event
			// is sent here (rollout Task 5). Nothing of it ran. A planner
			// that handed out the same token again gets one event for it.
			if next.Token != plan.Token {
				a.cancelPlanned(rt, next, false)
			}
			return plan, fmt.Errorf("%w: %v", errPlanStale, nerr)
		}
	}
	a.mu.Lock()
	e.plan, e.firstWake = next, next.FirstWake
	a.mu.Unlock()
	// The replaced plan's build never ran (its check refused it before the
	// exec), and afterWake now reports the replan: the replaced plan's
	// terminal event is sent here (rollout Task 5).
	if plan.Token != next.Token {
		a.cancelPlanned(rt, plan, false)
	}
	// The replanned build (and its fallback) is published before it execs
	// (spec §4.2, §8.5). A failed write execs neither: the wake fails with
	// errStagedBuildNotPublished, which is abandonment, never a failure.
	if err := a.publishBeforeExec(); err != nil {
		return next, err
	}
	return next, nil
}

// wakeOwned is the ownership check at a wake's exec (spec §7.4): the wake's
// lifecycle has not ended, and plan 3's ownedLocked holds for a waking
// entry (the entry is the host's, no fence, no shutdown; its lease is the
// wake's own, which planGate checks beside it).
func (a *Service) wakeOwned(ctx context.Context, e *actEntry) error {
	if err := ctx.Err(); err != nil {
		return err
	}
	a.mu.Lock()
	defer a.mu.Unlock()
	return a.ownedLocked(e, true)
}

// planGate is the gate every planned exec passes, B's and its fallback's
// alike, asked immediately before the exec (startPinnedAt's gate, or
// startWorker's before-exec hook): the wake's locally held lease
// (leaseLost; the entry has no replica yet, so ownedLocked cannot see it),
// the ownership check (wakeOwned), and the plan's selection gate for the
// build about to exec. Every refusal is abandonment, never a failure of the
// build: the lease and ownership refusals are gateNoOwnership, the
// selection gate's is errPlanStale (gateSuperseded), a cancelled wake is
// the lifecycle's. Check runs after a.mu is released.
func (a *Service) planGate(ctx context.Context, e *actEntry, lease Lease, plan WakePlan, build string) func() error {
	return func() error {
		if err := leaseLost(lease); err != nil {
			return err
		}
		if err := a.wakeOwned(ctx, e); err != nil {
			return err
		}
		if plan.Check != nil {
			if err := plan.Check(build); err != nil {
				return fmt.Errorf("%w: %v", errPlanStale, err)
			}
		}
		return nil
	}
}

func genOf(id release.StagedIdentity) Gen {
	return Gen{Build: id.Version, StagedPath: id.StagedPath, Digest: id.Digest, Demand: id.Demand}
}

// workerGenOf is a worker's generation as the scheduler sees it: its
// captured staged artifact, marked Worker, with no number. The scheduler
// compares a worker's generations by their artifact (rollout's
// acknowledged), because a respawn on the same entry changes the build
// without a number of its own.
func workerGenOf(id release.StagedIdentity) Gen {
	g := genOf(id)
	g.Worker = true
	return g
}

// withIdentity is g, the generation a planned exec started, carrying the
// staged identity it was booted from (PF10, spec §12 "Legacy sidecar
// binding"): the build, its staged path, digest and demand. The legacy
// sidecar co-spawned with it execs that staged path (startSidecar), and the
// scheduler records it as good by its digest.
func withIdentity(g Gen, id release.StagedIdentity) Gen {
	g.Build, g.StagedPath, g.Digest, g.Demand = id.Version, id.StagedPath, id.Digest, id.Demand
	return g
}

// GenEventKind names what happened to a planned wake.
type GenEventKind string

const (
	GenHealthy  GenEventKind = "healthy"  // the planned build is serving
	GenSettled  GenEventKind = "settled"  // a first wake survived its 10 s
	GenFailed   GenEventKind = "failed"   // the planned build failed, nothing fell back
	GenFellBack GenEventKind = "fellback" // the planned build failed, the good build serves
	// GenCancelled: a first wake's observation ended without a verdict (an
	// intentional stop, spec §6.4). It records nothing about the build; it
	// only closes the observation GenHealthy opened.
	GenCancelled GenEventKind = "cancelled"
)

// GenEvent is a planned wake's outcome, for the scheduler.
type GenEvent struct {
	Kind      GenEventKind
	Route     registry.Route
	Gen       Gen                    // the generation serving (healthy, settled, fellback)
	Failed    release.StagedIdentity // the build that failed (failed, fellback)
	Mode      SwapMode
	FirstWake bool
	Reason    string
	At        time.Time
	// FailedUnder is the plan's FailedUnder (failed, fellback): the
	// failing pointer's PromotedAt, read before the wake ran.
	FailedUnder time.Time
	// Token and ConfigRev are those of the plan the wake ran
	// (WakePlan.Token, WakePlan.ConfigRev; final review F4). Every event of
	// one planned wake carries the same pair, on every kind.
	Token     uint64
	ConfigRev string
	// Executed (cancelled only): the first wake's build ran, or may have,
	// before the wake ended with no verdict, and Gen.Build names it (final
	// review wave 2, N2). Always so for a cancelled observation, which
	// follows a healthy wake. A hold first wake that its lifecycle ended
	// after the exec (a stop, a fence, a failure after the spawn) is
	// reported the same way: the build may have migrated the database, so
	// the scheduler must not boot the good build there next.
	Executed bool
}

// The first-wake budget (spec §8.2). Vars so tests can shrink them.
var (
	firstWakeBudget      = 55 * time.Second
	firstWakeKillReserve = 25 * time.Second
	firstWakeKillBound   = 2 * time.Second
	firstWakeSettle      = 10 * time.Second
	// firstWakeMinVerdict is the least health window B is exec'd with on a
	// first wake that has a good build. B's window is fixed from the wake's
	// start, so a slow lease, restore or seeding eats it; below this floor
	// a failure would be the clock's, not the build's, and the good build
	// boots instead with nothing recorded against B.
	firstWakeMinVerdict = 20 * time.Second
	// outcomeFenceWait bounds how long a new wake waits for a planned wake's
	// outcome to be delivered before it plans (final review F3). OnGen's
	// work is in memory plus one local failure-book write, so it normally
	// returns in milliseconds; 5 s covers a slow disk with room to spare and
	// still leaves the waiting requests most of their wakeWaitBound (30 s).
	// It bounds the waiting request, never the restriction (final review
	// wave 2, N1): past it the wake is refused before it plans, and the
	// fence stays until the outcome is delivered. A later request retries.
	outcomeFenceWait = 5 * time.Second
)

// genKiller (plan 3 Task 2, opset.go) is plan 2's generation kill
// (ExecBackend.KillGen and its SplitBackend forward): cgroup.kill on
// generation n's child cgroup, then wait until it is empty. This file uses
// it and declares none.

// Current is the generation an awake instance serves, for the scheduler: a
// worker's is its captured spawn build (workerGenOf). A first wake under its
// 10 s observation reads as not awake, so the scheduler leaves it alone
// until the observation ends.
func (a *Service) Current(host string) (Gen, bool) {
	a.mu.Lock()
	defer a.mu.Unlock()
	e := a.entries[host]
	if e == nil || e.state != actAwake || e.firstWatch != nil {
		return Gen{}, false
	}
	if e.worker {
		return workerGenOf(e.workerServing), true
	}
	return e.gen, true
}

// BounceFor is Bounce for a scheduled attempt: the event names the attempt's
// cause rather than "carlos restart" (spec §9.4).
func (a *Service) BounceFor(host string, cause Cause) bool {
	a.mu.Lock()
	defer a.mu.Unlock()
	e := a.entries[host]
	if e == nil || e.state != actAwake {
		return false
	}
	emitEvent(a.Logs, e.route, "info", bounceText(e.worker, cause))
	a.beginStopLocked(e, stopWhyFor(cause))
	return true
}

// stopWhyFor is the run-interruption reason (run spec §5.3) for a scheduled
// attempt's bounce: a new build for release, install and retry, a restart
// otherwise.
func stopWhyFor(cause Cause) string {
	switch cause {
	case CauseRelease, CauseInstall, CauseRetry:
		return StopRelease
	}
	return StopRestart
}

// bounceText is the one place BounceFor words its event. The words are
// internal/rolloutwords, approved copy. The flag-off literal in
// bounceLocked is never edited; restart's cause phrase keeps the two
// identical.
func bounceText(worker bool, cause Cause) string {
	format := rolloutwords.EventBounceInstance
	if worker {
		format = rolloutwords.EventBounceSidecar
	}
	return fmt.Sprintf(format, rolloutwords.Cause(string(cause)))
}

// killFailed ends a failed first-wake build outright (spec §8.2 step 1),
// bounded by firstWakeKillBound, and reports whether its end is proven. An
// instance generation is killed with cgroup.kill on its own child and waited
// empty (genKiller). Any error, errNoGenContainment included, is unproven:
// without a generation child there is no kill that reaches what B forked,
// and the good build must never start beside it. A worker is not a
// generation child (plan 2 D4): its proven stop, bounded the same, stands in
// for the kill, and THE termination proof (genTerminated on a Worker
// generation) empties and checks its host cgroup. Never called with a.mu
// held.
func (a *Service) killFailed(rt registry.Route, g Gen, p Proc, worker bool) error {
	ctx, cancel := context.WithTimeout(context.Background(), firstWakeKillBound)
	defer cancel()
	if !worker {
		k, ok := a.backend.(genKiller)
		if !ok {
			return fmt.Errorf("%s: this backend cannot kill a generation", rt.Host)
		}
		if err := k.KillGen(ctx, rt, g.N); err != nil {
			return err
		}
		// The kill proved the end. The handle is still released through
		// its Stop, the only close a flat child's control listener gets
		// (backend_exec.go): left bound, it is the stale-socket trap
		// (#129). Its own bound, since the kill may have spent this one.
		sctx, scancel := context.WithTimeout(context.Background(), firstWakeKillBound)
		defer scancel()
		_ = p.Stop(sctx)
		return nil
	}
	if err := p.Stop(ctx); err != nil {
		return err
	}
	if !a.genTerminated(rt, Gen{N: g.N, Build: g.Build, Worker: true}, p) {
		return ErrStillRunning
	}
	return nil
}

// spawnPlanned is a planned wake's instance spawn (spec §8.2). The work
// before the exec (lease, restore, seed) has already run once and is shared.
//
// Every exec, B's and its fallback's, goes through startPinnedAt with
// planGate (PF7): the lease, the ownership and the plan's selection gate,
// asked at the exec. A refusal there runs nothing and is abandonment.
//
// Execution evidence comes from the spawn's result, as startPinnedChecked
// reads it: a returned process, or an unproven-start sentinel. Without it
// nothing of the build ran and nothing is recorded. With it, only what
// classifyStep calls the build's failure is recorded (a cancelled wake, a
// stale attempt and a mismatched exec are not), and only for a first wake.
//
// For a first wake in overlap mode with a good build, B's health window
// ends at the wake's start plus 55 s minus 25 s, a failed B is killed
// outright, and the good build gets whatever time remains. When B's end is
// not proven, nothing more starts. When less than firstWakeMinVerdict of
// B's window is left at its exec (a slow lease, restore or seeding), B is
// not exec'd at all: the good build boots instead and nothing is recorded,
// since a health check cut that short would fail B for the clock's sake.
func (a *Service) spawnPlanned(ctx context.Context, e *actEntry, rt registry.Route, plan WakePlan, w *wokenInstance, lease Lease) (Proc, string, Gen, error) {
	plan, err := a.checkPlan(ctx, e, rt, plan)
	if err != nil {
		return nil, "", Gen{}, err
	}
	canFall := plan.FirstWake && plan.Fallback != nil && plan.Mode == ModeOverlap
	boot := plan.Boot
	gate := a.planGate(ctx, e, lease, plan, boot.Version)
	var healthBy time.Time
	var left time.Duration
	if canFall {
		healthBy = e.wakeStart.Add(firstWakeBudget - firstWakeKillReserve)
		gate = verdictFloorGate(gate, healthBy, &left)
	}
	// Every exec here that runs the good build in another build's place is
	// recorded with the planner's selection (ruling R-PLANNED-RECOVERY): a
	// recovery boot (normally recorded at step 2b already), the good build
	// booted untried and the fallback. Each exec's own context only.
	proc, ver, g, err := a.startPinnedAt(withRecoverySelection(ctx, plan, rt, boot.Version), rt, genOf(boot), gate, healthBy)
	w.bootRan = err == nil || startMayHaveRun(proc, err)
	if proc == nil && errors.Is(err, errNoTimeForVerdict) {
		// B never ran: too little of its window was left for a verdict at
		// the exec. The good build boots in its place through the same
		// gate, and the wake is reported as the good build's ordinary wake
		// (w.untried): no failure of B, no fallback, no observation.
		// Operator log only; nothing happened to the member's build.
		good := *plan.Fallback
		log.Printf("activator: %s: first wake of %s had %v of its health window left at the exec, under the %v floor; booting the good build %s without trying it",
			rt.Host, boot.Version, left.Round(time.Millisecond), firstWakeMinVerdict, good.Version)
		w.untried = true
		gctx := WithPlannedSelection(ctx, PlannedSelection{Build: good.Version, InPlaceOf: boot.Version})
		gproc, gver, gg, gerr := a.startPinnedAt(gctx, rt, genOf(good), a.planGate(ctx, e, lease, plan, good.Version), time.Time{})
		if gerr != nil {
			return gproc, gver, gg, fmt.Errorf("the good build %s, booted without trying %s: %w", good.Version, boot.Version, gerr)
		}
		return gproc, gver, withIdentity(gg, good), nil
	}
	if err == nil {
		return proc, ver, withIdentity(g, boot), nil
	}
	// A nested start root accepted whose outcome is uncertain, with no
	// process back, may have run B (Astra R3 ruling 206, zdd-readiness
	// Task 6). In hold mode that is always B's failure: nothing falls back
	// from a hold, and a wake reported as cancelled would leave the build
	// free to boot again. In overlap mode only a unit that exited before it
	// was observed (errStartedThenExited) is a proven crash at the start and
	// fails B; any other uncertain overlap start stays "may have run"
	// (afterWake's GenCancelled with Executed), unless its kill is
	// unproven, which fails B below as before.
	uncertain := proc == nil && startMayHaveRun(nil, err)
	crashed := uncertain && errors.Is(err, errStartedThenExited)
	uncertainFailure := uncertain && (plan.Mode == ModeHold || crashed)
	executed := proc != nil || genStartUnproven(err)
	if !uncertainFailure && (!executed || classifyStep(ctx, err) != stepFailed || errors.Is(err, errPinMismatch)) {
		// Nothing of the build ran, or the wake's lifecycle ended it
		// (maintenance, Shutdown, a stop), or its gate refused, or the
		// backend ran something else: no failure of the build (spec §6.4,
		// §7.4). Nothing is recorded and nothing falls back; doWake's
		// cleanup stops a returned process and proves it gone.
		return proc, ver, g, err
	}
	if plan.FirstWake {
		w.failed, w.failReason = &boot, err.Error()
	}
	if !canFall {
		return proc, ver, g, err
	}
	if proc == nil {
		// No handle to kill. The good build starts only beside a B whose
		// end is proven (plan review finding 9): a crash at the start that
		// the backend's own kill proved gone (no errGenUnproven in the
		// chain). Every other overlap start that reaches here with no
		// process carries errGenUnproven: its end is unproven, so nothing
		// more starts, the wake serves nothing, and doWake keeps the lease
		// and the generation as an unproven wake.
		if !crashed || genStartUnproven(err) {
			return proc, ver, g, err
		}
	} else if kerr := a.killFailed(rt, g, proc, false); kerr != nil {
		return proc, ver, g, fmt.Errorf("%w; its end was not proven, so the good build was not started: %v", err, kerr)
	}
	good := *plan.Fallback
	fctx := WithPlannedSelection(ctx, PlannedSelection{Build: good.Version, InPlaceOf: boot.Version})
	fproc, fver, fg, ferr := a.startPinnedAt(fctx, rt, genOf(good), a.planGate(ctx, e, lease, plan, good.Version), time.Time{})
	if ferr != nil {
		if fproc == nil && !genStartUnproven(ferr) {
			// Nothing of the good build ran (its gate refused: a fence, a
			// shutdown, a lost lease or entry since B's exec). B's failure is
			// still recorded (w.failed); its process, proven gone, is what
			// doWake's cleanup proves once more.
			return proc, ver, g, fmt.Errorf("%w; the good build was not started: %v", err, ferr)
		}
		return fproc, fver, fg, fmt.Errorf("%w; the good build %s failed too: %w", err, good.Version, ferr)
	}
	w.fellBack = true
	return fproc, fver, withIdentity(fg, good), nil
}

// errNoTimeForVerdict: at B's exec, less than firstWakeMinVerdict of its
// health window was left. B is not exec'd; it is never a failure of B.
var errNoTimeForVerdict = errors.New("too little of the first wake's health window is left for a verdict")

// verdictFloorGate is gate with the first-wake floor asked after it, at
// the exec: B's window is the earlier of exec + spawnHealthTimeout and
// healthBy, less now. Below firstWakeMinVerdict it refuses with
// errNoTimeForVerdict and stores the window in *left for the log. The
// ownership, lease and plan checks go first, so their refusal (abandonment)
// is never mistaken for a lack of time.
func verdictFloorGate(gate func() error, healthBy time.Time, left *time.Duration) func() error {
	return func() error {
		if err := gate(); err != nil {
			return err
		}
		now := time.Now()
		end := now.Add(spawnHealthTimeout)
		if healthBy.Before(end) {
			end = healthBy
		}
		if *left = end.Sub(now); *left < firstWakeMinVerdict {
			return errNoTimeForVerdict
		}
		return nil
	}
}

// spawnPlannedWorker is spawnPlanned for a socketless child (spec §6.5).
// Each spawn uses startWorker's contract exactly as doWake's worker branch
// does (PF8): the build is published before its exec, planGate is asked at
// the exec (after the publication), and on success the captured identity is
// installed and the pending publication dropped in one critical section,
// then published again. A backend without the staged capability cannot
// exec a named build, so a planned worker is refused there before anything
// runs (errSwapNoGens). plannedWorkerReady stands in for the health check,
// and a failed release-cause first wake in overlap mode is replaced by the
// good build once the failed child's host cgroup is proven empty.
func (a *Service) spawnPlannedWorker(ctx context.Context, e *actEntry, rt registry.Route, plan WakePlan, env map[string]string, w *wokenInstance, lease Lease) (Proc, release.StagedIdentity, error) {
	if _, ok := a.backend.(stagedWorkerStarter); !ok {
		return nil, release.StagedIdentity{}, errSwapNoGens
	}
	plan, err := a.checkPlan(ctx, e, rt, plan)
	if err != nil {
		return nil, release.StagedIdentity{}, err
	}
	start := func(sctx context.Context, id release.StagedIdentity) (Proc, release.StagedIdentity, error) {
		p, got, pend, err := a.startWorker(sctx, rt, env, &id, a.planGate(ctx, e, lease, plan, id.Version))
		if err != nil {
			return p, got, err
		}
		a.mu.Lock()
		e.workerServing = got
		a.dropPendingGenerationLocked(pend)
		a.mu.Unlock()
		_ = a.persistGenerations()
		if got.StagedPath != id.StagedPath {
			return p, got, fmt.Errorf("%w: worker build %s from %q; the plan chose %s from %q", errPinMismatch, got.Version, got.StagedPath, id.Version, id.StagedPath)
		}
		return p, got, plannedWorkerReady(ctx, p)
	}
	canFall := plan.FirstWake && plan.Fallback != nil && plan.Mode == ModeOverlap
	boot := plan.Boot
	proc, id, err := start(withRecoverySelection(ctx, plan, rt, boot.Version), boot)
	w.bootRan = err == nil || startMayHaveRun(proc, err)
	if err == nil || proc == nil {
		// Ready, or nothing of the build ran: a refusal at the gate, an
		// unpublished build (ErrBuildNotPublished), a spawn error with no
		// child.
		return proc, id, err
	}
	if classifyStep(ctx, err) != stepFailed || errors.Is(err, errPinMismatch) {
		// A cancelled readiness window is the wake's lifecycle, not the
		// build (spec §6.4, §6.5): nothing recorded, nothing falls back.
		return proc, id, err
	}
	if plan.FirstWake {
		w.failed, w.failReason = &boot, err.Error()
	}
	if !canFall {
		return proc, id, err
	}
	if kerr := a.killFailed(rt, Gen{Build: id.Version}, proc, true); kerr != nil {
		return proc, id, fmt.Errorf("%w; its end was not proven, so the good build was not started: %v", err, kerr)
	}
	// The fallback is recorded with the planner's selection (ruling
	// R-PLANNED-RECOVERY), as spawnPlanned's is.
	fproc, fid, ferr := start(WithPlannedSelection(ctx, PlannedSelection{Build: plan.Fallback.Version, InPlaceOf: boot.Version}), *plan.Fallback)
	if ferr != nil {
		if fproc == nil {
			// Nothing of the good build ran: B's failure is still recorded,
			// and B's child, proven gone, is what doWake's cleanup proves.
			return proc, id, fmt.Errorf("%w; the good build was not started: %v", err, ferr)
		}
		return fproc, fid, fmt.Errorf("%w; the good build %s failed too: %w", err, plan.Fallback.Version, ferr)
	}
	w.fellBack = true
	return fproc, fid, nil
}

// plannedWorkerReady is spec §6.5's readiness for a socketless planned
// wake, the rule ReplaceWorker applies too: the child is ready once it is
// still running firstWakeSettle (10 s) after it started, and a clean exit
// inside that window is a one-shot's success. Any other exit fails. An early
// ready report does not shorten the window. The flag-off wake keeps
// awaitWorkerSettle (ready report, sidecarSettle, every early exit a
// failure), unchanged.
func plannedWorkerReady(ctx context.Context, p Proc) error {
	switch observeSettle(ctx, p, firstWakeSettle) {
	case settleOK:
		return nil
	case settleExited:
		if !procCrashed(p) {
			return nil
		}
		return errWorkerExitedEarly
	}
	if err := ctx.Err(); err != nil {
		return err
	}
	return context.Canceled
}

// outcomeFence counts a host's planned-wake outcomes that have been taken
// but not yet delivered to OnGen (final review F3). done closes when the
// count returns to zero.
type outcomeFence struct {
	n    int
	done chan struct{}
}

// holdOutcomeLocked takes one of host's outcome fences and returns its
// release. The release takes a.mu itself, so it is called off a.mu, after
// the outcome's OnGen has returned; only its first call counts. The caller
// holds a.mu.
func (a *Service) holdOutcomeLocked(host string) func() {
	if a.outcomeFences == nil {
		a.outcomeFences = make(map[string]*outcomeFence)
	}
	f := a.outcomeFences[host]
	if f == nil {
		f = &outcomeFence{done: make(chan struct{})}
		a.outcomeFences[host] = f
	}
	f.n++
	var once sync.Once
	return func() {
		once.Do(func() {
			a.mu.Lock()
			defer a.mu.Unlock()
			if f.n--; f.n == 0 {
				close(f.done)
				if a.outcomeFences[host] == f {
					delete(a.outcomeFences, host)
				}
			}
		})
	}
}

// errOutcomeUndelivered refuses a wake, before it plans, while an earlier
// planned wake's outcome is still being delivered past outcomeFenceWait
// (final review wave 2, N1). The fence fails closed: planning without the
// outcome could boot a build whose failure (a hold's included) is not yet
// recorded. Members get the maintenance fence's existing text
// (memberSafeWakeReason): the platform holds the host closed until its own
// work is done, and it reopens by itself.
var errOutcomeUndelivered = errors.New("an earlier planned wake's outcome is still being delivered")

// awaitOutcomes is a new wake's wait, before it plans, for host's
// planned-wake outcomes still being delivered (final review F3; spec §8.1,
// O7): a failed first wake's entry goes before afterWake's OnGen records
// the failure, and a first wake's observer reaches its verdict before
// OnGen hears it. Planning in that gap would pick the failed build again,
// with no record yet to hold it. The wait is bounded by outcomeFenceWait
// and the wake's own context; the fence is not (final review wave 2, N1).
// A wait that ends before the delivery refuses the wake: an error means
// the caller must not plan or launch anything. The fence stays until the
// outcome's OnGen returns, so the next request waits again. Called off
// a.mu.
func (a *Service) awaitOutcomes(ctx context.Context, host string) error {
	a.mu.Lock()
	f := a.outcomeFences[host]
	a.mu.Unlock()
	if f == nil {
		return nil
	}
	bound := time.NewTimer(outcomeFenceWait)
	defer bound.Stop()
	select {
	case <-f.done:
		return nil
	case <-ctx.Done():
		// Refused the same way: the outcome is still undelivered.
		return fmt.Errorf("%w: %w", errOutcomeUndelivered, ctx.Err())
	case <-bound.C:
		log.Printf("activator: %s: an earlier planned wake's outcome is still being delivered after %v; refusing this wake until it is", host, outcomeFenceWait)
		return errOutcomeUndelivered
	}
}

// outcomeOwedLocked reports whether wake w of e ends with a failure that
// afterWake will deliver (GenFailed, GenFellBack): the outcome fence covers
// it. The caller holds a.mu.
func (a *Service) outcomeOwedLocked(e *actEntry, w *wokenInstance, err error) bool {
	return a.OnGen != nil && w != nil && e.plan.planned() && (w.failed != nil || (err != nil && w.holdRanUnjudged(e.plan)))
}

// holdRanUnjudged reports whether a failed wake of plan ran (or may have
// run) its first wake's hold build without a verdict on it (final review
// wave 2, N2): its lifecycle ended it after the exec (a stop, a fence, a
// step after the spawn), or its start's outcome is uncertain. afterWake
// reports it as GenCancelled with Executed, and the outcome fence covers
// the delivery, so the next wake plans only once the scheduler holds the
// instance off its good build.
func (w *wokenInstance) holdRanUnjudged(plan WakePlan) bool {
	return w.bootRan && w.failed == nil && plan.FirstWake && plan.Mode == ModeHold
}

// emitPlanned is emitGen for an event of a wake that ran plan: it carries
// the plan's token and config revision (final review F4), so every event of
// one planned wake, on every path, names the same wake. Never called with
// a.mu held (emitGen's rule).
func (a *Service) emitPlanned(plan WakePlan, ev GenEvent) {
	ev.Token, ev.ConfigRev = plan.Token, plan.ConfigRev
	a.emitGen(ev)
}

// emitGen hands ev to OnGen. It is never called with a.mu held: OnGen
// reaches the scheduler's served, publishServing and writeGood, which take
// its goodMu and call back into Current.
func (a *Service) emitGen(ev GenEvent) {
	if a.OnGen != nil {
		a.OnGen(ev)
	}
}

// cancelPlanned reports that a planned wake of plan ended with no verdict
// on its build and nothing owed to the outcome fence (rollout Task 5, plan
// review #1): refused at its lease, its exec or its check, replaced by a
// replan, or (ran true) an overlap or ordinary wake its lifecycle ended
// after the exec. It records nothing; it is the wake's terminal event, which
// ends the scheduler's write-ahead intent for it. Only a plan with a token
// is reported, and only to an OnGen that is set, so a box with the flag off
// hears nothing. Never called with a.mu held (emitGen's rule).
func (a *Service) cancelPlanned(rt registry.Route, plan WakePlan, ran bool) {
	if a.OnGen == nil || plan.Token == 0 || !plan.planned() {
		return
	}
	ev := GenEvent{Kind: GenCancelled, Route: rt, Mode: plan.Mode, FirstWake: plan.FirstWake, At: time.Now(), Executed: ran}
	if ran {
		ev.Gen = genOf(plan.Boot)
	}
	a.emitPlanned(plan, ev)
}

// afterWake reports a planned wake's outcome. wake() calls it after
// e.ready has closed, without a.mu (emitGen's rule).
//
// Every planned wake with a token ends with exactly one terminal event for
// it (rollout Task 5): its failure, its fallback, its verdict or its
// cancellation here, or the first-wake observer's. A failed wake that owed
// nothing to the outcome fence (outcomeOwedLocked) reports its cancellation
// (cancelPlanned) after its entry is gone and admission has reopened; that
// is safe because the event records nothing, and a newer wake's intent
// replaces this one's, so its late arrival ends nothing it should not.
func (a *Service) afterWake(e *actEntry, w *wokenInstance, err error) {
	plan := e.plan // written only by this wake's own goroutine
	if a.OnGen == nil || !plan.planned() {
		return
	}
	if w == nil {
		// No result at all (a worker's token not minted, a claimed wake that
		// ran nothing of the planned build): nothing of it ran.
		if err != nil {
			a.cancelPlanned(e.route, plan, false)
		}
		return
	}
	now := time.Now()
	switch {
	case err != nil:
		switch {
		case w.failed != nil:
			a.emitPlanned(plan, GenEvent{Kind: GenFailed, Route: e.route, Failed: *w.failed, Mode: plan.Mode, FirstWake: true, Reason: w.failReason, At: now, FailedUnder: plan.FailedUnder})
		case w.holdRanUnjudged(plan):
			a.emitPlanned(plan, GenEvent{Kind: GenCancelled, Route: e.route, Gen: genOf(plan.Boot), Mode: plan.Mode, FirstWake: true, At: now, Executed: true})
		default:
			// Refused before the exec (its lease, a fence, its check), or an
			// overlap or ordinary wake ended after the exec with no verdict.
			a.cancelPlanned(e.route, plan, w.bootRan)
		}
	case w.fellBack:
		a.emitPlanned(plan, GenEvent{Kind: GenFellBack, Route: e.route, Gen: w.planned, Failed: *w.failed, Mode: plan.Mode, FirstWake: true, Reason: w.failReason, At: now, FailedUnder: plan.FailedUnder})
	case w.untried:
		// B was never exec'd (too little time for a verdict) and the good
		// build serves: reported exactly as a healthy wake of the good build
		// that is not a first wake, so the scheduler records it as served
		// and B stays a candidate for the next wake.
		a.emitPlanned(plan, GenEvent{Kind: GenHealthy, Route: e.route, Gen: w.planned, Mode: plan.Mode, At: now})
	default:
		a.emitPlanned(plan, GenEvent{Kind: GenHealthy, Route: e.route, Gen: w.planned, Mode: plan.Mode, FirstWake: plan.FirstWake, At: now})
		switch {
		case plan.FirstWake && e.worker:
			// A socketless first wake's readiness window (plannedWorkerReady,
			// spec §6.5) was its settle: it is settled now. A second window
			// would fail a build that was already ready, where §8.6 calls for
			// ordinary crash recovery.
			a.emitPlanned(plan, GenEvent{Kind: GenSettled, Route: e.route, Gen: w.planned, Mode: plan.Mode, FirstWake: true, At: now})
		case plan.FirstWake:
			a.watchFirstWake(e, w.planned, plan)
		}
	}
}

// armFirstWatchLocked registers a first wake's observation in the awake
// transition, before e.ready closes (spec §6.4): Current hides the
// generation from the instant it is reachable, and an exit or a stop
// before the observer goroutine runs is classified by it, never lost.
// wake() calls it after e.gen is set, and before the fence and bounce
// checks there, so a stop they begin cancels the observation. It also takes
// the host's outcome fence (final review F3), which watchFirstWake releases
// once its terminal event has been delivered: the entry can be stopped and
// a new wake begun before then, and that wake must not plan until the
// verdict is recorded. The caller holds a.mu.
func (a *Service) armFirstWatchLocked(e *actEntry, w *wokenInstance) {
	if a.Planner == nil || a.OnGen == nil || e.worker || !e.plan.planned() || !e.plan.FirstWake || w.fellBack || w.failed != nil || w.untried {
		return
	}
	ctx, cancel := context.WithCancel(context.Background())
	e.firstWatch, e.firstWatchCtx = cancel, ctx
	e.firstWatchRelease = a.holdOutcomeLocked(e.host)
}

// watchFirstWake is a first wake's 10 s observation (spec §6.4), on plan 3's
// observeSettle. The waiting requests do not sit through it. An intentional
// stop cancels it (beginStopLocked) and records nothing; an unexpected exit
// fails the build.
func (a *Service) watchFirstWake(e *actEntry, g Gen, plan WakePlan) {
	cancelled := func(rt registry.Route) {
		// Every observation GenHealthy opened is closed by an event; this
		// one says "no verdict", so the scheduler stops showing the host
		// as settling and records nothing (spec §6.4, §9.1). The build was
		// healthy, so it ran (Executed).
		a.emitPlanned(plan, GenEvent{Kind: GenCancelled, Route: rt, Gen: g, Mode: plan.Mode, FirstWake: true, At: time.Now(), Executed: true})
	}
	// The observation was registered in the awake transition
	// (armFirstWatchLocked), so a stop that began meanwhile has already
	// cancelled it, or, for a crash, marked unexpectedExit: observeSettle and
	// the classification below decide either way. Nothing registered means
	// no first-wake observation for this wake.
	a.mu.Lock()
	ctx, rt, proc := e.firstWatchCtx, e.route, e.proc
	a.mu.Unlock()
	if ctx == nil {
		cancelled(rt)
		return
	}
	go func() {
		r := observeSettle(ctx, proc, firstWakeSettle)
		a.mu.Lock()
		rt, unexpected, release := e.route, e.unexpectedExit, e.firstWatchRelease
		a.mu.Unlock()
		// end closes the observation: Current shows the generation again,
		// and a stop no longer has an observation to cancel.
		end := func() {
			a.mu.Lock()
			cancel := e.firstWatch
			e.firstWatch, e.firstWatchCtx, e.firstWatchRelease = nil, nil, nil
			a.mu.Unlock()
			if cancel != nil {
				cancel()
			}
		}
		// The verdict is delivered before the host's outcome fence is
		// released (final review F3): a new wake that began meanwhile plans
		// only once the scheduler has recorded it.
		defer func() {
			if release != nil {
				release()
			}
		}()
		exited := r == settleExited || (r == settleCancelled && unexpected)
		switch {
		case r == settleOK:
			// Ended first: the scheduler records a settled build as good
			// only while Current shows it (rollout's acknowledged).
			end()
			a.emitPlanned(plan, GenEvent{Kind: GenSettled, Route: rt, Gen: g, Mode: plan.Mode, FirstWake: true, At: time.Now()})
		case exited:
			// Delivered first, so nothing sees the failed generation as
			// current (and admits an attempt onto it) before the failure is
			// recorded.
			a.emitPlanned(plan, GenEvent{Kind: GenFailed, Route: rt, Failed: plan.Boot, Mode: plan.Mode, FirstWake: true,
				Reason: "exited within " + firstWakeSettle.String() + " of becoming healthy", At: time.Now(), FailedUnder: plan.FailedUnder})
			end()
		default: // an intentional stop cancelled it
			cancelled(rt)
			end()
		}
	}()
}

// claimedPlanResult carries a claims-on planned wake's outcome to afterWake.
// Plan 3's startAdmitted records the pinned build it saw fail
// (e.pinFailed) and, once the isolated launch opens, starts the good build
// in its place (StartPolicy.Fallback). w is the claim's result, nil when
// the wake failed. The returned value is what doClaimWake hands back to
// wake(). On this base every claims-on launch refuses before its exec hook,
// so pinFailed stays nil and a planned claimed wake reports nothing.
func (a *Service) claimedPlanResult(e *actEntry, w *wokenInstance) *wokenInstance {
	a.mu.Lock()
	failed, why, plan := e.pinFailed, e.pinFailReason, e.plan
	a.mu.Unlock()
	if !plan.planned() {
		return w
	}
	if w == nil {
		if failed == nil || !plan.FirstWake {
			return nil // nothing of the planned build ran: no failure of the build
		}
		return &wokenInstance{failed: &plan.Boot, failReason: why}
	}
	served := plan.Boot
	if failed != nil && plan.FirstWake && plan.Fallback != nil && w.spawnVersion == plan.Fallback.Version {
		served = *plan.Fallback
		w.failed, w.failReason, w.fellBack = &plan.Boot, why, true
	}
	w.planned = genOf(served)
	w.planned.N, w.planned.Socket, w.planned.Control = w.gen.N, w.gen.Socket, w.gen.Control
	return w
}