// swap.go is the overlap and hold swap (zero-downtime deploys spec §7.4,
// §7.5, §7.8): one attempt moves one awake instance to a new generation.
// Nothing calls Service.Swap until plan 5 wires CARLOS_SWAP. With plan 1's
// generations switch off, it refuses every attempt and spawns nothing.
package activator

import (
	"context"
	"errors"
	"fmt"
	"log"
	"time"

	"github.com/carlosframework/platform/internal/cgroup"
	"github.com/carlosframework/platform/internal/hostbox"
	"github.com/carlosframework/platform/internal/hostguard"
	"github.com/carlosframework/platform/internal/hostwrite"
	"github.com/carlosframework/platform/internal/instancectl"
	"github.com/carlosframework/platform/internal/registry"
)

// Cause is why an attempt exists (spec §6.3).
type Cause string

const (
	CauseRelease Cause = "release"
	CauseInstall Cause = "install"
	CauseRetry   Cause = "retry"
	CauseRestart Cause = "restart"
	CauseConfig  Cause = "config"
)

// SwapMode is how an awake instance moves to a new generation (spec §4.1).
type SwapMode string

const (
	ModeOverlap SwapMode = "overlap"
	ModeHold    SwapMode = "hold"
)

// Attempt is one try at moving one instance to a new generation (spec §6.3).
type Attempt struct {
	Host       string
	Cause      Cause
	Build      string
	StagedPath string
	Digest     string // expected sha256 of StagedPath, verified before exec
	Demand     string // release demand for the artifact: "hold" or ""
	Mode       SwapMode
	ConfigRev  string
	RestartID  string
	SchedRev   uint64 // scheduler revision; the newest-attempt check compares it
	Canary     bool   // observe the 10 s settle
}

// SwapOutcome is how an attempt ended.
type SwapOutcome string

const (
	SwapDone      SwapOutcome = "done"      // switched (overlap) or replaced (hold)
	SwapFailed    SwapOutcome = "failed"    // successor failed health or settle; incumbent still serving (overlap)
	SwapAbandoned SwapOutcome = "abandoned" // superseded or a re-check failed; not a build failure
	SwapDown      SwapOutcome = "down"      // hold mode failed: nothing serving
)

// SwapResult is Swap's answer. Reason is diagnostic text for the
// scheduler (plan 5); member wording is plan 6's.
type SwapResult struct {
	Outcome SwapOutcome
	Gen     Gen // the generation now current (on SwapDone)
	Reason  string
	// Drain is how long the retiring generation took from SIGTERM to its
	// proven exit (an overlap), or the old generation's stop (a hold). Zero
	// when nothing retired. The "swapped" event reports it (spec §9.4).
	Drain time.Duration
	// Mode is the mode the attempt actually ran in. It differs from
	// Attempt.Mode when an overlap attempt on a foreign-layout row ran as a
	// hold (spec §2.9): the scheduler records a failure under this mode, so
	// a failed hold is held down (spec §6.4, §8.3). Swap sets it.
	Mode SwapMode
	// Executed is the attempt's execution evidence (final review F8), one
	// side of a tri-state (final review wave 2, N4): false means it is
	// PROVEN that nothing of the attempt's build ran; true means it ran OR
	// it is not known whether it ran. True once a process of the build
	// started, whatever happened to it after (an overlap successor, a
	// hold's pinned wake, a worker's respawn child, installed or shed), and
	// also when a start was handed off and failed with its outcome
	// uncertain (startMayHaveRun) or when ReplaceWorker could not learn
	// what its respawn did. Only false lets an abandoned attempt count as
	// having spent nothing of what it was granted: the scheduler re-arms a
	// `carlos instances retry` allowance on it. Swap and ReplaceWorker set
	// it; every other result leaves it false, and each of those returns
	// before anything of the build could start.
	Executed bool
	// CleanExit is set by ReplaceWorker on a SwapDone whose replacement's
	// build ran to a clean exit inside the settle window: a one-shot's
	// success (spec §6.5). It has no generation the activator will go on
	// serving, so nothing acknowledges it afterwards; the scheduler records
	// it as the good build from this result (spec §8.5, Astra R3 flag
	// triage T9). A replacement still running at the end of the window is
	// SwapDone with CleanExit false.
	CleanExit bool
}

var (
	errSwapOff         = errors.New("swap: off on this box")
	errSwapBusy        = errors.New("swap: another generation operation is in flight")
	errSwapKind        = errors.New("swap: only exec instances overlap; workers and sidecars keep stop-then-start")
	errSuccessorExited = errors.New("swap: successor exited before the switch")
	errSwapMode        = errors.New("swap: unknown mode")
)

func abandon(err error) SwapResult  { return SwapResult{Outcome: SwapAbandoned, Reason: err.Error()} }
func failSwap(err error) SwapResult { return SwapResult{Outcome: SwapFailed, Reason: err.Error()} }

// stepOutcome is how one step of an attempt (a spawn, a health check, a
// readiness or settle window, a pinned wake) ended.
type stepOutcome int

const (
	stepOK        stepOutcome = iota
	stepCancelled             // the attempt's lifecycle ended it: its context, the barrier, a fence, an intentional stop
	stepStale                 // the attempt gate refused it: a newer attempt, a halted cohort, a lost lease, shutdown
	stepFailed                // the build failed: spawn, health or readiness
)

// classifyStep is THE outcome classifier (spec §6.4, §7.4): every path
// that might record a build failure asks it first, the swaps here and plan
// 5's planned wakes alike. Only stepFailed is a failure of the build; a
// cancelled or stale step records nothing and falls back to nothing.
// lifecycle is the attempt's own context (nil when the caller has none). A
// health deadline that expires is the build's failure (DeadlineExceeded);
// the lifecycle's own end, or a cancellation, is not.
func classifyStep(lifecycle context.Context, err error) stepOutcome {
	switch {
	case err == nil:
		return stepOK
	case lifecycle != nil && lifecycle.Err() != nil,
		errors.Is(err, context.Canceled), errors.Is(err, errSwapStopped), errors.Is(err, ErrFenced):
		return stepCancelled
	case verdictOf(err) != gateProceed:
		// The attempt gate's own refusal, wherever it surfaced: at Step 3a,
		// or as a claims validation inside the work at the exec. Nothing of
		// the build ran because of it (spec §7.4).
		return stepStale
	case errors.Is(err, ErrBuildNotPublished), errors.Is(err, errStagedBuildNotPublished),
		errors.Is(err, errBuildStateUnwritten), errors.Is(err, errStaleLink):
		// A spawn refused because its build could not be published before
		// the exec (ruling R19 item 3; the worker and the staged instance
		// sentinels, final review F9): nothing of the build ran, so it is
		// abandonment, never a failure of the build. Classified here and not
		// in verdictOf, which main's flag-off respawn reads; every caller of
		// classifyStep is flag-on (swaps, holds, planned wakes).
		// errBuildStateUnwritten (settings spec §4.2) is the same kind of
		// refusal; verdictOf also names it, so it is stale through either
		// arm, and named here beside its kin. errStaleLink (settings spec
		// §4.3 rule 3) is a held wake refusing, before its record and its
		// exec, an identity that is not its target: a planned wake's boot
		// never ran either.
		return stepStale
	}
	return stepFailed
}

// gateVerdict is the attempt gate's answer, typed (spec §6.3, §7.4).
type gateVerdict int

const (
	// gateProceed: the attempt is current and owns the host.
	gateProceed gateVerdict = iota
	// gateSuperseded: still the owner, but a newer attempt or a halted
	// cohort replaced this one. A restore of what it replaced may run, and
	// only behind a fresh ownership check at that exec.
	gateSuperseded
	// gateNoOwnership: the entry is not current, the lease is lost, a fence
	// or the agent's shutdown holds the host, or the claims validation
	// refused (a changed stamp, a durable fence, a changed claim). Nothing
	// may spawn, a restore included.
	gateNoOwnership
)

// verdictOf types an attempt-gate error. An error that is not the gate's
// (a failure of the build) is gateProceed: the gate did not refuse.
//
// Besides the gate's own refusals, eight kinds of refusal that stop a spawn
// before anything of the build runs are NoOwnership, so the one classifier
// reads them as abandonment, never as a failed build:
//   - plan 2's strict-host gate on placed spawns (index P2-8):
//     ErrStrictRequired, ErrStrictGroupInUse, ErrStrictCleanupPending;
//   - a closed claims-on launch (index P2-9): ErrClaimIsolationUnavailable.
//     It wraps hostbox.ErrUnsupported, which is not otherwise classified,
//     so it is named here;
//   - a row that cannot run a staged generation at all: errUnitNoGens,
//     errSwapNoGens;
//   - the cgroup manager refusing a generation start because the host's
//     parent cgroup holds a process directly (cgroup.ErrLegacyParentPopulated,
//     amadan #272): it checks before it creates the child or execs, and the
//     condition is the box's, not the build's;
//   - the cgroup manager refusing a start because the cgroup it would spawn
//     into took cgroup.kill and could not be made fresh
//     (cgroup.ErrKilledCgroupKept, amadan #272, Task 19): it refuses before
//     it builds the command, so nothing ran, and the condition is the box's;
//   - the cgroup manager refusing a generation start because a limit it
//     would run under could not be applied, or it could only be started
//     outside its cgroup and moved after (cgroup.ErrLimitNotApplied, zdd
//     readiness round 2 Task 7): it refuses before it builds the command
//     (or, for a failed placement, the shim proved it never reached exec),
//     so nothing ran, and the condition is the box's;
//   - root refusing the nested staged executable (instancectl.ErrStagedExe,
//     Task 9): it validates before it mounts or starts anything;
//   - a build state that could not be written before the exec
//     (errBuildStateUnwritten, settings spec §4.2). Unlike
//     ErrBuildNotPublished it is a new sentinel, so naming it here changes
//     nothing main did: a worker respawn it refuses logs a refusal, not
//     "failed to respawn" and a member error event, and the sweep asks
//     again; a ReplaceWorker attempt it refuses is abandoned and restores
//     the build it replaced at once (its veto switch names this sentinel
//     before the NoOwnership case), since nothing of the attempt ran.
//
// errSwapStopped (the barrier's cancel, from memberCurrentLocked) is
// NoOwnership too: the entry is stopping, so nothing may spawn for it.
// classifyStep reads it as stepCancelled before it asks this.
func verdictOf(err error) gateVerdict {
	switch {
	case err == nil:
		return gateProceed
	case errors.Is(err, errSwapSuperseded):
		return gateSuperseded
	case errors.Is(err, errSwapNotAwake), errors.Is(err, errSwapLeaseLost), errors.Is(err, errSwapFenced),
		errors.Is(err, errSwapShutdown), errors.Is(err, errSwapStopped), errors.Is(err, ErrFenced),
		errors.Is(err, hostguard.ErrStampMismatch), errors.Is(err, hostguard.ErrFenced),
		errors.Is(err, hostguard.ErrInactive), errors.Is(err, hostguard.ErrRetiredStamp),
		errors.Is(err, hostguard.ErrOldStamp), errors.Is(err, hostguard.ErrClaimIncarnation),
		errors.Is(err, hostguard.ErrRecordIncarnation), errors.Is(err, hostguard.ErrNotParticipant),
		errors.Is(err, hostguard.ErrNoClaim), errors.Is(err, hostguard.ErrNoClaims),
		errors.Is(err, hostguard.ErrNoHolder), errors.Is(err, hostguard.ErrMalformedClaim),
		errors.Is(err, hostbox.ErrChanged), errors.Is(err, errClaimsAdmission),
		errors.Is(err, ErrStrictRequired), errors.Is(err, ErrStrictGroupInUse), errors.Is(err, ErrStrictCleanupPending),
		errors.Is(err, ErrClaimIsolationUnavailable), errors.Is(err, cgroup.ErrLegacyParentPopulated),
		errors.Is(err, cgroup.ErrKilledCgroupKept), errors.Is(err, cgroup.ErrLimitNotApplied),
		errors.Is(err, errUnitNoGens), errors.Is(err, errSwapNoGens), errors.Is(err, instancectl.ErrStagedExe),
		errors.Is(err, errBuildStateUnwritten):
		return gateNoOwnership
	}
	return gateProceed
}

// errClaimsAdmission marks an error a claims-on spawn met before its exec
// hook ran (claimAdmission): the claim read failed (a store error, a
// timeout), the claim was absent or malformed, or the work's validation
// refused before the spawn. Nothing of the build ran, so it is never a
// failure of the build (spec §7.4).
var errClaimsAdmission = errors.New("the claims admission refused before the exec")

// claimAdmissionPrefix is the text claimAdmission puts before its cause's.
// memberSafeWakeReason removes it, so a member's event reads as the cause.
var claimAdmissionPrefix = errClaimsAdmission.Error() + ": "

// claimAdmission wraps err in errClaimsAdmission when the spawn never
// reached its exec hook (reached false). An error after the hook ran is the
// spawn's own, and is returned as it is.
func claimAdmission(err error, reached bool) error {
	if err == nil || reached {
		return err
	}
	return fmt.Errorf("%w: %w", errClaimsAdmission, err)
}

// stepResult is classifyStep for a swap step: a cancelled or stale step
// abandons the attempt (over is true); a failure of the build is the
// caller's to report (over is false).
func stepResult(lifecycle context.Context, err error) (SwapResult, bool) {
	switch classifyStep(lifecycle, err) {
	case stepCancelled:
		return abandon(errSwapStopped), true
	case stepStale:
		return abandon(err), true
	}
	return SwapResult{}, false
}

// Swap runs one attempt on an awake entry (spec §7.3–§7.5, §7.8). It
// blocks until the attempt is done, failed or abandoned. For an overlap
// that includes the retiring generation's drain. For a hold it lasts until
// the new generation is healthy or has failed.
//
// newest reports the scheduler's newest revision for a host. It is called
// with a.mu held, so it must not block and must not call back into the
// Service.
func (a *Service) Swap(ctx context.Context, at Attempt, newest func(host string) uint64) SwapResult {
	if !a.gens {
		return abandon(errSwapOff)
	}
	gb, ok := a.backend.(SwapBackend)
	if !ok {
		return abandon(errSwapNoGens)
	}
	if newest == nil {
		return abandon(errSwapSuperseded)
	}
	switch at.Mode {
	case ModeOverlap:
		if a.foreignRow(at.Host) {
			// A foreign row keeps its canonical socket (plan 1), so two
			// generations of it cannot coexist: its replacement is a hold
			// (spec §2.9, rev 17). Its result reads as a hold's, SwapDown
			// when the new build fails, and Mode tells the scheduler so: a
			// hold build that ran is held down (spec §6.4, §8.3).
			res := a.swapHold(ctx, at, newest)
			res.Mode = ModeHold
			return res
		}
		res := a.swapOverlap(ctx, gb, at, newest)
		res.Mode = ModeOverlap
		return res
	case ModeHold:
		res := a.swapHold(ctx, at, newest)
		res.Mode = ModeHold
		return res
	}
	return abandon(errSwapMode)
}

// EffectiveMode is the mode Swap runs an attempt of mode m on host in: an
// overlap on a foreign-layout row runs as a hold (foreignRow, spec §2.9),
// anything else as given. The scheduler writes it into the attempt's
// write-ahead intent before calling Swap (rollout Task 5, plan review
// finding 2), so an agent that crashes mid-swap reopens its failure book
// with the hold it was running. Called without a.mu held.
func (a *Service) EffectiveMode(host string, m SwapMode) SwapMode {
	if m == ModeOverlap && a.foreignRow(host) {
		return ModeHold
	}
	return m
}

// attemptCtx is the attempt's own lifetime. It ends with the caller's ctx,
// with the agent (Shutdown cancels lifeCtx), or with the barrier, which
// calls the returned cancel through the member.
func (a *Service) attemptCtx(ctx context.Context) (context.Context, context.CancelFunc) {
	mctx, cancel := context.WithCancel(ctx)
	if a.lifeCtx == nil {
		return mctx, cancel
	}
	stop := context.AfterFunc(a.lifeCtx, cancel)
	return mctx, func() { stop(); cancel() }
}

// genHealthy is the successor's health check, through its own handle:
// GET / below 500 within spawnHealthTimeout (spec §7.5 step 4).
func (a *Service) genHealthy(ctx context.Context, g Gen) error {
	if a.genHealth != nil {
		return a.genHealth(ctx, g)
	}
	if g.Socket == "" {
		return errors.New("generation has no socket")
	}
	return waitHealthyCtx(ctx, g.Socket, spawnHealthTimeout)
}

// spawnSuccessor is §7.5 step 3 in both modes. Claims off, it is the
// backend's staged spawn. Claims on, a claim-backed spawn would run inside
// the selected work, which validates the claim and stamp at the exec.
//
// gate is the attempt gate (memberCurrentLocked, taking a.mu itself). It
// runs immediately before every exec: claims off, right before
// StartStaged, and again inside it at the backend's final launch point,
// after the backend's preparation (withExecGate; final review F1). Claims
// on, it belongs after the runtime's Read and Prepare
// and again inside the selected work at the exec, as the spawn's before
// hook (plan 3 Task 10).
//
// Every claims-on launch is closed on this base (index P2-9): the isolated
// root launcher cannot carry claim reservations. swapOverlap refuses every
// claims-backed attempt before its raise (claimsSwapRefusal), so production
// never reaches the claims-on arm here; only the claimSpawn test seam, the
// stand-in for an open launch, does. Without the seam the arm still refuses,
// as a claims admission (NoOwnership, the attempt abandoned), and a
// claims-on spawn is never sent down the claims-off StartStaged.
func (a *Service) spawnSuccessor(ctx context.Context, gb SwapBackend, e *actEntry, rt registry.Route, work hostwrite.Work, g Gen, gate func() error) (Gen, Proc, error) {
	if hostwrite.Legacy(work) {
		if err := gate(); err != nil {
			return g, nil, err
		}
		// Settings spec §4.2: the successor's build is recorded before it
		// runs against the data, under the incumbent's lease. A refusal is
		// abandonment (classifyStep), never the build's failure.
		if err := a.recordBuild(ctx, rt.Host, g.Build); err != nil {
			return g, nil, err
		}
		return gb.StartStaged(withExecGate(ctx, gate), rt, g)
	}
	if a.claimSpawn != nil {
		return a.claimSpawn(ctx, e, rt, work, g)
	}
	return g, nil, claimAdmission(ErrClaimIsolationUnavailable, false)
}

// swapOverlap is §7.5 steps 1 to 7. The successor is registered in the
// operation set under the same lock that runs the §7.4 checks, so a
// terminal stop that begins later always sees it and joins it.
//
// Every exit after the admission leaves the set through dropMember (or, on
// success, promotes the successor and drops the retiring generation), and
// leaveSet restores a raised parent when the last member leaves. Nothing
// here blocks under a.mu except GenSwitched, which plan 1's lock contract
// puts in the same critical section as e.gen.
func (a *Service) swapOverlap(ctx context.Context, gb SwapBackend, at Attempt, newest func(string) uint64) (out SwapResult) {
	// The execution evidence (SwapResult.Executed): the backend returned a
	// successor process, or its error says the build may have run
	// (startMayHaveRun), whatever ends the attempt after that.
	var executed bool
	defer func() { out.Executed = executed }()
	// The bytes first. An artifact that is not what the attempt chose never
	// reaches admission, let alone exec (spec §4.2, §7.5 step 3).
	if err := verifyStaged(at.StagedPath, at.Digest); err != nil {
		return abandon(fmt.Errorf("staging: %w", err))
	}
	work, err := a.selectWork(ctx, at.Host)
	if err != nil {
		return abandon(fmt.Errorf("admission: %w", err))
	}
	if !hostwrite.Legacy(work) {
		// Claims on: refused before the admission and the raise, whatever
		// the runtime's capabilities (claimsSwapRefusal).
		if err := a.claimsSwapRefusal(true); err != nil {
			return abandon(err)
		}
	}
	mctx, cancel := a.attemptCtx(ctx)
	defer cancel()
	m := &genMember{
		gen:    Gen{N: nextGen(), Build: at.Build, StagedPath: at.StagedPath, Digest: at.Digest, Demand: at.Demand},
		role:   roleSpawning,
		cancel: cancel,
		done:   make(chan struct{}),
	}
	var (
		e         *actEntry
		rt        registry.Route
		incumbent Gen // the current generation at admission, for its tier
	)
	// Step 1: the checks and joining the set, inside the selected work. Only
	// the registration runs there. A claims-on spawn re-enters the work
	// itself (Task 10), and hostguard's flock does not nest. Registering
	// under the same lock as the checks is what lets a stop that begins
	// afterwards see the successor and join it.
	err = a.admit(ctx, work, func() error {
		a.mu.Lock()
		defer a.mu.Unlock()
		e = a.entries[at.Host]
		if err := a.swapChecksLocked(e, at, newest); err != nil {
			return err
		}
		if e.worker || e.route.Kind != "instance" {
			return errSwapKind
		}
		// A foreign row keeps its canonical socket (plan 1), so two
		// generations of it cannot coexist. It may still hold-swap. So
		// can a flat row whose generation path would not fit (rev 19
		// rule 4), which swapLayout reads as foreign.
		if l, ok := a.swapLayout(e.route); ok && l != LayoutFlat && l != LayoutNested {
			return errSwapLayout
		}
		// A parent still marked raised with no member is a restore that
		// failed and awaits the sweep's retry (final review F11): an overlap
		// raising it again now could have that retry lower it under the
		// successor.
		if len(e.ops.members) > 0 || e.ops.raised {
			return errSwapBusy
		}
		// A run's unit may be starting against the incumbent right now
		// (merge ruling M2). Admitting now could retire the incumbent under
		// a command that has not even started; the attempt is abandoned
		// instead, and the scheduler tries again once the start has
		// returned. From here on RunCmd waits for this successor rather
		// than starting a unit (switchPendingLocked).
		if cmdStartInFlightLocked(e) {
			return errSwapBusy
		}
		a.addMemberLocked(e, m)
		rt, incumbent = e.route, e.gen
		return nil
	})
	if err != nil {
		// The work may refuse after its body registered the member.
		if a.isMember(e, m) {
			a.dropMember(e, m)
		}
		return abandon(err)
	}

	// Each artifact's own tier, captured now (spec §7.3): the incumbent's
	// from the build it runs, the successor's from the attempt's build.
	// Neither is read from the live link, which root may repoint. They size
	// the overlap parent, the successor's child, and the restore (leaveSet
	// restores to the current generation's Tier).
	if incumbent.Tier == (cgroup.Limit{}) {
		incumbent.Tier = gb.GenTier(rt, incumbent.Build)
	}
	succTier := gb.GenTier(rt, at.Build)
	a.mu.Lock()
	if e.gen.N == incumbent.N {
		e.gen.Tier = incumbent.Tier
	}
	m.gen.Tier = succTier
	a.mu.Unlock()

	// Step 2: the parent raise, before the successor spawns (spec §7.3).
	// The parent is marked potentially changed BEFORE the raise is asked: a
	// raise that wrote memory.max and then failed a later write has changed
	// it, and the member's cleanup (leaveSet) must restore it (final review
	// F11).
	a.mu.Lock()
	e.ops.raised = true
	a.mu.Unlock()
	if err := gb.RaiseGenParent(ctx, rt, incumbent.Tier, succTier); err != nil {
		if parentUntouched(err) {
			// Refused before any write: there is nothing to restore.
			a.mu.Lock()
			e.ops.raised = false
			a.mu.Unlock()
		}
		a.dropMember(e, m) // restores a possibly changed parent (leaveSet)
		return abandon(fmt.Errorf("raise parent limit: %w", err))
	}

	// The successor's build is published before its exec (spec §4.2, §8.5):
	// it has been in the operation set since admission, so this write names
	// it. A failed write abandons the attempt; nothing of it ran, so it is
	// never a failure of the build.
	if err := a.publishBeforeExec(); err != nil {
		a.dropMember(e, m) // restores the raised parent (leaveSet)
		return abandon(err)
	}

	// Step 3a: the §7.4 checks again, at the exec (spec §6.3, §7.4). The
	// raise (and the pin publication just above this step) took time:
	// a newer attempt, a halted cohort, a lost lease or a fence since
	// admission means nothing execs. Claims on, this runs inside the
	// selected work as the admission did.
	gate := func() error {
		a.mu.Lock()
		defer a.mu.Unlock()
		return a.memberCurrentLocked(e, m, mctx, at, newest)
	}
	if err := a.admit(ctx, work, gate); err != nil {
		a.dropMember(e, m) // restores the raised parent (leaveSet)
		return abandon(err)
	}

	// Step 3: exec the attempt's staged build by path. The backend reports
	// what it ran, and anything but the attempt's choice is refused.
	g, proc, spawnErr := a.spawnSuccessor(mctx, gb, e, rt, work, m.gen, gate)
	executed = startMayHaveRun(proc, spawnErr)
	if spawnErr == nil && (g.N != m.gen.N || g.Build != m.gen.Build || g.StagedPath != m.gen.StagedPath) {
		spawnErr = fmt.Errorf("backend ran generation %d of %s from %q; the attempt chose %d of %s from %q",
			g.N, g.Build, g.StagedPath, m.gen.N, m.gen.Build, m.gen.StagedPath)
	}
	a.mu.Lock()
	m.proc = proc
	if spawnErr == nil {
		m.gen, m.role = g, roleSuccessor
	}
	a.mu.Unlock()
	if spawnErr != nil {
		a.dropMember(e, m)
		if res, over := stepResult(mctx, spawnErr); over {
			return res
		}
		return failSwap(fmt.Errorf("spawn: %w", spawnErr))
	}

	// Step 4: the health check, through the generation's own handle.
	if err := a.genHealthy(mctx, m.gen); err != nil {
		a.dropMember(e, m)
		if res, over := stepResult(mctx, err); over {
			return res
		}
		return failSwap(fmt.Errorf("health: %w", err))
	}

	// Step 4, the canary's half: the 10 s settle (spec §6.2). An
	// intentional stop of the entry cancels the observation through the
	// barrier, and that is not a failure (spec §6.4).
	if at.Canary {
		switch observeSettle(mctx, m.proc, swapSettle) {
		case settleCancelled:
			a.dropMember(e, m)
			return abandon(errSwapStopped)
		case settleExited:
			a.dropMember(e, m)
			return failSwap(errSettleExit)
		}
	}

	// Step 5: the switch. The checks run again, then, under the entry lock,
	// AppSocket, the serving build and the cache generation all move with
	// e.gen (contract C2).
	var (
		retiring *genMember
		switched bool
	)
	// The host's schedule write fence (final review F8): the switch waits
	// for the schedule writes in flight, keeps new ones out, and latches the
	// retiring generation in the same step, so no write of it lands after
	// the switch. Taken before a.mu (the fence's lock order).
	fence := a.schedFenceFor(at.Host)
	err = a.admit(ctx, work, func() error {
		fence.lock()
		defer fence.unlock()
		a.mu.Lock()
		defer a.mu.Unlock()
		if err := a.memberCurrentLocked(e, m, mctx, at, newest); err != nil {
			return err
		}
		if m.proc == nil || m.proc.Exited() {
			return errSuccessorExited
		}
		// Admission refused a run's start in flight and RunCmd starts none
		// while this successor is live, so none can be in flight here
		// (merge ruling M2). Checked again because a switch over one would
		// retire the generation that run is starting against.
		if cmdStartInFlightLocked(e) {
			return errSwapBusy
		}
		// The canonical symlink, the purge and the socket mode first. If the
		// link cannot move, nothing has switched yet. This runs under a.mu,
		// in the same critical section that sets e.gen below: plan 1's lock
		// contract for swapCanonicalLink.
		if err := gb.GenSwitched(e.route, m.gen, m.proc); err != nil {
			return err
		}
		retiring = &genMember{gen: e.gen, role: roleRetiring, proc: e.proc, done: make(chan struct{})}
		fence.retire(retiring.gen.N) // its schedule writes are refused from here, for good
		delete(e.ops.members, m.gen.N)
		a.addMemberLocked(e, retiring) // swapsim:M2
		e.gen, e.proc = m.gen, m.proc
		e.spawnVersion, e.staleSince, e.spawnedAt = m.gen.Build, time.Time{}, a.now()
		a.noteCurrent(e.host, m.gen.N) // plan 1's cache generation (C4)
		// Runs admitted against the retiring generation are interrupted
		// with it, as a scheduled restart's stop interrupts them (merge
		// ruling M2): a command never outlives the build it was started
		// from as that build retires.
		a.interruptCmdsLocked(e, retiring.gen.N, stopWhyFor(at.Cause))
		switched = true
		return nil
	})
	if err != nil && switched {
		// The work refused after its body had switched. The successor is
		// current and serving, so it is never dropped: the swap finishes,
		// and the retiring generation still drains below.
		log.Printf("activator: %s: the selected work reported %v after the switch to generation %d; the switch stands", at.Host, err, m.gen.N)
	} else if err != nil {
		a.dropMember(e, m)
		if errors.Is(err, errSuccessorExited) {
			return failSwap(err)
		}
		return abandon(err)
	}
	close(m.done) // promoted: it is current now, no longer a member
	// The retiring generation is no longer current. The edge closes its idle
	// connections now (spec §7.2), outside every activator lock (C4).
	a.genRetired(at.Host, retiring.gen)

	// Steps 6 and 7. The retiring generation drains under its own SIGTERM
	// handling, bounded by actStopTimeout. Once it is proven empty,
	// leaveSet restores the parent limit. The good build (§8.5) is plan 5's,
	// recorded from this result.
	a.mu.Lock()
	rt = e.route
	a.mu.Unlock()
	emitEvent(a.Logs, rt, "info", "instance stopping")
	res := SwapResult{Outcome: SwapDone, Gen: m.gen}
	// The drain is timed on the wall clock (not a.now, which tests fake), for
	// the "swapped" event's drain time (spec §9.4).
	drainStart := time.Now()
	proven := a.dropMember(e, retiring)
	res.Drain = time.Since(drainStart)
	if !proven {
		res.Reason = "retiring generation not yet proven stopped; the sweep finishes it"
	}
	// Step 8: a legacy co-spawned sidecar moves onto the new generation.
	a.replaceSidecar(e)
	return res
}