package activator import ( "context" "errors" "log" "sync" "time" "github.com/carlosframework/platform/internal/instancectl" "github.com/carlosframework/platform/internal/instancekey" "github.com/carlosframework/platform/internal/registry" ) // Why an instance stop interrupted a run (run spec §5.3). The text is what // the member reads after "the run was interrupted:" (RUN-C1 to RUN-C8); // beginStopLocked records one on every stop. const ( StopIdle = "the instance hibernated" StopRestart = "the instance was restarted" StopRelease = "the instance moved to a new release" StopExited = "the instance's process exited" StopLeaseLost = "the instance lost its database lease" StopFenced = "the instance was stopped for a move or a layout change" StopRetired = "the instance was taken off this box" StopShutdown = "the box's agent stopped" ) // ErrCmdClaim: claim control is armed on this box. Runs need the isolated // root protocol's reservations (#250), so they are refused (run spec §3.1). var ErrCmdClaim = errors.New("activator: claim control is armed; runs wait for the reservation protocol (#250)") // ErrCmdBusy: a run is already attached to the instance, or an earlier // run's unit was never proven settled. var ErrCmdBusy = errors.New("activator: a run is already attached to this instance") // ErrCmdNotInstance: runs are for instances; the route is a worker or // another kind. var ErrCmdNotInstance = errors.New("activator: runs are only for instances, and this route is not one") // CmdWakeError: the wake failed or did not finish within cmdWakeBound. // Reason is member-safe (memberSafeWakeReason). type CmdWakeError struct { Reason string Err error } func (e *CmdWakeError) Error() string { return e.Reason } func (e *CmdWakeError) Unwrap() error { return e.Err } // CmdStoppedError: the instance began stopping before the unit started. The // run is interrupted with Reason, never queued for later (run spec §5.3). type CmdStoppedError struct{ Reason string } func (e *CmdStoppedError) Error() string { return "activator: the instance stopped before the run started: " + e.Reason } // cmdWakeBound is how long RunCmd waits for a wake (run spec §5.3). var cmdWakeBound = 60 * time.Second // cmdStopBound bounds finishStop's wait for a run: the start in flight // (cmdStartBound), the stop (DeadlineControl), the cgroup (actStopTimeout) // and both streams (cmdAcceptGrace + cmdDrainBound each), with room. // Shutdown runs inside systemd's 90 s TimeoutStopSec for the agent: a // settle (about 62 s) fits; a start in flight on top may not, and an agent // killed mid-stop leaves the lease to its TTL. const cmdStopBound = 2 * time.Minute // cmdStartBound bounds RunCmd's StartCmd, well below cmdStopBound, so a // stop waiting on the start still has the settle's budget after it. const cmdStartBound = 45 * time.Second // CmdHandle is a started run, as the box's run pass sees it. type CmdHandle interface { Invocation() string Poll(ctx context.Context) (instancectl.CmdStatus, error) // Interrupted is the instance stop's reason when one stopped the unit, // "" otherwise, and when that stop was asked for: a command whose done // status the caller had before then finished on its own. Interrupted() (reason string, at time.Time) // Stop stops the unit and waits for it to settle and its streams to // end, leaving the instance running: a cancel, a timeout, or the end // of a command that exited. Safe to call more than once. Stop(ctx context.Context) error // Release clears the run from the entry once its result is written. Release() } // cmdHold is the entry's cmd field. type cmdHold struct { runID string // starting is set under a.mu by RunCmd's last awake check: from then // on a stop waits for the start. Before it, a stop does not wait: the // check will see the entry stopping and abandon the run. starting bool // cancelPrepare cancels spec.Prepare's context; beginStopLocked calls // it, under a.mu, so a stop interrupts a download. cancelPrepare context.CancelFunc started chan struct{} // closed once StartCmd returned, either way unit CmdUnit // written before started is closed; nil when no unit can be running now func() time.Time mu sync.Mutex asked bool // a stop has been asked for; under mu reason string askedAt time.Time // when the first stop was asked for; under mu // settling is the settle attempt in flight, nil when none is. A stop // joins it rather than starting a second one. Under mu. settling *cmdSettle // proven: a settle attempt succeeded, so nothing of the unit runs. It // is never unset. A failed attempt leaves it false, and the next stop // tries again (merge ruling M1): a failed settle is never cached as // the run's last word while the instance's ownership is given up. // Under mu. proven bool // lastTry is when the last settle attempt ended, by h.now; the sweep's // retry keeps actStopTimeout after it (retrySettle). Under mu. lastTry time.Time // gen is the generation the run was admitted against (merge ruling // M2): its number and its exe pin. RunCmd sets it under a.mu with // starting, and it is never written again. The command execs gen's // build; a swap or a scheduled restart that retires gen interrupts // the run (interruptCmdsLocked), and the generations file names gen's // build until the run is proven settled (entryGenerationsLocked). gen Gen // onProven, when set, runs once a settle has proven the unit gone, // with no lock held: RunCmd's republication of the generations file, // whose pin for gen may have been this run's alone. Set before the // unit starts and never written again. onProven func() } // cmdSettle is one settle attempt: done closes when it has ended, and err // is its result, written before done closes. type cmdSettle struct { done chan struct{} err error } func newCmdHold(runID string, cancelPrepare context.CancelFunc, now func() time.Time) *cmdHold { return &cmdHold{runID: runID, cancelPrepare: cancelPrepare, now: now, started: make(chan struct{})} } // stop waits for the start to return, records reason and the time if it is // the first stop asked for (the run's own Stop passes "", so an instance // stop after it is not what ended the run), and stops and settles the unit. // A unit already proven settled returns nil at once. Otherwise stop joins // the attempt in flight, or starts one, and returns that attempt's result: // an attempt that failed is tried again by the next stop. func (h *cmdHold) stop(ctx context.Context, reason string) error { select { case <-h.started: case <-ctx.Done(): return ctx.Err() } h.mu.Lock() if !h.asked { h.asked = true h.reason, h.askedAt = reason, h.now() } if h.unit == nil || h.proven { h.mu.Unlock() return nil } at := h.settling if at == nil { at = &cmdSettle{done: make(chan struct{})} h.settling = at go h.settle(at) } h.mu.Unlock() select { case <-at.done: return at.err case <-ctx.Done(): return ctx.Err() } } // settle is one attempt: the unit's StopAndSettle, unbounded by any // caller's context (StopAndSettle bounds itself), so a caller that gives // up waiting never cuts a settle short. func (h *cmdHold) settle(at *cmdSettle) { err := h.unit.StopAndSettle(context.Background()) h.mu.Lock() at.err = err if err == nil { h.proven = true } h.settling, h.lastTry = nil, h.now() h.mu.Unlock() close(at.done) if err == nil && h.onProven != nil { h.onProven() } } // retrySettle starts one settle attempt in the background, for the sweep // (re-review 6), when the unit has started and is not proven settled, no // attempt is in flight, and the last one ended at least every ago. A hold // still attached to its run is retried only once a stop has been asked // for it (an interruption, or the run's own Stop that failed): a running // command nobody asked to stop is left alone. A released hold is retried // whatever: its run is over, and only its proof is missing. The reason a // stop recorded is kept. It never blocks. func (h *cmdHold) retrySettle(released bool, every time.Duration) { select { case <-h.started: default: return } h.mu.Lock() defer h.mu.Unlock() if h.unit == nil || h.proven || h.settling != nil || (!released && !h.asked) || h.now().Sub(h.lastTry) < every { return } if !h.asked { h.asked, h.askedAt = true, h.now() } at := &cmdSettle{done: make(chan struct{})} h.settling = at go h.settle(at) } // provenSettled reports whether the hold leaves nothing running: the start // returned no unit, or a settle of the unit succeeded. A failed start that // reached the control service returns a unit, since the unit may be // running. func (h *cmdHold) provenSettled() bool { select { case <-h.started: default: return false } if h.unit == nil { return true } h.mu.Lock() defer h.mu.Unlock() return h.proven } type cmdRun struct { a *Service e *actEntry h *cmdHold once sync.Once } func (r *cmdRun) Invocation() string { return r.h.unit.Invocation() } func (r *cmdRun) Poll(ctx context.Context) (instancectl.CmdStatus, error) { return r.h.unit.Poll(ctx) } func (r *cmdRun) Interrupted() (string, time.Time) { r.h.mu.Lock() defer r.h.mu.Unlock() return r.h.reason, r.h.askedAt } func (r *cmdRun) Stop(ctx context.Context) error { return r.h.stop(ctx, "") } // Release clears the hold and stamps activity, so the instance idles a // full window after a run rather than hibernating at the next sweep. A // unit not proven settled (its Stop failed or never came) stays the // instance's: the next stop settles it before replication closes, and // keeps replication and the lease until it has (run spec §5.3, codex 1; // merge ruling M1). func (r *cmdRun) Release() { r.once.Do(func() { r.a.mu.Lock() if r.e.cmd == r.h { r.e.cmd = nil r.e.lastActive = r.a.now() if !r.h.provenSettled() { r.e.cmdLeft = append(r.e.cmdLeft, r.h) } } r.a.mu.Unlock() }) } // cmdBusyLocked reports whether e cannot take a run: one is attached, or // an earlier one's unit was never proven settled. It drops the released // holds that have since settled. The caller holds a.mu. func cmdBusyLocked(e *actEntry) bool { left := e.cmdLeft[:0] for _, h := range e.cmdLeft { if !h.provenSettled() { left = append(left, h) } } clear(e.cmdLeft[len(left):]) e.cmdLeft = left return e.cmd != nil || len(left) > 0 } // cmdHoldsLocked is every run of e whose unit may still be running: the // released runs not proven settled, and the attached run once it is // starting. An attached run not yet starting is not one: RunCmd's last // awake check sees a stop and abandons it before any unit starts. The // caller holds a.mu. func cmdHoldsLocked(e *actEntry) []*cmdHold { var holds []*cmdHold for _, h := range e.cmdLeft { if !h.provenSettled() { holds = append(holds, h) } } if e.cmd != nil && e.cmd.starting && !e.cmd.provenSettled() { holds = append(holds, e.cmd) } return holds } // cmdStartInFlightLocked reports whether e's run has passed RunCmd's last // check and its StartCmd has not returned: a unit may be starting against // e's current generation right now. An overlap swap is not admitted, and // does not switch, while one is (merge ruling M2). The caller holds a.mu. func cmdStartInFlightLocked(e *actEntry) bool { if e.cmd == nil || !e.cmd.starting { return false } select { case <-e.cmd.started: return false default: return true } } // switchPendingLocked is a live overlap successor of e: a member spawning // or health-checking whose owner has not given up on it (an unproven // member's attempt is over). While there is one, e's current generation // may retire at any moment, so RunCmd starts no unit against it, and waits // for the switch or the successor's end (merge ruling M2). nil when there // is none. The caller holds a.mu. func switchPendingLocked(e *actEntry) *genMember { for _, m := range e.ops.members { if (m.role == roleSpawning || m.role == roleSuccessor) && !m.unproven { return m } } return nil } // cmdOwnershipLocked is why e must not start a run now, or "": the entry // is the host's and awake with its lease (ownedLocked), and its current // generation's process is running. Each refusal is the stop reason it // will become (re-review 5). The caller holds a.mu. func cmdOwnershipLocked(a *Service, e *actEntry) string { switch err := a.ownedLocked(e, false); { case errors.Is(err, errSwapLeaseLost): return StopLeaseLost case errors.Is(err, errSwapFenced): return StopFenced case errors.Is(err, errSwapShutdown): return StopShutdown case err != nil: return StopIdle } if e.proc == nil || e.proc.Exited() { return StopExited } return "" } // cmdSwitchPoll is how often RunCmd re-checks a pending switch it waits on, // besides the successor's own done. var cmdSwitchPoll = 100 * time.Millisecond // interruptCmdsLocked interrupts every run of e admitted against // generation n, which is retiring (merge ruling M2): each unit is stopped // and settled off a.mu, and the run learns why, which the member reads as // interrupted (exit 125, RUN-C75). A settle that fails leaves the run // unproven: its own Stop or the instance's next stop tries again, and the // generations file keeps naming n's build until one succeeds. The caller // holds a.mu. func (a *Service) interruptCmdsLocked(e *actEntry, n uint64, why string) { for _, h := range cmdHoldsLocked(e) { if h.gen.N != n { continue } go func(h *cmdHold) { ctx, cancel := context.WithTimeout(context.Background(), cmdStopBound) defer cancel() if err := h.stop(ctx, why); err != nil { log.Printf("activator: %s: run %s on retired generation %d is not proven stopped: %v; the sweep tries again", e.host, h.runID, n, err) } }(h) } } // retryCmdSettlesLocked is the sweep's retry of every run of e whose stop // was asked for and is not proven (re-review 6): a run interrupted when a // swap retired its generation, a run whose own Stop failed, and a released // run left unsettled. It runs whatever state e is in, pending stop or // awake, so an instance that stays awake still gets its runs proven, their // pins dropped and later runs admitted. Each attempt runs off a.mu, one at // a time per run, actStopTimeout apart. The caller holds a.mu. func (a *Service) retryCmdSettlesLocked(e *actEntry) { for _, h := range e.cmdLeft { h.retrySettle(true, actStopTimeout) } if e.cmd != nil && e.cmd.starting { e.cmd.retrySettle(false, actStopTimeout) } } // settleCmds is a run's part of an instance's termination proof (run spec // §5.3; merge ruling M1): every run of e whose unit may still be running is // stopped and settled, bounded by cmdStopBound, and it reports whether none // may be running now. Proven means the unit stopped and its cgroup is empty // (cmdUnit.StopAndSettle). A settle that fails is not given up on: the stop // stays pending with replication and the lease held, and the sweep's next // pass calls this again (finishPendingStops). Never call it with a.mu held. func (a *Service) settleCmds(e *actEntry) bool { a.mu.Lock() holds, why := cmdHoldsLocked(e), e.stopWhy a.mu.Unlock() if len(holds) == 0 { return true } ctx, cancel := context.WithTimeout(context.Background(), cmdStopBound) defer cancel() proven := true for _, h := range holds { if err := h.stop(ctx, why); err != nil { log.Printf("activator: %s: run %s is not proven stopped: %v", e.host, h.runID, err) proven = false } } return proven } // RunCmd runs one command against rt's instance (run spec §5.2 and §5.3): // refused with claim control armed, for anything but an instance, or on a // layout other than nested; the instance woken through the ordinary wake // path, waiting up to 60 s; the run attached to the awake entry; // spec.Prepare run under the hold, its context cancelled by a stop; and // the unit started only while the entry is still awake. A stop that begins // before the last awake check does not wait for the run, which then ends // here (CmdStoppedError, no unit). One that begins after it waits in // finishStop for the start to return and then stops the unit before // replication closes. // // ctx is the run's own: its deadline and cancellation span the whole // admission, the wake, Prepare, any wait for a swap's switch and the last // checks (re-review 5). A run whose ctx ends before the start returns // ctx's error and starts nothing. func (a *Service) RunCmd(ctx context.Context, rt registry.Route, spec CmdSpec) (CmdHandle, error) { if a.cfg.HostWrites.Enabled { return nil, ErrCmdClaim } if rt.Kind != "instance" { return nil, ErrCmdNotInstance } cb, ok := a.backend.(CmdBackend) if !ok { return nil, ErrCmdNotNested } if c, ok := a.backend.(Classifier); !ok { return nil, ErrCmdNotNested } else if l, _ := c.Layout(rt); l != LayoutNested { return nil, ErrCmdNotNested } wctx, cancel := context.WithTimeout(ctx, cmdWakeBound) defer cancel() var release func() for { r, err := a.begin(wctx, rt, true, nil) if err == nil { release = r break } if errors.Is(err, errWakeTimeout) && wctx.Err() == nil { continue // begin's own bound is 30 s; a run waits 60 } if wctx.Err() != nil && ctx.Err() == nil { err = errWakeTimeout } return nil, &CmdWakeError{Reason: memberSafeWakeReason(err), Err: err} } pctx, pcancel := context.WithCancel(ctx) defer pcancel() h := newCmdHold(spec.RunID, pcancel, a.now) a.mu.Lock() e := a.entries[rt.Host] switch { case e == nil || e.state != actAwake: why := StopIdle if e != nil && e.stopWhy != "" { why = e.stopWhy } a.mu.Unlock() release() return nil, &CmdStoppedError{Reason: why} case e.worker: a.mu.Unlock() release() return nil, ErrCmdNotInstance case cmdBusyLocked(e): a.mu.Unlock() release() return nil, ErrCmdBusy } e.cmd = h a.mu.Unlock() release() // the hold, not the in-flight count, keeps the idle timer off now run := &cmdRun{a: a, e: e, h: h} abandon := func(err error) (CmdHandle, error) { close(h.started) run.Release() return nil, err } var prepErr error if spec.Prepare != nil { prepErr = spec.Prepare(pctx) } a.mu.Lock() // waited: the run has waited for a switch since launch eligibility was // last checked, by Prepare or by spec.Recheck. A wait can be long, so // eligibility is checked again before anything starts (re-review 5). waited := false for { if e.state != actAwake { why := e.stopWhy a.mu.Unlock() return abandon(&CmdStoppedError{Reason: why}) } if prepErr != nil { a.mu.Unlock() return abandon(prepErr) } // ctx carries the run's deadline and cancellation (the box's run // pass derives it from the record): a run cancelled or expired at // any point of its admission never starts (re-review 5). if err := ctx.Err(); err != nil { a.mu.Unlock() return abandon(err) } // Ownership, and a generation that is serving: the lease is held, // no fence or shutdown holds the host, and the current // generation's process is running. if why := cmdOwnershipLocked(a, e); why != "" { a.mu.Unlock() return abandon(&CmdStoppedError{Reason: why}) } // An overlap successor is on its way: the current generation may // retire before the unit is even up, so nothing starts against it. // The run waits for the switch (and then runs the successor's // build) or for the successor's end (merge ruling M2). A stop // cancels pctx, as do the run's own cancel and deadline, and the // checks above then abandon the run. if m := switchPendingLocked(e); m != nil { a.mu.Unlock() select { case <-m.done: case <-time.After(cmdSwitchPoll): case <-pctx.Done(): } waited = true a.mu.Lock() continue } if !waited || spec.Recheck == nil { break } // The run's own eligibility, which Prepare checked before the // wait, read again off a.mu (the record), then every check above // once more, since the entry may have moved meanwhile. a.mu.Unlock() prepErr = spec.Recheck(pctx) waited = false a.mu.Lock() } // Admitted against the current generation, under the lock a swap's // admission and switch take (merge ruling M2): no successor is live, // and a swap admitted from here on is refused until StartCmd returns // (cmdStartInFlightLocked). h.starting = true h.gen = e.gen h.onProven = func() { // The pin for h.gen is this run's alone once that generation is // no longer the entry's current one. a.mu.Lock() stale := a.entries[e.host] != e || e.gen.N != h.gen.N a.mu.Unlock() if stale { _ = a.persistGenerations() } } // What the command runs is what the instance serves now: the current // generation's recorded file, by path, never the live link, which a // repoint, a fallback or a held update can move off it (zero-downtime // spec §7.5). That holds whatever CARLOS_SWAP says, as it does for the // legacy sidecar (settings ruling R-EXEC-SIDECAR): since settings Task // 5c a flag-off wake runs its recorded file by path too, so a flag-off // command on the live link could run a different build than its // instance. Only a generation that names no file (a backend that cannot // pin one) leaves the run the live link. spec.serving = nil if h.gen.StagedPath != "" { spec.serving = &instancectl.StagedExe{Path: h.gen.StagedPath, Version: h.gen.Build, Digest: h.gen.Digest} } // And with the environment that generation launched with, from its // handle: never the delivered files, which may hold a config revision // it has not activated, a config swap queued or failed (merge ruling // M3). A handle that kept none gets none, and StartCmd refuses. spec.genEnv, spec.genKeyLayer = nil, instancekey.LayerNone if lp, ok := e.proc.(launchEnvProc); ok { spec.genEnv = lp.LaunchEnv() } if lp, ok := e.proc.(launchKeyLayerProc); ok { spec.genKeyLayer = lp.LaunchKeyLayer() } a.mu.Unlock() // One bound covers the build-state write and the start, so a stop // waiting on the start still has the settle's budget after it // (cmdStartBound). sctx, scancel := context.WithTimeout(ctx, cmdStartBound) defer scancel() // The build about to touch the instance's database is recorded first // (settings spec §4.2), as the sidecar's is (startSidecar): a command // runs its serving generation's build against the data, and the build // state must name that build before it does. A failed successor may // have left the record naming its own build; without this write a // command on the incumbent would run under a record that names // another, and a held recovery could pick the wrong one. A write that // fails refuses the launch (errBuildStateUnwritten): nothing starts. // // The starting hold is kept throughout. A stop that begins meanwhile // waits for this start to return, and no overlap swap is admitted // (cmdStartInFlightLocked), so the generation recorded is still the // one serving when the unit starts. The write takes time, so // ownership and the run's own cancel and deadline are checked again // after it, before anything starts. if err := a.recordBuild(sctx, rt.Host, h.gen.Build); err != nil { return abandon(err) } a.mu.Lock() var refused error switch { case e.state != actAwake: refused = &CmdStoppedError{Reason: e.stopWhy} case ctx.Err() != nil: refused = ctx.Err() case e.gen.N != h.gen.N: // Not reached while the starting hold keeps swaps out; a // generation that changed anyway is never started against. refused = errSwapBusy default: if why := cmdOwnershipLocked(a, e); why != "" { refused = &CmdStoppedError{Reason: why} } } a.mu.Unlock() if refused != nil { return abandon(refused) } u, err := cb.StartCmd(sctx, rt, spec) h.unit = u close(h.started) if err != nil { if u != nil { // The start reached the control service, so the unit may be // running: settle it now. Release keeps it for the next stop // if it does not settle. sctx, scancel := context.WithTimeout(context.WithoutCancel(ctx), cmdStopBound) _ = h.stop(sctx, "") scancel() } run.Release() return nil, err } a.mu.Lock() stopping, why := e.state != actAwake, e.stopWhy a.mu.Unlock() if stopping { // A stop began during the start. finishStop is waiting to stop the // unit, unless its wait already timed out: stop it here too (once // either way), so a late start never outlives the entry. go h.stop(context.Background(), why) //nolint:errcheck } return run, nil }