package host // The layout pass on the box (isolated-exec design §8.2): recovery at agent // start, before the activator can wake anything, and convergence to the // fleet's recorded layout intent on every serial move after that. The pass // itself lives in internal/layout; this file is how the agent builds it, // publishes it to the loops that run on other goroutines, and keeps it off // the serial loop's own goroutine. import ( "context" "fmt" "log" "os" "os/user" "strconv" "strings" "sync" "sync/atomic" "time" "github.com/carlosframework/platform/internal/activator" "github.com/carlosframework/platform/internal/layout" "github.com/carlosframework/platform/internal/provision" "github.com/carlosframework/platform/internal/registry" "github.com/carlosframework/platform/internal/release" "github.com/carlosframework/platform/internal/status" "github.com/carlosframework/platform/internal/version" ) // armedLayoutPass is the box's one layout pass. RunAgent stores it once, // after startup recovery and before the activator starts, and the serial // leg (fastConvergeLoop's goroutine) and the status snapshot read it. Nil // means no activator is armed, so there is nothing for a pass to move. var armedLayoutPass atomic.Pointer[layout.Pass] // layoutReload is the live edge's Reload, published by the edge's ready // hook, which runs after the pass is built. Until then the pass's reload is // a no-op and the edge's own 2s registry poll picks up a flipped row. var layoutReload atomic.Pointer[func()] // layoutIntentSeen is the last outcome of a layout intent read for this box: // "" when none is recorded or none addresses it, layout.IntentUnreadable // when the last read failed or named a mode this binary does not know. A // later successful read replaces whichever of the two it holds, so this // only ever needs to keep "no intent" apart from "we could not tell". var layoutIntentSeen atomic.Pointer[string] // lastLayoutIntent is the mode layoutIntentSeen holds, "" before any read. func lastLayoutIntent() string { if m := layoutIntentSeen.Load(); m != nil { return *m } return "" } func noteLayoutIntent(mode string) { layoutIntentSeen.Store(&mode) layoutDefaultSeen.Store(&layoutDefaultState{}) } // layoutDefaultEnv is the deployment's answer when no layout intent is // recorded (isolated-exec design §8.6, "Nested by default"): nested, or // flat for a deployment that is not ready to isolate its tenants. Read // from host.env like CARLOS_SYSTEM_CHANNEL, because it is a fact about the // box's deployment that an operator sets once at bootstrap, and the agent // reads every such fact the same way. const layoutDefaultEnv = "CARLOS_LAYOUT_DEFAULT" // layoutDefault reads CARLOS_LAYOUT_DEFAULT: "" and "nested" are nested, // "flat" is flat. Anything else is flat with the raw value returned as // invalid, so the caller can log it: an operator who typed the variable // meant to choose, and leaving the box as it was (flat) while saying so is // the reading that cannot surprise them. func layoutDefault(getenv func(string) string) (mode string, invalid string) { v := strings.ToLower(strings.TrimSpace(getenv(layoutDefaultEnv))) switch v { case "", release.LayoutModeNested: return release.LayoutModeNested, "" case release.LayoutModeFlat: return release.LayoutModeFlat, "" } return release.LayoutModeFlat, v } // layoutDefaultGate is the readiness the default consults before it stands // in for a nested intent: the same nestedGate the exec backend and the // layout pass refuse nested work with (instancefs.SelfCheck, the three // nosymfollow mounts, the control service's probe verdict). Stored when the // pass is built; nil means no activator is armed, so nothing could wake // nested and the default does not apply. var layoutDefaultGate atomic.Pointer[func() error] // layoutDefaultState is what status says about the default: Applied when // the box is nested by default rather than by a recorded intent, Blocked // with the reason when the deployment's default is nested but this box is // not ready for it. type layoutDefaultState struct { Applied bool Blocked string } var layoutDefaultSeen atomic.Pointer[layoutDefaultState] func lastLayoutDefault() layoutDefaultState { if d := layoutDefaultSeen.Load(); d != nil { return *d } return layoutDefaultState{} } // layoutBuild is the running build the default's floor check places; // a var so a test can name a release. var layoutBuild = func() string { return version.Build } // defaultLayoutIntent is the effective intent when none is recorded // (design §8.6, "Nested by default"). It is nested only on a box that is // READY: the deployment's default is nested (CARLOS_LAYOUT_DEFAULT), the // running build is at or above release.LayoutFloor when it can be placed // (a build that evaluates this rule carries the pass by construction, so // an unplaceable dev build passes; a mis-tagged release below the floor // does not), and the nested gate answers: SelfCheck, the mounts and the // probe verdict. Anything else is "" (flat, as before the default), with // the reason recorded for status so an operator can see why a box has // not moved on its own. The default never forces: an awake host gets its // grace exactly as under a recorded intent. func defaultLayoutIntent() (mode string, reason string) { def, invalid := layoutDefault(os.Getenv) if invalid != "" { return "", layoutDefaultEnv + "=" + invalid + " is not nested or flat; staying flat" } if def != release.LayoutModeNested { return "", "" } if b := layoutBuild(); release.BelowFloor(b, release.LayoutFloor) { return "", "this box runs " + b + ", below v" + release.LayoutFloor.String() + ", the first release on which nested instances wake" } gate := layoutDefaultGate.Load() if gate == nil { return "", "no activator is armed on this box, so nothing could wake nested" } if err := (*gate)(); err != nil { return "", "isolation not ready: " + err.Error() } return release.LayoutModeNested, "" } // noteLayoutDefault records the default's outcome for status: nested by // default when mode is nested, otherwise none with the reason it did not // apply. func noteLayoutDefault(mode, reason string) { if mode == release.LayoutModeNested { layoutIntentSeen.Store(&mode) layoutDefaultSeen.Store(&layoutDefaultState{Applied: true}) return } none := release.LayoutModeNone layoutIntentSeen.Store(&none) layoutDefaultSeen.Store(&layoutDefaultState{Blocked: reason}) } // layoutStatusReport builds the box's layout report for the status // snapshot (design §8.4, amendment R15): the mode this box last converged // to, and every instance-kind route's word from the armed pass's own // Classify — the same classification a wake would see, so status never // says something different from what the activator actually does. // // With no activator armed there is no pass and no Classify, but the box // is not silent either: it reports Mode "unmanaged" with every // instance-kind row judged by path shape alone (provision.NestedPaths), // because the one thing a reader needs to know about such a box is // whether a downgrade would strand a nested database, and that is a fact // about paths, not about wakes. Nothing wakes a hibernate-flagged row on // such a box, so no row there is managed, and the console's downgrade // guard (layoutguard.go) treats the report like any other: a row named // nested blocks, flat rows do not. Silence from a box at or above the // floor would otherwise block its fleet's rollback for good. // // routes is the same list the caller already read for its other probes; // this never lists the registry itself, mirroring every other probe in // this file. func layoutStatusReport(routes []registry.Route) *status.LayoutReport { pass := armedLayoutPass.Load() if pass == nil { return unmanagedLayoutReport(routes) } mode := lastLayoutIntent() if mode == "" { mode = "none" } def := lastLayoutDefault() rows := map[string]status.RowLayout{} for _, rt := range routes { // Only instance-kind rows: a sidecar (Kind "sidecar") is outside // the pass entirely (finding S6), and so is everything that is // neither an instance nor a sidecar (static, service). if rt.Kind != "instance" { continue } switch { case activator.ActivatorManaged(rt): word, reason := pass.Classify(rt, routes) // A flat host the pass keeps fenced and down after a refused // nested move classifies flat (no journal), but it does not // wake: status says so, loudly, with the refusal. if why, held := pass.Held(rt.Host); held && word == activator.LayoutFlat { word, reason = activator.LayoutFenced, "held down after a refused move to nested: "+why } rows[rt.Host] = status.RowLayout{Layout: string(word), Reason: reason, Note: pass.Note(rt.Host)} case rt.Socket != "" && !rt.Hibernate: // ActivatorManaged's own doc: an unflagged socket route is run // by another supervisor, nobody's to migrate or isolate. rows[rt.Host] = status.RowLayout{Layout: "external"} } } return &status.LayoutReport{Mode: mode, Rows: rows, Default: def.Applied, DefaultBlocked: def.Blocked} } // LayoutModeUnmanaged is the report Mode of a box with no activator armed: // no pass runs there and no hibernate-flagged row ever wakes, so its rows // are judged by path shape alone (see layoutStatusReport). const LayoutModeUnmanaged = "unmanaged" // unmanagedLayoutReport is layoutStatusReport's answer with no pass armed. // Instance-kind rows only, as the managed report: fenced when a journal or // hold marker names the host (a move was in flight, or a refused move kept // the host down, when the activator went away, and the files are still // where recovery expects them), nested by path shape, external for an // unflagged socket route, flat otherwise. The fenced check comes first, // as in activator.Classify: a mid-move host's row can still carry flat // paths while its files are already between two directories. func unmanagedLayoutReport(routes []registry.Route) *status.LayoutReport { rows := map[string]status.RowLayout{} dd := provisionTickConfig(os.Getenv).DataDir for _, rt := range routes { if rt.Kind != "instance" { continue } switch { case layout.JournalPresent(dd, rt.Host): rows[rt.Host] = status.RowLayout{Layout: string(activator.LayoutFenced), Reason: "layout migration in flight; see " + provision.LayoutJournalPath(dd, rt.Host) + " (no activator on this box to recover it)"} case layout.HoldMarkerPresent(dd, rt.Host): rows[rt.Host] = status.RowLayout{Layout: string(activator.LayoutFenced), Reason: "held down after a refused move to nested (no activator on this box to retry it)"} case provision.NestedPaths(dd, rt): rows[rt.Host] = status.RowLayout{Layout: string(activator.LayoutNested), Reason: "no activator on this box; judged by path shape"} case rt.Socket != "" && !rt.Hibernate: rows[rt.Host] = status.RowLayout{Layout: "external"} case rt.Hibernate && rt.Socket != "": rows[rt.Host] = status.RowLayout{Layout: string(activator.LayoutFlat)} } } return &status.LayoutReport{Mode: LayoutModeUnmanaged, Rows: rows} } // reloadRouting asks the live edge to rebuild its routes now, if there is // one yet. func reloadRouting() { if f := layoutReload.Load(); f != nil { (*f)() } } // serializeWithConfigSync runs f under the lock every SyncExec pass holds, // so config sync never renders a host's bundle into a directory the pass // is moving the host out of. The pass holds nothing but its own fence when // it asks, and SyncExec under this lock never waits on a fence or on the // pass (its bounce only starts a stop), so the two cannot wait on each // other. func serializeWithConfigSync(f func()) { configSyncMu.Lock() defer configSyncMu.Unlock() f() } // socketsGID is the carlos-sockets group's id, which a nested host's run // directory is given so the edge can reach its socket, or -1 to leave the // group alone where the group does not exist (a dev machine). func socketsGID() int { g, err := user.LookupGroup(socketsGroup) if err != nil { return -1 } n, err := strconv.Atoi(g.Gid) if err != nil { return -1 } return n } // nestedRowsPresent reports whether any activator-managed row on the box is // already at the nested paths. func nestedRowsPresent(rows []registry.Route) bool { dd := provisionTickConfig(os.Getenv).DataDir for _, rt := range rows { if activator.ActivatorManaged(rt) && provision.NestedPaths(dd, rt) { return true } } return false } // newLayoutPass builds the pass from what only RunAgent has in hand. // nestedOK must be the same gate the exec backend refuses nested wakes // with, so the pass never makes a nested tenant the box would refuse to // start. swept is whether the startup unit sweep succeeded; without it the // pass never trusts an unproven stop. // // DeferAwake is set because the only Converge caller is the serial leg, // which runs while tenants serve; startup recovery never reaches the code // that reads it. func newLayoutPass(reg *registry.Registry, svc *activator.Service, dd string, nestedOK func() error, swept bool) *layout.Pass { p := &layout.Pass{ Registry: reg, DataDir: dd, SocketsGID: socketsGID(), Reload: reloadRouting, Logf: log.Printf, SweptAtStart: swept, NestedOK: nestedOK, Serialize: serializeWithConfigSync, DeferAwake: true, } // Assigned only when non-nil: a nil *Service inside the interface would // not be a nil Fence, and the pass would call through it. if svc != nil { p.Fence = svc } // The pass fences from the box's generation source, the one the import // rollback's journal stores its generation from (fenceGenerations), so // a rollback resumed after a restart never meets this pass's fence // under its own number. A source that fails falls back to the pass's // own default, the wall clock. if reg != nil { next := fenceGenerations(reg) p.Generation = func() uint64 { if g, err := next(); err == nil { return g } return uint64(time.Now().UnixNano()) } } // The default's readiness is this same gate (defaultLayoutIntent). if nestedOK != nil { layoutDefaultGate.Store(&nestedOK) } return p } // startActivator is the order the activator comes up in. The startup unit // sweep first, so nothing an earlier agent left is running. Then layout // recovery, which runs even when the sweep failed (completing a journal // is exact) and is told whether it succeeded. Then Start, so no host is // woken while a journal says its files are between two directories. func startActivator(ctx context.Context, sweep func(context.Context) error, recoverLayout func(ctx context.Context, swept bool), start func()) { err := sweep(ctx) recoverLayout(ctx, err == nil) start() } // activatorStart keeps sweep, recovery and Start ordered. Legacy boxes run // the sequence in the background; claim boxes still call Start alone. // Ready releases requests and agent duties only after Start returns. The // returned channel also joins the instance-key pass on shutdown. type activatorStart struct { Configure func(context.Context) Ready func() Claims bool // claims.Writes.Enabled: a claim box Sweep func(context.Context) error // the instance-unit sweep Recover func(context.Context, bool) *layout.Pass // builds the pass and recovers under it; the bool is whether the sweep succeeded Routes func() ([]registry.Route, error) // the registry's List Route func(string) (*registry.Route, error) // the registry's Get: each row is read again in its host's slot Keys instanceKeyBackfiller // the exec backend Slot hostPreparationSlot // the service's PrepareScheduledToken Publish func(*layout.Pass) // armedLayoutPass.Store Start func() // the service's Start } // Run returns immediately on legacy boxes. Cancellation prevents a late // Start and reaches the key pass through the agent lifetime context. func (s activatorStart) Run(ctx context.Context) <-chan struct{} { done := make(chan struct{}) start := func() { if ctx.Err() == nil { s.Start() if s.Ready != nil { s.Ready() } } } if s.Claims { if s.Configure != nil { s.Configure(ctx) } start() close(done) return done } go func() { defer close(done) if s.Configure != nil { s.Configure(ctx) } if ctx.Err() != nil { return } startActivator(ctx, s.Sweep, func(ctx context.Context, swept bool) { if ctx.Err() == nil { s.Publish(s.Recover(ctx, swept)) } }, start) if ctx.Err() != nil { return } routes, err := s.Routes() if err != nil { log.Printf("agent: instance keys: list routes: %v (each host's next start mints its key)", err) return } backfillInstanceKeys(ctx, routes, keyBackfillInSlot{keys: s.Keys, slot: s.Slot, route: s.Route}) }() return done } // layoutIntentTimeout bounds the one intent read at startup, which the // activator's start waits on. const layoutIntentTimeout = 30 * time.Second // startupLayoutIntent reads the intent for startup recovery when there is // a journal to recover (startupRecoveryIntent). It is "" // when no intent is recorded or none addresses this box. It is // IntentUnreadable when the box cannot be sure: no deploy store, a failed // read, or a request whose mode this binary does not know. Under that // intent recovery holds a completed revert fenced rather than release a // host flat that the fleet may want nested. func startupLayoutIntent(ctx context.Context, store release.Store, storeErr error, boxLabel string) layout.Mode { if storeErr != nil { log.Printf("layout: intent -> unreadable: deploy store: %v", storeErr) noteLayoutIntent(string(layout.IntentUnreadable)) return layout.IntentUnreadable } ctx, cancel := context.WithTimeout(ctx, layoutIntentTimeout) defer cancel() in, found, err := release.LoadLayoutIntent(ctx, store, boxLabel) switch { case err != nil: log.Printf("layout: intent -> unreadable: %v", err) noteLayoutIntent(string(layout.IntentUnreadable)) return layout.IntentUnreadable case !found: // No recorded intent. The default is not asked its gate here: at // agent start the control service's probe has not run yet (its // /ready answers "not yet run" until the first async probe lands), // so the gate would say "not ready" on every start, and recovery // under "" drops every hold marker (RestoreHolds) and releases a // completed revert, handing a tenant that kept its move refused a // fresh grace to run flat. So when the deployment's default is // nested, startup recovery runs under IntentUnreadable, the // fail-closed intent: holds and journals are kept, nothing is // released flat, and the first serial move reads the intent with // the gate's real answer. The opt-out (CARLOS_LAYOUT_DEFAULT=flat) // keeps today's behaviour, dropping holds as a recorded flat does. if def, invalid := layoutDefault(os.Getenv); def == release.LayoutModeNested && invalid == "" { log.Printf("layout: no intent recorded at start -> holds and journals kept until the first serial move asks the isolation gate (nested by default)") noteLayoutDefault("", "agent starting: holds and journals kept until the first serial move asks the isolation gate") return layout.IntentUnreadable } mode, reason := defaultLayoutIntent() noteLayoutDefault(mode, reason) return layout.Mode(mode) case !release.ValidLayoutMode(in.Mode): log.Printf("layout: intent %s -> unreadable: mode %q is not flat or nested", in.ID, in.Mode) noteLayoutIntent(string(layout.IntentUnreadable)) return layout.IntentUnreadable } noteLayoutIntent(in.Mode) return layout.Mode(in.Mode) } // startupRecoveryIntent is the intent startup recovery runs under. The // intent only matters to a journal, so with none in /layout it is "" // and the deploy store is not opened: the activator's start does not wait // on a read it has no use for, and the published intent is left for the // first serial leg to set. A layout directory that cannot be listed may // hold journals, so it is read as usual. func startupRecoveryIntent(ctx context.Context, dataDir string, open func(context.Context) (release.Store, error), boxLabel string) layout.Mode { hosts, err := layout.JournalHosts(dataDir) // A hold marker is the other thing startup acts on (RestoreHolds), so // it too makes the intent worth reading. holds, herr := layout.HoldMarkers(dataDir) if err == nil && herr == nil && len(hosts) == 0 && len(holds) == 0 { return "" } if herr != nil { log.Printf("layout: list hold markers at start: %v; reading the intent anyway", herr) } if err != nil { log.Printf("layout: list journals at start: %v; reading the intent anyway", err) } ctx, cancel := context.WithTimeout(ctx, layoutIntentTimeout) defer cancel() store, err := open(ctx) return startupLayoutIntent(ctx, store, err, boxLabel) } // recoverLayoutAtStart runs the pass's recovery under the startup intent, // then restores the holds whose markers survived the restart, and logs // what it did. Both run before the activator can wake anything. func recoverLayoutAtStart(ctx context.Context, pass *layout.Pass, intent layout.Mode) { logLayoutResults(pass.Recover(ctx, intent)) logLayoutResults(pass.RestoreHolds(ctx, intent)) } // layoutLeg runs the layout pass off the serial loop. A pass can wait up to // a minute per host for a stop, and the restart, wake, schedule and system // legs and the fast sweep must not wait behind it. One pass runs at a time; // a serial move that lands while one is running marks the leg dirty, and // the leg runs exactly once more after it, however many moves landed. type layoutLeg struct { run func(context.Context) running atomic.Bool dirty atomic.Bool wg sync.WaitGroup retryMu sync.Mutex retryPending bool } // newLayoutLeg builds the leg around one run and, when a pass is already // armed, kicks it once. The leg otherwise runs only on a serial move or // its own retry timer, and startup recovery runs fail-closed without // asking the isolation gate (startupLayoutIntent), so a box that restarts // and then sees no console bump would keep what startup fenced for as // long as the fleet stayed quiet. The kick is the first evaluation with // the gate's real answer, and it depends on nothing but this process. // fastConvergeLoop builds the leg after startActivator has returned, so // the pass is armed by then on every box that has an activator; with none // armed the kick reads nothing (runLayoutLeg). func newLayoutLeg(ctx context.Context, run func(context.Context) bool) *layoutLeg { l := &layoutLeg{} l.run = func(ctx context.Context) { if run(ctx) { l.retryLater(ctx, awakeRetry) } } if armedLayoutPass.Load() != nil { l.kick(ctx) } return l } // kick asks for a pass and returns at once. func (l *layoutLeg) kick(ctx context.Context) { l.dirty.Store(true) if !l.running.CompareAndSwap(false, true) { return } l.wg.Add(1) go func() { defer l.wg.Done() for { for l.dirty.Swap(false) { l.run(ctx) } if !l.goIdle() { return } } }() } // goIdle marks the runner idle and reports whether it must run again. A // kick that landed after the runner's last look at dirty saw the leg // running and left its request there; the runner takes it back up, unless // that kick has already started a runner of its own. func (l *layoutLeg) goIdle() bool { l.running.Store(false) return l.dirty.Load() && l.running.CompareAndSwap(false, true) } // wait blocks until no pass is running or pending. func (l *layoutLeg) wait() { l.wg.Wait() } // runLayoutLeg is one run of the serial leg: the published pass, converged // to this box's intent. With no pass armed there is nothing to move, and // the intent is not read. func runLayoutLeg(ctx context.Context, store release.Store, boxLabel string) (retry bool) { pass := armedLayoutPass.Load() if pass == nil { return false } return layoutLegWith(ctx, store, boxLabel, func(ctx context.Context, mode layout.Mode, force bool) []layout.Result { // The pass is single-flight (the leg), so flipping its deferral // for one run races nothing. A forced intent stops awake hosts // now; an ordinary one gives them the pass's awake grace. pass.DeferAwake = !force return pass.Converge(ctx, mode) }) } // awakeRetry is how long after a run that deferred an awake host the leg // runs again on its own. The leg otherwise runs only on a serial move, // and the awake grace is a bound in wall time: without this a quiet // fleet would never come back to stop the host once its grace ran out. const awakeRetry = 10 * time.Minute // retryLater arms one timer to kick the leg again. A timer already // pending is enough; the run it starts re-evaluates every host. func (l *layoutLeg) retryLater(ctx context.Context, after time.Duration) { l.retryMu.Lock() defer l.retryMu.Unlock() if l.retryPending { return } l.retryPending = true time.AfterFunc(after, func() { l.retryMu.Lock() l.retryPending = false l.retryMu.Unlock() if ctx.Err() == nil { l.kick(ctx) } }) } // layoutLegWith is the leg's decision: read the intent, and converge only // when a request addresses this box with a mode this binary knows. An // intent that cannot be read moves nothing and publishes // layout.IntentUnreadable, overwriting whatever mode was last seen, since a // read that just failed is not proof the fleet still wants it; the next // serial move reads it again. // // It reports whether the run deferred an awake host, so the caller can // come back for it when its grace runs out. func layoutLegWith(ctx context.Context, store release.Store, boxLabel string, converge func(context.Context, layout.Mode, bool) []layout.Result) (retry bool) { in, found, err := release.LoadLayoutIntent(ctx, store, boxLabel) switch { case err != nil: log.Printf("layout: intent -> unreadable: %v; not converging this move", err) noteLayoutIntent(string(layout.IntentUnreadable)) return false case !found: // No recorded intent: the deployment's default stands in, on a // box that is ready for it (design §8.6, "Nested by default"). It // never forces, so an awake host keeps its grace; a box that is // not ready converges nothing and status says why. Logged once // per change of answer, not per serial move. mode, reason := defaultLayoutIntent() if prev := lastLayoutDefault(); prev.Blocked != reason || prev.Applied != (mode == release.LayoutModeNested) { if mode == release.LayoutModeNested { log.Printf("layout: no intent recorded -> nested by default (%s unset or nested; record `carlos system layout revert` to opt out)", layoutDefaultEnv) } else if reason != "" { log.Printf("layout: no intent recorded; nested by default is not in force -> %s", reason) } } noteLayoutDefault(mode, reason) if mode != release.LayoutModeNested { // A box the default has not reached yet is retried on the // leg's own timer (awakeRetry), for the same reason an awake // host is: the probe that gates it answers in wall time (its // first run lands seconds after the control service starts), // and a quiet fleet would otherwise never come back to ask. // The opt-out and a typo are not waiting on anything. def, invalid := layoutDefault(os.Getenv) return def == release.LayoutModeNested && invalid == "" } return logLayoutResults(converge(ctx, layout.ModeNested, false)) case !release.ValidLayoutMode(in.Mode): log.Printf("layout: intent %s -> unreadable: mode %q is not flat or nested; not converging this move", in.ID, in.Mode) noteLayoutIntent(string(layout.IntentUnreadable)) return false } noteLayoutIntent(in.Mode) return logLayoutResults(converge(ctx, layout.Mode(in.Mode), in.Force)) } // logLayoutResults logs every result that changed or withheld something, // and reports whether any host was deferred for being awake. Those hosts // are one line together: under a standing intent every serial move meets // them until they sleep or their grace runs out. // // It reports whether the leg should run again on its own: a host deferred // for being awake has a grace that runs out in wall time, and a held host // should come back within awakeRetry of an operator clearing its reason, // quiet fleet or not. func logLayoutResults(rs []layout.Result) (retry bool) { var awake []string for _, r := range rs { switch { case r.Outcome == "unchanged": case r.Outcome == "held": retry = true log.Printf("layout: %s: %s -> %s", r.Host, r.Outcome, r.Reason) case r.Outcome == "deferred" && strings.HasPrefix(r.Reason, "awake"): awake = append(awake, r.Host+" ("+r.Reason+")") case r.Host == "": log.Printf("layout: %s -> %s", r.Outcome, r.Reason) case r.Reason == "": log.Printf("layout: %s: %s", r.Host, r.Outcome) default: log.Printf("layout: %s: %s -> %s", r.Host, r.Outcome, r.Reason) } } if len(awake) > 0 { log.Printf("layout: deferred until asleep -> %s", strings.Join(awake, ", ")) } return retry || len(awake) > 0 } // mintLayoutIntent is provision.ReconcileConfig.Layout for this box: the // mode of the newest intent addressing its label, "" when none does. An // unreadable intent is an error, and the reconciler defers its mints // rather than minting flat on a box the fleet may want nested. The same // read the layout leg makes, so a mint and a pass agree. func mintLayoutIntent(ctx context.Context, store release.Store) (string, error) { return mintLayoutIntentWith(ctx, store, nil) } // mintLayoutIntentWith is mintLayoutIntent with the box's own rows in hand // (nil: no registry to ask). With no intent recorded and the deployment's // default nested, the default's gate is asked; and when the gate has not // yet answered (the first reconcile after an agent restart runs before the // control service's first probe) a box whose registry already holds an // activator-managed row at the nested paths counts as nested for mints // all the same: the box has evidently moved, and a flat row minted beside // nested siblings is the exposure the default exists to end. The gate // still guards every wake, so a nested row minted this way waits for it. func mintLayoutIntentWith(ctx context.Context, store release.Store, routes func() ([]registry.Route, error)) (string, error) { in, found, err := release.LoadLayoutIntent(ctx, store, BoxLabel()) switch { case err != nil: return "", err case !found: mode, _ := defaultLayoutIntent() if mode == release.LayoutModeNested { return mode, nil } if def, invalid := layoutDefault(os.Getenv); def == release.LayoutModeNested && invalid == "" && routes != nil { rows, err := routes() if err != nil { return "", fmt.Errorf("layout default: list routes: %w", err) } if nestedRowsPresent(rows) { return release.LayoutModeNested, nil } } return release.LayoutModeNone, nil case !release.ValidLayoutMode(in.Mode): return "", fmt.Errorf("layout intent %s names mode %q, which this binary does not know", in.ID, in.Mode) } return in.Mode, nil }