package host // The box agent's reads through the S3 change index (S3 change-index design // §4.3-§4.5, discussion #336). Armed only by CARLOS_CHANGE_INDEX=read; unset, // every pointer below stays nil and every loop reads the bucket exactly as // before. import ( "context" "crypto/ecdh" "crypto/ed25519" "fmt" "log" "math/rand/v2" "os" "sort" "strings" "sync" "sync/atomic" "time" "github.com/carlosframework/platform/internal/activator" "github.com/carlosframework/platform/internal/domains" "github.com/carlosframework/platform/internal/edgesfile" "github.com/carlosframework/platform/internal/hostwrite" "github.com/carlosframework/platform/internal/provision" "github.com/carlosframework/platform/internal/registry" "github.com/carlosframework/platform/internal/release" ) // changeIndexArmed reports whether this process reads through the change // index. Only the exact value "read" arms it (spec §7). func changeIndexArmed(getenv func(string) string) bool { return getenv("CARLOS_CHANGE_INDEX") == "read" } // armedChangeWatch is the process's one watch on the change index, and // armedRecordView the record view that follows it. RunAgent sets both once // at start-up when armed; nil means today's behaviour everywhere. var ( armedChangeWatch atomic.Pointer[release.ChangeWatch] armedRecordView atomic.Pointer[provision.RecordView] ) // backstopFullDue is the backstop's request for the unrestricted full pass: // set after each Rebuild, taken by fastTick on the fast converge loop's own // goroutine (fastDeps' maps are single-goroutine by construction). var backstopFullDue atomic.Bool // armChangeIndex arms the change index for this process: a watch on the // deployment store, and a record view over the same store. A fleet box // stays disarmed with the variable set: its store door has no conditional // read, so its watch would be unusable and every Refresh would walk anyway // (spec §3.4). The view's store must take no agent lock on any read path // (RecordView.mu is a leaf, spec §4.3): open hands back the plain deployment // store the other loops use. func armChangeIndex(ctx context.Context, getenv func(string) string, open func(context.Context) (release.Store, error)) *provision.RecordView { if !changeIndexArmed(getenv) { return nil } if shouldSkipPublisherElection(getenv) { log.Printf("agent: change index: CARLOS_CHANGE_INDEX=read is ignored on a fleet box; record reads stay on the walk") return nil } store, err := open(ctx) if err != nil { log.Printf("agent: change index: not armed: %v", err) return nil } watch := release.NewChangeWatch(store) v := provision.NewRecordView(store, watch) armedChangeWatch.Store(watch) armedRecordView.Store(v) log.Printf("agent: change index armed: record walks read the record view (backstop %s)", release.ChangeIndexBackstop) return v } // newChangeGate is a read gate over the armed watch, built by a loop when it // starts: RunAgent arms the index before it starts any loop, so the watch a // gate is built over is the one the process keeps. Disarmed, the watch is nil // and every read is due, as before the index (spec §5). func newChangeGate() *release.ReadGate { return newChangeGateOver(armedChangeWatch.Load()) } // newChangeGateOver is the one place the gated reads' maximum age is set: a // successful read stands in for the next one for at most the backstop. func newChangeGateOver(w *release.ChangeWatch) *release.ReadGate { return release.NewReadGate(w, release.ChangeIndexBackstop) } // viewUse remembers whether the last viewStore call used the view, so the // switch is logged once per change of state, never per tick. var viewUse struct { mu sync.Mutex last int // 0 none yet, 1 using the view, 2 walking } func noteViewUse(ok bool, why string) { state := 2 if ok { state = 1 } viewUse.mu.Lock() changed := viewUse.last != state viewUse.last = state viewUse.mu.Unlock() if !changed { return } if ok { log.Printf("agent: record view: record walks read the view") return } log.Printf("agent: record view: %s; record walks read the bucket until it can vouch again", why) } // viewStore is the store a record-family walker reads: the armed view's // current snapshot over live, refreshed first, when the refresh returned nil // and the snapshot is Complete. Otherwise ok is false and s is live, so the // caller walks the bucket as before the index (spec §3.4). // // Only record-family walkers take it (Reconcile, ReconcileSidecars, // steering), and the moves pass, which asks it only whether there is any // move work and acts on its own walk of the bucket (movesIdle). Its ListDirs of // apps// names only apps with record-family state, so app enumerators // (hostops.Apps, purge and activation sweeps, mail, economics) must never // get it. // // live must take no agent lock on any read path: RecordView.mu is a leaf // (lock order w.pass -> reconcileMu -> RecordView.mu). func viewStore(ctx context.Context, live release.Store) (s release.Store, epoch uint64, ok bool) { v := armedRecordView.Load() if v == nil || live == nil { return live, 0, false } err := v.Refresh(ctx) sn := v.Snapshot() switch { case err != nil: noteViewUse(false, fmt.Sprintf("refresh failed (%v)", err)) return live, 0, false case sn == nil || !sn.Complete: noteViewUse(false, "the snapshot is incomplete") return live, 0, false } noteViewUse(true, "") return sn.Store(live), sn.Epoch, true } // recordViewBackstopLoop is the backstop (spec §4.4): Rebuild at start and // every release.ChangeIndexBackstop, jittered by +-10%, each followed by the // unrestricted full pass, so every leg is re-run over everything at least // that often. The full pass runs on the fast converge loop's goroutine // (backstopFullDue); on a box where that loop is not armed only the // rebuild runs. func recordViewBackstopLoop(ctx context.Context, v *provision.RecordView) { for { recordViewBackstop(ctx, v, &backstopFullDue) select { case <-ctx.Done(): return case <-time.After(backstopEvery()): } } } // recordViewBackstop is one backstop round: the rebuild, then the request // for a full pass, whether or not the rebuild succeeded (a failed one // leaves the view unable to vouch, so that pass walks). func recordViewBackstop(ctx context.Context, v *provision.RecordView, due *atomic.Bool) { if err := v.Rebuild(ctx); err != nil { log.Printf("agent: record view backstop: %v", err) } due.Store(true) } // backstopEvery is release.ChangeIndexBackstop jittered by +-10%. func backstopEvery() time.Duration { b := release.ChangeIndexBackstop return b - b/10 + rand.N(b/5) } // indexScopes is the scopes of obs whose generation differs from acted's, // sorted, kept to the scopes a pass acts on (apps// and accounts/). // The control/ scopes belong to other readers. func indexScopes(acted, obs map[string]uint64) []string { var moved []string for scope, gen := range obs { if have, ok := acted[scope]; ok && have == gen { continue } if _, _, ok := provision.ScopeApp(scope); ok { moved = append(moved, scope) continue } if a, ok := strings.CutPrefix(scope, "accounts/"); ok && a != "" && !strings.Contains(a, "/") { moved = append(moved, scope) } } sort.Strings(moved) return moved } // movedRoutes keeps the routes whose (account, app) has an app scope in // moved. Account scopes keep nothing here: their effect on a box (an // account soft-deleted) is Reconcile's, which runs whole. func movedRoutes(moved []string) func(registry.Route) bool { apps := map[[2]string]bool{} for _, scope := range moved { if a, b, ok := provision.ScopeApp(scope); ok { apps[[2]string{a, b}] = true } } return func(rt registry.Route) bool { return apps[[2]string{release.RouteAccount(rt), rt.App}] } } // passLegs is the serial-triggered pass set's inputs (fastConvergeLoop), // shared by the full pass and the scoped one so the two run the same legs // in the same order and cannot drift. type passLegs struct { store release.Store reg *registry.Registry pub ed25519.PublicKey optDir string restart func(string) error opts []release.PassOption priv *ecdh.PrivateKey // nil: no config leg (no CARLOS_SECRETS_KEY) dd string svc *activator.Service writes hostwrite.Config bs *buildStates cfg func() provision.ReconcileConfig // read per pass, as provisionTickConfig always was } // passAfterReconcileHook, when set, runs in every pass right after its // reconcile leg. Tests only. var passAfterReconcileHook func() // full is the serial-triggered pass set, in spec order: mint routes // (reconcile), then reconcile aliases, then stage them (adopt), then deliver // their config, over every hosted route. func (l *passLegs) full(ctx context.Context) { l.run(ctx, "serial pass stages", nil) } // scoped is the pass an index-only change runs (spec §4.5, codex 10): the // same legs, with build states, adopt and config sync restricted to the // hosted routes whose app scope moved, and without the box-wide alias and // plans legs, so changing one app re-reads that app only. Reconcile and the moves pass // run whole, over the record view. Telemetry resolves every hosted app (its // sink takes a whole map), but its reads are gated on their own scopes // (telemetryReads), so only the moved apps' files are read. func (l *passLegs) scoped(ctx context.Context, moved []string) { l.run(ctx, "scoped pass stages", movedRoutes(moved)) } // run is one pass set; keep nil is the unrestricted full pass. Each error is // one log line, never fatal: the hourly floors stand underneath every leg. func (l *passLegs) run(ctx context.Context, what string, keep func(registry.Route) bool) { // Each stage's time, logged together when the set is slow, so the // journal says WHICH stage held a promote back. var took []string t := time.Now() mark := func(name string) { took = append(took, fmt.Sprintf("%s %s", name, time.Since(t).Round(100*time.Millisecond))) t = time.Now() } defer func(start time.Time) { logSlow(what, start, strings.Join(took, ", ")) }(time.Now()) if err := reconcileSerialized(ctx, l.store, l.reg, l.cfg()); err != nil { log.Printf("agent: fast converge reconcile: %v", err) } mark("reconcile") if hook := passAfterReconcileHook; hook != nil { hook() } // The registry's build-state copies, refreshed at every reconcile // (settings spec §4.2), before adopt reads them; refreshAll keeps // to its own budget, and the stage is timed apart from reconcile. // A scoped pass refreshes only the moved apps' hosts. if l.bs != nil { l.bs.refreshKept(ctx, keep) mark("build states") } telemetryPass(ctx, l.store, l.reg) mark("telemetry") // The plans pass is box-wide (the marker, then every served account's // limits record, published as one whole snapshot), so a scoped pass // leaves it out: the full pass, the instances floor and the backstop's // full pass run it. if keep == nil { plansPass(ctx, l.store, l.reg, os.Getenv) mark("plans") } // The moves pass takes its own snapshot inside w.pass (spec §4.3): it // is handed the plain store, never this pass's reconcile snapshot. runMovesPass(ctx, l.store) mark("moves") runRunsPass(ctx) mark("runs") // A serial move may be a forwarded db read: list the queue now. The // scoped pass nudges too: a request written through an indexed store // may move only the index. The nudge is a non-blocking channel send. nudgeDBReads() // Aliases are box-wide (every domain claim, every edges file), so the // scoped pass leaves them out: the serial-triggered full pass, the // instances floor and the backstop's full pass run them. if keep == nil { // Folded once per pass and handed in, never re-read inside the // loop: this leg runs on every serial bump (a seconds cadence), // and the fold is a LoadBoxes plus a List plus one Get per edges // file. On a fleet of under a dozen boxes that is a handful of // Gets per bump, which is accepted here rather than cached — a // cache would need its own staleness rule, and a STALE route set // is the one thing a security guard must not have. Aliases read // the domain claims, not the record families, so they read the // bucket. deploymentRoutes := edgesfile.DeploymentRoutes(ctx, l.store, time.Now()) if err := domains.ReconcileAliases(ctx, l.store, l.reg, time.Now, deploymentRoutes, &aliasClaimCache); err != nil { log.Printf("agent: fast converge alias pass: %v", err) } mark("aliases") } if err := release.PassFilter(ctx, l.reg, l.store, l.pub, l.optDir, l.restart, keep, l.opts...); err != nil { log.Printf("agent: fast converge adopt pass: %v", err) } mark("adopt") if l.priv != nil { if err := syncExecSerializedFiltered(ctx, keep, l.reg, l.store, l.priv, l.dd, l.svc, l.writes); err != nil { log.Printf("agent: fast converge config sync: %v", err) } mark("config") } }