package host import ( "context" "errors" "fmt" "log" "time" "github.com/carlosframework/platform/internal/activator" "github.com/carlosframework/platform/internal/registry" ) // The instance-key backfill (L21, plan 2026-10-10-instance-key-provision.md // ruling 7). Every launch path mints CARLOS_INSTANCE_KEY before it reads the // env, so no process is ever started without one. The backfill gives every // instance already on a box its key now, just after agent start, rather // than at its next start, so the rollout can be checked box by box before // any app adopts the key: a host that the backfill names "present" or // "minted" is ready. It runs in the background once the activator has // started (activatorStart.Run), never on the path the edge waits for. // // It runs only where the legacy activator writes .env itself. With // host claims on, every write to a host's installation runs under the // claim's guard, and the claim's launch source already mints the key under // that guard before every cold admission (claim_snapshot.go // EnsureClaimAdminToken), so a claim box is not backfilled here. // instanceKeyBackfiller is the activator surface the backfill needs. The // context bounds the one bucket read a mint makes (ruling 5: nothing is // minted until the row's configuration is delivered). type instanceKeyBackfiller interface { BackfillInstanceKey(context.Context, registry.Route) (activator.InstanceKeyOutcome, error) } // instanceKeyBackfillBudget bounds one pass. Each keyless row costs one // read of the bucket's configuration before its key is minted. The pass // runs in the background after the activator starts (activatorStart.Run), // so it never holds the edge up, but with the bucket unreachable the rows // after the budget still fail fast rather than each waiting out its own // read; every one of them mints at its next start. const instanceKeyBackfillBudget = time.Minute // hostPreparationSlot runs prepare in host's preparation slot, ordered // with the host's maintenance fence: (*activator.Service). // PrepareScheduledToken. It answers activator.ErrFenced without running // prepare when a fence is held for host, and a fence asked for while // prepare runs is installed only after it returns. Wakes do not wait on // the slot. type hostPreparationSlot func(host string, prepare func() (string, error)) (string, error) // errInstanceKeyRowSkipped is a row the backfill listed and, when its turn // came, did not write for: an operation held its host fenced, or its row // had changed (a move, a layout flip, a removal) so it is no longer one // the backfill selects. Counted with the skipped rows; its next start // mints the key if it still needs one. var errInstanceKeyRowSkipped = errors.New("skipped") // keyBackfillInSlot is the backfill's step for one row once the activator // is serving (the v0.26.1 hotfix moved the pass off the startup path). The // layout pass and every move fence a host before they move its files, and // install that fence only after any preparation already in the slot has // returned; the slot refuses a host already fenced. So inside the slot no // file of the host moves, and the row is read again there, so the write // goes to the paths the host has now, not those of the list taken before. // A start's own mint is not ordered by the slot: it and the backfill both // decide and append under the activator's adminTokenMu, create-only, so // whichever is second finds the key present. type keyBackfillInSlot struct { keys instanceKeyBackfiller slot hostPreparationSlot route func(string) (*registry.Route, error) } func (g keyBackfillInSlot) BackfillInstanceKey(ctx context.Context, rt registry.Route) (activator.InstanceKeyOutcome, error) { got, err := activator.InstanceKeyFailed, error(nil) _, serr := g.slot(rt.Host, func() (string, error) { fresh, rerr := g.route(rt.Host) switch { case rerr != nil: err = fmt.Errorf("read its row: %w", rerr) case fresh == nil || !instanceKeyRoute(*fresh): err = fmt.Errorf("%w: its row changed after the backfill listed it", errInstanceKeyRowSkipped) default: got, err = g.keys.BackfillInstanceKey(ctx, *fresh) } return "", nil }) if serr != nil { return activator.InstanceKeyFailed, fmt.Errorf("%w: an operation holds it (%v)", errInstanceKeyRowSkipped, serr) } return got, err } // instanceKeyRoute selects the rows the backfill writes for: ready, // activator-spawned instances, the rows execSyncable delivers config to, // without sidecars and without replicas (a replica is never minted a key, // ruling 26). A held row (moving, landing, restoring, releasing) belongs to // its operation and is skipped, as is a provisioning row whose first start // will mint. func instanceKeyRoute(rt registry.Route) bool { return rt.Kind == "instance" && rt.Host != "" && rt.Socket != "" && !rt.Platform && !rt.Replica && (rt.Status == "" || rt.Status == "ready") && rt.Backing != registry.BackingUnit && rt.Tunnel == "" && rt.Unit == "" } // instanceKeyBackfill counts one pass's outcomes. Deferred is a row whose // configuration sets a key this box has not delivered yet, or whose // configuration could not be read: not minted, by design, and minted or // found configured at its next start. type instanceKeyBackfill struct { Minted, Present, FromConfig, NoEnvFile, Deferred, Failed, Skipped int } // backfillInstanceKeys runs the backfill over routes, within // instanceKeyBackfillBudget, and logs one count line when it ends, // "agent: instance keys: ...": on a legacy box that is after the activator // has started, within the budget plus one row's read. It never fails the // agent: a host it cannot write is logged and counted, and that host's // next start mints its key (or refuses, loudly, if the same fault is // still there). Nothing logged names a key. // // The error is read before the outcome: a refused or failed row answers // InstanceKeyFailed (the zero outcome) beside its error, and counting an // outcome first would call a host ready whose key was never written. func backfillInstanceKeys(ctx context.Context, routes []registry.Route, b instanceKeyBackfiller) instanceKeyBackfill { ctx, cancel := context.WithTimeout(ctx, instanceKeyBackfillBudget) defer cancel() var st instanceKeyBackfill for _, rt := range routes { if ctx.Err() != nil { break } if !instanceKeyRoute(rt) { st.Skipped++ continue } got, err := b.BackfillInstanceKey(ctx, rt) switch { case errors.Is(err, errInstanceKeyRowSkipped): st.Skipped++ log.Printf("agent: instance key for %s: %v (its next start mints it)", rt.Host, err) continue case errors.Is(err, activator.ErrInstanceKeyDeferred): st.Deferred++ log.Printf("agent: instance key for %s: %v", rt.Host, err) continue case err != nil: st.Failed++ log.Printf("agent: instance key for %s: %v (its next start mints it, or refuses for the same reason)", rt.Host, err) continue } switch got { case activator.InstanceKeyMinted: st.Minted++ case activator.InstanceKeyPresent: st.Present++ case activator.InstanceKeyFromConfig: st.FromConfig++ case activator.InstanceKeyNoEnvFile: st.NoEnvFile++ case activator.InstanceKeyReplica: // Unreachable: instanceKeyRoute drops every replica row. Kept // so the counts always sum to the rows listed. st.Skipped++ } } log.Printf("agent: instance keys: %d minted, %d already present, %d set by app configuration, %d never started here, %d waiting for their configuration, %d failed, %d rows skipped", st.Minted, st.Present, st.FromConfig, st.NoEnvFile, st.Deferred, st.Failed, st.Skipped) return st }