// The root pass's D3 work (spec 2026-10-06-console-release-defences-design.md // §4.3, §4.5): look at every named-unit route's live build, once before // release.Pass and again, only where the pass changed it, after. // // - A build meeting observe's conditions (bound, active in one invocation // for the policy's after, hashed from its running executable, ready) // enters the app's healthy history, on every box, policy file or not. // That is what puts the build already live when a requirement is switched // on into the history without a restart. recordLive does it BEFORE // release.Pass (final review H4): on the first pass after a requirement // is switched on, the pass can adopt a proven head and replace the build // that has been running well for hours, and that build must already be in // the history, or a later rollback to it would wait for a dev proof. // - On a proving box (a prove block lists the app) the build the pass left // live is minted a proof (mintAndLog, after release.Pass), unless a refusal record names it or the bucket already // has one. Minting needs the root-only key, which is why it is here. // - A pass that reads a require entry leaves a proof-required/ marker // in the bake directory. On the hourly pass, an app with that marker, // healthy history and no require entry gets one line saying the // requirement is off: the break-glass trail. A box that never required a // proof has no marker and stays quiet. // // A policy file that does not parse is closed (spec §4.5): nothing is // minted, no marker is touched, and the pass says so every time. The live // build is still recorded as healthy at the default after, because that // record is evidence about the running bytes, not a reading of the policy. package host import ( "context" "crypto/ed25519" "errors" "fmt" "os" "path/filepath" "sort" "time" "github.com/carlosframework/platform/internal/activator" "github.com/carlosframework/platform/internal/registry" "github.com/carlosframework/platform/internal/release" "github.com/carlosframework/platform/internal/safefile" ) type proofRoot struct { mintCensus func() (bool, error) recordProgram func(liveBuild) error refusals func(release.HistoryBinding) ([]release.Refusal, error) obs proofObserver reg *registry.Registry store release.Store bakeDir string healthyDir string policyPath string readKey func(path string) (ed25519.PrivateKey, error) hourly bool // carlos-adopt.service, not -stage-only logf func(format string, a ...any) } func (r proofRoot) historyDir() string { if r.healthyDir != "" { return r.healthyDir } return r.bakeDir } // liveObservation is one look at a route's live build: the live link's // target when it was taken, and observe's answer. type liveObservation struct { target string build liveBuild err error } // proofObservations is recordLive's result, handed to mintAndLog so a route // the pass did not touch is observed once per pass, not twice. type proofObservations struct { pol *release.ProofPolicy perr error byHost map[string]liveObservation } // run is both halves back to back, for a caller with no pass between them. func (r proofRoot) run(ctx context.Context) { r.mintAndLog(ctx, r.recordLive(ctx)) } // around is RunAdopt's order: record the live builds, run the pass, then // mint on what the pass left live, including routes unaffected by a pass error. func (r proofRoot) around(ctx context.Context, pass func() error) error { obs := r.recordLive(ctx) err := pass() r.mintAndLog(ctx, obs) return err } // recordLive reads the policy, observes every named-unit route's live build, // and records each one that qualifies as healthy. func (r proofRoot) recordLive(ctx context.Context) proofObservations { pol, perr := release.LoadProofPolicy(r.policyPath) if perr != nil { r.logf("proof: %v; nothing is minted until it parses", perr) } obs := proofObservations{pol: pol, perr: perr, byHost: map[string]liveObservation{}} routes, err := r.reg.List() if err != nil { r.logf("proof: list routes: %v", err) return obs } for _, rt := range routes { if !release.Adoptable(rt) || rt.Unit == "" || rt.Unit == activator.UnitName(rt.Host) { continue } obs.byHost[rt.Host] = r.observeAndRecord(rt, pol.ProveAfter()) } return obs } // observeAndRecord observes rt's live build and, when it qualifies, adds it // to the app's healthy history. func (r proofRoot) observeAndRecord(rt registry.Route, after time.Duration) liveObservation { target, _ := os.Readlink(release.LiveLink(r.obs.optDir, rt.Host)) b, err := r.obs.observe(rt, after) o := liveObservation{target: target, build: b, err: err} if err != nil { return o } if err := release.RecordHealthy(r.historyDir(), b.binding.Route.Account, b.binding.Route.App, release.HealthyBuild{ SHA256: b.SHA256, Version: b.Version, How: "live", At: r.obs.now().UTC()}); err != nil { r.logf("proof: %s: recording the healthy build: %v", rt.Host, err) o.err = err } if r.recordProgram != nil { if err := r.recordProgram(b); err != nil { r.logf("proof: %s: recording the installed program: %v", rt.Host, err) o.err = errors.Join(o.err, err) } } return o } // stillLive reports whether o still describes rt's live build after the // pass: the live link names the same target and, for a build that passed, // the unit is the same process (a rollback can put the link back where it // was and restart the unit; that process has not run for after). func (r proofRoot) stillLive(rt registry.Route, o liveObservation) bool { if r.obs.resolve == nil { return false } binding, err := r.obs.resolve(context.Background(), rt) if err != nil || binding.Source != o.build.binding.Source { return false } rt = binding.Route target, err := os.Readlink(release.LiveLink(r.obs.optDir, rt.Host)) if err != nil || target != o.target { return false } if o.err != nil { return true } p, err := r.obs.sys.Show(rt.Unit) if err != nil || p.ActiveState != "active" || p.InvocationID != o.build.invocation || p.MainPID != o.build.pid { return false } start, err := r.obs.sys.StartTime(p.MainPID) return err == nil && start == o.build.start } // mintAndLog is the second half: mint on a proving box, leave the // requirement marker, and on the hourly pass say when a requirement is off. // A route whose live build the pass changed is observed again (and recorded, // if it qualifies); every other route reuses recordLive's observation. func (r proofRoot) mintAndLog(ctx context.Context, obs proofObservations) { pol, perr := obs.pol, obs.perr mintAllowed := r.refusals == nil && r.recordProgram == nil if perr == nil && pol != nil && pol.Prove != nil { var err error if r.mintCensus != nil { mintAllowed, err = r.mintCensus() } if err != nil { mintAllowed = false r.logf("proof: reading root census: %v; not minting", err) } else if !mintAllowed { r.logf("proof: minting paused until this box has a root census") } } routes, err := r.reg.List() if err != nil { r.logf("proof: list routes: %v", err) return } after := pol.ProveAfter() type acctApp struct{ acct, app string } seen := map[acctApp]bool{} for _, rt := range routes { if !release.Adoptable(rt) || rt.Unit == "" || rt.Unit == activator.UnitName(rt.Host) { continue } acct := release.RouteAccount(rt) seen[acctApp{acct, rt.App}] = true o, ok := obs.byHost[rt.Host] if !ok || !r.stillLive(rt, o) { o = r.observeAndRecord(rt, after) } if o.err != nil { if pol.Proves(rt.App) { r.logf("proof: %s: not minting yet: %v", rt.Host, o.err) } continue } if mintAllowed && perr == nil && pol.Proves(rt.App) { r.mint(ctx, pol, o.build.binding, o.build) } } if perr != nil { return } // The marker says this box has required a proof for the app at some // point. Written on every pass that reads a require entry, so a // requirement switched on and off between two hourly passes still leaves // its trail. if pol != nil { for _, e := range pol.Require { r.markRequired(e.App) } } if !r.hourly { return } keys := make([]acctApp, 0, len(seen)) for k := range seen { keys = append(keys, k) } sort.Slice(keys, func(i, j int) bool { return keys[i].acct+"/"+keys[i].app < keys[j].acct+"/"+keys[j].app }) for _, k := range keys { // Only where the marker exists is "off" news: a box that never // required a proof (Tito, a self-hoster) has nothing switched off. if len(pol.Requires(k.app)) > 0 { continue } if _, err := os.Stat(r.requiredMarker(k.app)); err != nil { continue } if n := len(release.ReadHealthy(r.historyDir(), k.acct, k.app)); n > 0 { r.logf("proof: requirement is off for %s on this box: %s has no require entry for it (%d healthy builds on record)", k.app, r.policyPath, n) } } } func (r proofRoot) requiredMarker(app string) string { return filepath.Join(r.bakeDir, "proof-required", app) } func (r proofRoot) markRequired(app string) { m := r.requiredMarker(app) if _, err := os.Stat(m); err == nil { return } if err := safefile.MkdirAll(filepath.Dir(m), 0o755); err != nil { r.logf("proof: writing the requirement marker %s: %v", m, err) return } if err := safefile.WriteFile(m, nil, 0o644); err != nil { r.logf("proof: writing the requirement marker %s: %v", m, err) } } func (r proofRoot) mint(ctx context.Context, pol *release.ProofPolicy, binding proofBinding, b liveBuild) { rt, acct := binding.Route, binding.Route.Account var refusals []release.Refusal var err error if r.refusals != nil { refusals, err = r.refusals(release.HistoryBinding{Host: rt.Host, Account: acct, App: rt.App}) } else { refusals, err = release.ListRefusals(r.bakeDir) } if err != nil { r.logf("proof: %s: reading refusal records: %v; not minting", rt.Host, err) return } for _, ref := range refusals { if ref.SHA256 == b.SHA256 { r.logf("proof: %s: not minting %s: this box refused it (%s)", rt.Host, b.Version, ref.Reason) return } } // Spec §4.3's last condition, and the reason a re-mint is never tried: // PutProof is create-only, and a second proof for one build would carry // a different proven_at and conflict. prover := pol.Prove.Prover _, err = release.GetProof(ctx, r.store, acct, rt.App, b.SHA256, prover) switch { case err == nil: return // already proven case !errors.Is(err, os.ErrNotExist): r.logf("proof: %s: reading the bucket for an existing proof: %v; not minting", rt.Host, err) return } key, err := r.readKey(pol.Prove.KeyFile) if err != nil { r.logf("proof: %s: %v; not minting", rt.Host, err) return } p, err := release.SignProof(key, release.Proof{App: rt.App, Platform: release.AdoptPlatform(), SHA256: b.SHA256, Version: b.Version, Prover: prover, ActiveSince: b.ActiveSince.UTC().Truncate(time.Second), ProvenAt: r.obs.now().UTC().Truncate(time.Second)}) if err != nil { r.logf("proof: %s: signing: %v", rt.Host, err) return } if err := release.PutProof(ctx, r.store, acct, p); err != nil { // A concurrent pass (the stage relay and the hourly timer can overlap // when the pass lock is unavailable) minted it between the check above // and this write. Its proof stands: already minted, not a failure. if _, gerr := release.GetProof(ctx, r.store, acct, rt.App, b.SHA256, prover); gerr == nil { r.logf("proof: %s: %s %s was already minted by another pass", rt.Host, rt.App, b.Version) return } r.logf("proof: %s: writing the proof: %v (the agent asks again in 2 minutes)", rt.Host, err) return } r.logf("proof: minted %s %s (sha256 %s) as %s, live since %s", rt.App, b.Version, b.SHA256, prover, b.ActiveSince.UTC().Format(time.RFC3339)) } // readProvingKey reads the root-only proving key, refusing one any other user // can read (spec §4.2: the key is root 0600). func readProvingKey(path string) (ed25519.PrivateKey, error) { fi, err := os.Stat(path) if err != nil { return nil, fmt.Errorf("proving key: %w", err) } if perm := fi.Mode().Perm(); perm&0o077 != 0 { return nil, fmt.Errorf("proving key %s is mode %04o; it must be readable by root only (0600)", path, perm) } b, err := safefile.ReadFile(path) if err != nil { return nil, fmt.Errorf("proving key: %w", err) } return release.ParseSigningKey(string(b), path) } func configureRootProof(pr *proofRoot, checks rootRestartChecks, open func() (*release.RootHistoryPass, error)) (*release.RootHistoryPass, error) { legacy, err := checks.allowLegacy() if err == nil && legacy { return nil, nil } pr.mintCensus = func() (bool, error) { legacy, err := checks.allowLegacy() return !legacy && err == nil, err } var history *release.RootHistoryPass pr.refusals = func(b release.HistoryBinding) ([]release.Refusal, error) { if history == nil { return nil, os.ErrPermission } return checks.mintRefusals(history.History, b) } if err != nil { return nil, err } history, err = open() if err == nil && history != nil { pr.recordProgram = func(b liveBuild) error { return recordRootProgram(history, b) } } return history, err }