package rollout import ( "crypto/rand" "encoding/hex" "encoding/json" "errors" "fmt" "maps" "math" "os" "path/filepath" "slices" "strings" "sync" "time" "github.com/carlosframework/platform/internal/activator" "github.com/carlosframework/platform/internal/release" ) // FailureFile is the failure records' home in the agent's data dir (spec // §8.4): on disk, so a record survives an agent restart. const FailureFile = "swap-failures.json" // FailureRecord holds one build off this box for instances that have not // served it (spec §8.4). Its scope is (Account, App, Build) on this box; // Host is the instance whose attempt failed, kept for status. type FailureRecord struct { Account string `json:"account"` App string `json:"app"` Build string `json:"build"` Host string `json:"host"` Cause string `json:"cause"` Mode string `json:"mode"` Reason string `json:"reason"` // PromotedAt is the PromotedAt of the channel pointer the build failed // under (spec rev 14 §8.4). A pointer naming the same build with a newer // one, which only `carlos promote --retry` or a fresh promotion can sign, // clears the record: the refusal records' rule (internal/release/ // refusal.go), so the two kinds of record behave alike. PromotedAt time.Time `json:"promoted_at"` At time.Time `json:"at"` // HeldDown are instances where a hold build executed and failed (spec // §6.4, §8.3): nothing falls back automatically on them, so their wakes // are refused until a newer build or a retry. HeldDown []string `json:"held_down,omitempty"` // Retried are instances `carlos instances retry` cleared: the record no // longer holds them, and their next attempt is a canary. Retried []string `json:"retried,omitempty"` // Epochs are the per-record epochs an agent before the crash-epoch hard // limit (readiness round 2, Task 1) wrote: the book's sequence values at // which the record last changed. They are read only so a reopen's // sequence starts past them (OpenFailureBook), then dropped, and never // written again: what a signed host retry names now is the app's // generation (FailureBook.gens). Epochs map[string]uint64 `json:"epochs,omitempty"` // MarkHoldRan asks Record to set Host's hold-ran marker onto this // record's build in the same save (final review wave 2, N2): an attempt // that failed in hold mode, ran its build, and was not held down. It is // an instruction to Record, never stored in the record. MarkHoldRan bool `json:"-"` // HoldRanAt is the marker's At for MarkHoldRan: when the attempt that // ran the build started (its dispatch). An instruction too, never stored. HoldRanAt time.Time `json:"-"` // HoldRanExec is the marker's Exec: the identity of the execution that // failed (HoldRan.Exec). An instruction, never stored. HoldRanExec string `json:"-"` // Seq is the book's snapshot (FailureBook.Snapshot) taken when the // failing execution started; zero when it is not known. Record compares // it with the retries since (Astra ruling 191, retry ordering): a failure // of an execution that began before a retry never takes the retry back. // An instruction, never stored. Seq uint64 `json:"-"` } // RecordResult is what Record did with a failure. failCohort runs the rest // of the cohort-failure transition (the halt and the supersession) for every // result but PreRetry (plan review finding 3). type RecordResult int const ( // Recorded: the build had no record; this failure is its first. Recorded RecordResult = iota // Merged: a record stood, and this failure changed it (its host's retry // spent, a hold-down added, a newer PromotedAt). Merged // Unchanged: a record stood, and this failure added nothing to it. Unchanged // PreRetry: the failing execution began before a retry that has since // cleared the build for its host or for the app (`carlos instances // retry`, `carlos promote --retry`). It is stale: nothing of the record // changes, and no halt or supersession may follow it. A hold-ran marker // it carries, or one in place of its hold-down, is still set (Record). PreRetry ) func (r RecordResult) String() string { switch r { case Recorded: return "recorded" case Merged: return "merged" case Unchanged: return "unchanged" case PreRetry: return "pre-retry" } return fmt.Sprintf("RecordResult(%d)", int(r)) } // HoldRan is an instance's "hold ran" marker (final review wave 2, N2): a // build that executed on it in hold mode and neither passed nor was held // down there (an attempt superseded or stopped after its exec, a first wake // whose observation was cancelled). It may have migrated the instance's // database, so nothing falls back on that instance automatically (spec // §6.4): while the marker stands the planner never boots the good build // (decideWake, planCheck), and a worker's replacement carries no fallback // onto it (executeWorker, Astra R3 N5). It is a restriction on the // instance, not a failure of the build: it blocks no sibling and halts no // cohort, and it lifts no failure record either (Astra R3 N7). A good-build // write decided after the marker was set (a pass whose execution began after // it, Astra R3 N6), a signed `carlos instances retry` naming the marker's // build (RetryHostAt, plan review finding 5), or a signed rollback pointer // newer than the marker (applyPointers, ruling 205) clears it. type HoldRan struct { Account string `json:"account"` App string `json:"app"` Build string `json:"build"` // Exec is the identity of the execution that set the marker: // "a-" for a scheduled attempt (dispatch.ExecID), // "w-" for a planned wake (Manager.wakeExecID). The // incarnation is the agent process's, so a run after a restart is never // taken for one before it (final review round 1, finding 1); a file an // earlier agent wrote holds "a" or "w", which no identity // handed out now equals. A marker belongs to one execution, not to its build (plan review // finding 4): another run of the same build renews the marker's Seq, so // a good write decided between the two runs clears nothing. Empty when // the caller could not tell, which counts as a new execution every time. Exec string `json:"exec,omitempty"` // Seq is the book's sequence when the marker was set (markLocked), so a // good write clears only markers set before its execution began // (ClearHoldRanBefore, Astra R3 N6). Zero in a file an earlier agent // wrote, which any snapshot is newer than. Seq uint64 `json:"seq,omitempty"` // At is when the execution that set the marker started: an attempt's // dispatch, a planned wake's plan (Task 3 fix round 1). Only a rollback // pointer signed after it lifts the marker (ruling 205, Astra R3): a // pointer from before the hold build started says nothing about // recovering from it, while `carlos rollback` signed during the // execution (the rollback that superseded it) does. It is not the time // the marker was recorded: that comes after the supersession, so the // rollback that caused it would always be older. Zero in a file an // earlier agent wrote, which any signed rollback pointer is newer than. At time.Time `json:"at,omitempty"` } // same reports whether h and o mark the same build of the same app. func (h HoldRan) same(o HoldRan) bool { return h.Account == o.Account && h.App == o.App && h.Build == o.Build } // Intent is an execution that may leave a failure behind, written to the // book before the execution runs anything (Astra R1 #5, R3 §5 item 2): an // adoption attempt, or a planned first wake. Its outcome's own book write // (Record, MarkHoldRan) comes later, and End removes the intent only after // it. A failure held only in memory because its save failed, followed by an // agent restart, therefore still finds the intent on disk, and // OpenFailureBook turns it into the failure record it may have been. type Intent struct { Account string `json:"account"` App string `json:"app"` Build string `json:"build"` // Mode is the mode the execution runs in: the effective one, after the // activator's own overrides (an overlap attempt on a foreign-layout row // runs as a hold, plan review finding 2), so a reconciled hold intent // holds the instance down. Mode string `json:"mode"` // ID is the execution's identity (HoldRan.Exec): "a-" // for an attempt (dispatch.ExecID), "w-" for a // planned wake (Manager.wakeExecID). A reconciled hold intent's marker carries it. ID string `json:"id"` // Start is when the execution started (dispatch.Started, a wake's plan // time): a reconciled hold intent's marker carries it as its At (ruling // 205). Zero when not known; reconciliation then uses the open time. Start time.Time `json:"start"` } // wakeIntent reports whether id is a planned wake's intent: "w" leads a // wake's identity, in this agent's form and an earlier one's alike. func wakeIntent(id string) bool { return strings.HasPrefix(id, "w") } // ErrIntentStands is Begin's refusal while another execution's intent stands // for the host: an attempt's, or a wake's that an attempt would replace. Its // outcome is not yet written, so replacing its intent would lose it across a // failed save and a restart. var ErrIntentStands = errors.New("another execution on this instance has not recorded its outcome yet") // ErrSeqExhausted is every save's answer once the book's sequence has run // out (seqLimit, fix wave 1, item 5): the book is never saved or published // again, so no generation repeats under its identity. Rotating the book's // identity alone does not reset an exhausted sequence: a reopen keeps the // highest saved sequence (highestSaved), so the next allocation exhausts // again. There is no supported reset yet; an exhausted book refuses managed // starts on an armed box, as an unreadable book does. The file must not be // deleted: it holds the failure records and hold-ran markers that keep // instances off builds that failed or ran on them. Reaching the limit takes // about 1.8e19 book events, so this is a guard, not an operating condition. var ErrSeqExhausted = errors.New("rollout: the failure book's sequence is exhausted; it is never saved or published again") // reconciledReason is a reconciled intent's Reason. reasonWord reads it as // a failed health check (status.ReasonHealth): the execution's verdict was // never recorded, which is all the box knows. const reconciledReason = "the agent restarted before this execution recorded its outcome" // failureSchema is the schema the book is written as, whatever the file was, // so an older agent refuses the file rather than read it and break what the // newer one keeps. Schema 2 added the intents (Task 5): an agent before it // accepts only schema 1 and would drop markers and intents. Schema 3 is the // crash-epoch hard limit (readiness round 2, Task 1; FailureBook's doc): an // agent before it accepts schema 1 and 2, and would keep the book's // identity, publish epochs it never saved and drop durable_epochs, so a // grant it issued could later match a generation this rule hands out. Either // way an agent downgrade with the flag on refuses managed starts until the // file is upgraded or removed. This agent reads schema 1, 2 and 3, and gives // a file of schema 1 or 2 a new identity (rule 4). const failureSchema = 3 type failureFile struct { Schema int `json:"schema"` Records []FailureRecord `json:"records"` // HoldRan is the hold-ran markers by host. It is additive to schema 1: an // agent that predates it ignores the key (encoding/json skips unknown // fields), and a file without it has no markers. HoldRan map[string]HoldRan `json:"hold_ran,omitempty"` // Seq is the book's sequence (FailureBook.seq), additive like HoldRan, // so a snapshot after a restart is newer than every marker before it. Seq uint64 `json:"seq,omitempty"` // Intents are the executions in flight by host (Intent), schema 2. Intents map[string]Intent `json:"intents,omitempty"` // Book is the book's identity (FailureBook.id), additive like HoldRan: // 16 random bytes, hex, made when the book is first created, and again // when a file from before the crash-epoch rule is adopted (Durable). Book string `json:"book"` // Gens are the apps' restriction generations (FailureBook.gens), keyed // account + "/" + app. Additive: an older agent ignores them. Gens map[string]uint64 `json:"gens,omitempty"` // Durable says the file was written under the crash-epoch rule (rule 4 // in FailureBook's doc): every generation it ever published was on disk // first. A file without it, or of schema 1 or 2, was written by an // earlier agent, which may have published epochs it never saved, so // OpenFailureBook gives such a book a new identity. Durable bool `json:"durable_epochs,omitempty"` } // highestSaved is the largest sequence value the file holds: its own seq, // every app's generation, every marker's, and every epoch an earlier agent // wrote into a record. A reopen's sequence starts there (rule 3). In a file // this rule wrote it is the file's own seq; the max keeps a hand-edited or // older file from handing out a value one of its entries already holds. func (f failureFile) highestSaved() uint64 { n := f.Seq for _, g := range f.Gens { n = max(n, g) } for _, h := range f.HoldRan { n = max(n, h.Seq) } for _, r := range f.Records { for _, e := range r.Epochs { n = max(n, e) } } return n } // randRead is crypto/rand's Read, the one source of a book's identity // (rule 5). A variable so a test can make it fail. var randRead = rand.Read // newBookID is a book's identity (FailureBook.id): 16 bytes from // crypto/rand, hex. It never falls back to anything weaker: an identity // two books could share would let a grant for one clear the other, so a // read that fails is an error for the caller (OpenFailureBook) to fail on. // (Since Go 1.24 crypto/rand.Read crashes the process rather than return an // error, so in production this path is a guard; the seam reaches it.) func newBookID() (string, error) { var b [16]byte if _, err := randRead(b[:]); err != nil { return "", fmt.Errorf("make the failure book's identity: %w", err) } return hex.EncodeToString(b[:]), nil } // genKey is an app's key in FailureBook.gens and failureFile.Gens. func genKey(acct, app string) string { return acct + "/" + app } // FailureBook is the box's failure records. An empty path keeps them in // memory only, which is what tests and the simulation use. // // Records handed out (Lookup, ForApp, List) are copies, inner slices // included, because a status reader holds them outside the lock while // RetryHost and Record edit the book. // // # The crash-epoch hard limit // // A signed `carlos instances retry` grant (release.HostRetryGrant) names a // book, an app and a generation, which the box's status published, and a // stale grant must never clear a later failure (Paul's ruling 2026-10-12, // readiness round 2 Task 1). The book keeps these rules: // // 1. One generation per app. gens[acct/app] takes bumpLocked's next value // (touchLocked) on every change to any restriction of that app: a // record created, a merge that changes a record, a host retried, a // record removed (ClearOnPointer, Prune), a marker of the app set, // renewed or replaced (markLocked; a replaced marker of another app // moves that app's too) or removed (ClearHoldRan, ClearHoldRanBefore, // ClearHoldRanOnPointer, RetryHostAt), an intent reconciled into a // record, and an epoch given at open to an older marker or an app // without a generation (epochOlderLocked). A call that changes nothing // (PreRetry, Unchanged, a repeated marker of the same execution) moves // nothing. The generation of build's restriction of host is the app's // when build's record blocks host or host's marker names build, else 0 // (restrictionEpochLocked). Prune drops an app's generation only when // the app has no record and no marker left; a later restriction of it // takes a fresh value, larger than every earlier one. // 2. Durable before use. dirty is true whenever memory differs from the // last save that landed: every call that changes recs, held, intents, // gens or seq saves under the same hold of mu (a failed save sets // dirty), or the open leaves dirty set. PublishedEpoch answers 0 while // dirty, and RetryHostAt first retries an owed save and clears nothing // while the book is still dirty. // 3. Monotonic across restarts. Every landed save writes the in-memory // seq, which only grows, and a reopen starts from the file's seq, maxed // with every saved generation, marker and epoch (highestSaved). So the // disk's seq only grows, every published generation is at most it, and // every value a later run hands out is larger. // 4. Adoption rotates the identity. A file of schema 1 or 2, or without // durable_epochs, written by an earlier agent that may have published // epochs it never saved, opens with a new identity and owes a save // (dirty), so nothing is published until the rotation is on disk, and // every grant issued before names the old identity. The book is written // as schema 3 (failureSchema), which an earlier agent refuses to open. // 5. The identity comes from crypto/rand only (newBookID). An open that // needs one and cannot get one fails. Only a memory-only book, which // tests and the simulation use, falls back to the clock. // 6. An owed save is retried on its own: the armed manager's pass and a // declined box's periodic pass call SaveIfDirty, and a declined box // saves a dirty book at startup. // // Proof sketch: a grant G names book X, app A and generation E, which // status published, so E was on X's disk (2, 4). RetryHostAt clears only // when X is this book, memory equals disk, and A's generation now equals E // with build still restricting host. Within a run every change takes a // fresh larger value (1), so equality means no change of A's restrictions // since publication. The values never wrap: past seqLimit the book is // exhausted and never saved or published again (ErrSeqExhausted). Across a crash the reopened book starts at the disk's // seq, which is at least E, and hands out larger values (3). A book written // before the rule has another identity (4), and another book has another // random identity (5). So G clears only the restriction state it was // issued for. // // The guarantee rests on two assumptions about the file, which the book // cannot check from inside: // // - Only an agent of this rule writes it. An earlier agent would keep the // identity while publishing epochs it never saved, and write the file // back without durable_epochs. Schema 3 enforces this against a // downgrade (failureSchema): an earlier agent refuses the file. A hand // edit or another tool writing it is outside the guarantee. // - It is never restored from an older copy. A snapshot, a backup or a // cloned data dir carries the identity with a lower seq than the disk // last held, so a reopen from it can hand out again a generation // already published, and a stale grant naming that value would match // (rule 3 fails); a clone also gives two books one identity (rule 5 // fails). Restoring or cloning a data dir must give the copy a new // identity: drop the file's "book" key, and the open makes one (rule 4). type FailureBook struct { write func([]byte) error path string mu sync.Mutex recs []FailureRecord // held is the hold-ran markers by host (HoldRan), saved with the records. held map[string]HoldRan // intents are the executions in flight by host (Intent), saved with the // records. reconciled is how many intents OpenFailureBook turned into // records. intents map[string]Intent reconciled int // dirty is set whenever memory differs from the last save that landed // (rule 2): a save failed, or the open changed something (a rotated // identity, reconciled intents, epochs given to older entries). Memory is // the truth meanwhile, and every later mutating call, Prune and // SaveIfDirty retry the save until it lands; nothing is published or // retried until then. What keeps a failure across an agent restart before // that is the execution's intent (Begin), saved before the execution ran // anything and removed only after its outcome was written (End): a // restart finds it on disk and records the failure from it (spec §8.4, // Astra R1 #5). dirty bool // now stamps a marker whose caller gave no At (markLocked). Nil means // time.Now; tests and the simulation set their own clock. now func() time.Time // seq orders the book's events that later outcomes are compared with // (Astra R3 N6, ruling 191): every marker set, every retry clear and // every app generation (gens) takes the next value, and an execution // takes a Snapshot (seq+1) when it starts. An outcome is older than an // event exactly when its snapshot is not greater than the event's value. // It is saved with the markers and the generations, so snapshots and // generations after a restart are newer than every one on disk (rule 3). // It never reaches seqLimit (bumpLocked). The cleared map below is not // saved; a host retry's clear moves the // app's generation, which is, so a spent grant stays spent across a // restart. seq uint64 // exhausted is set once a change needed a sequence value past the last // usable one (bumpLocked): from then on no save lands and nothing is // published (saveLocked), so memory never publishes a repeated value. exhausted bool // gens are the apps' restriction generations by genKey (rule 1), saved // with the records. A signed host retry names the one it was issued for. gens map[string]uint64 // cleared holds, in memory only, the sequence of the latest retry clear // per (account, app, build, host): `carlos instances retry` per host // (RetryHost), `carlos promote --retry` app-wide with host "" (a // ClearOnPointer that cleared a record). Executions in flight do not // survive an agent restart, so neither need these. Prune drops the // entries of builds no route installs, with their records. cleared map[[4]string]uint64 // id is the book's identity, saved with it (failureFile.Book). Status // publishes it beside every epoch (HostRollout.HeldBook), the console // signs it into its retry grant, and RetryHostAt clears only for a grant // naming it (final review round 1, finding 3, Astra): epochs are a // counter every book starts alike, so two books (a host relocated, a box // rebuilt) can give one value to two restrictions, and without the // identity a grant for one would clear the other. OpenFailureBook sets // it (rules 4 and 5); a memory-only book makes one on first use // (idLocked). id string } // ID is the book's identity (FailureBook.id). func (b *FailureBook) ID() string { b.mu.Lock() defer b.mu.Unlock() return b.idLocked() } // idLocked is the book's identity. Only a memory-only book (tests, the // simulation) reaches it without one, since OpenFailureBook always sets // it; such a book falls back to the clock if crypto/rand fails (rule 5). func (b *FailureBook) idLocked() string { if b.id == "" { id, err := newBookID() if err != nil { id = fmt.Sprintf("m%x", time.Now().UnixNano()) } b.id = id } return b.id } // touchLocked moves (acct, app)'s restriction generation to the book's next // sequence value (rule 1). The caller holds b.mu and saves. func (b *FailureBook) touchLocked(acct, app string) { if b.gens == nil { b.gens = map[string]uint64{} } b.gens[genKey(acct, app)] = b.bumpLocked() } func (b *FailureBook) clock() time.Time { if b.now != nil { return b.now() } return time.Now() } // cloneRec copies r's host lists so the copy shares nothing with r. A // record in memory has no Epochs (OpenFailureBook drops them). func cloneRec(r FailureRecord) FailureRecord { r.HeldDown = slices.Clone(r.HeldDown) r.Retried = slices.Clone(r.Retried) return r } // OpenFailureBook loads the book from dataDir. A file that exists but cannot // be read is an error, never an empty book: an empty book would move every // held instance straight back onto the build that failed. Intents left in // the file become failure records (reconcileLocked, Reconciled), which the // caller saves with Flush. // // The sequence starts at the largest value the file holds (rule 3 in // FailureBook's doc), so nothing handed out from here repeats a generation // status published: that was on disk. A book with no file gets a new // identity, and so does a file of schema 1 or 2 or without durable_epochs // (rule 4): an earlier agent wrote it and may have published epochs it // never saved. Either owes a save (dirty), as does an open that reconciled // intents or gave older entries epochs; nothing is published until it // lands. An identity crypto/rand cannot make is an error (rule 5): the // caller treats the book as unreadable. A book this rule wrote opens with // no save owed unless the open changed something. func OpenFailureBook(dataDir string) (*FailureBook, error) { b := &FailureBook{path: filepath.Join(dataDir, FailureFile)} raw, err := os.ReadFile(b.path) if errors.Is(err, os.ErrNotExist) { id, err := newBookID() if err != nil { return nil, fmt.Errorf("%s: %w", b.path, err) } b.id, b.dirty = id, true return b, nil } if err != nil { return nil, err } var f failureFile if err := json.Unmarshal(raw, &f); err != nil { return nil, fmt.Errorf("%s: %w", b.path, err) } // Schema 1 is an agent before Task 5 (no intents), 2 an agent before the // crash-epoch rule; 3 is written now. if f.Schema != 1 && f.Schema != 2 && f.Schema != failureSchema { return nil, fmt.Errorf("%s: unknown schema %d", b.path, f.Schema) } b.recs = f.Records b.held = f.HoldRan b.intents = f.Intents b.gens = f.Gens b.id = f.Book b.seq = f.highestSaved() // the reopen's sequence (rule 3); rolloutsim:M15 for i := range b.recs { // An earlier agent's per-record epochs: read for the sequence above, // never written again, and never what a grant is checked against. if b.recs[i].Epochs != nil { b.recs[i].Epochs = nil b.dirty = true } } if f.Schema < failureSchema || !f.Durable || b.id == "" { // Adoption of the rule (rule 4): grants issued before name the old // identity, or none. Only an agent of this rule writes schema 3, so a // file of an older schema is adopted even if it says durable_epochs. id, err := newBookID() if err != nil { return nil, fmt.Errorf("%s: %w", b.path, err) } b.id, b.dirty = id, true } b.mu.Lock() b.epochOlderLocked() b.reconcileLocked() b.mu.Unlock() return b, nil } // epochOlderLocked gives what an earlier agent wrote without a sequence // value one now: each marker with no Seq (by host, sorted), then each app // that has a record or a marker and no generation (sorted by key), so two // opens of the same file hand out the same values. A zero generation is // never accepted by a signed retry, so without one such a hold could not be // retried from the console. A marker given a Seq moves its app's generation // (rule 1). The book then owes a save (dirty), which the caller or the next // mutating call makes; nothing is written here. The caller holds b.mu. func (b *FailureBook) epochOlderLocked() { hosts := make([]string, 0, len(b.held)) for host, h := range b.held { if h.Seq == 0 { hosts = append(hosts, host) } } slices.Sort(hosts) apps := map[string][2]string{} for _, host := range hosts { h := b.held[host] h.Seq = b.bumpLocked() b.held[host] = h apps[genKey(h.Account, h.App)] = [2]string{h.Account, h.App} } for _, r := range b.recs { if b.gens[genKey(r.Account, r.App)] == 0 { apps[genKey(r.Account, r.App)] = [2]string{r.Account, r.App} } } for _, h := range b.held { if b.gens[genKey(h.Account, h.App)] == 0 { apps[genKey(h.Account, h.App)] = [2]string{h.Account, h.App} } } for _, k := range slices.Sorted(maps.Keys(apps)) { b.touchLocked(apps[k][0], apps[k][1]) b.dirty = true } } // reconcileLocked turns each intent read from the file into the failure // record its execution may have left (rule 3; spec §8.4, Astra R1 #5): its // build on its host, cause release, the intent's mode, stamped with the open // time, so only a pointer signed after the restart clears it. A hold intent // holds the instance down and sets its hold-ran marker too, with the // intent's identity and start (Task 4), as a hold that went down would have: // the build may have migrated the database, so nothing falls back there // automatically. An agent crash mid-execution therefore halts that build's // cohort until a retry, even when the execution would have passed (the // plan's ruling). The intents then go from memory and the book is owed a // save, which the caller makes (Flush); nothing is written here, so a // read-only open of a book never changes the file. The caller holds b.mu. func (b *FailureBook) reconcileLocked() { if len(b.intents) == 0 { b.intents = nil return } now := b.clock() hosts := make([]string, 0, len(b.intents)) for host := range b.intents { hosts = append(hosts, host) } slices.Sort(hosts) // one order for every open of the same file for _, host := range hosts { in := b.intents[host] r := FailureRecord{Account: in.Account, App: in.App, Build: in.Build, Host: host, Cause: string(activator.CauseRelease), Mode: in.Mode, Reason: reconciledReason, PromotedAt: now, At: now} if in.Mode == string(activator.ModeHold) { // A zero start leaves the marker's At to markLocked, which stamps // it with the book's clock: the open time. r.HeldDown, r.MarkHoldRan, r.HoldRanExec, r.HoldRanAt = []string{host}, true, in.ID, in.Start } b.recordLocked(r) } b.reconciled = len(hosts) b.intents = nil b.dirty = true } // Reconciled is how many intents OpenFailureBook turned into failure // records: executions an earlier agent began and never saw recorded. The // caller logs them and flushes the book (Flush). func (b *FailureBook) Reconciled() int { b.mu.Lock() defer b.mu.Unlock() return b.reconciled } // Flush saves the book now: after reconciliation, so the records it made // and the intents it removed reach the disk before anything runs. func (b *FailureBook) Flush() error { b.mu.Lock() defer b.mu.Unlock() return b.saveLocked() } // SaveIfDirty saves the book when a save is owed (dirty: a save failed, or // the open changed something), else does nothing and returns nil (rule 6). // The armed manager's pass and a declined box's periodic pass call it, so a // book whose disk recovers publishes its generations again without waiting // for another change, pointer or grant. func (b *FailureBook) SaveIfDirty() error { b.mu.Lock() defer b.mu.Unlock() return b.retryLocked() } // SaveOwed reports whether the book owes a save (dirty): while it does, // PublishedEpoch answers 0 and RetryHostAt clears nothing. func (b *FailureBook) SaveOwed() bool { b.mu.Lock() defer b.mu.Unlock() return b.dirty } // Begin writes host's intent and saves it before the execution runs // anything (rule 1). On a save error nothing of it stays in memory, the host's // earlier intent (if this one replaced it) is put back, and the error is // returned: the caller must not execute, since its failure could then be // lost across a failed save and a restart. // // One intent stands per host. A planned wake's intent replaces an older // wake's intent for the host: the activator delivers a planned wake's // outcome before a newer wake of the host plans (its outcome fence, // internal/activator awaitOutcomes, final review F3 and wave 2 N1), so the // older wake's intent is one whose outcome is written, or one whose build // never ran (GenCancelled, not executed), which an event on its way ends // anyway. Two plans for one host never race: the activator plans a host // only from its one waking entry. Any other standing intent refuses the // Begin (ErrIntentStands): an attempt's, whose outcome done() has yet to // write, or, for an attempt, a wake's, whose outcome may still be on its way // (a fallen-back wake's entry serves before its GenFellBack is delivered). // The refused execution is retried as any refused one is. func (b *FailureBook) Begin(host string, in Intent) error { if in.ID == "" { return errors.New("an intent needs its execution's identity") } b.mu.Lock() defer b.mu.Unlock() prev, had := b.intents[host] if had && prev.ID != in.ID && !(wakeIntent(prev.ID) && wakeIntent(in.ID)) { return fmt.Errorf("%s: %w (%s)", host, ErrIntentStands, prev.ID) } if b.intents == nil { b.intents = map[string]Intent{} } b.intents[host] = in if err := b.saveLocked(); err != nil { if had { b.intents[host] = prev } else { delete(b.intents, host) } return err } return nil } // End removes host's intent when it is id's, and saves (rule 2). It runs // after the execution's outcome is written to the book (Record, // MarkHoldRan), never before. Its save failing leaves the intent on disk, // which is the safe direction: a restart then records the failure the // outcome may also have written. An intent of another execution (a newer // wake that replaced this one's) is left alone. func (b *FailureBook) End(host, id string) error { b.mu.Lock() defer b.mu.Unlock() if cur, ok := b.intents[host]; !ok || id == "" || cur.ID != id { return b.retryLocked() } delete(b.intents, host) return b.saveLocked() } // saveLocked writes the book and records whether the file is now behind // memory, so a failure is retried by retryLocked rather than forgotten. It // writes the in-memory seq, which only grows (rule 3), the generations, and // durable_epochs, since everything this agent publishes is on disk first // (rule 2). func (b *FailureBook) saveLocked() error { if b.exhausted { b.dirty = true return ErrSeqExhausted } if b.path == "" && b.write == nil { return nil } raw, err := json.MarshalIndent(failureFile{Schema: failureSchema, Records: b.recs, HoldRan: b.held, Seq: b.seq, Intents: b.intents, Book: b.idLocked(), Gens: b.gens, Durable: true}, "", " ") if err == nil { if b.write != nil { err = b.write(raw) } else { err = release.WriteFileAtomic(b.path, raw, 0o644) } } b.dirty = err != nil return err } // retryLocked is the save a call that changed nothing still owes when an // earlier save failed. func (b *FailureBook) retryLocked() error { if !b.dirty { return nil } return b.saveLocked() } func (b *FailureBook) find(acct, app, build string) int { return slices.IndexFunc(b.recs, func(r FailureRecord) bool { return r.Account == acct && r.App == app && r.Build == build }) } // Snapshot is the book's sequence plus one, taken when an execution starts // (an attempt in scheduler.execute, a planned wake in Plan): every marker // and retry clear already in the book is older than it, and every one set // later is not (Astra R3 N6, ruling 191). Zero is never a snapshot: it // means "not known", which clears no marker and makes no outcome stale. func (b *FailureBook) Snapshot() uint64 { b.mu.Lock() defer b.mu.Unlock() if b.exhausted || b.seq >= seqLimit-1 { // At the boundary a snapshot would be seqLimit or more: "not known" // instead, the safe answer (fix wave 1, item 5). return 0 } return b.seq + 1 } // seqLimit bounds the book's sequence (fix wave 1, item 5): every value it // hands out, and every snapshot, is below it, so the sequence never wraps // and no generation repeats under one identity. The last value handed out // is seqLimit-1 (math.MaxUint64-2). const seqLimit = math.MaxUint64 - 1 // bumpLocked takes the book's next sequence value. At the boundary it hands // out nothing new: the book is exhausted and owes a save that never lands // (saveLocked), so nothing is published or cleared again, and the value it // returns (seqLimit, never a real one) is never seen outside memory. The // caller holds b.mu. func (b *FailureBook) bumpLocked() uint64 { if b.exhausted || b.seq >= seqLimit-1 { b.exhausted, b.dirty = true, true return seqLimit } b.seq++ return b.seq } // clearKey is a retry clear's key in FailureBook.cleared; host "" is the // app-wide clear of `carlos promote --retry`. func clearKey(acct, app, build, host string) [4]string { return [4]string{acct, app, build, host} } // preRetryLocked reports whether r's execution began before a retry that // cleared its build since, for its host or app-wide (ruling 191): its // snapshot is not newer than the clear. The caller holds b.mu. func (b *FailureBook) preRetryLocked(r FailureRecord) bool { if r.Seq == 0 { // rolloutsim:M9 return false } return r.Seq <= b.cleared[clearKey(r.Account, r.App, r.Build, r.Host)] || r.Seq <= b.cleared[clearKey(r.Account, r.App, r.Build, "")] } // clearedLocked notes a retry clear of build for host ("" app-wide) at the // book's next sequence value. The caller holds b.mu. func (b *FailureBook) clearedLocked(acct, app, build, host string) { if b.cleared == nil { b.cleared = map[[4]string]uint64{} } b.cleared[clearKey(acct, app, build, host)] = b.bumpLocked() } // Record writes r unless a record for its build already stands, in which case // the first failure stands and only r's host is merged in: it leaves Retried // (its retry was spent), joins HeldDown if r holds it down, and the record // keeps the newer of the two PromotedAt stamps, so only a pointer signed // after the latest failure clears it. The in-memory record is kept even when // the file write fails, so the caller's transition still blocks the build; // the error is for the journal, and the next call on the book (this one // again, if nothing new, or Prune) retries the save. Until a save lands, what // keeps the failure across an agent restart is the execution's intent // (Begin), still on disk because its End comes after this call: the restart // records the failure from it. r.MarkHoldRan sets r's host's hold-ran marker // in the same save. // // A failure of an execution that began before a retry of its build (r.Seq, // ruling 191) is stale: the record is neither created nor changed, the // host's retry is not spent and no hold-down is added, and Record reports // PreRetry so the caller runs none of the transition (plan review finding // 3). What the execution did to the instance's database still stands: a // requested hold-ran marker is set, and so is one in place of the hold-down // r asked for (a hold build that ran there and went down, Task 4 fix round // 1), with the execution's identity and start (HoldRanExec, HoldRanAt), so // nothing falls back onto the instance automatically (spec §6.4). A stale // hold run that went down is restricted as one that was superseded is. func (b *FailureBook) Record(r FailureRecord) (RecordResult, error) { b.mu.Lock() defer b.mu.Unlock() res, changed := b.recordLocked(r) if changed { return res, b.saveLocked() } return res, b.retryLocked() } // recordLocked is Record's change to memory, and reports whether it changed // anything to save. Reconciliation (reconcileLocked) shares it, so a // reconciled intent merges into a standing record as a live failure does. // The caller holds b.mu and saves. func (b *FailureBook) recordLocked(r FailureRecord) (RecordResult, bool) { stale := b.preRetryLocked(r) mark := r.MarkHoldRan || (stale && slices.Contains(r.HeldDown, r.Host)) marked := mark && b.markLocked(r.Host, HoldRan{Account: r.Account, App: r.App, Build: r.Build, Exec: r.HoldRanExec, At: r.HoldRanAt}) r.MarkHoldRan, r.HoldRanAt, r.HoldRanExec, r.Seq = false, time.Time{}, "", 0 if stale { return PreRetry, marked } i := b.find(r.Account, r.App, r.Build) if i < 0 { r.Retried, r.Epochs = nil, nil // A new record is a new restriction of every host of the app (rule 1). b.recs = append(b.recs, cloneRec(r)) b.touchLocked(r.Account, r.App) // rolloutsim:M17 return Recorded, true } cur := &b.recs[i] changed := false if j := slices.Index(cur.Retried, r.Host); j >= 0 { cur.Retried = slices.Delete(slices.Clone(cur.Retried), j, j+1) changed = true } if r.PromotedAt.After(cur.PromotedAt) { cur.PromotedAt = r.PromotedAt changed = true } for _, h := range r.HeldDown { if !slices.Contains(cur.HeldDown, h) { cur.HeldDown = append(slices.Clone(cur.HeldDown), h) changed = true } } if changed { // A merge with a change moves the app's generation, so a grant // signed before it clears nothing (plan review finding 13, rule 1). b.touchLocked(r.Account, r.App) return Merged, true } return Unchanged, marked } // markLocked sets host's hold-ran marker to h and reports whether that // changed anything. h.At is the execution's start, from the caller. A zero // one (a caller that could not know it) is stamped with the book's clock // now: later than any real start, so only a pointer signed later still // lifts the marker, the safe direction. // // A marker is an execution's (plan review finding 4). A repeated report of // the execution that set the standing marker (same build, same non-empty // Exec) changes nothing and writes nothing. Any other execution gets the // book's next sequence value, so a good write whose execution began before // it clears nothing (ClearHoldRanBefore). It keeps the later of the two // stamps, whatever the builds (Task 4 fix round 1): a late report of an // older execution never moves At back, so a rollback signed between the two // runs never looks newer than the marker. // // Every marker set, renewed or replaced moves its app's generation (rule // 1); a marker of another app it replaces (a host now of another app) moves // that app's too. The caller holds b.mu and saves. func (b *FailureBook) markLocked(host string, h HoldRan) bool { cur, ok := b.held[host] if ok && cur.same(h) && h.Exec != "" && cur.Exec == h.Exec { return false } if b.held == nil { b.held = map[string]HoldRan{} } if h.At.IsZero() { h.At = b.clock() } if ok && cur.At.After(h.At) { h.At = cur.At } h.Seq = b.bumpLocked() // rolloutsim:M12 if ok && (cur.Account != h.Account || cur.App != h.App) { b.touchLocked(cur.Account, cur.App) // the replaced marker's app lost a restriction } // The marker set, renewed or replaced: its app's generation moves. b.touchLocked(h.Account, h.App) // rolloutsim:M14 b.held[host] = h return true } // MarkHoldRan sets host's hold-ran marker (HoldRan) and saves it: the // outcome that set it changed nothing else in the book. A marker already // standing for another build is replaced: the newest hold build to run on // the instance is the one its database may now need. h.At is the // execution's start (markLocked). func (b *FailureBook) MarkHoldRan(host string, h HoldRan) error { b.mu.Lock() defer b.mu.Unlock() if !b.markLocked(host, h) { return b.retryLocked() } return b.saveLocked() } // HoldRanOn returns host's hold-ran marker, if one stands. func (b *FailureBook) HoldRanOn(host string) (HoldRan, bool) { b.mu.Lock() defer b.mu.Unlock() h, ok := b.held[host] return h, ok } // HoldRanCount is how many hold-ran markers stand. A box that declined // CARLOS_SWAP=overlap honours its book when it holds records or markers // (internal/host, honourDeclinedFailures). func (b *FailureBook) HoldRanCount() int { b.mu.Lock() defer b.mu.Unlock() return len(b.held) } // ClearHoldRan drops host's hold-ran marker, whatever it is, and reports // whether a marker stood. A host retry no longer uses it: its grant clears // only a marker naming the grant's build, at the grant's epoch // (RetryHostAt, plan review findings 5 and 13). func (b *FailureBook) ClearHoldRan(host string) (bool, error) { b.mu.Lock() defer b.mu.Unlock() cur, ok := b.held[host] if !ok { return false, b.retryLocked() } delete(b.held, host) b.touchLocked(cur.Account, cur.App) return true, b.saveLocked() } // ClearHoldRanBefore drops host's hold-ran marker for a good-build write // whose execution took snapshot snap (Astra R3 N6): only a marker set before // that execution began, which the instance has since passed over. A marker // set later is a hold build that ran after the write was decided (the write // was delayed, say, between its acknowledgement check and here), and it // stands. A zero snapshot (Bootstrap, a wake whose start is not known) // clears nothing. The check and the clear are one hold of the lock. func (b *FailureBook) ClearHoldRanBefore(host string, snap uint64) (bool, error) { b.mu.Lock() defer b.mu.Unlock() cur, ok := b.held[host] if !ok || snap == 0 || cur.Seq >= snap { // rolloutsim:M8 return false, b.retryLocked() } delete(b.held, host) b.touchLocked(cur.Account, cur.App) return true, b.saveLocked() } // ClearHoldRanOnPointer is an explicit rollback as this box sees it (ruling // 205, Astra R3): host's route installs its good build, and the signed // pointer of its channel names that build with promotedAt. The caller has // checked both. It clears host's marker only when the marker is (acct, // app)'s and older than the pointer: `carlos rollback` signs a fresh // pointer, while root's automatic sibling repoint signs none, so the // pointer the good build was first promoted under, from before the hold // build ran, lifts nothing. The check and the clear are one hold of the // lock, so a marker set by a newer run between the caller's reads and this // call is never the one cleared. func (b *FailureBook) ClearHoldRanOnPointer(host, acct, app string, promotedAt time.Time) (bool, error) { b.mu.Lock() defer b.mu.Unlock() cur, ok := b.held[host] if !ok || cur.Account != acct || cur.App != app || !promotedAt.After(cur.At) { return false, b.retryLocked() } delete(b.held, host) b.touchLocked(acct, app) return true, b.saveLocked() } // Lookup returns the record for (acct, app, build), if one stands. func (b *FailureBook) Lookup(acct, app, build string) (FailureRecord, bool) { b.mu.Lock() defer b.mu.Unlock() if i := b.find(acct, app, build); i >= 0 { return cloneRec(b.recs[i]), true } return FailureRecord{}, false } // Blocks reports whether host may not be moved onto build (spec §8.4). An // instance that has served the build (served) is never blocked, and neither // is one a host retry cleared. func (b *FailureBook) Blocks(acct, app, build, host string, served bool) bool { if served { return false } b.mu.Lock() defer b.mu.Unlock() i := b.find(acct, app, build) return i >= 0 && !slices.Contains(b.recs[i].Retried, host) } // Retried reports whether build's record stands and lists host as retried: // a `carlos instances retry` of host cleared it and no failure of host has // spent it since. The scheduler keeps a retry's canary entitlement only // while this holds (appQueue.dropRetries, final review round 2, finding 5). func (b *FailureBook) Retried(acct, app, build, host string) bool { b.mu.Lock() defer b.mu.Unlock() i := b.find(acct, app, build) return i >= 0 && slices.Contains(b.recs[i].Retried, host) } // HeldDown reports whether host's wakes are refused after a failed hold // attempt onto build (spec §6.4). func (b *FailureBook) HeldDown(acct, app, build, host string) bool { b.mu.Lock() defer b.mu.Unlock() i := b.find(acct, app, build) return i >= 0 && slices.Contains(b.recs[i].HeldDown, host) && !slices.Contains(b.recs[i].Retried, host) } // RetryHost is `carlos instances retry `: it clears the record for that // host only (spec §8.4). It checks no grant: the box reaches it through // RetryHostAt, once the console's signed grant has verified and its epoch // is still the restriction's. // // A retry that clears the host also notes a host clear at the book's next // sequence value (ruling 191): a failure of an execution of host that began // before it is then stale (Record), and takes nothing back. A retry of a // host already retried notes nothing (Task 4 fix round 1): the canary the // first retry let through may be running, and its failure must spend the // retry, or the host would be left neither held nor allowed through the // halt, with no later retry able to free it. func (b *FailureBook) RetryHost(acct, app, build, host string) (bool, error) { b.mu.Lock() defer b.mu.Unlock() if !b.retryHostLocked(acct, app, build, host) { return false, b.retryLocked() } return true, b.saveLocked() } // retryHostLocked is RetryHost's change to memory, and reports whether it // un-retried the host. It moves the app's generation (rule 1), saved with // the book: a grant that was spent here never matches again, even after a // reopen. The caller holds b.mu and saves. func (b *FailureBook) retryHostLocked(acct, app, build, host string) bool { i := b.find(acct, app, build) if i < 0 || slices.Contains(b.recs[i].Retried, host) { return false } b.clearedLocked(acct, app, build, host) cur := &b.recs[i] cur.Retried = append(slices.Clone(cur.Retried), host) if j := slices.Index(cur.HeldDown, host); j >= 0 { cur.HeldDown = slices.Delete(slices.Clone(cur.HeldDown), j, j+1) } b.touchLocked(acct, app) return true } // RestrictionEpoch is the generation of build's restriction of host as it // stands in memory (restrictionEpochLocked), saved or not. It is for logs // and tests: status publishes PublishedEpoch, never this. func (b *FailureBook) RestrictionEpoch(acct, app, build, host string) uint64 { b.mu.Lock() defer b.mu.Unlock() return b.restrictionEpochLocked(acct, app, build, host) } // PublishedEpoch is the generation of build's restriction of host that // status publishes beside HeldBy (HostRollout.HeldEpoch) and the console // signs into its retry grant: the app's generation when build's record // blocks host or host's marker names build, and the book's memory equals // its last landed save (rule 2). Zero otherwise: nothing of build restricts // host, or the book owes a save, so the value might not survive a crash. func (b *FailureBook) PublishedEpoch(acct, app, build, host string) uint64 { b.mu.Lock() defer b.mu.Unlock() if b.dirty { // durable before use (rule 2); rolloutsim:M13 return 0 } return b.restrictionEpochLocked(acct, app, build, host) } // restrictionEpochLocked is (acct, app)'s generation when build restricts // host, through its record (which blocks every host of the app it does not // list as retried) or host's hold-ran marker naming build; else 0 (rule 1). // The caller holds b.mu. func (b *FailureBook) restrictionEpochLocked(acct, app, build, host string) uint64 { i := b.find(acct, app, build) byRecord := i >= 0 && !slices.Contains(b.recs[i].Retried, host) m, ok := b.held[host] byMarker := ok && m.Account == acct && m.App == app && m.Build == build if !byRecord && !byMarker { return 0 } return b.gens[genKey(acct, app)] } // RetryHostAt is a signed `carlos instances retry ` grant (spec §8.4, // §11; HostRetryGrant) applied to the book: when the grant names this book // (book, the book's ID), the book owes no save, and epoch is build's // restriction of host now (the app's generation, PublishedEpoch), it clears // build's record for host, when that record holds host (RetryHost), and // host's hold-ran marker, when the marker names build (plan review finding // 5). A grant from before any restriction of the app last changed, a zero // epoch, or one for a build that restricts nothing clears nothing. Nor does // a grant for another book, or one that names no book (a console or a box // from before the identity): its epoch could be another book's (final // review round 1, finding 3). An owed save is retried first, and while it // fails nothing is cleared (rule 2): the generation in memory may be one no // disk holds. The check and the clears are one hold of the lock. now is the // generation at the check, for the caller's log; err is the save's. func (b *FailureBook) RetryHostAt(acct, app, build, host, book string, epoch uint64) (rec, mark bool, now uint64, err error) { b.mu.Lock() defer b.mu.Unlock() now = b.restrictionEpochLocked(acct, app, build, host) if err := b.retryLocked(); b.dirty { return false, false, now, err } if book == "" || book != b.idLocked() || epoch == 0 || now != epoch { // rolloutsim:M16 return false, false, now, nil } if m, ok := b.held[host]; ok && m.Account == acct && m.App == app && m.Build == build { delete(b.held, host) mark = true } rec = b.retryHostLocked(acct, app, build, host) if !rec && !mark { return false, false, now, nil } if !rec { // Only the marker was removed (no record held the host): still a // retry, so its place in the sequence is noted as one (final review // round 1, finding 4, Astra). A failure of an execution of build on // host that began before it is then stale (Record, PreRetry): no // record, no halt, though a late hold outcome still sets its marker. // retryHostLocked has noted it, and moved the generation, when it // cleared a record, and a retry that removed nothing notes nothing. b.clearedLocked(acct, app, build, host) b.touchLocked(acct, app) } return rec, mark, now, b.saveLocked() } // ClearOnPointer is `carlos promote --retry` as this box sees it: a signed // channel pointer naming build whose PromotedAt is newer than the record's // clears the record (spec rev 14 §8.4). The pointer's signature is verified // by the caller's read; nobody can forge a fresh one. A clear is noted // app-wide at the book's next sequence value (ruling 191), so a failure of // any instance's execution that began before it is stale (Record) and never // writes the record back. func (b *FailureBook) ClearOnPointer(acct, app, build string, promotedAt time.Time) (bool, error) { b.mu.Lock() defer b.mu.Unlock() i := b.find(acct, app, build) if i < 0 || !promotedAt.After(b.recs[i].PromotedAt) { return false, b.retryLocked() } b.recs = slices.Delete(b.recs, i, i+1) b.clearedLocked(acct, app, build, "") b.touchLocked(acct, app) return true, b.saveLocked() } // ForApp returns the app's records. func (b *FailureBook) ForApp(acct, app string) []FailureRecord { b.mu.Lock() defer b.mu.Unlock() var out []FailureRecord for _, r := range b.recs { if r.Account == acct && r.App == app { out = append(out, cloneRec(r)) } } return out } // Prune drops the records for builds no route on this box still has // installed. "The record stays for any instance still on the failed build" // (spec §8.4), and only for those. installed is called under the book's lock // and must take no lock of its own (nor call back into the book). The // scheduler runs Prune every sweep, so it also retries a save that failed. // The retry clears of such builds go too (they are in memory only): a stale // failure of one arriving after is a new record, which the next Prune drops // again. // // An app that lost a record and still has a record or a marker gets a new // generation; an app with neither left loses its generation, which is the // only way one is dropped (rule 1): nothing of it restricts any host, and a // later restriction takes a fresh value, larger than every earlier one. func (b *FailureBook) Prune(installed func(acct, app, build string) bool) error { b.mu.Lock() defer b.mu.Unlock() for k := range b.cleared { if !installed(k[0], k[1], k[2]) { delete(b.cleared, k) } } lost := map[string][2]string{} // apps that lost a record kept := b.recs[:0] for _, r := range b.recs { if installed(r.Account, r.App, r.Build) { kept = append(kept, r) } else { lost[genKey(r.Account, r.App)] = [2]string{r.Account, r.App} } } b.recs = kept standing := map[string]bool{} // apps with a record or a marker left for _, r := range b.recs { standing[genKey(r.Account, r.App)] = true } for _, h := range b.held { standing[genKey(h.Account, h.App)] = true } changed := false for k := range b.gens { if !standing[k] { delete(b.gens, k) changed = true } } for _, k := range slices.Sorted(maps.Keys(lost)) { if standing[k] { b.touchLocked(lost[k][0], lost[k][1]) } changed = true } if !changed { return b.retryLocked() } return b.saveLocked() } // List returns a copy of every record, for status. func (b *FailureBook) List() []FailureRecord { b.mu.Lock() defer b.mu.Unlock() out := make([]FailureRecord, len(b.recs)) for i, r := range b.recs { out[i] = cloneRec(r) } return out }