package rollout import ( "github.com/carlosframework/platform/internal/status" ) // Status is the box's rollout report (spec §8.4, §9.1), for the private // Metrics block. It reads the queue, the failure book and the config tracks // and asks the activator only about a host's process (Owns, Current), with no // Manager or scheduler lock held (the lock order on Manager): Manager.mu is // released before any scheduler.mu is taken, the host names are gathered under // each scheduler.mu and the activator is asked after it is released, and the // book is read under a scheduler.mu as the order allows. The hosts' serving // state is what the last agent pass and the last attempts recorded, except for // a host whose process is starting or stopping (the activator owns it and has // no current generation): that one is Transient, with no serving build, because // the cached one says nothing settled (Astra R3 section 5 item 10; plan review // finding 8). func (m *Manager) Status() status.RolloutReport { r := status.RolloutReport{Hosts: map[string]status.HostRollout{}, Apps: map[string]status.AppRollout{}} for _, f := range m.d.Book.List() { r.Failures = append(r.Failures, status.RolloutFailure{Account: f.Account, App: f.App, Build: f.Build, Host: f.Host, Mode: f.Mode, Reason: reasonWord(f.Reason), PromotedAt: f.PromotedAt, At: f.At, HeldDown: f.HeldDown}) } m.mu.Lock() scheds := make([]*scheduler, 0, len(m.apps)) for _, s := range m.apps { scheds = append(scheds, s) } observing := make(map[string]bool, len(m.observing)) for h := range m.observing { // whichever wake's token: the host is settling observing[h] = true } type cfgView struct { pending bool reason string } cfg := make(map[string]cfgView, len(m.cfg)) for h, t := range m.cfg { if t.delivered != "" && t.delivered != t.active { v := cfgView{pending: true} if t.lastErr != "" { // A reason word, never the raw text (which can name paths). v.reason = reasonWord(t.lastErr) } cfg[h] = v } } m.mu.Unlock() asked := m.askActivator(scheds) for _, s := range scheds { app := status.AppRollout{} s.mu.Lock() for host, h := range s.q.hosts { hr := status.HostRollout{Account: s.key.acct, App: s.key.app, Installed: h.wantBuild, Serving: h.serving, Good: h.good, Awake: h.serving != ""} // A host the activator was not asked about joined between the name // read and this loop: nothing says it is settled, so this report // calls it transient and the next one asks. if tr, ok := asked[host]; !ok || tr { hr.Serving, hr.Awake, hr.Transient = "", false, true } if v, ok := cfg[host]; ok { hr.ConfigPending, hr.ConfigError = true, v.reason } switch { case h.running != nil && (h.running.Canary || observing[host]): hr.Attempt, hr.AttemptBuild = status.AttemptSettling, h.running.ID.Build app.Settling++ case h.running != nil: hr.Attempt, hr.AttemptBuild = status.AttemptRunning, h.running.ID.Build app.Running++ case observing[host]: hr.Attempt, hr.AttemptBuild = status.AttemptSettling, h.wantBuild app.Settling++ default: if d, ok := s.q.pending(h); ok { hr.Attempt, hr.AttemptBuild, app.Queued = status.AttemptQueued, d.ID.Build, app.Queued+1 } } // A build held for this instance, with nothing else going on: the // failed words. A held-down instance is down. if hr.Attempt == "" && h.wantBuild != "" && h.serving != h.wantBuild { if rec, ok := m.d.Book.Lookup(s.key.acct, s.key.app, h.wantBuild); ok && m.d.Book.Blocks(s.key.acct, s.key.app, h.wantBuild, host, h.wantBuild == h.good) { hr.Attempt, hr.AttemptBuild, hr.HeldBy, hr.Reason = status.AttemptFailed, rec.Build, rec.Build, reasonWord(rec.Reason) // The generation the console signs into its retry grant // (plan review finding 13), 0 while the book owes a // save (the crash-epoch hard limit, rule 2). hr.HeldEpoch = m.d.Book.PublishedEpoch(s.key.acct, s.key.app, rec.Build, host) hr.HeldBook = m.d.Book.ID() // final review round 1, finding 3 hr.RetryV = status.HostRetryV // grants ride the host-retry log if m.d.Book.HeldDown(s.key.acct, s.key.app, h.wantBuild, host) { hr.Attempt = status.AttemptDown } } } // A hold build ran here without passing (final review wave 2, N2): // the instance is held to it, in the same words. Asleep with a // good build, its wakes are refused (decideHoldRan), so it is down // when a record holds its installed build, the marked build // included (Astra R3 N7), and when its installed build is the good // build, until a signed rollback lifts the marker (ruling 205), // unless the good build is the marked build itself, which its wake // boots. Awake, it serves what it serves. if mark, ok := holdRanFor(m.d.Book, s.key.acct, s.key.app, host); ok && h.serving == "" { switch { case hr.HeldBy != "" && h.good != "": hr.Attempt = status.AttemptDown case hr.Attempt == "": hr.HeldBy = mark.Build hr.HeldEpoch = m.d.Book.PublishedEpoch(s.key.acct, s.key.app, mark.Build, host) hr.HeldBook = m.d.Book.ID() hr.RetryV = status.HostRetryV if rec, ok := m.d.Book.Lookup(s.key.acct, s.key.app, mark.Build); ok { hr.Reason = reasonWord(rec.Reason) } // The marked build installed boots under its own rule, even // when it is the good build (Task 3 fix round 1). if h.good != "" && h.wantBuild == h.good && h.wantBuild != mark.Build { hr.Attempt = status.AttemptDown } } } r.Hosts[host] = hr } s.mu.Unlock() r.Apps[s.key.acct+"/"+s.key.app] = app } return r } // askActivator answers, for each host among the schedulers' at the moment // their names are read, whether its process is starting or stopping: the // activator owns the host and has no current generation for it (a wake or an // observation not yet current, a hibernation under way). A host the activator // does not own is asleep, and one it has a current generation for is awake; // both answer false. The names are read under each scheduler.mu and the // activator is asked after it is released (scheduler.view's rule), so no // Manager or scheduler lock is held across the ask. A host missing from the // answer joined after the names were read, and Status reports it transient. func (m *Manager) askActivator(scheds []*scheduler) map[string]bool { var names []string for _, s := range scheds { s.mu.Lock() for host := range s.q.hosts { names = append(names, host) } s.mu.Unlock() } out := make(map[string]bool, len(names)) for _, host := range names { if g, ok := m.d.Act.Current(host); ok && g.Build != "" { out[host] = false continue } out[host] = m.d.Act.Owns(host) } return out }