package engines import ( "fmt" "sort" ) // The launch gate (spec ยง5.1 "The invariant", "Setting the mark", "Starts // systemd makes on its own"). // // I1: a process of a build whose manifest declares doc_engine starts only // after the edge has fenced its group as internal. Every activator launch // path ends in the gate; systemd's own starts (Restart=always, socket // activation) execute the live link and pass no gate, so they are safe only // because adoption gates before it repoints and nothing else moves the link. // fenceWaitTicks is the gate's bound: 30 s of the edge's reloads, one per // tick. const fenceWaitTicks = 3 // gateRule names the rule that gates a launch path; systemd and socket // starts are not activator launches and have none. var gateRule = map[string]string{ "adopt": "gate-adopt", "swap": "gate-swap", "fallback": "gate-fallback", "recovery": "gate-recovery", "seed": "gate-seed", "cmd": "gate-cmd", "systemd": "", "socket": "", } // gateSeesDeclaring is what the gate reads from the build's manifest: its // declaration, or, when the manifest cannot be read, declaring. func (w *world) gateSeesDeclaring(ver int) bool { bd := w.builds[ver] if !bd.ManifestOK { return on("unreadable-declares") } return bd.Declares } // gate is the launch gate for a build about to start in group g: a build it // sees as declaring writes 1 if 1 is not already written, then always waits // for the edge's fence to cover the group's mark, within the bound. False is // a refused launch (the wake fails). func (w *world) gate(b *box, g string, ver int, path string) bool { if !w.gateSeesDeclaring(ver) { return true } if !w.dedicatedLaunchCheck(b, g, ver, path) { return false } wrote := false if w.groupMark(b, g) != markInternal { w.writeMark(b, g, markInternal) wrote = true } if !wrote && !on("gate-waits-when-written") { w.logf("box%d: gate(%s) %s v%d: 1 already written; not waiting", b.ID, path, g, ver) return true } for i := 0; i < fenceWaitTicks && !w.fenced(b, g); i++ { w.reload(b) } if !w.fenced(b, g) { w.logf("box%d: gate(%s) %s v%d: no fence within the bound; refused", b.ID, path, g, ver) return false } return true } // launch starts build ver in group g by path, through the gate where the // path has one. It returns false when the gate refuses. func (w *world) launch(b *box, g string, ver int, path string) bool { rule, known := gateRule[path] if !known { panic(fmt.Sprintf("launch: unknown path %q", path)) } // the gate holds the group's lock from its check through process // creation; --public holds the same lock for its whole run, so a launch // that finds it held waits (here: is refused this pass) if rule != "" && b.Locked[g] && on("clear-launch-lock") { w.logf("box%d: %s %s v%d: --public holds the group's lock; not this pass", b.ID, path, g, ver) return false } if rule != "" && on(rule) && !w.gate(b, g, ver, path) { return false } // a group keeps at most three processes: the oldest has exited n := 0 for _, p := range b.Procs { if p.Group == g { n++ } } if n >= 3 { for i, p := range b.Procs { if p.Group == g { b.Procs = append(b.Procs[:i], b.Procs[i+1:]...) w.logf("box%d: process %d of %s exited", b.ID, p.ID, g) break } } } w.stage(b, g, ver) // a build runs from /opt/carlos/versions: it is staged w.nextID++ p := &proc{ID: w.nextID, Group: g, Ver: ver, Slice: "inst:" + g} b.Procs = append(b.Procs, p) w.logf("box%d: %s started %s v%d as process %d", b.ID, path, g, ver, p.ID) w.applyTamper(b, p) w.checkStart(b, p) return true } // stopGroup stops every process of group g. func (w *world) stopGroup(b *box, g string) { kept := b.Procs[:0] for _, p := range b.Procs { if p.Group != g { kept = append(kept, p) } } b.Procs = kept } func (w *world) running(b *box, g string) bool { for _, p := range b.Procs { if p.Group == g { return true } } return false } // adopt makes ver the live release of group g: set-before (the gate, reached // before the repoint; on a timeout the link does not move and the mark stays // 1), then the repoint, then the restart, itself a gated launch. between runs // after the repoint and before the restart: whatever systemd does there. func (w *world) adopt(b *box, g string, ver int, between func()) bool { w.logf("box%d: adopt %s v%d (live v%d)", b.ID, g, ver, b.Live[g]) if !w.repoint(b, g, ver) { return false } if between != nil { between() } w.stopGroup(b, g) return w.launch(b, g, ver, "adopt") } // repointAwake is adoption that repoints the live link and does not restart: // an awake hibernating route (release/adopt.go:1174), or a generation // rollout that only schedules the swap. What ran keeps running. func (w *world) repointAwake(b *box, g string, ver int) bool { w.logf("box%d: adopt %s v%d without a restart (live v%d)", b.ID, g, ver, b.Live[g]) if !w.repoint(b, g, ver) { return false } w.logf("box%d: %s keeps its running processes until their next start", b.ID, g) return true } // repoint is adoption up to the repoint: the group's lock (public-holds-lock), // set-before (the gate, reached before the repoint; on a timeout the link // does not move and the mark stays 1), then the repoint. func (w *world) repoint(b *box, g string, ver int) bool { if b.Locked[g] && on("public-holds-lock") { w.logf("box%d: adopt %s v%d: --public holds the group's lock; not this pass", b.ID, g, ver) return false } if on("set-before-repoint") && !w.gate(b, g, ver, "adopt") { w.logf("box%d: adopt %s v%d: not repointed this pass", b.ID, g, ver) return false } b.Live[g] = ver w.stage(b, g, ver) if b.Good[g] == 0 { b.Good[g] = ver } w.logf("box%d: live link of %s -> v%d", b.ID, g, ver) return true } // stage records that build ver of group g is staged on the box. A version is // staged once per app (/opt/carlos/versions holds the app's versions): if // another instance of the app already staged it, it stays recorded there // only, and this group runs it from that copy. func (w *world) stage(b *box, g string, ver int) { for _, v := range w.stagedForApp(b, g) { if v == ver { return } } b.Staged[g] = append(b.Staged[g], ver) } // appGroups is group g and every other instance group on the box whose own // row has the same non-empty account/app, sorted. func (w *world) appGroups(b *box, g string) []string { own := b.Rows[g] if own == nil || own.App == "" { return []string{g} } out := []string{} seen := false for _, h := range w.groups(b) { if r := b.Rows[h]; r != nil && r.App == own.App { out = append(out, h) seen = seen || h == g } } if !seen { out = append(out, g) sort.Strings(out) } return out } // stagedForApp is every version of g's app staged on the box, whichever // instance of the app staged it. func (w *world) stagedForApp(b *box, g string) []int { var out []int for _, h := range w.appGroups(b, g) { out = append(out, b.Staged[h]...) } return out } // wake is the activator waking group g by its wake plan: "recovery" starts // the recovery build, "fallback" first fails to start the live build and // falls back; both choose GoodVersion. The triggering request, if any, is // delivered afterwards (or finds the fence cancelled it). It reports whether // there was a build to launch. func (w *world) wake(b *box, g string, mode string, trigger *req) bool { ver := b.Good[g] if mode == "fallback" { w.logf("box%d: wake %s: live v%d fails to start; falling back to good v%d", b.ID, g, b.Live[g], ver) } else { w.logf("box%d: wake %s: recovery build good v%d", b.ID, g, ver) } if ver != 0 { w.launch(b, g, ver, mode) } if trigger != nil { w.deliver(trigger) } return ver != 0 } // moveLive is anything other than adoption moving group g's live link (a // rollout tool, a hand repair). Only adoption, through the gate, and // --public may move a live link. func (w *world) moveLive(b *box, g string, ver int, who string) bool { if on("live-link-only-adoption") { w.logf("box%d: %s may not move the live link of %s; refused", b.ID, who, g) return false } b.Live[g] = ver w.logf("box%d: %s moved the live link of %s -> v%d", b.ID, who, g, ver) return true } // systemdRestart is Restart=always: the unit's main process exits and // systemd starts /opt/carlos/live/ again, with no activator. False // when the route has no live release. func (w *world) systemdRestart(b *box, g string) bool { if b.Live[g] == 0 { return false } w.stopGroup(b, g) w.launch(b, g, b.Live[g], "systemd") return true } // socketActivate is carlos-app@.socket starting a stopped unit from the live // link, with no activator. False when there is nothing to start. func (w *world) socketActivate(b *box, g string) bool { if b.Live[g] == 0 || w.running(b, g) { return false } w.launch(b, g, b.Live[g], "socket") return true } // ---------- the random walk ---------- func (w *world) pickBox() *box { return w.boxes[w.rng.Intn(len(w.boxes))] } // groups is every instance group of b's registry, sorted: the groups of the // instance rows that are not platform-minted (adoption skips minted rows, // release/adopt.go:1210; their processes are their source's). func (w *world) groups(b *box) []string { seen := map[string]bool{} var out []string for h, r := range b.Rows { if r.Kind != "instance" || r.Platform { continue } if g := w.group(b, h); !seen[g] { seen[g] = true out = append(out, g) } } sort.Strings(out) return out } func (w *world) pickGroup(b *box) string { gs := w.groups(b) if len(gs) == 0 { return "" } return gs[w.rng.Intn(len(gs))] } func (w *world) pickVer() int { return 1 + w.rng.Intn(nBuilds) } // randomPublic sends one public request to a host of b's edge map, by its // own name, an alias or a path map. func (w *world) randomPublic(b *box) *req { hosts := make([]string, 0, len(b.EdgeRows)) for h := range b.EdgeRows { hosts = append(hosts, h) } if len(hosts) == 0 { return nil } sort.Strings(hosts) h := hosts[w.rng.Intn(len(hosts))] paths := []string{"/"} for p := range b.EdgeRows[h].PathTo { paths = append(paths, p) } sort.Strings(paths[1:]) return w.admitPublic(b, h, paths[w.rng.Intn(len(paths))]) } // heldFor is the oldest held request on b whose proxy reaches group g. func (w *world) heldFor(b *box, g string) *req { for _, r := range b.Admitted { if r.Held && w.backendGroup(b, r.Backend) == g { return r } } return nil } func init() { actions = append(actions, action{name: "adopt", weight: 3, run: func(w *world) bool { b := w.pickBox() g := w.pickGroup(b) if g == "" { return false } ver := w.pickVer() var between func() switch w.rng.Intn(3) { case 1: between = func() { w.systemdRestart(b, g) } case 2: between = func() { if r := w.randomPublic(b); r != nil { w.socketActivate(b, w.backendGroup(b, r.Backend)) w.deliver(r) } } } w.adopt(b, g, ver, between) return true }}, action{name: "reload", weight: 4, run: func(w *world) bool { w.reload(w.pickBox()); return true }}, action{name: "reload-fails", weight: 1, run: func(w *world) bool { b := w.pickBox() b.ReloadFails = !b.ReloadFails && !w.calm() w.logf("box%d: reload fails = %v", b.ID, b.ReloadFails) return true }}, // 9, not 6 (Task 6): fence-box-wide's sweep catch needs a request held // through a service row that is deleted before a declaring adoption's // fence; at 6 the 3000-seed sweep caught it 7 times, at 9 12 times action{name: "public-request", weight: 9, run: func(w *world) bool { b := w.pickBox() r := w.randomPublic(b) if r == nil { return false } if len(b.Admitted) <= 8 && w.rng.Intn(2) == 0 { r.Held = true w.logf("box%d: request %d held in the activator's queue", b.ID, r.ID) return true } w.deliver(r) return true }}, action{name: "resume", weight: 3, run: func(w *world) bool { var held []*req for _, b := range w.boxes { for _, r := range b.Admitted { if r.Held { held = append(held, r) } } } if len(held) == 0 { return false } sort.Slice(held, func(i, j int) bool { return held[i].ID < held[j].ID }) w.deliver(held[w.rng.Intn(len(held))]) return true }}, action{name: "wake", weight: 3, run: func(w *world) bool { b := w.pickBox() g := w.pickGroup(b) if g == "" { return false } mode := "recovery" if w.rng.Intn(2) == 0 { mode = "fallback" } w.stopGroup(b, g) // the group had gone to sleep return w.wake(b, g, mode, w.heldFor(b, g)) }}, action{name: "swap", weight: 2, run: func(w *world) bool { b := w.pickBox() g := w.pickGroup(b) if g == "" { return false } if w.launch(b, g, w.pickVer(), "swap") { // the incoming generation serves; the outgoing ones stop newest := b.Procs[len(b.Procs)-1] kept := b.Procs[:0] for _, p := range b.Procs { if p.Group != g || p == newest { kept = append(kept, p) } } b.Procs = kept } return true }}, action{name: "seed", weight: 1, run: func(w *world) bool { b := w.pickBox() g := w.pickGroup(b) if g == "" { return false } w.stopGroup(b, g) w.launch(b, g, w.pickVer(), "seed") return true }}, action{name: "cmd", weight: 1, run: func(w *world) bool { b := w.pickBox() g := w.pickGroup(b) if g == "" { return false } w.launch(b, g, w.pickVer(), "cmd") return true }}, action{name: "systemd-restart", weight: 2, run: func(w *world) bool { b := w.pickBox() return w.systemdRestart(b, w.pickGroup(b)) }}, action{name: "socket-activation", weight: 2, run: func(w *world) bool { b := w.pickBox() g := w.pickGroup(b) w.stopGroup(b, g) // the unit exited idle; a connection arrives ok := w.socketActivate(b, g) if r := w.heldFor(b, g); r != nil { w.deliver(r) } return ok }}, action{name: "repoint-live", weight: 1, run: func(w *world) bool { b := w.pickBox() g := w.pickGroup(b) if g == "" { return false } w.moveLive(b, g, w.pickVer(), "a rollout tool") return true }}, ) }