package engines import ( "fmt" "sort" "strings" ) // The dedicated engine box and ingress (spec §5.9 "The engine box is // dedicated"; §5.1 "The edge is the only ingress"). // // I1 keeps a declaring build from starting before the edge refuses the // public, but the edge is not the only way to a process on its box. These // rules close the others: // // - ports-excluded: a release declaring doc_engine and ports is refused at // ship, and the box renders no ListenStream for an internal group. The // capability pass removes an existing one whenever the group's mark // closes (writeMark calls dropListeners), so it is gone before any fence // of that write completes. // - dedicated-launch-check: a declaring launch is refused on a box with a // proxying row outside its group (the gate, before it writes or waits). // - dedicated-edge-refusal: while an internal group is homed on the box, // the edge gives every proxying row outside that group the internal // refusal. // - ingress-precondition: the operator's firewall admits connections from // every source, public or private (the deployment's other boxes too), // only on the edge's ports and the ports the operator granted. It is a // world flag, not a box mechanism: the platform cannot see it. // // A proxying row is an instance or service row with a backend: the edge // proxies it to a process. Static rows (and the status host, the tunnel and // steering handlers, which the model does not have) are answered by the edge // itself. Alias rows and path maps resolve to their final row, and the edge // checks that one. // ---------- addresses ---------- // A TCP backend is on a box by its subnet, 10.0..:; a socket // path or 127.0.0.1 is reachable on its own box only. // boxAddr is box b's address. func boxAddr(b *box) string { return fmt.Sprintf("10.0.%d.1", b.ID) } // addrBox is the box an address backend is on, and its port; nil for a // socket path, loopback, or an address on no box. func (w *world) addrBox(backend string) (*box, string) { i := strings.LastIndex(backend, ":") if strings.HasPrefix(backend, "/") || i < 0 { return nil, "" } for _, b := range w.boxes { if strings.HasPrefix(backend[:i], fmt.Sprintf("10.0.%d.", b.ID)) { return b, backend[i+1:] } } return nil, "" } // ownPort is the port process p binds on its own, if it does (OwnPort). func ownPort(p *proc) string { return fmt.Sprint(20000 + p.ID) } // bindOwnPort is process p binding a port of its own on every interface: // the unit template allows it and the capability validator leaves high // ports to the app (spec §5.9, codex R12-1). func (w *world) bindOwnPort(b *box, p *proc) { p.OwnPort = true w.logf("box%d: process %d (%s v%d) binds port %s on every interface", b.ID, p.ID, p.Group, p.Ver, ownPort(p)) } // newest is group g's newest process on b, or nil. func (w *world) newest(b *box, g string) *proc { var p *proc for _, x := range b.Procs { if x.Group == g { p = x } } return p } // ---------- service rows ---------- // proxying: the edge proxies the row to a process. func proxying(r *row) bool { return (r.Kind == "instance" || r.Kind == "service") && r.Backend != "" } // addServiceRow is `carlos add` of a service row on box b whose upstream is // backend: a socket path or address on b, or, from box 1, box 0's address. // A service row is 0 by construction. func (w *world) addServiceRow(b *box, host, backend string) { b.Rows[host] = &row{Host: host, Kind: "service", Backend: backend, PathTo: map[string]string{}} b.Mark[host] = markPublic b.Wrote++ w.logf("box%d: registry change %d: service row %s -> %s", b.ID, b.Wrote, host, backend) } // deleteRow removes a service row. A request admitted through it keeps the // proxy it holds (the fence's business, fence-box-wide). func (w *world) deleteRow(b *box, host string) { delete(b.Rows, host) delete(b.Mark, host) b.Wrote++ w.logf("box%d: registry change %d: row %s deleted", b.ID, b.Wrote, host) } // removeGroup is the operator removing instance route g from b, as the // dedicated-box refusal's log line tells them to: its processes stop, its // rows and the rows minted from it go, and so do its listeners. The app's // staged versions stay on the box (/opt/carlos/versions is per app): any // recorded under g pass to another instance of the app. func (w *world) removeGroup(b *box, g string) { others := []string{} for _, h := range w.appGroups(b, g) { if h != g { others = append(others, h) } } if len(others) > 0 { for _, v := range b.Staged[g] { b.Staged[others[0]] = append(b.Staged[others[0]], v) } } w.stopGroup(b, g) for _, h := range hostsOf(b) { if r := b.Rows[h]; h == g || (r.Platform && r.Source == g) { delete(b.Rows, h) delete(b.Mark, h) } } for port, lg := range b.Listen { if lg == g { delete(b.Listen, port) } } delete(b.Live, g) delete(b.Good, g) delete(b.Staged, g) delete(b.SetAt, g) delete(b.PortGrant, g) b.Wrote++ w.logf("box%d: registry change %d: route %s removed with its minted rows", b.ID, b.Wrote, g) } // ---------- the dedicated box ---------- // dedicatedLaunchCheck is the launch gate's check before it writes or waits // for anything: a declaring launch in group g is refused while any proxying // row of the box's registry belongs to another group, and each such row is // named with the move or remove command. func (w *world) dedicatedLaunchCheck(b *box, g string, ver int, path string) bool { if !on("dedicated-launch-check") { return true } var outside []string for _, h := range hostsOf(b) { if r := b.Rows[h]; proxying(r) && w.group(b, h) != g { outside = append(outside, h) } } for _, h := range outside { w.logf("box%d: gate(%s) %s v%d: %s row %s proxies outside the group on the box that would home an engine; move or remove it", b.ID, path, g, ver, b.Rows[h].Kind, h) } if len(outside) > 0 { w.logf("box%d: gate(%s) %s v%d: refused: the box is not dedicated", b.ID, path, g, ver) return false } return true } // edgeHomed is every internal group homed on b, as the edge's map has it: a // group whose own route the edge serves 1 or NULL. The edge refuses NULL like // 1 (classify-null-refused), and so does this rule, release or not: a gate // writing 1 over a NULL group finds it fenced already and bumps no epoch, so // a request admitted through a row beside a NULL route would outlive a // declaring launch there (TestNullRouteHomesTheBox). The cost: a NULL route // on a shared box (an upgrade that cannot read its live manifest, a // provisioned route whose declaring first adoption the launch check refused) // takes every proxying neighbour out of service until it is classified. func (w *world) edgeHomed(b *box) map[string]bool { homed := map[string]bool{} for h, r := range b.EdgeRows { if r.Kind != "instance" || r.Platform { continue } if m := b.Serving[h]; m == markInternal || (m == markNull && on("classify-null-refused")) { homed[h] = true } } return homed } // dedicatedRefusal is the edge's refusal of final, the row a public request // resolved to: on a box that homes an internal group, a proxying row of any // other group gets the internal refusal. It returns why, or "" when the row // is served. func (w *world) dedicatedRefusal(b *box, final row) string { if !on("dedicated-edge-refusal") || !proxying(&final) { return "" } homed := w.edgeHomed(b) if len(homed) == 0 || homed[edgeGroup(b, final.Host)] { return "" } return fmt.Sprintf("%s row %s proxies outside %v, the internal group homed on the box; move or remove it", final.Kind, final.Host, sortedKeys(homed)) } func sortedKeys(m map[string]bool) []string { out := make([]string, 0, len(m)) for k := range m { out = append(out, k) } sort.Strings(out) return out } // ---------- ports ---------- // ship is `carlos ship` of build ver's release for group g, its capability // block declaring ports or not, then its adoption on b. A ports release // records the operator's grant (a port) and the capability pass runs. func (w *world) ship(b *box, g string, ver int, ports bool) bool { if ports && w.builds[ver].Declares && on("ports-excluded") { w.logf("ship %s v%d: refused: the release declares doc_engine and ports together", g, ver) return false } if !w.adopt(b, g, ver, nil) { return false } if ports && b.PortGrant[g] == "" { w.nextID++ b.PortGrant[g] = fmt.Sprint(40000 + w.nextID) w.logf("box%d: %s holds a ports grant: port %s", b.ID, g, b.PortGrant[g]) } w.capsPass(b) return true } // capsPass is the agent's capability pass: it renders each granted port's // ListenStream, except for a route of an internal group, whose listener it // removes. It fails closed: the route's own mark or any row of its group // being 1 or NULL is enough. func (w *world) capsPass(b *box) { gs := make([]string, 0, len(b.PortGrant)) for g := range b.PortGrant { gs = append(gs, g) } sort.Strings(gs) for _, g := range gs { if on("ports-excluded") && (b.Mark[g] != markPublic || w.closed(b, g)) { w.dropListeners(b, g) continue } if port := b.PortGrant[g]; b.Listen[port] != g { b.Listen[port] = g w.logf("box%d: capability pass renders ListenStream=%s for %s", b.ID, port, g) } } } // dropListeners removes group g's ListenStream sockets, stopping their // socket units, and logs the grant it is ignoring. writeMark calls it on // every write of 1 or NULL. func (w *world) dropListeners(b *box, g string) { if !on("ports-excluded") { return } ports := make([]string, 0, len(b.Listen)) for port, lg := range b.Listen { if lg == g { ports = append(ports, port) } } sort.Strings(ports) for _, port := range ports { delete(b.Listen, port) w.logf("box%d: %s is internal: ListenStream=%s removed, its ports grant ignored (revoke it with carlos caps)", b.ID, g, port) } } // ---------- ingress ---------- // directConnect is a connection to port on box b that does not go through // b's edge: from the internet, or from another box (a service row there // aimed at b's address). A ports listener takes it whatever the firewall // flag (the operator granted that port, so the firewall admits it); a stopped // unit is socket-activated from its live link. Otherwise it reaches a process // only if the ingress precondition is violated: one bound to the port on its // own, or a TCP route's backend. func (w *world) directConnect(b *box, port string, public bool) *proc { return w.connect(b, port, public, true) } // connect is directConnect, or with firewalled false the box's own traffic // (a local row's proxy to an address on its own box), which no firewall sees. func (w *world) connect(b *box, port string, public, firewalled bool) *proc { src := "private" if public { src = "public" } if g, ok := b.Listen[port]; ok { w.logf("box%d: %s connection to port %s: the ports listener of %s", b.ID, src, port, g) w.socketActivate(b, g) return w.reach(b, w.newest(b, g), public, "the ports listener on "+port) } if firewalled && on("ingress-precondition") { w.logf("box%d: %s connection to port %s: refused by the firewall (the edge's ports only)", b.ID, src, port) return nil } return w.reach(b, w.portProc(b, port), public, "port "+port) } // portProc is the process listening on port on b: one bound to it on its // own, else the newest process of the TCP route whose backend it is. func (w *world) portProc(b *box, port string) *proc { for i := len(b.Procs) - 1; i >= 0; i-- { if p := b.Procs[i]; p.OwnPort && ownPort(p) == port { return p } } for _, h := range hostsOf(b) { r := b.Rows[h] if r.Kind != "instance" || r.Platform { continue } if tb, pt := w.addrBox(r.Backend); tb == b && pt == port { return w.newest(b, w.group(b, h)) } } return nil } // reach records a connection past the edge reaching process p. INV-P, as in // deliver: a public connection never reaches a declaring build, nor a group // holding a loaded doc-engine grant. func (w *world) reach(b *box, p *proc, public bool, how string) *proc { if p == nil { w.logf("box%d: %s: nothing listens", b.ID, how) return nil } w.logf("box%d: %s reached process %d (%s v%d) past the edge", b.ID, how, p.ID, p.Group, p.Ver) w.events["delivered"]++ // a ports listener exists only because the group is public: a path the // upgrade's 0 opens; any other way past the edge is the ingress rule's listener := strings.HasPrefix(how, "the ports listener") if public && w.builds[p.Ver].Declares { w.failProc(b, p, "INV-P", listener, "box%d: a public connection through %s reached process %d of %s running v%d, which declares doc_engine, past the edge", b.ID, how, p.ID, p.Group, p.Ver) } else if g := w.grantLoaded(b, p.Group); public && g != "" { w.failProc(b, p, "INV-P", listener, "box%d: a public connection through %s reached process %d of %s running v%d, whose group holds the doc-engine grant (on %s), past the edge", b.ID, how, p.ID, p.Group, p.Ver, g) } return p } // deliverDirect is the proxy of request r on b whose backend is an address // no instance row of b carries: on another box, a direct connection there // (the firewall decides); on b itself, the process bound to that port. False // when the backend is no such address. func (w *world) deliverDirect(b *box, r *req) bool { tb, port := w.addrBox(r.Backend) if tb == nil { return false } w.logf("box%d: request %d proxies to %s, box%d's address", b.ID, r.ID, r.Backend, tb.ID) if p := w.connect(tb, port, r.Public, tb != b); p != nil { r.Reached = p.ID } return true } // ---------- INV-AV ---------- // A proxying row outside the internal group homed on its box is refused by // the dedicated-box rule on purpose (spec §5.9): INV-AV does not count it. // The row's true group decides, not the edge's resolution. func init() { avExemptions = append(avExemptions, func(w *world, b *box, host string) string { r := b.Rows[host] if r == nil || !proxying(r) { return "" } homed := w.edgeHomed(b) if len(homed) == 0 || homed[truthGroup(b, host)] { return "" } return fmt.Sprintf("a proxying row outside %v, the internal group homed on the box: refused by design (§5.9)", sortedKeys(homed)) }) } // ---------- the random walk ---------- // addressablePorts is every port on b a direct connection can name: TCP // routes' backends, ports processes bound on their own, ports listeners. func (w *world) addressablePorts(b *box) []string { set := map[string]bool{} for _, h := range hostsOf(b) { if r := b.Rows[h]; r.Kind == "instance" && !r.Platform { if tb, port := w.addrBox(r.Backend); tb == b { set[port] = true } } } for _, p := range b.Procs { if p.OwnPort { set[ownPort(p)] = true } } for port := range b.Listen { set[port] = true } return sortedKeys(set) } // localBackends is every upstream a local service row on b can name: its // instance routes' backends and the ports on b's own address. func (w *world) localBackends(b *box) []string { set := map[string]bool{"127.0.0.1:1": true} // nothing listens for _, h := range hostsOf(b) { if r := b.Rows[h]; r.Kind == "instance" && !r.Platform && r.Backend != "" { set[r.Backend] = true } } for _, port := range w.addressablePorts(b) { set[boxAddr(b)+":"+port] = true } return sortedKeys(set) } func init() { actions = append(actions, action{name: "add-service-row", weight: 2, run: func(w *world) bool { b := w.pickBox() if len(b.Rows) >= maxRows { return false } w.nextID++ host := fmt.Sprintf("svc%d.example", w.nextID) if b.ID == 1 && w.rng.Intn(2) == 0 { // aimed at box 0's address (codex R13-1) ports := append(w.addressablePorts(w.boxes[0]), "1") w.addServiceRow(b, host, boxAddr(w.boxes[0])+":"+ports[w.rng.Intn(len(ports))]) return true } bs := w.localBackends(b) w.addServiceRow(b, host, bs[w.rng.Intn(len(bs))]) return true }}, action{name: "delete-service-row", weight: 2, run: func(w *world) bool { type at struct { b *box host string } var rows []at for _, b := range w.boxes { for _, h := range hostsOf(b) { if b.Rows[h].Kind == "service" { rows = append(rows, at{b, h}) } } } if len(rows) == 0 { return false } x := rows[w.rng.Intn(len(rows))] w.deleteRow(x.b, x.host) return true }}, action{name: "own-port", weight: 1, run: func(w *world) bool { b := w.pickBox() if len(b.Procs) == 0 { return false } w.bindOwnPort(b, b.Procs[w.rng.Intn(len(b.Procs))]) return true }}, action{name: "direct-connect", weight: 2, run: func(w *world) bool { b := w.pickBox() ports := append(w.addressablePorts(b), "1") w.directConnect(b, ports[w.rng.Intn(len(ports))], true) return true }}, action{name: "ship-ports", weight: 1, run: func(w *world) bool { b := w.pickBox() g := w.pickGroup(b) if g == "" { return false } w.ship(b, g, w.pickVer(), true) return true }}, action{name: "caps-pass", weight: 1, run: func(w *world) bool { w.capsPass(w.pickBox()); return true }}, action{name: "remove-route", weight: 1, run: func(w *world) bool { // an instance route the walk added (the starting eng and shop stay) b := w.pickBox() var gs []string for _, g := range w.groups(b) { if g != eng && g != "shop" { gs = append(gs, g) } } if len(gs) == 0 { return false } w.removeGroup(b, gs[w.rng.Intn(len(gs))]) return true }}, ) }