package instancectl import ( "bufio" "bytes" "context" "errors" "fmt" "io/fs" "net/http" "net/netip" "os" "path/filepath" "sort" "strings" "time" "github.com/carlosframework/platform/internal/provision" ) // What the control service does beyond start and stop, so the isolation // the templates promise holds (#214, #178): it lays out each instance's // view of its own build, re-opens the box's DNS resolver through the // egress fence, and keeps tenants from filling the shared data disk. // Paths the instance units and the probe bind from (#214 item 2). The // templates name them literally; the units test pins both sides. const ( // LiveDir is where the adopt pass keeps each host's live link, and // where an instance finds its own build inside its namespace. LiveDir = "/opt/carlos/live" // SystemBinary is the platform binary every instance unit runs as // ExecStart=; the probe's stand-in build. SystemBinary = "/opt/carlos/carlos" // InstancesSlice is the parent of every per-host slice and of the // probe: an IPAddressAllow= on it reaches every instance unit. InstancesSlice = "carlos-instances.slice" ) // ensureExe writes the two sources an instance unit binds over its empty // /opt/carlos (#214 item 2), under root//: // // app -> target (the host's live link; the probe's: the platform binary) // live/ -> ../app // // PID 1 chases app to the build when it binds it at /opt/carlos/app, and // binds live/ at /opt/carlos/live, so inside the namespace // /opt/carlos/live/ resolves to the build and nothing else of the // version store exists. Both are links, never copies: a deploy repoints // the host's live link and the next start binds the new build, exactly as // before. Only root writes under root, so nothing here can be planted; // anything else found in live/ is removed all the same, because live/ is // bound into a tenant. func ensureExe(root, iid, target, name string) error { return ensureExeAs(root, iid, target, name, "app", "live") } // ensureExeAs is ensureExe with the two sources named: app and live under // root//. The cmd template binds its own pair (CmdExeApp, CmdExeLive; // merge ruling M4), so a command's start and an instance's or sidecar's // start never write a source the other is about to bind. Inside the // namespace both pairs are bound at the same places, so the link in live // is ../app either way. func ensureExeAs(root, iid, target, name, app, liveName string) error { if root == "" { return errors.New("no exe root") } if iid == "" || strings.ContainsAny(iid, "/\x00") || iid == "." || iid == ".." || name == "" || strings.ContainsAny(name, "/\x00") || name == "." || name == ".." { return fmt.Errorf("exe: %q/%q is not a name", iid, name) } dir := filepath.Join(root, iid) live := filepath.Join(dir, liveName) for _, d := range []string{root, dir, live} { if err := os.Mkdir(d, 0o755); err != nil && !errors.Is(err, fs.ErrExist) { return err } fi, err := os.Lstat(d) if err != nil { return err } if !fi.IsDir() { return fmt.Errorf("exe: %s is not a directory", d) } } if err := replaceLink(filepath.Join(dir, app), target); err != nil { return err } if err := replaceLink(filepath.Join(live, name), "../app"); err != nil { return err } entries, err := os.ReadDir(live) if err != nil { return err } for _, e := range entries { if e.Name() != name { if err := os.RemoveAll(filepath.Join(live, e.Name())); err != nil { return err } } } return nil } // replaceLink points path at target, atomically: a temp link renamed over // whatever was there, so a unit starting at that moment binds the old // target or the new one, never nothing. func replaceLink(path, target string) error { if cur, err := os.Readlink(path); err == nil && cur == target { return nil } tmp := path + ".tmp" _ = os.Remove(tmp) if err := os.Symlink(target, tmp); err != nil { return err } if err := os.Rename(tmp, path); err != nil { os.Remove(tmp) return err } return nil } // ensureHostExe is ensureExe for a host about to start. func (s *Server) ensureHostExe(host string) error { return ensureExe(s.ExeRoot, provision.InstanceID(host), filepath.Join(LiveDir, host), host) } // ParseResolvers returns the nameserver addresses of a resolv.conf, in // order, zones dropped. Lines it cannot read are skipped: the resolver // library skips them too. func ParseResolvers(b []byte) []netip.Addr { var out []netip.Addr sc := bufio.NewScanner(bytes.NewReader(b)) for sc.Scan() { f := strings.Fields(sc.Text()) if len(f) < 2 || f[0] != "nameserver" { continue } a, err := netip.ParseAddr(f[1]) if err != nil { continue } out = append(out, a.WithZone("").Unmap()) } return out } // ResolverAllowed says whether a nameserver may be re-opened through the // instance egress fence (#214 item 3), and if not, why. IPAddressAllow= // opens an address, not a port, and in both directions, so an allowed // address must be one where nothing but a resolver can answer: // // - no loopback address, systemd-resolved's stubs (127.0.0.53/54) // included: a tenant can bind the address itself and then reach every // listener on the box bound to all addresses through it. Instances // resolve through resolved's UPSTREAMS instead (the namespace binds // /run/systemd/resolve/resolv.conf over /etc/resolv.conf); // - no unspecified address, which means "this box"; // - no metadata service; // - no link-local address but Amazon's resolver, 169.254.169.253. An // allow cannot carry a zone, so fe80::1 would open that address on // every link, and link-local v4 is where IMDS and the time service // live. Boxes resolve through the VPC resolver; a link-local one is // refused and the probe's dns line says so. // // Everything else is opened as a single address: the VPC resolver (VPC // base + 2, inside RFC 1918), fd00:ec2::253, or a public one, unless it // is one of the box's own addresses, which only the box can say: // resolverAllow refuses those against localPrefixes, here, where the // allow is written (#126). func ResolverAllowed(a netip.Addr) (bool, string) { switch { case !a.IsValid(): return false, "not an address" case a == netip.MustParseAddr("127.0.0.53"), a == netip.MustParseAddr("127.0.0.54"): return false, "systemd-resolved's stub is on loopback; instances use its upstreams" case a.IsLoopback(): return false, "a loopback resolver shares its address with services on this box" case a.IsUnspecified(): return false, "the unspecified address is this box" case a == netip.MustParseAddr("169.254.169.254"), a == netip.MustParseAddr("fd00:ec2::254"): return false, "the metadata service is not a resolver" case a == netip.MustParseAddr("169.254.169.253"): return true, "" case a.IsLinkLocalUnicast(): return false, "a link-local resolver cannot be opened for one link only" } return true, "" } // resolverAllow is the IPAddressAllow= value for a resolv.conf: each // allowed nameserver as a single-address prefix, sorted and deduplicated // so the drop-in systemd writes does not churn. A nameserver inside one // of the box's local prefixes is left out, and named in refused. func resolverAllow(b []byte, local []netip.Prefix) (allow, refused []string) { seen := map[string]bool{} for _, a := range ParseResolvers(b) { if ok, _ := ResolverAllowed(a); !ok { continue } p := netip.PrefixFrom(a, a.BitLen()).String() if seen[p] { continue } seen[p] = true if lp, ok := isLocal(a, local); ok { refused = append(refused, fmt.Sprintf("%s is an address of this box (%s)", a, lp)) continue } allow = append(allow, p) } sort.Strings(allow) return allow, refused } // ResolvedConf is systemd-resolved's list of its upstreams. When it // exists, the namespace binds it over /etc/resolv.conf (the "-" bind in // the templates), so it is what instances resolve through and what the // allow must open. const ResolvedConf = "/run/systemd/resolve/resolv.conf" // resolverFile is the resolv.conf an instance sees: resolved's upstream // list when there is one, the box's own otherwise. Mirrors the template. func (s *Server) resolverFile() string { if s.ResolvedConf != "" { if _, err := os.Stat(s.ResolvedConf); err == nil { return s.ResolvedConf } } return s.ResolvConf } // allowResolvers re-opens the resolvers instances see on // carlos-instances.slice. The empty assignment first resets the list, so // a resolver that left the file is closed again. It runs before every // probe, so the allow is in place before the first start and follows a // resolver change within one probe period; the probe then checks the // result from inside. It returns the nameservers it refused for being // the box's own. When the box's addresses cannot be read, every resolver // is closed and the error returned: an allow that might name the box // itself is never written. func (s *Server) allowResolvers(ctx context.Context) (refused []string, err error) { b, err := os.ReadFile(s.resolverFile()) if err != nil { return nil, err } var allow []string local, lerr := s.localPrefixes() if lerr == nil { allow, refused = resolverAllow(b, local) } args := []string{"set-property", "--runtime", InstancesSlice, "IPAddressAllow="} if len(allow) > 0 { args = append(args, "IPAddressAllow="+strings.Join(allow, " ")) } out, err := s.Systemctl(ctx, args...) if err != nil { return nil, fmt.Errorf("%s: %s", strings.TrimSpace(out), err) } if lerr != nil { return nil, fmt.Errorf("this box's addresses: %v; every resolver is closed", lerr) } return refused, nil } // localPrefixes is LocalAddrs, refusing when there is none. func (s *Server) localPrefixes() ([]netip.Prefix, error) { if s.LocalAddrs == nil { return nil, errors.New("no way to list them") } return s.LocalAddrs() } // resolverLocalLine is the verdict on allowResolvers' refusals, folded // into the probe's report: the probe cannot make this check from inside // its unit, which cannot list the box's addresses (#126). func resolverLocalLine(refused []string) VerdictLine { if len(refused) > 0 { return VerdictLine{Check: "resolver-local", Verdict: "fail", Detail: strings.Join(refused, "; ")} } return VerdictLine{Check: "resolver-local", Verdict: "ok"} } // The data disk (#214 item 1). Every instance's directory, the registry, // the activator's state and litestream's shadow WAL share one filesystem, // /data, on the root volume. A per-tenant quota is not achievable there // today: ext4 project quotas need the "project" and "quota" features, // which tune2fs sets only on an unmounted filesystem, plus a prjquota // mount, and the root filesystem is never unmounted on a running box. // Every process that writes /data is uid carlos, so reserved blocks // cannot tell a tenant from the agent either. // // Nor can this service attribute usage well enough to stop a tenant for // it: space held by unlinked-but-open files appears in no directory, so // the guilty tenant can look small while honest ones look large, and a // stop from here bypasses the activator's fence and holds (a host mid- // migration). So the guard refuses, it never stops: below the floor no // instance starts or seeds, and a warning names the shortfall every // DiskEvery while it lasts. Instances already awake keep running. const minStartFloor = 1 << 30 // 1 GiB // DiskFloor is the free space a start needs on a filesystem of total // bytes: the larger of 1 GiB and 10%. func DiskFloor(total uint64) uint64 { return max(minStartFloor, total/10) } // ErrDiskFull is a start refused for disk space. var ErrDiskFull = errors.New("data disk too full") // diskRoom is nil when there is room to start an instance. func (s *Server) diskRoom() error { if s.Statfs == nil { return nil } avail, total, err := s.Statfs(s.DataRoot) if err != nil { return fmt.Errorf("statfs %s: %w", s.DataRoot, err) } if floor := DiskFloor(total); avail < floor { return fmt.Errorf("%w: %s free, below the %s floor", ErrDiskFull, humanBytes(avail), humanBytes(floor)) } return nil } // refuseStart answers a start the disk or the exe layout cannot take. exe // lays out the unit's view of its build (ensureHostExe, or ensureGenExe // for a generation). It reports whether it answered. func (s *Server) refuseStart(w http.ResponseWriter, host, unit string, exe func() error) bool { if err := s.diskRoom(); err != nil { s.logf("instance-control: %s: not starting %s: %v", host, unit, err) status := http.StatusBadGateway if errors.Is(err, ErrDiskFull) { status = http.StatusInsufficientStorage } http.Error(w, "not starting "+unit+": "+err.Error(), status) return true } if s.ExeRoot != "" && exe != nil { if err := exe(); err != nil { http.Error(w, "exe: "+err.Error(), http.StatusBadGateway) return true } } return false } // warnIfFull logs the alert while the disk is below the floor. It reports // whether it warned. func (s *Server) warnIfFull() bool { err := s.diskRoom() if errors.Is(err, ErrDiskFull) { s.logf("instance-control: disk: %v; refusing every instance start until there is room (awake instances keep running)", err) return true } if err != nil { s.logf("instance-control: disk: %v", err) } return false } // diskLoop runs warnIfFull every DiskEvery. func (s *Server) diskLoop(ctx context.Context) { if s.DiskEvery <= 0 { return } t := time.NewTicker(s.DiskEvery) defer t.Stop() for { select { case <-ctx.Done(): return case <-t.C: s.warnIfFull() } } } func humanBytes(n uint64) string { switch { case n >= 1<<30: return fmt.Sprintf("%.1f GiB", float64(n)/(1<<30)) case n >= 1<<20: return fmt.Sprintf("%.0f MiB", float64(n)/(1<<20)) } return fmt.Sprintf("%d bytes", n) }