package instanceexec import ( "errors" "fmt" "net" "net/netip" "os" "path/filepath" "strings" "syscall" "time" "github.com/carlosframework/platform/internal/instancectl" ) // What the probe unit's own resource section sets (the units test pins // the unit side), read back from its cgroup by the cpu-limits and // io-weight lines. const ( probeCPUMax = "50000 100000" // CPUQuota=50% probeCPUWeight = "50" probeIOWeight = "default 50" ) // resolverProbePort is the non-DNS port the resolver-ports line dials on // each allowed resolver: ssh, the listener a box is most likely to have // bound to all addresses. const resolverProbePort = 22 // Addresses the egress fence must refuse (#214 item 3), one per range, // none of them anything a box would bind. A send to each is answered by // the cgroup filter with EPERM before a packet leaves. var ( privateV4 = []string{"10.255.255.254", "172.31.255.254", "192.168.255.254", "100.127.255.254", "169.254.0.1", "198.18.0.1", "192.0.0.1"} // v6 has no route on most boxes; ENETUNREACH there is unverified. privateV6 = []string{"fd00::1", "fec0::1", "64:ff9b::a00:1"} ) // hardeningChecks are the verdict lines for #214 items 1-3 and #178. func hardeningChecks(e probeEnv, add func(check, verdict, detail string), pass func(check, detail string, ok bool)) { // Item 2: /opt/carlos holds the platform binary and this unit's own // build, and no version store. o1, ok1 := exactly(e.optRoot, "app", "carlos", "live") o2, ok2 := exactly(filepath.Join(e.optRoot, "live"), instancectl.ProbeDirName) pass("opt-own-build", o1+o2, ok1 && ok2) // Item 3: the box's resolvers are reachable through the fence. This // is also the live-stack control the EPERM lines below rest on. resolvers := map[string]bool{} var allowed []netip.Addr var dnsFail []string if b, err := os.ReadFile(e.resolvConf); err != nil { dnsFail = append(dnsFail, err.Error()) } else if addrs := instancectl.ParseResolvers(b); len(addrs) == 0 { dnsFail = append(dnsFail, "no nameserver in "+e.resolvConf) } else { for _, a := range addrs { resolvers[a.String()] = true if ok, why := instancectl.ResolverAllowed(a); !ok { dnsFail = append(dnsFail, a.String()+": "+why) continue } allowed = append(allowed, a) network := "udp4" if a.Is6() { network = "udp6" } if err := e.sendUDP(network, netip.AddrPortFrom(a, 53).String()); err != nil { dnsFail = append(dnsFail, fmt.Sprintf("%s: sendto = %v", a, err)) } } } pass("dns", strings.Join(dnsFail, "; "), len(dnsFail) == 0) // Item 3: an allowed resolver opens only a resolver. The allow is per // address, both ways and every port, so a non-DNS port on it must not // answer. That it is not an address of this box is the control // service's check (resolver-local), made where the allow is written: // listing the box's addresses takes netlink, which this unit's // RestrictAddressFamilies= refuses, as it does every instance's // (#126, the v0.25.0 roll). var portFail []string for _, a := range allowed { if err := e.connectTCP(netip.AddrPortFrom(a, resolverProbePort).String(), 2*time.Second); err == nil { portFail = append(portFail, fmt.Sprintf("%s: port %d answered", a, resolverProbePort)) } } pass("resolver-ports", strings.Join(portFail, "; "), len(portFail) == 0) // Item 3: loopback is closed. lo := e.sendUDP("udp4", "127.0.0.1:9") pass("loopback-v4", fmt.Sprintf("sendto = %v", lo), errors.Is(lo, syscall.EPERM)) lo6 := e.sendUDP("udp6", "[::1]:9") switch { case errors.Is(lo6, syscall.EPERM): add("loopback-v6", "ok", "") case noV6(lo6): add("loopback-v6", "unverified", fmt.Sprintf("no IPv6 on this box (%v); deny not exercised", lo6)) default: add("loopback-v6", "fail", fmt.Sprintf("sendto = %v", lo6)) } // Item 3: private and cloud-internal ranges are closed, except an // address the box resolves through, which is open on purpose. var v4Fail []string for _, a := range privateV4 { if resolvers[a] { continue } if err := e.sendUDP("udp4", net.JoinHostPort(a, "9")); !errors.Is(err, syscall.EPERM) { v4Fail = append(v4Fail, fmt.Sprintf("%s: sendto = %v", a, err)) } } pass("private-v4", strings.Join(v4Fail, "; "), len(v4Fail) == 0) var v6Fail, v6Unrouted []string for _, a := range privateV6 { if resolvers[a] { continue } err := e.sendUDP("udp6", net.JoinHostPort(a, "9")) switch { case errors.Is(err, syscall.EPERM): case noV6(err): v6Unrouted = append(v6Unrouted, a) default: v6Fail = append(v6Fail, fmt.Sprintf("%s: sendto = %v", a, err)) } } switch { case len(v6Fail) > 0: add("private-v6", "fail", strings.Join(v6Fail, "; ")) case len(v6Unrouted) > 0: add("private-v6", "unverified", "no route to "+strings.Join(v6Unrouted, ", ")+"; deny not exercised") default: add("private-v6", "ok", "") } // #178: an IPC namespace of its own, and no host mqueue mount. ino, ierr := e.ipcInode() m, mok := exactly(e.devMqueue) pass("ipc-private", fmt.Sprintf("ipc ns inode=%d err=%v %s", ino, ierr, m), ierr == nil && ino != initIPCInode && mok) // #178: the trampoline's abstract socket scope holds on this kernel. aerr := e.abstractScoped() pass("abstract-unix", fmt.Sprint(aerr), aerr == nil) // Item 1: CPU settings land on a unit under carlos-instances.slice. cg, cerr := e.selfCgroup() if cerr != nil { add("cpu-limits", "fail", cerr.Error()) add("io-weight", "fail", cerr.Error()) return } cpuMax := readTrim(filepath.Join(cg, "cpu.max")) cpuWeight := readTrim(filepath.Join(cg, "cpu.weight")) pass("cpu-limits", fmt.Sprintf("cpu.max=%q cpu.weight=%q", cpuMax, cpuWeight), cpuMax == probeCPUMax && cpuWeight == probeCPUWeight) // Item 1: the IO weight lands too, and says whether anything on this // box enforces it. io, ioErr := os.ReadFile(filepath.Join(cg, "io.weight")) ioFirst, _, _ := strings.Cut(strings.TrimSpace(string(io)), "\n") switch { case ioErr != nil: add("io-weight", "unverified", "no io.weight here ("+ioErr.Error()+"); IO is not weighted") case ioFirst != probeIOWeight: add("io-weight", "fail", fmt.Sprintf("io.weight=%q, want %q", ioFirst, probeIOWeight)) case !e.ioEnforced(cg): add("io-weight", "unverified", "weights are set, but no disk here has io.cost or BFQ to enforce them") default: add("io-weight", "ok", "") } } // noV6 is an answer that means the box has no IPv6 path for that // address, so the fence was never asked. func noV6(err error) bool { return errors.Is(err, syscall.ENETUNREACH) || errors.Is(err, syscall.EADDRNOTAVAIL) || errors.Is(err, syscall.EAFNOSUPPORT) } func readTrim(path string) string { b, err := os.ReadFile(path) if err != nil { return "<" + err.Error() + ">" } return strings.TrimSpace(string(b)) }