//go:build linux package instancefs import ( "bufio" "crypto/rand" "encoding/hex" "errors" "fmt" "io" "os" "path/filepath" "strconv" "strings" "golang.org/x/sys/unix" ) const resolveFlags = unix.RESOLVE_BENEATH | unix.RESOLVE_NO_SYMLINKS | unix.RESOLVE_NO_MAGICLINKS // openDir opens dir itself as a directory fd, refusing a symlink at ANY // component of dir, not only its final one: the instance directory's // parent is the agent's, but a same-uid process could have swapped an // intermediate component for a symlink before the mount option landed, // and openat2's RESOLVE_NO_SYMLINKS catches that the way a plain // O_NOFOLLOW open (which only refuses a symlink at the leaf) would not. // RESOLVE_BENEATH is deliberately absent here: dir is an absolute path // handed in by the caller, not a name resolved beneath some other fd, so // there is no "beneath" for it to stay under. func openDir(dir string) (int, error) { how := unix.OpenHow{ Flags: unix.O_PATH | unix.O_DIRECTORY | unix.O_CLOEXEC, Resolve: unix.RESOLVE_NO_SYMLINKS | unix.RESOLVE_NO_MAGICLINKS, } fd, err := unix.Openat2(unix.AT_FDCWD, dir, &how) if err != nil { if errors.Is(err, unix.ELOOP) { return -1, fmt.Errorf("%w: %s", ErrSymlink, dir) } return -1, &os.PathError{Op: "openat2", Path: dir, Err: err} } return fd, nil } // SelfCheck is the one call a process makes before it trusts the rest of // this package: it opens dir exactly as openDir does and reports whether // the kernel let the call through at all. It exists because a seccomp // filter is invisible from inside: the agent would learn of one from the // first nested open that fails, which is to say from a layout migration // half-way through its first host. ENOSYS is ErrOpenat2Unavailable, named // so a gate can say what to fix; any other error is dir's own and is // returned as it is. The probe unit of design ยง8.5 runs under the // instance templates' text and so cannot see the edge unit's; this is the // edge's own check, run by the agent in its own unit. func SelfCheck(dir string) error { fd, err := openDir(dir) if err != nil { if errors.Is(err, unix.ENOSYS) { return fmt.Errorf("%w: %s", ErrOpenat2Unavailable, dir) } return err } unix.Close(fd) return nil } // OpenBeneath opens name under dir with openat2, following no symlink at // any component and never leaving dir. Every open, whatever its flags, // carries O_NONBLOCK on the wire and is then checked with fstat: a FIFO // with no reader on the other end blocks a write-open indefinitely, and // one with no writer blocks a read-open indefinitely, and a tenant can // plant one anywhere under an instance directory the agent reads OR // writes by contract (its run directory, its env files, its admin token // file). O_NONBLOCK changes the two blocking cases without changing the // non-blocking ones: opening a regular file (the common case, and every // O_CREAT|O_EXCL caller's own fresh file) behaves identically either way, // a read-open of a FIFO with no writer now returns at once instead of // blocking, and a write-open of a FIFO with no reader answers ENXIO at // once instead of blocking. Whatever DOES come back with a valid fd (a // FIFO opened for read, or for write against an existing reader) is then // refused by the fstat check unless it is S_IFREG, so nothing non-regular // can be read or written through this function at all. func OpenBeneath(dir, name string, flag int, perm os.FileMode) (*os.File, error) { return openBeneathAs(dir, name, flag, perm, unix.S_IFREG) } // OpenDirBeneath opens name under dir as a directory, read-only, exactly as // OpenBeneath opens a file: openat2 with no symlink at any component of dir // or name, O_NONBLOCK, and then fstat, which accepts only a directory. A // name of "." is dir itself. The descriptor is the directory that was // vetted: a caller that works from it (an *at call, or OpenRootBeneath) // never resolves the path again, so a link swapped in after the open is // never followed. func OpenDirBeneath(dir, name string) (*os.File, error) { return openBeneathAs(dir, name, os.O_RDONLY|unix.O_DIRECTORY, 0, unix.S_IFDIR) } // OpenRootBeneath is OpenDirBeneath as an *os.Root, for code built on // os.Root (hostfiles.RestoreRoot). os.OpenRoot takes only a path and opens // it without O_NONBLOCK, so it would follow a link at the name and block // on a FIFO there. This opens the directory with OpenDirBeneath and hands // os.OpenRoot that descriptor's /proc/self/fd entry, a magic link the // kernel resolves to the very directory the descriptor holds: no // component is resolved twice. Without /proc it is an error, never a // fallback to the path. func OpenRootBeneath(dir, name string) (*os.Root, error) { f, err := OpenDirBeneath(dir, name) if err != nil { return nil, err } defer f.Close() betweenOpenAndRoot() r, err := os.OpenRoot(procSelfFD + strconv.Itoa(int(f.Fd()))) if err != nil { return nil, fmt.Errorf("instancefs: open %s as a root: %w", filepath.Join(dir, name), err) } return r, nil } // betweenOpenAndRoot is a test seam: where a path swapped after the open // would race a root opened by path. var betweenOpenAndRoot = func() {} // procSelfFD is where OpenRootBeneath finds its descriptor's magic link. A // test seam: a test points it at a missing directory to pin that no /proc // is an error, never a fallback to the path. var procSelfFD = "/proc/self/fd/" // OpenBeneathOrAbsent opens name under dir read-only as OpenBeneath does, // and is the one place "absent" is decided on ground someone else can // write. It opens dir once and keeps that descriptor: absent is true only // when the lookup of name beneath that very descriptor answers ENOENT. A // dir that cannot be opened (missing, a symlink, not a directory) is an // error wrapped in ErrDirUnopened, never absence: closing a check's // descriptor and opening the path again would read a directory renamed // away in between as a missing file. func OpenBeneathOrAbsent(dir, name string) (*os.File, bool, error) { rel, err := cleanRel(name) if err != nil { return nil, false, err } dfd, err := openDir(dir) if err != nil { return nil, false, fmt.Errorf("%w: %w", ErrDirUnopened, err) } defer unix.Close(dfd) betweenDirAndLookup() f, err := openAt(dfd, dir, rel, os.O_RDONLY, 0, unix.S_IFREG) if errors.Is(err, unix.ENOENT) { return nil, true, nil } if err != nil { return nil, false, err } return f, false, nil } // betweenDirAndLookup is a test seam: after OpenBeneathOrAbsent has opened // dir and before it looks name up beneath it. var betweenDirAndLookup = func() {} // openBeneathAs is OpenBeneath's open, accepting only the file type want // (S_IFREG or S_IFDIR) from fstat. func openBeneathAs(dir, name string, flag int, perm os.FileMode, want uint32) (*os.File, error) { rel, err := cleanRel(name) if err != nil { return nil, err } dfd, err := openDir(dir) if err != nil { return nil, err } defer unix.Close(dfd) return openAt(dfd, dir, rel, flag, perm, want) } // openAt is the open every function here shares: rel beneath the // directory descriptor dfd (which names dir, for errors), openat2 with no // symlink at any component, O_NONBLOCK, and then fstat, accepting only the // file type want. func openAt(dfd int, dir, rel string, flag int, perm os.FileMode, want uint32) (*os.File, error) { how := unix.OpenHow{ Flags: uint64(flag) | unix.O_NONBLOCK | unix.O_CLOEXEC, Mode: uint64(perm.Perm()), Resolve: resolveFlags, } fd, err := unix.Openat2(dfd, rel, &how) if err != nil { switch { case errors.Is(err, unix.ELOOP): return nil, fmt.Errorf("%w: %s", ErrSymlink, filepath.Join(dir, rel)) case errors.Is(err, unix.EXDEV): return nil, fmt.Errorf("%w: %s", ErrEscapes, filepath.Join(dir, rel)) } return nil, &os.PathError{Op: "openat2", Path: filepath.Join(dir, rel), Err: err} } path := filepath.Join(dir, rel) var st unix.Stat_t if err := unix.Fstat(fd, &st); err != nil { unix.Close(fd) return nil, &os.PathError{Op: "fstat", Path: path, Err: err} } if st.Mode&unix.S_IFMT != want { unix.Close(fd) if want == unix.S_IFDIR { return nil, fmt.Errorf("%w: %s", ErrNotDir, path) } return nil, fmt.Errorf("%w: %s", ErrNotRegular, path) } return os.NewFile(uintptr(fd), path), nil } // WriteAtomic writes data to a fresh O_EXCL temp file beside name and // renames it over name. rename(2) replaces the directory entry and never // follows a symlink at the destination, so a planted link is displaced, // not written through. The temp name is random, so no same-uid process // can pre-plant it (the O_EXCL refuses it if one does). func WriteAtomic(dir, name string, data []byte, perm os.FileMode) error { rel, err := cleanRel(name) if err != nil { return err } if strings.Contains(rel, "/") { return fmt.Errorf("%w: WriteAtomic takes a bare file name, got %q", ErrEscapes, name) } dfd, err := openDir(dir) if err != nil { return err } defer unix.Close(dfd) var rnd [8]byte if _, err := rand.Read(rnd[:]); err != nil { return err } tmp := "." + rel + ".tmp-" + hex.EncodeToString(rnd[:]) how := unix.OpenHow{ Flags: unix.O_WRONLY | unix.O_CREAT | unix.O_EXCL | unix.O_CLOEXEC, Mode: uint64(perm.Perm()), Resolve: resolveFlags, } fd, err := unix.Openat2(dfd, tmp, &how) if err != nil { return &os.PathError{Op: "openat2", Path: filepath.Join(dir, tmp), Err: err} } f := os.NewFile(uintptr(fd), filepath.Join(dir, tmp)) cleanup := func() { f.Close(); _ = unix.Unlinkat(dfd, tmp, 0) } if _, err := f.Write(data); err != nil { cleanup() return err } if err := f.Sync(); err != nil { cleanup() return err } if err := f.Close(); err != nil { _ = unix.Unlinkat(dfd, tmp, 0) return err } if err := unix.Renameat(dfd, tmp, dfd, rel); err != nil { _ = unix.Unlinkat(dfd, tmp, 0) return &os.PathError{Op: "renameat", Path: filepath.Join(dir, rel), Err: err} } return nil } // MountOptionOf reports whether mountpoint is a mountpoint whose per-mount // options include opt, by reading /proc/self/mountinfo. A path that is not // a mountpoint is (false, nil). func MountOptionOf(mountpoint, opt string) (bool, error) { f, err := os.Open("/proc/self/mountinfo") if err != nil { return false, err } defer f.Close() return mountOptionOf(f, mountpoint, opt) } // MountNoSymfollow reports whether mountpoint carries nosymfollow. It // predates MountOptionOf and stays as a thin wrapper because most callers // only ever ask this one question and naming it directly reads better at // the call site than spelling the option out every time. func MountNoSymfollow(mountpoint string) (bool, error) { return MountOptionOf(mountpoint, "nosymfollow") } // mountOptionOf is MountOptionOf's body, taking its mountinfo source as an // io.Reader so a test can feed it a fixture with stacked mount lines // instead of the live /proc/self/mountinfo. // // Stacked mounts at one mountpoint (a bind mount, then a later remount) // list oldest first in mountinfo, and the LAST line for a mountpoint is // the one actually in effect. Every line is therefore scanned to the end // and the last matching line's verdict remembered, rather than returning // on the first match: an earlier stacked mount carrying opt must not // outvote a later one that dropped it, or vice versa. func mountOptionOf(r io.Reader, mountpoint, opt string) (bool, error) { want := filepath.Clean(mountpoint) sc := bufio.NewScanner(r) verdict := false for sc.Scan() { line := sc.Text() mp, ok := mountinfoLineMountpoint(line) if !ok || mp != want { continue } verdict = mountinfoLineHas(line, want, opt) } return verdict, sc.Err() } // mountinfoLineMountpoint returns field 5 of a mountinfo line (the // mountpoint, octal-escaped), unescaped, and whether the line has enough // fields to have one at all. func mountinfoLineMountpoint(line string) (string, bool) { fields := strings.Fields(line) if len(fields) < 6 { return "", false } return unescapeMountinfo(fields[4]), true } // mountinfoLineHas parses one mountinfo line: field 5 is the mountpoint // (octal-escaped), field 6 the per-mount options. func mountinfoLineHas(line, mountpoint, option string) bool { fields := strings.Fields(line) if len(fields) < 6 { return false } if unescapeMountinfo(fields[4]) != mountpoint { return false } for _, o := range strings.Split(fields[5], ",") { if o == option { return true } } return false } // unescapeMountinfo undoes the \040-style escapes mountinfo applies to // spaces, tabs, newlines and backslashes in paths. func unescapeMountinfo(s string) string { if !strings.Contains(s, `\`) { return s } var b strings.Builder for i := 0; i < len(s); i++ { if s[i] == '\\' && i+3 < len(s) { var v int if _, err := fmt.Sscanf(s[i+1:i+4], "%o", &v); err == nil { b.WriteByte(byte(v)) i += 3 continue } } b.WriteByte(s[i]) } return b.String() }