//go:build linux package stopevidence import ( "bufio" "context" "errors" "fmt" "io/fs" "os" "os/exec" "path/filepath" "reflect" "strconv" "strings" "syscall" "testing" "time" ) // heldAndQuiet ends every holder's script: the shell says it has finished // starting, then blocks in its own read on a pipe nothing is written to. // // A scan must not meet a holder that is still starting. A process that has // just been exec'd opens and closes the linker cache, libc and the locale // files in turn, each on the first free descriptor. A scan that reads one of // them between its close and the next open sees a descriptor whose stat says // gone and whose entry is there, which is ErrIncomplete by design // (TestFDHoldersStatGoneWhileTheEntryRemains). The holders here used to be // `sleep 60`, scanned as soon as it was started, and the hard-link test // failed that way once under load (#274). const heldAndQuiet = "echo ready && read wait" // startHolder starts cmd, a shell whose script ends in heldAndQuiet, and // returns its pid once it has said ready. An error means it never did. func startHolder(t *testing.T, cmd *exec.Cmd) (int, error) { t.Helper() // Wait closes this end, and until then the child's read has nothing. if _, err := cmd.StdinPipe(); err != nil { t.Fatal(err) } r, w, err := os.Pipe() if err != nil { t.Fatal(err) } defer r.Close() cmd.Stdout = w err = cmd.Start() w.Close() if err != nil { return 0, err } t.Cleanup(func() { cmd.Process.Kill(); cmd.Wait() }) r.SetReadDeadline(time.Now().Add(10 * time.Second)) if line, err := bufio.NewReader(r).ReadString('\n'); line != "ready\n" { return 0, fmt.Errorf("the holder said %q before it stopped: %v", line, err) } return cmd.Process.Pid, nil } // holdOpen starts a child that holds path open as descriptor 3 and opens // nothing more. func holdOpen(t *testing.T, path string) int { t.Helper() f, err := os.Open(path) if err != nil { t.Fatal(err) } defer f.Close() cmd := exec.Command("sh", "-c", heldAndQuiet) cmd.ExtraFiles = []*os.File{f} pid, err := startHolder(t, cmd) if err != nil { t.Fatal(err) } return pid } // projectPIDs is a scan root holding only these processes' real /proc // entries, so the scan needs no access to other uids. func projectPIDs(t *testing.T, pids ...int) string { t.Helper() root := t.TempDir() for _, pid := range pids { p := strconv.Itoa(pid) if err := os.Symlink(filepath.Join("/proc", p), filepath.Join(root, p)); err != nil { t.Fatal(err) } } return root } func newDB(t *testing.T, dir string) string { t.Helper() if err := os.MkdirAll(dir, 0o755); err != nil { t.Fatal(err) } db := filepath.Join(dir, "h.example.com.db") if err := os.WriteFile(db, []byte("SQLite format 3\x00"), 0o644); err != nil { t.Fatal(err) } return db } func dbFilesOf(t *testing.T, db string) []string { t.Helper() files, err := dbPathsWithSidecars([]string{db}) if err != nil { t.Fatal(err) } return files } // Regression tests for FDHolders' identity matching (main 8e9ac1b8), the // scan Reconstruct's step 3 uses. Each holder is a real child process. // A hard link is the same file under another name: a holder that opened // the link holds the database. func TestFDIdentityFindsAHardLinkHeldByAChild(t *testing.T) { dir := t.TempDir() db := newDB(t, filepath.Join(dir, "data")) alias := filepath.Join(dir, "elsewhere", "alias.db") os.MkdirAll(filepath.Dir(alias), 0o755) if err := os.Link(db, alias); err != nil { t.Skip("no hard links here:", err) } pid := holdOpen(t, alias) root := projectPIDs(t, pid) got, err := FDHolders(root, dbFilesOf(t, db)) if err != nil || !reflect.DeepEqual(got, []int{pid}) { t.Fatalf("identity scan: %v %v, want [%d]", got, err, pid) } } // The database path the caller passes may run through a symlink (a data // directory linked elsewhere) while the holder's descriptor names the real // path. func TestFDIdentityFindsAHolderThroughASymlinkedDBPath(t *testing.T) { dir := t.TempDir() db := newDB(t, filepath.Join(dir, "real")) link := filepath.Join(dir, "data") if err := os.Symlink(filepath.Join(dir, "real"), link); err != nil { t.Fatal(err) } pid := holdOpen(t, db) root := projectPIDs(t, pid) viaLink := filepath.Join(link, filepath.Base(db)) got, err := FDHolders(root, dbFilesOf(t, viaLink)) if err != nil || !reflect.DeepEqual(got, []int{pid}) { t.Fatalf("identity scan through the symlinked path: %v %v, want [%d]", got, err, pid) } } // A holder in another mount namespace sees the database through a bind // mount at a different path; its descriptor's link text is that path. An // unprivileged user and mount namespace builds it; where the box forbids // that, the test skips. func TestFDIdentityFindsAHolderInAnotherMountNamespace(t *testing.T) { if _, err := exec.LookPath("unshare"); err != nil { t.Skip("no unshare:", err) } dir := t.TempDir() db := newDB(t, filepath.Join(dir, "real")) alias := filepath.Join(dir, "alias") if err := os.MkdirAll(alias, 0o755); err != nil { t.Fatal(err) } aliased := filepath.Join(alias, filepath.Base(db)) // The shell opens the database on descriptor 3 itself, after the mount, // and says ready only then: until that point descriptor 3 can be some // other file (the dynamic loader's libc, while unshare or sh starts). cmd := exec.Command("unshare", "--user", "--map-root-user", "--mount", "sh", "-c", `mount --bind "$1" "$2" && exec 3<"$3" && `+heldAndQuiet, "sh", filepath.Join(dir, "real"), alias, aliased) pid, err := startHolder(t, cmd) if err != nil { t.Skip("the namespace holder never came up holding the database (unprivileged user namespaces refused?):", err) } if target, _ := os.Readlink(filepath.Join("/proc", strconv.Itoa(pid), "fd", "3")); target != aliased { t.Logf("descriptor text %q (expected the bind path %q)", target, aliased) } root := projectPIDs(t, pid) got, err := FDHolders(root, dbFilesOf(t, db)) if err != nil || !reflect.DeepEqual(got, []int{pid}) { t.Fatalf("identity scan across a mount namespace: %v %v, want [%d]", got, err, pid) } } // A writer still holding a database that was deleted and replaced holds a // file that is no longer at the path; its link text names the path with // " (deleted)", and that is a hold, never a pass. func TestFDIdentityDeletedAndReplacedDatabaseIsAHold(t *testing.T) { db := newDB(t, t.TempDir()) pid := holdOpen(t, db) if err := os.Remove(db); err != nil { t.Fatal(err) } newDB(t, filepath.Dir(db)) root := projectPIDs(t, pid) got, err := FDHolders(root, dbFilesOf(t, db)) if err != nil || !reflect.DeepEqual(got, []int{pid}) { t.Fatalf("deleted and replaced: %v %v, want [%d]", got, err, pid) } } // A process holding something else is not a holder, and a database with // no WAL or SHM on disk is checked by the files that exist. func TestFDIdentityOtherFilesAreNotHolders(t *testing.T) { dir := t.TempDir() db := newDB(t, dir) other := filepath.Join(dir, "other.db") os.WriteFile(other, []byte("x"), 0o644) pid := holdOpen(t, other) got, err := FDHolders(projectPIDs(t, pid), dbFilesOf(t, db)) if err != nil || len(got) != 0 { t.Fatalf("other file: %v %v", got, err) } } type fakeInfo struct{ fs.FileInfo } func (fakeInfo) Sys() any { return nil } // aliasFaultFixture is a proc root with one descriptor on an unrelated // real file, and the database's files. func aliasFaultFixture(t *testing.T) (root, fdLink, db string, files []string) { t.Helper() dir := t.TempDir() db = newDB(t, filepath.Join(dir, "data")) other := filepath.Join(dir, "other") os.WriteFile(other, []byte("x"), 0o644) root = filepath.Join(dir, "proc") fdDir := filepath.Join(root, "20", "task", "20", "fd") os.MkdirAll(fdDir, 0o755) fdLink = filepath.Join(fdDir, "0") if err := os.Symlink(other, fdLink); err != nil { t.Fatal(err) } return root, fdLink, db, dbFilesOf(t, db) } // Every way the scan could fail to see a descriptor is an error, never a // pass. func TestFDIdentityFailsClosed(t *testing.T) { root, fdLink, db, files := aliasFaultFixture(t) real := realFDOps() for _, c := range []struct { name string mutate func(*fdOps) wantErr bool }{ {"fd stat refused", func(o *fdOps) { o.stat = func(p string) (os.FileInfo, error) { if p == fdLink { return nil, syscall.EACCES } return real.stat(p) } }, true}, {"fd closed between the reads", func(o *fdOps) { // The descriptor is gone: both the stat through the link and the // entry itself answer ENOENT. o.stat = func(p string) (os.FileInfo, error) { if p == fdLink { return nil, syscall.ENOENT } return real.stat(p) } o.lstat = func(p string) (os.FileInfo, error) { if p == fdLink { return nil, syscall.ENOENT } return real.lstat(p) } }, false}, {"fd link refused", func(o *fdOps) { o.readlink = func(string) (string, error) { return "", syscall.EACCES } }, true}, {"fd closed before the link read", func(o *fdOps) { o.readlink = func(string) (string, error) { return "", syscall.ENOENT } }, false}, {"fd with no identity", func(o *fdOps) { o.stat = func(p string) (os.FileInfo, error) { fi, err := real.stat(p) if p == fdLink && err == nil { return fakeInfo{fi}, nil } return fi, err } }, true}, {"database stat refused", func(o *fdOps) { o.stat = func(p string) (os.FileInfo, error) { if p == db { return nil, syscall.EACCES } return real.stat(p) } }, true}, {"database with no identity", func(o *fdOps) { o.stat = func(p string) (os.FileInfo, error) { fi, err := real.stat(p) if p == db && err == nil { return fakeInfo{fi}, nil } return fi, err } }, true}, } { ops := realFDOps() c.mutate(&ops) got, err := fdHolders(context.Background(), root, files, ops) if c.wantErr { if !errors.Is(err, ErrIncomplete) || got != nil { t.Fatalf("%s: %v %v, want ErrIncomplete", c.name, got, err) } continue } if err != nil || len(got) != 0 { t.Fatalf("%s: %v %v, want no holder", c.name, got, err) } } } // A descriptor whose stat through /proc answers ENOENT while its entry // remains is unreadable, not closed: main's FDHolders lstats the entry after // such a stat (9cf9f18f) and fails the scan with ErrIncomplete. func TestFDHoldersStatGoneWhileTheEntryRemains(t *testing.T) { root, fdLink, _, files := aliasFaultFixture(t) ops := realFDOps() real := ops.stat ops.stat = func(p string) (os.FileInfo, error) { if p == fdLink { return nil, syscall.ENOENT } return real(p) } got, err := fdHolders(context.Background(), root, files, ops) if !errors.Is(err, ErrIncomplete) || got != nil { t.Fatalf("%v %v, want ErrIncomplete", got, err) } } // Descriptors that are not paths (sockets, pipes, anonymous inodes) are // never followed: they cannot be the database, and a stat could block. func TestFDIdentitySkipsNonPathDescriptors(t *testing.T) { db := newDB(t, t.TempDir()) root := t.TempDir() fdDir := filepath.Join(root, "20", "task", "20", "fd") os.MkdirAll(fdDir, 0o755) for n, target := range []string{"socket:[123]", "pipe:[9]", "anon_inode:[eventfd]"} { os.Symlink(target, filepath.Join(fdDir, strconv.Itoa(n))) } ops := realFDOps() real := ops.stat ops.stat = func(p string) (os.FileInfo, error) { if strings.HasPrefix(p, root) && !strings.HasSuffix(p, "/20") && !strings.HasSuffix(p, "/task/20") { t.Fatalf("stat of a non-path descriptor %s", p) } return real(p) } if got, err := fdHolders(context.Background(), root, dbFilesOf(t, db), ops); err != nil || len(got) != 0 { t.Fatalf("%v %v", got, err) } } func TestFDIdentityInvalidInput(t *testing.T) { db := newDB(t, t.TempDir()) ctx := context.Background() for name, c := range map[string]struct { ctx context.Context root string paths []string }{ "nil context": {nil, "/proc", []string{db}}, "relative root": {ctx, "proc", []string{db}}, "no paths": {ctx, "/proc", nil}, "relative path": {ctx, "/proc", []string{"h.db"}}, } { if _, err := fdHolders(c.ctx, c.root, c.paths, realFDOps()); !errors.Is(err, ErrInvalidInput) { t.Fatalf("%s: %v", name, err) } } }