//go:build linux package cgroup import ( "errors" "fmt" "os" "os/exec" "path/filepath" "strings" "syscall" "testing" "time" ) // These run without a delegated cgroup: a temp directory is not cgroupfs, // so they exercise the failure paths. The kernel file covers the rest. // TestShimRefusesANonCgroupDirectory: the parent's own check (step 2) // refuses before anything starts, as a placement error. func TestShimRefusesANonCgroupDirectory(t *testing.T) { m := &Manager{root: t.TempDir(), logf: t.Logf} marker := filepath.Join(t.TempDir(), "ran") cmd := exec.Command("/bin/sh", "-c", "touch "+marker) refused, err := m.startViaShim(m.root, cmd, nil) if refused || !isPlacementError(err) { t.Fatalf("refused=%v err=%v, want a placement error", refused, err) } if cmd.Process != nil { t.Fatal("a process was started") } if _, err := os.Stat(marker); err == nil { t.Fatal("tenant code ran") } } // TestShimRefusesAPathOutsideCgroupfs: a cgroup2 directory is not enough; // the path must name it under /sys/fs/cgroup/, or the shim could not check // where it landed. func TestShimRefusesAPathOutsideCgroupfs(t *testing.T) { if _, err := os.Stat("/sys/fs/cgroup/cgroup.controllers"); err != nil { t.Skip("no cgroup2 mount at /sys/fs/cgroup") } m := &Manager{root: "/sys/fs/cgroup", logf: t.Logf} cmd := exec.Command("/bin/true") _, err := m.startViaShim("/sys/fs/cgroup", cmd, nil) if !isPlacementError(err) || cmd.Process != nil { t.Fatalf("err=%v process=%v, want a placement error and no process", err, cmd.Process) } } // TestShimUnavailableWithoutTheHook: a process whose main never ran the // hook never starts a shim. func TestShimUnavailableWithoutTheHook(t *testing.T) { if !shimHooked.Load() { t.Fatal("TestMain did not run the hook") } shimHooked.Store(false) t.Cleanup(func() { shimHooked.Store(true) }) m := &Manager{root: t.TempDir(), logf: t.Logf} cmd := exec.Command("/bin/true") refused, err := m.startViaShim(m.root, cmd, nil) if refused || !isPlacementError(err) { t.Fatalf("refused=%v err=%v, want a placement error", refused, err) } if cmd.Process != nil { t.Fatal("a process was started") } } // TestShimLookPathErrorIsTheCommands: a command whose LookPath failed // answers that error, as exec.Cmd.Start would, and starts nothing. func TestShimLookPathErrorIsTheCommands(t *testing.T) { m := &Manager{root: t.TempDir(), logf: t.Logf} cmd := exec.Command("carlos-no-such-command-anywhere") if cmd.Err == nil { t.Skip("LookPath found the command") } _, err := m.startViaShim(m.root, cmd, nil) if err != cmd.Err || isPlacementError(err) || cmd.Process != nil { t.Fatalf("err=%v, want the LookPath error %v and no process", err, cmd.Err) } } // TestShimPlacementFailureIsReported runs a real shim (this test binary, // through TestMain's hook) against a temp directory, calling the core with // a directory fd the prologue would have refused. The shim's own fstatfs // check (shim step 4) answers "P". func TestShimPlacementFailureIsReported(t *testing.T) { m := &Manager{root: t.TempDir(), logf: t.Logf} f, err := os.Open(m.root) if err != nil { t.Fatal(err) } defer f.Close() marker := filepath.Join(t.TempDir(), "ran") cmd := exec.Command("/bin/sh", "-c", "touch "+marker) _, err = m.startShimCore(m.root, f, "/nowhere", cmd, nil, time.Time{}) if !isPlacementError(err) { t.Fatalf("err=%v, want a placement error", err) } if !strings.Contains(err.Error(), "statfs") { t.Fatalf("err=%v, want the shim's statfs refusal", err) } if _, err := os.Stat(fmt.Sprintf("/proc/%d", cmd.Process.Pid)); err == nil { t.Fatal("the shim was not reaped") } if _, err := os.Stat(marker); err == nil { t.Fatal("tenant code ran") } if m.placementsUnder(m.root) != 0 { t.Fatal("the placement is still registered") } } // TestShimBeforeRefusalRegistersNothing: before is asked only once the // process is placed, so a start that cannot be placed never asks it, starts // nothing of the tenant, and leaves nothing registered. A refusal after a // real placement is TestShimKernelBeforeRunsAfterPlacement. func TestShimBeforeRefusalRegistersNothing(t *testing.T) { m := &Manager{root: t.TempDir(), logf: t.Logf} asked := 0 before := func() error { asked++; return errors.New("not yours any more") } marker := filepath.Join(t.TempDir(), "ran") cmd := exec.Command("/bin/sh", "-c", "touch "+marker) refused, err := m.startViaShim(m.root, cmd, before) if refused || !isPlacementError(err) || cmd.Process != nil { t.Fatalf("prologue: refused=%v err=%v process=%v", refused, err, cmd.Process) } f, err := os.Open(m.root) if err != nil { t.Fatal(err) } defer f.Close() cmd = exec.Command("/bin/sh", "-c", "touch "+marker) refused, err = m.startShimCore(m.root, f, "/nowhere", cmd, before, time.Time{}) if refused || !isPlacementError(err) { t.Fatalf("core: refused=%v err=%v, want a placement error", refused, err) } if asked != 0 { t.Fatalf("before was asked %d times for a process that was never placed", asked) } if _, err := os.Stat(marker); err == nil { t.Fatal("tenant code ran") } if m.placementsUnder(m.root) != 0 { t.Fatal("the placement is still registered") } } // TestShimRefusesNULInEnv: Start refuses an environment entry that holds a // NUL (Go issue #56284), where cmd.Environ() would silently drop it. The // core answers Start's own error, not a placement error, and starts and // registers nothing. With Env nil the NUL can arrive only through Dir, as // the PWD Start adds. func TestShimRefusesNULInEnv(t *testing.T) { const want = "exec: environment variable contains NUL" for _, c := range []struct { name string env []string dir string }{ {"env entry", []string{"A=1", "BAD=x\x00y", "B=2"}, ""}, {"env key", []string{"B\x00AD=x"}, ""}, {"PWD from dir", nil, "/no\x00where"}, } { t.Run(c.name, func(t *testing.T) { marker := filepath.Join(t.TempDir(), "ran") newCmd := func() *exec.Cmd { cmd := exec.Command("/bin/sh", "-c", "touch "+marker) cmd.Env = c.env cmd.Dir = c.dir return cmd } // The control: a direct start answers exactly this. direct := newCmd() if err := direct.Start(); err == nil || err.Error() != want { t.Fatalf("direct Start: err=%v, want %q", err, want) } m := &Manager{root: t.TempDir(), logf: t.Logf} f, err := os.Open(m.root) if err != nil { t.Fatal(err) } defer f.Close() cmd := newCmd() refused, err := m.startShimCore(m.root, f, "/nowhere", cmd, nil, time.Time{}) if refused || err == nil || err.Error() != want || isPlacementError(err) { t.Fatalf("refused=%v err=%v, want the command's error %q", refused, err, want) } if cmd.Process != nil { t.Fatal("a process was started") } if cmd.Path != "/bin/sh" { t.Fatalf("cmd was rewritten to %s", cmd.Path) } if _, err := os.Stat(marker); err == nil { t.Fatal("tenant code ran") } if m.placementsUnder(m.root) != 0 { t.Fatal("a placement was registered") } }) } } func TestShimStatusClassification(t *testing.T) { for _, c := range []struct { in string timedOut bool placed bool }{ {"A\n", false, true}, {"P procs: permission denied\n", false, false}, {"", false, false}, // died before placement {"", true, false}, // deadline before placement {"garbage\n", false, false}, {"A", false, false}, // no newline: not the shim's answer } { placed, err := classifyPlacement([]byte(c.in), c.timedOut, shimDeadline) if placed != c.placed { t.Fatalf("classifyPlacement(%q, %v) placed=%v, want %v", c.in, c.timedOut, placed, c.placed) } if !placed && !isPlacementError(err) { t.Fatalf("classifyPlacement(%q, %v) err=%v, want a placement error", c.in, c.timedOut, err) } if placed && err != nil { t.Fatalf("classifyPlacement(%q, %v) err=%v", c.in, c.timedOut, err) } } if _, err := classifyPlacement([]byte("P procs: permission denied\n"), false, shimDeadline); !strings.Contains(err.Error(), "procs: permission denied") { t.Fatalf("the shim's reason is lost: %v", err) } if _, err := classifyPlacement(nil, true, shimDeadline); !strings.Contains(err.Error(), "no answer") { t.Fatalf("the deadline is not named: %v", err) } if _, err := classifyPlacement(nil, false, shimDeadline); !strings.Contains(err.Error(), "exited before placement") { t.Fatalf("the early exit is not named: %v", err) } for _, c := range []struct { in string killed bool started bool op string // "" for a placement error, else the *os.PathError's Op errno syscall.Errno }{ {"R\n", false, true, "", 0}, {"R\n", true, true, "", 0}, // R then killed: may have exec'd, so started {"", false, false, "", 0}, // died after release, before R: placement error {"", true, false, "", 0}, // killed before R: placement error {"E chdir 2\n", false, false, "chdir", syscall.ENOENT}, {"R\nE exec 2\n", false, false, "fork/exec", syscall.ENOENT}, {"R\nE exec 7\n", false, false, "fork/exec", syscall.E2BIG}, {"garbage\n", false, false, "", 0}, {"R\ngarbage\n", false, true, "", 0}, // R and no E after it: started {"R", true, true, "", 0}, // R without its newline still counts } { started, err := classifyExec("/bin/app", "/wd", []byte(c.in), c.killed, shimDeadline) if started != c.started { t.Fatalf("classifyExec(%q, %v) started=%v, want %v", c.in, c.killed, started, c.started) } if started { if err != nil { t.Fatalf("classifyExec(%q, %v) err=%v", c.in, c.killed, err) } continue } if c.op == "" { if !isPlacementError(err) { t.Fatalf("classifyExec(%q, %v) err=%v, want a placement error", c.in, c.killed, err) } continue } var pe *os.PathError if !errors.As(err, &pe) || isPlacementError(err) { t.Fatalf("classifyExec(%q, %v) err=%v, want a *os.PathError", c.in, c.killed, err) } wantPath := "/bin/app" if c.op == "chdir" { wantPath = "/wd" } if pe.Op != c.op || pe.Path != wantPath || pe.Err != c.errno { t.Fatalf("classifyExec(%q, %v) = %#v, want Op %q Path %q Err %v", c.in, c.killed, pe, c.op, wantPath, c.errno) } } if _, err := classifyExec("/bin/app", "/wd", nil, false, shimDeadline); !strings.Contains(err.Error(), "exited before exec") { t.Fatalf("the early exit is not named: %v", err) } } // TestShimWaitCapsAtBy: with by unset or later than shimDeadline from now, // each wait is shimDeadline, as for every tenant start (Ruling 9); with by // sooner, the wait ends at by and the error names what was left of it. func TestShimWaitCapsAtBy(t *testing.T) { before := time.Now() d, within := shimWait(time.Time{}) if within != shimDeadline || d.Before(before.Add(shimDeadline)) || d.After(time.Now().Add(shimDeadline)) { t.Fatalf("no cap: deadline %v within %v, want shimDeadline from now", d.Sub(before), within) } late := time.Now().Add(shimDeadline + time.Hour) if d, within := shimWait(late); within != shimDeadline || !d.Before(late) { t.Fatalf("a later by: deadline %v within %v, want shimDeadline from now", d, within) } soon := time.Now().Add(time.Second) if d, within := shimWait(soon); !d.Equal(soon) || within <= 0 || within > time.Second { t.Fatalf("a sooner by: deadline %v within %v, want by and at most 1s", d, within) } past := time.Now().Add(-time.Second) if d, within := shimWait(past); !d.Equal(past) || within != 0 { t.Fatalf("a past by: deadline %v within %v, want by and 0", d, within) } if _, err := classifyPlacement(nil, true, 1500*time.Millisecond); !strings.Contains(err.Error(), "within 1.5s") { t.Fatalf("the capped wait is not named: %v", err) } if _, err := classifyExec("/bin/app", "/wd", nil, true, 1500*time.Millisecond); !strings.Contains(err.Error(), "within 1.5s") { t.Fatalf("the capped wait is not named: %v", err) } }