//go:build linux package activator import ( "context" "encoding/json" "errors" "net" "os" "os/exec" "path/filepath" "slices" "strconv" "strings" "time" "github.com/carlosframework/platform/internal/cgroup" "github.com/carlosframework/platform/internal/hostbox" "github.com/carlosframework/platform/internal/hostclaim" "github.com/carlosframework/platform/internal/hostwrite" "github.com/carlosframework/platform/internal/instancectl" "github.com/carlosframework/platform/internal/instancefs" "golang.org/x/sys/unix" ) // startIsolated is Start through the reservation-bearing root protocol // (claim Plan 11, R10). The agent writes the role's launch file and opens // its log socket, exactly as main's nested start does, then asks root to // run the reservation. It never builds a process of its own: the command it // returns names the unit's main process root reported, and the settle it // returns is root's stop of that same reservation. func (l RoleLaunch) startIsolated(ctx context.Context, role string, attempt hostclaim.WakeRetryAttempt, build func() *exec.Cmd, before func() error) (*exec.Cmd, *cgroup.Prepared, func(context.Context) error, RoleProducer, error) { fail := func(err error) (*exec.Cmd, *cgroup.Prepared, func(context.Context) error, RoleProducer, error) { return nil, nil, nil, RoleProducer{}, err } if role != "main" { return fail(hostbox.ErrRoleLaunchShape) } if l.Work == nil || l.Executable == nil || build == nil || before == nil || l.MemoryMax <= 0 || !cleanAbs(l.LaunchDir) || !cleanAbs(l.RunDir) || !cleanAbs(l.LogSocket) || attempt.ID == "" || attempt.InvocationID == "" || attempt.LaunchInputs == "" || attempt.Session != l.Target.Session || attempt.Box != l.Target.Box || attempt.Enrollment != l.Target.Enrollment || attempt.BootID != l.Target.BootID || attempt.ControllerID != l.Target.ControllerID { return fail(hostwrite.ErrUnavailable) } if l.InitializeUnlocked != nil { if err := l.InitializeUnlocked(ctx); err != nil { return fail(err) } } host := l.Target.Host var cmd *exec.Cmd var executable *os.File defer func() { if executable != nil { executable.Close() } }() reservation := hostbox.LaunchReservation{ID: attempt.ID + ":" + role, Role: role, InvocationID: attempt.InvocationID, Inputs: attempt.LaunchInputs, Target: l.Target, Group: hostbox.LaunchGroup{RelativePath: instancectl.ClaimUnitControlGroup(host)}} if l.Unit != nil { if !instancectl.ValidBoundClaimUnit(l.Unit.Unit) { return fail(hostbox.ErrRoleLaunchShape) } reservation.Group.RelativePath = instancectl.BoundClaimUnitControlGroup(host, l.Unit.Unit) } err := l.Work.With(ctx, l.Identity, func() error { cmd = build() if cmd == nil { return hostwrite.ErrUnavailable } var argv []string var socket string var err error if l.Unit != nil { if l.Spec == nil || len(cmd.Args) == 0 || !slices.Equal(cmd.Args[1:], l.Unit.Args) || !slices.Equal(cmd.Env, l.Unit.Env) || !slices.Equal(l.Spec.Args, l.Unit.Args) || !slices.Equal(l.Spec.Env, l.Unit.Env) { return hostbox.ErrRoleLaunchShape } argv = append([]string{filepath.Join(instancectl.LiveDir, host)}, l.Unit.Args...) } else { argv, socket, err = isolatedArgv(cmd, host, l.LaunchDir, l.RunDir) if err != nil { return err } } if l.Initialize != nil { if err = l.Initialize(ctx); err != nil { return err } } if executable, err = l.Executable(ctx, cmd); err != nil { return err } if executable == nil { return hostwrite.ErrUnavailable } var stat unix.Stat_t if err = unix.Fstat(int(executable.Fd()), &stat); err != nil { return err } if stat.Mode&unix.S_IFMT != unix.S_IFREG || stat.Mode&0111 == 0 || !cleanAbs(executable.Name()) { return hostwrite.ErrUnavailable } reservation.Executable = hostbox.LaunchFile{Device: uint64(stat.Dev), Inode: stat.Ino} // The nested start's own preparation (isolated-exec design ยง6.1): the // run directory, no stale socket, and the launch file the trampoline // execs. The file grants nothing the tenant could not write itself. if err = os.MkdirAll(l.RunDir, 0o750|os.ModeSetgid); err != nil { return err } if socket != "" { if err = os.Remove(socket); err != nil && !errors.Is(err, os.ErrNotExist) { return err } } raw, err := json.MarshalIndent(instancectl.Launch{Schema: instancectl.LaunchSchema, Host: host, Argv: argv, Env: append([]string(nil), cmd.Env...)}, "", " ") if err != nil { return err } return instancefs.WriteAtomic(l.LaunchDir, instancectl.LaunchFileName(instancectl.RoleRun), append(raw, '\n'), 0o600) }) if err != nil { return fail(err) } request := hostbox.RoleLaunchRequest{Spec: l.Spec, V: hostbox.RoleLaunchV, Op: hostbox.RoleLaunchStart, Reservation: &reservation, Executable: executable.Name(), MemoryMax: l.MemoryMax} stopRequest := hostbox.RoleLaunchRequest{V: hostbox.RoleLaunchV, Op: hostbox.RoleLaunchStop, Reservation: &reservation} if err = request.Validate(); err != nil { return fail(err) } logs, err := listenLogs(l.LogSocket, nil) if err != nil { return fail(err) } var producer RoleProducer if l.RegisterProducer != nil { if producer, err = l.RegisterProducer(ctx, role); err != nil { logs.Close() return fail(err) } } finish := func(err error) { if producer.Finish != nil { producer.Finish(err) } } // This hook runs after strict preparation and immediately before Start. if err = before(); err != nil { logs.Close() finish(nil) return fail(err) } settle := func(ctx context.Context) error { if err := l.Root.Stop(ctx, stopRequest); err != nil { return err } finish(nil) return logs.Close() } grant, err := l.Root.Launch(ctx, request) if err == nil { err = grant.Check(reservation) } var process *os.Process if err == nil { process, err = pinRootProcess(grant.PID, grant.ProcessStart) } if err != nil { // A refused, unanswered or unusable launch is stopped by its own // reservation, which root then refuses ever to start. Only a stop // that also failed leaves the caller a settle to retry. cleanup, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second) stopErr := settle(cleanup) cancel() if stopErr != nil { return nil, nil, settle, RoleProducer{}, errors.Join(err, stopErr) } return fail(err) } cmd.Process = process return cmd, nil, settle, producer, nil } func cleanAbs(p string) bool { return filepath.IsAbs(p) && filepath.Clean(p) == p } // isolatedArgv is the trampoline's argv for cmd: the unit's view of the // host's live build, then exactly --socket in the run directory and --db in // the instance directory, each once. Any other argument is a shape the // isolated unit does not carry. func isolatedArgv(cmd *exec.Cmd, host, launchDir, runDir string) ([]string, string, error) { if len(cmd.Args) == 0 { return nil, "", hostbox.ErrRoleLaunchShape } argv, socket, _, err := IsolatedArgv(host, launchDir, runDir, cmd.Args[1:], cmd.Env) return argv, socket, err } // PeerInRole verifies that conn's peer runs inside the unit root started // for cmd: the journaled unit reservation names cmd's process, its cgroup // still has the journaled identity, and the peer is a member of exactly that // group. The tenant answering the socket is the trampoline's child, so its // pid is never cmd's. func (l RoleLaunch) PeerInRole(ctx context.Context, conn net.Conn, cmd *exec.Cmd) error { unixConn, ok := conn.(*net.UnixConn) if l.Root == nil || l.Journal == nil || !ok || cmd == nil || cmd.Process == nil || !filepath.IsAbs(l.CgroupRoot) { return hostwrite.ErrUnavailable } raw, err := unixConn.SyscallConn() if err != nil { return err } var peer *unix.Ucred var sockErr error if err = raw.Control(func(fd uintptr) { peer, sockErr = unix.GetsockoptUcred(int(fd), unix.SOL_SOCKET, unix.SO_PEERCRED) }); err != nil { return err } if sockErr != nil { return sockErr } if peer == nil || peer.Pid <= 0 { return hostwrite.ErrUnavailable } inventory, err := l.Journal.Read(ctx, l.Target) if err != nil { return err } for _, r := range inventory.Reservations { if r.State != hostbox.UnitStarted || r.PID != cmd.Process.Pid { continue } start, err := roleProcessStart(r.PID) if err != nil { return err } if start != r.ProcessStart { return hostbox.ErrChanged } var stat unix.Stat_t if err = unix.Stat(filepath.Join(l.CgroupRoot, r.Group.RelativePath), &stat); err != nil { return err } if uint64(stat.Dev) != r.Group.Device || stat.Ino != r.Group.Inode { return hostbox.ErrChanged } membership, err := os.ReadFile("/proc/" + strconv.Itoa(int(peer.Pid)) + "/cgroup") if err != nil { return err } for _, line := range strings.Split(string(membership), "\n") { if line == "0::/"+r.Group.RelativePath { return nil } } return hostbox.ErrChanged } return hostbox.ErrChanged }