package instancectl import ( "context" "errors" "fmt" "os" "path" "path/filepath" "strconv" "strings" "time" "github.com/carlosframework/platform/internal/cgroup" "github.com/carlosframework/platform/internal/provision" ) // ClaimUnits runs a reserved main role through main's isolated run unit for // the claim root launch protocol (claim Plan 11, R10). It is the same // template, slice, tier and build view /start uses; what differs is that the // unit's /opt/carlos/app source is the exact build root verified, not the // host's live link, and that it refuses rather than joins a unit that is not // idle. Root calls it only after validating the reservation under the host // guard; it adds no request surface of its own. type ClaimUnits struct { BoundHost, BoundUnit string BoundAddr string Systemctl func(context.Context, ...string) (string, error) CgroupRoot string ExeRoot string UnitRoot string DataRoot string // Room is the disk floor check; nil skips it, as a nil Statfs does. Room func() error WaitBudget time.Duration Poll time.Duration } // ClaimUnit is one start: the host, the verified build and the slice tier. type ClaimUnit struct { Host string Executable string MemoryMax int64 } // ClaimUnitStarted is what systemd reported after the start. ControlGroup is // relative to the cgroup root. type ClaimUnitStarted struct { Unit, InvocationID, ControlGroup string MainPID int } var ( // ErrClaimUnitBusy refuses a start while the host's run unit is active, // changing state, or its cgroup still holds a process. ErrClaimUnitBusy = errors.New("instance control: the claim unit is not idle") // ErrClaimUnit refuses a start whose request or outcome it cannot vouch for. ErrClaimUnit = errors.New("instance control: the claim unit did not start as requested") ) // ClaimUnitControlGroup is the host's run unit cgroup, relative to the // cgroup root. func ClaimUnitControlGroup(host string) string { return path.Join(parentSlices, SliceName(host), UnitName(RoleRun, host)) } func claimTier(n int64) (cgroup.Limit, bool) { switch n { case cgroup.TierDefault.MemoryMax, cgroup.TierChildren.MemoryMax: return cgroup.Limit{MemoryMax: n}, true } return cgroup.Limit{}, false } func (u ClaimUnits) budget() time.Duration { if u.WaitBudget > 0 { return u.WaitBudget } return 30 * time.Second } func (u ClaimUnits) poll() time.Duration { if u.Poll > 0 { return u.Poll } return 50 * time.Millisecond } func (u ClaimUnits) show(ctx context.Context, unit string, props ...string) (map[string]string, error) { out, err := u.Systemctl(ctx, "show", "-p", strings.Join(props, ","), unit) if err != nil { return nil, fmt.Errorf("show %s: %w", unit, err) } return parseShow(out), nil } // cgroupEmpty reads the unit's cgroup.events: an absent group is empty. func (u ClaimUnits) cgroupEmpty(host string) error { _, group, err := u.Placement(host) if err != nil { return err } events := filepath.Join(u.CgroupRoot, group, "cgroup.events") b, err := os.ReadFile(events) if errors.Is(err, os.ErrNotExist) { return nil } if err != nil { return err } if strings.Contains(string(b), "populated 0") { return nil } return ErrClaimUnitBusy } func idleState(state string) bool { return state == "inactive" || state == "failed" } // Start starts host's run unit on the verified build. It refuses unless the // unit is idle, and stops any invocation it then cannot vouch for: one that // systemd did not newly create, or that runs outside the host's run cgroup. func (u ClaimUnits) Start(ctx context.Context, in ClaimUnit) (ClaimUnitStarted, error) { return u.start(ctx, in, nil, nil, nil) } // StartChecked runs the root check after all preparation, immediately before // systemctl start. It binds root's verified input snapshot over the mutable // tenant launch file. The ordinary Start path keeps its existing inputs. func (u ClaimUnits) StartChecked(ctx context.Context, in ClaimUnit, launch Launch, check func() error) (ClaimUnitStarted, error) { if check == nil { return ClaimUnitStarted{}, ErrClaimUnit } return u.start(ctx, in, &launch, check, nil) } func (u ClaimUnits) start(ctx context.Context, in ClaimUnit, launch *Launch, check func() error, permit *ClaimExecPermit) (ClaimUnitStarted, error) { tier, ok := claimTier(in.MemoryMax) if u.Systemctl == nil || u.CgroupRoot == "" || u.ExeRoot == "" || !ok || !provision.ValidHostShape(in.Host) || !filepath.IsAbs(in.Executable) || filepath.Clean(in.Executable) != in.Executable || in.Executable == SystemBinary || strings.HasPrefix(in.Executable, LiveDir+"/") { return ClaimUnitStarted{}, ErrClaimUnit } unit, group, err := u.Placement(in.Host) if err != nil { return ClaimUnitStarted{}, err } if u.BoundUnit != "" && (permit == nil || launch == nil || check == nil) { return ClaimUnitStarted{}, ErrClaimUnit } before, err := u.show(ctx, unit, "ActiveState", "InvocationID") if err != nil { return ClaimUnitStarted{}, err } if !idleState(before["ActiveState"]) { return ClaimUnitStarted{}, ErrClaimUnitBusy } if err := u.cgroupEmpty(in.Host); err != nil { return ClaimUnitStarted{}, err } if u.Room != nil { if err := u.Room(); err != nil { return ClaimUnitStarted{}, err } } if err := ensureExe(u.ExeRoot, provision.InstanceID(in.Host), in.Executable, in.Host); err != nil { return ClaimUnitStarted{}, err } if err := sliceTier(ctx, u.Systemctl, in.Host, tier); err != nil { return ClaimUnitStarted{}, err } var invocation string refuse := func(cause error) (ClaimUnitStarted, error) { cleanup, cancel := context.WithTimeout(context.WithoutCancel(ctx), u.budget()) defer cancel() if u.BoundUnit != "" { return ClaimUnitStarted{}, errors.Join(cause, u.StopInvocation(cleanup, in.Host, invocation)) } return ClaimUnitStarted{}, errors.Join(cause, u.Stop(cleanup, in.Host)) } if launch != nil { if err := u.freezeLaunch(ctx, in.Host, *launch); err != nil { return ClaimUnitStarted{}, err } } var gate *claimExecGate if permit != nil { gate, err = u.prepareExecGate(ctx, in.Host, *permit) if err != nil { return ClaimUnitStarted{}, errors.Join(err, u.clearLaunch(ctx, in.Host)) } defer gate.close() } if check != nil { if err := check(); err != nil { return ClaimUnitStarted{}, errors.Join(err, u.clearLaunch(ctx, in.Host)) } } if out, err := u.Systemctl(ctx, "start", unit); err != nil { return refuse(fmt.Errorf("start %s: %s: %w", unit, strings.TrimSpace(out), err)) } ready, cancel := context.WithTimeout(ctx, u.budget()) defer cancel() for { after, err := u.show(ready, unit, "InvocationID", "MainPID", "ControlGroup", "ActiveState") if err != nil { return refuse(err) } id := after["InvocationID"] if idleState(after["ActiveState"]) || (id != "" && id == before["InvocationID"]) || (invocation != "" && id != invocation) { return refuse(ErrClaimUnit) } if id != "" { invocation = id } pid, _ := strconv.Atoi(after["MainPID"]) if id != "" && pid > 0 && after["ActiveState"] == "active" { if after["ControlGroup"] != "/"+group { return refuse(ErrClaimUnit) } started := ClaimUnitStarted{Unit: unit, InvocationID: id, ControlGroup: group, MainPID: pid} if gate != nil { if err := gate.authorize(ready, u, &started, check); err != nil { cleanup, cancel := context.WithTimeout(context.WithoutCancel(ctx), u.budget()) defer cancel() if sameErr := u.Same(cleanup, started); sameErr != nil { return ClaimUnitStarted{}, errors.Join(err, u.Gone(cleanup, in.Host, started.InvocationID)) } return refuse(err) } } return started, nil } select { case <-ready.Done(): return refuse(ready.Err()) case <-time.After(u.poll()): } } } // Stop stops host's run unit and returns once systemd reports it inactive or // failed and its cgroup is empty, bounded by ctx and WaitBudget. func (u ClaimUnits) Stop(ctx context.Context, host string) error { if u.Systemctl == nil || u.CgroupRoot == "" || !provision.ValidHostShape(host) { return ErrClaimUnit } unit, _, err := u.Placement(host) if err != nil { return err } if out, err := u.Systemctl(ctx, "stop", unit); err != nil { return fmt.Errorf("stop %s: %s: %w", unit, strings.TrimSpace(out), err) } ctx, cancel := context.WithTimeout(ctx, u.budget()) defer cancel() for { state, err := u.show(ctx, unit, "ActiveState") if err == nil && idleState(state["ActiveState"]) { break } select { case <-ctx.Done(): return fmt.Errorf("%s: still %q: %w", unit, state["ActiveState"], ErrClaimUnitBusy) case <-time.After(u.poll()): } } if err := WaitCgroupEmpty(ctx, u.CgroupRoot, SliceName(host), unit, u.poll()); err != nil { return err } return u.clearLaunch(ctx, host) } // Gone proves an invocation of host's run unit has ended: systemd reports a // different invocation, or the unit idle with an empty cgroup. An empty // invocation asks only the second question. func (u ClaimUnits) Gone(ctx context.Context, host, invocation string) error { if u.Systemctl == nil || u.CgroupRoot == "" || !provision.ValidHostShape(host) { return ErrClaimUnit } unit, _, err := u.Placement(host) if err != nil { return err } state, err := u.show(ctx, unit, "ActiveState", "InvocationID") if err != nil { return err } if u.BoundUnit == "" && invocation != "" && state["InvocationID"] != "" && state["InvocationID"] != invocation { return nil } if !idleState(state["ActiveState"]) { return ErrClaimUnitBusy } if err := u.cgroupEmpty(host); err != nil { return err } return u.clearLaunch(ctx, host) } // DiskRoom is the start floor /start applies (#214 item 1), for the claim // launch: statfs of dataRoot, refused below DiskFloor. func DiskRoom(dataRoot string) func() error { return func() error { avail, total, err := statfsBytes(dataRoot) if err != nil { return fmt.Errorf("statfs %s: %w", dataRoot, err) } if floor := DiskFloor(total); avail < floor { return fmt.Errorf("%w: %s free, below the %s floor", ErrDiskFull, humanBytes(avail), humanBytes(floor)) } return nil } } // Same refuses replacement invocations and PID changes throughout readiness. func (u ClaimUnits) Same(ctx context.Context, s ClaimUnitStarted) error { if u.Systemctl == nil || s.Unit == "" || s.InvocationID == "" || s.MainPID <= 0 { return ErrClaimUnit } if u.BoundUnit != "" { unit, group, err := u.Placement(u.BoundHost) if err != nil || s.Unit != unit || s.ControlGroup != group { return ErrClaimUnit } } state, err := u.show(ctx, s.Unit, "InvocationID", "MainPID", "ControlGroup", "ActiveState") if err != nil { return err } if state["InvocationID"] != s.InvocationID || state["MainPID"] != strconv.Itoa(s.MainPID) || state["ControlGroup"] != "/"+s.ControlGroup || state["ActiveState"] != "active" { return ErrClaimUnit } return nil }