package instancectl import ( "context" "errors" "fmt" "io/fs" "net/http" "os" "path/filepath" "strconv" "strings" "syscall" "time" "github.com/carlosframework/platform/internal/cgroup" "github.com/carlosframework/platform/internal/provision" ) // Generation units (zero-downtime deploys spec §7.6, plan 2 D1). The // template carlos-instance-gen@:.service cannot key its slice and // data mounts by itself: systemd has no specifier for part of an instance // string. So before each start this service writes a runtime drop-in with // everything the template leaves out, checks that systemd took it // (reloading first only when it has not), and only then starts the unit. // ErrGenUnproven is StartGen's answer when the start failed after systemd // may have started the unit and the stop that followed did not prove the // unit's cgroup empty. The generation may still be running: its files are // kept, and the caller must not release anything it owns for it. var ErrGenUnproven = errors.New("instance control: generation termination unproven") // statusGenUnproven is the status that carries ErrGenUnproven. No other // verb answers it, so the client can tell it from an ordinary 502. const statusGenUnproven = http.StatusInternalServerError // DropInName is the drop-in's file name inside .d. const DropInName = "50-carlos-generation.conf" // GenDropIn is the drop-in's text. The paths are the templates' literal // ones (/data/instances, /run/carlos/instances, /run/carlos-control/exe). // The memory lines are the generation's own tier (spec §7.3). The // template's OOMPolicy=kill gives the unit's cgroup memory.oom.group=1. func GenDropIn(host string, n uint64, tier cgroup.Limit) string { key := GenKey(host, n) return genDropInHeader(host) + "BindReadOnlyPaths=" + exeRoot + key + "/app:/opt/carlos/app " + exeRoot + key + "/live:/opt/carlos/live\n" + "StandardOutput=file:" + GenLogSocket(host, n) + "\n" + tierLines(tier) } // exeRoot is the control service's exe root as the templates and drop-ins // name it, literally. const exeRoot = "/run/carlos-control/exe/" // genDropInHeader is the part of a generation drop-in that follows from the // host alone: its slice and the instance's own two directories. The binary // and rootfs generation drop-ins share it, so the two cannot drift. func genDropInHeader(host string) string { return "# Written by carlos instance-control for one generation; removed when it stops (zero-downtime plan 2).\n" + "[Service]\n" + hostMountLines(host) } // hostMountLines are the Slice=, BindPaths=, ReadWritePaths= and // WorkingDirectory= lines every per-start drop-in carries for host. func hostMountLines(host string) string { iid := provision.InstanceID(host) return "Slice=" + SliceName(host) + "\n" + "BindPaths=/data/instances/" + iid + " /run/carlos/instances/" + iid + "\n" + "ReadWritePaths=/data/instances/" + iid + " /run/carlos/instances/" + iid + "\n" + "WorkingDirectory=/data/instances/" + iid + "\n" } // tierLines are a generation's own memory lines (spec §7.3). func tierLines(tier cgroup.Limit) string { max := strconv.FormatInt(tier.MemoryMax, 10) return "MemoryMax=" + max + "\n" + "MemoryHigh=" + strconv.FormatInt(cgroup.HighOf(tier.MemoryMax), 10) + "\n" + "MemorySwapMax=" + max + "\n" } // writeGenDropIn writes the drop-in atomically: a temp file systemd never // reads (only *.conf is read), renamed into place. func (s *Server) writeGenDropIn(host string, n uint64, tier cgroup.Limit) error { return s.writeDropIn(GenUnitName(host, n), DropInName, GenDropIn(host, n, tier)) } // writeDropIn writes /.d/ atomically: a temp file // systemd never reads (only *.conf is read), renamed into place. func (s *Server) writeDropIn(unit, name, body string) error { dir := filepath.Join(s.DropInDir, unit+".d") if err := os.MkdirAll(dir, 0o755); err != nil { return err } tmp := filepath.Join(dir, "."+name+".tmp") if err := os.WriteFile(tmp, []byte(body), 0o644); err != nil { return err } return os.Rename(tmp, filepath.Join(dir, name)) } // writeDropInIfChanged is writeDropIn unless the file already holds body: // rewriting identical bytes moves the mtime, and systemd then reports // NeedDaemonReload=yes for a loaded unit whose name is reused (a worker's). // It reports whether it wrote. func (s *Server) writeDropInIfChanged(unit, name, body string) (bool, error) { if cur, err := os.ReadFile(filepath.Join(s.DropInDir, unit+".d", name)); err == nil && string(cur) == body { return false, nil } return true, s.writeDropIn(unit, name, body) } // ensureGenExe lays out generation n's view of its build under ExeRoot, // keyed by GenKey, as ensureHostExe does for the original template. Its // target is the host's live link, as today. Plan 3 binds the attempt's // staged path instead (spec §7.5 step 3). func (s *Server) ensureGenExe(host string, n uint64) error { return ensureExe(s.ExeRoot, GenKey(host, n), filepath.Join(LiveDir, host), host) } // A binary and a rootfs generation of one host and one number share // / (task 11 fixes E2a and E2a-reverse). Each // remover takes only its own entries and the key's directory only when // nothing else is left in it, so neither runtime's cleanup can take the // other's layout: // // - binary (removeGenFiles): the binary unit's drop-in directory; under // the key, the app link, its replaceLink temp (app.tmp) and the live // directory; a rootfs unit name is refused; // - rootfs (removeRootfsGenFiles): the rootfs unit's drop-in directory // (its own drop-in and the jit one); under the key, the rootfs/ // layout; called only behind the proof (removeRootfsUnitFiles). // // Both report whether anything was there to remove, so the sweep logs a // removal only when it removed something (fix K1), and both are // best-effort: a leftover is a few bytes in /run, never a running unit. // removeGenFiles removes a BINARY generation unit's files, by the unit's // name as systemd reports it (the rule above). The sweep logs its error, // so a leftover it cannot remove says why (Task 15). func (s *Server) removeGenFiles(unit string) (removed bool, err error) { esc, n, ok := ParseGenUnitName(unit) if !ok { if _, _, rootfs := ParseRootfsGenUnitName(unit); rootfs { return false, fmt.Errorf("%s is a rootfs generation: its files go through the rootfs proof", unit) } return false, nil } var errs []error if s.DropInDir != "" { took, err := removeTree(filepath.Join(s.DropInDir, unit+".d")) removed, errs = removed || took, append(errs, err) } iid, ok := Unescape(esc) if !ok || iid == "" || strings.ContainsAny(iid, "/\x00") || iid == "." || iid == ".." { return removed, errors.Join(errs...) } if s.ExeRoot != "" { dir := filepath.Join(s.ExeRoot, GenKeyOf(iid, n)) for _, name := range []string{"app", "app.tmp"} { took, err := removeEntry(filepath.Join(dir, name)) removed, errs = removed || took, append(errs, err) } took, err := removeTree(filepath.Join(dir, "live")) removed, errs = removed || took, append(errs, err) took, err = removeSharedKeyDir(dir) removed, errs = removed || took, append(errs, err) } return removed, errors.Join(errs...) } // removeRootfsGenFiles removes a rootfs generation unit's files, by the // unit's name (the rule above). Called only once removeRootfsUnitFiles has // proven the unit gone (task 11 fix E2). func (s *Server) removeRootfsGenFiles(unit string) (removed bool, err error) { esc, n, ok := ParseRootfsGenUnitName(unit) if !ok { return false, nil } // The .d directory holds the jit drop-in too (R11): its removal is // counted, so the next rootfs start reloads. removed = s.removeJITDropIn(unit) var errs []error if s.DropInDir != "" { took, err := removeTree(filepath.Join(s.DropInDir, unit+".d")) removed, errs = removed || took, append(errs, err) } iid, ok := Unescape(esc) if !ok || iid == "" || strings.ContainsAny(iid, "/\x00") || iid == "." || iid == ".." { return removed, errors.Join(errs...) } if s.ExeRoot != "" { dir := filepath.Join(s.ExeRoot, GenKeyOf(iid, n)) took, err := removeTree(filepath.Join(dir, RootfsExeDirName)) removed, errs = removed || took, append(errs, err) took, err = removeSharedKeyDir(dir) removed, errs = removed || took, append(errs, err) } return removed, errors.Join(errs...) } // removeSharedKeyDir removes a generation key's directory only when it is // empty: the other runtime's entries, if any, keep it. func removeSharedKeyDir(dir string) (removed bool, err error) { removed, err = removeEntry(dir) if isNotEmpty(err) { return false, nil } return removed, err } // removeEntry removes one file, link or empty directory and reports whether // it was there; absent is not an error. The guard // (rootfs_removal_guard_test.go) counts a call as a raw removal. func removeEntry(p string) (removed bool, err error) { if err := os.Remove(p); err != nil { if errors.Is(err, fs.ErrNotExist) { return false, nil } return false, err } return true, nil } // removeTree removes a directory tree, or a file, and reports whether it // was there; absent is not an error. No read: a non-empty directory shows // as the refusal of its plain removal. The guard counts a call as a raw // removal. func removeTree(p string) (removed bool, err error) { removed, err = removeEntry(p) if !isNotEmpty(err) { return removed, err } return true, os.RemoveAll(p) } // isNotEmpty reports a directory removal refused because the directory // still holds something. func isNotEmpty(err error) bool { return errors.Is(err, syscall.ENOTEMPTY) || errors.Is(err, syscall.EEXIST) } // slicePopulated reports whether anything runs in host's slice. Absent is // no. Unreadable is yes, so a raised limit is never lowered on a guess. func (s *Server) slicePopulated(host string) bool { b, err := os.ReadFile(filepath.Join(s.CgroupRoot, parentSlices, SliceName(host), "cgroup.events")) if errors.Is(err, os.ErrNotExist) { return false } if err != nil { return true } return !strings.Contains(string(b), "populated 0") } // genView is startGen's one look at systemd, a single show of the unit and // its host slice. applied reports whether systemd's view of unit is the // drop-in just written (GenDropIn). The unit's name fixes the host and // generation, and so every line of the drop-in but the tier's: the slice // says systemd read a drop-in at all, and the memory lines say it was this // tier's. NeedDaemonReload=no says no file changed since systemd loaded // the unit, though only by mtime, which is why the tier is checked as well // (Astra review: a reused, still-loaded generation and a clock that went // back). tierSet reports whether the slice already carries every value // setTier would give it, so a cold start can skip that call (wake latency, // pass 2c). Any failure to answer is a no to both. func (s *Server) genView(ctx context.Context, host, unit string, tier cgroup.Limit) (slice string, applied, tierSet bool) { out, err := s.Systemctl(ctx, "show", "-p", "Slice,NeedDaemonReload,MemoryMax,MemoryHigh,MemorySwapMax,CPUWeight,CPUQuotaPerSecUSec,IOWeight", unit, SliceName(host)) if err != nil { return "", false, false } blocks := strings.SplitN(strings.TrimLeft(out, "\n"), "\n\n", 2) m := parseShow(blocks[0]) max := strconv.FormatInt(tier.MemoryMax, 10) high := strconv.FormatInt(cgroup.HighOf(tier.MemoryMax), 10) applied = m["Slice"] == SliceName(host) && m["NeedDaemonReload"] == "no" && m["MemoryMax"] == max && m["MemorySwapMax"] == max && m["MemoryHigh"] == high if len(blocks) == 2 { tierSet = sliceAtTier(parseShow(blocks[1]), tier) } return m["Slice"], applied, tierSet } // sliceAtTier reports whether a slice's shown properties already carry // every value setTier would give it. func sliceAtTier(sl map[string]string, tier cgroup.Limit) bool { max := strconv.FormatInt(tier.MemoryMax, 10) high := strconv.FormatInt(cgroup.HighOf(tier.MemoryMax), 10) sh := cgroup.SharesOf(tier) return sl["MemoryMax"] == max && sl["MemorySwapMax"] == max && sl["MemoryHigh"] == high && sl["CPUWeight"] == strconv.Itoa(sh.CPUWeight) && sl["IOWeight"] == strconv.Itoa(sh.IOWeight) && sl["CPUQuotaPerSecUSec"] == cpuQuotaShown(sh.CPUQuotaPercent) } // cpuQuotaShown is CPUQuota=% as systemctl show prints its // CPUQuotaPerSecUSec (systemd 257): 500ms, 1s, 1.500000s. A format this // gets wrong only costs the set-property call it would have saved. func cpuQuotaShown(pct int) string { usec := int64(pct) * 10000 switch { case usec%1_000_000 == 0: return strconv.FormatInt(usec/1_000_000, 10) + "s" case usec < 1_000_000: return strconv.FormatInt(usec/1000, 10) + "ms" default: return strconv.FormatFloat(float64(usec)/1e6, 'f', 6, 64) + "s" } } // startGen is /start with a generation. The host slice is the parent (spec // §7.3): a cold start sets it to the tier. A start that finds the slice // populated leaves it alone, because a swap raised it with /parent before // the successor's start. func (s *Server) startGen(w http.ResponseWriter, r *http.Request, req request) { ctx := r.Context() unit := GenUnitName(req.host, req.gen) if s.DropInDir == "" { http.Error(w, "not starting "+unit+": generation units are not configured on this service", http.StatusNotImplemented) return } s.markGenStarting(unit, true) defer s.markGenStarting(unit, false) // Timed for the start's log line: a wake waits on all of this. began, reload := time.Now(), "skipped" // A refusal for the disk writes nothing. An exe layout that fails // half-way may have made its directory, so that failure cleans up too. // A staged executable is validated before anything is laid out // (zero-downtime plan 3, Task 9). if s.refuseStagedExe(w, req, unit) { return } checked := time.Now() if s.refuseStart(w, req.host, unit, func() error { err := s.ensureGenExeAt(req.host, req.gen, req.exe) if err != nil { s.removeGenFiles(unit) } return err }) { return } fail := func(status int, msg string) { s.removeGenFiles(unit) http.Error(w, msg, status) } laidOut := time.Now() if err := s.writeGenDropIn(req.host, req.gen, req.tier); err != nil { fail(http.StatusBadGateway, "drop-in: "+err.Error()) return } written := time.Now() // The reload is only for a unit systemd already holds loaded from older // files. A generation's unit is new and an inactive one is unloaded // straight away, so the next reference reads the drop-in fresh: the // common case needs no reload, which on a small box was a third of the // whole wake. The check is the same either way (genDropInApplied). slice, applied, tierSet := s.genView(ctx, req.host, unit, req.tier) if !applied { reloadStart := time.Now() if out, err := s.Systemctl(ctx, "daemon-reload"); err != nil { fail(http.StatusBadGateway, "daemon-reload: "+strings.TrimSpace(out+" "+err.Error())) return } if slice, applied, tierSet = s.genView(ctx, req.host, unit, req.tier); !applied { fail(http.StatusBadGateway, "not starting "+unit+": its drop-in did not apply (slice "+strconv.Quote(slice)+")") return } reload = time.Since(reloadStart).Round(time.Millisecond).String() } if !s.slicePopulated(req.host) && !tierSet { if err := s.setTier(ctx, req.host, req.tier); err != nil { fail(http.StatusBadGateway, "set-property: "+err.Error()) return } } // From the start on, systemd may have started the unit, whatever the // answer says. A failure past this point kills it and proves it empty // before its files go; one that cannot prove that keeps the files and // says so (statusGenUnproven). startAt := time.Now() if out, err := s.Systemctl(ctx, "start", unit); err != nil { s.abandonGen(ctx, w, req.host, unit, "start: "+strings.TrimSpace(out+" "+err.Error())) return } inv, err := s.Systemctl(ctx, "show", "-p", "InvocationID", "--value", unit) if err != nil { s.abandonGen(ctx, w, req.host, unit, "show: "+err.Error()) return } ms := func(d time.Duration) time.Duration { return d.Round(time.Millisecond) } s.logf("instance-control: %s: started %s memory.max=%d (prepared in %s: staged exe %s, layout %s, drop-in %s, check and tier %s; reload %s, start %s)", req.host, unit, req.tier.MemoryMax, ms(startAt.Sub(began)), ms(checked.Sub(began)), ms(laidOut.Sub(checked)), ms(written.Sub(laidOut)), ms(startAt.Sub(written)), reload, ms(time.Since(startAt))) writeJSON(w, http.StatusOK, StartResponse{InvocationID: strings.TrimSpace(inv)}) } // abandonGen answers a generation start that failed after systemctl start // was issued. The unit may be running, so it is killed and waited on, on a // context the client's hang-up cannot cancel. Only a stop that proved the // unit's cgroup empty removes its files; otherwise they stay, and the // answer is statusGenUnproven so the client knows the generation may live. func (s *Server) abandonGen(ctx context.Context, w http.ResponseWriter, host, unit, msg string) { s.abandonUnit(ctx, w, host, unit, msg, func(...string) error { _, _ = s.removeGenFiles(unit); return nil }) } // abandonUnit is abandonGen for any per-start unit: remove is what takes // its files once killGen has stopped it. For a rootfs unit remove is the // start's proof call (removeRootfsUnitFiles under the start's hold, task // 11 fixes E2 and E6), whose own proof (inactive or failed, and the // reported group empty too) can still refuse: then the files stay and the // answer is statusGenUnproven, as for a failed kill. A binary generation's // remove (abandonGen) is unchanged. remove's extra groups are the // misplaced end's; abandon names none. func (s *Server) abandonUnit(ctx context.Context, w http.ResponseWriter, host, unit, msg string, remove func(extra ...string) error) { dctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), DeadlineControl+s.waitBudget()) defer cancel() if err := s.killGen(dctx, host, unit); err != nil { s.logf("instance-control: %s: abandon %s: termination unproven, files kept: %v", host, unit, err) http.Error(w, msg+"; termination unproven: "+err.Error(), statusGenUnproven) return } s.resetGenFailed(dctx, unit) if err := remove(); err != nil { http.Error(w, msg+"; termination unproven: "+err.Error(), statusGenUnproven) return } http.Error(w, msg, http.StatusBadGateway) } // resetGenFailed clears a stopped generation unit's failed state, so a // generation that died on its own does not stay in systemctl --failed // once the control service has stopped it. It runs only after a stop or // kill that proved the unit gone. Best-effort: the stop already succeeded, // so a unit that is no longer loaded is ignored and any other failure is // logged, never turned into a failed stop. func (s *Server) resetGenFailed(ctx context.Context, unit string) { if out, err := s.Systemctl(ctx, "reset-failed", unit); err != nil && !strings.Contains(out+err.Error(), "not loaded") { s.logf("instance-control: reset-failed %s: %v", unit, err) } } // killGen kills a generation unit outright, stops it, and waits until its // cgroup is empty (spec §8.2 step 1). nil is the proof that nothing of it // runs. A kill that finds nothing to kill is not a failure: the stop and // the wait are the proof. func (s *Server) killGen(ctx context.Context, host, unit string) error { if out, err := s.Systemctl(ctx, "kill", "--signal=SIGKILL", unit); err != nil && !strings.Contains(out+err.Error(), "not loaded") { s.logf("instance-control: %s: kill %s: %v", host, unit, err) } if out, err := s.Systemctl(ctx, "stop", unit); err != nil { return errors.New("stop: " + strings.TrimSpace(out+" "+err.Error())) } wctx, cancel := context.WithTimeout(ctx, s.waitBudget()) defer cancel() if err := WaitCgroupEmpty(wctx, s.CgroupRoot, SliceName(host), unit, 50*time.Millisecond); err != nil { return errors.New("stop: " + err.Error()) } return nil } // stopGen is /stop with a generation. With kill=1 the generation is killed // outright (spec §8.2 step 1) and waited on until its cgroup is empty. // Either way the stop acts on that generation's unit only, and its files // go once it has stopped. func (s *Server) stopGen(w http.ResponseWriter, r *http.Request, req request) { ctx := r.Context() unit := GenUnitName(req.host, req.gen) if req.kill { if err := s.killGen(ctx, req.host, unit); err != nil { http.Error(w, err.Error(), http.StatusBadGateway) return } } else if out, err := s.Systemctl(ctx, "stop", unit); err != nil { http.Error(w, "stop: "+strings.TrimSpace(out+" "+err.Error()), http.StatusBadGateway) return } s.resetGenFailed(ctx, unit) _, _ = s.removeGenFiles(unit) w.WriteHeader(http.StatusNoContent) } func (s *Server) markGenStarting(unit string, on bool) { s.genMu.Lock() defer s.genMu.Unlock() if !on { if s.genStarting[unit]--; s.genStarting[unit] <= 0 { delete(s.genStarting, unit) } return } if s.genStarting == nil { s.genStarting = map[string]int{} } s.genStarting[unit]++ } // sweepGenLeftovers is the sweep's recovery for generation units that are // gone but kept their files: their stop's answer was lost, or the agent // that would have finalised them restarted. Such a unit is inactive, so // the sweep had nothing to stop. Each drop-in directory under DropInDir is // checked: a unit the sweep could not stop is skipped, as is one a start // is preparing. The rest lose their files only when systemd reports them // inactive or failed AND their cgroup is empty. No daemon-reload follows: // stopGen removes files the same way, and a generation's name is never // started again. // // Each generation executable directory under ExeRoot (@) is // checked the same way, so a generation that kept only its exe layout is // found too: its drop-in went and the exe's removal failed, which // removeGenFiles ignores (Task 15, Astra IMPL-2-R2 residual 4). A host's // own directory (, no '@') is never a generation key and is never // touched. On a box that never ran generations, ExeRoot holds only host // directories and this finds nothing. // // Rootfs generations and workers (rootfs apps plan 2) are swept the same // way: a rootfs generation's exe directory is judged by the rootfs gen unit // (genUnitOfExeKey, preflight 1.2), so one whose stop failed keeps its // files, and a worker's directory (@) by its worker unit. func (s *Server) sweepGenLeftovers(ctx context.Context, failed map[string]string) { var units []string seen := map[string]bool{} add := func(unit string) { if _, stuck := failed[unit]; stuck || seen[unit] { return } seen[unit] = true units = append(units, unit) } for _, e := range s.readSweepDir(s.DropInDir) { unit, ok := strings.CutSuffix(e.Name(), ".d") if !ok || !e.IsDir() { continue } if _, _, ok := ParseGenUnitName(unit); ok { add(unit) continue } if _, _, ok := ParseRootfsGenUnitName(unit); ok { add(unit) continue } if _, _, ok := ParseWorkerUnitName(unit); ok { add(unit) } } for _, e := range s.readSweepDir(s.ExeRoot) { if !e.IsDir() { continue } if unit, ok := s.genUnitOfExeKey(e.Name()); ok { add(unit) continue } if unit, ok := workerUnitOfExeKey(e.Name()); ok { add(unit) } } for _, unit := range units { s.sweepGenLeftover(ctx, unit) } } // readSweepDir lists dir for sweepGenLeftovers. An unset or absent // directory has nothing to sweep; any other failure is logged. func (s *Server) readSweepDir(dir string) []os.DirEntry { if dir == "" { return nil } entries, err := os.ReadDir(dir) if err != nil && !errors.Is(err, os.ErrNotExist) { s.logf("instance-control: sweep: %s: %v", dir, err) } return entries } // genUnitOfExeKey names the generation unit whose executable directory is // key: the rootfs generation unit when the directory holds a rootfs layout // (/rootfs/), the binary one otherwise (preflight 1.2), so the sweep // judges the directory by the unit that ran from it, and that unit's // remover (removeGenFiles for the binary name, removeRootfsGenFiles behind // the proof for the rootfs one) takes only its own entries under key. It // answers only when the unit's name maps back to exactly key, so the sweep // can never remove under some other directory. func (s *Server) genUnitOfExeKey(key string) (string, bool) { iid, n, ok := ParseGenKey(key) if !ok { return "", false } unit, parse := GenUnitNameOf(iid, n), ParseGenUnitName if fi, err := os.Lstat(filepath.Join(s.ExeRoot, key, RootfsExeDirName)); err == nil && fi.IsDir() { unit, parse = RootfsGenUnitNameOf(iid, n), ParseRootfsGenUnitName } esc, m, ok := parse(unit) if !ok || m != n { return "", false } if back, ok := Unescape(esc); !ok || GenKeyOf(back, n) != key { return "", false } return unit, true } // workerUnitOfExeKey is genUnitOfExeKey for a worker's directory // (@): the worker unit whose files removeWorkerFiles removes. func workerUnitOfExeKey(key string) (string, bool) { iid, name, ok := ParseWorkerKey(key) if !ok { return "", false } unit := WorkerUnitNameOf(iid, name) esc, back, ok := ParseWorkerUnitName(unit) if !ok || back != name { return "", false } if raw, ok := Unescape(esc); !ok || WorkerKeyOf(raw, name) != key { return "", false } return unit, true } func (s *Server) sweepGenLeftover(ctx context.Context, unit string) { // Held across the check and the removal, so a binary generation start // of the same unit cannot write fresh files in between. A rootfs unit's // lock does the same for it below. s.genMu.Lock() defer s.genMu.Unlock() if s.genStarting[unit] > 0 { return } // A rootfs unit's files go only under the unit's lock (task 11 fix // E6): a start in flight holds it, and the sweep leaves the files. var hold *rootfsUnitHold if IsRootfsUnit(unit) { var ok bool if hold, ok = s.tryHoldRootfsUnit(unit); !ok { return } defer hold.release() } if state := s.showValue(ctx, "ActiveState", unit); state != "inactive" && state != "failed" { return } _, esc, _ := ParseUnitName(unit) // One look at the cgroup, no wait: a populated one is not swept. wctx, cancel := context.WithTimeout(ctx, time.Millisecond) defer cancel() slice := "carlos-instances-" + esc + ".slice" var removed bool var rerr error if IsRootfsUnit(unit) { // And only through the one proof (task 11 fix E2), looked at once. sctx, scancel := context.WithTimeout(ctx, DeadlineControl) err := s.removeRootfsUnitFilesWith(sctx, wctx, hold, slice, unit, time.Millisecond, func() { if strings.HasPrefix(unit, UnitPrefixRootfsWorker) { removed, rerr = s.removeWorkerFiles(unit) } else { removed, rerr = s.removeRootfsGenFiles(unit) } }) scancel() if err != nil { return } } else { if err := WaitCgroupEmpty(wctx, s.CgroupRoot, slice, unit, time.Millisecond); err != nil { return } removed, rerr = s.removeGenFiles(unit) } if rerr != nil { s.logf("instance-control: sweep: could not remove the files of %s: %v", unit, rerr) return } if removed { s.logf("instance-control: sweep: removed the files of %s", unit) } } // handleParent sets the host slice's memory: the generation parent's // backstop (spec §7.3). The swap raises it before the successor starts and // restores it once the retiring generation has stopped. CPU and IO shares // stay at one tier (plan 2 D6). func (s *Server) handleParent(w http.ResponseWriter, r *http.Request) { q := r.URL.Query() host := q.Get("host") if !provision.ValidHostShape(host) { http.Error(w, "host "+strconv.Quote(host)+" is not a host", http.StatusBadRequest) return } n, err := strconv.ParseInt(q.Get("memory_max"), 10, 64) if err != nil || !cgroup.ValidParentLimit(cgroup.Limit{MemoryMax: n}) { http.Error(w, "memory_max "+strconv.Quote(q.Get("memory_max"))+" is not one tier or two", http.StatusBadRequest) return } max := strconv.FormatInt(n, 10) if out, err := s.Systemctl(r.Context(), "set-property", "--runtime", SliceName(host), "MemoryMax="+max, "MemoryHigh="+strconv.FormatInt(cgroup.HighOf(n), 10), "MemorySwapMax="+max); err != nil { http.Error(w, "set-property: "+strings.TrimSpace(out+" "+err.Error()), http.StatusBadGateway) return } w.WriteHeader(http.StatusNoContent) }