// Cache eviction for the on-box version store (discussion #111). // // StageArtifact writes every version this box is ever asked to run under // /versions//... and nothing removes one. Retention // (internal/host/retention.go, issue #136) prunes the BUCKET; the box's copy // is a cache of it, and an uncapped cache on a 20 GB root filled the flagship // on 2026-09-06 — 369 staged versions, 9.3 GB, at 40 to 60 new versions a day. // // Eviction, not retention: ensureStaged re-downloads a version that is // missing before it adopts it, so removing one costs a download, never a // deploy. What it must NOT do is remove a version something is USING, which is // the whole content of the pin set below. package release import ( "fmt" "io/fs" "os" "path/filepath" "time" ) // DefaultSweepGrace is how long an unpinned version survives after its last // write. Time-driven rather than size-driven on purpose: a size trigger only // fires once the disk is ALREADY tight, which is the state in which every // other thing on the box (the bake clock, the registry write, the next stage) // has begun to fail — see RunAdopt's placement comment. Six hours is long // enough to cover the window between staging a version and pointing a live // link at it (promotions, canaries and rollbacks all happen inside minutes), // and short enough that a day's shipping cannot accumulate. const DefaultSweepGrace = 6 * time.Hour // SweepResult is one sweep's accounting, for the log line. Errs holds the // per-version failures: a version that cannot be scanned or removed is // skipped, never fatal, because one unreadable tree must not stop the sweep // freeing the rest of the disk. type SweepResult struct { Scanned int // entries under /versions Skipped int // not a directory, or not a version name Pinned int // named by a live link or a registry row Young int // unpinned but written to inside the grace window Removed []string // version directories deleted Bytes int64 // bytes those directories held Errs []error } // SweepStaged removes every /versions/ that `pinned` does not // name and that nothing has written to for `grace`. // // The granularity is the VERSION directory, not the per-app subtree inside // it: a version dir is shared by every account and app staged at that version // (VersionPath), so one pin on any of them keeps the whole directory. Coarse // on purpose — the cost is a few megabytes held slightly longer, and the // alternative (deleting one app's binary out from under a shared version) // buys nothing a later sweep would not collect anyway. // // Entries that are not directories, and directories whose name ValidVersion // rejects, are left alone and counted in Skipped. Nothing this package wrote // has such a name, so such an entry is somebody else's — an operator's backup, // a hand-placed tree — and the sweep is not entitled to it. // // The returned error is reserved for "could not read the versions directory // at all", the one case in which the caller learned nothing and should not // treat an empty Removed list as "there was nothing to remove". func SweepStaged(optDir string, pinned map[string]bool, grace time.Duration, now time.Time) (SweepResult, error) { var res SweepResult vdir := versionsDir(optDir) entries, err := os.ReadDir(vdir) if err != nil { if os.IsNotExist(err) { // A box that has never staged anything. Not an error. return res, nil } return res, fmt.Errorf("read %s: %w", vdir, err) } for _, e := range entries { res.Scanned++ name := e.Name() // e.IsDir() is false for a symlink (ReadDir does not follow), which // is the answer we want: a link into the store is not a staged tree. if !e.IsDir() || !ValidVersion(name) { res.Skipped++ continue } if pinned[name] { res.Pinned++ continue } path := filepath.Join(vdir, name) newest, size, err := treeStats(path) if err != nil { res.Errs = append(res.Errs, fmt.Errorf("scan %s: %w", path, err)) continue } // A negative Sub (a file dated in the future, from a clock step or a // tarball's preserved mtime) reads as "younger than the grace", which // is the cautious direction. if now.Sub(newest) < grace { res.Young++ continue } if err := os.RemoveAll(path); err != nil { res.Errs = append(res.Errs, fmt.Errorf("remove %s: %w", path, err)) continue } res.Removed = append(res.Removed, name) res.Bytes += size } return res, nil } // treeStats reports the newest FILE mtime under root, and the bytes the // regular files hold. // // Files, not directories, and that is the subtle part: removing entries from a // directory bumps ITS mtime, so a tree the previous sweep half-deleted (a // RemoveAll that hit an error partway) would look freshly written and buy // itself another full grace period, every period, forever. The files that // survive carry the real age. // // A tree with no files left at all falls back to the newest directory mtime, // so an empty husk is still collectable rather than permanently ageless. func treeStats(root string) (newest time.Time, bytes int64, err error) { var dirNewest time.Time walkErr := filepath.WalkDir(root, func(_ string, d fs.DirEntry, err error) error { if err != nil { // A concurrent removal (an operator, an interrupted sweep) is not // this walk's problem: skip what vanished under it. if os.IsNotExist(err) { return nil } return err } info, err := d.Info() if err != nil { if os.IsNotExist(err) { return nil } return err } if d.IsDir() { if info.ModTime().After(dirNewest) { dirNewest = info.ModTime() } return nil } if info.ModTime().After(newest) { newest = info.ModTime() } if info.Mode().IsRegular() { bytes += info.Size() } return nil }) if walkErr != nil { return time.Time{}, 0, walkErr } if newest.IsZero() { newest = dirNewest } return newest, bytes, nil } // LivePins is the set of versions the box's live symlinks name — the hard // pins, because each one is a path some unit's ExecStart resolves through. // // Read from the LINKS and not from the registry's per-route Version column, // deliberately. The two agree on a healthy box, and the case where they do // not is exactly the incident's: PointLive succeeded, reg.SetVersions then // failed on a full /data, and the link is the only record of what is running. // Believing the registry there would delete a live binary. // // Every entry is read, including PointLive's `.tmp` leftovers from a // crashed swap: a leftover names a real staged tree and costs one version dir // of disk, which is a better trade than interpreting which links are current. // // A link this package cannot read at all is an error, not a skipped entry. // The sweep's caller declines to delete anything when pins cannot be // established — losing a pin is how a running app gets its binary removed, // so an unreadable live dir must stop the sweep and be loud, not quietly // narrow the pin set. func LivePins(optDir string) (map[string]bool, error) { pins := map[string]bool{} entries, err := os.ReadDir(liveDir(optDir)) if err != nil { if os.IsNotExist(err) { return pins, nil } return nil, fmt.Errorf("read %s: %w", liveDir(optDir), err) } for _, e := range entries { ver, err := LiveVersion(optDir, e.Name()) if err != nil { return nil, fmt.Errorf("read live link %s: %w", LiveLink(optDir, e.Name()), err) } if ver != "" { pins[ver] = true } } entries, err = os.ReadDir(filepath.Join(optDir, "previous")) if err != nil && !os.IsNotExist(err) { return nil, err } for _, e := range entries { target, err := os.Readlink(filepath.Join(optDir, "previous", e.Name())) if err != nil { return nil, err } if ver := versionOfTarget(optDir, target); ver != "" { pins[ver] = true } } return pins, nil }