package main import ( "context" "database/sql" "encoding/json" "fmt" "log" "net/http" "os" "path/filepath" "strconv" "strings" "time" ) // Reclaiming never-claimed spaces. // // Self-serve creation hands out a slot against the capacity cap the moment a // space is made. A space whose admin never completes setup — the setup invite // expires after 7 days — would otherwise sit there forever, holding a slot and a // disk. The reaper sweeps the cloud ledger for spaces older than a threshold, // asks the control daemon whether each was ever set up (see instanceStatus — the // signal is `setup_complete`, NOT member count), and deprovisions the ones that // never were. Deprovisioning ARCHIVES to /data/trash (deleteInstance never // destroys), so a false positive is recoverable; a companion trash purge // reclaims that archive after a longer window. // // Both are opt-in (off unless their env is set) and only ever touch spaces IN // the cloud ledger — operator-run instances (family.…, home, push, …) are never // in it, so they can never be candidates. // ---- "never claimed" signal (controld-side, root) ----------------------------- // instanceStatus reads a space's claimed-ness from its DB, READ-ONLY (safe // against the live instance — no migrations, no writes). For a hibernating // space this is also wake-free by construction: an asleep instance's local DB // is the warm cache the activator left at its last sleep (on one box it is the // newest truth — S3 only ever holds what was checkpointed FROM it), so the // answer comes straight off the file with no socket touched. A hibernating // space that has NEVER woken has no DB at all — os.Stat propagates not-exist // and the stats handler (controld.go) answers definitionally from the registry // instead: members 0, never set up, no content. // // "Never claimed" is NOT "zero members": add-instance pre-seeds the admin user // so the setup invite has an identity to ship, and claiming BECOMES that same // user — so a space has exactly one member whether or not anyone ever set it up. // (The pre-seed happens at setup-invite mint time, i.e. against a RUNNING // instance: for a hibernating create that's the create-path's readiness probe — // its first wake — so a woken-at-create space has the one admin; a never-woken // one has none.) The real signal is `setup_complete`: absent until the admin // finishes the setup conversation (markSetupComplete), then "1". members is // returned too, as a belt-and-suspenders guard (don't reap if anyone beyond the // seeded admin joined). func instanceStatus(host string) (members, people int, setupComplete, hasContent bool, err error) { path := filepath.Join(instanceDataDir, host+".db") if _, err := os.Stat(path); err != nil { return 0, 0, false, false, err // not-exist propagates → the caller answers unknown_host } conn, err := sql.Open("sqlite", "file:"+path+"?mode=ro&_pragma=busy_timeout(3000)") if err != nil { return 0, 0, false, false, err } defer conn.Close() conn.SetMaxOpenConns(1) ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() if err := conn.QueryRowContext(ctx, "SELECT COUNT(*) FROM users").Scan(&members); err != nil { return 0, 0, false, false, err } // people is the billable head count: members who aren't agents — per-seat // billing must never charge for Claude or Eleven. Like the tombstone note // below, this reads OTHER instances' DB files, which may predate is_agent; // a failed count falls back to the full member count rather than erroring // the whole stats read. people = members var np int if err := conn.QueryRowContext(ctx, "SELECT COUNT(*) FROM users WHERE COALESCE(is_agent, 0) = 0").Scan(&np); err == nil { people = np } var v string switch err := conn.QueryRowContext(ctx, "SELECT value FROM config WHERE key='setup_complete'").Scan(&v); err { case nil, sql.ErrNoRows: // absent = never set up default: return 0, 0, false, false, err } // hasContent guards against a false positive: any message or thread means the // space was actually used, so it's never reaped no matter what setup_complete // says. EXISTS short-circuits, so this stays O(1) even on a large DB. // NB: delete tombstones (#581) count as "content" here. That's deliberate: // this runs against OTHER instances' DB files, which may predate the // tombstone_of column (filtering on it would error), and a tombstone means // someone really used the space — erring protective is the guard's job. var hasMsg, hasThread int if err := conn.QueryRowContext(ctx, "SELECT EXISTS(SELECT 1 FROM messages), EXISTS(SELECT 1 FROM threads)").Scan(&hasMsg, &hasThread); err != nil { return 0, 0, false, false, err } return members, people, v == "1", hasMsg == 1 || hasThread == 1, nil } // ---- trash purge (controld-side, root) ---------------------------------------- // trashDirTime returns when a /data/trash entry was archived. deleteInstance // names them "-"; trust that suffix, falling back to the dir mtime. func trashDirTime(name string, info os.FileInfo) time.Time { if i := strings.LastIndexByte(name, '-'); i >= 0 { if sec, err := strconv.ParseInt(name[i+1:], 10, 64); err == nil && sec > 0 { return time.Unix(sec, 0) } } if info != nil { return info.ModTime() } return time.Time{} } // purgeTrash removes archived spaces older than keep, returning how many it // removed. Pure but for the filesystem, so it's directly testable. func purgeTrash(trashDir string, keep time.Duration, now time.Time) (int, error) { entries, err := os.ReadDir(trashDir) if os.IsNotExist(err) { return 0, nil } if err != nil { return 0, err } cutoff := now.Add(-keep) removed := 0 for _, e := range entries { if !e.IsDir() { continue } info, _ := e.Info() if trashDirTime(e.Name(), info).Before(cutoff) { if err := os.RemoveAll(filepath.Join(trashDir, e.Name())); err != nil { log.Printf("controld: trash purge %s: %v", e.Name(), err) continue } removed++ } } return removed, nil } // runTrashPurge sweeps /data/trash once at startup then daily. keep<=0 disables it. func runTrashPurge(keep time.Duration) { if keep <= 0 { return } trashDir := filepath.Join(instanceDataDir, "trash") log.Printf("controld: trash purge on — reclaims archived spaces older than %s", keep) for { if n, err := purgeTrash(trashDir, keep, time.Now()); err != nil { log.Printf("controld: trash purge: %v", err) } else if n > 0 { log.Printf("controld: trash purge removed %d archived space(s)", n) } time.Sleep(24 * time.Hour) } } // ---- the reaper (cloud-side) --------------------------------------------------- // instanceStats is what the control daemon reports about one space: the // reaper's guards plus the storage meter's byte count (meter.go). type instanceStats struct { Members int `json:"members"` // People is the non-agent member count — what per-seat billing charges // for (bots are free). Absent from pre-people controld responses, where // it decodes as 0; consumers fall back to Members. People int `json:"people"` SetupComplete bool `json:"setup_complete"` HasContent bool `json:"has_content"` StorageBytes int64 `json:"storage_bytes"` } // controlStats asks the control daemon for a space's stats. func (s *cloudServer) controlStats(host string) (instanceStats, error) { var st instanceStats req, err := http.NewRequest("GET", "http://control/instances/"+host+"/stats", nil) if err != nil { return st, err } req.Header.Set("Authorization", "Bearer "+s.controlToken) resp, err := s.control.Do(req) if err != nil { return st, err } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { return st, fmt.Errorf("control stats %s: HTTP %d", host, resp.StatusCode) } if err := json.NewDecoder(resp.Body).Decode(&st); err != nil { return st, err } return st, nil } // reapLoop sweeps every interval; enabled only when reapAfter>0. func (s *cloudServer) reapLoop(interval time.Duration) { if s.reapAfter <= 0 { return } mode := "" if s.reapDryRun { mode = " (dry-run — logs only, deletes nothing)" } log.Printf("cloud: reaper on%s — reclaims never-claimed spaces older than %s, sweeping every %s", mode, s.reapAfter, interval) for { s.reapOnce(time.Now()) time.Sleep(interval) } } // reapOnce reclaims never-claimed spaces older than reapAfter. It deletes only a // space the control daemon confirms was NEVER set up (setup_complete absent), that // nobody beyond the pre-seeded admin joined (members <= 1), AND that has zero // content (no messages or threads) — three independent guards, so a space with any // real use survives all of them. Anything set up / in use / with content / whose // status can't be read is left untouched (a stats error is a skip, never a reap). // A per-sweep cap (reapMax) bounds the blast radius of any future mis-classification // to a handful of recoverable archives before an operator would notice. Returns counts. func (s *cloudServer) reapOnce(now time.Time) (reaped, kept int) { cutoff := now.Add(-s.reapAfter).Unix() cands, err := s.db.spacesOlderThan(cutoff) if err != nil { log.Printf("cloud: reaper: ledger query: %v", err) return 0, 0 } for _, c := range cands { st, err := s.controlStats(c.Host) if err != nil { log.Printf("cloud: reaper: skip %s (stats unavailable): %v", c.Host, err) continue } if st.SetupComplete || st.Members > 1 || st.HasContent { kept++ continue // set up, someone else joined, or has content — never reap } created := time.Unix(c.CreatedAt, 0).UTC().Format(time.RFC3339) if s.reapDryRun { log.Printf("cloud: reaper: WOULD reclaim %s (id %d, never set up, created %s)", c.Host, c.ID, created) continue } if s.reapMax > 0 && reaped >= s.reapMax { log.Printf("cloud: reaper: per-sweep cap (%d) reached — deferring the rest to the next sweep", s.reapMax) break } status, body, err := s.controlDelete(c.Host, false) if err != nil { log.Printf("cloud: reaper: delete %s failed: %v", c.Host, err) continue } // unknown_host = already gone (an operator removed it); still clear the row. if status != http.StatusOK && body["error"] != "unknown_host" { log.Printf("cloud: reaper: delete %s: HTTP %d", c.Host, status) continue } if err := s.db.deleteSpaceByID(c.ID); err != nil { log.Printf("cloud: reaper: ledger delete %s (id %d): %v", c.Host, c.ID, err) } log.Printf("cloud: reaper: reclaimed never-claimed space %s (id %d, created %s)", c.Host, c.ID, created) reaped++ } if reaped > 0 || kept > 0 { log.Printf("cloud: reaper: swept %d candidate(s) — %d reclaimed, %d still in use", len(cands), reaped, kept) } return reaped, kept } // ---- self-reclaim (the create path's escape hatch) ------------------------------ // reclaimAge is how old one of the account's own spaces must be before a // quota-blocked create may reclaim it. Setup takes minutes, so an hour cleanly // separates "abandoned yesterday, retrying now" (reclaim) from "setting it up // in another tab right now" (never touch) — and the home page keeps the setup // link for continuing inside the window. const reclaimAge = time.Hour // reclaimOwnSpace frees at most one quota slot for (caller, requester) by // reclaiming their own oldest abandoned allocation: a pending row whose create // died between allocate and finalize, or a provisioned space that clears the // reaper's never-used guards (never set up, nobody joined, zero content) — // plus ownership and the age gate on top. Data still archives to /data/trash // via controld, same as the reaper. A stats error is a skip, never a reclaim. // Returns whether a slot was freed. func (s *cloudServer) reclaimOwnSpace(callerID, requester string) bool { cands, err := s.db.reclaimCandidates(callerID, requester, time.Now().Add(-reclaimAge).Unix()) if err != nil { log.Printf("cloud: self-reclaim: ledger query: %v", err) return false } for _, c := range cands { if c.Host == "" { // A pending row this old is a dead create (real ones finalize in // seconds); there is no instance behind it — ledger-only cleanup. if err := s.db.deleteSpaceByID(c.ID); err != nil { log.Printf("cloud: self-reclaim: clear stale pending row (id %d): %v", c.ID, err) continue } log.Printf("cloud: self-reclaim: cleared stale pending allocation (id %d, requester of caller %.8s…)", c.ID, callerID) return true } st, err := s.controlStats(c.Host) if err != nil { log.Printf("cloud: self-reclaim: skip %s (stats unavailable): %v", c.Host, err) continue } if st.SetupComplete || st.Members > 1 || st.HasContent { continue // in real use — the quota refusal stands } status, body, err := s.controlDelete(c.Host, false) if err != nil { log.Printf("cloud: self-reclaim: delete %s failed: %v", c.Host, err) continue } if status != http.StatusOK && body["error"] != "unknown_host" { log.Printf("cloud: self-reclaim: delete %s: HTTP %d", c.Host, status) continue } if err := s.db.deleteSpaceByID(c.ID); err != nil { log.Printf("cloud: self-reclaim: ledger delete %s (id %d): %v", c.Host, c.ID, err) } log.Printf("cloud: self-reclaim: reclaimed %s (id %d) — owner's own never-set-up space, making room for their new create", c.Host, c.ID) return true } return false }