package bot // The charter scheduler: how member bots act on their own clock, with no new // server surface. The server's existing scheduled-message queue is the alarm: // the charter host encrypts a bot.tick under its own SELF-THREAD key and asks // the server to deliver it at the due moment (miniapps.go). Delivery arrives // as an ordinary msg frame, handleTyped routes it here, the charter's task // runs, and the next occurrence is re-armed. Because the queue is server-side, // alarms survive sidecar restarts for free — every connect (and a slow // background tick) reconciles charters against pending alarms. The server // learns only (thread, time), which it already learns for every reminder. // // Hibernation (charter-wake registration): after every schedule recompute the // host also reports the EARLIEST next run to the instance (reportCharterDue → // POST /api/charter-next-due), which the router's due-time index folds in — // so a hibernating instance is woken for charter work even when the queue's // alarm row was consumed while no host was listening. And because a due-wake's // co-spawned host can lose the delivery race (the instance's scheduler fires // the tick before this host's cursor baseline), the connect-time reconcile // treats a live charter with no pending alarm as owing a run NOW — armed at // now, delivered within a scheduler tick, run through the normal path. import ( "time" "lchat/internal/e2ee" ) // tickPayload is the decrypted shape of one alarm. Bot is the charter id it // fires (the store id, not the slug). type tickPayload struct { Type string `json:"type"` // "bot.tick" Bot string `json:"bot"` TS int64 `json:"ts"` } // charterReconcile makes reality match the store: every live (un-paused) // charter has exactly one pending alarm in the self-thread; alarms for // deleted, paused, or duplicated charters are cancelled. atConnect runs on // the frame goroutine and may create the self-thread; the background ticker // passes false and only ever reads the snapshot. func (r *Runtime) charterReconcile(atConnect bool) { var selfID string var selfKey []byte if atConnect { id, key, err := r.ensureSelfThread() if err != nil { r.logf("charters: self thread: %v", err) return } selfID, selfKey = id, key } else { id, key, ok := r.selfThread() if !ok { return // never connected as host yet } selfID, selfKey = id, key } instKey, err := r.instanceKey() if err != nil { // Normal before any member has granted the instance key — nothing to // run yet anyway (charters are sealed under it). Nudge online holders // (an open tab grants on the spot), so the host heals without anyone // reloading. r.logf("charters: instance key not held yet (%v)", err) r.nudgeInstanceKey() return } entries, err := r.listCharters(instKey) if err != nil { r.logf("charters: list: %v", err) return } live := map[string]*Charter{} for _, e := range entries { if !e.Charter.Paused { live[e.ID] = e.Charter } } pending, err := r.listSchedules(selfID) if err != nil { r.logf("charters: pending: %v", err) return } armed := map[string]bool{} for _, p := range pending { var tp tickPayload if e2ee.DecryptJSON(selfKey, p.Blob, &tp) != nil || tp.Type != "bot.tick" { continue } if live[tp.Bot] == nil || armed[tp.Bot] { // An alarm for a bot that's gone/paused, or a duplicate (a crash // re-armed twice): cancel it. _ = r.unschedule(p.ID) continue } armed[tp.Bot] = true } now := time.Now() for id, c := range live { if armed[id] { continue } when := c.Schedule.next(now) if atConnect { // A live charter with NO pending alarm at connect means its alarm // fired while no host was listening (delivered into the self-thread // after the last host stopped — the hibernation due-wake case: the // instance's scheduler can beat this host's connect by a beat and // the cursor baseline swallows the tick) — or the alarm was never // armed at all. Either way the schedule owes a run NOW, not at the // next occurrence (a daily bot would otherwise skip a whole day // every time its wake lost that race). Arm at now: the server // delivers within one scheduler tick and the run flows through the // ordinary charterTick path — same crash-recovery ordering, no // second execution path. (charterTick's duplicate guard covers the // sliver where the original tick was in flight after all.) when = now } r.armCharter(selfID, selfKey, id, when) } // Report the charter due-time so the schedule survives hibernation: the // instance stores the earliest next run (a bare timestamp — the charters // themselves stay ciphertext to it) and the router's due-time index wakes // the sleeping instance for it. Reported on every reconcile — connect and // the slow tick — so a value from a dead host heals as soon as any host // connects. Zero (no live charters) is reported too: it clears a stale // value that would otherwise buy one pointless wake. r.reportCharterDue(earliestCharterDue(live, now)) } // earliestCharterDue is the soonest next occurrence across live charters // (unix seconds; 0 = none). Pure — the clock is a parameter, like // ScheduleSpec.next. It recomputes next(now) rather than reading the armed // alarms: an armed alarm may be sooner (e.g. an interval schedule armed a // while ago), but the alarm is ALSO a server-side scheduled message the // instance already folds into its next-due answer, so the earlier truth // still wins — this report is the backstop for exactly the alarms the queue // has already consumed. func earliestCharterDue(live map[string]*Charter, now time.Time) int64 { var due int64 for _, c := range live { if n := c.Schedule.next(now).Unix(); due == 0 || n < due { due = n } } return due } // reportCharterDue posts the earliest charter run to the instance // (POST /api/charter-next-due, memberbots.go). Best-effort — a failed report // costs promptness on the next hibernated occurrence (the instance wakes for // a member request instead), never correctness — and it rides r.client's // marked transport like every sidecar request, so it stamps no activity. func (r *Runtime) reportCharterDue(due int64) { if err := r.post("/api/charter-next-due", map[string]any{ "token": r.Session.Token, "due": due, }, nil); err != nil { r.logf("charters: report next due: %v", err) } } // recomputeCharterDue re-derives and reports the earliest charter run — for // call sites that just mutated the store (build, update, remove) rather than // folded it (botbuild.go). func (r *Runtime) recomputeCharterDue(instKey []byte) { entries, err := r.listCharters(instKey) if err != nil { r.logf("charters: recompute next due: %v", err) return } live := map[string]*Charter{} for _, e := range entries { if !e.Charter.Paused { live[e.ID] = e.Charter } } r.reportCharterDue(earliestCharterDue(live, time.Now())) } // armCharter queues one alarm for a charter. func (r *Runtime) armCharter(selfID string, selfKey []byte, charterID string, when time.Time) { payload := map[string]any{"type": "bot.tick", "bot": charterID, "ts": time.Now().UnixMilli()} if _, err := r.schedule(selfID, selfKey, payload, when); err != nil { r.logf("charters: arm %s: %v", charterID, err) } } // charterTick fires one delivered alarm (frame goroutine, via handleTyped). // The charter is re-read from the store first — it may have been edited, // paused, or deleted since the alarm was set — and the NEXT occurrence is // armed before the run, so a task that crashes the process can't silence the // bot forever (a duplicate alarm from that ordering is dropped by reconcile). func (r *Runtime) charterTick(selfID string, selfKey []byte, tp *tickPayload) { // Duplicate-tick guard: the connect-time catch-up arm (charterReconcile) // re-arms a charter whose alarm looks consumed-unheard, but a delivery can // land in the sliver between the connect's cursor baseline and the // reconcile's pending-alarm read — then BOTH the original tick and the // catch-up alarm arrive. Two ticks for one charter closer together than // any expressible schedule are that race, not a run owed twice: drop the // second without re-arming (the first already armed the next occurrence). if last, ok := r.charterRuns[tp.Bot]; ok && time.Since(last) < minEvery()/2 { r.logf("charters: tick %s: duplicate within %s — dropped", tp.Bot, minEvery()/2) return } instKey, err := r.instanceKey() if err != nil { r.logf("charters: tick %s: instance key: %v", tp.Bot, err) return } // One store read serves both needs: the ticked charter, and the live set // for the wake-time report below. entries, err := r.listCharters(instKey) if err != nil { r.logf("charters: tick %s: %v", tp.Bot, err) return } var c *Charter live := map[string]*Charter{} for _, e := range entries { if !e.Charter.Paused { live[e.ID] = e.Charter } if e.ID == tp.Bot { c = e.Charter } } if c == nil || c.Paused { // Deleted or paused since the alarm was set — don't re-arm, but do // refresh the report: this alarm may have been the earliest. r.reportCharterDue(earliestCharterDue(live, time.Now())) return } r.armCharter(selfID, selfKey, tp.Bot, c.Schedule.next(time.Now())) // Re-report after each run's re-arm, so a hibernating instance's stored // wake-time always tracks the freshest schedule. r.reportCharterDue(earliestCharterDue(live, time.Now())) r.charterRuns[tp.Bot] = time.Now() key, ok := r.threadKeyFor(c.Thread) if !ok { r.logf("charters: %s: no key for thread %s — skipping run", c.Name, c.Thread) return } r.logf("charters: running %s (%s)", c.Name, c.Schedule.Summary()) res, err := r.execTask(c.Title, &c.Task) switch { case err != nil: r.logf("charters: %s: %v", c.Name, err) r.charterRunFailed(selfID, selfKey, tp.Bot, c, key, taskResult{}) case res.sourcesDown(): // Before the empty check: an all-sources-failed run that also came // back textless should log (and, on the first failure, explain) the // real problem. r.logf("charters: %s: reached none of its sources (%d fetches)", c.Name, res.fetches) r.charterRunFailed(selfID, selfKey, tp.Bot, c, key, res) case res.text == "": r.logf("charters: %s: empty result — nothing posted", c.Name) r.charterRunFailed(selfID, selfKey, tp.Bot, c, key, taskResult{}) default: r.clearCharterFails(tp.Bot) r.postTaskResult(c.Thread, key, c.Title, &c.Task, res.text, res.card) r.meterInc("charter_runs", 1) // completed runs only — failures cost tokens but deliver nothing } } // How a failing charter behaves (the RTE-down case: a source 403s for a day, // an hourly bot): ONE notice per outage, not one per attempt, and the retries // space themselves out instead of hammering a dead feed on schedule. const charterBackoffCap = 24 * time.Hour // The one notice a failing bot posts, appended to the model's own explanation // of what went wrong. Two variants: interval schedules also stretch their // retries (backoffDelay), daily ones keep their day. const ( failNoteStretch = "I'll keep trying quietly — a little less often while my sources are unreachable — and post again as soon as I can read them." failNotePlain = "I'll keep trying quietly on my schedule and post again as soon as I can read my sources." ) // backoffDelay is how long after `fails` consecutive failed runs the next // attempt should wait: interval schedules double per failure (2×, 4×, 8×… // the interval) capped at a day; daily (at) schedules return 0 — their // cadence is already gentle, and the one-notice rule alone stops the spam. // Pure; the schedule must have been validated (compiled `every`). func backoffDelay(s *ScheduleSpec, fails int) time.Duration { if s.Every == "" || fails < 1 { return 0 } d := s.every for i := 0; i < fails && d < charterBackoffCap; i++ { d *= 2 } return min(d, charterBackoffCap) } // charterRunFailed folds one failed run into the backoff state: bump the // consecutive count, push the pre-armed next alarm out by the backoff, and // post the model's explanation — but only on the FIRST failure of an outage // (err/empty runs have no text and post nothing, exactly as before). func (r *Runtime) charterRunFailed(selfID string, selfKey []byte, botID string, c *Charter, key []byte, res taskResult) { n := r.bumpCharterFails(botID) if d := backoffDelay(&c.Schedule, n); d > 0 { // charterTick armed the next ordinary occurrence before the run (the // crash-safety ordering); replace it — a source that's down now is // usually still down one interval later. r.dropCharterAlarms(selfID, selfKey, botID) r.armCharter(selfID, selfKey, botID, time.Now().Add(d)) r.logf("charters: %s: failure #%d — next attempt in %s", c.Name, n, d) } else { r.logf("charters: %s: failure #%d", c.Name, n) } if n > 1 || res.text == "" { return } note := failNotePlain if c.Schedule.Every != "" { note = failNoteStretch } // The explanation posts without any card the run half-composed — a failure // notice should read as one, not dress up as an edition. r.postTaskResult(c.Thread, key, c.Title, &c.Task, res.text+"\n\n"+note, nil) } // bumpCharterFails / clearCharterFails track consecutive failed runs per // charter — in memory only (a restart forgives; worst case one extra notice). // Own lock: ticks run on the frame goroutine, but builds and removals clear // from their own goroutines. func (r *Runtime) bumpCharterFails(botID string) int { r.failMu.Lock() defer r.failMu.Unlock() r.charterFails[botID]++ return r.charterFails[botID] } func (r *Runtime) clearCharterFails(botID string) { r.failMu.Lock() defer r.failMu.Unlock() delete(r.charterFails, botID) } // dropCharterAlarms cancels every pending alarm for one charter in the // self-thread — schedule edits, retirement, and backoff re-arms all need it. func (r *Runtime) dropCharterAlarms(selfID string, selfKey []byte, botID string) { pending, err := r.listSchedules(selfID) if err != nil { return } for _, p := range pending { var tp tickPayload if e2ee.DecryptJSON(selfKey, p.Blob, &tp) == nil && tp.Type == "bot.tick" && tp.Bot == botID { _ = r.unschedule(p.ID) } } } // threadKeyFor returns the unwrapped key for a thread the bot holds. Frame // goroutine only (r.keys). func (r *Runtime) threadKeyFor(threadID string) ([]byte, bool) { e, ok := r.keys[threadID] if !ok { // A thread created since the last loadThreads (e.g. by our own builder // goroutine): one refresh is cheap and self-heals. r.loadThreads() e, ok = r.keys[threadID] } if !ok { return nil, false } return e.key, true }