package main // End-to-end test of charter failure backoff (internal/bot/chartersched.go): // a member bot whose every source is down posts ONE notice (the model's // explanation plus the keep-trying note), stays silent on further failed // attempts, and resumes normal editions the moment a source answers again. // The all-403 case is the important one — the run doesn't error, the model // writes an apology, and before backoff that apology posted every occurrence. import ( "bytes" "context" "encoding/json" "net/http" "net/http/httptest" "strings" "sync/atomic" "testing" "time" "lchat/internal/bot" "lchat/internal/e2ee" ) func TestCharterFailingRunBacksOffAndRecovers(t *testing.T) { t.Setenv("LCHAT_BOT_ALLOW_PRIVATE_FETCH", "1") t.Setenv("LCHAT_BOT_MIN_EVERY", "1s") // The "publication": down (403) until the test flips it healthy. var healthy, hits atomic.Int64 site := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { hits.Add(1) if healthy.Load() == 0 { w.WriteHeader(http.StatusForbidden) return } w.Write([]byte(`Budget talks resume`)) })) defer site.Close() // Each run is two model calls: a fetch, then the final text. Runs 1 and 2 // end in apologies (their fetch 403s); run 3 is the recovery edition (the // last response repeats if further ticks fire). fetchCall := []map[string]any{{"type": "tool_use", "id": "tu_f", "name": "fetch", "input": map[string]any{"url": site.URL + "/feed"}}} fakeAI := scriptedAnthropic(t, fetchCall, []map[string]any{{"type": "text", "text": "I couldn't reach the news feed this time."}}, fetchCall, []map[string]any{{"type": "text", "text": "Still can't reach the feed, I'm afraid."}}, fetchCall, []map[string]any{{"type": "text", "text": "Back in business: budget talks resume."}}, ) t.Setenv("ANTHROPIC_BASE_URL", fakeAI.URL) ts, s := newTestServer(t) owner := provisionNamed(t, ts.URL, "Owner", false) eleven := provisionNamed(t, ts.URL, "Eleven", true) ownerUID, _ := s.db.deviceTokenUser(owner.Token) instKey := grantInstanceKey(t, ts.URL, owner, eleven, s) threadID, threadKey := createThreadKind(t, ts.URL, s, "group", owner, eleven) charter := map[string]any{ "v": 1, "name": "flaky-news", "title": "Flaky News", "glyph": "📰", "schedule": map[string]any{"every": "1s"}, "task": map[string]any{ "prompt": "Fetch the feed and summarise it.", "tools": []any{"fetch"}, "fetch": map[string]any{"allow": []any{site.URL}}, }, "thread": threadID, "owner": ownerUID, } blob, err := e2ee.EncryptJSON(instKey, charter) if err != nil { t.Fatal(err) } body, _ := json.Marshal(map[string]string{"token": eleven.Token, "blob": blob, "owner_id": ownerUID}) req, _ := http.NewRequest(http.MethodPut, ts.URL+"/api/member-bots/bot-flaky1", bytes.NewReader(body)) req.Header.Set("Content-Type", "application/json") if resp, err := http.DefaultClient.Do(req); err != nil || resp.StatusCode != http.StatusOK { t.Fatalf("seed charter: %v %d", err, resp.StatusCode) } dir := t.TempDir() sessionPath := dir + "/eleven.session.json" if err := eleven.Save(sessionPath); err != nil { t.Fatal(err) } ready := make(chan struct{}) ctx, cancel := context.WithCancel(context.Background()) defer cancel() eng := &bot.Engine{Base: ts.URL, Session: eleven, SessionPath: sessionPath, Recipe: bot.DefaultRecipe(), AIKey: "test-key", Charters: true, Ready: func() { close(ready) }, Logf: t.Logf} go eng.Run(ctx) awaitReady(t, ready) // editions returns what the persona has posted so far. editions := func() []string { var out []string for _, rp := range botReplies(t, s, threadID, threadKey, "Flaky News") { out = append(out, rp.Text) } return out } // First failed run: exactly one notice — the model's apology plus the // keep-trying note (the interval variant, since this schedule stretches). waitFor(t, "first failure notice", func() bool { s.sweepDueScheduled() return len(editions()) == 1 }) if got := editions()[0]; !strings.Contains(got, "couldn't reach") || !strings.Contains(got, "keep trying quietly") { t.Fatalf("first notice should carry the apology + note: %q", got) } // Second failed run: the site sees another attempt, the thread doesn't. // (Ticks only fire when this test sweeps, so the count below is stable.) first := hits.Load() waitFor(t, "a backed-off second attempt", func() bool { s.sweepDueScheduled() return hits.Load() > first }) time.Sleep(300 * time.Millisecond) // let the run settle past its (non-)post decision if got := editions(); len(got) != 1 { t.Fatalf("a repeat failure must post nothing, got %d editions: %q", len(got), got) } // The source recovers: the next attempt posts a normal edition and the // outage leaves exactly one notice behind. (Two stages, because the second // failure pushed this attempt out to ~4s — one 5s waitFor would be tight.) healthy.Store(1) second := hits.Load() waitFor(t, "a recovery attempt reaching the site", func() bool { s.sweepDueScheduled() return hits.Load() > second }) waitFor(t, "recovery edition", func() bool { eds := editions() return len(eds) == 2 && strings.Contains(eds[1], "budget talks resume") }) if got := editions()[1]; strings.Contains(got, "keep trying") { t.Fatalf("the recovery edition must post clean, got %q", got) } }