package bakeoff import ( "net/http/httptest" "os" "os/exec" "path/filepath" "strings" "testing" "time" "bakeoff/harness/refapp/ref" ) func refServer(t *testing.T) *httptest.Server { t.Helper() srv := httptest.NewUnstartedServer(nil) srv.Start() srv.Config.Handler = ref.NewApp(srv.URL, "../shared") t.Cleanup(srv.Close) return srv } func TestSeed(t *testing.T) { srv := refServer(t) s, err := NewSession(srv.URL) if err != nil { t.Fatal(err) } ids, err := Seed(s, 50) if err != nil { t.Fatal(err) } if len(ids) != 50 { t.Fatalf("want 50 ids, got %d", len(ids)) } done, err := s.IDs("/completed") if err != nil { t.Fatal(err) } if len(done) != 25 { t.Fatalf("want 25 completed, got %d", len(done)) } if s.ToggleID != ids[0] { t.Fatalf("Seed must remember the toggle target: got %q, want %q", s.ToggleID, ids[0]) } if !strings.Contains(s.CookieHeader(), "=") { t.Fatalf("no cookie: %q", s.CookieHeader()) } } func TestParseOha(t *testing.T) { b, _ := os.ReadFile("testdata/oha.json") r, err := ParseOha(b, 200) if err != nil { t.Fatal(err) } if r.RPS <= 0 || r.P99Ms < r.P50Ms || r.P50Ms <= 0 { t.Fatalf("bad result %+v", r) } if r.ErrorRate != 0 { t.Fatalf("deadline aborts must not count as errors, got %v", r.ErrorRate) } } func TestParseOhaCountsErrors(t *testing.T) { b := []byte(`{"summary":{"requestsPerSec":10},"latencyPercentiles":{"p50":0.001,"p95":0.002,"p99":0.003}, "statusCodeDistribution":{"303":80,"500":10},"errorDistribution":{"connection refused":10,"aborted due to deadline":3}}`) r, err := ParseOha(b, 303) if err != nil { t.Fatal(err) } if r.ErrorRate != 0.2 { t.Fatalf("want 0.2, got %v", r.ErrorRate) } if r.P50Ms != 1 || r.P99Ms != 3 { t.Fatalf("want ms, got %+v", r) } if r.Status["500"] != 10 || r.Errors["connection refused"] != 10 || r.Errors["aborted due to deadline"] != 0 { t.Fatalf("want the raw breakdown kept (minus deadline aborts), got %v %v", r.Status, r.Errors) } } func TestParseClaudeLog(t *testing.T) { f, _ := os.Open("testdata/claude.jsonl") defer f.Close() tk, err := ParseClaudeLog(f) if err != nil { t.Fatal(err) } if tk.Output != 4 || tk.CacheWrite != 32396 || tk.Turns != 1 || tk.CostUSD < 0.25 || tk.Seconds <= 0 { t.Fatalf("bad tokens %+v", tk) } } func TestParseCodexLog(t *testing.T) { f, _ := os.Open("testdata/codex.jsonl") defer f.Close() tk, err := ParseCodexLog(f) if err != nil { t.Fatal(err) } if tk.Input != 16532 || tk.CacheRead != 12032 || tk.Output != 5 || tk.Turns != 1 { t.Fatalf("bad tokens %+v", tk) } } func TestCountCode(t *testing.T) { c, err := CountCode("testdata/codetree") if err != nil { t.Fatal(err) } want := CodeStats{Server: 5, Templates: 5, JS: 4, Tests: 2, Deps: 2} if c != want { t.Fatalf("got %+v, want %+v", c, want) } } func TestPageWeight(t *testing.T) { srv := refServer(t) s, _ := NewSession(srv.URL) w, err := PageWeight(s, "/") if err != nil { t.Fatal(err) } js, _ := os.ReadFile("refapp/ref/app.js") if w.JSBytes != len(js) || w.HTMLBytes <= 0 { t.Fatalf("got %+v, want JSBytes %d", w, len(js)) } } func TestMeasureTTFB(t *testing.T) { srv := refServer(t) s, _ := NewSession(srv.URL) st, err := MeasureTTFB(s, "/", 1, 5) if err != nil { t.Fatal(err) } if st.N != 5 || st.Min > st.P50 || st.P50 > st.Max || st.Min <= 0 { t.Fatalf("bad stats %+v", st) } } func TestBuildStats(t *testing.T) { dir := t.TempDir() for _, f := range []string{"Makefile", "src.go"} { b, _ := os.ReadFile("testdata/buildtree/" + f) os.WriteFile(filepath.Join(dir, f), b, 0o644) } b, err := BuildStats(dir, false, 1, 2) if err != nil { t.Fatal(err) } if !(b.ColdSeconds > b.WarmSeconds && b.WarmSeconds > b.EditSeconds && b.EditSeconds > 0) { t.Fatalf("want cold > warm > edit > 0, got %+v", b) } if b.Bytes != 6 { t.Fatalf("want 6 bytes, got %d", b.Bytes) } if _, err := os.Stat(filepath.Join(dir, ".unbumped")); err != nil { t.Fatal("unbump was not run") } last, _ := os.ReadFile(filepath.Join(dir, "dist", "history")) if !strings.Contains(string(last), "// bakeoff edit ") { t.Fatalf("the edit build must compile a unique comment line, built:\n%s", last) } src, _ := os.ReadFile(filepath.Join(dir, "src.go")) if string(src) != "package main\n\nconst marker = \"steady\"\n" { t.Fatalf("source not restored:\n%s", src) } if b.ColdRuns != 1 || b.Runs != 2 { t.Fatalf("want 1 cold run and 2 warm/edit runs recorded, got %+v", b) } } func TestUpdateRoundTrip(t *testing.T) { dir := t.TempDir() if err := UpdateIn(dir, "rust", func(r *AppResult) { r.Language = "Rust"; r.Wake = append(r.Wake, 12.5) }); err != nil { t.Fatal(err) } if err := UpdateIn(dir, "rust", func(r *AppResult) { r.Wake = append(r.Wake, 13) }); err != nil { t.Fatal(err) } r, err := LoadIn(dir, "rust") if err != nil { t.Fatal(err) } if r.Slug != "rust" || r.Language != "Rust" || len(r.Wake) != 2 { t.Fatalf("got %+v", r) } } func TestRunLoadAgainstRefapp(t *testing.T) { if testing.Short() { t.Skip("runs oha") } if _, err := exec.LookPath("oha"); err != nil { t.Skip("oha not installed") } srv := refServer(t) s, _ := NewSession(srv.URL) if _, err := Seed(s, 10); err != nil { t.Fatal(err) } for _, sc := range []string{"read", "toggle", "read-after"} { r, err := RunLoad(s, sc, 2, time.Second) if err != nil { t.Fatal(err) } if r.RPS <= 0 || r.ErrorRate != 0 || r.Scenario != sc || r.Concurrency != 2 { t.Fatalf("%s: bad result %+v", sc, r) } } } func TestParsePlaywright(t *testing.T) { f, _ := os.Open("testdata/playwright.json") defer f.Close() got, err := ParsePlaywright(f) if err != nil { t.Fatal(err) } if got["js-on"] != (Behaviour{Passed: 1, Failed: 1}) || got["js-off"] != (Behaviour{Failed: 1, Skipped: 1}) { t.Fatalf("got %+v (flaky counts as failed)", got) } } func TestSourceFiles(t *testing.T) { got, err := SourceFiles("testdata/codetree") if err != nil { t.Fatal(err) } want := []string{"main.go", "src/app.nim", "src/page.nimf", "static/app.js", "views/index.html"} if strings.Join(got, ",") != strings.Join(want, ",") { t.Fatalf("got %v, want %v", got, want) } } func TestMedianReadability(t *testing.T) { r, err := MedianReadability([]string{ `{"naming":4,"structure":3,"traceability":5,"errors":2,"comments":3,"overall":7,"notes":"a"}`, "```json\n{\"naming\":2,\"structure\":4,\"traceability\":4,\"errors\":3,\"comments\":3,\"overall\":6,\"notes\":\"b\"}\n```", `{"naming":5,"structure":5,"traceability":3,"errors":4,"comments":2,"overall":8,"notes":"c"}`, }) if err != nil { t.Fatal(err) } if r.Naming != 4 || r.Structure != 4 || r.Traceability != 4 || r.Errors != 3 || r.Comments != 3 || r.Overall != 7 || r.Notes != "a" { t.Fatalf("got %+v (notes come from the run whose overall is the median)", r) } } func TestParseFindings(t *testing.T) { got := ParseFindings("1. [spec] [a.go:1](/x/a.go:1): first\n continues here.\n\n2. [security] b: second\n") if len(got) != 2 || got[0] != "[spec] [a.go:1](/x/a.go:1): first continues here." || got[1] != "[security] b: second" { t.Fatalf("got %q", got) } if len(ParseFindings("No findings.")) != 0 { t.Fatal("No findings. must parse to none") } if !HasSecurity(got) || HasSecurity(got[:1]) { t.Fatal("HasSecurity wrong") } } func TestTokensSince(t *testing.T) { build := Tokens{Input: 82, Output: 56663, CacheRead: 4280334, CacheWrite: 132243, Turns: 46, Seconds: 693, CostUSD: 3.05} fixCumulative := Tokens{Input: 106, Output: 65224, CacheRead: 6096946, CacheWrite: 145431, Turns: 12, Seconds: 183, CostUSD: 3.69} got := fixCumulative.Since(build) want := Tokens{Input: 24, Output: 8561, CacheRead: 1816612, CacheWrite: 13188, Turns: 12, Seconds: 183} if got.Input != want.Input || got.Output != want.Output || got.CacheRead != want.CacheRead || got.CacheWrite != want.CacheWrite || got.Turns != 12 || got.Seconds != 183 || got.CostUSD < 0.63 || got.CostUSD > 0.65 { t.Fatalf("got %+v", got) } } func TestApplyRecordKeepsUnsetFields(t *testing.T) { r := AppResult{Language: "Go", Status: "LIVE", URL: "u", Version: "v1"} ApplyRecord(&r, Record{Version: "v2"}) if r.Language != "Go" || r.Status != "LIVE" || r.URL != "u" || r.Version != "v2" { t.Fatalf("got %+v", r) } } func TestParseClaudeLogSumsTurnsAcrossResults(t *testing.T) { f, _ := os.Open("testdata/claude-multi.jsonl") defer f.Close() tk, err := ParseClaudeLog(f) if err != nil { t.Fatal(err) } if tk.Turns != 5 || tk.Seconds != 1.5 || tk.Output != 140 || tk.CacheRead != 30 || tk.CostUSD != 1.5 { t.Fatalf("got %+v (usage and cost are cumulative: take the last; turns and time are per result: sum)", tk) } } // lr is a recorded ramp step; like real ones it carries a status // breakdown (completed responses), which is how a stall is told apart. func lr(c int, rps, p99, errs float64) LoadResult { return LoadResult{Scenario: "read", Concurrency: c, RPS: rps, P99Ms: p99, ErrorRate: errs, Status: map[string]int{"200": int(rps*10) + 1}} } func TestKneeIsFirstDoublingWithLittleGain(t *testing.T) { sweep := []LoadResult{lr(1, 20, 60, 0), lr(2, 40, 60, 0), lr(4, 79, 61, 0), lr(8, 150, 70, 0), lr(16, 160, 130, 0), lr(32, 162, 260, 0)} k := Knee(sweep) if !k.Found || k.Concurrency != 8 || k.PeakRPS != 162 { t.Fatalf("got %+v, want slowdown after 8 (16 added under 10%%), peak 162", k) } if k := Knee(sweep[:4]); k.Found || k.PeakRPS != 150 || k.MaxTested != 8 { t.Fatalf("got %+v, want not found within 8", k) } } func TestSweepStopsOnErrorsOrFlatThroughput(t *testing.T) { if !SweepShouldStop([]LoadResult{lr(1, 20, 60, 0), lr(2, 40, 60, 0.02)}) { t.Fatal("should stop on >1% errors") } if !SweepShouldStop([]LoadResult{lr(4, 100, 60, 0), lr(8, 105, 90, 0), lr(16, 108, 180, 0)}) { t.Fatal("should stop after two flat doublings") } if SweepShouldStop([]LoadResult{lr(4, 100, 60, 0), lr(8, 105, 90, 0), lr(16, 190, 90, 0)}) { t.Fatal("one flat step then growth: keep going") } } func TestKneeIgnoresAOneStepBlip(t *testing.T) { sweep := []LoadResult{lr(1, 58, 29, 0), lr(2, 108, 38, 0), lr(4, 98, 66, 0), lr(8, 184, 77, 0), lr(16, 314, 97, 0), lr(32, 325, 294, 0), lr(64, 324, 640, 0)} if k := Knee(sweep); !k.Found || k.Concurrency != 16 || k.PeakRPS != 325 { t.Fatalf("got %+v, want 16: the fewest connections from which no more load adds 10%%", k) } } func TestKneeStopsAtErrors(t *testing.T) { sweep := []LoadResult{lr(16, 400, 60, 0), lr(32, 800, 70, 0), lr(64, 1300, 90, 0), lr(128, 1360, 370, 0.40)} k := Knee(sweep) if k.ErrorsStatus != "" { t.Fatalf("no breakdown recorded: status should be empty, got %q", k.ErrorsStatus) } if !k.Found || k.Concurrency != 64 || k.ErrorsAt != 128 || k.PeakRPS != 1300 { t.Fatalf("got %+v: want slow-down 64 (last clean step), errors from 128, peak from clean steps only", k) } } func TestApplyRecordAppendsNoteOnce(t *testing.T) { r := AppResult{} ApplyRecord(&r, Record{Note: "exits under load"}) ApplyRecord(&r, Record{Note: "exits under load"}) if len(r.Notes) != 1 || r.Notes[0] != "exits under load" { t.Fatalf("got %q", r.Notes) } } func TestAsleepFromLogs(t *testing.T) { logs := `2026-10-24 09:50:52 [platform] todo-zig-2.bfj.oncarlos.com instance stopping 2026-10-24 09:50:53 [platform] todo-zig-2.bfj.oncarlos.com instance exited (exit 0) 2026-10-24 10:01:00 [platform] todo-zig.bfj.oncarlos.com instance woke in 1.1s (spawn 1s) (version x)` if !AsleepFromLogs(logs, "todo-zig-2.bfj.oncarlos.com") { t.Fatal("last event for the host is an exit: asleep") } logs += "\n2026-10-24 10:36:41 [platform] todo-zig-2.bfj.oncarlos.com instance woke in 1.156s (spawn 1.037s) (version 73268e0)" if AsleepFromLogs(logs, "todo-zig-2.bfj.oncarlos.com") { t.Fatal("last event is a wake: awake") } if AsleepFromLogs("", "todo-zig-2.bfj.oncarlos.com") { t.Fatal("no events: unknown, so not asleep") } } func TestParseProcStatus(t *testing.T) { status := "Name:\tapp\nVmHWM:\t 20480 kB\nVmRSS:\t 12288 kB\nThreads:\t4\n" rss, hwm, err := ParseProcStatus(status) if err != nil || rss != 12288 || hwm != 20480 { t.Fatalf("got rss %d hwm %d err %v", rss, hwm, err) } } func TestMeasureMemoryOfRefapp(t *testing.T) { if testing.Short() { t.Skip("builds and loads the reference app") } if _, err := exec.LookPath("oha"); err != nil { t.Skip("oha not installed") } bin := filepath.Join(t.TempDir(), "refapp") if out, err := exec.Command("go", "build", "-o", bin, "./refapp").CombinedOutput(); err != nil { t.Fatalf("%v\n%s", err, out) } shared, _ := filepath.Abs("../shared") m, err := MeasureMemory(bin, []string{"--shared", shared}, time.Second) if err != nil { t.Fatal(err) } if m.IdleKB <= 0 || m.PeakKB < m.IdleKB || m.AfterKB <= 0 { t.Fatalf("implausible %+v", m) } } func TestKneeNamesTheFailingStatus(t *testing.T) { bad := lr(128, 1360, 370, 0.40) bad.Status = map[string]int{"200": 600, "503": 400} k := Knee([]LoadResult{lr(32, 800, 70, 0), lr(64, 1300, 90, 0), bad}) if k.ErrorsStatus != "503" { t.Fatalf("want 503, got %q", k.ErrorsStatus) } bad.Status = map[string]int{"200": 600, "502": 300, "503": 100} if k := Knee([]LoadResult{lr(64, 1300, 90, 0), bad}); k.ErrorsStatus != "502" { t.Fatalf("want the most common failing status 502, got %q", k.ErrorsStatus) } } func TestKneeTreatsAStalledStepAsFailure(t *testing.T) { stall := lr(32, 3.2, 0, 0) stall.Status = nil ok := lr(16, 500, 40, 0) ok.Status = map[string]int{"200": 5000} k := Knee([]LoadResult{lr(8, 300, 30, 0), ok, stall, lr(64, 700, 90, 0)}) if k.ErrorsAt != 32 || k.ErrorsStatus != "stalled" || k.PeakRPS != 500 || k.Concurrency != 16 { t.Fatalf("a step that completed nothing is a failure, got %+v", k) } } func TestParseOhaNoCompletionsIsAllErrors(t *testing.T) { b := []byte(`{"summary":{"requestsPerSec":3.2},"latencyPercentiles":{},"statusCodeDistribution":{},"errorDistribution":{"aborted due to deadline":32}}`) r, err := ParseOha(b, 200) if err != nil { t.Fatal(err) } if r.ErrorRate != 1 || r.RPS != 0 { t.Fatalf("no completed response: want error rate 1 and 0 rps, got %+v", r) } }