// Command bakeoff measures one app and records the result in // results/.json. Run it from the repo root. // // bakeoff --slug [flags] package main import ( "encoding/json" "errors" "flag" "fmt" "io" "os" "os/exec" "path/filepath" "strconv" "strings" "time" b "bakeoff/harness" "bakeoff/harness/report" ) func main() { if len(os.Args) < 2 { usage() } cmd := os.Args[1] fs := flag.NewFlagSet(cmd, flag.ExitOnError) slug := fs.String("slug", "", "app slug, e.g. rust") url := fs.String("url", "", "app base URL (default https://todo-.bfj.oncarlos.com)") results := fs.String("results", "results", "results directory") dir := fs.String("dir", "", "app directory (default apps/)") runs := fs.Int("runs", 3, "build: warm and one-line-change runs") coldRuns := fs.Int("cold-runs", 1, "build: cold runs") dur := fs.Duration("duration", 10*time.Second, "load: length of each step") rest := fs.Duration("rest", 10*time.Second, "load: pause between steps") levels := fs.String("levels", "1,2,4,8,16,32,64,128,256,512,1024", "load: concurrency levels, doubling") phase := fs.String("phase", "", "tokens: build|fix|security|review") logs := fs.String("logs", "builders/logs", "tokens: log directory") since := fs.String("since", "", "tokens: log whose cumulative totals this resumed phase starts from (default: the previous phase's)") file := fs.String("file", "", "behaviour: Playwright JSON report") var rec b.Record fs.StringVar(&rec.Language, "language", "", "record: display name") fs.StringVar(&rec.Status, "status", "", "record: LIVE or BLOCKED ") fs.StringVar(&rec.Version, "version", "", "record: deployed version") fs.StringVar(&rec.Stack, "stack", "", "record: one-line stack summary") fs.StringVar(&rec.Source, "source", "", "record: source URL") fs.StringVar(&rec.Note, "note", "", "record: a note shown in the app's section (added once)") recURL := fs.String("app-url", "", "record: live URL") clearNotes := fs.Bool("clear-notes", false, "record: drop existing notes first") clearWake := fs.Bool("clear", false, "wake: drop recorded samples") preview := fs.Bool("preview", false, "report: tag strings for copy review") replies := fs.String("replies", "", "readability: comma-separated files, one scorer reply each") fs.Parse(os.Args[2:]) if *slug == "" && cmd != "report" && cmd != "writeup" { usage() } b.ResultsDir = *results if *url == "" { *url = b.URL(*slug) } if *dir == "" { *dir = filepath.Join("apps", *slug) } var err error switch cmd { case "ttfb": err = ttfb(*slug, *url) case "wake": if *clearWake { err = b.Update(*slug, func(r *b.AppResult) { r.Wake = nil }) break } err = wake(*slug, *url) case "load": err = load(*slug, *url, *levels, *dur, *rest) case "weight": err = weight(*slug, *url) case "code": err = code(*slug, *dir) case "build": err = build(*slug, *dir, *coldRuns, *runs) case "tokens": err = tokens(*slug, *logs, *phase, *since) case "behaviour": err = behaviour(*slug, *file) case "writeup": err = renderWriteup(*file) case "report": err = renderReport(*file, *preview) case "findings": err = findings(*slug, *file) case "probe": err = probe(*url, *levels, *dur) case "memory": err = memory(*slug, *dir, *dur) case "record": if rec.URL = *recURL; rec.URL == "" && strings.HasPrefix(rec.Status, "LIVE") { rec.URL = *url } err = b.Update(*slug, func(r *b.AppResult) { if *clearNotes { r.Notes = nil } b.ApplyRecord(r, rec) }) case "source": err = source(*dir) case "readability": err = readability(*slug, *replies) default: usage() } if err != nil { fmt.Fprintf(os.Stderr, "bakeoff %s %s: %v\n", cmd, *slug, err) os.Exit(1) } } func usage() { fmt.Fprintln(os.Stderr, "usage: bakeoff --slug [flags]") os.Exit(2) } func seeded(url string, n int) (*b.Session, error) { s, err := b.NewSession(url) if err != nil { return nil, err } if n > 0 { _, err = b.Seed(s, n) } return s, err } func ttfb(slug, url string) error { empty, err := seeded(url, 0) if err != nil { return err } e, err := b.MeasureTTFB(empty, "/", 3, 30) if err != nil { return err } full, err := seeded(url, 50) if err != nil { return err } f, err := b.MeasureTTFB(full, "/", 3, 30) if err != nil { return err } fmt.Printf("%s ttfb empty p50 %.1fms, 50 items p50 %.1fms\n", slug, e.P50, f.P50) return b.Update(slug, func(r *b.AppResult) { r.TTFBEmpty, r.TTFB50 = e, f }) } // wake takes one sample if the instance is asleep, and exits 3 if it is // awake, so a polling loop can tell "not asleep yet" from failure. func wake(slug, url string) error { asleep, err := b.IsAsleep(b.AppName(slug), b.Host(slug)) if err != nil { return err } if !asleep { fmt.Printf("%s is awake; no sample\n", slug) os.Exit(3) } // A fresh visitor, without first fetching / (that would wake it). s, err := b.SessionNoFetch(url) if err != nil { return err } ms, err := b.MeasureWake(s, "/") if err != nil { return err } fmt.Printf("%s wake %.1fms\n", slug, ms) return b.Update(slug, func(r *b.AppResult) { r.Wake = append(r.Wake, ms) }) } // load ramps each scenario through the levels, stopping early when the // app has stopped scaling or is failing (b.SweepShouldStop). func load(slug, url, levels string, d, rest time.Duration) error { s, err := seeded(url, 50) if err != nil { return err } var out []b.LoadResult for _, sc := range []string{"read", "toggle", "read-after"} { var sweep []b.LoadResult for _, l := range strings.Split(levels, ",") { c, err := strconv.Atoi(strings.TrimSpace(l)) if err != nil { return err } if len(out)+len(sweep) > 0 { time.Sleep(rest) } r, err := b.RunLoad(s, sc, c, d) if err != nil { // A step that can't run at all (oha failed, or the app is // down) is a step where every request failed: record it and // end this scenario, rather than losing the whole run. fmt.Printf("%s %s c=%d: step failed: %v\n", slug, sc, c, err) r = b.LoadResult{Scenario: sc, Concurrency: c, ErrorRate: 1, Errors: map[string]int{err.Error(): 1}} sweep = append(sweep, r) break } fmt.Printf("%s %s c=%d: %.0f rps, p50 %.1fms, p99 %.1fms, errors %.2f%%", slug, sc, c, r.RPS, r.P50Ms, r.P99Ms, r.ErrorRate*100) if r.ErrorRate > 0 { fmt.Printf(" status %v errors %v", r.Status, r.Errors) } fmt.Println() sweep = append(sweep, r) if b.SweepShouldStop(sweep) { break } } k := b.Knee(sweep) fmt.Printf("%s %s: peak %.0f rps; slows down after %d (found %v, tested to %d)\n", slug, sc, k.PeakRPS, k.Concurrency, k.Found, k.MaxTested) out = append(out, sweep...) } return b.Update(slug, func(r *b.AppResult) { r.Load = out }) } func weight(slug, url string) error { s, err := seeded(url, 50) if err != nil { return err } w, err := b.PageWeight(s, "/") if err != nil { return err } fmt.Printf("%s page weight (50 items): html %d bytes, js %d bytes\n", slug, w.HTMLBytes, w.JSBytes) return b.Update(slug, func(r *b.AppResult) { r.Weight = w }) } func code(slug, dir string) error { c, err := b.CountCode(dir) if err != nil { return err } fmt.Printf("%s code: %+v\n", slug, c) return b.Update(slug, func(r *b.AppResult) { r.Code = c }) } func build(slug, dir string, coldRuns, runs int) error { st, err := b.BuildStats(dir, strings.HasPrefix(slug, "go"), coldRuns, runs) if err != nil { return err } fmt.Printf("%s build: cold %.1fs, warm %.1fs, edit %.1fs, %d bytes on %s\n", slug, st.ColdSeconds, st.WarmSeconds, st.EditSeconds, st.Bytes, st.Host) return b.Update(slug, func(r *b.AppResult) { r.Build = st }) } func tokens(slug, logs, phase, since string) error { parse := b.ParseClaudeLog if phase == "review" { parse = b.ParseCodexLog } else if phase != "build" && phase != "fix" && phase != "security" { return errors.New("--phase must be build, fix, security or review") } files, _ := filepath.Glob(filepath.Join(logs, slug+"."+phase+"*.jsonl")) if len(files) == 0 { return fmt.Errorf("no %s logs for %s in %s", phase, slug, logs) } var total b.Tokens for _, f := range files { fh, err := os.Open(f) if err != nil { return err } t, err := parse(fh) fh.Close() if err != nil { return fmt.Errorf("%s: %w", f, err) } total = total.Add(t) } // Fix and security rounds resume the build session, whose result // events report usage cumulatively: subtract the phase before. if phase == "fix" || phase == "security" { prev := []string{"build"} if phase == "security" { prev = []string{"fix", "build"} } if since != "" { prev = []string{since} } for _, p := range prev { f := filepath.Join(logs, slug+"."+p+".jsonl") if since != "" { f = since } fh, err := os.Open(f) if err != nil { continue } before, err := b.ParseClaudeLog(fh) fh.Close() if err != nil { return fmt.Errorf("%s: %w", f, err) } total = total.Since(before) break } } fmt.Printf("%s %s tokens: %+v\n", slug, phase, total) return b.Update(slug, func(r *b.AppResult) { switch phase { case "build": r.TokensBuild = total case "fix": r.TokensFix = total case "security": r.TokensSecurity = total case "review": r.TokensReview = total } }) } func behaviour(slug, file string) error { var in io.Reader = os.Stdin if file != "" { f, err := os.Open(file) if err != nil { return err } defer f.Close() in = f } got, err := b.ParsePlaywright(in) if err != nil { return err } fmt.Printf("%s behaviour: js-on %+v, js-off %+v\n", slug, got["js-on"], got["js-off"]) return b.Update(slug, func(r *b.AppResult) { r.BehaviourJSOn, r.BehaviourJSOff = got["js-on"], got["js-off"] }) } // source prints the app's own code, file by file, for the readability scorer. func source(dir string) error { files, err := b.SourceFiles(dir) if err != nil { return err } for _, f := range files { body, err := os.ReadFile(filepath.Join(dir, f)) if err != nil { return err } fmt.Printf("===== %s =====\n%s\n", f, body) } return nil } func readability(slug, replies string) error { var texts []string for _, f := range strings.Split(replies, ",") { body, err := os.ReadFile(f) if err != nil { return err } texts = append(texts, string(body)) } r, err := b.MedianReadability(texts) if err != nil { return err } fmt.Printf("%s readability: overall %.0f/10 (naming %.0f, structure %.0f, traceability %.0f, errors %.0f, comments %.0f)\n", slug, r.Overall, r.Naming, r.Structure, r.Traceability, r.Errors, r.Comments) return b.Update(slug, func(a *b.AppResult) { a.Readability = r }) } // renderReport writes the report page to stdout from every results file, // in the contestants' fixed order, with prose from the copy file. func renderReport(copyFile string, preview bool) error { if copyFile == "" { copyFile = "copy-review/strings.json" } text, err := os.ReadFile(copyFile) if err != nil { return err } words, err := report.LoadStrings(text) if err != nil { return err } var rs []b.AppResult for _, a := range b.Apps { r, err := b.LoadIn(b.ResultsDir, a.Slug) if err != nil { return err } if r.Language == "" { r.Language = a.Language } if r.Status == "" { r.Status = "not built yet" } rs = append(rs, r) } if preview { return report.RenderPreview(rs, words, os.Stdout) } return report.Render(rs, words, os.Stdout) } // findings records what a review left unfixed and exits 4 if any of it // is a security finding (which earns the security-only round). func findings(slug, file string) error { text, err := os.ReadFile(file) if err != nil { return err } fs := b.ParseFindings(string(text)) fmt.Printf("%s: %d findings remain\n", slug, len(fs)) if err := b.Update(slug, func(r *b.AppResult) { r.ReviewFindings = fs }); err != nil { return err } if b.HasSecurity(fs) { fmt.Printf("%s: security findings remain\n", slug) os.Exit(4) } return nil } // probe runs one toggle step per level and prints oha's raw status and // error breakdown, to diagnose failures a ramp reports. func probe(url, levels string, d time.Duration) error { s, err := seeded(url, 50) if err != nil { return err } ids, err := s.IDs("/") if err != nil { return err } for _, l := range strings.Split(levels, ",") { out, err := exec.Command("oha", "--no-tui", "--output-format", "json", "-z", d.String(), "-c", strings.TrimSpace(l), "--redirect", "0", "-H", "Cookie: "+s.CookieHeader(), "-H", "Origin: "+url, "-m", "POST", "-T", "application/x-www-form-urlencoded", "-d", "filter=all", url+"/todos/"+ids[0]+"/toggle").Output() if err != nil { return err } var o struct { Status map[string]int `json:"statusCodeDistribution"` Errors map[string]int `json:"errorDistribution"` } json.Unmarshal(out, &o) fmt.Printf("c=%s status %v errors %v\n", l, o.Status, o.Errors) } return nil } // memory measures the app's own host build (dist/app, from make build) // locally: resident memory at rest, peak under load, and after. func memory(slug, dir string, d time.Duration) error { bin, err := filepath.Abs(filepath.Join(dir, "dist", "app")) if err != nil { return err } m, err := b.MeasureMemory(bin, nil, d) if err != nil { return err } fmt.Printf("%s memory: idle %d KB, peak %d KB, after %d KB\n", slug, m.IdleKB, m.PeakKB, m.AfterKB) return b.Update(slug, func(r *b.AppResult) { r.Memory = m }) } // renderWriteup writes docs/writeup.md (or --file) as the write-up page. func renderWriteup(file string) error { if file == "" { file = "docs/writeup.md" } src, err := os.ReadFile(file) if err != nil { return err } text, err := os.ReadFile("copy-review/strings.json") if err != nil { return err } words, err := report.LoadStrings(text) if err != nil { return err } return report.RenderWriteup(string(src), words, os.Stdout) }