package main import ( "encoding/csv" "encoding/json" "fmt" "os" "strconv" "strings" ) // readResults parses the CSV soffice wrote into plain strings, in row // order — pairing them back up with cases is main's job, by index. The // value read is field formulaColumnIndex (column D), NOT field 0: // columns A:C are the fixed support table the lookup cases reference // (see ods.go's lookupTableRows), so field 0 is "10", "20", "30", then // nothing. A short row means the formula cell exported nothing at all, // which checkResults treats as a failed run rather than an empty value. func readResults(path string) ([]string, error) { f, err := os.Open(path) if err != nil { return nil, err } defer f.Close() r := csv.NewReader(f) r.FieldsPerRecord = -1 rows, err := r.ReadAll() if err != nil { return nil, err } out := make([]string, len(rows)) for i, row := range rows { if len(row) > formulaColumnIndex { out[i] = row[formulaColumnIndex] } } return out, nil } // corpusValueJSON mirrors calc's own corpusValue wire format (see // internal/sheets/calc/corpus_test.go) — duplicated here rather than // imported, since gencorpus is a standalone tool and calc's copy is // unexported test-only plumbing that this package has no business // depending on. type corpusValueJSON struct { Kind string `json:"kind"` Num float64 `json:"num,omitempty"` Str string `json:"str,omitempty"` Bool bool `json:"bool,omitempty"` Err string `json:"err,omitempty"` } type corpusEntryJSON struct { Function string `json:"function"` Expr string `json:"expr"` Want corpusValueJSON `json:"want"` } type corpusFileJSON struct { Note string `json:"note"` Entries []corpusEntryJSON `json:"entries"` } // classify turns one CSV cell's text into corpusValueJSON. LibreOffice's // CSV export renders TRUE/FALSE for booleans, the six Excel error // strings verbatim for errors, a plain decimal for numbers, and anything // else as text. func classify(s string) corpusValueJSON { switch s { case "TRUE": return corpusValueJSON{Kind: "bool", Bool: true} case "FALSE": return corpusValueJSON{Kind: "bool", Bool: false} case "#DIV/0!", "#VALUE!", "#REF!", "#NAME?", "#NUM!", "#N/A": return corpusValueJSON{Kind: "err", Err: s} } if n, err := strconv.ParseFloat(s, 64); err == nil { return corpusValueJSON{Kind: "num", Num: n} } return corpusValueJSON{Kind: "str", Str: s} } // writeCorpusJSON writes the corpus file. It does NOT hand the whole // document to json.MarshalIndent: that puts every field of every entry // on its own line, so a regeneration that changed one value still // produces a several-hundred-line diff and the one row a reviewer needs // to see is buried in it. Each entry is written on a single line // instead, in the exact spacing the file has always had, so `git diff` // after a regeneration shows precisely the rows LibreOffice answered // differently and nothing else. Reviewing that diff is the whole point // of this tool's workflow (see main's closing message); a format that // defeats it is not a neutral choice. func writeCorpusJSON(path string, cases []corpusCase, results []string, version string) error { note, err := json.Marshal(corpusNote(path, version)) if err != nil { return err } var b strings.Builder b.WriteString("{\n \"note\": ") b.Write(note) b.WriteString(",\n \"entries\": [\n") for i, c := range cases { line, err := entryLine(corpusEntryJSON{Function: c.Function, Expr: c.Expr, Want: classify(results[i])}) if err != nil { return err } b.WriteString(" " + line) if i < len(cases)-1 { b.WriteString(",") } b.WriteString("\n") } b.WriteString(" ]\n}\n") return os.WriteFile(path, []byte(b.String()), 0o644) } // entryLine renders one entry as a single JSON object. Every scalar goes // through json.Marshal rather than fmt, so quoting and float formatting // are the encoding/json package's problem, not this function's; only the // whitespace between fields is chosen here. func entryLine(e corpusEntryJSON) (string, error) { fn, err := json.Marshal(e.Function) if err != nil { return "", err } expr, err := json.Marshal(e.Expr) if err != nil { return "", err } want, err := wantLine(e.Want) if err != nil { return "", err } return fmt.Sprintf(`{"function": %s, "expr": %s, "want": %s}`, fn, expr, want), nil } // wantLine renders one "want" value. The kind's own field is always // written, zero value included: corpusValueJSON's omitempty tags would // drop "num": 0 and "bool": false entirely, leaving a bare {"kind": // "num"} that reads like a value the tool failed to record rather than // like the number zero. A corpus is read by people deciding whether it // is trustworthy; it should not make its own honest answers look like // gaps. func wantLine(v corpusValueJSON) (string, error) { var payload any switch v.Kind { case "num": payload = v.Num case "str": payload = v.Str case "bool": payload = v.Bool case "err": payload = v.Err default: return `{"kind": "` + v.Kind + `"}`, nil } enc, err := json.Marshal(payload) if err != nil { return "", err } return fmt.Sprintf(`{"kind": %q, "%s": %s}`, v.Kind, v.Kind, enc), nil } // existingEntryCount reads path's current entry count, if the file // exists and parses — used by main to refuse a run that would silently // narrow an already-committed corpus. A missing or unparsable file // returns a non-nil error, which main treats as "nothing to compare // against" rather than a reason to fail. func existingEntryCount(path string) (int, error) { data, err := os.ReadFile(path) if err != nil { return 0, err } var cf corpusFileJSON if err := json.Unmarshal(data, &cf); err != nil { return 0, err } return len(cf.Entries), nil } // corpusNote is the provenance line written into the corpus file // itself. It names the exact LibreOffice build that computed the values // (captured from `soffice --version` at run time, not typed in by hand) // because "generated" without a version is not a reproducible claim, and // it names the one place where the recorded values are not literally // what LibreOffice printed. func corpusNote(path, version string) string { return "GENERATED by tools/gencorpus from a real headless " + version + " run: each entry's \"want\" is what that LibreOffice computed for that " + "exact formula, exported via `--convert-to csv`, not a value anyone " + "reasoned out. Recalculation on export is proven per run, not assumed: " + "every formula cell is written with a poisoned office:value cache (999999) " + "and the run fails if that value comes back. Two honest caveats. (1) The " + "values come through LibreOffice's CSV export, which renders a number at " + "roughly 15 significant digits with trailing zeros stripped -- SQRT(2) is " + "recorded as 1.4142135623731, not the full 1.4142135623730951; " + "corpus_test.go's valuesMatch compares numbers with a 1e-9 epsilon, so this " + "is a rendering artifact, not a disagreement. (2) LibreOffice Calc is not Excel. Where the two are known " + "to differ, the difference is recorded and reasoned about in " + "internal/sheets/xlsx/oracle_divergences_test.go, cross-checked against " + "excelize's independent formula engine -- this file records Calc's answer. " + "Regenerate with `go run ./tools/gencorpus -out " + path + "` (add -soffice /path/to/soffice if it is not on PATH) whenever a case is " + "added; commit the result and update corpusSHA256 in " + "internal/sheets/calc/corpus_test.go." }