package export_test // A small PDF reader, for tests only. // // The point of it is that a PDF export cannot be checked by comparing // bytes — the interesting properties are "a reader can find the // objects", "the pages are there", and "the text can be selected and // copied back out", and none of those is visible in a byte comparison. // This walks the file the way a reader does: through the trailer to // the cross-reference table, through the offsets in it to the objects, // and through each font's ToUnicode CMap to turn the glyph ids in a // content stream back into words. // // It follows the file's own xref offsets rather than scanning for // "obj", which means a wrong offset fails the test rather than being // routed around — the offsets are exactly the kind of thing that is // silently wrong in a hand-written PDF writer. import ( "bytes" "compress/zlib" "fmt" "io" "regexp" "strconv" "strings" "testing" ) type pdfObject struct { dict string stream []byte // already decompressed } type pdfFile struct { objects map[int]pdfObject } func readPDF(t *testing.T, data []byte) pdfFile { t.Helper() if !bytes.HasPrefix(data, []byte("%PDF-1.")) { t.Fatalf("not a PDF: starts with %q", firstLine(data)) } if !bytes.HasSuffix(bytes.TrimRight(data, "\n"), []byte("%%EOF")) { t.Fatal("file does not end with the end-of-file marker") } // startxref, then the offset on the following line. idx := bytes.LastIndex(data, []byte("startxref")) if idx < 0 { t.Fatal("no startxref") } var xrefAt int if _, err := fmt.Sscanf(string(data[idx:]), "startxref\n%d", &xrefAt); err != nil { t.Fatalf("unreadable startxref: %v", err) } if xrefAt <= 0 || xrefAt >= len(data) { t.Fatalf("startxref points outside the file: %d of %d", xrefAt, len(data)) } var first, count int if _, err := fmt.Sscanf(string(data[xrefAt:]), "xref\n%d %d", &first, &count); err != nil { t.Fatalf("unreadable xref header at %d: %v", xrefAt, err) } if first != 0 { t.Fatalf("xref starts at object %d, want 0", first) } // Entries are exactly 20 bytes each, starting after the two header // lines. Finding the entry table by scanning past the header keeps // this honest about the header's own length. entriesAt := xrefAt + strings.Index(string(data[xrefAt:]), "\n") + 1 entriesAt += strings.Index(string(data[entriesAt:]), "\n") + 1 out := pdfFile{objects: map[int]pdfObject{}} for i := 1; i < count; i++ { at := entriesAt + i*20 if at+20 > len(data) { t.Fatalf("xref entry %d runs past the end of the file", i) } entry := string(data[at : at+20]) off, err := strconv.Atoi(strings.TrimSpace(entry[:10])) if err != nil { t.Fatalf("xref entry %d is not a number: %q", i, entry) } out.objects[i] = readObject(t, data, i, off) } return out } var lengthRe = regexp.MustCompile(`/Length (\d+)`) func readObject(t *testing.T, data []byte, num, off int) pdfObject { t.Helper() if off <= 0 || off >= len(data) { t.Fatalf("object %d: offset %d is outside the file", num, off) } want := fmt.Sprintf("%d 0 obj\n", num) if !bytes.HasPrefix(data[off:], []byte(want)) { t.Fatalf("object %d: xref points at %q, not %q", num, firstLine(data[off:]), strings.TrimSpace(want)) } body := data[off+len(want):] streamAt := bytes.Index(body, []byte("stream\n")) endObj := bytes.Index(body, []byte("\nendobj")) if streamAt < 0 || (endObj >= 0 && endObj < streamAt) { if endObj < 0 { t.Fatalf("object %d: no endobj", num) } return pdfObject{dict: string(body[:endObj])} } dict := string(body[:streamAt]) m := lengthRe.FindStringSubmatch(dict) if m == nil { t.Fatalf("object %d: a stream with no /Length", num) } n, _ := strconv.Atoi(m[1]) start := streamAt + len("stream\n") if start+n > len(body) { t.Fatalf("object %d: /Length %d runs past the end of the file", num, n) } raw := body[start : start+n] if !strings.Contains(dict, "/FlateDecode") { return pdfObject{dict: dict, stream: raw} } zr, err := zlib.NewReader(bytes.NewReader(raw)) if err != nil { t.Fatalf("object %d: stream is not valid zlib: %v", num, err) } plain, err := io.ReadAll(zr) if err != nil { t.Fatalf("object %d: decompressing: %v", num, err) } return pdfObject{dict: dict, stream: plain} } func firstLine(b []byte) string { if i := bytes.IndexByte(b, '\n'); i >= 0 { b = b[:i] } if len(b) > 60 { b = b[:60] } return string(b) } // pages returns every page object's dictionary, in /Kids order. func (f pdfFile) pages(t *testing.T) []pdfObject { t.Helper() var pagesDict string for _, o := range f.objects { if strings.Contains(o.dict, "/Type /Pages") { pagesDict = o.dict } } if pagesDict == "" { t.Fatal("no /Type /Pages node") } kids := regexp.MustCompile(`(\d+) 0 R`).FindAllStringSubmatch( betweenBrackets(t, pagesDict, "/Kids ["), -1) var out []pdfObject for _, k := range kids { n, _ := strconv.Atoi(k[1]) o, ok := f.objects[n] if !ok { t.Fatalf("/Kids names object %d, which the xref does not list", n) } if !strings.Contains(o.dict, "/Type /Page") { t.Fatalf("object %d is in /Kids but is not a /Page", n) } out = append(out, o) } return out } func betweenBrackets(t *testing.T, s, open string) string { t.Helper() i := strings.Index(s, open) if i < 0 { t.Fatalf("%q not found in %q", open, s) } rest := s[i+len(open):] j := strings.Index(rest, "]") if j < 0 { t.Fatalf("unterminated %q", open) } return rest[:j] } var ( fontResRe = regexp.MustCompile(`/(F\d+) (\d+) 0 R`) contentsRe = regexp.MustCompile(`/Contents (\d+) 0 R`) toUnicodeRe = regexp.MustCompile(`/ToUnicode (\d+) 0 R`) bfcharRe = regexp.MustCompile(`<([0-9A-Fa-f]+)> <([0-9A-Fa-f]+)>`) // One drawn run, exactly as internal/pdf's Page.Text writes it: // the text matrix carries the baseline position, so the x and y a // run actually landed at are readable here and nowhere else. showRe = regexp.MustCompile( `/(F\d+) [\d.-]+ Tf 1 0 0 1 ([\d.-]+) ([\d.-]+) Tm <([0-9A-Fa-f]*)> Tj`) // One filled rectangle (Page.Rule) — the blockquote's edge is the // only non-text drawing this exporter does. ruleRe = regexp.MustCompile(`([\d.-]+) ([\d.-]+) ([\d.-]+) ([\d.-]+) re f`) ) // run is one drawn piece of text with where it was drawn: x and y are // the baseline's own coordinates, in points from the bottom-left of the // page, which is PDF's origin and not the layout's. type run struct { x, y float64 text string } // cmaps maps each font resource named by a page to its own ToUnicode // table — the glyph-id-to-rune mapping a reader uses to let a person // copy text out. func (f pdfFile) cmaps(t *testing.T, page pdfObject) map[string]map[uint16]rune { t.Helper() out := map[string]map[uint16]rune{} for _, m := range fontResRe.FindAllStringSubmatch(page.dict, -1) { n, _ := strconv.Atoi(m[2]) fontObj, ok := f.objects[n] if !ok { t.Fatalf("page resources name font object %d, which does not exist", n) } tu := toUnicodeRe.FindStringSubmatch(fontObj.dict) if tu == nil { t.Fatalf("font %s has no /ToUnicode — its text cannot be copied out", m[1]) } tn, _ := strconv.Atoi(tu[1]) cmap := map[uint16]rune{} for _, e := range bfcharRe.FindAllStringSubmatch(string(f.objects[tn].stream), -1) { gid, _ := strconv.ParseUint(e[1], 16, 16) cmap[uint16(gid)] = decodeUTF16BE(e[2]) } out[m[1]] = cmap } return out } func (f pdfFile) content(t *testing.T, page pdfObject) string { t.Helper() cm := contentsRe.FindStringSubmatch(page.dict) if cm == nil { t.Fatal("page has no /Contents") } cn, _ := strconv.Atoi(cm[1]) return string(f.objects[cn].stream) } // runs returns every drawn piece of text on the page WITH its baseline // position, in the order the content stream draws them. // // text below throws the positions away, which is why a whole-page // horizontal offset was invisible to every test in this package until // this existed: the words all copied back out correctly, they were just // drawn in the margin. func (f pdfFile) runs(t *testing.T, page pdfObject) []run { t.Helper() cmaps := f.cmaps(t, page) var out []run for _, m := range showRe.FindAllStringSubmatch(f.content(t, page), -1) { cmap, ok := cmaps[m[1]] if !ok { t.Fatalf("content stream uses font %s, which the page resources do not declare", m[1]) } x, err := strconv.ParseFloat(m[2], 64) if err != nil { t.Fatalf("unreadable x in the text matrix: %q", m[2]) } y, err := strconv.ParseFloat(m[3], 64) if err != nil { t.Fatalf("unreadable y in the text matrix: %q", m[3]) } var b strings.Builder hex := m[4] for i := 0; i+4 <= len(hex); i += 4 { gid, _ := strconv.ParseUint(hex[i:i+4], 16, 16) r, ok := cmap[uint16(gid)] if !ok { t.Errorf("glyph %d is drawn but has no ToUnicode entry", gid) continue } b.WriteRune(r) } out = append(out, run{x: x, y: y, text: b.String()}) } return out } // rect is one filled rectangle, in PDF coordinates. type rect struct{ x, y, w, h float64 } // rules returns every filled rectangle on the page — the blockquote // edge, which is drawn at an x of its own and would sit in the margin // alongside the text if that x were wrong. func (f pdfFile) rules(t *testing.T, page pdfObject) []rect { t.Helper() var out []rect for _, m := range ruleRe.FindAllStringSubmatch(f.content(t, page), -1) { var r rect for i, dst := range []*float64{&r.x, &r.y, &r.w, &r.h} { v, err := strconv.ParseFloat(m[i+1], 64) if err != nil { t.Fatalf("unreadable rectangle coordinate %q", m[i+1]) } *dst = v } out = append(out, r) } return out } // text reconstructs what a person would get by selecting everything on // the page and copying it: the glyph ids in the content stream, mapped // back through each font's own ToUnicode CMap. If that mapping is // wrong or missing, this returns mojibake, which is exactly what a // reader would show. func (f pdfFile) text(t *testing.T, page pdfObject) string { t.Helper() var b strings.Builder for _, r := range f.runs(t, page) { b.WriteString(r.text) } return b.String() } func decodeUTF16BE(hex string) rune { var units []rune for i := 0; i+4 <= len(hex); i += 4 { v, _ := strconv.ParseUint(hex[i:i+4], 16, 16) units = append(units, rune(v)) } if len(units) == 2 && units[0] >= 0xD800 && units[0] <= 0xDBFF { return ((units[0] - 0xD800) << 10) + (units[1] - 0xDC00) + 0x10000 } if len(units) == 0 { return 0 } return units[0] }