//! Rendering a ProseMirror-shaped Y.XmlFragment to text and HTML. //! //! This is the only place on the Rust side that knows ProseMirror's //! node vocabulary. It exists because the read view is server-rendered //! (spec §6.2: reading a document works with no script), so the server //! must be able to render what the browser's editor wrote. //! //! Unknown nodes render their children rather than disappearing. A //! future editor build can introduce a node this binary has never //! heard of, and a reader silently losing that text would be data //! loss, not a degradation. use std::fmt::Write as _; use yrs::types::xml::{XmlFragment, XmlOut, XmlTextRef}; use yrs::{Doc, GetString, ReadTxn, Text, Transact, Xml, XmlElementRef, XmlFragmentRef}; /// The CRDT root name. y-prosemirror's default, and part of the wire /// contract — changing it silently orphans every existing document. pub const ROOT: &str = "prosemirror"; fn root(doc: &Doc) -> XmlFragmentRef { doc.get_or_insert_xml_fragment(ROOT) } fn escape(s: &str, out: &mut String) { for c in s.chars() { match c { '&' => out.push_str("&"), '<' => out.push_str("<"), '>' => out.push_str(">"), '"' => out.push_str("""), '\'' => out.push_str("'"), _ => out.push(c), } } } /// Marks this build knows, in a fixed order so output is deterministic: /// `(mark name, opening markup, closing markup)`. A mark not listed here /// contributes no wrapper but never suppresses its text. /// /// Every one of these is attribute-free on the CRDT side — the mark's /// presence is the whole of its meaning — and the markup each one emits /// is a compile-time constant chosen by lookup from this table. That is /// what keeps `Store.HTML`'s `template.HTML` return honest /// (`internal/docs/store.go`): `fragment_html` escapes every text run /// AND puts no byte from a document into attribute position, so the /// markup is safe to write out unescaped. A mark carrying a value — /// `link`'s `href`, or a `fontFamily` naming its own face — would break /// that, and `POST /d/{sqid}/updates` accepts arbitrary client-supplied /// Yjs bytes that never went through ProseMirror's schema, so the value /// would be genuinely arbitrary. Adding one is a real decision about /// attribute escaping and (for `href`) a scheme allowlist, not a row in /// this table. /// /// Paragraph alignment (ALIGNMENTS below) is that same discipline /// applied to a property that genuinely cannot be a mark: it belongs to /// the block, so it arrives as an attribute WITH a value — and that /// value is used only as a key into a table whose right-hand side is a /// literal in this binary. /// /// The font and size marks are ordered ahead of the semantic ones so a /// run carrying both nests as `…` /// rather than the other way round. /// /// All three faces have a mark; "normal" size still has none. The rule /// is that the DEFAULT carries no mark — it is what a document is when /// nobody has said otherwise, and spending a mark on it would put a /// span around most of the text in most documents. What this table does /// NOT do is decide which face that is: the default is named by /// `internal/fonts`' `Default`, `views/show.html`'s `[docs-text]` rule /// and `fixtures/fonts.mjs`'s `DEFAULT_FONT`, and every face here is /// spelled out so that moving it is a change to those three and not to /// this one. `font_serif` exists for exactly that reason: serif used to /// BE the absence of a mark, and became unsayable the moment it stopped /// being the default. const MARKS: &[(&str, &str, &str)] = &[ ("font_serif", "", ""), ("font_sans", "", ""), ("font_mono", "", ""), // The point ladder. Eleven rows rather than one mark carrying a // number, and that is the whole of this table's discipline applied // to the feature people most expect to be a value: a mark spelling // "14pt" would put a document's own byte into attribute position, // which is exactly what the comment above forbids. Eleven // attribute-free marks spend no new safety property at all. // // 12pt is absent because it is the default, and the default carries // no mark — the same rule that keeps "normal" and the default face // out of this table. ("size_8", "", ""), ("size_9", "", ""), ("size_10", "", ""), ("size_11", "", ""), ("size_14", "", ""), ("size_18", "", ""), ("size_24", "", ""), ("size_36", "", ""), ("size_48", "", ""), ("size_72", "", ""), // The four-step scale the ladder replaced. These stay here // PERMANENTLY, and not out of politeness: documents in the wild // carry them, and a mark this table stops rendering is a mark whose // text silently loses its size for every reader without JavaScript // — the module comment's own "silence here is data loss". They are // simply no longer offered by the control, and the stylesheet maps // each to its nearest ladder step so one document does not read at // two sizes depending on which renderer looked at it. ("size_sm", "", ""), ("size_lg", "", ""), ("size_xl", "", ""), ("strong", "", ""), ("em", "", ""), ("code", "", ""), ]; /// Paragraph alignments this build knows: `(attribute value, class /// name)`. The first block-level formatting, and the first thing in /// this file that reads a document attribute whose value is not a /// number. /// /// That is why it is a table and not an interpolation. Alignment cannot /// be a mark — it belongs to the block, not the run — but it has to /// keep the property the MARKS table above exists to protect: nothing /// from a document may reach attribute position in `fragment_html`, /// whose output `internal/docs/store.go` returns as `template.HTML` and /// `views/show.html` writes into the page unescaped. So the value is /// only ever a key looked up here, and what gets emitted is the /// right-hand side, which is a literal in this binary. /// /// Closed, therefore, in the strong sense: a value that is not in this /// table — a typo, a hand-crafted update through /// `POST /d/{sqid}/updates`, or a future version's `justify` — is /// dropped entirely. Never escaped and emitted, never passed through. /// Adding `justify` here later is one row plus one CSS rule plus the /// arithmetic in `internal/export/pdf.go`; letting an unknown value /// through would be a decision about attribute escaping, which is the /// decision this shape refuses to have to make. /// /// `left` is deliberately absent. It is what a paragraph is when nobody /// has said otherwise, so it spends no class and no `attrs` — the same /// reasoning that keeps "normal" out of MARKS, and what makes /// a document written before alignment existed render byte-identically /// to one whose author chose left on purpose. const ALIGNMENTS: &[(&str, &str)] = &[ ("center", "docs-align-center"), ("right", "docs-align-right"), ]; /// The alignment value, if this build knows it. `None` for left, for an /// absent attribute, and for anything unrecognised — the three cases /// that all render the same way. fn alignment(txn: &T, e: &XmlElementRef) -> Option<&'static str> { let value = e.get_attribute(txn, "align")?; ALIGNMENTS .iter() .find(|(name, _)| *name == value) .map(|(name, _)| *name) } /// The deepest indent this build renders. Past eight half-inch steps a /// paragraph on US Letter has no measure left, and `fixtures/indent.mjs` /// states the same limit for the editor. const MAX_INDENT: u8 = 8; /// The indent level, if this block has one: a whole number from 1 to /// `MAX_INDENT`, or `None` for 0, for an absent attribute, and for /// anything that does not parse. /// /// A NUMBER, where `alignment` above handles a string, and that is the /// whole reason this is a separate function rather than another closed /// table. Alignment's values are strings, so the only safe thing to do /// with one is look it up and emit the table's own right-hand side. /// Indent's value is an integer, so it is handled the way `heading`'s /// level already is: parsed, then clamped. Both reach the property this /// module exists to hold — no byte from a document reaches attribute /// position — by different routes, and using the EXISTING route rather /// than inventing a third one is the point. A `u8` that has been through /// `clamp(1, 8)` can only be one of eight values, so interpolating it /// into a class name below is not the injection that interpolating an /// arbitrary attribute would be. /// /// Clamped rather than rejected, matching `heading`'s level: a document /// arriving through `POST /d/{sqid}/updates` with `indent` 400 renders /// at 8, and so does one with 4000000. Anything that is not a run of /// digits — `"420; drop"`, `" 3"`, `"3.0"`, an empty string — is `None` /// and renders flush left. `fixtures/indent.mjs`'s `normaliseIndent` /// applies the same two rules, deliberately, so the two agree on every /// input rather than only on the ones a real editor produces. /// /// (`heading`'s own level below still parses as `u8` and so still /// rejects 400 where it clamps 99. That is the shape this function had /// and stopped having; it is left alone here because changing it would /// change what existing documents render as, but it is not the model to /// copy.) fn indent_level(txn: &T, e: &XmlElementRef) -> Option { let raw = e.get_attribute(txn, "indent")?; // DIGITS ONLY, then parse wide, then clamp — and each of those // three is answering a disagreement with the editor rather than // being careful for its own sake. // // This parsed as `u8` for one review cycle, and the doc comment // above claimed "indent 400 renders at 8". It did not: 400 // overflows a u8, so the parse failed and the block rendered flush // left — while `fixtures/indent.mjs` used `Number()` and rendered // it at 8. The same split existed for " 3", "3.0", "1e1" and "0x3", // all of which JavaScript accepts and Rust rejected. No safety // property was ever spent (nothing here emits the byte either way), // but the claim that a document cannot indent one distance on // screen and another on paper was false for every one of them. // // Parsing as u32 makes "400" a number rather than an error, so it // clamps like every other too-large value; requiring digits makes // " 3" and "3.0" fail on BOTH sides, which is what indent.mjs's // matching strictness now guarantees. if raw.is_empty() || !raw.bytes().all(|b| b.is_ascii_digit()) { return None; } // unwrap_or(u32::MAX), not `.ok()?`. A string that passed the // digits-only check above and still fails to parse can only have // failed by being too large — so treating it as "as large as // possible" clamps it, where propagating the error would render it // flush left. That is the last corner where this and // fixtures/indent.mjs disagreed: JavaScript's Number() turns a // twenty-digit run into a float far above the maximum and clamps to // 8, and now so does this. let level = raw .parse::() .unwrap_or(u32::MAX) .min(MAX_INDENT as u32) as u8; if level == 0 { None } else { Some(level) } } /// The class `fragment_html` puts on an indented block. `None` when the /// block is not indented, so an un-indented paragraph emits exactly the /// markup it did before this existed. fn indent_class(txn: &T, e: &XmlElementRef) -> Option { indent_level(txn, e).map(|n| format!("docs-indent-{n}")) } /// The whole `class` attribute for a block, or `None` when it needs /// none. `extra` is whatever the caller already worked out (the /// alignment class, for a paragraph); the indent class is added here. /// /// One function rather than each call site joining its own, because /// the two classes have to be separated by exactly one space and a /// block with only one of them must emit no stray space — the sort of /// thing that is obviously right in each of two places and wrong in /// the one where they meet. fn block_class( txn: &T, e: &XmlElementRef, extra: Option<&'static str>, ) -> Option { match (extra, indent_class(txn, e)) { (Some(a), Some(i)) => Some(format!("{a} {i}")), (Some(a), None) => Some(a.to_string()), (None, Some(i)) => Some(i), (None, None) => None, } } /// The class `fragment_html` puts on an aligned paragraph, from the /// same table. fn align_class(txn: &T, e: &XmlElementRef) -> Option<&'static str> { let value = e.get_attribute(txn, "align")?; ALIGNMENTS .iter() .find(|(name, _)| *name == value) .map(|(_, class)| *class) } fn text_html(txn: &T, t: &XmlTextRef, out: &mut String) { // Diff gives runs with their attributes, which is where marks live. for chunk in t.diff(txn, yrs::types::text::YChange::identity) { let mut open = Vec::new(); if let Some(attrs) = &chunk.attributes { for (name, o, c) in MARKS { if attrs.contains_key(*name) { open.push((*o, *c)); } } } for (o, _) in &open { out.push_str(o); } if let yrs::Out::Any(yrs::Any::String(s)) = &chunk.insert { escape(s, out); } for (_, c) in open.iter().rev() { out.push_str(c); } } } fn node_html(txn: &T, node: &XmlOut, out: &mut String) { match node { XmlOut::Text(t) => text_html(txn, t, out), XmlOut::Element(e) => { let name = e.tag().to_string(); let (open, close) = match name.as_str() { // A paragraph can be aligned, indented, both, or // neither. Both classes are literals this binary builds // — one from the ALIGNMENTS table, one from a u8 that // has already been clamped to 1..=8 — so the joined // value can only be one of a small closed set of // strings, and nothing from the document reaches // attribute position. "paragraph" => match block_class(txn, e, align_class(txn, e)) { Some(class) => (format!("

"), "

".to_string()), None => ("

".to_string(), "

".to_string()), }, "heading" => { // Clamp: a corrupt or hostile level must not emit // or . let level = e .get_attribute(txn, "level") .and_then(|v| v.parse::().ok()) .unwrap_or(1) .clamp(1, 6); // Headings indent too — every word processor lets // them — but they do not align: alignment is on the // paragraph spec alone, here and in the editor's // schema, and adding it is a change to both. match block_class(txn, e, None) { Some(class) => ( format!(""), format!(""), ), None => (format!(""), format!("")), } } "bullet_list" => ("
    ".to_string(), "
".to_string()), "ordered_list" => ("
    ".to_string(), "
".to_string()), "list_item" => ("
  • ".to_string(), "
    • ".to_string()), "blockquote" => ("
    ".to_string(), "
    ".to_string()), // Shift+Enter's node (prosemirror-schema-basic's own // `hard_break`; the tag is the node's name because // y-prosemirror builds every non-text node as an // XmlElement named after its type). `
    ` is void, so // it has an opening and no closing markup — the loop // below still walks its children, which a real editor // never gives it and a client posting raw Yjs bytes // could, and they render after the break rather than // vanishing, per the module comment. "hard_break" => ("
    ".to_string(), String::new()), // Unknown: render children with no wrapper. See the // module comment — silence here is data loss. _ => (String::new(), String::new()), }; out.push_str(&open); for child in e.children(txn) { node_html(txn, &child, out); } out.push_str(&close); } XmlOut::Fragment(f) => { for child in f.children(txn) { node_html(txn, &child, out); } } } } /// The document rendered as HTML, escaped and ready to place inside the /// page container. pub fn fragment_html(doc: &Doc) -> String { let f = root(doc); let txn = doc.transact(); let mut out = String::new(); for child in f.children(&txn) { node_html(&txn, &child, &mut out); } out } fn node_text(txn: &T, node: &XmlOut, blocks: &mut Vec, cur: &mut String) { match node { XmlOut::Text(t) => cur.push_str(&t.get_string(txn)), XmlOut::Element(e) => { let name = e.tag().to_string(); // A hard break ends the current line and the block carries // on: one newline, where the block boundary below is a // blank line. It is deliberately NOT in the is_block set — // treating it as a block would turn one paragraph into two // wherever a person pressed Shift+Enter. if name == "hard_break" { cur.push('\n'); return; } let is_block = matches!( name.as_str(), "paragraph" | "heading" | "list_item" | "blockquote" ); // A text sibling that precedes this block with no block of // its own (unreachable through ProseMirror's own schema, // but reachable the moment a client posts arbitrary Yjs // bytes — see fragment_text's own doc comment) must not // run straight into this block's text with no separator: // flush whatever is already pending as its own segment // before this block contributes anything. if is_block && !cur.is_empty() { blocks.push(std::mem::take(cur)); } let mut had_children = false; for child in e.children(txn) { had_children = true; node_text(txn, &child, blocks, cur); } if is_block { if !cur.is_empty() { blocks.push(std::mem::take(cur)); } else if !had_children { // A structurally empty block (`

    `) still // renders as a blank paragraph in fragment_html; // mirror that here rather than silently dropping // it, which would make the two renderers disagree // about how many paragraphs a document has. A // block whose children already flushed their own // text (had_children but cur now empty, e.g. a // list_item wrapping a paragraph) must not get a // second, spurious empty entry — that is exactly // what had_children distinguishes. blocks.push(String::new()); } } } XmlOut::Fragment(f) => { for child in f.children(txn) { node_text(txn, &child, blocks, cur); } } } } /// The document as plain text, one blank line between blocks. Used by /// the list's preview and by anything that wants the words without the /// markup. /// /// Neither ProseMirror's own schema nor its `y-prosemirror` binding can /// produce a fragment with a bare text node preceding a block, or a /// structurally empty block — so, until this task, node_text's two /// edge cases here were unreachable in practice: a text sibling ahead /// of a block used to run straight into that block's own text with no /// separator (`"orphanactual"` for `[text("orphan"), paragraph(text("actual"))]`), /// and an empty `

    ` that fragment_html preserves used to vanish /// here instead of leaving a blank line. `POST /d/{sqid}/updates` /// (Task 5) is what makes them reachable: it accepts arbitrary /// client-supplied Yjs bytes, not only what a real ProseMirror editor /// would ever emit. Both are fixed in node_text above and covered by /// core/tests/fragment.rs. pub fn fragment_text(doc: &Doc) -> String { let f = root(doc); let txn = doc.transact(); let mut blocks = Vec::new(); let mut cur = String::new(); for child in f.children(&txn) { node_text(&txn, &child, &mut blocks, &mut cur); } if !cur.is_empty() { blocks.push(cur); } blocks.join("\n\n") } // --- the structured renderer --- // // `fragment_html` above answers "what does this document look like"; // `fragment_json` below answers "what is this document made of". The // export paths (`internal/export`, `internal/pdf`) need the second // question answered: where a list ends, which run is bold, which // paragraph is a heading. Re-deriving that in Go by parsing the markup // this file emits would mean a parser to keep honest against an // emitter, and Go's standard library has no HTML parser to do it with. // // The shape is ProseMirror's own JSON — `{"type":..,"attrs":..,"content":[..]}`, // text as `{"type":"text","text":..,"marks":[..]}` — because that is the // vocabulary the editor on the other side of the wire already speaks, // so a Go consumer and a future JS one describe the same document the // same way. Written by hand rather than through serde: the crate has // no serialisation dependency outside dev-dependencies, and this is // one escape function and three walks. /// JSON string escaping, per RFC 8259: the two mandatory escapes, the /// three shorthands, and `\u00XX` for everything else below U+0020. /// Everything at or above U+0020 other than `"` and `\` goes through /// as-is — the output is UTF-8 and JSON says that is legal. fn escape_json(s: &str, out: &mut String) { for c in s.chars() { match c { '"' => out.push_str("\\\""), '\\' => out.push_str("\\\\"), '\n' => out.push_str("\\n"), '\r' => out.push_str("\\r"), '\t' => out.push_str("\\t"), c if (c as u32) < 0x20 => { let _ = write!(out, "\\u{:04x}", c as u32); } c => out.push(c), } } } /// The mark names on one text run: the ones this build knows in `MARKS` /// order, then any it does not, sorted. /// /// The sort is not cosmetic. `Attrs` is a `HashMap`, so iterating it /// yields an order that varies between runs of the same binary on the /// same document — an unsorted tail would make this renderer's output /// nondeterministic, which would make every golden-file test on the Go /// side flake rather than fail. Unknown marks are carried through /// rather than dropped for the same reason unknown *nodes* render their /// children (see the module comment): a consumer that ignores a mark it /// does not recognise still has the text, and one that grows to /// understand it needs it to have been there. /// WHOEVER ADDS THE FIRST VALUED MARK: THIS FUNCTION IS WHERE THE /// ALLOWLIST GOES, AND IT MUST NOT ASSUME ONLY THAT MARK HAS A VALUE. /// /// This returns NAMES. Every mark's value — including an unknown mark's /// — is dropped here and reaches no consumer, which is why an unknown /// mark carried through by name is safe today: `fragment_html` finds no /// MARKS row and emits no wrapper, and `text_json` writes the name /// through `escape_json`. /// /// A valued mark changes that. The moment one value is allowed past /// this point, the shape of the question becomes "which marks may carry /// a value, and what may that value be" — and the tempting answer, that /// `link` is the one with an `href`, is wrong on arrival. This function /// collects marks it has never heard of, straight off the wire, and /// `POST /d/{sqid}/updates` accepts bytes that never went through /// ProseMirror's schema. So an unknown mark can carry an arbitrary /// value too, and an allowlist written around the known one would let /// it past. /// /// It is also the right place, and that is the other half of the note: /// this is the single point the HTML renderer and every JSON consumer /// agree on what marks a run carries, so a check here cannot be /// re-implemented differently by `markdown.go` and `pdf.go` the way one /// in `node_html` would have to be. Plan 7's D1 names it. /// /// (Found by the session that owns Plan 7, reading this file after the /// point-ladder work restructured around it — recorded here rather than /// only in that plan, because the plan may land long after someone else /// touches this function.) fn mark_names(attrs: &yrs::types::Attrs) -> Vec { let mut names: Vec = MARKS .iter() .filter(|(name, _, _)| attrs.contains_key(*name)) .map(|(name, _, _)| (*name).to_string()) .collect(); let mut unknown: Vec = attrs .keys() .map(|k| k.to_string()) .filter(|k| !MARKS.iter().any(|(name, _, _)| *name == k.as_str())) .collect(); unknown.sort(); names.append(&mut unknown); names } fn text_json(txn: &T, t: &XmlTextRef, out: &mut String, first: &mut bool) { for chunk in t.diff(txn, yrs::types::text::YChange::identity) { let yrs::Out::Any(yrs::Any::String(s)) = &chunk.insert else { // A non-string insert is an embed. `fragment_html` renders // nothing for one today, and inventing a JSON shape for // something with no renderer would be guessing at a schema // rather than reporting one. continue; }; if s.is_empty() { continue; } if !*first { out.push(','); } *first = false; out.push_str("{\"type\":\"text\",\"text\":\""); escape_json(s, out); out.push('"'); if let Some(attrs) = &chunk.attributes { let names = mark_names(attrs); if !names.is_empty() { out.push_str(",\"marks\":["); for (i, name) in names.iter().enumerate() { if i > 0 { out.push(','); } out.push_str("{\"type\":\""); escape_json(name, out); out.push_str("\"}"); } out.push(']'); } } out.push('}'); } } // Unlike the two renderers above, this one needs no case of its own // for `hard_break`: an element with no children already comes out as // `{"type":"hard_break","content":[]}`, which names the node and is // what internal/export reads. The empty `content` is how this renderer // spells every childless element (an empty paragraph included), so a // leaf is not a special shape here. fn node_json(txn: &T, node: &XmlOut, out: &mut String, first: &mut bool) { match node { XmlOut::Text(t) => text_json(txn, t, out, first), XmlOut::Element(e) => { let name = e.tag().to_string(); if !*first { out.push(','); } *first = false; out.push_str("{\"type\":\""); escape_json(&name, out); out.push('"'); // Attributes are collected and written ONCE, as a single // object. // // This used to be two independent `if name == …` arms, each // writing a complete `,"attrs":{…}`. That was correct only // for as long as no node could carry two attributes: a // heading has a level and a paragraph has an alignment, and // never both. Indent broke that — a paragraph can be // centred AND indented, and a heading can be indented too — // and the old shape would have emitted `"attrs"` twice in // one object. Every JSON parser accepts that and they do not // agree on which wins, so it would not have failed loudly; // internal/export would have quietly lost one of the two. // // Each value below is a number or a literal from a closed // table in this binary, never a byte from the document, so // nothing here needs escaping — the same property // `fragment_html` depends on, held for the same reason. let mut attrs: Vec = Vec::new(); if name == "heading" { // Clamped exactly as `node_html` clamps it, and for the // same reason: a corrupt or hostile level must not reach // a consumer as `h0` or `h99`. Both renderers clamping // independently is deliberate — a document must not // paginate as one level on screen and export as another. let level = e .get_attribute(txn, "level") .and_then(|v| v.parse::().ok()) .unwrap_or(1) .clamp(1, 6); attrs.push(format!("\"level\":{level}")); } if name == "paragraph" { // Same allowlist, same drop, and — like the heading // clamp above — arrived at independently in both // renderers, so a paragraph cannot read as centred on // screen and export flush left. `align` is the // allowlist's own key, a literal in this binary, so // nothing here needs escaping and nothing from the // document can appear. if let Some(align) = alignment(txn, e) { attrs.push(format!("\"align\":\"{align}\"")); } } if name == "paragraph" || name == "heading" { if let Some(indent) = indent_level(txn, e) { attrs.push(format!("\"indent\":{indent}")); } } if !attrs.is_empty() { let _ = write!(out, ",\"attrs\":{{{}}}", attrs.join(",")); } out.push_str(",\"content\":["); let mut inner = true; for child in e.children(txn) { node_json(txn, &child, out, &mut inner); } out.push_str("]}"); } XmlOut::Fragment(f) => { // Flattened into the parent's content, exactly as // `node_html` renders a nested fragment's children with no // wrapper of its own. for child in f.children(txn) { node_json(txn, &child, out, first); } } } } /// The document as ProseMirror-shaped JSON: `{"type":"doc","content":[..]}`. /// /// Deterministic for a given document state — see `mark_names` for the /// one place that took work — so a Go consumer's golden files are /// stable. pub fn fragment_json(doc: &Doc) -> String { let f = root(doc); let txn = doc.transact(); let mut out = String::from("{\"type\":\"doc\",\"content\":["); let mut first = true; for child in f.children(&txn) { node_json(&txn, &child, &mut out, &mut first); } out.push_str("]}"); out }