build: Implement formatForIngestion

2026-08-18 13:21:55 +00:00 · 2026-04-04 19:29:29 +05:30
parent 290ac89325
commit 88af87b7ae
7 changed files with 353 additions and 19 deletions
@@ -0,0 +1,335 @@
+---
+model: mimo-v2-pro
+---
+
+# Plan: HTML-to-Markdown Transformations for Ingestion
+
+## Summary
+
+Add configurable HTML-to-markdown transformations to `formatHTMLForIngestion.ts` to improve the quality of text stored in the vector index. Transformations convert ecmarkup/HTML elements (code blocks, cross-reference links, lists, inline code) to markdown syntax before the `HTMLTextSplitter` extracts plain text.
+
+## Config Interface
+
+```typescript
+interface FormatConfig {
+  codeBlocks: {
+    block: string[];    // fenced code (pre>code, emu-eqn block)
+    inline: string[];   // inline backticks (var, emu-val, emu-const, emu-eqn inline, code)
+    grammar: string[];  // fenced bnf (emu-grammar)
+  };
+  links: string[];      // [text](#hash) (emu-xref a)
+  lists: {
+    ordered: string[];  // 1. 2. 3. (ol)
+    unordered: string[];// - item (ul)
+  };
+  tables: string[];     // markdown tables (table, emu-table)
+}
+```
+
+Default config:
+```typescript
+const DEFAULT_CONFIG: FormatConfig = {
+  codeBlocks: {
+    block: ["pre>code", "emu-eqn:not([class*='inline'])"],
+    inline: ["var", "emu-val", "emu-const", "emu-eqn.inline"],
+    // Note: standalone <code> (outside <pre>) is rare; handle with :not(pre > *) check in converter
+    grammar: ["emu-grammar"],
+  },
+  links: ["emu-xref a"],
+  lists: {
+    ordered: ["ol"],
+    unordered: ["ul"],
+  },
+  tables: ["table", "emu-table"],
+};
+```
+
+## Transformation Order
+
+0. **`addNewlinesAfterBlocks($)`** — global newline injection (existing function, runs first)
+1. **Inline leaves**: `emu-xref a[href]` → links, `var`/`emu-val`/`emu-const` → inline code
+2. **Block leaves**: `pre>code` → fenced code, `emu-eqn` → fenced code
+3. **Structural parents**: `emu-grammar` → fenced bnf (uses `.text()` with pre-injected newlines), `ol/ul` → markdown lists, `table` → markdown tables
+
+## Implementation Details
+
+### Approach: Cheerio only
+
+All conversions use cheerio DOM manipulation (already in codebase). No external HTML-to-markdown library. Each conversion replaces DOM elements with text/markdown nodes. The existing `HTMLTextSplitter` processes the modified DOM unchanged.
+
+### New functions in `setup/utils/formatHTMLForIngestion.ts`
+
+Each function receives `cheerio.CheerioAPI` and modifies the DOM in-place.
+
+1. **`convertLinksToMarkdown($, selector: string)`** — selector default: `emu-xref a[href]`
+   - `$(selector).each(...)`: get href, strip filename, get text, replaceWith `<span>` containing `[text](#hash)`
+   - Safety: skip if href starts with `http`
+
+2. **`convertInlineCodeToMarkdown($, tags: string[])`**
+   - `$(tag).each(...)`: get `.text()`, replaceWith `<span>` containing `` `text` ``
+   - For `code` tags: skip if parent is `pre`
+
+3. **`convertBlockCodeToMarkdown($, tags: string[])`**
+   - For `pre>code`: get language from `class`, get `.text()` from code, replaceWith `<pre>` containing `` ```lang\ntext\n``` ``
+   - For `emu-eqn` block: get `.text()`, replaceWith `<pre>` containing `` ```\ntext\n``` ``
+
+4. **`convertGrammarToMarkdown($, selector: string)`**
+   - Walk inner tags (emu-nt, emu-t, emu-rhs, emu-geq), build text representation
+   - ReplaceWith `<pre>` containing `` ```bnf\ntext\n``` ``
+
+5. **`convertListsToMarkdown($, ordered: string[], unordered: string[])`**
+   - For each `ol`/`ul`: recursive function `listToMarkdown(elem, depth, isOrdered)`
+   - Process `li` children, add `1. ` / `- ` prefix with indentation
+   - ReplaceWith `<pre>` containing markdown text
+
+6. **`formatForIngestion($, config?)`** — orchestrator, calls all in order
+
+## Critical Implementation Code
+
+### Config interface
+
+```typescript
+export interface FormatConfig {
+  codeBlocks: {
+    block: string[];
+    inline: string[];
+    grammar: string[];
+  };
+  links: string[];
+  lists: {
+    ordered: string[];
+    unordered: string[];
+  };
+  tables: string[];
+}
+
+export const DEFAULT_CONFIG: FormatConfig = {
+  codeBlocks: {
+    block: ["pre>code", "emu-eqn:not([class*='inline'])"],
+    inline: ["var", "emu-val", "emu-const", "emu-eqn.inline"],
+    grammar: ["emu-grammar"],
+  },
+  links: ["emu-xref a"],
+  lists: {
+    ordered: ["ol"],
+    unordered: ["ul"],
+  },
+  tables: ["table", "emu-table"],
+};
+```
+
+### `convertLinksToMarkdown` — filename stripping
+
+```typescript
+export function convertLinksToMarkdown(
+  $: cheerio.CheerioAPI,
+  selector: string = "emu-xref a[href]",
+): void {
+  $(selector).each((_, elem) => {
+    const $a = $(elem);
+    const href = $a.attr("href") ?? "";
+
+    // Skip external links
+    if (href.startsWith("http")) return;
+
+    // Strip filename prefix: "abstract-operations.html#sec-tonumber" → "#sec-tonumber"
+    const hash = href.includes("#") ? `#${href.split("#")[1]}` : href;
+    const text = $a.text().trim();
+
+    if (!text || !hash) return;
+
+    $a.replaceWith(`<span class="link-markdown">[${text}](${hash})</span>`);
+  });
+}
+```
+
+### `convertInlineCodeToMarkdown` — backtick wrapping
+
+```typescript
+export function convertInlineCodeToMarkdown(
+  $: cheerio.CheerioAPI,
+  tags: string[],
+): void {
+  for (const tag of tags) {
+    $(tag).each((_, elem) => {
+      // Skip <code> inside <pre> (handled by block converter)
+      if (elem.name === "code" && $(elem).parent("pre").length > 0) return;
+
+      const text = $(elem).text().trim();
+      if (!text) return;
+      $(elem).replaceWith(`<span class="inline-code">\`${text}\`</span>`);
+    });
+  }
+}
+```
+
+### `convertBlockCodeToMarkdown` — fenced code blocks
+
+```typescript
+function extractLanguage(codeClass: string | undefined): string {
+  // "javascript hljs" → "javascript", "python hljs" → "python"
+  if (!codeClass) return "";
+  const match = codeClass.match(/^(\w+)/);
+  return match?.[1] ?? "";
+}
+
+export function convertBlockCodeToMarkdown(
+  $: cheerio.CheerioAPI,
+  tags: string[],
+): void {
+  // Handle pre>code (JS code examples)
+  $("pre > code").each((_, code) => {
+    const lang = extractLanguage($(code).attr("class"));
+    const text = $(code).text().trim();
+    const pre = $(code).parent("pre");
+    pre.replaceWith(
+      $(`<pre class="code-markdown">\`\`\`${lang}\n${text}\n\`\`\`</pre>`),
+    );
+  });
+
+  // Handle standalone emu-eqn (block-level equations)
+  $("emu-eqn:not([class*='inline'])").each((_, eqn) => {
+    const text = $(eqn).text().trim();
+    $(eqn).replaceWith(
+      $(`<pre class="code-markdown">\`\`\`\n${text}\n\`\`\`</pre>`),
+    );
+  });
+}
+```
+
+### `convertListsToMarkdown` — recursive list processing
+
+```typescript
+function listToMarkdown(
+  $: cheerio.CheerioAPI,
+  elem: cheerio.Element,
+  depth: number = 0,
+): string {
+  const $elem = $(elem);
+  const isOrdered = elem.name === "ol";
+  const indent = "  ".repeat(depth);
+  const lines: string[] = [];
+
+  $elem.children("li").each((i, li) => {
+    const prefix = isOrdered ? `${i + 1}. ` : "- ";
+    const $li = $(li);
+
+    // Recurse into nested lists first (depth-first)
+    $li.children("ol, ul").each((_, nested) => {
+      const nestedMarkdown = listToMarkdown($, nested, depth + 1);
+      $(nested).replaceWith($(`<pre class="list-processed">${nestedMarkdown}</pre>`));
+    });
+
+    // Now get full text (nested lists are replaced with <pre> containing markdown)
+    const itemText = $li.text().trim().replace(/\s+/g, " ");
+    lines.push(`${indent}${prefix}${itemText}`);
+  });
+
+  return lines.join("\n");
+}
+
+export function convertListsToMarkdown(
+  $: cheerio.CheerioAPI,
+  ordered: string[] = ["ol"],
+  unordered: string[] = ["ul"],
+): void {
+  const selectors = [...ordered, ...unordered].join(", ");
+  // Process from outermost — recursion handles depth-first nesting
+  $(selectors).each((_, elem) => {
+    // Skip if already processed (parent already handled this)
+    if ($(elem).hasClass("list-markdown")) return;
+    const markdown = listToMarkdown($, elem, 0);
+    $(elem).replaceWith($(`<pre class="list-markdown">${markdown}</pre>`));
+  });
+}
+```
+
+Key insight: recursion goes depth-first. When processing an `li`, nested `ol`/`ul` children are first replaced with `<pre class="list-processed">` containing their markdown. Then `$li.text()` includes that markdown. Outer `$(selectors).each(...)` skips already-processed elements.
+
+### `convertGrammarToMarkdown` — grammar production text
+
+`addNewlinesAfterBlocks` runs first globally, injecting `\n` after `emu-production` and `emu-rhs` (added to `BLOCK_ELEMENTS`). Then `.text()` on `emu-grammar` naturally preserves production/alternative boundaries.
+
+```typescript
+export function convertGrammarToMarkdown(
+  $: cheerio.CheerioAPI,
+  selector: string = "emu-grammar",
+): void {
+  $(selector).each((_, grammar) => {
+    const text = $(grammar).text().trim();
+    $(grammar).replaceWith(
+      $(`<pre class="code-markdown">\`\`\`bnf\n${text}\n\`\`\`</pre>`),
+    );
+  });
+}
+```
+
+Add `"emu-production"` and `"emu-rhs"` to `BLOCK_ELEMENTS` array.
+
+### `formatForIngestion` — orchestrator
+
+```typescript
+export function formatForIngestion(
+  $: cheerio.CheerioAPI,
+  config: Partial<FormatConfig> = {},
+): void {
+  const cfg = { ...DEFAULT_CONFIG, ...config };
+
+  // 1. Inject newlines globally (affects li, p, pre, emu-production, emu-rhs, etc.)
+  addNewlinesAfterBlocks($);
+
+  // 2. Inline leaves (links first — must be before lists/code destroy DOM)
+  convertLinksToMarkdown($, cfg.links.join(", "));
+
+  // 3. Inline code
+  convertInlineCodeToMarkdown($, cfg.codeBlocks.inline);
+
+  // 4. Block leaves (fenced code)
+  convertBlockCodeToMarkdown($, cfg.codeBlocks.block);
+
+  // 5. Structural parents (grammar, lists, tables)
+  convertGrammarToMarkdown($, cfg.codeBlocks.grammar.join(", "));
+  convertListsToMarkdown($, cfg.lists.ordered, cfg.lists.unordered);
+  convertTablesToMarkdown($);
+}
+```
+
+### Modify existing functions
+
+- **`convertTablesToMarkdown($)`**: No changes needed. `.text()` already extracts markdown text from pre-processed `<span>` elements.
+
+### Modify `setup/ingest.ts`
+
+- Replace individual calls to `convertTablesToMarkdown($)` and `addNewlinesAfterBlocks($)` with single call to `formatForIngestion($)`.
+- The child `emu-clause` replacement and deep nesting cleanup remain as-is (before `formatForIngestion`).
+
+Updated call order in `buildSpecDocuments()`:
+```typescript
+// 1. Child clause replacement (existing)
+$section.children("emu-clause").each(...);
+$section.find("emu-clause").remove();
+
+// 2. All formatting transformations (single call)
+formatForIngestion($);
+```
+
+## Files to Modify
+
+1. **`setup/utils/formatHTMLForIngestion.ts`** — add config interface, 5 new functions, orchestrating function
+2. **`setup/ingest.ts`** — replace individual calls with `formatForIngestion($)` call
+3. **New test file: `setup/utils/formatHTMLForIngestion.test.ts`** — unit tests for each transformation
+
+No new dependencies. Cheerio only.
+
+## Verification
+
+1. **Unit tests**: Write tests for each transformation with sample HTML snippets from the spec:
+   - `emu-xref a` → markdown link
+   - `var`/`emu-val`/`emu-const` → inline code
+   - `pre>code` → fenced code
+   - `emu-grammar` → fenced bnf
+   - `ol`/`ul` with nesting → markdown lists
+   - `table` → markdown table (with links inside cells preserved)
+   - Full pipeline via `formatForIngestion`
+2. **Manual verification**: Run `bun run setup/ingest.ts` and inspect output chunks for correct markdown syntax
+3. **Run lint/typecheck**: `bun run lint` and `bun run typecheck` (if available)