From 88af87b7aece507da84a6d5208d014395ba1191b Mon Sep 17 00:00:00 2001 From: bendtherules Date: Sat, 4 Apr 2026 19:29:29 +0530 Subject: [PATCH] build: Implement formatForIngestion --- .../1775291477517-lucky-garden.md | 0 .vscode/settings.json | 2 +- setup/build_graph.ts | 6 +- setup/html-add-internal-method-link.ts | 2 +- setup/ingest.ts | 27 ++- setup/utils/formatHTMLForIngestion.test.ts | 147 ++++++++++++++ setup/utils/formatHTMLForIngestion.ts | 188 +++++++++++++++++- 7 files changed, 353 insertions(+), 19 deletions(-) rename .opencode/plans/{ => archive}/1775291477517-lucky-garden.md (100%) create mode 100644 setup/utils/formatHTMLForIngestion.test.ts diff --git a/.opencode/plans/1775291477517-lucky-garden.md b/.opencode/plans/archive/1775291477517-lucky-garden.md similarity index 100% rename from .opencode/plans/1775291477517-lucky-garden.md rename to .opencode/plans/archive/1775291477517-lucky-garden.md diff --git a/.vscode/settings.json b/.vscode/settings.json index 4103ab6..7f2f527 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -8,7 +8,7 @@ "search.exclude": { "**/node_modules": true, "**/dist": true, - "**/storage": true, + "**/storage": true }, "files.exclude": { "**/node_modules": true, diff --git a/setup/build_graph.ts b/setup/build_graph.ts index 8f63177..f82e4d2 100644 --- a/setup/build_graph.ts +++ b/setup/build_graph.ts @@ -21,7 +21,11 @@ import { GRAPH_FILE } from "../constants"; */ async function buildGraph() { // Initialize a multi-graph (allows multiple edges between same nodes) - const graph = new Graph({ multi: true, type: "directed", allowSelfLoops: false }); + const graph = new Graph({ + multi: true, + type: "directed", + allowSelfLoops: false, + }); // Phase 1: Discover and parse specification HTML files const htmlFiles = await glob(path.join(SPEC_DIR, "*.html")); diff --git a/setup/html-add-internal-method-link.ts b/setup/html-add-internal-method-link.ts index 74c1e29..6ed6c99 100644 --- a/setup/html-add-internal-method-link.ts +++ b/setup/html-add-internal-method-link.ts @@ -24,7 +24,7 @@ const htmlCheerioApi = cheerio.load(htmlString); * the first `` of each row. The generated id follows the pattern: * `ask262-internal-method-` where `` is the text * content of the `` element with surrounding brackets stripped. - * + * * @param $ - The Cheerio parsing instance. */ function addInternalMethodIds($: CheerioAPI): void { diff --git a/setup/ingest.ts b/setup/ingest.ts index 2d7ec63..b0a2f90 100644 --- a/setup/ingest.ts +++ b/setup/ingest.ts @@ -10,10 +10,7 @@ import { glob } from "glob"; import ora from "ora"; import { EMBEDDING_MODEL, SPEC_DIR, STORAGE_DIR } from "../constants"; import { HTMLTextSplitter } from "./textsplitters"; -import { - addNewlinesAfterBlocks, - convertTablesToMarkdown, -} from "./utils/formatHTMLForIngestion"; +import { formatForIngestion } from "./utils/formatHTMLForIngestion"; const embeddings = new OllamaEmbeddings({ model: EMBEDDING_MODEL, @@ -153,8 +150,10 @@ async function buildSpecDocuments(): Promise { // Build children relationships for (const [id, section] of sectionMap) { if (section.parentId && sectionMap.has(section.parentId)) { - const parent = sectionMap.get(section.parentId)!; - parent.childrenIds.push(id); + const parent = sectionMap.get(section.parentId); + if (parent) { + parent.childrenIds.push(id); + } } } @@ -163,18 +162,17 @@ async function buildSpecDocuments(): Promise { // Parse the stored HTML and replace direct children with placeholders const $ = cheerio.load(`${section.html}`); - // Convert tables to markdown format for better text extraction - convertTablesToMarkdown($); - const $section = $.root(); // Find direct children emu-clause elements only $section.children("emu-clause").each((_, childElem) => { const childId = $(childElem).attr("id"); if (childId && sectionMap.has(childId)) { - const child = sectionMap.get(childId)!; - const placeholder = `[Subsection available: sectiontitle "${child.title}" at sectionid: \`${childId}\`]`; - $(childElem).replaceWith(placeholder); + const child = sectionMap.get(childId); + if (child) { + const placeholder = `[Subsection available: sectiontitle "${child.title}" at sectionid: \`${childId}\`]`; + $(childElem).replaceWith(placeholder); + } } else { $(childElem).remove(); } @@ -184,9 +182,8 @@ async function buildSpecDocuments(): Promise { // (shouldn't happen with proper HTML structure, but just in case) $section.find("emu-clause").remove(); - // Add newlines after block elements to preserve document structure - // This helps the text splitter maintain paragraph/section boundaries - addNewlinesAfterBlocks($); + // All formatting transformations (single call) + formatForIngestion($); // Skip sections that only have h1 left (no meaningful content) const hasOnlyH1 = diff --git a/setup/utils/formatHTMLForIngestion.test.ts b/setup/utils/formatHTMLForIngestion.test.ts new file mode 100644 index 0000000..7e3442b --- /dev/null +++ b/setup/utils/formatHTMLForIngestion.test.ts @@ -0,0 +1,147 @@ +import { describe, expect, test } from "bun:test"; +import * as cheerio from "cheerio"; +import { + convertBlockCodeToMarkdown, + convertGrammarToMarkdown, + convertInlineCodeToMarkdown, + convertLinksToMarkdown, + convertListsToMarkdown, + convertTablesToMarkdown, + DEFAULT_CONFIG, + formatForIngestion, +} from "./formatHTMLForIngestion"; + +describe("formatHTMLForIngestion", () => { + test("convertLinksToMarkdown converts emu-xref links to markdown and strips filenames", () => { + const html = `ToNumber`; + const $ = cheerio.load(html); + convertLinksToMarkdown($, DEFAULT_CONFIG.links.join(", ")); + expect($.html()).toContain( + '[ToNumber](#sec-tonumber)', + ); + }); + + test("convertLinksToMarkdown skips external links", () => { + const html = `External`; + const $ = cheerio.load(html); + convertLinksToMarkdown($, DEFAULT_CONFIG.links.join(", ")); + expect($.html()).toContain('External'); + }); + + test("convertInlineCodeToMarkdown converts var, emu-val, emu-const to inline code", () => { + const html = `
x y z w
`; + const $ = cheerio.load(html); + convertInlineCodeToMarkdown($, DEFAULT_CONFIG.codeBlocks.inline); + expect($.html()).toContain('`x`'); + expect($.html()).toContain('`y`'); + expect($.html()).toContain('`z`'); + expect($.html()).toContain('`w`'); + }); + + test("convertInlineCodeToMarkdown skips code inside pre", () => { + const html = `
x
`; + const $ = cheerio.load(html); + convertInlineCodeToMarkdown($, DEFAULT_CONFIG.codeBlocks.inline); + expect($.html()).toContain("
x
"); // unchanged + }); + + test("convertBlockCodeToMarkdown converts pre>code and emu-eqn", () => { + const html = `
+ 
const x = 1;
+ y = x + 1 + z = 2 +
`; + const $ = cheerio.load(html); + convertBlockCodeToMarkdown($, DEFAULT_CONFIG.codeBlocks.block); + expect($.html()).toContain( + '
```javascript\nconst x = 1;\n```
', + ); + expect($.html()).toContain( + '
```\ny = x + 1\n```
', + ); + expect($.html()).toContain('z = 2'); // unchanged + }); + + test("convertGrammarToMarkdown converts emu-grammar to fenced bnf", () => { + const html = `Statement :: BlockStatement`; + const $ = cheerio.load(html); + convertGrammarToMarkdown($, DEFAULT_CONFIG.codeBlocks.grammar.join(", ")); + expect($.html()).toContain( + '
```bnf\nStatement :: BlockStatement\n```
', + ); + }); + + test("convertListsToMarkdown converts ol and ul with nesting to markdown lists", () => { + const html = ` +
    +
  • Item 1
    • +
  • Item 2 +
      +
    1. Subitem 1
      2. +
    2. Subitem 2
      3. +
    +
    • +
+ `; + const $ = cheerio.load(html); + convertListsToMarkdown( + $, + DEFAULT_CONFIG.lists.ordered, + DEFAULT_CONFIG.lists.unordered, + ); + const result = $("pre.list-markdown").text(); + expect(result).toContain("- Item 1"); + expect(result).toContain("- Item 2"); + expect(result).toContain(" 1. Subitem 1"); + expect(result).toContain(" 2. Subitem 2"); + }); + + test("convertTablesToMarkdown converts tables to markdown tables", () => { + const html = ` + + + + + + + +
Col 1Col 2
Data 1Data 2
+ `; + const $ = cheerio.load(html); + convertTablesToMarkdown($); + const result = $("pre.table-markdown").text(); + expect(result).toContain("| Col 1 | Col 2 |"); + expect(result).toContain("| --- | --- |"); + expect(result).toContain("| Data 1 | Data 2 |"); + }); + + test("formatForIngestion runs the full pipeline", () => { + const html = ` +
+

See Example for x

+ 
let y = x;
+
    +
  • One
    • +
  • Two
    • +
+
+ `; + const $ = cheerio.load(html); + formatForIngestion($); + + // Check links + expect($.html()).toContain( + '[Example](#sec-example)', + ); + // Check inline code + expect($.html()).toContain('`x`'); + // Check block code + expect($.html()).toContain( + '
```javascript\nlet y = x;\n```
', + ); + // Check lists + expect($.html()).toContain( + '
- One\n- Two\n
', + ); + }); +}); diff --git a/setup/utils/formatHTMLForIngestion.ts b/setup/utils/formatHTMLForIngestion.ts index d519900..0fefdcd 100644 --- a/setup/utils/formatHTMLForIngestion.ts +++ b/setup/utils/formatHTMLForIngestion.ts @@ -30,8 +30,170 @@ export const BLOCK_ELEMENTS = [ "dl", "dt", "dd", + "emu-production", + "emu-rhs", ]; +export interface FormatConfig { + codeBlocks: { + block: string[]; + inline: string[]; + grammar: string[]; + }; + links: string[]; + lists: { + ordered: string[]; + unordered: string[]; + }; + tables: string[]; +} + +export const DEFAULT_CONFIG: FormatConfig = { + codeBlocks: { + block: ["pre>code", "emu-eqn:not([class*='inline'])"], + inline: ["var", "emu-val", "emu-const", "emu-eqn.inline", "code"], + grammar: ["emu-grammar"], + }, + links: ["emu-xref a"], + lists: { + ordered: ["ol"], + unordered: ["ul"], + }, + tables: ["table", "emu-table"], +}; + +export function convertLinksToMarkdown( + $: cheerio.CheerioAPI, + selector: string = "emu-xref a[href]", +): void { + $(selector).each((_, elem) => { + const $a = $(elem); + const href = $a.attr("href") ?? ""; + + // Skip external links + if (href.startsWith("http")) return; + + // Strip filename prefix: "abstract-operations.html#sec-tonumber" → "#sec-tonumber" + const hash = href.includes("#") ? `#${href.split("#")[1]}` : href; + const text = $a.text().trim(); + + if (!text || !hash) return; + + $a.replaceWith($(`[${text}](${hash})`)); + }); +} + +export function convertInlineCodeToMarkdown( + $: cheerio.CheerioAPI, + tags: string[], +): void { + for (const tag of tags) { + $(tag).each((_, elem) => { + // Skip inside 
 (handled by block converter)
+      if (
+        "name" in elem &&
+        elem.name === "code" &&
+        $(elem).parent("pre").length > 0
+      )
+        return;
+
+      const text = $(elem).text().trim();
+      if (!text) return;
+      $(elem).replaceWith($(`\`${text}\``));
+    });
+  }
+}
+
+function extractLanguage(codeClass: string | undefined): string {
+  // "javascript hljs" → "javascript", "python hljs" → "python"
+  if (!codeClass) return "";
+  const match = codeClass.match(/^(\w+)/);
+  return match?.[1] ?? "";
+}
+
+export function convertBlockCodeToMarkdown(
+  $: cheerio.CheerioAPI,
+  tags: string[],
+): void {
+  const selectors = tags.join(", ");
+  if (!selectors) return;
+
+  $(selectors).each((_, elem) => {
+    const $elem = $(elem);
+    const lang = extractLanguage($elem.attr("class"));
+    const text = $elem.text().trim();
+    $elem.replaceWith(
+      $(`
\`\`\`${lang}\n${text}\n\`\`\`
`),
+    );
+  });
+}
+
+function listToMarkdown(
+  $: cheerio.CheerioAPI,
+  elem: any,
+  depth: number = 0,
+): string {
+  const $elem = $(elem);
+  const isOrdered = elem.name === "ol";
+  const indent = "  ".repeat(depth);
+  const lines: string[] = [];
+
+  $elem.children("li").each((i, li) => {
+    const prefix = isOrdered ? `${i + 1}. ` : "- ";
+    const $li = $(li);
+
+    const nestedLists: string[] = [];
+
+    // Recurse into nested lists first (depth-first)
+    $li.children("ol, ul").each((_, nested) => {
+      const nestedMarkdown = listToMarkdown($, nested, depth + 1);
+      nestedLists.push(nestedMarkdown);
+      $(nested).remove();
+    });
+
+    // Now get full text
+    const itemText = $li.text().trim().replace(/\s+/g, " ");
+    lines.push(`${indent}${prefix}${itemText}`);
+
+    for (const nested of nestedLists) {
+      lines.push(nested);
+    }
+  });
+
+  return lines.join("\n");
+}
+
+export function convertListsToMarkdown(
+  $: cheerio.CheerioAPI,
+  ordered: string[] = ["ol"],
+  unordered: string[] = ["ul"],
+): void {
+  const selectors = [...ordered, ...unordered].join(", ");
+  if (!selectors) return;
+
+  // Process from outermost — recursion handles depth-first nesting
+  $(selectors).each((_, elem) => {
+    // Skip if already processed (parent already handled this)
+    if ($(elem).hasClass("list-markdown") || $(elem).hasClass("list-processed"))
+      return;
+    const markdown = listToMarkdown($, elem, 0);
+    $(elem).replaceWith($(`
\n${markdown}\n
`));
+  });
+}
+
+export function convertGrammarToMarkdown(
+  $: cheerio.CheerioAPI,
+  selector: string = "emu-grammar",
+): void {
+  if (!selector) return;
+  $(selector).each((_, grammar) => {
+    const text = $(grammar).text().trim();
+    $(grammar).replaceWith(
+      $(`
\`\`\`bnf\n${text}\n\`\`\`
`),
+    );
+  });
+}
+
 /**
  * Converts HTML tables to markdown table format.
  * This preserves table structure when extracting text from HTML.
@@ -95,7 +257,7 @@ export function convertTablesToMarkdown($: cheerio.CheerioAPI): void {
 
     // Replace table with markdown
     const markdown = mdLines.join("\n");
-    $table.replaceWith($(`
${markdown}
`));
+    $table.replaceWith($(`
\n${markdown}\n
`));
   });
 }
 
@@ -118,3 +280,27 @@ export function addNewlinesAfterBlocks(
       });
   }
 }
+
+export function formatForIngestion(
+  $: cheerio.CheerioAPI,
+  config: Partial = {},
+): void {
+  const cfg = { ...DEFAULT_CONFIG, ...config };
+
+  // 1. Inject newlines globally (affects li, p, pre, emu-production, emu-rhs, etc.)
+  addNewlinesAfterBlocks($);
+
+  // 2. Inline leaves (links first — must be before lists/code destroy DOM)
+  convertLinksToMarkdown($, cfg.links.join(", "));
+
+  // 3. Inline code
+  convertInlineCodeToMarkdown($, cfg.codeBlocks.inline);
+
+  // 4. Block leaves (fenced code)
+  convertBlockCodeToMarkdown($, cfg.codeBlocks.block);
+
+  // 5. Structural parents (grammar, lists, tables)
+  convertGrammarToMarkdown($, cfg.codeBlocks.grammar.join(", "));
+  convertListsToMarkdown($, cfg.lists.ordered, cfg.lists.unordered);
+  convertTablesToMarkdown($);
+}