mirror of
https://github.com/bendtherules/ask262.git
synced 2026-08-18 13:21:55 +00:00
build: Implement formatForIngestion
This commit is contained in:
Vendored
+1
-1
@@ -8,7 +8,7 @@
|
|||||||
"search.exclude": {
|
"search.exclude": {
|
||||||
"**/node_modules": true,
|
"**/node_modules": true,
|
||||||
"**/dist": true,
|
"**/dist": true,
|
||||||
"**/storage": true,
|
"**/storage": true
|
||||||
},
|
},
|
||||||
"files.exclude": {
|
"files.exclude": {
|
||||||
"**/node_modules": true,
|
"**/node_modules": true,
|
||||||
|
|||||||
@@ -21,7 +21,11 @@ import { GRAPH_FILE } from "../constants";
|
|||||||
*/
|
*/
|
||||||
async function buildGraph() {
|
async function buildGraph() {
|
||||||
// Initialize a multi-graph (allows multiple edges between same nodes)
|
// Initialize a multi-graph (allows multiple edges between same nodes)
|
||||||
const graph = new Graph({ multi: true, type: "directed", allowSelfLoops: false });
|
const graph = new Graph({
|
||||||
|
multi: true,
|
||||||
|
type: "directed",
|
||||||
|
allowSelfLoops: false,
|
||||||
|
});
|
||||||
|
|
||||||
// Phase 1: Discover and parse specification HTML files
|
// Phase 1: Discover and parse specification HTML files
|
||||||
const htmlFiles = await glob(path.join(SPEC_DIR, "*.html"));
|
const htmlFiles = await glob(path.join(SPEC_DIR, "*.html"));
|
||||||
|
|||||||
+12
-15
@@ -10,10 +10,7 @@ import { glob } from "glob";
|
|||||||
import ora from "ora";
|
import ora from "ora";
|
||||||
import { EMBEDDING_MODEL, SPEC_DIR, STORAGE_DIR } from "../constants";
|
import { EMBEDDING_MODEL, SPEC_DIR, STORAGE_DIR } from "../constants";
|
||||||
import { HTMLTextSplitter } from "./textsplitters";
|
import { HTMLTextSplitter } from "./textsplitters";
|
||||||
import {
|
import { formatForIngestion } from "./utils/formatHTMLForIngestion";
|
||||||
addNewlinesAfterBlocks,
|
|
||||||
convertTablesToMarkdown,
|
|
||||||
} from "./utils/formatHTMLForIngestion";
|
|
||||||
|
|
||||||
const embeddings = new OllamaEmbeddings({
|
const embeddings = new OllamaEmbeddings({
|
||||||
model: EMBEDDING_MODEL,
|
model: EMBEDDING_MODEL,
|
||||||
@@ -153,8 +150,10 @@ async function buildSpecDocuments(): Promise<Document[]> {
|
|||||||
// Build children relationships
|
// Build children relationships
|
||||||
for (const [id, section] of sectionMap) {
|
for (const [id, section] of sectionMap) {
|
||||||
if (section.parentId && sectionMap.has(section.parentId)) {
|
if (section.parentId && sectionMap.has(section.parentId)) {
|
||||||
const parent = sectionMap.get(section.parentId)!;
|
const parent = sectionMap.get(section.parentId);
|
||||||
parent.childrenIds.push(id);
|
if (parent) {
|
||||||
|
parent.childrenIds.push(id);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -163,18 +162,17 @@ async function buildSpecDocuments(): Promise<Document[]> {
|
|||||||
// Parse the stored HTML and replace direct children with placeholders
|
// Parse the stored HTML and replace direct children with placeholders
|
||||||
const $ = cheerio.load(`<body>${section.html}</body>`);
|
const $ = cheerio.load(`<body>${section.html}</body>`);
|
||||||
|
|
||||||
// Convert tables to markdown format for better text extraction
|
|
||||||
convertTablesToMarkdown($);
|
|
||||||
|
|
||||||
const $section = $.root();
|
const $section = $.root();
|
||||||
|
|
||||||
// Find direct children emu-clause elements only
|
// Find direct children emu-clause elements only
|
||||||
$section.children("emu-clause").each((_, childElem) => {
|
$section.children("emu-clause").each((_, childElem) => {
|
||||||
const childId = $(childElem).attr("id");
|
const childId = $(childElem).attr("id");
|
||||||
if (childId && sectionMap.has(childId)) {
|
if (childId && sectionMap.has(childId)) {
|
||||||
const child = sectionMap.get(childId)!;
|
const child = sectionMap.get(childId);
|
||||||
const placeholder = `[Subsection available: sectiontitle "${child.title}" at sectionid: \`${childId}\`]`;
|
if (child) {
|
||||||
$(childElem).replaceWith(placeholder);
|
const placeholder = `[Subsection available: sectiontitle "${child.title}" at sectionid: \`${childId}\`]`;
|
||||||
|
$(childElem).replaceWith(placeholder);
|
||||||
|
}
|
||||||
} else {
|
} else {
|
||||||
$(childElem).remove();
|
$(childElem).remove();
|
||||||
}
|
}
|
||||||
@@ -184,9 +182,8 @@ async function buildSpecDocuments(): Promise<Document[]> {
|
|||||||
// (shouldn't happen with proper HTML structure, but just in case)
|
// (shouldn't happen with proper HTML structure, but just in case)
|
||||||
$section.find("emu-clause").remove();
|
$section.find("emu-clause").remove();
|
||||||
|
|
||||||
// Add newlines after block elements to preserve document structure
|
// All formatting transformations (single call)
|
||||||
// This helps the text splitter maintain paragraph/section boundaries
|
formatForIngestion($);
|
||||||
addNewlinesAfterBlocks($);
|
|
||||||
|
|
||||||
// Skip sections that only have h1 left (no meaningful content)
|
// Skip sections that only have h1 left (no meaningful content)
|
||||||
const hasOnlyH1 =
|
const hasOnlyH1 =
|
||||||
|
|||||||
@@ -0,0 +1,147 @@
|
|||||||
|
import { describe, expect, test } from "bun:test";
|
||||||
|
import * as cheerio from "cheerio";
|
||||||
|
import {
|
||||||
|
convertBlockCodeToMarkdown,
|
||||||
|
convertGrammarToMarkdown,
|
||||||
|
convertInlineCodeToMarkdown,
|
||||||
|
convertLinksToMarkdown,
|
||||||
|
convertListsToMarkdown,
|
||||||
|
convertTablesToMarkdown,
|
||||||
|
DEFAULT_CONFIG,
|
||||||
|
formatForIngestion,
|
||||||
|
} from "./formatHTMLForIngestion";
|
||||||
|
|
||||||
|
describe("formatHTMLForIngestion", () => {
|
||||||
|
test("convertLinksToMarkdown converts emu-xref links to markdown and strips filenames", () => {
|
||||||
|
const html = `<emu-xref href="abstract-operations.html#sec-tonumber"><a href="abstract-operations.html#sec-tonumber">ToNumber</a></emu-xref>`;
|
||||||
|
const $ = cheerio.load(html);
|
||||||
|
convertLinksToMarkdown($, DEFAULT_CONFIG.links.join(", "));
|
||||||
|
expect($.html()).toContain(
|
||||||
|
'<span class="link-markdown">[ToNumber](#sec-tonumber)</span>',
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("convertLinksToMarkdown skips external links", () => {
|
||||||
|
const html = `<emu-xref href="https://example.com"><a href="https://example.com">External</a></emu-xref>`;
|
||||||
|
const $ = cheerio.load(html);
|
||||||
|
convertLinksToMarkdown($, DEFAULT_CONFIG.links.join(", "));
|
||||||
|
expect($.html()).toContain('<a href="https://example.com">External</a>');
|
||||||
|
});
|
||||||
|
|
||||||
|
test("convertInlineCodeToMarkdown converts var, emu-val, emu-const to inline code", () => {
|
||||||
|
const html = `<div><var>x</var> <emu-val>y</emu-val> <emu-const>z</emu-const> <code>w</code></div>`;
|
||||||
|
const $ = cheerio.load(html);
|
||||||
|
convertInlineCodeToMarkdown($, DEFAULT_CONFIG.codeBlocks.inline);
|
||||||
|
expect($.html()).toContain('<span class="inline-code">`x`</span>');
|
||||||
|
expect($.html()).toContain('<span class="inline-code">`y`</span>');
|
||||||
|
expect($.html()).toContain('<span class="inline-code">`z`</span>');
|
||||||
|
expect($.html()).toContain('<span class="inline-code">`w`</span>');
|
||||||
|
});
|
||||||
|
|
||||||
|
test("convertInlineCodeToMarkdown skips code inside pre", () => {
|
||||||
|
const html = `<pre><code>x</code></pre>`;
|
||||||
|
const $ = cheerio.load(html);
|
||||||
|
convertInlineCodeToMarkdown($, DEFAULT_CONFIG.codeBlocks.inline);
|
||||||
|
expect($.html()).toContain("<pre><code>x</code></pre>"); // unchanged
|
||||||
|
});
|
||||||
|
|
||||||
|
test("convertBlockCodeToMarkdown converts pre>code and emu-eqn", () => {
|
||||||
|
const html = `<div>
|
||||||
|
<pre><code class="javascript hljs">const x = 1;</code></pre>
|
||||||
|
<emu-eqn>y = x + 1</emu-eqn>
|
||||||
|
<emu-eqn class="inline">z = 2</emu-eqn>
|
||||||
|
</div>`;
|
||||||
|
const $ = cheerio.load(html);
|
||||||
|
convertBlockCodeToMarkdown($, DEFAULT_CONFIG.codeBlocks.block);
|
||||||
|
expect($.html()).toContain(
|
||||||
|
'<pre class="code-markdown">```javascript\nconst x = 1;\n```</pre>',
|
||||||
|
);
|
||||||
|
expect($.html()).toContain(
|
||||||
|
'<pre class="code-markdown">```\ny = x + 1\n```</pre>',
|
||||||
|
);
|
||||||
|
expect($.html()).toContain('<emu-eqn class="inline">z = 2</emu-eqn>'); // unchanged
|
||||||
|
});
|
||||||
|
|
||||||
|
test("convertGrammarToMarkdown converts emu-grammar to fenced bnf", () => {
|
||||||
|
const html = `<emu-grammar>Statement :: BlockStatement</emu-grammar>`;
|
||||||
|
const $ = cheerio.load(html);
|
||||||
|
convertGrammarToMarkdown($, DEFAULT_CONFIG.codeBlocks.grammar.join(", "));
|
||||||
|
expect($.html()).toContain(
|
||||||
|
'<pre class="code-markdown">```bnf\nStatement :: BlockStatement\n```</pre>',
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("convertListsToMarkdown converts ol and ul with nesting to markdown lists", () => {
|
||||||
|
const html = `
|
||||||
|
<ul>
|
||||||
|
<li>Item 1</li>
|
||||||
|
<li>Item 2
|
||||||
|
<ol>
|
||||||
|
<li>Subitem 1</li>
|
||||||
|
<li>Subitem 2</li>
|
||||||
|
</ol>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
`;
|
||||||
|
const $ = cheerio.load(html);
|
||||||
|
convertListsToMarkdown(
|
||||||
|
$,
|
||||||
|
DEFAULT_CONFIG.lists.ordered,
|
||||||
|
DEFAULT_CONFIG.lists.unordered,
|
||||||
|
);
|
||||||
|
const result = $("pre.list-markdown").text();
|
||||||
|
expect(result).toContain("- Item 1");
|
||||||
|
expect(result).toContain("- Item 2");
|
||||||
|
expect(result).toContain(" 1. Subitem 1");
|
||||||
|
expect(result).toContain(" 2. Subitem 2");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("convertTablesToMarkdown converts tables to markdown tables", () => {
|
||||||
|
const html = `
|
||||||
|
<table>
|
||||||
|
<thead>
|
||||||
|
<tr><th>Col 1</th><th>Col 2</th></tr>
|
||||||
|
</thead>
|
||||||
|
<tbody>
|
||||||
|
<tr><td>Data 1</td><td>Data 2</td></tr>
|
||||||
|
</tbody>
|
||||||
|
</table>
|
||||||
|
`;
|
||||||
|
const $ = cheerio.load(html);
|
||||||
|
convertTablesToMarkdown($);
|
||||||
|
const result = $("pre.table-markdown").text();
|
||||||
|
expect(result).toContain("| Col 1 | Col 2 |");
|
||||||
|
expect(result).toContain("| --- | --- |");
|
||||||
|
expect(result).toContain("| Data 1 | Data 2 |");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("formatForIngestion runs the full pipeline", () => {
|
||||||
|
const html = `
|
||||||
|
<div>
|
||||||
|
<p>See <emu-xref href="#sec-example"><a href="#sec-example">Example</a></emu-xref> for <var>x</var></p>
|
||||||
|
<pre><code class="javascript">let y = x;</code></pre>
|
||||||
|
<ul>
|
||||||
|
<li>One</li>
|
||||||
|
<li>Two</li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
`;
|
||||||
|
const $ = cheerio.load(html);
|
||||||
|
formatForIngestion($);
|
||||||
|
|
||||||
|
// Check links
|
||||||
|
expect($.html()).toContain(
|
||||||
|
'<span class="link-markdown">[Example](#sec-example)</span>',
|
||||||
|
);
|
||||||
|
// Check inline code
|
||||||
|
expect($.html()).toContain('<span class="inline-code">`x`</span>');
|
||||||
|
// Check block code
|
||||||
|
expect($.html()).toContain(
|
||||||
|
'<pre class="code-markdown">```javascript\nlet y = x;\n```</pre>',
|
||||||
|
);
|
||||||
|
// Check lists
|
||||||
|
expect($.html()).toContain(
|
||||||
|
'<pre class="list-markdown">- One\n- Two\n</pre>',
|
||||||
|
);
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -30,8 +30,170 @@ export const BLOCK_ELEMENTS = [
|
|||||||
"dl",
|
"dl",
|
||||||
"dt",
|
"dt",
|
||||||
"dd",
|
"dd",
|
||||||
|
"emu-production",
|
||||||
|
"emu-rhs",
|
||||||
];
|
];
|
||||||
|
|
||||||
|
export interface FormatConfig {
|
||||||
|
codeBlocks: {
|
||||||
|
block: string[];
|
||||||
|
inline: string[];
|
||||||
|
grammar: string[];
|
||||||
|
};
|
||||||
|
links: string[];
|
||||||
|
lists: {
|
||||||
|
ordered: string[];
|
||||||
|
unordered: string[];
|
||||||
|
};
|
||||||
|
tables: string[];
|
||||||
|
}
|
||||||
|
|
||||||
|
export const DEFAULT_CONFIG: FormatConfig = {
|
||||||
|
codeBlocks: {
|
||||||
|
block: ["pre>code", "emu-eqn:not([class*='inline'])"],
|
||||||
|
inline: ["var", "emu-val", "emu-const", "emu-eqn.inline", "code"],
|
||||||
|
grammar: ["emu-grammar"],
|
||||||
|
},
|
||||||
|
links: ["emu-xref a"],
|
||||||
|
lists: {
|
||||||
|
ordered: ["ol"],
|
||||||
|
unordered: ["ul"],
|
||||||
|
},
|
||||||
|
tables: ["table", "emu-table"],
|
||||||
|
};
|
||||||
|
|
||||||
|
export function convertLinksToMarkdown(
|
||||||
|
$: cheerio.CheerioAPI,
|
||||||
|
selector: string = "emu-xref a[href]",
|
||||||
|
): void {
|
||||||
|
$(selector).each((_, elem) => {
|
||||||
|
const $a = $(elem);
|
||||||
|
const href = $a.attr("href") ?? "";
|
||||||
|
|
||||||
|
// Skip external links
|
||||||
|
if (href.startsWith("http")) return;
|
||||||
|
|
||||||
|
// Strip filename prefix: "abstract-operations.html#sec-tonumber" → "#sec-tonumber"
|
||||||
|
const hash = href.includes("#") ? `#${href.split("#")[1]}` : href;
|
||||||
|
const text = $a.text().trim();
|
||||||
|
|
||||||
|
if (!text || !hash) return;
|
||||||
|
|
||||||
|
$a.replaceWith($(`<span class="link-markdown">[${text}](${hash})</span>`));
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
export function convertInlineCodeToMarkdown(
|
||||||
|
$: cheerio.CheerioAPI,
|
||||||
|
tags: string[],
|
||||||
|
): void {
|
||||||
|
for (const tag of tags) {
|
||||||
|
$(tag).each((_, elem) => {
|
||||||
|
// Skip <code> inside <pre> (handled by block converter)
|
||||||
|
if (
|
||||||
|
"name" in elem &&
|
||||||
|
elem.name === "code" &&
|
||||||
|
$(elem).parent("pre").length > 0
|
||||||
|
)
|
||||||
|
return;
|
||||||
|
|
||||||
|
const text = $(elem).text().trim();
|
||||||
|
if (!text) return;
|
||||||
|
$(elem).replaceWith($(`<span class="inline-code">\`${text}\`</span>`));
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function extractLanguage(codeClass: string | undefined): string {
|
||||||
|
// "javascript hljs" → "javascript", "python hljs" → "python"
|
||||||
|
if (!codeClass) return "";
|
||||||
|
const match = codeClass.match(/^(\w+)/);
|
||||||
|
return match?.[1] ?? "";
|
||||||
|
}
|
||||||
|
|
||||||
|
export function convertBlockCodeToMarkdown(
|
||||||
|
$: cheerio.CheerioAPI,
|
||||||
|
tags: string[],
|
||||||
|
): void {
|
||||||
|
const selectors = tags.join(", ");
|
||||||
|
if (!selectors) return;
|
||||||
|
|
||||||
|
$(selectors).each((_, elem) => {
|
||||||
|
const $elem = $(elem);
|
||||||
|
const lang = extractLanguage($elem.attr("class"));
|
||||||
|
const text = $elem.text().trim();
|
||||||
|
$elem.replaceWith(
|
||||||
|
$(`<pre class="code-markdown">\`\`\`${lang}\n${text}\n\`\`\`</pre>`),
|
||||||
|
);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function listToMarkdown(
|
||||||
|
$: cheerio.CheerioAPI,
|
||||||
|
elem: any,
|
||||||
|
depth: number = 0,
|
||||||
|
): string {
|
||||||
|
const $elem = $(elem);
|
||||||
|
const isOrdered = elem.name === "ol";
|
||||||
|
const indent = " ".repeat(depth);
|
||||||
|
const lines: string[] = [];
|
||||||
|
|
||||||
|
$elem.children("li").each((i, li) => {
|
||||||
|
const prefix = isOrdered ? `${i + 1}. ` : "- ";
|
||||||
|
const $li = $(li);
|
||||||
|
|
||||||
|
const nestedLists: string[] = [];
|
||||||
|
|
||||||
|
// Recurse into nested lists first (depth-first)
|
||||||
|
$li.children("ol, ul").each((_, nested) => {
|
||||||
|
const nestedMarkdown = listToMarkdown($, nested, depth + 1);
|
||||||
|
nestedLists.push(nestedMarkdown);
|
||||||
|
$(nested).remove();
|
||||||
|
});
|
||||||
|
|
||||||
|
// Now get full text
|
||||||
|
const itemText = $li.text().trim().replace(/\s+/g, " ");
|
||||||
|
lines.push(`${indent}${prefix}${itemText}`);
|
||||||
|
|
||||||
|
for (const nested of nestedLists) {
|
||||||
|
lines.push(nested);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
return lines.join("\n");
|
||||||
|
}
|
||||||
|
|
||||||
|
export function convertListsToMarkdown(
|
||||||
|
$: cheerio.CheerioAPI,
|
||||||
|
ordered: string[] = ["ol"],
|
||||||
|
unordered: string[] = ["ul"],
|
||||||
|
): void {
|
||||||
|
const selectors = [...ordered, ...unordered].join(", ");
|
||||||
|
if (!selectors) return;
|
||||||
|
|
||||||
|
// Process from outermost — recursion handles depth-first nesting
|
||||||
|
$(selectors).each((_, elem) => {
|
||||||
|
// Skip if already processed (parent already handled this)
|
||||||
|
if ($(elem).hasClass("list-markdown") || $(elem).hasClass("list-processed"))
|
||||||
|
return;
|
||||||
|
const markdown = listToMarkdown($, elem, 0);
|
||||||
|
$(elem).replaceWith($(`<pre class="list-markdown">\n${markdown}\n</pre>`));
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
export function convertGrammarToMarkdown(
|
||||||
|
$: cheerio.CheerioAPI,
|
||||||
|
selector: string = "emu-grammar",
|
||||||
|
): void {
|
||||||
|
if (!selector) return;
|
||||||
|
$(selector).each((_, grammar) => {
|
||||||
|
const text = $(grammar).text().trim();
|
||||||
|
$(grammar).replaceWith(
|
||||||
|
$(`<pre class="code-markdown">\`\`\`bnf\n${text}\n\`\`\`</pre>`),
|
||||||
|
);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Converts HTML tables to markdown table format.
|
* Converts HTML tables to markdown table format.
|
||||||
* This preserves table structure when extracting text from HTML.
|
* This preserves table structure when extracting text from HTML.
|
||||||
@@ -95,7 +257,7 @@ export function convertTablesToMarkdown($: cheerio.CheerioAPI): void {
|
|||||||
|
|
||||||
// Replace table with markdown
|
// Replace table with markdown
|
||||||
const markdown = mdLines.join("\n");
|
const markdown = mdLines.join("\n");
|
||||||
$table.replaceWith($(`<pre class="table-markdown">${markdown}</pre>`));
|
$table.replaceWith($(`<pre class="table-markdown">\n${markdown}\n</pre>`));
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -118,3 +280,27 @@ export function addNewlinesAfterBlocks(
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export function formatForIngestion(
|
||||||
|
$: cheerio.CheerioAPI,
|
||||||
|
config: Partial<FormatConfig> = {},
|
||||||
|
): void {
|
||||||
|
const cfg = { ...DEFAULT_CONFIG, ...config };
|
||||||
|
|
||||||
|
// 1. Inject newlines globally (affects li, p, pre, emu-production, emu-rhs, etc.)
|
||||||
|
addNewlinesAfterBlocks($);
|
||||||
|
|
||||||
|
// 2. Inline leaves (links first — must be before lists/code destroy DOM)
|
||||||
|
convertLinksToMarkdown($, cfg.links.join(", "));
|
||||||
|
|
||||||
|
// 3. Inline code
|
||||||
|
convertInlineCodeToMarkdown($, cfg.codeBlocks.inline);
|
||||||
|
|
||||||
|
// 4. Block leaves (fenced code)
|
||||||
|
convertBlockCodeToMarkdown($, cfg.codeBlocks.block);
|
||||||
|
|
||||||
|
// 5. Structural parents (grammar, lists, tables)
|
||||||
|
convertGrammarToMarkdown($, cfg.codeBlocks.grammar.join(", "));
|
||||||
|
convertListsToMarkdown($, cfg.lists.ordered, cfg.lists.unordered);
|
||||||
|
convertTablesToMarkdown($);
|
||||||
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user