mirror of
https://github.com/bendtherules/ask262.git
synced 2026-08-19 00:31:06 +00:00
refactor: rename files
This commit is contained in:
@@ -0,0 +1,388 @@
|
||||
import { describe, expect, test } from "bun:test";
|
||||
import * as cheerio from "cheerio";
|
||||
import {
|
||||
convertBlockCodeToMarkdown,
|
||||
convertGrammarToMarkdown,
|
||||
convertInlineCodeToMarkdown,
|
||||
convertLinksToMarkdown,
|
||||
convertListsToMarkdown,
|
||||
convertTablesToMarkdown,
|
||||
DEFAULT_CONFIG,
|
||||
formatForIngestion,
|
||||
} from "./formatHTMLForIngestion";
|
||||
|
||||
describe("formatHTMLForIngestion", () => {
|
||||
describe("convertLinksToMarkdown", () => {
|
||||
test("converts emu-xref links to markdown and strips filenames", () => {
|
||||
const html = `<emu-xref href="abstract-operations.html#sec-tonumber"><a href="abstract-operations.html#sec-tonumber">ToNumber</a></emu-xref>`;
|
||||
const $ = cheerio.load(html);
|
||||
convertLinksToMarkdown($, DEFAULT_CONFIG.links.join(", "));
|
||||
expect($.html()).toContain(
|
||||
'<span class="link-markdown">[ToNumber](#sec-tonumber)</span>',
|
||||
);
|
||||
});
|
||||
|
||||
test("skips external links", () => {
|
||||
const html = `<emu-xref href="https://example.com"><a href="https://example.com">External</a></emu-xref>`;
|
||||
const $ = cheerio.load(html);
|
||||
convertLinksToMarkdown($, DEFAULT_CONFIG.links.join(", "));
|
||||
expect($.html()).toContain('<a href="https://example.com">External</a>');
|
||||
});
|
||||
});
|
||||
|
||||
describe("convertInlineCodeToMarkdown", () => {
|
||||
test("converts var, emu-val, emu-const to inline code", () => {
|
||||
const html = `<div><var>x</var> <emu-val>y</emu-val> <emu-const>z</emu-const> <code>w</code></div>`;
|
||||
const $ = cheerio.load(html);
|
||||
convertInlineCodeToMarkdown($, DEFAULT_CONFIG.codeBlocks.inline);
|
||||
expect($.html()).toContain('<span class="inline-code">`x`</span>');
|
||||
expect($.html()).toContain('<span class="inline-code">`y`</span>');
|
||||
expect($.html()).toContain('<span class="inline-code">`z`</span>');
|
||||
expect($.html()).toContain('<span class="inline-code">`w`</span>');
|
||||
});
|
||||
|
||||
test("skips code inside pre", () => {
|
||||
const html = `<pre><code>x</code></pre>`;
|
||||
const $ = cheerio.load(html);
|
||||
convertInlineCodeToMarkdown($, DEFAULT_CONFIG.codeBlocks.inline);
|
||||
expect($.html()).toContain("<pre><code>x</code></pre>");
|
||||
});
|
||||
});
|
||||
|
||||
describe("convertBlockCodeToMarkdown", () => {
|
||||
test("converts pre>code and emu-eqn", () => {
|
||||
const html = `<div>
|
||||
<pre><code class="javascript hljs">const x = 1;</code></pre>
|
||||
<emu-eqn>y = x + 1</emu-eqn>
|
||||
<emu-eqn class="inline">z = 2</emu-eqn>
|
||||
</div>`;
|
||||
const $ = cheerio.load(html);
|
||||
convertBlockCodeToMarkdown($, DEFAULT_CONFIG.codeBlocks.block);
|
||||
expect($.html()).toContain(
|
||||
'<pre class="code-markdown">```javascript\nconst x = 1;\n```</pre>',
|
||||
);
|
||||
expect($.html()).toContain(
|
||||
'<pre class="code-markdown">```\ny = x + 1\n```</pre>',
|
||||
);
|
||||
expect($.html()).toContain('<emu-eqn class="inline">z = 2</emu-eqn>');
|
||||
});
|
||||
});
|
||||
|
||||
describe("convertGrammarToMarkdown", () => {
|
||||
test("converts emu-grammar to fenced bnf", () => {
|
||||
const html = `<emu-grammar>Statement :: BlockStatement</emu-grammar>`;
|
||||
const $ = cheerio.load(html);
|
||||
convertGrammarToMarkdown($, DEFAULT_CONFIG.codeBlocks.grammar.join(", "));
|
||||
expect($.html()).toContain(
|
||||
'<pre class="code-markdown">```bnf\nStatement :: BlockStatement\n```</pre>',
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe("convertListsToMarkdown", () => {
|
||||
test("converts ul with nested ol", () => {
|
||||
const html = `
|
||||
<ul>
|
||||
<li>Item 1</li>
|
||||
<li>Item 2
|
||||
<ol>
|
||||
<li>Subitem 1</li>
|
||||
<li>Subitem 2</li>
|
||||
</ol>
|
||||
</li>
|
||||
</ul>
|
||||
`;
|
||||
const $ = cheerio.load(html);
|
||||
convertListsToMarkdown(
|
||||
$,
|
||||
DEFAULT_CONFIG.lists.ordered,
|
||||
DEFAULT_CONFIG.lists.unordered,
|
||||
);
|
||||
const lines = $("pre.list-markdown")
|
||||
.text()
|
||||
.split("\n")
|
||||
.filter((l) => l.length > 0);
|
||||
expect(lines).toEqual([
|
||||
"- Item 1",
|
||||
"- Item 2",
|
||||
" A. Subitem 1",
|
||||
" B. Subitem 2",
|
||||
]);
|
||||
});
|
||||
|
||||
test("handles 3-level deep nesting (ol > ul > ol)", () => {
|
||||
const html = `
|
||||
<ol>
|
||||
<li>First</li>
|
||||
<li>Second
|
||||
<ul>
|
||||
<li>Alpha
|
||||
<ol>
|
||||
<li>Deep 1</li>
|
||||
<li>Deep 2</li>
|
||||
</ol>
|
||||
</li>
|
||||
<li>Beta</li>
|
||||
</ul>
|
||||
</li>
|
||||
</ol>
|
||||
`;
|
||||
const $ = cheerio.load(html);
|
||||
convertListsToMarkdown(
|
||||
$,
|
||||
DEFAULT_CONFIG.lists.ordered,
|
||||
DEFAULT_CONFIG.lists.unordered,
|
||||
);
|
||||
const lines = $("pre.list-markdown")
|
||||
.text()
|
||||
.split("\n")
|
||||
.filter((l) => l.length > 0);
|
||||
expect(lines).toEqual([
|
||||
"1. First",
|
||||
"2. Second",
|
||||
" * Alpha",
|
||||
" 1. Deep 1",
|
||||
" 2. Deep 2",
|
||||
" * Beta",
|
||||
]);
|
||||
});
|
||||
|
||||
test("handles multiple sibling nested lists in one item", () => {
|
||||
const html = `
|
||||
<ul>
|
||||
<li>Item
|
||||
<ol>
|
||||
<li>Ordered sub</li>
|
||||
</ol>
|
||||
<ul>
|
||||
<li>Unordered sub</li>
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
`;
|
||||
const $ = cheerio.load(html);
|
||||
convertListsToMarkdown(
|
||||
$,
|
||||
DEFAULT_CONFIG.lists.ordered,
|
||||
DEFAULT_CONFIG.lists.unordered,
|
||||
);
|
||||
const lines = $("pre.list-markdown")
|
||||
.text()
|
||||
.split("\n")
|
||||
.filter((l) => l.length > 0);
|
||||
expect(lines).toEqual([
|
||||
"- Item",
|
||||
" A. Ordered sub",
|
||||
" * Unordered sub",
|
||||
]);
|
||||
});
|
||||
|
||||
test("handles consecutive top-level lists", () => {
|
||||
const html = `
|
||||
<ul>
|
||||
<li>UL item 1</li>
|
||||
<li>UL item 2</li>
|
||||
</ul>
|
||||
<ol>
|
||||
<li>OL item 1</li>
|
||||
<li>OL item 2</li>
|
||||
</ol>
|
||||
`;
|
||||
const $ = cheerio.load(html);
|
||||
convertListsToMarkdown(
|
||||
$,
|
||||
DEFAULT_CONFIG.lists.ordered,
|
||||
DEFAULT_CONFIG.lists.unordered,
|
||||
);
|
||||
const results = $("pre.list-markdown");
|
||||
expect(results.length).toBe(2);
|
||||
|
||||
const ulLines = results
|
||||
.eq(0)
|
||||
.text()
|
||||
.split("\n")
|
||||
.filter((l) => l.length > 0);
|
||||
expect(ulLines).toEqual(["- UL item 1", "- UL item 2"]);
|
||||
|
||||
const olLines = results
|
||||
.eq(1)
|
||||
.text()
|
||||
.split("\n")
|
||||
.filter((l) => l.length > 0);
|
||||
expect(olLines).toEqual(["1. OL item 1", "2. OL item 2"]);
|
||||
});
|
||||
|
||||
test("preserves inline code inside list items", () => {
|
||||
const html = `
|
||||
<ul>
|
||||
<li>Call <code>foo()</code></li>
|
||||
<li>Use <var>x</var></li>
|
||||
</ul>
|
||||
`;
|
||||
const $ = cheerio.load(html);
|
||||
convertInlineCodeToMarkdown($, DEFAULT_CONFIG.codeBlocks.inline);
|
||||
convertListsToMarkdown(
|
||||
$,
|
||||
DEFAULT_CONFIG.lists.ordered,
|
||||
DEFAULT_CONFIG.lists.unordered,
|
||||
);
|
||||
const lines = $("pre.list-markdown")
|
||||
.text()
|
||||
.split("\n")
|
||||
.filter((l) => l.length > 0);
|
||||
expect(lines).toEqual(["- Call `foo()`", "- Use `x`"]);
|
||||
});
|
||||
|
||||
test("handles ol nested inside ol", () => {
|
||||
const html = `
|
||||
<ol>
|
||||
<li>First
|
||||
<ol>
|
||||
<li>Nested 1</li>
|
||||
<li>Nested 2</li>
|
||||
</ol>
|
||||
</li>
|
||||
<li>Second</li>
|
||||
</ol>
|
||||
`;
|
||||
const $ = cheerio.load(html);
|
||||
convertListsToMarkdown(
|
||||
$,
|
||||
DEFAULT_CONFIG.lists.ordered,
|
||||
DEFAULT_CONFIG.lists.unordered,
|
||||
);
|
||||
const lines = $("pre.list-markdown")
|
||||
.text()
|
||||
.split("\n")
|
||||
.filter((l) => l.length > 0);
|
||||
expect(lines).toEqual([
|
||||
"1. First",
|
||||
" A. Nested 1",
|
||||
" B. Nested 2",
|
||||
"2. Second",
|
||||
]);
|
||||
});
|
||||
|
||||
test("handles empty list", () => {
|
||||
const html = `<ul></ul>`;
|
||||
const $ = cheerio.load(html);
|
||||
convertListsToMarkdown(
|
||||
$,
|
||||
DEFAULT_CONFIG.lists.ordered,
|
||||
DEFAULT_CONFIG.lists.unordered,
|
||||
);
|
||||
const result = $("pre.list-markdown").text().trim();
|
||||
expect(result).toBe("");
|
||||
});
|
||||
});
|
||||
|
||||
describe("convertTablesToMarkdown", () => {
|
||||
test("converts tables to markdown tables", () => {
|
||||
const html = `
|
||||
<table>
|
||||
<thead>
|
||||
<tr><th>Col 1</th><th>Col 2</th></tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<tr><td>Data 1</td><td>Data 2</td></tr>
|
||||
</tbody>
|
||||
</table>
|
||||
`;
|
||||
const $ = cheerio.load(html);
|
||||
convertTablesToMarkdown($);
|
||||
const result = $("pre.table-markdown").text();
|
||||
expect(result).toContain("| Col 1 | Col 2 |");
|
||||
expect(result).toContain("| --- | --- |");
|
||||
expect(result).toContain("| Data 1 | Data 2 |");
|
||||
});
|
||||
});
|
||||
|
||||
describe("formatForIngestion", () => {
|
||||
test("runs the full pipeline", () => {
|
||||
const html = `
|
||||
<div>
|
||||
<p>See <emu-xref href="#sec-example"><a href="#sec-example">Example</a></emu-xref> for <var>x</var></p>
|
||||
<pre><code class="javascript">let y = x;</code></pre>
|
||||
<ul>
|
||||
<li>One</li>
|
||||
<li>Two</li>
|
||||
</ul>
|
||||
</div>
|
||||
`;
|
||||
const $ = cheerio.load(html);
|
||||
formatForIngestion($);
|
||||
|
||||
expect($.html()).toContain(
|
||||
'<span class="link-markdown">[Example](#sec-example)</span>',
|
||||
);
|
||||
expect($.html()).toContain('<span class="inline-code">`x`</span>');
|
||||
expect($.html()).toContain(
|
||||
'<pre class="code-markdown">```javascript\nlet y = x;\n```</pre>',
|
||||
);
|
||||
expect($.html()).toContain(
|
||||
'<pre class="list-markdown">- One\n- Two\n</pre>',
|
||||
);
|
||||
});
|
||||
|
||||
test("preserves nested list indentation through the full pipeline", () => {
|
||||
const html = `
|
||||
<ol>
|
||||
<li>First</li>
|
||||
<li>Second
|
||||
<ul>
|
||||
<li>Alpha
|
||||
<ol>
|
||||
<li>Deep 1</li>
|
||||
<li>Deep 2</li>
|
||||
</ol>
|
||||
</li>
|
||||
<li>Beta</li>
|
||||
</ul>
|
||||
</li>
|
||||
</ol>
|
||||
`;
|
||||
const $ = cheerio.load(html);
|
||||
formatForIngestion($);
|
||||
|
||||
const lines = $("pre.list-markdown")
|
||||
.text()
|
||||
.split("\n")
|
||||
.filter((l) => l.length > 0);
|
||||
expect(lines).toEqual([
|
||||
"1. First",
|
||||
"2. Second",
|
||||
" * Alpha",
|
||||
" 1. Deep 1",
|
||||
" 2. Deep 2",
|
||||
" * Beta",
|
||||
]);
|
||||
});
|
||||
|
||||
test("preserves ol nested inside ol through the full pipeline", () => {
|
||||
const html = `
|
||||
<ol>
|
||||
<li>First
|
||||
<ol>
|
||||
<li>Nested 1</li>
|
||||
<li>Nested 2</li>
|
||||
</ol>
|
||||
</li>
|
||||
<li>Second</li>
|
||||
</ol>
|
||||
`;
|
||||
const $ = cheerio.load(html);
|
||||
formatForIngestion($);
|
||||
|
||||
const lines = $("pre.list-markdown")
|
||||
.text()
|
||||
.split("\n")
|
||||
.filter((l) => l.length > 0);
|
||||
expect(lines).toEqual([
|
||||
"1. First",
|
||||
" A. Nested 1",
|
||||
" B. Nested 2",
|
||||
"2. Second",
|
||||
]);
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,330 @@
|
||||
import type * as cheerio from "cheerio";
|
||||
|
||||
/**
|
||||
* List of block-level HTML elements that should have newlines added after them
|
||||
* to preserve document structure during text extraction.
|
||||
*/
|
||||
export const BLOCK_ELEMENTS = [
|
||||
"p",
|
||||
"div",
|
||||
"h1",
|
||||
"h2",
|
||||
"h3",
|
||||
"h4",
|
||||
"h5",
|
||||
"h6",
|
||||
"li",
|
||||
"td",
|
||||
"th",
|
||||
"pre",
|
||||
"blockquote",
|
||||
"section",
|
||||
"emu-clause",
|
||||
"emu-note",
|
||||
"emu-example",
|
||||
"emu-table",
|
||||
"figure",
|
||||
"figcaption",
|
||||
"ul",
|
||||
"ol",
|
||||
"dl",
|
||||
"dt",
|
||||
"dd",
|
||||
"emu-production",
|
||||
"emu-rhs",
|
||||
];
|
||||
|
||||
export interface FormatConfig {
|
||||
codeBlocks: {
|
||||
block: string[];
|
||||
inline: string[];
|
||||
grammar: string[];
|
||||
};
|
||||
links: string[];
|
||||
lists: {
|
||||
ordered: string[];
|
||||
unordered: string[];
|
||||
};
|
||||
tables: string[];
|
||||
}
|
||||
|
||||
export const DEFAULT_CONFIG: FormatConfig = {
|
||||
codeBlocks: {
|
||||
block: ["pre>code", "emu-eqn:not([class*='inline'])"],
|
||||
inline: ["var", "emu-val", "emu-const", "emu-eqn.inline", "code"],
|
||||
grammar: ["emu-grammar"],
|
||||
},
|
||||
links: ["emu-xref a"],
|
||||
lists: {
|
||||
ordered: ["ol"],
|
||||
unordered: ["ul"],
|
||||
},
|
||||
tables: ["table", "emu-table"],
|
||||
};
|
||||
|
||||
export function convertLinksToMarkdown(
|
||||
$: cheerio.CheerioAPI,
|
||||
selector: string = "emu-xref a[href]",
|
||||
): void {
|
||||
$(selector).each((_, elem) => {
|
||||
const $a = $(elem);
|
||||
const href = $a.attr("href") ?? "";
|
||||
|
||||
// Skip external links
|
||||
if (href.startsWith("http")) return;
|
||||
|
||||
// Strip filename prefix: "abstract-operations.html#sec-tonumber" → "#sec-tonumber"
|
||||
const hash = href.includes("#") ? `#${href.split("#")[1]}` : href;
|
||||
const text = $a.text().trim();
|
||||
|
||||
if (!text || !hash) return;
|
||||
|
||||
$a.replaceWith($(`<span class="link-markdown">[${text}](${hash})</span>`));
|
||||
});
|
||||
}
|
||||
|
||||
export function convertInlineCodeToMarkdown(
|
||||
$: cheerio.CheerioAPI,
|
||||
tags: string[],
|
||||
): void {
|
||||
for (const tag of tags) {
|
||||
$(tag).each((_, elem) => {
|
||||
// Skip <code> inside <pre> (handled by block converter)
|
||||
if (
|
||||
"name" in elem &&
|
||||
elem.name === "code" &&
|
||||
$(elem).parent("pre").length > 0
|
||||
)
|
||||
return;
|
||||
|
||||
const text = $(elem).text().trim();
|
||||
if (!text) return;
|
||||
$(elem).replaceWith($(`<span class="inline-code">\`${text}\`</span>`));
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
function extractLanguage(codeClass: string | undefined): string {
|
||||
// "javascript hljs" → "javascript", "python hljs" → "python"
|
||||
if (!codeClass) return "";
|
||||
const match = codeClass.match(/^(\w+)/);
|
||||
return match?.[1] ?? "";
|
||||
}
|
||||
|
||||
export function convertBlockCodeToMarkdown(
|
||||
$: cheerio.CheerioAPI,
|
||||
tags: string[],
|
||||
): void {
|
||||
const selectors = tags.join(", ");
|
||||
if (!selectors) return;
|
||||
|
||||
$(selectors).each((_, elem) => {
|
||||
const $elem = $(elem);
|
||||
const lang = extractLanguage($elem.attr("class"));
|
||||
const text = $elem.text().trim();
|
||||
$elem.replaceWith(
|
||||
$(`<pre class="code-markdown">\`\`\`${lang}\n${text}\n\`\`\`</pre>`),
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the markdown list item prefix based on list type and nesting depth.
|
||||
* Alternates markers to visually distinguish nesting levels:
|
||||
* - Ordered lists: `1.` at even depth, `A.` at odd depth
|
||||
* - Unordered lists: `-` at even depth, `*` at odd depth
|
||||
*
|
||||
* @param isOrdered - Whether the list is ordered (ol) or unordered (ul)
|
||||
* @param depth - Nesting depth (0 = top-level)
|
||||
* @param index - Zero-based item index within its list
|
||||
* @returns Prefix string like "1. ", "A. ", "- ", or "* "
|
||||
*/
|
||||
function getListItemPrefix(
|
||||
isOrdered: boolean,
|
||||
depth: number,
|
||||
index: number,
|
||||
): string {
|
||||
if (isOrdered) {
|
||||
return depth % 2 === 0
|
||||
? `${index + 1}. `
|
||||
: `${String.fromCharCode(65 + index)}. `;
|
||||
}
|
||||
return depth % 2 === 0 ? "- " : "* ";
|
||||
}
|
||||
|
||||
function listToMarkdown(
|
||||
$: cheerio.CheerioAPI,
|
||||
elem: any,
|
||||
depth: number = 0,
|
||||
): string {
|
||||
const $elem = $(elem);
|
||||
const isOrdered = elem.name === "ol";
|
||||
const indent = " ".repeat(depth);
|
||||
const lines: string[] = [];
|
||||
|
||||
$elem.children("li").each((i, li) => {
|
||||
const prefix = getListItemPrefix(isOrdered, depth, i);
|
||||
const $li = $(li);
|
||||
|
||||
const nestedLists: string[] = [];
|
||||
|
||||
// Recurse into nested lists first (depth-first)
|
||||
$li.children("ol, ul").each((_, nested) => {
|
||||
const nestedMarkdown = listToMarkdown($, nested, depth + 1);
|
||||
nestedLists.push(nestedMarkdown);
|
||||
$(nested).remove();
|
||||
});
|
||||
|
||||
// Now get full text
|
||||
const itemText = $li.text().trim().replace(/\s+/g, " ");
|
||||
lines.push(`${indent}${prefix}${itemText}`);
|
||||
|
||||
for (const nested of nestedLists) {
|
||||
lines.push(nested);
|
||||
}
|
||||
});
|
||||
|
||||
return lines.join("\n");
|
||||
}
|
||||
|
||||
export function convertListsToMarkdown(
|
||||
$: cheerio.CheerioAPI,
|
||||
ordered: string[] = ["ol"],
|
||||
unordered: string[] = ["ul"],
|
||||
): void {
|
||||
const selectors = [...ordered, ...unordered].join(", ");
|
||||
if (!selectors) return;
|
||||
|
||||
// Process from outermost — recursion handles depth-first nesting
|
||||
$(selectors).each((_, elem) => {
|
||||
// Skip if already processed (parent already handled this)
|
||||
if ($(elem).hasClass("list-markdown") || $(elem).hasClass("list-processed"))
|
||||
return;
|
||||
const markdown = listToMarkdown($, elem, 0);
|
||||
$(elem).replaceWith($(`<pre class="list-markdown">\n${markdown}\n</pre>`));
|
||||
});
|
||||
}
|
||||
|
||||
export function convertGrammarToMarkdown(
|
||||
$: cheerio.CheerioAPI,
|
||||
selector: string = "emu-grammar",
|
||||
): void {
|
||||
if (!selector) return;
|
||||
$(selector).each((_, grammar) => {
|
||||
const text = $(grammar).text().trim();
|
||||
$(grammar).replaceWith(
|
||||
$(`<pre class="code-markdown">\`\`\`bnf\n${text}\n\`\`\`</pre>`),
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Converts HTML tables to markdown table format.
|
||||
* This preserves table structure when extracting text from HTML.
|
||||
* Replaces the original table with a <pre> element containing the markdown.
|
||||
*
|
||||
* @param $ - Cheerio API instance
|
||||
*/
|
||||
export function convertTablesToMarkdown($: cheerio.CheerioAPI): void {
|
||||
$("table, emu-table").each((_, tableElem) => {
|
||||
const $table = $(tableElem);
|
||||
const rows: string[][] = [];
|
||||
|
||||
// Extract header rows
|
||||
$table.find("thead tr").each((_, rowElem) => {
|
||||
const row: string[] = [];
|
||||
$(rowElem)
|
||||
.find("th, td")
|
||||
.each((_, cellElem) => {
|
||||
row.push($(cellElem).text().trim().replace(/\|/g, "\\|"));
|
||||
});
|
||||
if (row.length > 0) rows.push(row);
|
||||
});
|
||||
|
||||
// Extract body rows
|
||||
$table.find("tbody tr, tr").each((_, rowElem) => {
|
||||
// Skip if already processed as header
|
||||
if ($(rowElem).parent("thead").length > 0) return;
|
||||
|
||||
const row: string[] = [];
|
||||
$(rowElem)
|
||||
.find("td, th")
|
||||
.each((_, cellElem) => {
|
||||
row.push($(cellElem).text().trim().replace(/\|/g, "\\|"));
|
||||
});
|
||||
if (row.length > 0) rows.push(row);
|
||||
});
|
||||
|
||||
if (rows.length === 0) return;
|
||||
|
||||
// Determine max columns
|
||||
const maxCols = Math.max(...rows.map((r) => r.length));
|
||||
|
||||
// Build markdown table
|
||||
const mdLines: string[] = [];
|
||||
|
||||
// Header row
|
||||
if (rows.length > 0) {
|
||||
const header = rows[0].concat(Array(maxCols - rows[0].length).fill(""));
|
||||
mdLines.push("| " + header.join(" | ") + " |");
|
||||
}
|
||||
|
||||
// Separator
|
||||
mdLines.push("|" + Array(maxCols).fill(" --- ").join("|") + "|");
|
||||
|
||||
// Data rows (skip header if we have more rows)
|
||||
const dataRows = rows.length > 1 ? rows.slice(1) : [];
|
||||
for (const row of dataRows) {
|
||||
const padded = row.concat(Array(maxCols - row.length).fill(""));
|
||||
mdLines.push("| " + padded.join(" | ") + " |");
|
||||
}
|
||||
|
||||
// Replace table with markdown
|
||||
const markdown = mdLines.join("\n");
|
||||
$table.replaceWith($(`<pre class="table-markdown">\n${markdown}\n</pre>`));
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Adds newlines after specified block elements to preserve document structure.
|
||||
* This helps text splitters maintain paragraph/section boundaries.
|
||||
*
|
||||
* @param $ - Cheerio API instance
|
||||
* @param elements - Array of element tag names to add newlines after
|
||||
*/
|
||||
export function addNewlinesAfterBlocks(
|
||||
$: cheerio.CheerioAPI,
|
||||
elements: string[] = BLOCK_ELEMENTS,
|
||||
): void {
|
||||
for (const tag of elements) {
|
||||
$.root()
|
||||
.find(tag)
|
||||
.each((_, el) => {
|
||||
$(el).append("\n");
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
export function formatForIngestion(
|
||||
$: cheerio.CheerioAPI,
|
||||
config: Partial<FormatConfig> = {},
|
||||
): void {
|
||||
const cfg = { ...DEFAULT_CONFIG, ...config };
|
||||
|
||||
// 1. Inject newlines globally (affects li, p, pre, emu-production, emu-rhs, etc.)
|
||||
addNewlinesAfterBlocks($);
|
||||
|
||||
// 2. Inline leaves (links first — must be before lists/code destroy DOM)
|
||||
convertLinksToMarkdown($, cfg.links.join(", "));
|
||||
|
||||
// 3. Inline code
|
||||
convertInlineCodeToMarkdown($, cfg.codeBlocks.inline);
|
||||
|
||||
// 4. Block leaves (fenced code)
|
||||
convertBlockCodeToMarkdown($, cfg.codeBlocks.block);
|
||||
|
||||
// 5. Structural parents (grammar, lists, tables)
|
||||
convertGrammarToMarkdown($, cfg.codeBlocks.grammar.join(", "));
|
||||
convertListsToMarkdown($, cfg.lists.ordered, cfg.lists.unordered);
|
||||
convertTablesToMarkdown($);
|
||||
}
|
||||
Reference in New Issue
Block a user