mirror of
https://github.com/bendtherules/ask262.git
synced 2026-08-18 21:31:46 +00:00
fix - Fix normalizeWhitespace removing indents in nested lists
This commit is contained in:
@@ -12,136 +12,377 @@ import {
|
||||
} from "./formatHTMLForIngestion";
|
||||
|
||||
describe("formatHTMLForIngestion", () => {
|
||||
test("convertLinksToMarkdown converts emu-xref links to markdown and strips filenames", () => {
|
||||
const html = `<emu-xref href="abstract-operations.html#sec-tonumber"><a href="abstract-operations.html#sec-tonumber">ToNumber</a></emu-xref>`;
|
||||
const $ = cheerio.load(html);
|
||||
convertLinksToMarkdown($, DEFAULT_CONFIG.links.join(", "));
|
||||
expect($.html()).toContain(
|
||||
'<span class="link-markdown">[ToNumber](#sec-tonumber)</span>',
|
||||
);
|
||||
describe("convertLinksToMarkdown", () => {
|
||||
test("converts emu-xref links to markdown and strips filenames", () => {
|
||||
const html = `<emu-xref href="abstract-operations.html#sec-tonumber"><a href="abstract-operations.html#sec-tonumber">ToNumber</a></emu-xref>`;
|
||||
const $ = cheerio.load(html);
|
||||
convertLinksToMarkdown($, DEFAULT_CONFIG.links.join(", "));
|
||||
expect($.html()).toContain(
|
||||
'<span class="link-markdown">[ToNumber](#sec-tonumber)</span>',
|
||||
);
|
||||
});
|
||||
|
||||
test("skips external links", () => {
|
||||
const html = `<emu-xref href="https://example.com"><a href="https://example.com">External</a></emu-xref>`;
|
||||
const $ = cheerio.load(html);
|
||||
convertLinksToMarkdown($, DEFAULT_CONFIG.links.join(", "));
|
||||
expect($.html()).toContain('<a href="https://example.com">External</a>');
|
||||
});
|
||||
});
|
||||
|
||||
test("convertLinksToMarkdown skips external links", () => {
|
||||
const html = `<emu-xref href="https://example.com"><a href="https://example.com">External</a></emu-xref>`;
|
||||
const $ = cheerio.load(html);
|
||||
convertLinksToMarkdown($, DEFAULT_CONFIG.links.join(", "));
|
||||
expect($.html()).toContain('<a href="https://example.com">External</a>');
|
||||
describe("convertInlineCodeToMarkdown", () => {
|
||||
test("converts var, emu-val, emu-const to inline code", () => {
|
||||
const html = `<div><var>x</var> <emu-val>y</emu-val> <emu-const>z</emu-const> <code>w</code></div>`;
|
||||
const $ = cheerio.load(html);
|
||||
convertInlineCodeToMarkdown($, DEFAULT_CONFIG.codeBlocks.inline);
|
||||
expect($.html()).toContain('<span class="inline-code">`x`</span>');
|
||||
expect($.html()).toContain('<span class="inline-code">`y`</span>');
|
||||
expect($.html()).toContain('<span class="inline-code">`z`</span>');
|
||||
expect($.html()).toContain('<span class="inline-code">`w`</span>');
|
||||
});
|
||||
|
||||
test("skips code inside pre", () => {
|
||||
const html = `<pre><code>x</code></pre>`;
|
||||
const $ = cheerio.load(html);
|
||||
convertInlineCodeToMarkdown($, DEFAULT_CONFIG.codeBlocks.inline);
|
||||
expect($.html()).toContain("<pre><code>x</code></pre>");
|
||||
});
|
||||
});
|
||||
|
||||
test("convertInlineCodeToMarkdown converts var, emu-val, emu-const to inline code", () => {
|
||||
const html = `<div><var>x</var> <emu-val>y</emu-val> <emu-const>z</emu-const> <code>w</code></div>`;
|
||||
const $ = cheerio.load(html);
|
||||
convertInlineCodeToMarkdown($, DEFAULT_CONFIG.codeBlocks.inline);
|
||||
expect($.html()).toContain('<span class="inline-code">`x`</span>');
|
||||
expect($.html()).toContain('<span class="inline-code">`y`</span>');
|
||||
expect($.html()).toContain('<span class="inline-code">`z`</span>');
|
||||
expect($.html()).toContain('<span class="inline-code">`w`</span>');
|
||||
describe("convertBlockCodeToMarkdown", () => {
|
||||
test("converts pre>code and emu-eqn", () => {
|
||||
const html = `<div>
|
||||
<pre><code class="javascript hljs">const x = 1;</code></pre>
|
||||
<emu-eqn>y = x + 1</emu-eqn>
|
||||
<emu-eqn class="inline">z = 2</emu-eqn>
|
||||
</div>`;
|
||||
const $ = cheerio.load(html);
|
||||
convertBlockCodeToMarkdown($, DEFAULT_CONFIG.codeBlocks.block);
|
||||
expect($.html()).toContain(
|
||||
'<pre class="code-markdown">```javascript\nconst x = 1;\n```</pre>',
|
||||
);
|
||||
expect($.html()).toContain(
|
||||
'<pre class="code-markdown">```\ny = x + 1\n```</pre>',
|
||||
);
|
||||
expect($.html()).toContain('<emu-eqn class="inline">z = 2</emu-eqn>');
|
||||
});
|
||||
});
|
||||
|
||||
test("convertInlineCodeToMarkdown skips code inside pre", () => {
|
||||
const html = `<pre><code>x</code></pre>`;
|
||||
const $ = cheerio.load(html);
|
||||
convertInlineCodeToMarkdown($, DEFAULT_CONFIG.codeBlocks.inline);
|
||||
expect($.html()).toContain("<pre><code>x</code></pre>"); // unchanged
|
||||
describe("convertGrammarToMarkdown", () => {
|
||||
test("converts emu-grammar to fenced bnf", () => {
|
||||
const html = `<emu-grammar>Statement :: BlockStatement</emu-grammar>`;
|
||||
const $ = cheerio.load(html);
|
||||
convertGrammarToMarkdown($, DEFAULT_CONFIG.codeBlocks.grammar.join(", "));
|
||||
expect($.html()).toContain(
|
||||
'<pre class="code-markdown">```bnf\nStatement :: BlockStatement\n```</pre>',
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
test("convertBlockCodeToMarkdown converts pre>code and emu-eqn", () => {
|
||||
const html = `<div>
|
||||
<pre><code class="javascript hljs">const x = 1;</code></pre>
|
||||
<emu-eqn>y = x + 1</emu-eqn>
|
||||
<emu-eqn class="inline">z = 2</emu-eqn>
|
||||
</div>`;
|
||||
const $ = cheerio.load(html);
|
||||
convertBlockCodeToMarkdown($, DEFAULT_CONFIG.codeBlocks.block);
|
||||
expect($.html()).toContain(
|
||||
'<pre class="code-markdown">```javascript\nconst x = 1;\n```</pre>',
|
||||
);
|
||||
expect($.html()).toContain(
|
||||
'<pre class="code-markdown">```\ny = x + 1\n```</pre>',
|
||||
);
|
||||
expect($.html()).toContain('<emu-eqn class="inline">z = 2</emu-eqn>'); // unchanged
|
||||
});
|
||||
|
||||
test("convertGrammarToMarkdown converts emu-grammar to fenced bnf", () => {
|
||||
const html = `<emu-grammar>Statement :: BlockStatement</emu-grammar>`;
|
||||
const $ = cheerio.load(html);
|
||||
convertGrammarToMarkdown($, DEFAULT_CONFIG.codeBlocks.grammar.join(", "));
|
||||
expect($.html()).toContain(
|
||||
'<pre class="code-markdown">```bnf\nStatement :: BlockStatement\n```</pre>',
|
||||
);
|
||||
});
|
||||
|
||||
test("convertListsToMarkdown converts ol and ul with nesting to markdown lists", () => {
|
||||
const html = `
|
||||
<ul>
|
||||
<li>Item 1</li>
|
||||
<li>Item 2
|
||||
<ol>
|
||||
<li>Subitem 1</li>
|
||||
<li>Subitem 2</li>
|
||||
</ol>
|
||||
</li>
|
||||
</ul>
|
||||
`;
|
||||
const $ = cheerio.load(html);
|
||||
convertListsToMarkdown(
|
||||
$,
|
||||
DEFAULT_CONFIG.lists.ordered,
|
||||
DEFAULT_CONFIG.lists.unordered,
|
||||
);
|
||||
const result = $("pre.list-markdown").text();
|
||||
expect(result).toContain("- Item 1");
|
||||
expect(result).toContain("- Item 2");
|
||||
expect(result).toContain(" 1. Subitem 1");
|
||||
expect(result).toContain(" 2. Subitem 2");
|
||||
});
|
||||
|
||||
test("convertTablesToMarkdown converts tables to markdown tables", () => {
|
||||
const html = `
|
||||
<table>
|
||||
<thead>
|
||||
<tr><th>Col 1</th><th>Col 2</th></tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<tr><td>Data 1</td><td>Data 2</td></tr>
|
||||
</tbody>
|
||||
</table>
|
||||
`;
|
||||
const $ = cheerio.load(html);
|
||||
convertTablesToMarkdown($);
|
||||
const result = $("pre.table-markdown").text();
|
||||
expect(result).toContain("| Col 1 | Col 2 |");
|
||||
expect(result).toContain("| --- | --- |");
|
||||
expect(result).toContain("| Data 1 | Data 2 |");
|
||||
});
|
||||
|
||||
test("formatForIngestion runs the full pipeline", () => {
|
||||
const html = `
|
||||
<div>
|
||||
<p>See <emu-xref href="#sec-example"><a href="#sec-example">Example</a></emu-xref> for <var>x</var></p>
|
||||
<pre><code class="javascript">let y = x;</code></pre>
|
||||
describe("convertListsToMarkdown", () => {
|
||||
test("converts ul with nested ol", () => {
|
||||
const html = `
|
||||
<ul>
|
||||
<li>One</li>
|
||||
<li>Two</li>
|
||||
<li>Item 1</li>
|
||||
<li>Item 2
|
||||
<ol>
|
||||
<li>Subitem 1</li>
|
||||
<li>Subitem 2</li>
|
||||
</ol>
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
`;
|
||||
const $ = cheerio.load(html);
|
||||
formatForIngestion($);
|
||||
`;
|
||||
const $ = cheerio.load(html);
|
||||
convertListsToMarkdown(
|
||||
$,
|
||||
DEFAULT_CONFIG.lists.ordered,
|
||||
DEFAULT_CONFIG.lists.unordered,
|
||||
);
|
||||
const lines = $("pre.list-markdown")
|
||||
.text()
|
||||
.split("\n")
|
||||
.filter((l) => l.length > 0);
|
||||
expect(lines).toEqual([
|
||||
"- Item 1",
|
||||
"- Item 2",
|
||||
" 1. Subitem 1",
|
||||
" 2. Subitem 2",
|
||||
]);
|
||||
});
|
||||
|
||||
// Check links
|
||||
expect($.html()).toContain(
|
||||
'<span class="link-markdown">[Example](#sec-example)</span>',
|
||||
);
|
||||
// Check inline code
|
||||
expect($.html()).toContain('<span class="inline-code">`x`</span>');
|
||||
// Check block code
|
||||
expect($.html()).toContain(
|
||||
'<pre class="code-markdown">```javascript\nlet y = x;\n```</pre>',
|
||||
);
|
||||
// Check lists
|
||||
expect($.html()).toContain(
|
||||
'<pre class="list-markdown">- One\n- Two\n</pre>',
|
||||
);
|
||||
test("handles 3-level deep nesting (ol > ul > ol)", () => {
|
||||
const html = `
|
||||
<ol>
|
||||
<li>First</li>
|
||||
<li>Second
|
||||
<ul>
|
||||
<li>Alpha
|
||||
<ol>
|
||||
<li>Deep 1</li>
|
||||
<li>Deep 2</li>
|
||||
</ol>
|
||||
</li>
|
||||
<li>Beta</li>
|
||||
</ul>
|
||||
</li>
|
||||
</ol>
|
||||
`;
|
||||
const $ = cheerio.load(html);
|
||||
convertListsToMarkdown(
|
||||
$,
|
||||
DEFAULT_CONFIG.lists.ordered,
|
||||
DEFAULT_CONFIG.lists.unordered,
|
||||
);
|
||||
const lines = $("pre.list-markdown")
|
||||
.text()
|
||||
.split("\n")
|
||||
.filter((l) => l.length > 0);
|
||||
expect(lines).toEqual([
|
||||
"1. First",
|
||||
"2. Second",
|
||||
" - Alpha",
|
||||
" 1. Deep 1",
|
||||
" 2. Deep 2",
|
||||
" - Beta",
|
||||
]);
|
||||
});
|
||||
|
||||
test("handles multiple sibling nested lists in one item", () => {
|
||||
const html = `
|
||||
<ul>
|
||||
<li>Item
|
||||
<ol>
|
||||
<li>Ordered sub</li>
|
||||
</ol>
|
||||
<ul>
|
||||
<li>Unordered sub</li>
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
`;
|
||||
const $ = cheerio.load(html);
|
||||
convertListsToMarkdown(
|
||||
$,
|
||||
DEFAULT_CONFIG.lists.ordered,
|
||||
DEFAULT_CONFIG.lists.unordered,
|
||||
);
|
||||
const lines = $("pre.list-markdown")
|
||||
.text()
|
||||
.split("\n")
|
||||
.filter((l) => l.length > 0);
|
||||
expect(lines).toEqual([
|
||||
"- Item",
|
||||
" 1. Ordered sub",
|
||||
" - Unordered sub",
|
||||
]);
|
||||
});
|
||||
|
||||
test("handles consecutive top-level lists", () => {
|
||||
const html = `
|
||||
<ul>
|
||||
<li>UL item 1</li>
|
||||
<li>UL item 2</li>
|
||||
</ul>
|
||||
<ol>
|
||||
<li>OL item 1</li>
|
||||
<li>OL item 2</li>
|
||||
</ol>
|
||||
`;
|
||||
const $ = cheerio.load(html);
|
||||
convertListsToMarkdown(
|
||||
$,
|
||||
DEFAULT_CONFIG.lists.ordered,
|
||||
DEFAULT_CONFIG.lists.unordered,
|
||||
);
|
||||
const results = $("pre.list-markdown");
|
||||
expect(results.length).toBe(2);
|
||||
|
||||
const ulLines = results
|
||||
.eq(0)
|
||||
.text()
|
||||
.split("\n")
|
||||
.filter((l) => l.length > 0);
|
||||
expect(ulLines).toEqual(["- UL item 1", "- UL item 2"]);
|
||||
|
||||
const olLines = results
|
||||
.eq(1)
|
||||
.text()
|
||||
.split("\n")
|
||||
.filter((l) => l.length > 0);
|
||||
expect(olLines).toEqual(["1. OL item 1", "2. OL item 2"]);
|
||||
});
|
||||
|
||||
test("preserves inline code inside list items", () => {
|
||||
const html = `
|
||||
<ul>
|
||||
<li>Call <code>foo()</code></li>
|
||||
<li>Use <var>x</var></li>
|
||||
</ul>
|
||||
`;
|
||||
const $ = cheerio.load(html);
|
||||
convertInlineCodeToMarkdown($, DEFAULT_CONFIG.codeBlocks.inline);
|
||||
convertListsToMarkdown(
|
||||
$,
|
||||
DEFAULT_CONFIG.lists.ordered,
|
||||
DEFAULT_CONFIG.lists.unordered,
|
||||
);
|
||||
const lines = $("pre.list-markdown")
|
||||
.text()
|
||||
.split("\n")
|
||||
.filter((l) => l.length > 0);
|
||||
expect(lines).toEqual(["- Call `foo()`", "- Use `x`"]);
|
||||
});
|
||||
|
||||
test("handles ol nested inside ol", () => {
|
||||
const html = `
|
||||
<ol>
|
||||
<li>First
|
||||
<ol>
|
||||
<li>Nested 1</li>
|
||||
<li>Nested 2</li>
|
||||
</ol>
|
||||
</li>
|
||||
<li>Second</li>
|
||||
</ol>
|
||||
`;
|
||||
const $ = cheerio.load(html);
|
||||
convertListsToMarkdown(
|
||||
$,
|
||||
DEFAULT_CONFIG.lists.ordered,
|
||||
DEFAULT_CONFIG.lists.unordered,
|
||||
);
|
||||
const lines = $("pre.list-markdown")
|
||||
.text()
|
||||
.split("\n")
|
||||
.filter((l) => l.length > 0);
|
||||
expect(lines).toEqual([
|
||||
"1. First",
|
||||
" 1. Nested 1",
|
||||
" 2. Nested 2",
|
||||
"2. Second",
|
||||
]);
|
||||
});
|
||||
|
||||
test("handles empty list", () => {
|
||||
const html = `<ul></ul>`;
|
||||
const $ = cheerio.load(html);
|
||||
convertListsToMarkdown(
|
||||
$,
|
||||
DEFAULT_CONFIG.lists.ordered,
|
||||
DEFAULT_CONFIG.lists.unordered,
|
||||
);
|
||||
const result = $("pre.list-markdown").text().trim();
|
||||
expect(result).toBe("");
|
||||
});
|
||||
});
|
||||
|
||||
describe("convertTablesToMarkdown", () => {
|
||||
test("converts tables to markdown tables", () => {
|
||||
const html = `
|
||||
<table>
|
||||
<thead>
|
||||
<tr><th>Col 1</th><th>Col 2</th></tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<tr><td>Data 1</td><td>Data 2</td></tr>
|
||||
</tbody>
|
||||
</table>
|
||||
`;
|
||||
const $ = cheerio.load(html);
|
||||
convertTablesToMarkdown($);
|
||||
const result = $("pre.table-markdown").text();
|
||||
expect(result).toContain("| Col 1 | Col 2 |");
|
||||
expect(result).toContain("| --- | --- |");
|
||||
expect(result).toContain("| Data 1 | Data 2 |");
|
||||
});
|
||||
});
|
||||
|
||||
describe("formatForIngestion", () => {
|
||||
test("runs the full pipeline", () => {
|
||||
const html = `
|
||||
<div>
|
||||
<p>See <emu-xref href="#sec-example"><a href="#sec-example">Example</a></emu-xref> for <var>x</var></p>
|
||||
<pre><code class="javascript">let y = x;</code></pre>
|
||||
<ul>
|
||||
<li>One</li>
|
||||
<li>Two</li>
|
||||
</ul>
|
||||
</div>
|
||||
`;
|
||||
const $ = cheerio.load(html);
|
||||
formatForIngestion($);
|
||||
|
||||
expect($.html()).toContain(
|
||||
'<span class="link-markdown">[Example](#sec-example)</span>',
|
||||
);
|
||||
expect($.html()).toContain('<span class="inline-code">`x`</span>');
|
||||
expect($.html()).toContain(
|
||||
'<pre class="code-markdown">```javascript\nlet y = x;\n```</pre>',
|
||||
);
|
||||
expect($.html()).toContain(
|
||||
'<pre class="list-markdown">- One\n- Two\n</pre>',
|
||||
);
|
||||
});
|
||||
|
||||
test("preserves nested list indentation through the full pipeline", () => {
|
||||
const html = `
|
||||
<ol>
|
||||
<li>First</li>
|
||||
<li>Second
|
||||
<ul>
|
||||
<li>Alpha
|
||||
<ol>
|
||||
<li>Deep 1</li>
|
||||
<li>Deep 2</li>
|
||||
</ol>
|
||||
</li>
|
||||
<li>Beta</li>
|
||||
</ul>
|
||||
</li>
|
||||
</ol>
|
||||
`;
|
||||
const $ = cheerio.load(html);
|
||||
formatForIngestion($);
|
||||
|
||||
const lines = $("pre.list-markdown")
|
||||
.text()
|
||||
.split("\n")
|
||||
.filter((l) => l.length > 0);
|
||||
expect(lines).toEqual([
|
||||
"1. First",
|
||||
"2. Second",
|
||||
" - Alpha",
|
||||
" 1. Deep 1",
|
||||
" 2. Deep 2",
|
||||
" - Beta",
|
||||
]);
|
||||
});
|
||||
|
||||
test("preserves ol nested inside ol through the full pipeline", () => {
|
||||
const html = `
|
||||
<ol>
|
||||
<li>First
|
||||
<ol>
|
||||
<li>Nested 1</li>
|
||||
<li>Nested 2</li>
|
||||
</ol>
|
||||
</li>
|
||||
<li>Second</li>
|
||||
</ol>
|
||||
`;
|
||||
const $ = cheerio.load(html);
|
||||
formatForIngestion($);
|
||||
|
||||
const lines = $("pre.list-markdown")
|
||||
.text()
|
||||
.split("\n")
|
||||
.filter((l) => l.length > 0);
|
||||
expect(lines).toEqual([
|
||||
"1. First",
|
||||
" 1. Nested 1",
|
||||
" 2. Nested 2",
|
||||
"2. Second",
|
||||
]);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user