import { describe, expect, test } from "bun:test";
import * as cheerio from "cheerio";
import {
convertBlockCodeToMarkdown,
convertGrammarToMarkdown,
convertInlineCodeToMarkdown,
convertLinksToMarkdown,
convertListsToMarkdown,
convertTablesToMarkdown,
DEFAULT_CONFIG,
formatForIngestion,
} from "./formatHTMLForIngestion";
describe("formatHTMLForIngestion", () => {
describe("convertLinksToMarkdown", () => {
test("converts emu-xref links to markdown and strips filenames", () => {
const html = `ToNumber`;
const $ = cheerio.load(html);
convertLinksToMarkdown($, DEFAULT_CONFIG.links.join(", "));
expect($.html()).toContain(
'[ToNumber](#sec-tonumber)',
);
});
test("skips external links", () => {
const html = `External`;
const $ = cheerio.load(html);
convertLinksToMarkdown($, DEFAULT_CONFIG.links.join(", "));
expect($.html()).toContain('External');
});
});
describe("convertInlineCodeToMarkdown", () => {
test("converts var, emu-val, emu-const to inline code", () => {
const html = `
x y z w
`;
const $ = cheerio.load(html);
convertInlineCodeToMarkdown($, DEFAULT_CONFIG.codeBlocks.inline);
expect($.html()).toContain('`x`');
expect($.html()).toContain('`y`');
expect($.html()).toContain('`z`');
expect($.html()).toContain('`w`');
});
test("skips code inside pre", () => {
const html = `x
`;
const $ = cheerio.load(html);
convertInlineCodeToMarkdown($, DEFAULT_CONFIG.codeBlocks.inline);
expect($.html()).toContain("x
");
});
});
describe("convertBlockCodeToMarkdown", () => {
test("converts pre>code and emu-eqn", () => {
const html = `
const x = 1;
y = x + 1
z = 2
`;
const $ = cheerio.load(html);
convertBlockCodeToMarkdown($, DEFAULT_CONFIG.codeBlocks.block);
expect($.html()).toContain(
'```javascript\nconst x = 1;\n```
',
);
expect($.html()).toContain(
'```\ny = x + 1\n```
',
);
expect($.html()).toContain('z = 2');
});
});
describe("convertGrammarToMarkdown", () => {
test("converts emu-grammar to fenced bnf", () => {
const html = `Statement :: BlockStatement`;
const $ = cheerio.load(html);
convertGrammarToMarkdown($, DEFAULT_CONFIG.codeBlocks.grammar.join(", "));
expect($.html()).toContain(
'```bnf\nStatement :: BlockStatement\n```
',
);
});
});
describe("convertListsToMarkdown", () => {
test("converts ul with nested ol", () => {
const html = `
- Item 1
- Item 2
- Subitem 1
- Subitem 2
`;
const $ = cheerio.load(html);
convertListsToMarkdown(
$,
DEFAULT_CONFIG.lists.ordered,
DEFAULT_CONFIG.lists.unordered,
);
const lines = $("pre.list-markdown")
.text()
.split("\n")
.filter((l) => l.length > 0);
expect(lines).toEqual([
"- Item 1",
"- Item 2",
" 1. Subitem 1",
" 2. Subitem 2",
]);
});
test("handles 3-level deep nesting (ol > ul > ol)", () => {
const html = `
- First
- Second
`;
const $ = cheerio.load(html);
convertListsToMarkdown(
$,
DEFAULT_CONFIG.lists.ordered,
DEFAULT_CONFIG.lists.unordered,
);
const lines = $("pre.list-markdown")
.text()
.split("\n")
.filter((l) => l.length > 0);
expect(lines).toEqual([
"1. First",
"2. Second",
" - Alpha",
" 1. Deep 1",
" 2. Deep 2",
" - Beta",
]);
});
test("handles multiple sibling nested lists in one item", () => {
const html = `
`;
const $ = cheerio.load(html);
convertListsToMarkdown(
$,
DEFAULT_CONFIG.lists.ordered,
DEFAULT_CONFIG.lists.unordered,
);
const lines = $("pre.list-markdown")
.text()
.split("\n")
.filter((l) => l.length > 0);
expect(lines).toEqual([
"- Item",
" 1. Ordered sub",
" - Unordered sub",
]);
});
test("handles consecutive top-level lists", () => {
const html = `
- OL item 1
- OL item 2
`;
const $ = cheerio.load(html);
convertListsToMarkdown(
$,
DEFAULT_CONFIG.lists.ordered,
DEFAULT_CONFIG.lists.unordered,
);
const results = $("pre.list-markdown");
expect(results.length).toBe(2);
const ulLines = results
.eq(0)
.text()
.split("\n")
.filter((l) => l.length > 0);
expect(ulLines).toEqual(["- UL item 1", "- UL item 2"]);
const olLines = results
.eq(1)
.text()
.split("\n")
.filter((l) => l.length > 0);
expect(olLines).toEqual(["1. OL item 1", "2. OL item 2"]);
});
test("preserves inline code inside list items", () => {
const html = `
`;
const $ = cheerio.load(html);
convertInlineCodeToMarkdown($, DEFAULT_CONFIG.codeBlocks.inline);
convertListsToMarkdown(
$,
DEFAULT_CONFIG.lists.ordered,
DEFAULT_CONFIG.lists.unordered,
);
const lines = $("pre.list-markdown")
.text()
.split("\n")
.filter((l) => l.length > 0);
expect(lines).toEqual(["- Call `foo()`", "- Use `x`"]);
});
test("handles ol nested inside ol", () => {
const html = `
- First
- Nested 1
- Nested 2
- Second
`;
const $ = cheerio.load(html);
convertListsToMarkdown(
$,
DEFAULT_CONFIG.lists.ordered,
DEFAULT_CONFIG.lists.unordered,
);
const lines = $("pre.list-markdown")
.text()
.split("\n")
.filter((l) => l.length > 0);
expect(lines).toEqual([
"1. First",
" 1. Nested 1",
" 2. Nested 2",
"2. Second",
]);
});
test("handles empty list", () => {
const html = ``;
const $ = cheerio.load(html);
convertListsToMarkdown(
$,
DEFAULT_CONFIG.lists.ordered,
DEFAULT_CONFIG.lists.unordered,
);
const result = $("pre.list-markdown").text().trim();
expect(result).toBe("");
});
});
describe("convertTablesToMarkdown", () => {
test("converts tables to markdown tables", () => {
const html = `
`;
const $ = cheerio.load(html);
convertTablesToMarkdown($);
const result = $("pre.table-markdown").text();
expect(result).toContain("| Col 1 | Col 2 |");
expect(result).toContain("| --- | --- |");
expect(result).toContain("| Data 1 | Data 2 |");
});
});
describe("formatForIngestion", () => {
test("runs the full pipeline", () => {
const html = `
`;
const $ = cheerio.load(html);
formatForIngestion($);
expect($.html()).toContain(
'[Example](#sec-example)',
);
expect($.html()).toContain('`x`');
expect($.html()).toContain(
'```javascript\nlet y = x;\n```
',
);
expect($.html()).toContain(
'- One\n- Two\n
',
);
});
test("preserves nested list indentation through the full pipeline", () => {
const html = `
- First
- Second
`;
const $ = cheerio.load(html);
formatForIngestion($);
const lines = $("pre.list-markdown")
.text()
.split("\n")
.filter((l) => l.length > 0);
expect(lines).toEqual([
"1. First",
"2. Second",
" - Alpha",
" 1. Deep 1",
" 2. Deep 2",
" - Beta",
]);
});
test("preserves ol nested inside ol through the full pipeline", () => {
const html = `
- First
- Nested 1
- Nested 2
- Second
`;
const $ = cheerio.load(html);
formatForIngestion($);
const lines = $("pre.list-markdown")
.text()
.split("\n")
.filter((l) => l.length > 0);
expect(lines).toEqual([
"1. First",
" 1. Nested 1",
" 2. Nested 2",
"2. Second",
]);
});
});
});