diff --git a/setup/textsplitters/HtmlTextSplitter.test.ts b/setup/textsplitters/HtmlTextSplitter.test.ts index bdd3ef1..c737d9c 100644 --- a/setup/textsplitters/HtmlTextSplitter.test.ts +++ b/setup/textsplitters/HtmlTextSplitter.test.ts @@ -50,16 +50,16 @@ describe("HTMLTextSplitter", () => { expect(chunks).toEqual(["Hello\nWorld"]); }); - test("normalizes repeated whitespace in emitted text", async () => { + test("preserves whitespace as-is from HTML text content", async () => { const splitter = new HTMLTextSplitter({ chunkSize: 64, }); const chunks = await splitter.splitText( - "
Hello\n\n World
\tAgain
", + "
Hello\n\n World
Again
", ); - expect(chunks).toEqual(["Hello\n\nWorld\nAgain"]); + expect(chunks).toEqual(["Hello\n\n World \nAgain"]); }); test("treats separators as soft hints until size pressure exists", async () => { @@ -325,4 +325,25 @@ describe("HTMLTextSplitter", () => { expect(chunks).toEqual(["alphabetgamma"]); }); + + test("preserves nested list indentation from formatForIngestion output", async () => { + const splitter = new HTMLTextSplitter({ + chunkSize: 200, + }); + + const html = [ + '
',
+      "1. First",
+      "  1. Nested 1",
+      "  2. Nested 2",
+      "2. Second",
+      "
", + ].join("\n"); + + const chunks = await splitter.splitText(html); + + expect(chunks).toEqual([ + "1. First\n 1. Nested 1\n 2. Nested 2\n2. Second", + ]); + }); }); diff --git a/setup/textsplitters/HtmlTextSplitter.ts b/setup/textsplitters/HtmlTextSplitter.ts index 854d0f1..37ef7e4 100644 --- a/setup/textsplitters/HtmlTextSplitter.ts +++ b/setup/textsplitters/HtmlTextSplitter.ts @@ -111,7 +111,7 @@ export class HTMLTextSplitter extends TextSplitter { ...fields, chunkOverlap: 0, keepSeparator: false, - lengthFunction: (text: string) => this.normalizeWhitespace(text).length, + lengthFunction: (text: string) => text.trim().length, }); this.separators = fields?.separators ?? []; @@ -229,22 +229,6 @@ export class HTMLTextSplitter extends TextSplitter { return node.type === "tag" && BLOCKISH_TAGS.has(node.name); } - /** - * Collapses internal whitespace and trims leading/trailing whitespace. - * Preserves newlines between block elements while normalizing spaces. - * - * Needed so text extraction can preserve raw adjacency first and normalize - * only once after boundary-aware joining. - */ - private normalizeWhitespace(text: string): string { - return text - .replace(/\n[ \t]+/g, "\n") // Remove leading spaces after newlines - .replace(/[ \t]+\n/g, "\n") // Remove trailing spaces before newlines - .replace(/\n{3,}/g, "\n\n") // Collapse 3+ newlines to 2 - .replace(/[ \t]{2,}/g, " ") // Collapse multiple spaces/tabs to one - .trim(); - } - /** * Joins multiple nodes into the normalized text the splitter actually emits. * @@ -273,7 +257,7 @@ export class HTMLTextSplitter extends TextSplitter { previousNode = node; } - return this.normalizeWhitespace(result); + return result.trim(); } /** diff --git a/setup/utils/formatHTMLForIngestion.test.ts b/setup/utils/formatHTMLForIngestion.test.ts index 7e3442b..870f24a 100644 --- a/setup/utils/formatHTMLForIngestion.test.ts +++ b/setup/utils/formatHTMLForIngestion.test.ts @@ -12,136 +12,377 @@ import { } from "./formatHTMLForIngestion"; describe("formatHTMLForIngestion", () => { - test("convertLinksToMarkdown converts emu-xref links to markdown and strips filenames", () => { - const html = `ToNumber`; - const $ = cheerio.load(html); - convertLinksToMarkdown($, DEFAULT_CONFIG.links.join(", ")); - expect($.html()).toContain( - '[ToNumber](#sec-tonumber)', - ); + describe("convertLinksToMarkdown", () => { + test("converts emu-xref links to markdown and strips filenames", () => { + const html = `ToNumber`; + const $ = cheerio.load(html); + convertLinksToMarkdown($, DEFAULT_CONFIG.links.join(", ")); + expect($.html()).toContain( + '[ToNumber](#sec-tonumber)', + ); + }); + + test("skips external links", () => { + const html = `External`; + const $ = cheerio.load(html); + convertLinksToMarkdown($, DEFAULT_CONFIG.links.join(", ")); + expect($.html()).toContain('External'); + }); }); - test("convertLinksToMarkdown skips external links", () => { - const html = `External`; - const $ = cheerio.load(html); - convertLinksToMarkdown($, DEFAULT_CONFIG.links.join(", ")); - expect($.html()).toContain('External'); + describe("convertInlineCodeToMarkdown", () => { + test("converts var, emu-val, emu-const to inline code", () => { + const html = `
x y z w
`; + const $ = cheerio.load(html); + convertInlineCodeToMarkdown($, DEFAULT_CONFIG.codeBlocks.inline); + expect($.html()).toContain('`x`'); + expect($.html()).toContain('`y`'); + expect($.html()).toContain('`z`'); + expect($.html()).toContain('`w`'); + }); + + test("skips code inside pre", () => { + const html = `
x
`; + const $ = cheerio.load(html); + convertInlineCodeToMarkdown($, DEFAULT_CONFIG.codeBlocks.inline); + expect($.html()).toContain("
x
"); + }); }); - test("convertInlineCodeToMarkdown converts var, emu-val, emu-const to inline code", () => { - const html = `
x y z w
`; - const $ = cheerio.load(html); - convertInlineCodeToMarkdown($, DEFAULT_CONFIG.codeBlocks.inline); - expect($.html()).toContain('`x`'); - expect($.html()).toContain('`y`'); - expect($.html()).toContain('`z`'); - expect($.html()).toContain('`w`'); + describe("convertBlockCodeToMarkdown", () => { + test("converts pre>code and emu-eqn", () => { + const html = `
+ 
const x = 1;
+ y = x + 1 + z = 2 +
`; + const $ = cheerio.load(html); + convertBlockCodeToMarkdown($, DEFAULT_CONFIG.codeBlocks.block); + expect($.html()).toContain( + '
```javascript\nconst x = 1;\n```
', + ); + expect($.html()).toContain( + '
```\ny = x + 1\n```
', + ); + expect($.html()).toContain('z = 2'); + }); }); - test("convertInlineCodeToMarkdown skips code inside pre", () => { - const html = `
x
`; - const $ = cheerio.load(html); - convertInlineCodeToMarkdown($, DEFAULT_CONFIG.codeBlocks.inline); - expect($.html()).toContain("
x
"); // unchanged + describe("convertGrammarToMarkdown", () => { + test("converts emu-grammar to fenced bnf", () => { + const html = `Statement :: BlockStatement`; + const $ = cheerio.load(html); + convertGrammarToMarkdown($, DEFAULT_CONFIG.codeBlocks.grammar.join(", ")); + expect($.html()).toContain( + '
```bnf\nStatement :: BlockStatement\n```
', + ); + }); }); - test("convertBlockCodeToMarkdown converts pre>code and emu-eqn", () => { - const html = `
- 
const x = 1;
- y = x + 1 - z = 2 -
`; - const $ = cheerio.load(html); - convertBlockCodeToMarkdown($, DEFAULT_CONFIG.codeBlocks.block); - expect($.html()).toContain( - '
```javascript\nconst x = 1;\n```
', - ); - expect($.html()).toContain( - '
```\ny = x + 1\n```
', - ); - expect($.html()).toContain('z = 2'); // unchanged - }); - - test("convertGrammarToMarkdown converts emu-grammar to fenced bnf", () => { - const html = `Statement :: BlockStatement`; - const $ = cheerio.load(html); - convertGrammarToMarkdown($, DEFAULT_CONFIG.codeBlocks.grammar.join(", ")); - expect($.html()).toContain( - '
```bnf\nStatement :: BlockStatement\n```
', - ); - }); - - test("convertListsToMarkdown converts ol and ul with nesting to markdown lists", () => { - const html = ` - - `; - const $ = cheerio.load(html); - convertListsToMarkdown( - $, - DEFAULT_CONFIG.lists.ordered, - DEFAULT_CONFIG.lists.unordered, - ); - const result = $("pre.list-markdown").text(); - expect(result).toContain("- Item 1"); - expect(result).toContain("- Item 2"); - expect(result).toContain(" 1. Subitem 1"); - expect(result).toContain(" 2. Subitem 2"); - }); - - test("convertTablesToMarkdown converts tables to markdown tables", () => { - const html = ` - - - - - - - -
Col 1Col 2
Data 1Data 2
- `; - const $ = cheerio.load(html); - convertTablesToMarkdown($); - const result = $("pre.table-markdown").text(); - expect(result).toContain("| Col 1 | Col 2 |"); - expect(result).toContain("| --- | --- |"); - expect(result).toContain("| Data 1 | Data 2 |"); - }); - - test("formatForIngestion runs the full pipeline", () => { - const html = ` -
-

See Example for x

- 
let y = x;
+ describe("convertListsToMarkdown", () => { + test("converts ul with nested ol", () => { + const html = ` -
- `; - const $ = cheerio.load(html); - formatForIngestion($); + `; + const $ = cheerio.load(html); + convertListsToMarkdown( + $, + DEFAULT_CONFIG.lists.ordered, + DEFAULT_CONFIG.lists.unordered, + ); + const lines = $("pre.list-markdown") + .text() + .split("\n") + .filter((l) => l.length > 0); + expect(lines).toEqual([ + "- Item 1", + "- Item 2", + " 1. Subitem 1", + " 2. Subitem 2", + ]); + }); - // Check links - expect($.html()).toContain( - '[Example](#sec-example)', - ); - // Check inline code - expect($.html()).toContain('`x`'); - // Check block code - expect($.html()).toContain( - '
```javascript\nlet y = x;\n```
', - ); - // Check lists - expect($.html()).toContain( - '
- One\n- Two\n
', - ); + test("handles 3-level deep nesting (ol > ul > ol)", () => { + const html = ` +
    +
  1. First
    2. +
  2. Second + +
    3. +
+ `; + const $ = cheerio.load(html); + convertListsToMarkdown( + $, + DEFAULT_CONFIG.lists.ordered, + DEFAULT_CONFIG.lists.unordered, + ); + const lines = $("pre.list-markdown") + .text() + .split("\n") + .filter((l) => l.length > 0); + expect(lines).toEqual([ + "1. First", + "2. Second", + " - Alpha", + " 1. Deep 1", + " 2. Deep 2", + " - Beta", + ]); + }); + + test("handles multiple sibling nested lists in one item", () => { + const html = ` + + `; + const $ = cheerio.load(html); + convertListsToMarkdown( + $, + DEFAULT_CONFIG.lists.ordered, + DEFAULT_CONFIG.lists.unordered, + ); + const lines = $("pre.list-markdown") + .text() + .split("\n") + .filter((l) => l.length > 0); + expect(lines).toEqual([ + "- Item", + " 1. Ordered sub", + " - Unordered sub", + ]); + }); + + test("handles consecutive top-level lists", () => { + const html = ` + +
    +
  1. OL item 1
    2. +
  2. OL item 2
    3. +
+ `; + const $ = cheerio.load(html); + convertListsToMarkdown( + $, + DEFAULT_CONFIG.lists.ordered, + DEFAULT_CONFIG.lists.unordered, + ); + const results = $("pre.list-markdown"); + expect(results.length).toBe(2); + + const ulLines = results + .eq(0) + .text() + .split("\n") + .filter((l) => l.length > 0); + expect(ulLines).toEqual(["- UL item 1", "- UL item 2"]); + + const olLines = results + .eq(1) + .text() + .split("\n") + .filter((l) => l.length > 0); + expect(olLines).toEqual(["1. OL item 1", "2. OL item 2"]); + }); + + test("preserves inline code inside list items", () => { + const html = ` + + `; + const $ = cheerio.load(html); + convertInlineCodeToMarkdown($, DEFAULT_CONFIG.codeBlocks.inline); + convertListsToMarkdown( + $, + DEFAULT_CONFIG.lists.ordered, + DEFAULT_CONFIG.lists.unordered, + ); + const lines = $("pre.list-markdown") + .text() + .split("\n") + .filter((l) => l.length > 0); + expect(lines).toEqual(["- Call `foo()`", "- Use `x`"]); + }); + + test("handles ol nested inside ol", () => { + const html = ` +
    +
  1. First +
      +
    1. Nested 1
      2. +
    2. Nested 2
      3. +
    +
    2. +
  2. Second
    3. +
+ `; + const $ = cheerio.load(html); + convertListsToMarkdown( + $, + DEFAULT_CONFIG.lists.ordered, + DEFAULT_CONFIG.lists.unordered, + ); + const lines = $("pre.list-markdown") + .text() + .split("\n") + .filter((l) => l.length > 0); + expect(lines).toEqual([ + "1. First", + " 1. Nested 1", + " 2. Nested 2", + "2. Second", + ]); + }); + + test("handles empty list", () => { + const html = ``; + const $ = cheerio.load(html); + convertListsToMarkdown( + $, + DEFAULT_CONFIG.lists.ordered, + DEFAULT_CONFIG.lists.unordered, + ); + const result = $("pre.list-markdown").text().trim(); + expect(result).toBe(""); + }); + }); + + describe("convertTablesToMarkdown", () => { + test("converts tables to markdown tables", () => { + const html = ` + + + + + + + +
Col 1Col 2
Data 1Data 2
+ `; + const $ = cheerio.load(html); + convertTablesToMarkdown($); + const result = $("pre.table-markdown").text(); + expect(result).toContain("| Col 1 | Col 2 |"); + expect(result).toContain("| --- | --- |"); + expect(result).toContain("| Data 1 | Data 2 |"); + }); + }); + + describe("formatForIngestion", () => { + test("runs the full pipeline", () => { + const html = ` +
+

See Example for x

+ 
let y = x;
+ +
+ `; + const $ = cheerio.load(html); + formatForIngestion($); + + expect($.html()).toContain( + '[Example](#sec-example)', + ); + expect($.html()).toContain('`x`'); + expect($.html()).toContain( + '
```javascript\nlet y = x;\n```
', + ); + expect($.html()).toContain( + '
- One\n- Two\n
', + ); + }); + + test("preserves nested list indentation through the full pipeline", () => { + const html = ` +
    +
  1. First
    2. +
  2. Second + +
    3. +
+ `; + const $ = cheerio.load(html); + formatForIngestion($); + + const lines = $("pre.list-markdown") + .text() + .split("\n") + .filter((l) => l.length > 0); + expect(lines).toEqual([ + "1. First", + "2. Second", + " - Alpha", + " 1. Deep 1", + " 2. Deep 2", + " - Beta", + ]); + }); + + test("preserves ol nested inside ol through the full pipeline", () => { + const html = ` +
    +
  1. First +
      +
    1. Nested 1
      2. +
    2. Nested 2
      3. +
    +
    2. +
  2. Second
    3. +
+ `; + const $ = cheerio.load(html); + formatForIngestion($); + + const lines = $("pre.list-markdown") + .text() + .split("\n") + .filter((l) => l.length > 0); + expect(lines).toEqual([ + "1. First", + " 1. Nested 1", + " 2. Nested 2", + "2. Second", + ]); + }); }); });