fix - Fix normalizeWhitespace removing indents in nested lists

setup/textsplitters/HtmlTextSplitter.test.ts

@@ -50,13 +50,13 @@ describe("HTMLTextSplitter", () => {
     expect(chunks).toEqual(["Hello\nWorld"]);
   });
 
-  test("normalizes repeated whitespace in emitted text", async () => {
+  test("preserves whitespace as-is from HTML text content", async () => {
     const splitter = new HTMLTextSplitter({
       chunkSize: 64,
     });
 
     const chunks = await splitter.splitText(
-      "<div>  Hello\n\n   World  </div><div>\tAgain</div>",
+      "<div>  Hello\n\n   World  </div><div>Again</div>",
     );
 
     expect(chunks).toEqual(["Hello\n\n   World  \nAgain"]);
@@ -325,4 +325,25 @@ describe("HTMLTextSplitter", () => {
 
     expect(chunks).toEqual(["alphabetgamma"]);
   });
+
+  test("preserves nested list indentation from formatForIngestion output", async () => {
+    const splitter = new HTMLTextSplitter({
+      chunkSize: 200,
+    });
+
+    const html = [
+      '<pre class="list-markdown">',
+      "1. First",
+      "  1. Nested 1",
+      "  2. Nested 2",
+      "2. Second",
+      "</pre>",
+    ].join("\n");
+
+    const chunks = await splitter.splitText(html);
+
+    expect(chunks).toEqual([
+      "1. First\n  1. Nested 1\n  2. Nested 2\n2. Second",
+    ]);
+  });
 });

setup/textsplitters/HtmlTextSplitter.ts

@@ -111,7 +111,7 @@ export class HTMLTextSplitter extends TextSplitter {
       ...fields,
       chunkOverlap: 0,
       keepSeparator: false,
-      lengthFunction: (text: string) => this.normalizeWhitespace(text).length,
+      lengthFunction: (text: string) => text.trim().length,
     });
 
     this.separators = fields?.separators ?? [];
@@ -229,22 +229,6 @@ export class HTMLTextSplitter extends TextSplitter {
     return node.type === "tag" && BLOCKISH_TAGS.has(node.name);
   }
 
-  /**
-   * Collapses internal whitespace and trims leading/trailing whitespace.
-   * Preserves newlines between block elements while normalizing spaces.
-   *
-   * Needed so text extraction can preserve raw adjacency first and normalize
-   * only once after boundary-aware joining.
-   */
-  private normalizeWhitespace(text: string): string {
-    return text
-      .replace(/\n[ \t]+/g, "\n") // Remove leading spaces after newlines
-      .replace(/[ \t]+\n/g, "\n") // Remove trailing spaces before newlines
-      .replace(/\n{3,}/g, "\n\n") // Collapse 3+ newlines to 2
-      .replace(/[ \t]{2,}/g, " ") // Collapse multiple spaces/tabs to one
-      .trim();
-  }
-
   /**
    * Joins multiple nodes into the normalized text the splitter actually emits.
    *
@@ -273,7 +257,7 @@ export class HTMLTextSplitter extends TextSplitter {
       previousNode = node;
     }
 
-    return this.normalizeWhitespace(result);
+    return result.trim();
   }
 
   /**

setup/utils/formatHTMLForIngestion.test.ts

@@ -12,7 +12,8 @@ import {
 } from "./formatHTMLForIngestion";
 
 describe("formatHTMLForIngestion", () => {
-  test("convertLinksToMarkdown converts emu-xref links to markdown and strips filenames", () => {
+  describe("convertLinksToMarkdown", () => {
+    test("converts emu-xref links to markdown and strips filenames", () => {
       const html = `<emu-xref href="abstract-operations.html#sec-tonumber"><a href="abstract-operations.html#sec-tonumber">ToNumber</a></emu-xref>`;
       const $ = cheerio.load(html);
       convertLinksToMarkdown($, DEFAULT_CONFIG.links.join(", "));
@@ -21,14 +22,16 @@ describe("formatHTMLForIngestion", () => {
       );
     });
 
-  test("convertLinksToMarkdown skips external links", () => {
+    test("skips external links", () => {
       const html = `<emu-xref href="https://example.com"><a href="https://example.com">External</a></emu-xref>`;
       const $ = cheerio.load(html);
       convertLinksToMarkdown($, DEFAULT_CONFIG.links.join(", "));
       expect($.html()).toContain('<a href="https://example.com">External</a>');
     });
+  });
 
-  test("convertInlineCodeToMarkdown converts var, emu-val, emu-const to inline code", () => {
+  describe("convertInlineCodeToMarkdown", () => {
+    test("converts var, emu-val, emu-const to inline code", () => {
       const html = `<div><var>x</var> <emu-val>y</emu-val> <emu-const>z</emu-const> <code>w</code></div>`;
       const $ = cheerio.load(html);
       convertInlineCodeToMarkdown($, DEFAULT_CONFIG.codeBlocks.inline);
@@ -38,14 +41,16 @@ describe("formatHTMLForIngestion", () => {
       expect($.html()).toContain('<span class="inline-code">`w`</span>');
     });
 
-  test("convertInlineCodeToMarkdown skips code inside pre", () => {
+    test("skips code inside pre", () => {
       const html = `<pre><code>x</code></pre>`;
       const $ = cheerio.load(html);
       convertInlineCodeToMarkdown($, DEFAULT_CONFIG.codeBlocks.inline);
-    expect($.html()).toContain("<pre><code>x</code></pre>"); // unchanged
+      expect($.html()).toContain("<pre><code>x</code></pre>");
+    });
   });
 
-  test("convertBlockCodeToMarkdown converts pre>code and emu-eqn", () => {
+  describe("convertBlockCodeToMarkdown", () => {
+    test("converts pre>code and emu-eqn", () => {
       const html = `<div>
         <pre><code class="javascript hljs">const x = 1;</code></pre>
         <emu-eqn>y = x + 1</emu-eqn>
@@ -59,10 +64,12 @@ describe("formatHTMLForIngestion", () => {
       expect($.html()).toContain(
         '<pre class="code-markdown">```\ny = x + 1\n```</pre>',
       );
-    expect($.html()).toContain('<emu-eqn class="inline">z = 2</emu-eqn>'); // unchanged
+      expect($.html()).toContain('<emu-eqn class="inline">z = 2</emu-eqn>');
+    });
   });
 
-  test("convertGrammarToMarkdown converts emu-grammar to fenced bnf", () => {
+  describe("convertGrammarToMarkdown", () => {
+    test("converts emu-grammar to fenced bnf", () => {
       const html = `<emu-grammar>Statement :: BlockStatement</emu-grammar>`;
       const $ = cheerio.load(html);
       convertGrammarToMarkdown($, DEFAULT_CONFIG.codeBlocks.grammar.join(", "));
@@ -70,8 +77,10 @@ describe("formatHTMLForIngestion", () => {
         '<pre class="code-markdown">```bnf\nStatement :: BlockStatement\n```</pre>',
       );
     });
+  });
 
-  test("convertListsToMarkdown converts ol and ul with nesting to markdown lists", () => {
+  describe("convertListsToMarkdown", () => {
+    test("converts ul with nested ol", () => {
       const html = `
         <ul>
           <li>Item 1</li>
@@ -89,14 +98,186 @@ describe("formatHTMLForIngestion", () => {
         DEFAULT_CONFIG.lists.ordered,
         DEFAULT_CONFIG.lists.unordered,
       );
-    const result = $("pre.list-markdown").text();
+      const lines = $("pre.list-markdown")
-    expect(result).toContain("- Item 1");
+        .text()
-    expect(result).toContain("- Item 2");
+        .split("\n")
-    expect(result).toContain("  1. Subitem 1");
+        .filter((l) => l.length > 0);
-    expect(result).toContain("  2. Subitem 2");
+      expect(lines).toEqual([
+        "- Item 1",
+        "- Item 2",
+        "  1. Subitem 1",
+        "  2. Subitem 2",
+      ]);
     });
 
-  test("convertTablesToMarkdown converts tables to markdown tables", () => {
+    test("handles 3-level deep nesting (ol > ul > ol)", () => {
+      const html = `
+        <ol>
+          <li>First</li>
+          <li>Second
+            <ul>
+              <li>Alpha
+                <ol>
+                  <li>Deep 1</li>
+                  <li>Deep 2</li>
+                </ol>
+              </li>
+              <li>Beta</li>
+            </ul>
+          </li>
+        </ol>
+      `;
+      const $ = cheerio.load(html);
+      convertListsToMarkdown(
+        $,
+        DEFAULT_CONFIG.lists.ordered,
+        DEFAULT_CONFIG.lists.unordered,
+      );
+      const lines = $("pre.list-markdown")
+        .text()
+        .split("\n")
+        .filter((l) => l.length > 0);
+      expect(lines).toEqual([
+        "1. First",
+        "2. Second",
+        "  - Alpha",
+        "    1. Deep 1",
+        "    2. Deep 2",
+        "  - Beta",
+      ]);
+    });
+
+    test("handles multiple sibling nested lists in one item", () => {
+      const html = `
+        <ul>
+          <li>Item
+            <ol>
+              <li>Ordered sub</li>
+            </ol>
+            <ul>
+              <li>Unordered sub</li>
+            </ul>
+          </li>
+        </ul>
+      `;
+      const $ = cheerio.load(html);
+      convertListsToMarkdown(
+        $,
+        DEFAULT_CONFIG.lists.ordered,
+        DEFAULT_CONFIG.lists.unordered,
+      );
+      const lines = $("pre.list-markdown")
+        .text()
+        .split("\n")
+        .filter((l) => l.length > 0);
+      expect(lines).toEqual([
+        "- Item",
+        "  1. Ordered sub",
+        "  - Unordered sub",
+      ]);
+    });
+
+    test("handles consecutive top-level lists", () => {
+      const html = `
+        <ul>
+          <li>UL item 1</li>
+          <li>UL item 2</li>
+        </ul>
+        <ol>
+          <li>OL item 1</li>
+          <li>OL item 2</li>
+        </ol>
+      `;
+      const $ = cheerio.load(html);
+      convertListsToMarkdown(
+        $,
+        DEFAULT_CONFIG.lists.ordered,
+        DEFAULT_CONFIG.lists.unordered,
+      );
+      const results = $("pre.list-markdown");
+      expect(results.length).toBe(2);
+
+      const ulLines = results
+        .eq(0)
+        .text()
+        .split("\n")
+        .filter((l) => l.length > 0);
+      expect(ulLines).toEqual(["- UL item 1", "- UL item 2"]);
+
+      const olLines = results
+        .eq(1)
+        .text()
+        .split("\n")
+        .filter((l) => l.length > 0);
+      expect(olLines).toEqual(["1. OL item 1", "2. OL item 2"]);
+    });
+
+    test("preserves inline code inside list items", () => {
+      const html = `
+        <ul>
+          <li>Call <code>foo()</code></li>
+          <li>Use <var>x</var></li>
+        </ul>
+      `;
+      const $ = cheerio.load(html);
+      convertInlineCodeToMarkdown($, DEFAULT_CONFIG.codeBlocks.inline);
+      convertListsToMarkdown(
+        $,
+        DEFAULT_CONFIG.lists.ordered,
+        DEFAULT_CONFIG.lists.unordered,
+      );
+      const lines = $("pre.list-markdown")
+        .text()
+        .split("\n")
+        .filter((l) => l.length > 0);
+      expect(lines).toEqual(["- Call `foo()`", "- Use `x`"]);
+    });
+
+    test("handles ol nested inside ol", () => {
+      const html = `
+        <ol>
+          <li>First
+            <ol>
+              <li>Nested 1</li>
+              <li>Nested 2</li>
+            </ol>
+          </li>
+          <li>Second</li>
+        </ol>
+      `;
+      const $ = cheerio.load(html);
+      convertListsToMarkdown(
+        $,
+        DEFAULT_CONFIG.lists.ordered,
+        DEFAULT_CONFIG.lists.unordered,
+      );
+      const lines = $("pre.list-markdown")
+        .text()
+        .split("\n")
+        .filter((l) => l.length > 0);
+      expect(lines).toEqual([
+        "1. First",
+        "  1. Nested 1",
+        "  2. Nested 2",
+        "2. Second",
+      ]);
+    });
+
+    test("handles empty list", () => {
+      const html = `<ul></ul>`;
+      const $ = cheerio.load(html);
+      convertListsToMarkdown(
+        $,
+        DEFAULT_CONFIG.lists.ordered,
+        DEFAULT_CONFIG.lists.unordered,
+      );
+      const result = $("pre.list-markdown").text().trim();
+      expect(result).toBe("");
+    });
+  });
+
+  describe("convertTablesToMarkdown", () => {
+    test("converts tables to markdown tables", () => {
       const html = `
         <table>
           <thead>
@@ -114,8 +295,10 @@ describe("formatHTMLForIngestion", () => {
       expect(result).toContain("| --- | --- |");
       expect(result).toContain("| Data 1 | Data 2 |");
     });
+  });
 
-  test("formatForIngestion runs the full pipeline", () => {
+  describe("formatForIngestion", () => {
+    test("runs the full pipeline", () => {
       const html = `
         <div>
           <p>See <emu-xref href="#sec-example"><a href="#sec-example">Example</a></emu-xref> for <var>x</var></p>
@@ -129,19 +312,77 @@ describe("formatHTMLForIngestion", () => {
       const $ = cheerio.load(html);
       formatForIngestion($);
 
-    // Check links
       expect($.html()).toContain(
         '<span class="link-markdown">[Example](#sec-example)</span>',
       );
-    // Check inline code
       expect($.html()).toContain('<span class="inline-code">`x`</span>');
-    // Check block code
       expect($.html()).toContain(
         '<pre class="code-markdown">```javascript\nlet y = x;\n```</pre>',
       );
-    // Check lists
       expect($.html()).toContain(
         '<pre class="list-markdown">- One\n- Two\n</pre>',
       );
     });
+
+    test("preserves nested list indentation through the full pipeline", () => {
+      const html = `
+        <ol>
+          <li>First</li>
+          <li>Second
+            <ul>
+              <li>Alpha
+                <ol>
+                  <li>Deep 1</li>
+                  <li>Deep 2</li>
+                </ol>
+              </li>
+              <li>Beta</li>
+            </ul>
+          </li>
+        </ol>
+      `;
+      const $ = cheerio.load(html);
+      formatForIngestion($);
+
+      const lines = $("pre.list-markdown")
+        .text()
+        .split("\n")
+        .filter((l) => l.length > 0);
+      expect(lines).toEqual([
+        "1. First",
+        "2. Second",
+        "  - Alpha",
+        "    1. Deep 1",
+        "    2. Deep 2",
+        "  - Beta",
+      ]);
+    });
+
+    test("preserves ol nested inside ol through the full pipeline", () => {
+      const html = `
+        <ol>
+          <li>First
+            <ol>
+              <li>Nested 1</li>
+              <li>Nested 2</li>
+            </ol>
+          </li>
+          <li>Second</li>
+        </ol>
+      `;
+      const $ = cheerio.load(html);
+      formatForIngestion($);
+
+      const lines = $("pre.list-markdown")
+        .text()
+        .split("\n")
+        .filter((l) => l.length > 0);
+      expect(lines).toEqual([
+        "1. First",
+        "  1. Nested 1",
+        "  2. Nested 2",
+        "2. Second",
+      ]);
+    });
+  });
 });