fix - Fix normalizeWhitespace removing indents in nested lists

setup/textsplitters/HtmlTextSplitter.test.ts

@@ -50,16 +50,16 @@ describe("HTMLTextSplitter", () => {
     expect(chunks).toEqual(["Hello\nWorld"]);
   });
 
-  test("normalizes repeated whitespace in emitted text", async () => {
+  test("preserves whitespace as-is from HTML text content", async () => {
     const splitter = new HTMLTextSplitter({
       chunkSize: 64,
     });
 
     const chunks = await splitter.splitText(
-      "<div>  Hello\n\n   World  </div><div>\tAgain</div>",
+      "<div>  Hello\n\n   World  </div><div>Again</div>",
     );
 
-    expect(chunks).toEqual(["Hello\n\nWorld\nAgain"]);
+    expect(chunks).toEqual(["Hello\n\n   World  \nAgain"]);
   });
 
   test("treats separators as soft hints until size pressure exists", async () => {
@@ -325,4 +325,25 @@ describe("HTMLTextSplitter", () => {
 
     expect(chunks).toEqual(["alphabetgamma"]);
   });
+
+  test("preserves nested list indentation from formatForIngestion output", async () => {
+    const splitter = new HTMLTextSplitter({
+      chunkSize: 200,
+    });
+
+    const html = [
+      '<pre class="list-markdown">',
+      "1. First",
+      "  1. Nested 1",
+      "  2. Nested 2",
+      "2. Second",
+      "</pre>",
+    ].join("\n");
+
+    const chunks = await splitter.splitText(html);
+
+    expect(chunks).toEqual([
+      "1. First\n  1. Nested 1\n  2. Nested 2\n2. Second",
+    ]);
+  });
 });

setup/textsplitters/HtmlTextSplitter.ts

@@ -111,7 +111,7 @@ export class HTMLTextSplitter extends TextSplitter {
       ...fields,
       chunkOverlap: 0,
       keepSeparator: false,
-      lengthFunction: (text: string) => this.normalizeWhitespace(text).length,
+      lengthFunction: (text: string) => text.trim().length,
     });
 
     this.separators = fields?.separators ?? [];
@@ -229,22 +229,6 @@ export class HTMLTextSplitter extends TextSplitter {
     return node.type === "tag" && BLOCKISH_TAGS.has(node.name);
   }
 
-  /**
-   * Collapses internal whitespace and trims leading/trailing whitespace.
-   * Preserves newlines between block elements while normalizing spaces.
-   *
-   * Needed so text extraction can preserve raw adjacency first and normalize
-   * only once after boundary-aware joining.
-   */
-  private normalizeWhitespace(text: string): string {
-    return text
-      .replace(/\n[ \t]+/g, "\n") // Remove leading spaces after newlines
-      .replace(/[ \t]+\n/g, "\n") // Remove trailing spaces before newlines
-      .replace(/\n{3,}/g, "\n\n") // Collapse 3+ newlines to 2
-      .replace(/[ \t]{2,}/g, " ") // Collapse multiple spaces/tabs to one
-      .trim();
-  }
-
   /**
    * Joins multiple nodes into the normalized text the splitter actually emits.
    *
@@ -273,7 +257,7 @@ export class HTMLTextSplitter extends TextSplitter {
       previousNode = node;
     }
 
-    return this.normalizeWhitespace(result);
+    return result.trim();
   }
 
   /**

setup/utils/formatHTMLForIngestion.test.ts

@@ -12,136 +12,377 @@ import {
 } from "./formatHTMLForIngestion";
 
 describe("formatHTMLForIngestion", () => {
-  test("convertLinksToMarkdown converts emu-xref links to markdown and strips filenames", () => {
+  describe("convertLinksToMarkdown", () => {
-    const html = `<emu-xref href="abstract-operations.html#sec-tonumber"><a href="abstract-operations.html#sec-tonumber">ToNumber</a></emu-xref>`;
+    test("converts emu-xref links to markdown and strips filenames", () => {
-    const $ = cheerio.load(html);
+      const html = `<emu-xref href="abstract-operations.html#sec-tonumber"><a href="abstract-operations.html#sec-tonumber">ToNumber</a></emu-xref>`;
-    convertLinksToMarkdown($, DEFAULT_CONFIG.links.join(", "));
+      const $ = cheerio.load(html);
-    expect($.html()).toContain(
+      convertLinksToMarkdown($, DEFAULT_CONFIG.links.join(", "));
-      '<span class="link-markdown">[ToNumber](#sec-tonumber)</span>',
+      expect($.html()).toContain(
-    );
+        '<span class="link-markdown">[ToNumber](#sec-tonumber)</span>',
+      );
+    });
+
+    test("skips external links", () => {
+      const html = `<emu-xref href="https://example.com"><a href="https://example.com">External</a></emu-xref>`;
+      const $ = cheerio.load(html);
+      convertLinksToMarkdown($, DEFAULT_CONFIG.links.join(", "));
+      expect($.html()).toContain('<a href="https://example.com">External</a>');
+    });
   });
 
-  test("convertLinksToMarkdown skips external links", () => {
+  describe("convertInlineCodeToMarkdown", () => {
-    const html = `<emu-xref href="https://example.com"><a href="https://example.com">External</a></emu-xref>`;
+    test("converts var, emu-val, emu-const to inline code", () => {
-    const $ = cheerio.load(html);
+      const html = `<div><var>x</var> <emu-val>y</emu-val> <emu-const>z</emu-const> <code>w</code></div>`;
-    convertLinksToMarkdown($, DEFAULT_CONFIG.links.join(", "));
+      const $ = cheerio.load(html);
-    expect($.html()).toContain('<a href="https://example.com">External</a>');
+      convertInlineCodeToMarkdown($, DEFAULT_CONFIG.codeBlocks.inline);
+      expect($.html()).toContain('<span class="inline-code">`x`</span>');
+      expect($.html()).toContain('<span class="inline-code">`y`</span>');
+      expect($.html()).toContain('<span class="inline-code">`z`</span>');
+      expect($.html()).toContain('<span class="inline-code">`w`</span>');
+    });
+
+    test("skips code inside pre", () => {
+      const html = `<pre><code>x</code></pre>`;
+      const $ = cheerio.load(html);
+      convertInlineCodeToMarkdown($, DEFAULT_CONFIG.codeBlocks.inline);
+      expect($.html()).toContain("<pre><code>x</code></pre>");
+    });
   });
 
-  test("convertInlineCodeToMarkdown converts var, emu-val, emu-const to inline code", () => {
+  describe("convertBlockCodeToMarkdown", () => {
-    const html = `<div><var>x</var> <emu-val>y</emu-val> <emu-const>z</emu-const> <code>w</code></div>`;
+    test("converts pre>code and emu-eqn", () => {
-    const $ = cheerio.load(html);
+      const html = `<div>
-    convertInlineCodeToMarkdown($, DEFAULT_CONFIG.codeBlocks.inline);
+        <pre><code class="javascript hljs">const x = 1;</code></pre>
-    expect($.html()).toContain('<span class="inline-code">`x`</span>');
+        <emu-eqn>y = x + 1</emu-eqn>
-    expect($.html()).toContain('<span class="inline-code">`y`</span>');
+        <emu-eqn class="inline">z = 2</emu-eqn>
-    expect($.html()).toContain('<span class="inline-code">`z`</span>');
+      </div>`;
-    expect($.html()).toContain('<span class="inline-code">`w`</span>');
+      const $ = cheerio.load(html);
+      convertBlockCodeToMarkdown($, DEFAULT_CONFIG.codeBlocks.block);
+      expect($.html()).toContain(
+        '<pre class="code-markdown">```javascript\nconst x = 1;\n```</pre>',
+      );
+      expect($.html()).toContain(
+        '<pre class="code-markdown">```\ny = x + 1\n```</pre>',
+      );
+      expect($.html()).toContain('<emu-eqn class="inline">z = 2</emu-eqn>');
+    });
   });
 
-  test("convertInlineCodeToMarkdown skips code inside pre", () => {
+  describe("convertGrammarToMarkdown", () => {
-    const html = `<pre><code>x</code></pre>`;
+    test("converts emu-grammar to fenced bnf", () => {
-    const $ = cheerio.load(html);
+      const html = `<emu-grammar>Statement :: BlockStatement</emu-grammar>`;
-    convertInlineCodeToMarkdown($, DEFAULT_CONFIG.codeBlocks.inline);
+      const $ = cheerio.load(html);
-    expect($.html()).toContain("<pre><code>x</code></pre>"); // unchanged
+      convertGrammarToMarkdown($, DEFAULT_CONFIG.codeBlocks.grammar.join(", "));
+      expect($.html()).toContain(
+        '<pre class="code-markdown">```bnf\nStatement :: BlockStatement\n```</pre>',
+      );
+    });
   });
 
-  test("convertBlockCodeToMarkdown converts pre>code and emu-eqn", () => {
+  describe("convertListsToMarkdown", () => {
-    const html = `<div>
+    test("converts ul with nested ol", () => {
-      <pre><code class="javascript hljs">const x = 1;</code></pre>
+      const html = `
-      <emu-eqn>y = x + 1</emu-eqn>
-      <emu-eqn class="inline">z = 2</emu-eqn>
-    </div>`;
-    const $ = cheerio.load(html);
-    convertBlockCodeToMarkdown($, DEFAULT_CONFIG.codeBlocks.block);
-    expect($.html()).toContain(
-      '<pre class="code-markdown">```javascript\nconst x = 1;\n```</pre>',
-    );
-    expect($.html()).toContain(
-      '<pre class="code-markdown">```\ny = x + 1\n```</pre>',
-    );
-    expect($.html()).toContain('<emu-eqn class="inline">z = 2</emu-eqn>'); // unchanged
-  });
-
-  test("convertGrammarToMarkdown converts emu-grammar to fenced bnf", () => {
-    const html = `<emu-grammar>Statement :: BlockStatement</emu-grammar>`;
-    const $ = cheerio.load(html);
-    convertGrammarToMarkdown($, DEFAULT_CONFIG.codeBlocks.grammar.join(", "));
-    expect($.html()).toContain(
-      '<pre class="code-markdown">```bnf\nStatement :: BlockStatement\n```</pre>',
-    );
-  });
-
-  test("convertListsToMarkdown converts ol and ul with nesting to markdown lists", () => {
-    const html = `
-      <ul>
-        <li>Item 1</li>
-        <li>Item 2
-          <ol>
-            <li>Subitem 1</li>
-            <li>Subitem 2</li>
-          </ol>
-        </li>
-      </ul>
-    `;
-    const $ = cheerio.load(html);
-    convertListsToMarkdown(
-      $,
-      DEFAULT_CONFIG.lists.ordered,
-      DEFAULT_CONFIG.lists.unordered,
-    );
-    const result = $("pre.list-markdown").text();
-    expect(result).toContain("- Item 1");
-    expect(result).toContain("- Item 2");
-    expect(result).toContain("  1. Subitem 1");
-    expect(result).toContain("  2. Subitem 2");
-  });
-
-  test("convertTablesToMarkdown converts tables to markdown tables", () => {
-    const html = `
-      <table>
-        <thead>
-          <tr><th>Col 1</th><th>Col 2</th></tr>
-        </thead>
-        <tbody>
-          <tr><td>Data 1</td><td>Data 2</td></tr>
-        </tbody>
-      </table>
-    `;
-    const $ = cheerio.load(html);
-    convertTablesToMarkdown($);
-    const result = $("pre.table-markdown").text();
-    expect(result).toContain("| Col 1 | Col 2 |");
-    expect(result).toContain("| --- | --- |");
-    expect(result).toContain("| Data 1 | Data 2 |");
-  });
-
-  test("formatForIngestion runs the full pipeline", () => {
-    const html = `
-      <div>
-        <p>See <emu-xref href="#sec-example"><a href="#sec-example">Example</a></emu-xref> for <var>x</var></p>
-        <pre><code class="javascript">let y = x;</code></pre>
         <ul>
-          <li>One</li>
+          <li>Item 1</li>
-          <li>Two</li>
+          <li>Item 2
+            <ol>
+              <li>Subitem 1</li>
+              <li>Subitem 2</li>
+            </ol>
+          </li>
         </ul>
-      </div>
+      `;
-    `;
+      const $ = cheerio.load(html);
-    const $ = cheerio.load(html);
+      convertListsToMarkdown(
-    formatForIngestion($);
+        $,
+        DEFAULT_CONFIG.lists.ordered,
+        DEFAULT_CONFIG.lists.unordered,
+      );
+      const lines = $("pre.list-markdown")
+        .text()
+        .split("\n")
+        .filter((l) => l.length > 0);
+      expect(lines).toEqual([
+        "- Item 1",
+        "- Item 2",
+        "  1. Subitem 1",
+        "  2. Subitem 2",
+      ]);
+    });
 
-    // Check links
+    test("handles 3-level deep nesting (ol > ul > ol)", () => {
-    expect($.html()).toContain(
+      const html = `
-      '<span class="link-markdown">[Example](#sec-example)</span>',
+        <ol>
-    );
+          <li>First</li>
-    // Check inline code
+          <li>Second
-    expect($.html()).toContain('<span class="inline-code">`x`</span>');
+            <ul>
-    // Check block code
+              <li>Alpha
-    expect($.html()).toContain(
+                <ol>
-      '<pre class="code-markdown">```javascript\nlet y = x;\n```</pre>',
+                  <li>Deep 1</li>
-    );
+                  <li>Deep 2</li>
-    // Check lists
+                </ol>
-    expect($.html()).toContain(
+              </li>
-      '<pre class="list-markdown">- One\n- Two\n</pre>',
+              <li>Beta</li>
-    );
+            </ul>
+          </li>
+        </ol>
+      `;
+      const $ = cheerio.load(html);
+      convertListsToMarkdown(
+        $,
+        DEFAULT_CONFIG.lists.ordered,
+        DEFAULT_CONFIG.lists.unordered,
+      );
+      const lines = $("pre.list-markdown")
+        .text()
+        .split("\n")
+        .filter((l) => l.length > 0);
+      expect(lines).toEqual([
+        "1. First",
+        "2. Second",
+        "  - Alpha",
+        "    1. Deep 1",
+        "    2. Deep 2",
+        "  - Beta",
+      ]);
+    });
+
+    test("handles multiple sibling nested lists in one item", () => {
+      const html = `
+        <ul>
+          <li>Item
+            <ol>
+              <li>Ordered sub</li>
+            </ol>
+            <ul>
+              <li>Unordered sub</li>
+            </ul>
+          </li>
+        </ul>
+      `;
+      const $ = cheerio.load(html);
+      convertListsToMarkdown(
+        $,
+        DEFAULT_CONFIG.lists.ordered,
+        DEFAULT_CONFIG.lists.unordered,
+      );
+      const lines = $("pre.list-markdown")
+        .text()
+        .split("\n")
+        .filter((l) => l.length > 0);
+      expect(lines).toEqual([
+        "- Item",
+        "  1. Ordered sub",
+        "  - Unordered sub",
+      ]);
+    });
+
+    test("handles consecutive top-level lists", () => {
+      const html = `
+        <ul>
+          <li>UL item 1</li>
+          <li>UL item 2</li>
+        </ul>
+        <ol>
+          <li>OL item 1</li>
+          <li>OL item 2</li>
+        </ol>
+      `;
+      const $ = cheerio.load(html);
+      convertListsToMarkdown(
+        $,
+        DEFAULT_CONFIG.lists.ordered,
+        DEFAULT_CONFIG.lists.unordered,
+      );
+      const results = $("pre.list-markdown");
+      expect(results.length).toBe(2);
+
+      const ulLines = results
+        .eq(0)
+        .text()
+        .split("\n")
+        .filter((l) => l.length > 0);
+      expect(ulLines).toEqual(["- UL item 1", "- UL item 2"]);
+
+      const olLines = results
+        .eq(1)
+        .text()
+        .split("\n")
+        .filter((l) => l.length > 0);
+      expect(olLines).toEqual(["1. OL item 1", "2. OL item 2"]);
+    });
+
+    test("preserves inline code inside list items", () => {
+      const html = `
+        <ul>
+          <li>Call <code>foo()</code></li>
+          <li>Use <var>x</var></li>
+        </ul>
+      `;
+      const $ = cheerio.load(html);
+      convertInlineCodeToMarkdown($, DEFAULT_CONFIG.codeBlocks.inline);
+      convertListsToMarkdown(
+        $,
+        DEFAULT_CONFIG.lists.ordered,
+        DEFAULT_CONFIG.lists.unordered,
+      );
+      const lines = $("pre.list-markdown")
+        .text()
+        .split("\n")
+        .filter((l) => l.length > 0);
+      expect(lines).toEqual(["- Call `foo()`", "- Use `x`"]);
+    });
+
+    test("handles ol nested inside ol", () => {
+      const html = `
+        <ol>
+          <li>First
+            <ol>
+              <li>Nested 1</li>
+              <li>Nested 2</li>
+            </ol>
+          </li>
+          <li>Second</li>
+        </ol>
+      `;
+      const $ = cheerio.load(html);
+      convertListsToMarkdown(
+        $,
+        DEFAULT_CONFIG.lists.ordered,
+        DEFAULT_CONFIG.lists.unordered,
+      );
+      const lines = $("pre.list-markdown")
+        .text()
+        .split("\n")
+        .filter((l) => l.length > 0);
+      expect(lines).toEqual([
+        "1. First",
+        "  1. Nested 1",
+        "  2. Nested 2",
+        "2. Second",
+      ]);
+    });
+
+    test("handles empty list", () => {
+      const html = `<ul></ul>`;
+      const $ = cheerio.load(html);
+      convertListsToMarkdown(
+        $,
+        DEFAULT_CONFIG.lists.ordered,
+        DEFAULT_CONFIG.lists.unordered,
+      );
+      const result = $("pre.list-markdown").text().trim();
+      expect(result).toBe("");
+    });
+  });
+
+  describe("convertTablesToMarkdown", () => {
+    test("converts tables to markdown tables", () => {
+      const html = `
+        <table>
+          <thead>
+            <tr><th>Col 1</th><th>Col 2</th></tr>
+          </thead>
+          <tbody>
+            <tr><td>Data 1</td><td>Data 2</td></tr>
+          </tbody>
+        </table>
+      `;
+      const $ = cheerio.load(html);
+      convertTablesToMarkdown($);
+      const result = $("pre.table-markdown").text();
+      expect(result).toContain("| Col 1 | Col 2 |");
+      expect(result).toContain("| --- | --- |");
+      expect(result).toContain("| Data 1 | Data 2 |");
+    });
+  });
+
+  describe("formatForIngestion", () => {
+    test("runs the full pipeline", () => {
+      const html = `
+        <div>
+          <p>See <emu-xref href="#sec-example"><a href="#sec-example">Example</a></emu-xref> for <var>x</var></p>
+          <pre><code class="javascript">let y = x;</code></pre>
+          <ul>
+            <li>One</li>
+            <li>Two</li>
+          </ul>
+        </div>
+      `;
+      const $ = cheerio.load(html);
+      formatForIngestion($);
+
+      expect($.html()).toContain(
+        '<span class="link-markdown">[Example](#sec-example)</span>',
+      );
+      expect($.html()).toContain('<span class="inline-code">`x`</span>');
+      expect($.html()).toContain(
+        '<pre class="code-markdown">```javascript\nlet y = x;\n```</pre>',
+      );
+      expect($.html()).toContain(
+        '<pre class="list-markdown">- One\n- Two\n</pre>',
+      );
+    });
+
+    test("preserves nested list indentation through the full pipeline", () => {
+      const html = `
+        <ol>
+          <li>First</li>
+          <li>Second
+            <ul>
+              <li>Alpha
+                <ol>
+                  <li>Deep 1</li>
+                  <li>Deep 2</li>
+                </ol>
+              </li>
+              <li>Beta</li>
+            </ul>
+          </li>
+        </ol>
+      `;
+      const $ = cheerio.load(html);
+      formatForIngestion($);
+
+      const lines = $("pre.list-markdown")
+        .text()
+        .split("\n")
+        .filter((l) => l.length > 0);
+      expect(lines).toEqual([
+        "1. First",
+        "2. Second",
+        "  - Alpha",
+        "    1. Deep 1",
+        "    2. Deep 2",
+        "  - Beta",
+      ]);
+    });
+
+    test("preserves ol nested inside ol through the full pipeline", () => {
+      const html = `
+        <ol>
+          <li>First
+            <ol>
+              <li>Nested 1</li>
+              <li>Nested 2</li>
+            </ol>
+          </li>
+          <li>Second</li>
+        </ol>
+      `;
+      const $ = cheerio.load(html);
+      formatForIngestion($);
+
+      const lines = $("pre.list-markdown")
+        .text()
+        .split("\n")
+        .filter((l) => l.length > 0);
+      expect(lines).toEqual([
+        "1. First",
+        "  1. Nested 1",
+        "  2. Nested 2",
+        "2. Second",
+      ]);
+    });
   });
 });