fix - Fix normalizeWhitespace removing indents in nested lists

setup/textsplitters/HtmlTextSplitter.test.ts

@@ -50,16 +50,16 @@ describe("HTMLTextSplitter", () => {
     expect(chunks).toEqual(["Hello\nWorld"]);
   });
 
-  test("normalizes repeated whitespace in emitted text", async () => {
+  test("preserves whitespace as-is from HTML text content", async () => {
     const splitter = new HTMLTextSplitter({
       chunkSize: 64,
     });
 
     const chunks = await splitter.splitText(
-      "<div>  Hello\n\n   World  </div><div>\tAgain</div>",
+      "<div>  Hello\n\n   World  </div><div>Again</div>",
     );
 
-    expect(chunks).toEqual(["Hello\n\nWorld\nAgain"]);
+    expect(chunks).toEqual(["Hello\n\n   World  \nAgain"]);
   });
 
   test("treats separators as soft hints until size pressure exists", async () => {
@@ -325,4 +325,25 @@ describe("HTMLTextSplitter", () => {
 
     expect(chunks).toEqual(["alphabetgamma"]);
   });
+
+  test("preserves nested list indentation from formatForIngestion output", async () => {
+    const splitter = new HTMLTextSplitter({
+      chunkSize: 200,
+    });
+
+    const html = [
+      '<pre class="list-markdown">',
+      "1. First",
+      "  1. Nested 1",
+      "  2. Nested 2",
+      "2. Second",
+      "</pre>",
+    ].join("\n");
+
+    const chunks = await splitter.splitText(html);
+
+    expect(chunks).toEqual([
+      "1. First\n  1. Nested 1\n  2. Nested 2\n2. Second",
+    ]);
+  });
 });

setup/textsplitters/HtmlTextSplitter.ts

@@ -111,7 +111,7 @@ export class HTMLTextSplitter extends TextSplitter {
       ...fields,
       chunkOverlap: 0,
       keepSeparator: false,
-      lengthFunction: (text: string) => this.normalizeWhitespace(text).length,
+      lengthFunction: (text: string) => text.trim().length,
     });
 
     this.separators = fields?.separators ?? [];
@@ -229,22 +229,6 @@ export class HTMLTextSplitter extends TextSplitter {
     return node.type === "tag" && BLOCKISH_TAGS.has(node.name);
   }
 
-  /**
-   * Collapses internal whitespace and trims leading/trailing whitespace.
-   * Preserves newlines between block elements while normalizing spaces.
-   *
-   * Needed so text extraction can preserve raw adjacency first and normalize
-   * only once after boundary-aware joining.
-   */
-  private normalizeWhitespace(text: string): string {
-    return text
-      .replace(/\n[ \t]+/g, "\n") // Remove leading spaces after newlines
-      .replace(/[ \t]+\n/g, "\n") // Remove trailing spaces before newlines
-      .replace(/\n{3,}/g, "\n\n") // Collapse 3+ newlines to 2
-      .replace(/[ \t]{2,}/g, " ") // Collapse multiple spaces/tabs to one
-      .trim();
-  }
-
   /**
    * Joins multiple nodes into the normalized text the splitter actually emits.
    *
@@ -273,7 +257,7 @@ export class HTMLTextSplitter extends TextSplitter {
       previousNode = node;
     }
 
-    return this.normalizeWhitespace(result);
+    return result.trim();
   }
 
   /**