fix(agent-tools, setup): standardize metadata keys to lowercase and update HTML ingestion

- Renamed `partIndex`/`totalParts` to `partindex`/`totalparts` in `section_retriever.ts`, `spec_retriever.ts`, and document creation logic. - Adjusted sorting logic to use the new `partindex` field. - Updated metadata handling and string interpolation to reference the lowercase keys. - Fixed import path for `HTMLTextSplitter` and moved the file to `setup/textsplitters/HtmlTextSplitter.ts`. - Added HTML preprocessing utilities (`addNewlinesAfterBlocks`, `convertTablesToMarkdown`) in ingestion workflow. - Integrated table-to‑markdown conversion and newline insertion to improve text splitting and document structure.
2026-08-18 13:21:55 +00:00 · 2026-04-04 12:33:05 +05:30
parent 822597e0e2
commit ff1c97af44
8 changed files with 176 additions and 37 deletions
@@ -41,10 +41,10 @@ export function createSectionRetrieverTool(table: Table) {
          .limit(100)
          .toArray();

-        // Sort by partIndex to maintain order (nulls last for single-part sections)
+        // Sort by partindex to maintain order (nulls last for single-part sections)
        const sortedResults = results.sort((a: unknown, b: unknown) => {
-          const aIndex = (a as { partIndex?: number }).partIndex ?? Infinity;
-          const bIndex = (b as { partIndex?: number }).partIndex ?? Infinity;
+          const aIndex = (a as { partindex?: number }).partindex ?? Infinity;
+          const bIndex = (b as { partindex?: number }).partindex ?? Infinity;
          return aIndex - bIndex;
        });

@@ -46,8 +46,8 @@ export function createSpecRetrieverTool(
          type: r.type,
          parentsectionid: r.parentsectionid,
          childrensectionids: r.childrensectionids,
-          partIndex: r.partIndex,
-          totalParts: r.totalParts,
+          partindex: r.partindex,
+          totalparts: r.totalparts,
        },
      }));

@@ -77,8 +77,8 @@ export function createSpecRetrieverTool(
          const sectionId = meta?.sectionid || "unknown";
          const sectionTitle = meta?.sectiontitle || "unknown";
          const partInfo =
-            meta?.partIndex !== null && meta?.partIndex !== undefined
-              ? ` [part ${(meta.partIndex as number) + 1}/${meta.totalParts}]`
+            meta?.partindex !== null && meta?.partindex !== undefined
+              ? ` [part ${(meta.partindex as number) + 1}/${meta.totalparts}]`
              : "";
          return `--- Section: ${sectionId} | "${sectionTitle}"${partInfo} (score: ${r.score.toFixed(2)}) ---\n${r.document.pageContent}`;
        })
@@ -9,7 +9,11 @@ import * as cheerio from "cheerio";
 import { glob } from "glob";
 import ora from "ora";
 import { EMBEDDING_MODEL, SPEC_DIR, STORAGE_DIR } from "../constants";
-import { HTMLTextSplitter } from "../textsplitters";
+import { HTMLTextSplitter } from "./textsplitters";
+import {
+  addNewlinesAfterBlocks,
+  convertTablesToMarkdown,
+} from "./utils/formatHTMLForIngestion";

 const embeddings = new OllamaEmbeddings({
  model: EMBEDDING_MODEL,
@@ -157,7 +161,12 @@ async function buildSpecDocuments(): Promise<Document[]> {
    // Second pass: create documents with formatted text
    for (const [id, section] of sectionMap) {
      // Parse the stored HTML and replace direct children with placeholders
-      const $section = cheerio.load(`<body>${section.html}</body>`).root();
+      const $ = cheerio.load(`<body>${section.html}</body>`);
+
+      // Convert tables to markdown format for better text extraction
+      convertTablesToMarkdown($);
+
+      const $section = $.root();

      // Find direct children emu-clause elements only
      $section.children("emu-clause").each((_, childElem) => {
@@ -175,6 +184,10 @@ async function buildSpecDocuments(): Promise<Document[]> {
      // (shouldn't happen with proper HTML structure, but just in case)
      $section.find("emu-clause").remove();

+      // Add newlines after block elements to preserve document structure
+      // This helps the text splitter maintain paragraph/section boundaries
+      addNewlinesAfterBlocks($);
+
      // Skip sections that only have h1 left (no meaningful content)
      const hasOnlyH1 =
        $section.children().length === 1 &&
@@ -255,8 +268,8 @@ async function buildSpecDocuments(): Promise<Document[]> {
              type: "specification",
              parentsectionid: section.parentId,
              childrensectionids: section.childrenIds,
-              partIndex: chunk.index,
-              totalParts: chunkData.length,
+              partindex: chunk.index,
+              totalparts: chunkData.length,
            },
          }),
        );
@@ -169,8 +169,8 @@ export class HTMLTextSplitter extends TextSplitter {
            pageContent: `${chunkHeader}${chunk}`,
            metadata: {
              ...document.metadata,
-              partIndex: index,
-              totalParts: chunks.length,
+              partindex: index,
+              totalparts: chunks.length,
            },
          }),
        );
@@ -231,12 +231,18 @@ export class HTMLTextSplitter extends TextSplitter {

  /**
   * Collapses internal whitespace and trims leading/trailing whitespace.
+   * Preserves newlines between block elements while normalizing spaces.
   *
   * Needed so text extraction can preserve raw adjacency first and normalize
   * only once after boundary-aware joining.
   */
  private normalizeWhitespace(text: string): string {
-    return text.replace(/\s+/g, " ").trim();
+    return text
+      .replace(/\n[ \t]+/g, "\n") // Remove leading spaces after newlines
+      .replace(/[ \t]+\n/g, "\n") // Remove trailing spaces before newlines
+      .replace(/\n{3,}/g, "\n\n") // Collapse 3+ newlines to 2
+      .replace(/[ \t]{2,}/g, " ") // Collapse multiple spaces/tabs to one
+      .trim();
  }

  /**
@@ -260,7 +266,7 @@ export class HTMLTextSplitter extends TextSplitter {
        previousNode &&
        (this.isBlockishNode(previousNode) || this.isBlockishNode(node))
      ) {
-        result += " ";
+        result += "\n"; // Add newline between block elements for better structure
      }

      result += text;
@@ -0,0 +1 @@
+export * from "./HtmlTextSplitter";
@@ -0,0 +1,120 @@
+import type * as cheerio from "cheerio";
+
+/**
+ * List of block-level HTML elements that should have newlines added after them
+ * to preserve document structure during text extraction.
+ */
+export const BLOCK_ELEMENTS = [
+  "p",
+  "div",
+  "h1",
+  "h2",
+  "h3",
+  "h4",
+  "h5",
+  "h6",
+  "li",
+  "td",
+  "th",
+  "pre",
+  "blockquote",
+  "section",
+  "emu-clause",
+  "emu-note",
+  "emu-example",
+  "emu-table",
+  "figure",
+  "figcaption",
+  "ul",
+  "ol",
+  "dl",
+  "dt",
+  "dd",
+];
+
+/**
+ * Converts HTML tables to markdown table format.
+ * This preserves table structure when extracting text from HTML.
+ * Replaces the original table with a <pre> element containing the markdown.
+ *
+ * @param $ - Cheerio API instance
+ */
+export function convertTablesToMarkdown($: cheerio.CheerioAPI): void {
+  $("table, emu-table").each((_, tableElem) => {
+    const $table = $(tableElem);
+    const rows: string[][] = [];
+
+    // Extract header rows
+    $table.find("thead tr").each((_, rowElem) => {
+      const row: string[] = [];
+      $(rowElem)
+        .find("th, td")
+        .each((_, cellElem) => {
+          row.push($(cellElem).text().trim().replace(/\|/g, "\\|"));
+        });
+      if (row.length > 0) rows.push(row);
+    });
+
+    // Extract body rows
+    $table.find("tbody tr, tr").each((_, rowElem) => {
+      // Skip if already processed as header
+      if ($(rowElem).parent("thead").length > 0) return;
+
+      const row: string[] = [];
+      $(rowElem)
+        .find("td, th")
+        .each((_, cellElem) => {
+          row.push($(cellElem).text().trim().replace(/\|/g, "\\|"));
+        });
+      if (row.length > 0) rows.push(row);
+    });
+
+    if (rows.length === 0) return;
+
+    // Determine max columns
+    const maxCols = Math.max(...rows.map((r) => r.length));
+
+    // Build markdown table
+    const mdLines: string[] = [];
+
+    // Header row
+    if (rows.length > 0) {
+      const header = rows[0].concat(Array(maxCols - rows[0].length).fill(""));
+      mdLines.push("| " + header.join(" | ") + " |");
+    }
+
+    // Separator
+    mdLines.push("|" + Array(maxCols).fill(" --- ").join("|") + "|");
+
+    // Data rows (skip header if we have more rows)
+    const dataRows = rows.length > 1 ? rows.slice(1) : [];
+    for (const row of dataRows) {
+      const padded = row.concat(Array(maxCols - row.length).fill(""));
+      mdLines.push("| " + padded.join(" | ") + " |");
+    }
+
+    // Replace table with markdown
+    const markdown = mdLines.join("\n");
+    $table.replaceWith($(`<pre class="table-markdown">${markdown}</pre>`));
+  });
+}
+
+/**
+ * Adds newlines after specified block elements to preserve document structure.
+ * This helps text splitters maintain paragraph/section boundaries.
+ *
+ * @param $ - Cheerio API instance
+ * @param elements - Array of element tag names to add newlines after
+ */
+export function addNewlinesAfterBlocks(
+  $: cheerio.CheerioAPI,
+  elements: string[] = BLOCK_ELEMENTS,
+): void {
+  for (const tag of elements) {
+    $.root()
+      .find(tag)
+      .each((_, el) => {
+        $(el).append("\n");
+      });
+  }
+}
@@ -1,6 +1,6 @@
 import { describe, expect, test } from "bun:test";
 import { Document } from "@langchain/core/documents";
-import { HTMLTextSplitter } from "../../textsplitters";
+import { HTMLTextSplitter } from "../../setup/textsplitters";

 describe("HTMLTextSplitter", () => {
  test("keeps small HTML in a single chunk", async () => {
@@ -13,7 +13,7 @@ describe("HTMLTextSplitter", () => {
      "<section><h2>Title</h2><p>Short body text</p></section>",
    );

-    expect(chunks).toEqual(["Title Short body text"]);
+    expect(chunks).toEqual(["Title\nShort body text"]);
  });

  test("does not add synthetic spaces between inline tags", async () => {
@@ -47,7 +47,7 @@ describe("HTMLTextSplitter", () => {

    const chunks = await splitter.splitText("<div>Hello</div><div>World</div>");

-    expect(chunks).toEqual(["Hello World"]);
+    expect(chunks).toEqual(["Hello\nWorld"]);
  });

  test("normalizes repeated whitespace in emitted text", async () => {
@@ -59,7 +59,7 @@ describe("HTMLTextSplitter", () => {
      "<div>  Hello\n\n   World  </div><div>\tAgain</div>",
    );

-    expect(chunks).toEqual(["Hello World Again"]);
+    expect(chunks).toEqual(["Hello\n\nWorld\nAgain"]);
  });

  test("treats separators as soft hints until size pressure exists", async () => {
@@ -78,7 +78,7 @@ describe("HTMLTextSplitter", () => {
      ].join(""),
    );

-    expect(chunks).toEqual(["Intro text", "Section Title Body text"]);
+    expect(chunks).toEqual(["Intro text", "Section Title\nBody text"]);
  });

  test("groups consecutive separators into later section boundaries", async () => {
@@ -91,7 +91,7 @@ describe("HTMLTextSplitter", () => {
      "<section><h2>A</h2><h3>B</h3><p>Body</p></section>",
    );

-    expect(chunks).toEqual(["A", "B Body"]);
+    expect(chunks).toEqual(["A", "B\nBody"]);
  });

  test("keeps protected content intact up to maxChunkSize", async () => {
@@ -140,7 +140,7 @@ describe("HTMLTextSplitter", () => {
      '<div class="keep"><h2>Title</h2><p>Body text</p></div>',
    );

-    expect(chunks).toEqual(["Title Body text"]);
+    expect(chunks).toEqual(["Title\nBody text"]);
  });

  test("force-splits oversized leaf text when maxChunkSize is finite", async () => {
@@ -212,14 +212,14 @@ describe("HTMLTextSplitter", () => {
    expect(documents[0].pageContent).toBe("Intro text");
    expect(documents[0].metadata).toMatchObject({
      source: "sample.html",
-      partIndex: 0,
-      totalParts: 2,
+      partindex: 0,
+      totalparts: 2,
    });
-    expect(documents[1].pageContent).toBe("Section Title Body text");
+    expect(documents[1].pageContent).toBe("Section Title\nBody text");
    expect(documents[1].metadata).toMatchObject({
      source: "sample.html",
-      partIndex: 1,
-      totalParts: 2,
+      partindex: 1,
+      totalparts: 2,
    });
  });

@@ -244,7 +244,7 @@ describe("HTMLTextSplitter", () => {

    expect(documents).toHaveLength(2);
    expect(documents[0].pageContent).toBe("Header: Intro text");
-    expect(documents[1].pageContent).toBe("Header: Section Title Body text");
+    expect(documents[1].pageContent).toBe("Header: Section Title\nBody text");
  });

  test("returns no documents when source content produces no chunks", async () => {
@@ -284,23 +284,23 @@ describe("HTMLTextSplitter", () => {
    expect(documents).toHaveLength(4);
    expect(documents[0].metadata).toMatchObject({
      source: "first.html",
-      partIndex: 0,
-      totalParts: 2,
+      partindex: 0,
+      totalparts: 2,
    });
    expect(documents[1].metadata).toMatchObject({
      source: "first.html",
-      partIndex: 1,
-      totalParts: 2,
+      partindex: 1,
+      totalparts: 2,
    });
    expect(documents[2].metadata).toMatchObject({
      source: "second.html",
-      partIndex: 0,
-      totalParts: 2,
+      partindex: 0,
+      totalparts: 2,
    });
    expect(documents[3].metadata).toMatchObject({
      source: "second.html",
-      partIndex: 1,
-      totalParts: 2,
+      partindex: 1,
+      totalparts: 2,
    });
  });

@@ -1 +0,0 @@
-export * from "./HTMLTextSplitter";