From ff1c97af4493a7d7eaba0ae3d40cb1596ed8023a Mon Sep 17 00:00:00 2001 From: bendtherules Date: Sat, 4 Apr 2026 12:33:05 +0530 Subject: [PATCH] fix(agent-tools, setup): standardize metadata keys to lowercase and update HTML ingestion MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Renamed `partIndex`/`totalParts` to `partindex`/`totalparts` in `section_retriever.ts`, `spec_retriever.ts`, and document creation logic. - Adjusted sorting logic to use the new `partindex` field. - Updated metadata handling and string interpolation to reference the lowercase keys. - Fixed import path for `HTMLTextSplitter` and moved the file to `setup/textsplitters/HtmlTextSplitter.ts`. - Added HTML preprocessing utilities (`addNewlinesAfterBlocks`, `convertTablesToMarkdown`) in ingestion workflow. - Integrated table-to‑markdown conversion and newline insertion to improve text splitting and document structure. --- agent_tools/section_retriever.ts | 6 +- agent_tools/spec_retriever.ts | 8 +- setup/ingest.ts | 21 ++- .../textsplitters/HtmlTextSplitter.ts | 14 +- setup/textsplitters/index.ts | 1 + setup/utils/formatHTMLForIngestion.ts | 120 ++++++++++++++++++ test/textsplitters/HTMLTextSplitter.test.ts | 42 +++--- textsplitters/index.ts | 1 - 8 files changed, 176 insertions(+), 37 deletions(-) rename textsplitters/HTMLTextSplitter.ts => setup/textsplitters/HtmlTextSplitter.ts (95%) create mode 100644 setup/textsplitters/index.ts create mode 100644 setup/utils/formatHTMLForIngestion.ts delete mode 100644 textsplitters/index.ts diff --git a/agent_tools/section_retriever.ts b/agent_tools/section_retriever.ts index f659444..5d4b336 100644 --- a/agent_tools/section_retriever.ts +++ b/agent_tools/section_retriever.ts @@ -41,10 +41,10 @@ export function createSectionRetrieverTool(table: Table) { .limit(100) .toArray(); - // Sort by partIndex to maintain order (nulls last for single-part sections) + // Sort by partindex to maintain order (nulls last for single-part sections) const sortedResults = results.sort((a: unknown, b: unknown) => { - const aIndex = (a as { partIndex?: number }).partIndex ?? Infinity; - const bIndex = (b as { partIndex?: number }).partIndex ?? Infinity; + const aIndex = (a as { partindex?: number }).partindex ?? Infinity; + const bIndex = (b as { partindex?: number }).partindex ?? Infinity; return aIndex - bIndex; }); diff --git a/agent_tools/spec_retriever.ts b/agent_tools/spec_retriever.ts index 5c74133..71de4ec 100644 --- a/agent_tools/spec_retriever.ts +++ b/agent_tools/spec_retriever.ts @@ -46,8 +46,8 @@ export function createSpecRetrieverTool( type: r.type, parentsectionid: r.parentsectionid, childrensectionids: r.childrensectionids, - partIndex: r.partIndex, - totalParts: r.totalParts, + partindex: r.partindex, + totalparts: r.totalparts, }, })); @@ -77,8 +77,8 @@ export function createSpecRetrieverTool( const sectionId = meta?.sectionid || "unknown"; const sectionTitle = meta?.sectiontitle || "unknown"; const partInfo = - meta?.partIndex !== null && meta?.partIndex !== undefined - ? ` [part ${(meta.partIndex as number) + 1}/${meta.totalParts}]` + meta?.partindex !== null && meta?.partindex !== undefined + ? ` [part ${(meta.partindex as number) + 1}/${meta.totalparts}]` : ""; return `--- Section: ${sectionId} | "${sectionTitle}"${partInfo} (score: ${r.score.toFixed(2)}) ---\n${r.document.pageContent}`; }) diff --git a/setup/ingest.ts b/setup/ingest.ts index 907ad7c..2d7ec63 100644 --- a/setup/ingest.ts +++ b/setup/ingest.ts @@ -9,7 +9,11 @@ import * as cheerio from "cheerio"; import { glob } from "glob"; import ora from "ora"; import { EMBEDDING_MODEL, SPEC_DIR, STORAGE_DIR } from "../constants"; -import { HTMLTextSplitter } from "../textsplitters"; +import { HTMLTextSplitter } from "./textsplitters"; +import { + addNewlinesAfterBlocks, + convertTablesToMarkdown, +} from "./utils/formatHTMLForIngestion"; const embeddings = new OllamaEmbeddings({ model: EMBEDDING_MODEL, @@ -157,7 +161,12 @@ async function buildSpecDocuments(): Promise { // Second pass: create documents with formatted text for (const [id, section] of sectionMap) { // Parse the stored HTML and replace direct children with placeholders - const $section = cheerio.load(`${section.html}`).root(); + const $ = cheerio.load(`${section.html}`); + + // Convert tables to markdown format for better text extraction + convertTablesToMarkdown($); + + const $section = $.root(); // Find direct children emu-clause elements only $section.children("emu-clause").each((_, childElem) => { @@ -175,6 +184,10 @@ async function buildSpecDocuments(): Promise { // (shouldn't happen with proper HTML structure, but just in case) $section.find("emu-clause").remove(); + // Add newlines after block elements to preserve document structure + // This helps the text splitter maintain paragraph/section boundaries + addNewlinesAfterBlocks($); + // Skip sections that only have h1 left (no meaningful content) const hasOnlyH1 = $section.children().length === 1 && @@ -255,8 +268,8 @@ async function buildSpecDocuments(): Promise { type: "specification", parentsectionid: section.parentId, childrensectionids: section.childrenIds, - partIndex: chunk.index, - totalParts: chunkData.length, + partindex: chunk.index, + totalparts: chunkData.length, }, }), ); diff --git a/textsplitters/HTMLTextSplitter.ts b/setup/textsplitters/HtmlTextSplitter.ts similarity index 95% rename from textsplitters/HTMLTextSplitter.ts rename to setup/textsplitters/HtmlTextSplitter.ts index 3a26469..854d0f1 100644 --- a/textsplitters/HTMLTextSplitter.ts +++ b/setup/textsplitters/HtmlTextSplitter.ts @@ -169,8 +169,8 @@ export class HTMLTextSplitter extends TextSplitter { pageContent: `${chunkHeader}${chunk}`, metadata: { ...document.metadata, - partIndex: index, - totalParts: chunks.length, + partindex: index, + totalparts: chunks.length, }, }), ); @@ -231,12 +231,18 @@ export class HTMLTextSplitter extends TextSplitter { /** * Collapses internal whitespace and trims leading/trailing whitespace. + * Preserves newlines between block elements while normalizing spaces. * * Needed so text extraction can preserve raw adjacency first and normalize * only once after boundary-aware joining. */ private normalizeWhitespace(text: string): string { - return text.replace(/\s+/g, " ").trim(); + return text + .replace(/\n[ \t]+/g, "\n") // Remove leading spaces after newlines + .replace(/[ \t]+\n/g, "\n") // Remove trailing spaces before newlines + .replace(/\n{3,}/g, "\n\n") // Collapse 3+ newlines to 2 + .replace(/[ \t]{2,}/g, " ") // Collapse multiple spaces/tabs to one + .trim(); } /** @@ -260,7 +266,7 @@ export class HTMLTextSplitter extends TextSplitter { previousNode && (this.isBlockishNode(previousNode) || this.isBlockishNode(node)) ) { - result += " "; + result += "\n"; // Add newline between block elements for better structure } result += text; diff --git a/setup/textsplitters/index.ts b/setup/textsplitters/index.ts new file mode 100644 index 0000000..d9d4edf --- /dev/null +++ b/setup/textsplitters/index.ts @@ -0,0 +1 @@ +export * from "./HtmlTextSplitter"; diff --git a/setup/utils/formatHTMLForIngestion.ts b/setup/utils/formatHTMLForIngestion.ts new file mode 100644 index 0000000..d519900 --- /dev/null +++ b/setup/utils/formatHTMLForIngestion.ts @@ -0,0 +1,120 @@ +import type * as cheerio from "cheerio"; + +/** + * List of block-level HTML elements that should have newlines added after them + * to preserve document structure during text extraction. + */ +export const BLOCK_ELEMENTS = [ + "p", + "div", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "li", + "td", + "th", + "pre", + "blockquote", + "section", + "emu-clause", + "emu-note", + "emu-example", + "emu-table", + "figure", + "figcaption", + "ul", + "ol", + "dl", + "dt", + "dd", +]; + +/** + * Converts HTML tables to markdown table format. + * This preserves table structure when extracting text from HTML. + * Replaces the original table with a 
 element containing the markdown.
+ *
+ * @param $ - Cheerio API instance
+ */
+export function convertTablesToMarkdown($: cheerio.CheerioAPI): void {
+  $("table, emu-table").each((_, tableElem) => {
+    const $table = $(tableElem);
+    const rows: string[][] = [];
+
+    // Extract header rows
+    $table.find("thead tr").each((_, rowElem) => {
+      const row: string[] = [];
+      $(rowElem)
+        .find("th, td")
+        .each((_, cellElem) => {
+          row.push($(cellElem).text().trim().replace(/\|/g, "\\|"));
+        });
+      if (row.length > 0) rows.push(row);
+    });
+
+    // Extract body rows
+    $table.find("tbody tr, tr").each((_, rowElem) => {
+      // Skip if already processed as header
+      if ($(rowElem).parent("thead").length > 0) return;
+
+      const row: string[] = [];
+      $(rowElem)
+        .find("td, th")
+        .each((_, cellElem) => {
+          row.push($(cellElem).text().trim().replace(/\|/g, "\\|"));
+        });
+      if (row.length > 0) rows.push(row);
+    });
+
+    if (rows.length === 0) return;
+
+    // Determine max columns
+    const maxCols = Math.max(...rows.map((r) => r.length));
+
+    // Build markdown table
+    const mdLines: string[] = [];
+
+    // Header row
+    if (rows.length > 0) {
+      const header = rows[0].concat(Array(maxCols - rows[0].length).fill(""));
+      mdLines.push("| " + header.join(" | ") + " |");
+    }
+
+    // Separator
+    mdLines.push("|" + Array(maxCols).fill(" --- ").join("|") + "|");
+
+    // Data rows (skip header if we have more rows)
+    const dataRows = rows.length > 1 ? rows.slice(1) : [];
+    for (const row of dataRows) {
+      const padded = row.concat(Array(maxCols - row.length).fill(""));
+      mdLines.push("| " + padded.join(" | ") + " |");
+    }
+
+    // Replace table with markdown
+    const markdown = mdLines.join("\n");
+    $table.replaceWith($(`
${markdown}
`));
+  });
+}
+
+/**
+ * Adds newlines after specified block elements to preserve document structure.
+ * This helps text splitters maintain paragraph/section boundaries.
+ *
+ * @param $ - Cheerio API instance
+ * @param elements - Array of element tag names to add newlines after
+ */
+export function addNewlinesAfterBlocks(
+  $: cheerio.CheerioAPI,
+  elements: string[] = BLOCK_ELEMENTS,
+): void {
+  for (const tag of elements) {
+    $.root()
+      .find(tag)
+      .each((_, el) => {
+        $(el).append("\n");
+      });
+  }
+}
diff --git a/test/textsplitters/HTMLTextSplitter.test.ts b/test/textsplitters/HTMLTextSplitter.test.ts
index b5e74ed..e02db97 100644
--- a/test/textsplitters/HTMLTextSplitter.test.ts
+++ b/test/textsplitters/HTMLTextSplitter.test.ts
@@ -1,6 +1,6 @@
 import { describe, expect, test } from "bun:test";
 import { Document } from "@langchain/core/documents";
-import { HTMLTextSplitter } from "../../textsplitters";
+import { HTMLTextSplitter } from "../../setup/textsplitters";
 
 describe("HTMLTextSplitter", () => {
   test("keeps small HTML in a single chunk", async () => {
@@ -13,7 +13,7 @@ describe("HTMLTextSplitter", () => {
       "
Title
Short body text
",
     );
 
-    expect(chunks).toEqual(["Title Short body text"]);
+    expect(chunks).toEqual(["Title\nShort body text"]);
   });
 
   test("does not add synthetic spaces between inline tags", async () => {
@@ -47,7 +47,7 @@ describe("HTMLTextSplitter", () => {
 
     const chunks = await splitter.splitText("
Hello
World
");
 
-    expect(chunks).toEqual(["Hello World"]);
+    expect(chunks).toEqual(["Hello\nWorld"]);
   });
 
   test("normalizes repeated whitespace in emitted text", async () => {
@@ -59,7 +59,7 @@ describe("HTMLTextSplitter", () => {
       "
  Hello\n\n   World  
\tAgain
",
     );
 
-    expect(chunks).toEqual(["Hello World Again"]);
+    expect(chunks).toEqual(["Hello\n\nWorld\nAgain"]);
   });
 
   test("treats separators as soft hints until size pressure exists", async () => {
@@ -78,7 +78,7 @@ describe("HTMLTextSplitter", () => {
       ].join(""),
     );
 
-    expect(chunks).toEqual(["Intro text", "Section Title Body text"]);
+    expect(chunks).toEqual(["Intro text", "Section Title\nBody text"]);
   });
 
   test("groups consecutive separators into later section boundaries", async () => {
@@ -91,7 +91,7 @@ describe("HTMLTextSplitter", () => {
       "
A
B
Body
",
     );
 
-    expect(chunks).toEqual(["A", "B Body"]);
+    expect(chunks).toEqual(["A", "B\nBody"]);
   });
 
   test("keeps protected content intact up to maxChunkSize", async () => {
@@ -140,7 +140,7 @@ describe("HTMLTextSplitter", () => {
       '
Title
Body text
',
     );
 
-    expect(chunks).toEqual(["Title Body text"]);
+    expect(chunks).toEqual(["Title\nBody text"]);
   });
 
   test("force-splits oversized leaf text when maxChunkSize is finite", async () => {
@@ -212,14 +212,14 @@ describe("HTMLTextSplitter", () => {
     expect(documents[0].pageContent).toBe("Intro text");
     expect(documents[0].metadata).toMatchObject({
       source: "sample.html",
-      partIndex: 0,
-      totalParts: 2,
+      partindex: 0,
+      totalparts: 2,
     });
-    expect(documents[1].pageContent).toBe("Section Title Body text");
+    expect(documents[1].pageContent).toBe("Section Title\nBody text");
     expect(documents[1].metadata).toMatchObject({
       source: "sample.html",
-      partIndex: 1,
-      totalParts: 2,
+      partindex: 1,
+      totalparts: 2,
     });
   });
 
@@ -244,7 +244,7 @@ describe("HTMLTextSplitter", () => {
 
     expect(documents).toHaveLength(2);
     expect(documents[0].pageContent).toBe("Header: Intro text");
-    expect(documents[1].pageContent).toBe("Header: Section Title Body text");
+    expect(documents[1].pageContent).toBe("Header: Section Title\nBody text");
   });
 
   test("returns no documents when source content produces no chunks", async () => {
@@ -284,23 +284,23 @@ describe("HTMLTextSplitter", () => {
     expect(documents).toHaveLength(4);
     expect(documents[0].metadata).toMatchObject({
       source: "first.html",
-      partIndex: 0,
-      totalParts: 2,
+      partindex: 0,
+      totalparts: 2,
     });
     expect(documents[1].metadata).toMatchObject({
       source: "first.html",
-      partIndex: 1,
-      totalParts: 2,
+      partindex: 1,
+      totalparts: 2,
     });
     expect(documents[2].metadata).toMatchObject({
       source: "second.html",
-      partIndex: 0,
-      totalParts: 2,
+      partindex: 0,
+      totalparts: 2,
     });
     expect(documents[3].metadata).toMatchObject({
       source: "second.html",
-      partIndex: 1,
-      totalParts: 2,
+      partindex: 1,
+      totalparts: 2,
     });
   });
 
diff --git a/textsplitters/index.ts b/textsplitters/index.ts
deleted file mode 100644
index 82e713d..0000000
--- a/textsplitters/index.ts
+++ /dev/null
@@ -1 +0,0 @@
-export * from "./HTMLTextSplitter";