bendtherules/ask262
1
0
Fork 0
mirror of https://github.com/bendtherules/ask262.git synced 2026-08-18 13:21:55 +00:00
Code Issues 1 Packages Projects Releases Wiki Activity

fix(agent-tools, setup): standardize metadata keys to lowercase and update HTML ingestion

Browse Source 
- Renamed `partIndex`/`totalParts` to `partindex`/`totalparts` in `section_retriever.ts`, `spec_retriever.ts`, and document creation logic.
- Adjusted sorting logic to use the new `partindex` field.
- Updated metadata handling and string interpolation to reference the lowercase keys.
- Fixed import path for `HTMLTextSplitter` and moved the file to `setup/textsplitters/HtmlTextSplitter.ts`.
- Added HTML preprocessing utilities (`addNewlinesAfterBlocks`, `convertTablesToMarkdown`) in ingestion workflow.
- Integrated table-to‑markdown conversion and newline insertion to improve text splitting and document structure.
This commit is contained in:
bendtherules
2026-04-04 12:33:05 +05:30
parent 822597e0e2
commit ff1c97af44
8 changed files with 176 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files
agent_tools/section_retriever.ts
+3 -3
View File
@@ -41,10 +41,10 @@ export function createSectionRetrieverTool(table: Table) {
.limit(100) .limit(100)
.toArray(); .toArray();
// Sort by partIndex to maintain order (nulls last for single-part sections) // Sort by partindex to maintain order (nulls last for single-part sections)
const sortedResults = results.sort((a: unknown, b: unknown) => { const sortedResults = results.sort((a: unknown, b: unknown) => {
const aIndex = (a as { partIndex?: number }).partIndex ?? Infinity; const aIndex = (a as { partindex?: number }).partindex ?? Infinity;
const bIndex = (b as { partIndex?: number }).partIndex ?? Infinity; const bIndex = (b as { partindex?: number }).partindex ?? Infinity;
return aIndex - bIndex; return aIndex - bIndex;
}); });
agent_tools/spec_retriever.ts
+4 -4
View File
@@ -46,8 +46,8 @@ export function createSpecRetrieverTool(
type: r.type, type: r.type,
parentsectionid: r.parentsectionid, parentsectionid: r.parentsectionid,
childrensectionids: r.childrensectionids, childrensectionids: r.childrensectionids,
partIndex: r.partIndex, partindex: r.partindex,
totalParts: r.totalParts, totalparts: r.totalparts,
}, },
})); }));
@@ -77,8 +77,8 @@ export function createSpecRetrieverTool(
const sectionId = meta?.sectionid || "unknown"; const sectionId = meta?.sectionid || "unknown";
const sectionTitle = meta?.sectiontitle || "unknown"; const sectionTitle = meta?.sectiontitle || "unknown";
const partInfo = const partInfo =
meta?.partIndex !== null && meta?.partIndex !== undefined meta?.partindex !== null && meta?.partindex !== undefined
? ` [part ${(meta.partIndex as number) + 1}/${meta.totalParts}]` ? ` [part ${(meta.partindex as number) + 1}/${meta.totalparts}]`
: ""; : "";
return `--- Section: ${sectionId} | "${sectionTitle}"${partInfo} (score: ${r.score.toFixed(2)}) ---\n${r.document.pageContent}`; return `--- Section: ${sectionId} | "${sectionTitle}"${partInfo} (score: ${r.score.toFixed(2)}) ---\n${r.document.pageContent}`;
}) })
setup/ingest.ts
+17 -4
View File
@@ -9,7 +9,11 @@ import * as cheerio from "cheerio";
import { glob } from "glob"; import { glob } from "glob";
import ora from "ora"; import ora from "ora";
import { EMBEDDING_MODEL, SPEC_DIR, STORAGE_DIR } from "../constants"; import { EMBEDDING_MODEL, SPEC_DIR, STORAGE_DIR } from "../constants";
import { HTMLTextSplitter } from "../textsplitters"; import { HTMLTextSplitter } from "./textsplitters";
import {
addNewlinesAfterBlocks,
convertTablesToMarkdown,
} from "./utils/formatHTMLForIngestion";
const embeddings = new OllamaEmbeddings({ const embeddings = new OllamaEmbeddings({
model: EMBEDDING_MODEL, model: EMBEDDING_MODEL,
@@ -157,7 +161,12 @@ async function buildSpecDocuments(): Promise<Document[]> {
// Second pass: create documents with formatted text // Second pass: create documents with formatted text
for (const [id, section] of sectionMap) { for (const [id, section] of sectionMap) {
// Parse the stored HTML and replace direct children with placeholders // Parse the stored HTML and replace direct children with placeholders
const $section = cheerio.load(`<body>${section.html}</body>`).root(); const $ = cheerio.load(`<body>${section.html}</body>`);
// Convert tables to markdown format for better text extraction
convertTablesToMarkdown($);
const $section = $.root();
// Find direct children emu-clause elements only // Find direct children emu-clause elements only
$section.children("emu-clause").each((_, childElem) => { $section.children("emu-clause").each((_, childElem) => {
@@ -175,6 +184,10 @@ async function buildSpecDocuments(): Promise<Document[]> {
// (shouldn't happen with proper HTML structure, but just in case) // (shouldn't happen with proper HTML structure, but just in case)
$section.find("emu-clause").remove(); $section.find("emu-clause").remove();
// Add newlines after block elements to preserve document structure
// This helps the text splitter maintain paragraph/section boundaries
addNewlinesAfterBlocks($);
// Skip sections that only have h1 left (no meaningful content) // Skip sections that only have h1 left (no meaningful content)
const hasOnlyH1 = const hasOnlyH1 =
$section.children().length === 1 && $section.children().length === 1 &&
@@ -255,8 +268,8 @@ async function buildSpecDocuments(): Promise<Document[]> {
type: "specification", type: "specification",
parentsectionid: section.parentId, parentsectionid: section.parentId,
childrensectionids: section.childrenIds, childrensectionids: section.childrenIds,
partIndex: chunk.index, partindex: chunk.index,
totalParts: chunkData.length, totalparts: chunkData.length,
}, },
}), }),
); );
textsplitters/HTMLTextSplitter.ts → setup/textsplitters/HtmlTextSplitter.ts
+10 -4
View File
@@ -169,8 +169,8 @@ export class HTMLTextSplitter extends TextSplitter {
pageContent: `${chunkHeader}${chunk}`, pageContent: `${chunkHeader}${chunk}`,
metadata: { metadata: {
...document.metadata, ...document.metadata,
partIndex: index, partindex: index,
totalParts: chunks.length, totalparts: chunks.length,
}, },
}), }),
); );
@@ -231,12 +231,18 @@ export class HTMLTextSplitter extends TextSplitter {
/** /**
* Collapses internal whitespace and trims leading/trailing whitespace. * Collapses internal whitespace and trims leading/trailing whitespace.
* Preserves newlines between block elements while normalizing spaces.
* *
* Needed so text extraction can preserve raw adjacency first and normalize * Needed so text extraction can preserve raw adjacency first and normalize
* only once after boundary-aware joining. * only once after boundary-aware joining.
*/ */
private normalizeWhitespace(text: string): string { private normalizeWhitespace(text: string): string {
return text.replace(/\s+/g, " ").trim(); return text
.replace(/\n[ \t]+/g, "\n") // Remove leading spaces after newlines
.replace(/[ \t]+\n/g, "\n") // Remove trailing spaces before newlines
.replace(/\n{3,}/g, "\n\n") // Collapse 3+ newlines to 2
.replace(/[ \t]{2,}/g, " ") // Collapse multiple spaces/tabs to one
.trim();
} }
/** /**
@@ -260,7 +266,7 @@ export class HTMLTextSplitter extends TextSplitter {
previousNode && previousNode &&
(this.isBlockishNode(previousNode) || this.isBlockishNode(node)) (this.isBlockishNode(previousNode) || this.isBlockishNode(node))
) { ) {
result += " "; result += "\n"; // Add newline between block elements for better structure
} }
result += text; result += text;
setup/textsplitters/index.ts
+1
View File
@@ -0,0 +1 @@
export * from "./HtmlTextSplitter";
setup/utils/formatHTMLForIngestion.ts
+120
View File
@@ -0,0 +1,120 @@
import type * as cheerio from "cheerio";
/**
* List of block-level HTML elements that should have newlines added after them
* to preserve document structure during text extraction.
*/
export const BLOCK_ELEMENTS = [
"p",
"div",
"h1",
"h2",
"h3",
"h4",
"h5",
"h6",
"li",
"td",
"th",
"pre",
"blockquote",
"section",
"emu-clause",
"emu-note",
"emu-example",
"emu-table",
"figure",
"figcaption",
"ul",
"ol",
"dl",
"dt",
"dd",
];
/**
* Converts HTML tables to markdown table format.
* This preserves table structure when extracting text from HTML.
* Replaces the original table with a <pre> element containing the markdown.
*
* @param $ - Cheerio API instance
*/
export function convertTablesToMarkdown($: cheerio.CheerioAPI): void {
$("table, emu-table").each((_, tableElem) => {
const $table = $(tableElem);
const rows: string[][] = [];
// Extract header rows
$table.find("thead tr").each((_, rowElem) => {
const row: string[] = [];
$(rowElem)
.find("th, td")
.each((_, cellElem) => {
row.push($(cellElem).text().trim().replace(/\|/g, "\\|"));
});
if (row.length > 0) rows.push(row);
});
// Extract body rows
$table.find("tbody tr, tr").each((_, rowElem) => {
// Skip if already processed as header
if ($(rowElem).parent("thead").length > 0) return;
const row: string[] = [];
$(rowElem)
.find("td, th")
.each((_, cellElem) => {
row.push($(cellElem).text().trim().replace(/\|/g, "\\|"));
});
if (row.length > 0) rows.push(row);
});
if (rows.length === 0) return;
// Determine max columns
const maxCols = Math.max(...rows.map((r) => r.length));
// Build markdown table
const mdLines: string[] = [];
// Header row
if (rows.length > 0) {
const header = rows[0].concat(Array(maxCols - rows[0].length).fill(""));
mdLines.push("| " + header.join(" | ") + " |");
}
// Separator
mdLines.push("|" + Array(maxCols).fill(" --- ").join("|") + "|");
// Data rows (skip header if we have more rows)
const dataRows = rows.length > 1 ? rows.slice(1) : [];
for (const row of dataRows) {
const padded = row.concat(Array(maxCols - row.length).fill(""));
mdLines.push("| " + padded.join(" | ") + " |");
}
// Replace table with markdown
const markdown = mdLines.join("\n");
$table.replaceWith($(`<pre class="table-markdown">${markdown}</pre>`));
});
}
/**
* Adds newlines after specified block elements to preserve document structure.
* This helps text splitters maintain paragraph/section boundaries.
*
* @param $ - Cheerio API instance
* @param elements - Array of element tag names to add newlines after
*/
export function addNewlinesAfterBlocks(
$: cheerio.CheerioAPI,
elements: string[] = BLOCK_ELEMENTS,
): void {
for (const tag of elements) {
$.root()
.find(tag)
.each((_, el) => {
$(el).append("\n");
});
}
}
test/textsplitters/HTMLTextSplitter.test.ts
+21 -21
View File
@@ -1,6 +1,6 @@
import { describe, expect, test } from "bun:test"; import { describe, expect, test } from "bun:test";
import { Document } from "@langchain/core/documents"; import { Document } from "@langchain/core/documents";
import { HTMLTextSplitter } from "../../textsplitters"; import { HTMLTextSplitter } from "../../setup/textsplitters";
describe("HTMLTextSplitter", () => { describe("HTMLTextSplitter", () => {
test("keeps small HTML in a single chunk", async () => { test("keeps small HTML in a single chunk", async () => {
@@ -13,7 +13,7 @@ describe("HTMLTextSplitter", () => {
"<section><h2>Title</h2><p>Short body text</p></section>", "<section><h2>Title</h2><p>Short body text</p></section>",
); );
expect(chunks).toEqual(["Title Short body text"]); expect(chunks).toEqual(["Title\nShort body text"]);
}); });
test("does not add synthetic spaces between inline tags", async () => { test("does not add synthetic spaces between inline tags", async () => {
@@ -47,7 +47,7 @@ describe("HTMLTextSplitter", () => {
const chunks = await splitter.splitText("<div>Hello</div><div>World</div>"); const chunks = await splitter.splitText("<div>Hello</div><div>World</div>");
expect(chunks).toEqual(["Hello World"]); expect(chunks).toEqual(["Hello\nWorld"]);
}); });
test("normalizes repeated whitespace in emitted text", async () => { test("normalizes repeated whitespace in emitted text", async () => {
@@ -59,7 +59,7 @@ describe("HTMLTextSplitter", () => {
"<div> Hello\n\n World </div><div>\tAgain</div>", "<div> Hello\n\n World </div><div>\tAgain</div>",
); );
expect(chunks).toEqual(["Hello World Again"]); expect(chunks).toEqual(["Hello\n\nWorld\nAgain"]);
}); });
test("treats separators as soft hints until size pressure exists", async () => { test("treats separators as soft hints until size pressure exists", async () => {
@@ -78,7 +78,7 @@ describe("HTMLTextSplitter", () => {
].join(""), ].join(""),
); );
expect(chunks).toEqual(["Intro text", "Section Title Body text"]); expect(chunks).toEqual(["Intro text", "Section Title\nBody text"]);
}); });
test("groups consecutive separators into later section boundaries", async () => { test("groups consecutive separators into later section boundaries", async () => {
@@ -91,7 +91,7 @@ describe("HTMLTextSplitter", () => {
"<section><h2>A</h2><h3>B</h3><p>Body</p></section>", "<section><h2>A</h2><h3>B</h3><p>Body</p></section>",
); );
expect(chunks).toEqual(["A", "B Body"]); expect(chunks).toEqual(["A", "B\nBody"]);
}); });
test("keeps protected content intact up to maxChunkSize", async () => { test("keeps protected content intact up to maxChunkSize", async () => {
@@ -140,7 +140,7 @@ describe("HTMLTextSplitter", () => {
'<div class="keep"><h2>Title</h2><p>Body text</p></div>', '<div class="keep"><h2>Title</h2><p>Body text</p></div>',
); );
expect(chunks).toEqual(["Title Body text"]); expect(chunks).toEqual(["Title\nBody text"]);
}); });
test("force-splits oversized leaf text when maxChunkSize is finite", async () => { test("force-splits oversized leaf text when maxChunkSize is finite", async () => {
@@ -212,14 +212,14 @@ describe("HTMLTextSplitter", () => {
expect(documents[0].pageContent).toBe("Intro text"); expect(documents[0].pageContent).toBe("Intro text");
expect(documents[0].metadata).toMatchObject({ expect(documents[0].metadata).toMatchObject({
source: "sample.html", source: "sample.html",
partIndex: 0, partindex: 0,
totalParts: 2, totalparts: 2,
}); });
expect(documents[1].pageContent).toBe("Section Title Body text"); expect(documents[1].pageContent).toBe("Section Title\nBody text");
expect(documents[1].metadata).toMatchObject({ expect(documents[1].metadata).toMatchObject({
source: "sample.html", source: "sample.html",
partIndex: 1, partindex: 1,
totalParts: 2, totalparts: 2,
}); });
}); });
@@ -244,7 +244,7 @@ describe("HTMLTextSplitter", () => {
expect(documents).toHaveLength(2); expect(documents).toHaveLength(2);
expect(documents[0].pageContent).toBe("Header: Intro text"); expect(documents[0].pageContent).toBe("Header: Intro text");
expect(documents[1].pageContent).toBe("Header: Section Title Body text"); expect(documents[1].pageContent).toBe("Header: Section Title\nBody text");
}); });
test("returns no documents when source content produces no chunks", async () => { test("returns no documents when source content produces no chunks", async () => {
@@ -284,23 +284,23 @@ describe("HTMLTextSplitter", () => {
expect(documents).toHaveLength(4); expect(documents).toHaveLength(4);
expect(documents[0].metadata).toMatchObject({ expect(documents[0].metadata).toMatchObject({
source: "first.html", source: "first.html",
partIndex: 0, partindex: 0,
totalParts: 2, totalparts: 2,
}); });
expect(documents[1].metadata).toMatchObject({ expect(documents[1].metadata).toMatchObject({
source: "first.html", source: "first.html",
partIndex: 1, partindex: 1,
totalParts: 2, totalparts: 2,
}); });
expect(documents[2].metadata).toMatchObject({ expect(documents[2].metadata).toMatchObject({
source: "second.html", source: "second.html",
partIndex: 0, partindex: 0,
totalParts: 2, totalparts: 2,
}); });
expect(documents[3].metadata).toMatchObject({ expect(documents[3].metadata).toMatchObject({
source: "second.html", source: "second.html",
partIndex: 1, partindex: 1,
totalParts: 2, totalparts: 2,
}); });
}); });
textsplitters/index.ts
-1
View File
@@ -1 +0,0 @@
export * from "./HTMLTextSplitter";