bendtherules/ask262
1
0
Fork 0
mirror of https://github.com/bendtherules/ask262.git synced 2026-08-18 13:21:55 +00:00
Code Issues 1 Packages Projects Releases Wiki Activity

fix(agent-tools, setup): standardize metadata keys to lowercase and update HTML ingestion

Browse Source 
- Renamed `partIndex`/`totalParts` to `partindex`/`totalparts` in `section_retriever.ts`, `spec_retriever.ts`, and document creation logic.
- Adjusted sorting logic to use the new `partindex` field.
- Updated metadata handling and string interpolation to reference the lowercase keys.
- Fixed import path for `HTMLTextSplitter` and moved the file to `setup/textsplitters/HtmlTextSplitter.ts`.
- Added HTML preprocessing utilities (`addNewlinesAfterBlocks`, `convertTablesToMarkdown`) in ingestion workflow.
- Integrated table-to‑markdown conversion and newline insertion to improve text splitting and document structure.
This commit is contained in:
bendtherules
2026-04-04 12:33:05 +05:30
parent 822597e0e2
commit ff1c97af44
8 changed files with 176 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files
setup/ingest.ts
+17 -4
View File
@@ -9,7 +9,11 @@ import * as cheerio from "cheerio";
import { glob } from "glob";
import ora from "ora";
import { EMBEDDING_MODEL, SPEC_DIR, STORAGE_DIR } from "../constants";
import { HTMLTextSplitter } from "../textsplitters";
import { HTMLTextSplitter } from "./textsplitters";
import {
addNewlinesAfterBlocks,
convertTablesToMarkdown,
} from "./utils/formatHTMLForIngestion";
const embeddings = new OllamaEmbeddings({
model: EMBEDDING_MODEL,
@@ -157,7 +161,12 @@ async function buildSpecDocuments(): Promise<Document[]> {
// Second pass: create documents with formatted text
for (const [id, section] of sectionMap) {
// Parse the stored HTML and replace direct children with placeholders
const $section = cheerio.load(`<body>${section.html}</body>`).root();
const $ = cheerio.load(`<body>${section.html}</body>`);
// Convert tables to markdown format for better text extraction
convertTablesToMarkdown($);
const $section = $.root();
// Find direct children emu-clause elements only
$section.children("emu-clause").each((_, childElem) => {
@@ -175,6 +184,10 @@ async function buildSpecDocuments(): Promise<Document[]> {
// (shouldn't happen with proper HTML structure, but just in case)
$section.find("emu-clause").remove();
// Add newlines after block elements to preserve document structure
// This helps the text splitter maintain paragraph/section boundaries
addNewlinesAfterBlocks($);
// Skip sections that only have h1 left (no meaningful content)
const hasOnlyH1 =
$section.children().length === 1 &&
@@ -255,8 +268,8 @@ async function buildSpecDocuments(): Promise<Document[]> {
type: "specification",
parentsectionid: section.parentId,
childrensectionids: section.childrenIds,
partIndex: chunk.index,
totalParts: chunkData.length,
partindex: chunk.index,
totalparts: chunkData.length,
},
}),
);