bendtherules/ask262
1
0
Fork 0
mirror of https://github.com/bendtherules/ask262.git synced 2026-08-18 13:21:55 +00:00
Code Issues 1 Packages Projects Releases Wiki Activity

ingest - Consume HTMLTextSplitter

Browse Source
This commit is contained in:
bendtherules
2026-04-03 13:19:46 +05:30
parent ec5669dcd7
commit 6e5e7a7f4a
1 changed files with 66 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files
setup/ingest.ts
+66 -64
View File
@@ -5,43 +5,40 @@ import * as lancedbSdk from "@lancedb/lancedb";
import { Index } from "@lancedb/lancedb"; import { Index } from "@lancedb/lancedb";
import { Document } from "@langchain/core/documents"; import { Document } from "@langchain/core/documents";
import { OllamaEmbeddings } from "@langchain/ollama"; import { OllamaEmbeddings } from "@langchain/ollama";
import { RecursiveCharacterTextSplitter } from "@langchain/textsplitters";
import * as cheerio from "cheerio"; import * as cheerio from "cheerio";
import { glob } from "glob"; import { glob } from "glob";
import ora from "ora"; import ora from "ora";
import { EMBEDDING_MODEL, SPEC_DIR, STORAGE_DIR } from "../constants"; import { EMBEDDING_MODEL, SPEC_DIR, STORAGE_DIR } from "../constants";
import { HTMLTextSplitter } from "../textsplitters";
// TODO: Debug small content chunks
const embeddings = new OllamaEmbeddings({ const embeddings = new OllamaEmbeddings({
model: EMBEDDING_MODEL, model: EMBEDDING_MODEL,
}); });
const htmlSplitter = new RecursiveCharacterTextSplitter({ const htmlSplitter = new HTMLTextSplitter({
chunkSize: 8192, // ~2048 tokens, keeps most algorithms intact chunkSize: 8192,
chunkOverlap: 200, // Increased overlap for better continuity maxChunkSize: 12288,
chunkOverlap: 0,
separators: [ separators: [
"<emu-note", "emu-note",
"<emu-example", "emu-example",
"<emu-table", "emu-table",
"<emu-grammar", "td",
"<td", "h3",
// Finally, try to split along HTML tags "h4",
"<h3", "h5",
"<h4", "h6",
"<h5", "p",
"<h6", "div",
"<p", "br",
"<div", "li",
"<br", "ul",
"<li", "ol",
"<span",
"<ul",
"<ol",
], ],
neverBreakWithin: ["emu-grammar", "emu-alg"],
}); });
const LARGE_DOC_THRESHOLD = 9500; const LARGE_DOC_THRESHOLD = htmlSplitter.chunkSize + 100;
const BATCH_SIZE = 100; const BATCH_SIZE = 100;
async function generateEmbeddingsWithProgress( async function generateEmbeddingsWithProgress(
@@ -108,7 +105,7 @@ async function buildSpecDocuments(): Promise<Document[]> {
for (const file of htmlFiles) { for (const file of htmlFiles) {
const content = fs.readFileSync(file, "utf-8"); const content = fs.readFileSync(file, "utf-8");
const $ = cheerio.load(content); const $ = cheerio.load(`<body>${content}</body>`);
// First pass: collect all sections and build hierarchy // First pass: collect all sections and build hierarchy
const sectionMap = new Map< const sectionMap = new Map<
@@ -153,7 +150,7 @@ async function buildSpecDocuments(): Promise<Document[]> {
// Second pass: create documents with formatted text // Second pass: create documents with formatted text
for (const [id, section] of sectionMap) { for (const [id, section] of sectionMap) {
// Parse the stored HTML and replace direct children with placeholders // Parse the stored HTML and replace direct children with placeholders
const $section = cheerio.load(section.html).root(); const $section = cheerio.load(`<body>${section.html}</body>`).root();
// Find direct children emu-clause elements only // Find direct children emu-clause elements only
$section.children("emu-clause").each((_, childElem) => { $section.children("emu-clause").each((_, childElem) => {
@@ -179,50 +176,37 @@ async function buildSpecDocuments(): Promise<Document[]> {
const hasMinimalContent = textContent.length <= section.title.length + 10; // title + small buffer const hasMinimalContent = textContent.length <= section.title.length + 10; // title + small buffer
if (hasOnlyH1 || hasMinimalContent) { if (hasOnlyH1 || hasMinimalContent) {
// console.log(
// ` Skipping section ${id} - only contains heading, no substantive content`,
// );
continue; continue;
} }
// Get HTML content with inline placeholders for splitting // Get HTML content with inline placeholders for splitting
const sectionHtml = $section.html() || ""; const sectionHtml = $section.html() || "";
// Create temp document with HTML content for splitting // Split HTML into normalized text chunks.
const tempDoc = new Document({ const chunks = await htmlSplitter.splitText(sectionHtml);
pageContent: sectionHtml,
metadata: { // First pass: collect all chunk data
source: path.basename(file), // Only mark chunks as "small" if the original section content was long enough
sectionid: id, // to reasonably split (more than 100 chars). This prevents false positives
sectiontitle: section.title, // when the entire section was just naturally brief.
type: "specification", const chunkData = chunks.map((chunk, idx) => {
}, return {
index: idx,
text: chunk,
size: chunk.length,
isSmall: chunk.length < 50 && textContent.length > 100,
};
}); });
// Split the document // Print warnings for small chunks
const chunks = await htmlSplitter.splitDocuments([tempDoc]); const smallChunks = chunkData.filter((c) => c.isSmall);
if (smallChunks.length > 0) {
// Log if large section wasn't split properly const cleanedHtml = sectionHtml.replace(/\s+/g, " ").trim();
if (sectionHtml.length > LARGE_DOC_THRESHOLD && chunks.length === 1) { for (const chunk of smallChunks) {
console.warn(
` ⚠️ Section ${id} (${section.title}) is large (${sectionHtml.length} chars) but was NOT split (1 chunk)`,
);
}
for (let i = 0; i < chunks.length; i++) {
const chunk = chunks[i];
// Extract text from HTML chunk
const chunkText = cheerio.load(chunk.pageContent).text().trim();
// Warn if chunk is very small
const MIN_CHUNK_SIZE = 50;
if (chunkText.length < MIN_CHUNK_SIZE) {
console.warn( console.warn(
` ⚠️ WARNING: Chunk ${i + 1}/${chunks.length} for section ${id} is very small (${chunkText.length} chars)`, ` ⚠️ WARNING: Chunk ${chunk.index + 1}/${chunks.length} for section ${id} is very small (${chunk.size} chars)`,
); );
console.warn(` Chunk content: "${chunk.pageContent}"`); console.warn(` Chunk content: "${chunk.text}"`);
// Clean up whitespace in HTML for cleaner log output
const cleanedHtml = sectionHtml.replace(/\s+/g, " ").trim();
console.warn( console.warn(
` Original text that was split (${sectionHtml.length} chars):`, ` Original text that was split (${sectionHtml.length} chars):`,
); );
@@ -230,10 +214,28 @@ async function buildSpecDocuments(): Promise<Document[]> {
` "${cleanedHtml.slice(0, 500)}${cleanedHtml.length > 500 ? "... [truncated]" : ""}"`, ` "${cleanedHtml.slice(0, 500)}${cleanedHtml.length > 500 ? "... [truncated]" : ""}"`,
); );
} }
// Print summary after all warnings
const chunkSizes = chunkData
.map((c) => `${c.index + 1}:${c.size}`)
.join(", ");
const totalChunkSize = chunkData.reduce((sum, c) => sum + c.size, 0);
console.warn(
` All chunk sizes: [${chunkSizes}] (total: ${totalChunkSize} chars)`,
);
}
// Print warnings for large sections wasn't split properly
if (chunkData.length === 1 && chunkData[0].size > LARGE_DOC_THRESHOLD) {
console.warn(
` 🛑 Section ${id} (${section.title}) is large (${chunkData[0].size} chars) but was NOT split (1 chunk)`,
);
}
// Create documents
for (const chunk of chunkData) {
documents.push( documents.push(
new Document({ new Document({
pageContent: chunkText, pageContent: chunk.text,
metadata: { metadata: {
source: path.basename(file), source: path.basename(file),
sectionid: id, sectionid: id,
@@ -241,8 +243,8 @@ async function buildSpecDocuments(): Promise<Document[]> {
type: "specification", type: "specification",
parentsectionid: section.parentId, parentsectionid: section.parentId,
childrensectionids: section.childrenIds, childrensectionids: section.childrenIds,
partIndex: chunks.length > 1 ? i : null, partIndex: chunk.index,
totalParts: chunks.length, totalParts: chunkData.length,
}, },
}), }),
); );
@@ -264,7 +266,7 @@ async function main() {
const id = doc.metadata.sectionid || "unknown"; const id = doc.metadata.sectionid || "unknown";
const title = doc.metadata.sectiontitle || "unknown"; const title = doc.metadata.sectiontitle || "unknown";
console.warn( console.warn(
`⚠️ Warning: Document ${id} "${title}" is large (${doc.pageContent.length} chars)`, `🚨 Warning: Document ${id} "${title}" is large (${doc.pageContent.length} chars)`,
); );
} }
} }