From 85d888c7ba2418632286614f0506d0f4d092b58f Mon Sep 17 00:00:00 2001 From: bendtherules Date: Wed, 1 Apr 2026 12:49:57 +0530 Subject: [PATCH] feat(ingest): add splitter, metadata and qwen3-embedding MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit add HTML‑aware text splitter, raise batch size and large‑doc threshold, enhance progress spinner and error logging, rename ingestSpec to buildSpecDocuments and introduce hierarchical section processing. --- setup/ingest.ts | 252 +++++++++++++++++++++++++++++------------------- 1 file changed, 155 insertions(+), 97 deletions(-) diff --git a/setup/ingest.ts b/setup/ingest.ts index 5e4ed0c..df615f1 100644 --- a/setup/ingest.ts +++ b/setup/ingest.ts @@ -12,18 +12,37 @@ import ora from "ora"; import { SPEC_DIR, STORAGE_DIR } from "../constants"; const embeddings = new OllamaEmbeddings({ - model: "nomic-embed-text-v2-moe", + model: "qwen3-embedding:0.6b", }); -const textSplitter = new RecursiveCharacterTextSplitter({ +const htmlSplitter = new RecursiveCharacterTextSplitter({ chunkSize: 4096, chunkOverlap: 100, - separators: ["\n\n", "\n", ". ", " ", ""], + separators: [ + " doc.pageContent); - const batchVectors = await embeddings.embedDocuments(batchTexts); - - vectors.push(...batchVectors); + // Update spinner before processing batch const currentDoc = batch[0]; const progress = `${i + batch.length}/${total}`; - const meta = - currentDoc.metadata.sectiontitle || currentDoc.metadata.sectionid || ""; - const truncatedMeta = meta.length > 40 ? `${meta.slice(0, 37)}...` : meta; - spinner.text = `Generating embeddings (${progress}): ${truncatedMeta}`; + const sectionId = currentDoc.metadata.sectionid || "unknown"; + const contentLength = currentDoc.pageContent.length; + spinner.text = `Generating embeddings (${progress}): ${sectionId} (${contentLength} chars)`; + + const batchVectors = await embeddings.embedDocuments(batchTexts); + vectors.push(...batchVectors); } spinner.succeed(`Generated ${vectors.length} embeddings`); } catch (error) { + const failedDoc = documents[currentIndex]; + const failedSectionId = failedDoc?.metadata?.sectionid || "unknown"; + const failedSectionTitle = failedDoc?.metadata?.sectiontitle || "unknown"; + const failedContentLength = failedDoc?.pageContent?.length || 0; spinner.fail(`Failed to generate embeddings: ${error}`); + console.error( + `Debug: Failed on section ${failedSectionId} "${failedSectionTitle}" (${failedContentLength} chars)`, + ); throw error; } @@ -74,7 +102,7 @@ function askUser(question: string): Promise { }); } -async function ingestSpec(): Promise { +async function buildSpecDocuments(): Promise { const htmlFiles = await glob(path.join(SPEC_DIR, "*.html")); const documents: Document[] = []; @@ -82,102 +110,125 @@ async function ingestSpec(): Promise { const content = fs.readFileSync(file, "utf-8"); const $ = cheerio.load(content); - $("emu-clause").each((_i, elem) => { + // First pass: collect all sections and build hierarchy + const sectionMap = new Map< + string, + { + title: string; + parentId: string | null; + childrenIds: string[]; + html: string; + } + >(); + + $("#spec-container emu-clause").each((_i, elem) => { const id = $(elem).attr("id"); const title = $(elem).find("h1").first().text().trim(); - const text = $(elem) - .clone() - .children("emu-clause") - .remove() - .end() - .text() - .trim(); - if (!id || !title || !text) { + if (!id || !title) { return; } - // For large documents, break down by structural tags - if (text.length > LARGE_DOC_THRESHOLD) { - let subDocsCreated = false; - const $section = $(elem).clone(); - $section.children("emu-clause").remove(); + // Get parent section id + const parentElem = $(elem).parent("emu-clause"); + const parentId = + parentElem.length > 0 ? parentElem.attr("id") || null : null; - // Extract content from each breakdown tag type - for (const tagName of BREAKDOWN_TAGS) { - let partCounter = 1; - $section.find(tagName).each((_, subElem) => { - const subText = $(subElem).text().trim(); - const subId = `${id}-${tagName}-part-${partCounter}`; - partCounter++; + sectionMap.set(id, { + title, + parentId, + childrenIds: [], + html: $(elem).html() || "", + }); + }); - if (subText) { - documents.push( - new Document({ - pageContent: subText, - metadata: { - source: file, - sectionid: subId, - sectiontitle: `${title} [${tagName}]`, - type: "specification", - parentsectionid: id, - breakdowntag: tagName, - }, - }), - ); - subDocsCreated = true; - } - }); + // Build children relationships + for (const [id, section] of sectionMap) { + if (section.parentId && sectionMap.has(section.parentId)) { + const parent = sectionMap.get(section.parentId)!; + parent.childrenIds.push(id); + } + } + + // Second pass: create documents with formatted text + for (const [id, section] of sectionMap) { + // Parse the stored HTML and replace direct children with placeholders + const $section = cheerio.load(section.html).root(); + + // Find direct children emu-clause elements only + $section.children("emu-clause").each((_, childElem) => { + const childId = $(childElem).attr("id"); + if (childId && sectionMap.has(childId)) { + const child = sectionMap.get(childId)!; + const placeholder = `[Subsection available: sectiontitle "${child.title}" at sectionid: \`${childId}\`]`; + $(childElem).replaceWith(placeholder); + } else { + $(childElem).remove(); + } + }); + + // Remove any nested emu-clause elements that weren't direct children + // (shouldn't happen with proper HTML structure, but just in case) + $section.find("emu-clause").remove(); + + // Get HTML content with inline placeholders for splitting + const sectionHtml = $section.html() || ""; + + // Create temp document with HTML content for splitting + const tempDoc = new Document({ + pageContent: sectionHtml, + metadata: { + source: path.basename(file), + sectionid: id, + sectiontitle: section.title, + type: "specification", + }, + }); + + // Split the document + const chunks = await htmlSplitter.splitDocuments([tempDoc]); + + // Log if large section wasn't split properly + if (sectionHtml.length > LARGE_DOC_THRESHOLD && chunks.length === 1) { + console.warn( + ` ⚠️ Section ${id} (${section.title}) is large (${sectionHtml.length} chars) but was NOT split (1 chunk)`, + ); + } + for (let i = 0; i < chunks.length; i++) { + const chunk = chunks[i]; + // Extract text from HTML chunk + let chunkText = cheerio.load(chunk.pageContent).text().trim(); + + // Add part reference if multiple chunks (comes first in text) + if (chunks.length > 1) { + const partRef = `[This is partial section of: sectiontitle "${section.title}" with sectionid: \`${id}\`. This is part ${i + 1} of ${chunks.length}. Use same sectionid to find other parts.]\n`; + chunkText = partRef + chunkText; } - // Extract remaining content (text outside breakdown tags) - const $remaining = $section.clone(); - for (const tagName of BREAKDOWN_TAGS) { - $remaining.find(tagName).remove(); + // Add parent reference at the beginning (if exists) - comes after part reference + if (section.parentId && sectionMap.has(section.parentId)) { + const parent = sectionMap.get(section.parentId)!; + chunkText = + `[Parent section available: sectiontitle "${parent.title}" at sectionid: \`${section.parentId}\`]\n` + + chunkText; } - const remainingText = $remaining.text().trim(); - if (remainingText) { - documents.push( - new Document({ - pageContent: remainingText, - metadata: { - source: file, - sectionid: `${id}-prose-part-1`, - sectiontitle: `${title} [prose]`, - type: "specification", - parentsectionid: id, - breakdowntag: "prose", - }, - }), - ); - } - } else { - // Add the full section document (for smaller sections or when no breakdown happened) documents.push( new Document({ - pageContent: text, + pageContent: chunkText, metadata: { - source: file, + source: path.basename(file), sectionid: id, - sectiontitle: title, + sectiontitle: section.title, type: "specification", - parentsectionid: null, - breakdowntag: null, + parentsectionid: section.parentId, + childrensectionids: section.childrenIds, + partIndex: chunks.length > 1 ? i : null, + totalParts: chunks.length, }, }), ); } - - }); - } - - // Log warnings for any large documents in the final collection - for (const doc of documents) { - if (doc.pageContent.length > LARGE_DOC_THRESHOLD) { - const id = doc.metadata.sectionid || "unknown"; - const size = doc.pageContent.length; - console.warn(`⚠️ Warning: Final document ${id} is large (${size} chars)`); } } @@ -185,13 +236,20 @@ async function ingestSpec(): Promise { } async function main() { - console.log("Ingesting specification..."); - const specDocs = await ingestSpec(); - console.log(`Ingested ${specDocs.length} specification sections.`); + console.log("Building specification documents..."); + const specDocs = await buildSpecDocuments(); + console.log(`Built ${specDocs.length} specification documents.`); - console.log("Splitting documents into chunks..."); - const splitDocs = await textSplitter.splitDocuments(specDocs); - console.log(`Total chunks generated: ${splitDocs.length}`); + // Check for any large documents + for (const doc of specDocs) { + if (doc.pageContent.length > LARGE_DOC_THRESHOLD) { + const id = doc.metadata.sectionid || "unknown"; + const title = doc.metadata.sectiontitle || "unknown"; + console.warn( + `⚠️ Warning: Document ${id} "${title}" is large (${doc.pageContent.length} chars)`, + ); + } + } const db = await lancedbSdk.connect(STORAGE_DIR); @@ -218,11 +276,11 @@ async function main() { } console.log("Generating embeddings..."); - const vectors = await generateEmbeddingsWithProgress(splitDocs); + const vectors = await generateEmbeddingsWithProgress(specDocs); console.log("Creating table with documents..."); // Prepare data records with vector, text, and metadata - const data = splitDocs.map((doc, i) => ({ + const data = specDocs.map((doc, i) => ({ vector: vectors[i], text: doc.pageContent, ...doc.metadata,