refactor(ingest): remove part and parent reference in chunk

- Remove construction of `partRef` and parent section annotations from chunk text
- Directly extract and trim text from HTML chunk using cheerio
- Streamline document creation logic without additional references.
This commit is contained in:
2026-04-01 18:33:57 +05:30
parent b3381508c3
commit 1ff1038898
+2 -15
View File
@@ -194,24 +194,11 @@ async function buildSpecDocuments(): Promise<Document[]> {
` ⚠️ Section ${id} (${section.title}) is large (${sectionHtml.length} chars) but was NOT split (1 chunk)`, ` ⚠️ Section ${id} (${section.title}) is large (${sectionHtml.length} chars) but was NOT split (1 chunk)`,
); );
} }
for (let i = 0; i < chunks.length; i++) { for (let i = 0; i < chunks.length; i++) {
const chunk = chunks[i]; const chunk = chunks[i];
// Extract text from HTML chunk // Extract text from HTML chunk
let chunkText = cheerio.load(chunk.pageContent).text().trim(); const chunkText = cheerio.load(chunk.pageContent).text().trim();
// Add part reference if multiple chunks (comes first in text)
if (chunks.length > 1) {
const partRef = `[This is partial section of: sectiontitle "${section.title}" with sectionid: \`${id}\`. This is part ${i + 1} of ${chunks.length}. Use same sectionid to find other parts.]\n`;
chunkText = partRef + chunkText;
}
// Add parent reference at the beginning (if exists) - comes after part reference
if (section.parentId && sectionMap.has(section.parentId)) {
const parent = sectionMap.get(section.parentId)!;
chunkText =
`[Parent section available: sectiontitle "${parent.title}" at sectionid: \`${section.parentId}\`]\n` +
chunkText;
}
documents.push( documents.push(
new Document({ new Document({