mirror of
https://github.com/bendtherules/ask262.git
synced 2026-08-18 13:21:55 +00:00
refactor(ingest): remove part and parent reference in chunk
- Remove construction of `partRef` and parent section annotations from chunk text - Directly extract and trim text from HTML chunk using cheerio - Streamline document creation logic without additional references.
This commit is contained in:
+2
-15
@@ -194,24 +194,11 @@ async function buildSpecDocuments(): Promise<Document[]> {
|
||||
` ⚠️ Section ${id} (${section.title}) is large (${sectionHtml.length} chars) but was NOT split (1 chunk)`,
|
||||
);
|
||||
}
|
||||
|
||||
for (let i = 0; i < chunks.length; i++) {
|
||||
const chunk = chunks[i];
|
||||
// Extract text from HTML chunk
|
||||
let chunkText = cheerio.load(chunk.pageContent).text().trim();
|
||||
|
||||
// Add part reference if multiple chunks (comes first in text)
|
||||
if (chunks.length > 1) {
|
||||
const partRef = `[This is partial section of: sectiontitle "${section.title}" with sectionid: \`${id}\`. This is part ${i + 1} of ${chunks.length}. Use same sectionid to find other parts.]\n`;
|
||||
chunkText = partRef + chunkText;
|
||||
}
|
||||
|
||||
// Add parent reference at the beginning (if exists) - comes after part reference
|
||||
if (section.parentId && sectionMap.has(section.parentId)) {
|
||||
const parent = sectionMap.get(section.parentId)!;
|
||||
chunkText =
|
||||
`[Parent section available: sectiontitle "${parent.title}" at sectionid: \`${section.parentId}\`]\n` +
|
||||
chunkText;
|
||||
}
|
||||
const chunkText = cheerio.load(chunk.pageContent).text().trim();
|
||||
|
||||
documents.push(
|
||||
new Document({
|
||||
|
||||
Reference in New Issue
Block a user