mirror of
https://github.com/bendtherules/ask262.git
synced 2026-08-18 13:21:55 +00:00
refactor(ingest): remove part and parent reference in chunk
- Remove construction of `partRef` and parent section annotations from chunk text - Directly extract and trim text from HTML chunk using cheerio - Streamline document creation logic without additional references.
This commit is contained in:
+2
-15
@@ -194,24 +194,11 @@ async function buildSpecDocuments(): Promise<Document[]> {
|
|||||||
` ⚠️ Section ${id} (${section.title}) is large (${sectionHtml.length} chars) but was NOT split (1 chunk)`,
|
` ⚠️ Section ${id} (${section.title}) is large (${sectionHtml.length} chars) but was NOT split (1 chunk)`,
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
for (let i = 0; i < chunks.length; i++) {
|
for (let i = 0; i < chunks.length; i++) {
|
||||||
const chunk = chunks[i];
|
const chunk = chunks[i];
|
||||||
// Extract text from HTML chunk
|
// Extract text from HTML chunk
|
||||||
let chunkText = cheerio.load(chunk.pageContent).text().trim();
|
const chunkText = cheerio.load(chunk.pageContent).text().trim();
|
||||||
|
|
||||||
// Add part reference if multiple chunks (comes first in text)
|
|
||||||
if (chunks.length > 1) {
|
|
||||||
const partRef = `[This is partial section of: sectiontitle "${section.title}" with sectionid: \`${id}\`. This is part ${i + 1} of ${chunks.length}. Use same sectionid to find other parts.]\n`;
|
|
||||||
chunkText = partRef + chunkText;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Add parent reference at the beginning (if exists) - comes after part reference
|
|
||||||
if (section.parentId && sectionMap.has(section.parentId)) {
|
|
||||||
const parent = sectionMap.get(section.parentId)!;
|
|
||||||
chunkText =
|
|
||||||
`[Parent section available: sectiontitle "${parent.title}" at sectionid: \`${section.parentId}\`]\n` +
|
|
||||||
chunkText;
|
|
||||||
}
|
|
||||||
|
|
||||||
documents.push(
|
documents.push(
|
||||||
new Document({
|
new Document({
|
||||||
|
|||||||
Reference in New Issue
Block a user