refactor(ingest): restructure section document creation logic and add warnings for oversized documents

- Move full‑section Document creation into an `else` block so it only occurs when no sub‑documents were created.
- Remove premature return that skipped adding the full section.
- Add post‑processing loop to log a warning for any final document exceeding `LARGE_DOC_THRESHOLD`.
This commit is contained in:
2026-03-31 13:54:07 +05:30
parent ac518b7b77
commit 3b6e9a9d8f
+13 -7
View File
@@ -152,13 +152,7 @@ async function ingestSpec(): Promise<Document[]> {
}), }),
); );
} }
} else {
// Skip adding the full section since we've broken it into parts
if (subDocsCreated || remainingText) {
return;
}
}
// Add the full section document (for smaller sections or when no breakdown happened) // Add the full section document (for smaller sections or when no breakdown happened)
documents.push( documents.push(
new Document({ new Document({
@@ -173,8 +167,20 @@ async function ingestSpec(): Promise<Document[]> {
}, },
}), }),
); );
}
}); });
} }
// Log warnings for any large documents in the final collection
for (const doc of documents) {
if (doc.pageContent.length > LARGE_DOC_THRESHOLD) {
const id = doc.metadata.sectionid || "unknown";
const size = doc.pageContent.length;
console.warn(`⚠️ Warning: Final document ${id} is large (${size} chars)`);
}
}
return documents; return documents;
} }