mirror of
https://github.com/bendtherules/ask262.git
synced 2026-08-18 13:21:55 +00:00
refactor(ingest): restructure section document creation logic and add warnings for oversized documents
- Move full‑section Document creation into an `else` block so it only occurs when no sub‑documents were created. - Remove premature return that skipped adding the full section. - Add post‑processing loop to log a warning for any final document exceeding `LARGE_DOC_THRESHOLD`.
This commit is contained in:
+13
-7
@@ -152,13 +152,7 @@ async function ingestSpec(): Promise<Document[]> {
|
|||||||
}),
|
}),
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
} else {
|
||||||
// Skip adding the full section since we've broken it into parts
|
|
||||||
if (subDocsCreated || remainingText) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Add the full section document (for smaller sections or when no breakdown happened)
|
// Add the full section document (for smaller sections or when no breakdown happened)
|
||||||
documents.push(
|
documents.push(
|
||||||
new Document({
|
new Document({
|
||||||
@@ -173,8 +167,20 @@ async function ingestSpec(): Promise<Document[]> {
|
|||||||
},
|
},
|
||||||
}),
|
}),
|
||||||
);
|
);
|
||||||
|
}
|
||||||
|
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Log warnings for any large documents in the final collection
|
||||||
|
for (const doc of documents) {
|
||||||
|
if (doc.pageContent.length > LARGE_DOC_THRESHOLD) {
|
||||||
|
const id = doc.metadata.sectionid || "unknown";
|
||||||
|
const size = doc.pageContent.length;
|
||||||
|
console.warn(`⚠️ Warning: Final document ${id} is large (${size} chars)`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
return documents;
|
return documents;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user