refactor(verify-db): top‑5 largest document - consider all chunks

- Replace per‑chunk sorting with aggregation of total size per section.
- Compute total characters and chunk count for each document.
- Sort sections by total size and display the top five, including part count when applicable.
This commit is contained in:
2026-04-04 12:40:11 +05:30
parent e6992a174b
commit 9bf174ddc0
+29 -8
View File
@@ -157,15 +157,36 @@ async function showSummary(table: Table) {
console.log(` Max: ${Math.max(...sizes)} chars`); console.log(` Max: ${Math.max(...sizes)} chars`);
} }
// Show top 5 largest using sorted query // Show top 5 largest documents (by total section size, not individual chunks)
const sortedBySize = [...allRecords].sort( const sectionSizes = new Map<
(a, b) => b.text.length - a.text.length, string,
); { sectionid: string; title: string; totalSize: number; chunks: number }
console.log(`\n Top 5 largest documents:`); >();
for (let i = 0; i < Math.min(5, sortedBySize.length); i++) { for (const r of allRecords) {
const r = sortedBySize[i]; const existing = sectionSizes.get(r.sectionid);
if (existing) {
existing.totalSize += r.text.length;
existing.chunks += 1;
} else {
sectionSizes.set(r.sectionid, {
sectionid: r.sectionid,
title: r.sectiontitle,
totalSize: r.text.length,
chunks: 1,
});
}
}
const sortedSections = Array.from(sectionSizes.values())
.sort((a, b) => b.totalSize - a.totalSize)
.slice(0, 5);
console.log(`\n Top 5 largest documents (by total size):`);
for (let i = 0; i < sortedSections.length; i++) {
const s = sortedSections[i];
const chunkInfo = s.chunks > 1 ? ` (${s.chunks} parts)` : "";
console.log( console.log(
` ${i + 1}. ${r.sectionid} (chunk ${r.partindex + 1}/${r.totalparts}): ${r.text.length} chars`, ` ${i + 1}. ${s.sectionid}${chunkInfo}: ${s.totalSize.toLocaleString()} chars`,
); );
} }
} }