From 3b6e9a9d8f735c77199967d6c44ff52afad2c3b7 Mon Sep 17 00:00:00 2001
From: bendtherules <abhasbhattacharya2@gmail.com>
Date: Tue, 31 Mar 2026 13:54:07 +0530
Subject: [PATCH] refactor(ingest): restructure section document creation logic
 and add warnings for oversized documents
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- Move full‑section Document creation into an `else` block so it only occurs when no sub‑documents were created.
- Remove premature return that skipped adding the full section.
- Add post‑processing loop to log a warning for any final document exceeding `LARGE_DOC_THRESHOLD`.
---
 setup/ingest.ts | 44 +++++++++++++++++++++++++-------------------
 1 file changed, 25 insertions(+), 19 deletions(-)

diff --git a/setup/ingest.ts b/setup/ingest.ts
index 672dcc8..5e4ed0c 100644
--- a/setup/ingest.ts
+++ b/setup/ingest.ts
@@ -152,29 +152,35 @@ async function ingestSpec(): Promise<Document[]> {
             }),
           );
         }
-
-        // Skip adding the full section since we've broken it into parts
-        if (subDocsCreated || remainingText) {
-          return;
-        }
+      } else {
+        // Add the full section document (for smaller sections or when no breakdown happened)
+        documents.push(
+          new Document({
+            pageContent: text,
+            metadata: {
+              source: file,
+              sectionid: id,
+              sectiontitle: title,
+              type: "specification",
+              parentsectionid: null,
+              breakdowntag: null,
+            },
+          }),
+        );
       }
 
-      // Add the full section document (for smaller sections or when no breakdown happened)
-      documents.push(
-        new Document({
-          pageContent: text,
-          metadata: {
-            source: file,
-            sectionid: id,
-            sectiontitle: title,
-            type: "specification",
-            parentsectionid: null,
-            breakdowntag: null,
-          },
-        }),
-      );
     });
   }
+
+  // Log warnings for any large documents in the final collection
+  for (const doc of documents) {
+    if (doc.pageContent.length > LARGE_DOC_THRESHOLD) {
+      const id = doc.metadata.sectionid || "unknown";
+      const size = doc.pageContent.length;
+      console.warn(`⚠️  Warning: Final document ${id} is large (${size} chars)`);
+    }
+  }
+
   return documents;
 }