feat: add VS Code configuration for debugging bun

- Add .vscode/extensions.json recommending oven.bun-vscode.
- Add .vscode/launch.json with Bun debug configurations for various scripts.
- Add .vscode/settings.json for TypeScript SDK, debug options, and file exclusions.
- Remove .vscode/ entry from .gitignore.
- Refactor ingest.ts: increase chunk size to 8192 and overlap to 200, raise large document threshold, remove <h1>/<h2> separators, skip sections containing only headings or minimal content, and add warnings for very small chunks.

.gitignore

@@ -19,7 +19,6 @@ yarn-error.log*
 # Coverage reports
 coverage/
 # Test output
-.vscode/
 .idea/
 *.log
 # Optional: package lock files if you prefer not to commit them

.vscode/extensions.json

@@ -0,0 +1,3 @@
+{
+  "recommendations": ["oven.bun-vscode"]
+}

.vscode/launch.json

@@ -0,0 +1,52 @@
+{
+  "version": "0.2.0",
+  "configurations": [
+    {
+      "name": "Debug: Agent",
+      "type": "bun",
+      "request": "launch",
+      "program": "${workspaceFolder}/agent.ts",
+      "args": ["How does array DefineOwnProperty work?"],
+      "cwd": "${workspaceFolder}",
+      "stopOnEntry": false,
+      "watchMode": false
+    },
+    {
+      "name": "Debug: Ingest",
+      "type": "bun",
+      "request": "launch",
+      "program": "${workspaceFolder}/setup/ingest.ts",
+      "cwd": "${workspaceFolder}",
+      "stopOnEntry": false,
+      "watchMode": false
+    },
+    {
+      "name": "Debug: Test Spec Retriever",
+      "type": "bun",
+      "request": "launch",
+      "program": "${workspaceFolder}/test/manual/test-spec-retriever.ts",
+      "args": ["array.[[DefineOwnProperty]]"],
+      "cwd": "${workspaceFolder}",
+      "stopOnEntry": false,
+      "watchMode": false
+    },
+    {
+      "name": "Debug: Build Graph",
+      "type": "bun",
+      "request": "launch",
+      "program": "${workspaceFolder}/setup/build_graph.ts",
+      "cwd": "${workspaceFolder}",
+      "stopOnEntry": false,
+      "watchMode": false
+    },
+    {
+      "name": "Debug Current File",
+      "type": "bun",
+      "request": "launch",
+      "program": "${file}",
+      "cwd": "${workspaceFolder}",
+      "stopOnEntry": false,
+      "watchMode": false
+    }
+  ]
+}

.vscode/settings.json

@@ -0,0 +1,19 @@
+{
+  "typescript.tsdk": "./node_modules/typescript/lib",
+  "typescript.enablePromptUseWorkspaceTsdk": true,
+  "debug.javascript.autoAttachFilter": "smart",
+  "debug.javascript.terminalOptions": {
+    "skipFiles": ["<node_internals>/**"]
+  },
+  "search.exclude": {
+    "**/node_modules": true,
+    "**/dist": true,
+    "**/storage": true,
+    "**/spec-built": true,
+    "**/engine262": true
+  },
+  "files.exclude": {
+    "**/node_modules": true,
+    "**/dist": true
+  }
+}

setup/ingest.ts

@@ -11,13 +11,15 @@ import { glob } from "glob";
 import ora from "ora";
 import { EMBEDDING_MODEL, SPEC_DIR, STORAGE_DIR } from "../constants";
 
+// TODO: Debug small content chunks
+
 const embeddings = new OllamaEmbeddings({
   model: EMBEDDING_MODEL,
 });
 
 const htmlSplitter = new RecursiveCharacterTextSplitter({
-  chunkSize: 4096,
-  chunkOverlap: 100,
+  chunkSize: 8192, // ~2048 tokens, keeps most algorithms intact
+  chunkOverlap: 200, // Increased overlap for better continuity
   separators: [
     "<emu-note",
     "<emu-example",
@@ -25,8 +27,6 @@ const htmlSplitter = new RecursiveCharacterTextSplitter({
     "<emu-grammar",
     "<td",
     // Finally, try to split along HTML tags
-    "<h1",
-    "<h2",
     "<h3",
     "<h4",
     "<h5",
@@ -41,7 +41,7 @@ const htmlSplitter = new RecursiveCharacterTextSplitter({
   ],
 });
 
-const LARGE_DOC_THRESHOLD = 5500;
+const LARGE_DOC_THRESHOLD = 9500;
 const BATCH_SIZE = 100;
 
 async function generateEmbeddingsWithProgress(
@@ -171,6 +171,20 @@ async function buildSpecDocuments(): Promise<Document[]> {
       // (shouldn't happen with proper HTML structure, but just in case)
       $section.find("emu-clause").remove();
 
+      // Skip sections that only have h1 left (no meaningful content)
+      const hasOnlyH1 =
+        $section.children().length === 1 &&
+        $section.children("h1").length === 1;
+      const textContent = $section.text().trim();
+      const hasMinimalContent = textContent.length <= section.title.length + 10; // title + small buffer
+
+      if (hasOnlyH1 || hasMinimalContent) {
+        // console.log(
+        //   `  Skipping section ${id} - only contains heading, no substantive content`,
+        // );
+        continue;
+      }
+
       // Get HTML content with inline placeholders for splitting
       const sectionHtml = $section.html() || "";
 
@@ -200,6 +214,23 @@ async function buildSpecDocuments(): Promise<Document[]> {
         // Extract text from HTML chunk
         const chunkText = cheerio.load(chunk.pageContent).text().trim();
 
+        // Warn if chunk is very small
+        const MIN_CHUNK_SIZE = 50;
+        if (chunkText.length < MIN_CHUNK_SIZE) {
+          console.warn(
+            `  ⚠️ WARNING: Chunk ${i + 1}/${chunks.length} for section ${id} is very small (${chunkText.length} chars)`,
+          );
+          console.warn(`     Chunk content: "${chunk.pageContent}"`);
+          // Clean up whitespace in HTML for cleaner log output
+          const cleanedHtml = sectionHtml.replace(/\s+/g, " ").trim();
+          console.warn(
+            `     Original text that was split (${sectionHtml.length} chars):`,
+          );
+          console.warn(
+            `     "${cleanedHtml.slice(0, 500)}${cleanedHtml.length > 500 ? "... [truncated]" : ""}"`,
+          );
+        }
+
         documents.push(
           new Document({
             pageContent: chunkText,