From 818bccceadc0053d510310f9da5fff764226ef16 Mon Sep 17 00:00:00 2001 From: bendtherules Date: Thu, 2 Apr 2026 13:33:00 +0530 Subject: [PATCH] feat: add VS Code configuration for debugging bun - Add .vscode/extensions.json recommending oven.bun-vscode. - Add .vscode/launch.json with Bun debug configurations for various scripts. - Add .vscode/settings.json for TypeScript SDK, debug options, and file exclusions. - Remove .vscode/ entry from .gitignore. - Refactor ingest.ts: increase chunk size to 8192 and overlap to 200, raise large document threshold, remove

/

separators, skip sections containing only headings or minimal content, and add warnings for very small chunks. --- .gitignore | 1 - .vscode/extensions.json | 3 +++ .vscode/launch.json | 52 +++++++++++++++++++++++++++++++++++++++++ .vscode/settings.json | 19 +++++++++++++++ setup/ingest.ts | 41 ++++++++++++++++++++++++++++---- 5 files changed, 110 insertions(+), 6 deletions(-) create mode 100644 .vscode/extensions.json create mode 100644 .vscode/launch.json create mode 100644 .vscode/settings.json diff --git a/.gitignore b/.gitignore index 8c84582..2fb4c2f 100644 --- a/.gitignore +++ b/.gitignore @@ -19,7 +19,6 @@ yarn-error.log* # Coverage reports coverage/ # Test output -.vscode/ .idea/ *.log # Optional: package lock files if you prefer not to commit them diff --git a/.vscode/extensions.json b/.vscode/extensions.json new file mode 100644 index 0000000..2e0417c --- /dev/null +++ b/.vscode/extensions.json @@ -0,0 +1,3 @@ +{ + "recommendations": ["oven.bun-vscode"] +} diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 0000000..9b67ebb --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,52 @@ +{ + "version": "0.2.0", + "configurations": [ + { + "name": "Debug: Agent", + "type": "bun", + "request": "launch", + "program": "${workspaceFolder}/agent.ts", + "args": ["How does array DefineOwnProperty work?"], + "cwd": "${workspaceFolder}", + "stopOnEntry": false, + "watchMode": false + }, + { + "name": "Debug: Ingest", + "type": "bun", + "request": "launch", + "program": "${workspaceFolder}/setup/ingest.ts", + "cwd": "${workspaceFolder}", + "stopOnEntry": false, + "watchMode": false + }, + { + "name": "Debug: Test Spec Retriever", + "type": "bun", + "request": "launch", + "program": "${workspaceFolder}/test/manual/test-spec-retriever.ts", + "args": ["array.[[DefineOwnProperty]]"], + "cwd": "${workspaceFolder}", + "stopOnEntry": false, + "watchMode": false + }, + { + "name": "Debug: Build Graph", + "type": "bun", + "request": "launch", + "program": "${workspaceFolder}/setup/build_graph.ts", + "cwd": "${workspaceFolder}", + "stopOnEntry": false, + "watchMode": false + }, + { + "name": "Debug Current File", + "type": "bun", + "request": "launch", + "program": "${file}", + "cwd": "${workspaceFolder}", + "stopOnEntry": false, + "watchMode": false + } + ] +} diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..9d254c4 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,19 @@ +{ + "typescript.tsdk": "./node_modules/typescript/lib", + "typescript.enablePromptUseWorkspaceTsdk": true, + "debug.javascript.autoAttachFilter": "smart", + "debug.javascript.terminalOptions": { + "skipFiles": ["/**"] + }, + "search.exclude": { + "**/node_modules": true, + "**/dist": true, + "**/storage": true, + "**/spec-built": true, + "**/engine262": true + }, + "files.exclude": { + "**/node_modules": true, + "**/dist": true + } +} diff --git a/setup/ingest.ts b/setup/ingest.ts index d5b9c95..77b72c0 100644 --- a/setup/ingest.ts +++ b/setup/ingest.ts @@ -11,13 +11,15 @@ import { glob } from "glob"; import ora from "ora"; import { EMBEDDING_MODEL, SPEC_DIR, STORAGE_DIR } from "../constants"; +// TODO: Debug small content chunks + const embeddings = new OllamaEmbeddings({ model: EMBEDDING_MODEL, }); const htmlSplitter = new RecursiveCharacterTextSplitter({ - chunkSize: 4096, - chunkOverlap: 100, + chunkSize: 8192, // ~2048 tokens, keeps most algorithms intact + chunkOverlap: 200, // Increased overlap for better continuity separators: [ " { // (shouldn't happen with proper HTML structure, but just in case) $section.find("emu-clause").remove(); + // Skip sections that only have h1 left (no meaningful content) + const hasOnlyH1 = + $section.children().length === 1 && + $section.children("h1").length === 1; + const textContent = $section.text().trim(); + const hasMinimalContent = textContent.length <= section.title.length + 10; // title + small buffer + + if (hasOnlyH1 || hasMinimalContent) { + // console.log( + // ` Skipping section ${id} - only contains heading, no substantive content`, + // ); + continue; + } + // Get HTML content with inline placeholders for splitting const sectionHtml = $section.html() || ""; @@ -200,6 +214,23 @@ async function buildSpecDocuments(): Promise { // Extract text from HTML chunk const chunkText = cheerio.load(chunk.pageContent).text().trim(); + // Warn if chunk is very small + const MIN_CHUNK_SIZE = 50; + if (chunkText.length < MIN_CHUNK_SIZE) { + console.warn( + ` ⚠️ WARNING: Chunk ${i + 1}/${chunks.length} for section ${id} is very small (${chunkText.length} chars)`, + ); + console.warn(` Chunk content: "${chunk.pageContent}"`); + // Clean up whitespace in HTML for cleaner log output + const cleanedHtml = sectionHtml.replace(/\s+/g, " ").trim(); + console.warn( + ` Original text that was split (${sectionHtml.length} chars):`, + ); + console.warn( + ` "${cleanedHtml.slice(0, 500)}${cleanedHtml.length > 500 ? "... [truncated]" : ""}"`, + ); + } + documents.push( new Document({ pageContent: chunkText,