diff --git a/.gitignore b/.gitignore index 8c84582..2fb4c2f 100644 --- a/.gitignore +++ b/.gitignore @@ -19,7 +19,6 @@ yarn-error.log* # Coverage reports coverage/ # Test output -.vscode/ .idea/ *.log # Optional: package lock files if you prefer not to commit them diff --git a/.vscode/extensions.json b/.vscode/extensions.json new file mode 100644 index 0000000..2e0417c --- /dev/null +++ b/.vscode/extensions.json @@ -0,0 +1,3 @@ +{ + "recommendations": ["oven.bun-vscode"] +} diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 0000000..9b67ebb --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,52 @@ +{ + "version": "0.2.0", + "configurations": [ + { + "name": "Debug: Agent", + "type": "bun", + "request": "launch", + "program": "${workspaceFolder}/agent.ts", + "args": ["How does array DefineOwnProperty work?"], + "cwd": "${workspaceFolder}", + "stopOnEntry": false, + "watchMode": false + }, + { + "name": "Debug: Ingest", + "type": "bun", + "request": "launch", + "program": "${workspaceFolder}/setup/ingest.ts", + "cwd": "${workspaceFolder}", + "stopOnEntry": false, + "watchMode": false + }, + { + "name": "Debug: Test Spec Retriever", + "type": "bun", + "request": "launch", + "program": "${workspaceFolder}/test/manual/test-spec-retriever.ts", + "args": ["array.[[DefineOwnProperty]]"], + "cwd": "${workspaceFolder}", + "stopOnEntry": false, + "watchMode": false + }, + { + "name": "Debug: Build Graph", + "type": "bun", + "request": "launch", + "program": "${workspaceFolder}/setup/build_graph.ts", + "cwd": "${workspaceFolder}", + "stopOnEntry": false, + "watchMode": false + }, + { + "name": "Debug Current File", + "type": "bun", + "request": "launch", + "program": "${file}", + "cwd": "${workspaceFolder}", + "stopOnEntry": false, + "watchMode": false + } + ] +} diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..9d254c4 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,19 @@ +{ + "typescript.tsdk": "./node_modules/typescript/lib", + "typescript.enablePromptUseWorkspaceTsdk": true, + "debug.javascript.autoAttachFilter": "smart", + "debug.javascript.terminalOptions": { + "skipFiles": ["/**"] + }, + "search.exclude": { + "**/node_modules": true, + "**/dist": true, + "**/storage": true, + "**/spec-built": true, + "**/engine262": true + }, + "files.exclude": { + "**/node_modules": true, + "**/dist": true + } +} diff --git a/setup/ingest.ts b/setup/ingest.ts index d5b9c95..77b72c0 100644 --- a/setup/ingest.ts +++ b/setup/ingest.ts @@ -11,13 +11,15 @@ import { glob } from "glob"; import ora from "ora"; import { EMBEDDING_MODEL, SPEC_DIR, STORAGE_DIR } from "../constants"; +// TODO: Debug small content chunks + const embeddings = new OllamaEmbeddings({ model: EMBEDDING_MODEL, }); const htmlSplitter = new RecursiveCharacterTextSplitter({ - chunkSize: 4096, - chunkOverlap: 100, + chunkSize: 8192, // ~2048 tokens, keeps most algorithms intact + chunkOverlap: 200, // Increased overlap for better continuity separators: [ " { // (shouldn't happen with proper HTML structure, but just in case) $section.find("emu-clause").remove(); + // Skip sections that only have h1 left (no meaningful content) + const hasOnlyH1 = + $section.children().length === 1 && + $section.children("h1").length === 1; + const textContent = $section.text().trim(); + const hasMinimalContent = textContent.length <= section.title.length + 10; // title + small buffer + + if (hasOnlyH1 || hasMinimalContent) { + // console.log( + // ` Skipping section ${id} - only contains heading, no substantive content`, + // ); + continue; + } + // Get HTML content with inline placeholders for splitting const sectionHtml = $section.html() || ""; @@ -200,6 +214,23 @@ async function buildSpecDocuments(): Promise { // Extract text from HTML chunk const chunkText = cheerio.load(chunk.pageContent).text().trim(); + // Warn if chunk is very small + const MIN_CHUNK_SIZE = 50; + if (chunkText.length < MIN_CHUNK_SIZE) { + console.warn( + ` ⚠️ WARNING: Chunk ${i + 1}/${chunks.length} for section ${id} is very small (${chunkText.length} chars)`, + ); + console.warn(` Chunk content: "${chunk.pageContent}"`); + // Clean up whitespace in HTML for cleaner log output + const cleanedHtml = sectionHtml.replace(/\s+/g, " ").trim(); + console.warn( + ` Original text that was split (${sectionHtml.length} chars):`, + ); + console.warn( + ` "${cleanedHtml.slice(0, 500)}${cleanedHtml.length > 500 ? "... [truncated]" : ""}"`, + ); + } + documents.push( new Document({ pageContent: chunkText,