From b6dbcfe91a823ead3ef6a5f2c99128e33dc0ee13 Mon Sep 17 00:00:00 2001 From: bendtherules Date: Fri, 27 Mar 2026 17:33:58 +0530 Subject: [PATCH] build: migrate to Typescript + biome --- agent.mjs => agent.ts | 64 +++++----- biome.json | 9 ++ bun.lock | 197 ++++++++++++++++++++++++++++++ constants.mjs | 2 - constants.ts | 2 + engine262/src/parser/unicode.d.ts | 30 ++--- engine262/src/syntax-error.d.ts | 4 +- package.json | 10 +- setup/build_graph.mjs | 107 ---------------- setup/build_graph.ts | 123 +++++++++++++++++++ setup/{ingest.mjs => ingest.ts} | 105 +++++++++------- tsconfig.json | 17 +++ 12 files changed, 466 insertions(+), 204 deletions(-) rename agent.mjs => agent.ts (68%) create mode 100644 biome.json create mode 100644 bun.lock delete mode 100644 constants.mjs create mode 100644 constants.ts delete mode 100644 setup/build_graph.mjs create mode 100644 setup/build_graph.ts rename setup/{ingest.mjs => ingest.ts} (51%) create mode 100644 tsconfig.json diff --git a/agent.mjs b/agent.ts similarity index 68% rename from agent.mjs rename to agent.ts index 10c06cd..081165a 100644 --- a/agent.mjs +++ b/agent.ts @@ -1,35 +1,35 @@ -import fs from 'fs'; -import { - VectorStoreIndex, - storageContextFromDefaults, +import fs from "node:fs"; +import { + VectorStoreIndex, + storageContextFromDefaults, Settings, QueryEngineTool, - ReActAgent -} from 'llamaindex'; -import { OllamaEmbedding } from '@llamaindex/ollama'; -import { OpenAI } from '@llamaindex/openai'; -import { Graph } from 'graphology'; + ReActAgent, +} from "llamaindex"; +import { OllamaEmbedding } from "@llamaindex/ollama"; +import { OpenAI } from "@llamaindex/openai"; +import { Graph } from "graphology"; -import { STORAGE_DIR, GRAPH_FILE } from './constants.mjs'; +import { STORAGE_DIR, GRAPH_FILE } from "./constants.ts"; // Configure Settings Settings.embedModel = new OllamaEmbedding({ model: "nomic-embed-text-v2-moe", }); -const config = JSON.parse(fs.readFileSync('./config.json', 'utf-8')); +const config = JSON.parse(fs.readFileSync("./config.json", "utf-8")); const apiKey = config.NVIDIA_API_KEY; const baseURL = config.NVIDIA_API_BASE; if (!apiKey) { - console.warn("Please set NVIDIA_API_KEY in config.json."); + console.warn("Please set NVIDIA_API_KEY in config.json."); } const llm = new OpenAI({ - model: "openai/gpt-oss-120b", - apiKey: apiKey, - baseURL: baseURL, - temperature: 0 + model: "openai/gpt-oss-120b", + apiKey: apiKey, + baseURL: baseURL, + temperature: 0, }); Settings.llm = llm; @@ -43,7 +43,7 @@ async function main() { storageContext, }); - const graphData = JSON.parse(fs.readFileSync(GRAPH_FILE, 'utf-8')); + const graphData = JSON.parse(fs.readFileSync(GRAPH_FILE, "utf-8")); const graph = new Graph({ multi: true }); graph.import(graphData); @@ -53,14 +53,16 @@ async function main() { queryEngine, metadata: { name: "spec_retriever", - description: "Queries the language specification for text content about specific sections or topics. Use this to get the detailed text of a section.", + description: + "Queries the language specification for text content about specific sections or topics. Use this to get the detailed text of a section.", }, }); const graphTool = { metadata: { name: "graph_explorer", - description: "Explores structural relationships between specification sections and implementation code (functions). Use this to find which spec section a function implements. Input: section ID or function name.", + description: + "Explores structural relationships between specification sections and implementation code (functions). Use this to find which spec section a function implements. Input: section ID or function name.", parameters: { type: "object", properties: { @@ -76,9 +78,9 @@ async function main() { console.log(`[Tool: graph_explorer] Querying for: ${query}`); let nodeId = query; if (!graph.hasNode(nodeId)) { - if (graph.hasNode(`func-${query}`)) { - nodeId = `func-${query}`; - } + if (graph.hasNode(`func-${query}`)) { + nodeId = `func-${query}`; + } } if (graph.hasNode(nodeId)) { @@ -88,16 +90,16 @@ async function main() { if (nodeAttr.title) result += `- Title: ${nodeAttr.title}\n`; if (nodeAttr.file) result += `- File: ${nodeAttr.file}\n`; result += `\nConnected parts:\n`; - neighbors.forEach(neighbor => { + neighbors.forEach((neighbor) => { const attr = graph.getNodeAttributes(neighbor); const edges = graph.edges(nodeId, neighbor); const edgeAttr = graph.getEdgeAttributes(edges[0]); - result += `- ${neighbor} (${attr.type}) via ${edgeAttr.type}${attr.title ? ': ' + attr.title : ''}\n`; + result += `- ${neighbor} (${attr.type}) via ${edgeAttr.type}${attr.title ? ": " + attr.title : ""}\n`; }); return result; } return `No information found in graph for ${query}. Use spec_retriever to search text.`; - } + }, }; const agent = new ReActAgent({ @@ -108,21 +110,23 @@ async function main() { Your goal is to explain how specific parts of the language work by combining information from the provided tools. CRITICAL INSTRUCTIONS: -1. ALWAYS prefer using the provided tools ('spec_retriever' and 'graph_explorer') to answer questions. -2. Do NOT rely on your internal knowledge of JavaScript or the ECMAScript specification. +1. ALWAYS prefer using the provided tools ('spec_retriever' and 'graph_explorer') to answer questions. +2. Do NOT rely on your internal knowledge of JavaScript or the ECMAScript specification. 3. If the user asks about a function, you MUST first use 'graph_explorer' to find the associated specification section. 4. You MUST then use 'spec_retriever' to read the actual text of that specification section before answering. 5. Base your explanations ONLY on the information retrieved from the tools. -6. If the tools do not provide enough information, state that clearly rather than guessing from your internal knowledge.` +6. If the tools do not provide enough information, state that clearly rather than guessing from your internal knowledge.`, }); console.log("Agent is ready!"); - const message = process.argv[2] || "Which spec section does Evaluate_IfStatement implement? and what does that section say?"; + const message = + process.argv[2] || + "Which spec section does Evaluate_IfStatement implement? and what does that section say?"; console.log(`User: ${message}`); const response = await agent.chat({ - message: message + message: message, }); console.log("\n--- Agent Response ---\n"); diff --git a/biome.json b/biome.json new file mode 100644 index 0000000..82bcdcf --- /dev/null +++ b/biome.json @@ -0,0 +1,9 @@ +{ + "formatter": { + "indentStyle": "space", + "indentWidth": 2, + "lineWidth": 80, + "lineEnding": "lf" + }, + "files": { "includes": ["**", "!!spec-built/**", "!!engine262/**"] } +} diff --git a/bun.lock b/bun.lock new file mode 100644 index 0000000..ac3e1f1 --- /dev/null +++ b/bun.lock @@ -0,0 +1,197 @@ +{ + "lockfileVersion": 1, + "configVersion": 0, + "workspaces": { + "": { + "name": "gemini-3-1-pro-preview", + "dependencies": { + "@llamaindex/core": "^0.6.22", + "@llamaindex/env": "^0.1.30", + "@llamaindex/node-parser": "^2.0.22", + "@llamaindex/ollama": "^0.1.23", + "@llamaindex/openai": "^0.4.22", + "acorn": "^8.16.0", + "cheerio": "^1.2.0", + "glob": "^13.0.6", + "graphology": "^0.26.0", + "llamaindex": "^0.12.1", + }, + "devDependencies": { + "@biomejs/biome": "^2.4.9", + }, + }, + }, + "packages": { + "@aws-crypto/sha256-js": ["@aws-crypto/sha256-js@5.2.0", "", { "dependencies": { "@aws-crypto/util": "^5.2.0", "@aws-sdk/types": "^3.222.0", "tslib": "^2.6.2" } }, "sha512-FFQQyu7edu4ufvIZ+OadFpHHOt+eSTBaYaki44c+akjg7qZg9oOQeLlk77F6tSYqjDAFClrHJk9tMf0HdVyOvA=="], + + "@aws-crypto/util": ["@aws-crypto/util@5.2.0", "", { "dependencies": { "@aws-sdk/types": "^3.222.0", "@smithy/util-utf8": "^2.0.0", "tslib": "^2.6.2" } }, "sha512-4RkU9EsI6ZpBve5fseQlGNUWKMa1RLPQ1dnjnQoe07ldfIzcsGb5hC5W0Dm7u423KWzawlrpbjXBrXCEv9zazQ=="], + + "@aws-sdk/types": ["@aws-sdk/types@3.973.6", "", { "dependencies": { "@smithy/types": "^4.13.1", "tslib": "^2.6.2" } }, "sha512-Atfcy4E++beKtwJHiDln2Nby8W/mam64opFPTiHEqgsthqeydFS1pY+OUlN1ouNOmf8ArPU/6cDS65anOP3KQw=="], + + "@biomejs/biome": ["@biomejs/biome@2.4.9", "", { "optionalDependencies": { "@biomejs/cli-darwin-arm64": "2.4.9", "@biomejs/cli-darwin-x64": "2.4.9", "@biomejs/cli-linux-arm64": "2.4.9", "@biomejs/cli-linux-arm64-musl": "2.4.9", "@biomejs/cli-linux-x64": "2.4.9", "@biomejs/cli-linux-x64-musl": "2.4.9", "@biomejs/cli-win32-arm64": "2.4.9", "@biomejs/cli-win32-x64": "2.4.9" }, "bin": { "biome": "bin/biome" } }, "sha512-wvZW92FrwitTcacvCBT8xdAbfbxWfDLwjYMmU3djjqQTh7Ni4ZdiWIT/x5VcZ+RQuxiKzIOzi5D+dcyJDFZMsA=="], + + "@biomejs/cli-darwin-arm64": ["@biomejs/cli-darwin-arm64@2.4.9", "", { "os": "darwin", "cpu": "arm64" }, "sha512-d5G8Gf2RpH5pYwiHLPA+UpG3G9TLQu4WM+VK6sfL7K68AmhcEQ9r+nkj/DvR/GYhYox6twsHUtmWWWIKfcfQQA=="], + + "@biomejs/cli-darwin-x64": ["@biomejs/cli-darwin-x64@2.4.9", "", { "os": "darwin", "cpu": "x64" }, "sha512-LNCLNgqDMG7BLdc3a8aY/dwKPK7+R8/JXJoXjCvZh2gx8KseqBdFDKbhrr7HCWF8SzNhbTaALhTBoh/I6rf9lA=="], + + "@biomejs/cli-linux-arm64": ["@biomejs/cli-linux-arm64@2.4.9", "", { "os": "linux", "cpu": "arm64" }, "sha512-4adnkAUi6K4C/emPRgYznMOcLlUqZdXWM6aIui4VP4LraE764g6Q4YguygnAUoxKjKIXIWPteKMgRbN0wsgwcg=="], + + "@biomejs/cli-linux-arm64-musl": ["@biomejs/cli-linux-arm64-musl@2.4.9", "", { "os": "linux", "cpu": "arm64" }, "sha512-8RCww5xnPn2wpK4L/QDGDOW0dq80uVWfppPxHIUg6mOs9B6gRmqPp32h1Ls3T8GnW8Wo5A8u7vpTwz4fExN+sw=="], + + "@biomejs/cli-linux-x64": ["@biomejs/cli-linux-x64@2.4.9", "", { "os": "linux", "cpu": "x64" }, "sha512-L10na7POF0Ks/cgLFNF1ZvIe+X4onLkTi5oP9hY+Rh60Q+7fWzKDDCeGyiHUFf1nGIa9dQOOUPGe2MyYg8nMSQ=="], + + "@biomejs/cli-linux-x64-musl": ["@biomejs/cli-linux-x64-musl@2.4.9", "", { "os": "linux", "cpu": "x64" }, "sha512-5TD+WS9v5vzXKzjetF0hgoaNFHMcpQeBUwKKVi3JbG1e9UCrFuUK3Gt185fyTzvRdwYkJJEMqglRPjmesmVv4A=="], + + "@biomejs/cli-win32-arm64": ["@biomejs/cli-win32-arm64@2.4.9", "", { "os": "win32", "cpu": "arm64" }, "sha512-aDZr0RBC3sMGJOU10BvG7eZIlWLK/i51HRIfScE2lVhfts2dQTreowLiJJd+UYg/tHKxS470IbzpuKmd0MiD6g=="], + + "@biomejs/cli-win32-x64": ["@biomejs/cli-win32-x64@2.4.9", "", { "os": "win32", "cpu": "x64" }, "sha512-NS4g/2G9SoQ4ktKtz31pvyc/rmgzlcIDCGU/zWbmHJAqx6gcRj2gj5Q/guXhoWTzCUaQZDIqiCQXHS7BcGYc0w=="], + + "@finom/zod-to-json-schema": ["@finom/zod-to-json-schema@3.24.11", "", { "peerDependencies": { "zod": "^4.0.14" } }, "sha512-fL656yBPiWebtfGItvtXLWrFNGlF1NcDFS0WdMQXMs9LluVg0CfT5E2oXYp0pidl0vVG53XkW55ysijNkU5/hA=="], + + "@llamaindex/core": ["@llamaindex/core@0.6.22", "", { "dependencies": { "@finom/zod-to-json-schema": "3.24.11", "@llamaindex/env": "0.1.30", "@types/node": "^24.0.13", "magic-bytes.js": "^1.10.0", "zod": "^4.1.5" } }, "sha512-/BXyemkvpxMaUhOkbwJ2PTvzKjSWkL8+6QLpz/n+pk8xBwMMe1GVBgli/J57gCyi8GbrlBafBj6GaPOgWub2Eg=="], + + "@llamaindex/env": ["@llamaindex/env@0.1.30", "", { "dependencies": { "@aws-crypto/sha256-js": "^5.2.0", "js-tiktoken": "^1.0.12", "pathe": "^1.1.2" }, "peerDependencies": { "@huggingface/transformers": "^3.5.0", "gpt-tokenizer": "^2.5.0" }, "optionalPeers": ["@huggingface/transformers", "gpt-tokenizer"] }, "sha512-y6kutMcCevzbmexUgz+HXf7KiZemzAoFEYSjAILfR+cG6FmYSF8XvLbGOB34Kx8mlRi7EI8rZXpezJ5qCqOyZg=="], + + "@llamaindex/node-parser": ["@llamaindex/node-parser@2.0.22", "", { "dependencies": { "html-to-text": "^9.0.5" }, "peerDependencies": { "@llamaindex/core": "0.6.22", "@llamaindex/env": "0.1.30", "tree-sitter": "^0.22.0", "web-tree-sitter": "^0.24.3" } }, "sha512-uj5O89WShAAyiSZ8f8tU7hnLJ6pSmlY2a6hkAOs8odkUgT87dEqaPHpsK7w0iJdEFiob7GoLeRhv2K624FooXg=="], + + "@llamaindex/ollama": ["@llamaindex/ollama@0.1.23", "", { "dependencies": { "ollama": "^0.5.16", "remeda": "^2.17.3" }, "peerDependencies": { "@llamaindex/core": "0.6.22", "@llamaindex/env": "0.1.30" } }, "sha512-Tk5TnnPhjBnedWrJA1TCdhC/aJAJscAw/FDQG89fa/7m9p53XviI14K0a0cYLYxqVqDUh9J/KI/JYmYwYjhPmg=="], + + "@llamaindex/openai": ["@llamaindex/openai@0.4.22", "", { "dependencies": { "openai": "^5.12.0" }, "peerDependencies": { "@llamaindex/core": "0.6.22", "@llamaindex/env": "0.1.30" } }, "sha512-jVaSscK7kyBM0wj3vceG2HPnFreSuOvkBYGFovFdS1heCyahlGDoQ1cLAUWqQ/J9Njfrt/PZTp3ICP+APl+zig=="], + + "@llamaindex/workflow": ["@llamaindex/workflow@1.1.24", "", { "dependencies": { "@llamaindex/workflow-core": "^1.3.2" }, "peerDependencies": { "@llamaindex/core": "0.6.22", "@llamaindex/env": "0.1.30" } }, "sha512-VyKsbRkFlnT5dRNKbgLXQV+ZpQ+CAFgmC9LaZv6hD/fIKo6wq1wQW/ZqLZgZt569xeHgxmrXPB6KHdqn/AhPbQ=="], + + "@llamaindex/workflow-core": ["@llamaindex/workflow-core@1.3.3", "", { "peerDependencies": { "@modelcontextprotocol/sdk": "^1.7.0", "hono": "^4.7.4", "next": "^15.2.2", "p-retry": "^6.2.1", "rxjs": "^7.8.2", "zod": "^3.25.0 || ^4.0.0" }, "optionalPeers": ["@modelcontextprotocol/sdk", "hono", "next", "p-retry", "rxjs"] }, "sha512-WJIcD4K2suGbNkwU5CC70jKKrA5tARba42nMs8Pou1RGzmoxqg+K+b7vyLBmiDtImR8P40YLmkayCIRVQPBmsg=="], + + "@selderee/plugin-htmlparser2": ["@selderee/plugin-htmlparser2@0.11.0", "", { "dependencies": { "domhandler": "^5.0.3", "selderee": "^0.11.0" } }, "sha512-P33hHGdldxGabLFjPPpaTxVolMrzrcegejx+0GxjrIb9Zv48D8yAIA/QTDR2dFl7Uz7urX8aX6+5bCZslr+gWQ=="], + + "@smithy/is-array-buffer": ["@smithy/is-array-buffer@2.2.0", "", { "dependencies": { "tslib": "^2.6.2" } }, "sha512-GGP3O9QFD24uGeAXYUjwSTXARoqpZykHadOmA8G5vfJPK0/DC67qa//0qvqrJzL1xc8WQWX7/yc7fwudjPHPhA=="], + + "@smithy/types": ["@smithy/types@4.13.1", "", { "dependencies": { "tslib": "^2.6.2" } }, "sha512-787F3yzE2UiJIQ+wYW1CVg2odHjmaWLGksnKQHUrK/lYZSEcy1msuLVvxaR/sI2/aDe9U+TBuLsXnr3vod1g0g=="], + + "@smithy/util-buffer-from": ["@smithy/util-buffer-from@2.2.0", "", { "dependencies": { "@smithy/is-array-buffer": "^2.2.0", "tslib": "^2.6.2" } }, "sha512-IJdWBbTcMQ6DA0gdNhh/BwrLkDR+ADW5Kr1aZmd4k3DIF6ezMV4R2NIAmT08wQJ3yUK82thHWmC/TnK/wpMMIA=="], + + "@smithy/util-utf8": ["@smithy/util-utf8@2.3.0", "", { "dependencies": { "@smithy/util-buffer-from": "^2.2.0", "tslib": "^2.6.2" } }, "sha512-R8Rdn8Hy72KKcebgLiv8jQcQkXoLMOGGv5uI1/k0l+snqkOzQ1R0ChUBCxWMlBsFMekWjq0wRudIweFs7sKT5A=="], + + "@types/lodash": ["@types/lodash@4.17.24", "", {}, "sha512-gIW7lQLZbue7lRSWEFql49QJJWThrTFFeIMJdp3eH4tKoxm1OvEPg02rm4wCCSHS0cL3/Fizimb35b7k8atwsQ=="], + + "@types/node": ["@types/node@24.12.0", "", { "dependencies": { "undici-types": "~7.16.0" } }, "sha512-GYDxsZi3ChgmckRT9HPU0WEhKLP08ev/Yfcq2AstjrDASOYCSXeyjDsHg4v5t4jOj7cyDX3vmprafKlWIG9MXQ=="], + + "acorn": ["acorn@8.16.0", "", { "bin": "bin/acorn" }, "sha512-UVJyE9MttOsBQIDKw1skb9nAwQuR5wuGD3+82K6JgJlm/Y+KI92oNsMNGZCYdDsVtRHSak0pcV5Dno5+4jh9sw=="], + + "balanced-match": ["balanced-match@4.0.4", "", {}, "sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA=="], + + "base64-js": ["base64-js@1.5.1", "", {}, "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA=="], + + "boolbase": ["boolbase@1.0.0", "", {}, "sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww=="], + + "brace-expansion": ["brace-expansion@5.0.5", "", { "dependencies": { "balanced-match": "^4.0.2" } }, "sha512-VZznLgtwhn+Mact9tfiwx64fA9erHH/MCXEUfB/0bX/6Fz6ny5EGTXYltMocqg4xFAQZtnO3DHWWXi8RiuN7cQ=="], + + "cheerio": ["cheerio@1.2.0", "", { "dependencies": { "cheerio-select": "^2.1.0", "dom-serializer": "^2.0.0", "domhandler": "^5.0.3", "domutils": "^3.2.2", "encoding-sniffer": "^0.2.1", "htmlparser2": "^10.1.0", "parse5": "^7.3.0", "parse5-htmlparser2-tree-adapter": "^7.1.0", "parse5-parser-stream": "^7.1.2", "undici": "^7.19.0", "whatwg-mimetype": "^4.0.0" } }, "sha512-WDrybc/gKFpTYQutKIK6UvfcuxijIZfMfXaYm8NMsPQxSYvf+13fXUJ4rztGGbJcBQ/GF55gvrZ0Bc0bj/mqvg=="], + + "cheerio-select": ["cheerio-select@2.1.0", "", { "dependencies": { "boolbase": "^1.0.0", "css-select": "^5.1.0", "css-what": "^6.1.0", "domelementtype": "^2.3.0", "domhandler": "^5.0.3", "domutils": "^3.0.1" } }, "sha512-9v9kG0LvzrlcungtnJtpGNxY+fzECQKhK4EGJX2vByejiMX84MFNQw4UxPJl3bFbTMw+Dfs37XaIkCwTZfLh4g=="], + + "css-select": ["css-select@5.2.2", "", { "dependencies": { "boolbase": "^1.0.0", "css-what": "^6.1.0", "domhandler": "^5.0.2", "domutils": "^3.0.1", "nth-check": "^2.0.1" } }, "sha512-TizTzUddG/xYLA3NXodFM0fSbNizXjOKhqiQQwvhlspadZokn1KDy0NZFS0wuEubIYAV5/c1/lAr0TaaFXEXzw=="], + + "css-what": ["css-what@6.2.2", "", {}, "sha512-u/O3vwbptzhMs3L1fQE82ZSLHQQfto5gyZzwteVIEyeaY5Fc7R4dapF/BvRoSYFeqfBk4m0V1Vafq5Pjv25wvA=="], + + "deepmerge": ["deepmerge@4.3.1", "", {}, "sha512-3sUqbMEc77XqpdNO7FRyRog+eW3ph+GYCbj+rK+uYyRMuwsVy0rMiVtPn+QJlKFvWP/1PYpapqYn0Me2knFn+A=="], + + "dom-serializer": ["dom-serializer@2.0.0", "", { "dependencies": { "domelementtype": "^2.3.0", "domhandler": "^5.0.2", "entities": "^4.2.0" } }, "sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg=="], + + "domelementtype": ["domelementtype@2.3.0", "", {}, "sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw=="], + + "domhandler": ["domhandler@5.0.3", "", { "dependencies": { "domelementtype": "^2.3.0" } }, "sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w=="], + + "domutils": ["domutils@3.2.2", "", { "dependencies": { "dom-serializer": "^2.0.0", "domelementtype": "^2.3.0", "domhandler": "^5.0.3" } }, "sha512-6kZKyUajlDuqlHKVX1w7gyslj9MPIXzIFiz/rGu35uC1wMi+kMhQwGhl4lt9unC9Vb9INnY9Z3/ZA3+FhASLaw=="], + + "encoding-sniffer": ["encoding-sniffer@0.2.1", "", { "dependencies": { "iconv-lite": "^0.6.3", "whatwg-encoding": "^3.1.1" } }, "sha512-5gvq20T6vfpekVtqrYQsSCFZ1wEg5+wW0/QaZMWkFr6BqD3NfKs0rLCx4rrVlSWJeZb5NBJgVLswK/w2MWU+Gw=="], + + "entities": ["entities@4.5.0", "", {}, "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw=="], + + "events": ["events@3.3.0", "", {}, "sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q=="], + + "glob": ["glob@13.0.6", "", { "dependencies": { "minimatch": "^10.2.2", "minipass": "^7.1.3", "path-scurry": "^2.0.2" } }, "sha512-Wjlyrolmm8uDpm/ogGyXZXb1Z+Ca2B8NbJwqBVg0axK9GbBeoS7yGV6vjXnYdGm6X53iehEuxxbyiKp8QmN4Vw=="], + + "graphology": ["graphology@0.26.0", "", { "dependencies": { "events": "^3.3.0" }, "peerDependencies": { "graphology-types": ">=0.24.0" } }, "sha512-8SSImzgUUYC89Z042s+0r/vMibY7GX/Emz4LDO5e7jYXhuoWfHISPFJYjpRLUSJGq6UQ6xlenvX1p/hJdfXuXg=="], + + "graphology-types": ["graphology-types@0.24.8", "", {}, "sha512-hDRKYXa8TsoZHjgEaysSRyPdT6uB78Ci8WnjgbStlQysz7xR52PInxNsmnB7IBOM1BhikxkNyCVEFgmPKnpx3Q=="], + + "html-to-text": ["html-to-text@9.0.5", "", { "dependencies": { "@selderee/plugin-htmlparser2": "^0.11.0", "deepmerge": "^4.3.1", "dom-serializer": "^2.0.0", "htmlparser2": "^8.0.2", "selderee": "^0.11.0" } }, "sha512-qY60FjREgVZL03vJU6IfMV4GDjGBIoOyvuFdpBDIX9yTlDw0TjxVBQp+P8NvpdIXNJvfWBTNul7fsAQJq2FNpg=="], + + "htmlparser2": ["htmlparser2@10.1.0", "", { "dependencies": { "domelementtype": "^2.3.0", "domhandler": "^5.0.3", "domutils": "^3.2.2", "entities": "^7.0.1" } }, "sha512-VTZkM9GWRAtEpveh7MSF6SjjrpNVNNVJfFup7xTY3UpFtm67foy9HDVXneLtFVt4pMz5kZtgNcvCniNFb1hlEQ=="], + + "iconv-lite": ["iconv-lite@0.6.3", "", { "dependencies": { "safer-buffer": ">= 2.1.2 < 3.0.0" } }, "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw=="], + + "js-tiktoken": ["js-tiktoken@1.0.21", "", { "dependencies": { "base64-js": "^1.5.1" } }, "sha512-biOj/6M5qdgx5TKjDnFT1ymSpM5tbd3ylwDtrQvFQSu0Z7bBYko2dF+W/aUkXUPuk6IVpRxk/3Q2sHOzGlS36g=="], + + "leac": ["leac@0.6.0", "", {}, "sha512-y+SqErxb8h7nE/fiEX07jsbuhrpO9lL8eca7/Y1nuWV2moNlXhyd59iDGcRf6moVyDMbmTNzL40SUyrFU/yDpg=="], + + "llamaindex": ["llamaindex@0.12.1", "", { "dependencies": { "@llamaindex/core": "0.6.22", "@llamaindex/env": "0.1.30", "@llamaindex/node-parser": "2.0.22", "@llamaindex/workflow": "1.1.24", "@types/lodash": "^4.17.7", "@types/node": "^24.0.13", "lodash": "^4.17.21", "magic-bytes.js": "^1.10.0" } }, "sha512-/tXXITk/iVGBycOFaDhev6dgTBIr6Ycu4FoPIt6A5JcEAiB6ujONjiV36flVXUR8JdqwMtS767XMjV+36nV4yQ=="], + + "lodash": ["lodash@4.17.23", "", {}, "sha512-LgVTMpQtIopCi79SJeDiP0TfWi5CNEc/L/aRdTh3yIvmZXTnheWpKjSZhnvMl8iXbC1tFg9gdHHDMLoV7CnG+w=="], + + "lru-cache": ["lru-cache@11.2.7", "", {}, "sha512-aY/R+aEsRelme17KGQa/1ZSIpLpNYYrhcrepKTZgE+W3WM16YMCaPwOHLHsmopZHELU0Ojin1lPVxKR0MihncA=="], + + "magic-bytes.js": ["magic-bytes.js@1.13.0", "", {}, "sha512-afO2mnxW7GDTXMm5/AoN1WuOcdoKhtgXjIvHmobqTD1grNplhGdv3PFOyjCVmrnOZBIT/gD/koDKpYG+0mvHcg=="], + + "minimatch": ["minimatch@10.2.4", "", { "dependencies": { "brace-expansion": "^5.0.2" } }, "sha512-oRjTw/97aTBN0RHbYCdtF1MQfvusSIBQM0IZEgzl6426+8jSC0nF1a/GmnVLpfB9yyr6g6FTqWqiZVbxrtaCIg=="], + + "minipass": ["minipass@7.1.3", "", {}, "sha512-tEBHqDnIoM/1rXME1zgka9g6Q2lcoCkxHLuc7ODJ5BxbP5d4c2Z5cGgtXAku59200Cx7diuHTOYfSBD8n6mm8A=="], + + "node-addon-api": ["node-addon-api@8.7.0", "", {}, "sha512-9MdFxmkKaOYVTV+XVRG8ArDwwQ77XIgIPyKASB1k3JPq3M8fGQQQE3YpMOrKm6g//Ktx8ivZr8xo1Qmtqub+GA=="], + + "node-gyp-build": ["node-gyp-build@4.8.4", "", { "bin": { "node-gyp-build": "bin.js", "node-gyp-build-optional": "optional.js", "node-gyp-build-test": "build-test.js" } }, "sha512-LA4ZjwlnUblHVgq0oBF3Jl/6h/Nvs5fzBLwdEF4nuxnFdsfajde4WfxtJr3CaiH+F6ewcIB/q4jQ4UzPyid+CQ=="], + + "nth-check": ["nth-check@2.1.1", "", { "dependencies": { "boolbase": "^1.0.0" } }, "sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w=="], + + "ollama": ["ollama@0.5.18", "", { "dependencies": { "whatwg-fetch": "^3.6.20" } }, "sha512-lTFqTf9bo7Cd3hpF6CviBe/DEhewjoZYd9N/uCe7O20qYTvGqrNOFOBDj3lbZgFWHUgDv5EeyusYxsZSLS8nvg=="], + + "openai": ["openai@5.23.2", "", { "peerDependencies": { "ws": "^8.18.0", "zod": "^3.23.8" }, "optionalPeers": ["ws"], "bin": "bin/cli" }, "sha512-MQBzmTulj+MM5O8SKEk/gL8a7s5mktS9zUtAkU257WjvobGc9nKcBuVwjyEEcb9SI8a8Y2G/mzn3vm9n1Jlleg=="], + + "parse5": ["parse5@7.3.0", "", { "dependencies": { "entities": "^6.0.0" } }, "sha512-IInvU7fabl34qmi9gY8XOVxhYyMyuH2xUNpb2q8/Y+7552KlejkRvqvD19nMoUW/uQGGbqNpA6Tufu5FL5BZgw=="], + + "parse5-htmlparser2-tree-adapter": ["parse5-htmlparser2-tree-adapter@7.1.0", "", { "dependencies": { "domhandler": "^5.0.3", "parse5": "^7.0.0" } }, "sha512-ruw5xyKs6lrpo9x9rCZqZZnIUntICjQAd0Wsmp396Ul9lN/h+ifgVV1x1gZHi8euej6wTfpqX8j+BFQxF0NS/g=="], + + "parse5-parser-stream": ["parse5-parser-stream@7.1.2", "", { "dependencies": { "parse5": "^7.0.0" } }, "sha512-JyeQc9iwFLn5TbvvqACIF/VXG6abODeB3Fwmv/TGdLk2LfbWkaySGY72at4+Ty7EkPZj854u4CrICqNk2qIbow=="], + + "parseley": ["parseley@0.12.1", "", { "dependencies": { "leac": "^0.6.0", "peberminta": "^0.9.0" } }, "sha512-e6qHKe3a9HWr0oMRVDTRhKce+bRO8VGQR3NyVwcjwrbhMmFCX9KszEV35+rn4AdilFAq9VPxP/Fe1wC9Qjd2lw=="], + + "path-scurry": ["path-scurry@2.0.2", "", { "dependencies": { "lru-cache": "^11.0.0", "minipass": "^7.1.2" } }, "sha512-3O/iVVsJAPsOnpwWIeD+d6z/7PmqApyQePUtCndjatj/9I5LylHvt5qluFaBT3I5h3r1ejfR056c+FCv+NnNXg=="], + + "pathe": ["pathe@1.1.2", "", {}, "sha512-whLdWMYL2TwI08hn8/ZqAbrVemu0LNaNNJZX73O6qaIdCTfXutsLhMkjdENX0qhsQ9uIimo4/aQOmXkoon2nDQ=="], + + "peberminta": ["peberminta@0.9.0", "", {}, "sha512-XIxfHpEuSJbITd1H3EeQwpcZbTLHc+VVr8ANI9t5sit565tsI4/xK3KWTUFE2e6QiangUkh3B0jihzmGnNrRsQ=="], + + "remeda": ["remeda@2.33.6", "", {}, "sha512-tazDGH7s75kUPGBKLvhgBEHMgW+TdDFhjUAMdQj57IoWz6HsGa5D2RX5yDUz6IIqiRRvZiaEHzCzWdTeixc/Kg=="], + + "safer-buffer": ["safer-buffer@2.1.2", "", {}, "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg=="], + + "selderee": ["selderee@0.11.0", "", { "dependencies": { "parseley": "^0.12.0" } }, "sha512-5TF+l7p4+OsnP8BCCvSyZiSPc4x4//p5uPwK8TCnVPJYRmU2aYKMpOXvw8zM5a5JvuuCGN1jmsMwuU2W02ukfA=="], + + "tree-sitter": ["tree-sitter@0.22.4", "", { "dependencies": { "node-addon-api": "^8.3.0", "node-gyp-build": "^4.8.4" } }, "sha512-usbHZP9/oxNsUY65MQUsduGRqDHQOou1cagUSwjhoSYAmSahjQDAVsh9s+SlZkn8X8+O1FULRGwHu7AFP3kjzg=="], + + "tslib": ["tslib@2.8.1", "", {}, "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w=="], + + "undici": ["undici@7.24.6", "", {}, "sha512-Xi4agocCbRzt0yYMZGMA6ApD7gvtUFaxm4ZmeacWI4cZxaF6C+8I8QfofC20NAePiB/IcvZmzkJ7XPa471AEtA=="], + + "undici-types": ["undici-types@7.16.0", "", {}, "sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw=="], + + "web-tree-sitter": ["web-tree-sitter@0.24.7", "", {}, "sha512-CdC/TqVFbXqR+C51v38hv6wOPatKEUGxa39scAeFSm98wIhZxAYonhRQPSMmfZ2w7JDI0zQDdzdmgtNk06/krQ=="], + + "whatwg-encoding": ["whatwg-encoding@3.1.1", "", { "dependencies": { "iconv-lite": "0.6.3" } }, "sha512-6qN4hJdMwfYBtE3YBTTHhoeuUrDBPZmbQaxWAqSALV/MeEnR5z1xd8UKud2RAkFoPkmB+hli1TZSnyi84xz1vQ=="], + + "whatwg-fetch": ["whatwg-fetch@3.6.20", "", {}, "sha512-EqhiFU6daOA8kpjOWTL0olhVOF3i7OrFzSYiGsEMB8GcXS+RrzauAERX65xMeNWVqxA6HXH2m69Z9LaKKdisfg=="], + + "whatwg-mimetype": ["whatwg-mimetype@4.0.0", "", {}, "sha512-QaKxh0eNIi2mE9p2vEdzfagOKHCcj1pJ56EEHGQOVxp8r9/iszLUUV7v89x9O1p/T+NlTM5W7jW6+cz4Fq1YVg=="], + + "zod": ["zod@4.3.6", "", {}, "sha512-rftlrkhHZOcjDwkGlnUtZZkvaPHCsDATp4pGpuOOMDaTdDDXF91wuVDJoWoPsKX/3YPQ5fHuF3STjcYyKr+Qhg=="], + + "html-to-text/htmlparser2": ["htmlparser2@8.0.2", "", { "dependencies": { "domelementtype": "^2.3.0", "domhandler": "^5.0.3", "domutils": "^3.0.1", "entities": "^4.4.0" } }, "sha512-GYdjWKDkbRLkZ5geuHs5NY1puJ+PXwP7+fHPRz06Eirsb9ugf6d8kkXav6ADhcODhFFPMIXyxkxSuMf3D6NCFA=="], + + "htmlparser2/entities": ["entities@7.0.1", "", {}, "sha512-TWrgLOFUQTH994YUyl1yT4uyavY5nNB5muff+RtWaqNVCAK408b5ZnnbNAUEWLTCpum9w6arT70i1XdQ4UeOPA=="], + + "openai/zod": ["zod@3.25.76", "", {}, "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ=="], + + "parse5/entities": ["entities@6.0.1", "", {}, "sha512-aN97NXWF6AWBTahfVOIrB/NShkzi5H7F9r1s9mD3cDj4Ko5f2qhhVoYMibXF7GlLveb/D2ioWay8lxI97Ven3g=="], + } +} diff --git a/constants.mjs b/constants.mjs deleted file mode 100644 index e29e204..0000000 --- a/constants.mjs +++ /dev/null @@ -1,2 +0,0 @@ -export const STORAGE_DIR = './storage'; -export const GRAPH_FILE = './graphology/graph.json'; \ No newline at end of file diff --git a/constants.ts b/constants.ts new file mode 100644 index 0000000..e4bde96 --- /dev/null +++ b/constants.ts @@ -0,0 +1,2 @@ +export const STORAGE_DIR = "./storage"; +export const GRAPH_FILE = "./graphology/graph.json"; diff --git a/engine262/src/parser/unicode.d.ts b/engine262/src/parser/unicode.d.ts index 7469fe2..9caaa7a 100644 --- a/engine262/src/parser/unicode.d.ts +++ b/engine262/src/parser/unicode.d.ts @@ -1,20 +1,20 @@ -declare module '@unicode/unicode-16.0.0/Binary_Property/ID_Start/regex.js' { - let regex: RegExp; - export default regex; +declare module "@unicode/unicode-16.0.0/Binary_Property/ID_Start/regex.js" { + let regex: RegExp; + export default regex; } -declare module '@unicode/unicode-16.0.0/Binary_Property/ID_Continue/regex.js' { - let regex: RegExp; - export default regex; +declare module "@unicode/unicode-16.0.0/Binary_Property/ID_Continue/regex.js" { + let regex: RegExp; + export default regex; } -declare module '@unicode/unicode-16.0.0/General_Category/Space_Separator/regex.js' { - let regex: RegExp; - export default regex; +declare module "@unicode/unicode-16.0.0/General_Category/Space_Separator/regex.js" { + let regex: RegExp; + export default regex; } -declare module '@unicode/unicode-16.0.0/Case_Folding/C/symbols.js' { - let data: Map; - export default data; +declare module "@unicode/unicode-16.0.0/Case_Folding/C/symbols.js" { + let data: Map; + export default data; } -declare module '@unicode/unicode-16.0.0/Case_Folding/S/symbols.js' { - let data: Map; - export default data; +declare module "@unicode/unicode-16.0.0/Case_Folding/S/symbols.js" { + let data: Map; + export default data; } diff --git a/engine262/src/syntax-error.d.ts b/engine262/src/syntax-error.d.ts index 3e9138d..4b9242a 100644 --- a/engine262/src/syntax-error.d.ts +++ b/engine262/src/syntax-error.d.ts @@ -1,5 +1,5 @@ // eslint-disable-next-line no-unused-vars interface SyntaxError { - decoration?: string - position?: number + decoration?: string; + position?: number; } diff --git a/package.json b/package.json index c613148..e1c9b80 100644 --- a/package.json +++ b/package.json @@ -3,6 +3,11 @@ "version": "1.0.0", "main": "index.js", "scripts": { + "lint": "biome check .", + "format": "biome format .", + "ingest": "bun run setup/ingest.ts", + "agent": "bun run agent.ts", + "build": "bun run setup/build_graph.ts", "test": "echo \"Error: no test specified\" && exit 1" }, "keywords": [], @@ -21,5 +26,8 @@ "glob": "^13.0.6", "graphology": "^0.26.0", "llamaindex": "^0.12.1" + }, + "devDependencies": { + "@biomejs/biome": "^2.4.9" } -} +} \ No newline at end of file diff --git a/setup/build_graph.mjs b/setup/build_graph.mjs deleted file mode 100644 index 0f382aa..0000000 --- a/setup/build_graph.mjs +++ /dev/null @@ -1,107 +0,0 @@ -import fs from 'fs'; -import path from 'path'; -import { glob } from 'glob'; -import * as cheerio from 'cheerio'; -import { Graph } from 'graphology'; - -const SPEC_DIR = './spec-built/multipage'; -const CODE_DIR = './engine262/src'; -import { GRAPH_FILE } from '../constants.mjs'; - -async function buildGraph() { - const graph = new Graph({ multi: true }); - const htmlFiles = await glob(path.join(SPEC_DIR, '*.html')); - console.log(`Found ${htmlFiles.length} specification HTML file(s) in ${SPEC_DIR}`); - if (htmlFiles.length === 0) { - console.warn(`Warning: No specification HTML files found in ${SPEC_DIR}`); - } - - console.log("Processing specification for graph..."); - for (const file of htmlFiles) { - const fileName = path.basename(file); - const content = fs.readFileSync(file, 'utf-8'); - const $ = cheerio.load(content); - - $('emu-clause').each((i, elem) => { - const id = $(elem).attr('id'); - const title = $(elem).find('h1').first().text().trim(); - - if (id) { - if (!graph.hasNode(id)) { - graph.addNode(id, { title, type: 'SpecSection', file: fileName }); - } - - // Extract internal links - $(elem).find('a[href]').each((j, link) => { - const href = $(link).attr('href'); - if (href && href.includes('#')) { - const [targetFile, targetId] = href.split('#'); - // We only care about links to other sections for now - if (targetId && targetId.startsWith('sec-')) { - // Add relationship later if nodes exist - } - } - }); - } - }); - } - - // Add edges for internal links - for (const file of htmlFiles) { - const content = fs.readFileSync(file, 'utf-8'); - const $ = cheerio.load(content); - $('emu-clause').each((i, elem) => { - const sourceId = $(elem).attr('id'); - if (sourceId && graph.hasNode(sourceId)) { - $(elem).find('a[href]').each((j, link) => { - const href = $(link).attr('href'); - if (href && href.includes('#')) { - const [, targetId] = href.split('#'); - if (targetId && graph.hasNode(targetId) && sourceId !== targetId) { - if (!graph.hasEdge(sourceId, targetId)) { - graph.addEdge(sourceId, targetId, { type: 'LINKS_TO' }); - } - } - } - }); - } - }); - } - - console.log("Processing code for graph..."); - const jsFiles = await glob(path.join(CODE_DIR, '**/*.mts')); - console.log(`Found ${jsFiles.length} code file(s) in ${CODE_DIR}`); - if (jsFiles.length === 0) { - console.warn(`Warning: No code files found in ${CODE_DIR}`); - } - - for (const file of jsFiles) { - const fileName = path.basename(file); - const content = fs.readFileSync(file, 'utf-8'); - - // Simple heuristic: look for function names starting with Evaluate_ - const functionMatches = content.matchAll(/export function\*? (Evaluate_([a-zA-Z0-9_]+))/g); - for (const match of functionMatches) { - const fullFuncName = match[1]; - const shortName = match[2]; - const funcNodeId = `func-${fullFuncName}`; - - if (!graph.hasNode(funcNodeId)) { - graph.addNode(funcNodeId, { name: fullFuncName, type: 'JSFunction', file: fileName }); - } - - // Try to link to a spec section - // Heuristic: Evaluate_IfStatement -> sec-if-statement - const specId = `sec-${shortName.replace(/([a-z0-9])([A-Z])/g, '$1-$2').toLowerCase()}`; - if (graph.hasNode(specId)) { - graph.addEdge(funcNodeId, specId, { type: 'IMPLEMENTS' }); - } - } - } - - console.log(`Graph built with ${graph.order} nodes and ${graph.size} edges.`); - fs.writeFileSync(GRAPH_FILE, JSON.stringify(graph.export(), null, 2)); - console.log(`Graph saved to ${GRAPH_FILE}`); -} - -buildGraph().catch(console.error); diff --git a/setup/build_graph.ts b/setup/build_graph.ts new file mode 100644 index 0000000..817e0a6 --- /dev/null +++ b/setup/build_graph.ts @@ -0,0 +1,123 @@ +import fs from "fs"; +import path from "path"; +import { glob } from "glob"; +import * as cheerio from "cheerio"; +import { Graph } from "graphology"; + +const SPEC_DIR = "./spec-built/multipage"; +const CODE_DIR = "./engine262/src"; +import { GRAPH_FILE } from "../constants.ts"; + +async function buildGraph() { + const graph = new Graph({ multi: true }); + const htmlFiles = await glob(path.join(SPEC_DIR, "*.html")); + console.log( + `Found ${htmlFiles.length} specification HTML file(s) in ${SPEC_DIR}`, + ); + if (htmlFiles.length === 0) { + console.warn(`Warning: No specification HTML files found in ${SPEC_DIR}`); + } + + console.log("Processing specification for graph..."); + for (const file of htmlFiles) { + const fileName = path.basename(file); + const content = fs.readFileSync(file, "utf-8"); + const $ = cheerio.load(content); + + $("emu-clause").each((i, elem) => { + const id = $(elem).attr("id"); + const title = $(elem).find("h1").first().text().trim(); + + if (id) { + if (!graph.hasNode(id)) { + graph.addNode(id, { title, type: "SpecSection", file: fileName }); + } + + // Extract internal links + $(elem) + .find("a[href]") + .each((j, link) => { + const href = $(link).attr("href"); + if (href && href.includes("#")) { + const [targetFile, targetId] = href.split("#"); + // We only care about links to other sections for now + if (targetId && targetId.startsWith("sec-")) { + // Add relationship later if nodes exist + } + } + }); + } + }); + } + + // Add edges for internal links + for (const file of htmlFiles) { + const content = fs.readFileSync(file, "utf-8"); + const $ = cheerio.load(content); + $("emu-clause").each((i, elem) => { + const sourceId = $(elem).attr("id"); + if (sourceId && graph.hasNode(sourceId)) { + $(elem) + .find("a[href]") + .each((j, link) => { + const href = $(link).attr("href"); + if (href && href.includes("#")) { + const [, targetId] = href.split("#"); + if ( + targetId && + graph.hasNode(targetId) && + sourceId !== targetId + ) { + if (!graph.hasEdge(sourceId, targetId)) { + graph.addEdge(sourceId, targetId, { type: "LINKS_TO" }); + } + } + } + }); + } + }); + } + + console.log("Processing code for graph..."); + const jsFiles = await glob(path.join(CODE_DIR, "**/*.mts")); + console.log(`Found ${jsFiles.length} code file(s) in ${CODE_DIR}`); + if (jsFiles.length === 0) { + console.warn(`Warning: No code files found in ${CODE_DIR}`); + } + + for (const file of jsFiles) { + const fileName = path.basename(file); + const content = fs.readFileSync(file, "utf-8"); + + // Simple heuristic: look for function names starting with Evaluate_ + const functionMatches = content.matchAll( + /export function\*? (Evaluate_([a-zA-Z0-9_]+))/g, + ); + for (const match of functionMatches) { + const fullFuncName = match[1]; + const shortName = match[2]; + const funcNodeId = `func-${fullFuncName}`; + + if (!graph.hasNode(funcNodeId)) { + graph.addNode(funcNodeId, { + name: fullFuncName, + type: "JSFunction", + file: fileName, + }); + } + + // Try to link to a spec section + // Heuristic: Evaluate_IfStatement -> sec-if-statement + const specId = `sec-${shortName.replace(/([a-z0-9])([A-Z])/g, "$1-$2").toLowerCase()}`; + if (graph.hasNode(specId)) { + graph.addEdge(funcNodeId, specId, { type: "IMPLEMENTS" }); + } + } + } + + console.log(`Graph built with ${graph.order} nodes and ${graph.size} edges.`); + fs.writeFileSync(GRAPH_FILE, JSON.stringify(graph.export(), null, 2)); + console.log(`Graph saved to ${GRAPH_FILE}`); +} + +buildGraph().catch(console.error); diff --git a/setup/ingest.mjs b/setup/ingest.ts similarity index 51% rename from setup/ingest.mjs rename to setup/ingest.ts index 2c2eb3f..c87dc36 100644 --- a/setup/ingest.mjs +++ b/setup/ingest.ts @@ -1,52 +1,63 @@ -import fs from 'fs'; -import path from 'path'; -import { glob } from 'glob'; -import * as cheerio from 'cheerio'; -import { - Document, - VectorStoreIndex, - Settings, - storageContextFromDefaults, - SentenceSplitter -} from 'llamaindex'; -import { OllamaEmbedding } from '@llamaindex/ollama'; +import fs from "fs"; +import path from "path"; +import { glob } from "glob"; +import * as cheerio from "cheerio"; +import { + Document, + VectorStoreIndex, + Settings, + storageContextFromDefaults, + SentenceSplitter, +} from "llamaindex"; +import { OllamaEmbedding } from "@llamaindex/ollama"; // Configure Settings Settings.embedModel = new OllamaEmbedding({ model: "nomic-embed-text-v2-moe", }); -const SPEC_DIR = './spec-built/multipage'; -const CODE_DIR = './engine262/src'; -import { STORAGE_DIR } from '../constants.mjs'; +const SPEC_DIR = "./spec-built/multipage"; +const CODE_DIR = "./engine262/src"; +import { STORAGE_DIR } from "../constants.ts"; // Initialize a SentenceSplitter with even smaller chunk size -const sentenceSplitter = new SentenceSplitter({ chunkSize: 256, chunkOverlap: 20 }); +const sentenceSplitter = new SentenceSplitter({ + chunkSize: 256, + chunkOverlap: 20, +}); async function ingestSpec() { - const htmlFiles = await glob(path.join(SPEC_DIR, '*.html')); + const htmlFiles = await glob(path.join(SPEC_DIR, "*.html")); const documents = []; for (const file of htmlFiles) { - const content = fs.readFileSync(file, 'utf-8'); + const content = fs.readFileSync(file, "utf-8"); const $ = cheerio.load(content); - $('emu-clause').each((i, elem) => { - const id = $(elem).attr('id'); - const title = $(elem).find('h1').first().text().trim(); + $("emu-clause").each((i, elem) => { + const id = $(elem).attr("id"); + const title = $(elem).find("h1").first().text().trim(); // Only extract immediate text to avoid excessive chunking of child sections - const text = $(elem).clone().children('emu-clause').remove().end().text().trim(); + const text = $(elem) + .clone() + .children("emu-clause") + .remove() + .end() + .text() + .trim(); if (id && title && text) { - documents.push(new Document({ - text, - metadata: { - source: file, - sectionId: id, - sectionTitle: title, - type: 'specification' - } - })); + documents.push( + new Document({ + text, + metadata: { + source: file, + sectionId: id, + sectionTitle: title, + type: "specification", + }, + }), + ); } }); } @@ -65,11 +76,13 @@ async function main() { console.log(`Total raw nodes generated: ${rawNodes.length}`); // Safety filter to ensure no node exceeds context limit - const nodes = rawNodes.filter(node => { + const nodes = rawNodes.filter((node) => { const contentLen = node.getContent().length; if (contentLen > 2000) { - console.warn(`Skipping node with length ${contentLen} from ${node.metadata.source || 'unknown'}`); - return false; + console.warn( + `Skipping node with length ${contentLen} from ${node.metadata.source || "unknown"}`, + ); + return false; } return true; }); @@ -81,10 +94,10 @@ async function main() { }); console.log("Building index (this might take a while with local Ollama)..."); - + const BATCH_SIZE = 50; let index; - + // Try to load existing index if any try { index = await VectorStoreIndex.init({ @@ -97,19 +110,17 @@ async function main() { for (let i = 0; i < nodes.length; i += BATCH_SIZE) { const batch = nodes.slice(i, i + BATCH_SIZE); - console.log(`Processing batch ${i / BATCH_SIZE + 1} / ${Math.ceil(nodes.length / BATCH_SIZE)}...`); - + console.log( + `Processing batch ${i / BATCH_SIZE + 1} / ${Math.ceil(nodes.length / BATCH_SIZE)}...`, + ); + if (!index) { - index = await VectorStoreIndex.init({ - storageContext, - nodes: batch - }); + index = await VectorStoreIndex.init({ + storageContext, + nodes: batch, + }); } else { - // Here we'd ideally skip nodes already in the index, - // but for simplicity we'll just continue or assume - // we're starting fresh for this run if index was null. - // To truly resume, we need more logic. - await index.insertNodes(batch); + await index.insertNodes(batch); } } diff --git a/tsconfig.json b/tsconfig.json new file mode 100644 index 0000000..273637b --- /dev/null +++ b/tsconfig.json @@ -0,0 +1,17 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "esnext", + "moduleResolution": "node", + "esModuleInterop": true, + "allowSyntheticDefaultImports": true, + "strict": true, + "noEmitOnError": true, + "resolveJsonModule": true, + "outDir": "./dist", + "skipLibCheck": true, + "forceConsistentCasingInFileNames": true + }, + "include": ["**/*.ts"], + "exclude": ["node_modules", "dist"] +} \ No newline at end of file