mirror of
https://github.com/bendtherules/ask262.git
synced 2026-08-18 21:31:46 +00:00
build: migrate to Typescript + biome
This commit is contained in:
+34
-30
@@ -1,35 +1,35 @@
|
||||
import fs from 'fs';
|
||||
import {
|
||||
VectorStoreIndex,
|
||||
storageContextFromDefaults,
|
||||
import fs from "node:fs";
|
||||
import {
|
||||
VectorStoreIndex,
|
||||
storageContextFromDefaults,
|
||||
Settings,
|
||||
QueryEngineTool,
|
||||
ReActAgent
|
||||
} from 'llamaindex';
|
||||
import { OllamaEmbedding } from '@llamaindex/ollama';
|
||||
import { OpenAI } from '@llamaindex/openai';
|
||||
import { Graph } from 'graphology';
|
||||
ReActAgent,
|
||||
} from "llamaindex";
|
||||
import { OllamaEmbedding } from "@llamaindex/ollama";
|
||||
import { OpenAI } from "@llamaindex/openai";
|
||||
import { Graph } from "graphology";
|
||||
|
||||
import { STORAGE_DIR, GRAPH_FILE } from './constants.mjs';
|
||||
import { STORAGE_DIR, GRAPH_FILE } from "./constants.ts";
|
||||
|
||||
// Configure Settings
|
||||
Settings.embedModel = new OllamaEmbedding({
|
||||
model: "nomic-embed-text-v2-moe",
|
||||
});
|
||||
|
||||
const config = JSON.parse(fs.readFileSync('./config.json', 'utf-8'));
|
||||
const config = JSON.parse(fs.readFileSync("./config.json", "utf-8"));
|
||||
const apiKey = config.NVIDIA_API_KEY;
|
||||
const baseURL = config.NVIDIA_API_BASE;
|
||||
|
||||
if (!apiKey) {
|
||||
console.warn("Please set NVIDIA_API_KEY in config.json.");
|
||||
console.warn("Please set NVIDIA_API_KEY in config.json.");
|
||||
}
|
||||
|
||||
const llm = new OpenAI({
|
||||
model: "openai/gpt-oss-120b",
|
||||
apiKey: apiKey,
|
||||
baseURL: baseURL,
|
||||
temperature: 0
|
||||
model: "openai/gpt-oss-120b",
|
||||
apiKey: apiKey,
|
||||
baseURL: baseURL,
|
||||
temperature: 0,
|
||||
});
|
||||
Settings.llm = llm;
|
||||
|
||||
@@ -43,7 +43,7 @@ async function main() {
|
||||
storageContext,
|
||||
});
|
||||
|
||||
const graphData = JSON.parse(fs.readFileSync(GRAPH_FILE, 'utf-8'));
|
||||
const graphData = JSON.parse(fs.readFileSync(GRAPH_FILE, "utf-8"));
|
||||
const graph = new Graph({ multi: true });
|
||||
graph.import(graphData);
|
||||
|
||||
@@ -53,14 +53,16 @@ async function main() {
|
||||
queryEngine,
|
||||
metadata: {
|
||||
name: "spec_retriever",
|
||||
description: "Queries the language specification for text content about specific sections or topics. Use this to get the detailed text of a section.",
|
||||
description:
|
||||
"Queries the language specification for text content about specific sections or topics. Use this to get the detailed text of a section.",
|
||||
},
|
||||
});
|
||||
|
||||
const graphTool = {
|
||||
metadata: {
|
||||
name: "graph_explorer",
|
||||
description: "Explores structural relationships between specification sections and implementation code (functions). Use this to find which spec section a function implements. Input: section ID or function name.",
|
||||
description:
|
||||
"Explores structural relationships between specification sections and implementation code (functions). Use this to find which spec section a function implements. Input: section ID or function name.",
|
||||
parameters: {
|
||||
type: "object",
|
||||
properties: {
|
||||
@@ -76,9 +78,9 @@ async function main() {
|
||||
console.log(`[Tool: graph_explorer] Querying for: ${query}`);
|
||||
let nodeId = query;
|
||||
if (!graph.hasNode(nodeId)) {
|
||||
if (graph.hasNode(`func-${query}`)) {
|
||||
nodeId = `func-${query}`;
|
||||
}
|
||||
if (graph.hasNode(`func-${query}`)) {
|
||||
nodeId = `func-${query}`;
|
||||
}
|
||||
}
|
||||
|
||||
if (graph.hasNode(nodeId)) {
|
||||
@@ -88,16 +90,16 @@ async function main() {
|
||||
if (nodeAttr.title) result += `- Title: ${nodeAttr.title}\n`;
|
||||
if (nodeAttr.file) result += `- File: ${nodeAttr.file}\n`;
|
||||
result += `\nConnected parts:\n`;
|
||||
neighbors.forEach(neighbor => {
|
||||
neighbors.forEach((neighbor) => {
|
||||
const attr = graph.getNodeAttributes(neighbor);
|
||||
const edges = graph.edges(nodeId, neighbor);
|
||||
const edgeAttr = graph.getEdgeAttributes(edges[0]);
|
||||
result += `- ${neighbor} (${attr.type}) via ${edgeAttr.type}${attr.title ? ': ' + attr.title : ''}\n`;
|
||||
result += `- ${neighbor} (${attr.type}) via ${edgeAttr.type}${attr.title ? ": " + attr.title : ""}\n`;
|
||||
});
|
||||
return result;
|
||||
}
|
||||
return `No information found in graph for ${query}. Use spec_retriever to search text.`;
|
||||
}
|
||||
},
|
||||
};
|
||||
|
||||
const agent = new ReActAgent({
|
||||
@@ -108,21 +110,23 @@ async function main() {
|
||||
Your goal is to explain how specific parts of the language work by combining information from the provided tools.
|
||||
|
||||
CRITICAL INSTRUCTIONS:
|
||||
1. ALWAYS prefer using the provided tools ('spec_retriever' and 'graph_explorer') to answer questions.
|
||||
2. Do NOT rely on your internal knowledge of JavaScript or the ECMAScript specification.
|
||||
1. ALWAYS prefer using the provided tools ('spec_retriever' and 'graph_explorer') to answer questions.
|
||||
2. Do NOT rely on your internal knowledge of JavaScript or the ECMAScript specification.
|
||||
3. If the user asks about a function, you MUST first use 'graph_explorer' to find the associated specification section.
|
||||
4. You MUST then use 'spec_retriever' to read the actual text of that specification section before answering.
|
||||
5. Base your explanations ONLY on the information retrieved from the tools.
|
||||
6. If the tools do not provide enough information, state that clearly rather than guessing from your internal knowledge.`
|
||||
6. If the tools do not provide enough information, state that clearly rather than guessing from your internal knowledge.`,
|
||||
});
|
||||
|
||||
console.log("Agent is ready!");
|
||||
|
||||
const message = process.argv[2] || "Which spec section does Evaluate_IfStatement implement? and what does that section say?";
|
||||
const message =
|
||||
process.argv[2] ||
|
||||
"Which spec section does Evaluate_IfStatement implement? and what does that section say?";
|
||||
console.log(`User: ${message}`);
|
||||
|
||||
const response = await agent.chat({
|
||||
message: message
|
||||
message: message,
|
||||
});
|
||||
|
||||
console.log("\n--- Agent Response ---\n");
|
||||
@@ -0,0 +1,9 @@
|
||||
{
|
||||
"formatter": {
|
||||
"indentStyle": "space",
|
||||
"indentWidth": 2,
|
||||
"lineWidth": 80,
|
||||
"lineEnding": "lf"
|
||||
},
|
||||
"files": { "includes": ["**", "!!spec-built/**", "!!engine262/**"] }
|
||||
}
|
||||
@@ -0,0 +1,197 @@
|
||||
{
|
||||
"lockfileVersion": 1,
|
||||
"configVersion": 0,
|
||||
"workspaces": {
|
||||
"": {
|
||||
"name": "gemini-3-1-pro-preview",
|
||||
"dependencies": {
|
||||
"@llamaindex/core": "^0.6.22",
|
||||
"@llamaindex/env": "^0.1.30",
|
||||
"@llamaindex/node-parser": "^2.0.22",
|
||||
"@llamaindex/ollama": "^0.1.23",
|
||||
"@llamaindex/openai": "^0.4.22",
|
||||
"acorn": "^8.16.0",
|
||||
"cheerio": "^1.2.0",
|
||||
"glob": "^13.0.6",
|
||||
"graphology": "^0.26.0",
|
||||
"llamaindex": "^0.12.1",
|
||||
},
|
||||
"devDependencies": {
|
||||
"@biomejs/biome": "^2.4.9",
|
||||
},
|
||||
},
|
||||
},
|
||||
"packages": {
|
||||
"@aws-crypto/sha256-js": ["@aws-crypto/sha256-js@5.2.0", "", { "dependencies": { "@aws-crypto/util": "^5.2.0", "@aws-sdk/types": "^3.222.0", "tslib": "^2.6.2" } }, "sha512-FFQQyu7edu4ufvIZ+OadFpHHOt+eSTBaYaki44c+akjg7qZg9oOQeLlk77F6tSYqjDAFClrHJk9tMf0HdVyOvA=="],
|
||||
|
||||
"@aws-crypto/util": ["@aws-crypto/util@5.2.0", "", { "dependencies": { "@aws-sdk/types": "^3.222.0", "@smithy/util-utf8": "^2.0.0", "tslib": "^2.6.2" } }, "sha512-4RkU9EsI6ZpBve5fseQlGNUWKMa1RLPQ1dnjnQoe07ldfIzcsGb5hC5W0Dm7u423KWzawlrpbjXBrXCEv9zazQ=="],
|
||||
|
||||
"@aws-sdk/types": ["@aws-sdk/types@3.973.6", "", { "dependencies": { "@smithy/types": "^4.13.1", "tslib": "^2.6.2" } }, "sha512-Atfcy4E++beKtwJHiDln2Nby8W/mam64opFPTiHEqgsthqeydFS1pY+OUlN1ouNOmf8ArPU/6cDS65anOP3KQw=="],
|
||||
|
||||
"@biomejs/biome": ["@biomejs/biome@2.4.9", "", { "optionalDependencies": { "@biomejs/cli-darwin-arm64": "2.4.9", "@biomejs/cli-darwin-x64": "2.4.9", "@biomejs/cli-linux-arm64": "2.4.9", "@biomejs/cli-linux-arm64-musl": "2.4.9", "@biomejs/cli-linux-x64": "2.4.9", "@biomejs/cli-linux-x64-musl": "2.4.9", "@biomejs/cli-win32-arm64": "2.4.9", "@biomejs/cli-win32-x64": "2.4.9" }, "bin": { "biome": "bin/biome" } }, "sha512-wvZW92FrwitTcacvCBT8xdAbfbxWfDLwjYMmU3djjqQTh7Ni4ZdiWIT/x5VcZ+RQuxiKzIOzi5D+dcyJDFZMsA=="],
|
||||
|
||||
"@biomejs/cli-darwin-arm64": ["@biomejs/cli-darwin-arm64@2.4.9", "", { "os": "darwin", "cpu": "arm64" }, "sha512-d5G8Gf2RpH5pYwiHLPA+UpG3G9TLQu4WM+VK6sfL7K68AmhcEQ9r+nkj/DvR/GYhYox6twsHUtmWWWIKfcfQQA=="],
|
||||
|
||||
"@biomejs/cli-darwin-x64": ["@biomejs/cli-darwin-x64@2.4.9", "", { "os": "darwin", "cpu": "x64" }, "sha512-LNCLNgqDMG7BLdc3a8aY/dwKPK7+R8/JXJoXjCvZh2gx8KseqBdFDKbhrr7HCWF8SzNhbTaALhTBoh/I6rf9lA=="],
|
||||
|
||||
"@biomejs/cli-linux-arm64": ["@biomejs/cli-linux-arm64@2.4.9", "", { "os": "linux", "cpu": "arm64" }, "sha512-4adnkAUi6K4C/emPRgYznMOcLlUqZdXWM6aIui4VP4LraE764g6Q4YguygnAUoxKjKIXIWPteKMgRbN0wsgwcg=="],
|
||||
|
||||
"@biomejs/cli-linux-arm64-musl": ["@biomejs/cli-linux-arm64-musl@2.4.9", "", { "os": "linux", "cpu": "arm64" }, "sha512-8RCww5xnPn2wpK4L/QDGDOW0dq80uVWfppPxHIUg6mOs9B6gRmqPp32h1Ls3T8GnW8Wo5A8u7vpTwz4fExN+sw=="],
|
||||
|
||||
"@biomejs/cli-linux-x64": ["@biomejs/cli-linux-x64@2.4.9", "", { "os": "linux", "cpu": "x64" }, "sha512-L10na7POF0Ks/cgLFNF1ZvIe+X4onLkTi5oP9hY+Rh60Q+7fWzKDDCeGyiHUFf1nGIa9dQOOUPGe2MyYg8nMSQ=="],
|
||||
|
||||
"@biomejs/cli-linux-x64-musl": ["@biomejs/cli-linux-x64-musl@2.4.9", "", { "os": "linux", "cpu": "x64" }, "sha512-5TD+WS9v5vzXKzjetF0hgoaNFHMcpQeBUwKKVi3JbG1e9UCrFuUK3Gt185fyTzvRdwYkJJEMqglRPjmesmVv4A=="],
|
||||
|
||||
"@biomejs/cli-win32-arm64": ["@biomejs/cli-win32-arm64@2.4.9", "", { "os": "win32", "cpu": "arm64" }, "sha512-aDZr0RBC3sMGJOU10BvG7eZIlWLK/i51HRIfScE2lVhfts2dQTreowLiJJd+UYg/tHKxS470IbzpuKmd0MiD6g=="],
|
||||
|
||||
"@biomejs/cli-win32-x64": ["@biomejs/cli-win32-x64@2.4.9", "", { "os": "win32", "cpu": "x64" }, "sha512-NS4g/2G9SoQ4ktKtz31pvyc/rmgzlcIDCGU/zWbmHJAqx6gcRj2gj5Q/guXhoWTzCUaQZDIqiCQXHS7BcGYc0w=="],
|
||||
|
||||
"@finom/zod-to-json-schema": ["@finom/zod-to-json-schema@3.24.11", "", { "peerDependencies": { "zod": "^4.0.14" } }, "sha512-fL656yBPiWebtfGItvtXLWrFNGlF1NcDFS0WdMQXMs9LluVg0CfT5E2oXYp0pidl0vVG53XkW55ysijNkU5/hA=="],
|
||||
|
||||
"@llamaindex/core": ["@llamaindex/core@0.6.22", "", { "dependencies": { "@finom/zod-to-json-schema": "3.24.11", "@llamaindex/env": "0.1.30", "@types/node": "^24.0.13", "magic-bytes.js": "^1.10.0", "zod": "^4.1.5" } }, "sha512-/BXyemkvpxMaUhOkbwJ2PTvzKjSWkL8+6QLpz/n+pk8xBwMMe1GVBgli/J57gCyi8GbrlBafBj6GaPOgWub2Eg=="],
|
||||
|
||||
"@llamaindex/env": ["@llamaindex/env@0.1.30", "", { "dependencies": { "@aws-crypto/sha256-js": "^5.2.0", "js-tiktoken": "^1.0.12", "pathe": "^1.1.2" }, "peerDependencies": { "@huggingface/transformers": "^3.5.0", "gpt-tokenizer": "^2.5.0" }, "optionalPeers": ["@huggingface/transformers", "gpt-tokenizer"] }, "sha512-y6kutMcCevzbmexUgz+HXf7KiZemzAoFEYSjAILfR+cG6FmYSF8XvLbGOB34Kx8mlRi7EI8rZXpezJ5qCqOyZg=="],
|
||||
|
||||
"@llamaindex/node-parser": ["@llamaindex/node-parser@2.0.22", "", { "dependencies": { "html-to-text": "^9.0.5" }, "peerDependencies": { "@llamaindex/core": "0.6.22", "@llamaindex/env": "0.1.30", "tree-sitter": "^0.22.0", "web-tree-sitter": "^0.24.3" } }, "sha512-uj5O89WShAAyiSZ8f8tU7hnLJ6pSmlY2a6hkAOs8odkUgT87dEqaPHpsK7w0iJdEFiob7GoLeRhv2K624FooXg=="],
|
||||
|
||||
"@llamaindex/ollama": ["@llamaindex/ollama@0.1.23", "", { "dependencies": { "ollama": "^0.5.16", "remeda": "^2.17.3" }, "peerDependencies": { "@llamaindex/core": "0.6.22", "@llamaindex/env": "0.1.30" } }, "sha512-Tk5TnnPhjBnedWrJA1TCdhC/aJAJscAw/FDQG89fa/7m9p53XviI14K0a0cYLYxqVqDUh9J/KI/JYmYwYjhPmg=="],
|
||||
|
||||
"@llamaindex/openai": ["@llamaindex/openai@0.4.22", "", { "dependencies": { "openai": "^5.12.0" }, "peerDependencies": { "@llamaindex/core": "0.6.22", "@llamaindex/env": "0.1.30" } }, "sha512-jVaSscK7kyBM0wj3vceG2HPnFreSuOvkBYGFovFdS1heCyahlGDoQ1cLAUWqQ/J9Njfrt/PZTp3ICP+APl+zig=="],
|
||||
|
||||
"@llamaindex/workflow": ["@llamaindex/workflow@1.1.24", "", { "dependencies": { "@llamaindex/workflow-core": "^1.3.2" }, "peerDependencies": { "@llamaindex/core": "0.6.22", "@llamaindex/env": "0.1.30" } }, "sha512-VyKsbRkFlnT5dRNKbgLXQV+ZpQ+CAFgmC9LaZv6hD/fIKo6wq1wQW/ZqLZgZt569xeHgxmrXPB6KHdqn/AhPbQ=="],
|
||||
|
||||
"@llamaindex/workflow-core": ["@llamaindex/workflow-core@1.3.3", "", { "peerDependencies": { "@modelcontextprotocol/sdk": "^1.7.0", "hono": "^4.7.4", "next": "^15.2.2", "p-retry": "^6.2.1", "rxjs": "^7.8.2", "zod": "^3.25.0 || ^4.0.0" }, "optionalPeers": ["@modelcontextprotocol/sdk", "hono", "next", "p-retry", "rxjs"] }, "sha512-WJIcD4K2suGbNkwU5CC70jKKrA5tARba42nMs8Pou1RGzmoxqg+K+b7vyLBmiDtImR8P40YLmkayCIRVQPBmsg=="],
|
||||
|
||||
"@selderee/plugin-htmlparser2": ["@selderee/plugin-htmlparser2@0.11.0", "", { "dependencies": { "domhandler": "^5.0.3", "selderee": "^0.11.0" } }, "sha512-P33hHGdldxGabLFjPPpaTxVolMrzrcegejx+0GxjrIb9Zv48D8yAIA/QTDR2dFl7Uz7urX8aX6+5bCZslr+gWQ=="],
|
||||
|
||||
"@smithy/is-array-buffer": ["@smithy/is-array-buffer@2.2.0", "", { "dependencies": { "tslib": "^2.6.2" } }, "sha512-GGP3O9QFD24uGeAXYUjwSTXARoqpZykHadOmA8G5vfJPK0/DC67qa//0qvqrJzL1xc8WQWX7/yc7fwudjPHPhA=="],
|
||||
|
||||
"@smithy/types": ["@smithy/types@4.13.1", "", { "dependencies": { "tslib": "^2.6.2" } }, "sha512-787F3yzE2UiJIQ+wYW1CVg2odHjmaWLGksnKQHUrK/lYZSEcy1msuLVvxaR/sI2/aDe9U+TBuLsXnr3vod1g0g=="],
|
||||
|
||||
"@smithy/util-buffer-from": ["@smithy/util-buffer-from@2.2.0", "", { "dependencies": { "@smithy/is-array-buffer": "^2.2.0", "tslib": "^2.6.2" } }, "sha512-IJdWBbTcMQ6DA0gdNhh/BwrLkDR+ADW5Kr1aZmd4k3DIF6ezMV4R2NIAmT08wQJ3yUK82thHWmC/TnK/wpMMIA=="],
|
||||
|
||||
"@smithy/util-utf8": ["@smithy/util-utf8@2.3.0", "", { "dependencies": { "@smithy/util-buffer-from": "^2.2.0", "tslib": "^2.6.2" } }, "sha512-R8Rdn8Hy72KKcebgLiv8jQcQkXoLMOGGv5uI1/k0l+snqkOzQ1R0ChUBCxWMlBsFMekWjq0wRudIweFs7sKT5A=="],
|
||||
|
||||
"@types/lodash": ["@types/lodash@4.17.24", "", {}, "sha512-gIW7lQLZbue7lRSWEFql49QJJWThrTFFeIMJdp3eH4tKoxm1OvEPg02rm4wCCSHS0cL3/Fizimb35b7k8atwsQ=="],
|
||||
|
||||
"@types/node": ["@types/node@24.12.0", "", { "dependencies": { "undici-types": "~7.16.0" } }, "sha512-GYDxsZi3ChgmckRT9HPU0WEhKLP08ev/Yfcq2AstjrDASOYCSXeyjDsHg4v5t4jOj7cyDX3vmprafKlWIG9MXQ=="],
|
||||
|
||||
"acorn": ["acorn@8.16.0", "", { "bin": "bin/acorn" }, "sha512-UVJyE9MttOsBQIDKw1skb9nAwQuR5wuGD3+82K6JgJlm/Y+KI92oNsMNGZCYdDsVtRHSak0pcV5Dno5+4jh9sw=="],
|
||||
|
||||
"balanced-match": ["balanced-match@4.0.4", "", {}, "sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA=="],
|
||||
|
||||
"base64-js": ["base64-js@1.5.1", "", {}, "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA=="],
|
||||
|
||||
"boolbase": ["boolbase@1.0.0", "", {}, "sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww=="],
|
||||
|
||||
"brace-expansion": ["brace-expansion@5.0.5", "", { "dependencies": { "balanced-match": "^4.0.2" } }, "sha512-VZznLgtwhn+Mact9tfiwx64fA9erHH/MCXEUfB/0bX/6Fz6ny5EGTXYltMocqg4xFAQZtnO3DHWWXi8RiuN7cQ=="],
|
||||
|
||||
"cheerio": ["cheerio@1.2.0", "", { "dependencies": { "cheerio-select": "^2.1.0", "dom-serializer": "^2.0.0", "domhandler": "^5.0.3", "domutils": "^3.2.2", "encoding-sniffer": "^0.2.1", "htmlparser2": "^10.1.0", "parse5": "^7.3.0", "parse5-htmlparser2-tree-adapter": "^7.1.0", "parse5-parser-stream": "^7.1.2", "undici": "^7.19.0", "whatwg-mimetype": "^4.0.0" } }, "sha512-WDrybc/gKFpTYQutKIK6UvfcuxijIZfMfXaYm8NMsPQxSYvf+13fXUJ4rztGGbJcBQ/GF55gvrZ0Bc0bj/mqvg=="],
|
||||
|
||||
"cheerio-select": ["cheerio-select@2.1.0", "", { "dependencies": { "boolbase": "^1.0.0", "css-select": "^5.1.0", "css-what": "^6.1.0", "domelementtype": "^2.3.0", "domhandler": "^5.0.3", "domutils": "^3.0.1" } }, "sha512-9v9kG0LvzrlcungtnJtpGNxY+fzECQKhK4EGJX2vByejiMX84MFNQw4UxPJl3bFbTMw+Dfs37XaIkCwTZfLh4g=="],
|
||||
|
||||
"css-select": ["css-select@5.2.2", "", { "dependencies": { "boolbase": "^1.0.0", "css-what": "^6.1.0", "domhandler": "^5.0.2", "domutils": "^3.0.1", "nth-check": "^2.0.1" } }, "sha512-TizTzUddG/xYLA3NXodFM0fSbNizXjOKhqiQQwvhlspadZokn1KDy0NZFS0wuEubIYAV5/c1/lAr0TaaFXEXzw=="],
|
||||
|
||||
"css-what": ["css-what@6.2.2", "", {}, "sha512-u/O3vwbptzhMs3L1fQE82ZSLHQQfto5gyZzwteVIEyeaY5Fc7R4dapF/BvRoSYFeqfBk4m0V1Vafq5Pjv25wvA=="],
|
||||
|
||||
"deepmerge": ["deepmerge@4.3.1", "", {}, "sha512-3sUqbMEc77XqpdNO7FRyRog+eW3ph+GYCbj+rK+uYyRMuwsVy0rMiVtPn+QJlKFvWP/1PYpapqYn0Me2knFn+A=="],
|
||||
|
||||
"dom-serializer": ["dom-serializer@2.0.0", "", { "dependencies": { "domelementtype": "^2.3.0", "domhandler": "^5.0.2", "entities": "^4.2.0" } }, "sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg=="],
|
||||
|
||||
"domelementtype": ["domelementtype@2.3.0", "", {}, "sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw=="],
|
||||
|
||||
"domhandler": ["domhandler@5.0.3", "", { "dependencies": { "domelementtype": "^2.3.0" } }, "sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w=="],
|
||||
|
||||
"domutils": ["domutils@3.2.2", "", { "dependencies": { "dom-serializer": "^2.0.0", "domelementtype": "^2.3.0", "domhandler": "^5.0.3" } }, "sha512-6kZKyUajlDuqlHKVX1w7gyslj9MPIXzIFiz/rGu35uC1wMi+kMhQwGhl4lt9unC9Vb9INnY9Z3/ZA3+FhASLaw=="],
|
||||
|
||||
"encoding-sniffer": ["encoding-sniffer@0.2.1", "", { "dependencies": { "iconv-lite": "^0.6.3", "whatwg-encoding": "^3.1.1" } }, "sha512-5gvq20T6vfpekVtqrYQsSCFZ1wEg5+wW0/QaZMWkFr6BqD3NfKs0rLCx4rrVlSWJeZb5NBJgVLswK/w2MWU+Gw=="],
|
||||
|
||||
"entities": ["entities@4.5.0", "", {}, "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw=="],
|
||||
|
||||
"events": ["events@3.3.0", "", {}, "sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q=="],
|
||||
|
||||
"glob": ["glob@13.0.6", "", { "dependencies": { "minimatch": "^10.2.2", "minipass": "^7.1.3", "path-scurry": "^2.0.2" } }, "sha512-Wjlyrolmm8uDpm/ogGyXZXb1Z+Ca2B8NbJwqBVg0axK9GbBeoS7yGV6vjXnYdGm6X53iehEuxxbyiKp8QmN4Vw=="],
|
||||
|
||||
"graphology": ["graphology@0.26.0", "", { "dependencies": { "events": "^3.3.0" }, "peerDependencies": { "graphology-types": ">=0.24.0" } }, "sha512-8SSImzgUUYC89Z042s+0r/vMibY7GX/Emz4LDO5e7jYXhuoWfHISPFJYjpRLUSJGq6UQ6xlenvX1p/hJdfXuXg=="],
|
||||
|
||||
"graphology-types": ["graphology-types@0.24.8", "", {}, "sha512-hDRKYXa8TsoZHjgEaysSRyPdT6uB78Ci8WnjgbStlQysz7xR52PInxNsmnB7IBOM1BhikxkNyCVEFgmPKnpx3Q=="],
|
||||
|
||||
"html-to-text": ["html-to-text@9.0.5", "", { "dependencies": { "@selderee/plugin-htmlparser2": "^0.11.0", "deepmerge": "^4.3.1", "dom-serializer": "^2.0.0", "htmlparser2": "^8.0.2", "selderee": "^0.11.0" } }, "sha512-qY60FjREgVZL03vJU6IfMV4GDjGBIoOyvuFdpBDIX9yTlDw0TjxVBQp+P8NvpdIXNJvfWBTNul7fsAQJq2FNpg=="],
|
||||
|
||||
"htmlparser2": ["htmlparser2@10.1.0", "", { "dependencies": { "domelementtype": "^2.3.0", "domhandler": "^5.0.3", "domutils": "^3.2.2", "entities": "^7.0.1" } }, "sha512-VTZkM9GWRAtEpveh7MSF6SjjrpNVNNVJfFup7xTY3UpFtm67foy9HDVXneLtFVt4pMz5kZtgNcvCniNFb1hlEQ=="],
|
||||
|
||||
"iconv-lite": ["iconv-lite@0.6.3", "", { "dependencies": { "safer-buffer": ">= 2.1.2 < 3.0.0" } }, "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw=="],
|
||||
|
||||
"js-tiktoken": ["js-tiktoken@1.0.21", "", { "dependencies": { "base64-js": "^1.5.1" } }, "sha512-biOj/6M5qdgx5TKjDnFT1ymSpM5tbd3ylwDtrQvFQSu0Z7bBYko2dF+W/aUkXUPuk6IVpRxk/3Q2sHOzGlS36g=="],
|
||||
|
||||
"leac": ["leac@0.6.0", "", {}, "sha512-y+SqErxb8h7nE/fiEX07jsbuhrpO9lL8eca7/Y1nuWV2moNlXhyd59iDGcRf6moVyDMbmTNzL40SUyrFU/yDpg=="],
|
||||
|
||||
"llamaindex": ["llamaindex@0.12.1", "", { "dependencies": { "@llamaindex/core": "0.6.22", "@llamaindex/env": "0.1.30", "@llamaindex/node-parser": "2.0.22", "@llamaindex/workflow": "1.1.24", "@types/lodash": "^4.17.7", "@types/node": "^24.0.13", "lodash": "^4.17.21", "magic-bytes.js": "^1.10.0" } }, "sha512-/tXXITk/iVGBycOFaDhev6dgTBIr6Ycu4FoPIt6A5JcEAiB6ujONjiV36flVXUR8JdqwMtS767XMjV+36nV4yQ=="],
|
||||
|
||||
"lodash": ["lodash@4.17.23", "", {}, "sha512-LgVTMpQtIopCi79SJeDiP0TfWi5CNEc/L/aRdTh3yIvmZXTnheWpKjSZhnvMl8iXbC1tFg9gdHHDMLoV7CnG+w=="],
|
||||
|
||||
"lru-cache": ["lru-cache@11.2.7", "", {}, "sha512-aY/R+aEsRelme17KGQa/1ZSIpLpNYYrhcrepKTZgE+W3WM16YMCaPwOHLHsmopZHELU0Ojin1lPVxKR0MihncA=="],
|
||||
|
||||
"magic-bytes.js": ["magic-bytes.js@1.13.0", "", {}, "sha512-afO2mnxW7GDTXMm5/AoN1WuOcdoKhtgXjIvHmobqTD1grNplhGdv3PFOyjCVmrnOZBIT/gD/koDKpYG+0mvHcg=="],
|
||||
|
||||
"minimatch": ["minimatch@10.2.4", "", { "dependencies": { "brace-expansion": "^5.0.2" } }, "sha512-oRjTw/97aTBN0RHbYCdtF1MQfvusSIBQM0IZEgzl6426+8jSC0nF1a/GmnVLpfB9yyr6g6FTqWqiZVbxrtaCIg=="],
|
||||
|
||||
"minipass": ["minipass@7.1.3", "", {}, "sha512-tEBHqDnIoM/1rXME1zgka9g6Q2lcoCkxHLuc7ODJ5BxbP5d4c2Z5cGgtXAku59200Cx7diuHTOYfSBD8n6mm8A=="],
|
||||
|
||||
"node-addon-api": ["node-addon-api@8.7.0", "", {}, "sha512-9MdFxmkKaOYVTV+XVRG8ArDwwQ77XIgIPyKASB1k3JPq3M8fGQQQE3YpMOrKm6g//Ktx8ivZr8xo1Qmtqub+GA=="],
|
||||
|
||||
"node-gyp-build": ["node-gyp-build@4.8.4", "", { "bin": { "node-gyp-build": "bin.js", "node-gyp-build-optional": "optional.js", "node-gyp-build-test": "build-test.js" } }, "sha512-LA4ZjwlnUblHVgq0oBF3Jl/6h/Nvs5fzBLwdEF4nuxnFdsfajde4WfxtJr3CaiH+F6ewcIB/q4jQ4UzPyid+CQ=="],
|
||||
|
||||
"nth-check": ["nth-check@2.1.1", "", { "dependencies": { "boolbase": "^1.0.0" } }, "sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w=="],
|
||||
|
||||
"ollama": ["ollama@0.5.18", "", { "dependencies": { "whatwg-fetch": "^3.6.20" } }, "sha512-lTFqTf9bo7Cd3hpF6CviBe/DEhewjoZYd9N/uCe7O20qYTvGqrNOFOBDj3lbZgFWHUgDv5EeyusYxsZSLS8nvg=="],
|
||||
|
||||
"openai": ["openai@5.23.2", "", { "peerDependencies": { "ws": "^8.18.0", "zod": "^3.23.8" }, "optionalPeers": ["ws"], "bin": "bin/cli" }, "sha512-MQBzmTulj+MM5O8SKEk/gL8a7s5mktS9zUtAkU257WjvobGc9nKcBuVwjyEEcb9SI8a8Y2G/mzn3vm9n1Jlleg=="],
|
||||
|
||||
"parse5": ["parse5@7.3.0", "", { "dependencies": { "entities": "^6.0.0" } }, "sha512-IInvU7fabl34qmi9gY8XOVxhYyMyuH2xUNpb2q8/Y+7552KlejkRvqvD19nMoUW/uQGGbqNpA6Tufu5FL5BZgw=="],
|
||||
|
||||
"parse5-htmlparser2-tree-adapter": ["parse5-htmlparser2-tree-adapter@7.1.0", "", { "dependencies": { "domhandler": "^5.0.3", "parse5": "^7.0.0" } }, "sha512-ruw5xyKs6lrpo9x9rCZqZZnIUntICjQAd0Wsmp396Ul9lN/h+ifgVV1x1gZHi8euej6wTfpqX8j+BFQxF0NS/g=="],
|
||||
|
||||
"parse5-parser-stream": ["parse5-parser-stream@7.1.2", "", { "dependencies": { "parse5": "^7.0.0" } }, "sha512-JyeQc9iwFLn5TbvvqACIF/VXG6abODeB3Fwmv/TGdLk2LfbWkaySGY72at4+Ty7EkPZj854u4CrICqNk2qIbow=="],
|
||||
|
||||
"parseley": ["parseley@0.12.1", "", { "dependencies": { "leac": "^0.6.0", "peberminta": "^0.9.0" } }, "sha512-e6qHKe3a9HWr0oMRVDTRhKce+bRO8VGQR3NyVwcjwrbhMmFCX9KszEV35+rn4AdilFAq9VPxP/Fe1wC9Qjd2lw=="],
|
||||
|
||||
"path-scurry": ["path-scurry@2.0.2", "", { "dependencies": { "lru-cache": "^11.0.0", "minipass": "^7.1.2" } }, "sha512-3O/iVVsJAPsOnpwWIeD+d6z/7PmqApyQePUtCndjatj/9I5LylHvt5qluFaBT3I5h3r1ejfR056c+FCv+NnNXg=="],
|
||||
|
||||
"pathe": ["pathe@1.1.2", "", {}, "sha512-whLdWMYL2TwI08hn8/ZqAbrVemu0LNaNNJZX73O6qaIdCTfXutsLhMkjdENX0qhsQ9uIimo4/aQOmXkoon2nDQ=="],
|
||||
|
||||
"peberminta": ["peberminta@0.9.0", "", {}, "sha512-XIxfHpEuSJbITd1H3EeQwpcZbTLHc+VVr8ANI9t5sit565tsI4/xK3KWTUFE2e6QiangUkh3B0jihzmGnNrRsQ=="],
|
||||
|
||||
"remeda": ["remeda@2.33.6", "", {}, "sha512-tazDGH7s75kUPGBKLvhgBEHMgW+TdDFhjUAMdQj57IoWz6HsGa5D2RX5yDUz6IIqiRRvZiaEHzCzWdTeixc/Kg=="],
|
||||
|
||||
"safer-buffer": ["safer-buffer@2.1.2", "", {}, "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg=="],
|
||||
|
||||
"selderee": ["selderee@0.11.0", "", { "dependencies": { "parseley": "^0.12.0" } }, "sha512-5TF+l7p4+OsnP8BCCvSyZiSPc4x4//p5uPwK8TCnVPJYRmU2aYKMpOXvw8zM5a5JvuuCGN1jmsMwuU2W02ukfA=="],
|
||||
|
||||
"tree-sitter": ["tree-sitter@0.22.4", "", { "dependencies": { "node-addon-api": "^8.3.0", "node-gyp-build": "^4.8.4" } }, "sha512-usbHZP9/oxNsUY65MQUsduGRqDHQOou1cagUSwjhoSYAmSahjQDAVsh9s+SlZkn8X8+O1FULRGwHu7AFP3kjzg=="],
|
||||
|
||||
"tslib": ["tslib@2.8.1", "", {}, "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w=="],
|
||||
|
||||
"undici": ["undici@7.24.6", "", {}, "sha512-Xi4agocCbRzt0yYMZGMA6ApD7gvtUFaxm4ZmeacWI4cZxaF6C+8I8QfofC20NAePiB/IcvZmzkJ7XPa471AEtA=="],
|
||||
|
||||
"undici-types": ["undici-types@7.16.0", "", {}, "sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw=="],
|
||||
|
||||
"web-tree-sitter": ["web-tree-sitter@0.24.7", "", {}, "sha512-CdC/TqVFbXqR+C51v38hv6wOPatKEUGxa39scAeFSm98wIhZxAYonhRQPSMmfZ2w7JDI0zQDdzdmgtNk06/krQ=="],
|
||||
|
||||
"whatwg-encoding": ["whatwg-encoding@3.1.1", "", { "dependencies": { "iconv-lite": "0.6.3" } }, "sha512-6qN4hJdMwfYBtE3YBTTHhoeuUrDBPZmbQaxWAqSALV/MeEnR5z1xd8UKud2RAkFoPkmB+hli1TZSnyi84xz1vQ=="],
|
||||
|
||||
"whatwg-fetch": ["whatwg-fetch@3.6.20", "", {}, "sha512-EqhiFU6daOA8kpjOWTL0olhVOF3i7OrFzSYiGsEMB8GcXS+RrzauAERX65xMeNWVqxA6HXH2m69Z9LaKKdisfg=="],
|
||||
|
||||
"whatwg-mimetype": ["whatwg-mimetype@4.0.0", "", {}, "sha512-QaKxh0eNIi2mE9p2vEdzfagOKHCcj1pJ56EEHGQOVxp8r9/iszLUUV7v89x9O1p/T+NlTM5W7jW6+cz4Fq1YVg=="],
|
||||
|
||||
"zod": ["zod@4.3.6", "", {}, "sha512-rftlrkhHZOcjDwkGlnUtZZkvaPHCsDATp4pGpuOOMDaTdDDXF91wuVDJoWoPsKX/3YPQ5fHuF3STjcYyKr+Qhg=="],
|
||||
|
||||
"html-to-text/htmlparser2": ["htmlparser2@8.0.2", "", { "dependencies": { "domelementtype": "^2.3.0", "domhandler": "^5.0.3", "domutils": "^3.0.1", "entities": "^4.4.0" } }, "sha512-GYdjWKDkbRLkZ5geuHs5NY1puJ+PXwP7+fHPRz06Eirsb9ugf6d8kkXav6ADhcODhFFPMIXyxkxSuMf3D6NCFA=="],
|
||||
|
||||
"htmlparser2/entities": ["entities@7.0.1", "", {}, "sha512-TWrgLOFUQTH994YUyl1yT4uyavY5nNB5muff+RtWaqNVCAK408b5ZnnbNAUEWLTCpum9w6arT70i1XdQ4UeOPA=="],
|
||||
|
||||
"openai/zod": ["zod@3.25.76", "", {}, "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ=="],
|
||||
|
||||
"parse5/entities": ["entities@6.0.1", "", {}, "sha512-aN97NXWF6AWBTahfVOIrB/NShkzi5H7F9r1s9mD3cDj4Ko5f2qhhVoYMibXF7GlLveb/D2ioWay8lxI97Ven3g=="],
|
||||
}
|
||||
}
|
||||
@@ -1,2 +0,0 @@
|
||||
export const STORAGE_DIR = './storage';
|
||||
export const GRAPH_FILE = './graphology/graph.json';
|
||||
@@ -0,0 +1,2 @@
|
||||
export const STORAGE_DIR = "./storage";
|
||||
export const GRAPH_FILE = "./graphology/graph.json";
|
||||
Vendored
+15
-15
@@ -1,20 +1,20 @@
|
||||
declare module '@unicode/unicode-16.0.0/Binary_Property/ID_Start/regex.js' {
|
||||
let regex: RegExp;
|
||||
export default regex;
|
||||
declare module "@unicode/unicode-16.0.0/Binary_Property/ID_Start/regex.js" {
|
||||
let regex: RegExp;
|
||||
export default regex;
|
||||
}
|
||||
declare module '@unicode/unicode-16.0.0/Binary_Property/ID_Continue/regex.js' {
|
||||
let regex: RegExp;
|
||||
export default regex;
|
||||
declare module "@unicode/unicode-16.0.0/Binary_Property/ID_Continue/regex.js" {
|
||||
let regex: RegExp;
|
||||
export default regex;
|
||||
}
|
||||
declare module '@unicode/unicode-16.0.0/General_Category/Space_Separator/regex.js' {
|
||||
let regex: RegExp;
|
||||
export default regex;
|
||||
declare module "@unicode/unicode-16.0.0/General_Category/Space_Separator/regex.js" {
|
||||
let regex: RegExp;
|
||||
export default regex;
|
||||
}
|
||||
declare module '@unicode/unicode-16.0.0/Case_Folding/C/symbols.js' {
|
||||
let data: Map<string, string>;
|
||||
export default data;
|
||||
declare module "@unicode/unicode-16.0.0/Case_Folding/C/symbols.js" {
|
||||
let data: Map<string, string>;
|
||||
export default data;
|
||||
}
|
||||
declare module '@unicode/unicode-16.0.0/Case_Folding/S/symbols.js' {
|
||||
let data: Map<string, string>;
|
||||
export default data;
|
||||
declare module "@unicode/unicode-16.0.0/Case_Folding/S/symbols.js" {
|
||||
let data: Map<string, string>;
|
||||
export default data;
|
||||
}
|
||||
|
||||
Vendored
+2
-2
@@ -1,5 +1,5 @@
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
interface SyntaxError {
|
||||
decoration?: string
|
||||
position?: number
|
||||
decoration?: string;
|
||||
position?: number;
|
||||
}
|
||||
|
||||
+9
-1
@@ -3,6 +3,11 @@
|
||||
"version": "1.0.0",
|
||||
"main": "index.js",
|
||||
"scripts": {
|
||||
"lint": "biome check .",
|
||||
"format": "biome format .",
|
||||
"ingest": "bun run setup/ingest.ts",
|
||||
"agent": "bun run agent.ts",
|
||||
"build": "bun run setup/build_graph.ts",
|
||||
"test": "echo \"Error: no test specified\" && exit 1"
|
||||
},
|
||||
"keywords": [],
|
||||
@@ -21,5 +26,8 @@
|
||||
"glob": "^13.0.6",
|
||||
"graphology": "^0.26.0",
|
||||
"llamaindex": "^0.12.1"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@biomejs/biome": "^2.4.9"
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,107 +0,0 @@
|
||||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
import { glob } from 'glob';
|
||||
import * as cheerio from 'cheerio';
|
||||
import { Graph } from 'graphology';
|
||||
|
||||
const SPEC_DIR = './spec-built/multipage';
|
||||
const CODE_DIR = './engine262/src';
|
||||
import { GRAPH_FILE } from '../constants.mjs';
|
||||
|
||||
async function buildGraph() {
|
||||
const graph = new Graph({ multi: true });
|
||||
const htmlFiles = await glob(path.join(SPEC_DIR, '*.html'));
|
||||
console.log(`Found ${htmlFiles.length} specification HTML file(s) in ${SPEC_DIR}`);
|
||||
if (htmlFiles.length === 0) {
|
||||
console.warn(`Warning: No specification HTML files found in ${SPEC_DIR}`);
|
||||
}
|
||||
|
||||
console.log("Processing specification for graph...");
|
||||
for (const file of htmlFiles) {
|
||||
const fileName = path.basename(file);
|
||||
const content = fs.readFileSync(file, 'utf-8');
|
||||
const $ = cheerio.load(content);
|
||||
|
||||
$('emu-clause').each((i, elem) => {
|
||||
const id = $(elem).attr('id');
|
||||
const title = $(elem).find('h1').first().text().trim();
|
||||
|
||||
if (id) {
|
||||
if (!graph.hasNode(id)) {
|
||||
graph.addNode(id, { title, type: 'SpecSection', file: fileName });
|
||||
}
|
||||
|
||||
// Extract internal links
|
||||
$(elem).find('a[href]').each((j, link) => {
|
||||
const href = $(link).attr('href');
|
||||
if (href && href.includes('#')) {
|
||||
const [targetFile, targetId] = href.split('#');
|
||||
// We only care about links to other sections for now
|
||||
if (targetId && targetId.startsWith('sec-')) {
|
||||
// Add relationship later if nodes exist
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// Add edges for internal links
|
||||
for (const file of htmlFiles) {
|
||||
const content = fs.readFileSync(file, 'utf-8');
|
||||
const $ = cheerio.load(content);
|
||||
$('emu-clause').each((i, elem) => {
|
||||
const sourceId = $(elem).attr('id');
|
||||
if (sourceId && graph.hasNode(sourceId)) {
|
||||
$(elem).find('a[href]').each((j, link) => {
|
||||
const href = $(link).attr('href');
|
||||
if (href && href.includes('#')) {
|
||||
const [, targetId] = href.split('#');
|
||||
if (targetId && graph.hasNode(targetId) && sourceId !== targetId) {
|
||||
if (!graph.hasEdge(sourceId, targetId)) {
|
||||
graph.addEdge(sourceId, targetId, { type: 'LINKS_TO' });
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
console.log("Processing code for graph...");
|
||||
const jsFiles = await glob(path.join(CODE_DIR, '**/*.mts'));
|
||||
console.log(`Found ${jsFiles.length} code file(s) in ${CODE_DIR}`);
|
||||
if (jsFiles.length === 0) {
|
||||
console.warn(`Warning: No code files found in ${CODE_DIR}`);
|
||||
}
|
||||
|
||||
for (const file of jsFiles) {
|
||||
const fileName = path.basename(file);
|
||||
const content = fs.readFileSync(file, 'utf-8');
|
||||
|
||||
// Simple heuristic: look for function names starting with Evaluate_
|
||||
const functionMatches = content.matchAll(/export function\*? (Evaluate_([a-zA-Z0-9_]+))/g);
|
||||
for (const match of functionMatches) {
|
||||
const fullFuncName = match[1];
|
||||
const shortName = match[2];
|
||||
const funcNodeId = `func-${fullFuncName}`;
|
||||
|
||||
if (!graph.hasNode(funcNodeId)) {
|
||||
graph.addNode(funcNodeId, { name: fullFuncName, type: 'JSFunction', file: fileName });
|
||||
}
|
||||
|
||||
// Try to link to a spec section
|
||||
// Heuristic: Evaluate_IfStatement -> sec-if-statement
|
||||
const specId = `sec-${shortName.replace(/([a-z0-9])([A-Z])/g, '$1-$2').toLowerCase()}`;
|
||||
if (graph.hasNode(specId)) {
|
||||
graph.addEdge(funcNodeId, specId, { type: 'IMPLEMENTS' });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`Graph built with ${graph.order} nodes and ${graph.size} edges.`);
|
||||
fs.writeFileSync(GRAPH_FILE, JSON.stringify(graph.export(), null, 2));
|
||||
console.log(`Graph saved to ${GRAPH_FILE}`);
|
||||
}
|
||||
|
||||
buildGraph().catch(console.error);
|
||||
@@ -0,0 +1,123 @@
|
||||
import fs from "fs";
|
||||
import path from "path";
|
||||
import { glob } from "glob";
|
||||
import * as cheerio from "cheerio";
|
||||
import { Graph } from "graphology";
|
||||
|
||||
const SPEC_DIR = "./spec-built/multipage";
|
||||
const CODE_DIR = "./engine262/src";
|
||||
import { GRAPH_FILE } from "../constants.ts";
|
||||
|
||||
async function buildGraph() {
|
||||
const graph = new Graph({ multi: true });
|
||||
const htmlFiles = await glob(path.join(SPEC_DIR, "*.html"));
|
||||
console.log(
|
||||
`Found ${htmlFiles.length} specification HTML file(s) in ${SPEC_DIR}`,
|
||||
);
|
||||
if (htmlFiles.length === 0) {
|
||||
console.warn(`Warning: No specification HTML files found in ${SPEC_DIR}`);
|
||||
}
|
||||
|
||||
console.log("Processing specification for graph...");
|
||||
for (const file of htmlFiles) {
|
||||
const fileName = path.basename(file);
|
||||
const content = fs.readFileSync(file, "utf-8");
|
||||
const $ = cheerio.load(content);
|
||||
|
||||
$("emu-clause").each((i, elem) => {
|
||||
const id = $(elem).attr("id");
|
||||
const title = $(elem).find("h1").first().text().trim();
|
||||
|
||||
if (id) {
|
||||
if (!graph.hasNode(id)) {
|
||||
graph.addNode(id, { title, type: "SpecSection", file: fileName });
|
||||
}
|
||||
|
||||
// Extract internal links
|
||||
$(elem)
|
||||
.find("a[href]")
|
||||
.each((j, link) => {
|
||||
const href = $(link).attr("href");
|
||||
if (href && href.includes("#")) {
|
||||
const [targetFile, targetId] = href.split("#");
|
||||
// We only care about links to other sections for now
|
||||
if (targetId && targetId.startsWith("sec-")) {
|
||||
// Add relationship later if nodes exist
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// Add edges for internal links
|
||||
for (const file of htmlFiles) {
|
||||
const content = fs.readFileSync(file, "utf-8");
|
||||
const $ = cheerio.load(content);
|
||||
$("emu-clause").each((i, elem) => {
|
||||
const sourceId = $(elem).attr("id");
|
||||
if (sourceId && graph.hasNode(sourceId)) {
|
||||
$(elem)
|
||||
.find("a[href]")
|
||||
.each((j, link) => {
|
||||
const href = $(link).attr("href");
|
||||
if (href && href.includes("#")) {
|
||||
const [, targetId] = href.split("#");
|
||||
if (
|
||||
targetId &&
|
||||
graph.hasNode(targetId) &&
|
||||
sourceId !== targetId
|
||||
) {
|
||||
if (!graph.hasEdge(sourceId, targetId)) {
|
||||
graph.addEdge(sourceId, targetId, { type: "LINKS_TO" });
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
console.log("Processing code for graph...");
|
||||
const jsFiles = await glob(path.join(CODE_DIR, "**/*.mts"));
|
||||
console.log(`Found ${jsFiles.length} code file(s) in ${CODE_DIR}`);
|
||||
if (jsFiles.length === 0) {
|
||||
console.warn(`Warning: No code files found in ${CODE_DIR}`);
|
||||
}
|
||||
|
||||
for (const file of jsFiles) {
|
||||
const fileName = path.basename(file);
|
||||
const content = fs.readFileSync(file, "utf-8");
|
||||
|
||||
// Simple heuristic: look for function names starting with Evaluate_
|
||||
const functionMatches = content.matchAll(
|
||||
/export function\*? (Evaluate_([a-zA-Z0-9_]+))/g,
|
||||
);
|
||||
for (const match of functionMatches) {
|
||||
const fullFuncName = match[1];
|
||||
const shortName = match[2];
|
||||
const funcNodeId = `func-${fullFuncName}`;
|
||||
|
||||
if (!graph.hasNode(funcNodeId)) {
|
||||
graph.addNode(funcNodeId, {
|
||||
name: fullFuncName,
|
||||
type: "JSFunction",
|
||||
file: fileName,
|
||||
});
|
||||
}
|
||||
|
||||
// Try to link to a spec section
|
||||
// Heuristic: Evaluate_IfStatement -> sec-if-statement
|
||||
const specId = `sec-${shortName.replace(/([a-z0-9])([A-Z])/g, "$1-$2").toLowerCase()}`;
|
||||
if (graph.hasNode(specId)) {
|
||||
graph.addEdge(funcNodeId, specId, { type: "IMPLEMENTS" });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`Graph built with ${graph.order} nodes and ${graph.size} edges.`);
|
||||
fs.writeFileSync(GRAPH_FILE, JSON.stringify(graph.export(), null, 2));
|
||||
console.log(`Graph saved to ${GRAPH_FILE}`);
|
||||
}
|
||||
|
||||
buildGraph().catch(console.error);
|
||||
@@ -1,52 +1,63 @@
|
||||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
import { glob } from 'glob';
|
||||
import * as cheerio from 'cheerio';
|
||||
import {
|
||||
Document,
|
||||
VectorStoreIndex,
|
||||
Settings,
|
||||
storageContextFromDefaults,
|
||||
SentenceSplitter
|
||||
} from 'llamaindex';
|
||||
import { OllamaEmbedding } from '@llamaindex/ollama';
|
||||
import fs from "fs";
|
||||
import path from "path";
|
||||
import { glob } from "glob";
|
||||
import * as cheerio from "cheerio";
|
||||
import {
|
||||
Document,
|
||||
VectorStoreIndex,
|
||||
Settings,
|
||||
storageContextFromDefaults,
|
||||
SentenceSplitter,
|
||||
} from "llamaindex";
|
||||
import { OllamaEmbedding } from "@llamaindex/ollama";
|
||||
|
||||
// Configure Settings
|
||||
Settings.embedModel = new OllamaEmbedding({
|
||||
model: "nomic-embed-text-v2-moe",
|
||||
});
|
||||
|
||||
const SPEC_DIR = './spec-built/multipage';
|
||||
const CODE_DIR = './engine262/src';
|
||||
import { STORAGE_DIR } from '../constants.mjs';
|
||||
const SPEC_DIR = "./spec-built/multipage";
|
||||
const CODE_DIR = "./engine262/src";
|
||||
import { STORAGE_DIR } from "../constants.ts";
|
||||
|
||||
// Initialize a SentenceSplitter with even smaller chunk size
|
||||
const sentenceSplitter = new SentenceSplitter({ chunkSize: 256, chunkOverlap: 20 });
|
||||
const sentenceSplitter = new SentenceSplitter({
|
||||
chunkSize: 256,
|
||||
chunkOverlap: 20,
|
||||
});
|
||||
|
||||
async function ingestSpec() {
|
||||
const htmlFiles = await glob(path.join(SPEC_DIR, '*.html'));
|
||||
const htmlFiles = await glob(path.join(SPEC_DIR, "*.html"));
|
||||
const documents = [];
|
||||
|
||||
for (const file of htmlFiles) {
|
||||
const content = fs.readFileSync(file, 'utf-8');
|
||||
const content = fs.readFileSync(file, "utf-8");
|
||||
const $ = cheerio.load(content);
|
||||
|
||||
$('emu-clause').each((i, elem) => {
|
||||
const id = $(elem).attr('id');
|
||||
const title = $(elem).find('h1').first().text().trim();
|
||||
$("emu-clause").each((i, elem) => {
|
||||
const id = $(elem).attr("id");
|
||||
const title = $(elem).find("h1").first().text().trim();
|
||||
// Only extract immediate text to avoid excessive chunking of child sections
|
||||
const text = $(elem).clone().children('emu-clause').remove().end().text().trim();
|
||||
const text = $(elem)
|
||||
.clone()
|
||||
.children("emu-clause")
|
||||
.remove()
|
||||
.end()
|
||||
.text()
|
||||
.trim();
|
||||
|
||||
if (id && title && text) {
|
||||
documents.push(new Document({
|
||||
text,
|
||||
metadata: {
|
||||
source: file,
|
||||
sectionId: id,
|
||||
sectionTitle: title,
|
||||
type: 'specification'
|
||||
}
|
||||
}));
|
||||
documents.push(
|
||||
new Document({
|
||||
text,
|
||||
metadata: {
|
||||
source: file,
|
||||
sectionId: id,
|
||||
sectionTitle: title,
|
||||
type: "specification",
|
||||
},
|
||||
}),
|
||||
);
|
||||
}
|
||||
});
|
||||
}
|
||||
@@ -65,11 +76,13 @@ async function main() {
|
||||
console.log(`Total raw nodes generated: ${rawNodes.length}`);
|
||||
|
||||
// Safety filter to ensure no node exceeds context limit
|
||||
const nodes = rawNodes.filter(node => {
|
||||
const nodes = rawNodes.filter((node) => {
|
||||
const contentLen = node.getContent().length;
|
||||
if (contentLen > 2000) {
|
||||
console.warn(`Skipping node with length ${contentLen} from ${node.metadata.source || 'unknown'}`);
|
||||
return false;
|
||||
console.warn(
|
||||
`Skipping node with length ${contentLen} from ${node.metadata.source || "unknown"}`,
|
||||
);
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
});
|
||||
@@ -81,10 +94,10 @@ async function main() {
|
||||
});
|
||||
|
||||
console.log("Building index (this might take a while with local Ollama)...");
|
||||
|
||||
|
||||
const BATCH_SIZE = 50;
|
||||
let index;
|
||||
|
||||
|
||||
// Try to load existing index if any
|
||||
try {
|
||||
index = await VectorStoreIndex.init({
|
||||
@@ -97,19 +110,17 @@ async function main() {
|
||||
|
||||
for (let i = 0; i < nodes.length; i += BATCH_SIZE) {
|
||||
const batch = nodes.slice(i, i + BATCH_SIZE);
|
||||
console.log(`Processing batch ${i / BATCH_SIZE + 1} / ${Math.ceil(nodes.length / BATCH_SIZE)}...`);
|
||||
|
||||
console.log(
|
||||
`Processing batch ${i / BATCH_SIZE + 1} / ${Math.ceil(nodes.length / BATCH_SIZE)}...`,
|
||||
);
|
||||
|
||||
if (!index) {
|
||||
index = await VectorStoreIndex.init({
|
||||
storageContext,
|
||||
nodes: batch
|
||||
});
|
||||
index = await VectorStoreIndex.init({
|
||||
storageContext,
|
||||
nodes: batch,
|
||||
});
|
||||
} else {
|
||||
// Here we'd ideally skip nodes already in the index,
|
||||
// but for simplicity we'll just continue or assume
|
||||
// we're starting fresh for this run if index was null.
|
||||
// To truly resume, we need more logic.
|
||||
await index.insertNodes(batch);
|
||||
await index.insertNodes(batch);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"target": "ES2022",
|
||||
"module": "esnext",
|
||||
"moduleResolution": "node",
|
||||
"esModuleInterop": true,
|
||||
"allowSyntheticDefaultImports": true,
|
||||
"strict": true,
|
||||
"noEmitOnError": true,
|
||||
"resolveJsonModule": true,
|
||||
"outDir": "./dist",
|
||||
"skipLibCheck": true,
|
||||
"forceConsistentCasingInFileNames": true
|
||||
},
|
||||
"include": ["**/*.ts"],
|
||||
"exclude": ["node_modules", "dist"]
|
||||
}
|
||||
Reference in New Issue
Block a user