mirror of
https://github.com/bendtherules/ask262.git
synced 2026-08-18 13:21:55 +00:00
build: migrate to Typescript + biome
This commit is contained in:
+22
-18
@@ -1,23 +1,23 @@
|
|||||||
import fs from 'fs';
|
import fs from "node:fs";
|
||||||
import {
|
import {
|
||||||
VectorStoreIndex,
|
VectorStoreIndex,
|
||||||
storageContextFromDefaults,
|
storageContextFromDefaults,
|
||||||
Settings,
|
Settings,
|
||||||
QueryEngineTool,
|
QueryEngineTool,
|
||||||
ReActAgent
|
ReActAgent,
|
||||||
} from 'llamaindex';
|
} from "llamaindex";
|
||||||
import { OllamaEmbedding } from '@llamaindex/ollama';
|
import { OllamaEmbedding } from "@llamaindex/ollama";
|
||||||
import { OpenAI } from '@llamaindex/openai';
|
import { OpenAI } from "@llamaindex/openai";
|
||||||
import { Graph } from 'graphology';
|
import { Graph } from "graphology";
|
||||||
|
|
||||||
import { STORAGE_DIR, GRAPH_FILE } from './constants.mjs';
|
import { STORAGE_DIR, GRAPH_FILE } from "./constants.ts";
|
||||||
|
|
||||||
// Configure Settings
|
// Configure Settings
|
||||||
Settings.embedModel = new OllamaEmbedding({
|
Settings.embedModel = new OllamaEmbedding({
|
||||||
model: "nomic-embed-text-v2-moe",
|
model: "nomic-embed-text-v2-moe",
|
||||||
});
|
});
|
||||||
|
|
||||||
const config = JSON.parse(fs.readFileSync('./config.json', 'utf-8'));
|
const config = JSON.parse(fs.readFileSync("./config.json", "utf-8"));
|
||||||
const apiKey = config.NVIDIA_API_KEY;
|
const apiKey = config.NVIDIA_API_KEY;
|
||||||
const baseURL = config.NVIDIA_API_BASE;
|
const baseURL = config.NVIDIA_API_BASE;
|
||||||
|
|
||||||
@@ -29,7 +29,7 @@ const llm = new OpenAI({
|
|||||||
model: "openai/gpt-oss-120b",
|
model: "openai/gpt-oss-120b",
|
||||||
apiKey: apiKey,
|
apiKey: apiKey,
|
||||||
baseURL: baseURL,
|
baseURL: baseURL,
|
||||||
temperature: 0
|
temperature: 0,
|
||||||
});
|
});
|
||||||
Settings.llm = llm;
|
Settings.llm = llm;
|
||||||
|
|
||||||
@@ -43,7 +43,7 @@ async function main() {
|
|||||||
storageContext,
|
storageContext,
|
||||||
});
|
});
|
||||||
|
|
||||||
const graphData = JSON.parse(fs.readFileSync(GRAPH_FILE, 'utf-8'));
|
const graphData = JSON.parse(fs.readFileSync(GRAPH_FILE, "utf-8"));
|
||||||
const graph = new Graph({ multi: true });
|
const graph = new Graph({ multi: true });
|
||||||
graph.import(graphData);
|
graph.import(graphData);
|
||||||
|
|
||||||
@@ -53,14 +53,16 @@ async function main() {
|
|||||||
queryEngine,
|
queryEngine,
|
||||||
metadata: {
|
metadata: {
|
||||||
name: "spec_retriever",
|
name: "spec_retriever",
|
||||||
description: "Queries the language specification for text content about specific sections or topics. Use this to get the detailed text of a section.",
|
description:
|
||||||
|
"Queries the language specification for text content about specific sections or topics. Use this to get the detailed text of a section.",
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
|
|
||||||
const graphTool = {
|
const graphTool = {
|
||||||
metadata: {
|
metadata: {
|
||||||
name: "graph_explorer",
|
name: "graph_explorer",
|
||||||
description: "Explores structural relationships between specification sections and implementation code (functions). Use this to find which spec section a function implements. Input: section ID or function name.",
|
description:
|
||||||
|
"Explores structural relationships between specification sections and implementation code (functions). Use this to find which spec section a function implements. Input: section ID or function name.",
|
||||||
parameters: {
|
parameters: {
|
||||||
type: "object",
|
type: "object",
|
||||||
properties: {
|
properties: {
|
||||||
@@ -88,16 +90,16 @@ async function main() {
|
|||||||
if (nodeAttr.title) result += `- Title: ${nodeAttr.title}\n`;
|
if (nodeAttr.title) result += `- Title: ${nodeAttr.title}\n`;
|
||||||
if (nodeAttr.file) result += `- File: ${nodeAttr.file}\n`;
|
if (nodeAttr.file) result += `- File: ${nodeAttr.file}\n`;
|
||||||
result += `\nConnected parts:\n`;
|
result += `\nConnected parts:\n`;
|
||||||
neighbors.forEach(neighbor => {
|
neighbors.forEach((neighbor) => {
|
||||||
const attr = graph.getNodeAttributes(neighbor);
|
const attr = graph.getNodeAttributes(neighbor);
|
||||||
const edges = graph.edges(nodeId, neighbor);
|
const edges = graph.edges(nodeId, neighbor);
|
||||||
const edgeAttr = graph.getEdgeAttributes(edges[0]);
|
const edgeAttr = graph.getEdgeAttributes(edges[0]);
|
||||||
result += `- ${neighbor} (${attr.type}) via ${edgeAttr.type}${attr.title ? ': ' + attr.title : ''}\n`;
|
result += `- ${neighbor} (${attr.type}) via ${edgeAttr.type}${attr.title ? ": " + attr.title : ""}\n`;
|
||||||
});
|
});
|
||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
return `No information found in graph for ${query}. Use spec_retriever to search text.`;
|
return `No information found in graph for ${query}. Use spec_retriever to search text.`;
|
||||||
}
|
},
|
||||||
};
|
};
|
||||||
|
|
||||||
const agent = new ReActAgent({
|
const agent = new ReActAgent({
|
||||||
@@ -113,16 +115,18 @@ CRITICAL INSTRUCTIONS:
|
|||||||
3. If the user asks about a function, you MUST first use 'graph_explorer' to find the associated specification section.
|
3. If the user asks about a function, you MUST first use 'graph_explorer' to find the associated specification section.
|
||||||
4. You MUST then use 'spec_retriever' to read the actual text of that specification section before answering.
|
4. You MUST then use 'spec_retriever' to read the actual text of that specification section before answering.
|
||||||
5. Base your explanations ONLY on the information retrieved from the tools.
|
5. Base your explanations ONLY on the information retrieved from the tools.
|
||||||
6. If the tools do not provide enough information, state that clearly rather than guessing from your internal knowledge.`
|
6. If the tools do not provide enough information, state that clearly rather than guessing from your internal knowledge.`,
|
||||||
});
|
});
|
||||||
|
|
||||||
console.log("Agent is ready!");
|
console.log("Agent is ready!");
|
||||||
|
|
||||||
const message = process.argv[2] || "Which spec section does Evaluate_IfStatement implement? and what does that section say?";
|
const message =
|
||||||
|
process.argv[2] ||
|
||||||
|
"Which spec section does Evaluate_IfStatement implement? and what does that section say?";
|
||||||
console.log(`User: ${message}`);
|
console.log(`User: ${message}`);
|
||||||
|
|
||||||
const response = await agent.chat({
|
const response = await agent.chat({
|
||||||
message: message
|
message: message,
|
||||||
});
|
});
|
||||||
|
|
||||||
console.log("\n--- Agent Response ---\n");
|
console.log("\n--- Agent Response ---\n");
|
||||||
@@ -0,0 +1,9 @@
|
|||||||
|
{
|
||||||
|
"formatter": {
|
||||||
|
"indentStyle": "space",
|
||||||
|
"indentWidth": 2,
|
||||||
|
"lineWidth": 80,
|
||||||
|
"lineEnding": "lf"
|
||||||
|
},
|
||||||
|
"files": { "includes": ["**", "!!spec-built/**", "!!engine262/**"] }
|
||||||
|
}
|
||||||
@@ -0,0 +1,197 @@
|
|||||||
|
{
|
||||||
|
"lockfileVersion": 1,
|
||||||
|
"configVersion": 0,
|
||||||
|
"workspaces": {
|
||||||
|
"": {
|
||||||
|
"name": "gemini-3-1-pro-preview",
|
||||||
|
"dependencies": {
|
||||||
|
"@llamaindex/core": "^0.6.22",
|
||||||
|
"@llamaindex/env": "^0.1.30",
|
||||||
|
"@llamaindex/node-parser": "^2.0.22",
|
||||||
|
"@llamaindex/ollama": "^0.1.23",
|
||||||
|
"@llamaindex/openai": "^0.4.22",
|
||||||
|
"acorn": "^8.16.0",
|
||||||
|
"cheerio": "^1.2.0",
|
||||||
|
"glob": "^13.0.6",
|
||||||
|
"graphology": "^0.26.0",
|
||||||
|
"llamaindex": "^0.12.1",
|
||||||
|
},
|
||||||
|
"devDependencies": {
|
||||||
|
"@biomejs/biome": "^2.4.9",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
"packages": {
|
||||||
|
"@aws-crypto/sha256-js": ["@aws-crypto/sha256-js@5.2.0", "", { "dependencies": { "@aws-crypto/util": "^5.2.0", "@aws-sdk/types": "^3.222.0", "tslib": "^2.6.2" } }, "sha512-FFQQyu7edu4ufvIZ+OadFpHHOt+eSTBaYaki44c+akjg7qZg9oOQeLlk77F6tSYqjDAFClrHJk9tMf0HdVyOvA=="],
|
||||||
|
|
||||||
|
"@aws-crypto/util": ["@aws-crypto/util@5.2.0", "", { "dependencies": { "@aws-sdk/types": "^3.222.0", "@smithy/util-utf8": "^2.0.0", "tslib": "^2.6.2" } }, "sha512-4RkU9EsI6ZpBve5fseQlGNUWKMa1RLPQ1dnjnQoe07ldfIzcsGb5hC5W0Dm7u423KWzawlrpbjXBrXCEv9zazQ=="],
|
||||||
|
|
||||||
|
"@aws-sdk/types": ["@aws-sdk/types@3.973.6", "", { "dependencies": { "@smithy/types": "^4.13.1", "tslib": "^2.6.2" } }, "sha512-Atfcy4E++beKtwJHiDln2Nby8W/mam64opFPTiHEqgsthqeydFS1pY+OUlN1ouNOmf8ArPU/6cDS65anOP3KQw=="],
|
||||||
|
|
||||||
|
"@biomejs/biome": ["@biomejs/biome@2.4.9", "", { "optionalDependencies": { "@biomejs/cli-darwin-arm64": "2.4.9", "@biomejs/cli-darwin-x64": "2.4.9", "@biomejs/cli-linux-arm64": "2.4.9", "@biomejs/cli-linux-arm64-musl": "2.4.9", "@biomejs/cli-linux-x64": "2.4.9", "@biomejs/cli-linux-x64-musl": "2.4.9", "@biomejs/cli-win32-arm64": "2.4.9", "@biomejs/cli-win32-x64": "2.4.9" }, "bin": { "biome": "bin/biome" } }, "sha512-wvZW92FrwitTcacvCBT8xdAbfbxWfDLwjYMmU3djjqQTh7Ni4ZdiWIT/x5VcZ+RQuxiKzIOzi5D+dcyJDFZMsA=="],
|
||||||
|
|
||||||
|
"@biomejs/cli-darwin-arm64": ["@biomejs/cli-darwin-arm64@2.4.9", "", { "os": "darwin", "cpu": "arm64" }, "sha512-d5G8Gf2RpH5pYwiHLPA+UpG3G9TLQu4WM+VK6sfL7K68AmhcEQ9r+nkj/DvR/GYhYox6twsHUtmWWWIKfcfQQA=="],
|
||||||
|
|
||||||
|
"@biomejs/cli-darwin-x64": ["@biomejs/cli-darwin-x64@2.4.9", "", { "os": "darwin", "cpu": "x64" }, "sha512-LNCLNgqDMG7BLdc3a8aY/dwKPK7+R8/JXJoXjCvZh2gx8KseqBdFDKbhrr7HCWF8SzNhbTaALhTBoh/I6rf9lA=="],
|
||||||
|
|
||||||
|
"@biomejs/cli-linux-arm64": ["@biomejs/cli-linux-arm64@2.4.9", "", { "os": "linux", "cpu": "arm64" }, "sha512-4adnkAUi6K4C/emPRgYznMOcLlUqZdXWM6aIui4VP4LraE764g6Q4YguygnAUoxKjKIXIWPteKMgRbN0wsgwcg=="],
|
||||||
|
|
||||||
|
"@biomejs/cli-linux-arm64-musl": ["@biomejs/cli-linux-arm64-musl@2.4.9", "", { "os": "linux", "cpu": "arm64" }, "sha512-8RCww5xnPn2wpK4L/QDGDOW0dq80uVWfppPxHIUg6mOs9B6gRmqPp32h1Ls3T8GnW8Wo5A8u7vpTwz4fExN+sw=="],
|
||||||
|
|
||||||
|
"@biomejs/cli-linux-x64": ["@biomejs/cli-linux-x64@2.4.9", "", { "os": "linux", "cpu": "x64" }, "sha512-L10na7POF0Ks/cgLFNF1ZvIe+X4onLkTi5oP9hY+Rh60Q+7fWzKDDCeGyiHUFf1nGIa9dQOOUPGe2MyYg8nMSQ=="],
|
||||||
|
|
||||||
|
"@biomejs/cli-linux-x64-musl": ["@biomejs/cli-linux-x64-musl@2.4.9", "", { "os": "linux", "cpu": "x64" }, "sha512-5TD+WS9v5vzXKzjetF0hgoaNFHMcpQeBUwKKVi3JbG1e9UCrFuUK3Gt185fyTzvRdwYkJJEMqglRPjmesmVv4A=="],
|
||||||
|
|
||||||
|
"@biomejs/cli-win32-arm64": ["@biomejs/cli-win32-arm64@2.4.9", "", { "os": "win32", "cpu": "arm64" }, "sha512-aDZr0RBC3sMGJOU10BvG7eZIlWLK/i51HRIfScE2lVhfts2dQTreowLiJJd+UYg/tHKxS470IbzpuKmd0MiD6g=="],
|
||||||
|
|
||||||
|
"@biomejs/cli-win32-x64": ["@biomejs/cli-win32-x64@2.4.9", "", { "os": "win32", "cpu": "x64" }, "sha512-NS4g/2G9SoQ4ktKtz31pvyc/rmgzlcIDCGU/zWbmHJAqx6gcRj2gj5Q/guXhoWTzCUaQZDIqiCQXHS7BcGYc0w=="],
|
||||||
|
|
||||||
|
"@finom/zod-to-json-schema": ["@finom/zod-to-json-schema@3.24.11", "", { "peerDependencies": { "zod": "^4.0.14" } }, "sha512-fL656yBPiWebtfGItvtXLWrFNGlF1NcDFS0WdMQXMs9LluVg0CfT5E2oXYp0pidl0vVG53XkW55ysijNkU5/hA=="],
|
||||||
|
|
||||||
|
"@llamaindex/core": ["@llamaindex/core@0.6.22", "", { "dependencies": { "@finom/zod-to-json-schema": "3.24.11", "@llamaindex/env": "0.1.30", "@types/node": "^24.0.13", "magic-bytes.js": "^1.10.0", "zod": "^4.1.5" } }, "sha512-/BXyemkvpxMaUhOkbwJ2PTvzKjSWkL8+6QLpz/n+pk8xBwMMe1GVBgli/J57gCyi8GbrlBafBj6GaPOgWub2Eg=="],
|
||||||
|
|
||||||
|
"@llamaindex/env": ["@llamaindex/env@0.1.30", "", { "dependencies": { "@aws-crypto/sha256-js": "^5.2.0", "js-tiktoken": "^1.0.12", "pathe": "^1.1.2" }, "peerDependencies": { "@huggingface/transformers": "^3.5.0", "gpt-tokenizer": "^2.5.0" }, "optionalPeers": ["@huggingface/transformers", "gpt-tokenizer"] }, "sha512-y6kutMcCevzbmexUgz+HXf7KiZemzAoFEYSjAILfR+cG6FmYSF8XvLbGOB34Kx8mlRi7EI8rZXpezJ5qCqOyZg=="],
|
||||||
|
|
||||||
|
"@llamaindex/node-parser": ["@llamaindex/node-parser@2.0.22", "", { "dependencies": { "html-to-text": "^9.0.5" }, "peerDependencies": { "@llamaindex/core": "0.6.22", "@llamaindex/env": "0.1.30", "tree-sitter": "^0.22.0", "web-tree-sitter": "^0.24.3" } }, "sha512-uj5O89WShAAyiSZ8f8tU7hnLJ6pSmlY2a6hkAOs8odkUgT87dEqaPHpsK7w0iJdEFiob7GoLeRhv2K624FooXg=="],
|
||||||
|
|
||||||
|
"@llamaindex/ollama": ["@llamaindex/ollama@0.1.23", "", { "dependencies": { "ollama": "^0.5.16", "remeda": "^2.17.3" }, "peerDependencies": { "@llamaindex/core": "0.6.22", "@llamaindex/env": "0.1.30" } }, "sha512-Tk5TnnPhjBnedWrJA1TCdhC/aJAJscAw/FDQG89fa/7m9p53XviI14K0a0cYLYxqVqDUh9J/KI/JYmYwYjhPmg=="],
|
||||||
|
|
||||||
|
"@llamaindex/openai": ["@llamaindex/openai@0.4.22", "", { "dependencies": { "openai": "^5.12.0" }, "peerDependencies": { "@llamaindex/core": "0.6.22", "@llamaindex/env": "0.1.30" } }, "sha512-jVaSscK7kyBM0wj3vceG2HPnFreSuOvkBYGFovFdS1heCyahlGDoQ1cLAUWqQ/J9Njfrt/PZTp3ICP+APl+zig=="],
|
||||||
|
|
||||||
|
"@llamaindex/workflow": ["@llamaindex/workflow@1.1.24", "", { "dependencies": { "@llamaindex/workflow-core": "^1.3.2" }, "peerDependencies": { "@llamaindex/core": "0.6.22", "@llamaindex/env": "0.1.30" } }, "sha512-VyKsbRkFlnT5dRNKbgLXQV+ZpQ+CAFgmC9LaZv6hD/fIKo6wq1wQW/ZqLZgZt569xeHgxmrXPB6KHdqn/AhPbQ=="],
|
||||||
|
|
||||||
|
"@llamaindex/workflow-core": ["@llamaindex/workflow-core@1.3.3", "", { "peerDependencies": { "@modelcontextprotocol/sdk": "^1.7.0", "hono": "^4.7.4", "next": "^15.2.2", "p-retry": "^6.2.1", "rxjs": "^7.8.2", "zod": "^3.25.0 || ^4.0.0" }, "optionalPeers": ["@modelcontextprotocol/sdk", "hono", "next", "p-retry", "rxjs"] }, "sha512-WJIcD4K2suGbNkwU5CC70jKKrA5tARba42nMs8Pou1RGzmoxqg+K+b7vyLBmiDtImR8P40YLmkayCIRVQPBmsg=="],
|
||||||
|
|
||||||
|
"@selderee/plugin-htmlparser2": ["@selderee/plugin-htmlparser2@0.11.0", "", { "dependencies": { "domhandler": "^5.0.3", "selderee": "^0.11.0" } }, "sha512-P33hHGdldxGabLFjPPpaTxVolMrzrcegejx+0GxjrIb9Zv48D8yAIA/QTDR2dFl7Uz7urX8aX6+5bCZslr+gWQ=="],
|
||||||
|
|
||||||
|
"@smithy/is-array-buffer": ["@smithy/is-array-buffer@2.2.0", "", { "dependencies": { "tslib": "^2.6.2" } }, "sha512-GGP3O9QFD24uGeAXYUjwSTXARoqpZykHadOmA8G5vfJPK0/DC67qa//0qvqrJzL1xc8WQWX7/yc7fwudjPHPhA=="],
|
||||||
|
|
||||||
|
"@smithy/types": ["@smithy/types@4.13.1", "", { "dependencies": { "tslib": "^2.6.2" } }, "sha512-787F3yzE2UiJIQ+wYW1CVg2odHjmaWLGksnKQHUrK/lYZSEcy1msuLVvxaR/sI2/aDe9U+TBuLsXnr3vod1g0g=="],
|
||||||
|
|
||||||
|
"@smithy/util-buffer-from": ["@smithy/util-buffer-from@2.2.0", "", { "dependencies": { "@smithy/is-array-buffer": "^2.2.0", "tslib": "^2.6.2" } }, "sha512-IJdWBbTcMQ6DA0gdNhh/BwrLkDR+ADW5Kr1aZmd4k3DIF6ezMV4R2NIAmT08wQJ3yUK82thHWmC/TnK/wpMMIA=="],
|
||||||
|
|
||||||
|
"@smithy/util-utf8": ["@smithy/util-utf8@2.3.0", "", { "dependencies": { "@smithy/util-buffer-from": "^2.2.0", "tslib": "^2.6.2" } }, "sha512-R8Rdn8Hy72KKcebgLiv8jQcQkXoLMOGGv5uI1/k0l+snqkOzQ1R0ChUBCxWMlBsFMekWjq0wRudIweFs7sKT5A=="],
|
||||||
|
|
||||||
|
"@types/lodash": ["@types/lodash@4.17.24", "", {}, "sha512-gIW7lQLZbue7lRSWEFql49QJJWThrTFFeIMJdp3eH4tKoxm1OvEPg02rm4wCCSHS0cL3/Fizimb35b7k8atwsQ=="],
|
||||||
|
|
||||||
|
"@types/node": ["@types/node@24.12.0", "", { "dependencies": { "undici-types": "~7.16.0" } }, "sha512-GYDxsZi3ChgmckRT9HPU0WEhKLP08ev/Yfcq2AstjrDASOYCSXeyjDsHg4v5t4jOj7cyDX3vmprafKlWIG9MXQ=="],
|
||||||
|
|
||||||
|
"acorn": ["acorn@8.16.0", "", { "bin": "bin/acorn" }, "sha512-UVJyE9MttOsBQIDKw1skb9nAwQuR5wuGD3+82K6JgJlm/Y+KI92oNsMNGZCYdDsVtRHSak0pcV5Dno5+4jh9sw=="],
|
||||||
|
|
||||||
|
"balanced-match": ["balanced-match@4.0.4", "", {}, "sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA=="],
|
||||||
|
|
||||||
|
"base64-js": ["base64-js@1.5.1", "", {}, "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA=="],
|
||||||
|
|
||||||
|
"boolbase": ["boolbase@1.0.0", "", {}, "sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww=="],
|
||||||
|
|
||||||
|
"brace-expansion": ["brace-expansion@5.0.5", "", { "dependencies": { "balanced-match": "^4.0.2" } }, "sha512-VZznLgtwhn+Mact9tfiwx64fA9erHH/MCXEUfB/0bX/6Fz6ny5EGTXYltMocqg4xFAQZtnO3DHWWXi8RiuN7cQ=="],
|
||||||
|
|
||||||
|
"cheerio": ["cheerio@1.2.0", "", { "dependencies": { "cheerio-select": "^2.1.0", "dom-serializer": "^2.0.0", "domhandler": "^5.0.3", "domutils": "^3.2.2", "encoding-sniffer": "^0.2.1", "htmlparser2": "^10.1.0", "parse5": "^7.3.0", "parse5-htmlparser2-tree-adapter": "^7.1.0", "parse5-parser-stream": "^7.1.2", "undici": "^7.19.0", "whatwg-mimetype": "^4.0.0" } }, "sha512-WDrybc/gKFpTYQutKIK6UvfcuxijIZfMfXaYm8NMsPQxSYvf+13fXUJ4rztGGbJcBQ/GF55gvrZ0Bc0bj/mqvg=="],
|
||||||
|
|
||||||
|
"cheerio-select": ["cheerio-select@2.1.0", "", { "dependencies": { "boolbase": "^1.0.0", "css-select": "^5.1.0", "css-what": "^6.1.0", "domelementtype": "^2.3.0", "domhandler": "^5.0.3", "domutils": "^3.0.1" } }, "sha512-9v9kG0LvzrlcungtnJtpGNxY+fzECQKhK4EGJX2vByejiMX84MFNQw4UxPJl3bFbTMw+Dfs37XaIkCwTZfLh4g=="],
|
||||||
|
|
||||||
|
"css-select": ["css-select@5.2.2", "", { "dependencies": { "boolbase": "^1.0.0", "css-what": "^6.1.0", "domhandler": "^5.0.2", "domutils": "^3.0.1", "nth-check": "^2.0.1" } }, "sha512-TizTzUddG/xYLA3NXodFM0fSbNizXjOKhqiQQwvhlspadZokn1KDy0NZFS0wuEubIYAV5/c1/lAr0TaaFXEXzw=="],
|
||||||
|
|
||||||
|
"css-what": ["css-what@6.2.2", "", {}, "sha512-u/O3vwbptzhMs3L1fQE82ZSLHQQfto5gyZzwteVIEyeaY5Fc7R4dapF/BvRoSYFeqfBk4m0V1Vafq5Pjv25wvA=="],
|
||||||
|
|
||||||
|
"deepmerge": ["deepmerge@4.3.1", "", {}, "sha512-3sUqbMEc77XqpdNO7FRyRog+eW3ph+GYCbj+rK+uYyRMuwsVy0rMiVtPn+QJlKFvWP/1PYpapqYn0Me2knFn+A=="],
|
||||||
|
|
||||||
|
"dom-serializer": ["dom-serializer@2.0.0", "", { "dependencies": { "domelementtype": "^2.3.0", "domhandler": "^5.0.2", "entities": "^4.2.0" } }, "sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg=="],
|
||||||
|
|
||||||
|
"domelementtype": ["domelementtype@2.3.0", "", {}, "sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw=="],
|
||||||
|
|
||||||
|
"domhandler": ["domhandler@5.0.3", "", { "dependencies": { "domelementtype": "^2.3.0" } }, "sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w=="],
|
||||||
|
|
||||||
|
"domutils": ["domutils@3.2.2", "", { "dependencies": { "dom-serializer": "^2.0.0", "domelementtype": "^2.3.0", "domhandler": "^5.0.3" } }, "sha512-6kZKyUajlDuqlHKVX1w7gyslj9MPIXzIFiz/rGu35uC1wMi+kMhQwGhl4lt9unC9Vb9INnY9Z3/ZA3+FhASLaw=="],
|
||||||
|
|
||||||
|
"encoding-sniffer": ["encoding-sniffer@0.2.1", "", { "dependencies": { "iconv-lite": "^0.6.3", "whatwg-encoding": "^3.1.1" } }, "sha512-5gvq20T6vfpekVtqrYQsSCFZ1wEg5+wW0/QaZMWkFr6BqD3NfKs0rLCx4rrVlSWJeZb5NBJgVLswK/w2MWU+Gw=="],
|
||||||
|
|
||||||
|
"entities": ["entities@4.5.0", "", {}, "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw=="],
|
||||||
|
|
||||||
|
"events": ["events@3.3.0", "", {}, "sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q=="],
|
||||||
|
|
||||||
|
"glob": ["glob@13.0.6", "", { "dependencies": { "minimatch": "^10.2.2", "minipass": "^7.1.3", "path-scurry": "^2.0.2" } }, "sha512-Wjlyrolmm8uDpm/ogGyXZXb1Z+Ca2B8NbJwqBVg0axK9GbBeoS7yGV6vjXnYdGm6X53iehEuxxbyiKp8QmN4Vw=="],
|
||||||
|
|
||||||
|
"graphology": ["graphology@0.26.0", "", { "dependencies": { "events": "^3.3.0" }, "peerDependencies": { "graphology-types": ">=0.24.0" } }, "sha512-8SSImzgUUYC89Z042s+0r/vMibY7GX/Emz4LDO5e7jYXhuoWfHISPFJYjpRLUSJGq6UQ6xlenvX1p/hJdfXuXg=="],
|
||||||
|
|
||||||
|
"graphology-types": ["graphology-types@0.24.8", "", {}, "sha512-hDRKYXa8TsoZHjgEaysSRyPdT6uB78Ci8WnjgbStlQysz7xR52PInxNsmnB7IBOM1BhikxkNyCVEFgmPKnpx3Q=="],
|
||||||
|
|
||||||
|
"html-to-text": ["html-to-text@9.0.5", "", { "dependencies": { "@selderee/plugin-htmlparser2": "^0.11.0", "deepmerge": "^4.3.1", "dom-serializer": "^2.0.0", "htmlparser2": "^8.0.2", "selderee": "^0.11.0" } }, "sha512-qY60FjREgVZL03vJU6IfMV4GDjGBIoOyvuFdpBDIX9yTlDw0TjxVBQp+P8NvpdIXNJvfWBTNul7fsAQJq2FNpg=="],
|
||||||
|
|
||||||
|
"htmlparser2": ["htmlparser2@10.1.0", "", { "dependencies": { "domelementtype": "^2.3.0", "domhandler": "^5.0.3", "domutils": "^3.2.2", "entities": "^7.0.1" } }, "sha512-VTZkM9GWRAtEpveh7MSF6SjjrpNVNNVJfFup7xTY3UpFtm67foy9HDVXneLtFVt4pMz5kZtgNcvCniNFb1hlEQ=="],
|
||||||
|
|
||||||
|
"iconv-lite": ["iconv-lite@0.6.3", "", { "dependencies": { "safer-buffer": ">= 2.1.2 < 3.0.0" } }, "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw=="],
|
||||||
|
|
||||||
|
"js-tiktoken": ["js-tiktoken@1.0.21", "", { "dependencies": { "base64-js": "^1.5.1" } }, "sha512-biOj/6M5qdgx5TKjDnFT1ymSpM5tbd3ylwDtrQvFQSu0Z7bBYko2dF+W/aUkXUPuk6IVpRxk/3Q2sHOzGlS36g=="],
|
||||||
|
|
||||||
|
"leac": ["leac@0.6.0", "", {}, "sha512-y+SqErxb8h7nE/fiEX07jsbuhrpO9lL8eca7/Y1nuWV2moNlXhyd59iDGcRf6moVyDMbmTNzL40SUyrFU/yDpg=="],
|
||||||
|
|
||||||
|
"llamaindex": ["llamaindex@0.12.1", "", { "dependencies": { "@llamaindex/core": "0.6.22", "@llamaindex/env": "0.1.30", "@llamaindex/node-parser": "2.0.22", "@llamaindex/workflow": "1.1.24", "@types/lodash": "^4.17.7", "@types/node": "^24.0.13", "lodash": "^4.17.21", "magic-bytes.js": "^1.10.0" } }, "sha512-/tXXITk/iVGBycOFaDhev6dgTBIr6Ycu4FoPIt6A5JcEAiB6ujONjiV36flVXUR8JdqwMtS767XMjV+36nV4yQ=="],
|
||||||
|
|
||||||
|
"lodash": ["lodash@4.17.23", "", {}, "sha512-LgVTMpQtIopCi79SJeDiP0TfWi5CNEc/L/aRdTh3yIvmZXTnheWpKjSZhnvMl8iXbC1tFg9gdHHDMLoV7CnG+w=="],
|
||||||
|
|
||||||
|
"lru-cache": ["lru-cache@11.2.7", "", {}, "sha512-aY/R+aEsRelme17KGQa/1ZSIpLpNYYrhcrepKTZgE+W3WM16YMCaPwOHLHsmopZHELU0Ojin1lPVxKR0MihncA=="],
|
||||||
|
|
||||||
|
"magic-bytes.js": ["magic-bytes.js@1.13.0", "", {}, "sha512-afO2mnxW7GDTXMm5/AoN1WuOcdoKhtgXjIvHmobqTD1grNplhGdv3PFOyjCVmrnOZBIT/gD/koDKpYG+0mvHcg=="],
|
||||||
|
|
||||||
|
"minimatch": ["minimatch@10.2.4", "", { "dependencies": { "brace-expansion": "^5.0.2" } }, "sha512-oRjTw/97aTBN0RHbYCdtF1MQfvusSIBQM0IZEgzl6426+8jSC0nF1a/GmnVLpfB9yyr6g6FTqWqiZVbxrtaCIg=="],
|
||||||
|
|
||||||
|
"minipass": ["minipass@7.1.3", "", {}, "sha512-tEBHqDnIoM/1rXME1zgka9g6Q2lcoCkxHLuc7ODJ5BxbP5d4c2Z5cGgtXAku59200Cx7diuHTOYfSBD8n6mm8A=="],
|
||||||
|
|
||||||
|
"node-addon-api": ["node-addon-api@8.7.0", "", {}, "sha512-9MdFxmkKaOYVTV+XVRG8ArDwwQ77XIgIPyKASB1k3JPq3M8fGQQQE3YpMOrKm6g//Ktx8ivZr8xo1Qmtqub+GA=="],
|
||||||
|
|
||||||
|
"node-gyp-build": ["node-gyp-build@4.8.4", "", { "bin": { "node-gyp-build": "bin.js", "node-gyp-build-optional": "optional.js", "node-gyp-build-test": "build-test.js" } }, "sha512-LA4ZjwlnUblHVgq0oBF3Jl/6h/Nvs5fzBLwdEF4nuxnFdsfajde4WfxtJr3CaiH+F6ewcIB/q4jQ4UzPyid+CQ=="],
|
||||||
|
|
||||||
|
"nth-check": ["nth-check@2.1.1", "", { "dependencies": { "boolbase": "^1.0.0" } }, "sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w=="],
|
||||||
|
|
||||||
|
"ollama": ["ollama@0.5.18", "", { "dependencies": { "whatwg-fetch": "^3.6.20" } }, "sha512-lTFqTf9bo7Cd3hpF6CviBe/DEhewjoZYd9N/uCe7O20qYTvGqrNOFOBDj3lbZgFWHUgDv5EeyusYxsZSLS8nvg=="],
|
||||||
|
|
||||||
|
"openai": ["openai@5.23.2", "", { "peerDependencies": { "ws": "^8.18.0", "zod": "^3.23.8" }, "optionalPeers": ["ws"], "bin": "bin/cli" }, "sha512-MQBzmTulj+MM5O8SKEk/gL8a7s5mktS9zUtAkU257WjvobGc9nKcBuVwjyEEcb9SI8a8Y2G/mzn3vm9n1Jlleg=="],
|
||||||
|
|
||||||
|
"parse5": ["parse5@7.3.0", "", { "dependencies": { "entities": "^6.0.0" } }, "sha512-IInvU7fabl34qmi9gY8XOVxhYyMyuH2xUNpb2q8/Y+7552KlejkRvqvD19nMoUW/uQGGbqNpA6Tufu5FL5BZgw=="],
|
||||||
|
|
||||||
|
"parse5-htmlparser2-tree-adapter": ["parse5-htmlparser2-tree-adapter@7.1.0", "", { "dependencies": { "domhandler": "^5.0.3", "parse5": "^7.0.0" } }, "sha512-ruw5xyKs6lrpo9x9rCZqZZnIUntICjQAd0Wsmp396Ul9lN/h+ifgVV1x1gZHi8euej6wTfpqX8j+BFQxF0NS/g=="],
|
||||||
|
|
||||||
|
"parse5-parser-stream": ["parse5-parser-stream@7.1.2", "", { "dependencies": { "parse5": "^7.0.0" } }, "sha512-JyeQc9iwFLn5TbvvqACIF/VXG6abODeB3Fwmv/TGdLk2LfbWkaySGY72at4+Ty7EkPZj854u4CrICqNk2qIbow=="],
|
||||||
|
|
||||||
|
"parseley": ["parseley@0.12.1", "", { "dependencies": { "leac": "^0.6.0", "peberminta": "^0.9.0" } }, "sha512-e6qHKe3a9HWr0oMRVDTRhKce+bRO8VGQR3NyVwcjwrbhMmFCX9KszEV35+rn4AdilFAq9VPxP/Fe1wC9Qjd2lw=="],
|
||||||
|
|
||||||
|
"path-scurry": ["path-scurry@2.0.2", "", { "dependencies": { "lru-cache": "^11.0.0", "minipass": "^7.1.2" } }, "sha512-3O/iVVsJAPsOnpwWIeD+d6z/7PmqApyQePUtCndjatj/9I5LylHvt5qluFaBT3I5h3r1ejfR056c+FCv+NnNXg=="],
|
||||||
|
|
||||||
|
"pathe": ["pathe@1.1.2", "", {}, "sha512-whLdWMYL2TwI08hn8/ZqAbrVemu0LNaNNJZX73O6qaIdCTfXutsLhMkjdENX0qhsQ9uIimo4/aQOmXkoon2nDQ=="],
|
||||||
|
|
||||||
|
"peberminta": ["peberminta@0.9.0", "", {}, "sha512-XIxfHpEuSJbITd1H3EeQwpcZbTLHc+VVr8ANI9t5sit565tsI4/xK3KWTUFE2e6QiangUkh3B0jihzmGnNrRsQ=="],
|
||||||
|
|
||||||
|
"remeda": ["remeda@2.33.6", "", {}, "sha512-tazDGH7s75kUPGBKLvhgBEHMgW+TdDFhjUAMdQj57IoWz6HsGa5D2RX5yDUz6IIqiRRvZiaEHzCzWdTeixc/Kg=="],
|
||||||
|
|
||||||
|
"safer-buffer": ["safer-buffer@2.1.2", "", {}, "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg=="],
|
||||||
|
|
||||||
|
"selderee": ["selderee@0.11.0", "", { "dependencies": { "parseley": "^0.12.0" } }, "sha512-5TF+l7p4+OsnP8BCCvSyZiSPc4x4//p5uPwK8TCnVPJYRmU2aYKMpOXvw8zM5a5JvuuCGN1jmsMwuU2W02ukfA=="],
|
||||||
|
|
||||||
|
"tree-sitter": ["tree-sitter@0.22.4", "", { "dependencies": { "node-addon-api": "^8.3.0", "node-gyp-build": "^4.8.4" } }, "sha512-usbHZP9/oxNsUY65MQUsduGRqDHQOou1cagUSwjhoSYAmSahjQDAVsh9s+SlZkn8X8+O1FULRGwHu7AFP3kjzg=="],
|
||||||
|
|
||||||
|
"tslib": ["tslib@2.8.1", "", {}, "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w=="],
|
||||||
|
|
||||||
|
"undici": ["undici@7.24.6", "", {}, "sha512-Xi4agocCbRzt0yYMZGMA6ApD7gvtUFaxm4ZmeacWI4cZxaF6C+8I8QfofC20NAePiB/IcvZmzkJ7XPa471AEtA=="],
|
||||||
|
|
||||||
|
"undici-types": ["undici-types@7.16.0", "", {}, "sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw=="],
|
||||||
|
|
||||||
|
"web-tree-sitter": ["web-tree-sitter@0.24.7", "", {}, "sha512-CdC/TqVFbXqR+C51v38hv6wOPatKEUGxa39scAeFSm98wIhZxAYonhRQPSMmfZ2w7JDI0zQDdzdmgtNk06/krQ=="],
|
||||||
|
|
||||||
|
"whatwg-encoding": ["whatwg-encoding@3.1.1", "", { "dependencies": { "iconv-lite": "0.6.3" } }, "sha512-6qN4hJdMwfYBtE3YBTTHhoeuUrDBPZmbQaxWAqSALV/MeEnR5z1xd8UKud2RAkFoPkmB+hli1TZSnyi84xz1vQ=="],
|
||||||
|
|
||||||
|
"whatwg-fetch": ["whatwg-fetch@3.6.20", "", {}, "sha512-EqhiFU6daOA8kpjOWTL0olhVOF3i7OrFzSYiGsEMB8GcXS+RrzauAERX65xMeNWVqxA6HXH2m69Z9LaKKdisfg=="],
|
||||||
|
|
||||||
|
"whatwg-mimetype": ["whatwg-mimetype@4.0.0", "", {}, "sha512-QaKxh0eNIi2mE9p2vEdzfagOKHCcj1pJ56EEHGQOVxp8r9/iszLUUV7v89x9O1p/T+NlTM5W7jW6+cz4Fq1YVg=="],
|
||||||
|
|
||||||
|
"zod": ["zod@4.3.6", "", {}, "sha512-rftlrkhHZOcjDwkGlnUtZZkvaPHCsDATp4pGpuOOMDaTdDDXF91wuVDJoWoPsKX/3YPQ5fHuF3STjcYyKr+Qhg=="],
|
||||||
|
|
||||||
|
"html-to-text/htmlparser2": ["htmlparser2@8.0.2", "", { "dependencies": { "domelementtype": "^2.3.0", "domhandler": "^5.0.3", "domutils": "^3.0.1", "entities": "^4.4.0" } }, "sha512-GYdjWKDkbRLkZ5geuHs5NY1puJ+PXwP7+fHPRz06Eirsb9ugf6d8kkXav6ADhcODhFFPMIXyxkxSuMf3D6NCFA=="],
|
||||||
|
|
||||||
|
"htmlparser2/entities": ["entities@7.0.1", "", {}, "sha512-TWrgLOFUQTH994YUyl1yT4uyavY5nNB5muff+RtWaqNVCAK408b5ZnnbNAUEWLTCpum9w6arT70i1XdQ4UeOPA=="],
|
||||||
|
|
||||||
|
"openai/zod": ["zod@3.25.76", "", {}, "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ=="],
|
||||||
|
|
||||||
|
"parse5/entities": ["entities@6.0.1", "", {}, "sha512-aN97NXWF6AWBTahfVOIrB/NShkzi5H7F9r1s9mD3cDj4Ko5f2qhhVoYMibXF7GlLveb/D2ioWay8lxI97Ven3g=="],
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,2 +0,0 @@
|
|||||||
export const STORAGE_DIR = './storage';
|
|
||||||
export const GRAPH_FILE = './graphology/graph.json';
|
|
||||||
@@ -0,0 +1,2 @@
|
|||||||
|
export const STORAGE_DIR = "./storage";
|
||||||
|
export const GRAPH_FILE = "./graphology/graph.json";
|
||||||
Vendored
+5
-5
@@ -1,20 +1,20 @@
|
|||||||
declare module '@unicode/unicode-16.0.0/Binary_Property/ID_Start/regex.js' {
|
declare module "@unicode/unicode-16.0.0/Binary_Property/ID_Start/regex.js" {
|
||||||
let regex: RegExp;
|
let regex: RegExp;
|
||||||
export default regex;
|
export default regex;
|
||||||
}
|
}
|
||||||
declare module '@unicode/unicode-16.0.0/Binary_Property/ID_Continue/regex.js' {
|
declare module "@unicode/unicode-16.0.0/Binary_Property/ID_Continue/regex.js" {
|
||||||
let regex: RegExp;
|
let regex: RegExp;
|
||||||
export default regex;
|
export default regex;
|
||||||
}
|
}
|
||||||
declare module '@unicode/unicode-16.0.0/General_Category/Space_Separator/regex.js' {
|
declare module "@unicode/unicode-16.0.0/General_Category/Space_Separator/regex.js" {
|
||||||
let regex: RegExp;
|
let regex: RegExp;
|
||||||
export default regex;
|
export default regex;
|
||||||
}
|
}
|
||||||
declare module '@unicode/unicode-16.0.0/Case_Folding/C/symbols.js' {
|
declare module "@unicode/unicode-16.0.0/Case_Folding/C/symbols.js" {
|
||||||
let data: Map<string, string>;
|
let data: Map<string, string>;
|
||||||
export default data;
|
export default data;
|
||||||
}
|
}
|
||||||
declare module '@unicode/unicode-16.0.0/Case_Folding/S/symbols.js' {
|
declare module "@unicode/unicode-16.0.0/Case_Folding/S/symbols.js" {
|
||||||
let data: Map<string, string>;
|
let data: Map<string, string>;
|
||||||
export default data;
|
export default data;
|
||||||
}
|
}
|
||||||
|
|||||||
Vendored
+2
-2
@@ -1,5 +1,5 @@
|
|||||||
// eslint-disable-next-line no-unused-vars
|
// eslint-disable-next-line no-unused-vars
|
||||||
interface SyntaxError {
|
interface SyntaxError {
|
||||||
decoration?: string
|
decoration?: string;
|
||||||
position?: number
|
position?: number;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -3,6 +3,11 @@
|
|||||||
"version": "1.0.0",
|
"version": "1.0.0",
|
||||||
"main": "index.js",
|
"main": "index.js",
|
||||||
"scripts": {
|
"scripts": {
|
||||||
|
"lint": "biome check .",
|
||||||
|
"format": "biome format .",
|
||||||
|
"ingest": "bun run setup/ingest.ts",
|
||||||
|
"agent": "bun run agent.ts",
|
||||||
|
"build": "bun run setup/build_graph.ts",
|
||||||
"test": "echo \"Error: no test specified\" && exit 1"
|
"test": "echo \"Error: no test specified\" && exit 1"
|
||||||
},
|
},
|
||||||
"keywords": [],
|
"keywords": [],
|
||||||
@@ -21,5 +26,8 @@
|
|||||||
"glob": "^13.0.6",
|
"glob": "^13.0.6",
|
||||||
"graphology": "^0.26.0",
|
"graphology": "^0.26.0",
|
||||||
"llamaindex": "^0.12.1"
|
"llamaindex": "^0.12.1"
|
||||||
|
},
|
||||||
|
"devDependencies": {
|
||||||
|
"@biomejs/biome": "^2.4.9"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1,107 +0,0 @@
|
|||||||
import fs from 'fs';
|
|
||||||
import path from 'path';
|
|
||||||
import { glob } from 'glob';
|
|
||||||
import * as cheerio from 'cheerio';
|
|
||||||
import { Graph } from 'graphology';
|
|
||||||
|
|
||||||
const SPEC_DIR = './spec-built/multipage';
|
|
||||||
const CODE_DIR = './engine262/src';
|
|
||||||
import { GRAPH_FILE } from '../constants.mjs';
|
|
||||||
|
|
||||||
async function buildGraph() {
|
|
||||||
const graph = new Graph({ multi: true });
|
|
||||||
const htmlFiles = await glob(path.join(SPEC_DIR, '*.html'));
|
|
||||||
console.log(`Found ${htmlFiles.length} specification HTML file(s) in ${SPEC_DIR}`);
|
|
||||||
if (htmlFiles.length === 0) {
|
|
||||||
console.warn(`Warning: No specification HTML files found in ${SPEC_DIR}`);
|
|
||||||
}
|
|
||||||
|
|
||||||
console.log("Processing specification for graph...");
|
|
||||||
for (const file of htmlFiles) {
|
|
||||||
const fileName = path.basename(file);
|
|
||||||
const content = fs.readFileSync(file, 'utf-8');
|
|
||||||
const $ = cheerio.load(content);
|
|
||||||
|
|
||||||
$('emu-clause').each((i, elem) => {
|
|
||||||
const id = $(elem).attr('id');
|
|
||||||
const title = $(elem).find('h1').first().text().trim();
|
|
||||||
|
|
||||||
if (id) {
|
|
||||||
if (!graph.hasNode(id)) {
|
|
||||||
graph.addNode(id, { title, type: 'SpecSection', file: fileName });
|
|
||||||
}
|
|
||||||
|
|
||||||
// Extract internal links
|
|
||||||
$(elem).find('a[href]').each((j, link) => {
|
|
||||||
const href = $(link).attr('href');
|
|
||||||
if (href && href.includes('#')) {
|
|
||||||
const [targetFile, targetId] = href.split('#');
|
|
||||||
// We only care about links to other sections for now
|
|
||||||
if (targetId && targetId.startsWith('sec-')) {
|
|
||||||
// Add relationship later if nodes exist
|
|
||||||
}
|
|
||||||
}
|
|
||||||
});
|
|
||||||
}
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
// Add edges for internal links
|
|
||||||
for (const file of htmlFiles) {
|
|
||||||
const content = fs.readFileSync(file, 'utf-8');
|
|
||||||
const $ = cheerio.load(content);
|
|
||||||
$('emu-clause').each((i, elem) => {
|
|
||||||
const sourceId = $(elem).attr('id');
|
|
||||||
if (sourceId && graph.hasNode(sourceId)) {
|
|
||||||
$(elem).find('a[href]').each((j, link) => {
|
|
||||||
const href = $(link).attr('href');
|
|
||||||
if (href && href.includes('#')) {
|
|
||||||
const [, targetId] = href.split('#');
|
|
||||||
if (targetId && graph.hasNode(targetId) && sourceId !== targetId) {
|
|
||||||
if (!graph.hasEdge(sourceId, targetId)) {
|
|
||||||
graph.addEdge(sourceId, targetId, { type: 'LINKS_TO' });
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
});
|
|
||||||
}
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
console.log("Processing code for graph...");
|
|
||||||
const jsFiles = await glob(path.join(CODE_DIR, '**/*.mts'));
|
|
||||||
console.log(`Found ${jsFiles.length} code file(s) in ${CODE_DIR}`);
|
|
||||||
if (jsFiles.length === 0) {
|
|
||||||
console.warn(`Warning: No code files found in ${CODE_DIR}`);
|
|
||||||
}
|
|
||||||
|
|
||||||
for (const file of jsFiles) {
|
|
||||||
const fileName = path.basename(file);
|
|
||||||
const content = fs.readFileSync(file, 'utf-8');
|
|
||||||
|
|
||||||
// Simple heuristic: look for function names starting with Evaluate_
|
|
||||||
const functionMatches = content.matchAll(/export function\*? (Evaluate_([a-zA-Z0-9_]+))/g);
|
|
||||||
for (const match of functionMatches) {
|
|
||||||
const fullFuncName = match[1];
|
|
||||||
const shortName = match[2];
|
|
||||||
const funcNodeId = `func-${fullFuncName}`;
|
|
||||||
|
|
||||||
if (!graph.hasNode(funcNodeId)) {
|
|
||||||
graph.addNode(funcNodeId, { name: fullFuncName, type: 'JSFunction', file: fileName });
|
|
||||||
}
|
|
||||||
|
|
||||||
// Try to link to a spec section
|
|
||||||
// Heuristic: Evaluate_IfStatement -> sec-if-statement
|
|
||||||
const specId = `sec-${shortName.replace(/([a-z0-9])([A-Z])/g, '$1-$2').toLowerCase()}`;
|
|
||||||
if (graph.hasNode(specId)) {
|
|
||||||
graph.addEdge(funcNodeId, specId, { type: 'IMPLEMENTS' });
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
console.log(`Graph built with ${graph.order} nodes and ${graph.size} edges.`);
|
|
||||||
fs.writeFileSync(GRAPH_FILE, JSON.stringify(graph.export(), null, 2));
|
|
||||||
console.log(`Graph saved to ${GRAPH_FILE}`);
|
|
||||||
}
|
|
||||||
|
|
||||||
buildGraph().catch(console.error);
|
|
||||||
@@ -0,0 +1,123 @@
|
|||||||
|
import fs from "fs";
|
||||||
|
import path from "path";
|
||||||
|
import { glob } from "glob";
|
||||||
|
import * as cheerio from "cheerio";
|
||||||
|
import { Graph } from "graphology";
|
||||||
|
|
||||||
|
const SPEC_DIR = "./spec-built/multipage";
|
||||||
|
const CODE_DIR = "./engine262/src";
|
||||||
|
import { GRAPH_FILE } from "../constants.ts";
|
||||||
|
|
||||||
|
async function buildGraph() {
|
||||||
|
const graph = new Graph({ multi: true });
|
||||||
|
const htmlFiles = await glob(path.join(SPEC_DIR, "*.html"));
|
||||||
|
console.log(
|
||||||
|
`Found ${htmlFiles.length} specification HTML file(s) in ${SPEC_DIR}`,
|
||||||
|
);
|
||||||
|
if (htmlFiles.length === 0) {
|
||||||
|
console.warn(`Warning: No specification HTML files found in ${SPEC_DIR}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log("Processing specification for graph...");
|
||||||
|
for (const file of htmlFiles) {
|
||||||
|
const fileName = path.basename(file);
|
||||||
|
const content = fs.readFileSync(file, "utf-8");
|
||||||
|
const $ = cheerio.load(content);
|
||||||
|
|
||||||
|
$("emu-clause").each((i, elem) => {
|
||||||
|
const id = $(elem).attr("id");
|
||||||
|
const title = $(elem).find("h1").first().text().trim();
|
||||||
|
|
||||||
|
if (id) {
|
||||||
|
if (!graph.hasNode(id)) {
|
||||||
|
graph.addNode(id, { title, type: "SpecSection", file: fileName });
|
||||||
|
}
|
||||||
|
|
||||||
|
// Extract internal links
|
||||||
|
$(elem)
|
||||||
|
.find("a[href]")
|
||||||
|
.each((j, link) => {
|
||||||
|
const href = $(link).attr("href");
|
||||||
|
if (href && href.includes("#")) {
|
||||||
|
const [targetFile, targetId] = href.split("#");
|
||||||
|
// We only care about links to other sections for now
|
||||||
|
if (targetId && targetId.startsWith("sec-")) {
|
||||||
|
// Add relationship later if nodes exist
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
// Add edges for internal links
|
||||||
|
for (const file of htmlFiles) {
|
||||||
|
const content = fs.readFileSync(file, "utf-8");
|
||||||
|
const $ = cheerio.load(content);
|
||||||
|
$("emu-clause").each((i, elem) => {
|
||||||
|
const sourceId = $(elem).attr("id");
|
||||||
|
if (sourceId && graph.hasNode(sourceId)) {
|
||||||
|
$(elem)
|
||||||
|
.find("a[href]")
|
||||||
|
.each((j, link) => {
|
||||||
|
const href = $(link).attr("href");
|
||||||
|
if (href && href.includes("#")) {
|
||||||
|
const [, targetId] = href.split("#");
|
||||||
|
if (
|
||||||
|
targetId &&
|
||||||
|
graph.hasNode(targetId) &&
|
||||||
|
sourceId !== targetId
|
||||||
|
) {
|
||||||
|
if (!graph.hasEdge(sourceId, targetId)) {
|
||||||
|
graph.addEdge(sourceId, targetId, { type: "LINKS_TO" });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log("Processing code for graph...");
|
||||||
|
const jsFiles = await glob(path.join(CODE_DIR, "**/*.mts"));
|
||||||
|
console.log(`Found ${jsFiles.length} code file(s) in ${CODE_DIR}`);
|
||||||
|
if (jsFiles.length === 0) {
|
||||||
|
console.warn(`Warning: No code files found in ${CODE_DIR}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
for (const file of jsFiles) {
|
||||||
|
const fileName = path.basename(file);
|
||||||
|
const content = fs.readFileSync(file, "utf-8");
|
||||||
|
|
||||||
|
// Simple heuristic: look for function names starting with Evaluate_
|
||||||
|
const functionMatches = content.matchAll(
|
||||||
|
/export function\*? (Evaluate_([a-zA-Z0-9_]+))/g,
|
||||||
|
);
|
||||||
|
for (const match of functionMatches) {
|
||||||
|
const fullFuncName = match[1];
|
||||||
|
const shortName = match[2];
|
||||||
|
const funcNodeId = `func-${fullFuncName}`;
|
||||||
|
|
||||||
|
if (!graph.hasNode(funcNodeId)) {
|
||||||
|
graph.addNode(funcNodeId, {
|
||||||
|
name: fullFuncName,
|
||||||
|
type: "JSFunction",
|
||||||
|
file: fileName,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
// Try to link to a spec section
|
||||||
|
// Heuristic: Evaluate_IfStatement -> sec-if-statement
|
||||||
|
const specId = `sec-${shortName.replace(/([a-z0-9])([A-Z])/g, "$1-$2").toLowerCase()}`;
|
||||||
|
if (graph.hasNode(specId)) {
|
||||||
|
graph.addEdge(funcNodeId, specId, { type: "IMPLEMENTS" });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log(`Graph built with ${graph.order} nodes and ${graph.size} edges.`);
|
||||||
|
fs.writeFileSync(GRAPH_FILE, JSON.stringify(graph.export(), null, 2));
|
||||||
|
console.log(`Graph saved to ${GRAPH_FILE}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
buildGraph().catch(console.error);
|
||||||
@@ -1,52 +1,63 @@
|
|||||||
import fs from 'fs';
|
import fs from "fs";
|
||||||
import path from 'path';
|
import path from "path";
|
||||||
import { glob } from 'glob';
|
import { glob } from "glob";
|
||||||
import * as cheerio from 'cheerio';
|
import * as cheerio from "cheerio";
|
||||||
import {
|
import {
|
||||||
Document,
|
Document,
|
||||||
VectorStoreIndex,
|
VectorStoreIndex,
|
||||||
Settings,
|
Settings,
|
||||||
storageContextFromDefaults,
|
storageContextFromDefaults,
|
||||||
SentenceSplitter
|
SentenceSplitter,
|
||||||
} from 'llamaindex';
|
} from "llamaindex";
|
||||||
import { OllamaEmbedding } from '@llamaindex/ollama';
|
import { OllamaEmbedding } from "@llamaindex/ollama";
|
||||||
|
|
||||||
// Configure Settings
|
// Configure Settings
|
||||||
Settings.embedModel = new OllamaEmbedding({
|
Settings.embedModel = new OllamaEmbedding({
|
||||||
model: "nomic-embed-text-v2-moe",
|
model: "nomic-embed-text-v2-moe",
|
||||||
});
|
});
|
||||||
|
|
||||||
const SPEC_DIR = './spec-built/multipage';
|
const SPEC_DIR = "./spec-built/multipage";
|
||||||
const CODE_DIR = './engine262/src';
|
const CODE_DIR = "./engine262/src";
|
||||||
import { STORAGE_DIR } from '../constants.mjs';
|
import { STORAGE_DIR } from "../constants.ts";
|
||||||
|
|
||||||
// Initialize a SentenceSplitter with even smaller chunk size
|
// Initialize a SentenceSplitter with even smaller chunk size
|
||||||
const sentenceSplitter = new SentenceSplitter({ chunkSize: 256, chunkOverlap: 20 });
|
const sentenceSplitter = new SentenceSplitter({
|
||||||
|
chunkSize: 256,
|
||||||
|
chunkOverlap: 20,
|
||||||
|
});
|
||||||
|
|
||||||
async function ingestSpec() {
|
async function ingestSpec() {
|
||||||
const htmlFiles = await glob(path.join(SPEC_DIR, '*.html'));
|
const htmlFiles = await glob(path.join(SPEC_DIR, "*.html"));
|
||||||
const documents = [];
|
const documents = [];
|
||||||
|
|
||||||
for (const file of htmlFiles) {
|
for (const file of htmlFiles) {
|
||||||
const content = fs.readFileSync(file, 'utf-8');
|
const content = fs.readFileSync(file, "utf-8");
|
||||||
const $ = cheerio.load(content);
|
const $ = cheerio.load(content);
|
||||||
|
|
||||||
$('emu-clause').each((i, elem) => {
|
$("emu-clause").each((i, elem) => {
|
||||||
const id = $(elem).attr('id');
|
const id = $(elem).attr("id");
|
||||||
const title = $(elem).find('h1').first().text().trim();
|
const title = $(elem).find("h1").first().text().trim();
|
||||||
// Only extract immediate text to avoid excessive chunking of child sections
|
// Only extract immediate text to avoid excessive chunking of child sections
|
||||||
const text = $(elem).clone().children('emu-clause').remove().end().text().trim();
|
const text = $(elem)
|
||||||
|
.clone()
|
||||||
|
.children("emu-clause")
|
||||||
|
.remove()
|
||||||
|
.end()
|
||||||
|
.text()
|
||||||
|
.trim();
|
||||||
|
|
||||||
if (id && title && text) {
|
if (id && title && text) {
|
||||||
documents.push(new Document({
|
documents.push(
|
||||||
|
new Document({
|
||||||
text,
|
text,
|
||||||
metadata: {
|
metadata: {
|
||||||
source: file,
|
source: file,
|
||||||
sectionId: id,
|
sectionId: id,
|
||||||
sectionTitle: title,
|
sectionTitle: title,
|
||||||
type: 'specification'
|
type: "specification",
|
||||||
}
|
},
|
||||||
}));
|
}),
|
||||||
|
);
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
@@ -65,10 +76,12 @@ async function main() {
|
|||||||
console.log(`Total raw nodes generated: ${rawNodes.length}`);
|
console.log(`Total raw nodes generated: ${rawNodes.length}`);
|
||||||
|
|
||||||
// Safety filter to ensure no node exceeds context limit
|
// Safety filter to ensure no node exceeds context limit
|
||||||
const nodes = rawNodes.filter(node => {
|
const nodes = rawNodes.filter((node) => {
|
||||||
const contentLen = node.getContent().length;
|
const contentLen = node.getContent().length;
|
||||||
if (contentLen > 2000) {
|
if (contentLen > 2000) {
|
||||||
console.warn(`Skipping node with length ${contentLen} from ${node.metadata.source || 'unknown'}`);
|
console.warn(
|
||||||
|
`Skipping node with length ${contentLen} from ${node.metadata.source || "unknown"}`,
|
||||||
|
);
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
return true;
|
return true;
|
||||||
@@ -97,18 +110,16 @@ async function main() {
|
|||||||
|
|
||||||
for (let i = 0; i < nodes.length; i += BATCH_SIZE) {
|
for (let i = 0; i < nodes.length; i += BATCH_SIZE) {
|
||||||
const batch = nodes.slice(i, i + BATCH_SIZE);
|
const batch = nodes.slice(i, i + BATCH_SIZE);
|
||||||
console.log(`Processing batch ${i / BATCH_SIZE + 1} / ${Math.ceil(nodes.length / BATCH_SIZE)}...`);
|
console.log(
|
||||||
|
`Processing batch ${i / BATCH_SIZE + 1} / ${Math.ceil(nodes.length / BATCH_SIZE)}...`,
|
||||||
|
);
|
||||||
|
|
||||||
if (!index) {
|
if (!index) {
|
||||||
index = await VectorStoreIndex.init({
|
index = await VectorStoreIndex.init({
|
||||||
storageContext,
|
storageContext,
|
||||||
nodes: batch
|
nodes: batch,
|
||||||
});
|
});
|
||||||
} else {
|
} else {
|
||||||
// Here we'd ideally skip nodes already in the index,
|
|
||||||
// but for simplicity we'll just continue or assume
|
|
||||||
// we're starting fresh for this run if index was null.
|
|
||||||
// To truly resume, we need more logic.
|
|
||||||
await index.insertNodes(batch);
|
await index.insertNodes(batch);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -0,0 +1,17 @@
|
|||||||
|
{
|
||||||
|
"compilerOptions": {
|
||||||
|
"target": "ES2022",
|
||||||
|
"module": "esnext",
|
||||||
|
"moduleResolution": "node",
|
||||||
|
"esModuleInterop": true,
|
||||||
|
"allowSyntheticDefaultImports": true,
|
||||||
|
"strict": true,
|
||||||
|
"noEmitOnError": true,
|
||||||
|
"resolveJsonModule": true,
|
||||||
|
"outDir": "./dist",
|
||||||
|
"skipLibCheck": true,
|
||||||
|
"forceConsistentCasingInFileNames": true
|
||||||
|
},
|
||||||
|
"include": ["**/*.ts"],
|
||||||
|
"exclude": ["node_modules", "dist"]
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user