bendtherules/ask262
1
0
Fork 0
mirror of https://github.com/bendtherules/ask262.git synced 2026-08-18 13:21:55 +00:00
Code Issues 1 Packages Projects Releases Wiki Activity

build: migrate to Typescript + biome

Browse Source
This commit is contained in:
bendtherules
2026-03-27 17:33:58 +05:30
parent 3f0c4c217a
commit b6dbcfe91a
12 changed files with 466 additions and 204 deletions
Download Patch File Download Diff File Expand all files Collapse all files
setup/build_graph.mjs
-107
View File
@@ -1,107 +0,0 @@
import fs from 'fs';
import path from 'path';
import { glob } from 'glob';
import * as cheerio from 'cheerio';
import { Graph } from 'graphology';
const SPEC_DIR = './spec-built/multipage';
const CODE_DIR = './engine262/src';
import { GRAPH_FILE } from '../constants.mjs';
async function buildGraph() {
const graph = new Graph({ multi: true });
const htmlFiles = await glob(path.join(SPEC_DIR, '*.html'));
console.log(`Found ${htmlFiles.length} specification HTML file(s) in ${SPEC_DIR}`);
if (htmlFiles.length === 0) {
console.warn(`Warning: No specification HTML files found in ${SPEC_DIR}`);
}
console.log("Processing specification for graph...");
for (const file of htmlFiles) {
const fileName = path.basename(file);
const content = fs.readFileSync(file, 'utf-8');
const $ = cheerio.load(content);
$('emu-clause').each((i, elem) => {
const id = $(elem).attr('id');
const title = $(elem).find('h1').first().text().trim();
if (id) {
if (!graph.hasNode(id)) {
graph.addNode(id, { title, type: 'SpecSection', file: fileName });
}
// Extract internal links
$(elem).find('a[href]').each((j, link) => {
const href = $(link).attr('href');
if (href && href.includes('#')) {
const [targetFile, targetId] = href.split('#');
// We only care about links to other sections for now
if (targetId && targetId.startsWith('sec-')) {
// Add relationship later if nodes exist
}
}
});
}
});
}
// Add edges for internal links
for (const file of htmlFiles) {
const content = fs.readFileSync(file, 'utf-8');
const $ = cheerio.load(content);
$('emu-clause').each((i, elem) => {
const sourceId = $(elem).attr('id');
if (sourceId && graph.hasNode(sourceId)) {
$(elem).find('a[href]').each((j, link) => {
const href = $(link).attr('href');
if (href && href.includes('#')) {
const [, targetId] = href.split('#');
if (targetId && graph.hasNode(targetId) && sourceId !== targetId) {
if (!graph.hasEdge(sourceId, targetId)) {
graph.addEdge(sourceId, targetId, { type: 'LINKS_TO' });
}
}
}
});
}
});
}
console.log("Processing code for graph...");
const jsFiles = await glob(path.join(CODE_DIR, '**/*.mts'));
console.log(`Found ${jsFiles.length} code file(s) in ${CODE_DIR}`);
if (jsFiles.length === 0) {
console.warn(`Warning: No code files found in ${CODE_DIR}`);
}
for (const file of jsFiles) {
const fileName = path.basename(file);
const content = fs.readFileSync(file, 'utf-8');
// Simple heuristic: look for function names starting with Evaluate_
const functionMatches = content.matchAll(/export function\*? (Evaluate_([a-zA-Z0-9_]+))/g);
for (const match of functionMatches) {
const fullFuncName = match[1];
const shortName = match[2];
const funcNodeId = `func-${fullFuncName}`;
if (!graph.hasNode(funcNodeId)) {
graph.addNode(funcNodeId, { name: fullFuncName, type: 'JSFunction', file: fileName });
}
// Try to link to a spec section
// Heuristic: Evaluate_IfStatement -> sec-if-statement
const specId = `sec-${shortName.replace(/([a-z0-9])([A-Z])/g, '$1-$2').toLowerCase()}`;
if (graph.hasNode(specId)) {
graph.addEdge(funcNodeId, specId, { type: 'IMPLEMENTS' });
}
}
}
console.log(`Graph built with ${graph.order} nodes and ${graph.size} edges.`);
fs.writeFileSync(GRAPH_FILE, JSON.stringify(graph.export(), null, 2));
console.log(`Graph saved to ${GRAPH_FILE}`);
}
buildGraph().catch(console.error);
setup/build_graph.ts
+123
View File
@@ -0,0 +1,123 @@
import fs from "fs";
import path from "path";
import { glob } from "glob";
import * as cheerio from "cheerio";
import { Graph } from "graphology";
const SPEC_DIR = "./spec-built/multipage";
const CODE_DIR = "./engine262/src";
import { GRAPH_FILE } from "../constants.ts";
async function buildGraph() {
const graph = new Graph({ multi: true });
const htmlFiles = await glob(path.join(SPEC_DIR, "*.html"));
console.log(
`Found ${htmlFiles.length} specification HTML file(s) in ${SPEC_DIR}`,
);
if (htmlFiles.length === 0) {
console.warn(`Warning: No specification HTML files found in ${SPEC_DIR}`);
}
console.log("Processing specification for graph...");
for (const file of htmlFiles) {
const fileName = path.basename(file);
const content = fs.readFileSync(file, "utf-8");
const $ = cheerio.load(content);
$("emu-clause").each((i, elem) => {
const id = $(elem).attr("id");
const title = $(elem).find("h1").first().text().trim();
if (id) {
if (!graph.hasNode(id)) {
graph.addNode(id, { title, type: "SpecSection", file: fileName });
}
// Extract internal links
$(elem)
.find("a[href]")
.each((j, link) => {
const href = $(link).attr("href");
if (href && href.includes("#")) {
const [targetFile, targetId] = href.split("#");
// We only care about links to other sections for now
if (targetId && targetId.startsWith("sec-")) {
// Add relationship later if nodes exist
}
}
});
}
});
}
// Add edges for internal links
for (const file of htmlFiles) {
const content = fs.readFileSync(file, "utf-8");
const $ = cheerio.load(content);
$("emu-clause").each((i, elem) => {
const sourceId = $(elem).attr("id");
if (sourceId && graph.hasNode(sourceId)) {
$(elem)
.find("a[href]")
.each((j, link) => {
const href = $(link).attr("href");
if (href && href.includes("#")) {
const [, targetId] = href.split("#");
if (
targetId &&
graph.hasNode(targetId) &&
sourceId !== targetId
) {
if (!graph.hasEdge(sourceId, targetId)) {
graph.addEdge(sourceId, targetId, { type: "LINKS_TO" });
}
}
}
});
}
});
}
console.log("Processing code for graph...");
const jsFiles = await glob(path.join(CODE_DIR, "**/*.mts"));
console.log(`Found ${jsFiles.length} code file(s) in ${CODE_DIR}`);
if (jsFiles.length === 0) {
console.warn(`Warning: No code files found in ${CODE_DIR}`);
}
for (const file of jsFiles) {
const fileName = path.basename(file);
const content = fs.readFileSync(file, "utf-8");
// Simple heuristic: look for function names starting with Evaluate_
const functionMatches = content.matchAll(
/export function\*? (Evaluate_([a-zA-Z0-9_]+))/g,
);
for (const match of functionMatches) {
const fullFuncName = match[1];
const shortName = match[2];
const funcNodeId = `func-${fullFuncName}`;
if (!graph.hasNode(funcNodeId)) {
graph.addNode(funcNodeId, {
name: fullFuncName,
type: "JSFunction",
file: fileName,
});
}
// Try to link to a spec section
// Heuristic: Evaluate_IfStatement -> sec-if-statement
const specId = `sec-${shortName.replace(/([a-z0-9])([A-Z])/g, "$1-$2").toLowerCase()}`;
if (graph.hasNode(specId)) {
graph.addEdge(funcNodeId, specId, { type: "IMPLEMENTS" });
}
}
}
console.log(`Graph built with ${graph.order} nodes and ${graph.size} edges.`);
fs.writeFileSync(GRAPH_FILE, JSON.stringify(graph.export(), null, 2));
console.log(`Graph saved to ${GRAPH_FILE}`);
}
buildGraph().catch(console.error);
setup/ingest.mjs → setup/ingest.ts
+58 -47
View File
@@ -1,52 +1,63 @@
import fs from 'fs';
import path from 'path';
import { glob } from 'glob';
import * as cheerio from 'cheerio';
import {
Document,
VectorStoreIndex,
Settings,
storageContextFromDefaults,
SentenceSplitter
} from 'llamaindex';
import { OllamaEmbedding } from '@llamaindex/ollama';
import fs from "fs";
import path from "path";
import { glob } from "glob";
import * as cheerio from "cheerio";
import {
Document,
VectorStoreIndex,
Settings,
storageContextFromDefaults,
SentenceSplitter,
} from "llamaindex";
import { OllamaEmbedding } from "@llamaindex/ollama";
// Configure Settings
Settings.embedModel = new OllamaEmbedding({
model: "nomic-embed-text-v2-moe",
});
const SPEC_DIR = './spec-built/multipage';
const CODE_DIR = './engine262/src';
import { STORAGE_DIR } from '../constants.mjs';
const SPEC_DIR = "./spec-built/multipage";
const CODE_DIR = "./engine262/src";
import { STORAGE_DIR } from "../constants.ts";
// Initialize a SentenceSplitter with even smaller chunk size
const sentenceSplitter = new SentenceSplitter({ chunkSize: 256, chunkOverlap: 20 });
const sentenceSplitter = new SentenceSplitter({
chunkSize: 256,
chunkOverlap: 20,
});
async function ingestSpec() {
const htmlFiles = await glob(path.join(SPEC_DIR, '*.html'));
const htmlFiles = await glob(path.join(SPEC_DIR, "*.html"));
const documents = [];
for (const file of htmlFiles) {
const content = fs.readFileSync(file, 'utf-8');
const content = fs.readFileSync(file, "utf-8");
const $ = cheerio.load(content);
$('emu-clause').each((i, elem) => {
const id = $(elem).attr('id');
const title = $(elem).find('h1').first().text().trim();
$("emu-clause").each((i, elem) => {
const id = $(elem).attr("id");
const title = $(elem).find("h1").first().text().trim();
// Only extract immediate text to avoid excessive chunking of child sections
const text = $(elem).clone().children('emu-clause').remove().end().text().trim();
const text = $(elem)
.clone()
.children("emu-clause")
.remove()
.end()
.text()
.trim();
if (id && title && text) {
documents.push(new Document({
text,
metadata: {
source: file,
sectionId: id,
sectionTitle: title,
type: 'specification'
}
}));
documents.push(
new Document({
text,
metadata: {
source: file,
sectionId: id,
sectionTitle: title,
type: "specification",
},
}),
);
}
});
}
@@ -65,11 +76,13 @@ async function main() {
console.log(`Total raw nodes generated: ${rawNodes.length}`);
// Safety filter to ensure no node exceeds context limit
const nodes = rawNodes.filter(node => {
const nodes = rawNodes.filter((node) => {
const contentLen = node.getContent().length;
if (contentLen > 2000) {
console.warn(`Skipping node with length ${contentLen} from ${node.metadata.source || 'unknown'}`);
return false;
console.warn(
`Skipping node with length ${contentLen} from ${node.metadata.source || "unknown"}`,
);
return false;
}
return true;
});
@@ -81,10 +94,10 @@ async function main() {
});
console.log("Building index (this might take a while with local Ollama)...");
const BATCH_SIZE = 50;
let index;
// Try to load existing index if any
try {
index = await VectorStoreIndex.init({
@@ -97,19 +110,17 @@ async function main() {
for (let i = 0; i < nodes.length; i += BATCH_SIZE) {
const batch = nodes.slice(i, i + BATCH_SIZE);
console.log(`Processing batch ${i / BATCH_SIZE + 1} / ${Math.ceil(nodes.length / BATCH_SIZE)}...`);
console.log(
`Processing batch ${i / BATCH_SIZE + 1} / ${Math.ceil(nodes.length / BATCH_SIZE)}...`,
);
if (!index) {
index = await VectorStoreIndex.init({
storageContext,
nodes: batch
});
index = await VectorStoreIndex.init({
storageContext,
nodes: batch,
});
} else {
// Here we'd ideally skip nodes already in the index,
// but for simplicity we'll just continue or assume
// we're starting fresh for this run if index was null.
// To truly resume, we need more logic.
await index.insertNodes(batch);
await index.insertNodes(batch);
}
}