build: EvaluateInEngine262 tool

.opencode/package.json

@@ -3,4 +3,4 @@
   "dependencies": {
     "@opencode-ai/plugin": "1.3.17"
   }
-}
+}

AGENTS.md

@@ -29,6 +29,7 @@ ask262/
 │   │   ├── specRetriever.ts  # Vector search tool
 │   │   ├── sectionRetriever.ts   # Section chunk retrieval tool
 │   │   ├── graphExplorer.ts  # Knowledge graph navigation tool
+│   │   ├── evaluateInEngine262.ts  # Execute JS and capture spec marks
 │   │   └── reranker.ts       # Document reranking utility
 │   └── setup/                # Data ingestion and graph building
 │       ├── ingest.ts         # Ingests spec HTML into vector index
@@ -39,8 +40,9 @@ ask262/
 │       └── utils/            # Formatting utilities
 │   └── test/                 # Manual verification tests
 │       └── manual/
-│           ├── verify-db.ts  # Verify database contents
-│           └── test-spec-retriever.ts
+│           ├── verify-db.ts              # Verify database contents
+│           ├── test-spec-retriever.ts    # Test spec retriever tool
+│           └── test-evaluate-in-engine262.ts  # Test evaluate tool
 ├── config.json               # API keys and endpoints (user-created)
 ├── spec-built/multipage/     # ECMAScript spec HTML files
 ├── engine262/src/            # JavaScript engine implementation
@@ -61,6 +63,8 @@ bun run lint:fix        # Fix auto-fixable issues
 bun run format:fix      # Format code with Biome
 bun run type-check      # TypeScript check (no emit)
 bun test                # Run all tests
+bun run test-evaluate   # Test evaluate in engine262 tool
+bun run test-spec-retriever "query"  # Test spec retriever tool with query
 bun run agent "Query"   # Run agent with question
 ```
 

package.json

@@ -11,6 +11,8 @@
     "format:fix": "biome format . --write",
     "ingest": "bun run src/setup/ingest.ts",
     "verify-db": "bun run src/test/manual/verify-db.ts",
+    "test-evaluate": "bun run src/test/manual/test-evaluate-in-engine262.ts",
+    "test-spec-retriever": "bun run src/test/manual/test-spec-retriever.ts",
     "agent": "bun run src/agent.ts",
     "build": "bun run src/setup/buildGraph.ts",
     "test": "bun test"

src/agent-tools/evaluateInEngine262.ts

@@ -0,0 +1,154 @@
+/**
+ * Evaluate JavaScript code in engine262 and capture spec section marks.
+ * Executes code in the engine262 JavaScript engine and returns the captured
+ * ECMAScript spec section marks as JSON.
+ */
+
+import { DynamicStructuredTool } from "@langchain/core/tools";
+import { z } from "zod";
+
+const evaluateSchema = z.object({
+  code: z.string().describe("JavaScript code to execute in engine262"),
+});
+
+// Type definitions for engine262 module
+interface MarkData {
+  readonly sectionIds: string[];
+  readonly fileRelativePath: string;
+  readonly lineNumber: number;
+  readonly important: boolean;
+}
+
+// biome-ignore lint/suspicious/noExplicitAny: engine262 is an external module without TypeScript types
+let engine262Module: any = null;
+
+/**
+ * Lazy load engine262 module
+ */
+async function loadEngine262() {
+  if (!engine262Module) {
+    // Dynamic import of engine262 from local path
+    engine262Module = await import("../../engine262/lib/engine262.mjs");
+  }
+  return engine262Module;
+}
+
+/**
+ * Creates the evaluateInEngine262 tool.
+ * Executes JavaScript code in engine262 and captures spec section marks.
+ * @returns The evaluate_in_engine262 tool instance
+ */
+export function createEvaluateInEngine262Tool() {
+  return new DynamicStructuredTool({
+    name: "evaluate_in_engine262",
+    description:
+      "Executes JavaScript code in the engine262 JavaScript engine and captures which ECMAScript specification sections are hit during execution. Returns the full marks array as JSON. Useful for understanding how specific JavaScript operations map to the ECMAScript spec.",
+    schema: evaluateSchema,
+    func: async ({ code }) => {
+      console.log(`[Tool: evaluate_in_engine262] Executing code...`);
+
+      try {
+        const engine = await loadEngine262();
+        const ask262Debug = engine.ask262Debug as {
+          marks: MarkData[];
+          startTrace: () => void;
+          stopTrace: () => void;
+          startImportant: () => void;
+          stopImportant: () => void;
+        };
+        const Agent = engine.Agent;
+        const ManagedRealm = engine.ManagedRealm;
+        const setSurroundingAgent = engine.setSurroundingAgent;
+        const OrdinaryObjectCreate = engine.OrdinaryObjectCreate;
+        const CreateBuiltinFunction = engine.CreateBuiltinFunction;
+        const CreateDataProperty = engine.CreateDataProperty;
+        const Value = engine.Value;
+        const skipDebugger = engine.skipDebugger;
+
+        // Reset marks from previous runs
+        ask262Debug.marks = [];
+
+        // Set up agent and realm
+        const agent = new Agent();
+        setSurroundingAgent(agent);
+        const realm = new ManagedRealm();
+
+        // Expose ask262Debug controls to the evaluated code
+        realm.scope(() => {
+          const debugObj = OrdinaryObjectCreate(
+            agent.intrinsic("%Object.prototype%"),
+          );
+          skipDebugger(
+            CreateDataProperty(
+              realm.GlobalObject,
+              Value("ask262Debug"),
+              debugObj,
+            ),
+          );
+
+          const startImportant = CreateBuiltinFunction(
+            () => {
+              ask262Debug.startImportant();
+              return Value.undefined;
+            },
+            0,
+            Value("startImportant"),
+            [],
+          );
+          skipDebugger(
+            CreateDataProperty(
+              debugObj,
+              Value("startImportant"),
+              startImportant,
+            ),
+          );
+
+          const stopImportant = CreateBuiltinFunction(
+            () => {
+              ask262Debug.stopImportant();
+              return Value.undefined;
+            },
+            0,
+            Value("stopImportant"),
+            [],
+          );
+          skipDebugger(
+            CreateDataProperty(debugObj, Value("stopImportant"), stopImportant),
+          );
+        });
+
+        // Start tracing
+        ask262Debug.startTrace();
+
+        // Execute the code
+        realm.evaluateScript(code);
+
+        // Stop tracing
+        ask262Debug.stopTrace();
+
+        // Get captured marks
+        const marks = ask262Debug.marks;
+
+        console.log(
+          `[Tool: evaluate_in_engine262] Captured ${marks.length} unique marks`,
+        );
+
+        // Filter and group marks by important flag
+        const importantMarks = marks.filter((m) => m.important);
+        const otherMarks = marks.filter((m) => !m.important);
+
+        // Extract sectionIds, remove fileRelativePath and lineNumber
+        const result = {
+          importantSections: importantMarks.map((m) => m.sectionIds),
+          otherSections: otherMarks.map((m) => m.sectionIds),
+        };
+
+        // Return compressed JSON
+        return JSON.stringify(result);
+      } catch (error) {
+        console.error(`[Tool: evaluate_in_engine262] Error: ${error}`);
+        return `Error executing code in engine262: ${error instanceof Error ? error.message : String(error)}`;
+      }
+    },
+  });
+}

src/agent-tools/index.ts

@@ -3,6 +3,7 @@
  * Exports all tool factory functions and utilities.
  */
 
+export { createEvaluateInEngine262Tool } from "./evaluateInEngine262";
 export { createGraphExplorerTool } from "./graphExplorer";
 export { type RerankResult, rerankDocuments } from "./reranker";
 export { createSectionRetrieverTool } from "./sectionRetriever";

src/agent.ts

@@ -7,6 +7,7 @@ import { ChatOpenAI } from "@langchain/openai";
 import Graph from "graphology";
 import { AgentExecutor, createReactAgent } from "langchain/agents";
 import {
+  createEvaluateInEngine262Tool,
   createGraphExplorerTool,
   createSectionRetrieverTool,
   createSpecRetrieverTool,
@@ -44,12 +45,13 @@ Available tools: {tool_names}
 {tools}
 
 CRITICAL INSTRUCTIONS:
-1. ALWAYS prefer using the provided tools ('spec_retriever', 'fetch_section_chunks', and 'graph_explorer') to answer questions.
+1. ALWAYS prefer using the provided tools ('spec_retriever', 'fetch_section_chunks', 'graph_explorer', and 'evaluate_in_engine262') to answer questions.
 2. Do NOT rely on your internal knowledge of JavaScript or the ECMAScript specification.
 3. If the user asks about a function, you MUST first use 'graph_explorer' to find the associated specification section.
 4. You MUST then use 'fetch_section_chunks' to read the actual text of that specification section before answering.
-5. Base your explanations ONLY on the information retrieved from the tools.
-6. If the tools do not provide enough information, state that clearly rather than guessing from your internal knowledge.
+5. When the user provides JavaScript code or asks about runtime behavior, use 'evaluate_in_engine262' to execute the code and see which spec sections are hit during execution.
+6. Base your explanations ONLY on the information retrieved from the tools.
+7. If the tools do not provide enough information, state that clearly rather than guessing from your internal knowledge.
 
 {agent_scratchpad}`;
 
@@ -73,16 +75,17 @@ async function main() {
   const specRetrieverTool = createSpecRetrieverTool(table, embeddings);
   const sectionRetrieverTool = createSectionRetrieverTool(table);
   const graphTool = createGraphExplorerTool(graph);
+  const evaluateTool = createEvaluateInEngine262Tool();
 
   const agent = await createReactAgent({
     llm,
-    tools: [specRetrieverTool, sectionRetrieverTool, graphTool],
+    tools: [specRetrieverTool, sectionRetrieverTool, graphTool, evaluateTool],
     prompt,
   });
 
   const agentExecutor = new AgentExecutor({
     agent,
-    tools: [specRetrieverTool, sectionRetrieverTool, graphTool],
+    tools: [specRetrieverTool, sectionRetrieverTool, graphTool, evaluateTool],
   });
 
   console.log("Agent is ready!");

src/test/manual/test-evaluate-in-engine262.ts

@@ -0,0 +1,81 @@
+#!/usr/bin/env bun
+/**
+ * Manual test script for evaluateInEngine262 agent tool.
+ * Tests that the tool properly executes JavaScript code in engine262
+ * and captures spec section marks.
+ *
+ * Usage: bun run src/test/manual/test-evaluate-in-engine262.ts ["your JavaScript code"]
+ */
+
+import { createEvaluateInEngine262Tool } from "../../agent-tools";
+
+interface EvaluateResult {
+  importantSections: string[][];
+  otherSections: string[][];
+}
+
+async function main() {
+  // Get test code from command line or use default
+  const testCode =
+    process.argv[2] ||
+    `
+    // Test Array.prototype.every
+    [1, 2, 3].every(x => x > 0);
+    
+    ask262Debug.startImportant();
+    // Test Proxy creation
+    new Proxy({}, {});
+    ask262Debug.stopImportant();
+  `;
+
+  console.log("=== Testing evaluateInEngine262 Tool ===\n");
+  console.log("Test code:");
+  console.log("---");
+  console.log(testCode);
+  console.log("---\n");
+
+  console.log("Creating tool...\n");
+  const evaluateTool = createEvaluateInEngine262Tool();
+
+  console.log("Executing tool...\n");
+  try {
+    const result = await evaluateTool.func({ code: testCode });
+
+    // Parse and verify results
+    const parsed: EvaluateResult = JSON.parse(result);
+
+    const importantCount = parsed.importantSections.length;
+    const otherCount = parsed.otherSections.length;
+    const totalCount = importantCount + otherCount;
+
+    console.log(`\n✓ Captured ${totalCount} marks`);
+    console.log(`  (${importantCount} important, ${otherCount} other)`);
+    console.log("");
+
+    // Flatten and dedupe section IDs
+    const importantIds = new Set(parsed.importantSections.flat());
+    const otherIds = new Set(
+      parsed.otherSections.flat().filter((id) => !importantIds.has(id)),
+    );
+
+    const totalUnique = importantIds.size + otherIds.size;
+
+    // Show important sections first
+    const sortedIds = [...importantIds, ...otherIds];
+
+    sortedIds.slice(0, 50).forEach((id) => {
+      const isImportant = importantIds.has(id);
+      const marker = isImportant ? "[IMPORTANT] " : "            ";
+      console.log(`  ${marker}${id}`);
+    });
+
+    if (totalUnique > 50) {
+      console.log(`\n  ... and ${totalUnique - 50} more`);
+    }
+  } catch (error) {
+    console.error("Error:", error);
+    process.exit(1);
+  }
+}
+
+main().catch(console.error);