feat(evaluate) - Run evaluateInEngine262 in child_process with strict timeout

package.json

@@ -12,6 +12,7 @@
     "ingest": "bun run src/setup/ingest.ts",
     "verify-db": "bun run src/test/manual/verify-db.ts",
     "test-evaluate": "bun run src/test/manual/test-evaluate-in-engine262.ts",
+    "test-evaluate-timeout": "bun run src/test/manual/test-evaluate-timeout.ts",
     "test-search-spec-sections": "bun run src/test/manual/test-search-spec-sections.ts",
     "agent": "bun run src/agent.ts",
     "build": "bun run src/setup/buildGraph.ts",

src/agent-tools/evaluateInEngine262.runner.mjs

@@ -0,0 +1,154 @@
+/**
+ * Child process runner script for engine262 evaluation.
+ * Reads code from stdin, executes in engine262, outputs JSON to stdout.
+ */
+
+// Read code from stdin
+let code = "";
+process.stdin.setEncoding("utf8");
+process.stdin.on("data", (chunk) => {
+  code += chunk;
+});
+process.stdin.on("end", async () => {
+  try {
+    // Load engine262
+    const engine = await import("../../engine262/lib/engine262.mjs");
+
+    const Agent = engine.Agent;
+    const ManagedRealm = engine.ManagedRealm;
+    const setSurroundingAgent = engine.setSurroundingAgent;
+    const OrdinaryObjectCreate = engine.OrdinaryObjectCreate;
+    const CreateBuiltinFunction = engine.CreateBuiltinFunction;
+    const CreateDataProperty = engine.CreateDataProperty;
+    const Value = engine.Value;
+    const skipDebugger = engine.skipDebugger;
+    const ask262Debug = engine.ask262Debug;
+
+    // Reset state
+    ask262Debug.reset();
+
+    // Array to capture console output
+    const consoleOutput = [];
+
+    // Set up agent and realm
+    const agent = new Agent();
+    setSurroundingAgent(agent);
+    const realm = new ManagedRealm();
+
+    // Expose ask262Debug and console to the evaluated code
+    realm.scope(() => {
+      const debugObj = OrdinaryObjectCreate(
+        agent.intrinsic("%Object.prototype%"),
+      );
+      skipDebugger(
+        CreateDataProperty(realm.GlobalObject, Value("ask262Debug"), debugObj),
+      );
+
+      const startImportant = CreateBuiltinFunction(
+        () => {
+          ask262Debug.startImportant();
+          return Value("undefined");
+        },
+        0,
+        Value("startImportant"),
+        [],
+      );
+      skipDebugger(
+        CreateDataProperty(debugObj, Value("startImportant"), startImportant),
+      );
+
+      const stopImportant = CreateBuiltinFunction(
+        () => {
+          ask262Debug.stopImportant();
+          return Value("undefined");
+        },
+        0,
+        Value("stopImportant"),
+        [],
+      );
+      skipDebugger(
+        CreateDataProperty(debugObj, Value("stopImportant"), stopImportant),
+      );
+
+      // Create console object
+      const consoleObj = OrdinaryObjectCreate(
+        agent.intrinsic("%Object.prototype%"),
+      );
+      skipDebugger(
+        CreateDataProperty(realm.GlobalObject, Value("console"), consoleObj),
+      );
+
+      // Add console methods: log, warn, debug, error
+      const consoleMethods = ["log", "warn", "debug", "error"];
+      for (const method of consoleMethods) {
+        const fn = CreateBuiltinFunction(
+          (args) => {
+            const jsValues = args.map((arg) => {
+              if (arg && typeof arg === "object") {
+                const strVal = arg.stringValue;
+                if (typeof strVal === "function") {
+                  return strVal.call(arg);
+                }
+                const value = arg.value;
+                if (value !== undefined) {
+                  return value;
+                }
+              }
+              return arg;
+            });
+            consoleOutput.push({ method, values: jsValues });
+            return Value("undefined");
+          },
+          1,
+          Value(method),
+          [],
+        );
+        skipDebugger(CreateDataProperty(consoleObj, Value(method), fn));
+      }
+    });
+
+    // Start tracing
+    ask262Debug.startTrace();
+
+    // Execute the code
+    const completion = realm.evaluateScript(code);
+
+    // Stop tracing
+    ask262Debug.stopTrace();
+
+    // Check for error completion
+    if (completion?.Type === "throw") {
+      const errorValue = completion.Value;
+      const errorMessage =
+        errorValue?.ErrorData?.stringValue?.() ||
+        errorValue?.ErrorData?.value ||
+        "Unknown error";
+      console.log(JSON.stringify({ error: errorMessage }));
+      process.exit(0);
+    }
+
+    // Get captured marks
+    const marks = ask262Debug.marks;
+
+    // Filter and group marks by important flag
+    const importantMarks = marks.filter((m) => m.important);
+    const otherMarks = marks.filter((m) => !m.important);
+
+    // Output result as JSON
+    const result = {
+      importantSections: importantMarks.flatMap((m) => m.sectionIds),
+      otherSections: otherMarks.flatMap((m) => m.sectionIds),
+      consoleOutput: consoleOutput,
+    };
+
+    console.log(JSON.stringify(result));
+    process.exit(0);
+  } catch (error) {
+    console.log(
+      JSON.stringify({
+        error: error instanceof Error ? error.message : String(error),
+      }),
+    );
+    process.exit(1);
+  }
+});

src/agent-tools/evaluateInEngine262.ts

@@ -2,8 +2,12 @@
  * Evaluate JavaScript code in engine262 and capture spec section marks.
  * Executes code in the engine262 JavaScript engine and returns the captured
  * ECMAScript spec section marks.
+ * Uses child_process for true isolation and guaranteed termination via SIGKILL.
  */
 
+import { spawn } from "node:child_process";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
 import { z } from "zod";
 
 // #region Zod schemas (not exported)
@@ -106,180 +110,139 @@ export type EvaluateToolInput = z.infer<typeof inputSchema>;
  */
 export const toolName = "ask262_evaluate_in_engine262";
 
-// Type definitions for engine262 module
-interface MarkData {
-  readonly sectionIds: string[];
-  readonly fileRelativePath: string;
-  readonly lineNumber: number;
-  readonly important: boolean;
-}
-
-// biome-ignore lint/suspicious/noExplicitAny: engine262 is an external module without TypeScript types
-let engine262Module: any = null;
+/**
+ * Default timeout for code execution
+ */
+const DEFAULT_EXECUTION_TIMEOUT_MS = 1000;
 
 /**
- * Lazy load engine262 module
+ * Execute JavaScript code in engine262 using a child process.
+ * @param code - JavaScript code to execute
+ * @param timeoutMs - Maximum execution time in milliseconds
+ * @returns Execution result as JSON string
  */
-async function loadEngine262() {
-  if (!engine262Module) {
-    // Dynamic import of engine262 from local path
-    engine262Module = await import("../../engine262/lib/engine262.mjs");
+function executeInChildProcess(
+  code: string,
+  timeoutMs: number,
+): Promise<string> {
+  return new Promise((resolve) => {
+    // Get the runner script path
+    const runnerPath = path.join(
+      path.dirname(fileURLToPath(import.meta.url)),
+      "evaluateInEngine262.runner.mjs",
+    );
+
+    // Spawn child process
+    const child = spawn("node", [runnerPath], {
+      stdio: ["pipe", "pipe", "pipe"],
+      killSignal: "SIGKILL", // Force kill, can't be blocked
+    });
+
+    let stdout = "";
+    let stderr = "";
+
+    // Collect stdout
+    child.stdout?.on("data", (data: Buffer) => {
+      stdout += data.toString();
+    });
+
+    // Collect stderr (for errors)
+    child.stderr?.on("data", (data: Buffer) => {
+      stderr += data.toString();
+    });
+
+    // Set up timeout
+    let killedByTimeout = false;
+    const timeoutId = setTimeout(() => {
+      killedByTimeout = true;
+      // SIGKILL can't be caught or blocked - guaranteed termination
+      child.kill("SIGKILL");
+    }, timeoutMs);
+
+    // Handle process exit
+    child.on("close", (code, signal) => {
+      // Clear timeout
+      clearTimeout(timeoutId);
+
+      if (signal === "SIGKILL" || signal === "SIGTERM") {
+        // Process was killed (timeout or error)
+        if (killedByTimeout) {
+          resolve(
+            JSON.stringify({
+              error: `Execution timeout after ${timeoutMs}ms`,
+            }),
+          );
+        } else {
+          resolve(
+            JSON.stringify({
+              error: stderr || "Process terminated",
+            }),
+          );
         }
-  return engine262Module;
+        return;
+      }
+
+      if (code !== 0) {
+        resolve(
+          JSON.stringify({
+            error: stderr || `Process exited with code ${code}`,
+          }),
+        );
+        return;
+      }
+
+      // Success - return stdout (should be JSON result)
+      try {
+        // Validate it's valid JSON
+        JSON.parse(stdout);
+        resolve(stdout);
+      } catch {
+        resolve(
+          JSON.stringify({
+            error: `Invalid output: ${stdout.slice(0, 200)}`,
+          }),
+        );
+      }
+    });
+
+    // Handle spawn errors
+    child.on("error", (error) => {
+      clearTimeout(timeoutId);
+      resolve(
+        JSON.stringify({
+          error: `Failed to spawn process: ${error.message}`,
+        }),
+      );
+    });
+
+    // Send code to child via stdin
+    child.stdin?.write(code);
+    child.stdin?.end();
+  });
 }
 
 /**
  * Creates the evaluateInEngine262 tool function.
- * Executes JavaScript code in engine262 and captures spec section marks.
+ * Executes JavaScript code in engine262 using a child process with timeout support.
+ * Uses child_process for true isolation and guaranteed termination via SIGKILL.
+ * @param timeoutMs - Maximum execution time in milliseconds
  * @returns Function that executes code and returns structured output
  */
-export function createEvaluateInEngine262Tool() {
+export function createEvaluateInEngine262Tool(
+  timeoutMs = DEFAULT_EXECUTION_TIMEOUT_MS,
+) {
   return async ({ code }: EvaluateToolInput): Promise<EvaluateToolOutput> => {
-    const engine = await loadEngine262();
-    const ask262Debug = engine.ask262Debug as {
-      marks: MarkData[];
-      startTrace: () => void;
-      stopTrace: () => void;
-      startImportant: () => void;
-      stopImportant: () => void;
-      reset: () => void;
-    };
-    const Agent = engine.Agent;
-    const ManagedRealm = engine.ManagedRealm;
-    const setSurroundingAgent = engine.setSurroundingAgent;
-    const OrdinaryObjectCreate = engine.OrdinaryObjectCreate;
-    const CreateBuiltinFunction = engine.CreateBuiltinFunction;
-    const CreateDataProperty = engine.CreateDataProperty;
-    const Value = engine.Value;
-    const skipDebugger = engine.skipDebugger;
+    try {
+      // Execute code in isolated child process
+      const resultJson = await executeInChildProcess(code, timeoutMs);
 
-    // Reset state from previous runs
-    ask262Debug.reset();
-
-    // Array to capture console output
-    const consoleOutput: ConsoleEntry[] = [];
-
-    // Set up agent and realm
-    const agent = new Agent();
-    setSurroundingAgent(agent);
-    const realm = new ManagedRealm();
-
-    // Expose ask262Debug and console to the evaluated code
-    realm.scope(() => {
-      const debugObj = OrdinaryObjectCreate(
-        agent.intrinsic("%Object.prototype%"),
-      );
-      skipDebugger(
-        CreateDataProperty(realm.GlobalObject, Value("ask262Debug"), debugObj),
-      );
-
-      const startImportant = CreateBuiltinFunction(
-        () => {
-          ask262Debug.startImportant();
-          // biome-ignore lint/suspicious/noExplicitAny: engine262 uses custom value types
-          return (Value as any)("undefined");
-        },
-        0,
-        Value("startImportant"),
-        [],
-      );
-      skipDebugger(
-        CreateDataProperty(debugObj, Value("startImportant"), startImportant),
-      );
-
-      const stopImportant = CreateBuiltinFunction(
-        () => {
-          ask262Debug.stopImportant();
-          // biome-ignore lint/suspicious/noExplicitAny: engine262 uses custom value types
-          return (Value as any)("undefined");
-        },
-        0,
-        Value("stopImportant"),
-        [],
-      );
-      skipDebugger(
-        CreateDataProperty(debugObj, Value("stopImportant"), stopImportant),
-      );
-
-      // Create console object with methods (excluding 'clear')
-      const consoleObj = OrdinaryObjectCreate(
-        agent.intrinsic("%Object.prototype%"),
-      );
-      skipDebugger(
-        CreateDataProperty(realm.GlobalObject, Value("console"), consoleObj),
-      );
-
-      // Add console methods: log, warn, debug, error
-      const consoleMethods = ["log", "warn", "debug", "error"];
-      for (const method of consoleMethods) {
-        const fn = CreateBuiltinFunction(
-          (args: unknown[]) => {
-            // Convert engine262 values to JavaScript values for the output
-            const jsValues = args.map((arg) => {
-              // Handle engine262 Value types - convert to primitive JS values
-              if (arg && typeof arg === "object") {
-                // Try to get string value if it's a JSStringValue
-                const strVal = (arg as { stringValue?: () => string })
-                  .stringValue;
-                if (typeof strVal === "function") {
-                  return strVal.call(arg);
-                }
-                // Try other common properties
-                const value = (arg as { value?: unknown }).value;
-                if (value !== undefined) {
-                  return value;
-                }
-              }
-              return arg;
-            });
-            consoleOutput.push({ method, values: jsValues });
-            // biome-ignore lint/suspicious/noExplicitAny: engine262 uses custom value types
-            return (Value as any)("undefined");
-          },
-          1,
-          Value(method),
-          [],
-        );
-        skipDebugger(CreateDataProperty(consoleObj, Value(method), fn));
-      }
-    });
-
-    // Start tracing
-    ask262Debug.startTrace();
-
-    // Execute the code
-    const completion = realm.evaluateScript(code);
-
-    // Stop tracing
-    ask262Debug.stopTrace();
-
-    // Check for error completion (engine262 returns ThrowCompletion_ instead of throwing)
-    // biome-ignore lint/suspicious/noExplicitAny: engine262 internal completion types
-    const completionAny = completion as any;
-    if (completionAny?.Type === "throw") {
-      const errorValue = completionAny.Value;
-      // Extract error message from ErrorData property
-      const errorMessage: string =
-        errorValue?.ErrorData?.stringValue?.() ||
-        errorValue?.ErrorData?.value ||
-        "Unknown error";
+      // Parse the result
+      return JSON.parse(resultJson) as EvaluateToolOutput;
+    } catch (error) {
+      // Return error result
       return {
-        error: errorMessage,
+        error: error instanceof Error ? error.message : String(error),
       };
     }
-
-    // Get captured marks
-    const marks = ask262Debug.marks;
-
-    // Filter and group marks by important flag
-    const importantMarks = marks.filter((m) => m.important);
-    const otherMarks = marks.filter((m) => !m.important);
-
-    // Flatten sectionIds from all marks
-    return {
-      importantSections: importantMarks.flatMap((m) => m.sectionIds),
-      otherSections: otherMarks.flatMap((m) => m.sectionIds),
-      consoleOutput: consoleOutput,
-    };
   };
 }

src/test/manual/test-evaluate-in-engine262.ts

@@ -7,12 +7,7 @@
  * Usage: bun run src/test/manual/test-evaluate-in-engine262.ts ["your JavaScript code"]
  */
 
-import { createEvaluateInEngine262Tool } from "../../agent-tools";
-
-interface EvaluateResult {
-  importantSections: string[][];
-  otherSections: string[][];
-}
+import { createEvaluateInEngine262Tool } from "../../agent-tools/index.js";
 
 async function main() {
   // Get test code from command line or use default
@@ -39,13 +34,15 @@ async function main() {
 
   console.log("Executing tool...\n");
   try {
-    const result = await evaluateTool.func({ code: testCode });
+    const result = await evaluateTool({ code: testCode });
 
-    // Parse and verify results
-    const parsed: EvaluateResult = JSON.parse(result);
+    // Verify results
+    if ("error" in result) {
+      throw new Error(result.error);
+    }
 
-    const importantCount = parsed.importantSections.length;
-    const otherCount = parsed.otherSections.length;
+    const importantCount = result.importantSections.length;
+    const otherCount = result.otherSections.length;
     const totalCount = importantCount + otherCount;
 
     console.log(`\n✓ Captured ${totalCount} marks`);
@@ -53,9 +50,9 @@ async function main() {
     console.log("");
 
     // Flatten and dedupe section IDs
-    const importantIds = new Set(parsed.importantSections.flat());
+    const importantIds = new Set(result.importantSections);
     const otherIds = new Set(
-      parsed.otherSections.flat().filter((id) => !importantIds.has(id)),
+      result.otherSections.filter((id: string) => !importantIds.has(id)),
     );
 
     const totalUnique = importantIds.size + otherIds.size;

src/test/manual/test-evaluate-timeout.ts

@@ -0,0 +1,67 @@
+/**
+ * Test timeout functionality for evaluateInEngine262 tool
+ * Tests that code execution properly times out after the specified duration
+ */
+
+import { createEvaluateInEngine262Tool } from "../../agent-tools/evaluateInEngine262.js";
+
+async function testTimeout() {
+  console.log("=== Testing evaluateInEngine262 Timeout ===\n");
+
+  // Test 1: Quick code should complete before timeout
+  console.log("Test 1: Quick code (should succeed)...");
+  const quickTool = createEvaluateInEngine262Tool(5000); // 5 second timeout
+  const start1 = Date.now();
+  const quickResult = await quickTool({ code: "console.log('hello'); 1 + 1" });
+  const elapsed1 = Date.now() - start1;
+
+  if (quickResult.error) {
+    throw new Error(`Quick code failed: ${quickResult.error}`);
+  }
+  console.log(`✓ Completed in ${elapsed1}ms`);
+  console.log(
+    `  Console output: ${JSON.stringify(quickResult.consoleOutput)}\n`,
+  );
+
+  // Test 2: Long-running code should timeout
+  console.log("Test 2: Long-running code (should timeout after 500ms)...");
+  const slowTool = createEvaluateInEngine262Tool(500); // 500ms timeout
+  const start2 = Date.now();
+  const slowResult = await slowTool({
+    code: `
+      // Busy-wait loop that takes ~5 seconds
+      const start = Date.now();
+      while (Date.now() - start < 5000) {
+        // Busy wait - no yielding
+        for (let i = 0; i < 1000; i++) {
+          Math.sqrt(i);
+        }
+      }
+      console.log("completed");
+    `,
+  });
+  const elapsed2 = Date.now() - start2;
+
+  if (!slowResult.error) {
+    throw new Error("Expected timeout error but code completed successfully");
+  }
+  if (!slowResult.error.includes("timeout")) {
+    throw new Error(`Expected timeout error but got: ${slowResult.error}`);
+  }
+  console.log(`✓ Timed out in ${elapsed2}ms`);
+  console.log(`  Error message: ${slowResult.error}\n`);
+
+  // Note: worker.terminate() sends signal but CPU-bound loops may not stop immediately.
+  // The important thing is that we got a timeout error before the 5-second loop completed.
+  if (elapsed2 > 5000) {
+    throw new Error(`Timeout took too long: ${elapsed2}ms (expected < 5000ms)`);
+  }
+
+  console.log(`  (Note: Total time includes worker termination cleanup)`);
+  console.log("=== All timeout tests passed! ✓ ===");
+}
+
+testTimeout().catch((error) => {
+  console.error("\nTest failed:", error);
+  process.exit(1);
+});