From 07a4d4957d9fcd614ae0367d0e74100507cc854a Mon Sep 17 00:00:00 2001
From: bendtherules <abhasbhattacharya2@gmail.com>
Date: Tue, 14 Apr 2026 11:07:59 +0530
Subject: [PATCH] feat(agent-tools/mcp-server): unify evaluate output schema
 and improve error handling

- Replace union output schema with a single combined schema containing optional fields for success and error cases, addressing MCP SDK limitation with union types.
- Add `importantSections`, `otherSections`, and `consoleOutput` as optional fields in the new schema.
- Update exported `outputSchema` and `EvaluateToolOutput` type to use the combined schema.
- Refactor execution logic to handle engine262's `ThrowCompletion_` instead of relying on thrown exceptions, extracting a readable error message.
- Adjust MCP server error detection to check `result.error !== undefined` rather than using the `in` operator.
- Update related comments and documentation to reflect the new schema design and error handling approach.
---
 src/agent-tools/evaluateInEngine262.ts | 58 +++++++++++++++++++-------
 src/mcp-server.ts                      |  2 +-
 2 files changed, 45 insertions(+), 15 deletions(-)

diff --git a/src/agent-tools/evaluateInEngine262.ts b/src/agent-tools/evaluateInEngine262.ts
index f4e9c6c..4be59e8 100644
--- a/src/agent-tools/evaluateInEngine262.ts
+++ b/src/agent-tools/evaluateInEngine262.ts
@@ -23,12 +23,33 @@ const evaluateSuccessOutputSchema = z.object({
   consoleOutput: z
     .array(consoleEntrySchema)
     .describe("Console output captured during execution"),
+  error: z.undefined().optional(),
 });
 
 const evaluateErrorOutputSchema = z.object({
+  importantSections: z.undefined().optional(),
+  otherSections: z.undefined().optional(),
+  consoleOutput: z.undefined().optional(),
   error: z.string().describe("Error message when execution fails"),
 });
 
+// Combined output schema (single object with optional fields because MCP SDK doesn't accept union schemas)
+const evaluateOutputSchemaCombined = z.object({
+  importantSections: z
+    .array(z.string())
+    .optional()
+    .describe("Important spec sections hit during execution (absent on error)"),
+  otherSections: z
+    .array(z.string())
+    .optional()
+    .describe("Other spec sections hit during execution (absent on error)"),
+  consoleOutput: z
+    .array(consoleEntrySchema)
+    .optional()
+    .describe("Console output captured during execution (absent on error)"),
+  error: z.string().optional().describe("Error message when execution fails"),
+});
+
 // #endregion
 
 // #region Exported Zod schemas
@@ -57,12 +78,11 @@ export const inputSchema = z.object({
 });
 
 /**
- * Output schema for the evaluate tool (union of success and error outputs).
+ * Output schema for the evaluate tool.
+ * Uses a single object schema with optional fields for MCP SDK compatibility
+ * (MCP SDK doesn't support union schemas in outputSchema).
  */
-export const outputSchema = z.union([
-  evaluateSuccessOutputSchema,
-  evaluateErrorOutputSchema,
-]);
+export const outputSchema = evaluateOutputSchemaCombined;
 
 // #endregion
 
@@ -74,7 +94,8 @@ export type EvaluateSuccessOutput = z.infer<typeof evaluateSuccessOutputSchema>;
 
 export type EvaluateErrorOutput = z.infer<typeof evaluateErrorOutputSchema>;
 
-export type EvaluateToolOutput = z.infer<typeof outputSchema>;
+/** Combined tool output type (success or error) */
+export type EvaluateToolOutput = EvaluateSuccessOutput | EvaluateErrorOutput;
 
 export type EvaluateToolInput = z.infer<typeof inputSchema>;
 
@@ -226,18 +247,27 @@ export function createEvaluateInEngine262Tool() {
     // Start tracing
     ask262Debug.startTrace();
 
-    try {
-      // Execute the code - only this part can fail
-      realm.evaluateScript(code);
-    } catch (error) {
-      return {
-        error: error instanceof Error ? error.message : String(error),
-      };
-    }
+    // Execute the code
+    const completion = realm.evaluateScript(code);
 
     // Stop tracing
     ask262Debug.stopTrace();
 
+    // Check for error completion (engine262 returns ThrowCompletion_ instead of throwing)
+    // biome-ignore lint/suspicious/noExplicitAny: engine262 internal completion types
+    const completionAny = completion as any;
+    if (completionAny?.Type === "throw") {
+      const errorValue = completionAny.Value;
+      // Extract error message from ErrorData property
+      const errorMessage: string =
+        errorValue?.ErrorData?.stringValue?.() ||
+        errorValue?.ErrorData?.value ||
+        "Unknown error";
+      return {
+        error: errorMessage,
+      };
+    }
+
     // Get captured marks
     const marks = ask262Debug.marks;
 
diff --git a/src/mcp-server.ts b/src/mcp-server.ts
index ffc59c7..6732a97 100644
--- a/src/mcp-server.ts
+++ b/src/mcp-server.ts
@@ -159,7 +159,7 @@ async function main() {
     },
     async ({ code }: EvaluateToolMCPInput): Promise<EvaluateToolMCPOutput> => {
       const result = await evaluateTool({ code });
-      const isError = "error" in result;
+      const isError = result.error !== undefined;
       const text = isError ? result.error : JSON.stringify(result, null, 2);
       return {
         content: [{ type: "text", text }],