fix: deduplicate section IDs in evaluateInEngine262

src/agent-tools/evaluateInEngine262.runner.mjs

@@ -139,14 +139,21 @@ process.stdin.on("end", async () => {
     const marks = ask262Debug.marks;
     console.error(`[RUNNER] Captured ${marks.length} marks`);
 
-    // Filter and group marks by important flag
+    // Filter and group marks by important flag, then deduplicate
-    const importantMarks = marks.filter((m) => m.important);
+    const importantIds = new Set(
-    const otherMarks = marks.filter((m) => !m.important);
+      marks.filter((m) => m.important).flatMap((m) => m.sectionIds),
+    );
+    const otherIds = new Set(
+      marks.filter((m) => !m.important).flatMap((m) => m.sectionIds),
+    );
+
+    // Remove duplicates from otherIds that exist in importantIds
+    const dedupedOtherIds = [...otherIds].filter((id) => !importantIds.has(id));
 
     // Output result as JSON
     const result = {
-      importantSections: importantMarks.flatMap((m) => m.sectionIds),
+      importantSections: [...importantIds],
-      otherSections: otherMarks.flatMap((m) => m.sectionIds),
+      otherSections: dedupedOtherIds,
       consoleOutput: consoleOutput,
     };
 

src/test/evaluateInEngine262.test.ts

@@ -0,0 +1,135 @@
+/**
+ * Tests for the evaluateInEngine262 tool.
+ * Tests JavaScript code execution in engine262 and spec section capture.
+ */
+
+import { describe, expect, test } from "bun:test";
+import { createEvaluateInEngine262Tool } from "../agent-tools/index.js";
+
+describe("evaluateInEngine262", () => {
+  const evaluateTool = createEvaluateInEngine262Tool();
+
+  test("should execute simple arithmetic", async () => {
+    const result = await evaluateTool({ code: "1 + 1" });
+
+    expect(result.error).toBeUndefined();
+    expect(result.importantSections).toBeArray();
+    expect(result.otherSections).toBeArray();
+    expect(result.consoleOutput).toBeArray();
+  });
+
+  test("should capture spec sections for array operations", async () => {
+    const result = await evaluateTool({ code: "[1, 2, 3].map(x => x * 2)" });
+
+    expect(result.error).toBeUndefined();
+    expect(result.otherSections?.length).toBeGreaterThan(0);
+
+    // Should include array-related spec sections
+    const sectionIds = [
+      ...(result.importantSections || []),
+      ...(result.otherSections || []),
+    ];
+    expect(sectionIds.some((id) => id.includes("array"))).toBe(true);
+  });
+
+  test("should return error for syntax errors", async () => {
+    const result = await evaluateTool({ code: "function {" });
+
+    expect(result.error).toBeDefined();
+    expect(result.error).toContain("SyntaxError");
+  });
+
+  test("should return error for runtime errors", async () => {
+    const result = await evaluateTool({ code: "null.foo" });
+
+    expect(result.error).toBeDefined();
+    expect(result.error).toContain("TypeError");
+  });
+
+  test("should return error for reference errors", async () => {
+    const result = await evaluateTool({ code: "arr.push(1)" });
+
+    expect(result.error).toBeDefined();
+    expect(result.error).toContain("ReferenceError");
+  });
+
+  test("should capture console output", async () => {
+    const result = await evaluateTool({
+      code: "console.log('hello', 123); console.warn('test')",
+    });
+
+    expect(result.error).toBeUndefined();
+    expect(result.consoleOutput?.length).toBeGreaterThan(0);
+
+    const logEntry = result.consoleOutput?.find(
+      (entry) => entry.method === "log",
+    );
+    expect(logEntry).toBeDefined();
+  });
+
+  test("should capture important sections with ask262Debug", async () => {
+    const result = await evaluateTool({
+      code: `
+        ask262Debug.startImportant();
+        [1, 2, 3].filter(x => x > 1);
+        ask262Debug.stopImportant();
+      `,
+    });
+
+    expect(result.error).toBeUndefined();
+    expect(result.importantSections?.length).toBeGreaterThan(0);
+  });
+
+  test("should handle empty code", async () => {
+    const result = await evaluateTool({ code: "" });
+
+    // Empty code should either succeed or have a specific error
+    expect(result).toBeDefined();
+  });
+
+  test("should handle complex code with multiple operations", async () => {
+    const result = await evaluateTool({
+      code: `
+        const arr = [1, 2, 3];
+        const doubled = arr.map(x => x * 2);
+        const filtered = doubled.filter(x => x > 2);
+        filtered.reduce((a, b) => a + b, 0);
+      `,
+    });
+
+    expect(result.error).toBeUndefined();
+    expect(result.otherSections?.length).toBeGreaterThan(0);
+  });
+
+  test("should timeout after 1 second by default", async () => {
+    const result = await evaluateTool({
+      code: "while (true) {}", // Infinite loop
+    });
+
+    expect(result.error).toBeDefined();
+    expect(result.error).toContain("timeout");
+  });
+
+  test("should return deduplicated section IDs", async () => {
+    const result = await evaluateTool({
+      code: "[1].map(x => x).map(x => x)",
+    });
+
+    expect(result.error).toBeUndefined();
+
+    // Check that there are no duplicates within or between arrays
+    const importantSet = new Set(result.importantSections || []);
+    const otherSet = new Set(result.otherSections || []);
+
+    // No duplicates within important
+    expect(importantSet.size).toBe((result.importantSections || []).length);
+
+    // No duplicates within other
+    expect(otherSet.size).toBe((result.otherSections || []).length);
+
+    // No overlap between important and other
+    for (const id of result.importantSections || []) {
+      expect(otherSet.has(id)).toBe(false);
+    }
+  });
+});

src/test/getSectionContent.test.ts

@@ -0,0 +1,122 @@
+/**
+ * Tests for the getSectionContent tool.
+ * Tests retrieval of full section content by section ID.
+ */
+
+import { describe, expect, test } from "bun:test";
+import { createGetSectionContentTool } from "../agent-tools/index.js";
+import { createMockTable, defaultTestData } from "./utils/mock.js";
+
+describe("getSectionContent", () => {
+  const mockTable = createMockTable(defaultTestData);
+  const getContentTool = createGetSectionContentTool(mockTable);
+
+  test("should return content for a single valid section", async () => {
+    const result = await getContentTool({
+      sectionIds: ["sec-if-statement"],
+      recursive: false,
+    });
+
+    expect(result.sections).toBeArray();
+    expect(result.sections.length).toBe(1);
+    expect(result.sections[0].found).toBe(true);
+    expect(result.sections[0].sectionId).toBe("sec-if-statement");
+    expect(result.sections[0].content).toBeString();
+    expect(result.sections[0].content.length).toBeGreaterThan(0);
+  });
+
+  test("should return content for multiple sections", async () => {
+    const result = await getContentTool({
+      sectionIds: ["sec-if-statement", "sec-for-statement"],
+      recursive: false,
+    });
+
+    expect(result.sections.length).toBe(2);
+    expect(result.sections[0].found).toBe(true);
+    expect(result.sections[1].found).toBe(true);
+  });
+
+  test("should return found: false for non-existent sections", async () => {
+    const result = await getContentTool({
+      sectionIds: ["sec-does-not-exist"],
+      recursive: false,
+    });
+
+    expect(result.sections.length).toBe(1);
+    expect(result.sections[0].found).toBe(false);
+    expect(result.sections[0].error).toBeDefined();
+    expect(result.sections[0].content).toBe("");
+  });
+
+  test("should handle mix of existing and non-existing sections", async () => {
+    const result = await getContentTool({
+      sectionIds: ["sec-if-statement", "sec-does-not-exist"],
+      recursive: false,
+    });
+
+    expect(result.sections.length).toBe(2);
+    expect(result.sections[0].found).toBe(true);
+    expect(result.sections[1].found).toBe(false);
+  });
+
+  test("should include section metadata", async () => {
+    const result = await getContentTool({
+      sectionIds: ["sec-if-statement"],
+      recursive: false,
+    });
+
+    const section = result.sections[0];
+    expect(section.sectionId).toBe("sec-if-statement");
+    expect(section.sectionTitle).toBeDefined();
+    expect(section.found).toBe(true);
+  });
+
+  test("should return empty array for empty sectionIds", async () => {
+    const result = await getContentTool({
+      sectionIds: [],
+      recursive: false,
+    });
+
+    expect(result.sections).toBeArray();
+    expect(result.sections.length).toBe(0);
+  });
+
+  test("should include partIndex and totalParts when available", async () => {
+    const result = await getContentTool({
+      sectionIds: ["sec-try-statement"],
+      recursive: false,
+    });
+
+    const section = result.sections[0];
+    if (section.found && section.partIndex !== undefined) {
+      expect(section.partIndex).toBeNumber();
+      expect(section.totalParts).toBeNumber();
+    }
+  });
+
+  test("should fetch children when recursive is true", async () => {
+    const result = await getContentTool({
+      sectionIds: ["sec-catch-clause"],
+      recursive: true,
+    });
+
+    // Should include at least the requested section
+    expect(result.sections.length).toBeGreaterThan(0);
+    const requestedSection = result.sections.find(
+      (s) => s.sectionId === "sec-catch-clause",
+    );
+    expect(requestedSection).toBeDefined();
+    expect(requestedSection?.found).toBe(true);
+  });
+
+  test("should preserve input order in output", async () => {
+    const sectionIds = ["sec-for-statement", "sec-if-statement"];
+    const result = await getContentTool({
+      sectionIds,
+      recursive: false,
+    });
+
+    expect(result.sections[0].sectionId).toBe("sec-for-statement");
+    expect(result.sections[1].sectionId).toBe("sec-if-statement");
+  });
+});

src/test/manual/test-nonexistent-sections.ts

@@ -1,70 +0,0 @@
-#!/usr/bin/env bun
-/**
- * Test to verify non-existing section IDs return found: false
- */
-
-import * as lancedbSdk from "@lancedb/lancedb";
-import { createGetSectionContentTool } from "../../agent-tools/getSectionContent.js";
-import { STORAGE_DIR } from "../../constants.js";
-
-async function main() {
-  console.log("Testing getSectionContent with non-existing section IDs...\n");
-
-  try {
-    const db = await lancedbSdk.connect(STORAGE_DIR);
-    const table = await db.openTable("spec_vectors");
-
-    const getSectionContentTool = createGetSectionContentTool(table);
-
-    // Test with mix of existing and non-existing section IDs
-    const result = await getSectionContentTool({
-      sectionIds: [
-        "sec-non-existent-12345", // Should not exist
-        "sec-also-fake-99999", // Should not exist
-      ],
-      recursive: false,
-    });
-
-    console.log("=== RESULT ===");
-    console.log(JSON.stringify(result, null, 2));
-    console.log("\n=== VERIFICATION ===");
-
-    // Verify all sections are returned
-    if (result.sections.length !== 2) {
-      console.error(`❌ Expected 2 sections, got ${result.sections.length}`);
-      process.exit(1);
-    }
-
-    // Verify non-existing sections have found: false
-    for (const section of result.sections) {
-      if (section.found !== false) {
-        console.error(
-          `❌ Section ${section.sectionId} should have found: false`,
-        );
-        process.exit(1);
-      }
-      if (!section.error) {
-        console.error(
-          `❌ Section ${section.sectionId} should have error message`,
-        );
-        process.exit(1);
-      }
-      if (section.content !== "") {
-        console.error(
-          `❌ Section ${section.sectionId} should have empty content`,
-        );
-        process.exit(1);
-      }
-      console.log(
-        `✅ ${section.sectionId}: found=false, error="${section.error}"`,
-      );
-    }
-
-    console.log("\n✅ All non-existing sections correctly return found: false");
-  } catch (error) {
-    console.error("❌ Test failed:", error);
-    process.exit(1);
-  }
-}
-
-main().catch(console.error);

src/test/searchSpecSections.test.ts

@@ -0,0 +1,96 @@
+/**
+ * Tests for the searchSpecSections tool.
+ * Tests vector search functionality to find relevant spec sections.
+ */
+
+import { describe, expect, test } from "bun:test";
+import { createSearchSpecSectionsTool } from "../agent-tools/index.js";
+import {
+  createMockEmbeddings,
+  createMockTable,
+  defaultTestData,
+} from "./utils/mock.js";
+
+describe("searchSpecSections", () => {
+  const mockTable = createMockTable(defaultTestData);
+  const mockEmbeddings = createMockEmbeddings();
+  const searchTool = createSearchSpecSectionsTool(mockTable, mockEmbeddings);
+
+  test("should return results for a valid query", async () => {
+    const result = await searchTool({ query: "array map" });
+
+    expect(result.results).toBeArray();
+    expect(result.results.length).toBeGreaterThan(0);
+    expect(result.results.length).toBeLessThanOrEqual(5); // Default limit
+  });
+
+  test("should return section metadata without content", async () => {
+    const result = await searchTool({ query: "if statement" });
+
+    const firstResult = result.results[0];
+    expect(firstResult).toHaveProperty("sectionId");
+    expect(firstResult).toHaveProperty("sectionTitle");
+    expect(firstResult).toHaveProperty("vectorDistance");
+    expect(firstResult).toHaveProperty("partIndex");
+    expect(firstResult).toHaveProperty("totalParts");
+    expect(firstResult).not.toHaveProperty("content");
+  });
+
+  test("should return valid section IDs", async () => {
+    const result = await searchTool({ query: "loop iteration" });
+
+    for (const item of result.results) {
+      expect(item.sectionId).toBeString();
+      expect(item.sectionId).toMatch(/^sec-/);
+    }
+  });
+
+  test("should return valid section titles", async () => {
+    const result = await searchTool({ query: "exception handling" });
+
+    for (const item of result.results) {
+      expect(item.sectionTitle).toBeString();
+      expect(item.sectionTitle.length).toBeGreaterThan(0);
+    }
+  });
+
+  test("should return vectorDistance as a number", async () => {
+    const result = await searchTool({ query: "javascript" });
+
+    for (const item of result.results) {
+      expect(item.vectorDistance).toBeNumber();
+      expect(item.vectorDistance).toBeGreaterThanOrEqual(0);
+    }
+  });
+
+  test("should handle empty query", async () => {
+    const result = await searchTool({ query: "" });
+
+    expect(result.results).toBeArray();
+    // May return results or empty depending on mock implementation
+  });
+
+  test("should handle complex queries", async () => {
+    const result = await searchTool({
+      query: "how does array prototype map method work with callbacks",
+    });
+
+    expect(result.results).toBeArray();
+    expect(result.results.length).toBeGreaterThan(0);
+  });
+
+  test("should return partIndex and totalParts as numbers or null", async () => {
+    const result = await searchTool({ query: "try catch" });
+
+    for (const item of result.results) {
+      if (item.partIndex !== null) {
+        expect(item.partIndex).toBeNumber();
+        expect(item.partIndex).toBeGreaterThanOrEqual(0);
+      }
+      if (item.totalParts !== null) {
+        expect(item.totalParts).toBeNumber();
+        expect(item.totalParts).toBeGreaterThan(0);
+      }
+    }
+  });
+});

src/test/utils/mock.ts

@@ -0,0 +1,132 @@
+/**
+ * Test utilities for mocking LanceDB and embeddings.
+ * Provides helper functions to create mock Table instances for testing.
+ */
+
+import type { Table } from "@lancedb/lancedb";
+import type { Embeddings } from "@langchain/core/embeddings";
+
+/**
+ * Mock embeddings instance for testing.
+ * Returns predictable vectors based on input text.
+ */
+export function createMockEmbeddings(): Embeddings {
+  return {
+    embedQuery: async (text: string): Promise<number[]> => {
+      // Return a simple hash-based vector for testing
+      const vector = new Array(128).fill(0);
+      for (let i = 0; i < text.length; i++) {
+        vector[i % 128] += text.charCodeAt(i) / 1000;
+      }
+      return vector;
+    },
+    embedDocuments: async (documents: string[]): Promise<number[][]> => {
+      return Promise.all(
+        documents.map((doc) => createMockEmbeddings().embedQuery(doc)),
+      );
+    },
+  } as Embeddings;
+}
+
+/**
+ * Create a mock LanceDB table with test data.
+ * @param testData - Array of test documents to return from queries
+ */
+export function createMockTable(testData: MockTableData[]): Table {
+  return {
+    search: (_queryVector: number[]) => ({
+      limit: (_n: number) => ({
+        toArray: async () => {
+          // Return first n results, sorted by a simple distance calculation
+          return testData
+            .map((data) => ({
+              ...data,
+              _distance: Math.random() * 0.5, // Random distance for testing
+            }))
+            .slice(0, _n);
+        },
+      }),
+    }),
+    query: () => ({
+      where: (condition: string) => ({
+        limit: (_n: number) => ({
+          toArray: async () => {
+            // Parse sectionid from condition like "sectionid = 'sec-xxx'"
+            const match = condition.match(/sectionid = ['"]([^'"]+)['"]/);
+            const sectionId = match ? match[1] : null;
+
+            if (sectionId) {
+              return testData
+                .filter((data) => data.sectionid === sectionId)
+                .map((data) => ({
+                  ...data,
+                }));
+            }
+            return [];
+          },
+        }),
+      }),
+    }),
+  } as unknown as Table;
+}
+
+/**
+ * Mock table data structure
+ */
+export interface MockTableData {
+  sectionid: string;
+  sectiontitle: string;
+  text: string;
+  partindex?: number;
+  totalparts?: number;
+  childrensectionids?: string[];
+}
+
+/**
+ * Default test data for spec sections
+ */
+export const defaultTestData: MockTableData[] = [
+  {
+    sectionid: "sec-if-statement",
+    sectiontitle: "The if Statement",
+    text: "The if statement evaluates a condition and executes a block if true.",
+    partindex: 0,
+    totalparts: 1,
+  },
+  {
+    sectionid: "sec-for-statement",
+    sectiontitle: "The for Statement",
+    text: "The for statement creates a loop with initialization, condition, and increment.",
+    partindex: 0,
+    totalparts: 1,
+  },
+  {
+    sectionid: "sec-array.prototype.map",
+    sectiontitle: "Array.prototype.map",
+    text: "The map method creates a new array by applying a function to each element.",
+    partindex: 0,
+    totalparts: 1,
+  },
+  {
+    sectionid: "sec-array.prototype.filter",
+    sectiontitle: "Array.prototype.filter",
+    text: "The filter method creates a new array with elements that pass the test.",
+    partindex: 0,
+    totalparts: 1,
+  },
+  {
+    sectionid: "sec-try-statement",
+    sectiontitle: "The try Statement",
+    text: "The try statement marks a block of statements to try, with catch and finally clauses.",
+    partindex: 0,
+    totalparts: 2,
+  },
+  {
+    sectionid: "sec-catch-clause",
+    sectiontitle: "The catch Clause",
+    text: "The catch clause provides exception handling for the try block.",
+    partindex: 1,
+    totalparts: 2,
+    childrensectionids: ["sec-try-statement"],
+  },
+];