diff --git a/src/agent-tools/evaluateInEngine262.runner.mjs b/src/agent-tools/evaluateInEngine262.runner.mjs index f0ba0e5..1087ca1 100644 --- a/src/agent-tools/evaluateInEngine262.runner.mjs +++ b/src/agent-tools/evaluateInEngine262.runner.mjs @@ -139,14 +139,21 @@ process.stdin.on("end", async () => { const marks = ask262Debug.marks; console.error(`[RUNNER] Captured ${marks.length} marks`); - // Filter and group marks by important flag - const importantMarks = marks.filter((m) => m.important); - const otherMarks = marks.filter((m) => !m.important); + // Filter and group marks by important flag, then deduplicate + const importantIds = new Set( + marks.filter((m) => m.important).flatMap((m) => m.sectionIds), + ); + const otherIds = new Set( + marks.filter((m) => !m.important).flatMap((m) => m.sectionIds), + ); + + // Remove duplicates from otherIds that exist in importantIds + const dedupedOtherIds = [...otherIds].filter((id) => !importantIds.has(id)); // Output result as JSON const result = { - importantSections: importantMarks.flatMap((m) => m.sectionIds), - otherSections: otherMarks.flatMap((m) => m.sectionIds), + importantSections: [...importantIds], + otherSections: dedupedOtherIds, consoleOutput: consoleOutput, }; diff --git a/src/test/evaluateInEngine262.test.ts b/src/test/evaluateInEngine262.test.ts new file mode 100644 index 0000000..c3c4d84 --- /dev/null +++ b/src/test/evaluateInEngine262.test.ts @@ -0,0 +1,135 @@ +/** + * Tests for the evaluateInEngine262 tool. + * Tests JavaScript code execution in engine262 and spec section capture. + */ + +import { describe, expect, test } from "bun:test"; +import { createEvaluateInEngine262Tool } from "../agent-tools/index.js"; + +describe("evaluateInEngine262", () => { + const evaluateTool = createEvaluateInEngine262Tool(); + + test("should execute simple arithmetic", async () => { + const result = await evaluateTool({ code: "1 + 1" }); + + expect(result.error).toBeUndefined(); + expect(result.importantSections).toBeArray(); + expect(result.otherSections).toBeArray(); + expect(result.consoleOutput).toBeArray(); + }); + + test("should capture spec sections for array operations", async () => { + const result = await evaluateTool({ code: "[1, 2, 3].map(x => x * 2)" }); + + expect(result.error).toBeUndefined(); + expect(result.otherSections?.length).toBeGreaterThan(0); + + // Should include array-related spec sections + const sectionIds = [ + ...(result.importantSections || []), + ...(result.otherSections || []), + ]; + expect(sectionIds.some((id) => id.includes("array"))).toBe(true); + }); + + test("should return error for syntax errors", async () => { + const result = await evaluateTool({ code: "function {" }); + + expect(result.error).toBeDefined(); + expect(result.error).toContain("SyntaxError"); + }); + + test("should return error for runtime errors", async () => { + const result = await evaluateTool({ code: "null.foo" }); + + expect(result.error).toBeDefined(); + expect(result.error).toContain("TypeError"); + }); + + test("should return error for reference errors", async () => { + const result = await evaluateTool({ code: "arr.push(1)" }); + + expect(result.error).toBeDefined(); + expect(result.error).toContain("ReferenceError"); + }); + + test("should capture console output", async () => { + const result = await evaluateTool({ + code: "console.log('hello', 123); console.warn('test')", + }); + + expect(result.error).toBeUndefined(); + expect(result.consoleOutput?.length).toBeGreaterThan(0); + + const logEntry = result.consoleOutput?.find( + (entry) => entry.method === "log", + ); + expect(logEntry).toBeDefined(); + }); + + test("should capture important sections with ask262Debug", async () => { + const result = await evaluateTool({ + code: ` + ask262Debug.startImportant(); + [1, 2, 3].filter(x => x > 1); + ask262Debug.stopImportant(); + `, + }); + + expect(result.error).toBeUndefined(); + expect(result.importantSections?.length).toBeGreaterThan(0); + }); + + test("should handle empty code", async () => { + const result = await evaluateTool({ code: "" }); + + // Empty code should either succeed or have a specific error + expect(result).toBeDefined(); + }); + + test("should handle complex code with multiple operations", async () => { + const result = await evaluateTool({ + code: ` + const arr = [1, 2, 3]; + const doubled = arr.map(x => x * 2); + const filtered = doubled.filter(x => x > 2); + filtered.reduce((a, b) => a + b, 0); + `, + }); + + expect(result.error).toBeUndefined(); + expect(result.otherSections?.length).toBeGreaterThan(0); + }); + + test("should timeout after 1 second by default", async () => { + const result = await evaluateTool({ + code: "while (true) {}", // Infinite loop + }); + + expect(result.error).toBeDefined(); + expect(result.error).toContain("timeout"); + }); + + test("should return deduplicated section IDs", async () => { + const result = await evaluateTool({ + code: "[1].map(x => x).map(x => x)", + }); + + expect(result.error).toBeUndefined(); + + // Check that there are no duplicates within or between arrays + const importantSet = new Set(result.importantSections || []); + const otherSet = new Set(result.otherSections || []); + + // No duplicates within important + expect(importantSet.size).toBe((result.importantSections || []).length); + + // No duplicates within other + expect(otherSet.size).toBe((result.otherSections || []).length); + + // No overlap between important and other + for (const id of result.importantSections || []) { + expect(otherSet.has(id)).toBe(false); + } + }); +}); diff --git a/src/test/getSectionContent.test.ts b/src/test/getSectionContent.test.ts new file mode 100644 index 0000000..4119258 --- /dev/null +++ b/src/test/getSectionContent.test.ts @@ -0,0 +1,122 @@ +/** + * Tests for the getSectionContent tool. + * Tests retrieval of full section content by section ID. + */ + +import { describe, expect, test } from "bun:test"; +import { createGetSectionContentTool } from "../agent-tools/index.js"; +import { createMockTable, defaultTestData } from "./utils/mock.js"; + +describe("getSectionContent", () => { + const mockTable = createMockTable(defaultTestData); + const getContentTool = createGetSectionContentTool(mockTable); + + test("should return content for a single valid section", async () => { + const result = await getContentTool({ + sectionIds: ["sec-if-statement"], + recursive: false, + }); + + expect(result.sections).toBeArray(); + expect(result.sections.length).toBe(1); + expect(result.sections[0].found).toBe(true); + expect(result.sections[0].sectionId).toBe("sec-if-statement"); + expect(result.sections[0].content).toBeString(); + expect(result.sections[0].content.length).toBeGreaterThan(0); + }); + + test("should return content for multiple sections", async () => { + const result = await getContentTool({ + sectionIds: ["sec-if-statement", "sec-for-statement"], + recursive: false, + }); + + expect(result.sections.length).toBe(2); + expect(result.sections[0].found).toBe(true); + expect(result.sections[1].found).toBe(true); + }); + + test("should return found: false for non-existent sections", async () => { + const result = await getContentTool({ + sectionIds: ["sec-does-not-exist"], + recursive: false, + }); + + expect(result.sections.length).toBe(1); + expect(result.sections[0].found).toBe(false); + expect(result.sections[0].error).toBeDefined(); + expect(result.sections[0].content).toBe(""); + }); + + test("should handle mix of existing and non-existing sections", async () => { + const result = await getContentTool({ + sectionIds: ["sec-if-statement", "sec-does-not-exist"], + recursive: false, + }); + + expect(result.sections.length).toBe(2); + expect(result.sections[0].found).toBe(true); + expect(result.sections[1].found).toBe(false); + }); + + test("should include section metadata", async () => { + const result = await getContentTool({ + sectionIds: ["sec-if-statement"], + recursive: false, + }); + + const section = result.sections[0]; + expect(section.sectionId).toBe("sec-if-statement"); + expect(section.sectionTitle).toBeDefined(); + expect(section.found).toBe(true); + }); + + test("should return empty array for empty sectionIds", async () => { + const result = await getContentTool({ + sectionIds: [], + recursive: false, + }); + + expect(result.sections).toBeArray(); + expect(result.sections.length).toBe(0); + }); + + test("should include partIndex and totalParts when available", async () => { + const result = await getContentTool({ + sectionIds: ["sec-try-statement"], + recursive: false, + }); + + const section = result.sections[0]; + if (section.found && section.partIndex !== undefined) { + expect(section.partIndex).toBeNumber(); + expect(section.totalParts).toBeNumber(); + } + }); + + test("should fetch children when recursive is true", async () => { + const result = await getContentTool({ + sectionIds: ["sec-catch-clause"], + recursive: true, + }); + + // Should include at least the requested section + expect(result.sections.length).toBeGreaterThan(0); + const requestedSection = result.sections.find( + (s) => s.sectionId === "sec-catch-clause", + ); + expect(requestedSection).toBeDefined(); + expect(requestedSection?.found).toBe(true); + }); + + test("should preserve input order in output", async () => { + const sectionIds = ["sec-for-statement", "sec-if-statement"]; + const result = await getContentTool({ + sectionIds, + recursive: false, + }); + + expect(result.sections[0].sectionId).toBe("sec-for-statement"); + expect(result.sections[1].sectionId).toBe("sec-if-statement"); + }); +}); diff --git a/src/test/manual/test-nonexistent-sections.ts b/src/test/manual/test-nonexistent-sections.ts deleted file mode 100644 index c005eb4..0000000 --- a/src/test/manual/test-nonexistent-sections.ts +++ /dev/null @@ -1,70 +0,0 @@ -#!/usr/bin/env bun -/** - * Test to verify non-existing section IDs return found: false - */ - -import * as lancedbSdk from "@lancedb/lancedb"; -import { createGetSectionContentTool } from "../../agent-tools/getSectionContent.js"; -import { STORAGE_DIR } from "../../constants.js"; - -async function main() { - console.log("Testing getSectionContent with non-existing section IDs...\n"); - - try { - const db = await lancedbSdk.connect(STORAGE_DIR); - const table = await db.openTable("spec_vectors"); - - const getSectionContentTool = createGetSectionContentTool(table); - - // Test with mix of existing and non-existing section IDs - const result = await getSectionContentTool({ - sectionIds: [ - "sec-non-existent-12345", // Should not exist - "sec-also-fake-99999", // Should not exist - ], - recursive: false, - }); - - console.log("=== RESULT ==="); - console.log(JSON.stringify(result, null, 2)); - console.log("\n=== VERIFICATION ==="); - - // Verify all sections are returned - if (result.sections.length !== 2) { - console.error(`❌ Expected 2 sections, got ${result.sections.length}`); - process.exit(1); - } - - // Verify non-existing sections have found: false - for (const section of result.sections) { - if (section.found !== false) { - console.error( - `❌ Section ${section.sectionId} should have found: false`, - ); - process.exit(1); - } - if (!section.error) { - console.error( - `❌ Section ${section.sectionId} should have error message`, - ); - process.exit(1); - } - if (section.content !== "") { - console.error( - `❌ Section ${section.sectionId} should have empty content`, - ); - process.exit(1); - } - console.log( - `✅ ${section.sectionId}: found=false, error="${section.error}"`, - ); - } - - console.log("\n✅ All non-existing sections correctly return found: false"); - } catch (error) { - console.error("❌ Test failed:", error); - process.exit(1); - } -} - -main().catch(console.error); diff --git a/src/test/searchSpecSections.test.ts b/src/test/searchSpecSections.test.ts new file mode 100644 index 0000000..8c73de2 --- /dev/null +++ b/src/test/searchSpecSections.test.ts @@ -0,0 +1,96 @@ +/** + * Tests for the searchSpecSections tool. + * Tests vector search functionality to find relevant spec sections. + */ + +import { describe, expect, test } from "bun:test"; +import { createSearchSpecSectionsTool } from "../agent-tools/index.js"; +import { + createMockEmbeddings, + createMockTable, + defaultTestData, +} from "./utils/mock.js"; + +describe("searchSpecSections", () => { + const mockTable = createMockTable(defaultTestData); + const mockEmbeddings = createMockEmbeddings(); + const searchTool = createSearchSpecSectionsTool(mockTable, mockEmbeddings); + + test("should return results for a valid query", async () => { + const result = await searchTool({ query: "array map" }); + + expect(result.results).toBeArray(); + expect(result.results.length).toBeGreaterThan(0); + expect(result.results.length).toBeLessThanOrEqual(5); // Default limit + }); + + test("should return section metadata without content", async () => { + const result = await searchTool({ query: "if statement" }); + + const firstResult = result.results[0]; + expect(firstResult).toHaveProperty("sectionId"); + expect(firstResult).toHaveProperty("sectionTitle"); + expect(firstResult).toHaveProperty("vectorDistance"); + expect(firstResult).toHaveProperty("partIndex"); + expect(firstResult).toHaveProperty("totalParts"); + expect(firstResult).not.toHaveProperty("content"); + }); + + test("should return valid section IDs", async () => { + const result = await searchTool({ query: "loop iteration" }); + + for (const item of result.results) { + expect(item.sectionId).toBeString(); + expect(item.sectionId).toMatch(/^sec-/); + } + }); + + test("should return valid section titles", async () => { + const result = await searchTool({ query: "exception handling" }); + + for (const item of result.results) { + expect(item.sectionTitle).toBeString(); + expect(item.sectionTitle.length).toBeGreaterThan(0); + } + }); + + test("should return vectorDistance as a number", async () => { + const result = await searchTool({ query: "javascript" }); + + for (const item of result.results) { + expect(item.vectorDistance).toBeNumber(); + expect(item.vectorDistance).toBeGreaterThanOrEqual(0); + } + }); + + test("should handle empty query", async () => { + const result = await searchTool({ query: "" }); + + expect(result.results).toBeArray(); + // May return results or empty depending on mock implementation + }); + + test("should handle complex queries", async () => { + const result = await searchTool({ + query: "how does array prototype map method work with callbacks", + }); + + expect(result.results).toBeArray(); + expect(result.results.length).toBeGreaterThan(0); + }); + + test("should return partIndex and totalParts as numbers or null", async () => { + const result = await searchTool({ query: "try catch" }); + + for (const item of result.results) { + if (item.partIndex !== null) { + expect(item.partIndex).toBeNumber(); + expect(item.partIndex).toBeGreaterThanOrEqual(0); + } + if (item.totalParts !== null) { + expect(item.totalParts).toBeNumber(); + expect(item.totalParts).toBeGreaterThan(0); + } + } + }); +}); diff --git a/src/test/utils/mock.ts b/src/test/utils/mock.ts new file mode 100644 index 0000000..6b9d53b --- /dev/null +++ b/src/test/utils/mock.ts @@ -0,0 +1,132 @@ +/** + * Test utilities for mocking LanceDB and embeddings. + * Provides helper functions to create mock Table instances for testing. + */ + +import type { Table } from "@lancedb/lancedb"; +import type { Embeddings } from "@langchain/core/embeddings"; + +/** + * Mock embeddings instance for testing. + * Returns predictable vectors based on input text. + */ +export function createMockEmbeddings(): Embeddings { + return { + embedQuery: async (text: string): Promise => { + // Return a simple hash-based vector for testing + const vector = new Array(128).fill(0); + for (let i = 0; i < text.length; i++) { + vector[i % 128] += text.charCodeAt(i) / 1000; + } + return vector; + }, + embedDocuments: async (documents: string[]): Promise => { + return Promise.all( + documents.map((doc) => createMockEmbeddings().embedQuery(doc)), + ); + }, + } as Embeddings; +} + +/** + * Create a mock LanceDB table with test data. + * @param testData - Array of test documents to return from queries + */ +export function createMockTable(testData: MockTableData[]): Table { + return { + search: (_queryVector: number[]) => ({ + limit: (_n: number) => ({ + toArray: async () => { + // Return first n results, sorted by a simple distance calculation + return testData + .map((data) => ({ + ...data, + _distance: Math.random() * 0.5, // Random distance for testing + })) + .slice(0, _n); + }, + }), + }), + query: () => ({ + where: (condition: string) => ({ + limit: (_n: number) => ({ + toArray: async () => { + // Parse sectionid from condition like "sectionid = 'sec-xxx'" + const match = condition.match(/sectionid = ['"]([^'"]+)['"]/); + const sectionId = match ? match[1] : null; + + if (sectionId) { + return testData + .filter((data) => data.sectionid === sectionId) + .map((data) => ({ + ...data, + })); + } + return []; + }, + }), + }), + }), + } as unknown as Table; +} + +/** + * Mock table data structure + */ +export interface MockTableData { + sectionid: string; + sectiontitle: string; + text: string; + partindex?: number; + totalparts?: number; + childrensectionids?: string[]; +} + +/** + * Default test data for spec sections + */ +export const defaultTestData: MockTableData[] = [ + { + sectionid: "sec-if-statement", + sectiontitle: "The if Statement", + text: "The if statement evaluates a condition and executes a block if true.", + partindex: 0, + totalparts: 1, + }, + { + sectionid: "sec-for-statement", + sectiontitle: "The for Statement", + text: "The for statement creates a loop with initialization, condition, and increment.", + partindex: 0, + totalparts: 1, + }, + { + sectionid: "sec-array.prototype.map", + sectiontitle: "Array.prototype.map", + text: "The map method creates a new array by applying a function to each element.", + partindex: 0, + totalparts: 1, + }, + { + sectionid: "sec-array.prototype.filter", + sectiontitle: "Array.prototype.filter", + text: "The filter method creates a new array with elements that pass the test.", + partindex: 0, + totalparts: 1, + }, + { + sectionid: "sec-try-statement", + sectiontitle: "The try Statement", + text: "The try statement marks a block of statements to try, with catch and finally clauses.", + partindex: 0, + totalparts: 2, + }, + { + sectionid: "sec-catch-clause", + sectiontitle: "The catch Clause", + text: "The catch clause provides exception handling for the try block.", + partindex: 1, + totalparts: 2, + childrensectionids: ["sec-try-statement"], + }, +];