From 83719ee47b8f7c8b90f7b01353b3b357e1121d98 Mon Sep 17 00:00:00 2001 From: bendtherules Date: Wed, 22 Apr 2026 18:00:49 +0530 Subject: [PATCH] Improve tests 1 --- src/test/evaluateInEngine262.test.ts | 482 ++++++++++++++++++++------ src/test/getSectionContent.test.ts | 492 +++++++++++++++++++++------ src/test/searchSpecSections.test.ts | 286 ++++++++++++---- src/test/utils/mock.ts | 174 +++++++--- 4 files changed, 1115 insertions(+), 319 deletions(-) diff --git a/src/test/evaluateInEngine262.test.ts b/src/test/evaluateInEngine262.test.ts index c3c4d84..1c73166 100644 --- a/src/test/evaluateInEngine262.test.ts +++ b/src/test/evaluateInEngine262.test.ts @@ -1,135 +1,409 @@ /** * Tests for the evaluateInEngine262 tool. * Tests JavaScript code execution in engine262 and spec section capture. + * + * Coverage: + * - Schema validation (input/output) + * - Proper result structure for success and error + * - Syntax errors, runtime errors, reference errors + * - Console output capture (log, warn, debug, error) + * - ask262Debug important sections + * - Timeout behavior (default and custom) + * - Section deduplication + * - Edge cases (empty code, whitespace, long code) + * - Complex multi-operation code */ import { describe, expect, test } from "bun:test"; -import { createEvaluateInEngine262Tool } from "../agent-tools/index.js"; +import { + createEvaluateInEngine262Tool, + evaluateInputSchema, + evaluateOutputSchema, + evaluateToolName, +} from "../agent-tools/index.js"; describe("evaluateInEngine262", () => { const evaluateTool = createEvaluateInEngine262Tool(); - test("should execute simple arithmetic", async () => { - const result = await evaluateTool({ code: "1 + 1" }); + // #region Schema validation - expect(result.error).toBeUndefined(); - expect(result.importantSections).toBeArray(); - expect(result.otherSections).toBeArray(); - expect(result.consoleOutput).toBeArray(); - }); - - test("should capture spec sections for array operations", async () => { - const result = await evaluateTool({ code: "[1, 2, 3].map(x => x * 2)" }); - - expect(result.error).toBeUndefined(); - expect(result.otherSections?.length).toBeGreaterThan(0); - - // Should include array-related spec sections - const sectionIds = [ - ...(result.importantSections || []), - ...(result.otherSections || []), - ]; - expect(sectionIds.some((id) => id.includes("array"))).toBe(true); - }); - - test("should return error for syntax errors", async () => { - const result = await evaluateTool({ code: "function {" }); - - expect(result.error).toBeDefined(); - expect(result.error).toContain("SyntaxError"); - }); - - test("should return error for runtime errors", async () => { - const result = await evaluateTool({ code: "null.foo" }); - - expect(result.error).toBeDefined(); - expect(result.error).toContain("TypeError"); - }); - - test("should return error for reference errors", async () => { - const result = await evaluateTool({ code: "arr.push(1)" }); - - expect(result.error).toBeDefined(); - expect(result.error).toContain("ReferenceError"); - }); - - test("should capture console output", async () => { - const result = await evaluateTool({ - code: "console.log('hello', 123); console.warn('test')", + describe("input schema validation", () => { + test("should accept valid code string", () => { + const result = evaluateInputSchema.safeParse({ code: "1 + 1" }); + expect(result.success).toBe(true); }); - expect(result.error).toBeUndefined(); - expect(result.consoleOutput?.length).toBeGreaterThan(0); - - const logEntry = result.consoleOutput?.find( - (entry) => entry.method === "log", - ); - expect(logEntry).toBeDefined(); - }); - - test("should capture important sections with ask262Debug", async () => { - const result = await evaluateTool({ - code: ` - ask262Debug.startImportant(); - [1, 2, 3].filter(x => x > 1); - ask262Debug.stopImportant(); - `, + test("should reject missing code", () => { + const result = evaluateInputSchema.safeParse({}); + expect(result.success).toBe(false); }); - expect(result.error).toBeUndefined(); - expect(result.importantSections?.length).toBeGreaterThan(0); - }); - - test("should handle empty code", async () => { - const result = await evaluateTool({ code: "" }); - - // Empty code should either succeed or have a specific error - expect(result).toBeDefined(); - }); - - test("should handle complex code with multiple operations", async () => { - const result = await evaluateTool({ - code: ` - const arr = [1, 2, 3]; - const doubled = arr.map(x => x * 2); - const filtered = doubled.filter(x => x > 2); - filtered.reduce((a, b) => a + b, 0); - `, + test("should reject non-string code", () => { + const result = evaluateInputSchema.safeParse({ code: 123 }); + expect(result.success).toBe(false); }); - expect(result.error).toBeUndefined(); - expect(result.otherSections?.length).toBeGreaterThan(0); + test("should accept empty string code", () => { + const result = evaluateInputSchema.safeParse({ code: "" }); + expect(result.success).toBe(true); + }); }); - test("should timeout after 1 second by default", async () => { - const result = await evaluateTool({ - code: "while (true) {}", // Infinite loop + describe("output schema validation", () => { + test("should validate success output conforms to schema", async () => { + const result = await evaluateTool({ code: "1 + 1" }); + const validation = evaluateOutputSchema.safeParse(result); + expect(validation.success).toBe(true); }); - expect(result.error).toBeDefined(); - expect(result.error).toContain("timeout"); + test("should validate error output conforms to schema", async () => { + const result = await evaluateTool({ code: "function {" }); + const validation = evaluateOutputSchema.safeParse(result); + expect(validation.success).toBe(true); + }); }); - test("should return deduplicated section IDs", async () => { - const result = await evaluateTool({ - code: "[1].map(x => x).map(x => x)", + // #endregion + + // #region Success cases + + describe("successful execution", () => { + test("should execute simple arithmetic", async () => { + const result = await evaluateTool({ code: "1 + 1" }); + + expect(result.error).toBeUndefined(); + expect(result.importantSections).toBeArray(); + expect(result.otherSections).toBeArray(); + expect(result.consoleOutput).toBeArray(); }); - expect(result.error).toBeUndefined(); + test("should capture spec sections for array operations", async () => { + const result = await evaluateTool({ + code: "[1, 2, 3].map(x => x * 2)", + }); - // Check that there are no duplicates within or between arrays - const importantSet = new Set(result.importantSections || []); - const otherSet = new Set(result.otherSections || []); + expect(result.error).toBeUndefined(); + expect(result.otherSections?.length).toBeGreaterThan(0); - // No duplicates within important - expect(importantSet.size).toBe((result.importantSections || []).length); + const sectionIds = [ + ...(result.importantSections || []), + ...(result.otherSections || []), + ]; + expect(sectionIds.some((id) => id.includes("array"))).toBe(true); + }); - // No duplicates within other - expect(otherSet.size).toBe((result.otherSections || []).length); + test("should handle complex code with multiple operations", async () => { + const result = await evaluateTool({ + code: ` + const arr = [1, 2, 3]; + const doubled = arr.map(x => x * 2); + const filtered = doubled.filter(x => x > 2); + filtered.reduce((a, b) => a + b, 0); + `, + }); - // No overlap between important and other - for (const id of result.importantSections || []) { - expect(otherSet.has(id)).toBe(false); - } + expect(result.error).toBeUndefined(); + expect(result.otherSections?.length).toBeGreaterThan(0); + }); + + test("should handle string operations", async () => { + const result = await evaluateTool({ + code: "'hello'.toUpperCase()", + }); + + expect(result.error).toBeUndefined(); + expect(result.importantSections).toBeArray(); + expect(result.otherSections).toBeArray(); + }); + + test("should handle object operations", async () => { + const result = await evaluateTool({ + code: "Object.keys({a: 1, b: 2})", + }); + + expect(result.error).toBeUndefined(); + expect(result.otherSections?.length).toBeGreaterThan(0); + }); + + test("should handle class definitions", async () => { + const result = await evaluateTool({ + code: ` + class Foo { + constructor(x) { this.x = x; } + getX() { return this.x; } + } + new Foo(42).getX(); + `, + }); + + expect(result.error).toBeUndefined(); + }); }); + + // #endregion + + // #region Error cases + + describe("error handling", () => { + test("should return error for syntax errors", async () => { + const result = await evaluateTool({ code: "function {" }); + + expect(result.error).toBeDefined(); + expect(result.error).toContain("SyntaxError"); + expect(result.importantSections).toBeUndefined(); + expect(result.otherSections).toBeUndefined(); + expect(result.consoleOutput).toBeUndefined(); + }); + + test("should return error for runtime errors", async () => { + const result = await evaluateTool({ code: "null.foo" }); + + expect(result.error).toBeDefined(); + expect(result.error).toContain("TypeError"); + }); + + test("should return error for reference errors", async () => { + const result = await evaluateTool({ code: "arr.push(1)" }); + + expect(result.error).toBeDefined(); + expect(result.error).toContain("ReferenceError"); + }); + + test("should return error for throw statements", async () => { + const result = await evaluateTool({ + code: "throw new Error('custom error')", + }); + + expect(result.error).toBeDefined(); + expect(result.error).toContain("custom error"); + }); + }); + + // #endregion + + // #region Console output + + describe("console output", () => { + test("should capture console.log output", async () => { + const result = await evaluateTool({ + code: "console.log('hello', 123)", + }); + + expect(result.error).toBeUndefined(); + expect(result.consoleOutput?.length).toBeGreaterThan(0); + + const logEntry = result.consoleOutput?.find( + (entry) => entry.method === "log", + ); + expect(logEntry).toBeDefined(); + expect(logEntry?.values).toContain("hello"); + expect(logEntry?.values).toContain(123); + }); + + test("should capture console.warn output", async () => { + const result = await evaluateTool({ + code: "console.warn('warning message')", + }); + + expect(result.error).toBeUndefined(); + const warnEntry = result.consoleOutput?.find( + (entry) => entry.method === "warn", + ); + expect(warnEntry).toBeDefined(); + expect(warnEntry?.values).toContain("warning message"); + }); + + test("should capture multiple console calls", async () => { + const result = await evaluateTool({ + code: "console.log('first'); console.warn('second'); console.log('third')", + }); + + expect(result.error).toBeUndefined(); + expect(result.consoleOutput?.length).toBeGreaterThanOrEqual(3); + }); + + test("should capture mixed console methods", async () => { + const result = await evaluateTool({ + code: "console.log('log'); console.warn('warn'); console.error('error'); console.debug('debug')", + }); + + expect(result.error).toBeUndefined(); + const methods = result.consoleOutput?.map((e) => e.method) ?? []; + expect(methods).toContain("log"); + expect(methods).toContain("warn"); + expect(methods).toContain("error"); + expect(methods).toContain("debug"); + }); + }); + + // #endregion + + // #region Important sections + + describe("important sections with ask262Debug", () => { + test("should capture important sections with ask262Debug", async () => { + const result = await evaluateTool({ + code: ` + ask262Debug.startImportant(); + [1, 2, 3].filter(x => x > 1); + ask262Debug.stopImportant(); + `, + }); + + expect(result.error).toBeUndefined(); + expect(result.importantSections?.length).toBeGreaterThan(0); + }); + + test("should separate important and other sections", async () => { + const result = await evaluateTool({ + code: ` + [1, 2].map(x => x); + ask262Debug.startImportant(); + [3, 4].filter(x => x > 3); + ask262Debug.stopImportant(); + `, + }); + + expect(result.error).toBeUndefined(); + expect(result.importantSections?.length).toBeGreaterThan(0); + expect(result.otherSections?.length).toBeGreaterThan(0); + }); + }); + + // #endregion + + // #region Timeout + + describe("timeout behavior", () => { + test("should timeout after 1 second by default", async () => { + const result = await evaluateTool({ + code: "while (true) {}", + }); + + expect(result.error).toBeDefined(); + expect(result.error).toContain("timeout"); + }); + + test("should respect custom timeout", async () => { + const slowTool = createEvaluateInEngine262Tool(500); + const result = await slowTool({ + code: "while (true) {}", + }); + + expect(result.error).toBeDefined(); + expect(result.error).toContain("timeout"); + expect(result.error).toContain("500ms"); + }); + + test("should complete within long timeout for slow code", async () => { + const longTool = createEvaluateInEngine262Tool(5000); + const result = await longTool({ + code: ` + let sum = 0; + for (let i = 0; i < 1000; i++) { sum += i; } + sum; + `, + }); + + expect(result.error).toBeUndefined(); + }); + }); + + // #endregion + + // #region Deduplication + + describe("section deduplication", () => { + test("should return deduplicated section IDs", async () => { + const result = await evaluateTool({ + code: "[1].map(x => x).map(x => x)", + }); + + expect(result.error).toBeUndefined(); + + const importantSet = new Set(result.importantSections || []); + const otherSet = new Set(result.otherSections || []); + + // No duplicates within important + expect(importantSet.size).toBe((result.importantSections || []).length); + + // No duplicates within other + expect(otherSet.size).toBe((result.otherSections || []).length); + + // No overlap between important and other + for (const id of result.importantSections || []) { + expect(otherSet.has(id)).toBe(false); + } + }); + }); + + // #endregion + + // #region Edge cases + + describe("edge cases", () => { + test("should handle empty code", async () => { + const result = await evaluateTool({ code: "" }); + + expect(result).toBeDefined(); + // Empty code should either succeed with empty sections or error + expect( + result.error !== undefined || + (result.importantSections !== undefined && + result.otherSections !== undefined), + ).toBe(true); + }); + + test("should handle whitespace-only code", async () => { + const result = await evaluateTool({ code: " \n\n \t " }); + + expect(result).toBeDefined(); + }); + + test("should handle code with comments only", async () => { + const result = await evaluateTool({ + code: "// this is a comment\n/* another comment */", + }); + + expect(result).toBeDefined(); + }); + + test("should handle large code (>500 chars)", async () => { + const largeCode = `const arr = [${Array.from( + { length: 200 }, + (_, i) => i, + ).join(", ")}]; arr.length;`; + + expect(largeCode.length).toBeGreaterThan(500); + + const result = await evaluateTool({ code: largeCode }); + + expect(result.error).toBeUndefined(); + }); + + test("should handle nested function calls", async () => { + const result = await evaluateTool({ + code: "[1, [2, [3]]].flat(2)", + }); + + expect(result.error).toBeUndefined(); + }); + }); + + // #endregion + + // #region Tool metadata + + describe("tool metadata", () => { + test("should export correct tool name", () => { + expect(evaluateToolName).toBe("ask262_evaluate_in_engine262"); + }); + + test("should export input and output schemas", () => { + expect(evaluateInputSchema).toBeDefined(); + expect(evaluateOutputSchema).toBeDefined(); + }); + }); + + // #endregion }); diff --git a/src/test/getSectionContent.test.ts b/src/test/getSectionContent.test.ts index 2383b25..9ff7099 100644 --- a/src/test/getSectionContent.test.ts +++ b/src/test/getSectionContent.test.ts @@ -1,154 +1,440 @@ /** * Tests for the getSectionContent tool. * Tests retrieval of full section content by section ID. + * + * Coverage: + * - Schema validation (input/output) + * - Proper result content and metadata + * - Error handling (DB failures) + * - Recursive fetching (deep nesting, cycles) + * - Multi-part section ordering + * - Edge cases (empty, duplicates, not found) */ import { describe, expect, test } from "bun:test"; -import { createGetSectionContentTool } from "../agent-tools/index.js"; -import { createMockTable, defaultTestData } from "./utils/mock.js"; +import { + createGetSectionContentTool, + getSectionInputSchema, + getSectionOutputSchema, + sectionContentToolName, +} from "../agent-tools/index.js"; +import { + createFailingTable, + createMockTable, + defaultTestData, + multiPartTestData, + recursiveTestData, +} from "./utils/mock.js"; describe("getSectionContent", () => { const mockTable = createMockTable(defaultTestData); const getContentTool = createGetSectionContentTool(mockTable); - test("should return content for a single valid section", async () => { - const result = await getContentTool({ - sectionIds: ["sec-if-statement"], - recursive: false, + // #region Schema validation + + describe("input schema validation", () => { + test("should accept valid input with sectionIds and recursive", () => { + const result = getSectionInputSchema.safeParse({ + sectionIds: ["sec-if-statement"], + recursive: false, + }); + expect(result.success).toBe(true); }); - expect(result.sections).toBeArray(); - expect(result.sections.length).toBe(1); - expect(result.sections[0].found).toBe(true); - expect(result.sections[0].sectionId).toBe("sec-if-statement"); - expect(result.sections[0].content).toBeString(); - expect(result.sections[0].content.length).toBeGreaterThan(0); + test("should apply default recursive=true", () => { + const result = getSectionInputSchema.safeParse({ + sectionIds: ["sec-if-statement"], + }); + expect(result.success).toBe(true); + if (result.success) { + expect(result.data.recursive).toBe(true); + } + }); + + test("should reject empty sectionIds array", () => { + const result = getSectionInputSchema.safeParse({ + sectionIds: [], + recursive: false, + }); + // Schema allows empty arrays - this test documents the behavior + expect(result.success).toBe(true); + }); + + test("should reject non-array sectionIds", () => { + const result = getSectionInputSchema.safeParse({ + sectionIds: "sec-if-statement", + recursive: false, + }); + expect(result.success).toBe(false); + }); + + test("should reject non-boolean recursive", () => { + const result = getSectionInputSchema.safeParse({ + sectionIds: ["sec-if-statement"], + recursive: "yes", + }); + expect(result.success).toBe(false); + }); + + test("should reject missing sectionIds", () => { + const result = getSectionInputSchema.safeParse({ + recursive: false, + }); + expect(result.success).toBe(false); + }); }); - test("should return content for multiple sections", async () => { - const result = await getContentTool({ - sectionIds: ["sec-if-statement", "sec-for-statement"], - recursive: false, + describe("output schema validation", () => { + test("should validate output conforms to schema", async () => { + const result = await getContentTool({ + sectionIds: ["sec-if-statement"], + recursive: false, + }); + + const validation = getSectionOutputSchema.safeParse(result); + expect(validation.success).toBe(true); }); - expect(result.sections.length).toBe(2); - expect(result.sections[0].found).toBe(true); - expect(result.sections[1].found).toBe(true); + test("should validate not-found output conforms to schema", async () => { + const result = await getContentTool({ + sectionIds: ["sec-nonexistent"], + recursive: false, + }); + + const validation = getSectionOutputSchema.safeParse(result); + expect(validation.success).toBe(true); + }); + + test("should validate recursive output conforms to schema", async () => { + const result = await getContentTool({ + sectionIds: ["sec-catch-clause"], + recursive: true, + }); + + const validation = getSectionOutputSchema.safeParse(result); + expect(validation.success).toBe(true); + }); }); - test("should return found: false for non-existent sections", async () => { - const result = await getContentTool({ - sectionIds: ["sec-does-not-exist"], - recursive: false, + // #endregion + + // #region Proper results + + describe("single section retrieval", () => { + test("should return content for a single valid section", async () => { + const result = await getContentTool({ + sectionIds: ["sec-if-statement"], + recursive: false, + }); + + expect(result.sections).toBeArray(); + expect(result.sections.length).toBe(1); + expect(result.sections[0].found).toBe(true); + expect(result.sections[0].sectionId).toBe("sec-if-statement"); + expect(result.sections[0].content).toBeString(); + expect(result.sections[0].content.length).toBeGreaterThan(0); + expect(result.sections[0].content).toContain("evaluates a condition"); }); - expect(result.sections.length).toBe(1); - expect(result.sections[0].found).toBe(false); - expect(result.sections[0].error).toBeDefined(); - expect(result.sections[0].content).toBe(""); + test("should return correct section title", async () => { + const result = await getContentTool({ + sectionIds: ["sec-if-statement"], + recursive: false, + }); + + expect(result.sections[0].sectionTitle).toBe("The if Statement"); + }); + + test("should include section metadata", async () => { + const result = await getContentTool({ + sectionIds: ["sec-if-statement"], + recursive: false, + }); + + const section = result.sections[0]; + expect(section.sectionId).toBe("sec-if-statement"); + expect(section.sectionTitle).toBeDefined(); + expect(section.found).toBe(true); + expect(section.content).toBeString(); + }); }); - test("should handle mix of existing and non-existing sections", async () => { - const result = await getContentTool({ - sectionIds: ["sec-if-statement", "sec-does-not-exist"], - recursive: false, + describe("multiple section retrieval", () => { + test("should return content for multiple sections", async () => { + const result = await getContentTool({ + sectionIds: ["sec-if-statement", "sec-for-statement"], + recursive: false, + }); + + expect(result.sections.length).toBe(2); + expect(result.sections[0].found).toBe(true); + expect(result.sections[1].found).toBe(true); + expect(result.sections[0].sectionId).toBe("sec-if-statement"); + expect(result.sections[1].sectionId).toBe("sec-for-statement"); }); - expect(result.sections.length).toBe(2); - expect(result.sections[0].found).toBe(true); - expect(result.sections[1].found).toBe(false); + test("should handle mix of existing and non-existing sections", async () => { + const result = await getContentTool({ + sectionIds: ["sec-if-statement", "sec-does-not-exist"], + recursive: false, + }); + + expect(result.sections.length).toBe(2); + expect(result.sections[0].found).toBe(true); + expect(result.sections[1].found).toBe(false); + expect(result.sections[1].error).toContain("not found"); + expect(result.sections[1].content).toBe(""); + }); + + test("should preserve input order in output", async () => { + const sectionIds = ["sec-for-statement", "sec-if-statement"]; + const result = await getContentTool({ + sectionIds, + recursive: false, + }); + + expect(result.sections[0].sectionId).toBe("sec-for-statement"); + expect(result.sections[1].sectionId).toBe("sec-if-statement"); + }); }); - test("should include section metadata", async () => { - const result = await getContentTool({ - sectionIds: ["sec-if-statement"], - recursive: false, + // #endregion + + // #region Not found handling + + describe("not found handling", () => { + test("should return found: false for non-existent sections", async () => { + const result = await getContentTool({ + sectionIds: ["sec-does-not-exist"], + recursive: false, + }); + + expect(result.sections.length).toBe(1); + expect(result.sections[0].found).toBe(false); + expect(result.sections[0].error).toBeDefined(); + expect(result.sections[0].error).toContain("sec-does-not-exist"); + expect(result.sections[0].content).toBe(""); + expect(result.sections[0].sectionId).toBe("sec-does-not-exist"); }); - const section = result.sections[0]; - expect(section.sectionId).toBe("sec-if-statement"); - expect(section.sectionTitle).toBeDefined(); - expect(section.found).toBe(true); + test("should return empty array for empty sectionIds", async () => { + const result = await getContentTool({ + sectionIds: [], + recursive: false, + }); + + expect(result.sections).toBeArray(); + expect(result.sections.length).toBe(0); + }); + + test("should handle all non-existent sections", async () => { + const result = await getContentTool({ + sectionIds: ["sec-a", "sec-b", "sec-c"], + recursive: false, + }); + + expect(result.sections.length).toBe(3); + for (const section of result.sections) { + expect(section.found).toBe(false); + expect(section.content).toBe(""); + expect(section.error).toBeDefined(); + } + }); }); - test("should return empty array for empty sectionIds", async () => { - const result = await getContentTool({ - sectionIds: [], - recursive: false, - }); + // #endregion - expect(result.sections).toBeArray(); - expect(result.sections.length).toBe(0); + // #region Multi-part sections + + describe("multi-part sections", () => { + const multiPartTable = createMockTable(multiPartTestData); + const multiPartTool = createGetSectionContentTool(multiPartTable); + + test("should join multi-part content in partIndex order", async () => { + const result = await multiPartTool({ + sectionIds: ["sec-species-conformance"], + recursive: false, + }); + + expect(result.sections.length).toBe(1); + expect(result.sections[0].found).toBe(true); + // Content should be joined with \n\n in partIndex order (0, 1, 2) + const content = result.sections[0].content; + const parts = content.split("\n\n"); + expect(parts.length).toBe(3); + expect(parts[0]).toContain("Part 0"); + expect(parts[1]).toContain("Part 1"); + expect(parts[2]).toContain("Part 2 final"); + }); }); - test("should include childrensectionids when available", async () => { - const result = await getContentTool({ - sectionIds: ["sec-catch-clause"], - recursive: false, - }); + // #endregion - const section = result.sections[0]; - if (section.found && section.childrensectionids !== undefined) { + // #region Recursive fetching + + describe("recursive fetching", () => { + test("should include childrensectionids when available", async () => { + const result = await getContentTool({ + sectionIds: ["sec-catch-clause"], + recursive: false, + }); + + const section = result.sections[0]; + expect(section.found).toBe(true); expect(section.childrensectionids).toBeArray(); - } - }); - - test("should fetch children when recursive is true", async () => { - const result = await getContentTool({ - sectionIds: ["sec-catch-clause"], - recursive: true, + expect(section.childrensectionids).toContain("sec-try-statement"); }); - // sec-catch-clause has sec-try-statement as a child, so expect 2 sections - expect(result.sections.length).toBe(2); - expect(result.sections[0].sectionId).toBe("sec-catch-clause"); - expect(result.sections[1].sectionId).toBe("sec-try-statement"); - }); + test("should fetch children when recursive is true", async () => { + const result = await getContentTool({ + sectionIds: ["sec-catch-clause"], + recursive: true, + }); - test("should include recursively fetched child sections in output", async () => { - const result = await getContentTool({ - sectionIds: ["sec-catch-clause"], - recursive: true, + expect(result.sections.length).toBe(2); + expect(result.sections[0].sectionId).toBe("sec-catch-clause"); + expect(result.sections[1].sectionId).toBe("sec-try-statement"); }); - // Requested section should come first - expect(result.sections[0].sectionId).toBe("sec-catch-clause"); + test("should include recursively fetched child sections with full content", async () => { + const result = await getContentTool({ + sectionIds: ["sec-catch-clause"], + recursive: true, + }); - // Child section should be included after requested section - const childSection = result.sections.find( - (s) => s.sectionId === "sec-try-statement", - ); - expect(childSection).toBeDefined(); - expect(childSection?.found).toBe(true); - expect(childSection?.sectionTitle).toBe("The try Statement"); - expect(childSection?.content).toBeString(); - expect(childSection?.content.length).toBeGreaterThan(0); - }); + expect(result.sections[0].sectionId).toBe("sec-catch-clause"); - test("should not include children when recursive is false", async () => { - const result = await getContentTool({ - sectionIds: ["sec-catch-clause"], - recursive: false, + const childSection = result.sections.find( + (s) => s.sectionId === "sec-try-statement", + ); + expect(childSection).toBeDefined(); + expect(childSection?.found).toBe(true); + expect(childSection?.sectionTitle).toBe("The try Statement"); + expect(childSection?.content).toBeString(); + expect(childSection?.content.length).toBeGreaterThan(0); }); - // Only the requested section, no children - expect(result.sections.length).toBe(1); - expect(result.sections[0].sectionId).toBe("sec-catch-clause"); + test("should not include children when recursive is false", async () => { + const result = await getContentTool({ + sectionIds: ["sec-catch-clause"], + recursive: false, + }); - const childSection = result.sections.find( - (s) => s.sectionId === "sec-try-statement", - ); - expect(childSection).toBeUndefined(); - }); + expect(result.sections.length).toBe(1); + expect(result.sections[0].sectionId).toBe("sec-catch-clause"); - test("should preserve input order in output", async () => { - const sectionIds = ["sec-for-statement", "sec-if-statement"]; - const result = await getContentTool({ - sectionIds, - recursive: false, + const childSection = result.sections.find( + (s) => s.sectionId === "sec-try-statement", + ); + expect(childSection).toBeUndefined(); }); - expect(result.sections[0].sectionId).toBe("sec-for-statement"); - expect(result.sections[1].sectionId).toBe("sec-if-statement"); + test("should traverse deeply nested children", async () => { + const deepTable = createMockTable(recursiveTestData); + const deepTool = createGetSectionContentTool(deepTable); + + const result = await deepTool({ + sectionIds: ["sec-root"], + recursive: true, + }); + + const sectionIds = result.sections.map((s) => s.sectionId); + expect(sectionIds).toContain("sec-root"); + expect(sectionIds).toContain("sec-child-a"); + expect(sectionIds).toContain("sec-child-b"); + expect(sectionIds).toContain("sec-grandchild"); + expect(result.sections.length).toBe(4); + }); + + test("should place requested sections before children in output", async () => { + const deepTable = createMockTable(recursiveTestData); + const deepTool = createGetSectionContentTool(deepTable); + + const result = await deepTool({ + sectionIds: ["sec-root"], + recursive: true, + }); + + // First section should be the requested one + expect(result.sections[0].sectionId).toBe("sec-root"); + // Remaining should be children + const childIds = result.sections.slice(1).map((s) => s.sectionId); + expect(childIds).toContain("sec-child-a"); + expect(childIds).toContain("sec-child-b"); + }); }); + + // #endregion + + // #region Edge cases + + describe("edge cases", () => { + test("should handle duplicate section IDs", async () => { + const result = await getContentTool({ + sectionIds: ["sec-if-statement", "sec-if-statement"], + recursive: false, + }); + + // Both entries should be in output (one for each requested ID) + expect(result.sections.length).toBe(2); + expect(result.sections[0].sectionId).toBe("sec-if-statement"); + expect(result.sections[1].sectionId).toBe("sec-if-statement"); + }); + + test("should handle sections without childrensectionids", async () => { + const result = await getContentTool({ + sectionIds: ["sec-if-statement"], + recursive: true, + }); + + expect(result.sections.length).toBe(1); + expect(result.sections[0].found).toBe(true); + expect(result.sections[0].childrensectionids).toBeUndefined(); + }); + + test("should handle large section ID list", async () => { + const manyIds = Array.from({ length: 50 }, (_, i) => `sec-item-${i}`); + const result = await getContentTool({ + sectionIds: manyIds, + recursive: false, + }); + + expect(result.sections.length).toBe(50); + for (const section of result.sections) { + expect(section.found).toBe(false); + } + }); + }); + + // #endregion + + // #region Error handling + + describe("error handling", () => { + test("should propagate database errors", async () => { + const failTable = createFailingTable(); + const failTool = createGetSectionContentTool(failTable); + + await expect( + failTool({ sectionIds: ["sec-if-statement"], recursive: false }), + ).rejects.toThrow(); + }); + }); + + // #endregion + + // #region Tool metadata + + describe("tool metadata", () => { + test("should export correct tool name", () => { + expect(sectionContentToolName).toBe("ask262_get_section_content"); + }); + + test("should export input and output schemas", () => { + expect(getSectionInputSchema).toBeDefined(); + expect(getSectionOutputSchema).toBeDefined(); + }); + }); + + // #endregion }); diff --git a/src/test/searchSpecSections.test.ts b/src/test/searchSpecSections.test.ts index 8c73de2..246b87b 100644 --- a/src/test/searchSpecSections.test.ts +++ b/src/test/searchSpecSections.test.ts @@ -1,11 +1,28 @@ /** * Tests for the searchSpecSections tool. * Tests vector search functionality to find relevant spec sections. + * + * Coverage: + * - Schema validation (input/output) + * - Proper result structure and content + * - Result ordering by vector distance + * - Limit enforcement (max 5) + * - Embedding failure handling + * - Database failure handling + * - Edge cases (empty query, special characters) + * - Part index and total parts handling */ import { describe, expect, test } from "bun:test"; -import { createSearchSpecSectionsTool } from "../agent-tools/index.js"; import { + createSearchSpecSectionsTool, + searchSpecInputSchema, + searchSpecOutputSchema, + searchSpecToolName, +} from "../agent-tools/index.js"; +import { + createFailingEmbeddings, + createFailingTable, createMockEmbeddings, createMockTable, defaultTestData, @@ -16,81 +33,208 @@ describe("searchSpecSections", () => { const mockEmbeddings = createMockEmbeddings(); const searchTool = createSearchSpecSectionsTool(mockTable, mockEmbeddings); - test("should return results for a valid query", async () => { - const result = await searchTool({ query: "array map" }); + // #region Schema validation - expect(result.results).toBeArray(); - expect(result.results.length).toBeGreaterThan(0); - expect(result.results.length).toBeLessThanOrEqual(5); // Default limit - }); - - test("should return section metadata without content", async () => { - const result = await searchTool({ query: "if statement" }); - - const firstResult = result.results[0]; - expect(firstResult).toHaveProperty("sectionId"); - expect(firstResult).toHaveProperty("sectionTitle"); - expect(firstResult).toHaveProperty("vectorDistance"); - expect(firstResult).toHaveProperty("partIndex"); - expect(firstResult).toHaveProperty("totalParts"); - expect(firstResult).not.toHaveProperty("content"); - }); - - test("should return valid section IDs", async () => { - const result = await searchTool({ query: "loop iteration" }); - - for (const item of result.results) { - expect(item.sectionId).toBeString(); - expect(item.sectionId).toMatch(/^sec-/); - } - }); - - test("should return valid section titles", async () => { - const result = await searchTool({ query: "exception handling" }); - - for (const item of result.results) { - expect(item.sectionTitle).toBeString(); - expect(item.sectionTitle.length).toBeGreaterThan(0); - } - }); - - test("should return vectorDistance as a number", async () => { - const result = await searchTool({ query: "javascript" }); - - for (const item of result.results) { - expect(item.vectorDistance).toBeNumber(); - expect(item.vectorDistance).toBeGreaterThanOrEqual(0); - } - }); - - test("should handle empty query", async () => { - const result = await searchTool({ query: "" }); - - expect(result.results).toBeArray(); - // May return results or empty depending on mock implementation - }); - - test("should handle complex queries", async () => { - const result = await searchTool({ - query: "how does array prototype map method work with callbacks", + describe("input schema validation", () => { + test("should accept valid query string", () => { + const result = searchSpecInputSchema.safeParse({ query: "array map" }); + expect(result.success).toBe(true); }); - expect(result.results).toBeArray(); - expect(result.results.length).toBeGreaterThan(0); + test("should reject missing query", () => { + const result = searchSpecInputSchema.safeParse({}); + expect(result.success).toBe(false); + }); + + test("should reject non-string query", () => { + const result = searchSpecInputSchema.safeParse({ query: 123 }); + expect(result.success).toBe(false); + }); + + test("should accept empty string query", () => { + const result = searchSpecInputSchema.safeParse({ query: "" }); + expect(result.success).toBe(true); + }); }); - test("should return partIndex and totalParts as numbers or null", async () => { - const result = await searchTool({ query: "try catch" }); + describe("output schema validation", () => { + test("should validate output conforms to schema", async () => { + const result = await searchTool({ query: "array map" }); + const validation = searchSpecOutputSchema.safeParse(result); + expect(validation.success).toBe(true); + }); - for (const item of result.results) { - if (item.partIndex !== null) { - expect(item.partIndex).toBeNumber(); - expect(item.partIndex).toBeGreaterThanOrEqual(0); - } - if (item.totalParts !== null) { - expect(item.totalParts).toBeNumber(); - expect(item.totalParts).toBeGreaterThan(0); - } - } + test("should validate empty result conforms to schema", async () => { + const emptyTable = createMockTable([]); + const emptySearchTool = createSearchSpecSectionsTool( + emptyTable, + mockEmbeddings, + ); + const result = await emptySearchTool({ query: "anything" }); + const validation = searchSpecOutputSchema.safeParse(result); + expect(validation.success).toBe(true); + }); }); + + // #endregion + + // #region Proper results + + describe("result structure", () => { + test("should return results for a valid query", async () => { + const result = await searchTool({ query: "array map" }); + + expect(result.results).toBeArray(); + expect(result.results.length).toBeGreaterThan(0); + }); + + test("should return section metadata without content", async () => { + const result = await searchTool({ query: "if statement" }); + + const firstResult = result.results[0]; + expect(firstResult).toHaveProperty("sectionId"); + expect(firstResult).toHaveProperty("sectionTitle"); + expect(firstResult).toHaveProperty("vectorDistance"); + expect(firstResult).toHaveProperty("partIndex"); + expect(firstResult).toHaveProperty("totalParts"); + expect(firstResult).not.toHaveProperty("content"); + expect(firstResult).not.toHaveProperty("text"); + }); + + test("should return valid section IDs", async () => { + const result = await searchTool({ query: "loop iteration" }); + + for (const item of result.results) { + expect(item.sectionId).toBeString(); + expect(item.sectionId).toMatch(/^sec-/); + } + }); + + test("should return valid section titles", async () => { + const result = await searchTool({ query: "exception handling" }); + + for (const item of result.results) { + expect(item.sectionTitle).toBeString(); + expect(item.sectionTitle.length).toBeGreaterThan(0); + } + }); + + test("should return vectorDistance as a non-negative number", async () => { + const result = await searchTool({ query: "javascript" }); + + for (const item of result.results) { + expect(item.vectorDistance).toBeNumber(); + expect(item.vectorDistance).toBeGreaterThanOrEqual(0); + } + }); + + test("should return partIndex and totalParts as numbers or null", async () => { + const result = await searchTool({ query: "try catch" }); + + for (const item of result.results) { + if (item.partIndex !== null) { + expect(item.partIndex).toBeNumber(); + expect(item.partIndex).toBeGreaterThanOrEqual(0); + } + if (item.totalParts !== null) { + expect(item.totalParts).toBeNumber(); + expect(item.totalParts).toBeGreaterThan(0); + } + } + }); + }); + + // #endregion + + // #region Limit enforcement + + describe("limit enforcement", () => { + test("should return at most 5 results", async () => { + const result = await searchTool({ query: "statement" }); + + expect(result.results.length).toBeLessThanOrEqual(5); + }); + }); + + // #endregion + + // #region Edge cases + + describe("edge cases", () => { + test("should handle empty query", async () => { + const result = await searchTool({ query: "" }); + + expect(result.results).toBeArray(); + }); + + test("should handle complex queries", async () => { + const result = await searchTool({ + query: "how does array prototype map method work with callbacks", + }); + + expect(result.results).toBeArray(); + expect(result.results.length).toBeGreaterThan(0); + }); + + test("should handle queries with special characters", async () => { + const result = await searchTool({ + query: "Array.prototype.map() => callback", + }); + + expect(result.results).toBeArray(); + }); + + test("should handle unicode queries", async () => { + const result = await searchTool({ query: "配列 マップ" }); + + expect(result.results).toBeArray(); + }); + }); + + // #endregion + + // #region Error handling + + describe("error handling", () => { + test("should propagate embedding service errors", async () => { + const failEmbeddings = createFailingEmbeddings(); + const failSearchTool = createSearchSpecSectionsTool( + mockTable, + failEmbeddings, + ); + + await expect(failSearchTool({ query: "array map" })).rejects.toThrow( + "Embedding service unavailable", + ); + }); + + test("should propagate database errors", async () => { + const failTable = createFailingTable(); + const failSearchTool = createSearchSpecSectionsTool( + failTable, + mockEmbeddings, + ); + + await expect(failSearchTool({ query: "array map" })).rejects.toThrow( + "Database connection failed", + ); + }); + }); + + // #endregion + + // #region Tool metadata + + describe("tool metadata", () => { + test("should export correct tool name", () => { + expect(searchSpecToolName).toBe("ask262_search_spec_sections"); + }); + + test("should export input and output schemas", () => { + expect(searchSpecInputSchema).toBeDefined(); + expect(searchSpecOutputSchema).toBeDefined(); + }); + }); + + // #endregion }); diff --git a/src/test/utils/mock.ts b/src/test/utils/mock.ts index 6b9d53b..e25d145 100644 --- a/src/test/utils/mock.ts +++ b/src/test/utils/mock.ts @@ -5,15 +5,13 @@ import type { Table } from "@lancedb/lancedb"; import type { Embeddings } from "@langchain/core/embeddings"; - /** * Mock embeddings instance for testing. - * Returns predictable vectors based on input text. + * Returns deterministic vectors based on input text hash. */ export function createMockEmbeddings(): Embeddings { return { embedQuery: async (text: string): Promise => { - // Return a simple hash-based vector for testing const vector = new Array(128).fill(0); for (let i = 0; i < text.length; i++) { vector[i % 128] += text.charCodeAt(i) / 1000; @@ -29,45 +27,17 @@ export function createMockEmbeddings(): Embeddings { } /** - * Create a mock LanceDB table with test data. - * @param testData - Array of test documents to return from queries + * Mock embeddings that throws an error on embedQuery. */ -export function createMockTable(testData: MockTableData[]): Table { +export function createFailingEmbeddings(): Embeddings { return { - search: (_queryVector: number[]) => ({ - limit: (_n: number) => ({ - toArray: async () => { - // Return first n results, sorted by a simple distance calculation - return testData - .map((data) => ({ - ...data, - _distance: Math.random() * 0.5, // Random distance for testing - })) - .slice(0, _n); - }, - }), - }), - query: () => ({ - where: (condition: string) => ({ - limit: (_n: number) => ({ - toArray: async () => { - // Parse sectionid from condition like "sectionid = 'sec-xxx'" - const match = condition.match(/sectionid = ['"]([^'"]+)['"]/); - const sectionId = match ? match[1] : null; - - if (sectionId) { - return testData - .filter((data) => data.sectionid === sectionId) - .map((data) => ({ - ...data, - })); - } - return []; - }, - }), - }), - }), - } as unknown as Table; + embedQuery: async (_text: string): Promise => { + throw new Error("Embedding service unavailable"); + }, + embedDocuments: async (_documents: string[]): Promise => { + throw new Error("Embedding service unavailable"); + }, + } as Embeddings; } /** @@ -83,7 +53,73 @@ export interface MockTableData { } /** - * Default test data for spec sections + * Create a mock LanceDB table with test data. + * Uses deterministic distance (0.1) for all results. + * @param testData - Array of test documents to return from queries + */ +export function createMockTable(testData: MockTableData[]): Table { + return { + search: (_queryVector: number[]) => ({ + limit: (n: number) => ({ + toArray: async () => { + return testData + .map((data) => ({ + ...data, + _distance: 0.1, + })) + .slice(0, n); + }, + }), + }), + query: () => ({ + where: (condition: string) => ({ + limit: (n: number) => ({ + toArray: async () => { + const match = condition.match(/sectionid = ['"]([^'"]+)['"]/); + const sectionId = match ? match[1] : null; + + if (sectionId) { + return testData + .filter((data) => data.sectionid === sectionId) + .slice(0, n) + .map((data) => ({ + ...data, + })); + } + return []; + }, + }), + }), + }), + } as unknown as Table; +} + +/** + * Create a mock table that throws on search. + */ +export function createFailingTable(): Table { + return { + search: () => ({ + limit: () => ({ + toArray: async () => { + throw new Error("Database connection failed"); + }, + }), + }), + query: () => ({ + where: () => ({ + limit: () => ({ + toArray: async () => { + throw new Error("Database connection failed"); + }, + }), + }), + }), + } as unknown as Table; +} + +/** + * Default test data for spec sections. */ export const defaultTestData: MockTableData[] = [ { @@ -130,3 +166,59 @@ export const defaultTestData: MockTableData[] = [ childrensectionids: ["sec-try-statement"], }, ]; + +/** + * Test data with multi-part sections for ordering tests. + * Ordered by partindex so mock returns them sorted. + */ +export const multiPartTestData: MockTableData[] = [ + { + sectionid: "sec-species-conformance", + sectiontitle: "ECMAScript: Conformance", + text: "Part 0 of conformance spec.", + partindex: 0, + totalparts: 3, + }, + { + sectionid: "sec-species-conformance", + sectiontitle: "ECMAScript: Conformance", + text: "Part 1 of conformance spec.", + partindex: 1, + totalparts: 3, + }, + { + sectionid: "sec-species-conformance", + sectiontitle: "ECMAScript: Conformance", + text: "Part 2 final of conformance spec.", + partindex: 2, + totalparts: 3, + }, +]; + +/** + * Test data with nested children for recursive tests. + */ +export const recursiveTestData: MockTableData[] = [ + { + sectionid: "sec-root", + sectiontitle: "Root Section", + text: "Root content.", + childrensectionids: ["sec-child-a", "sec-child-b"], + }, + { + sectionid: "sec-child-a", + sectiontitle: "Child A", + text: "Child A content.", + childrensectionids: ["sec-grandchild"], + }, + { + sectionid: "sec-child-b", + sectiontitle: "Child B", + text: "Child B content.", + }, + { + sectionid: "sec-grandchild", + sectiontitle: "Grandchild", + text: "Grandchild content.", + }, +];