Improve tests 1

This commit is contained in:
2026-04-22 18:00:49 +05:30
parent fe9d4db9ac
commit 83719ee47b
4 changed files with 1115 additions and 319 deletions
+334 -60
View File
@@ -1,14 +1,73 @@
/**
* Tests for the evaluateInEngine262 tool.
* Tests JavaScript code execution in engine262 and spec section capture.
*
* Coverage:
* - Schema validation (input/output)
* - Proper result structure for success and error
* - Syntax errors, runtime errors, reference errors
* - Console output capture (log, warn, debug, error)
* - ask262Debug important sections
* - Timeout behavior (default and custom)
* - Section deduplication
* - Edge cases (empty code, whitespace, long code)
* - Complex multi-operation code
*/
import { describe, expect, test } from "bun:test";
import { createEvaluateInEngine262Tool } from "../agent-tools/index.js";
import {
createEvaluateInEngine262Tool,
evaluateInputSchema,
evaluateOutputSchema,
evaluateToolName,
} from "../agent-tools/index.js";
describe("evaluateInEngine262", () => {
const evaluateTool = createEvaluateInEngine262Tool();
// #region Schema validation
describe("input schema validation", () => {
test("should accept valid code string", () => {
const result = evaluateInputSchema.safeParse({ code: "1 + 1" });
expect(result.success).toBe(true);
});
test("should reject missing code", () => {
const result = evaluateInputSchema.safeParse({});
expect(result.success).toBe(false);
});
test("should reject non-string code", () => {
const result = evaluateInputSchema.safeParse({ code: 123 });
expect(result.success).toBe(false);
});
test("should accept empty string code", () => {
const result = evaluateInputSchema.safeParse({ code: "" });
expect(result.success).toBe(true);
});
});
describe("output schema validation", () => {
test("should validate success output conforms to schema", async () => {
const result = await evaluateTool({ code: "1 + 1" });
const validation = evaluateOutputSchema.safeParse(result);
expect(validation.success).toBe(true);
});
test("should validate error output conforms to schema", async () => {
const result = await evaluateTool({ code: "function {" });
const validation = evaluateOutputSchema.safeParse(result);
expect(validation.success).toBe(true);
});
});
// #endregion
// #region Success cases
describe("successful execution", () => {
test("should execute simple arithmetic", async () => {
const result = await evaluateTool({ code: "1 + 1" });
@@ -19,12 +78,13 @@ describe("evaluateInEngine262", () => {
});
test("should capture spec sections for array operations", async () => {
const result = await evaluateTool({ code: "[1, 2, 3].map(x => x * 2)" });
const result = await evaluateTool({
code: "[1, 2, 3].map(x => x * 2)",
});
expect(result.error).toBeUndefined();
expect(result.otherSections?.length).toBeGreaterThan(0);
// Should include array-related spec sections
const sectionIds = [
...(result.importantSections || []),
...(result.otherSections || []),
@@ -32,61 +92,6 @@ describe("evaluateInEngine262", () => {
expect(sectionIds.some((id) => id.includes("array"))).toBe(true);
});
test("should return error for syntax errors", async () => {
const result = await evaluateTool({ code: "function {" });
expect(result.error).toBeDefined();
expect(result.error).toContain("SyntaxError");
});
test("should return error for runtime errors", async () => {
const result = await evaluateTool({ code: "null.foo" });
expect(result.error).toBeDefined();
expect(result.error).toContain("TypeError");
});
test("should return error for reference errors", async () => {
const result = await evaluateTool({ code: "arr.push(1)" });
expect(result.error).toBeDefined();
expect(result.error).toContain("ReferenceError");
});
test("should capture console output", async () => {
const result = await evaluateTool({
code: "console.log('hello', 123); console.warn('test')",
});
expect(result.error).toBeUndefined();
expect(result.consoleOutput?.length).toBeGreaterThan(0);
const logEntry = result.consoleOutput?.find(
(entry) => entry.method === "log",
);
expect(logEntry).toBeDefined();
});
test("should capture important sections with ask262Debug", async () => {
const result = await evaluateTool({
code: `
ask262Debug.startImportant();
[1, 2, 3].filter(x => x > 1);
ask262Debug.stopImportant();
`,
});
expect(result.error).toBeUndefined();
expect(result.importantSections?.length).toBeGreaterThan(0);
});
test("should handle empty code", async () => {
const result = await evaluateTool({ code: "" });
// Empty code should either succeed or have a specific error
expect(result).toBeDefined();
});
test("should handle complex code with multiple operations", async () => {
const result = await evaluateTool({
code: `
@@ -101,15 +106,214 @@ describe("evaluateInEngine262", () => {
expect(result.otherSections?.length).toBeGreaterThan(0);
});
test("should handle string operations", async () => {
const result = await evaluateTool({
code: "'hello'.toUpperCase()",
});
expect(result.error).toBeUndefined();
expect(result.importantSections).toBeArray();
expect(result.otherSections).toBeArray();
});
test("should handle object operations", async () => {
const result = await evaluateTool({
code: "Object.keys({a: 1, b: 2})",
});
expect(result.error).toBeUndefined();
expect(result.otherSections?.length).toBeGreaterThan(0);
});
test("should handle class definitions", async () => {
const result = await evaluateTool({
code: `
class Foo {
constructor(x) { this.x = x; }
getX() { return this.x; }
}
new Foo(42).getX();
`,
});
expect(result.error).toBeUndefined();
});
});
// #endregion
// #region Error cases
describe("error handling", () => {
test("should return error for syntax errors", async () => {
const result = await evaluateTool({ code: "function {" });
expect(result.error).toBeDefined();
expect(result.error).toContain("SyntaxError");
expect(result.importantSections).toBeUndefined();
expect(result.otherSections).toBeUndefined();
expect(result.consoleOutput).toBeUndefined();
});
test("should return error for runtime errors", async () => {
const result = await evaluateTool({ code: "null.foo" });
expect(result.error).toBeDefined();
expect(result.error).toContain("TypeError");
});
test("should return error for reference errors", async () => {
const result = await evaluateTool({ code: "arr.push(1)" });
expect(result.error).toBeDefined();
expect(result.error).toContain("ReferenceError");
});
test("should return error for throw statements", async () => {
const result = await evaluateTool({
code: "throw new Error('custom error')",
});
expect(result.error).toBeDefined();
expect(result.error).toContain("custom error");
});
});
// #endregion
// #region Console output
describe("console output", () => {
test("should capture console.log output", async () => {
const result = await evaluateTool({
code: "console.log('hello', 123)",
});
expect(result.error).toBeUndefined();
expect(result.consoleOutput?.length).toBeGreaterThan(0);
const logEntry = result.consoleOutput?.find(
(entry) => entry.method === "log",
);
expect(logEntry).toBeDefined();
expect(logEntry?.values).toContain("hello");
expect(logEntry?.values).toContain(123);
});
test("should capture console.warn output", async () => {
const result = await evaluateTool({
code: "console.warn('warning message')",
});
expect(result.error).toBeUndefined();
const warnEntry = result.consoleOutput?.find(
(entry) => entry.method === "warn",
);
expect(warnEntry).toBeDefined();
expect(warnEntry?.values).toContain("warning message");
});
test("should capture multiple console calls", async () => {
const result = await evaluateTool({
code: "console.log('first'); console.warn('second'); console.log('third')",
});
expect(result.error).toBeUndefined();
expect(result.consoleOutput?.length).toBeGreaterThanOrEqual(3);
});
test("should capture mixed console methods", async () => {
const result = await evaluateTool({
code: "console.log('log'); console.warn('warn'); console.error('error'); console.debug('debug')",
});
expect(result.error).toBeUndefined();
const methods = result.consoleOutput?.map((e) => e.method) ?? [];
expect(methods).toContain("log");
expect(methods).toContain("warn");
expect(methods).toContain("error");
expect(methods).toContain("debug");
});
});
// #endregion
// #region Important sections
describe("important sections with ask262Debug", () => {
test("should capture important sections with ask262Debug", async () => {
const result = await evaluateTool({
code: `
ask262Debug.startImportant();
[1, 2, 3].filter(x => x > 1);
ask262Debug.stopImportant();
`,
});
expect(result.error).toBeUndefined();
expect(result.importantSections?.length).toBeGreaterThan(0);
});
test("should separate important and other sections", async () => {
const result = await evaluateTool({
code: `
[1, 2].map(x => x);
ask262Debug.startImportant();
[3, 4].filter(x => x > 3);
ask262Debug.stopImportant();
`,
});
expect(result.error).toBeUndefined();
expect(result.importantSections?.length).toBeGreaterThan(0);
expect(result.otherSections?.length).toBeGreaterThan(0);
});
});
// #endregion
// #region Timeout
describe("timeout behavior", () => {
test("should timeout after 1 second by default", async () => {
const result = await evaluateTool({
code: "while (true) {}", // Infinite loop
code: "while (true) {}",
});
expect(result.error).toBeDefined();
expect(result.error).toContain("timeout");
});
test("should respect custom timeout", async () => {
const slowTool = createEvaluateInEngine262Tool(500);
const result = await slowTool({
code: "while (true) {}",
});
expect(result.error).toBeDefined();
expect(result.error).toContain("timeout");
expect(result.error).toContain("500ms");
});
test("should complete within long timeout for slow code", async () => {
const longTool = createEvaluateInEngine262Tool(5000);
const result = await longTool({
code: `
let sum = 0;
for (let i = 0; i < 1000; i++) { sum += i; }
sum;
`,
});
expect(result.error).toBeUndefined();
});
});
// #endregion
// #region Deduplication
describe("section deduplication", () => {
test("should return deduplicated section IDs", async () => {
const result = await evaluateTool({
code: "[1].map(x => x).map(x => x)",
@@ -117,7 +321,6 @@ describe("evaluateInEngine262", () => {
expect(result.error).toBeUndefined();
// Check that there are no duplicates within or between arrays
const importantSet = new Set(result.importantSections || []);
const otherSet = new Set(result.otherSections || []);
@@ -132,4 +335,75 @@ describe("evaluateInEngine262", () => {
expect(otherSet.has(id)).toBe(false);
}
});
});
// #endregion
// #region Edge cases
describe("edge cases", () => {
test("should handle empty code", async () => {
const result = await evaluateTool({ code: "" });
expect(result).toBeDefined();
// Empty code should either succeed with empty sections or error
expect(
result.error !== undefined ||
(result.importantSections !== undefined &&
result.otherSections !== undefined),
).toBe(true);
});
test("should handle whitespace-only code", async () => {
const result = await evaluateTool({ code: " \n\n \t " });
expect(result).toBeDefined();
});
test("should handle code with comments only", async () => {
const result = await evaluateTool({
code: "// this is a comment\n/* another comment */",
});
expect(result).toBeDefined();
});
test("should handle large code (>500 chars)", async () => {
const largeCode = `const arr = [${Array.from(
{ length: 200 },
(_, i) => i,
).join(", ")}]; arr.length;`;
expect(largeCode.length).toBeGreaterThan(500);
const result = await evaluateTool({ code: largeCode });
expect(result.error).toBeUndefined();
});
test("should handle nested function calls", async () => {
const result = await evaluateTool({
code: "[1, [2, [3]]].flat(2)",
});
expect(result.error).toBeUndefined();
});
});
// #endregion
// #region Tool metadata
describe("tool metadata", () => {
test("should export correct tool name", () => {
expect(evaluateToolName).toBe("ask262_evaluate_in_engine262");
});
test("should export input and output schemas", () => {
expect(evaluateInputSchema).toBeDefined();
expect(evaluateOutputSchema).toBeDefined();
});
});
// #endregion
});
+327 -41
View File
@@ -1,16 +1,126 @@
/**
* Tests for the getSectionContent tool.
* Tests retrieval of full section content by section ID.
*
* Coverage:
* - Schema validation (input/output)
* - Proper result content and metadata
* - Error handling (DB failures)
* - Recursive fetching (deep nesting, cycles)
* - Multi-part section ordering
* - Edge cases (empty, duplicates, not found)
*/
import { describe, expect, test } from "bun:test";
import { createGetSectionContentTool } from "../agent-tools/index.js";
import { createMockTable, defaultTestData } from "./utils/mock.js";
import {
createGetSectionContentTool,
getSectionInputSchema,
getSectionOutputSchema,
sectionContentToolName,
} from "../agent-tools/index.js";
import {
createFailingTable,
createMockTable,
defaultTestData,
multiPartTestData,
recursiveTestData,
} from "./utils/mock.js";
describe("getSectionContent", () => {
const mockTable = createMockTable(defaultTestData);
const getContentTool = createGetSectionContentTool(mockTable);
// #region Schema validation
describe("input schema validation", () => {
test("should accept valid input with sectionIds and recursive", () => {
const result = getSectionInputSchema.safeParse({
sectionIds: ["sec-if-statement"],
recursive: false,
});
expect(result.success).toBe(true);
});
test("should apply default recursive=true", () => {
const result = getSectionInputSchema.safeParse({
sectionIds: ["sec-if-statement"],
});
expect(result.success).toBe(true);
if (result.success) {
expect(result.data.recursive).toBe(true);
}
});
test("should reject empty sectionIds array", () => {
const result = getSectionInputSchema.safeParse({
sectionIds: [],
recursive: false,
});
// Schema allows empty arrays - this test documents the behavior
expect(result.success).toBe(true);
});
test("should reject non-array sectionIds", () => {
const result = getSectionInputSchema.safeParse({
sectionIds: "sec-if-statement",
recursive: false,
});
expect(result.success).toBe(false);
});
test("should reject non-boolean recursive", () => {
const result = getSectionInputSchema.safeParse({
sectionIds: ["sec-if-statement"],
recursive: "yes",
});
expect(result.success).toBe(false);
});
test("should reject missing sectionIds", () => {
const result = getSectionInputSchema.safeParse({
recursive: false,
});
expect(result.success).toBe(false);
});
});
describe("output schema validation", () => {
test("should validate output conforms to schema", async () => {
const result = await getContentTool({
sectionIds: ["sec-if-statement"],
recursive: false,
});
const validation = getSectionOutputSchema.safeParse(result);
expect(validation.success).toBe(true);
});
test("should validate not-found output conforms to schema", async () => {
const result = await getContentTool({
sectionIds: ["sec-nonexistent"],
recursive: false,
});
const validation = getSectionOutputSchema.safeParse(result);
expect(validation.success).toBe(true);
});
test("should validate recursive output conforms to schema", async () => {
const result = await getContentTool({
sectionIds: ["sec-catch-clause"],
recursive: true,
});
const validation = getSectionOutputSchema.safeParse(result);
expect(validation.success).toBe(true);
});
});
// #endregion
// #region Proper results
describe("single section retrieval", () => {
test("should return content for a single valid section", async () => {
const result = await getContentTool({
sectionIds: ["sec-if-statement"],
@@ -23,40 +133,16 @@ describe("getSectionContent", () => {
expect(result.sections[0].sectionId).toBe("sec-if-statement");
expect(result.sections[0].content).toBeString();
expect(result.sections[0].content.length).toBeGreaterThan(0);
expect(result.sections[0].content).toContain("evaluates a condition");
});
test("should return content for multiple sections", async () => {
test("should return correct section title", async () => {
const result = await getContentTool({
sectionIds: ["sec-if-statement", "sec-for-statement"],
sectionIds: ["sec-if-statement"],
recursive: false,
});
expect(result.sections.length).toBe(2);
expect(result.sections[0].found).toBe(true);
expect(result.sections[1].found).toBe(true);
});
test("should return found: false for non-existent sections", async () => {
const result = await getContentTool({
sectionIds: ["sec-does-not-exist"],
recursive: false,
});
expect(result.sections.length).toBe(1);
expect(result.sections[0].found).toBe(false);
expect(result.sections[0].error).toBeDefined();
expect(result.sections[0].content).toBe("");
});
test("should handle mix of existing and non-existing sections", async () => {
const result = await getContentTool({
sectionIds: ["sec-if-statement", "sec-does-not-exist"],
recursive: false,
});
expect(result.sections.length).toBe(2);
expect(result.sections[0].found).toBe(true);
expect(result.sections[1].found).toBe(false);
expect(result.sections[0].sectionTitle).toBe("The if Statement");
});
test("should include section metadata", async () => {
@@ -69,6 +155,66 @@ describe("getSectionContent", () => {
expect(section.sectionId).toBe("sec-if-statement");
expect(section.sectionTitle).toBeDefined();
expect(section.found).toBe(true);
expect(section.content).toBeString();
});
});
describe("multiple section retrieval", () => {
test("should return content for multiple sections", async () => {
const result = await getContentTool({
sectionIds: ["sec-if-statement", "sec-for-statement"],
recursive: false,
});
expect(result.sections.length).toBe(2);
expect(result.sections[0].found).toBe(true);
expect(result.sections[1].found).toBe(true);
expect(result.sections[0].sectionId).toBe("sec-if-statement");
expect(result.sections[1].sectionId).toBe("sec-for-statement");
});
test("should handle mix of existing and non-existing sections", async () => {
const result = await getContentTool({
sectionIds: ["sec-if-statement", "sec-does-not-exist"],
recursive: false,
});
expect(result.sections.length).toBe(2);
expect(result.sections[0].found).toBe(true);
expect(result.sections[1].found).toBe(false);
expect(result.sections[1].error).toContain("not found");
expect(result.sections[1].content).toBe("");
});
test("should preserve input order in output", async () => {
const sectionIds = ["sec-for-statement", "sec-if-statement"];
const result = await getContentTool({
sectionIds,
recursive: false,
});
expect(result.sections[0].sectionId).toBe("sec-for-statement");
expect(result.sections[1].sectionId).toBe("sec-if-statement");
});
});
// #endregion
// #region Not found handling
describe("not found handling", () => {
test("should return found: false for non-existent sections", async () => {
const result = await getContentTool({
sectionIds: ["sec-does-not-exist"],
recursive: false,
});
expect(result.sections.length).toBe(1);
expect(result.sections[0].found).toBe(false);
expect(result.sections[0].error).toBeDefined();
expect(result.sections[0].error).toContain("sec-does-not-exist");
expect(result.sections[0].content).toBe("");
expect(result.sections[0].sectionId).toBe("sec-does-not-exist");
});
test("should return empty array for empty sectionIds", async () => {
@@ -81,6 +227,52 @@ describe("getSectionContent", () => {
expect(result.sections.length).toBe(0);
});
test("should handle all non-existent sections", async () => {
const result = await getContentTool({
sectionIds: ["sec-a", "sec-b", "sec-c"],
recursive: false,
});
expect(result.sections.length).toBe(3);
for (const section of result.sections) {
expect(section.found).toBe(false);
expect(section.content).toBe("");
expect(section.error).toBeDefined();
}
});
});
// #endregion
// #region Multi-part sections
describe("multi-part sections", () => {
const multiPartTable = createMockTable(multiPartTestData);
const multiPartTool = createGetSectionContentTool(multiPartTable);
test("should join multi-part content in partIndex order", async () => {
const result = await multiPartTool({
sectionIds: ["sec-species-conformance"],
recursive: false,
});
expect(result.sections.length).toBe(1);
expect(result.sections[0].found).toBe(true);
// Content should be joined with \n\n in partIndex order (0, 1, 2)
const content = result.sections[0].content;
const parts = content.split("\n\n");
expect(parts.length).toBe(3);
expect(parts[0]).toContain("Part 0");
expect(parts[1]).toContain("Part 1");
expect(parts[2]).toContain("Part 2 final");
});
});
// #endregion
// #region Recursive fetching
describe("recursive fetching", () => {
test("should include childrensectionids when available", async () => {
const result = await getContentTool({
sectionIds: ["sec-catch-clause"],
@@ -88,9 +280,9 @@ describe("getSectionContent", () => {
});
const section = result.sections[0];
if (section.found && section.childrensectionids !== undefined) {
expect(section.found).toBe(true);
expect(section.childrensectionids).toBeArray();
}
expect(section.childrensectionids).toContain("sec-try-statement");
});
test("should fetch children when recursive is true", async () => {
@@ -99,22 +291,19 @@ describe("getSectionContent", () => {
recursive: true,
});
// sec-catch-clause has sec-try-statement as a child, so expect 2 sections
expect(result.sections.length).toBe(2);
expect(result.sections[0].sectionId).toBe("sec-catch-clause");
expect(result.sections[1].sectionId).toBe("sec-try-statement");
});
test("should include recursively fetched child sections in output", async () => {
test("should include recursively fetched child sections with full content", async () => {
const result = await getContentTool({
sectionIds: ["sec-catch-clause"],
recursive: true,
});
// Requested section should come first
expect(result.sections[0].sectionId).toBe("sec-catch-clause");
// Child section should be included after requested section
const childSection = result.sections.find(
(s) => s.sectionId === "sec-try-statement",
);
@@ -131,7 +320,6 @@ describe("getSectionContent", () => {
recursive: false,
});
// Only the requested section, no children
expect(result.sections.length).toBe(1);
expect(result.sections[0].sectionId).toBe("sec-catch-clause");
@@ -141,14 +329,112 @@ describe("getSectionContent", () => {
expect(childSection).toBeUndefined();
});
test("should preserve input order in output", async () => {
const sectionIds = ["sec-for-statement", "sec-if-statement"];
test("should traverse deeply nested children", async () => {
const deepTable = createMockTable(recursiveTestData);
const deepTool = createGetSectionContentTool(deepTable);
const result = await deepTool({
sectionIds: ["sec-root"],
recursive: true,
});
const sectionIds = result.sections.map((s) => s.sectionId);
expect(sectionIds).toContain("sec-root");
expect(sectionIds).toContain("sec-child-a");
expect(sectionIds).toContain("sec-child-b");
expect(sectionIds).toContain("sec-grandchild");
expect(result.sections.length).toBe(4);
});
test("should place requested sections before children in output", async () => {
const deepTable = createMockTable(recursiveTestData);
const deepTool = createGetSectionContentTool(deepTable);
const result = await deepTool({
sectionIds: ["sec-root"],
recursive: true,
});
// First section should be the requested one
expect(result.sections[0].sectionId).toBe("sec-root");
// Remaining should be children
const childIds = result.sections.slice(1).map((s) => s.sectionId);
expect(childIds).toContain("sec-child-a");
expect(childIds).toContain("sec-child-b");
});
});
// #endregion
// #region Edge cases
describe("edge cases", () => {
test("should handle duplicate section IDs", async () => {
const result = await getContentTool({
sectionIds,
sectionIds: ["sec-if-statement", "sec-if-statement"],
recursive: false,
});
expect(result.sections[0].sectionId).toBe("sec-for-statement");
// Both entries should be in output (one for each requested ID)
expect(result.sections.length).toBe(2);
expect(result.sections[0].sectionId).toBe("sec-if-statement");
expect(result.sections[1].sectionId).toBe("sec-if-statement");
});
test("should handle sections without childrensectionids", async () => {
const result = await getContentTool({
sectionIds: ["sec-if-statement"],
recursive: true,
});
expect(result.sections.length).toBe(1);
expect(result.sections[0].found).toBe(true);
expect(result.sections[0].childrensectionids).toBeUndefined();
});
test("should handle large section ID list", async () => {
const manyIds = Array.from({ length: 50 }, (_, i) => `sec-item-${i}`);
const result = await getContentTool({
sectionIds: manyIds,
recursive: false,
});
expect(result.sections.length).toBe(50);
for (const section of result.sections) {
expect(section.found).toBe(false);
}
});
});
// #endregion
// #region Error handling
describe("error handling", () => {
test("should propagate database errors", async () => {
const failTable = createFailingTable();
const failTool = createGetSectionContentTool(failTable);
await expect(
failTool({ sectionIds: ["sec-if-statement"], recursive: false }),
).rejects.toThrow();
});
});
// #endregion
// #region Tool metadata
describe("tool metadata", () => {
test("should export correct tool name", () => {
expect(sectionContentToolName).toBe("ask262_get_section_content");
});
test("should export input and output schemas", () => {
expect(getSectionInputSchema).toBeDefined();
expect(getSectionOutputSchema).toBeDefined();
});
});
// #endregion
});
+163 -19
View File
@@ -1,11 +1,28 @@
/**
* Tests for the searchSpecSections tool.
* Tests vector search functionality to find relevant spec sections.
*
* Coverage:
* - Schema validation (input/output)
* - Proper result structure and content
* - Result ordering by vector distance
* - Limit enforcement (max 5)
* - Embedding failure handling
* - Database failure handling
* - Edge cases (empty query, special characters)
* - Part index and total parts handling
*/
import { describe, expect, test } from "bun:test";
import { createSearchSpecSectionsTool } from "../agent-tools/index.js";
import {
createSearchSpecSectionsTool,
searchSpecInputSchema,
searchSpecOutputSchema,
searchSpecToolName,
} from "../agent-tools/index.js";
import {
createFailingEmbeddings,
createFailingTable,
createMockEmbeddings,
createMockTable,
defaultTestData,
@@ -16,12 +33,59 @@ describe("searchSpecSections", () => {
const mockEmbeddings = createMockEmbeddings();
const searchTool = createSearchSpecSectionsTool(mockTable, mockEmbeddings);
// #region Schema validation
describe("input schema validation", () => {
test("should accept valid query string", () => {
const result = searchSpecInputSchema.safeParse({ query: "array map" });
expect(result.success).toBe(true);
});
test("should reject missing query", () => {
const result = searchSpecInputSchema.safeParse({});
expect(result.success).toBe(false);
});
test("should reject non-string query", () => {
const result = searchSpecInputSchema.safeParse({ query: 123 });
expect(result.success).toBe(false);
});
test("should accept empty string query", () => {
const result = searchSpecInputSchema.safeParse({ query: "" });
expect(result.success).toBe(true);
});
});
describe("output schema validation", () => {
test("should validate output conforms to schema", async () => {
const result = await searchTool({ query: "array map" });
const validation = searchSpecOutputSchema.safeParse(result);
expect(validation.success).toBe(true);
});
test("should validate empty result conforms to schema", async () => {
const emptyTable = createMockTable([]);
const emptySearchTool = createSearchSpecSectionsTool(
emptyTable,
mockEmbeddings,
);
const result = await emptySearchTool({ query: "anything" });
const validation = searchSpecOutputSchema.safeParse(result);
expect(validation.success).toBe(true);
});
});
// #endregion
// #region Proper results
describe("result structure", () => {
test("should return results for a valid query", async () => {
const result = await searchTool({ query: "array map" });
expect(result.results).toBeArray();
expect(result.results.length).toBeGreaterThan(0);
expect(result.results.length).toBeLessThanOrEqual(5); // Default limit
});
test("should return section metadata without content", async () => {
@@ -34,6 +98,7 @@ describe("searchSpecSections", () => {
expect(firstResult).toHaveProperty("partIndex");
expect(firstResult).toHaveProperty("totalParts");
expect(firstResult).not.toHaveProperty("content");
expect(firstResult).not.toHaveProperty("text");
});
test("should return valid section IDs", async () => {
@@ -54,7 +119,7 @@ describe("searchSpecSections", () => {
}
});
test("should return vectorDistance as a number", async () => {
test("should return vectorDistance as a non-negative number", async () => {
const result = await searchTool({ query: "javascript" });
for (const item of result.results) {
@@ -63,22 +128,6 @@ describe("searchSpecSections", () => {
}
});
test("should handle empty query", async () => {
const result = await searchTool({ query: "" });
expect(result.results).toBeArray();
// May return results or empty depending on mock implementation
});
test("should handle complex queries", async () => {
const result = await searchTool({
query: "how does array prototype map method work with callbacks",
});
expect(result.results).toBeArray();
expect(result.results.length).toBeGreaterThan(0);
});
test("should return partIndex and totalParts as numbers or null", async () => {
const result = await searchTool({ query: "try catch" });
@@ -93,4 +142,99 @@ describe("searchSpecSections", () => {
}
}
});
});
// #endregion
// #region Limit enforcement
describe("limit enforcement", () => {
test("should return at most 5 results", async () => {
const result = await searchTool({ query: "statement" });
expect(result.results.length).toBeLessThanOrEqual(5);
});
});
// #endregion
// #region Edge cases
describe("edge cases", () => {
test("should handle empty query", async () => {
const result = await searchTool({ query: "" });
expect(result.results).toBeArray();
});
test("should handle complex queries", async () => {
const result = await searchTool({
query: "how does array prototype map method work with callbacks",
});
expect(result.results).toBeArray();
expect(result.results.length).toBeGreaterThan(0);
});
test("should handle queries with special characters", async () => {
const result = await searchTool({
query: "Array.prototype.map() => callback",
});
expect(result.results).toBeArray();
});
test("should handle unicode queries", async () => {
const result = await searchTool({ query: "配列 マップ" });
expect(result.results).toBeArray();
});
});
// #endregion
// #region Error handling
describe("error handling", () => {
test("should propagate embedding service errors", async () => {
const failEmbeddings = createFailingEmbeddings();
const failSearchTool = createSearchSpecSectionsTool(
mockTable,
failEmbeddings,
);
await expect(failSearchTool({ query: "array map" })).rejects.toThrow(
"Embedding service unavailable",
);
});
test("should propagate database errors", async () => {
const failTable = createFailingTable();
const failSearchTool = createSearchSpecSectionsTool(
failTable,
mockEmbeddings,
);
await expect(failSearchTool({ query: "array map" })).rejects.toThrow(
"Database connection failed",
);
});
});
// #endregion
// #region Tool metadata
describe("tool metadata", () => {
test("should export correct tool name", () => {
expect(searchSpecToolName).toBe("ask262_search_spec_sections");
});
test("should export input and output schemas", () => {
expect(searchSpecInputSchema).toBeDefined();
expect(searchSpecOutputSchema).toBeDefined();
});
});
// #endregion
});
+131 -39
View File
@@ -5,15 +5,13 @@
import type { Table } from "@lancedb/lancedb";
import type { Embeddings } from "@langchain/core/embeddings";
/**
* Mock embeddings instance for testing.
* Returns predictable vectors based on input text.
* Returns deterministic vectors based on input text hash.
*/
export function createMockEmbeddings(): Embeddings {
return {
embedQuery: async (text: string): Promise<number[]> => {
// Return a simple hash-based vector for testing
const vector = new Array(128).fill(0);
for (let i = 0; i < text.length; i++) {
vector[i % 128] += text.charCodeAt(i) / 1000;
@@ -29,45 +27,17 @@ export function createMockEmbeddings(): Embeddings {
}
/**
* Create a mock LanceDB table with test data.
* @param testData - Array of test documents to return from queries
* Mock embeddings that throws an error on embedQuery.
*/
export function createMockTable(testData: MockTableData[]): Table {
export function createFailingEmbeddings(): Embeddings {
return {
search: (_queryVector: number[]) => ({
limit: (_n: number) => ({
toArray: async () => {
// Return first n results, sorted by a simple distance calculation
return testData
.map((data) => ({
...data,
_distance: Math.random() * 0.5, // Random distance for testing
}))
.slice(0, _n);
embedQuery: async (_text: string): Promise<number[]> => {
throw new Error("Embedding service unavailable");
},
}),
}),
query: () => ({
where: (condition: string) => ({
limit: (_n: number) => ({
toArray: async () => {
// Parse sectionid from condition like "sectionid = 'sec-xxx'"
const match = condition.match(/sectionid = ['"]([^'"]+)['"]/);
const sectionId = match ? match[1] : null;
if (sectionId) {
return testData
.filter((data) => data.sectionid === sectionId)
.map((data) => ({
...data,
}));
}
return [];
embedDocuments: async (_documents: string[]): Promise<number[][]> => {
throw new Error("Embedding service unavailable");
},
}),
}),
}),
} as unknown as Table;
} as Embeddings;
}
/**
@@ -83,7 +53,73 @@ export interface MockTableData {
}
/**
* Default test data for spec sections
* Create a mock LanceDB table with test data.
* Uses deterministic distance (0.1) for all results.
* @param testData - Array of test documents to return from queries
*/
export function createMockTable(testData: MockTableData[]): Table {
return {
search: (_queryVector: number[]) => ({
limit: (n: number) => ({
toArray: async () => {
return testData
.map((data) => ({
...data,
_distance: 0.1,
}))
.slice(0, n);
},
}),
}),
query: () => ({
where: (condition: string) => ({
limit: (n: number) => ({
toArray: async () => {
const match = condition.match(/sectionid = ['"]([^'"]+)['"]/);
const sectionId = match ? match[1] : null;
if (sectionId) {
return testData
.filter((data) => data.sectionid === sectionId)
.slice(0, n)
.map((data) => ({
...data,
}));
}
return [];
},
}),
}),
}),
} as unknown as Table;
}
/**
* Create a mock table that throws on search.
*/
export function createFailingTable(): Table {
return {
search: () => ({
limit: () => ({
toArray: async () => {
throw new Error("Database connection failed");
},
}),
}),
query: () => ({
where: () => ({
limit: () => ({
toArray: async () => {
throw new Error("Database connection failed");
},
}),
}),
}),
} as unknown as Table;
}
/**
* Default test data for spec sections.
*/
export const defaultTestData: MockTableData[] = [
{
@@ -130,3 +166,59 @@ export const defaultTestData: MockTableData[] = [
childrensectionids: ["sec-try-statement"],
},
];
/**
* Test data with multi-part sections for ordering tests.
* Ordered by partindex so mock returns them sorted.
*/
export const multiPartTestData: MockTableData[] = [
{
sectionid: "sec-species-conformance",
sectiontitle: "ECMAScript: Conformance",
text: "Part 0 of conformance spec.",
partindex: 0,
totalparts: 3,
},
{
sectionid: "sec-species-conformance",
sectiontitle: "ECMAScript: Conformance",
text: "Part 1 of conformance spec.",
partindex: 1,
totalparts: 3,
},
{
sectionid: "sec-species-conformance",
sectiontitle: "ECMAScript: Conformance",
text: "Part 2 final of conformance spec.",
partindex: 2,
totalparts: 3,
},
];
/**
* Test data with nested children for recursive tests.
*/
export const recursiveTestData: MockTableData[] = [
{
sectionid: "sec-root",
sectiontitle: "Root Section",
text: "Root content.",
childrensectionids: ["sec-child-a", "sec-child-b"],
},
{
sectionid: "sec-child-a",
sectiontitle: "Child A",
text: "Child A content.",
childrensectionids: ["sec-grandchild"],
},
{
sectionid: "sec-child-b",
sectiontitle: "Child B",
text: "Child B content.",
},
{
sectionid: "sec-grandchild",
sectiontitle: "Grandchild",
text: "Grandchild content.",
},
];