mirror of
https://github.com/bendtherules/ask262.git
synced 2026-08-18 13:21:55 +00:00
refactor: rename files
This commit is contained in:
@@ -0,0 +1,60 @@
|
||||
/**
|
||||
* Graph explorer tool.
|
||||
* Explores structural relationships between specification sections and implementation code.
|
||||
*/
|
||||
|
||||
import { DynamicStructuredTool } from "@langchain/core/tools";
|
||||
import type Graph from "graphology";
|
||||
import { z } from "zod";
|
||||
|
||||
const graphExplorerSchema = z.object({
|
||||
query: z
|
||||
.string()
|
||||
.describe(
|
||||
"The section ID or function name to explore in the graph (e.g., 'Evaluate_IfStatement' or 'sec-if-statement')",
|
||||
),
|
||||
});
|
||||
|
||||
/**
|
||||
* Creates the graph explorer tool.
|
||||
* @param graph - Graphology graph instance
|
||||
*/
|
||||
export function createGraphExplorerTool(graph: Graph) {
|
||||
return new DynamicStructuredTool({
|
||||
name: "graph_explorer",
|
||||
description:
|
||||
"Explores structural relationships between specification sections and implementation code (functions). Use this to find which spec section a function implements.",
|
||||
schema: graphExplorerSchema,
|
||||
func: async ({ query }) => {
|
||||
console.log(`[Tool: graph_explorer] Querying for: ${query}`);
|
||||
|
||||
let nodeId = query;
|
||||
if (!graph.hasNode(nodeId)) {
|
||||
if (graph.hasNode(`func-${query}`)) {
|
||||
nodeId = `func-${query}`;
|
||||
}
|
||||
}
|
||||
|
||||
if (graph.hasNode(nodeId)) {
|
||||
const neighbors = graph.neighbors(nodeId);
|
||||
const nodeAttr = graph.getNodeAttributes(nodeId);
|
||||
|
||||
let result = `Information for ${nodeId} (${nodeAttr.type}):\n`;
|
||||
if (nodeAttr.title) result += `- Title: ${nodeAttr.title}\n`;
|
||||
if (nodeAttr.file) result += `- File: ${nodeAttr.file}\n`;
|
||||
result += `\nConnected parts:\n`;
|
||||
|
||||
neighbors.forEach((neighbor) => {
|
||||
const attr = graph.getNodeAttributes(neighbor);
|
||||
const edges = graph.edges(nodeId, neighbor);
|
||||
const edgeAttr = graph.getEdgeAttributes(edges[0]);
|
||||
result += `- ${neighbor} (${attr.type}) via ${edgeAttr.type}${attr.title ? `: ${attr.title}` : ""}\n`;
|
||||
});
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
return `No information found in graph for ${query}. Use spec_retriever to search text.`;
|
||||
},
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
/**
|
||||
* Agent tools index file.
|
||||
* Exports all tool factory functions and utilities.
|
||||
*/
|
||||
|
||||
export { createGraphExplorerTool } from "./graphExplorer";
|
||||
export { type RerankResult, rerankDocuments } from "./reranker";
|
||||
export { createSectionRetrieverTool } from "./sectionRetriever";
|
||||
export { createSpecRetrieverTool } from "./specRetriever";
|
||||
@@ -0,0 +1,68 @@
|
||||
/**
|
||||
* Reranking utility for document relevance scoring.
|
||||
* Uses Ollama's reranker API to score documents against a query.
|
||||
*/
|
||||
|
||||
import { RERANKER_MODEL } from "../constants";
|
||||
|
||||
const OLLAMA_HOST = process.env.OLLAMA_HOST || "http://localhost:11434";
|
||||
|
||||
export interface RerankResult<T> {
|
||||
document: T;
|
||||
score: number;
|
||||
index: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Rerank documents based on relevance to the query using Ollama's reranker.
|
||||
* @param query - The search query
|
||||
* @param documents - Array of documents to rerank
|
||||
* @returns Array of reranked documents with scores
|
||||
*/
|
||||
export async function rerankDocuments<T extends { pageContent: string }>(
|
||||
query: string,
|
||||
documents: T[],
|
||||
): Promise<RerankResult<T>[]> {
|
||||
try {
|
||||
const response = await fetch(`${OLLAMA_HOST}/api/rerank`, {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
model: RERANKER_MODEL,
|
||||
query: query,
|
||||
documents: documents.map((d) => d.pageContent),
|
||||
}),
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
console.warn(
|
||||
`Reranker API failed: ${response.statusText}. Returning all documents.`,
|
||||
);
|
||||
return documents.map((doc, i) => ({
|
||||
document: doc,
|
||||
score: 1.0,
|
||||
index: i,
|
||||
}));
|
||||
}
|
||||
|
||||
const data = await response.json();
|
||||
if (!data.results || !Array.isArray(data.results)) {
|
||||
return documents.map((doc, i) => ({
|
||||
document: doc,
|
||||
score: 1.0,
|
||||
index: i,
|
||||
}));
|
||||
}
|
||||
|
||||
return data.results.map(
|
||||
(result: { index: number; relevance_score: number }) => ({
|
||||
document: documents[result.index],
|
||||
score: result.relevance_score,
|
||||
index: result.index,
|
||||
}),
|
||||
);
|
||||
} catch (error) {
|
||||
console.warn(`Reranker error: ${error}. Returning all documents.`);
|
||||
return documents.map((doc, i) => ({ document: doc, score: 1.0, index: i }));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,75 @@
|
||||
/**
|
||||
* Section chunk retriever tool.
|
||||
* Retrieves all text chunks from a specific specification section by sectionid.
|
||||
*/
|
||||
|
||||
import type { Table } from "@lancedb/lancedb";
|
||||
import { DynamicStructuredTool } from "@langchain/core/tools";
|
||||
import { z } from "zod";
|
||||
|
||||
const sectionRetrieverSchema = z.object({
|
||||
sectionId: z
|
||||
.string()
|
||||
.describe("The section ID (e.g., 'sec-if-statement') to fetch chunks for"),
|
||||
});
|
||||
|
||||
/**
|
||||
* Creates the section retriever tool.
|
||||
* @param table - LanceDB table containing spec vectors
|
||||
*/
|
||||
export function createSectionRetrieverTool(table: Table) {
|
||||
return new DynamicStructuredTool({
|
||||
name: "fetch_section_chunks",
|
||||
description:
|
||||
"Retrieves all text chunks from a specific specification section by sectionid. " +
|
||||
"Supports recursive fetching - if a section has children, it will fetch all descendants. " +
|
||||
"Use this to get complete content when you see 'Subsection available' or 'partial section' references.",
|
||||
schema: sectionRetrieverSchema,
|
||||
func: async ({ sectionId }) => {
|
||||
const allDocs: string[] = [];
|
||||
const queue: string[] = [sectionId];
|
||||
const visited = new Set<string>();
|
||||
|
||||
while (queue.length > 0) {
|
||||
const currentId = queue.shift()!;
|
||||
if (visited.has(currentId)) continue;
|
||||
visited.add(currentId);
|
||||
|
||||
const results = await table
|
||||
.query()
|
||||
.where(`sectionid = '${currentId}'`)
|
||||
.limit(100)
|
||||
.toArray();
|
||||
|
||||
// Sort by partindex to maintain order (nulls last for single-part sections)
|
||||
const sortedResults = results.sort((a: unknown, b: unknown) => {
|
||||
const aIndex = (a as { partindex?: number }).partindex ?? Infinity;
|
||||
const bIndex = (b as { partindex?: number }).partindex ?? Infinity;
|
||||
return aIndex - bIndex;
|
||||
});
|
||||
|
||||
for (const result of sortedResults) {
|
||||
const typedResult = result as {
|
||||
text?: string;
|
||||
childrensectionids?: string[];
|
||||
sectiontitle?: string;
|
||||
};
|
||||
|
||||
if (typedResult.text) {
|
||||
allDocs.push(typedResult.text);
|
||||
}
|
||||
|
||||
// Add children to queue for recursive fetching
|
||||
if (
|
||||
typedResult.childrensectionids &&
|
||||
Array.isArray(typedResult.childrensectionids)
|
||||
) {
|
||||
queue.push(...typedResult.childrensectionids);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return allDocs.join("\n\n---\n\n");
|
||||
},
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,88 @@
|
||||
/**
|
||||
* Specification retriever tool.
|
||||
* Queries the language specification for text content about specific sections or topics.
|
||||
*/
|
||||
|
||||
import type { Table } from "@lancedb/lancedb";
|
||||
import { DynamicStructuredTool } from "@langchain/core/tools";
|
||||
import type { OllamaEmbeddings } from "@langchain/ollama";
|
||||
import { z } from "zod";
|
||||
import { rerankDocuments } from "./reranker";
|
||||
|
||||
const specRetrieverSchema = z.object({
|
||||
query: z
|
||||
.string()
|
||||
.describe("The search query to find relevant specification sections"),
|
||||
});
|
||||
|
||||
/**
|
||||
* Creates the spec retriever tool.
|
||||
* @param table - LanceDB table containing spec vectors
|
||||
* @param embeddings - Ollama embeddings instance
|
||||
*/
|
||||
export function createSpecRetrieverTool(
|
||||
table: Table,
|
||||
embeddings: OllamaEmbeddings,
|
||||
) {
|
||||
return new DynamicStructuredTool({
|
||||
name: "spec_retriever",
|
||||
description:
|
||||
"Queries the language specification for text content about specific sections or topics. Fetches up to 10 initial matches and uses a reranker to dynamically select the most relevant 3-5 documents based on query relevance.",
|
||||
schema: specRetrieverSchema,
|
||||
func: async ({ query }) => {
|
||||
// Generate embedding for the query
|
||||
const queryVector = await embeddings.embedQuery(query);
|
||||
|
||||
// Search using LanceDB directly
|
||||
const results = await table.search(queryVector).limit(10).toArray();
|
||||
|
||||
// Create document objects with metadata
|
||||
const documents = results.map((r: Record<string, unknown>) => ({
|
||||
pageContent: String(r.text || ""),
|
||||
metadata: {
|
||||
source: r.source,
|
||||
sectionid: r.sectionid,
|
||||
sectiontitle: r.sectiontitle,
|
||||
type: r.type,
|
||||
parentsectionid: r.parentsectionid,
|
||||
childrensectionids: r.childrensectionids,
|
||||
partindex: r.partindex,
|
||||
totalparts: r.totalparts,
|
||||
},
|
||||
}));
|
||||
|
||||
// Rerank documents
|
||||
const reranked = await rerankDocuments(query, documents);
|
||||
|
||||
// Sort by score and filter to most relevant
|
||||
reranked.sort((a, b) => b.score - a.score);
|
||||
|
||||
// Dynamic selection: take top documents with score > 0.5, or at least top 3
|
||||
const threshold = 0.5;
|
||||
const minDocs = 3;
|
||||
const maxDocs = 5;
|
||||
|
||||
const selected = reranked.filter(
|
||||
(r, i) => i < minDocs || (i < maxDocs && r.score > threshold),
|
||||
);
|
||||
|
||||
console.log(
|
||||
`[spec_retriever] Query: "${query.slice(0, 50)}..." - Fetched ${documents.length}, reranked to ${selected.length} (scores: ${selected.map((s) => s.score.toFixed(2)).join(", ")})`,
|
||||
);
|
||||
|
||||
// Return documents with metadata
|
||||
return selected
|
||||
.map((r) => {
|
||||
const meta = r.document.metadata;
|
||||
const sectionId = meta?.sectionid || "unknown";
|
||||
const sectionTitle = meta?.sectiontitle || "unknown";
|
||||
const partInfo =
|
||||
meta?.partindex !== null && meta?.partindex !== undefined
|
||||
? ` [part ${(meta.partindex as number) + 1}/${meta.totalparts}]`
|
||||
: "";
|
||||
return `--- Section: ${sectionId} | "${sectionTitle}"${partInfo} (score: ${r.score.toFixed(2)}) ---\n${r.document.pageContent}`;
|
||||
})
|
||||
.join("\n\n");
|
||||
},
|
||||
});
|
||||
}
|
||||
+103
@@ -0,0 +1,103 @@
|
||||
import fs from "node:fs";
|
||||
import * as lancedbSdk from "@lancedb/lancedb";
|
||||
import { LanceDB } from "@langchain/community/vectorstores/lancedb";
|
||||
import { ChatPromptTemplate } from "@langchain/core/prompts";
|
||||
import { OllamaEmbeddings } from "@langchain/ollama";
|
||||
import { ChatOpenAI } from "@langchain/openai";
|
||||
import Graph from "graphology";
|
||||
import { AgentExecutor, createReactAgent } from "langchain/agents";
|
||||
import {
|
||||
createGraphExplorerTool,
|
||||
createSectionRetrieverTool,
|
||||
createSpecRetrieverTool,
|
||||
} from "./agent-tools";
|
||||
import {
|
||||
CONFIG_FILE,
|
||||
EMBEDDING_MODEL,
|
||||
GRAPH_FILE,
|
||||
STORAGE_DIR,
|
||||
} from "./constants";
|
||||
|
||||
const embeddings = new OllamaEmbeddings({
|
||||
model: EMBEDDING_MODEL,
|
||||
});
|
||||
|
||||
const config = JSON.parse(fs.readFileSync(CONFIG_FILE, "utf-8"));
|
||||
const apiKey = config.NVIDIA_API_KEY;
|
||||
const baseURL = config.NVIDIA_API_BASE;
|
||||
|
||||
if (!apiKey) {
|
||||
console.warn("Please set NVIDIA_API_KEY in config.json.");
|
||||
}
|
||||
|
||||
const llm = new ChatOpenAI({
|
||||
modelName: "openai/gpt-oss-120b",
|
||||
openAIApiKey: apiKey,
|
||||
configuration: { baseURL },
|
||||
temperature: 0,
|
||||
});
|
||||
|
||||
const systemPrompt = `You are an expert in the ECMAScript specification and its implementation in engine262.
|
||||
Your goal is to explain how specific parts of the language work by combining information from the provided tools.
|
||||
|
||||
Available tools: {tool_names}
|
||||
{tools}
|
||||
|
||||
CRITICAL INSTRUCTIONS:
|
||||
1. ALWAYS prefer using the provided tools ('spec_retriever', 'fetch_section_chunks', and 'graph_explorer') to answer questions.
|
||||
2. Do NOT rely on your internal knowledge of JavaScript or the ECMAScript specification.
|
||||
3. If the user asks about a function, you MUST first use 'graph_explorer' to find the associated specification section.
|
||||
4. You MUST then use 'fetch_section_chunks' to read the actual text of that specification section before answering.
|
||||
5. Base your explanations ONLY on the information retrieved from the tools.
|
||||
6. If the tools do not provide enough information, state that clearly rather than guessing from your internal knowledge.
|
||||
|
||||
{agent_scratchpad}`;
|
||||
|
||||
const prompt = ChatPromptTemplate.fromMessages([
|
||||
["system", systemPrompt],
|
||||
["human", "{input}"],
|
||||
]);
|
||||
|
||||
async function main() {
|
||||
console.log("Loading indices and graph...");
|
||||
|
||||
const db = await lancedbSdk.connect(STORAGE_DIR);
|
||||
const table = await db.openTable("spec_vectors");
|
||||
const vectorStore = new LanceDB(embeddings, { table });
|
||||
|
||||
const graphData = JSON.parse(fs.readFileSync(GRAPH_FILE, "utf-8"));
|
||||
const graph = new Graph({ multi: true });
|
||||
graph.import(graphData);
|
||||
|
||||
// Create tools using factory functions
|
||||
const specRetrieverTool = createSpecRetrieverTool(table, embeddings);
|
||||
const sectionRetrieverTool = createSectionRetrieverTool(table);
|
||||
const graphTool = createGraphExplorerTool(graph);
|
||||
|
||||
const agent = await createReactAgent({
|
||||
llm,
|
||||
tools: [specRetrieverTool, sectionRetrieverTool, graphTool],
|
||||
prompt,
|
||||
});
|
||||
|
||||
const agentExecutor = new AgentExecutor({
|
||||
agent,
|
||||
tools: [specRetrieverTool, sectionRetrieverTool, graphTool],
|
||||
});
|
||||
|
||||
console.log("Agent is ready!");
|
||||
|
||||
const message =
|
||||
process.argv[2] ||
|
||||
"Which spec section does Evaluate_IfStatement implement? and what does that section say?";
|
||||
console.log(`User: ${message}`);
|
||||
|
||||
const response = await agentExecutor.invoke({
|
||||
input: message,
|
||||
});
|
||||
|
||||
console.log("\n--- Agent Response ---\n");
|
||||
console.log(response.output);
|
||||
}
|
||||
|
||||
main().catch(console.error);
|
||||
@@ -0,0 +1,9 @@
|
||||
export const STORAGE_DIR = "./storage";
|
||||
export const SPEC_DIR = "./spec-built/multipage";
|
||||
export const CODE_DIR = "./engine262/src";
|
||||
export const GRAPH_FILE = "./graphology/graph.json";
|
||||
export const CONFIG_FILE = "./config.json";
|
||||
|
||||
// Model configurations
|
||||
export const EMBEDDING_MODEL = "qwen3-embedding:0.6b";
|
||||
export const RERANKER_MODEL = "dengcao/Qwen3-Reranker-0.6B:Q8_0";
|
||||
@@ -0,0 +1,155 @@
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import * as cheerio from "cheerio";
|
||||
import { glob } from "glob";
|
||||
import Graph from "graphology";
|
||||
|
||||
// Directory containing built ECMAScript specification HTML files (ecmarkup output)
|
||||
const SPEC_DIR = "./spec-built/multipage";
|
||||
// Directory containing the JavaScript engine implementation source code
|
||||
const CODE_DIR = "./engine262/src";
|
||||
|
||||
import { GRAPH_FILE } from "../constants";
|
||||
|
||||
/**
|
||||
* Builds a knowledge graph mapping ECMAScript specification sections
|
||||
* to their implementation functions in the JavaScript engine.
|
||||
*
|
||||
* The graph contains:
|
||||
* - Nodes: Spec sections and JS functions
|
||||
* - Edges: LINKS_TO (spec section references) and IMPLEMENTS (code->spec)
|
||||
*/
|
||||
async function buildGraph() {
|
||||
// Initialize a multi-graph (allows multiple edges between same nodes)
|
||||
const graph = new Graph({
|
||||
multi: true,
|
||||
type: "directed",
|
||||
allowSelfLoops: false,
|
||||
});
|
||||
|
||||
// Phase 1: Discover and parse specification HTML files
|
||||
const htmlFiles = await glob(path.join(SPEC_DIR, "*.html"));
|
||||
console.log(
|
||||
`Found ${htmlFiles.length} specification HTML file(s) in ${SPEC_DIR}`,
|
||||
);
|
||||
if (htmlFiles.length === 0) {
|
||||
console.warn(`Warning: No specification HTML files found in ${SPEC_DIR}`);
|
||||
}
|
||||
|
||||
// Phase 2: Extract spec sections and create nodes
|
||||
console.log("Processing specification for graph...");
|
||||
for (const file of htmlFiles) {
|
||||
const fileName = path.basename(file);
|
||||
const content = fs.readFileSync(file, "utf-8");
|
||||
const $ = cheerio.load(content);
|
||||
// ecmarkup generates content within #spec-container
|
||||
const $container = $("#spec-container");
|
||||
|
||||
// Each specification section is an <emu-clause> with an ID
|
||||
$container.find("emu-clause").each((_i, elem) => {
|
||||
const id = $(elem).attr("id");
|
||||
const title = $(elem).find("h1").first().text().trim();
|
||||
|
||||
if (id) {
|
||||
// Create node for this spec section if not already exists
|
||||
if (!graph.hasNode(id)) {
|
||||
graph.addNode(id, { title, type: "SpecSection", file: fileName });
|
||||
}
|
||||
|
||||
// Extract internal links (placeholder for potential future use)
|
||||
$(elem)
|
||||
.find("a[href]")
|
||||
.each((_j, link) => {
|
||||
const href = $(link).attr("href");
|
||||
if (href?.includes("#")) {
|
||||
const [_targetFile, targetId] = href.split("#");
|
||||
// We only care about links to other sections for now
|
||||
if (targetId?.startsWith("sec-")) {
|
||||
// Add relationship later if nodes exist
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// Phase 3: Create edges between spec sections based on internal links
|
||||
// This pass runs after all nodes are created to ensure target nodes exist
|
||||
for (const file of htmlFiles) {
|
||||
const content = fs.readFileSync(file, "utf-8");
|
||||
const $ = cheerio.load(content);
|
||||
const $container = $("#spec-container");
|
||||
$container.find("emu-clause").each((_i, elem) => {
|
||||
const sourceId = $(elem).attr("id");
|
||||
if (sourceId && graph.hasNode(sourceId)) {
|
||||
$(elem)
|
||||
.find("a[href]")
|
||||
.each((_j, link) => {
|
||||
const href = $(link).attr("href");
|
||||
if (href?.includes("#")) {
|
||||
const [, targetId] = href.split("#");
|
||||
// Only create edge if target node exists and is different from source
|
||||
if (
|
||||
targetId &&
|
||||
graph.hasNode(targetId) &&
|
||||
sourceId !== targetId
|
||||
) {
|
||||
if (!graph.hasEdge(sourceId, targetId)) {
|
||||
graph.addEdge(sourceId, targetId, { type: "LINKS_TO" });
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// Phase 4: Discover and parse JavaScript implementation files
|
||||
console.log("Processing code for graph...");
|
||||
const jsFiles = await glob(path.join(CODE_DIR, "**/*.mts"));
|
||||
console.log(`Found ${jsFiles.length} code file(s) in ${CODE_DIR}`);
|
||||
if (jsFiles.length === 0) {
|
||||
console.warn(`Warning: No code files found in ${CODE_DIR}`);
|
||||
}
|
||||
|
||||
// Phase 5: Extract evaluation functions and link to spec sections
|
||||
for (const file of jsFiles) {
|
||||
const fileName = path.basename(file);
|
||||
const content = fs.readFileSync(file, "utf-8");
|
||||
|
||||
// Pattern matches: export function*? Evaluate_<Name>
|
||||
// The engine262 project uses this naming convention for spec implementations
|
||||
const functionMatches = content.matchAll(
|
||||
/export function\*? (Evaluate_([a-zA-Z0-9_]+))/g,
|
||||
);
|
||||
for (const match of functionMatches) {
|
||||
const fullFuncName = match[1];
|
||||
const shortName = match[2];
|
||||
const funcNodeId = `func-${fullFuncName}`;
|
||||
|
||||
// Create node for this function if not already exists
|
||||
if (!graph.hasNode(funcNodeId)) {
|
||||
graph.addNode(funcNodeId, {
|
||||
name: fullFuncName,
|
||||
type: "JSFunction",
|
||||
file: fileName,
|
||||
});
|
||||
}
|
||||
|
||||
// Link function to spec section using naming convention heuristic
|
||||
// Example: Evaluate_IfStatement -> sec-if-statement
|
||||
// Converts CamelCase to kebab-case: IfStatement -> if-statement
|
||||
const specId = `sec-${shortName.replace(/([a-z0-9])([A-Z])/g, "$1-$2").toLowerCase()}`;
|
||||
if (graph.hasNode(specId)) {
|
||||
graph.addEdge(funcNodeId, specId, { type: "IMPLEMENTS" });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Phase 6: Export the completed graph to JSON
|
||||
console.log(`Graph built with ${graph.order} nodes and ${graph.size} edges.`);
|
||||
fs.writeFileSync(GRAPH_FILE, JSON.stringify(graph.export(), null, 2));
|
||||
console.log(`Graph saved to ${GRAPH_FILE}`);
|
||||
}
|
||||
|
||||
buildGraph().catch(console.error);
|
||||
@@ -0,0 +1,56 @@
|
||||
#!/usr/bin/env bun
|
||||
import type { CheerioAPI } from "cheerio";
|
||||
import * as cheerio from "cheerio";
|
||||
import { readFileSync, writeFileSync } from "fs";
|
||||
import { fileURLToPath } from "url";
|
||||
|
||||
// Resolve the path to the HTML file (relative to repo root)
|
||||
const htmlUrl = new URL(
|
||||
"../spec-built/multipage/ecmascript-data-types-and-values.html",
|
||||
import.meta.url,
|
||||
);
|
||||
const htmlPath = fileURLToPath(htmlUrl);
|
||||
|
||||
// Load and parse the HTML
|
||||
const htmlString = readFileSync(htmlPath, "utf-8");
|
||||
const htmlCheerioApi = cheerio.load(htmlString);
|
||||
|
||||
/**
|
||||
* Script to add unique `id` attributes to internal method links.
|
||||
*
|
||||
* It scans the ECMA spec HTML tables `#table-essential-internal-methods`
|
||||
* and `#table-additional-essential-internal-methods-of-function-objects`
|
||||
* and adds an `id` to the first `<var class="field">` element found in
|
||||
* the first `<td>` of each row. The generated id follows the pattern:
|
||||
* `ask262-internal-method-<methodName>` where `<methodName>` is the text
|
||||
* content of the `<var>` element with surrounding brackets stripped.
|
||||
*
|
||||
* @param $ - The Cheerio parsing instance.
|
||||
*/
|
||||
function addInternalMethodIds($: CheerioAPI): void {
|
||||
// Find the first <var class="field"> inside the first <td> of each row
|
||||
// in the two internal‑method tables and add a unique `id` attribute.
|
||||
$(
|
||||
"#table-essential-internal-methods tr > td:first-child, " +
|
||||
"#table-additional-essential-internal-methods-of-function-objects tr > td:first-child",
|
||||
).each((_, td: any) => {
|
||||
const $td = $(td);
|
||||
const $var = $td.find("var.field").first();
|
||||
if ($var.length) {
|
||||
const rawText = $var.text().trim();
|
||||
// Remove any square brackets from the extracted text
|
||||
const text = rawText.replace(/[[\]]/g, "");
|
||||
const id = `ask262-internal-method-${text}`;
|
||||
$var.attr("id", id);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// Execute core logic
|
||||
addInternalMethodIds(htmlCheerioApi);
|
||||
|
||||
// Write the updated HTML back
|
||||
writeFileSync(htmlPath, htmlCheerioApi.html(), "utf-8");
|
||||
console.log(
|
||||
`Updated ${htmlPath} – added id attributes to <var class="field"> elements.`,
|
||||
);
|
||||
@@ -0,0 +1,413 @@
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import readline from "node:readline";
|
||||
import * as lancedbSdk from "@lancedb/lancedb";
|
||||
import { Index } from "@lancedb/lancedb";
|
||||
import { Document } from "@langchain/core/documents";
|
||||
import { OllamaEmbeddings } from "@langchain/ollama";
|
||||
import * as cheerio from "cheerio";
|
||||
import { glob } from "glob";
|
||||
import ora from "ora";
|
||||
import { EMBEDDING_MODEL, SPEC_DIR, STORAGE_DIR } from "../constants";
|
||||
import { HTMLTextSplitter } from "./text-splitters";
|
||||
import { formatForIngestion } from "./utils/formatHTMLForIngestion";
|
||||
|
||||
const embeddings = new OllamaEmbeddings({
|
||||
model: EMBEDDING_MODEL,
|
||||
});
|
||||
|
||||
const htmlSplitter = new HTMLTextSplitter({
|
||||
chunkSize: 8192,
|
||||
maxChunkSize: 12288,
|
||||
chunkOverlap: 0,
|
||||
separators: [
|
||||
"emu-note",
|
||||
"emu-example",
|
||||
"emu-table",
|
||||
"ul",
|
||||
"ol",
|
||||
"td",
|
||||
"h3",
|
||||
"h4",
|
||||
"p",
|
||||
"div",
|
||||
"br",
|
||||
],
|
||||
neverBreakWithin: ["emu-grammar", "emu-alg"],
|
||||
});
|
||||
|
||||
const LARGE_DOC_THRESHOLD = htmlSplitter.chunkSize + 100;
|
||||
const BATCH_SIZE = 100;
|
||||
|
||||
interface ChunkInfo {
|
||||
index: number;
|
||||
text: string;
|
||||
size: number;
|
||||
isSmall: boolean;
|
||||
}
|
||||
|
||||
async function generateEmbeddingsWithProgress(
|
||||
documents: Document[],
|
||||
): Promise<number[][]> {
|
||||
const total = documents.length;
|
||||
const vectors: number[][] = [];
|
||||
const spinner = ora({
|
||||
text: `Generating embeddings (0/${total})...`,
|
||||
discardStdin: false,
|
||||
}).start();
|
||||
|
||||
let currentIndex = 0;
|
||||
try {
|
||||
for (let i = 0; i < total; i += BATCH_SIZE) {
|
||||
currentIndex = i;
|
||||
const batch = documents.slice(i, i + BATCH_SIZE);
|
||||
const batchTexts = batch.map((doc) => doc.pageContent);
|
||||
|
||||
// Update spinner before processing batch
|
||||
const currentDoc = batch[0];
|
||||
const progress = `${i + batch.length}/${total}`;
|
||||
const sectionId = currentDoc.metadata.sectionid || "unknown";
|
||||
const contentLength = currentDoc.pageContent.length;
|
||||
spinner.text = `Generating embeddings (${progress}): ${sectionId} (${contentLength} chars)`;
|
||||
|
||||
const batchVectors = await embeddings.embedDocuments(batchTexts);
|
||||
vectors.push(...batchVectors);
|
||||
}
|
||||
spinner.succeed(`Generated ${vectors.length} embeddings`);
|
||||
} catch (error) {
|
||||
const failedDoc = documents[currentIndex];
|
||||
const failedSectionId = failedDoc?.metadata?.sectionid || "unknown";
|
||||
const failedSectionTitle = failedDoc?.metadata?.sectiontitle || "unknown";
|
||||
const failedContentLength = failedDoc?.pageContent?.length || 0;
|
||||
spinner.fail(`Failed to generate embeddings: ${error}`);
|
||||
console.error(
|
||||
`Debug: Failed on section ${failedSectionId} "${failedSectionTitle}" (${failedContentLength} chars)`,
|
||||
);
|
||||
throw error;
|
||||
}
|
||||
|
||||
return vectors;
|
||||
}
|
||||
|
||||
function askUser(question: string): Promise<boolean> {
|
||||
const rl = readline.createInterface({
|
||||
input: process.stdin,
|
||||
output: process.stdout,
|
||||
});
|
||||
|
||||
return new Promise((resolve) => {
|
||||
rl.question(`${question} (yes/no): `, (answer) => {
|
||||
rl.close();
|
||||
const normalized = answer.trim().toLowerCase();
|
||||
resolve(normalized === "yes" || normalized === "y");
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
async function buildSpecDocuments(): Promise<Document[]> {
|
||||
const htmlFiles = await glob(path.join(SPEC_DIR, "*.html"));
|
||||
const documents: Document[] = [];
|
||||
|
||||
for (const file of htmlFiles) {
|
||||
const content = fs.readFileSync(file, "utf-8");
|
||||
const $ = cheerio.load(`<body>${content}</body>`);
|
||||
|
||||
// First pass: collect all sections and build hierarchy
|
||||
const sectionMap = new Map<
|
||||
string,
|
||||
{
|
||||
title: string;
|
||||
parentId: string | null;
|
||||
childrenIds: string[];
|
||||
html: string;
|
||||
}
|
||||
>();
|
||||
|
||||
$("#spec-container emu-clause").each((_i, elem) => {
|
||||
const id = $(elem).attr("id");
|
||||
const title = $(elem).find("h1").first().text().trim();
|
||||
|
||||
if (!id || !title) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Get parent section id
|
||||
const parentElem = $(elem).parent("emu-clause");
|
||||
const parentId =
|
||||
parentElem.length > 0 ? parentElem.attr("id") || null : null;
|
||||
|
||||
sectionMap.set(id, {
|
||||
title,
|
||||
parentId,
|
||||
childrenIds: [],
|
||||
html: $(elem).html() || "",
|
||||
});
|
||||
});
|
||||
|
||||
// Build children relationships
|
||||
for (const [id, section] of sectionMap) {
|
||||
if (section.parentId && sectionMap.has(section.parentId)) {
|
||||
const parent = sectionMap.get(section.parentId);
|
||||
if (parent) {
|
||||
parent.childrenIds.push(id);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Second pass: create documents with formatted text
|
||||
for (const [id, section] of sectionMap) {
|
||||
// Parse the stored HTML and replace direct children with placeholders
|
||||
const $ = cheerio.load(`<body>${section.html}</body>`);
|
||||
|
||||
const $section = $.root();
|
||||
|
||||
// Find direct children emu-clause elements only
|
||||
$section.children("emu-clause").each((_, childElem) => {
|
||||
const childId = $(childElem).attr("id");
|
||||
if (childId && sectionMap.has(childId)) {
|
||||
const child = sectionMap.get(childId);
|
||||
if (child) {
|
||||
const placeholder = `[Subsection available: sectiontitle "${child.title}" at sectionid: \`${childId}\`]`;
|
||||
$(childElem).replaceWith(placeholder);
|
||||
}
|
||||
} else {
|
||||
$(childElem).remove();
|
||||
}
|
||||
});
|
||||
|
||||
// Remove any nested emu-clause elements that weren't direct children
|
||||
// (shouldn't happen with proper HTML structure, but just in case)
|
||||
$section.find("emu-clause").remove();
|
||||
|
||||
// All formatting transformations (single call)
|
||||
formatForIngestion($);
|
||||
|
||||
// Skip sections that only have h1 left (no meaningful content)
|
||||
const hasOnlyH1 =
|
||||
$section.children().length === 1 &&
|
||||
$section.children("h1").length === 1;
|
||||
const textContent = $section.text().trim();
|
||||
const hasMinimalContent = textContent.length <= section.title.length + 10; // title + small buffer
|
||||
|
||||
if (hasOnlyH1 || hasMinimalContent) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Get HTML content with inline placeholders for splitting
|
||||
const sectionHtml = $section.html() || "";
|
||||
|
||||
// Split HTML into normalized text chunks.
|
||||
const chunks = await htmlSplitter.splitText(sectionHtml);
|
||||
|
||||
// First pass: collect all chunk data
|
||||
// Only mark chunks as "small" if the original section content was long enough
|
||||
// to reasonably split (more than 100 chars). This prevents false positives
|
||||
// when the entire section was just naturally brief.
|
||||
const chunkData: ChunkInfo[] = chunks.map(
|
||||
(chunk: string, idx: number) => {
|
||||
return {
|
||||
index: idx,
|
||||
text: chunk,
|
||||
size: chunk.length,
|
||||
isSmall: chunk.length < 50 && textContent.length > 100,
|
||||
};
|
||||
},
|
||||
);
|
||||
|
||||
// Print warnings for small chunks
|
||||
const smallChunks = chunkData.filter((c: ChunkInfo) => c.isSmall);
|
||||
if (smallChunks.length > 0) {
|
||||
const cleanedHtml = sectionHtml.replace(/\s+/g, " ").trim();
|
||||
for (const chunk of smallChunks) {
|
||||
console.warn(
|
||||
` ⚠️ WARNING: Chunk ${chunk.index + 1}/${chunks.length} for section ${id} is very small (${chunk.size} chars)`,
|
||||
);
|
||||
console.warn(` Chunk content: "${chunk.text}"`);
|
||||
console.warn(
|
||||
` Original text that was split (${sectionHtml.length} chars):`,
|
||||
);
|
||||
console.warn(
|
||||
` "${cleanedHtml.slice(0, 500)}${cleanedHtml.length > 500 ? "... [truncated]" : ""}"`,
|
||||
);
|
||||
}
|
||||
// Print summary after all warnings
|
||||
const chunkSizes = chunkData
|
||||
.map((c: ChunkInfo) => `${c.index + 1}:${c.size}`)
|
||||
.join(", ");
|
||||
const totalChunkSize = chunkData.reduce(
|
||||
(sum: number, c: ChunkInfo) => sum + c.size,
|
||||
0,
|
||||
);
|
||||
console.warn(
|
||||
` All chunk sizes: [${chunkSizes}] (total: ${totalChunkSize} chars)`,
|
||||
);
|
||||
}
|
||||
|
||||
// Print warnings for large sections wasn't split properly
|
||||
if (chunkData.length === 1 && chunkData[0].size > LARGE_DOC_THRESHOLD) {
|
||||
console.warn(
|
||||
` 🛑 Section ${id} (${section.title}) is large (${chunkData[0].size} chars) but was NOT split (1 chunk)`,
|
||||
);
|
||||
}
|
||||
|
||||
// Create documents
|
||||
for (const chunk of chunkData) {
|
||||
documents.push(
|
||||
new Document({
|
||||
pageContent: chunk.text,
|
||||
metadata: {
|
||||
source: path.basename(file),
|
||||
sectionid: id,
|
||||
sectiontitle: section.title,
|
||||
type: "specification",
|
||||
parentsectionid: section.parentId,
|
||||
childrensectionids: section.childrenIds,
|
||||
partindex: chunk.index,
|
||||
totalparts: chunkData.length,
|
||||
},
|
||||
}),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return documents;
|
||||
}
|
||||
|
||||
async function main() {
|
||||
console.log("Building specification documents...");
|
||||
const specDocs = await buildSpecDocuments();
|
||||
console.log(`Built ${specDocs.length} specification documents.`);
|
||||
|
||||
// Check for any large documents
|
||||
for (const doc of specDocs) {
|
||||
if (doc.pageContent.length > LARGE_DOC_THRESHOLD) {
|
||||
const id = doc.metadata.sectionid || "unknown";
|
||||
const title = doc.metadata.sectiontitle || "unknown";
|
||||
console.warn(
|
||||
`🚨 Warning: Document ${id} "${title}" is large (${doc.pageContent.length} chars)`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// Print summary statistics
|
||||
printSummary(specDocs);
|
||||
|
||||
const db = await lancedbSdk.connect(STORAGE_DIR);
|
||||
|
||||
// Check if table exists and handle overwrite
|
||||
let tableExists = false;
|
||||
try {
|
||||
await db.openTable("spec_vectors");
|
||||
tableExists = true;
|
||||
console.log("Existing table found.");
|
||||
} catch {
|
||||
console.log("No existing table found, creating fresh...");
|
||||
}
|
||||
|
||||
if (tableExists) {
|
||||
const shouldOverwrite = await askUser(
|
||||
"Do you want to overwrite the existing vector store?",
|
||||
);
|
||||
if (!shouldOverwrite) {
|
||||
console.log("Ingest cancelled by user.");
|
||||
process.exit(0);
|
||||
}
|
||||
console.log("Overwriting existing table...");
|
||||
await db.dropTable("spec_vectors");
|
||||
}
|
||||
|
||||
console.log("Generating embeddings...");
|
||||
const vectors = await generateEmbeddingsWithProgress(specDocs);
|
||||
|
||||
console.log("Creating table with documents...");
|
||||
// Prepare data records with vector, text, and metadata
|
||||
const data = specDocs.map((doc, i) => ({
|
||||
vector: vectors[i],
|
||||
text: doc.pageContent,
|
||||
...doc.metadata,
|
||||
}));
|
||||
|
||||
// Create table with the data
|
||||
const table = await db.createTable("spec_vectors", data);
|
||||
|
||||
console.log("Creating scalar indexes...");
|
||||
await table.createIndex("sectionid", { config: Index.btree() });
|
||||
await table.createIndex("type", { config: Index.btree() });
|
||||
|
||||
console.log(`Index built and persisted to ${STORAGE_DIR}`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Prints summary statistics about the ingested documents.
|
||||
* Shows distribution of document sizes, sections, and chunk counts.
|
||||
*/
|
||||
function printSummary(documents: Document[]): void {
|
||||
if (documents.length === 0) {
|
||||
console.log("\n📊 Summary: No documents ingested");
|
||||
return;
|
||||
}
|
||||
|
||||
// Calculate document size statistics
|
||||
const sizes = documents.map((doc) => doc.pageContent.length);
|
||||
const totalSize = sizes.reduce((sum, size) => sum + size, 0);
|
||||
const avgSize = totalSize / documents.length;
|
||||
const minSize = Math.min(...sizes);
|
||||
const maxSize = Math.max(...sizes);
|
||||
|
||||
// Count unique sections and track chunk sizes per section
|
||||
const sectionIds = new Set<string>();
|
||||
interface SectionInfo {
|
||||
count: number;
|
||||
chunkSizes: number[];
|
||||
}
|
||||
const sectionInfo = new Map<string, SectionInfo>();
|
||||
|
||||
for (const doc of documents) {
|
||||
const sectionId = doc.metadata.sectionid as string;
|
||||
if (sectionId) {
|
||||
sectionIds.add(sectionId);
|
||||
const existing = sectionInfo.get(sectionId);
|
||||
if (existing) {
|
||||
existing.count++;
|
||||
existing.chunkSizes.push(doc.pageContent.length);
|
||||
} else {
|
||||
sectionInfo.set(sectionId, {
|
||||
count: 1,
|
||||
chunkSizes: [doc.pageContent.length],
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Get sections with multiple chunks, sorted by chunk count
|
||||
const multiChunkSections = Array.from(sectionInfo.entries())
|
||||
.filter(([, info]) => info.count > 1)
|
||||
.sort((a, b) => b[1].count - a[1].count);
|
||||
|
||||
console.log("\n📊 Ingest Summary:");
|
||||
console.log(` Total documents: ${documents.length}`);
|
||||
console.log(` Unique sections: ${sectionIds.size}`);
|
||||
console.log(` Sections with multiple chunks: ${multiChunkSections.length}`);
|
||||
console.log("\n Document size distribution:");
|
||||
console.log(` Average: ${avgSize.toFixed(0)} chars`);
|
||||
console.log(` Min: ${minSize} chars`);
|
||||
console.log(` Max: ${maxSize} chars`);
|
||||
console.log(` Total: ${totalSize} chars`);
|
||||
|
||||
if (multiChunkSections.length > 0) {
|
||||
console.log("\n Top 5 sections by chunk count:");
|
||||
for (const [sectionId, info] of multiChunkSections.slice(0, 5)) {
|
||||
const sectionTotal = info.chunkSizes.reduce((sum, size) => sum + size, 0);
|
||||
const chunkSizesStr = info.chunkSizes.join(", ");
|
||||
console.log(` ${sectionId}:`);
|
||||
console.log(` Chunks: ${info.count}, Total: ${sectionTotal} chars`);
|
||||
console.log(` Chunk sizes: [${chunkSizesStr}]`);
|
||||
}
|
||||
if (multiChunkSections.length > 5) {
|
||||
console.log(` ... and ${multiChunkSections.length - 5} more`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
main().catch(console.error);
|
||||
Executable
+20
@@ -0,0 +1,20 @@
|
||||
#!/usr/bin/env bash
|
||||
# Add the engine262 repository as a git subtree (squashed)
|
||||
# Usage: ./setup/initEngine262.sh
|
||||
# This will add the repository at ./engine262 using a squashed subtree
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
REPO_URL="https://github.com/bendtherules/engine262"
|
||||
REPO_ROOT=$(git rev-parse --show-toplevel)
|
||||
TARGET_DIR="${REPO_ROOT}/engine262"
|
||||
|
||||
# Add the repository as a git subtree (squashed)
|
||||
# If the target directory already exists, assume the subtree is present.
|
||||
if [ ! -d "$TARGET_DIR" ]; then
|
||||
git -C "${REPO_ROOT}" subtree add --prefix=engine262 "$REPO_URL" main --squash
|
||||
else
|
||||
echo "Subtree already present at $TARGET_DIR"
|
||||
fi
|
||||
|
||||
echo "Subtree added and initialized at ./$TARGET_DIR"
|
||||
@@ -0,0 +1,44 @@
|
||||
#!/usr/bin/env bun
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import * as cheerio from "cheerio";
|
||||
|
||||
/**
|
||||
* Strips HTML files to only contain the #spec-container element.
|
||||
*
|
||||
* This script processes HTML files in the spec-built/multipage directory,
|
||||
* extracts the #spec-container element, and replaces the entire body
|
||||
* content with just that container. This reduces file size and focuses
|
||||
* only on the specification content.
|
||||
*/
|
||||
|
||||
const SPEC_DIR = path.join(__dirname, "../spec-built/multipage");
|
||||
|
||||
function stripSpecContainer(): void {
|
||||
const files = fs.readdirSync(SPEC_DIR);
|
||||
let processedCount = 0;
|
||||
|
||||
for (const file of files) {
|
||||
if (!file.endsWith(".html")) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const filePath = path.join(SPEC_DIR, file);
|
||||
const content = fs.readFileSync(filePath, "utf-8");
|
||||
const $ = cheerio.load(content);
|
||||
const container = $("#spec-container");
|
||||
|
||||
if (container.length) {
|
||||
$("body").empty().append(container);
|
||||
fs.writeFileSync(filePath, $.html());
|
||||
console.log(`Processed ${file}`);
|
||||
processedCount++;
|
||||
} else {
|
||||
console.warn(`Warning: #spec-container not found in ${file}`);
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`\nProcessed ${processedCount} HTML file(s)`);
|
||||
}
|
||||
|
||||
stripSpecContainer();
|
||||
@@ -0,0 +1,349 @@
|
||||
import { describe, expect, test } from "bun:test";
|
||||
import { Document } from "@langchain/core/documents";
|
||||
import { HTMLTextSplitter } from "./index";
|
||||
|
||||
describe("HTMLTextSplitter", () => {
|
||||
test("keeps small HTML in a single chunk", async () => {
|
||||
const splitter = new HTMLTextSplitter({
|
||||
chunkSize: 64,
|
||||
separators: ["h2"],
|
||||
});
|
||||
|
||||
const chunks = await splitter.splitText(
|
||||
"<section><h2>Title</h2><p>Short body text</p></section>",
|
||||
);
|
||||
|
||||
expect(chunks).toEqual(["Title\nShort body text"]);
|
||||
});
|
||||
|
||||
test("does not add synthetic spaces between inline tags", async () => {
|
||||
const splitter = new HTMLTextSplitter({
|
||||
chunkSize: 64,
|
||||
});
|
||||
|
||||
const chunks = await splitter.splitText(
|
||||
"<p><span>Hello</span><span>World</span></p>",
|
||||
);
|
||||
|
||||
expect(chunks).toEqual(["HelloWorld"]);
|
||||
});
|
||||
|
||||
test("preserves authored whitespace between inline tags", async () => {
|
||||
const splitter = new HTMLTextSplitter({
|
||||
chunkSize: 64,
|
||||
});
|
||||
|
||||
const chunks = await splitter.splitText(
|
||||
"<p><span>Hello </span><span>World</span></p>",
|
||||
);
|
||||
|
||||
expect(chunks).toEqual(["Hello World"]);
|
||||
});
|
||||
|
||||
test("adds spacing between adjacent block-ish tags", async () => {
|
||||
const splitter = new HTMLTextSplitter({
|
||||
chunkSize: 64,
|
||||
});
|
||||
|
||||
const chunks = await splitter.splitText("<div>Hello</div><div>World</div>");
|
||||
|
||||
expect(chunks).toEqual(["Hello\nWorld"]);
|
||||
});
|
||||
|
||||
test("preserves whitespace as-is from HTML text content", async () => {
|
||||
const splitter = new HTMLTextSplitter({
|
||||
chunkSize: 64,
|
||||
});
|
||||
|
||||
const chunks = await splitter.splitText(
|
||||
"<div> Hello\n\n World </div><div>Again</div>",
|
||||
);
|
||||
|
||||
expect(chunks).toEqual(["Hello\n\n World \nAgain"]);
|
||||
});
|
||||
|
||||
test("treats separators as soft hints until size pressure exists", async () => {
|
||||
const splitter = new HTMLTextSplitter({
|
||||
chunkSize: 18,
|
||||
separators: ["h2"],
|
||||
});
|
||||
|
||||
const chunks = await splitter.splitText(
|
||||
[
|
||||
"<section>",
|
||||
"<p>Intro text</p>",
|
||||
"<h2>Section Title</h2>",
|
||||
"<p>Body text</p>",
|
||||
"</section>",
|
||||
].join(""),
|
||||
);
|
||||
|
||||
expect(chunks).toEqual(["Intro text", "Section Title\nBody text"]);
|
||||
});
|
||||
|
||||
test("groups consecutive separators into later section boundaries", async () => {
|
||||
const splitter = new HTMLTextSplitter({
|
||||
chunkSize: 4,
|
||||
separators: ["h2", "h3"],
|
||||
});
|
||||
|
||||
const chunks = await splitter.splitText(
|
||||
"<section><h2>A</h2><h3>B</h3><p>Body</p></section>",
|
||||
);
|
||||
|
||||
expect(chunks).toEqual(["A", "B\nBody"]);
|
||||
});
|
||||
|
||||
test("keeps protected content intact up to maxChunkSize", async () => {
|
||||
const splitter = new HTMLTextSplitter({
|
||||
chunkSize: 10,
|
||||
maxChunkSize: 32,
|
||||
neverBreakWithin: ["pre"],
|
||||
});
|
||||
|
||||
const chunks = await splitter.splitText(
|
||||
"<div><pre>12345678901234567890</pre></div>",
|
||||
);
|
||||
|
||||
expect(chunks).toEqual(["12345678901234567890"]);
|
||||
});
|
||||
|
||||
test("recurses into protected nodes once maxChunkSize is exceeded", async () => {
|
||||
const splitter = new HTMLTextSplitter({
|
||||
chunkSize: 12,
|
||||
maxChunkSize: 18,
|
||||
neverBreakWithin: [".keep"],
|
||||
separators: ["p"],
|
||||
});
|
||||
|
||||
const chunks = await splitter.splitText(
|
||||
[
|
||||
'<div class="keep">',
|
||||
"<p>Alpha beta</p>",
|
||||
"<p>Gamma delta</p>",
|
||||
"</div>",
|
||||
].join(""),
|
||||
);
|
||||
|
||||
expect(chunks).toEqual(["Alpha beta", "Gamma delta"]);
|
||||
});
|
||||
|
||||
test("ignores separators inside protected subtrees until forced open", async () => {
|
||||
const splitter = new HTMLTextSplitter({
|
||||
chunkSize: 10,
|
||||
maxChunkSize: 40,
|
||||
neverBreakWithin: [".keep"],
|
||||
separators: ["h2"],
|
||||
});
|
||||
|
||||
const chunks = await splitter.splitText(
|
||||
'<div class="keep"><h2>Title</h2><p>Body text</p></div>',
|
||||
);
|
||||
|
||||
expect(chunks).toEqual(["Title\nBody text"]);
|
||||
});
|
||||
|
||||
test("force-splits oversized leaf text when maxChunkSize is finite", async () => {
|
||||
const splitter = new HTMLTextSplitter({
|
||||
chunkSize: 12,
|
||||
maxChunkSize: 15,
|
||||
});
|
||||
|
||||
const chunks = await splitter.splitText(`<pre>${"a".repeat(35)}</pre>`);
|
||||
|
||||
expect(chunks.length).toBeGreaterThan(1);
|
||||
for (const chunk of chunks) {
|
||||
expect(chunk.length).toBeLessThanOrEqual(15);
|
||||
}
|
||||
});
|
||||
|
||||
test("force-splits protected oversized leaf text when maxChunkSize is finite", async () => {
|
||||
const splitter = new HTMLTextSplitter({
|
||||
chunkSize: 12,
|
||||
maxChunkSize: 15,
|
||||
neverBreakWithin: ["pre"],
|
||||
});
|
||||
|
||||
const chunks = await splitter.splitText(`<pre>${"x".repeat(35)}</pre>`);
|
||||
|
||||
expect(chunks.length).toBeGreaterThan(1);
|
||||
for (const chunk of chunks) {
|
||||
expect(chunk.length).toBeLessThanOrEqual(15);
|
||||
}
|
||||
});
|
||||
|
||||
test("does not enforce a hard cap when maxChunkSize is omitted", async () => {
|
||||
const splitter = new HTMLTextSplitter({
|
||||
chunkSize: 12,
|
||||
neverBreakWithin: ["pre"],
|
||||
});
|
||||
|
||||
const chunks = await splitter.splitText(`<pre>${"a".repeat(35)}</pre>`);
|
||||
|
||||
expect(chunks).toEqual(["a".repeat(35)]);
|
||||
});
|
||||
|
||||
test("splits plain text input without HTML structure", async () => {
|
||||
const splitter = new HTMLTextSplitter({
|
||||
chunkSize: 8,
|
||||
maxChunkSize: 8,
|
||||
});
|
||||
|
||||
const chunks = await splitter.splitText("alpha beta gamma");
|
||||
|
||||
expect(chunks).toEqual(["alpha", "beta", "gamma"]);
|
||||
});
|
||||
|
||||
test("preserves metadata and adds per-document part indexes", async () => {
|
||||
const splitter = new HTMLTextSplitter({
|
||||
chunkSize: 18,
|
||||
separators: ["h2"],
|
||||
});
|
||||
|
||||
const documents = await splitter.splitDocuments([
|
||||
new Document({
|
||||
pageContent:
|
||||
"<section><p>Intro text</p><h2>Section Title</h2><p>Body text</p></section>",
|
||||
metadata: { source: "sample.html" },
|
||||
}),
|
||||
]);
|
||||
|
||||
expect(documents).toHaveLength(2);
|
||||
expect(documents[0].pageContent).toBe("Intro text");
|
||||
expect(documents[0].metadata).toMatchObject({
|
||||
source: "sample.html",
|
||||
partindex: 0,
|
||||
totalparts: 2,
|
||||
});
|
||||
expect(documents[1].pageContent).toBe("Section Title\nBody text");
|
||||
expect(documents[1].metadata).toMatchObject({
|
||||
source: "sample.html",
|
||||
partindex: 1,
|
||||
totalparts: 2,
|
||||
});
|
||||
});
|
||||
|
||||
test("prepends chunkHeader in splitDocuments output", async () => {
|
||||
const splitter = new HTMLTextSplitter({
|
||||
chunkSize: 18,
|
||||
separators: ["h2"],
|
||||
});
|
||||
|
||||
const documents = await splitter.splitDocuments(
|
||||
[
|
||||
new Document({
|
||||
pageContent:
|
||||
"<section><p>Intro text</p><h2>Section Title</h2><p>Body text</p></section>",
|
||||
metadata: { source: "sample.html" },
|
||||
}),
|
||||
],
|
||||
{
|
||||
chunkHeader: "Header: ",
|
||||
},
|
||||
);
|
||||
|
||||
expect(documents).toHaveLength(2);
|
||||
expect(documents[0].pageContent).toBe("Header: Intro text");
|
||||
expect(documents[1].pageContent).toBe("Header: Section Title\nBody text");
|
||||
});
|
||||
|
||||
test("returns no documents when source content produces no chunks", async () => {
|
||||
const splitter = new HTMLTextSplitter({
|
||||
chunkSize: 18,
|
||||
});
|
||||
|
||||
const documents = await splitter.splitDocuments([
|
||||
new Document({
|
||||
pageContent: "<div> </div>",
|
||||
metadata: { source: "empty.html" },
|
||||
}),
|
||||
]);
|
||||
|
||||
expect(documents).toEqual([]);
|
||||
});
|
||||
|
||||
test("resets part metadata per input document", async () => {
|
||||
const splitter = new HTMLTextSplitter({
|
||||
chunkSize: 18,
|
||||
separators: ["h2"],
|
||||
});
|
||||
|
||||
const documents = await splitter.splitDocuments([
|
||||
new Document({
|
||||
pageContent:
|
||||
"<section><p>Intro text</p><h2>Section Title</h2><p>Body text</p></section>",
|
||||
metadata: { source: "first.html" },
|
||||
}),
|
||||
new Document({
|
||||
pageContent:
|
||||
"<section><p>Lead text</p><h2>Second Title</h2><p>More text</p></section>",
|
||||
metadata: { source: "second.html" },
|
||||
}),
|
||||
]);
|
||||
|
||||
expect(documents).toHaveLength(4);
|
||||
expect(documents[0].metadata).toMatchObject({
|
||||
source: "first.html",
|
||||
partindex: 0,
|
||||
totalparts: 2,
|
||||
});
|
||||
expect(documents[1].metadata).toMatchObject({
|
||||
source: "first.html",
|
||||
partindex: 1,
|
||||
totalparts: 2,
|
||||
});
|
||||
expect(documents[2].metadata).toMatchObject({
|
||||
source: "second.html",
|
||||
partindex: 0,
|
||||
totalparts: 2,
|
||||
});
|
||||
expect(documents[3].metadata).toMatchObject({
|
||||
source: "second.html",
|
||||
partindex: 1,
|
||||
totalparts: 2,
|
||||
});
|
||||
});
|
||||
|
||||
test("rejects chunkOverlap", () => {
|
||||
expect(
|
||||
() =>
|
||||
new HTMLTextSplitter({
|
||||
chunkSize: 32,
|
||||
chunkOverlap: 4,
|
||||
}),
|
||||
).toThrow("does not support chunkOverlap");
|
||||
});
|
||||
|
||||
test("keeps adjacent inline text contiguous when no whitespace exists", async () => {
|
||||
const splitter = new HTMLTextSplitter({
|
||||
chunkSize: 100,
|
||||
});
|
||||
|
||||
const chunks = await splitter.splitText(
|
||||
"<span>alpha</span><span>bet</span><span>gamma</span>",
|
||||
);
|
||||
|
||||
expect(chunks).toEqual(["alphabetgamma"]);
|
||||
});
|
||||
|
||||
test("preserves nested list indentation from formatForIngestion output", async () => {
|
||||
const splitter = new HTMLTextSplitter({
|
||||
chunkSize: 200,
|
||||
});
|
||||
|
||||
const html = [
|
||||
'<pre class="list-markdown">',
|
||||
"1. First",
|
||||
" A. Nested 1",
|
||||
" B. Nested 2",
|
||||
"2. Second",
|
||||
"</pre>",
|
||||
].join("\n");
|
||||
|
||||
const chunks = await splitter.splitText(html);
|
||||
|
||||
expect(chunks).toEqual([
|
||||
"1. First\n A. Nested 1\n B. Nested 2\n2. Second",
|
||||
]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,513 @@
|
||||
import { Document } from "@langchain/core/documents";
|
||||
import {
|
||||
RecursiveCharacterTextSplitter,
|
||||
TextSplitter,
|
||||
type TextSplitterChunkHeaderOptions,
|
||||
type TextSplitterParams,
|
||||
} from "@langchain/textsplitters";
|
||||
import * as cheerio from "cheerio";
|
||||
import type { AnyNode } from "domhandler";
|
||||
|
||||
export interface HTMLTextSplitterParams extends TextSplitterParams {
|
||||
/**
|
||||
* CSS selectors that act as preferred structural split points.
|
||||
* They are only used when a subtree needs to be broken down.
|
||||
*/
|
||||
separators?: string[];
|
||||
|
||||
/**
|
||||
* CSS selectors for nodes that should stay atomic unless maxChunkSize forces recursion.
|
||||
*/
|
||||
neverBreakWithin?: string[];
|
||||
|
||||
/**
|
||||
* Absolute hard limit for chunk text length.
|
||||
* Defaults to Infinity if omitted.
|
||||
*/
|
||||
maxChunkSize?: number;
|
||||
}
|
||||
|
||||
interface Segment {
|
||||
text: string;
|
||||
isProtected: boolean;
|
||||
sourceKind: "node" | "forced-split";
|
||||
}
|
||||
|
||||
const BLOCKISH_TAGS = new Set([
|
||||
"address",
|
||||
"article",
|
||||
"aside",
|
||||
"blockquote",
|
||||
"br",
|
||||
"dd",
|
||||
"div",
|
||||
"dl",
|
||||
"dt",
|
||||
"emu-clause",
|
||||
"emu-example",
|
||||
"emu-grammar",
|
||||
"emu-note",
|
||||
"emu-table",
|
||||
"fieldset",
|
||||
"figcaption",
|
||||
"figure",
|
||||
"footer",
|
||||
"form",
|
||||
"h1",
|
||||
"h2",
|
||||
"h3",
|
||||
"h4",
|
||||
"h5",
|
||||
"h6",
|
||||
"header",
|
||||
"hr",
|
||||
"li",
|
||||
"main",
|
||||
"nav",
|
||||
"ol",
|
||||
"p",
|
||||
"pre",
|
||||
"section",
|
||||
"table",
|
||||
"tbody",
|
||||
"td",
|
||||
"th",
|
||||
"thead",
|
||||
"tr",
|
||||
"ul",
|
||||
]);
|
||||
|
||||
/**
|
||||
* Splits HTML input into normalized text chunks.
|
||||
*
|
||||
* Important: this splitter does not preserve HTML markup in output.
|
||||
* It uses parsed HTML structure for traversal and sizing, but emits text only.
|
||||
*
|
||||
* @example
|
||||
* ```ts
|
||||
* const splitter = new HTMLTextSplitter({
|
||||
* chunkSize: 32,
|
||||
* separators: ["h2"],
|
||||
* neverBreakWithin: ["pre"],
|
||||
* });
|
||||
*
|
||||
* const chunks = await splitter.splitText(
|
||||
* "<section><h2>Title</h2><p>Body</p></section>",
|
||||
* );
|
||||
* // => ["Title Body"]
|
||||
* ```
|
||||
*/
|
||||
export class HTMLTextSplitter extends TextSplitter {
|
||||
separators: string[];
|
||||
neverBreakWithin: string[];
|
||||
maxChunkSize: number;
|
||||
|
||||
constructor(fields?: Partial<HTMLTextSplitterParams>) {
|
||||
if ((fields?.chunkOverlap ?? 0) !== 0) {
|
||||
throw new Error("HTMLTextSplitter does not support chunkOverlap.");
|
||||
}
|
||||
|
||||
super({
|
||||
...fields,
|
||||
chunkOverlap: 0,
|
||||
keepSeparator: false,
|
||||
lengthFunction: (text: string) => text.trim().length,
|
||||
});
|
||||
|
||||
this.separators = fields?.separators ?? [];
|
||||
this.neverBreakWithin = fields?.neverBreakWithin ?? [];
|
||||
this.maxChunkSize = fields?.maxChunkSize ?? Number.POSITIVE_INFINITY;
|
||||
}
|
||||
|
||||
/**
|
||||
* Splits HTML input into normalized text chunks.
|
||||
*
|
||||
* Important: this method emits text only. HTML markup is used for traversal
|
||||
* and sizing, but is not preserved in returned chunks.
|
||||
*/
|
||||
async splitText(text: string): Promise<string[]> {
|
||||
const $ = cheerio.load(text);
|
||||
const rootNodes =
|
||||
$("body").length > 0
|
||||
? $("body").contents().toArray()
|
||||
: $.root().contents().toArray();
|
||||
const roots = this.getMeaningfulNodes(rootNodes);
|
||||
|
||||
if (roots.length === 0) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const rootGroups = this.groupChildrenForDecomposition($, roots, false);
|
||||
const segments: Segment[] = [];
|
||||
|
||||
for (const group of rootGroups) {
|
||||
segments.push(...(await this.collectSegmentsFromNodes($, group, false)));
|
||||
}
|
||||
|
||||
return this.mergeSegments(segments);
|
||||
}
|
||||
|
||||
/**
|
||||
* Splits HTML documents into text-only child documents.
|
||||
*
|
||||
* Important: child document `pageContent` contains normalized text, not HTML.
|
||||
* Parent metadata is preserved and per-document part metadata is added.
|
||||
*/
|
||||
async splitDocuments(
|
||||
documents: Document[],
|
||||
chunkHeaderOptions?: TextSplitterChunkHeaderOptions,
|
||||
): Promise<Document[]> {
|
||||
const splitDocs: Document[] = [];
|
||||
const chunkHeader = chunkHeaderOptions?.chunkHeader ?? "";
|
||||
|
||||
for (const document of documents) {
|
||||
const chunks = await this.splitText(document.pageContent);
|
||||
|
||||
for (const [index, chunk] of chunks.entries()) {
|
||||
splitDocs.push(
|
||||
new Document({
|
||||
pageContent: `${chunkHeader}${chunk}`,
|
||||
metadata: {
|
||||
...document.metadata,
|
||||
partindex: index,
|
||||
totalparts: chunks.length,
|
||||
},
|
||||
}),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
return splitDocs;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns true when a tag node matches any configured CSS selector.
|
||||
*
|
||||
* Needed so separator and protected-node checks share the same selector logic
|
||||
* and non-tag nodes are ignored safely.
|
||||
*/
|
||||
private matchesAny(
|
||||
$: cheerio.CheerioAPI,
|
||||
node: AnyNode,
|
||||
selectors: string[],
|
||||
): boolean {
|
||||
if (node.type !== "tag") {
|
||||
return false;
|
||||
}
|
||||
|
||||
for (const selector of selectors) {
|
||||
if ($(node).is(selector)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Extracts the canonical text value for a node.
|
||||
*
|
||||
* Needed so sizing and emitted output use the same trimmed text semantics for
|
||||
* both element nodes and raw text nodes.
|
||||
*/
|
||||
private getNodeText($: cheerio.CheerioAPI, node: AnyNode): string {
|
||||
if (node.type === "text") {
|
||||
return node.data ?? "";
|
||||
}
|
||||
|
||||
return $(node).text();
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns whether a node should introduce a visible boundary when adjacent
|
||||
* text is concatenated.
|
||||
*
|
||||
* Needed so inline elements such as `span` do not get synthetic spaces while
|
||||
* block-ish elements still remain readable in emitted text.
|
||||
*/
|
||||
private isBlockishNode(node: AnyNode): boolean {
|
||||
return node.type === "tag" && BLOCKISH_TAGS.has(node.name);
|
||||
}
|
||||
|
||||
/**
|
||||
* Joins multiple nodes into the normalized text the splitter actually emits.
|
||||
*
|
||||
* Needed so grouped nodes are measured and emitted with the same spacing rules
|
||||
* used later during chunk merging.
|
||||
*/
|
||||
private getNodesText($: cheerio.CheerioAPI, nodes: AnyNode[]): string {
|
||||
let result = "";
|
||||
let previousNode: AnyNode | null = null;
|
||||
|
||||
for (const node of nodes) {
|
||||
const text = this.getNodeText($, node);
|
||||
if (!text) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (
|
||||
result.length > 0 &&
|
||||
previousNode &&
|
||||
(this.isBlockishNode(previousNode) || this.isBlockishNode(node))
|
||||
) {
|
||||
result += "\n"; // Add newline between block elements for better structure
|
||||
}
|
||||
|
||||
result += text;
|
||||
previousNode = node;
|
||||
}
|
||||
|
||||
return result.trim();
|
||||
}
|
||||
|
||||
/**
|
||||
* Filters child nodes down to the recursion units the splitter can handle.
|
||||
*
|
||||
* Needed to recurse through DOM structure without carrying empty whitespace-only
|
||||
* text nodes or unsupported node types through the algorithm.
|
||||
*/
|
||||
private getMeaningfulNodes(nodes: AnyNode[]): AnyNode[] {
|
||||
return nodes.filter((node) => {
|
||||
if (node.type === "text") {
|
||||
return (node.data ?? "").trim().length > 0;
|
||||
}
|
||||
|
||||
return node.type === "tag";
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the child nodes that are meaningful recursion units.
|
||||
*
|
||||
* Needed to recurse through DOM structure without carrying empty whitespace-only
|
||||
* text nodes or unsupported node types through the algorithm.
|
||||
*/
|
||||
private getChildNodesForRecursion(node: AnyNode): AnyNode[] {
|
||||
if (!("children" in node) || !Array.isArray(node.children)) {
|
||||
return [];
|
||||
}
|
||||
|
||||
return this.getMeaningfulNodes(node.children);
|
||||
}
|
||||
|
||||
/**
|
||||
* Groups child nodes so separator nodes start a new group.
|
||||
*
|
||||
* Needed because separators represent preferred section boundaries. Grouping
|
||||
* makes "separator plus following content" explicit during recursion.
|
||||
*/
|
||||
private groupChildrenForDecomposition(
|
||||
$: cheerio.CheerioAPI,
|
||||
childNodes: AnyNode[],
|
||||
inProtectedTree: boolean,
|
||||
): AnyNode[][] {
|
||||
if (childNodes.length === 0) {
|
||||
return [];
|
||||
}
|
||||
|
||||
if (inProtectedTree || this.separators.length === 0) {
|
||||
return [childNodes];
|
||||
}
|
||||
|
||||
const groups: AnyNode[][] = [];
|
||||
let currentGroup: AnyNode[] = [];
|
||||
|
||||
const flush = () => {
|
||||
if (currentGroup.length > 0) {
|
||||
groups.push(currentGroup);
|
||||
currentGroup = [];
|
||||
}
|
||||
};
|
||||
|
||||
for (const child of childNodes) {
|
||||
if (this.matchesAny($, child, this.separators)) {
|
||||
flush();
|
||||
}
|
||||
|
||||
currentGroup.push(child);
|
||||
}
|
||||
|
||||
flush();
|
||||
|
||||
return groups;
|
||||
}
|
||||
|
||||
/**
|
||||
* Collects text segments for a single node or grouped sibling nodes.
|
||||
*
|
||||
* Needed so separator-led groups can be treated as one structural unit before
|
||||
* the splitter decides whether it must recurse deeper.
|
||||
*/
|
||||
private async collectSegmentsFromNodes(
|
||||
$: cheerio.CheerioAPI,
|
||||
nodes: AnyNode[],
|
||||
inProtectedAncestor: boolean,
|
||||
): Promise<Segment[]> {
|
||||
const text = this.getNodesText($, nodes);
|
||||
|
||||
if (!text) {
|
||||
return [];
|
||||
}
|
||||
|
||||
if (nodes.length > 1) {
|
||||
const startsWithSeparator = this.matchesAny($, nodes[0], this.separators);
|
||||
|
||||
if (
|
||||
text.length <= this.chunkSize ||
|
||||
(startsWithSeparator && text.length <= this.maxChunkSize)
|
||||
) {
|
||||
return [{ text, isProtected: false, sourceKind: "node" }];
|
||||
}
|
||||
|
||||
const segments: Segment[] = [];
|
||||
for (const node of nodes) {
|
||||
segments.push(
|
||||
...(await this.collectSegmentsFromNodes(
|
||||
$,
|
||||
[node],
|
||||
inProtectedAncestor,
|
||||
)),
|
||||
);
|
||||
}
|
||||
|
||||
return segments.length > 0
|
||||
? segments
|
||||
: await this.forceSplitText(text, false);
|
||||
}
|
||||
|
||||
const [node] = nodes;
|
||||
const isProtected =
|
||||
!inProtectedAncestor && this.matchesAny($, node, this.neverBreakWithin);
|
||||
const childNodes = this.getChildNodesForRecursion(node);
|
||||
|
||||
if (childNodes.length > 0) {
|
||||
const childGroups = this.groupChildrenForDecomposition(
|
||||
$,
|
||||
childNodes,
|
||||
isProtected || inProtectedAncestor,
|
||||
);
|
||||
const normalizedChildText = this.getNodesText($, childGroups.flat());
|
||||
|
||||
if (normalizedChildText.length <= this.chunkSize) {
|
||||
return [{ text: normalizedChildText, isProtected, sourceKind: "node" }];
|
||||
}
|
||||
|
||||
if (isProtected && normalizedChildText.length <= this.maxChunkSize) {
|
||||
return [{ text: normalizedChildText, isProtected, sourceKind: "node" }];
|
||||
}
|
||||
|
||||
const segments: Segment[] = [];
|
||||
for (const group of childGroups) {
|
||||
segments.push(
|
||||
...(await this.collectSegmentsFromNodes(
|
||||
$,
|
||||
group,
|
||||
isProtected || inProtectedAncestor,
|
||||
)),
|
||||
);
|
||||
}
|
||||
|
||||
return segments.length > 0
|
||||
? segments
|
||||
: await this.forceSplitText(normalizedChildText, isProtected);
|
||||
}
|
||||
|
||||
if (text.length <= this.chunkSize) {
|
||||
return [{ text, isProtected, sourceKind: "node" }];
|
||||
}
|
||||
|
||||
if (isProtected && text.length <= this.maxChunkSize) {
|
||||
return [{ text, isProtected, sourceKind: "node" }];
|
||||
}
|
||||
|
||||
return this.forceSplitText(text, isProtected);
|
||||
}
|
||||
|
||||
/**
|
||||
* Applies the final plain-text fallback when a node cannot be decomposed further.
|
||||
*
|
||||
* Needed to enforce a finite hard cap with `RecursiveCharacterTextSplitter`
|
||||
* once DOM structure is exhausted.
|
||||
*/
|
||||
private async forceSplitText(
|
||||
text: string,
|
||||
isProtected: boolean,
|
||||
): Promise<Segment[]> {
|
||||
const normalizedText = text.trim();
|
||||
|
||||
if (!normalizedText) {
|
||||
return [];
|
||||
}
|
||||
|
||||
if (!Number.isFinite(this.maxChunkSize)) {
|
||||
return [
|
||||
{
|
||||
text: normalizedText,
|
||||
isProtected,
|
||||
sourceKind: "forced-split",
|
||||
},
|
||||
];
|
||||
}
|
||||
|
||||
const fallback = new RecursiveCharacterTextSplitter({
|
||||
chunkSize: this.maxChunkSize,
|
||||
chunkOverlap: 0,
|
||||
keepSeparator: false,
|
||||
lengthFunction: (value: string) => value.trim().length,
|
||||
});
|
||||
|
||||
const chunks = await fallback.splitText(normalizedText);
|
||||
|
||||
return chunks
|
||||
.map((chunk) => chunk.trim())
|
||||
.filter(Boolean)
|
||||
.map((chunk) => ({
|
||||
text: chunk,
|
||||
isProtected,
|
||||
sourceKind: "forced-split" as const,
|
||||
}));
|
||||
}
|
||||
|
||||
/**
|
||||
* Merges flat segments into final chunks using `chunkSize` as a soft target.
|
||||
*
|
||||
* Needed to keep chunk assembly separate from DOM traversal and to normalize the
|
||||
* final text output by joining segment text with single spaces.
|
||||
*/
|
||||
private mergeSegments(segments: Segment[]): string[] {
|
||||
const chunks: string[] = [];
|
||||
let currentParts: string[] = [];
|
||||
let currentLength = 0;
|
||||
|
||||
const flush = () => {
|
||||
if (currentParts.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
chunks.push(currentParts.join(" "));
|
||||
currentParts = [];
|
||||
currentLength = 0;
|
||||
};
|
||||
|
||||
for (const segment of segments) {
|
||||
const nextLength =
|
||||
currentLength === 0
|
||||
? segment.text.length
|
||||
: currentLength + 1 + segment.text.length;
|
||||
|
||||
if (currentParts.length > 0 && nextLength > this.chunkSize) {
|
||||
flush();
|
||||
}
|
||||
|
||||
currentParts.push(segment.text);
|
||||
currentLength =
|
||||
currentLength === 0
|
||||
? segment.text.length
|
||||
: currentLength + 1 + segment.text.length;
|
||||
}
|
||||
|
||||
flush();
|
||||
|
||||
return chunks;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
export * from "./HtmlTextSplitter";
|
||||
@@ -0,0 +1,388 @@
|
||||
import { describe, expect, test } from "bun:test";
|
||||
import * as cheerio from "cheerio";
|
||||
import {
|
||||
convertBlockCodeToMarkdown,
|
||||
convertGrammarToMarkdown,
|
||||
convertInlineCodeToMarkdown,
|
||||
convertLinksToMarkdown,
|
||||
convertListsToMarkdown,
|
||||
convertTablesToMarkdown,
|
||||
DEFAULT_CONFIG,
|
||||
formatForIngestion,
|
||||
} from "./formatHTMLForIngestion";
|
||||
|
||||
describe("formatHTMLForIngestion", () => {
|
||||
describe("convertLinksToMarkdown", () => {
|
||||
test("converts emu-xref links to markdown and strips filenames", () => {
|
||||
const html = `<emu-xref href="abstract-operations.html#sec-tonumber"><a href="abstract-operations.html#sec-tonumber">ToNumber</a></emu-xref>`;
|
||||
const $ = cheerio.load(html);
|
||||
convertLinksToMarkdown($, DEFAULT_CONFIG.links.join(", "));
|
||||
expect($.html()).toContain(
|
||||
'<span class="link-markdown">[ToNumber](#sec-tonumber)</span>',
|
||||
);
|
||||
});
|
||||
|
||||
test("skips external links", () => {
|
||||
const html = `<emu-xref href="https://example.com"><a href="https://example.com">External</a></emu-xref>`;
|
||||
const $ = cheerio.load(html);
|
||||
convertLinksToMarkdown($, DEFAULT_CONFIG.links.join(", "));
|
||||
expect($.html()).toContain('<a href="https://example.com">External</a>');
|
||||
});
|
||||
});
|
||||
|
||||
describe("convertInlineCodeToMarkdown", () => {
|
||||
test("converts var, emu-val, emu-const to inline code", () => {
|
||||
const html = `<div><var>x</var> <emu-val>y</emu-val> <emu-const>z</emu-const> <code>w</code></div>`;
|
||||
const $ = cheerio.load(html);
|
||||
convertInlineCodeToMarkdown($, DEFAULT_CONFIG.codeBlocks.inline);
|
||||
expect($.html()).toContain('<span class="inline-code">`x`</span>');
|
||||
expect($.html()).toContain('<span class="inline-code">`y`</span>');
|
||||
expect($.html()).toContain('<span class="inline-code">`z`</span>');
|
||||
expect($.html()).toContain('<span class="inline-code">`w`</span>');
|
||||
});
|
||||
|
||||
test("skips code inside pre", () => {
|
||||
const html = `<pre><code>x</code></pre>`;
|
||||
const $ = cheerio.load(html);
|
||||
convertInlineCodeToMarkdown($, DEFAULT_CONFIG.codeBlocks.inline);
|
||||
expect($.html()).toContain("<pre><code>x</code></pre>");
|
||||
});
|
||||
});
|
||||
|
||||
describe("convertBlockCodeToMarkdown", () => {
|
||||
test("converts pre>code and emu-eqn", () => {
|
||||
const html = `<div>
|
||||
<pre><code class="javascript hljs">const x = 1;</code></pre>
|
||||
<emu-eqn>y = x + 1</emu-eqn>
|
||||
<emu-eqn class="inline">z = 2</emu-eqn>
|
||||
</div>`;
|
||||
const $ = cheerio.load(html);
|
||||
convertBlockCodeToMarkdown($, DEFAULT_CONFIG.codeBlocks.block);
|
||||
expect($.html()).toContain(
|
||||
'<pre class="code-markdown">```javascript\nconst x = 1;\n```</pre>',
|
||||
);
|
||||
expect($.html()).toContain(
|
||||
'<pre class="code-markdown">```\ny = x + 1\n```</pre>',
|
||||
);
|
||||
expect($.html()).toContain('<emu-eqn class="inline">z = 2</emu-eqn>');
|
||||
});
|
||||
});
|
||||
|
||||
describe("convertGrammarToMarkdown", () => {
|
||||
test("converts emu-grammar to fenced bnf", () => {
|
||||
const html = `<emu-grammar>Statement :: BlockStatement</emu-grammar>`;
|
||||
const $ = cheerio.load(html);
|
||||
convertGrammarToMarkdown($, DEFAULT_CONFIG.codeBlocks.grammar.join(", "));
|
||||
expect($.html()).toContain(
|
||||
'<pre class="code-markdown">```bnf\nStatement :: BlockStatement\n```</pre>',
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe("convertListsToMarkdown", () => {
|
||||
test("converts ul with nested ol", () => {
|
||||
const html = `
|
||||
<ul>
|
||||
<li>Item 1</li>
|
||||
<li>Item 2
|
||||
<ol>
|
||||
<li>Subitem 1</li>
|
||||
<li>Subitem 2</li>
|
||||
</ol>
|
||||
</li>
|
||||
</ul>
|
||||
`;
|
||||
const $ = cheerio.load(html);
|
||||
convertListsToMarkdown(
|
||||
$,
|
||||
DEFAULT_CONFIG.lists.ordered,
|
||||
DEFAULT_CONFIG.lists.unordered,
|
||||
);
|
||||
const lines = $("pre.list-markdown")
|
||||
.text()
|
||||
.split("\n")
|
||||
.filter((l) => l.length > 0);
|
||||
expect(lines).toEqual([
|
||||
"- Item 1",
|
||||
"- Item 2",
|
||||
" A. Subitem 1",
|
||||
" B. Subitem 2",
|
||||
]);
|
||||
});
|
||||
|
||||
test("handles 3-level deep nesting (ol > ul > ol)", () => {
|
||||
const html = `
|
||||
<ol>
|
||||
<li>First</li>
|
||||
<li>Second
|
||||
<ul>
|
||||
<li>Alpha
|
||||
<ol>
|
||||
<li>Deep 1</li>
|
||||
<li>Deep 2</li>
|
||||
</ol>
|
||||
</li>
|
||||
<li>Beta</li>
|
||||
</ul>
|
||||
</li>
|
||||
</ol>
|
||||
`;
|
||||
const $ = cheerio.load(html);
|
||||
convertListsToMarkdown(
|
||||
$,
|
||||
DEFAULT_CONFIG.lists.ordered,
|
||||
DEFAULT_CONFIG.lists.unordered,
|
||||
);
|
||||
const lines = $("pre.list-markdown")
|
||||
.text()
|
||||
.split("\n")
|
||||
.filter((l) => l.length > 0);
|
||||
expect(lines).toEqual([
|
||||
"1. First",
|
||||
"2. Second",
|
||||
" * Alpha",
|
||||
" 1. Deep 1",
|
||||
" 2. Deep 2",
|
||||
" * Beta",
|
||||
]);
|
||||
});
|
||||
|
||||
test("handles multiple sibling nested lists in one item", () => {
|
||||
const html = `
|
||||
<ul>
|
||||
<li>Item
|
||||
<ol>
|
||||
<li>Ordered sub</li>
|
||||
</ol>
|
||||
<ul>
|
||||
<li>Unordered sub</li>
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
`;
|
||||
const $ = cheerio.load(html);
|
||||
convertListsToMarkdown(
|
||||
$,
|
||||
DEFAULT_CONFIG.lists.ordered,
|
||||
DEFAULT_CONFIG.lists.unordered,
|
||||
);
|
||||
const lines = $("pre.list-markdown")
|
||||
.text()
|
||||
.split("\n")
|
||||
.filter((l) => l.length > 0);
|
||||
expect(lines).toEqual([
|
||||
"- Item",
|
||||
" A. Ordered sub",
|
||||
" * Unordered sub",
|
||||
]);
|
||||
});
|
||||
|
||||
test("handles consecutive top-level lists", () => {
|
||||
const html = `
|
||||
<ul>
|
||||
<li>UL item 1</li>
|
||||
<li>UL item 2</li>
|
||||
</ul>
|
||||
<ol>
|
||||
<li>OL item 1</li>
|
||||
<li>OL item 2</li>
|
||||
</ol>
|
||||
`;
|
||||
const $ = cheerio.load(html);
|
||||
convertListsToMarkdown(
|
||||
$,
|
||||
DEFAULT_CONFIG.lists.ordered,
|
||||
DEFAULT_CONFIG.lists.unordered,
|
||||
);
|
||||
const results = $("pre.list-markdown");
|
||||
expect(results.length).toBe(2);
|
||||
|
||||
const ulLines = results
|
||||
.eq(0)
|
||||
.text()
|
||||
.split("\n")
|
||||
.filter((l) => l.length > 0);
|
||||
expect(ulLines).toEqual(["- UL item 1", "- UL item 2"]);
|
||||
|
||||
const olLines = results
|
||||
.eq(1)
|
||||
.text()
|
||||
.split("\n")
|
||||
.filter((l) => l.length > 0);
|
||||
expect(olLines).toEqual(["1. OL item 1", "2. OL item 2"]);
|
||||
});
|
||||
|
||||
test("preserves inline code inside list items", () => {
|
||||
const html = `
|
||||
<ul>
|
||||
<li>Call <code>foo()</code></li>
|
||||
<li>Use <var>x</var></li>
|
||||
</ul>
|
||||
`;
|
||||
const $ = cheerio.load(html);
|
||||
convertInlineCodeToMarkdown($, DEFAULT_CONFIG.codeBlocks.inline);
|
||||
convertListsToMarkdown(
|
||||
$,
|
||||
DEFAULT_CONFIG.lists.ordered,
|
||||
DEFAULT_CONFIG.lists.unordered,
|
||||
);
|
||||
const lines = $("pre.list-markdown")
|
||||
.text()
|
||||
.split("\n")
|
||||
.filter((l) => l.length > 0);
|
||||
expect(lines).toEqual(["- Call `foo()`", "- Use `x`"]);
|
||||
});
|
||||
|
||||
test("handles ol nested inside ol", () => {
|
||||
const html = `
|
||||
<ol>
|
||||
<li>First
|
||||
<ol>
|
||||
<li>Nested 1</li>
|
||||
<li>Nested 2</li>
|
||||
</ol>
|
||||
</li>
|
||||
<li>Second</li>
|
||||
</ol>
|
||||
`;
|
||||
const $ = cheerio.load(html);
|
||||
convertListsToMarkdown(
|
||||
$,
|
||||
DEFAULT_CONFIG.lists.ordered,
|
||||
DEFAULT_CONFIG.lists.unordered,
|
||||
);
|
||||
const lines = $("pre.list-markdown")
|
||||
.text()
|
||||
.split("\n")
|
||||
.filter((l) => l.length > 0);
|
||||
expect(lines).toEqual([
|
||||
"1. First",
|
||||
" A. Nested 1",
|
||||
" B. Nested 2",
|
||||
"2. Second",
|
||||
]);
|
||||
});
|
||||
|
||||
test("handles empty list", () => {
|
||||
const html = `<ul></ul>`;
|
||||
const $ = cheerio.load(html);
|
||||
convertListsToMarkdown(
|
||||
$,
|
||||
DEFAULT_CONFIG.lists.ordered,
|
||||
DEFAULT_CONFIG.lists.unordered,
|
||||
);
|
||||
const result = $("pre.list-markdown").text().trim();
|
||||
expect(result).toBe("");
|
||||
});
|
||||
});
|
||||
|
||||
describe("convertTablesToMarkdown", () => {
|
||||
test("converts tables to markdown tables", () => {
|
||||
const html = `
|
||||
<table>
|
||||
<thead>
|
||||
<tr><th>Col 1</th><th>Col 2</th></tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<tr><td>Data 1</td><td>Data 2</td></tr>
|
||||
</tbody>
|
||||
</table>
|
||||
`;
|
||||
const $ = cheerio.load(html);
|
||||
convertTablesToMarkdown($);
|
||||
const result = $("pre.table-markdown").text();
|
||||
expect(result).toContain("| Col 1 | Col 2 |");
|
||||
expect(result).toContain("| --- | --- |");
|
||||
expect(result).toContain("| Data 1 | Data 2 |");
|
||||
});
|
||||
});
|
||||
|
||||
describe("formatForIngestion", () => {
|
||||
test("runs the full pipeline", () => {
|
||||
const html = `
|
||||
<div>
|
||||
<p>See <emu-xref href="#sec-example"><a href="#sec-example">Example</a></emu-xref> for <var>x</var></p>
|
||||
<pre><code class="javascript">let y = x;</code></pre>
|
||||
<ul>
|
||||
<li>One</li>
|
||||
<li>Two</li>
|
||||
</ul>
|
||||
</div>
|
||||
`;
|
||||
const $ = cheerio.load(html);
|
||||
formatForIngestion($);
|
||||
|
||||
expect($.html()).toContain(
|
||||
'<span class="link-markdown">[Example](#sec-example)</span>',
|
||||
);
|
||||
expect($.html()).toContain('<span class="inline-code">`x`</span>');
|
||||
expect($.html()).toContain(
|
||||
'<pre class="code-markdown">```javascript\nlet y = x;\n```</pre>',
|
||||
);
|
||||
expect($.html()).toContain(
|
||||
'<pre class="list-markdown">- One\n- Two\n</pre>',
|
||||
);
|
||||
});
|
||||
|
||||
test("preserves nested list indentation through the full pipeline", () => {
|
||||
const html = `
|
||||
<ol>
|
||||
<li>First</li>
|
||||
<li>Second
|
||||
<ul>
|
||||
<li>Alpha
|
||||
<ol>
|
||||
<li>Deep 1</li>
|
||||
<li>Deep 2</li>
|
||||
</ol>
|
||||
</li>
|
||||
<li>Beta</li>
|
||||
</ul>
|
||||
</li>
|
||||
</ol>
|
||||
`;
|
||||
const $ = cheerio.load(html);
|
||||
formatForIngestion($);
|
||||
|
||||
const lines = $("pre.list-markdown")
|
||||
.text()
|
||||
.split("\n")
|
||||
.filter((l) => l.length > 0);
|
||||
expect(lines).toEqual([
|
||||
"1. First",
|
||||
"2. Second",
|
||||
" * Alpha",
|
||||
" 1. Deep 1",
|
||||
" 2. Deep 2",
|
||||
" * Beta",
|
||||
]);
|
||||
});
|
||||
|
||||
test("preserves ol nested inside ol through the full pipeline", () => {
|
||||
const html = `
|
||||
<ol>
|
||||
<li>First
|
||||
<ol>
|
||||
<li>Nested 1</li>
|
||||
<li>Nested 2</li>
|
||||
</ol>
|
||||
</li>
|
||||
<li>Second</li>
|
||||
</ol>
|
||||
`;
|
||||
const $ = cheerio.load(html);
|
||||
formatForIngestion($);
|
||||
|
||||
const lines = $("pre.list-markdown")
|
||||
.text()
|
||||
.split("\n")
|
||||
.filter((l) => l.length > 0);
|
||||
expect(lines).toEqual([
|
||||
"1. First",
|
||||
" A. Nested 1",
|
||||
" B. Nested 2",
|
||||
"2. Second",
|
||||
]);
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,330 @@
|
||||
import type * as cheerio from "cheerio";
|
||||
|
||||
/**
|
||||
* List of block-level HTML elements that should have newlines added after them
|
||||
* to preserve document structure during text extraction.
|
||||
*/
|
||||
export const BLOCK_ELEMENTS = [
|
||||
"p",
|
||||
"div",
|
||||
"h1",
|
||||
"h2",
|
||||
"h3",
|
||||
"h4",
|
||||
"h5",
|
||||
"h6",
|
||||
"li",
|
||||
"td",
|
||||
"th",
|
||||
"pre",
|
||||
"blockquote",
|
||||
"section",
|
||||
"emu-clause",
|
||||
"emu-note",
|
||||
"emu-example",
|
||||
"emu-table",
|
||||
"figure",
|
||||
"figcaption",
|
||||
"ul",
|
||||
"ol",
|
||||
"dl",
|
||||
"dt",
|
||||
"dd",
|
||||
"emu-production",
|
||||
"emu-rhs",
|
||||
];
|
||||
|
||||
export interface FormatConfig {
|
||||
codeBlocks: {
|
||||
block: string[];
|
||||
inline: string[];
|
||||
grammar: string[];
|
||||
};
|
||||
links: string[];
|
||||
lists: {
|
||||
ordered: string[];
|
||||
unordered: string[];
|
||||
};
|
||||
tables: string[];
|
||||
}
|
||||
|
||||
export const DEFAULT_CONFIG: FormatConfig = {
|
||||
codeBlocks: {
|
||||
block: ["pre>code", "emu-eqn:not([class*='inline'])"],
|
||||
inline: ["var", "emu-val", "emu-const", "emu-eqn.inline", "code"],
|
||||
grammar: ["emu-grammar"],
|
||||
},
|
||||
links: ["emu-xref a"],
|
||||
lists: {
|
||||
ordered: ["ol"],
|
||||
unordered: ["ul"],
|
||||
},
|
||||
tables: ["table", "emu-table"],
|
||||
};
|
||||
|
||||
export function convertLinksToMarkdown(
|
||||
$: cheerio.CheerioAPI,
|
||||
selector: string = "emu-xref a[href]",
|
||||
): void {
|
||||
$(selector).each((_, elem) => {
|
||||
const $a = $(elem);
|
||||
const href = $a.attr("href") ?? "";
|
||||
|
||||
// Skip external links
|
||||
if (href.startsWith("http")) return;
|
||||
|
||||
// Strip filename prefix: "abstract-operations.html#sec-tonumber" → "#sec-tonumber"
|
||||
const hash = href.includes("#") ? `#${href.split("#")[1]}` : href;
|
||||
const text = $a.text().trim();
|
||||
|
||||
if (!text || !hash) return;
|
||||
|
||||
$a.replaceWith($(`<span class="link-markdown">[${text}](${hash})</span>`));
|
||||
});
|
||||
}
|
||||
|
||||
export function convertInlineCodeToMarkdown(
|
||||
$: cheerio.CheerioAPI,
|
||||
tags: string[],
|
||||
): void {
|
||||
for (const tag of tags) {
|
||||
$(tag).each((_, elem) => {
|
||||
// Skip <code> inside <pre> (handled by block converter)
|
||||
if (
|
||||
"name" in elem &&
|
||||
elem.name === "code" &&
|
||||
$(elem).parent("pre").length > 0
|
||||
)
|
||||
return;
|
||||
|
||||
const text = $(elem).text().trim();
|
||||
if (!text) return;
|
||||
$(elem).replaceWith($(`<span class="inline-code">\`${text}\`</span>`));
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
function extractLanguage(codeClass: string | undefined): string {
|
||||
// "javascript hljs" → "javascript", "python hljs" → "python"
|
||||
if (!codeClass) return "";
|
||||
const match = codeClass.match(/^(\w+)/);
|
||||
return match?.[1] ?? "";
|
||||
}
|
||||
|
||||
export function convertBlockCodeToMarkdown(
|
||||
$: cheerio.CheerioAPI,
|
||||
tags: string[],
|
||||
): void {
|
||||
const selectors = tags.join(", ");
|
||||
if (!selectors) return;
|
||||
|
||||
$(selectors).each((_, elem) => {
|
||||
const $elem = $(elem);
|
||||
const lang = extractLanguage($elem.attr("class"));
|
||||
const text = $elem.text().trim();
|
||||
$elem.replaceWith(
|
||||
$(`<pre class="code-markdown">\`\`\`${lang}\n${text}\n\`\`\`</pre>`),
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the markdown list item prefix based on list type and nesting depth.
|
||||
* Alternates markers to visually distinguish nesting levels:
|
||||
* - Ordered lists: `1.` at even depth, `A.` at odd depth
|
||||
* - Unordered lists: `-` at even depth, `*` at odd depth
|
||||
*
|
||||
* @param isOrdered - Whether the list is ordered (ol) or unordered (ul)
|
||||
* @param depth - Nesting depth (0 = top-level)
|
||||
* @param index - Zero-based item index within its list
|
||||
* @returns Prefix string like "1. ", "A. ", "- ", or "* "
|
||||
*/
|
||||
function getListItemPrefix(
|
||||
isOrdered: boolean,
|
||||
depth: number,
|
||||
index: number,
|
||||
): string {
|
||||
if (isOrdered) {
|
||||
return depth % 2 === 0
|
||||
? `${index + 1}. `
|
||||
: `${String.fromCharCode(65 + index)}. `;
|
||||
}
|
||||
return depth % 2 === 0 ? "- " : "* ";
|
||||
}
|
||||
|
||||
function listToMarkdown(
|
||||
$: cheerio.CheerioAPI,
|
||||
elem: any,
|
||||
depth: number = 0,
|
||||
): string {
|
||||
const $elem = $(elem);
|
||||
const isOrdered = elem.name === "ol";
|
||||
const indent = " ".repeat(depth);
|
||||
const lines: string[] = [];
|
||||
|
||||
$elem.children("li").each((i, li) => {
|
||||
const prefix = getListItemPrefix(isOrdered, depth, i);
|
||||
const $li = $(li);
|
||||
|
||||
const nestedLists: string[] = [];
|
||||
|
||||
// Recurse into nested lists first (depth-first)
|
||||
$li.children("ol, ul").each((_, nested) => {
|
||||
const nestedMarkdown = listToMarkdown($, nested, depth + 1);
|
||||
nestedLists.push(nestedMarkdown);
|
||||
$(nested).remove();
|
||||
});
|
||||
|
||||
// Now get full text
|
||||
const itemText = $li.text().trim().replace(/\s+/g, " ");
|
||||
lines.push(`${indent}${prefix}${itemText}`);
|
||||
|
||||
for (const nested of nestedLists) {
|
||||
lines.push(nested);
|
||||
}
|
||||
});
|
||||
|
||||
return lines.join("\n");
|
||||
}
|
||||
|
||||
export function convertListsToMarkdown(
|
||||
$: cheerio.CheerioAPI,
|
||||
ordered: string[] = ["ol"],
|
||||
unordered: string[] = ["ul"],
|
||||
): void {
|
||||
const selectors = [...ordered, ...unordered].join(", ");
|
||||
if (!selectors) return;
|
||||
|
||||
// Process from outermost — recursion handles depth-first nesting
|
||||
$(selectors).each((_, elem) => {
|
||||
// Skip if already processed (parent already handled this)
|
||||
if ($(elem).hasClass("list-markdown") || $(elem).hasClass("list-processed"))
|
||||
return;
|
||||
const markdown = listToMarkdown($, elem, 0);
|
||||
$(elem).replaceWith($(`<pre class="list-markdown">\n${markdown}\n</pre>`));
|
||||
});
|
||||
}
|
||||
|
||||
export function convertGrammarToMarkdown(
|
||||
$: cheerio.CheerioAPI,
|
||||
selector: string = "emu-grammar",
|
||||
): void {
|
||||
if (!selector) return;
|
||||
$(selector).each((_, grammar) => {
|
||||
const text = $(grammar).text().trim();
|
||||
$(grammar).replaceWith(
|
||||
$(`<pre class="code-markdown">\`\`\`bnf\n${text}\n\`\`\`</pre>`),
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Converts HTML tables to markdown table format.
|
||||
* This preserves table structure when extracting text from HTML.
|
||||
* Replaces the original table with a <pre> element containing the markdown.
|
||||
*
|
||||
* @param $ - Cheerio API instance
|
||||
*/
|
||||
export function convertTablesToMarkdown($: cheerio.CheerioAPI): void {
|
||||
$("table, emu-table").each((_, tableElem) => {
|
||||
const $table = $(tableElem);
|
||||
const rows: string[][] = [];
|
||||
|
||||
// Extract header rows
|
||||
$table.find("thead tr").each((_, rowElem) => {
|
||||
const row: string[] = [];
|
||||
$(rowElem)
|
||||
.find("th, td")
|
||||
.each((_, cellElem) => {
|
||||
row.push($(cellElem).text().trim().replace(/\|/g, "\\|"));
|
||||
});
|
||||
if (row.length > 0) rows.push(row);
|
||||
});
|
||||
|
||||
// Extract body rows
|
||||
$table.find("tbody tr, tr").each((_, rowElem) => {
|
||||
// Skip if already processed as header
|
||||
if ($(rowElem).parent("thead").length > 0) return;
|
||||
|
||||
const row: string[] = [];
|
||||
$(rowElem)
|
||||
.find("td, th")
|
||||
.each((_, cellElem) => {
|
||||
row.push($(cellElem).text().trim().replace(/\|/g, "\\|"));
|
||||
});
|
||||
if (row.length > 0) rows.push(row);
|
||||
});
|
||||
|
||||
if (rows.length === 0) return;
|
||||
|
||||
// Determine max columns
|
||||
const maxCols = Math.max(...rows.map((r) => r.length));
|
||||
|
||||
// Build markdown table
|
||||
const mdLines: string[] = [];
|
||||
|
||||
// Header row
|
||||
if (rows.length > 0) {
|
||||
const header = rows[0].concat(Array(maxCols - rows[0].length).fill(""));
|
||||
mdLines.push("| " + header.join(" | ") + " |");
|
||||
}
|
||||
|
||||
// Separator
|
||||
mdLines.push("|" + Array(maxCols).fill(" --- ").join("|") + "|");
|
||||
|
||||
// Data rows (skip header if we have more rows)
|
||||
const dataRows = rows.length > 1 ? rows.slice(1) : [];
|
||||
for (const row of dataRows) {
|
||||
const padded = row.concat(Array(maxCols - row.length).fill(""));
|
||||
mdLines.push("| " + padded.join(" | ") + " |");
|
||||
}
|
||||
|
||||
// Replace table with markdown
|
||||
const markdown = mdLines.join("\n");
|
||||
$table.replaceWith($(`<pre class="table-markdown">\n${markdown}\n</pre>`));
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Adds newlines after specified block elements to preserve document structure.
|
||||
* This helps text splitters maintain paragraph/section boundaries.
|
||||
*
|
||||
* @param $ - Cheerio API instance
|
||||
* @param elements - Array of element tag names to add newlines after
|
||||
*/
|
||||
export function addNewlinesAfterBlocks(
|
||||
$: cheerio.CheerioAPI,
|
||||
elements: string[] = BLOCK_ELEMENTS,
|
||||
): void {
|
||||
for (const tag of elements) {
|
||||
$.root()
|
||||
.find(tag)
|
||||
.each((_, el) => {
|
||||
$(el).append("\n");
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
export function formatForIngestion(
|
||||
$: cheerio.CheerioAPI,
|
||||
config: Partial<FormatConfig> = {},
|
||||
): void {
|
||||
const cfg = { ...DEFAULT_CONFIG, ...config };
|
||||
|
||||
// 1. Inject newlines globally (affects li, p, pre, emu-production, emu-rhs, etc.)
|
||||
addNewlinesAfterBlocks($);
|
||||
|
||||
// 2. Inline leaves (links first — must be before lists/code destroy DOM)
|
||||
convertLinksToMarkdown($, cfg.links.join(", "));
|
||||
|
||||
// 3. Inline code
|
||||
convertInlineCodeToMarkdown($, cfg.codeBlocks.inline);
|
||||
|
||||
// 4. Block leaves (fenced code)
|
||||
convertBlockCodeToMarkdown($, cfg.codeBlocks.block);
|
||||
|
||||
// 5. Structural parents (grammar, lists, tables)
|
||||
convertGrammarToMarkdown($, cfg.codeBlocks.grammar.join(", "));
|
||||
convertListsToMarkdown($, cfg.lists.ordered, cfg.lists.unordered);
|
||||
convertTablesToMarkdown($);
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
/**
|
||||
* Manual test script for spec_retriever agent tool
|
||||
* Usage: bun run src/test/manual/test-spec-retriever.ts "your search query"
|
||||
*/
|
||||
|
||||
import * as lancedbSdk from "@lancedb/lancedb";
|
||||
import { OllamaEmbeddings } from "@langchain/ollama";
|
||||
import { createSpecRetrieverTool } from "../../agent-tools";
|
||||
import { EMBEDDING_MODEL, STORAGE_DIR } from "../../constants";
|
||||
|
||||
async function main() {
|
||||
const query = process.argv[2] || "array.[[DefineOwnProperty]]";
|
||||
|
||||
console.log(`Testing spec_retriever with query: "${query}"`);
|
||||
console.log("Loading database and embeddings...\n");
|
||||
|
||||
const embeddings = new OllamaEmbeddings({
|
||||
model: EMBEDDING_MODEL,
|
||||
});
|
||||
|
||||
const db = await lancedbSdk.connect(STORAGE_DIR);
|
||||
const table = await db.openTable("spec_vectors");
|
||||
|
||||
console.log("Creating tool...\n");
|
||||
const specRetrieverTool = createSpecRetrieverTool(table, embeddings);
|
||||
|
||||
console.log("Executing tool...\n");
|
||||
const result = await specRetrieverTool.func({ query });
|
||||
|
||||
console.log("=== TOOL OUTPUT ===");
|
||||
console.log(result);
|
||||
console.log("\n=== END OUTPUT ===");
|
||||
}
|
||||
|
||||
main().catch(console.error);
|
||||
@@ -0,0 +1,452 @@
|
||||
#!/usr/bin/env bun
|
||||
/**
|
||||
* Manual verification script for inspecting LanceDB documents/chunks.
|
||||
*
|
||||
* Usage:
|
||||
* bun run src/test/manual/verify-db.ts # Show summary (default command)
|
||||
* bun run src/test/manual/verify-db.ts summary # Show database summary
|
||||
* bun run src/test/manual/verify-db.ts list # List all sections
|
||||
* bun run src/test/manual/verify-db.ts section <id> # Show chunks for a section
|
||||
* bun run src/test/manual/verify-db.ts search "query" # Search by vector similarity (semantic search)
|
||||
* bun run src/test/manual/verify-db.ts tree # Show full section hierarchy
|
||||
* bun run src/test/manual/verify-db.ts tree sec-ecmascript-language-source-code # Show subtree from section
|
||||
* bun run src/test/manual/verify-db.ts sample # Show random samples
|
||||
* bun run src/test/manual/verify-db.ts large # Show large documents
|
||||
* bun run src/test/manual/verify-db.ts --help # Show help
|
||||
* bun run src/test/manual/verify-db.ts <command> --help # Show help for specific command
|
||||
*/
|
||||
|
||||
import fs from "node:fs";
|
||||
import type { Table } from "@lancedb/lancedb";
|
||||
import * as lancedbSdk from "@lancedb/lancedb";
|
||||
import { OllamaEmbeddings } from "@langchain/ollama";
|
||||
import { Command } from "commander";
|
||||
import { EMBEDDING_MODEL, STORAGE_DIR } from "../../constants";
|
||||
|
||||
const embeddings = new OllamaEmbeddings({
|
||||
model: EMBEDDING_MODEL,
|
||||
});
|
||||
|
||||
interface DocumentRecord {
|
||||
vector: number[];
|
||||
text: string;
|
||||
source: string;
|
||||
sectionid: string;
|
||||
sectiontitle: string;
|
||||
type: string;
|
||||
parentsectionid: string | null;
|
||||
childrensectionids: string[];
|
||||
partindex: number;
|
||||
totalparts: number;
|
||||
}
|
||||
|
||||
const program = new Command()
|
||||
.name("verify-db")
|
||||
.description(
|
||||
"Manual verification script for inspecting LanceDB documents/chunks",
|
||||
)
|
||||
.version("1.0.0");
|
||||
|
||||
async function getTable(): Promise<Table> {
|
||||
// Check if database exists
|
||||
if (!fs.existsSync(STORAGE_DIR)) {
|
||||
console.error(`❌ Storage directory not found: ${STORAGE_DIR}`);
|
||||
console.log("\nRun 'bun run ingest' first to create the database.");
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const db = await lancedbSdk.connect(STORAGE_DIR);
|
||||
|
||||
// Check if table exists
|
||||
try {
|
||||
return await db.openTable("spec_vectors");
|
||||
} catch (error) {
|
||||
console.error("❌ Table 'spec_vectors' not found in database");
|
||||
console.log("\nRun 'bun run ingest' first to populate the database.");
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
program
|
||||
.command("summary")
|
||||
.description("Show database summary statistics")
|
||||
.action(async () => {
|
||||
const table = await getTable();
|
||||
await showSummary(table);
|
||||
});
|
||||
|
||||
program
|
||||
.command("list")
|
||||
.description("List all sections with chunk counts")
|
||||
.action(async () => {
|
||||
const table = await getTable();
|
||||
await listSections(table);
|
||||
});
|
||||
|
||||
program
|
||||
.command("section <id>")
|
||||
.description("Show all chunks for a specific section")
|
||||
.action(async (sectionId: string) => {
|
||||
const table = await getTable();
|
||||
await showSection(table, sectionId);
|
||||
});
|
||||
|
||||
program
|
||||
.command("search <query>")
|
||||
.description("Search by vector similarity (semantic search)")
|
||||
.option("-l, --limit <number>", "Number of results to show", "5")
|
||||
.action(async (query: string, options: { limit: string }) => {
|
||||
const table = await getTable();
|
||||
await vectorSearch(table, query, parseInt(options.limit));
|
||||
});
|
||||
|
||||
program
|
||||
.command("tree [section]")
|
||||
.description("Show section hierarchy tree")
|
||||
.action(async (section: string | undefined) => {
|
||||
const table = await getTable();
|
||||
await showTree(table, section);
|
||||
});
|
||||
|
||||
program
|
||||
.command("sample")
|
||||
.description("Show random samples")
|
||||
.option("-n, --count <number>", "Number of samples to show", "3")
|
||||
.action(async (options: { count: string }) => {
|
||||
const table = await getTable();
|
||||
await showSamples(table, parseInt(options.count));
|
||||
});
|
||||
|
||||
program
|
||||
.command("large")
|
||||
.description("Show documents larger than minimum size")
|
||||
.option("-m, --min <number>", "Minimum size in characters", "5000")
|
||||
.action(async (options: { min: string }) => {
|
||||
const table = await getTable();
|
||||
await showLargeDocs(table, parseInt(options.min));
|
||||
});
|
||||
|
||||
async function showSummary(table: Table) {
|
||||
console.log("\n📊 Database Summary:");
|
||||
|
||||
// Get total count efficiently using countRows()
|
||||
const totalCount = await table.countRows();
|
||||
console.log(` Total documents: ${totalCount}`);
|
||||
|
||||
// Count unique sections
|
||||
const allRecords = (await table.query().toArray()) as DocumentRecord[];
|
||||
const sections = new Map<string, number>();
|
||||
const sizes = allRecords.map((r) => r.text.length);
|
||||
|
||||
for (const record of allRecords) {
|
||||
sections.set(record.sectionid, (sections.get(record.sectionid) || 0) + 1);
|
||||
}
|
||||
|
||||
console.log(` Unique sections: ${sections.size}`);
|
||||
console.log(
|
||||
` Multi-chunk sections: ${Array.from(sections.values()).filter((c) => c > 1).length}`,
|
||||
);
|
||||
console.log(`\n Size distribution:`);
|
||||
if (sizes.length === 0) {
|
||||
console.log(" No documents to calculate size distribution");
|
||||
} else {
|
||||
console.log(
|
||||
` Average: ${(sizes.reduce((a, b) => a + b, 0) / sizes.length).toFixed(0)} chars`,
|
||||
);
|
||||
console.log(` Min: ${Math.min(...sizes)} chars`);
|
||||
console.log(` Max: ${Math.max(...sizes)} chars`);
|
||||
}
|
||||
|
||||
// Show top 5 largest documents (by total section size, not individual chunks)
|
||||
const sectionSizes = new Map<
|
||||
string,
|
||||
{ sectionid: string; title: string; totalSize: number; chunks: number }
|
||||
>();
|
||||
for (const r of allRecords) {
|
||||
const existing = sectionSizes.get(r.sectionid);
|
||||
if (existing) {
|
||||
existing.totalSize += r.text.length;
|
||||
existing.chunks += 1;
|
||||
} else {
|
||||
sectionSizes.set(r.sectionid, {
|
||||
sectionid: r.sectionid,
|
||||
title: r.sectiontitle,
|
||||
totalSize: r.text.length,
|
||||
chunks: 1,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
const sortedSections = Array.from(sectionSizes.values())
|
||||
.sort((a, b) => b.totalSize - a.totalSize)
|
||||
.slice(0, 5);
|
||||
|
||||
console.log(`\n Top 5 largest documents (by total size):`);
|
||||
for (let i = 0; i < sortedSections.length; i++) {
|
||||
const s = sortedSections[i];
|
||||
const chunkInfo = s.chunks > 1 ? ` (${s.chunks} parts)` : "";
|
||||
console.log(
|
||||
` ${i + 1}. ${s.sectionid}${chunkInfo}: ${s.totalSize.toLocaleString()} chars`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async function listSections(table: Table) {
|
||||
// Query all records to properly count chunks per section
|
||||
const records = (await table
|
||||
.query()
|
||||
.select(["sectionid", "sectiontitle"])
|
||||
.toArray()) as DocumentRecord[];
|
||||
|
||||
const sectionMap = new Map<string, { title: string; chunks: number }>();
|
||||
|
||||
for (const record of records) {
|
||||
const existing = sectionMap.get(record.sectionid);
|
||||
if (existing) {
|
||||
// Increment chunk count for this section
|
||||
existing.chunks++;
|
||||
} else {
|
||||
sectionMap.set(record.sectionid, {
|
||||
title: record.sectiontitle,
|
||||
chunks: 1,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
const sorted = Array.from(sectionMap.entries()).sort(
|
||||
(a, b) => b[1].chunks - a[1].chunks,
|
||||
);
|
||||
|
||||
console.log(`\n📑 All Sections (${sorted.length} total):\n`);
|
||||
console.log("ID | Title | Chunks");
|
||||
console.log("-".repeat(80));
|
||||
|
||||
for (const [id, info] of sorted) {
|
||||
const title =
|
||||
info.title.length > 50 ? info.title.slice(0, 47) + "..." : info.title;
|
||||
console.log(
|
||||
`${id.padEnd(30)} | ${title.padEnd(50)} | ${info.chunks.toString().padStart(3)}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async function showSection(table: Table, sectionId: string) {
|
||||
// Query with where() for efficient database filtering
|
||||
const records = (await table
|
||||
.query()
|
||||
.where(`sectionid = '${sectionId}'`)
|
||||
.toArray()) as DocumentRecord[];
|
||||
|
||||
if (records.length === 0) {
|
||||
console.error(`❌ No records found for section: ${sectionId}`);
|
||||
console.error(
|
||||
'\nTip: Use "bun run test/manual/verify-db.ts list" to see all sections',
|
||||
);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
// Sort by part index
|
||||
const sectionRecords = records.sort((a, b) => a.partindex - b.partindex);
|
||||
const first = sectionRecords[0];
|
||||
|
||||
console.log(`\n📄 Section: ${sectionId}`);
|
||||
console.log(` Title: ${first.sectiontitle}`);
|
||||
console.log(` Source: ${first.source}`);
|
||||
console.log(` Parent: ${first.parentsectionid || "none"}`);
|
||||
|
||||
// Handle childrensectionids which comes back as an Apache Arrow Vector
|
||||
const childrenIds = getChildrenIds(first);
|
||||
const childrenStr = childrenIds.length > 0 ? childrenIds.join(", ") : "none";
|
||||
console.log(` Children: ${childrenStr}`);
|
||||
console.log(` Total parts: ${first.totalparts}`);
|
||||
console.log(
|
||||
` Total size: ${sectionRecords.reduce((sum, r) => sum + r.text.length, 0)} chars\n`,
|
||||
);
|
||||
|
||||
for (const record of sectionRecords) {
|
||||
console.log(`─`.repeat(80));
|
||||
if (record.totalparts > 1) {
|
||||
console.log(
|
||||
`Part ${record.partindex + 1}/${record.totalparts} (${record.text.length} chars):\n`,
|
||||
);
|
||||
} else {
|
||||
console.log(`Content (${record.text.length} chars):\n`);
|
||||
}
|
||||
console.log(record.text);
|
||||
console.log();
|
||||
}
|
||||
}
|
||||
|
||||
async function vectorSearch(table: Table, query: string, limit: number) {
|
||||
console.log(`\n🔍 Vector similarity search for "${query}"...`);
|
||||
console.log(" Generating embedding...");
|
||||
|
||||
const queryVector = await embeddings.embedQuery(query);
|
||||
|
||||
console.log(" Searching database...");
|
||||
// Use table.vectorSearch() which is the explicit/convenience method for vector search
|
||||
// Note: fastSearch() is available on Query but not VectorQuery, so we use standard search
|
||||
const results = (await table
|
||||
.vectorSearch(queryVector)
|
||||
.limit(limit)
|
||||
.toArray()) as DocumentRecord[];
|
||||
|
||||
console.log(`\n Top ${limit} most similar documents:\n`);
|
||||
|
||||
for (let i = 0; i < results.length; i++) {
|
||||
const r = results[i];
|
||||
const preview = r.text.replace(/\s+/g, " ").slice(0, 200);
|
||||
console.log(`${i + 1}. ${r.sectionid} (${r.sectiontitle})`);
|
||||
console.log(
|
||||
` Chunk ${r.partindex + 1}/${r.totalparts} (${r.text.length} chars)`,
|
||||
);
|
||||
console.log(` ${preview}${r.text.length > 200 ? "..." : ""}\n`);
|
||||
}
|
||||
}
|
||||
|
||||
async function showTree(table: Table, sectionId?: string) {
|
||||
// Query all records for hierarchy analysis
|
||||
const records = (await table
|
||||
.query()
|
||||
.select([
|
||||
"sectionid",
|
||||
"sectiontitle",
|
||||
"parentsectionid",
|
||||
"childrensectionids",
|
||||
])
|
||||
.toArray()) as DocumentRecord[];
|
||||
|
||||
// Build hierarchy map
|
||||
const sectionMap = new Map<string, DocumentRecord>();
|
||||
const rootSections: DocumentRecord[] = [];
|
||||
|
||||
for (const record of records) {
|
||||
if (!sectionMap.has(record.sectionid)) {
|
||||
sectionMap.set(record.sectionid, record);
|
||||
if (!record.parentsectionid) {
|
||||
rootSections.push(record);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// If a specific section is requested, show only that subtree
|
||||
if (sectionId) {
|
||||
const startSection = sectionMap.get(sectionId);
|
||||
if (!startSection) {
|
||||
console.error(`❌ Section not found: ${sectionId}`);
|
||||
console.error(
|
||||
'\nTip: Use "bun run test/manual/verify-db.ts list" to see all sections',
|
||||
);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
console.log(`\n🌳 Section Hierarchy for ${sectionId}:\n`);
|
||||
printTreeNode(startSection, sectionMap, "", true);
|
||||
return;
|
||||
}
|
||||
|
||||
console.log(`\n🌳 Section Hierarchy (${sectionMap.size} sections):\n`);
|
||||
|
||||
// Sort root sections
|
||||
rootSections.sort((a, b) => a.sectionid.localeCompare(b.sectionid));
|
||||
|
||||
for (let i = 0; i < rootSections.length; i++) {
|
||||
const isLast = i === rootSections.length - 1;
|
||||
printTreeNode(rootSections[i], sectionMap, "", isLast);
|
||||
}
|
||||
}
|
||||
|
||||
function printTreeNode(
|
||||
node: DocumentRecord,
|
||||
sectionMap: Map<string, DocumentRecord>,
|
||||
prefix: string,
|
||||
isLast: boolean,
|
||||
): void {
|
||||
const connector = isLast ? "└── " : "├── ";
|
||||
const title =
|
||||
node.sectiontitle.length > 50
|
||||
? node.sectiontitle.slice(0, 47) + "..."
|
||||
: node.sectiontitle;
|
||||
console.log(`${prefix}${connector}${node.sectionid}`);
|
||||
console.log(`${prefix}${isLast ? " " : "│ "} ${title}`);
|
||||
|
||||
// Get children
|
||||
const children: DocumentRecord[] = [];
|
||||
const childIds = getChildrenIds(node);
|
||||
for (const childId of childIds) {
|
||||
const child = sectionMap.get(childId);
|
||||
if (child) {
|
||||
children.push(child);
|
||||
}
|
||||
}
|
||||
|
||||
children.sort((a, b) => a.sectionid.localeCompare(b.sectionid));
|
||||
|
||||
const childPrefix = prefix + (isLast ? " " : "│ ");
|
||||
for (let i = 0; i < children.length; i++) {
|
||||
const isLastChild = i === children.length - 1;
|
||||
printTreeNode(children[i], sectionMap, childPrefix, isLastChild);
|
||||
}
|
||||
}
|
||||
|
||||
function getChildrenIds(node: DocumentRecord): string[] {
|
||||
// Array.from() works on both plain arrays and Apache Arrow Vectors
|
||||
// because Arrow Vectors implement [Symbol.iterator]
|
||||
return Array.from(node.childrensectionids as Iterable<string>);
|
||||
}
|
||||
|
||||
async function showSamples(table: Table, count: number) {
|
||||
// Query all records but limit fields
|
||||
const records = (await table
|
||||
.query()
|
||||
.select(["sectionid", "sectiontitle", "partindex", "totalparts", "text"])
|
||||
.toArray()) as DocumentRecord[];
|
||||
|
||||
const shuffled = [...records].sort(() => 0.5 - Math.random());
|
||||
const samples = shuffled.slice(0, count);
|
||||
|
||||
console.log(`\n🎲 ${count} Random Samples:\n`);
|
||||
|
||||
for (let i = 0; i < samples.length; i++) {
|
||||
const r = samples[i];
|
||||
console.log(`─`.repeat(80));
|
||||
console.log(`Sample ${i + 1}: ${r.sectionid}`);
|
||||
console.log(`Title: ${r.sectiontitle}`);
|
||||
console.log(
|
||||
`Chunk: ${r.partindex + 1}/${r.totalparts} (${r.text.length} chars)\n`,
|
||||
);
|
||||
console.log(r.text.slice(0, 400));
|
||||
if (r.text.length > 400) {
|
||||
console.log(`\n... (${r.text.length - 400} more characters)`);
|
||||
}
|
||||
console.log();
|
||||
}
|
||||
}
|
||||
|
||||
async function showLargeDocs(table: Table, minSize: number) {
|
||||
// Query with limit to text field only
|
||||
const records = (await table
|
||||
.query()
|
||||
.select(["sectionid", "partindex", "totalparts", "text"])
|
||||
.toArray()) as DocumentRecord[];
|
||||
|
||||
const large = records
|
||||
.filter((r) => r.text.length > minSize)
|
||||
.sort((a, b) => b.text.length - a.text.length);
|
||||
|
||||
console.log(
|
||||
`\n📏 Documents larger than ${minSize} chars (${large.length} found):\n`,
|
||||
);
|
||||
|
||||
for (const r of large) {
|
||||
console.log(
|
||||
`${r.sectionid} (chunk ${r.partindex + 1}/${r.totalparts}): ${r.text.length} chars`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
if (process.argv.length <= 2) {
|
||||
program.help();
|
||||
} else {
|
||||
program.parse();
|
||||
}
|
||||
Reference in New Issue
Block a user