mirror of
https://github.com/bendtherules/ask262.git
synced 2026-08-18 13:21:55 +00:00
Migrate to langchain
This commit is contained in:
@@ -1,25 +1,19 @@
|
||||
import fs from "node:fs";
|
||||
import { OllamaEmbedding } from "@llamaindex/ollama";
|
||||
import { OpenAI } from "@llamaindex/openai";
|
||||
import * as lancedbSdk from "@lancedb/lancedb";
|
||||
import { LanceDB } from "@langchain/community/vectorstores/lancedb";
|
||||
import { ChatPromptTemplate } from "@langchain/core/prompts";
|
||||
import { DynamicTool } from "@langchain/core/tools";
|
||||
import { OllamaEmbeddings } from "@langchain/ollama";
|
||||
import { ChatOpenAI } from "@langchain/openai";
|
||||
import Graph from "graphology";
|
||||
import {
|
||||
QueryEngineTool,
|
||||
ReActAgent,
|
||||
Settings,
|
||||
storageContextFromDefaults,
|
||||
VectorStoreIndex,
|
||||
} from "llamaindex";
|
||||
import { AgentExecutor, createReactAgent } from "langchain/agents";
|
||||
|
||||
import { GRAPH_FILE, STORAGE_DIR } from "./constants";
|
||||
|
||||
// Configure LlamaIndex to use local Ollama embeddings for semantic search
|
||||
// This enables the query engine to perform similarity searches without external APIs
|
||||
Settings.embedModel = new OllamaEmbedding({
|
||||
const embeddings = new OllamaEmbeddings({
|
||||
model: "nomic-embed-text-v2-moe",
|
||||
});
|
||||
|
||||
// Load API configuration from config.json
|
||||
// Expects NVIDIA_API_KEY and NVIDIA_API_BASE for accessing NVIDIA's API endpoint
|
||||
const config = JSON.parse(fs.readFileSync("./config.json", "utf-8"));
|
||||
const apiKey = config.NVIDIA_API_KEY;
|
||||
const baseURL = config.NVIDIA_API_BASE;
|
||||
@@ -28,81 +22,71 @@ if (!apiKey) {
|
||||
console.warn("Please set NVIDIA_API_KEY in config.json.");
|
||||
}
|
||||
|
||||
// Initialize the LLM using NVIDIA's OpenAI-compatible API endpoint
|
||||
// Model: openai/gpt-oss-120b with temperature 0 for deterministic responses
|
||||
const llm = new OpenAI({
|
||||
model: "openai/gpt-oss-120b",
|
||||
apiKey: apiKey,
|
||||
baseURL: baseURL,
|
||||
const llm = new ChatOpenAI({
|
||||
modelName: "openai/gpt-oss-120b",
|
||||
openAIApiKey: apiKey,
|
||||
configuration: { baseURL },
|
||||
temperature: 0,
|
||||
});
|
||||
Settings.llm = llm;
|
||||
|
||||
/**
|
||||
* Main function that initializes and runs the ECMAScript specification agent.
|
||||
*
|
||||
* The agent combines two information sources:
|
||||
* 1. Vector search index (spec_retriever) - for semantic text search across spec sections
|
||||
* 2. Graph knowledge base (graph_explorer) - for structural relationships between sections and code
|
||||
*/
|
||||
const systemPrompt = `You are an expert in the ECMAScript specification and its implementation in engine262.
|
||||
Your goal is to explain how specific parts of the language work by combining information from the provided tools.
|
||||
|
||||
CRITICAL INSTRUCTIONS:
|
||||
1. ALWAYS prefer using the provided tools ('spec_retriever', 'fetch_section_chunks', and 'graph_explorer') to answer questions.
|
||||
2. Do NOT rely on your internal knowledge of JavaScript or the ECMAScript specification.
|
||||
3. If the user asks about a function, you MUST first use 'graph_explorer' to find the associated specification section.
|
||||
4. You MUST then use 'fetch_section_chunks' to read the actual text of that specification section before answering.
|
||||
5. Base your explanations ONLY on the information retrieved from the tools.
|
||||
6. If the tools do not provide enough information, state that clearly rather than guessing from your internal knowledge.`;
|
||||
|
||||
const prompt = ChatPromptTemplate.fromMessages([
|
||||
["system", systemPrompt],
|
||||
["human", "{input}"],
|
||||
]);
|
||||
|
||||
async function main() {
|
||||
console.log("Loading indices and graph...");
|
||||
|
||||
// Load the vector index from disk containing embedded spec sections
|
||||
const storageContext = await storageContextFromDefaults({
|
||||
persistDir: STORAGE_DIR,
|
||||
});
|
||||
const db = await lancedbSdk.connect(STORAGE_DIR);
|
||||
const table = await db.openTable("spec_vectors");
|
||||
const vectorStore = new LanceDB(embeddings, { table });
|
||||
|
||||
const index = await VectorStoreIndex.init({
|
||||
storageContext,
|
||||
});
|
||||
|
||||
// Load the knowledge graph mapping spec sections to implementation functions
|
||||
const graphData = JSON.parse(fs.readFileSync(GRAPH_FILE, "utf-8"));
|
||||
const graph = new Graph({ multi: true });
|
||||
graph.import(graphData);
|
||||
|
||||
// Create a query engine with top-3 similarity results for text retrieval
|
||||
const queryEngine = index.asQueryEngine({ similarityTopK: 3 });
|
||||
|
||||
/**
|
||||
* Tool for retrieving specification text via vector similarity search.
|
||||
* Used to get detailed content of specific sections based on semantic queries.
|
||||
*/
|
||||
const queryEngineTool = new QueryEngineTool({
|
||||
queryEngine,
|
||||
metadata: {
|
||||
const specRetrieverTool = new DynamicTool({
|
||||
name: "spec_retriever",
|
||||
description:
|
||||
"Queries the language specification for text content about specific sections or topics. Use this to get the detailed text of a section.",
|
||||
func: async (query: string) => {
|
||||
const results = await vectorStore.similaritySearch(query, 3);
|
||||
return results.map((r) => r.pageContent).join("\n\n");
|
||||
},
|
||||
});
|
||||
|
||||
/**
|
||||
* Tool for exploring the knowledge graph connecting spec sections to implementation.
|
||||
* Enables structural navigation: finding which spec section a function implements
|
||||
* or which functions implement a spec section.
|
||||
*/
|
||||
const graphTool = {
|
||||
metadata: {
|
||||
const sectionRetrieverTool = new DynamicTool({
|
||||
name: "fetch_section_chunks",
|
||||
description:
|
||||
"Retrieves all text chunks from a specific specification section by sectionId. Use after finding a sectionId via spec_retriever.",
|
||||
func: async (sectionId: string) => {
|
||||
const results = await table
|
||||
.query()
|
||||
.where(`sectionid = '${sectionId}'`)
|
||||
.limit(100)
|
||||
.toArray();
|
||||
return results.map((r: { text: string }) => r.text).join("\n\n");
|
||||
},
|
||||
});
|
||||
|
||||
const graphTool = new DynamicTool({
|
||||
name: "graph_explorer",
|
||||
description:
|
||||
"Explores structural relationships between specification sections and implementation code (functions). Use this to find which spec section a function implements. Input: section ID or function name.",
|
||||
parameters: {
|
||||
type: "object",
|
||||
properties: {
|
||||
query: {
|
||||
type: "string",
|
||||
description: "The section ID or function name to explore.",
|
||||
},
|
||||
},
|
||||
required: ["query"],
|
||||
},
|
||||
},
|
||||
call: async ({ query }: { query: string }) => {
|
||||
func: async (query: string) => {
|
||||
console.log(`[Tool: graph_explorer] Querying for: ${query}`);
|
||||
|
||||
// Try exact node match, or prepend 'func-' prefix for function names
|
||||
let nodeId = query;
|
||||
if (!graph.hasNode(nodeId)) {
|
||||
if (graph.hasNode(`func-${query}`)) {
|
||||
@@ -111,7 +95,6 @@ async function main() {
|
||||
}
|
||||
|
||||
if (graph.hasNode(nodeId)) {
|
||||
// Collect node info and all connected nodes
|
||||
const neighbors = graph.neighbors(nodeId);
|
||||
const nodeAttr = graph.getNodeAttributes(nodeId);
|
||||
|
||||
@@ -120,7 +103,6 @@ async function main() {
|
||||
if (nodeAttr.file) result += `- File: ${nodeAttr.file}\n`;
|
||||
result += `\nConnected parts:\n`;
|
||||
|
||||
// List all connected nodes with their relationship types
|
||||
neighbors.forEach((neighbor) => {
|
||||
const attr = graph.getNodeAttributes(neighbor);
|
||||
const edges = graph.edges(nodeId, neighbor);
|
||||
@@ -133,48 +115,32 @@ async function main() {
|
||||
|
||||
return `No information found in graph for ${query}. Use spec_retriever to search text.`;
|
||||
},
|
||||
};
|
||||
});
|
||||
|
||||
/**
|
||||
* ReAct agent that reasons about ECMAScript specification.
|
||||
*
|
||||
* The agent follows this workflow:
|
||||
* 1. For function queries: graph_explorer → spec_retriever → explanation
|
||||
* 2. For section queries: spec_retriever → explanation
|
||||
*
|
||||
* Critical constraints ensure tool-based answers rather than internal knowledge.
|
||||
*/
|
||||
const agent = new ReActAgent({
|
||||
tools: [queryEngineTool, graphTool],
|
||||
llm: llm,
|
||||
verbose: true,
|
||||
systemPrompt: `You are an expert in the ECMAScript specification and its implementation in engine262.
|
||||
Your goal is to explain how specific parts of the language work by combining information from the provided tools.
|
||||
const agent = await createReactAgent({
|
||||
llm,
|
||||
tools: [specRetrieverTool, sectionRetrieverTool, graphTool],
|
||||
prompt,
|
||||
});
|
||||
|
||||
CRITICAL INSTRUCTIONS:
|
||||
1. ALWAYS prefer using the provided tools ('spec_retriever' and 'graph_explorer') to answer questions.
|
||||
2. Do NOT rely on your internal knowledge of JavaScript or the ECMAScript specification.
|
||||
3. If the user asks about a function, you MUST first use 'graph_explorer' to find the associated specification section.
|
||||
4. You MUST then use 'spec_retriever' to read the actual text of that specification section before answering.
|
||||
5. Base your explanations ONLY on the information retrieved from the tools.
|
||||
6. If the tools do not provide enough information, state that clearly rather than guessing from your internal knowledge.`,
|
||||
const agentExecutor = new AgentExecutor({
|
||||
agent,
|
||||
tools: [specRetrieverTool, sectionRetrieverTool, graphTool],
|
||||
});
|
||||
|
||||
console.log("Agent is ready!");
|
||||
|
||||
// Accept user query from command line argument, or use default question
|
||||
const message =
|
||||
process.argv[2] ||
|
||||
"Which spec section does Evaluate_IfStatement implement? and what does that section say?";
|
||||
console.log(`User: ${message}`);
|
||||
|
||||
// Execute the agent with the user's query
|
||||
const response = await agent.chat({
|
||||
message: message,
|
||||
const response = await agentExecutor.invoke({
|
||||
input: message,
|
||||
});
|
||||
|
||||
console.log("\n--- Agent Response ---\n");
|
||||
console.log(response.toString());
|
||||
console.log(response.output);
|
||||
}
|
||||
|
||||
main().catch(console.error);
|
||||
|
||||
+6
-6
@@ -19,16 +19,16 @@
|
||||
"description": "",
|
||||
"type": "module",
|
||||
"dependencies": {
|
||||
"@llamaindex/core": "^0.6.22",
|
||||
"@llamaindex/env": "^0.1.30",
|
||||
"@llamaindex/node-parser": "^2.0.22",
|
||||
"@llamaindex/ollama": "^0.1.23",
|
||||
"@llamaindex/openai": "^0.4.22",
|
||||
"@lancedb/lancedb": "^0.5.0",
|
||||
"@langchain/community": "0.2.0",
|
||||
"@langchain/core": "^0.2.0",
|
||||
"@langchain/ollama": "^0.1.0",
|
||||
"@langchain/openai": "^0.1.0",
|
||||
"acorn": "^8.16.0",
|
||||
"cheerio": "^1.2.0",
|
||||
"glob": "^13.0.6",
|
||||
"graphology": "^0.26.0",
|
||||
"llamaindex": "^0.12.1"
|
||||
"langchain": "^0.2.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"typescript": "^5.5.2",
|
||||
|
||||
+50
-161
@@ -1,37 +1,29 @@
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import readline from "node:readline";
|
||||
import { OllamaEmbedding } from "@llamaindex/ollama";
|
||||
import * as lancedbSdk from "@lancedb/lancedb";
|
||||
import { Index } from "@lancedb/lancedb";
|
||||
import { LanceDB } from "@langchain/community/vectorstores/lancedb";
|
||||
import { Document } from "@langchain/core/documents";
|
||||
import { OllamaEmbeddings } from "@langchain/ollama";
|
||||
import { RecursiveCharacterTextSplitter } from "@langchain/textsplitters";
|
||||
import * as cheerio from "cheerio";
|
||||
import { glob } from "glob";
|
||||
import {
|
||||
Document,
|
||||
SentenceSplitter,
|
||||
Settings,
|
||||
storageContextFromDefaults,
|
||||
VectorStoreIndex,
|
||||
} from "llamaindex";
|
||||
import { SPEC_DIR, STORAGE_DIR } from "../constants";
|
||||
|
||||
// Configure LlamaIndex to use local Ollama embeddings
|
||||
// This creates vector embeddings for semantic search without external APIs
|
||||
Settings.embedModel = new OllamaEmbedding({
|
||||
const embeddings = new OllamaEmbeddings({
|
||||
model: "nomic-embed-text-v2-moe",
|
||||
});
|
||||
|
||||
// Text chunking configuration for larger chunks with more context preservation
|
||||
// Larger chunks reduce total number of nodes while still fitting within
|
||||
// the embedding model's 8192 token limit (~2048 chars ≈ 512 tokens)
|
||||
const sentenceSplitter = new SentenceSplitter({
|
||||
chunkSize: 2048,
|
||||
chunkOverlap: 50,
|
||||
const textSplitter = new RecursiveCharacterTextSplitter({
|
||||
chunkSize: 4096,
|
||||
chunkOverlap: 100,
|
||||
separators: ["\n\n", "\n", ". ", " ", ""],
|
||||
});
|
||||
|
||||
/**
|
||||
* Prompts the user for confirmation via stdin.
|
||||
* @param question - The question to display to the user
|
||||
* @returns Promise that resolves to true if user confirms (yes/y), false otherwise
|
||||
*/
|
||||
const BREAKDOWN_TAGS = ["emu-table", "emu-grammar"] as const;
|
||||
const LARGE_DOC_THRESHOLD = 5000;
|
||||
|
||||
function askUser(question: string): Promise<boolean> {
|
||||
const rl = readline.createInterface({
|
||||
input: process.stdin,
|
||||
@@ -47,24 +39,7 @@ function askUser(question: string): Promise<boolean> {
|
||||
});
|
||||
}
|
||||
|
||||
// Tags to extract from large sections for finer-grained chunking
|
||||
// Extend this array to add more tag types for breakdown
|
||||
const BREAKDOWN_TAGS = ["emu-table", "emu-grammar"] as const;
|
||||
const LARGE_DOC_THRESHOLD = 5000;
|
||||
|
||||
/**
|
||||
* Extracts ECMAScript specification sections from HTML files and converts them
|
||||
* to Documents for vector indexing. Each section (emu-clause) becomes a separate
|
||||
* document with metadata for tracking.
|
||||
*
|
||||
* For large sections (> 5000 chars), attempts to break them down by extracting
|
||||
* content from specific structural tags (emu-table, emu-grammar, etc.) to create
|
||||
* more focused chunks. Falls back to the full section text if no breakdown tags
|
||||
* are found.
|
||||
*
|
||||
* @returns Array of Documents ready for indexing
|
||||
*/
|
||||
async function ingestSpec() {
|
||||
async function ingestSpec(): Promise<Document[]> {
|
||||
const htmlFiles = await glob(path.join(SPEC_DIR, "*.html"));
|
||||
const documents: Document[] = [];
|
||||
|
||||
@@ -75,7 +50,6 @@ async function ingestSpec() {
|
||||
$("emu-clause").each((_i, elem) => {
|
||||
const id = $(elem).attr("id");
|
||||
const title = $(elem).find("h1").first().text().trim();
|
||||
// Only extract immediate text to avoid excessive chunking of child sections
|
||||
const text = $(elem)
|
||||
.clone()
|
||||
.children("emu-clause")
|
||||
@@ -88,7 +62,7 @@ async function ingestSpec() {
|
||||
return;
|
||||
}
|
||||
|
||||
// For large documents, attempt to break down by structural tags
|
||||
// For large documents, break down by structural tags
|
||||
if (text.length > LARGE_DOC_THRESHOLD) {
|
||||
let subDocsCreated = false;
|
||||
const $section = $(elem).clone();
|
||||
@@ -105,14 +79,14 @@ async function ingestSpec() {
|
||||
if (subText) {
|
||||
documents.push(
|
||||
new Document({
|
||||
text: subText,
|
||||
pageContent: subText,
|
||||
metadata: {
|
||||
source: file,
|
||||
sectionId: subId,
|
||||
sectionTitle: `${title} [${tagName}]`,
|
||||
sectionid: subId,
|
||||
sectiontitle: `${title} [${tagName}]`,
|
||||
type: "specification",
|
||||
parentSectionId: id,
|
||||
breakdownTag: tagName,
|
||||
parentsectionid: id,
|
||||
breakdowntag: tagName,
|
||||
},
|
||||
}),
|
||||
);
|
||||
@@ -131,14 +105,14 @@ async function ingestSpec() {
|
||||
if (remainingText) {
|
||||
documents.push(
|
||||
new Document({
|
||||
text: remainingText,
|
||||
pageContent: remainingText,
|
||||
metadata: {
|
||||
source: file,
|
||||
sectionId: `${id}-prose-part-1`,
|
||||
sectionTitle: `${title} [prose]`,
|
||||
sectionid: `${id}-prose-part-1`,
|
||||
sectiontitle: `${title} [prose]`,
|
||||
type: "specification",
|
||||
parentSectionId: id,
|
||||
breakdownTag: "prose",
|
||||
parentsectionid: id,
|
||||
breakdowntag: "prose",
|
||||
},
|
||||
}),
|
||||
);
|
||||
@@ -148,18 +122,19 @@ async function ingestSpec() {
|
||||
if (subDocsCreated || remainingText) {
|
||||
return;
|
||||
}
|
||||
// Otherwise, fall through to add the full section document
|
||||
}
|
||||
|
||||
// Add the full section document (for smaller sections or when no breakdown happened)
|
||||
documents.push(
|
||||
new Document({
|
||||
text,
|
||||
pageContent: text,
|
||||
metadata: {
|
||||
source: file,
|
||||
sectionId: id,
|
||||
sectionTitle: title,
|
||||
sectionid: id,
|
||||
sectiontitle: title,
|
||||
type: "specification",
|
||||
parentsectionid: null,
|
||||
breakdowntag: null,
|
||||
},
|
||||
}),
|
||||
);
|
||||
@@ -168,99 +143,21 @@ async function ingestSpec() {
|
||||
return documents;
|
||||
}
|
||||
|
||||
/**
|
||||
* Main execution pipeline:
|
||||
* 1. Ingest specification HTML files and convert to documents
|
||||
* 2. Split documents into smaller text chunks (nodes)
|
||||
* 3. Filter out oversized chunks that could exceed LLM context limits
|
||||
* 4. Build a vector index in batches to handle large document sets
|
||||
* 5. Persist the index to disk for later retrieval
|
||||
*/
|
||||
async function main() {
|
||||
console.log("Ingesting specification...");
|
||||
const specDocs = await ingestSpec();
|
||||
console.log(`Ingested ${specDocs.length} specification sections.`);
|
||||
|
||||
console.log("Splitting documents into nodes...");
|
||||
console.log("Splitting documents into chunks...");
|
||||
const splitDocs = await textSplitter.splitDocuments(specDocs);
|
||||
console.log(`Total chunks generated: ${splitDocs.length}`);
|
||||
|
||||
// Debug: Log largest documents (over 2000 chars) to diagnose oversized nodes
|
||||
const largeDocs = specDocs
|
||||
.map((doc, i) => ({
|
||||
index: i,
|
||||
length: doc.text.length,
|
||||
sectionId: doc.metadata.sectionId,
|
||||
}))
|
||||
.filter((doc) => doc.length > 2000)
|
||||
.sort((a, b) => b.length - a.length)
|
||||
.slice(0, 50);
|
||||
const db = await lancedbSdk.connect(STORAGE_DIR);
|
||||
|
||||
if (largeDocs.length > 0) {
|
||||
console.log("\nDebug: Largest documents (> 2000 chars):");
|
||||
largeDocs.forEach((doc) => {
|
||||
console.log(
|
||||
` Doc ${doc.index}: ${doc.length} chars, section: ${doc.sectionId}`,
|
||||
);
|
||||
});
|
||||
const remaining =
|
||||
specDocs.filter((doc) => doc.text.length > 2000).length -
|
||||
largeDocs.length;
|
||||
if (remaining > 0) {
|
||||
console.log(` ... and ${remaining} more large documents`);
|
||||
}
|
||||
} else {
|
||||
console.log("\nDebug: No documents over 2000 chars found");
|
||||
}
|
||||
|
||||
const rawNodes = sentenceSplitter.getNodesFromDocuments(specDocs);
|
||||
console.log(`Total raw nodes generated: ${rawNodes.length}`);
|
||||
|
||||
// Debug: Log node size distribution
|
||||
const nodeSizes = rawNodes.map((n) => n.getContent().length);
|
||||
const maxNodeSize = Math.max(...nodeSizes);
|
||||
const avgNodeSize = nodeSizes.reduce((a, b) => a + b, 0) / nodeSizes.length;
|
||||
console.log(
|
||||
`\nDebug: Node size stats - Max: ${maxNodeSize}, Avg: ${Math.round(avgNodeSize)}`,
|
||||
);
|
||||
|
||||
// Safety filter to ensure no node exceeds context limit
|
||||
// Filter threshold set to chunkSize + buffer for metadata overhead
|
||||
const MAX_NODE_LENGTH = 2500;
|
||||
let skippedCount = 0;
|
||||
const nodes = rawNodes.filter((node) => {
|
||||
const contentLen = node.getContent().length;
|
||||
if (contentLen > MAX_NODE_LENGTH) {
|
||||
skippedCount++;
|
||||
if (skippedCount <= 3) {
|
||||
console.warn(
|
||||
`Skipping node with length ${contentLen} from ${node.metadata.source || "unknown"} (section: ${node.metadata.sectionId})`,
|
||||
);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
});
|
||||
|
||||
if (skippedCount > 3) {
|
||||
console.warn(` ... and ${skippedCount - 3} more nodes skipped`);
|
||||
}
|
||||
console.log(`Total valid nodes for indexing: ${nodes.length}`);
|
||||
|
||||
console.log("Creating storage context...");
|
||||
const storageContext = await storageContextFromDefaults({
|
||||
persistDir: STORAGE_DIR,
|
||||
});
|
||||
|
||||
console.log("Building index (this might take a while with local Ollama)...");
|
||||
|
||||
const BATCH_SIZE = 50;
|
||||
let index: VectorStoreIndex | null = null;
|
||||
|
||||
// Try to load existing index if any
|
||||
let table: lancedbSdk.Table;
|
||||
try {
|
||||
index = await VectorStoreIndex.init({
|
||||
storageContext,
|
||||
});
|
||||
console.log("Existing index found.");
|
||||
table = await db.openTable("spec_vectors");
|
||||
console.log("Existing table found.");
|
||||
const shouldOverwrite = await askUser(
|
||||
"Do you want to overwrite the existing vector store?",
|
||||
);
|
||||
@@ -268,29 +165,21 @@ async function main() {
|
||||
console.log("Ingest cancelled by user.");
|
||||
process.exit(0);
|
||||
}
|
||||
console.log("Overwriting existing index...");
|
||||
// Reset index to null so we create a fresh one
|
||||
index = null;
|
||||
} catch (_e) {
|
||||
console.log("No existing index found, starting fresh.");
|
||||
console.log("Overwriting existing table...");
|
||||
await db.dropTable("spec_vectors");
|
||||
table = await db.createTable("spec_vectors", []);
|
||||
} catch {
|
||||
console.log("No existing table found, creating fresh...");
|
||||
table = await db.createTable("spec_vectors", []);
|
||||
}
|
||||
|
||||
// Process nodes in batches to avoid overwhelming the embedding service
|
||||
for (let i = 0; i < nodes.length; i += BATCH_SIZE) {
|
||||
const batch = nodes.slice(i, i + BATCH_SIZE);
|
||||
console.log(
|
||||
`Processing batch ${i / BATCH_SIZE + 1} / ${Math.ceil(nodes.length / BATCH_SIZE)}...`,
|
||||
);
|
||||
console.log("Creating scalar indexes...");
|
||||
await table.createIndex("sectionid", { config: Index.btree() });
|
||||
await table.createIndex("type", { config: Index.btree() });
|
||||
|
||||
if (!index) {
|
||||
index = await VectorStoreIndex.init({
|
||||
storageContext,
|
||||
nodes: batch,
|
||||
});
|
||||
} else {
|
||||
await index.insertNodes(batch);
|
||||
}
|
||||
}
|
||||
console.log("Storing documents with embeddings...");
|
||||
const vectorStore = new LanceDB(embeddings, { table });
|
||||
await vectorStore.addDocuments(splitDocs);
|
||||
|
||||
console.log(`Index built and persisted to ${STORAGE_DIR}`);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user