diff --git a/.opencode/plans/archive/1775113983739-sunny-knight.md b/.opencode/plans/archive/1775113983739-sunny-knight.md new file mode 100644 index 0000000..2ff4b52 --- /dev/null +++ b/.opencode/plans/archive/1775113983739-sunny-knight.md @@ -0,0 +1,556 @@ +--- +model: openai/gpt-5.4 +--- + +# HTMLTextSplitter Implementation Plan + +## Overview + +Build a new `HTMLTextSplitter` that extends LangChain's `TextSplitter` and splits HTML by traversing the parsed DOM tree instead of slicing raw HTML strings. + +This version is intentionally narrow: +- size metric uses cheerio text extraction only +- separators are CSS selectors +- `neverBreakWithin` is respected unless `maxChunkSize` forces a deeper split +- no `chunkOverlap` support +- output is text chunks, not reconstructed partial HTML + +## Confirmed Decisions + +1. Use a traversal-based algorithm, not source offsets. +2. Only split when needed for size control. +3. Separator matches are soft structural hints, not unconditional split points. +4. If a separator subtree is too large, recurse into that subtree. +5. `neverBreakWithin` wins unless a node exceeds `maxChunkSize`; then recurse as a last resort. +6. Ignore separator matches inside protected subtrees during normal traversal. +7. V1 emits text chunks, not valid partial HTML. +8. Traverse both element nodes and text nodes. +9. Normalize output after boundary-aware joining so block-ish tags create readable spacing without adding synthetic spaces between inline tags. +10. When a direct child matches a separator during decomposition, it stays with the following content. +11. `partIndex` and `totalParts` are tracked per input document. +12. `chunkOverlap` is not supported. +13. Scope is the splitter itself, with ingest integration called out as a follow-up. + +## Goals + +1. Extend `TextSplitter` from `@langchain/textsplitters`. +2. Accept HTML input and parse with `cheerio`. +3. Use CSS selectors for `separators` and `neverBreakWithin`. +4. Use `chunkSize` as a soft target and `maxChunkSize` as a hard limit. +5. Produce chunks whose content is text derived from the traversed HTML. +6. Recurse only when needed; avoid unnecessary fragmentation. + +## Non-Goals + +1. Preserving exact HTML markup in chunk output. +2. Supporting `chunkOverlap`. +3. Building a full `setup/ingest.ts` migration in this plan. +4. Reproducing browser-accurate `innerText` behavior. + +## Public Interface + +```typescript +import { type Document } from "@langchain/core/documents"; +import { TextSplitter, type TextSplitterParams } from "@langchain/textsplitters"; + +export interface HTMLTextSplitterParams extends TextSplitterParams { + /** + * CSS selectors that act as preferred structural split points. + * They are only used when a subtree needs to be broken down. + */ + separators?: string[]; + + /** + * CSS selectors for nodes that should stay atomic unless maxChunkSize forces recursion. + */ + neverBreakWithin?: string[]; + + /** + * Absolute hard limit for chunk text length. + * Defaults to Infinity if omitted. + */ + maxChunkSize?: number; +} + +export class HTMLTextSplitter extends TextSplitter { + constructor(fields?: Partial); + + /** + * Splits HTML input into normalized text chunks. + * + * Important: this splitter does not preserve HTML markup in output. + * It uses parsed HTML structure for traversal and sizing, but emits text only. + */ + splitText(text: string): Promise; + + /** + * Splits HTML documents into text-only child documents. + * + * Important: child document `pageContent` contains normalized text, not HTML. + * Parent metadata is preserved and per-document part metadata is added. + */ + splitDocuments(documents: Document[]): Promise; +} +``` + +## Output Semantics + +1. `splitText()` accepts HTML and returns `string[]` text chunks. +2. Each chunk is text content extracted from a DOM subtree or subtree fragment. +3. `splitDocuments()` preserves parent metadata and adds per-document `partIndex` and `totalParts`. +4. Final chunk text is normalized after DOM-aware joining; block-ish tag boundaries may add spacing, while inline tag boundaries do not add synthetic spaces. + +## Size Semantics + +1. `chunkSize` is the target size. +2. `maxChunkSize` is the hard cap when provided. +3. The size function is based on the same normalized text content used for output segments. +4. Segment text is derived from cheerio text extraction, joined with DOM-aware boundaries, then whitespace-normalized before measurement. +5. No separate text-length implementation should be maintained outside that semantic. +6. If `maxChunkSize` is omitted, treat it as `Infinity`. + +## Core Algorithm + +### Mental Model + +Treat each DOM node as a unit that can either: +- fit as-is +- stay atomic because it is protected and still within `maxChunkSize` +- be recursively decomposed into children because it is too large + +Separators only matter when a node must be broken down. They influence recursion order; they do not force a split on their own. + +### Traversal Strategy + +1. Parse the HTML with cheerio. +2. Select a traversal root: + - use `body` children if a body exists + - otherwise use root children +3. Walk top-level nodes in document order. +4. Convert each node into one or more text segments using recursive decomposition. +5. Merge resulting segments into final chunks while respecting `chunkSize` and `maxChunkSize`. + +### Recursive Decomposition Rules + +For a node `N`: + +1. Compute `nodeText` using the shared `getNodeText()` helper so both element nodes and text nodes follow the same trimming semantics. +2. If `nodeText` is empty, skip it. +3. If `nodeText.length <= chunkSize`, emit it as one segment. +4. If `N` matches `neverBreakWithin` and `nodeText.length <= maxChunkSize`, emit it as one segment. +5. Otherwise, recurse into children. +6. If `N` has no element/text children that can further subdivide the content, force-split the node text with `RecursiveCharacterTextSplitter`. + +### Separator-Aware Recursion + +When recursing into a node with child elements: + +1. Partition children into traversal groups. +2. Prefer group boundaries at direct child elements matching `separators`. +3. A separator child starts a new group and stays with the following sibling content rather than the preceding content. +4. If no separator child exists, recurse in plain DOM order. +5. Ignore separator matches inside any subtree currently treated as protected. + +## Internal Types + +```typescript +interface Segment { + text: string; + isProtected: boolean; + sourceKind: "node" | "forced-split"; +} +``` + +V1 should keep internal types local to the implementation file unless reuse appears during build. + +## Detailed Behavior + +### 1. Protected Nodes + +If a node matches `neverBreakWithin`, keep it atomic while its text length is `<= maxChunkSize`. Recurse into it only if it exceeds `maxChunkSize`. + +If `maxChunkSize` is `Infinity`, protected nodes are never forced open by hard-cap rules. + +### 2. Forced Splitting + +When recursion reaches a node whose text still exceeds `maxChunkSize`, and there is no useful child structure left, use `RecursiveCharacterTextSplitter` on the extracted text. + +Use it with: +- `chunkSize: this.maxChunkSize` +- `chunkOverlap: 0` + +This is the final fallback that guarantees the hard limit. + +If `maxChunkSize` is `Infinity`, this fallback is not used for hard-cap enforcement. + +### 3. Merging + +After recursion produces a flat list of text segments: + +1. Start a new chunk. +2. Append the next segment if the combined length stays within `chunkSize`. +3. If appending would exceed `chunkSize`, flush the current chunk and start a new one. +4. If any individual segment is already larger than `chunkSize` but not larger than `maxChunkSize`, allow that segment to become its own chunk. +5. Join merged segment text using DOM-aware boundaries and whitespace normalization. +6. No overlap is applied. + +When `maxChunkSize` is `Infinity`, a single segment may grow beyond `chunkSize` if it cannot be decomposed further without violating other rules. + +## Pseudocode + +```typescript +async splitText(html: string): Promise { + const $ = cheerio.load(html); + const roots = $("body").length > 0 ? $("body").contents().toArray() : $.root().contents().toArray(); + + const segments: Segment[] = []; + for (const node of roots) { + segments.push(...(await this.collectSegments($, node, false))); + } + + return this.mergeSegments(segments); +} + +private async collectSegments( + $: cheerio.CheerioAPI, + node: cheerio.AnyNode, + inProtectedAncestor: boolean, +): Promise { + const text = this.getNodeText($, node); + if (!text) return []; + + const isProtected = !inProtectedAncestor && this.matchesAny($, node, this.neverBreakWithin); + + if (text.length <= this.chunkSize) { + return [{ text, isProtected, sourceKind: "node" }]; + } + + if (isProtected && text.length <= this.maxChunkSize) { + return [{ text, isProtected, sourceKind: "node" }]; + } + + const childNodes = this.getChildNodesForRecursion(node); + if (childNodes.length === 0) { + return this.forceSplitText(text); + } + + const childGroups = this.groupChildrenForDecomposition($, childNodes, isProtected || inProtectedAncestor); + const segments: Segment[] = []; + for (const group of childGroups) { + for (const child of group) { + segments.push(...(await this.collectSegments($, child, isProtected || inProtectedAncestor))); + } + } + + return segments.length > 0 ? segments : this.forceSplitText(text); +} +``` + +## Utils Pseudocode + +```typescript +/** + * Returns true when a tag node matches any configured CSS selector. + * + * Needed so separator and protected-node checks share the same selector logic + * and non-tag nodes are ignored safely. + */ +private matchesAny( + $: cheerio.CheerioAPI, + node: cheerio.AnyNode, + selectors: string[], +): boolean { + if (node.type !== "tag") { + return false; + } + + for (const selector of selectors) { + if ($(node).is(selector)) { + return true; + } + } + + return false; +} + +/** + * Extracts the canonical text value for a node. + * + * Needed so sizing and emitted output use the same trimmed text semantics for + * both element nodes and raw text nodes. + */ +private getNodeText($: cheerio.CheerioAPI, node: cheerio.AnyNode): string { + if (node.type === "text") { + return (node.data ?? "").trim(); + } + + return $(node).text().trim(); +} + +/** + * Returns the child nodes that are meaningful recursion units. + * + * Needed to recurse through DOM structure without carrying empty whitespace-only + * text nodes or unsupported node types through the algorithm. + */ +private getChildNodesForRecursion(node: cheerio.AnyNode): cheerio.AnyNode[] { + if (!("children" in node) || !Array.isArray(node.children)) { + return []; + } + + return node.children.filter((child) => { + if (child.type === "text") { + return (child.data ?? "").trim().length > 0; + } + + return child.type === "tag"; + }); +} + +/** + * Groups child nodes so separator nodes start a new group. + * + * Needed because separators represent preferred section boundaries. Grouping + * makes "separator plus following content" explicit during recursion. + */ +private groupChildrenForDecomposition( + $: cheerio.CheerioAPI, + childNodes: cheerio.AnyNode[], + inProtectedTree: boolean, +): cheerio.AnyNode[][] { + if (inProtectedTree || this.separators.length === 0) { + return [childNodes]; + } + + const groups: cheerio.AnyNode[][] = []; + let currentGroup: cheerio.AnyNode[] = []; + + const flush = () => { + if (currentGroup.length > 0) { + groups.push(currentGroup); + currentGroup = []; + } + }; + + for (const child of childNodes) { + if (this.matchesAny($, child, this.separators)) { + flush(); + } + + currentGroup.push(child); + } + + flush(); + return groups; +} + +/** + * Applies the final plain-text fallback when a node cannot be decomposed further. + * + * Needed to enforce a finite hard cap with `RecursiveCharacterTextSplitter` + * once DOM structure is exhausted. + */ +private async forceSplitText(text: string): Promise { + if (!Number.isFinite(this.maxChunkSize)) { + return [{ text: text.trim(), isProtected: false, sourceKind: "forced-split" }]; + } + + const fallback = new RecursiveCharacterTextSplitter({ + chunkSize: this.maxChunkSize, + chunkOverlap: 0, + }); + + const chunks = await fallback.splitText(text); + return chunks + .map((chunk) => chunk.trim()) + .filter(Boolean) + .map((chunk) => ({ + text: chunk, + isProtected: false, + sourceKind: "forced-split" as const, + })); +} + +/** + * Merges flat segments into final chunks using `chunkSize` as a soft target. + * + * Needed to keep chunk assembly separate from DOM traversal and to normalize the + * final text output by joining segment text with single spaces. + */ +private mergeSegments(segments: Segment[]): string[] { + const chunks: string[] = []; + let currentParts: string[] = []; + let currentLength = 0; + + const flush = () => { + if (currentParts.length === 0) { + return; + } + + chunks.push(currentParts.join(" ")); + currentParts = []; + currentLength = 0; + }; + + for (const segment of segments) { + const nextLength = currentLength === 0 + ? segment.text.length + : currentLength + 1 + segment.text.length; + + if (currentParts.length > 0 && nextLength > this.chunkSize) { + flush(); + } + + currentParts.push(segment.text); + currentLength = currentLength === 0 + ? segment.text.length + : currentLength + 1 + segment.text.length; + } + + flush(); + return chunks; +} +``` + +## Implementation Plan + +### Phase 1: Class Skeleton + +1. Create `textsplitters/html_text_splitter.ts`. +2. Define `HTMLTextSplitterParams`. +3. Extend `TextSplitter`. +4. Reject unsupported overlap in the constructor: + - if `chunkOverlap !== 0`, throw a clear error +5. Default `maxChunkSize` to `Infinity` when omitted. + +### Phase 2: Tree Traversal Helpers + +1. `matchesAny()` for selector matching. +2. `getNodeText()` using cheerio text extraction and trimming semantics shared with output. +3. `getChildNodesForRecursion()` to expose both text and element children in DOM order. +4. `groupChildrenForDecomposition()` to build explicit child groups, with separator nodes grouped with following content. + +### Phase 3: Recursive Collection + +1. Implement `collectSegments()`. +2. Preserve document order. +3. Skip empty/whitespace-only results. + +### Phase 4: Final Fallback + +1. Implement `forceSplitText()` with `RecursiveCharacterTextSplitter`. +2. Use `maxChunkSize` as the chunk size when it is finite. +3. Use `chunkOverlap: 0`. +4. Trim and drop empty results. + +### Phase 5: Merge And Document Support + +1. Merge segments into final chunks using `chunkSize` as the soft target. +2. Implement `splitDocuments()` by mapping each input document through `splitText()`. +3. Preserve original metadata and add per-document `partIndex` / `totalParts`. + +## Edge Cases + +### Empty HTML + +Return `[]`. + +### Plain Text Input + +Cheerio will still parse it; return one or more text chunks based on size. + +### Deeply Nested Protected Content + +Keep it atomic unless it exceeds `maxChunkSize`; then recurse into its children and only force-split as a last resort. + +If `maxChunkSize` is `Infinity`, it remains atomic unless some ancestor decomposition requires visiting its children. + +### Separator Inside Protected Content + +Ignore the separator during normal traversal. + +### Large Text Node With No Useful Child Structure + +Use text fallback splitting. + +### Consecutive Small Nodes + +Merge them up to `chunkSize`. + +### A Single Segment Between chunkSize And maxChunkSize + +Allow it as a standalone chunk. + +## Testing Strategy + +Use Bun's built-in test runner and add automated tests under `test/textsplitters/`. + +### Minimum Verification + +1. `bunx tsc --noEmit` +2. automated splitter tests under `test/textsplitters/` +3. `bun run setup/ingest.ts` on a small fixture or targeted spec file if practical + +### Behavior Cases To Verify + +1. Returns one chunk when content is under `chunkSize`. +2. Ignores HTML tag length when using cheerio text extraction. +3. Does not split at separators unless size pressure requires it. +4. Keeps protected nodes intact up to `maxChunkSize`. +5. Recurses into protected nodes larger than `maxChunkSize`. +6. Uses fallback splitting for oversized leaf text. +7. Preserves metadata in `splitDocuments()`. +8. Throws when `chunkOverlap !== 0`. +9. Leaves hard-cap fallback disabled when `maxChunkSize` is omitted. + +The implementation should also update `package.json` with a real `bun test` command so these tests can run in CI and locally. + +### Example Important Cases + +```typescript +// separator is a soft hint, not an unconditional boundary +const splitter = new HTMLTextSplitter({ + chunkSize: 100, + maxChunkSize: 100, + separators: ["h2"], +}); + +await splitter.splitText("

A

short body

"); +// expected: one chunk because no size pressure exists +``` + +```typescript +// protected node stays atomic until maxChunkSize is exceeded +const splitter = new HTMLTextSplitter({ + chunkSize: 20, + maxChunkSize: 80, + neverBreakWithin: ["pre"], +}); + +await splitter.splitText("
...60 chars...
"); +// expected: one chunk +``` + +## Repository Integration Notes + +The eventual migration in `setup/ingest.ts` should account for these changes: + +1. output will already be text chunks, so downstream `cheerio.load(chunk.pageContent).text()` cleanup may become redundant +2. `keepSeparator` is removed from the new design +3. `chunkOverlap` must remain `0` +4. current warnings around chunk sizes should be rechecked after migration + +## Success Criteria + +1. No raw HTML offset slicing is used anywhere. +2. The splitter traverses DOM structure recursively. +3. Separators act as soft decomposition hints only. +4. Protected nodes remain atomic unless `maxChunkSize` is exceeded. +5. No output chunk exceeds `maxChunkSize` when a finite hard cap is configured. +6. No overlap behavior is implemented or implied. +7. `splitDocuments()` preserves metadata and returns chunked documents. +8. The implementation can be verified in this repo without assuming a missing test framework. + +## Open Questions + +No remaining open questions in the current plan. diff --git a/Readme.md b/Readme.md index e55aae1..b7b8d39 100644 --- a/Readme.md +++ b/Readme.md @@ -12,7 +12,7 @@ This project implements a RAG-based AI chat agent to explore the ECMAScript spec 1. **Install dependencies**: ```bash - npm install + bun install ``` 2. **Prepare environment**: @@ -26,13 +26,18 @@ This project implements a RAG-based AI chat agent to explore the ECMAScript spec 4. **Ingest data**: ```bash - node ingest.mjs + bun run ingest ``` - *Note: This will take significant time as it generates local embeddings via Ollama for both the spec and the implementation.* + *Note: This will take significant time as it generates local embeddings via Ollama for both the spec and the implementation.* 5. **Build graph**: ```bash - node build_graph.mjs + bun run build + ``` + +6. **Run tests**: + ```bash + bun test ``` ## Usage @@ -40,7 +45,7 @@ This project implements a RAG-based AI chat agent to explore the ECMAScript spec Ask the agent questions about how code relates to the specification: ```bash -node agent.mjs "Explain how the 'if' statement works and show its implementation." +bun run agent "Explain how the 'if' statement works and show its implementation." ``` The agent will use tools to search the specification, explore the implementation code, and navigate the relationships between them using the graph. diff --git a/bun.lock b/bun.lock index 318a534..02798b8 100644 --- a/bun.lock +++ b/bun.lock @@ -19,6 +19,7 @@ }, "devDependencies": { "@biomejs/biome": "^2.4.9", + "bun-types": "^1.3.11", "typescript": "^5.5.2", }, }, @@ -70,7 +71,7 @@ "@types/command-line-usage": ["@types/command-line-usage@5.0.4", "", {}, "sha512-BwR5KP3Es/CSht0xqBcUXS3qCAUVXwpRKsV2+arxeb65atasuXG9LykC9Ab10Cw3s2raH92ZqOeILaQbsB2ACg=="], - "@types/node": ["@types/node@20.19.37", "", { "dependencies": { "undici-types": "~6.21.0" } }, "sha512-8kzdPJ3FsNsVIurqBs7oodNnCEVbni9yUEkaHbgptDACOPW04jimGagZ51E6+lXUwJjgnBw+hyko/lkFWCldqw=="], + "@types/node": ["@types/node@24.12.0", "", { "dependencies": { "undici-types": "~7.16.0" } }, "sha512-GYDxsZi3ChgmckRT9HPU0WEhKLP08ev/Yfcq2AstjrDASOYCSXeyjDsHg4v5t4jOj7cyDX3vmprafKlWIG9MXQ=="], "@types/node-fetch": ["@types/node-fetch@2.6.13", "", { "dependencies": { "@types/node": "*", "form-data": "^4.0.4" } }, "sha512-QGpRVpzSaUs30JBSGPjOg4Uveu384erbHBoT1zeONvyCfwQxIkUshLAOqN/k9EjGviPRmWTTe6aH2qySWKTVSw=="], @@ -106,6 +107,8 @@ "brace-expansion": ["brace-expansion@5.0.5", "", { "dependencies": { "balanced-match": "^4.0.2" } }, "sha512-VZznLgtwhn+Mact9tfiwx64fA9erHH/MCXEUfB/0bX/6Fz6ny5EGTXYltMocqg4xFAQZtnO3DHWWXi8RiuN7cQ=="], + "bun-types": ["bun-types@1.3.11", "", { "dependencies": { "@types/node": "*" } }, "sha512-1KGPpoxQWl9f6wcZh57LvrPIInQMn2TQ7jsgxqpRzg+l0QPOFvJVH7HmvHo/AiPgwXy+/Thf6Ov3EdVn1vOabg=="], + "call-bind-apply-helpers": ["call-bind-apply-helpers@1.0.2", "", { "dependencies": { "es-errors": "^1.3.0", "function-bind": "^1.1.2" } }, "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ=="], "camelcase": ["camelcase@6.3.0", "", {}, "sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA=="], @@ -316,7 +319,7 @@ "undici": ["undici@7.24.6", "", {}, "sha512-Xi4agocCbRzt0yYMZGMA6ApD7gvtUFaxm4ZmeacWI4cZxaF6C+8I8QfofC20NAePiB/IcvZmzkJ7XPa471AEtA=="], - "undici-types": ["undici-types@6.21.0", "", {}, "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ=="], + "undici-types": ["undici-types@7.16.0", "", {}, "sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw=="], "uuid": ["uuid@9.0.1", "", { "bin": { "uuid": "dist/bin/uuid" } }, "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA=="], @@ -348,7 +351,7 @@ "@langchain/ollama/uuid": ["uuid@10.0.0", "", { "bin": { "uuid": "dist/bin/uuid" } }, "sha512-8XkAphELsDnEGrDxUOHB3RGvXz6TeuYSGEZBOjtTtPm2lwhGBjLgOzLHB63IUWfBpNucQjND6d3AOudO+H3RWQ=="], - "@types/node-fetch/@types/node": ["@types/node@24.12.0", "", { "dependencies": { "undici-types": "~7.16.0" } }, "sha512-GYDxsZi3ChgmckRT9HPU0WEhKLP08ev/Yfcq2AstjrDASOYCSXeyjDsHg4v5t4jOj7cyDX3vmprafKlWIG9MXQ=="], + "apache-arrow/@types/node": ["@types/node@20.19.37", "", { "dependencies": { "undici-types": "~6.21.0" } }, "sha512-8kzdPJ3FsNsVIurqBs7oodNnCEVbni9yUEkaHbgptDACOPW04jimGagZ51E6+lXUwJjgnBw+hyko/lkFWCldqw=="], "chalk-template/chalk": ["chalk@4.1.2", "", { "dependencies": { "ansi-styles": "^4.1.0", "supports-color": "^7.1.0" } }, "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA=="], @@ -368,7 +371,7 @@ "table-layout/array-back": ["array-back@6.2.3", "", {}, "sha512-SGDvmg6QTYiTxCBkYVmThcoa67uLl35pyzRHdpCGBOcqFy6BtwnphoFPk7LhJshD+Yk1Kt35WGWeZPTgwR4Fhw=="], - "@types/node-fetch/@types/node/undici-types": ["undici-types@7.16.0", "", {}, "sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw=="], + "apache-arrow/@types/node/undici-types": ["undici-types@6.21.0", "", {}, "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ=="], "chalk-template/chalk/ansi-styles": ["ansi-styles@4.3.0", "", { "dependencies": { "color-convert": "^2.0.1" } }, "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg=="], diff --git a/bunfig.toml b/bunfig.toml new file mode 100644 index 0000000..d34602b --- /dev/null +++ b/bunfig.toml @@ -0,0 +1,3 @@ +[test] +# Only run tests in the test/ directory, ignoring engine262/ and other subdirectories +root = "./test" diff --git a/package.json b/package.json index d0984da..0460e32 100644 --- a/package.json +++ b/package.json @@ -11,7 +11,7 @@ "ingest": "bun run setup/ingest.ts", "agent": "bun run agent.ts", "build": "bun run setup/build_graph.ts", - "test": "echo \"Error: no test specified\" && exit 1" + "test": "bun test" }, "keywords": [], "author": "", @@ -32,7 +32,8 @@ "ora": "^9.3.0" }, "devDependencies": { - "typescript": "^5.5.2", - "@biomejs/biome": "^2.4.9" + "@biomejs/biome": "^2.4.9", + "bun-types": "^1.3.11", + "typescript": "^5.5.2" } } diff --git a/test/textsplitters/html_text_splitter.test.ts b/test/textsplitters/html_text_splitter.test.ts new file mode 100644 index 0000000..1dab8b8 --- /dev/null +++ b/test/textsplitters/html_text_splitter.test.ts @@ -0,0 +1,289 @@ +import { describe, expect, test } from "bun:test"; +import { Document } from "@langchain/core/documents"; +import { HTMLTextSplitter } from "../../textsplitters"; + +describe("HTMLTextSplitter", () => { + test("keeps small HTML in a single chunk", async () => { + const splitter = new HTMLTextSplitter({ + chunkSize: 64, + separators: ["h2"], + }); + + const chunks = await splitter.splitText( + "

Title

Short body text

", + ); + + expect(chunks).toEqual(["Title Short body text"]); + }); + + test("does not add synthetic spaces between inline tags", async () => { + const splitter = new HTMLTextSplitter({ + chunkSize: 64, + }); + + const chunks = await splitter.splitText( + "

HelloWorld

", + ); + + expect(chunks).toEqual(["HelloWorld"]); + }); + + test("preserves authored whitespace between inline tags", async () => { + const splitter = new HTMLTextSplitter({ + chunkSize: 64, + }); + + const chunks = await splitter.splitText( + "

Hello World

", + ); + + expect(chunks).toEqual(["Hello World"]); + }); + + test("adds spacing between adjacent block-ish tags", async () => { + const splitter = new HTMLTextSplitter({ + chunkSize: 64, + }); + + const chunks = await splitter.splitText("
Hello
World
"); + + expect(chunks).toEqual(["Hello World"]); + }); + + test("normalizes repeated whitespace in emitted text", async () => { + const splitter = new HTMLTextSplitter({ + chunkSize: 64, + }); + + const chunks = await splitter.splitText( + "
Hello\n\n World
\tAgain
", + ); + + expect(chunks).toEqual(["Hello World Again"]); + }); + + test("treats separators as soft hints until size pressure exists", async () => { + const splitter = new HTMLTextSplitter({ + chunkSize: 18, + separators: ["h2"], + }); + + const chunks = await splitter.splitText( + [ + "
", + "

Intro text

", + "

Section Title

", + "

Body text

", + "
", + ].join(""), + ); + + expect(chunks).toEqual(["Intro text", "Section Title Body text"]); + }); + + test("groups consecutive separators into later section boundaries", async () => { + const splitter = new HTMLTextSplitter({ + chunkSize: 4, + separators: ["h2", "h3"], + }); + + const chunks = await splitter.splitText( + "

A

B

Body

", + ); + + expect(chunks).toEqual(["A", "B Body"]); + }); + + test("keeps protected content intact up to maxChunkSize", async () => { + const splitter = new HTMLTextSplitter({ + chunkSize: 10, + maxChunkSize: 32, + neverBreakWithin: ["pre"], + }); + + const chunks = await splitter.splitText( + "
12345678901234567890
", + ); + + expect(chunks).toEqual(["12345678901234567890"]); + }); + + test("recurses into protected nodes once maxChunkSize is exceeded", async () => { + const splitter = new HTMLTextSplitter({ + chunkSize: 12, + maxChunkSize: 18, + neverBreakWithin: [".keep"], + separators: ["p"], + }); + + const chunks = await splitter.splitText( + [ + '
', + "

Alpha beta

", + "

Gamma delta

", + "
", + ].join(""), + ); + + expect(chunks).toEqual(["Alpha beta", "Gamma delta"]); + }); + + test("ignores separators inside protected subtrees until forced open", async () => { + const splitter = new HTMLTextSplitter({ + chunkSize: 10, + maxChunkSize: 40, + neverBreakWithin: [".keep"], + separators: ["h2"], + }); + + const chunks = await splitter.splitText( + '

Title

Body text

', + ); + + expect(chunks).toEqual(["Title Body text"]); + }); + + test("force-splits oversized leaf text when maxChunkSize is finite", async () => { + const splitter = new HTMLTextSplitter({ + chunkSize: 12, + maxChunkSize: 15, + }); + + const chunks = await splitter.splitText(`
${"a".repeat(35)}
`); + + expect(chunks.length).toBeGreaterThan(1); + for (const chunk of chunks) { + expect(chunk.length).toBeLessThanOrEqual(15); + } + }); + + test("force-splits protected oversized leaf text when maxChunkSize is finite", async () => { + const splitter = new HTMLTextSplitter({ + chunkSize: 12, + maxChunkSize: 15, + neverBreakWithin: ["pre"], + }); + + const chunks = await splitter.splitText(`
${"x".repeat(35)}
`); + + expect(chunks.length).toBeGreaterThan(1); + for (const chunk of chunks) { + expect(chunk.length).toBeLessThanOrEqual(15); + } + }); + + test("does not enforce a hard cap when maxChunkSize is omitted", async () => { + const splitter = new HTMLTextSplitter({ + chunkSize: 12, + neverBreakWithin: ["pre"], + }); + + const chunks = await splitter.splitText(`
${"a".repeat(35)}
`); + + expect(chunks).toEqual(["a".repeat(35)]); + }); + + test("splits plain text input without HTML structure", async () => { + const splitter = new HTMLTextSplitter({ + chunkSize: 8, + maxChunkSize: 8, + }); + + const chunks = await splitter.splitText("alpha beta gamma"); + + expect(chunks).toEqual(["alpha", "beta", "gamma"]); + }); + + test("preserves metadata and adds per-document part indexes", async () => { + const splitter = new HTMLTextSplitter({ + chunkSize: 18, + separators: ["h2"], + }); + + const documents = await splitter.splitDocuments([ + new Document({ + pageContent: + "

Intro text

Section Title

Body text

", + metadata: { source: "sample.html" }, + }), + ]); + + expect(documents).toHaveLength(2); + expect(documents[0].pageContent).toBe("Intro text"); + expect(documents[0].metadata).toMatchObject({ + source: "sample.html", + partIndex: 0, + totalParts: 2, + }); + expect(documents[1].pageContent).toBe("Section Title Body text"); + expect(documents[1].metadata).toMatchObject({ + source: "sample.html", + partIndex: 1, + totalParts: 2, + }); + }); + + test("resets part metadata per input document", async () => { + const splitter = new HTMLTextSplitter({ + chunkSize: 18, + separators: ["h2"], + }); + + const documents = await splitter.splitDocuments([ + new Document({ + pageContent: + "

Intro text

Section Title

Body text

", + metadata: { source: "first.html" }, + }), + new Document({ + pageContent: + "

Lead text

Second Title

More text

", + metadata: { source: "second.html" }, + }), + ]); + + expect(documents).toHaveLength(4); + expect(documents[0].metadata).toMatchObject({ + source: "first.html", + partIndex: 0, + totalParts: 2, + }); + expect(documents[1].metadata).toMatchObject({ + source: "first.html", + partIndex: 1, + totalParts: 2, + }); + expect(documents[2].metadata).toMatchObject({ + source: "second.html", + partIndex: 0, + totalParts: 2, + }); + expect(documents[3].metadata).toMatchObject({ + source: "second.html", + partIndex: 1, + totalParts: 2, + }); + }); + + test("rejects chunkOverlap", () => { + expect( + () => + new HTMLTextSplitter({ + chunkSize: 32, + chunkOverlap: 4, + }), + ).toThrow("does not support chunkOverlap"); + }); + + test("keeps adjacent inline text contiguous when no whitespace exists", async () => { + const splitter = new HTMLTextSplitter({ + chunkSize: 100, + }); + + const chunks = await splitter.splitText( + "alphabetgamma", + ); + + expect(chunks).toEqual(["alphabetgamma"]); + }); +}); diff --git a/textsplitters/html_text_splitter.ts b/textsplitters/html_text_splitter.ts new file mode 100644 index 0000000..3a26469 --- /dev/null +++ b/textsplitters/html_text_splitter.ts @@ -0,0 +1,523 @@ +import { Document } from "@langchain/core/documents"; +import { + RecursiveCharacterTextSplitter, + TextSplitter, + type TextSplitterChunkHeaderOptions, + type TextSplitterParams, +} from "@langchain/textsplitters"; +import * as cheerio from "cheerio"; +import type { AnyNode } from "domhandler"; + +export interface HTMLTextSplitterParams extends TextSplitterParams { + /** + * CSS selectors that act as preferred structural split points. + * They are only used when a subtree needs to be broken down. + */ + separators?: string[]; + + /** + * CSS selectors for nodes that should stay atomic unless maxChunkSize forces recursion. + */ + neverBreakWithin?: string[]; + + /** + * Absolute hard limit for chunk text length. + * Defaults to Infinity if omitted. + */ + maxChunkSize?: number; +} + +interface Segment { + text: string; + isProtected: boolean; + sourceKind: "node" | "forced-split"; +} + +const BLOCKISH_TAGS = new Set([ + "address", + "article", + "aside", + "blockquote", + "br", + "dd", + "div", + "dl", + "dt", + "emu-clause", + "emu-example", + "emu-grammar", + "emu-note", + "emu-table", + "fieldset", + "figcaption", + "figure", + "footer", + "form", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "header", + "hr", + "li", + "main", + "nav", + "ol", + "p", + "pre", + "section", + "table", + "tbody", + "td", + "th", + "thead", + "tr", + "ul", +]); + +/** + * Splits HTML input into normalized text chunks. + * + * Important: this splitter does not preserve HTML markup in output. + * It uses parsed HTML structure for traversal and sizing, but emits text only. + * + * @example + * ```ts + * const splitter = new HTMLTextSplitter({ + * chunkSize: 32, + * separators: ["h2"], + * neverBreakWithin: ["pre"], + * }); + * + * const chunks = await splitter.splitText( + * "

Title

Body

", + * ); + * // => ["Title Body"] + * ``` + */ +export class HTMLTextSplitter extends TextSplitter { + separators: string[]; + neverBreakWithin: string[]; + maxChunkSize: number; + + constructor(fields?: Partial) { + if ((fields?.chunkOverlap ?? 0) !== 0) { + throw new Error("HTMLTextSplitter does not support chunkOverlap."); + } + + super({ + ...fields, + chunkOverlap: 0, + keepSeparator: false, + lengthFunction: (text: string) => this.normalizeWhitespace(text).length, + }); + + this.separators = fields?.separators ?? []; + this.neverBreakWithin = fields?.neverBreakWithin ?? []; + this.maxChunkSize = fields?.maxChunkSize ?? Number.POSITIVE_INFINITY; + } + + /** + * Splits HTML input into normalized text chunks. + * + * Important: this method emits text only. HTML markup is used for traversal + * and sizing, but is not preserved in returned chunks. + */ + async splitText(text: string): Promise { + const $ = cheerio.load(text); + const rootNodes = + $("body").length > 0 + ? $("body").contents().toArray() + : $.root().contents().toArray(); + const roots = this.getMeaningfulNodes(rootNodes); + + if (roots.length === 0) { + return []; + } + + const rootGroups = this.groupChildrenForDecomposition($, roots, false); + const segments: Segment[] = []; + + for (const group of rootGroups) { + segments.push(...(await this.collectSegmentsFromNodes($, group, false))); + } + + return this.mergeSegments(segments); + } + + /** + * Splits HTML documents into text-only child documents. + * + * Important: child document `pageContent` contains normalized text, not HTML. + * Parent metadata is preserved and per-document part metadata is added. + */ + async splitDocuments( + documents: Document[], + chunkHeaderOptions?: TextSplitterChunkHeaderOptions, + ): Promise { + const splitDocs: Document[] = []; + const chunkHeader = chunkHeaderOptions?.chunkHeader ?? ""; + + for (const document of documents) { + const chunks = await this.splitText(document.pageContent); + + for (const [index, chunk] of chunks.entries()) { + splitDocs.push( + new Document({ + pageContent: `${chunkHeader}${chunk}`, + metadata: { + ...document.metadata, + partIndex: index, + totalParts: chunks.length, + }, + }), + ); + } + } + + return splitDocs; + } + + /** + * Returns true when a tag node matches any configured CSS selector. + * + * Needed so separator and protected-node checks share the same selector logic + * and non-tag nodes are ignored safely. + */ + private matchesAny( + $: cheerio.CheerioAPI, + node: AnyNode, + selectors: string[], + ): boolean { + if (node.type !== "tag") { + return false; + } + + for (const selector of selectors) { + if ($(node).is(selector)) { + return true; + } + } + + return false; + } + + /** + * Extracts the canonical text value for a node. + * + * Needed so sizing and emitted output use the same trimmed text semantics for + * both element nodes and raw text nodes. + */ + private getNodeText($: cheerio.CheerioAPI, node: AnyNode): string { + if (node.type === "text") { + return node.data ?? ""; + } + + return $(node).text(); + } + + /** + * Returns whether a node should introduce a visible boundary when adjacent + * text is concatenated. + * + * Needed so inline elements such as `span` do not get synthetic spaces while + * block-ish elements still remain readable in emitted text. + */ + private isBlockishNode(node: AnyNode): boolean { + return node.type === "tag" && BLOCKISH_TAGS.has(node.name); + } + + /** + * Collapses internal whitespace and trims leading/trailing whitespace. + * + * Needed so text extraction can preserve raw adjacency first and normalize + * only once after boundary-aware joining. + */ + private normalizeWhitespace(text: string): string { + return text.replace(/\s+/g, " ").trim(); + } + + /** + * Joins multiple nodes into the normalized text the splitter actually emits. + * + * Needed so grouped nodes are measured and emitted with the same spacing rules + * used later during chunk merging. + */ + private getNodesText($: cheerio.CheerioAPI, nodes: AnyNode[]): string { + let result = ""; + let previousNode: AnyNode | null = null; + + for (const node of nodes) { + const text = this.getNodeText($, node); + if (!text) { + continue; + } + + if ( + result.length > 0 && + previousNode && + (this.isBlockishNode(previousNode) || this.isBlockishNode(node)) + ) { + result += " "; + } + + result += text; + previousNode = node; + } + + return this.normalizeWhitespace(result); + } + + /** + * Filters child nodes down to the recursion units the splitter can handle. + * + * Needed to recurse through DOM structure without carrying empty whitespace-only + * text nodes or unsupported node types through the algorithm. + */ + private getMeaningfulNodes(nodes: AnyNode[]): AnyNode[] { + return nodes.filter((node) => { + if (node.type === "text") { + return (node.data ?? "").trim().length > 0; + } + + return node.type === "tag"; + }); + } + + /** + * Returns the child nodes that are meaningful recursion units. + * + * Needed to recurse through DOM structure without carrying empty whitespace-only + * text nodes or unsupported node types through the algorithm. + */ + private getChildNodesForRecursion(node: AnyNode): AnyNode[] { + if (!("children" in node) || !Array.isArray(node.children)) { + return []; + } + + return this.getMeaningfulNodes(node.children); + } + + /** + * Groups child nodes so separator nodes start a new group. + * + * Needed because separators represent preferred section boundaries. Grouping + * makes "separator plus following content" explicit during recursion. + */ + private groupChildrenForDecomposition( + $: cheerio.CheerioAPI, + childNodes: AnyNode[], + inProtectedTree: boolean, + ): AnyNode[][] { + if (childNodes.length === 0) { + return []; + } + + if (inProtectedTree || this.separators.length === 0) { + return [childNodes]; + } + + const groups: AnyNode[][] = []; + let currentGroup: AnyNode[] = []; + + const flush = () => { + if (currentGroup.length > 0) { + groups.push(currentGroup); + currentGroup = []; + } + }; + + for (const child of childNodes) { + if (this.matchesAny($, child, this.separators)) { + flush(); + } + + currentGroup.push(child); + } + + flush(); + + return groups; + } + + /** + * Collects text segments for a single node or grouped sibling nodes. + * + * Needed so separator-led groups can be treated as one structural unit before + * the splitter decides whether it must recurse deeper. + */ + private async collectSegmentsFromNodes( + $: cheerio.CheerioAPI, + nodes: AnyNode[], + inProtectedAncestor: boolean, + ): Promise { + const text = this.getNodesText($, nodes); + + if (!text) { + return []; + } + + if (nodes.length > 1) { + const startsWithSeparator = this.matchesAny($, nodes[0], this.separators); + + if ( + text.length <= this.chunkSize || + (startsWithSeparator && text.length <= this.maxChunkSize) + ) { + return [{ text, isProtected: false, sourceKind: "node" }]; + } + + const segments: Segment[] = []; + for (const node of nodes) { + segments.push( + ...(await this.collectSegmentsFromNodes( + $, + [node], + inProtectedAncestor, + )), + ); + } + + return segments.length > 0 + ? segments + : await this.forceSplitText(text, false); + } + + const [node] = nodes; + const isProtected = + !inProtectedAncestor && this.matchesAny($, node, this.neverBreakWithin); + const childNodes = this.getChildNodesForRecursion(node); + + if (childNodes.length > 0) { + const childGroups = this.groupChildrenForDecomposition( + $, + childNodes, + isProtected || inProtectedAncestor, + ); + const normalizedChildText = this.getNodesText($, childGroups.flat()); + + if (normalizedChildText.length <= this.chunkSize) { + return [{ text: normalizedChildText, isProtected, sourceKind: "node" }]; + } + + if (isProtected && normalizedChildText.length <= this.maxChunkSize) { + return [{ text: normalizedChildText, isProtected, sourceKind: "node" }]; + } + + const segments: Segment[] = []; + for (const group of childGroups) { + segments.push( + ...(await this.collectSegmentsFromNodes( + $, + group, + isProtected || inProtectedAncestor, + )), + ); + } + + return segments.length > 0 + ? segments + : await this.forceSplitText(normalizedChildText, isProtected); + } + + if (text.length <= this.chunkSize) { + return [{ text, isProtected, sourceKind: "node" }]; + } + + if (isProtected && text.length <= this.maxChunkSize) { + return [{ text, isProtected, sourceKind: "node" }]; + } + + return this.forceSplitText(text, isProtected); + } + + /** + * Applies the final plain-text fallback when a node cannot be decomposed further. + * + * Needed to enforce a finite hard cap with `RecursiveCharacterTextSplitter` + * once DOM structure is exhausted. + */ + private async forceSplitText( + text: string, + isProtected: boolean, + ): Promise { + const normalizedText = text.trim(); + + if (!normalizedText) { + return []; + } + + if (!Number.isFinite(this.maxChunkSize)) { + return [ + { + text: normalizedText, + isProtected, + sourceKind: "forced-split", + }, + ]; + } + + const fallback = new RecursiveCharacterTextSplitter({ + chunkSize: this.maxChunkSize, + chunkOverlap: 0, + keepSeparator: false, + lengthFunction: (value: string) => value.trim().length, + }); + + const chunks = await fallback.splitText(normalizedText); + + return chunks + .map((chunk) => chunk.trim()) + .filter(Boolean) + .map((chunk) => ({ + text: chunk, + isProtected, + sourceKind: "forced-split" as const, + })); + } + + /** + * Merges flat segments into final chunks using `chunkSize` as a soft target. + * + * Needed to keep chunk assembly separate from DOM traversal and to normalize the + * final text output by joining segment text with single spaces. + */ + private mergeSegments(segments: Segment[]): string[] { + const chunks: string[] = []; + let currentParts: string[] = []; + let currentLength = 0; + + const flush = () => { + if (currentParts.length === 0) { + return; + } + + chunks.push(currentParts.join(" ")); + currentParts = []; + currentLength = 0; + }; + + for (const segment of segments) { + const nextLength = + currentLength === 0 + ? segment.text.length + : currentLength + 1 + segment.text.length; + + if (currentParts.length > 0 && nextLength > this.chunkSize) { + flush(); + } + + currentParts.push(segment.text); + currentLength = + currentLength === 0 + ? segment.text.length + : currentLength + 1 + segment.text.length; + } + + flush(); + + return chunks; + } +} diff --git a/textsplitters/index.ts b/textsplitters/index.ts new file mode 100644 index 0000000..2e941ef --- /dev/null +++ b/textsplitters/index.ts @@ -0,0 +1 @@ +export * from "./html_text_splitter"; diff --git a/tsconfig.json b/tsconfig.json index dc91905..1b17cd2 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -3,6 +3,7 @@ "target": "ES2022", "module": "esnext", "moduleResolution": "node", + "types": ["bun-types"], "esModuleInterop": true, "allowSyntheticDefaultImports": true, "strict": true,