mirror of
https://github.com/bendtherules/ask262.git
synced 2026-08-18 13:21:55 +00:00
fix(agent-tools, setup): standardize metadata keys to lowercase and update HTML ingestion
- Renamed `partIndex`/`totalParts` to `partindex`/`totalparts` in `section_retriever.ts`, `spec_retriever.ts`, and document creation logic. - Adjusted sorting logic to use the new `partindex` field. - Updated metadata handling and string interpolation to reference the lowercase keys. - Fixed import path for `HTMLTextSplitter` and moved the file to `setup/textsplitters/HtmlTextSplitter.ts`. - Added HTML preprocessing utilities (`addNewlinesAfterBlocks`, `convertTablesToMarkdown`) in ingestion workflow. - Integrated table-to‑markdown conversion and newline insertion to improve text splitting and document structure.
This commit is contained in:
+17
-4
@@ -9,7 +9,11 @@ import * as cheerio from "cheerio";
|
||||
import { glob } from "glob";
|
||||
import ora from "ora";
|
||||
import { EMBEDDING_MODEL, SPEC_DIR, STORAGE_DIR } from "../constants";
|
||||
import { HTMLTextSplitter } from "../textsplitters";
|
||||
import { HTMLTextSplitter } from "./textsplitters";
|
||||
import {
|
||||
addNewlinesAfterBlocks,
|
||||
convertTablesToMarkdown,
|
||||
} from "./utils/formatHTMLForIngestion";
|
||||
|
||||
const embeddings = new OllamaEmbeddings({
|
||||
model: EMBEDDING_MODEL,
|
||||
@@ -157,7 +161,12 @@ async function buildSpecDocuments(): Promise<Document[]> {
|
||||
// Second pass: create documents with formatted text
|
||||
for (const [id, section] of sectionMap) {
|
||||
// Parse the stored HTML and replace direct children with placeholders
|
||||
const $section = cheerio.load(`<body>${section.html}</body>`).root();
|
||||
const $ = cheerio.load(`<body>${section.html}</body>`);
|
||||
|
||||
// Convert tables to markdown format for better text extraction
|
||||
convertTablesToMarkdown($);
|
||||
|
||||
const $section = $.root();
|
||||
|
||||
// Find direct children emu-clause elements only
|
||||
$section.children("emu-clause").each((_, childElem) => {
|
||||
@@ -175,6 +184,10 @@ async function buildSpecDocuments(): Promise<Document[]> {
|
||||
// (shouldn't happen with proper HTML structure, but just in case)
|
||||
$section.find("emu-clause").remove();
|
||||
|
||||
// Add newlines after block elements to preserve document structure
|
||||
// This helps the text splitter maintain paragraph/section boundaries
|
||||
addNewlinesAfterBlocks($);
|
||||
|
||||
// Skip sections that only have h1 left (no meaningful content)
|
||||
const hasOnlyH1 =
|
||||
$section.children().length === 1 &&
|
||||
@@ -255,8 +268,8 @@ async function buildSpecDocuments(): Promise<Document[]> {
|
||||
type: "specification",
|
||||
parentsectionid: section.parentId,
|
||||
childrensectionids: section.childrenIds,
|
||||
partIndex: chunk.index,
|
||||
totalParts: chunkData.length,
|
||||
partindex: chunk.index,
|
||||
totalparts: chunkData.length,
|
||||
},
|
||||
}),
|
||||
);
|
||||
|
||||
@@ -0,0 +1,529 @@
|
||||
import { Document } from "@langchain/core/documents";
|
||||
import {
|
||||
RecursiveCharacterTextSplitter,
|
||||
TextSplitter,
|
||||
type TextSplitterChunkHeaderOptions,
|
||||
type TextSplitterParams,
|
||||
} from "@langchain/textsplitters";
|
||||
import * as cheerio from "cheerio";
|
||||
import type { AnyNode } from "domhandler";
|
||||
|
||||
export interface HTMLTextSplitterParams extends TextSplitterParams {
|
||||
/**
|
||||
* CSS selectors that act as preferred structural split points.
|
||||
* They are only used when a subtree needs to be broken down.
|
||||
*/
|
||||
separators?: string[];
|
||||
|
||||
/**
|
||||
* CSS selectors for nodes that should stay atomic unless maxChunkSize forces recursion.
|
||||
*/
|
||||
neverBreakWithin?: string[];
|
||||
|
||||
/**
|
||||
* Absolute hard limit for chunk text length.
|
||||
* Defaults to Infinity if omitted.
|
||||
*/
|
||||
maxChunkSize?: number;
|
||||
}
|
||||
|
||||
interface Segment {
|
||||
text: string;
|
||||
isProtected: boolean;
|
||||
sourceKind: "node" | "forced-split";
|
||||
}
|
||||
|
||||
const BLOCKISH_TAGS = new Set([
|
||||
"address",
|
||||
"article",
|
||||
"aside",
|
||||
"blockquote",
|
||||
"br",
|
||||
"dd",
|
||||
"div",
|
||||
"dl",
|
||||
"dt",
|
||||
"emu-clause",
|
||||
"emu-example",
|
||||
"emu-grammar",
|
||||
"emu-note",
|
||||
"emu-table",
|
||||
"fieldset",
|
||||
"figcaption",
|
||||
"figure",
|
||||
"footer",
|
||||
"form",
|
||||
"h1",
|
||||
"h2",
|
||||
"h3",
|
||||
"h4",
|
||||
"h5",
|
||||
"h6",
|
||||
"header",
|
||||
"hr",
|
||||
"li",
|
||||
"main",
|
||||
"nav",
|
||||
"ol",
|
||||
"p",
|
||||
"pre",
|
||||
"section",
|
||||
"table",
|
||||
"tbody",
|
||||
"td",
|
||||
"th",
|
||||
"thead",
|
||||
"tr",
|
||||
"ul",
|
||||
]);
|
||||
|
||||
/**
|
||||
* Splits HTML input into normalized text chunks.
|
||||
*
|
||||
* Important: this splitter does not preserve HTML markup in output.
|
||||
* It uses parsed HTML structure for traversal and sizing, but emits text only.
|
||||
*
|
||||
* @example
|
||||
* ```ts
|
||||
* const splitter = new HTMLTextSplitter({
|
||||
* chunkSize: 32,
|
||||
* separators: ["h2"],
|
||||
* neverBreakWithin: ["pre"],
|
||||
* });
|
||||
*
|
||||
* const chunks = await splitter.splitText(
|
||||
* "<section><h2>Title</h2><p>Body</p></section>",
|
||||
* );
|
||||
* // => ["Title Body"]
|
||||
* ```
|
||||
*/
|
||||
export class HTMLTextSplitter extends TextSplitter {
|
||||
separators: string[];
|
||||
neverBreakWithin: string[];
|
||||
maxChunkSize: number;
|
||||
|
||||
constructor(fields?: Partial<HTMLTextSplitterParams>) {
|
||||
if ((fields?.chunkOverlap ?? 0) !== 0) {
|
||||
throw new Error("HTMLTextSplitter does not support chunkOverlap.");
|
||||
}
|
||||
|
||||
super({
|
||||
...fields,
|
||||
chunkOverlap: 0,
|
||||
keepSeparator: false,
|
||||
lengthFunction: (text: string) => this.normalizeWhitespace(text).length,
|
||||
});
|
||||
|
||||
this.separators = fields?.separators ?? [];
|
||||
this.neverBreakWithin = fields?.neverBreakWithin ?? [];
|
||||
this.maxChunkSize = fields?.maxChunkSize ?? Number.POSITIVE_INFINITY;
|
||||
}
|
||||
|
||||
/**
|
||||
* Splits HTML input into normalized text chunks.
|
||||
*
|
||||
* Important: this method emits text only. HTML markup is used for traversal
|
||||
* and sizing, but is not preserved in returned chunks.
|
||||
*/
|
||||
async splitText(text: string): Promise<string[]> {
|
||||
const $ = cheerio.load(text);
|
||||
const rootNodes =
|
||||
$("body").length > 0
|
||||
? $("body").contents().toArray()
|
||||
: $.root().contents().toArray();
|
||||
const roots = this.getMeaningfulNodes(rootNodes);
|
||||
|
||||
if (roots.length === 0) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const rootGroups = this.groupChildrenForDecomposition($, roots, false);
|
||||
const segments: Segment[] = [];
|
||||
|
||||
for (const group of rootGroups) {
|
||||
segments.push(...(await this.collectSegmentsFromNodes($, group, false)));
|
||||
}
|
||||
|
||||
return this.mergeSegments(segments);
|
||||
}
|
||||
|
||||
/**
|
||||
* Splits HTML documents into text-only child documents.
|
||||
*
|
||||
* Important: child document `pageContent` contains normalized text, not HTML.
|
||||
* Parent metadata is preserved and per-document part metadata is added.
|
||||
*/
|
||||
async splitDocuments(
|
||||
documents: Document[],
|
||||
chunkHeaderOptions?: TextSplitterChunkHeaderOptions,
|
||||
): Promise<Document[]> {
|
||||
const splitDocs: Document[] = [];
|
||||
const chunkHeader = chunkHeaderOptions?.chunkHeader ?? "";
|
||||
|
||||
for (const document of documents) {
|
||||
const chunks = await this.splitText(document.pageContent);
|
||||
|
||||
for (const [index, chunk] of chunks.entries()) {
|
||||
splitDocs.push(
|
||||
new Document({
|
||||
pageContent: `${chunkHeader}${chunk}`,
|
||||
metadata: {
|
||||
...document.metadata,
|
||||
partindex: index,
|
||||
totalparts: chunks.length,
|
||||
},
|
||||
}),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
return splitDocs;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns true when a tag node matches any configured CSS selector.
|
||||
*
|
||||
* Needed so separator and protected-node checks share the same selector logic
|
||||
* and non-tag nodes are ignored safely.
|
||||
*/
|
||||
private matchesAny(
|
||||
$: cheerio.CheerioAPI,
|
||||
node: AnyNode,
|
||||
selectors: string[],
|
||||
): boolean {
|
||||
if (node.type !== "tag") {
|
||||
return false;
|
||||
}
|
||||
|
||||
for (const selector of selectors) {
|
||||
if ($(node).is(selector)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Extracts the canonical text value for a node.
|
||||
*
|
||||
* Needed so sizing and emitted output use the same trimmed text semantics for
|
||||
* both element nodes and raw text nodes.
|
||||
*/
|
||||
private getNodeText($: cheerio.CheerioAPI, node: AnyNode): string {
|
||||
if (node.type === "text") {
|
||||
return node.data ?? "";
|
||||
}
|
||||
|
||||
return $(node).text();
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns whether a node should introduce a visible boundary when adjacent
|
||||
* text is concatenated.
|
||||
*
|
||||
* Needed so inline elements such as `span` do not get synthetic spaces while
|
||||
* block-ish elements still remain readable in emitted text.
|
||||
*/
|
||||
private isBlockishNode(node: AnyNode): boolean {
|
||||
return node.type === "tag" && BLOCKISH_TAGS.has(node.name);
|
||||
}
|
||||
|
||||
/**
|
||||
* Collapses internal whitespace and trims leading/trailing whitespace.
|
||||
* Preserves newlines between block elements while normalizing spaces.
|
||||
*
|
||||
* Needed so text extraction can preserve raw adjacency first and normalize
|
||||
* only once after boundary-aware joining.
|
||||
*/
|
||||
private normalizeWhitespace(text: string): string {
|
||||
return text
|
||||
.replace(/\n[ \t]+/g, "\n") // Remove leading spaces after newlines
|
||||
.replace(/[ \t]+\n/g, "\n") // Remove trailing spaces before newlines
|
||||
.replace(/\n{3,}/g, "\n\n") // Collapse 3+ newlines to 2
|
||||
.replace(/[ \t]{2,}/g, " ") // Collapse multiple spaces/tabs to one
|
||||
.trim();
|
||||
}
|
||||
|
||||
/**
|
||||
* Joins multiple nodes into the normalized text the splitter actually emits.
|
||||
*
|
||||
* Needed so grouped nodes are measured and emitted with the same spacing rules
|
||||
* used later during chunk merging.
|
||||
*/
|
||||
private getNodesText($: cheerio.CheerioAPI, nodes: AnyNode[]): string {
|
||||
let result = "";
|
||||
let previousNode: AnyNode | null = null;
|
||||
|
||||
for (const node of nodes) {
|
||||
const text = this.getNodeText($, node);
|
||||
if (!text) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (
|
||||
result.length > 0 &&
|
||||
previousNode &&
|
||||
(this.isBlockishNode(previousNode) || this.isBlockishNode(node))
|
||||
) {
|
||||
result += "\n"; // Add newline between block elements for better structure
|
||||
}
|
||||
|
||||
result += text;
|
||||
previousNode = node;
|
||||
}
|
||||
|
||||
return this.normalizeWhitespace(result);
|
||||
}
|
||||
|
||||
/**
|
||||
* Filters child nodes down to the recursion units the splitter can handle.
|
||||
*
|
||||
* Needed to recurse through DOM structure without carrying empty whitespace-only
|
||||
* text nodes or unsupported node types through the algorithm.
|
||||
*/
|
||||
private getMeaningfulNodes(nodes: AnyNode[]): AnyNode[] {
|
||||
return nodes.filter((node) => {
|
||||
if (node.type === "text") {
|
||||
return (node.data ?? "").trim().length > 0;
|
||||
}
|
||||
|
||||
return node.type === "tag";
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the child nodes that are meaningful recursion units.
|
||||
*
|
||||
* Needed to recurse through DOM structure without carrying empty whitespace-only
|
||||
* text nodes or unsupported node types through the algorithm.
|
||||
*/
|
||||
private getChildNodesForRecursion(node: AnyNode): AnyNode[] {
|
||||
if (!("children" in node) || !Array.isArray(node.children)) {
|
||||
return [];
|
||||
}
|
||||
|
||||
return this.getMeaningfulNodes(node.children);
|
||||
}
|
||||
|
||||
/**
|
||||
* Groups child nodes so separator nodes start a new group.
|
||||
*
|
||||
* Needed because separators represent preferred section boundaries. Grouping
|
||||
* makes "separator plus following content" explicit during recursion.
|
||||
*/
|
||||
private groupChildrenForDecomposition(
|
||||
$: cheerio.CheerioAPI,
|
||||
childNodes: AnyNode[],
|
||||
inProtectedTree: boolean,
|
||||
): AnyNode[][] {
|
||||
if (childNodes.length === 0) {
|
||||
return [];
|
||||
}
|
||||
|
||||
if (inProtectedTree || this.separators.length === 0) {
|
||||
return [childNodes];
|
||||
}
|
||||
|
||||
const groups: AnyNode[][] = [];
|
||||
let currentGroup: AnyNode[] = [];
|
||||
|
||||
const flush = () => {
|
||||
if (currentGroup.length > 0) {
|
||||
groups.push(currentGroup);
|
||||
currentGroup = [];
|
||||
}
|
||||
};
|
||||
|
||||
for (const child of childNodes) {
|
||||
if (this.matchesAny($, child, this.separators)) {
|
||||
flush();
|
||||
}
|
||||
|
||||
currentGroup.push(child);
|
||||
}
|
||||
|
||||
flush();
|
||||
|
||||
return groups;
|
||||
}
|
||||
|
||||
/**
|
||||
* Collects text segments for a single node or grouped sibling nodes.
|
||||
*
|
||||
* Needed so separator-led groups can be treated as one structural unit before
|
||||
* the splitter decides whether it must recurse deeper.
|
||||
*/
|
||||
private async collectSegmentsFromNodes(
|
||||
$: cheerio.CheerioAPI,
|
||||
nodes: AnyNode[],
|
||||
inProtectedAncestor: boolean,
|
||||
): Promise<Segment[]> {
|
||||
const text = this.getNodesText($, nodes);
|
||||
|
||||
if (!text) {
|
||||
return [];
|
||||
}
|
||||
|
||||
if (nodes.length > 1) {
|
||||
const startsWithSeparator = this.matchesAny($, nodes[0], this.separators);
|
||||
|
||||
if (
|
||||
text.length <= this.chunkSize ||
|
||||
(startsWithSeparator && text.length <= this.maxChunkSize)
|
||||
) {
|
||||
return [{ text, isProtected: false, sourceKind: "node" }];
|
||||
}
|
||||
|
||||
const segments: Segment[] = [];
|
||||
for (const node of nodes) {
|
||||
segments.push(
|
||||
...(await this.collectSegmentsFromNodes(
|
||||
$,
|
||||
[node],
|
||||
inProtectedAncestor,
|
||||
)),
|
||||
);
|
||||
}
|
||||
|
||||
return segments.length > 0
|
||||
? segments
|
||||
: await this.forceSplitText(text, false);
|
||||
}
|
||||
|
||||
const [node] = nodes;
|
||||
const isProtected =
|
||||
!inProtectedAncestor && this.matchesAny($, node, this.neverBreakWithin);
|
||||
const childNodes = this.getChildNodesForRecursion(node);
|
||||
|
||||
if (childNodes.length > 0) {
|
||||
const childGroups = this.groupChildrenForDecomposition(
|
||||
$,
|
||||
childNodes,
|
||||
isProtected || inProtectedAncestor,
|
||||
);
|
||||
const normalizedChildText = this.getNodesText($, childGroups.flat());
|
||||
|
||||
if (normalizedChildText.length <= this.chunkSize) {
|
||||
return [{ text: normalizedChildText, isProtected, sourceKind: "node" }];
|
||||
}
|
||||
|
||||
if (isProtected && normalizedChildText.length <= this.maxChunkSize) {
|
||||
return [{ text: normalizedChildText, isProtected, sourceKind: "node" }];
|
||||
}
|
||||
|
||||
const segments: Segment[] = [];
|
||||
for (const group of childGroups) {
|
||||
segments.push(
|
||||
...(await this.collectSegmentsFromNodes(
|
||||
$,
|
||||
group,
|
||||
isProtected || inProtectedAncestor,
|
||||
)),
|
||||
);
|
||||
}
|
||||
|
||||
return segments.length > 0
|
||||
? segments
|
||||
: await this.forceSplitText(normalizedChildText, isProtected);
|
||||
}
|
||||
|
||||
if (text.length <= this.chunkSize) {
|
||||
return [{ text, isProtected, sourceKind: "node" }];
|
||||
}
|
||||
|
||||
if (isProtected && text.length <= this.maxChunkSize) {
|
||||
return [{ text, isProtected, sourceKind: "node" }];
|
||||
}
|
||||
|
||||
return this.forceSplitText(text, isProtected);
|
||||
}
|
||||
|
||||
/**
|
||||
* Applies the final plain-text fallback when a node cannot be decomposed further.
|
||||
*
|
||||
* Needed to enforce a finite hard cap with `RecursiveCharacterTextSplitter`
|
||||
* once DOM structure is exhausted.
|
||||
*/
|
||||
private async forceSplitText(
|
||||
text: string,
|
||||
isProtected: boolean,
|
||||
): Promise<Segment[]> {
|
||||
const normalizedText = text.trim();
|
||||
|
||||
if (!normalizedText) {
|
||||
return [];
|
||||
}
|
||||
|
||||
if (!Number.isFinite(this.maxChunkSize)) {
|
||||
return [
|
||||
{
|
||||
text: normalizedText,
|
||||
isProtected,
|
||||
sourceKind: "forced-split",
|
||||
},
|
||||
];
|
||||
}
|
||||
|
||||
const fallback = new RecursiveCharacterTextSplitter({
|
||||
chunkSize: this.maxChunkSize,
|
||||
chunkOverlap: 0,
|
||||
keepSeparator: false,
|
||||
lengthFunction: (value: string) => value.trim().length,
|
||||
});
|
||||
|
||||
const chunks = await fallback.splitText(normalizedText);
|
||||
|
||||
return chunks
|
||||
.map((chunk) => chunk.trim())
|
||||
.filter(Boolean)
|
||||
.map((chunk) => ({
|
||||
text: chunk,
|
||||
isProtected,
|
||||
sourceKind: "forced-split" as const,
|
||||
}));
|
||||
}
|
||||
|
||||
/**
|
||||
* Merges flat segments into final chunks using `chunkSize` as a soft target.
|
||||
*
|
||||
* Needed to keep chunk assembly separate from DOM traversal and to normalize the
|
||||
* final text output by joining segment text with single spaces.
|
||||
*/
|
||||
private mergeSegments(segments: Segment[]): string[] {
|
||||
const chunks: string[] = [];
|
||||
let currentParts: string[] = [];
|
||||
let currentLength = 0;
|
||||
|
||||
const flush = () => {
|
||||
if (currentParts.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
chunks.push(currentParts.join(" "));
|
||||
currentParts = [];
|
||||
currentLength = 0;
|
||||
};
|
||||
|
||||
for (const segment of segments) {
|
||||
const nextLength =
|
||||
currentLength === 0
|
||||
? segment.text.length
|
||||
: currentLength + 1 + segment.text.length;
|
||||
|
||||
if (currentParts.length > 0 && nextLength > this.chunkSize) {
|
||||
flush();
|
||||
}
|
||||
|
||||
currentParts.push(segment.text);
|
||||
currentLength =
|
||||
currentLength === 0
|
||||
? segment.text.length
|
||||
: currentLength + 1 + segment.text.length;
|
||||
}
|
||||
|
||||
flush();
|
||||
|
||||
return chunks;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
export * from "./HtmlTextSplitter";
|
||||
@@ -0,0 +1,120 @@
|
||||
import type * as cheerio from "cheerio";
|
||||
|
||||
/**
|
||||
* List of block-level HTML elements that should have newlines added after them
|
||||
* to preserve document structure during text extraction.
|
||||
*/
|
||||
export const BLOCK_ELEMENTS = [
|
||||
"p",
|
||||
"div",
|
||||
"h1",
|
||||
"h2",
|
||||
"h3",
|
||||
"h4",
|
||||
"h5",
|
||||
"h6",
|
||||
"li",
|
||||
"td",
|
||||
"th",
|
||||
"pre",
|
||||
"blockquote",
|
||||
"section",
|
||||
"emu-clause",
|
||||
"emu-note",
|
||||
"emu-example",
|
||||
"emu-table",
|
||||
"figure",
|
||||
"figcaption",
|
||||
"ul",
|
||||
"ol",
|
||||
"dl",
|
||||
"dt",
|
||||
"dd",
|
||||
];
|
||||
|
||||
/**
|
||||
* Converts HTML tables to markdown table format.
|
||||
* This preserves table structure when extracting text from HTML.
|
||||
* Replaces the original table with a <pre> element containing the markdown.
|
||||
*
|
||||
* @param $ - Cheerio API instance
|
||||
*/
|
||||
export function convertTablesToMarkdown($: cheerio.CheerioAPI): void {
|
||||
$("table, emu-table").each((_, tableElem) => {
|
||||
const $table = $(tableElem);
|
||||
const rows: string[][] = [];
|
||||
|
||||
// Extract header rows
|
||||
$table.find("thead tr").each((_, rowElem) => {
|
||||
const row: string[] = [];
|
||||
$(rowElem)
|
||||
.find("th, td")
|
||||
.each((_, cellElem) => {
|
||||
row.push($(cellElem).text().trim().replace(/\|/g, "\\|"));
|
||||
});
|
||||
if (row.length > 0) rows.push(row);
|
||||
});
|
||||
|
||||
// Extract body rows
|
||||
$table.find("tbody tr, tr").each((_, rowElem) => {
|
||||
// Skip if already processed as header
|
||||
if ($(rowElem).parent("thead").length > 0) return;
|
||||
|
||||
const row: string[] = [];
|
||||
$(rowElem)
|
||||
.find("td, th")
|
||||
.each((_, cellElem) => {
|
||||
row.push($(cellElem).text().trim().replace(/\|/g, "\\|"));
|
||||
});
|
||||
if (row.length > 0) rows.push(row);
|
||||
});
|
||||
|
||||
if (rows.length === 0) return;
|
||||
|
||||
// Determine max columns
|
||||
const maxCols = Math.max(...rows.map((r) => r.length));
|
||||
|
||||
// Build markdown table
|
||||
const mdLines: string[] = [];
|
||||
|
||||
// Header row
|
||||
if (rows.length > 0) {
|
||||
const header = rows[0].concat(Array(maxCols - rows[0].length).fill(""));
|
||||
mdLines.push("| " + header.join(" | ") + " |");
|
||||
}
|
||||
|
||||
// Separator
|
||||
mdLines.push("|" + Array(maxCols).fill(" --- ").join("|") + "|");
|
||||
|
||||
// Data rows (skip header if we have more rows)
|
||||
const dataRows = rows.length > 1 ? rows.slice(1) : [];
|
||||
for (const row of dataRows) {
|
||||
const padded = row.concat(Array(maxCols - row.length).fill(""));
|
||||
mdLines.push("| " + padded.join(" | ") + " |");
|
||||
}
|
||||
|
||||
// Replace table with markdown
|
||||
const markdown = mdLines.join("\n");
|
||||
$table.replaceWith($(`<pre class="table-markdown">${markdown}</pre>`));
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Adds newlines after specified block elements to preserve document structure.
|
||||
* This helps text splitters maintain paragraph/section boundaries.
|
||||
*
|
||||
* @param $ - Cheerio API instance
|
||||
* @param elements - Array of element tag names to add newlines after
|
||||
*/
|
||||
export function addNewlinesAfterBlocks(
|
||||
$: cheerio.CheerioAPI,
|
||||
elements: string[] = BLOCK_ELEMENTS,
|
||||
): void {
|
||||
for (const tag of elements) {
|
||||
$.root()
|
||||
.find(tag)
|
||||
.each((_, el) => {
|
||||
$(el).append("\n");
|
||||
});
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user