import type * as cheerio from "cheerio"; /** * List of block-level HTML elements that should have newlines added after them * to preserve document structure during text extraction. */ export const BLOCK_ELEMENTS = [ "p", "div", "h1", "h2", "h3", "h4", "h5", "h6", "li", "td", "th", "pre", "blockquote", "section", "emu-clause", "emu-note", "emu-example", "emu-table", "figure", "figcaption", "ul", "ol", "dl", "dt", "dd", ]; /** * Converts HTML tables to markdown table format. * This preserves table structure when extracting text from HTML. * Replaces the original table with a
element containing the markdown.
*
* @param $ - Cheerio API instance
*/
export function convertTablesToMarkdown($: cheerio.CheerioAPI): void {
$("table, emu-table").each((_, tableElem) => {
const $table = $(tableElem);
const rows: string[][] = [];
// Extract header rows
$table.find("thead tr").each((_, rowElem) => {
const row: string[] = [];
$(rowElem)
.find("th, td")
.each((_, cellElem) => {
row.push($(cellElem).text().trim().replace(/\|/g, "\\|"));
});
if (row.length > 0) rows.push(row);
});
// Extract body rows
$table.find("tbody tr, tr").each((_, rowElem) => {
// Skip if already processed as header
if ($(rowElem).parent("thead").length > 0) return;
const row: string[] = [];
$(rowElem)
.find("td, th")
.each((_, cellElem) => {
row.push($(cellElem).text().trim().replace(/\|/g, "\\|"));
});
if (row.length > 0) rows.push(row);
});
if (rows.length === 0) return;
// Determine max columns
const maxCols = Math.max(...rows.map((r) => r.length));
// Build markdown table
const mdLines: string[] = [];
// Header row
if (rows.length > 0) {
const header = rows[0].concat(Array(maxCols - rows[0].length).fill(""));
mdLines.push("| " + header.join(" | ") + " |");
}
// Separator
mdLines.push("|" + Array(maxCols).fill(" --- ").join("|") + "|");
// Data rows (skip header if we have more rows)
const dataRows = rows.length > 1 ? rows.slice(1) : [];
for (const row of dataRows) {
const padded = row.concat(Array(maxCols - row.length).fill(""));
mdLines.push("| " + padded.join(" | ") + " |");
}
// Replace table with markdown
const markdown = mdLines.join("\n");
$table.replaceWith($(`${markdown}`));
});
}
/**
* Adds newlines after specified block elements to preserve document structure.
* This helps text splitters maintain paragraph/section boundaries.
*
* @param $ - Cheerio API instance
* @param elements - Array of element tag names to add newlines after
*/
export function addNewlinesAfterBlocks(
$: cheerio.CheerioAPI,
elements: string[] = BLOCK_ELEMENTS,
): void {
for (const tag of elements) {
$.root()
.find(tag)
.each((_, el) => {
$(el).append("\n");
});
}
}