mirror of
https://github.com/bendtherules/ask262.git
synced 2026-08-18 13:21:55 +00:00
plan(HTML‑to‑Markdown): add sample code
This commit is contained in:
@@ -44,23 +44,12 @@ const DEFAULT_CONFIG: FormatConfig = {
|
||||
};
|
||||
```
|
||||
|
||||
## Transformation Order (leaf-to-parent)
|
||||
## Transformation Order
|
||||
|
||||
1. **Inline leaves** (processed first, no children affected):
|
||||
- `emu-xref a[href]` → `<span class="link-markdown">[text](#hash)</span>`. Target the inner `<a>` tag (always has href, unlike `emu-xref` which may lack it).
|
||||
- Strip filename prefix from href (e.g., `abstract-operations.html#sec-tonumber` → `#sec-tonumber`)
|
||||
- Discard `e-user-code` class.
|
||||
- Replace `<a>` inside `emu-xref` (leaving wrapper `emu-xref` to be ignored by text splitter)
|
||||
- `var`, `emu-val`, `emu-const`, `code`, inline `emu-eqn` → `<span class="inline-code">`text`</span>`. Strip inner elements, plain text only.
|
||||
2. **Block leaves**:
|
||||
- `pre>code` → `<pre class="code-markdown">```language\ntext\n```</pre>`. Language from `class` attribute. Strip hljs spans. Plain text.
|
||||
- block `emu-eqn` → `<pre class="code-markdown">```\ntext\n```</pre>`. No language tag. Strip inner elements.
|
||||
3. **Structural parents** (children already resolved):
|
||||
- `emu-grammar` → `<pre class="code-markdown">```bnf\ntext\n```</pre>`. Extract text from grammar inner tags (emu-nt, emu-t, etc.).
|
||||
- `ol` → `<pre class="list-markdown">1. item\n 1. nested</pre>`. Nested via 2-space indentation.
|
||||
- `ul` → `<pre class="list-markdown">- item\n - nested</pre>`. Nested via 2-space indentation.
|
||||
- `table`, `emu-table` → `<pre class="table-markdown">...existing markdown...</pre>`. Existing `convertTablesToMarkdown` works unchanged — `.text()` already extracts markdown text from pre-processed spans.
|
||||
4. **`addNewlinesAfterBlocks($)`** (existing, unchanged — operates on remaining DOM elements)
|
||||
0. **`addNewlinesAfterBlocks($)`** — global newline injection (existing function, runs first)
|
||||
1. **Inline leaves**: `emu-xref a[href]` → links, `var`/`emu-val`/`emu-const` → inline code
|
||||
2. **Block leaves**: `pre>code` → fenced code, `emu-eqn` → fenced code
|
||||
3. **Structural parents**: `emu-grammar` → fenced bnf (uses `.text()` with pre-injected newlines), `ol/ul` → markdown lists, `table` → markdown tables
|
||||
|
||||
## Implementation Details
|
||||
|
||||
@@ -95,6 +84,216 @@ Each function receives `cheerio.CheerioAPI` and modifies the DOM in-place.
|
||||
|
||||
6. **`formatForIngestion($, config?)`** — orchestrator, calls all in order
|
||||
|
||||
## Critical Implementation Code
|
||||
|
||||
### Config interface
|
||||
|
||||
```typescript
|
||||
export interface FormatConfig {
|
||||
codeBlocks: {
|
||||
block: string[];
|
||||
inline: string[];
|
||||
grammar: string[];
|
||||
};
|
||||
links: string[];
|
||||
lists: {
|
||||
ordered: string[];
|
||||
unordered: string[];
|
||||
};
|
||||
tables: string[];
|
||||
}
|
||||
|
||||
export const DEFAULT_CONFIG: FormatConfig = {
|
||||
codeBlocks: {
|
||||
block: ["pre>code", "emu-eqn:not([class*='inline'])"],
|
||||
inline: ["var", "emu-val", "emu-const", "emu-eqn.inline"],
|
||||
grammar: ["emu-grammar"],
|
||||
},
|
||||
links: ["emu-xref a"],
|
||||
lists: {
|
||||
ordered: ["ol"],
|
||||
unordered: ["ul"],
|
||||
},
|
||||
tables: ["table", "emu-table"],
|
||||
};
|
||||
```
|
||||
|
||||
### `convertLinksToMarkdown` — filename stripping
|
||||
|
||||
```typescript
|
||||
export function convertLinksToMarkdown(
|
||||
$: cheerio.CheerioAPI,
|
||||
selector: string = "emu-xref a[href]",
|
||||
): void {
|
||||
$(selector).each((_, elem) => {
|
||||
const $a = $(elem);
|
||||
const href = $a.attr("href") ?? "";
|
||||
|
||||
// Skip external links
|
||||
if (href.startsWith("http")) return;
|
||||
|
||||
// Strip filename prefix: "abstract-operations.html#sec-tonumber" → "#sec-tonumber"
|
||||
const hash = href.includes("#") ? `#${href.split("#")[1]}` : href;
|
||||
const text = $a.text().trim();
|
||||
|
||||
if (!text || !hash) return;
|
||||
|
||||
$a.replaceWith(`<span class="link-markdown">[${text}](${hash})</span>`);
|
||||
});
|
||||
}
|
||||
```
|
||||
|
||||
### `convertInlineCodeToMarkdown` — backtick wrapping
|
||||
|
||||
```typescript
|
||||
export function convertInlineCodeToMarkdown(
|
||||
$: cheerio.CheerioAPI,
|
||||
tags: string[],
|
||||
): void {
|
||||
for (const tag of tags) {
|
||||
$(tag).each((_, elem) => {
|
||||
// Skip <code> inside <pre> (handled by block converter)
|
||||
if (elem.name === "code" && $(elem).parent("pre").length > 0) return;
|
||||
|
||||
const text = $(elem).text().trim();
|
||||
if (!text) return;
|
||||
$(elem).replaceWith(`<span class="inline-code">\`${text}\`</span>`);
|
||||
});
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### `convertBlockCodeToMarkdown` — fenced code blocks
|
||||
|
||||
```typescript
|
||||
function extractLanguage(codeClass: string | undefined): string {
|
||||
// "javascript hljs" → "javascript", "python hljs" → "python"
|
||||
if (!codeClass) return "";
|
||||
const match = codeClass.match(/^(\w+)/);
|
||||
return match?.[1] ?? "";
|
||||
}
|
||||
|
||||
export function convertBlockCodeToMarkdown(
|
||||
$: cheerio.CheerioAPI,
|
||||
tags: string[],
|
||||
): void {
|
||||
// Handle pre>code (JS code examples)
|
||||
$("pre > code").each((_, code) => {
|
||||
const lang = extractLanguage($(code).attr("class"));
|
||||
const text = $(code).text().trim();
|
||||
const pre = $(code).parent("pre");
|
||||
pre.replaceWith(
|
||||
$(`<pre class="code-markdown">\`\`\`${lang}\n${text}\n\`\`\`</pre>`),
|
||||
);
|
||||
});
|
||||
|
||||
// Handle standalone emu-eqn (block-level equations)
|
||||
$("emu-eqn:not([class*='inline'])").each((_, eqn) => {
|
||||
const text = $(eqn).text().trim();
|
||||
$(eqn).replaceWith(
|
||||
$(`<pre class="code-markdown">\`\`\`\n${text}\n\`\`\`</pre>`),
|
||||
);
|
||||
});
|
||||
}
|
||||
```
|
||||
|
||||
### `convertListsToMarkdown` — recursive list processing
|
||||
|
||||
```typescript
|
||||
function listToMarkdown(
|
||||
$: cheerio.CheerioAPI,
|
||||
elem: cheerio.Element,
|
||||
depth: number = 0,
|
||||
): string {
|
||||
const $elem = $(elem);
|
||||
const isOrdered = elem.name === "ol";
|
||||
const indent = " ".repeat(depth);
|
||||
const lines: string[] = [];
|
||||
|
||||
$elem.children("li").each((i, li) => {
|
||||
const prefix = isOrdered ? `${i + 1}. ` : "- ";
|
||||
const $li = $(li);
|
||||
|
||||
// Recurse into nested lists first (depth-first)
|
||||
$li.children("ol, ul").each((_, nested) => {
|
||||
const nestedMarkdown = listToMarkdown($, nested, depth + 1);
|
||||
$(nested).replaceWith($(`<pre class="list-processed">${nestedMarkdown}</pre>`));
|
||||
});
|
||||
|
||||
// Now get full text (nested lists are replaced with <pre> containing markdown)
|
||||
const itemText = $li.text().trim().replace(/\s+/g, " ");
|
||||
lines.push(`${indent}${prefix}${itemText}`);
|
||||
});
|
||||
|
||||
return lines.join("\n");
|
||||
}
|
||||
|
||||
export function convertListsToMarkdown(
|
||||
$: cheerio.CheerioAPI,
|
||||
ordered: string[] = ["ol"],
|
||||
unordered: string[] = ["ul"],
|
||||
): void {
|
||||
const selectors = [...ordered, ...unordered].join(", ");
|
||||
// Process from outermost — recursion handles depth-first nesting
|
||||
$(selectors).each((_, elem) => {
|
||||
// Skip if already processed (parent already handled this)
|
||||
if ($(elem).hasClass("list-markdown")) return;
|
||||
const markdown = listToMarkdown($, elem, 0);
|
||||
$(elem).replaceWith($(`<pre class="list-markdown">${markdown}</pre>`));
|
||||
});
|
||||
}
|
||||
```
|
||||
|
||||
Key insight: recursion goes depth-first. When processing an `li`, nested `ol`/`ul` children are first replaced with `<pre class="list-processed">` containing their markdown. Then `$li.text()` includes that markdown. Outer `$(selectors).each(...)` skips already-processed elements.
|
||||
|
||||
### `convertGrammarToMarkdown` — grammar production text
|
||||
|
||||
`addNewlinesAfterBlocks` runs first globally, injecting `\n` after `emu-production` and `emu-rhs` (added to `BLOCK_ELEMENTS`). Then `.text()` on `emu-grammar` naturally preserves production/alternative boundaries.
|
||||
|
||||
```typescript
|
||||
export function convertGrammarToMarkdown(
|
||||
$: cheerio.CheerioAPI,
|
||||
selector: string = "emu-grammar",
|
||||
): void {
|
||||
$(selector).each((_, grammar) => {
|
||||
const text = $(grammar).text().trim();
|
||||
$(grammar).replaceWith(
|
||||
$(`<pre class="code-markdown">\`\`\`bnf\n${text}\n\`\`\`</pre>`),
|
||||
);
|
||||
});
|
||||
}
|
||||
```
|
||||
|
||||
Add `"emu-production"` and `"emu-rhs"` to `BLOCK_ELEMENTS` array.
|
||||
|
||||
### `formatForIngestion` — orchestrator
|
||||
|
||||
```typescript
|
||||
export function formatForIngestion(
|
||||
$: cheerio.CheerioAPI,
|
||||
config: Partial<FormatConfig> = {},
|
||||
): void {
|
||||
const cfg = { ...DEFAULT_CONFIG, ...config };
|
||||
|
||||
// 1. Inject newlines globally (affects li, p, pre, emu-production, emu-rhs, etc.)
|
||||
addNewlinesAfterBlocks($);
|
||||
|
||||
// 2. Inline leaves (links first — must be before lists/code destroy DOM)
|
||||
convertLinksToMarkdown($, cfg.links.join(", "));
|
||||
|
||||
// 3. Inline code
|
||||
convertInlineCodeToMarkdown($, cfg.codeBlocks.inline);
|
||||
|
||||
// 4. Block leaves (fenced code)
|
||||
convertBlockCodeToMarkdown($, cfg.codeBlocks.block);
|
||||
|
||||
// 5. Structural parents (grammar, lists, tables)
|
||||
convertGrammarToMarkdown($, cfg.codeBlocks.grammar.join(", "));
|
||||
convertListsToMarkdown($, cfg.lists.ordered, cfg.lists.unordered);
|
||||
convertTablesToMarkdown($);
|
||||
}
|
||||
```
|
||||
|
||||
### Modify existing functions
|
||||
|
||||
- **`convertTablesToMarkdown($)`**: No changes needed. `.text()` already extracts markdown text from pre-processed `<span>` elements.
|
||||
|
||||
Reference in New Issue
Block a user