fix(agent-tools, setup): standardize metadata keys to lowercase and update HTML ingestion

- Renamed `partIndex`/`totalParts` to `partindex`/`totalparts` in `section_retriever.ts`, `spec_retriever.ts`, and document creation logic.
- Adjusted sorting logic to use the new `partindex` field.
- Updated metadata handling and string interpolation to reference the lowercase keys.
- Fixed import path for `HTMLTextSplitter` and moved the file to `setup/textsplitters/HtmlTextSplitter.ts`.
- Added HTML preprocessing utilities (`addNewlinesAfterBlocks`, `convertTablesToMarkdown`) in ingestion workflow.
- Integrated table-to‑markdown conversion and newline insertion to improve text splitting and document structure.

agent_tools/section_retriever.ts

@@ -41,10 +41,10 @@ export function createSectionRetrieverTool(table: Table) {
           .limit(100)
           .toArray();
 
-        // Sort by partIndex to maintain order (nulls last for single-part sections)
+        // Sort by partindex to maintain order (nulls last for single-part sections)
         const sortedResults = results.sort((a: unknown, b: unknown) => {
-          const aIndex = (a as { partIndex?: number }).partIndex ?? Infinity;
-          const bIndex = (b as { partIndex?: number }).partIndex ?? Infinity;
+          const aIndex = (a as { partindex?: number }).partindex ?? Infinity;
+          const bIndex = (b as { partindex?: number }).partindex ?? Infinity;
           return aIndex - bIndex;
         });
 

agent_tools/spec_retriever.ts

@@ -46,8 +46,8 @@ export function createSpecRetrieverTool(
           type: r.type,
           parentsectionid: r.parentsectionid,
           childrensectionids: r.childrensectionids,
-          partIndex: r.partIndex,
-          totalParts: r.totalParts,
+          partindex: r.partindex,
+          totalparts: r.totalparts,
         },
       }));
 
@@ -77,8 +77,8 @@ export function createSpecRetrieverTool(
           const sectionId = meta?.sectionid || "unknown";
           const sectionTitle = meta?.sectiontitle || "unknown";
           const partInfo =
-            meta?.partIndex !== null && meta?.partIndex !== undefined
-              ? ` [part ${(meta.partIndex as number) + 1}/${meta.totalParts}]`
+            meta?.partindex !== null && meta?.partindex !== undefined
+              ? ` [part ${(meta.partindex as number) + 1}/${meta.totalparts}]`
               : "";
           return `--- Section: ${sectionId} | "${sectionTitle}"${partInfo} (score: ${r.score.toFixed(2)}) ---\n${r.document.pageContent}`;
         })