mirror of
https://github.com/bendtherules/ask262.git
synced 2026-08-18 13:21:55 +00:00
test(textsplitters): add more tests
- Verify that `splitDocuments` correctly prepends the provided `chunkHeader` to each chunk. - Ensure `splitDocuments` returns an empty array when the source HTML yields no chunks.
This commit is contained in:
@@ -223,6 +223,45 @@ describe("HTMLTextSplitter", () => {
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test("prepends chunkHeader in splitDocuments output", async () => {
|
||||||
|
const splitter = new HTMLTextSplitter({
|
||||||
|
chunkSize: 18,
|
||||||
|
separators: ["h2"],
|
||||||
|
});
|
||||||
|
|
||||||
|
const documents = await splitter.splitDocuments(
|
||||||
|
[
|
||||||
|
new Document({
|
||||||
|
pageContent:
|
||||||
|
"<section><p>Intro text</p><h2>Section Title</h2><p>Body text</p></section>",
|
||||||
|
metadata: { source: "sample.html" },
|
||||||
|
}),
|
||||||
|
],
|
||||||
|
{
|
||||||
|
chunkHeader: "Header: ",
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(documents).toHaveLength(2);
|
||||||
|
expect(documents[0].pageContent).toBe("Header: Intro text");
|
||||||
|
expect(documents[1].pageContent).toBe("Header: Section Title Body text");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("returns no documents when source content produces no chunks", async () => {
|
||||||
|
const splitter = new HTMLTextSplitter({
|
||||||
|
chunkSize: 18,
|
||||||
|
});
|
||||||
|
|
||||||
|
const documents = await splitter.splitDocuments([
|
||||||
|
new Document({
|
||||||
|
pageContent: "<div> </div>",
|
||||||
|
metadata: { source: "empty.html" },
|
||||||
|
}),
|
||||||
|
]);
|
||||||
|
|
||||||
|
expect(documents).toEqual([]);
|
||||||
|
});
|
||||||
|
|
||||||
test("resets part metadata per input document", async () => {
|
test("resets part metadata per input document", async () => {
|
||||||
const splitter = new HTMLTextSplitter({
|
const splitter = new HTMLTextSplitter({
|
||||||
chunkSize: 18,
|
chunkSize: 18,
|
||||||
|
|||||||
Reference in New Issue
Block a user