Skip to content

Commit 82baa4a

Browse files
committed
Fix test comments and update TokenTextSplitterTest for enhanced metadata
- Correct comment: 'excluding' -> 'including' chunk-specific fields - Update TokenTextSplitterTest to handle new metadata fields - Ensure all tests pass with enhanced TextSplitter functionality Signed-off-by: Hyeri1ee <[email protected]>
1 parent f7c1f5e commit 82baa4a

File tree

2 files changed

+17
-4
lines changed

2 files changed

+17
-4
lines changed

spring-ai-commons/src/test/java/org/springframework/ai/transformer/splitter/TextSplitterTests.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -82,7 +82,7 @@ public void testSplitText() {
8282
assertThat(chunks.get(3).getText())
8383
.isEqualTo("choose. It isn’t the lack of an exit, but the abundance of exits that is so disorienting.");
8484

85-
// Verify that the original metadata is copied to all chunks (excluding
85+
// Verify that the original metadata is copied to all chunks (including
8686
// chunk-specific fields)
8787
assertThat(chunks.get(0).getMetadata()).containsKeys("key1", "key2", "parent_document_id", "chunk_index",
8888
"total_chunks");

spring-ai-commons/src/test/java/org/springframework/ai/transformer/splitter/TokenTextSplitterTest.java

Lines changed: 16 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -104,9 +104,22 @@ public void testTokenTextSplitterBuilderWithAllFields() {
104104
assertThat(chunks.get(4).getText()).isEqualTo("It isn’t the lack of an exit, but");
105105
assertThat(chunks.get(5).getText()).isEqualTo("the abundance of exits that is so disorienting");
106106

107-
// Verify that the same, merged metadata is copied to all chunks.
108-
assertThat(chunks.get(0).getMetadata()).isEqualTo(chunks.get(1).getMetadata());
109-
assertThat(chunks.get(2).getMetadata()).isEqualTo(chunks.get(3).getMetadata());
107+
// Verify that the original metadata is copied to all chunks (including
108+
// chunk-specific fields)
109+
assertThat(chunks.get(0).getMetadata()).containsKeys("key1", "key2", "parent_document_id", "chunk_index",
110+
"total_chunks");
111+
assertThat(chunks.get(1).getMetadata()).containsKeys("key1", "key2", "parent_document_id", "chunk_index",
112+
"total_chunks");
113+
assertThat(chunks.get(2).getMetadata()).containsKeys("key2", "key3", "parent_document_id", "chunk_index",
114+
"total_chunks");
115+
assertThat(chunks.get(3).getMetadata()).containsKeys("key2", "key3", "parent_document_id", "chunk_index",
116+
"total_chunks");
117+
118+
// Verify chunk indices are correct
119+
assertThat(chunks.get(0).getMetadata().get("chunk_index")).isEqualTo(0);
120+
assertThat(chunks.get(1).getMetadata().get("chunk_index")).isEqualTo(1);
121+
assertThat(chunks.get(2).getMetadata().get("chunk_index")).isEqualTo(0);
122+
assertThat(chunks.get(3).getMetadata().get("chunk_index")).isEqualTo(1);
110123

111124
assertThat(chunks.get(0).getMetadata()).containsKeys("key1", "key2").doesNotContainKeys("key3");
112125
assertThat(chunks.get(2).getMetadata()).containsKeys("key2", "key3").doesNotContainKeys("key1");

0 commit comments

Comments
 (0)