merged main

Author: dantelmomsft

.github/workflows/nightly-jobs.yaml

@@ -1,4 +1,4 @@
-name: Nightly Jobs
+name: Nightly and CICD Jobs
 
 on:
   pull_request:
@@ -61,6 +61,12 @@ jobs:
           java-version: '17'
           cache: 'maven'
 
+      - name: Verify Indexer project
+        run: |
+          echo "Testing indexer project."
+          cd ./app/indexer
+          ./mvnw test
+
       - name: Build Spring Boot App
         run: |
           echo "Building Spring Boot app."

app/indexer/core/src/main/java/com/microsoft/openai/samples/indexer/parser/TextSplitter.java

@@ -17,11 +17,18 @@ public class TextSplitter {
     public TextSplitter(boolean verbose) {
         this.sentenceEndings = new ArrayList<>();
         this.sentenceEndings.add(".");
+        this.sentenceEndings.add("。");
+        this.sentenceEndings.add("．");
         this.sentenceEndings.add("!");
         this.sentenceEndings.add("?");
+        this.sentenceEndings.add("‼");
+        this.sentenceEndings.add("⁇");
+        this.sentenceEndings.add("⁈");
+        this.sentenceEndings.add("⁉");
 
         this.wordBreaks = new ArrayList<>();
         this.wordBreaks.add(",");
+        this.wordBreaks.add("、");
         this.wordBreaks.add(";");
         this.wordBreaks.add(":");
         this.wordBreaks.add(" ");
@@ -49,6 +56,12 @@ public List<SplitPage> splitPages(List<Page> pages) {
         int length = allText.length();
         int start = 0;
         int end = length;
+
+        if (length <= maxSectionLength) {
+            splitPages.add(new SplitPage(findPage(start, pages), allText.toString()));
+            return splitPages;
+        }
+
         while (start + sectionOverlap < length) {
             int lastWord = -1;
             end = start + maxSectionLength;

app/indexer/core/src/test/java/com/microsoft/openai/samples/indexer/parser/TextSplitterTest.java

@@ -0,0 +1,22 @@
+package com.microsoft.openai.samples.indexer.parser;
+
+import com.microsoft.openai.samples.indexer.SplitPage;
+import org.junit.jupiter.api.Test;
+
+import java.util.List;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+class TextSplitterTest {
+
+    @Test
+    void testSplitTinyPages() {
+        List<Page> testPages = List.of(new Page[]{
+                new Page(1, 0, "hello, world")
+        });
+        TextSplitter splitter = new TextSplitter(false);
+        List<SplitPage> result = splitter.splitPages(testPages);
+        assertEquals(1, result.size());
+        assertEquals("hello, world", result.get(0).getText());
+    }
+}