Merge pull request #77 from Azure-Samples/tonybaloney-patch-1

Add ideographic and full-width punctuation to splitter

Author: dantelmomsft

app/indexer/core/src/main/java/com/microsoft/openai/samples/indexer/parser/TextSplitter.java

@@ -17,11 +17,18 @@ public class TextSplitter {
     public TextSplitter(boolean verbose) {
         this.sentenceEndings = new ArrayList<>();
         this.sentenceEndings.add(".");
+        this.sentenceEndings.add("。");
+        this.sentenceEndings.add("．");
         this.sentenceEndings.add("!");
         this.sentenceEndings.add("?");
+        this.sentenceEndings.add("‼");
+        this.sentenceEndings.add("⁇");
+        this.sentenceEndings.add("⁈");
+        this.sentenceEndings.add("⁉");
 
         this.wordBreaks = new ArrayList<>();
         this.wordBreaks.add(",");
+        this.wordBreaks.add("、");
         this.wordBreaks.add(";");
         this.wordBreaks.add(":");
         this.wordBreaks.add(" ");