OPENNLP-1745: SentenceDetector - Add Junit test for useTokenEnd = false

Author: NishantShri4

opennlp-tools/src/test/java/opennlp/tools/sentdetect/SentenceDetectorMEGermanTest.java

@@ -20,12 +20,15 @@
 import java.io.IOException;
 import java.util.Locale;
 
-import org.junit.jupiter.api.Assertions;
-import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.Test;
 
 import opennlp.tools.dictionary.Dictionary;
 
+import static org.junit.jupiter.api.Assertions.assertAll;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.fail;
+
 /**
  * Tests for the {@link SentenceDetectorME} class.
  * <p>
@@ -45,19 +48,25 @@ public class SentenceDetectorMEGermanTest extends AbstractSentenceDetectorTest {
 
   private static SentenceModel sentdetectModel;
 
-  @BeforeAll
-  public static void prepareResources() throws IOException {
-    Dictionary abbreviationDict = loadAbbDictionary(Locale.GERMAN);
-    SentenceDetectorFactory factory = new SentenceDetectorFactory(
-            "deu", true, abbreviationDict, EOS_CHARS);
-    sentdetectModel = train(factory, Locale.GERMAN);
-    Assertions.assertNotNull(sentdetectModel);
-    Assertions.assertEquals("deu", sentdetectModel.getLanguage());
+  private void prepareResources(boolean useTokenEnd) {
+    try {
+      Dictionary abbreviationDict = loadAbbDictionary(Locale.GERMAN);
+      SentenceDetectorFactory factory = new SentenceDetectorFactory(
+          "deu", useTokenEnd, abbreviationDict, EOS_CHARS);
+
+      sentdetectModel = train(factory, Locale.GERMAN);
+      assertNotNull(sentdetectModel);
+      assertEquals("deu", sentdetectModel.getLanguage());
+    } catch (IOException  ex) {
+      fail("Couldn't train the SentenceModel using test data. Exception: " + ex.getMessage());
+    }
   }
 
   // Example taken from 'Sentences_DE.txt'
   @Test
   void testSentDetectWithInlineAbbreviationsEx1() {
+    prepareResources(true);
+
     final String sent1 = "Ein Traum, zu dessen Bildung eine besonders starke Verdichtung beigetragen, " +
             "wird für diese Untersuchung das günstigste Material sein.";
     // Here we have two abbreviations "S. = Seite" and "ff. = folgende (Plural)"
@@ -66,40 +75,64 @@ void testSentDetectWithInlineAbbreviationsEx1() {
     SentenceDetectorME sentDetect = new SentenceDetectorME(sentdetectModel);
     String sampleSentences = sent1 + " " + sent2;
     String[] sents = sentDetect.sentDetect(sampleSentences);
-    Assertions.assertEquals(2, sents.length);
-    Assertions.assertEquals(sent1, sents[0]);
-    Assertions.assertEquals(sent2, sents[1]);
+    assertEquals(2, sents.length);
+    assertEquals(sent1, sents[0]);
+    assertEquals(sent2, sents[1]);
     double[] probs = sentDetect.getSentenceProbabilities();
-    Assertions.assertEquals(2, probs.length);
+    assertEquals(2, probs.length);
   }
 
   // Reduced example taken from 'Sentences_DE.txt'
   @Test
   void testSentDetectWithInlineAbbreviationsEx2() {
+    prepareResources(true);
+
     // Here we have three abbreviations: "S. = Seite", "vgl. = vergleiche", and "f. = folgende (Singular)"
     final String sent1 = "Die farbige Tafel, die ich aufschlage, " +
             "geht (vgl. die Analyse S. 185 f.) auf ein neues Thema ein.";
 
     SentenceDetectorME sentDetect = new SentenceDetectorME(sentdetectModel);
     String[] sents = sentDetect.sentDetect(sent1);
-    Assertions.assertEquals(1, sents.length);
-    Assertions.assertEquals(sent1, sents[0]);
+    assertEquals(1, sents.length);
+    assertEquals(sent1, sents[0]);
     double[] probs = sentDetect.getSentenceProbabilities();
-    Assertions.assertEquals(1, probs.length);
+    assertEquals(1, probs.length);
   }
 
   // Modified example deduced from 'Sentences_DE.txt'
   @Test
   void testSentDetectWithInlineAbbreviationsEx3() {
+    prepareResources(true);
+
     // Here we have two abbreviations "z. B. = zum Beispiel" and "S. = Seite"
     final String sent1 = "Die farbige Tafel, die ich aufschlage, " +
             "geht (z. B. die Analyse S. 185) auf ein neues Thema ein.";
 
     SentenceDetectorME sentDetect = new SentenceDetectorME(sentdetectModel);
     String[] sents = sentDetect.sentDetect(sent1);
-    Assertions.assertEquals(1, sents.length);
-    Assertions.assertEquals(sent1, sents[0]);
+    assertEquals(1, sents.length);
+    assertEquals(sent1, sents[0]);
+    double[] probs = sentDetect.getSentenceProbabilities();
+    assertEquals(1, probs.length);
+  }
+
+  @Test
+  void testSentDetectWithUseTokenEndFalse() {
+    prepareResources(false);
+
+    final String sent1 =  "Träume sind eine Verbindung von Gedanken.";
+    final String sent2 =  "Verschiedene Gedanken sind während der Traumformation aktiv.";
+
+    SentenceDetectorME sentDetect = new SentenceDetectorME(sentdetectModel);
+    //There is no blank space before start of the second sentence.
+    String[] sents = sentDetect.sentDetect(sent1 + sent2);
     double[] probs = sentDetect.getSentenceProbabilities();
-    Assertions.assertEquals(1, probs.length);
+    assertAll(
+        () -> assertEquals(2, sents.length),
+        () -> assertEquals(sent1, sents[0]),
+        () -> assertEquals(sent1, sents[0]),
+        () -> assertEquals(sent2, sents[1]),
+        () -> assertEquals(2, probs.length)
+    );
   }
 }