OPENNLP-1745 : SentenceDetector - Add Junit test for useTokenEnd = false

Author: NishantShri4

.asf.yaml

@@ -12,6 +12,7 @@ github:
         dismiss_stale_reviews: false
         require_code_owner_reviews: false
         required_approving_review_count: 1
+    opennlp-2.x:  {}
   autolink_jira:
     - OPENNLP
   custom_subjects:

.github/workflows/maven.yml

@@ -34,7 +34,7 @@ jobs:
         java: [ 17, 21 ]
         experimental: [false]
         include:
-          - java: 24-ea
+          - java: 25-ea
             os: ubuntu-latest
             experimental: true
 
@@ -49,11 +49,11 @@ jobs:
         restore-keys: |
           ${{ runner.os }}-maven-
     - name: Set up JDK ${{ matrix.java }}
-      uses: actions/setup-java@v4
+      uses: actions/setup-java@3a4f6e1af504cf6a31855fa899c6aa5355ba6c12 # v4.7.0
       with:
         distribution: temurin
         java-version: ${{ matrix.java }}
     - name: Build with Maven
-      run: mvn -V clean test install --no-transfer-progress -Pjacoco -Pci
+      run: mvn -V clean test install --show-version --batch-mode --no-transfer-progress -Pjacoco -Pci
     - name: Jacoco
       run: mvn jacoco:report

NOTICE

@@ -61,9 +61,9 @@ List of third-party dependencies grouped by their license type.
 
     MIT License
 
-        * onnx-runtime (com.microsoft.onnxruntime:onnxruntime_gpu:1.20.0 - https://microsoft.github.io/onnxruntime/)
-        * onnx-runtime (com.microsoft.onnxruntime:onnxruntime:1.20.0 - https://microsoft.github.io/onnxruntime/)
-        * SLF4J API Module (org.slf4j:slf4j-api:2.0.16 - http://www.slf4j.org)
+        * onnx-runtime (com.microsoft.onnxruntime:onnxruntime_gpu:1.22.0 - https://microsoft.github.io/onnxruntime/)
+        * onnx-runtime (com.microsoft.onnxruntime:onnxruntime:1.22.0 - https://microsoft.github.io/onnxruntime/)
+        * SLF4J API Module (org.slf4j:slf4j-api:2.0.17 - http://www.slf4j.org)
 
     The MIT License (MIT)
 

README.md

@@ -71,10 +71,10 @@ Currently, the library has different packages:
 * `opennlp-tools` : The core toolkit.
 * `opennlp-tools-models` : A set of classes to load [OpenNLP models](https://github.com/apache/opennlp-models) from the classpath.
 * `opennlp-uima` : A set of [Apache UIMA](https://uima.apache.org) annotators.
-* `opennlp-morfologik-addon` : An addon for Morfologik
-* `opennlp-dl` : OpenNLP interface implementations for ONNX models using the `onnxruntime` dependency.
+* `opennlp-morfologik-addon` : An addon for _Morfologik_.
+* `opennlp-dl` : OpenNLP interface implementations for [ONNX](https://onnx.ai) models using the `onnxruntime` dependency.
 * `opennlp-dl-gpu` : Replaces `onnxruntime` with the `onnxruntime_gpu` dependency to support GPU acceleration.
-* `opennlp-sandbox`: Other projects in progress are found in the [sandbox](https://github.com/apache/opennlp-sandbox)
+* `opennlp-sandbox`: Other projects in progress reside in the [sandbox](https://github.com/apache/opennlp-sandbox).           
 
 ## Getting Started
 
@@ -104,6 +104,28 @@ compile group: "org.apache.opennlp", name: "opennlp-tools", version: "${opennlp.
 
 For more details please check our [documentation](http://opennlp.apache.org/docs/)
 
+## Branches and Merging Strategy
+
+To support ongoing development and stable maintenance of Apache OpenNLP, the project follows a dual-branch model:
+
+### Branch overview
+
+- **`main`**: Development branch for version **3.0** and beyond. All feature development and 3.x releases occur here.
+- **`opennlp-2.x`**: Maintains the stable **2.x** release line. This branch will receive selective updates and patch releases.
+
+### Workflow summary
+
+- Feature development
+  - New features targeting versions 3.0+ are developed on feature branches _off_ `main` and merged _into_ `main`.
+- Bug fixes and dependency updates
+  - Relevant fixes or dependency updates from `main` may be cherry-picked into `opennlp-2.x` as needed.
+- Releases
+  - **3.x** releases are made from the `main` branch.
+  - **2.x** releases are made from the `opennlp-2.x` branch.
+- Release tags
+  - Release tags are applied directly to the appropriate version branch (`main` for 3.x or `opennlp-2.x` for 2.x). 
+  - The presence of a version branch does not affect the tagging or visibility of releases.
+
 ## Building OpenNLP
 
 At least JDK 17 and Maven 3.3.9 are required to build the library.
@@ -114,12 +136,14 @@ After cloning the repository go into the destination directory and run:
 mvn install
 ```
 
-### Additional Developement Information
+### Additional Development Information
 
-- [Building and Integrating Snowball Stemmer for OpenNLP](dev/Snowball-Stemmer.md)
+- Building and integrating [Snowball Stemmer](dev/Snowball-Stemmer.md) for OpenNLP.
 
 ## Contributing
 
-The Apache OpenNLP project is developed by volunteers and is always looking for new contributors to work on all parts of the project. Every contribution is welcome and needed to make it better. A contribution can be anything from a small documentation typo fix to a new component.
+The Apache OpenNLP project is developed by volunteers and is always looking for new contributors to work on all parts of the project. 
+Every contribution is welcome and needed to make it better. 
+A contribution can be anything from a small documentation typo fix to a new component.
 
 If you would like to get involved please follow the instructions [here](https://github.com/apache/opennlp/blob/main/.github/CONTRIBUTING.md)

opennlp-distr/pom.xml

@@ -24,7 +24,7 @@
   <parent>
     <groupId>org.apache.opennlp</groupId>
     <artifactId>opennlp</artifactId>
-    <version>2.5.4-SNAPSHOT</version>
+    <version>3.0.0-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 

opennlp-dl-gpu/pom.xml

@@ -24,7 +24,7 @@
   <parent>
     <groupId>org.apache.opennlp</groupId>
     <artifactId>opennlp</artifactId>
-    <version>2.5.4-SNAPSHOT</version>
+    <version>3.0.0-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
   <groupId>org.apache.opennlp</groupId>

opennlp-dl/pom.xml

@@ -24,7 +24,7 @@
   <parent>
     <groupId>org.apache.opennlp</groupId>
     <artifactId>opennlp</artifactId>
-    <version>2.5.4-SNAPSHOT</version>
+    <version>3.0.0-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
   <groupId>org.apache.opennlp</groupId>

opennlp-docs/pom.xml

@@ -24,7 +24,7 @@
   <parent>
 	<groupId>org.apache.opennlp</groupId>
 	<artifactId>opennlp</artifactId>
-	<version>2.5.4-SNAPSHOT</version>
+	<version>3.0.0-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 

opennlp-docs/src/docbkx/model-loading.xml

@@ -78,18 +78,18 @@ under the License.
 
             <programlisting language="java">
                 <![CDATA[
-final ClassgraphModelFinder finder = new ClassgraphModelFinder(); // or use new SimpleClassPathModelFinder()
+final ClassgraphModelFinder finder = new ClassgraphModelFinder(); // or use: new SimpleClassPathModelFinder()
 final ClassPathModelLoader loader = new ClassPathModelLoader();
 final Set<ClassPathModelEntry> models = finder.findModels(false);
 for(ClassPathModelEntry entry : models) {
 
    final ClassPathModel model = loader.load(entry);
 
    if(model != null) {
-                System.out.println(model.getModelName());
-                System.out.println(model.getModelSHA256());
-                System.out.println(model.getModelVersion());
-                System.out.println(model.getModeLanguage());
+      System.out.println(model.getModelName());
+      System.out.println(model.getModelSHA256());
+      System.out.println(model.getModelVersion());
+      System.out.println(model.getModelLanguage());
       // do something with the model by consuming the byte array
    }
 }]]>
@@ -120,23 +120,23 @@ model.language=${model.language}
             Make sure to replace the values accordingly and configure your build tool to include the binary model and the <emphasis>model.properties</emphasis>
             in the resulting JAR file.
 
-            To load such a custom model, you may need to adjust the pattern for classpath scanning. For example, if you name your model "custom-opennlp-model",
+            To load such a custom model, you may need to adjust the pattern for classpath scanning. For example, if you name the model "custom-opennlp-model",
             you need the following code to successfully find and load it:
 
             <programlisting language="java">
                 <![CDATA[
-final ClassgraphModelFinder finder = new ClassgraphModelFinder("custom-opennlp-model.jar"); // or use new SimpleClassPathModelFinder("custom-opennlp-model.jar")
+final ClassgraphModelFinder finder = new ClassgraphModelFinder("custom-opennlp-model.jar"); // or use: new SimpleClassPathModelFinder("custom-opennlp-model.jar")
 final ClassPathModelLoader loader = new ClassPathModelLoader();
 final Set<ClassPathModelEntry> models = finder.findModels(false);
 for(ClassPathModelEntry entry : models) {
 
    final ClassPathModel model = loader.load(entry);
 
    if(model != null) {
-                System.out.println(model.getModelName());
-                System.out.println(model.getModelSHA256());
-                System.out.println(model.getModelVersion());
-                System.out.println(model.getModeLanguage());
+      System.out.println(model.getModelName());
+      System.out.println(model.getModelSHA256());
+      System.out.println(model.getModelVersion());
+      System.out.println(model.getModelLanguage());
       // do something with the model by consuming the byte array
    }
 }]]>

opennlp-docs/src/docbkx/postagger.xml

@@ -237,11 +237,114 @@ try (OutputStream modelOut = new BufferedOutputStream(new FileOutputStream(model
 		</para>
 		<para>
 		The dictionary is defined in a xml format and can be created and stored with the POSDictionary class.
-		Please for now checkout the javadoc and source code of that class.
+		Below is an example to train a custom model using a tag dictionary.
 		</para>
-		<para>Note: The format should be documented and sample code should show how to use the dictionary.
-			  Any contributions are very welcome. If you want to contribute please contact us on the mailing list
-			  or comment on the jira issue <ulink url="https://issues.apache.org/jira/browse/OPENNLP-287">OPENNLP-287</ulink>.
+		<para>
+		Sample POS Training material (file : en-custom-pos.train)
+			<screen>
+				<![CDATA[
+It_PRON is_OTHER spring_PROPN season_NOUN. The_DET flowers_NOUN are_OTHER red_ADJ and_CCONJ yellow_ADJ ._PUNCT
+Red_NOUN is_OTHER my_DET favourite_ADJ colour_NOUN ._PUNCT]]>
+			</screen>
+		</para>
+		<para>
+		Sample Tag Dictionary (file : dictionary.xml)
+			<programlisting language="xml">
+				<![CDATA[
+<?xml version="1.0" encoding="UTF-8"?>
+ <dictionary case_sensitive="false">
+  <entry tags="PRON">
+    <token>It</token>
+  </entry>
+  <entry tags="OTHER">
+    <token>is</token>
+  </entry>
+  <entry tags="PROPN">
+    <token>Spring</token>
+  </entry>
+  <entry tags="NOUN">
+    <token>season</token>
+  </entry>
+  <entry tags="DET">
+    <token>the</token>
+  </entry>
+  <entry tags="NOUN">
+    <token>flowers</token>
+  </entry>
+  <entry tags="OTHER">
+    <token>are</token>
+  </entry>
+  <entry tags="NOUN">
+    <token>red</token>
+  </entry>
+  <entry tags="CCONJ">
+    <token>and</token>
+  </entry>
+  <entry tags="NOUN">
+    <token>yellow</token>
+  </entry>
+  <entry tags="PRON">
+    <token>my</token>
+  </entry>
+  <entry tags="ADJ">
+    <token>favourite</token>
+  </entry>
+  <entry tags="NOUN">
+    <token>colour</token>
+  </entry>
+  <entry tags="PUNCT">
+    <token>.</token>
+  </entry>
+</dictionary>]]>
+			</programlisting>
+		</para>
+		<para>Sample code to train a model using above tag dictionary
+			<programlisting language="java">
+			<![CDATA[
+POSModel model = null;
+	try {
+		ObjectStream<String> lineStream = new PlainTextByLineStream(
+				new MarkableFileInputStreamFactory(new File("en-custom-pos.train")), StandardCharsets.UTF_8);
+
+		ObjectStream<POSSample> sampleStream = new WordTagSampleStream(lineStream);
+
+		TrainingParameters params = ModelUtil.createDefaultTrainingParameters();
+		params.put(TrainingParameters.CUTOFF_PARAM, 0);
+
+		POSTaggerFactory factory = new POSTaggerFactory();
+		TagDictionary dict = factory.createTagDictionary(new File("dictionary.xml"));
+		factory.setTagDictionary(dict);
+
+		model = POSTaggerME.train("eng", sampleStream, params, factory);
+
+		OutputStream modelOut = new BufferedOutputStream(new FileOutputStream("en-custom-pos-maxent.bin"));
+		model.serialize(modelOut);
+
+	} catch (IOException e) {
+		e.printStackTrace();
+	}]]>
+			</programlisting>
+		</para>
+		<para>
+		The custom model is then used to tag a sequence.
+		<programlisting language="java">
+			<![CDATA[
+String[] sent = new String[]{"Spring", "is", "my", "favourite", "season", "."};
+String[] tags = tagger.tag(sent);
+Arrays.stream(tags).forEach(k -> System.out.print(k + " "));]]>
+		</programlisting>
+		</para>
+		<para>
+			<literallayout>
+				Input
+				    Sentence:	Spring is my favourite season.
+
+				Output
+				    POS Tags using the custom model (en-custom-pos-maxent.bin): PROPN OTHER PRON ADJ NOUN PUNCT
+
+				Output with the default model
+				    POS Tags using the default model (opennlp-en-ud-ewt-pos-1.2-2.5.0.bin):	NOUN AUX PRON ADJ NOUN PUNCT
+			</literallayout>
 		</para>
 		</section>
 		</section>