ICU-22789 Add Segmenter API to conveniently wrap BreakIterator in ICU4J

See #3237

Author: echeran

icu4j/main/core/src/main/java/com/ibm/icu/segmenter/BoundaryIteratorOfInts.java

@@ -0,0 +1,85 @@
+// © 2025 and later: Unicode, Inc. and others.
+// License & terms of use: https://www.unicode.org/copyright.html
+
+package com.ibm.icu.segmenter;
+
+import com.ibm.icu.text.BreakIterator;
+import com.ibm.icu.segmenter.Segments.IterationDirection;
+
+/**
+ * An iterator of segmentation boundaries that can operate in either the forwards or reverse
+ * direction.
+ *
+ * <p>When constructed to operate in the forwards direction, the iterator will return all boundaries
+ * that are strictly after the input index value provided to the constructor. However, when
+ * constructed to operate in the backwards direction, if the input index is already a segmentation
+ * boundary, then it will be included as the first value that the iterator returns as it iterates
+ * backwards.
+ */
+class BoundaryIteratorOfInts {
+  private BreakIterator breakIter;
+  private IterationDirection direction;
+  private int currIdx;
+
+  BoundaryIteratorOfInts(BreakIterator breakIter, CharSequence sourceSequence, IterationDirection direction, int startIdx) {
+    this.breakIter = breakIter;
+    this.direction = direction;
+
+    if (direction == IterationDirection.FORWARDS) {
+      currIdx = breakIter.following(startIdx);
+    } else {
+      assert direction == IterationDirection.BACKWARDS;
+
+      // When iterating backwards over boundaries, adjust the initial index to be the boundary
+      // that is either startIdx or else the one right before startIdx.
+      //
+      // Note: we have to set the initial index indirectly because there is no way to statelessly
+      // query whether an index is on a boundary. Instead, BreakIterator.isBoundary() will mutate
+      // state when the input is not on a boundary, before it returns the value indicating a
+      // boundary.
+      int sourceLength = sourceSequence.length();
+      if (startIdx == 0) {
+        currIdx = breakIter.first();
+      } else if (startIdx == sourceLength) {
+        currIdx = breakIter.last();
+      } else {
+        boolean isOnBoundary =
+            0 <= startIdx
+                && startIdx <= sourceLength
+                && breakIter.isBoundary(startIdx);
+
+        // The previous call to BreakIterator.isBoundary(startIdx) will have advanced breakIter's
+        // current position forwards to the next boundary if the argument, startIdx, is not a
+        // boundary. Therefore, in that case, we have to move back to the previous boundary.
+        //
+        // BreakIterator.isBoundary(startIdx) should have cached the surrounding 2 boundaries in the
+        // BreakIterator, which means that BreakIterator.preceding(startIdx) shouldn't cost
+        // significant extra time.
+        //
+        // BreakIterator.preceding(startIdx) is used in initialization instead of a simple call to
+        // BreakIterator.previous() since BreakIterator.preceding() can accept arguments larger than
+        // the last boundary and return the last boundary, whereas .previous() would return DONE.
+        // Thus, .preceding() provides symmetrical behavior to .following(), which we use in the
+        // forwards direction.
+        currIdx = isOnBoundary ? startIdx : breakIter.preceding(startIdx);
+      }
+    }
+  }
+
+  public boolean hasNext() {
+    return currIdx != BreakIterator.DONE;
+  }
+
+  public Integer next() {
+    int result = currIdx;
+
+    if (direction == IterationDirection.FORWARDS) {
+      currIdx = breakIter.next();
+    } else {
+      assert direction == IterationDirection.BACKWARDS;
+      currIdx = breakIter.previous();
+    }
+
+    return result;
+  }
+}

icu4j/main/core/src/main/java/com/ibm/icu/segmenter/BoundarySpliterator.java

@@ -0,0 +1,64 @@
+// © 2025 and later: Unicode, Inc. and others.
+// License & terms of use: https://www.unicode.org/copyright.html
+
+package com.ibm.icu.segmenter;
+
+import com.ibm.icu.text.BreakIterator;
+import com.ibm.icu.segmenter.Segments.IterationDirection;
+import java.util.Spliterator;
+import java.util.function.IntConsumer;
+
+class BoundarySpliterator implements Spliterator.OfInt {
+
+  private final BoundaryIteratorOfInts iter;
+
+  BoundarySpliterator(BreakIterator breakIter, CharSequence sourceSequence, IterationDirection direction, int startIdx) {
+    iter = new BoundaryIteratorOfInts(breakIter, sourceSequence, direction, startIdx);
+  }
+
+  @Override
+  public OfInt trySplit() {
+    // The elements of the Stream represent an iteration through a string, and is thus inherently
+    // stateful. Therefore, splitting this Stream does not make sense. Ex: splitting the Stream
+    // is tantamount to discarding the segment subtended by the end value (index into the input
+    // string) of one substream and the beginning value of the next substream.
+    return null;
+  }
+
+  @Override
+  public long estimateSize() {
+    // The number of segments per input size depends on language, script, and
+    // the content of the input string, and thus is hard to estimate without
+    // sacrificing performance. Thus, returning `Long.MAX_VALUE`, according
+    // to the API, to mean "unknown, or too expensive to compute".
+    return Long.MAX_VALUE;
+  }
+
+  @Override
+  public int characteristics() {
+    return
+        // BreakIterator always advances
+        Spliterator.DISTINCT
+        // The design of the Segmenter API is to provide an immutable view of
+        // segmentation by preventing the input string from mutating
+        // in the underlying BreakIterator.
+        | Spliterator.IMMUTABLE
+        // primitive int is non-null
+        | Spliterator.NONNULL
+        // BreakIterator always advances, and in a single direction.
+        | Spliterator.ORDERED;
+  }
+
+  @Override
+  public boolean tryAdvance(IntConsumer action) {
+    if (action == null) {
+      throw new NullPointerException();
+    }
+    if (iter.hasNext()) {
+      action.accept(iter.next());
+      return true;
+    } else {
+      return false;
+    }
+  }
+}

icu4j/main/core/src/main/java/com/ibm/icu/segmenter/LocalizedSegmenter.java

@@ -0,0 +1,137 @@
+// © 2025 and later: Unicode, Inc. and others.
+// License & terms of use: https://www.unicode.org/copyright.html
+
+package com.ibm.icu.segmenter;
+
+import com.ibm.icu.text.BreakIterator;
+import com.ibm.icu.util.ULocale;
+import java.util.Locale;
+import java.util.stream.IntStream;
+import java.util.stream.Stream;
+
+/**
+ * Performs segmentation according to the rules defined for the locale.
+ */
+public class LocalizedSegmenter implements Segmenter {
+
+  private BreakIterator breakIterPrototype;
+
+  /**
+   * Returns a {@link Segments} object that encapsulates the segmentation of the input
+   * {@code CharSequence}. The {@code Segments} object, in turn, provides the main APIs to support
+   * traversal over the resulting segments and boundaries via the Java {@code Stream} abstraction.
+   * @param s input {@code CharSequence} on which segmentation is performed. The input must not be
+   *     modified while using the resulting {@code Segments} object.
+   * @return A {@code Segments} object with APIs to access the results of segmentation, including
+   *     APIs that return {@code Stream}s of the segments and boundaries.
+   * @draft ICU 78
+   */
+  @Override
+  public Segments segment(CharSequence s) {
+    return new SegmentsImpl(breakIterPrototype, s);
+  }
+
+  /**
+   * @return a builder for constructing {@code LocalizedSegmenter}
+   * @draft ICU 78
+   */
+  public static Builder builder() {
+    return new Builder();
+  }
+
+  private LocalizedSegmenter(ULocale locale, SegmentationType segmentationType) {
+    switch (segmentationType) {
+      case LINE:
+        breakIterPrototype = BreakIterator.getLineInstance(locale);
+        break;
+      case SENTENCE:
+        breakIterPrototype = BreakIterator.getSentenceInstance(locale);
+        break;
+      case WORD:
+        breakIterPrototype = BreakIterator.getWordInstance(locale);
+        break;
+      case GRAPHEME_CLUSTER:
+        breakIterPrototype = BreakIterator.getCharacterInstance(locale);
+        break;
+    }
+  }
+
+  /**
+   * The type of segmentation to be performed. See the ICU User Guide page
+   * <a
+   * href="https://unicode-org.github.io/icu/userguide/boundaryanalysis/#four-types-of-breakiterator">Boundary Analysis</a>
+   * for further details.
+   * @draft ICU 78
+   */
+  public enum SegmentationType {
+    GRAPHEME_CLUSTER,
+    WORD,
+    LINE,
+    SENTENCE,
+  }
+
+  /**
+   * Builder for {@link LocalizedSegmenter}
+   * @draft ICU 78
+   */
+  public static class Builder {
+
+    private ULocale locale = ULocale.ROOT;
+
+    private SegmentationType segmentationType = null;
+
+    private Builder() { }
+
+    /**
+     * Set the locale for which segmentation rules will be loaded
+     * @param locale an ICU locale object
+     * @draft ICU 78
+     */
+    public Builder setLocale(ULocale locale) {
+      if (locale == null) {
+        throw new IllegalArgumentException("locale cannot be set to null.");
+      }
+      this.locale = locale;
+      return this;
+    }
+
+    /**
+     * Set the locale for which segmentation rules will be loaded
+     * @param locale a Java locale object
+     * @draft ICU 78
+     */
+    public Builder setLocale(Locale locale) {
+      if (locale == null) {
+        throw new IllegalArgumentException("locale cannot be set to null.");
+      }
+      this.locale = ULocale.forLocale(locale);
+      return this;
+    }
+
+    /**
+     * Set the segmentation type to be performed.
+     * @param segmentationType
+     * @draft ICU 78
+     */
+    public Builder setSegmentationType(SegmentationType segmentationType) {
+      if (segmentationType == null) {
+        throw new IllegalArgumentException("segmentationType cannot be set to null.");
+      }
+      this.segmentationType = segmentationType;
+      return this;
+    }
+
+    /**
+     * Builds the {@code Segmenter}
+     * @return the constructed {@code Segmenter} instance
+     * @draft ICU 78
+     */
+    public Segmenter build() {
+      if (segmentationType == null) {
+        throw new IllegalArgumentException("segmentationType is null and must be set to a specific value.");
+      }
+      return new LocalizedSegmenter(locale, segmentationType);
+    }
+
+  }
+}

icu4j/main/core/src/main/java/com/ibm/icu/segmenter/RuleBasedSegmenter.java

@@ -0,0 +1,90 @@
+// © 2025 and later: Unicode, Inc. and others.
+// License & terms of use: https://www.unicode.org/copyright.html
+
+package com.ibm.icu.segmenter;
+
+import com.ibm.icu.text.BreakIterator;
+import com.ibm.icu.text.RuleBasedBreakIterator;
+import java.io.InputStream;
+import java.util.stream.IntStream;
+import java.util.stream.Stream;
+
+/**
+ * Performs segmentation according to the provided rule string. The rule string must follow the
+ * same guidelines as for {@link RuleBasedBreakIterator#RuleBasedBreakIterator(String)}.
+ * @draft ICU 78
+ */
+public class RuleBasedSegmenter implements Segmenter {
+
+  private final BreakIterator breakIterPrototype;
+
+  /**
+   * Returns a {@link Segments} object that encapsulates the segmentation of the input
+   * {@code CharSequence}. The {@code Segments} object, in turn, provides the main APIs to support
+   * traversal over the resulting segments and boundaries via the Java {@code Stream} abstraction.
+   * @param s input {@code CharSequence} on which segmentation is performed. The input must not be
+   *     modified while using the resulting {@code Segments} object.
+   * @return A {@code Segments} object with APIs to access the results of segmentation, including
+   *     APIs that return {@code Stream}s of the segments and boundaries.
+   * @draft ICU 78
+   */
+  @Override
+  public Segments segment(CharSequence s) {
+    return new SegmentsImpl(breakIterPrototype, s);
+  }
+
+  /**
+   * @return a builder for constructing {@code RuleBasedSegmenter}
+   * @draft ICU 78
+   */
+  public static Builder builder() {
+    return new Builder();
+  }
+
+  private RuleBasedSegmenter(BreakIterator breakIter) {
+    breakIterPrototype = breakIter;
+  }
+
+  /**
+   * Builder for {@link RuleBasedSegmenter}
+   * @draft ICU 78
+   */
+  public static class Builder {
+
+    private BreakIterator breakIter = null;
+
+    private Builder() { }
+
+    /**
+     * Sets the rule string for segmentation.
+     * @param rules rule string.  The rule string must follow the same guidelines as for
+     *     {@link RuleBasedBreakIterator#getInstanceFromCompiledRules(InputStream)}.
+     * @draft ICU 78
+     */
+    public Builder setRules(String rules) {
+      if (rules == null) {
+        throw new IllegalArgumentException("rules cannot be set to null.");
+      }
+      try {
+        breakIter = new RuleBasedBreakIterator(rules);
+        return this;
+      } catch (RuntimeException rte) {
+        throw new IllegalArgumentException("The provided rule string is invalid"
+            + " or there was an error in creating the RuleBasedSegmenter.", rte);
+      }
+    }
+
+    /**
+     * Builds the {@code Segmenter}
+     * @return the constructed {@code Segmenter} instance
+     * @draft ICU 78
+     */
+    public Segmenter build() {
+      if (breakIter == null) {
+        throw new IllegalArgumentException("A rule string must be set.");
+      } else {
+        return new RuleBasedSegmenter(breakIter);
+      }
+    }
+  }
+}