Share some code between SourceSplitter and LineBreaker

Also, use "offset" consistently to align with
Lucene.

Author: idodeclare

opengrok-indexer/src/main/java/org/opengrok/indexer/analysis/CtagsReader.java

@@ -19,7 +19,7 @@
 
 /*
  * Copyright (c) 2005, 2019, Oracle and/or its affiliates. All rights reserved.
- * Portions Copyright (c) 2017-2018, Chris Fraire <[email protected]>.
+ * Portions Copyright (c) 2017-2018, 2020, Chris Fraire <[email protected]>.
  */
 
 package org.opengrok.indexer.analysis;
@@ -395,9 +395,9 @@ private CpatIndex bestIndexOfTag(int lineno, String whole, String str) {
 
         int woff = strictIndexOf(whole, str);
         if (woff < 0) {
-            /**
+            /*
              * When a splitter is available, search the entire line.
-             * (N.b. use 0-offset vs ctags's 1-offset.)
+             * (N.b. use 0-based indexing vs ctags's 1-based.)
              */
             String cut = trySplitterCut(lineno - 1, 1);
             if (cut == null || !cut.startsWith(whole)) {
@@ -512,9 +512,9 @@ private CpatIndex bestIndexOfArg(int lineno, String whole, String arg) {
             return new CpatIndex(lineno, s, e);
         }
 
-        /**
+        /*
          * When a splitter is available, search the next several lines.
-         * (N.b. use 0-offset vs ctags's 1-offset.)
+         * (N.b. use 0-based indexing vs ctags's 1-based.)
          */
         String cut = trySplitterCut(lineno - 1, MAX_CUT_LINES);
         if (cut == null || !cut.startsWith(whole)) {
@@ -640,22 +640,22 @@ private PatResult strictMatch(String whole, String substr, Pattern pat) {
     }
 
     /**
-     * Finds the line with the longest content from {@code midx}.
+     * Finds the line with the longest content from {@code cut}.
      * <p>
      * The {@link Definitions} tag model is based on a match within a line.
      * "signature" fields, however, can be condensed from multiple lines; and a
      * fuzzy match can therefore span multiple lines.
      */
     private CpatIndex bestLineOfMatch(int lineno, PatResult pr, String cut) {
-        // (N.b. use 0-offset vs ctags's 1-offset.)
-        int lpos = splitter.getPosition(lineno - 1);
-        int mpos = lpos + pr.start;
-        int moff = splitter.findLineOffset(mpos);
-        int zpos = lpos + pr.end - 1;
-        int zoff = splitter.findLineOffset(zpos);
+        // (N.b. use 0-based indexing vs ctags's 1-based.)
+        int lineOff = splitter.getOffset(lineno - 1);
+        int mOff = lineOff + pr.start;
+        int mIndex = splitter.findLineIndex(mOff);
+        int zOff = lineOff + pr.end - 1;
+        int zIndex = splitter.findLineIndex(zOff);
 
         int t = tabSize;
-        int resoff = moff;
+        int resIndex = mIndex;
         int contentLength = 0;
         /**
          * Initialize the following just to silence warnings but with values
@@ -664,31 +664,31 @@ private CpatIndex bestLineOfMatch(int lineno, PatResult pr, String cut) {
         String whole = "";
         int s = 0;
         int e = 1;
-        /**
-         * Iterate to determine the length of the portion of `midx' that
-         * is contained within each line.
+        /*
+         * Iterate to determine the length of the portion of cut that is
+         * contained within each line.
          */
-        for (int ioff = moff; ioff <= zoff; ++ioff) {
-            String iwhole = splitter.getLine(ioff);
-            int ioffpos = splitter.getPosition(ioff);
-            int iendpos = ioffpos + iwhole.length();
-            int i_s = pr.start + lpos < ioffpos ? ioffpos : pr.start + lpos;
-            int i_e = pr.end + lpos > iendpos ? iendpos : pr.end + lpos;
-            if (i_e - i_s > contentLength) {
-                contentLength = i_e - i_s;
-                resoff = ioff;
+        for (int lIndex = mIndex; lIndex <= zIndex; ++lIndex) {
+            String iwhole = splitter.getLine(lIndex);
+            int lOff = splitter.getOffset(lIndex);
+            int lOffZ = lOff + iwhole.length();
+            int offStart = Math.max(pr.start + lineOff, lOff);
+            int offEnd = Math.min(pr.end + lineOff, lOffZ);
+            if (offEnd - offStart > contentLength) {
+                contentLength = offEnd - offStart;
+                resIndex = lIndex;
                 whole = iwhole;
                 // (The following are not yet adjusted for tabs.)
-                s = i_s - ioffpos;
-                e = i_e - ioffpos;
+                s = offStart - lOff;
+                e = offEnd - lOff;
             }
         }
 
         if (s >= 0 && s < whole.length() && e >= 0 && e <= whole.length()) {
             s = ExpandTabsReader.translate(whole, s, t);
             e = ExpandTabsReader.translate(whole, e, t);
-            // (N.b. use ctags's 1-offset.)
-            return new CpatIndex(resoff + 1, s, e);
+            // (N.b. use ctags's 1-based indexing.)
+            return new CpatIndex(resIndex + 1, s, e);
         }
 
         /**

opengrok-indexer/src/main/java/org/opengrok/indexer/analysis/plain/DefinitionsTokenStream.java

@@ -18,7 +18,7 @@
  */
 
 /*
- * Copyright (c) 2018, Chris Fraire <[email protected]>.
+ * Copyright (c) 2018, 2020, Chris Fraire <[email protected]>.
  */
 
 package org.opengrok.indexer.analysis.plain;
@@ -118,7 +118,7 @@ private void createTokens(Definitions defs, LineBreaker brk) {
 
             if (lineno >= 0 && lineno < brk.count() && tag.symbol != null &&
                     tag.text != null) {
-                int lineoff = brk.getPosition(lineno);
+                int lineoff = brk.getOffset(lineno);
                 if (tag.lineStart >= 0) {
                     PendingToken tok = new PendingToken(tag.symbol, lineoff +
                         tag.lineStart, lineoff + tag.lineEnd);

opengrok-indexer/src/main/java/org/opengrok/indexer/search/context/ContextFormatter.java

@@ -18,7 +18,7 @@
  */
 
 /*
- * Copyright (c) 2018, Chris Fraire <[email protected]>.
+ * Copyright (c) 2018, 2020, Chris Fraire <[email protected]>.
  */
 
 package org.opengrok.indexer.search.context;
@@ -323,7 +323,7 @@ private void writeScope(int lineOffset, Appendable dest)
             throws IOException {
         Scopes.Scope scope = null;
         if (scopes != null) {
-            // N.b. use ctags 1-offset vs 0-offset.
+            // N.b. use ctags 1-based indexing vs 0-based.
             scope = scopes.getScope(lineOffset + 1);
         }
         if (scope != null && scope != scopes.getScope(-1)) {
@@ -340,7 +340,7 @@ private void writeScope(int lineOffset, Appendable dest)
     private void writeTag(int lineOffset, Appendable dest, List<String> marks)
             throws IOException {
         if (defs != null) {
-            // N.b. use ctags 1-offset vs 0-offset.
+            // N.b. use ctags 1-based indexing vs 0-based.
             List<Tag> linetags =  defs.getTags(lineOffset + 1);
             if (linetags != null) {
                 Tag pickedTag = findTagForMark(linetags, marks);

opengrok-indexer/src/main/java/org/opengrok/indexer/search/context/PassageConverter.java

@@ -18,7 +18,7 @@
  */
 
 /*
- * Copyright (c) 2018, Chris Fraire <[email protected]>.
+ * Copyright (c) 2018, 2020, Chris Fraire <[email protected]>.
  */
 
 package org.opengrok.indexer.search.context;
@@ -75,11 +75,11 @@ public SortedMap<Integer, LineHighlight> convert(Passage[] passages,
                 continue;
             }
 
-            int m = splitter.findLineOffset(start);
+            int m = splitter.findLineIndex(start);
             if (m < 0) {
                 continue;
             }
-            int n = splitter.findLineOffset(end - 1);
+            int n = splitter.findLineIndex(end - 1);
             if (n < 0) {
                 continue;
             }
@@ -97,23 +97,23 @@ public SortedMap<Integer, LineHighlight> convert(Passage[] passages,
             // Create LineHighlight entries for passage matches.
             for (int i = 0; i < passage.getNumMatches(); ++i) {
                 int mstart = passage.getMatchStarts()[i];
-                int mm = splitter.findLineOffset(mstart);
+                int mm = splitter.findLineIndex(mstart);
                 int mend = passage.getMatchEnds()[i];
-                int nn = splitter.findLineOffset(mend - 1);
+                int nn = splitter.findLineIndex(mend - 1);
                 if (mstart < mend && mm >= m && mm <= n && nn >= m && nn <= n) {
                     if (mm == nn) {
-                        int lbeg = splitter.getPosition(mm);
+                        int lbeg = splitter.getOffset(mm);
                         int lstart = mstart - lbeg;
                         int lend = mend - lbeg;
                         LineHighlight lhigh = res.get(mm);
                         lhigh.addMarkup(PhraseHighlight.create(lstart, lend));
                     } else {
-                        int lbeg = splitter.getPosition(mm);
+                        int lbeg = splitter.getOffset(mm);
                         int loff = mstart - lbeg;
                         LineHighlight lhigh = res.get(mm);
                         lhigh.addMarkup(PhraseHighlight.createStarter(loff));
 
-                        lbeg = splitter.getPosition(nn);
+                        lbeg = splitter.getOffset(nn);
                         loff = mend - lbeg;
                         lhigh = res.get(nn);
                         lhigh.addMarkup(PhraseHighlight.createEnder(loff));

opengrok-indexer/src/main/java/org/opengrok/indexer/util/LineBreaker.java

@@ -18,16 +18,13 @@
  */
 
 /*
- * Copyright (c) 2018, Chris Fraire <[email protected]>.
+ * Copyright (c) 2018, 2020, Chris Fraire <[email protected]>.
  */
 
 package org.opengrok.indexer.util;
 
-import java.io.BufferedReader;
 import java.io.IOException;
-import java.io.InputStream;
 import java.io.Reader;
-import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
 import java.util.List;
 import org.opengrok.indexer.analysis.StreamSource;
@@ -40,6 +37,7 @@
 public class LineBreaker {
 
     private int length;
+    private int count;
     private int[] lineOffsets;
 
     /**
@@ -65,29 +63,13 @@ public void reset(StreamSource src, ReaderWrapper wrapper)
             throw new IllegalArgumentException("`src' is null");
         }
 
-        length = 0;
-        lineOffsets = null;
-
-        try (InputStream in = src.getStream();
-            Reader rdr = IOUtils.createBOMStrippedReader(in,
-                StandardCharsets.UTF_8.name())) {
-            Reader intermediate = null;
-            if (wrapper != null) {
-                intermediate = wrapper.get(rdr);
-            }
-
-            try (BufferedReader brdr = new BufferedReader(
-                    intermediate != null ? intermediate : rdr)) {
-                reset(brdr);
-            } finally {
-                if (intermediate != null) {
-                    intermediate.close();
-                }
-            }
-        }
+        SplitterUtil.reset(this::reset, src, wrapper);
     }
 
     private void reset(Reader reader) throws IOException {
+        length = 0;
+        lineOffsets = null;
+
         List<Integer> newOffsets = new ArrayList<>();
         newOffsets.add(0);
 
@@ -124,6 +106,12 @@ private void reset(Reader reader) throws IOException {
             }
         }
 
+        count = newOffsets.size();
+        if (newOffsets.get(newOffsets.size() - 1) < length) {
+            newOffsets.add(length);
+            // Do not increment count.
+        }
+
         lineOffsets = new int[newOffsets.size()];
         for (int i = 0; i < lineOffsets.length; ++i) {
             lineOffsets[i] = newOffsets.get(i);
@@ -139,28 +127,44 @@ public int originalLength() {
     }
 
     /**
-     * Gets the number of broken lines.
-     * @return value
+     * Gets the number of split lines.
      */
     public int count() {
         if (lineOffsets == null) {
             throw new IllegalStateException("reset() did not succeed");
         }
-        return lineOffsets.length;
+        return count;
     }
 
     /**
-     * Gets the starting document character position of the line at the
-     * specified offset.
-     * @param offset greater than or equal to zero and less than or equal to
+     * Gets the starting document character offset of the line at the
+     * specified index in the lines list.
+     * @param index greater than or equal to zero and less than or equal to
      * {@link #count()}
-     * @return line length, including the end-of-line token
-     * @throws IllegalArgumentException if {@code offset} is out of bounds
+     * @return line starting offset
+     * @throws IllegalArgumentException if {@code index} is out of bounds
      */
-    public int getPosition(int offset) {
-        if (offset < 0 || lineOffsets == null || offset >= lineOffsets.length) {
-            throw new IllegalArgumentException("`offset' is out of bounds");
+    public int getOffset(int index) {
+        if (lineOffsets == null) {
+            throw new IllegalStateException("reset() did not succeed");
+        }
+        if (index < 0 || index >= lineOffsets.length) {
+            throw new IllegalArgumentException("index is out of bounds");
+        }
+        return lineOffsets[index];
+    }
+
+    /**
+     * Find the line index for the specified document offset.
+     * @param offset greater than or equal to zero and less than
+     * {@link #originalLength()}.
+     * @return -1 if {@code offset} is beyond the document bounds; otherwise,
+     * a valid index
+     */
+    public int findLineIndex(int offset) {
+        if (lineOffsets == null) {
+            throw new IllegalStateException("reset() did not succeed");
         }
-        return lineOffsets[offset];
+        return SplitterUtil.findLineIndex(length, lineOffsets, offset);
     }
 }