fixes for FilePath searches being broken (in QueryBuilder),

cleanups, path tokenizer fixes and tests
fixes issue #9

Author: Lubos Kosco

src/org/opensolaris/opengrok/analysis/AnalyzerGuru.java

@@ -71,11 +71,12 @@
 import org.opensolaris.opengrok.history.HistoryException;
 import org.opensolaris.opengrok.history.HistoryGuru;
 import org.opensolaris.opengrok.history.HistoryReader;
+import org.opensolaris.opengrok.search.QueryBuilder;
 import org.opensolaris.opengrok.web.Util;
 
 /**
  * Manages and provides Analyzers as needed. Please see
- * <a href="http://www.opensolaris.org/os/project/opengrok/manual/internals/">
+ * <a href="https://github.com/OpenGrok/OpenGrok/wiki/OpenGrok-Internals">
  * this</a> page for a great description of the purpose of the AnalyzerGuru.
  *
  * Created on September 22, 2005
@@ -89,27 +90,27 @@ public class AnalyzerGuru {
 
     /** Map from file names to analyzer factories. */
     private static final Map<String, FileAnalyzerFactory>
-        FILE_NAMES = new HashMap<String, FileAnalyzerFactory>();
+        FILE_NAMES = new HashMap<>();
 
     /** Map from file extensions to analyzer factories. */
     private static final Map<String, FileAnalyzerFactory>
-        ext = new HashMap<String, FileAnalyzerFactory>();
+        ext = new HashMap<>();
 
     // @TODO: have a comparator
     /** Map from magic strings to analyzer factories. */
     private static final SortedMap<String, FileAnalyzerFactory>
-        magics = new TreeMap<String, FileAnalyzerFactory>();
+        magics = new TreeMap<>();
 
     /**
      * List of matcher objects which can be used to determine which analyzer
      * factory to use.
      */
     private static final List<FileAnalyzerFactory.Matcher>
-        matchers = new ArrayList<FileAnalyzerFactory.Matcher>();
+        matchers = new ArrayList<>();
 
     /** List of all registered {@code FileAnalyzerFactory} instances. */
     private static final List<FileAnalyzerFactory>
-        factories = new ArrayList<FileAnalyzerFactory>();
+        factories = new ArrayList<>();
 
     public static final Reader dummyR = new StringReader("");
     public static final String dummyS = "";
@@ -237,41 +238,41 @@ public static FileAnalyzer getAnalyzer(InputStream in, String file) throws IOExc
      * @param xrefOut Where to write the xref (possibly {@code null})
      * @return The Lucene document to add to the index database
      * @throws java.io.IOException If an exception occurs while collecting the
-     *                             datas
+     *                             data
      */
     public Document getDocument(File file, String path,
                                 FileAnalyzer fa, Writer xrefOut)
             throws IOException {
         Document doc = new Document();
         String date = DateTools.timeToString(file.lastModified(),
             DateTools.Resolution.MILLISECOND);
-        doc.add(new Field("u", Util.path2uid(path, date),
+        doc.add(new Field(QueryBuilder.U, Util.path2uid(path, date),
             string_ft_stored_nanalyzed_norms));
-        doc.add(new Field("fullpath", file.getAbsolutePath(),
+        doc.add(new Field(QueryBuilder.FULLPATH, file.getAbsolutePath(),
             string_ft_nstored_nanalyzed_norms));
 
         try {
             HistoryReader hr = HistoryGuru.getInstance().getHistoryReader(file);
             if (hr != null) {
-                doc.add(new TextField("hist", hr));
+                doc.add(new TextField(QueryBuilder.HIST, hr));
                 // date = hr.getLastCommentDate() //RFE
             }
         } catch (HistoryException e) {
             OpenGrokLogger.getLogger().log(Level.WARNING, "An error occurred while reading history: ", e);
         }    
-        doc.add(new Field("date", date, string_ft_stored_nanalyzed_norms));
+        doc.add(new Field(QueryBuilder.DATE, date, string_ft_stored_nanalyzed_norms));
         if (path != null) {
-            doc.add(new TextField("path", path, Store.YES));
+            doc.add(new TextField(QueryBuilder.PATH, path, Store.YES));
             Project project = Project.getProject(path);
             if (project != null) {
-                doc.add(new TextField("project", project.getPath(), Store.YES));
+                doc.add(new TextField(QueryBuilder.PROJECT, project.getPath(), Store.YES));
             }
         }
 
         if (fa != null) {
             Genre g = fa.getGenre();
             if (g == Genre.PLAIN || g == Genre.XREFABLE || g == Genre.HTML) {
-                doc.add(new Field("t", g.typeName(), string_ft_stored_nanalyzed_norms
+                doc.add(new Field(QueryBuilder.T, g.typeName(), string_ft_stored_nanalyzed_norms
                     ));
             }                   
             fa.analyze(doc, StreamSource.fromFile(file), xrefOut);
@@ -301,9 +302,9 @@ public static String getContentType(InputStream in, String file) throws IOExcept
     }
 
     /**
-     * Write a browsable version of the file
+     * Write a browse-able version of the file
      *
-     * @param factory The analyzer factory for this filetype
+     * @param factory The analyzer factory for this file type
      * @param in The input stream containing the data
      * @param out Where to write the result
      * @param defs definitions for the source file, if available
@@ -329,7 +330,7 @@ public static void writeXref(FileAnalyzerFactory factory, Reader in,
     /**
      * Get the genre of a file
      *
-     * @param file The file to inpect
+     * @param file The file to inspect
      * @return The genre suitable to decide how to display the file
      */
     public static Genre getGenre(String file) {
@@ -440,7 +441,7 @@ public static FileAnalyzerFactory find(InputStream in, String file)
      */
     public static FileAnalyzerFactory find(String file) {
         String path = file;
-        int i = 0;
+        int i;
         if (((i = path.lastIndexOf('/')) > 0 || (i = path.lastIndexOf('\\')) > 0)
             && (i + 1 < path.length())) {
             path = path.substring(i + 1);
@@ -458,7 +459,7 @@ public static FileAnalyzerFactory find(String file) {
     }
 
     /**
-     * Finds a suitable analyser class for the data in this stream
+     * Finds a suitable analyzer class for the data in this stream
      *
      * @param in The stream containing the data to analyze
      * @return the analyzer factory to use
@@ -497,7 +498,7 @@ public static FileAnalyzerFactory find(InputStream in) throws IOException {
     }
 
     /**
-     * Finds a suitable analyser class for a magic signature
+     * Finds a suitable analyzer class for a magic signature
      *
      * @param signature the magic signature look up
      * @return the analyzer factory to use
@@ -536,7 +537,7 @@ private static FileAnalyzerFactory find(byte[] signature)
 
     /** Byte-order markers. */
     private static final Map<String, byte[]> BOMS =
-            new HashMap<String, byte[]>();
+            new HashMap<>();
     static {
         BOMS.put("UTF-8", new byte[] {(byte) 0xEF, (byte) 0xBB, (byte) 0xBF});
         BOMS.put("UTF-16BE", new byte[] {(byte) 0xFE, (byte) 0xFF});

src/org/opensolaris/opengrok/analysis/CompatibleAnalyser.java

@@ -26,6 +26,7 @@
 import org.apache.lucene.analysis.Analyzer;
 import org.opensolaris.opengrok.analysis.plain.PlainFullTokenizer;
 import org.opensolaris.opengrok.analysis.plain.PlainSymbolTokenizer;
+import org.opensolaris.opengrok.search.QueryBuilder;
 
 public class CompatibleAnalyser extends Analyzer {
 
@@ -36,16 +37,16 @@ public CompatibleAnalyser() {
     @Override
     protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
         switch (fieldName) {
-            case "full":
+            case QueryBuilder.FULL:
                 return new TokenStreamComponents(new PlainFullTokenizer(reader));
-            case "refs":
+            case QueryBuilder.REFS:
                 return new TokenStreamComponents(new PlainSymbolTokenizer(reader));
-            case "defs":
+            case QueryBuilder.DEFS:
                 return new TokenStreamComponents(new PlainSymbolTokenizer(reader));
-            case "path":
-            case "project":
+            case QueryBuilder.PATH:
+            case QueryBuilder.PROJECT:
                 return new TokenStreamComponents(new PathTokenizer(reader));
-            case "hist":
+            case QueryBuilder.HIST:
                 return new HistoryAnalyzer().createComponents(fieldName, reader);
             default:
                 return new TokenStreamComponents(new PlainFullTokenizer(reader));

src/org/opensolaris/opengrok/analysis/PathTokenizer.java

@@ -27,31 +27,54 @@
 import java.util.Arrays;
 import org.apache.lucene.analysis.Tokenizer;
 import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
+import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
 
+/**
+ * Tokenizer for paths filenames and extensions Input:
+ *
+ * <pre>
+ *  /topdir/subdir/filename.ext
+ * </pre>
+ *
+ * Output:
+ *
+ * <pre>
+ *  topdir
+ *  subdir
+ *  filename
+ *  .
+ *  ext
+ * </pre>
+ */
 public class PathTokenizer extends Tokenizer {
 
-    // below should be '/' since we try to convert even windows file separators to unix ones
-    private static final char dirSep = '/';
-    private boolean dot = false;
-    private static final char ADOT[]={'.'};
+    // below should be '/' since we try to convert even windows file separators 
+    // to unix ones
+    public static final char DEFAULT_DELIMITER = '/';
     private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
+    private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);    
+    private int startPosition = 0;
+    private final char delimiter;
+    private int charsRead = 0;
+    private boolean dot = false;
+    private static final char cdot = '.';
 
     public PathTokenizer(Reader input) {
-        super(input);        
-    }
-
-    @Override
-    public void reset() throws IOException {
-        super.reset();
-        dot = false;
+        super(input);
+        this.delimiter = DEFAULT_DELIMITER;        
     }
 
     @Override
     public final boolean incrementToken() throws IOException {
         clearAttributes();
         if (dot) {
-            dot = false;
-            termAtt.copyBuffer(ADOT,0,1);
+            dot = false;            
+            termAtt.setEmpty();
+            termAtt.append(cdot);
+            termAtt.setLength(1);
+            offsetAtt.setOffset(correctOffset(startPosition), correctOffset(startPosition + 1));
+            startPosition++;
             return true;
         }
 
@@ -60,22 +83,42 @@ public final boolean incrementToken() throws IOException {
         int i = 0;
         do {
             c = input.read();
+            charsRead++;
             if (c == -1) {
                 return false;
             }
-        } while (c == dirSep);
+        } while (c == delimiter);
 
         do {
             if (i >= buf.length) {
                 buf = Arrays.copyOf(buf, buf.length * 2);
             }
             buf[i++] = Character.toLowerCase((char) c);
             c = input.read();
-        } while (c != dirSep && c != '.' && !Character.isWhitespace(c) && c != -1);
-        if (c == '.') {
+            charsRead++;
+        } while ( c != delimiter && c != cdot && !Character.isWhitespace(c) && c != -1);
+        if (c == cdot) {
             dot = true;
-        }
+        }        
         termAtt.copyBuffer(buf, 0, i);
+        termAtt.setLength(i);
+        offsetAtt.setOffset(correctOffset(startPosition), correctOffset(startPosition + i));
+        startPosition = startPosition + i + 1;
         return true;
     }
+
+    @Override
+    public final void end() {
+        // set final offset
+        int finalOffset = correctOffset(charsRead);
+        offsetAtt.setOffset(finalOffset, finalOffset);
+    }
+
+    @Override
+    public void reset() throws IOException {
+        super.reset();
+        dot=false;
+        charsRead = 0;
+        startPosition = 0;
+    }
 }

src/org/opensolaris/opengrok/index/IndexDatabase.java

@@ -99,6 +99,9 @@ public class IndexDatabase {
     private Ctags ctags;
     private LockFactory lockfact;
     private final BytesRef emptyBR = new BytesRef("");
+    
+    //Directory where we store indexes
+    private static final String INDEX_DIR="index";
 
     /**
      * Create a new instance of the Index Database. Use this constructor if you
@@ -143,7 +146,7 @@ public static void updateAll(ExecutorService executor) throws IOException {
      */
     static void updateAll(ExecutorService executor, IndexChangedListener listener) throws IOException {
         RuntimeEnvironment env = RuntimeEnvironment.getInstance();
-        List<IndexDatabase> dbs = new ArrayList<IndexDatabase>();
+        List<IndexDatabase> dbs = new ArrayList<>();
 
         if (env.hasProjects()) {
             for (Project project : env.getProjects()) {
@@ -182,7 +185,7 @@ public void run() {
      */
     public static void update(ExecutorService executor, IndexChangedListener listener, List<String> paths) throws IOException {
         RuntimeEnvironment env = RuntimeEnvironment.getInstance();
-        List<IndexDatabase> dbs = new ArrayList<IndexDatabase>();
+        List<IndexDatabase> dbs = new ArrayList<>();
 
         for (String path : paths) {
             Project project = Project.getProject(path);
@@ -236,7 +239,7 @@ public void run() {
     private void initialize() throws IOException {
         synchronized (this) {
             RuntimeEnvironment env = RuntimeEnvironment.getInstance();
-            File indexDir = new File(env.getDataRootFile(), "index");
+            File indexDir = new File(env.getDataRootFile(), INDEX_DIR);
             File spellDir = new File(env.getDataRootFile(), "spellIndex");
             if (project != null) {
                 indexDir = new File(indexDir, project.getPath());
@@ -267,10 +270,10 @@ private void initialize() throws IOException {
             if (env.isGenerateHtml()) {
                 xrefDir = new File(env.getDataRootFile(), "xref");
             }
-            listeners = new ArrayList<IndexChangedListener>();
+            listeners = new ArrayList<>();
             dirtyFile = new File(indexDir, "dirty");
             dirty = dirtyFile.exists();
-            directories = new ArrayList<String>();
+            directories = new ArrayList<>();
         }
     }
 
@@ -363,7 +366,7 @@ public void update() throws IOException, HistoryException {
                 if (numDocs > 0) {
                     Fields uFields = MultiFields.getFields(reader);//reader.getTermVectors(0);
                     terms = uFields.terms(QueryBuilder.U);
-                }                
+                }
 
                 try {
                     if (numDocs > 0) {
@@ -444,7 +447,7 @@ public void update() throws IOException, HistoryException {
      * @throws IOException if an error occurs
      */
     static void optimizeAll(ExecutorService executor) throws IOException {
-        List<IndexDatabase> dbs = new ArrayList<IndexDatabase>();
+        List<IndexDatabase> dbs = new ArrayList<>();
         RuntimeEnvironment env = RuntimeEnvironment.getInstance();
         if (env.hasProjects()) {
             for (Project project : env.getProjects()) {
@@ -805,7 +808,7 @@ private boolean isLocal(String path) {
      *
      */
     private int indexDown(File dir, String parent, boolean count_only, int cur_count, int est_total) throws IOException {
-        int lcur_count = cur_count;        
+        int lcur_count = cur_count;
         if (isInterrupted()) {
             return lcur_count;
         }
@@ -959,7 +962,7 @@ public static void listAllFiles(List<String> subFiles) throws IOException {
     public void listFiles() throws IOException {
         IndexReader ireader = null;
         TermsEnum iter=null;
-        Terms terms = null;        
+        Terms terms = null;
 
         try {
             ireader = DirectoryReader.open(indexDirectory); // open existing index
@@ -1064,7 +1067,7 @@ public static IndexReader getIndexReader(String path) {
         IndexReader ret = null;
 
         RuntimeEnvironment env = RuntimeEnvironment.getInstance();
-        File indexDir = new File(env.getDataRootFile(), "index");
+        File indexDir = new File(env.getDataRootFile(), INDEX_DIR);
 
         if (env.hasProjects()) {
             Project p = Project.getProject(path);