cleanup spellchecker code

use Direct spellchecker to avoid another index creation
closes #637

Author: Lubos Kosco

src/org/opensolaris/opengrok/index/IndexDatabase.java

@@ -51,8 +51,6 @@
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.TopDocs;
-import org.apache.lucene.search.spell.LuceneDictionary;
-import org.apache.lucene.search.spell.SpellChecker;
 import org.apache.lucene.store.FSDirectory;
 import org.apache.lucene.store.LockFactory;
 import org.apache.lucene.store.NoLockFactory;
@@ -82,8 +80,7 @@
 public class IndexDatabase {
 
     private Project project;
-    private FSDirectory indexDirectory;
-    private FSDirectory spellDirectory;
+    private FSDirectory indexDirectory;    
     private IndexWriter writer;
     private TermsEnum uidIter;
     private IgnoredNames ignoredNames;
@@ -241,31 +238,22 @@ public void run() {
     private void initialize() throws IOException {
         synchronized (this) {
             RuntimeEnvironment env = RuntimeEnvironment.getInstance();
-            File indexDir = new File(env.getDataRootFile(), INDEX_DIR);
-            File spellDir = new File(env.getDataRootFile(), "spellIndex");
+            File indexDir = new File(env.getDataRootFile(), INDEX_DIR);            
             if (project != null) {
-                indexDir = new File(indexDir, project.getPath());
-                spellDir = new File(spellDir, project.getPath());
+                indexDir = new File(indexDir, project.getPath());                
             }
 
             if (!indexDir.exists() && !indexDir.mkdirs()) {
                 // to avoid race conditions, just recheck..
                 if (!indexDir.exists()) {
                     throw new FileNotFoundException("Failed to create root directory [" + indexDir.getAbsolutePath() + "]");
                 }
-            }
-
-            if (!spellDir.exists() && !spellDir.mkdirs()) {
-                if (!spellDir.exists()) {
-                    throw new FileNotFoundException("Failed to create root directory [" + spellDir.getAbsolutePath() + "]");
-                }
-            }
+            }            
 
             if (!env.isUsingLuceneLocking()) {
                 lockfact = NoLockFactory.getNoLockFactory();
             }
-            indexDirectory = FSDirectory.open(indexDir, lockfact);
-            spellDirectory = FSDirectory.open(spellDir, lockfact);
+            indexDirectory = FSDirectory.open(indexDir, lockfact);            
             ignoredNames = env.getIgnoredNames();
             includedNames = env.getIncludedNames();
             analyzerGuru = new AnalyzerGuru();
@@ -426,8 +414,7 @@ public void update() throws IOException, HistoryException {
         if (!isInterrupted() && isDirty()) {
             if (RuntimeEnvironment.getInstance().isOptimizeDatabase()) {
                 optimize();
-            }
-            createSpellingSuggestions();
+            }            
             RuntimeEnvironment env = RuntimeEnvironment.getInstance();
             File timestamp = new File(env.getDataRootFile(), "timestamp");
             if (timestamp.exists()) {
@@ -518,40 +505,7 @@ public void optimize() {
             }
         }
     }
-
-    /**
-     * Generate a spelling suggestion for the definitions stored in defs
-     */
-    public void createSpellingSuggestions() {
-        IndexReader indexReader = null;
-        SpellChecker checker;
-
-        try {
-            log.info("Generating spelling suggestion index ... ");
-            indexReader = DirectoryReader.open(indexDirectory);
-            checker = new SpellChecker(spellDirectory);
-            //TODO below seems only to index "defs" , possible bug ?
-            Analyzer analyzer = AnalyzerGuru.getAnalyzer();
-            IndexWriterConfig iwc = new IndexWriterConfig(SearchEngine.LUCENE_VERSION, analyzer);
-            iwc.setOpenMode(OpenMode.CREATE_OR_APPEND);
-            checker.indexDictionary(new LuceneDictionary(indexReader, QueryBuilder.DEFS), iwc, false);
-            log.info("done");
-        } catch (IOException e) {
-            log.log(Level.SEVERE, "ERROR: Generating spelling: {0}", e);
-        } finally {
-            if (indexReader != null) {
-                try {
-                    indexReader.close();
-                } catch (IOException e) {
-                    log.log(Level.WARNING, "An error occured while closing reader", e);
-                }
-            }
-            if (spellDirectory != null) {
-                spellDirectory.close();
-            }
-        }
-    }
-
+    
     private boolean isDirty() {
         synchronized (lock) {
             return dirty;

src/org/opensolaris/opengrok/search/Summarizer.java

@@ -285,7 +285,7 @@ private Token[] getTokens(String text) throws IOException {
         //FIXME somehow integrate below cycle to getSummary to save the cloning and memory,
         //also creating Tokens is suboptimal with 3.0.0 , this whole class could be replaced by highlighter
         ArrayList<Token> result = new ArrayList<Token>();
-        TokenStream ts = analyzer.tokenStream("full", new StringReader(text));
+        TokenStream ts = analyzer.tokenStream("full", text);
         CharTermAttribute term = ts.addAttribute(CharTermAttribute.class);
         OffsetAttribute offset = ts.addAttribute(OffsetAttribute.class);
         while(ts.incrementToken()) {
@@ -297,7 +297,7 @@ private Token[] getTokens(String text) throws IOException {
 
 
     /**
-     * Get the terms from a query and adds them to hightlite
+     * Get the terms from a query and adds them to highlight
      * a stream of tokens
      *
      * @param query

src/org/opensolaris/opengrok/web/SearchHelper.java

@@ -27,7 +27,6 @@
 import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -42,9 +41,12 @@
 import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.MultiReader;
+import org.apache.lucene.index.Term;
 import org.apache.lucene.queryparser.classic.ParseException;
 import org.apache.lucene.search.*;
-import org.apache.lucene.search.spell.SpellChecker;
+import org.apache.lucene.search.spell.DirectSpellChecker;
+import org.apache.lucene.search.spell.SuggestMode;
+import org.apache.lucene.search.spell.SuggestWord;
 import org.apache.lucene.store.FSDirectory;
 import org.opensolaris.opengrok.OpenGrokLogger;
 import org.opensolaris.opengrok.analysis.CompatibleAnalyser;
@@ -64,6 +66,10 @@
  */
 public class SearchHelper {
 
+    /**
+     * max number of words to suggest for spellcheck
+     */
+    public int SPELLCHECK_SUGGEST_WORD_COUNT=5;
     /**
      * opengrok's data root: used to find the search index file
      */
@@ -128,7 +134,7 @@ public class SearchHelper {
      * the searcher used to open/search the index. Automatically set via
      * {@link #prepareExec(SortedSet)}.
      */
-    public IndexSearcher searcher;
+    public IndexSearcher searcher;    
     /**
      * list of docs which result from the executing the query
      */
@@ -147,6 +153,10 @@ public class SearchHelper {
      * {@link #prepareExec(SortedSet)}.
      */
     protected Sort sort;
+    /**
+     * the spellchecker object
+     */
+    protected DirectSpellChecker checker;    
     /**
      * projects to use to setup indexer searchers. Usually setup via
      * {@link #prepareExec(SortedSet)}.
@@ -211,7 +221,8 @@ public class SearchHelper {
     public static Set<Map.Entry<String, String>> getFileTypeDescirptions() {
         return fileTypeDescription.entrySet();
     }
-    
+        
+    File indexDir;
     /**
      * Create the searcher to use wrt. to currently set parameters and the given
      * projects. Does not produce any {@link #redirect} link. It also does
@@ -235,13 +246,13 @@ public SearchHelper prepareExec(SortedSet<String> projects) {
         }
         // the Query created by the QueryBuilder
         try {
+	    indexDir=new File(dataRoot, "index");
             query = builder.build();
             if (projects == null) {
                 errorMsg = "No project selected!";
                 return this;
             }
-            this.projects = projects;
-            File indexDir = new File(dataRoot, "index");
+            this.projects = projects;            
             if (projects.isEmpty()) {
                 //no project setup
                 FSDirectory dir = FSDirectory.open(indexDir);
@@ -285,6 +296,7 @@ public SearchHelper prepareExec(SortedSet<String> projects) {
                     sort = Sort.RELEVANCE;
                     break;
             }
+	    checker=new DirectSpellChecker();
         } catch (ParseException e) {
             errorMsg = PARSE_ERROR_MSG + e.getMessage();
         } catch (FileNotFoundException e) {
@@ -357,17 +369,20 @@ public SearchHelper executeQuery() {
     }
     private static final Pattern TABSPACE = Pattern.compile("[\t ]+");
 
-    private static void getSuggestion(String term, SpellChecker checker,
+    private void getSuggestion(Term term, IndexReader ir,
             List<String> result) throws IOException {
         if (term == null) {
             return;
         }
-        String[] toks = TABSPACE.split(term, 0);
+        String[] toks = TABSPACE.split(term.text(), 0);
         for (int j = 0; j < toks.length; j++) {
-            if (toks[j].length() <= 3) {
-                continue;
-            }
-            result.addAll(Arrays.asList(checker.suggestSimilar(toks[j].toLowerCase(), 5)));
+         //TODO below seems to be case insensitive ... for refs/defs this is bad
+	    SuggestWord[] words=checker.suggestSimilar(
+		    new Term(term.field(),toks[j]), SPELLCHECK_SUGGEST_WORD_COUNT, ir, 
+		    SuggestMode.SUGGEST_ALWAYS);	    
+	    for (SuggestWord w: words) {
+              result.add(w.string);
+	    }
         }
     }
 
@@ -379,74 +394,78 @@ private static void getSuggestion(String term, SpellChecker checker,
      * <li>{@link #projects}</li> <li>{@link #dataRoot}</li>
      * <li>{@link #builder}</li> </ul>
      *
-     * @return a possible empty list of sugeestions.
+     * @return a possible empty list of suggestions.
      */
     public List<Suggestion> getSuggestions() {
         if (projects == null) {
-            return new ArrayList<Suggestion>(0);
+            return new ArrayList<>(0);
         }
-        File[] spellIndex = null;
+	String name[];
         if (projects.isEmpty()) {
-            spellIndex = new File[]{new File(dataRoot, "spellIndex")};
+            name=new String[]{"/"};
         } else if (projects.size() == 1) {
-            spellIndex = new File[]{
-                new File(dataRoot, "spellIndex/" + projects.first())
-            };
+		name=new String[]{projects.first()};
         } else {
-            spellIndex = new File[projects.size()];
-            int ii = 0;
-            File indexDir = new File(dataRoot, "spellIndex");
+            name = new String[projects.size()];
+            int ii = 0;            
             for (String proj : projects) {
-                spellIndex[ii++] = new File(indexDir, proj);
+                name[ii++] = proj;
             }
         }
-        List<Suggestion> res = new ArrayList<Suggestion>();
-        List<String> dummy = new ArrayList<String>();
-        for (int idx = 0; idx < spellIndex.length; idx++) {
-            if (!spellIndex[idx].exists()) {
-                continue;
-            }
-            FSDirectory spellDirectory = null;
-            SpellChecker checker = null;
-            Suggestion s = new Suggestion(spellIndex[idx].getName());
+        List<Suggestion> res = new ArrayList<>();
+        List<String> dummy = new ArrayList<>();
+	FSDirectory dir;
+	IndexReader ir=null;
+	Term t;
+	for (int idx = 0; idx < name.length; idx++) {
+            Suggestion s = new Suggestion(name[idx]);	    
             try {
-                spellDirectory = FSDirectory.open(spellIndex[idx]);
-                checker = new SpellChecker(spellDirectory);
-                getSuggestion(builder.getFreetext(), checker, dummy);
+	        dir = FSDirectory.open(new File(indexDir, name[idx]));	    
+		ir = DirectoryReader.open(dir);
+		if (builder.getFreetext()!=null && 
+			!builder.getFreetext().isEmpty()) {
+		t=new Term(QueryBuilder.FULL,builder.getFreetext());
+                getSuggestion(t, ir, dummy);
                 s.freetext = dummy.toArray(new String[dummy.size()]);
                 dummy.clear();
-                getSuggestion(builder.getRefs(), checker, dummy);
+		}
+		if (builder.getRefs()!=null && !builder.getRefs().isEmpty()) {
+		t=new Term(QueryBuilder.REFS,builder.getRefs());
+                getSuggestion(t, ir, dummy);
                 s.refs = dummy.toArray(new String[dummy.size()]);
                 dummy.clear();
-                // TODO it seems the only true spellchecker is for
-                // below field, see IndexDatabase
-                // createspellingsuggestions ...
-                getSuggestion(builder.getDefs(), checker, dummy);
+		}
+                if (builder.getDefs()!=null && !builder.getDefs().isEmpty()) {
+		t=new Term(QueryBuilder.DEFS,builder.getDefs());
+                getSuggestion(t, ir, dummy);
                 s.defs = dummy.toArray(new String[dummy.size()]);
                 dummy.clear();
-                if (s.freetext.length > 0 || s.defs.length > 0 || s.refs.length > 0) {
+		}
+		//TODO suggest also for path and history?
+                if ((s.freetext!=null && s.freetext.length > 0) || 
+			(s.defs!=null && s.defs.length > 0) || 
+			(s.refs!=null && s.refs.length > 0) ) {
                     res.add(s);
                 }
             } catch (IOException e) {
-                log.log(Level.WARNING, "Got excption while getting spelling suggestions: ", e);
+                log.log(Level.WARNING, "Got exception while getting "
+			+ "spelling suggestions: ", e);
             } finally {
-                if (spellDirectory != null) {
-                    spellDirectory.close();
-                }
-                if (checker != null) {
-                    try {
-                        checker.close();
-                    } catch (Exception x) {
-                        log.log(Level.WARNING, "Got excption while closing spelling suggestions: ", x);
-                    }
-                }
-            }
-        }
+                if (ir != null) {
+			try {
+				ir.close();
+			} catch (IOException ex) {
+				log.log(Level.WARNING, "Got exception while "
+					+ "getting spelling suggestions: ", ex);
+			}
+               }
+	    }	
+	}    
         return res;
     }
 
     /**
-     * Prepare the fields to support printing a fullblown summary. Does nothing
+     * Prepare the fields to support printing a full blown summary. Does nothing
      * if {@link #redirect} or {@link #errorMsg} have a none-{@code null} value.
      *
      * <p> Parameters which should be populated/set at this time: <ul>

web/search.jsp

@@ -133,18 +133,24 @@ include file="menu.jspf"
         List<Suggestion> hints = searchHelper.getSuggestions();
         for (Suggestion hint : hints) {
         %><p><font color="#cc0000">Did you mean (for <%= hint.name %>)</font>:<%
-            for (String word : hint.freetext) {
+	  if (hint.freetext!=null) { 
+	    for (String word : hint.freetext) {
             %> <a href="search?q=<%= Util.URIEncode(word) %>"><%=
                 Util.htmlize(word) %></a> &nbsp; <%
-            }
-            for (String word : hint.refs) {
+	    }
+	  }
+	  if (hint.refs!=null)  {
+	    for (String word : hint.refs) {
             %> <a href="search?refs=<%= Util.URIEncode(word) %>"><%=
                 Util.htmlize(word) %></a> &nbsp; <%
-            }
-            for (String word : hint.defs) {
+	    }
+	  }
+	  if (hint.defs!=null) {
+	    for (String word : hint.defs) {
             %> <a href="search?defs=<%= Util.URIEncode(word) %>"><%=
                 Util.htmlize(word) %></a> &nbsp; <%
             }
+	  }
         %></p><%
         }
         %>
@@ -233,4 +239,4 @@ include file="menu.jspf"
 
 include file="foot.jspf"
 
-%>
+%>