Minor API improvements and some tests

Author: ahornace

src/org/opensolaris/opengrok/web/api/v1/suggester/provider/service/impl/SuggesterServiceImpl.java

@@ -32,6 +32,7 @@
 import org.apache.lucene.util.BytesRef;
 import org.opengrok.suggest.LookupResultItem;
 import org.opengrok.suggest.Suggester;
+import org.opengrok.suggest.Suggester.NamedIndexDir;
 import org.opengrok.suggest.Suggester.NamedIndexReader;
 import org.opengrok.suggest.query.SuggesterQuery;
 import org.opensolaris.opengrok.configuration.Configuration;
@@ -189,13 +190,17 @@ public void refresh(final String project) {
                 logger.log(Level.FINE, "Cannot refresh {0} because suggester is not initialized", project);
                 return;
             }
-            suggester.rebuild(Collections.singletonList(Paths.get(config.getDataRoot(), IndexDatabase.INDEX_DIR,
-                    p.getPath())));
+            suggester.rebuild(Collections.singleton(getNamedIndexDir(p)));
         } finally {
             lock.readLock().unlock();
         }
     }
 
+    private NamedIndexDir getNamedIndexDir(final Project p) {
+        Path indexPath = Paths.get(env.getConfiguration().getDataRoot(), IndexDatabase.INDEX_DIR, p.getPath());
+        return new NamedIndexDir(p.getName(), indexPath);
+    }
+
     /** {@inheritDoc} */
     @Override
     public void delete(final String project) {
@@ -281,17 +286,15 @@ private void initSuggester() {
         }).start();
     }
 
-    private static List<Path> getAllProjectIndexDirs() {
-        Configuration config = RuntimeEnvironment.getInstance().getConfiguration();
-
-        if (config.isProjectsEnabled()) {
-            return RuntimeEnvironment.getInstance().getProjectList().stream()
+    private Collection<NamedIndexDir> getAllProjectIndexDirs() {
+        if (env.getConfiguration().isProjectsEnabled()) {
+            return env.getProjectList().stream()
                     .filter(Project::isIndexed)
-                    .map(project -> Paths.get(config.getDataRoot(), IndexDatabase.INDEX_DIR, project.getPath()))
+                    .map(this::getNamedIndexDir)
                     .collect(Collectors.toList());
         } else {
-            return Collections.singletonList(Paths.get(
-                    RuntimeEnvironment.getInstance().getDataRootPath(), IndexDatabase.INDEX_DIR));
+            Path indexDir = Paths.get(env.getDataRootPath(), IndexDatabase.INDEX_DIR);
+            return Collections.singleton(new NamedIndexDir("", indexDir));
         }
     }
 

suggester/pom.xml

@@ -15,14 +15,17 @@
             <groupId>org.apache.lucene</groupId>
             <artifactId>lucene-core</artifactId>
         </dependency>
+
         <dependency>
             <groupId>org.apache.lucene</groupId>
             <artifactId>lucene-analyzers-common</artifactId>
         </dependency>
+
         <dependency>
             <groupId>org.apache.lucene</groupId>
             <artifactId>lucene-queryparser</artifactId>
         </dependency>
+
         <dependency>
             <groupId>org.apache.lucene</groupId>
             <artifactId>lucene-suggest</artifactId>
@@ -72,6 +75,13 @@
             <version>1.7.25</version>
         </dependency>
 
+        <dependency>
+            <groupId>org.awaitility</groupId>
+            <artifactId>awaitility</artifactId>
+            <version>3.1.0</version>
+            <scope>test</scope>
+        </dependency>
+
     </dependencies>
 
     <build>
@@ -103,6 +113,7 @@
                     </execution>
                 </executions>
             </plugin>
+
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-surefire-plugin</artifactId>

suggester/src/main/java/org/opengrok/suggest/Suggester.java

@@ -63,7 +63,7 @@ public final class Suggester implements Closeable {
 
     private static final Logger logger = Logger.getLogger(Suggester.class.getName());
 
-    private final Map<String, FieldWFSTCollection> projectData = new ConcurrentHashMap<>();
+    private final Map<String, SuggesterProjectData> projectData = new ConcurrentHashMap<>();
 
     private final Object lock = new Object();
 
@@ -115,9 +115,9 @@ public Suggester(
 
     /**
      * Initializes suggester data for specified indexes. The data is initialized asynchronously.
-     * @param luceneIndexes paths where Lucene indexes are stored
+     * @param luceneIndexes paths to lucene indexes and name with which the index should be associated
      */
-    public void init(final Collection<Path> luceneIndexes) {
+    public void init(final Collection<NamedIndexDir> luceneIndexes) {
         if (luceneIndexes == null || luceneIndexes.isEmpty()) {
             logger.log(Level.INFO, "No index directories found, exiting...");
             return;
@@ -131,17 +131,17 @@ public void init(final Collection<Path> luceneIndexes) {
 
             ExecutorService executorService = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors());
 
-            for (Path indexDir : luceneIndexes) {
+            for (NamedIndexDir indexDir : luceneIndexes) {
                 submitInitIfIndexExists(executorService, indexDir);
             }
 
             shutdownAndAwaitTermination(executorService, "Suggester successfully initialized");
         }
     }
 
-    private void submitInitIfIndexExists(final ExecutorService executorService, final Path indexDir) {
+    private void submitInitIfIndexExists(final ExecutorService executorService, final NamedIndexDir indexDir) {
         try {
-            if (indexExists(indexDir)) {
+            if (indexExists(indexDir.path)) {
                 executorService.submit(getInitRunnable(indexDir));
             } else {
                 logger.log(Level.FINE, "Index in {0} directory does not exist, skipping...", indexDir);
@@ -151,17 +151,17 @@ private void submitInitIfIndexExists(final ExecutorService executorService, fina
         }
     }
 
-    private Runnable getInitRunnable(final Path indexDir) {
+    private Runnable getInitRunnable(final NamedIndexDir indexDir) {
         return () -> {
             try {
                 Instant start = Instant.now();
                 logger.log(Level.FINE, "Initializing {0}", indexDir);
 
-                FieldWFSTCollection wfst = new FieldWFSTCollection(FSDirectory.open(indexDir),
-                        getSuggesterDir(indexDir), allowMostPopular, allowedFields);
+                SuggesterProjectData wfst = new SuggesterProjectData(FSDirectory.open(indexDir.path),
+                        getSuggesterDir(indexDir.name), allowMostPopular, allowedFields);
                 wfst.init();
                 if (projectsEnabled) {
-                    projectData.put(indexDir.getFileName().toString(), wfst);
+                    projectData.put(indexDir.name, wfst);
                 } else {
                     projectData.put(PROJECTS_DISABLED_KEY, wfst);
                 }
@@ -174,9 +174,9 @@ private Runnable getInitRunnable(final Path indexDir) {
         };
     }
 
-    private Path getSuggesterDir(final Path indexDir) {
+    private Path getSuggesterDir(final String indexDirName) {
         if (projectsEnabled) {
-            return suggesterDir.toPath().resolve(indexDir.getFileName());
+            return suggesterDir.toPath().resolve(indexDirName);
         } else {
             return this.suggesterDir.toPath();
         }
@@ -200,9 +200,9 @@ private void shutdownAndAwaitTermination(final ExecutorService executorService,
 
     /**
      * Rebuilds the data structures for specified indexes.
-     * @param indexDirs paths where Lucene indexes are stored
+     * @param indexDirs paths to lucene indexes and name with which the index should be associated
      */
-    public void rebuild(final List<Path> indexDirs) {
+    public void rebuild(final Collection<NamedIndexDir> indexDirs) {
         if (indexDirs == null || indexDirs.isEmpty()) {
             logger.log(Level.INFO, "Not rebuilding suggester data because no index directories were specified");
             return;
@@ -213,10 +213,10 @@ public void rebuild(final List<Path> indexDirs) {
 
             ExecutorService executorService = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors());
 
-            for (Path indexDir : indexDirs) {
-                FieldWFSTCollection fieldsWFST = projectData.get(indexDir.getFileName().toString());
-                if (fieldsWFST != null) {
-                    executorService.submit(getRebuildRunnable(fieldsWFST));
+            for (NamedIndexDir indexDir : indexDirs) {
+                SuggesterProjectData data = this.projectData.get(indexDir.name);
+                if (data != null) {
+                    executorService.submit(getRebuildRunnable(data));
                 } else {
                     submitInitIfIndexExists(executorService, indexDir);
                 }
@@ -226,36 +226,35 @@ public void rebuild(final List<Path> indexDirs) {
         }
     }
 
-    private Runnable getRebuildRunnable(final FieldWFSTCollection fieldsWFST) {
+    private Runnable getRebuildRunnable(final SuggesterProjectData data) {
         return () -> {
             try {
                 Instant start = Instant.now();
-                logger.log(Level.FINE, "Rebuilding {0}", fieldsWFST);
-                fieldsWFST.rebuild();
+                logger.log(Level.FINE, "Rebuilding {0}", data);
+                data.rebuild();
 
                 Duration d = Duration.between(start, Instant.now());
-                logger.log(Level.FINE, "Rebuild of {0} finished, took {1}", new Object[] {fieldsWFST, d});
+                logger.log(Level.FINE, "Rebuild of {0} finished, took {1}", new Object[] {data, d});
             } catch (Exception e) {
                 logger.log(Level.SEVERE, "Could not rebuild suggester", e);
             }
         };
     }
 
     /**
-     * Removes the data associated with the names {@code projectNames}.
-     * @param projectNames names of the indexes to delete (name is determined by the name of the Lucene index
-     * directory)
+     * Removes the data associated with the provided names.
+     * @param names names of the indexes to delete
      */
-    public void remove(final Iterable<String> projectNames) {
-        if (projectNames == null) {
+    public void remove(final Iterable<String> names) {
+        if (names == null) {
             return;
         }
 
         synchronized (lock) {
-            logger.log(Level.INFO, "Removing following suggesters: {0}", projectNames);
+            logger.log(Level.INFO, "Removing following suggesters: {0}", names);
 
-            for (String suggesterName : projectNames) {
-                FieldWFSTCollection collection = projectData.get(suggesterName);
+            for (String suggesterName : names) {
+                SuggesterProjectData collection = projectData.get(suggesterName);
                 if (collection == null) {
                     logger.log(Level.WARNING, "Unknown suggester {0}", suggesterName);
                     continue;
@@ -290,7 +289,7 @@ public List<LookupResultItem> search(
 
         List<LookupResultItem> results = readers.parallelStream().flatMap(namedIndexReader -> {
 
-            FieldWFSTCollection data = projectData.get(namedIndexReader.name);
+            SuggesterProjectData data = projectData.get(namedIndexReader.name);
             if (data == null) {
                 logger.log(Level.FINE, "{0} not yet initialized", namedIndexReader.name);
                 return Stream.empty();
@@ -385,7 +384,7 @@ public void increaseSearchCount(final String project, final Term term, final int
         if (!allowMostPopular) {
             return;
         }
-        FieldWFSTCollection data;
+        SuggesterProjectData data;
         if (!projectsEnabled) {
             data = projectData.get(PROJECTS_DISABLED_KEY);
         } else {
@@ -408,7 +407,7 @@ public List<Entry<BytesRef, Integer>> getSearchCounts(
             final int page,
             final int pageSize
     ) {
-        FieldWFSTCollection data = projectData.get(project);
+        SuggesterProjectData data = projectData.get(project);
         if (data == null) {
             logger.log(Level.FINE, "Cannot retrieve search counts because data for project {0} were not found",
                     project);
@@ -432,16 +431,65 @@ public void close() {
         });
     }
 
+    /**
+     * Model classes for holding project name and path to ist index directory.
+     */
+    public static class NamedIndexDir {
+
+        /**
+         * Name of the project.
+         */
+        private final String name;
+
+        /**
+         * Path to index directory for project with name {@link #name}.
+         */
+        private final Path path;
+
+        public NamedIndexDir(final String name, final Path path) {
+            if (name == null) {
+                throw new IllegalArgumentException("Name cannot be null");
+            }
+            if (path == null) {
+                throw new IllegalArgumentException("Path cannot be null");
+            }
+
+            this.name = name;
+            this.path = path;
+        }
+
+        public String getName() {
+            return name;
+        }
+
+        public Path getPath() {
+            return path;
+        }
+    }
+
     /**
      * Model class to hold the project name and its {@link IndexReader}.
      */
     public static class NamedIndexReader {
 
+        /**
+         * Name of the project.
+         */
         private final String name;
 
+        /**
+         * IndexReader of the project with {@link #name}.
+         */
         private final IndexReader reader;
 
         public NamedIndexReader(final String name, final IndexReader reader) {
+            if (name == null) {
+                throw new IllegalArgumentException("Name cannot be null");
+            }
+            if (reader == null) {
+                throw new IllegalArgumentException("Reader cannot be null");
+            }
+
             this.name = name;
             this.reader = reader;
         }

suggester/src/main/java/org/opengrok/suggest/FieldWFSTCollection.java renamed to suggester/src/main/java/org/opengrok/suggest/SuggesterProjectData.java

@@ -64,9 +64,9 @@
 /**
  * Holds all the necessary data for one index directory. In the context of OpenGrok it is one project.
  */
-class FieldWFSTCollection implements Closeable {
+class SuggesterProjectData implements Closeable {
 
-    private static final Logger logger = Logger.getLogger(FieldWFSTCollection.class.getName());
+    private static final Logger logger = Logger.getLogger(SuggesterProjectData.class.getName());
 
     private static final int MAX_TERM_SIZE = Short.MAX_VALUE - 3;
 
@@ -98,7 +98,7 @@ class FieldWFSTCollection implements Closeable {
 
     private Set<String> fields;
 
-    FieldWFSTCollection(
+    SuggesterProjectData(
             final Directory indexDir,
             final Path suggesterDir,
             final boolean allowMostPopular,
@@ -275,6 +275,7 @@ private void createSuggesterDir() throws IOException {
     }
 
     private void initSearchCountMap() throws IOException {
+        searchCountMaps.values().forEach(PopularityMap::close);
         searchCountMaps.clear();
 
         for (String field : fields) {
@@ -325,6 +326,7 @@ private double getAverageLength(final String field) {
         if (averageLengths.containsKey(field)) {
             return averageLengths.get(field);
         }
+        logger.log(Level.FINE, "Could not determine average length for field {0}, using default one", field);
         return AVERAGE_LENGTH_DEFAULT;
     }
 
@@ -509,7 +511,7 @@ public List<Entry<BytesRef, Integer>> getSearchCountsSorted(final String field,
 
     @Override
     public String toString() {
-        return "FieldWFSTCollection{" +
+        return "SuggesterProjectData{" +
                 "indexDir=" + indexDir +
                 ", suggesterDir=" + suggesterDir +
                 ", allowMostPopular=" + allowMostPopular +

suggester/src/main/java/org/opengrok/suggest/SuggesterSearcher.java

@@ -76,7 +76,7 @@ class SuggesterSearcher extends IndexSearcher {
 
     /**
      * Returns the suggestions for generic {@link SuggesterQuery} (almost all except lone
-     * {@link org.opengrok.suggest.query.SuggesterPrefixQuery} for which check {@link FieldWFSTCollection}).
+     * {@link org.opengrok.suggest.query.SuggesterPrefixQuery} for which see {@link SuggesterProjectData}).
      * @param query query on which the suggestions depend
      * @param project name of the project
      * @param suggesterQuery query for the suggestions

suggester/src/main/java/org/opengrok/suggest/popular/PopularityMap.java

@@ -47,4 +47,8 @@ public interface PopularityMap extends PopularityCounter, AutoCloseable {
      */
     List<Entry<BytesRef, Integer>> getPopularityData(int page, int pageSize);
 
+    /** {@inheritDoc} */
+    @Override
+    void close();
+
 }