remove historycache entries automatically (#1675)

fixes #767

Author: Vladimir Kotal

src/org/opensolaris/opengrok/history/FileHistoryCache.java

@@ -715,6 +715,30 @@ public void clear(Repository repository) {
         }
     }
 
+    @Override
+    public void clearFile(String path) {
+        RuntimeEnvironment env = RuntimeEnvironment.getInstance();
+        File historyFile;
+        try {
+            historyFile = getCachedFile(new File(env.getSourceRootPath() + path));
+        } catch (HistoryException ex) {
+            LOGGER.log(Level.WARNING, "cannot get history file for file " + path, ex);
+            return;
+        }
+        File parent = historyFile.getParentFile();
+
+        if (!historyFile.delete() && historyFile.exists()) {
+            LOGGER.log(Level.WARNING,
+                "Failed to remove obsolete history cache-file: {0}",
+                historyFile.getAbsolutePath());
+        }
+
+        if (parent.delete()) {
+            LOGGER.log(Level.FINE, "Removed empty history cache dir:{0}",
+                parent.getAbsolutePath());
+        }
+    }
+
     @Override
     public String getInfo() {
         return getClass().getSimpleName();

src/org/opensolaris/opengrok/history/HistoryCache.java

@@ -18,7 +18,7 @@
  */
 
 /*
- * Copyright (c) 2006, 2014, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2006, 2017, Oracle and/or its affiliates. All rights reserved.
  */
 package org.opensolaris.opengrok.history;
 
@@ -125,6 +125,12 @@ Map<String, Date> getLastModifiedTimes(
      */
     void clear(Repository repository) throws HistoryException;
 
+    /**
+     * Clear entry for single file from history cache.
+     * @param file path to the file relative to the source root
+     */
+    void clearFile(String file);
+
     /**
      * Get a string with information about the history cache.
      *

src/org/opensolaris/opengrok/history/HistoryGuru.java

@@ -81,6 +81,7 @@ private HistoryGuru() {
         HistoryCache cache = null;
         RuntimeEnvironment env = RuntimeEnvironment.getInstance();
         scanningDepth = env.getScanningDepth();
+
         if (env.useHistoryCache()) {
             cache = new FileHistoryCache();
 
@@ -627,14 +628,17 @@ public void createCache(Collection<String> repositories) {
      * @throws HistoryException 
      */
     public List<Repository> clearCache(Collection<String> repositories) throws HistoryException {
-        List<Repository> repos = getReposFromString(repositories);
+        List<Repository> repos = new ArrayList<>();
         HistoryCache cache = historyCache;
-        if (cache == null) {
-            cache = new FileHistoryCache();
+
+        if (!useCache()) {
+            return repos;
         }
-        for (Repository r : repos) {
+
+        for (Repository r : getReposFromString(repositories)) {
             try {
                 cache.clear(r);
+                repos.add(r);
                 LOGGER.log(Level.INFO,
                         "History cache for {0} cleared.", r.getDirectoryName());
             } catch (HistoryException e) {
@@ -647,13 +651,25 @@ public List<Repository> clearCache(Collection<String> repositories) throws Histo
         return repos;
     }
 
+    public void clearCacheFile(String path) {
+        if (!useCache()) {
+            return;
+        }
+
+        historyCache.clearFile(path);
+    }
+
     /**
      * Remove history data for a list of repositories and invalidate the list
      * of repositories accordingly.
      * @param repositories list of repository paths
      * @throws HistoryException 
      */
     public void removeCache(Collection<String> repositories) throws HistoryException {
+        if (!useCache()) {
+            return;
+        }
+
         List<Repository> repos = clearCache(repositories);
         invalidateRepositories(repos);
     }

src/org/opensolaris/opengrok/index/IndexDatabase.java

@@ -35,7 +35,9 @@
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Comparator;
+import java.util.HashSet;
 import java.util.List;
+import java.util.Set;
 import java.util.concurrent.ExecutorService;
 import java.util.logging.Level;
 import java.util.logging.Logger;
@@ -592,9 +594,13 @@ private void removeXrefFile(String path) {
         }
     }
 
+    private void removeHistoryFile(String path) {
+        HistoryGuru.getInstance().clearCacheFile(path);
+    }
+
     /**
-     * Remove a stale file (uidIter.term().text()) from the index database (and
-     * the xref file)
+     * Remove a stale file (uidIter.term().text()) from the index database,
+     * history cache and xref.
      *
      * @throws java.io.IOException if an error occurs
      */
@@ -610,6 +616,7 @@ private void removeFile() throws IOException {
         writer.commit();
 
         removeXrefFile(path);
+        removeHistoryFile(path);
 
         setDirty();
         for (IndexChangedListener listener : listeners) {
@@ -951,28 +958,32 @@ public void removeIndexChangedListener(IndexChangedListener listener) {
     }
 
     /**
-     * List all files in all of the index databases
+     * Get all files in all of the index databases.
      *
      * @throws IOException if an error occurs
+     * @return set of files
      */
-    public static void listAllFiles() throws IOException {
-        listAllFiles(null);
+    public static Set<String> getAllFiles() throws IOException {
+        return getAllFiles(null);
     }
 
     /**
-     * List all files in some of the index databases
+     * List all files in some of the index databases.
      *
      * @param subFiles Subdirectories for the various projects to list the files
      * for (or null or an empty list to dump all projects)
      * @throws IOException if an error occurs
+     * @return set of files in the index databases specified by the subFiles parameter
      */
-    public static void listAllFiles(List<String> subFiles) throws IOException {
+    public static Set<String> getAllFiles(List<String> subFiles) throws IOException {
+        Set<String> files = new HashSet<>();
         RuntimeEnvironment env = RuntimeEnvironment.getInstance();
+
         if (env.hasProjects()) {
             if (subFiles == null || subFiles.isEmpty()) {
                 for (Project project : env.getProjectList()) {
                     IndexDatabase db = new IndexDatabase(project);
-                    db.listFiles();
+                    files.addAll(db.getFiles());
                 }
             } else {
                 for (String path : subFiles) {
@@ -981,25 +992,29 @@ public static void listAllFiles(List<String> subFiles) throws IOException {
                         LOGGER.log(Level.WARNING, "Could not find a project for \"{0}\"", path);
                     } else {
                         IndexDatabase db = new IndexDatabase(project);
-                        db.listFiles();
+                        files.addAll(db.getFiles());
                     }
                 }
             }
         } else {
             IndexDatabase db = new IndexDatabase();
-            db.listFiles();
+            files = db.getFiles();
         }
+
+        return files;
     }
 
     /**
-     * List all of the files in this index database
+     * Get all files in this index database.
      *
      * @throws IOException If an IO error occurs while reading from the database
+     * @return set of files in this index database
      */
-    public void listFiles() throws IOException {
+    public Set<String> getFiles() throws IOException {
         IndexReader ireader = null;
         TermsEnum iter;
         Terms terms = null;
+        Set<String> files = new HashSet<>();
 
         try {
             ireader = DirectoryReader.open(indexDirectory); // open existing index
@@ -1010,12 +1025,37 @@ public void listFiles() throws IOException {
             }
             iter = terms.iterator(); // init uid iterator
             while (iter != null && iter.term() != null) {
-                LOGGER.fine(Util.uid2url(iter.term().utf8ToString()));
-                BytesRef next=iter.next();
-                if (next==null) {iter=null;}
+                files.add(Util.uid2url(iter.term().utf8ToString()));
+                BytesRef next = iter.next();
+                if (next == null) {
+                    iter = null;
+                }
             }
         } finally {
+            if (ireader != null) {
+                try {
+                    ireader.close();
+                } catch (IOException e) {
+                    LOGGER.log(Level.WARNING, "An error occured while closing index reader", e);
+                }
+            }
+        }
 
+        return files;
+    }
+
+    /**
+     * Get number of documents in this index database.
+     * @return number of documents
+     */
+    public int getNumFiles() throws IOException {
+        IndexReader ireader = null;
+        int numDocs = 0;
+        
+        try {
+            ireader = DirectoryReader.open(indexDirectory); // open existing index
+            numDocs = ireader.numDocs();
+        } finally {
             if (ireader != null) {
                 try {
                     ireader.close();
@@ -1024,6 +1064,8 @@ public void listFiles() throws IOException {
                 }
             }
         }
+        
+        return numDocs;
     }
 
     static void listFrequentTokens() throws IOException {

src/org/opensolaris/opengrok/index/Indexer.java

@@ -799,8 +799,11 @@ public void prepareIndexer(RuntimeEnvironment env,
                   LOGGER.info("Done...");
               }
         }
+
         if (listFiles) {
-            IndexDatabase.listAllFiles(subFiles);
+            for (String file : IndexDatabase.getAllFiles(subFiles)) {
+                LOGGER.fine(file);
+            }
         }
 
         if (createDict) {

test/org/opensolaris/opengrok/history/FileHistoryCacheTest.java

@@ -353,7 +353,7 @@ public void testStoreAndGet() throws Exception {
      * rename+change file.
      * 
      * The scenario goes as follows:
-     * - create repo
+     * - create Mercurial repository
      * - perform full reindex
      * - add changesets which renamed and modify a file
      * - perform incremental reindex