History cache control per project

opengrok-indexer/src/main/java/org/opengrok/indexer/configuration/Project.java

@@ -84,10 +84,15 @@ public class Project implements Comparable<Project>, Nameable, Serializable {
     private Boolean handleRenamedFiles = null;
 
     /**
-     * This flag enables/disables per-project history cache.
+     * This flag enables/disables per-project history queries.
      */
     private Boolean historyEnabled = null;
 
+    /**
+     * This flag enables/disables per-project history cache.
+     */
+    private Boolean historyCacheEnabled = null;
+
     /**
      * This flag enables/disables per-project annotation cache.
      */
@@ -301,6 +306,20 @@ public void setHistoryEnabled(boolean flag) {
         this.historyEnabled = flag;
     }
 
+    /**
+     * @return true if this project should have history cache.
+     */
+    public boolean isHistoryCacheEnabled() {
+        return historyCacheEnabled != null && historyCacheEnabled;
+    }
+
+    /**
+     * @param flag true if project should have history cache, false otherwise.
+     */
+    public void setHistoryCacheEnabled(boolean flag) {
+        this.historyCacheEnabled = flag;
+    }
+
     /**
      * @return true if this project should have annotation cache.
      */
@@ -494,11 +513,16 @@ public final void completeWithDefaults() {
             setHandleRenamedFiles(env.isHandleHistoryOfRenamedFiles());
         }
 
-        // Allow project to override global setting of history cache generation.
+        // Allow project to override global setting of history queries.
         if (historyEnabled == null) {
             setHistoryEnabled(env.isHistoryEnabled());
         }
 
+        // Allow project to override global setting of history cache generation.
+        if (historyCacheEnabled == null) {
+            setHistoryCacheEnabled(env.useHistoryCache());
+        }
+
         // Allow project to override global setting of annotation cache generation.
         if (annotationCacheEnabled == null) {
             setAnnotationCacheEnabled(env.isAnnotationCacheEnabled());

opengrok-indexer/src/main/java/org/opengrok/indexer/history/HistoryGuru.java

@@ -135,6 +135,11 @@ private HistoryGuru() {
         repositoryLookup = RepositoryLookup.cached();
     }
 
+    @VisibleForTesting
+    HistoryCache getHistoryCache() {
+        return historyCache;
+    }
+
     /**
      * Set annotation cache to its default implementation.
      * @return {@link AnnotationCache} instance or {@code null} on error
@@ -164,18 +169,16 @@ static AnnotationCache initializeAnnotationCache() {
      * @return {@link HistoryCache} instance
      */
     private HistoryCache initializeHistoryCache() {
-        HistoryCache historyCacheResult = null;
-        if (env.useHistoryCache()) {
-            historyCacheResult = new FileHistoryCache();
+        HistoryCache historyCacheResult = new FileHistoryCache();
 
-            try {
-                historyCacheResult.initialize();
-            } catch (CacheException he) {
-                LOGGER.log(Level.WARNING, "Failed to initialize the history cache", he);
-                // Failed to initialize, run without a history cache.
-                historyCacheResult = null;
-            }
+        try {
+            historyCacheResult.initialize();
+        } catch (CacheException he) {
+            LOGGER.log(Level.WARNING, "Failed to initialize the history cache", he);
+            // Failed to initialize, run without a history cache.
+            historyCacheResult = null;
         }
+
         return historyCacheResult;
     }
 
@@ -188,13 +191,24 @@ public static HistoryGuru getInstance() {
         return INSTANCE;
     }
 
-    /**
-     * Return whether cache should be used for the history log.
-     *
-     * @return {@code true} if the history cache has been enabled and initialized, {@code false} otherwise
-     */
-    private boolean useHistoryCache() {
-        return historyCache != null;
+    private boolean useHistoryCache(File file) {
+        if (historyCache == null) {
+            return false;
+        }
+
+        return useHistoryCache(getRepository(file));
+    }
+
+    private boolean useHistoryCache(@Nullable Repository repository) {
+        if (historyCache == null || repository == null) {
+            return false;
+        }
+
+        if (!historyCache.supportsRepository(repository)) {
+            return false;
+        }
+
+        return repository.isHistoryCacheEnabled();
     }
 
     /**
@@ -418,7 +432,7 @@ boolean isRepoHistoryEligible(Repository repo, File file, boolean ui) {
     private History getHistoryFromCache(File file, Repository repository, boolean withFiles)
             throws CacheException {
 
-        if (useHistoryCache() && historyCache.supportsRepository(repository)) {
+        if (useHistoryCache(repository)) {
             return historyCache.get(file, repository, withFiles);
         }
 
@@ -428,7 +442,7 @@ private History getHistoryFromCache(File file, Repository repository, boolean wi
     @Nullable
     private HistoryEntry getLastHistoryEntryFromCache(File file, Repository repository) throws CacheException {
 
-        if (useHistoryCache() && historyCache.supportsRepository(repository)) {
+        if (useHistoryCache(repository)) {
             return historyCache.getLastHistoryEntry(file);
         }
 
@@ -451,6 +465,9 @@ public HistoryEntry getLastHistoryEntry(File file, boolean ui, boolean fallback)
                 launderLog(file.toString())));
         final File dir = file.isDirectory() ? file : file.getParentFile();
         final Repository repository = getRepository(dir);
+        if (repository == null) {
+            return null;
+        }
         final String meterName = "history.entry.latest";
 
         try {
@@ -479,7 +496,7 @@ public HistoryEntry getLastHistoryEntry(File file, boolean ui, boolean fallback)
         if (!isRepoHistoryEligible(repository, file, ui)) {
             statistics.report(LOGGER, Level.FINEST,
                     String.format("cannot retrieve the last history entry for ''%s'' in %s because of settings",
-                    launderLog(file.toString()), repository), meterName);
+                            launderLog(file.toString()), repository), meterName);
             return null;
         }
 
@@ -691,7 +708,7 @@ private static boolean repositoryHasHistory(File file, Repository repo) {
      * @return if there is history cache entry for the file
      */
     public boolean hasHistoryCacheForFile(File file) {
-        if (!useHistoryCache()) {
+        if (!useHistoryCache(file)) {
             LOGGER.finest(() -> String.format("history cache is off for '%s' to check history cache presence",
                     launderLog(file.toString())));
             return false;
@@ -823,15 +840,15 @@ public boolean fillLastHistoryEntries(File directory, List<DirectoryEntry> entri
             return true;
         }
 
-        if (!useHistoryCache()) {
-            LOGGER.finest(() -> String.format("history cache is disabled for '%s' to retrieve last modified times",
+        Repository repository = getRepository(directory);
+        if (repository == null) {
+            LOGGER.finest(() -> String.format("cannot find repository for '%s' to retrieve last modified times",
                     launderLog(directory.toString())));
             return true;
         }
 
-        Repository repository = getRepository(directory);
-        if (repository == null) {
-            LOGGER.finest(() -> String.format("cannot find repository for '%s' to retrieve last modified times",
+        if (!useHistoryCache(repository)) {
+            LOGGER.finest(() -> String.format("history cache is disabled for '%s' to retrieve last modified times",
                     launderLog(directory.toString())));
             return true;
         }
@@ -1036,9 +1053,17 @@ public void storeHistory(File file, History history) {
     }
 
     private void createHistoryCache(Repository repository, String sinceRevision) throws CacheException, HistoryException {
+        if (!repository.isHistoryCacheEnabled()) {
+            LOGGER.log(Level.INFO,
+                    "Skipping history cache creation for {0} and its subdirectories: history cache disabled",
+                    repository);
+            return;
+        }
+
         if (!repository.isHistoryEnabled()) {
             LOGGER.log(Level.INFO,
-                    "Skipping history cache creation for {0} and its subdirectories", repository);
+                    "Skipping history cache creation for {0} and its subdirectories: history disabled",
+                    repository);
             return;
         }
 
@@ -1139,9 +1164,14 @@ private Map<Repository, Optional<Exception>> createHistoryCacheReal(Collection<R
      * @return map of repository to optional exception
      */
     public Map<Repository, Optional<Exception>> createHistoryCache(Collection<String> repositories) {
-        if (!useHistoryCache()) {
+        if (repositories.stream().
+                map(e -> new File(env.getSourceRootPath(), e)).
+                map(this::getRepository).
+                filter(Objects::nonNull).
+                noneMatch(RepositoryInfo::isHistoryCacheEnabled)) {
             return Collections.emptyMap();
         }
+
         return createHistoryCacheReal(getReposFromString(repositories));
     }
 
@@ -1151,12 +1181,12 @@ public Map<Repository, Optional<Exception>> createHistoryCache(Collection<String
      * @param removeHistory whether to remove history cache entry for the path
      */
     public void clearHistoryCacheFile(String path, boolean removeHistory) {
-        if (!useHistoryCache()) {
+        Repository repository = getRepository(new File(env.getSourceRootFile(), path));
+        if (repository == null) {
             return;
         }
 
-        Repository repository = getRepository(new File(env.getSourceRootFile(), path));
-        if (repository == null) {
+        if (!useHistoryCache(repository)) {
             return;
         }
 
@@ -1230,7 +1260,7 @@ public void clearAnnotationCacheFile(String path) {
      * @return list of repository names
      */
     public List<String> removeHistoryCache(Collection<RepositoryInfo> repositories) {
-        if (!useHistoryCache()) {
+        if (repositories.stream().noneMatch(RepositoryInfo::isHistoryCacheEnabled)) {
             return List.of();
         }
 
@@ -1258,7 +1288,7 @@ public List<String> removeAnnotationCache(Collection<RepositoryInfo> repositorie
      * @return map of repository to optional exception
      */
     public Map<Repository, Optional<Exception>> createHistoryCache() {
-        if (!useHistoryCache()) {
+        if (repositories.values().stream().noneMatch(RepositoryInfo::isHistoryCacheEnabled)) {
             return Collections.emptyMap();
         }
 

opengrok-indexer/src/main/java/org/opengrok/indexer/history/RepositoryInfo.java

@@ -78,6 +78,8 @@ public class RepositoryInfo implements Serializable {
     @DTOElement
     private boolean historyEnabled;
     @DTOElement
+    private boolean historyCacheEnabled;
+    @DTOElement
     private boolean annotationCacheEnabled;
     @DTOElement
     private boolean mergeCommitsEnabled;
@@ -105,6 +107,7 @@ public RepositoryInfo(RepositoryInfo orig) {
         this.branch = orig.branch;
         this.currentVersion = orig.currentVersion;
         this.historyEnabled = orig.historyEnabled;
+        this.historyCacheEnabled = orig.historyCacheEnabled;
         this.annotationCacheEnabled = orig.annotationCacheEnabled;
         this.handleRenamedFiles = orig.handleRenamedFiles;
         this.mergeCommitsEnabled = orig.mergeCommitsEnabled;
@@ -164,7 +167,7 @@ public void setHistoryBasedReindex(boolean flag) {
     }
 
     /**
-     * @return true if the repository should have history cache.
+     * @return true if the repository should support history queries.
      */
     public boolean isHistoryEnabled() {
         return this.historyEnabled;
@@ -174,6 +177,17 @@ public void setHistoryEnabled(boolean flag) {
         this.historyEnabled = flag;
     }
 
+    /**
+     * @return true if the history for this repository should be stored in history cache.
+     */
+    public boolean isHistoryCacheEnabled() {
+        return this.historyCacheEnabled;
+    }
+
+    public void setHistoryCacheEnabled(boolean flag) {
+        this.historyCacheEnabled = flag;
+    }
+
     public boolean isAnnotationCacheEnabled() {
         return annotationCacheEnabled;
     }
@@ -381,6 +395,7 @@ public void fillFromProject() {
         Project proj = Project.getProject(getDirectoryNameRelative());
         if (proj != null) {
             setHistoryEnabled(proj.isHistoryEnabled());
+            setHistoryCacheEnabled(proj.isHistoryCacheEnabled());
             setAnnotationCacheEnabled(proj.isAnnotationCacheEnabled());
             setHandleRenamedFiles(proj.isHandleRenamedFiles());
             setMergeCommitsEnabled(proj.isMergeCommitsEnabled());
@@ -391,6 +406,7 @@ public void fillFromProject() {
             RuntimeEnvironment env = RuntimeEnvironment.getInstance();
 
             setHistoryEnabled(env.isHistoryEnabled());
+            setHistoryCacheEnabled(env.useHistoryCache());
             setAnnotationCacheEnabled(env.isAnnotationCacheEnabled());
             setHandleRenamedFiles(env.isHandleHistoryOfRenamedFiles());
             setMergeCommitsEnabled(env.isMergeCommitsEnabled());
@@ -433,9 +449,20 @@ public String toString() {
         stringBuilder.append(",");
 
         if (!isHistoryEnabled()) {
+            stringBuilder.append("history=off");
+        } else {
+            stringBuilder.append("history=on");
+        }
+
+        stringBuilder.append(",");
+        if (!isHistoryCacheEnabled()) {
             stringBuilder.append("historyCache=off");
         } else {
             stringBuilder.append("historyCache=on,");
+        }
+
+        if (isHandleRenamedFiles()) {
+            stringBuilder.append(",");
             stringBuilder.append("renamed=");
             stringBuilder.append(this.isHandleRenamedFiles());
         }

opengrok-indexer/src/main/java/org/opengrok/indexer/index/Indexer.java

@@ -1123,13 +1123,12 @@ public Map<Repository, Optional<Exception>> prepareIndexer(RuntimeEnvironment en
      * @param repositories       list of repository paths relative to source root
      * @return map of repository to exception
      * @throws IndexerException indexer exception
-     * @throws IOException      I/O exception
      */
     public Map<Repository, Optional<Exception>> prepareIndexer(RuntimeEnvironment env,
                                                                Set<String> searchPaths,
                                                                boolean addProjects,
                                                                boolean createHistoryCache,
-                                                               List<String> repositories) throws IndexerException, IOException {
+                                                               List<String> repositories) throws IndexerException {
 
         if (!env.validateUniversalCtags()) {
             throw new IndexerException("Could not find working Universal ctags. " +