Annotation cache (#4059)

fixes #4028

Author: vladak

apiary.apib

@@ -384,8 +384,24 @@ This will also mark the project as not indexed.
 ### delete history cache for a project [DELETE]
 
 This will delete history cache for a project.
+Return list of repository paths for which the cache was successfully deleted.
 
-+ Response 204
++ Response 200 (application/json)
+  + Body
+
+            ["/project/repository1","/project/repository2"]
+
+## Project annotation cache management [/projects/{project}/annotationcache]
+
+### delete annotation cache for a project [DELETE]
+
+This will delete annotation cache for a project.
+Return list of repository paths for which the cache was successfully deleted.
+
++ Response 200 (application/json)
+  + Body
+
+            ["/project/repository1","/project/repository2"]
 
 ## Project metadata management [/projects/{project}/indexed]
 

opengrok-indexer/src/main/java/org/opengrok/indexer/analysis/AnalyzerGuru.java

@@ -593,28 +593,12 @@ public void populateDocument(Document doc, File file, String path, AbstractAnaly
                 new BytesRef(file.getAbsolutePath())));
 
         if (RuntimeEnvironment.getInstance().isHistoryEnabled()) {
-            try {
-                HistoryGuru histGuru = HistoryGuru.getInstance();
-                HistoryReader hr = histGuru.getHistoryReader(file);
-                if (hr != null) {
-                    doc.add(new TextField(QueryBuilder.HIST, hr));
-                    History history;
-                    if ((history = histGuru.getHistory(file)) != null) {
-                        List<HistoryEntry> historyEntries = history.getHistoryEntries(1, 0);
-                        if (!historyEntries.isEmpty()) {
-                            HistoryEntry histEntry = historyEntries.get(0);
-                            doc.add(new TextField(QueryBuilder.LASTREV, histEntry.getRevision(), Store.YES));
-                        }
-                    }
-                }
-            } catch (HistoryException e) {
-                LOGGER.log(Level.WARNING, "An error occurred while reading history: ", e);
-            }
+            populateDocumentHistory(doc, file);
         }
         doc.add(new Field(QueryBuilder.DATE, date, string_ft_stored_nanalyzed_norms));
         doc.add(new SortedDocValuesField(QueryBuilder.DATE, new BytesRef(date)));
 
-        // `path' is not null, as it was passed to Util.path2uid() above.
+        // 'path' is not null, as it was passed to Util.path2uid() above.
         doc.add(new TextField(QueryBuilder.PATH, path, Store.YES));
         Project project = Project.getProject(path);
         if (project != null) {
@@ -630,8 +614,7 @@ public void populateDocument(Document doc, File file, String path, AbstractAnaly
         String fileParent = fpath.getParent();
         if (fileParent != null && fileParent.length() > 0) {
             String normalizedPath = QueryBuilder.normalizeDirPath(fileParent);
-            StringField npstring = new StringField(QueryBuilder.DIRPATH,
-                normalizedPath, Store.NO);
+            StringField npstring = new StringField(QueryBuilder.DIRPATH, normalizedPath, Store.NO);
             doc.add(npstring);
         }
 
@@ -647,6 +630,26 @@ public void populateDocument(Document doc, File file, String path, AbstractAnaly
         }
     }
 
+    private static void populateDocumentHistory(Document doc, File file) {
+        try {
+            HistoryGuru histGuru = HistoryGuru.getInstance();
+            HistoryReader hr = histGuru.getHistoryReader(file);
+            if (hr != null) {
+                doc.add(new TextField(QueryBuilder.HIST, hr));
+                History history;
+                if ((history = histGuru.getHistory(file)) != null) {
+                    List<HistoryEntry> historyEntries = history.getHistoryEntries(1, 0);
+                    if (!historyEntries.isEmpty()) {
+                        HistoryEntry histEntry = historyEntries.get(0);
+                        doc.add(new TextField(QueryBuilder.LASTREV, histEntry.getRevision(), Store.YES));
+                    }
+                }
+            }
+        } catch (HistoryException e) {
+            LOGGER.log(Level.WARNING, "An error occurred while reading history: ", e);
+        }
+    }
+
     /**
      * Write a browse-able version of the file.
      *

opengrok-indexer/src/main/java/org/opengrok/indexer/configuration/Configuration.java

@@ -18,7 +18,7 @@
  */
 
 /*
- * Copyright (c) 2007, 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2007, 2022, Oracle and/or its affiliates. All rights reserved.
  * Portions Copyright (c) 2017, 2020, Chris Fraire <[email protected]>.
  * Portions Copyright (c) 2020, Aleksandr Kirillov <[email protected]>.
  */
@@ -107,6 +107,10 @@ public final class Configuration {
      * Should the history log be cached?
      */
     private boolean historyCache;
+    /**
+     * Should the latest annotation be cached?
+     */
+    private boolean annotationCacheEnabled;
     /**
      * flag to generate history. This is bigger hammer than @{code historyCache}
      * above. If set to false, no history query will be ever made and the webapp
@@ -517,6 +521,7 @@ public Configuration() {
         cmds = new HashMap<>();
         setAllowLeadingWildcard(true);
         setAllowedSymlinks(new HashSet<>());
+        setAnnotationCacheEnabled(false);
         setAuthenticationTokens(new HashSet<>());
         setAuthorizationWatchdogEnabled(false);
         //setBugPage("http://bugs.myserver.org/bugdatabase/view_bug.do?bug_id=");
@@ -798,6 +803,25 @@ public boolean isHistoryCache() {
         return historyCache;
     }
 
+    /**
+     * Should the latest annotation be cached?
+     *
+     * @return {@code true} if a {@code AnnotationCache} implementation should be
+     * used, {@code false} otherwise
+     */
+    public boolean isAnnotationCacheEnabled() {
+        return annotationCacheEnabled;
+    }
+
+    /**
+     * Set whether latest annotation should be cached.
+     *
+     * @param useCache if {@code true} enable annotation cache
+     */
+    public void setAnnotationCacheEnabled(boolean useCache) {
+        this.annotationCacheEnabled = useCache;
+    }
+
     /**
      * Set whether history should be cached.
      *

opengrok-indexer/src/main/java/org/opengrok/indexer/configuration/Project.java

@@ -88,6 +88,11 @@ public class Project implements Comparable<Project>, Nameable, Serializable {
      */
     private Boolean historyEnabled = null;
 
+    /**
+     * This flag enables/disables per-project annotation cache.
+     */
+    private Boolean annotationCacheEnabled = null;
+
     /**
      * This flag enables/disables per project merge commits.
      */
@@ -141,8 +146,7 @@ public Project(String name) {
     }
 
     /**
-     * Create a project with given name and path and default configuration
-     * values.
+     * Create a project with given name and path and default configuration values.
      *
      * @param name the name of the project
      * @param path the path of the project relative to the source root
@@ -297,6 +301,20 @@ public void setHistoryEnabled(boolean flag) {
         this.historyEnabled = flag;
     }
 
+    /**
+     * @return true if this project should have annotation cache.
+     */
+    public boolean isAnnotationCacheEnabled() {
+        return annotationCacheEnabled != null && annotationCacheEnabled;
+    }
+
+    /**
+     * @param flag true if project should have annotation cache, false otherwise.
+     */
+    public void setAnnotationCacheEnabled(boolean flag) {
+        this.annotationCacheEnabled = flag;
+    }
+
     /**
      * @param flag true if project's repositories should deal with merge commits.
      */
@@ -353,6 +371,7 @@ public void clearProperties() {
         historyBasedReindex = null;
         mergeCommitsEnabled = null;
         historyEnabled = null;
+        annotationCacheEnabled = null;
         handleRenamedFiles = null;
     }
 
@@ -480,6 +499,11 @@ public final void completeWithDefaults() {
             setHistoryEnabled(env.isHistoryEnabled());
         }
 
+        // Allow project to override global setting of annotation cache generation.
+        if (annotationCacheEnabled == null) {
+            setAnnotationCacheEnabled(env.isAnnotationCacheEnabled());
+        }
+
         // Allow project to override global setting of navigate window.
         if (navigateWindowEnabled == null) {
             setNavigateWindowEnabled(env.isNavigateWindowEnabled());

opengrok-indexer/src/main/java/org/opengrok/indexer/configuration/RuntimeEnvironment.java

@@ -715,6 +715,24 @@ public void setUseHistoryCache(boolean useHistoryCache) {
         syncWriteConfiguration(useHistoryCache, Configuration::setHistoryCache);
     }
 
+    /**
+     * Is annotation cache currently enabled?
+     *
+     * @return whether annotation cache is enabled
+     */
+    public boolean isAnnotationCacheEnabled() {
+        return syncReadConfiguration(Configuration::isAnnotationCacheEnabled);
+    }
+
+    /**
+     * Specify whether annotation cache should be generated during indexing.
+     *
+     * @param useAnnotationCache set false if you do not want to use annotation cache
+     */
+    public void setAnnotationCacheEnabled(boolean useAnnotationCache) {
+        syncWriteConfiguration(useAnnotationCache, Configuration::setAnnotationCacheEnabled);
+    }
+
     /**
      * Should we generate HTML or not during the indexing phase.
      *
@@ -736,8 +754,7 @@ public void setGenerateHtml(boolean generateHtml) {
     /**
      * Set if we should compress the xref files or not.
      *
-     * @param compressXref set to true if the generated html files should be
-     * compressed
+     * @param compressXref set to true if the generated html files should be compressed
      */
     public void setCompressXref(boolean compressXref) {
         syncWriteConfiguration(compressXref, Configuration::setCompressXref);
@@ -1456,12 +1473,12 @@ public void setHistoryBasedReindex(boolean flag) {
         syncWriteConfiguration(flag, Configuration::setHistoryBasedReindex);
     }
 
-    public FileCollector getFileCollector(String name) {
-        return fileCollectorMap.get(name);
+    public FileCollector getFileCollector(String projectName) {
+        return fileCollectorMap.get(projectName);
     }
 
-    public void setFileCollector(String name, FileCollector fileCollector) {
-        fileCollectorMap.put(name, fileCollector);
+    public void setFileCollector(String projectName, FileCollector fileCollector) {
+        fileCollectorMap.put(projectName, fileCollector);
     }
 
     @VisibleForTesting

opengrok-indexer/src/main/java/org/opengrok/indexer/history/AbstractCache.java

@@ -0,0 +1,128 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * See LICENSE.txt included in this distribution for the specific
+ * language governing permissions and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at LICENSE.txt.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+
+/*
+ * Copyright (c) 2005, 2022, Oracle and/or its affiliates. All rights reserved.
+ */
+package org.opengrok.indexer.history;
+
+import org.opengrok.indexer.configuration.RuntimeEnvironment;
+import org.opengrok.indexer.util.ForbiddenSymlinkException;
+import org.opengrok.indexer.util.TandemPath;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.logging.Level;
+
+/**
+ * Class to hold code shared between various cache implementations,
+ * notably {@link FileHistoryCache} and {@link FileAnnotationCache}.
+ */
+public abstract class AbstractCache implements Cache {
+
+    public boolean hasCacheForFile(File file) throws HistoryException {
+        try {
+            return getCachedFile(file).exists();
+        } catch (ForbiddenSymlinkException ex) {
+            LOGGER.log(Level.FINER, ex.getMessage());
+            return false;
+        }
+    }
+
+    /**
+     * Get a <code>File</code> object describing the cache file.
+     *
+     * @param file the file to find the cache for
+     * @return file that might contain cached object for <code>file</code>
+     */
+    File getCachedFile(File file) throws HistoryException, ForbiddenSymlinkException {
+
+        RuntimeEnvironment env = RuntimeEnvironment.getInstance();
+
+        StringBuilder sb = new StringBuilder();
+        sb.append(env.getDataRootPath());
+        sb.append(File.separatorChar);
+        sb.append(getCacheDirName());
+
+        try {
+            String add = env.getPathRelativeToSourceRoot(file);
+            if (add.length() == 0) {
+                add = File.separator;
+            }
+            sb.append(add);
+        } catch (IOException e) {
+            throw new HistoryException("Failed to get path relative to source root for " + file, e);
+        }
+
+        return new File(TandemPath.join(sb.toString(), ".gz"));
+    }
+
+    public List<String> clearCache(Collection<RepositoryInfo> repositories) {
+        List<String> clearedRepos = new ArrayList<>();
+
+        for (RepositoryInfo repo : repositories) {
+            try {
+                this.clear(repo);
+                clearedRepos.add(repo.getDirectoryNameRelative());
+                LOGGER.log(Level.INFO, "{1} cache for ''{0}'' cleared.",
+                        new Object[]{repo.getDirectoryName(), this.getInfo()});
+            } catch (HistoryException e) {
+                LOGGER.log(Level.WARNING,
+                        "Clearing cache for repository {0} failed: {1}",
+                        new Object[]{repo.getDirectoryName(), e.getLocalizedMessage()});
+            }
+        }
+
+        return clearedRepos;
+    }
+
+    /**
+     * Attempt to delete file with its parent.
+     * @param file file to delete
+     */
+    static void clearWithParent(File file) {
+        File parent = file.getParentFile();
+
+        if (!file.delete() && file.exists()) {
+            LOGGER.log(Level.WARNING, "Failed to remove obsolete cache-file: {0}", file.getAbsolutePath());
+        }
+
+        if (parent.delete()) {
+            LOGGER.log(Level.FINE, "Removed empty cache dir:{0}", parent.getAbsolutePath());
+        }
+    }
+
+    public void clearFile(String path) {
+        File historyFile;
+        try {
+            historyFile = getCachedFile(new File(RuntimeEnvironment.getInstance().getSourceRootPath() + path));
+        } catch (ForbiddenSymlinkException ex) {
+            LOGGER.log(Level.FINER, ex.getMessage());
+            return;
+        } catch (HistoryException ex) {
+            LOGGER.log(Level.WARNING, String.format("cannot get history file for file %s", path), ex);
+            return;
+        }
+
+        clearWithParent(historyFile);
+    }
+}