ignore history should be per project tunable

fixes #1728

Author: Vladimir Kotal

OpenGrok

@@ -308,9 +308,6 @@ DefaultInstanceConfiguration()
 
     # OPTIONAL: Ignore these patterns as names of files or directories
     #OPENGROK_IGNORE_PATTERNS="-i dummy"
-    # To ignore skipping just the history cache creation for a particular
-    # directory and all of it's subdirectories, touch an empty
-    # .opengrok_skip_history file at the root of that directory
 
     # OPTIONAL: Enable Projects
     #           (Every directory in SRC_ROOT is considered a separate project)

README.md

@@ -119,9 +119,9 @@ by OpenGrok.
 
 Note that OpenGrok ignores symbolic links.
 
-If you want to skip indexing the history of a particular directory
-(and all of it's subdirectories), you can touch `.opengrok_skip_history` file
-at the root of that directory.
+If you want to skip indexing the history of a particular directory,
+use per project settings as per
+https://github.com/oracle/opengrok/wiki/Per-project-configuration
 If you want to disable history generation for all repositories globally, then
 set `OPENGROK_GENERATE_HISTORY` environment variable to `off` during indexing.
 

src/org/opensolaris/opengrok/configuration/Project.java

@@ -73,6 +73,11 @@ public class Project implements Comparable<Project>, Nameable, Serializable {
      */
     private Boolean handleRenamedFiles = null;
 
+    /**
+     * This flag enables/disables per-project history cache.
+     */
+    private Boolean historyEnabled = null;
+    
     /**
      * This marks the project as (not)ready before initial index is done. this
      * is to avoid all/multi-project searches referencing this project from
@@ -184,7 +189,7 @@ public void setName(String name) {
      * that you should ALWAYS prefix the path with current file.separator ,
      * current environment should always have it set up
      *
-     * @param path the relative path from source sroot where this project is
+     * @param path the relative path from source root where this project is
      * located.
      */
     public void setPath(String path) {
@@ -248,6 +253,20 @@ public void setHandleRenamedFiles(boolean flag) {
         this.handleRenamedFiles = flag;
     }
 
+    /**
+     * @return true if this project should have history cache.
+     */
+    public boolean isHistoryEnabled() {
+        return historyEnabled != null && historyEnabled;
+    }
+    
+    /**
+     * @param flag true if project should have history cache, false otherwise.
+     */
+    public void setHistoryEnabled(boolean flag) {
+        this.historyEnabled = flag;
+    }
+    
     /**
      * Return groups where this project belongs
      *
@@ -302,6 +321,11 @@ final public void completeWithDefaults(Configuration cfg) {
         if (handleRenamedFiles == null) {
             setHandleRenamedFiles(cfg.isHandleHistoryOfRenamedFiles());
         }
+        
+        // Allow project to override global setting of history cache generation.
+        if (historyEnabled == null) {
+            setHistoryEnabled(cfg.isHistoryEnabled());
+        }
     }
 
     /**

src/org/opensolaris/opengrok/configuration/RuntimeEnvironment.java

@@ -803,7 +803,7 @@ public void removeRepositories() {
     public void setRepositories(String dir) {
         List<RepositoryInfo> repos = new ArrayList<>(HistoryGuru.getInstance().
                 addRepositories(new File[]{new File(dir)},
-                RuntimeEnvironment.getInstance().getIgnoredNames()));
+                    RuntimeEnvironment.getInstance().getIgnoredNames()));
         RuntimeEnvironment.getInstance().setRepositories(repos);
     }
 
@@ -1236,18 +1236,6 @@ public void setHandleHistoryOfRenamedFiles(boolean enable) {
     public boolean isHandleHistoryOfRenamedFiles() {
         return threadConfig.get().isHandleHistoryOfRenamedFiles();
     }
-
-    public boolean isHandleHistoryOfRenamedFilesForProject(Project project) {
-        if (hasProjects() && project != null) {
-            return project.isHandleRenamedFiles();
-        } else {
-            return isHandleHistoryOfRenamedFiles();
-        }
-    }
-    
-    public boolean isHandleHistoryOfRenamedFilesFor(String path) {
-        return isHandleHistoryOfRenamedFilesForProject(Project.getProject(path));
-    }
 
     public void setRevisionMessageCollapseThreshold(int threshold) {
         threadConfig.get().setRevisionMessageCollapseThreshold(threshold);

src/org/opensolaris/opengrok/history/FileHistoryCache.java

@@ -54,7 +54,6 @@
 import java.util.logging.Logger;
 import java.util.zip.GZIPInputStream;
 import java.util.zip.GZIPOutputStream;
-import org.opensolaris.opengrok.configuration.Project;
 import org.opensolaris.opengrok.configuration.RuntimeEnvironment;
 import org.opensolaris.opengrok.logger.LoggerFactory;
 import org.opensolaris.opengrok.util.ForbiddenSymlinkException;
@@ -69,8 +68,8 @@ class FileHistoryCache implements HistoryCache {
 
     private final Object lock = new Object();
 
-    private final static String historyCacheDirName = "historycache";
-    private final static String latestRevFileName = "OpenGroklatestRev";
+    private final static String HISTORY_CACHE_DIR_NAME = "historycache";
+    private final static String LATEST_REV_FILE_NAME = "OpenGroklatestRev";
     private final static String DIRECTORY_FILE_PREFIX = "OpenGrokDirHist";
 
     private boolean historyIndexDone = false;
@@ -201,7 +200,7 @@ private static File getCachedFile(File file) throws HistoryException,
         StringBuilder sb = new StringBuilder();
         sb.append(env.getDataRootPath());
         sb.append(File.separatorChar);
-        sb.append(historyCacheDirName);
+        sb.append(HISTORY_CACHE_DIR_NAME);
 
         try {
             String add = env.getPathRelativeToSourceRoot(file);
@@ -589,7 +588,8 @@ public History get(File file, Repository repository, boolean withFiles)
          * fetched in the first phase of indexing.
          */
         RuntimeEnvironment env = RuntimeEnvironment.getInstance();
-        if (isHistoryIndexDone() && repository.hasHistoryForDirectories() &&
+        if (isHistoryIndexDone() && repository.isHistoryEnabled() &&
+            repository.hasHistoryForDirectories() &&
             !env.isFetchHistoryWhenNotInCache()) {
                 return null;
         }
@@ -651,7 +651,7 @@ public boolean hasCacheForDirectory(File directory, Repository repository)
         }
         RuntimeEnvironment env = RuntimeEnvironment.getInstance();
         File dir = env.getDataRootFile();
-        dir = new File(dir, FileHistoryCache.historyCacheDirName);
+        dir = new File(dir, FileHistoryCache.HISTORY_CACHE_DIR_NAME);
         try {
             dir = new File(dir, env.getPathRelativeToSourceRoot(
                 new File(repos.getDirectoryName())));
@@ -692,14 +692,14 @@ public String getRepositoryHistDataDirname(Repository repository) {
         }
 
         return env.getDataRootPath() + File.separatorChar
-            + FileHistoryCache.historyCacheDirName
+            + FileHistoryCache.HISTORY_CACHE_DIR_NAME
             + repoDirBasename;
     }
 
     private String getRepositoryCachedRevPath(Repository repository) {
         String histDir = getRepositoryHistDataDirname(repository);
         if (histDir == null) return null;
-        return histDir + File.separatorChar + latestRevFileName;
+        return histDir + File.separatorChar + LATEST_REV_FILE_NAME;
     }
 
     /**

src/org/opensolaris/opengrok/history/HistoryGuru.java

@@ -250,7 +250,7 @@ public History getHistory(File file, boolean withFiles, boolean ui)
                 || (rscm == RemoteSCM.ON)
                 || (ui || ((rscm == RemoteSCM.DIRBASED) && (repo != null) && repo.hasHistoryForDirectories()));
 
-        if (repo != null && repo.isWorking() && repo.fileHasHistory(file)
+        if (repo != null && repo.isHistoryEnabled() && repo.isWorking() && repo.fileHasHistory(file)
                 && (!repo.isRemote() || doRemote)) {
 
             if (useCache() && historyCache.supportsRepository(repo)) {
@@ -387,14 +387,6 @@ private Collection<RepositoryInfo> addRepositories(File[] files,
             String path;
             try {
                 path = file.getCanonicalPath();
-                File skipRepository = new File(path, ".opengrok_skip_history");
-                // Should potential repository be ignored?
-                if (skipRepository.exists()) {
-                    LOGGER.log(Level.INFO,
-                        "Skipping history cache creation for {0} and its subdirectories",
-                        file.getAbsolutePath());
-                    continue;
-                }
 
                 Repository repository = null;
                 try {
@@ -576,6 +568,12 @@ private void createCache(Repository repository, String sinceRevision) {
         String path = repository.getDirectoryName();
         String type = repository.getClass().getSimpleName();
 
+        if (!repository.isHistoryEnabled()) {
+            LOGGER.log(Level.INFO,
+                    "Skipping history cache creation of {0} repository in {1} and its subdirectories",
+                    new Object[]{type, path});
+        }
+        
         if (repository.isWorking()) {
             boolean verbose = RuntimeEnvironment.getInstance().isVerbose();
             Statistics elapsed = new Statistics();

src/org/opensolaris/opengrok/history/RepositoryFactory.java

@@ -161,6 +161,8 @@ public static Repository getRepository(File file, boolean interactive)
                     repo.buildTagList(file, interactive);
                 }
 
+                repo.fillFromProject();
+                
                 break;
             }
         }

src/org/opensolaris/opengrok/history/RepositoryInfo.java

@@ -30,6 +30,7 @@
 import java.util.Objects;
 import java.util.logging.Level;
 import java.util.logging.Logger;
+import org.opensolaris.opengrok.configuration.Project;
 import org.opensolaris.opengrok.configuration.RuntimeEnvironment;
 import org.opensolaris.opengrok.logger.LoggerFactory;
 import org.opensolaris.opengrok.util.ClassUtil;
@@ -67,6 +68,7 @@ public class RepositoryInfo implements Serializable {
     protected String currentVersion;
 
     private boolean handleRenamedFiles;
+    private boolean historyEnabled;
 
     /**
      * format used for printing the date in {@code currentVersion}
@@ -98,6 +100,24 @@ public boolean isHandleRenamedFiles() {
         return this.handleRenamedFiles;
     }
 
+    /**
+     * @param flag true if the repository should handle renamed files, false otherwise.
+     */
+    public void setHandleRenamedFiles(boolean flag) {
+        this.handleRenamedFiles = flag;
+    }
+    
+    /**
+     * @return true if the repository should have history cache.
+     */
+    public boolean isHistoryEnabled() {
+        return this.historyEnabled;
+    }
+    
+    public void setHistoryEnabled(boolean flag) {
+        this.historyEnabled = flag;
+    }
+    
     /**
      * @return relative path to source root
      */
@@ -111,9 +131,6 @@ public String getDirectoryNameRelative() {
      */
     public void setDirectoryNameRelative(String dir) {
         this.directoryNameRelative = dir;
-        
-        handleRenamedFiles = RuntimeEnvironment.getInstance().
-                isHandleHistoryOfRenamedFilesFor(dir);
     }
 
     /**
@@ -256,6 +273,22 @@ public void setCurrentVersion(String currentVersion) {
         this.currentVersion = currentVersion;
     }
 
+    /**
+     * Fill configurable properties from associated project (if any) or Configuration.
+     */
+    public void fillFromProject() {
+        Project proj = Project.getProject(getDirectoryNameRelative());
+        if (proj != null) {
+            setHistoryEnabled(proj.isHistoryEnabled());
+            setHandleRenamedFiles(proj.isHandleRenamedFiles());
+        } else {
+            RuntimeEnvironment env = RuntimeEnvironment.getInstance();
+            
+            setHistoryEnabled(env.isHistoryEnabled());
+            setHandleRenamedFiles(env.isHandleHistoryOfRenamedFiles());
+        }
+    }
+    
     @Override
     public boolean equals(Object obj) {
         if (!(obj instanceof RepositoryInfo)) {

src/org/opensolaris/opengrok/index/IgnoredFiles.java

@@ -55,7 +55,6 @@ public final class IgnoredFiles extends Filter {
         ".vsmdi", // Visual Studio tests
         "*.dll",
         "*.DLL",
-        ".opengrok_skip_history",
     };
 
     public IgnoredFiles() {

src/org/opensolaris/opengrok/index/Indexer.java

@@ -884,6 +884,8 @@ public void prepareIndexer(RuntimeEnvironment env,
             throw new IndexerException("Internal error, zapCache shouldn't be null");
         }
 
+        // Projects need to be created first since when adding repositories below,
+        // some of the project properties might be needed for that.
         if (addProjects) {
             File files[] = env.getSourceRootFile().listFiles();
             Map<String,Project> projects = env.getProjects();
@@ -921,11 +923,10 @@ public void prepareIndexer(RuntimeEnvironment env,
         if (searchRepositories || listRepoPaths || !zapCache.isEmpty()) {
             LOGGER.log(Level.INFO, "Scanning for repositories...");
             long start = System.currentTimeMillis();
-            if (env.isHistoryEnabled()) {
-                env.setRepositories(env.getSourceRootPath());
-            }
+            env.setRepositories(env.getSourceRootPath());
             long time = (System.currentTimeMillis() - start) / 1000;
             LOGGER.log(Level.INFO, "Done scanning for repositories ({0}s)", time);
+            
             if (listRepoPaths || !zapCache.isEmpty()) {
                 List<RepositoryInfo> repos = env.getRepositories();
                 String prefix = env.getSourceRootPath();
@@ -990,18 +991,17 @@ public void prepareIndexer(RuntimeEnvironment env,
             }
         }
 
-        if (env.isHistoryEnabled()) {
-            if (repositories != null && !repositories.isEmpty()) {
-                LOGGER.log(Level.INFO, "Generating history cache for repositories: " +
-                    repositories.stream().collect(Collectors.joining(",")));
-                HistoryGuru.getInstance().createCache(repositories);
-                LOGGER.info("Done...");
-              } else {
-                  LOGGER.log(Level.INFO, "Generating history cache for all repositories ...");
-                  HistoryGuru.getInstance().createCache();
-                  LOGGER.info("Done...");
-              }
-        }
+        // Even if history is disabled globally, it can be enabled for some repositories.
+        if (repositories != null && !repositories.isEmpty()) {
+            LOGGER.log(Level.INFO, "Generating history cache for repositories: " +
+                repositories.stream().collect(Collectors.joining(",")));
+            HistoryGuru.getInstance().createCache(repositories);
+            LOGGER.info("Done...");
+          } else {
+              LOGGER.log(Level.INFO, "Generating history cache for all repositories ...");
+              HistoryGuru.getInstance().createCache();
+              LOGGER.info("Done...");
+          }
 
         if (listFiles) {
             for (String file : IndexDatabase.getAllFiles(subFiles)) {