Merge pull request #3951 from vladak/truly_incremental_reindex

history based reindex

Author: vladak

dev/checkstyle/suppressions.xml

@@ -18,7 +18,7 @@ information: Portions Copyright [yyyy] [name of copyright owner]
 
 CDDL HEADER END
 
-Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved.
+Copyright (c) 2018, 2002, Oracle and/or its affiliates. All rights reserved.
 Portions Copyright (c) 2018-2020, Chris Fraire <[email protected]>.
 
 -->
@@ -43,7 +43,7 @@ Portions Copyright (c) 2018-2020, Chris Fraire <[email protected]>.
         |Context\.java|HistoryContext\.java|Suggester\.java|
 	|ProjectHelperTestBase\.java|SearchHelper\.java" />
 
-    <suppress checks="FileLength" files="RuntimeEnvironment\.java" />
+    <suppress checks="FileLength" files="RuntimeEnvironment\.java|IndexDatabase\.java" />
 
     <suppress checks="MethodLength" files="Indexer\.java|IndexDatabase\.java|AuthorizationFrameworkTest\.java" />
 

opengrok-indexer/src/main/java/org/opengrok/indexer/configuration/Configuration.java

@@ -300,6 +300,8 @@ public final class Configuration {
     private int connectTimeout = -1;    // connect timeout in seconds
     private int apiTimeout = -1;    // API timeout in seconds
 
+    private boolean historyBasedReindex;
+
     /*
      * types of handling history for remote SCM repositories:
      *  ON - index history and display it in webapp
@@ -576,6 +578,7 @@ public Configuration() {
         setTagsEnabled(false);
         //setUserPage("http://www.myserver.org/viewProfile.jspa?username=");
         // Set to empty string so we can append it to the URL unconditionally later.
+        setHistoryBasedReindex(true);
         setUserPageSuffix("");
         setWebappLAF("default");
         // webappCtags is default(boolean)
@@ -1412,6 +1415,14 @@ public void setApiTimeout(int apiTimeout) {
         this.apiTimeout = apiTimeout;
     }
 
+    public boolean isHistoryBasedReindex() {
+        return historyBasedReindex;
+    }
+
+    public void setHistoryBasedReindex(boolean flag) {
+        historyBasedReindex = flag;
+    }
+
     /**
      * Write the current configuration to a file.
      *
@@ -1524,4 +1535,45 @@ private static Configuration decodeObject(InputStream in) throws IOException {
 
         return conf;
     }
+
+    public static class ConfigurationException extends Exception {
+        static final long serialVersionUID = -1;
+
+        public ConfigurationException(String message) {
+            super(message);
+        }
+    }
+
+    /**
+     * Check if configuration is populated and self-consistent.
+     * @throws ConfigurationException on error
+     */
+    public void checkConfiguration() throws ConfigurationException {
+
+        if (getSourceRoot() == null) {
+            throw new ConfigurationException("Source root is not specified.");
+        }
+
+        if (getDataRoot() == null) {
+            throw new ConfigurationException("Data root is not specified.");
+        }
+
+        if (!new File(getSourceRoot()).canRead()) {
+            throw new ConfigurationException("Source root directory '" + getSourceRoot() + "' must be readable.");
+        }
+
+        if (!new File(getDataRoot()).canWrite()) {
+            throw new ConfigurationException("Data root directory '" + getDataRoot() + "' must be writable.");
+        }
+
+        if (!isHistoryEnabled() && isHistoryBasedReindex()) {
+            LOGGER.log(Level.INFO, "History based reindex is on, however history is off. " +
+                    "History has to be enabled for history based reindex.");
+        }
+
+        if (!isHistoryCache() && isHistoryBasedReindex()) {
+            LOGGER.log(Level.INFO, "History based reindex is on, however history cache is off. " +
+                    "History cache has to be enabled for history based reindex.");
+        }
+    }
 }

opengrok-indexer/src/main/java/org/opengrok/indexer/configuration/Project.java

@@ -18,7 +18,7 @@
  */
 
 /*
- * Copyright (c) 2006, 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2006, 2022, Oracle and/or its affiliates. All rights reserved.
  * Portions Copyright (c) 2018, Chris Fraire <[email protected]>.
  */
 package org.opengrok.indexer.configuration;
@@ -34,6 +34,7 @@
 import java.util.logging.Logger;
 import java.util.regex.PatternSyntaxException;
 
+import org.jetbrains.annotations.VisibleForTesting;
 import org.opengrok.indexer.logger.LoggerFactory;
 import org.opengrok.indexer.util.ClassUtil;
 import org.opengrok.indexer.util.ForbiddenSymlinkException;
@@ -99,6 +100,11 @@ public class Project implements Comparable<Project>, Nameable, Serializable {
      */
     private boolean indexed = false;
 
+    /**
+     * This flag sets per-project reindex based on traversing SCM history.
+     */
+    private Boolean historyBasedReindex = null;
+
     /**
      * Set of groups which match this project.
      */
@@ -289,6 +295,28 @@ public void setMergeCommitsEnabled(boolean flag) {
         this.mergeCommitsEnabled = flag;
     }
 
+    /**
+     * @return true if this project handles renamed files.
+     */
+    public boolean isHistoryBasedReindex() {
+        return historyBasedReindex != null && historyBasedReindex;
+    }
+
+    /**
+     * @param flag true if project should handle renamed files, false otherwise.
+     */
+    public void setHistoryBasedReindex(boolean flag) {
+        this.historyBasedReindex = flag;
+    }
+
+    @VisibleForTesting
+    public void clearProperties() {
+        historyBasedReindex = null;
+        mergeCommitsEnabled = null;
+        historyEnabled = null;
+        handleRenamedFiles = null;
+    }
+
     /**
      * Return groups where this project belongs.
      *
@@ -436,6 +464,10 @@ public final void completeWithDefaults() {
         if (reviewPattern == null) {
             setReviewPattern(env.getReviewPattern());
         }
+
+        if (historyBasedReindex == null) {
+            setHistoryBasedReindex(env.isHistoryBasedReindex());
+        }
     }
 
     /**
@@ -476,8 +508,7 @@ public static Project getProject(String path) {
      * Get the project for a specific file.
      *
      * @param file the file to lookup
-     * @return the project that this file belongs to (or null if the file
-     * doesn't belong to a project)
+     * @return the project that this file belongs to (or {@code null} if the file doesn't belong to a project)
      */
     public static Project getProject(File file) {
         Project ret = null;

opengrok-indexer/src/main/java/org/opengrok/indexer/configuration/RuntimeEnvironment.java

@@ -36,6 +36,7 @@
 import java.util.Collection;
 import java.util.Collections;
 import java.util.Date;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
@@ -62,8 +63,10 @@
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.FSDirectory;
 import org.apache.lucene.util.NamedThreadFactory;
+import org.jetbrains.annotations.VisibleForTesting;
 import org.opengrok.indexer.authorization.AuthorizationFramework;
 import org.opengrok.indexer.authorization.AuthorizationStack;
+import org.opengrok.indexer.history.FileCollector;
 import org.opengrok.indexer.history.HistoryGuru;
 import org.opengrok.indexer.history.RepositoryInfo;
 import org.opengrok.indexer.index.IndexDatabase;
@@ -137,6 +140,12 @@ public List<String> getSubFiles() {
 
     private final List<String> subFiles = new ArrayList<>();
 
+    /**
+     * Maps project name to FileCollector object. This is used to pass the list of files acquired when
+     * generating history cache in the first phase of indexing to the second phase of indexing.
+     */
+    private final Map<String, FileCollector> fileCollectorMap = new HashMap<>();
+
     /**
      * Creates a new instance of RuntimeEnvironment. Private to ensure a
      * singleton anti-pattern.
@@ -465,7 +474,7 @@ public List<Project> getProjectList() {
     /**
      * Get project map.
      *
-     * @return a Map with all of the projects
+     * @return a Map with all the projects
      */
     public Map<String, Project> getProjects() {
         return syncReadConfiguration(Configuration::getProjects);
@@ -1417,6 +1426,27 @@ public void setConnectTimeout(int connectTimeout) {
         syncWriteConfiguration(connectTimeout, Configuration::setConnectTimeout);
     }
 
+    public boolean isHistoryBasedReindex() {
+        return syncReadConfiguration(Configuration::isHistoryBasedReindex);
+    }
+
+    public void setHistoryBasedReindex(boolean flag) {
+        syncWriteConfiguration(flag, Configuration::setHistoryBasedReindex);
+    }
+
+    public FileCollector getFileCollector(String name) {
+        return fileCollectorMap.get(name);
+    }
+
+    public void setFileCollector(String name, FileCollector fileCollector) {
+        fileCollectorMap.put(name, fileCollector);
+    }
+
+    @VisibleForTesting
+    public void clearFileCollector() {
+        fileCollectorMap.clear();
+    }
+
     /**
      * Read an configuration file and set it as the current configuration.
      *
@@ -1491,7 +1521,8 @@ public void writeConfiguration(String host) throws IOException, InterruptedExcep
      * Project with some repository information is considered as a repository
      * otherwise it is just a simple project.
      */
-    private void generateProjectRepositoriesMap() throws IOException {
+    @VisibleForTesting
+    public void generateProjectRepositoriesMap() throws IOException {
         repository_map.clear();
         for (RepositoryInfo r : getRepositories()) {
             Project proj;

opengrok-indexer/src/main/java/org/opengrok/indexer/history/ChangesetVisitor.java

@@ -0,0 +1,33 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * See LICENSE.txt included in this distribution for the specific
+ * language governing permissions and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at LICENSE.txt.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+
+/*
+ * Copyright (c) 2022, Oracle and/or its affiliates. All rights reserved.
+ */
+package org.opengrok.indexer.history;
+
+import java.util.function.Consumer;
+
+public abstract class ChangesetVisitor implements Consumer<RepositoryWithHistoryTraversal.ChangesetInfo> {
+    boolean consumeMergeChangesets;
+
+    protected ChangesetVisitor(boolean consumeMergeChangesets) {
+        this.consumeMergeChangesets = consumeMergeChangesets;
+    }
+}

opengrok-indexer/src/main/java/org/opengrok/indexer/history/FileCollector.java

@@ -0,0 +1,66 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * See LICENSE.txt included in this distribution for the specific
+ * language governing permissions and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at LICENSE.txt.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+
+/*
+ * Copyright (c) 2022, Oracle and/or its affiliates. All rights reserved.
+ */
+package org.opengrok.indexer.history;
+
+import java.util.Collection;
+import java.util.SortedSet;
+import java.util.TreeSet;
+
+/**
+ * This class is meant to collect files that were touched in some way by SCM update.
+ * The visitor argument contains the files separated based on the type of modification performed,
+ * however the consumer of this class is not interested in this classification.
+ * This is because when incrementally indexing a bunch of changesets,
+ * in one changeset a file may be deleted, only to be re-added in the next changeset etc.
+ */
+public class FileCollector extends ChangesetVisitor {
+    private final SortedSet<String> files;
+
+    /**
+     * Assumes comparing in the same way as {@code org.opengrok.indexer.index.IndexDatabase#FILENAME_COMPARATOR}.
+     */
+    public FileCollector(boolean consumeMergeChangesets) {
+        super(consumeMergeChangesets);
+        files = new TreeSet<>();
+    }
+
+    public void accept(RepositoryWithHistoryTraversal.ChangesetInfo changesetInfo) {
+        if (changesetInfo.renamedFiles != null) {
+            files.addAll(changesetInfo.renamedFiles);
+        }
+        if (changesetInfo.files != null) {
+            files.addAll(changesetInfo.files);
+        }
+        if (changesetInfo.deletedFiles != null) {
+            files.addAll(changesetInfo.deletedFiles);
+        }
+    }
+
+    public SortedSet<String> getFiles() {
+        return files;
+    }
+
+    void addFiles(Collection<String> files) {
+        this.files.addAll(files);
+    }
+}