Project centric approach

fixes #1390

Author: Vladimir Kotal

OpenGrok

@@ -105,7 +105,7 @@ Usage()
 
     exec >&2
     echo ""
-    echo "Usage: ${progname} <deploy|derby|update|updateQuietly|usage|help>"
+    echo "Usage: ${progname} <bootstrap|deploy|derby|update|updateQuietly|usage|help>"
     echo "       ${progname} index [<directory>]"
     echo "       ${progname} indexpart <src_root> <directory1> [..]"
     echo "       ${progname} clearHistory <src_root> <repository_name>"
@@ -331,7 +331,7 @@ DefaultInstanceConfiguration()
 
     # OPTIONAL: JVM Options
     JAVA_OPTS="${JAVA_OPTS:--Xmx2048m}"
-    
+
     # OPTIONAL: Full Path to History Utilities
     BK="`Which bk`"
     HG="`Which hg`"
@@ -822,6 +822,8 @@ CommonInvocation()
         ${CTAGS_OPTIONS_FILE:+-o} ${CTAGS_OPTIONS_FILE}         	\
         ${OPENGROK_FLUSH_RAM_BUFFER_SIZE} ${SKIN} ${LEADING_WILDCARD}	\
         ${READ_XML_CONF}                                        	\
+        ${WEBAPP_CONFIG}						\
+        ${WEBAPP_CONTEXT}						\
         "${@}"
 }
 
@@ -831,8 +833,6 @@ StdInvocation()
         -W ${XML_CONFIGURATION}	\
 	${SCAN_FOR_REPOSITORY}	\
 	${ENABLE_PROJECTS}	\
-        ${WEBAPP_CONFIG}	\
-        ${WEBAPP_CONTEXT}	\
         -s "${SRC_ROOT}"	\
 	-d "${DATA_ROOT}"	\
 	"${@}"
@@ -1031,6 +1031,12 @@ case "${1}" in
         UpdateDescriptionCache
         ;;
 
+    bootstrap)
+        ValidateConfiguration
+        CreateRuntimeRequirements
+	StdInvocation "-y"
+	;;
+
     index)
 	if [ $# -gt 2 ]
 	then

README.txt

@@ -143,6 +143,13 @@ Currently supported message types:
                  reloads all authorization plugins.
 5) RefreshMesssage (refresh)
     Sent at the end of partial reindex to trigger refresh of SearcherManagers.
+6) ProjectMessage
+    Used for adding/deleting projects and partial (per-project) reindex.
+    	- "add" adds project(s) and its repositories to the configuration.
+	  If the project already exists, refresh list of its repositories.
+	- "delete" removes project(s) and its repositores from the configuration
+	  Also deletes its data under data root (but not the source code).
+	- "indexed" mark the project(s) as indexed so it becomes visible in the UI
 
 4. OpenGrok install
 -----------------
@@ -314,27 +321,49 @@ projects to finish syncing before running the indexer. To overcome this
 limitation, the index of each project can be created just after the
 mirroring of this project finishes.
 
-Thus, the overall approach would be:
+Indexing of all projects (i.e. 'OpenGrok index /var/opengrok/src') in
+order to discover new and remove deleted projects from the configuration can be
+avoided. One can start with no index and no projects and then incrementally add
+them and index them.
 
-  1. create initial index of all the source code
+It basically works like this:
 
-     This will produce configuration.xml, optionally by combining the
-     discovered projects with read-only configuration (as specified
-     with OPENGROK_READ_XML_CONFIGURATION). This step has to be performed
-     only once - during the initial OpenGrok setup.
+  0. bootstrap initial configuration:
 
-  2. mirror and index all projects in parallel
+     OpenGrok bootstrap
 
-     This is done by running indexpart command of the OpenGrok script and
-     specifying the configuration.xml written in previous step as
-     OPENGROK_READ_XML_CONFIGURATION. The configuration will help the indexer
-     to discover source/data root and project to source path mapping.
+        - this will create /var/opengrok/etc/configuration.xml with basic set of
+ 	 properties. If more is needed use:
 
-  3. perform complete reindex (like in step 1)
+ 	   Messages -n config -t set "set propertyXY = valueFOO"
 
-     Once all the pre-existing projects are mirrored and indexed, run full
-     indexer to discover projects which have been added or deleted.
-     This will produce new configuration.xml.
+  1. add a new project "foo":
+
+     Messages -t foo -n project add
+
+     - the project foo is now visible in the configuration however is not yet
+       searchable. Store the config in a file so that indexer can see the project
+       and its repositories:
+
+         Messages -n config -t getconf > /var/opengrok/etc/configuration.xml
+
+  2. index the project. It will become searchable after that.
+
+     OPENGROK_READ_XML_CONFIGURATION=/var/opengrok/etc/configuration.xml \
+         OpenGrok indexpart /var/opengrok/src /foo
+
+  3. make the project "indexed" status of the project persistent so that if
+     webapp is redeployed the project will be still visible:
+
+     Messages -n config -t getconf > /var/opengrok/etc/configuration.xml
+
+
+If an add message is sent that matches existing project, the list of
+repositories for that project will be refreshed. This is handy when repositories
+are added/deleted.
+
+5.2.3.1 Logging when running partial reindex
+--------------------------------------------
 
 When running the indexer the logs are being written to single file. Since
 multiple indexers are being run in parallel in step 2, their logs have to
@@ -348,7 +377,7 @@ Note the path component 'myproj' which separates the logs for given
 project to this directory. The creation of the per-project directory and the
 logging.properties file can be easily done in a script.
 
-The command used in step 2 can look like this:
+The command used in step 2 can thus look like this:
 
   OPENGROK_LOGGER_CONFIG_PATH=/var/opengrok/myproj.logging \
      OPENGROK_READ_XML_CONFIGURATION=/var/opengrok/etc/configuration.xml \

src/org/opensolaris/opengrok/configuration/Configuration.java

@@ -47,6 +47,7 @@
 import java.util.Map;
 import java.util.Set;
 import java.util.TreeSet;
+import java.util.concurrent.ConcurrentHashMap;
 import java.util.function.Predicate;
 import java.util.logging.Level;
 import java.util.logging.Logger;
@@ -177,6 +178,7 @@ public final class Configuration {
     private int commandTimeout; // in seconds
     private int indexRefreshPeriod; // in seconds
     private boolean scopesEnabled;
+    private boolean projectsEnabled;
     private boolean foldingEnabled;
     private String statisticsFilePath;
     /*
@@ -381,7 +383,7 @@ public Configuration() {
         setPluginDirectory(null);
         setPluginStack(new AuthorizationStack(AuthControlFlag.REQUIRED, "default stack"));
         setPrintProgress(false);
-        setProjects(new HashMap<>());
+        setProjects(new ConcurrentHashMap<>());
         setQuickContextScan(true);
         //below can cause an outofmemory error, since it is defaulting to NO LIMIT
         setRamBufferSize(defaultRamBufferSize); //MB
@@ -668,6 +670,10 @@ public void setRepositories(List<RepositoryInfo> repositories) {
         this.repositories = repositories;
     }
 
+    public void addRepositories(List<RepositoryInfo> repositories) {
+        this.repositories.addAll(repositories);
+    }
+
     public String getUrlPrefix() {
         return urlPrefix;
     }
@@ -911,27 +917,6 @@ public void setCurrentIndexedCollapseThreshold(int currentIndexedCollapseThresho
         this.currentIndexedCollapseThreshold = currentIndexedCollapseThreshold;
     }
 
-    private transient Date lastModified;
-
-    /**
-     * Get the date of the last index update.
-     *
-     * @return the time of the last index update.
-     */
-    public Date getDateForLastIndexRun() {
-        if (lastModified == null) {
-            File timestamp = new File(getDataRoot(), "timestamp");
-            if (timestamp.exists()) {
-                lastModified = new Date(timestamp.lastModified());
-            }
-        }
-        return lastModified;
-    }
-
-    public void refreshDateForLastIndexRun() {
-        lastModified = null;
-    }
-
     /**
      * Get the contents of a file or empty string if the file cannot be read.
      */
@@ -1137,6 +1122,14 @@ public int getMaxSearchThreadCount() {
     public void setMaxSearchThreadCount(int count) {
         this.MaxSearchThreadCount = count;
     }
+    
+    public boolean isProjectsEnabled() {
+        return projectsEnabled;
+    }
+
+    public void setProjectsEnabled(boolean flag) {
+        this.projectsEnabled = flag;
+    }
 
     /**
      * Write the current configuration to a file

src/org/opensolaris/opengrok/configuration/IndexTimestamp.java

@@ -0,0 +1,76 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * See LICENSE.txt included in this distribution for the specific
+ * language governing permissions and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at LICENSE.txt.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+
+ /*
+ * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved.
+ */
+package org.opensolaris.opengrok.configuration;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Date;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import org.opensolaris.opengrok.logger.LoggerFactory;
+
+/**
+ * 
+ */
+public class IndexTimestamp {
+    private transient Date lastModified;
+    
+    private static final Logger LOGGER = LoggerFactory.getLogger(IndexTimestamp.class);
+
+    /**
+     * Get the date of the last index update.
+     *
+     * @return the time of the last index update.
+     */
+    public Date getDateForLastIndexRun() {
+        RuntimeEnvironment env = RuntimeEnvironment.getInstance();
+        if (lastModified == null) {
+            File timestamp = new File(env.getDataRootFile(), "timestamp");
+            if (timestamp.exists()) {
+                lastModified = new Date(timestamp.lastModified());
+            }
+        }
+        return lastModified;
+    }
+
+    public void refreshDateForLastIndexRun() {
+        lastModified = null;
+    }
+
+    public void stamp() throws IOException {
+        RuntimeEnvironment env = RuntimeEnvironment.getInstance();
+        File timestamp = new File(env.getDataRootFile(), "timestamp");
+        String purpose = "used for timestamping the index database.";
+        if (timestamp.exists()) {
+            if (!timestamp.setLastModified(System.currentTimeMillis())) {
+                LOGGER.log(Level.WARNING, "Failed to set last modified time on ''{0}'', {1}",
+                    new Object[]{timestamp.getAbsolutePath(), purpose});
+            }
+        } else {
+            if (!timestamp.createNewFile()) {
+                LOGGER.log(Level.WARNING, "Failed to create file ''{0}'', {1}",
+                    new Object[]{timestamp.getAbsolutePath(), purpose});
+            }
+        }
+    }
+}

src/org/opensolaris/opengrok/configuration/Project.java

@@ -48,6 +48,13 @@ public class Project implements Comparable<Project>, Nameable {
      */
     private int tabSize;
 
+    /**
+     * This marks the project as (not)ready before initial index is done.
+     * this is to avoid all/multi-project searches referencing this project
+     * from failing.
+     */
+    private boolean indexed = false;
+
     /**
      * Set of groups which match this project.
      */
@@ -85,6 +92,10 @@ public String getPath() {
         return path;
     }
 
+    public boolean isIndexed() {
+        return indexed;
+    }
+
     /**
      * Get the project id
      *
@@ -130,6 +141,10 @@ public void setPath(String path) {
         this.path = path;
     }
 
+    public void setIndexed(boolean flag) {
+        this.indexed = flag;
+    }
+
     /**
      * Set tab size for this project. Used for expanding tabs to spaces in
      * xrefs.
@@ -214,7 +229,7 @@ public static Project getProject(String path) {
     public static Project getProject(File file) {
         Project ret = null;
         try {
-            ret = getProject(RuntimeEnvironment.getInstance().getPathRelativeToSourceRoot(file, 0));
+            ret = getProject(RuntimeEnvironment.getInstance().getPathRelativeToSourceRoot(file));
         } catch (FileNotFoundException e) { // NOPMD
             // ignore if not under source root
         } catch (IOException e) { // NOPMD