Add indexParallel(), using Ctags object pool, for faster indexing

Also, restrict (new) PendingFileCompleter as
package-private not public.

Author: idodeclare

OpenGrok

@@ -110,6 +110,8 @@
 #                                 increase it to 256MB, but do increase JVM to
 #                                 4/8/16GB ! Lucene defaults to 8 threads.
 #                                 Increase JVM memory as noted using JAVA_OPTS
+#   - OPENGROK_PARALLELISM        Default is 0, meaning that parallelism is
+#                                   determined from available processors.
 #   - OPENGROK_LOGGER_CONFIG_PATH Set path to custom logging.properties file.
 #   - OPENGROK_SUBVERSION_USERNAME name of the user that should be used for
 #                                 fetching the history from subversion
@@ -862,6 +864,7 @@ CommonInvocation()
         ${OPENGROK_MANDOC:+--mandoc} ${OPENGROK_MANDOC}			\
         ${CTAGS_OPTIONS_FILE:+-o} ${CTAGS_OPTIONS_FILE}         	\
         ${OPENGROK_FLUSH_RAM_BUFFER_SIZE} ${SKIN} ${LEADING_WILDCARD}	\
+        ${OPENGROK_PARALLELISM:+--threads} ${OPENGROK_PARALLELISM}	\
         ${READ_XML_CONF}                                        	\
         ${WEBAPP_CONFIG}						\
         ${WEBAPP_CONTEXT}						\

src/org/opensolaris/opengrok/analysis/Ctags.java

@@ -58,6 +58,14 @@ public class Ctags {
 
     private boolean junit_testing = false;
 
+    public boolean isClosed() {
+        return ctags != null && !ctags.isAlive();
+    }
+
+    public String getBinary() {
+        return binary;
+    }
+
     public void setBinary(String binary) {
         this.binary = binary;
     }

src/org/opensolaris/opengrok/analysis/CtagsValidator.java

@@ -0,0 +1,48 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * See LICENSE.txt included in this distribution for the specific
+ * language governing permissions and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at LICENSE.txt.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+
+/*
+ * Copyright (c) 2017, Chris Fraire <[email protected]>.
+ */
+
+package org.opensolaris.opengrok.analysis;
+
+import java.io.IOException;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import org.opensolaris.opengrok.logger.LoggerFactory;
+import org.opensolaris.opengrok.util.ObjectValidator;
+
+public final class CtagsValidator implements ObjectValidator<Ctags> {
+
+    private static final Logger LOGGER = LoggerFactory.getLogger(
+        CtagsValidator.class);
+
+    public boolean isValid(Ctags ctags) {
+        return ctags != null && !ctags.isClosed();
+    }
+
+    public void invalidate(Ctags ctags) {
+        try {
+            if (ctags != null) ctags.close();
+        } catch (IOException ex) {
+            LOGGER.log(Level.FINE, "Error closing ctags", ex);
+        }
+    }
+}

src/org/opensolaris/opengrok/configuration/Configuration.java

@@ -182,6 +182,7 @@ public final class Configuration {
     private String luceneLocking = LuceneLockName.OFF;
     private boolean compressXref;
     private boolean indexVersionedFilesOnly;
+    private int indexingParallelism;
     private boolean tagsEnabled;
     private int hitsPerPage;
     private int cachePages;
@@ -971,6 +972,14 @@ public void setIndexVersionedFilesOnly(boolean indexVersionedFilesOnly) {
         this.indexVersionedFilesOnly = indexVersionedFilesOnly;
     }
 
+    public int getIndexingParallelism() {
+        return indexingParallelism;
+    }
+
+    public void setIndexingParallelism(int value) {
+        this.indexingParallelism = value > 0 ? value : 0;
+    }
+
     public boolean isTagsEnabled() {
         return this.tagsEnabled;
     }

src/org/opensolaris/opengrok/configuration/RuntimeEnvironment.java

@@ -1059,6 +1059,17 @@ public void setIndexVersionedFilesOnly(boolean indexVersionedFilesOnly) {
         threadConfig.get().setIndexVersionedFilesOnly(indexVersionedFilesOnly);
     }
 
+    /**
+     * Gets the value of {@link Configuration#getIndexingParallelism()} -- or
+     * if zero, then as a default gets the number of available processors.
+     * @return a natural number >= 1
+     */
+    public int getIndexingParallelism() {
+        int parallelism = threadConfig.get().getIndexingParallelism();
+        return parallelism < 1 ? Runtime.getRuntime().availableProcessors() :
+            parallelism;
+    }
+
     public boolean isTagsEnabled() {
         return threadConfig.get().isTagsEnabled();
     }