Merge pull request #1936 from idodeclare/feature/deferred_ops

Feature/deferred ops

Author: tarzanek

OpenGrok

@@ -44,6 +44,8 @@
 #                                 options for CTags program (for its --options
 #                                 switch), default is DATA_ROOT/etc/ctags.config
 #   - OPENGROK_MANDOC             Full path to mandoc(1) binary
+#   - OPENGROK_LOCKING            Locking mode on|off|simple|native (default off)
+#                                 ("on" is an alias for "simple")
 #   - JAVA_HOME                   Full Path to Java Installation Root
 #   - JAVA                        Full Path to java binary (to enable 64bit JDK)
 #   - JAVA_OPTS                   Java options (e.g. for JVM memory increase
@@ -109,6 +111,8 @@
 #                                 increase it to 256MB, but do increase JVM to
 #                                 4/8/16GB ! Lucene defaults to 8 threads.
 #                                 Increase JVM memory as noted using JAVA_OPTS
+#   - OPENGROK_PARALLELISM        Default is 0, meaning that parallelism is
+#                                   determined from available processors.
 #   - OPENGROK_LOGGER_CONFIG_PATH Set path to custom logging.properties file.
 #   - OPENGROK_SUBVERSION_USERNAME name of the user that should be used for
 #                                 fetching the history from subversion
@@ -861,10 +865,12 @@ CommonInvocation()
 	${RENAMED_FILES_HISTORY}					\
         ${SCAN_DEPTH}                                           	\
         ${PROGRESS}                                             	\
+        ${OPENGROK_LOCKING:+--lock} ${OPENGROK_LOCKING}        		\
         ${OPENGROK_CTAGS:+-c} ${OPENGROK_CTAGS}               		\
         ${OPENGROK_MANDOC:+--mandoc} ${OPENGROK_MANDOC}			\
         ${CTAGS_OPTIONS_FILE:+-o} ${CTAGS_OPTIONS_FILE}         	\
         ${OPENGROK_FLUSH_RAM_BUFFER_SIZE} ${SKIN} ${LEADING_WILDCARD}	\
+        ${OPENGROK_PARALLELISM:+--threads} ${OPENGROK_PARALLELISM}	\
         ${READ_XML_CONF}                                        	\
         ${WEBAPP_CONFIG}						\
         ${WEBAPP_CONTEXT}						\
@@ -1039,17 +1045,15 @@ case "${1}" in
     update)
         ValidateConfiguration
         CreateRuntimeRequirements
-        UpdateGeneratedData
-        UpdateDescriptionCache
+        UpdateGeneratedData && UpdateDescriptionCache
         ;;
 
     updateQuietly)
         ValidateConfiguration
         CreateRuntimeRequirements
         QUIET="-q"
         VERBOSE=""
-        UpdateGeneratedData
-        UpdateDescriptionCache
+        UpdateGeneratedData && UpdateDescriptionCache
         ;;
 
     updateDesc)
@@ -1073,8 +1077,7 @@ case "${1}" in
         fi
         ValidateConfiguration
         CreateRuntimeRequirements
-        UpdateGeneratedData
-        UpdateDescriptionCache
+        UpdateGeneratedData && UpdateDescriptionCache
         ;;
 
     indexpart)
@@ -1112,3 +1115,4 @@ case "${1}" in
         Usage
         ;;
 esac
+exit $?

src/org/opensolaris/opengrok/analysis/AnalyzerGuru.java

@@ -393,10 +393,11 @@ public static FileAnalyzer getAnalyzer(InputStream in, String file) throws IOExc
      * @param fa The analyzer to use on the file
      * @param xrefOut Where to write the xref (possibly {@code null})
      * @throws IOException If an exception occurs while collecting the data
+     * @throws InterruptedException if a timeout occurs
      */
     public void populateDocument(Document doc, File file, String path,
-            FileAnalyzer fa, Writer xrefOut)
-            throws IOException {
+        FileAnalyzer fa, Writer xrefOut) throws IOException,
+            InterruptedException {
 
         String date = DateTools.timeToString(file.lastModified(),
                 DateTools.Resolution.MILLISECOND);

src/org/opensolaris/opengrok/analysis/Ctags.java

@@ -19,7 +19,7 @@
 
  /*
  * Copyright (c) 2005, 2017, Oracle and/or its affiliates. All rights reserved.
- * Portions Copyright (c) 2017, Chris Fraire <[email protected]>.
+ * Portions Copyright (c) 2017-2018, Chris Fraire <[email protected]>.
  */
 package org.opensolaris.opengrok.analysis;
 
@@ -39,25 +39,40 @@
 import org.opensolaris.opengrok.util.IOUtils;
 
 /**
- * Provides Ctags by having a running instance of ctags
+ * Provides Ctags by having a running subprocess of ctags.
  *
  * @author Chandan
  */
 public class Ctags {
 
     private static final Logger LOGGER = LoggerFactory.getLogger(Ctags.class);
 
+    private volatile boolean closing;
     private Process ctags;
+    private Thread errThread;
     private OutputStreamWriter ctagsIn;
     private BufferedReader ctagsOut;
     private static final String CTAGS_FILTER_TERMINATOR = "__ctags_done_with_file__";
     //default: setCtags(System.getProperty("org.opensolaris.opengrok.analysis.Ctags", "ctags"));
     private String binary;
     private String CTagsExtraOptionsFile = null;
-    private ProcessBuilder processBuilder;
 
     private boolean junit_testing = false;
 
+    /**
+     * Gets a value indicating if a subprocess of ctags was started and it is
+     * not alive.
+     * @return {@code true} if the instance should be considered closed and no
+     * longer usable.
+     */
+    public boolean isClosed() {
+        return ctags != null && !ctags.isAlive();
+    }
+
+    public String getBinary() {
+        return binary;
+    }
+
     public void setBinary(String binary) {
         this.binary = binary;
     }
@@ -69,14 +84,16 @@ public void setCTagsExtraOptionsFile(String CTagsExtraOptionsFile) {
     public void close() throws IOException {
         IOUtils.close(ctagsIn);
         if (ctags != null) {
+            closing = true;
             LOGGER.log(Level.FINE, "Destroying ctags command");
-            ctags.destroy();
+            ctags.destroyForcibly();
         }
     }
 
     private void initialize() throws IOException {
         RuntimeEnvironment env = RuntimeEnvironment.getInstance();
-        if (processBuilder == null) {
+        ProcessBuilder processBuilder;
+        if (true) {
             List<String> command = new ArrayList<>();
 
             command.add(binary);
@@ -284,58 +301,65 @@ private void initialize() throws IOException {
         ctagsIn = new OutputStreamWriter(ctags.getOutputStream());
         ctagsOut = new BufferedReader(new InputStreamReader(ctags.getInputStream()));
 
-        Thread errThread = new Thread(new Runnable() {
+        errThread = new Thread(new Runnable() {
 
             @Override
             public void run() {
-                StringBuilder sb = new StringBuilder();
                 try (BufferedReader error = new BufferedReader(
                         new InputStreamReader(ctags.getErrorStream()))) {
                     String s;
                     while ((s = error.readLine()) != null) {
-                        sb.append(s);
-                        sb.append('\n');
+                        if (s.length() > 0) {
+                            LOGGER.log(Level.WARNING, "Error from ctags: {0}",
+                                s);
+                        }
+                        if (closing) break;
                     }
                 } catch (IOException exp) {
                     LOGGER.log(Level.WARNING, "Got an exception reading ctags error stream: ", exp);
                 }
-                if (sb.length() > 0) {
-                    LOGGER.log(Level.WARNING, "Error from ctags: {0}", sb.toString());
-                }
             }
         });
         errThread.setDaemon(true);
         errThread.start();
     }
 
-    public Definitions doCtags(String file) throws IOException {
-        boolean ctagsRunning = false;
+    public Definitions doCtags(String file) throws IOException,
+            InterruptedException {
+        if (file.length() < 1 || "\n".equals(file)) return null;
+
         if (ctags != null) {
             try {
                 int exitValue = ctags.exitValue();
                 // If it is possible to retrieve exit value without exception
-                // this means the ctags process is dead so we must restart it.
-                ctagsRunning = false;
+                // this means the ctags process is dead.
                 LOGGER.log(Level.WARNING, "Ctags process exited with exit value {0}",
                         exitValue);
+                // Throw the following to indicate non-I/O error for retry.
+                throw new InterruptedException("ctags died");
             } catch (IllegalThreadStateException exp) {
-                ctagsRunning = true;
                 // The ctags process is still running.
             }
-        }
-
-        if (!ctagsRunning) {
+        } else {
             initialize();
         }
 
-        Definitions ret = null;
-        if (file.length() > 0 && !"\n".equals(file)) {
-            //log.fine("doing >" + file + "<");
+        CtagsReader rdr = new CtagsReader();
+        Definitions ret;
+        try {
             ctagsIn.write(file);
+            if (Thread.interrupted()) throw new InterruptedException("write()");
             ctagsIn.flush();
-            CtagsReader rdr = new CtagsReader();
+            if (Thread.interrupted()) throw new InterruptedException("flush()");
             readTags(rdr);
             ret = rdr.getDefinitions();
+        } catch (IOException ex) {
+            /*
+             * In case the ctags process had to be destroyed, possibly pre-empt
+             * the IOException with an InterruptedException.
+             */
+            if (Thread.interrupted()) throw new InterruptedException("I/O");
+            throw ex;
         }
 
         return ret;
@@ -383,15 +407,23 @@ public void destroy() {
         };
 
         CtagsReader rdr = new CtagsReader();
-        readTags(rdr);
+        try {
+            readTags(rdr);
+        } catch (InterruptedException ex) {
+            LOGGER.log(Level.SEVERE, "readTags() test", ex);
+        }
         Definitions ret = rdr.getDefinitions();
         return ret;
     }
 
-    private void readTags(CtagsReader reader) {
+    private void readTags(CtagsReader reader) throws InterruptedException {
         try {
             do {
                 String tagLine = ctagsOut.readLine();
+                if (Thread.interrupted()) {
+                    throw new InterruptedException("readLine()");
+                }
+
                 //log.fine("Tagline:-->" + tagLine+"<----ONELINE");
                 if (tagLine == null) {
                     if (!junit_testing) {
@@ -421,6 +453,8 @@ private void readTags(CtagsReader reader) {
 
                 reader.readLine(tagLine);
             } while (true);
+        } catch (InterruptedException e) {
+            throw e;
         } catch (Exception e) {
             LOGGER.log(Level.WARNING, "CTags parsing problem: ", e);
         }

src/org/opensolaris/opengrok/analysis/CtagsValidator.java

@@ -0,0 +1,48 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * See LICENSE.txt included in this distribution for the specific
+ * language governing permissions and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at LICENSE.txt.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+
+/*
+ * Copyright (c) 2017, Chris Fraire <[email protected]>.
+ */
+
+package org.opensolaris.opengrok.analysis;
+
+import java.io.IOException;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import org.opensolaris.opengrok.logger.LoggerFactory;
+import org.opensolaris.opengrok.util.ObjectValidator;
+
+public final class CtagsValidator implements ObjectValidator<Ctags> {
+
+    private static final Logger LOGGER = LoggerFactory.getLogger(
+        CtagsValidator.class);
+
+    public boolean isValid(Ctags ctags) {
+        return ctags != null && !ctags.isClosed();
+    }
+
+    public void invalidate(Ctags ctags) {
+        try {
+            if (ctags != null) ctags.close();
+        } catch (IOException ex) {
+            LOGGER.log(Level.FINE, "Error closing ctags", ex);
+        }
+    }
+}

src/org/opensolaris/opengrok/analysis/FileAnalyzer.java

@@ -201,8 +201,10 @@ public String getFileTypeName() {
      * @param src the input data source
      * @param xrefOut where to write the xref (may be {@code null})
      * @throws IOException if any I/O error
+     * @throws InterruptedException if a timeout occurs
      */
-    public void analyze(Document doc, StreamSource src, Writer xrefOut) throws IOException {
+    public void analyze(Document doc, StreamSource src, Writer xrefOut)
+            throws IOException, InterruptedException {
         // not used
     }
 

src/org/opensolaris/opengrok/analysis/archive/BZip2Analyzer.java

@@ -19,6 +19,7 @@
 
 /*
  * Copyright (c) 2005, 2017, Oracle and/or its affiliates. All rights reserved.
+ * Portions Copyright (c) 2017, Chris Fraire <[email protected]>.
  */
 package org.opensolaris.opengrok.analysis.archive;
 
@@ -57,7 +58,8 @@ protected BZip2Analyzer(FileAnalyzerFactory factory) {
     private FileAnalyzer fa;
 
     @Override
-    public void analyze(Document doc, StreamSource src, Writer xrefOut) throws IOException {
+    public void analyze(Document doc, StreamSource src, Writer xrefOut)
+            throws IOException, InterruptedException {
         StreamSource bzSrc = wrap(src);
         String path = doc.get("path");
         if (path != null

src/org/opensolaris/opengrok/analysis/archive/GZIPAnalyzer.java

@@ -19,6 +19,7 @@
 
 /*
  * Copyright (c) 2005, 2017, Oracle and/or its affiliates. All rights reserved.
+ * Portions Copyright (c) 2017, Chris Fraire <[email protected]>.
  */
 package org.opensolaris.opengrok.analysis.archive;
 
@@ -63,7 +64,8 @@ protected GZIPAnalyzer(FileAnalyzerFactory factory) {
     private FileAnalyzer fa;
 
     @Override
-    public void analyze(Document doc, StreamSource src, Writer xrefOut) throws IOException {
+    public void analyze(Document doc, StreamSource src, Writer xrefOut)
+            throws IOException, InterruptedException {
         StreamSource gzSrc = wrap(src);
         String path = doc.get("path");
         if (path != null

src/org/opensolaris/opengrok/analysis/plain/AbstractSourceCodeAnalyzer.java

@@ -19,6 +19,7 @@
 
 /*
  * Copyright (c) 2012, 2017 Oracle and/or its affiliates. All rights reserved.
+ * Portions Copyright (c) 2017, Chris Fraire <[email protected]>.
  */
 package org.opensolaris.opengrok.analysis.plain;
 
@@ -69,9 +70,4 @@ protected AbstractSourceCodeAnalyzer(FileAnalyzerFactory factory,
      */
     @Override
     protected abstract JFlexXref newXref(Reader reader);
-
-    @Override
-    public void analyze(Document doc, StreamSource src, Writer xrefOut) throws IOException {
-        super.analyze(doc, src, xrefOut);
-    }        
 }

src/org/opensolaris/opengrok/analysis/plain/PlainAnalyzer.java

@@ -85,7 +85,8 @@ protected Reader getReader(InputStream stream) throws IOException {
     }
 
     @Override
-    public void analyze(Document doc, StreamSource src, Writer xrefOut) throws IOException {
+    public void analyze(Document doc, StreamSource src, Writer xrefOut)
+            throws IOException, InterruptedException {
         Definitions defs = null;
 
         doc.add(new TextField(QueryBuilder.FULL, getReader(src.getStream())));