Support ctags for historical revisions

- Add webappCtags configuration flag, with
  --webappCtags switch, to indicate if the webapp
  is eligible to run ctags.
- Add Repository.getHistoryGet() override to
  allow sub-classes to override to avoid a full
  in-memory version; and override for Git and
  Mercurial.
- Revise BoundedBlockingObjectPool as LIFO-to-FIFO
  so the webapp does not start extraneous
  instances; Indexer switches to FIFO performance
  when the queue pool is emptied.
- make IndexerParallelizer a lazy property of
  RuntimeEnvironment, and make the executors of
  IndexerParallelizer also lazy properties. Move
  the history-related executors into
  IndexerParallelizer so the lifecycle of all
  indexing/history executors are controlled in
  the same class.

Author: idodeclare

dev/checkstyle/suppressions.xml

@@ -1,5 +1,27 @@
 <?xml version="1.0"?>
+<!--
 
+CDDL HEADER START
+
+The contents of this file are subject to the terms of the
+Common Development and Distribution License (the "License").
+You may not use this file except in compliance with the License.
+
+See LICENSE.txt included in this distribution for the specific
+language governing permissions and limitations under the License.
+
+When distributing Covered Code, include this CDDL HEADER in each
+file and include the License file at LICENSE.txt.
+If applicable, add the following below this CDDL HEADER, with the
+fields enclosed by brackets "[]" replaced with your own identifying
+information: Portions Copyright [yyyy] [name of copyright owner]
+
+CDDL HEADER END
+
+Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved.
+Portions Copyright (c) 2018, Chris Fraire <[email protected]>.
+
+-->
 <!DOCTYPE suppressions PUBLIC
         "-//Checkstyle//DTD SuppressionFilter Configuration 1.1//EN"
         "https://checkstyle.org/dtds/suppressions_1_1.dtd">
@@ -9,7 +31,8 @@
         |CustomExactPhraseScorer\.java|PhrasePositions\.java|PhraseQueue\.java|Summarizer\.java|
         |Summary\.java|OGKUnifiedHighlighter\.java|ResponseHeaderFilter\.java|BoundedBlockingObjectPool\.java|
         |ObjectPool\.java|ObjectValidator\.java|ObjectFactory\.java|AbstractObjectPool\.java|
-        |BlockingObjectPool\.java|OGKTextVecField\.java|OGKTextField\.java" />
+        |BlockingObjectPool\.java|OGKTextVecField\.java|OGKTextField\.java|
+        |LazilyInstantiate\.java" />
 
     <suppress checks="ParameterNumber" files="CtagsReader\.java|Definitions\.java|PerlLexHelper\.java|
         |JFlexXrefUtils\.java|RubyLexHelper\.java|FileAnalyzerFactory\.java|SearchController\.java|

opengrok-indexer/src/main/java/org/opengrok/indexer/configuration/Configuration.java

@@ -107,6 +107,7 @@ public final class Configuration {
      * Path to {@code ctags} binary.
      */
     private String ctags;
+    private boolean webappCtags;
 
     /**
      * A defined value to specify the mandoc binary or else null so that mandoc
@@ -475,6 +476,7 @@ public Configuration() {
         // unconditionally later.
         setUserPageSuffix("");
         setWebappLAF("default");
+        // webappCtags is default(boolean)
     }
 
     public String getRepoCmd(String clazzName) {
@@ -983,6 +985,20 @@ public void setWebappLAF(String webappLAF) {
         this.webappLAF = webappLAF;
     }
 
+    /**
+     * Gets a value indicating if the web app should run ctags as necessary.
+     */
+    public boolean isWebappCtags() {
+        return webappCtags;
+    }
+
+    /**
+     * Sets a value indicating if the web app should run ctags as necessary.
+     */
+    public void setWebappCtags(boolean value) {
+        this.webappCtags = value;
+    }
+
     public RemoteSCM getRemoteScmSupported() {
         return remoteScmSupported;
     }

opengrok-indexer/src/main/java/org/opengrok/indexer/configuration/RuntimeEnvironment.java

@@ -43,7 +43,6 @@
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.ThreadFactory;
-import java.util.concurrent.TimeUnit;
 import java.util.concurrent.locks.Lock;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
 import java.util.logging.Level;
@@ -65,9 +64,11 @@
 import org.opengrok.indexer.index.Filter;
 import org.opengrok.indexer.index.IgnoredNames;
 import org.opengrok.indexer.index.IndexDatabase;
+import org.opengrok.indexer.index.IndexerParallelizer;
 import org.opengrok.indexer.logger.LoggerFactory;
 import org.opengrok.indexer.util.CtagsUtil;
 import org.opengrok.indexer.util.ForbiddenSymlinkException;
+import org.opengrok.indexer.util.LazilyInstantiate;
 import org.opengrok.indexer.util.PathUtils;
 import org.opengrok.indexer.web.Prefix;
 import org.opengrok.indexer.web.Statistics;
@@ -88,11 +89,10 @@ public final class RuntimeEnvironment {
     private static final String URL_PREFIX = "/source" + Prefix.SEARCH_R + "?";
 
     private Configuration configuration;
-    private ReentrantReadWriteLock configLock;
+    private final ReentrantReadWriteLock configLock;
+    private final LazilyInstantiate<IndexerParallelizer> lzIndexerParallelizer;
+    private final LazilyInstantiate<ExecutorService> lzSearchExecutor;
     private static final RuntimeEnvironment instance = new RuntimeEnvironment();
-    private static ExecutorService historyExecutor = null;
-    private static ExecutorService historyRenamedExecutor = null;
-    private static ExecutorService searchExecutor = null;
 
     private final Map<Project, List<RepositoryInfo>> repository_map = new ConcurrentHashMap<>();
     private final Map<String, SearcherManager> searcherManagerMap = new ConcurrentHashMap<>();
@@ -127,43 +127,21 @@ private RuntimeEnvironment() {
         configuration = new Configuration();
         configLock = new ReentrantReadWriteLock();
         watchDog = new WatchDogService();
+        lzIndexerParallelizer = LazilyInstantiate.using(() ->
+                new IndexerParallelizer(this));
+        lzSearchExecutor = LazilyInstantiate.using(() -> newSearchExecutor());
     }
 
     /** Instance of authorization framework.*/
     private AuthorizationFramework authFramework;
 
-    /* Get thread pool used for top-level repository history generation. */
-    public static synchronized ExecutorService getHistoryExecutor() {
-        if (historyExecutor == null) {
-            historyExecutor = Executors.newFixedThreadPool(getInstance().getHistoryParallelism(),
-                    runnable -> {
-                        Thread thread = Executors.defaultThreadFactory().newThread(runnable);
-                        thread.setName("history-handling-" + thread.getId());
-                        return thread;
-                    });
-        }
-
-        return historyExecutor;
-    }
-
-    /* Get thread pool used for history generation of renamed files. */
-    public static synchronized ExecutorService getHistoryRenamedExecutor() {
-        if (historyRenamedExecutor == null) {
-            historyRenamedExecutor = Executors.newFixedThreadPool(getInstance().getHistoryRenamedParallelism(),
-                    runnable -> {
-                        Thread thread = Executors.defaultThreadFactory().newThread(runnable);
-                        thread.setName("renamed-handling-" + thread.getId());
-                        return thread;
-                    });
-        }
-
-        return historyRenamedExecutor;
+    /** Gets the thread pool used for multi-project searches. */
+    public ExecutorService getSearchExecutor() {
+        return lzSearchExecutor.get();
     }
 
-    /* Get thread pool used for multi-project searches. */
-    public synchronized ExecutorService getSearchExecutor() {
-        if (searchExecutor == null) {
-            searchExecutor = Executors.newFixedThreadPool(
+    private ExecutorService newSearchExecutor() {
+        return Executors.newFixedThreadPool(
                 this.getMaxSearchThreadCount(),
                 new ThreadFactory() {
                 @Override
@@ -173,23 +151,6 @@ public Thread newThread(Runnable runnable) {
                     return thread;
                 }
             });
-        }
-
-        return searchExecutor;
-    }
-
-    public static synchronized void freeHistoryExecutor() {
-        historyExecutor = null;
-    }
-
-    public static synchronized void destroyRenamedHistoryExecutor() throws InterruptedException {
-        if (historyRenamedExecutor != null) {
-            historyRenamedExecutor.shutdown();
-            // All the jobs should be completed by now however for testing
-            // we would like to make sure the threads are gone.
-            historyRenamedExecutor.awaitTermination(1, TimeUnit.MINUTES);
-            historyRenamedExecutor = null;
-        }
     }
 
     /**
@@ -203,6 +164,10 @@ public static RuntimeEnvironment getInstance() {
 
     public WatchDogService watchDog;
 
+    public IndexerParallelizer getIndexerParallelizer() {
+        return lzIndexerParallelizer.get();
+    }
+
     private String getCanonicalPath(String s) {
         if (s == null) { return null; }
         try {
@@ -1092,6 +1057,14 @@ public void setWebappLAF(String laf) {
         setConfigurationValue("webappLAF", laf);
     }
 
+    /**
+     * Gets a value indicating if the web app should run ctags as necessary.
+     * @return the value of {@link Configuration#isWebappCtags()}
+     */
+    public boolean isWebappCtags() {
+        return (boolean)getConfigurationValue("webappCtags");
+    }
+
     public Configuration.RemoteSCM getRemoteScmSupported() {
         return (Configuration.RemoteSCM)getConfigurationValue("remoteScmSupported");
     }

opengrok-indexer/src/main/java/org/opengrok/indexer/history/FileHistoryCache.java

@@ -333,7 +333,7 @@ private History mergeOldAndNewHistory(File cacheFile, History histNew, Repositor
     /**
      * Store history object (encoded as XML and compressed with gzip) in a file.
      *
-     * @param history history object to store
+     * @param histNew history object to store
      * @param file file to store the history object into
      * @param repo repository for the file
      * @param mergeHistory whether to merge the history with existing or
@@ -536,9 +536,7 @@ public void store(History history, Repository repository)
         final CountDownLatch latch = new CountDownLatch(renamed_map.size());
         AtomicInteger renamedFileHistoryCount = new AtomicInteger();
         for (final Map.Entry<String, List<HistoryEntry>> map_entry : renamed_map.entrySet()) {
-            RuntimeEnvironment.getHistoryRenamedExecutor().submit(new Runnable() {
-                @Override
-                public void run() {
+            env.getIndexerParallelizer().getHistoryRenamedExecutor().submit(() -> {
                     try {
                         doFileHistory(map_entry.getKey(), map_entry.getValue(),
                             env, repositoryF,
@@ -552,7 +550,6 @@ public void run() {
                     } finally {
                         latch.countDown();
                     }
-                }
             });
         }
 

opengrok-indexer/src/main/java/org/opengrok/indexer/history/GitRepository.java

@@ -19,17 +19,19 @@
 
 /*
  * Copyright (c) 2008, 2019, Oracle and/or its affiliates. All rights reserved.
- * Portions Copyright (c) 2017, Chris Fraire <[email protected]>.
+ * Portions Copyright (c) 2017-2018, Chris Fraire <[email protected]>.
  */
 package org.opengrok.indexer.history;
 
 import java.io.BufferedReader;
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
 import java.io.File;
+import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
+import java.io.OutputStream;
 import java.io.Reader;
 import java.nio.file.Paths;
 import java.text.ParseException;
@@ -46,6 +48,7 @@
 import java.util.regex.Pattern;
 import org.opengrok.indexer.configuration.RuntimeEnvironment;
 import org.opengrok.indexer.logger.LoggerFactory;
+import org.opengrok.indexer.util.BufferSink;
 import org.opengrok.indexer.util.Executor;
 import org.opengrok.indexer.util.StringUtils;
 
@@ -171,14 +174,18 @@ Executor getRenamedFilesExecutor(final File file, String sinceRevision) throws I
     /**
      * Try to get file contents for given revision.
      *
+     * @param sink a required target sink
+     * @param outIterations a required out array for storing the number of calls
+     * made to {@code sink}
      * @param fullpath full pathname of the file
      * @param rev revision
-     * @return contents of the file in revision rev
+     * @return {@code true} if any contents were found
      */
-    private InputStream getHistoryRev(String fullpath, String rev) {
-        InputStream ret = null;
-        File directory = new File(getDirectoryName());
+    private boolean getHistoryRev(BufferSink sink, int[] outIterations,
+            String fullpath, String rev) {
 
+        outIterations[0] = 0;
+        File directory = new File(getDirectoryName());
         try {
             /*
              * Be careful, git uses only forward slashes in its command and output (not in file path).
@@ -197,13 +204,13 @@ private InputStream getHistoryRev(String fullpath, String rev) {
                     RuntimeEnvironment.getInstance().getInteractiveCommandTimeout());
             int status = executor.exec();
 
-            ByteArrayOutputStream out = new ByteArrayOutputStream();
             byte[] buffer = new byte[32 * 1024];
             try (InputStream in = executor.getOutputStream()) {
                 int len;
                 while ((len = in.read(buffer)) != -1) {
                     if (len > 0) {
-                        out.write(buffer, 0, len);
+                        outIterations[0]++;
+                        sink.write(buffer, 0, len);
                     }
                 }
             }
@@ -212,22 +219,48 @@ private InputStream getHistoryRev(String fullpath, String rev) {
              * If exit value of the process was not 0 then the file did
              * not exist or internal git error occured.
              */
-            if (status == 0) {
-                ret = new ByteArrayInputStream(out.toByteArray());
-            } else {
-                ret = null;
-            }
+            return status == 0;
         } catch (Exception exp) {
             LOGGER.log(Level.SEVERE,
                     "Failed to get history for file {0} in revision {1}: ",
                         new Object[]{fullpath, rev, exp.getClass().toString(), exp});
+            return false;
         }
+    }
 
-        return ret;
+    /**
+     * Gets the contents of a specific version of a named file into the
+     * specified target without a full, in-memory buffer.
+     *
+     * @param target a required target file which will be overwritten
+     * @param parent the name of the directory containing the file
+     * @param basename the name of the file to get
+     * @param rev the revision to get
+     * @return {@code true} if contents were found
+     * @throws java.io.IOException if an I/O error occurs
+     */
+    @Override
+    public boolean getHistoryGet(File target, String parent, String basename,
+            String rev) throws IOException {
+        try (OutputStream out = new FileOutputStream(target)) {
+            return getHistoryGet((buf, offset, n) -> out.write(buf, offset, n),
+                    parent, basename, rev);
+        }
     }
 
     @Override
-    public InputStream getHistoryGet(String parent, String basename, String rev) {
+    public InputStream getHistoryGet(String parent, String basename,
+            String rev) {
+        ByteArrayOutputStream out = new ByteArrayOutputStream();
+        if (getHistoryGet((buf, offset, n) -> out.write(buf, offset, n),
+                parent, basename, rev)) {
+            return new ByteArrayInputStream(out.toByteArray());
+        }
+        return null;
+    }
+
+    protected boolean getHistoryGet(BufferSink sink, String parent,
+            String basename, String rev) {
         String fullpath;
         try {
             fullpath = new File(parent, basename).getCanonicalPath();
@@ -238,12 +271,13 @@ public String get() {
                     return String.format("Failed to get canonical path: %s/%s", parent, basename);
                 }
             });
-            return null;
+            return false;
         }
 
-        InputStream ret = getHistoryRev(fullpath, rev);
-
-        if (ret == null) {
+        int[] iterations = new int[1];
+        boolean ret = getHistoryRev((buf, offset, n) -> sink.write(buf, offset,
+                n), iterations, fullpath, rev);
+        if (!ret && iterations[0] < 1) {
             /*
              * If we failed to get the contents it might be that the file was
              * renamed so we need to find its original name in that revision
@@ -259,10 +293,10 @@ public String get() {
                         return String.format("Failed to get original revision: %s/%s (revision %s)", parent, basename, rev);
                     }
                 });
-                return null;
+                return false;
             }
             if (origpath != null) {
-                ret = getHistoryRev(origpath, rev);
+                ret = getHistoryRev(sink, iterations, origpath, rev);
             }
         }