timeout for command execution

fixes #429
fixes #487

Author: vladak

src/org/opensolaris/opengrok/configuration/Configuration.java

@@ -119,6 +119,7 @@ public final class Configuration {
     private boolean chattyStatusPage;
     private final Map<String, String> cmds;
     private int tabSize;
+    private int command_timeout;
     private static final Logger logger = Logger.getLogger(Configuration.class.getName());
 
     /**
@@ -161,6 +162,14 @@ public void setScanningDepth(int scanningDepth) {
         this.scanningDepth = scanningDepth;
     }
 
+    public int getCommandTimeout() {
+        return command_timeout;
+    }
+
+    public void setCommandTimeout(int timeout) {
+        this.command_timeout = timeout;
+    }
+    
     /**
      * Creates a new instance of Configuration
      */
@@ -202,6 +211,7 @@ public Configuration() {
         cmds = new HashMap<String, String>();
         setSourceRoot(null);
         setDataRoot(null);
+        setCommandTimeout(600); // 10 minutes
     }
 
     public String getRepoCmd(String clazzName) {

src/org/opensolaris/opengrok/configuration/RuntimeEnvironment.java

@@ -103,6 +103,14 @@ public void setScanningDepth(int scanningDepth) {
         threadConfig.get().setScanningDepth(scanningDepth);
     }
 
+    public int getCommandTimeout() {
+        return threadConfig.get().getCommandTimeout();
+    }
+
+    public void setCommandTimeout(int timeout) {
+        threadConfig.get().setCommandTimeout(timeout);
+    }
+
     /**
      * Get the path to the where the index database is stored
      *
@@ -894,7 +902,8 @@ public void run() {
 
                             if (obj instanceof Configuration) {
                                 setConfiguration((Configuration) obj);
-                                log.log(Level.INFO, "Configuration updated: {0}", configuration.getSourceRoot());
+                                log.log(Level.INFO, "Configuration updated: {0}",
+                                    configuration.getSourceRoot());
                             }
                         } catch (IOException e) {
                             log.log(Level.SEVERE, "Error reading config file: ", e);

src/org/opensolaris/opengrok/util/Executor.java

@@ -33,9 +33,12 @@
 import java.lang.Thread.UncaughtExceptionHandler;
 import java.util.Arrays;
 import java.util.List;
+import java.util.Timer;
+import java.util.TimerTask;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 
+import org.opensolaris.opengrok.configuration.RuntimeEnvironment;
 import org.opensolaris.opengrok.OpenGrokLogger;
 
 /**
@@ -110,12 +113,21 @@ public int exec(boolean reportExceptions) {
      */
     public int exec(final boolean reportExceptions, StreamHandler handler) {
         int ret = -1;
-
         ProcessBuilder processBuilder = new ProcessBuilder(cmdList);
+        final String cmd_str = processBuilder.command().toString();
+        final String dir_str;
+        File cwd = processBuilder.directory();
+        if (cwd == null) {
+            dir_str = System.getProperty("user.dir");
+        } else {
+            dir_str = cwd.toString();
+        }
+      
         if (workingDirectory != null) {
             processBuilder.directory(workingDirectory);
             if (processBuilder.environment().containsKey("PWD")) {
-                processBuilder.environment().put("PWD", workingDirectory.getAbsolutePath());
+                processBuilder.environment().put("PWD",
+                    workingDirectory.getAbsolutePath());
             }
         }
 
@@ -129,6 +141,7 @@ public int exec(final boolean reportExceptions, StreamHandler handler) {
         Process process = null;
         try {
             process = processBuilder.start();
+            final Process proc = process;
 
             final InputStream errorStream = process.getErrorStream();
             final SpoolHandler err = new SpoolHandler();
@@ -148,9 +161,24 @@ public void run() {
             });
             thread.start();
 
+            /*
+             * Setup timer so if the process get stuck we can terminate it and
+             * make progress instead of hanging the whole indexer.
+             */
+            Timer t = new Timer();
+            t.schedule(new TimerTask() {
+                @Override public void run() {
+                    OpenGrokLogger.getLogger().log(Level.INFO,
+                        "Terminating process of command {0} in directory {1} " +
+                        "due to timeout", new Object[]{cmd_str, dir_str});
+                    proc.destroy();
+                }
+            }, RuntimeEnvironment.getInstance().getCommandTimeout() * 1000);
+            
             handler.processStream(process.getInputStream());
 
             ret = process.waitFor();
+            t.cancel();
             process = null;
             thread.join();
             stderr = err.getBytes();
@@ -175,17 +203,12 @@ public void run() {
         }
 
         if (ret != 0 && reportExceptions) {
-            int MAX_MSG_SZ = 512; /* limit to avoid floodding the logs */
+            int MAX_MSG_SZ = 512; /* limit to avoid flooding the logs */
             StringBuilder msg = new StringBuilder("Non-zero exit status ")
                     .append(ret).append(" from command ")
-                    .append(processBuilder.command().toString())
-                    .append(" in directory ");
-            File cwd = processBuilder.directory();
-            if (cwd == null) {
-                msg.append(System.getProperty("user.dir"));
-            } else {
-                msg.append(cwd.toString());
-            }
+                    .append(cmd_str)
+                    .append(" in directory ")
+                    .append(dir_str);
             if (stderr != null && stderr.length > 0) {
                     msg.append(": ");
                     if (stderr.length > MAX_MSG_SZ) {