show config info in bpipe dev output, and display tail output from failed commands at end of pipeline

Author: ssadedin

src/main/groovy/bpipe/Command.groovy

@@ -116,6 +116,8 @@ class Command implements Serializable {
      */
     List<String> shell 
 
+    transient OutputLog outputLog
+    
     File dir
 
     transient List<CommandDependency> dependencies

src/main/groovy/bpipe/CommandFailedException.java

@@ -33,15 +33,25 @@
  */
 class CommandFailedException extends PipelineError {
 
-    Command command;
+    private static final long serialVersionUID = 1L;
+
+    private Command command;
 
     public CommandFailedException(String arg0, PipelineContext ctx, Command command) {
         super(arg0, ctx);
-        this.command = command;
+        this.setCommand(command);
     }
 
     public CommandFailedException(Throwable arg0) {
         super(arg0);
     }
+
+    public Command getCommand() {
+        return command;
+    }
+
+    public void setCommand(Command command) {
+        this.command = command;
+    }
 
 }

src/main/groovy/bpipe/CommandManager.groovy

@@ -151,6 +151,8 @@ class CommandManager {
 
         command.name = name
         command.executor = wrapped
+        command.outputLog = commandLog
+
         wrapped.start(cfg, command, commandLog, commandLog)
 
 		this.commandIds[cmdExec] = command.id

src/main/groovy/bpipe/OutputLog.groovy

@@ -57,8 +57,9 @@ class OutputLog implements Appendable {
     StringBuilder buffer = new StringBuilder()
 
     // New fields for maintaining a rolling tail of the last N lines
-    int maxTailLines = 5
-    List<String> tailBuffer = []
+    int maxTailLines = 5
+
+    ArrayDeque<String> tailBuffer = new ArrayDeque(maxTailLines)
 
     String branch
     String prefix
@@ -90,7 +91,7 @@ class OutputLog implements Appendable {
      * @param output
      */
     void buffer(CharSequence output) {
-        output.eachLine { String line ->
+        output.eachLine { String line ->
             bufferLine(line)
         }
     }
@@ -112,11 +113,12 @@ class OutputLog implements Appendable {
 
     }
 
-    void bufferLine(String line) {
-        // Update rolling tail buffer
-        tailBuffer << line
-        if(tailBuffer.size() > maxTailLines) {
-            tailBuffer.remove(0)
+    void bufferLine(String line) {
+
+        // Update rolling tail buffer
+        tailBuffer.addLast(line)
+        if(tailBuffer.size() > maxTailLines) {
+            tailBuffer.removeFirst()
         }
 
         // In dev mode, no output log at all, just print raw output
@@ -157,9 +159,10 @@ class OutputLog implements Appendable {
         buffer.append(arg0,arg1,arg2)
         this.flush()
         return this;
-    }
-    List<String> getLastTailLines() {
-        return tailBuffer.clone()
+    }
+
+    Iterable<String> getLastTailLines() {
+        return tailBuffer
     }
 }
 

src/main/groovy/bpipe/Pipeline.groovy

@@ -53,6 +53,7 @@ import org.codehaus.groovy.runtime.ReverseListIterator
 import bpipe.graph.Graph;
 import bpipe.storage.StorageLayer
 import bpipe.storage.UnknownStoragePipelineFile
+import static org.fusesource.jansi.Ansi.*
 
 import static bpipe.Utils.isContainer 
 import static bpipe.Utils.unbox 
@@ -930,14 +931,27 @@ public class Pipeline implements ResourceRequestor {
                 String branchPart = pe.ctx?.branch?.firstNonTrivialName ? " ($pe.ctx.branch.firstNonTrivialName) " : ""
                 String stageName = pe.ctx?.stageName
                 if(stageName && stageName != "Unknown") {
-                    return "In stage ${pe.ctx?.stageName}$branchPart: " + pe.message
+                    String mainMessage = "In stage ${pe.ctx?.stageName}$branchPart: " + pe.message
+                    
+                    if(pe instanceof CommandFailedException) {
+                        OutputLog o = pe.getCommand().outputLog
+                        if(o?.getLastTailLines()) {
+                            mainMessage += "\n\nTrailing lines:${ansi().fgRed()}\n\n    " + o.getLastTailLines().join('\n    ')  + ansi().fgDefault() + '\n'
+                        }
+                        else {
+                            mainMessage += "\nNo output recorded from command\n"
+                        }
+                    }
+                    
+                    return mainMessage
                 }
                 else {
                     return pe.message
                 }
             }
-            else 
+            else {
                 return e.message
+            }
         }.join('\n')
     }
 

src/main/groovy/bpipe/PipelineContext.groovy

@@ -2091,7 +2091,13 @@ class PipelineContext {
                 String threadsMsg = ''
                 if(prettyCmd.contains('__bpipe_lazy_resource_threads__')) {
                     prettyCmd = prettyCmd.replaceAll('__bpipe_lazy_resource_threads__', "${ansi().fgGreen()}\\\${threads}${ansi().fgDefault()}")
-                    threadsMsg = "${ansi().bold()}Note:${ansi().boldOff()} ${ansi().fgGreen()}\${threads}${ansi().fgDefault()} will be assigned at runtime based on available concurrency"
+                    threadsMsg = "${ansi().bold()}Note:${ansi().boldOff()} ${ansi().fgGreen()}\${threads}${ansi().fgDefault()} will be assigned at runtime based on available concurrency\n"
+                }
+
+                String memoryMsg = ""
+                if(prettyCmd.contains('__bpipe_lazy_resource_memory__')) {
+                    prettyCmd = prettyCmd.replaceAll('__bpipe_lazy_resource_memory__', "${ansi().fgGreen()}\\\${memory}${ansi().fgDefault()}")
+                    memoryMsg = "${ansi().bold()}Note:${ansi().boldOff()} ${ansi().fgGreen()}\${memory}${ansi().fgDefault()} will be assigned based on configuration\n"
                 }
 
                 String msg = branch.name ? "Stage $stageName in branch $branch.name would execute:\n\n        $prettyCmd" : "Would execute $prettyCmd"
@@ -2107,7 +2113,7 @@ class PipelineContext {
                 if(configObject?.containsKey('container') && configObject.container) {
                     ConfigObject container = configObject['container']
                     if(container)
-                        containerMsg = "Will execute in container: " + container + condaInfo
+                        containerMsg = "Will execute in container: " + container + condaInfo + '\n'
                 }
                 else {
                     if(condaInfo)
@@ -2117,20 +2123,20 @@ class PipelineContext {
                 println msg
 
                 println threadsMsg
+
+                if(memoryMsg)
+                    println memoryMsg
 
                 if(containerMsg)
                     println containerMsg
-                String memoryMsg = ""
-                if(prettyCmd.contains('__bpipe_lazy_resource_memory__')) {
-                    prettyCmd = prettyCmd.replaceAll('__bpipe_lazy_resource_memory__', "${ansi().fgGreen()}\\\${memory}${ansi().fgDefault()}")
-                    memoryMsg = "${ansi().bold()}Note:${ansi().boldOff()} ${ansi().fgGreen()}\${memory}${ansi().fgDefault()} will be assigned at runtime"
+
+                if(config) {
+                    println("Config".padRight(12) + ": " + config)
                 }
-                if(memoryMsg)
-                    println memoryMsg
 
-                for(key in ['Executor','Memory','Walltime','Modules']) {
+                for(key in ['Executor','Queue','Memory','Walltime','Modules']) {
                     if(configObject.containsKey(key.toLowerCase()))
-                        println((key.padRight(12)) + ": " + configObject[key.toLowerCase()])
+                        println((key.take(11).padRight(12)) + ": " + configObject[key.toLowerCase()])
                 }
 
                 println "\n${ansi().fgBlue()}Waiting for changes or <enter> to continue ....${ansi().fgDefault()}\n"
@@ -2153,7 +2159,8 @@ class PipelineContext {
 
         log.info "Command $c.id in branch $branch failed with exit code $c.exitCode"
 
-        throw new CommandFailedException("Command $c.id in stage $stageName failed with exit status = $c.exitCode : \n\n$c.command", this, c)
+        String message = "Command $c.id in stage $stageName failed with exit status = $c.exitCode : \n\n$c.command"
+        throw new CommandFailedException(message, this, c)
       }
 
       if(!this.probeMode)

src/main/groovy/bpipe/Utils.groovy

@@ -48,6 +48,7 @@ import java.util.logging.Level;
 import java.util.logging.Logger;
 import java.util.regex.Pattern
 import org.codehaus.groovy.runtime.StackTraceUtils
+import static org.fusesource.jansi.Ansi.*
 
 import bpipe.storage.StorageLayer
 
@@ -1097,6 +1098,17 @@ class Utils {
 
             String result = (e.ctx ? " $e.ctx.stageName $branch " : "").center(width,"-") + "\n\n" + e.message + "\n"
 
+            if(e instanceof CommandFailedException) {
+                OutputLog o = e.getCommand().outputLog
+                if(o?.getLastTailLines()) {
+                    result += "\n\nTrailing lines:${ansi().fgRed()}\n\n    " + o.getLastTailLines().join('\n    ')  + ansi().fgDefault() + '\n'
+                }
+                else {
+                    result += "\nNo output recorded from command\n"
+                }
+            }
+            
+            
 //            if(branchHierarchy) {
 //                result = result + "\n$branchInfo\n"
 //            }