Fix the livy 0.4 submission log empty output line for stderr/yarn diag

Signed-off-by: Wei Zhang <[email protected]>

Author: wezhang

Utils/hdinsight-node-common/src/com/microsoft/azure/hdinsight/spark/common/SparkBatchJob.java

@@ -665,6 +665,13 @@ public String getSparkDriverHost() throws IOException {
     }
 
     public Observable<SimpleImmutableEntry<MessageInfoType, String>> getSubmissionLog() {
+        // Those lines are carried per response,
+        // if there is no value followed, the line should not be sent to console
+        final Set<String> ignoredEmptyLines = new HashSet<>(Arrays.asList(
+                "stdout:",
+                "stderr:",
+                "yarn diagnostics:"));
+
         return Observable.create(ob -> {
             try {
                 int start = 0;
@@ -685,7 +692,9 @@ public Observable<SimpleImmutableEntry<MessageInfoType, String>> getSubmissionLo
                                     "Bad spark log response: " + httpResponse.getMessage()));
 
                     // To subscriber
-                    sparkJobLog.getLog().forEach(line -> ob.onNext(new SimpleImmutableEntry<>(Log, line)));
+                    sparkJobLog.getLog().stream()
+                            .filter(line -> !ignoredEmptyLines.contains(line.trim().toLowerCase()))
+                            .forEach(line -> ob.onNext(new SimpleImmutableEntry<>(Log, line)));
 
                     linesGot = sparkJobLog.getLog().size();
                     start += linesGot;