Merge pull request apache-spark-on-k8s#330 from palantir/ds/merge-test-results-scala

Cache & merge scala test results, bump parallelism

Author: dansanduleac

.circleci/config.yml

@@ -12,6 +12,7 @@ test-defaults: &test-defaults
   <<: *defaults
   environment:
     CIRCLE_TEST_REPORTS: /tmp/circle-test-reports
+    TEST_RESULTS_FILE: /tmp/test-results/results.json
 
 
 all-branches-and-tags: &all-branches-and-tags
@@ -239,7 +240,7 @@ jobs:
   run-scala-tests:
     <<: *test-defaults
     # project/CirclePlugin.scala does its own test splitting in SBT based on CIRCLE_NODE_INDEX, CIRCLE_NODE_TOTAL
-    parallelism: 6
+    parallelism: 12
     # Spark runs a lot of tests in parallel, we need 16 GB of RAM for this
     resource_class: xlarge
     steps:
@@ -258,11 +259,42 @@ jobs:
       - restore_cache:
           keys:
             - v2-home-sbt-{{ checksum "build/sbt" }}-{{ checksum "project/target/streams/$global/update/$global/streams/update_cache_2.10/inputs" }}
+      - restore_cache:
+          keys:
+            - v1-test-results-{{ .Branch }}-{{ .BuildNum }}
+            - v1-test-results-{{ .Branch }}-
+            - v1-test-results-master-
+      - run:
+          name: Merge cached test results with results provided by circle (if any)
+          command: |
+            [[ -f "$CIRCLE_INTERNAL_TASK_DATA/circle-test-results/results.json" ]] && circle_exists=1
+            [[ -f "$TEST_RESULTS_FILE" ]] && cached_exists=1
+            if [[ -z "$circle_exists" ]] || [[ -z "$cached_exists" ]]; then
+              # Only one exists, CirclePlugin will try to look for both so we're good.
+              exit 0
+            fi
+
+            # Otherwise, combine the two, preferring newer results from circle test results
+            echo "Found both cached and circle test results, merging"
+            jq -s 'def meld:
+               reduce .[] as $o
+                 ({}; reduce ($o|keys)[] as $key (.; .[$key] += [$o[$key]] ));
+              def grp: map([{key: "\(.source)~\(.classname)", value: .}] | from_entries) | meld;
+              ((.[0].tests | grp) * (.[1].tests | grp)) | map(values) | flatten | {tests: .}' \
+                "$TEST_RESULTS_FILE" \
+                "$CIRCLE_INTERNAL_TASK_DATA/circle-test-results/results.json" \
+                > /tmp/new-test-results.json
+            # Overwrite the previously cached results with the merged file
+            mv -f /tmp/new-test-results.json "$TEST_RESULTS_FILE"
       - run:
           name: Run all tests
           command: ./dev/run-scala-tests.py \
               | tee -a "/tmp/run-scala-tests.log"
           no_output_timeout: 15m
+      - save_cache:
+          key: v1-test-results-{{ .Branch }}-{{ .BuildNum }}
+          paths:
+            - "/tmp/test-results"
       - store_artifacts:
           path: /tmp/run-scala-tests.log
           destination: run-scala-tests.log

project/CirclePlugin.scala

@@ -77,15 +77,20 @@ object CirclePlugin extends AutoPlugin {
         log.info(s"Discovered tests in these projects: ${byProject.map(_.project.project)}")
 
         import collection.JavaConverters._
-        // Get timings and sum them up by TestKey = (source, classname)
-        val testResultsFile = sys.env.get("CIRCLE_INTERNAL_TASK_DATA")
+        val fromCircle = sys.env.get("CIRCLE_INTERNAL_TASK_DATA")
             .map(taskData => file(taskData) / "circle-test-results/results.json")
-            .filter(file => file.exists())
+        val fromCached = sys.env.get("TEST_RESULTS_FILE").map(file)
+
+        val testResultsFile = List(fromCached, fromCircle)
+            .collectFirst {
+              case Some(file) if file.exists() =>
+                log.info(s"Using circle test results to determine test packing: $file")
+                file
+            }
+        // Get timings and sum them up by TestKey = (source, classname)
         val testTimings = try {
-          testResultsFile.fold {
-            log.warn("Couldn't find circle test results file, using naive test packing")
-          } {
-            file => log.info(s"Using circle test results to determine test packing: $file")
+          if (testResultsFile.isEmpty) {
+            log.warn("Couldn't find any circle test results file, using naive test packing")
           }
           testResultsFile
               .map(file => {