Merge branch 'runner-improvements'

* runner-improvements:
  Add stop on first fail option
  Provide a good err msg for invalid test executables

Author: jamesls

bin/jp-compliance

@@ -78,10 +78,11 @@ class ComplianceTestRunner(object):
         self.tests = tests
         self.jp_executable = exe
 
-    def run_tests(self):
+    def run_tests(self, stop_first_fail):
         for test_case in self._test_cases():
             if self._should_run(test_case):
-                self._run_test(test_case)
+                if not self._run_test(test_case) and stop_first_fail:
+                    return
 
     def _should_run(self, test_case):
         if not self.tests:
@@ -125,9 +126,13 @@ class ComplianceTestRunner(object):
     def _run_test(self, test_case):
         command = shlex.split(self.jp_executable)
         command.append(test_case['expression'])
-        process = subprocess.Popen(command, stdout=subprocess.PIPE,
-                                   stderr=subprocess.PIPE,
-                                   stdin=subprocess.PIPE)
+        try:
+            process = subprocess.Popen(command, stdout=subprocess.PIPE,
+                                    stderr=subprocess.PIPE,
+                                    stdin=subprocess.PIPE)
+        except Exception, e:
+            raise RuntimeError('Could not execute test executable "%s": '
+                               '%s' % (' '.join(command), e))
         process.stdin.write(json.dumps(test_case['given']))
         process.stdin.flush()
         stdout, stderr = process.communicate()
@@ -139,17 +144,21 @@ class ComplianceTestRunner(object):
             expected = test_case['result']
             if not actual == expected:
                 self._show_failure(actual, test_case)
+                return False
             else:
                 sys.stdout.write('.')
                 sys.stdout.flush()
+                return True
         else:
             error_type = test_case['error']
             # For errors, we expect the error type on stderr.
             if error_type not in stderr:
                 self._show_failure_for_error(stderr, test_case)
+                return False
             else:
                 sys.stdout.write('.')
                 sys.stdout.flush()
+                return True
 
     def _show_failure(self, actual, test_case):
         test_case['actual'] = json.dumps(actual)
@@ -214,14 +223,22 @@ def main():
                               'These values can then be used with the '
                               '"-t/--tests" argument.  If this argument is '
                               'specified, no tests will actually be run.'))
+    parser.add_argument('-s', '--stop-first-fail', action='store_true',
+                        help='Stop running tests after a single test fails.')
     args = parser.parse_args()
     runner = ComplianceTestRunner(args.exe, args.tests)
     if args.list:
         display_available_tests(runner.get_compliance_test_files())
     else:
-        runner.run_tests()
+        try:
+            runner.run_tests(args.stop_first_fail)
+        except Exception, e:
+            sys.stderr.write(str(e))
+            sys.stderr.write("\n")
+            return 1
     sys.stdout.write('\n')
+    return 0
 
 
 if __name__ == '__main__':
-    main()
+    sys.exit(main())