Merge #10053: [test] Allow functional test cases to be skipped

0c1ade6 Skip rpcbind_test if OS/network requirements are not met. (John Newbery)
232b666 Allow test cases to be skipped (John Newbery)

Tree-SHA512: d90c956ba6e27e53f422cba6267bdcc60faef9370a7e66b7f6480137f84d9a813442ac477b20fbbc540be2b4636928be910c46e221570ab3b9a5b9f0f11f7fc8

Author: MarcoFalke

test/functional/rpcbind_test.py

@@ -4,6 +4,9 @@
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
 """Test running bitcoind with the -rpcbind and -rpcallowip options."""
 
+import socket
+import sys
+
 from test_framework.test_framework import BitcoinTestFramework
 from test_framework.util import *
 from test_framework.netutil import *
@@ -52,15 +55,26 @@ def run_allowip_test(self, allow_ips, rpchost, rpcport):
 
     def run_test(self):
         # due to OS-specific network stats queries, this test works only on Linux
-        assert(sys.platform.startswith('linux'))
+        if not sys.platform.startswith('linux'):
+            self.log.warning("This test can only be run on linux. Skipping test.")
+            sys.exit(self.TEST_EXIT_SKIPPED)
         # find the first non-loopback interface for testing
         non_loopback_ip = None
         for name,ip in all_interfaces():
             if ip != '127.0.0.1':
                 non_loopback_ip = ip
                 break
         if non_loopback_ip is None:
-            assert(not 'This test requires at least one non-loopback IPv4 interface')
+            self.log.warning("This test requires at least one non-loopback IPv4 interface. Skipping test.")
+            sys.exit(self.TEST_EXIT_SKIPPED)
+        try:
+            s = socket.socket(socket.AF_INET6, socket.SOCK_DGRAM)
+            s.connect(("::1",1))
+            s.close
+        except OSError:
+            self.log.warning("This test requires IPv6 support. Skipping test.")
+            sys.exit(self.TEST_EXIT_SKIPPED)
+
         self.log.info("Using interface %s for testing" % non_loopback_ip)
 
         defaultport = rpc_port(0)

test/functional/test_framework/test_framework.py

@@ -27,9 +27,12 @@
 )
 from .authproxy import JSONRPCException
 
-
 class BitcoinTestFramework(object):
 
+    TEST_EXIT_PASSED = 0
+    TEST_EXIT_FAILED = 1
+    TEST_EXIT_SKIPPED = 77
+
     def __init__(self):
         self.num_nodes = 4
         self.setup_clean_chain = False
@@ -182,11 +185,11 @@ def main(self):
                     print("".join(deque(open(f), MAX_LINES_TO_PRINT)))
         if success:
             self.log.info("Tests successful")
-            sys.exit(0)
+            sys.exit(self.TEST_EXIT_PASSED)
         else:
             self.log.error("Test failed. Test logging available at %s/test_framework.log", self.options.tmpdir)
             logging.shutdown()
-            sys.exit(1)
+            sys.exit(self.TEST_EXIT_FAILED)
 
     def _start_logging(self):
         # Add logger and logging handlers

test/functional/test_runner.py

@@ -24,6 +24,9 @@
 import tempfile
 import re
 
+TEST_EXIT_PASSED = 0
+TEST_EXIT_SKIPPED = 77
+
 BASE_SCRIPTS= [
     # Scripts that are run by the travis build process.
     # Longest test should go first, to favor running tests in parallel
@@ -245,20 +248,20 @@ def run_tests(test_list, src_dir, build_dir, exeext, jobs=1, enable_coverage=Fal
     job_queue = TestHandler(jobs, tests_dir, test_list, flags)
 
     max_len_name = len(max(test_list, key=len))
-    results = BOLD[1] + "%s | %s | %s\n\n" % ("TEST".ljust(max_len_name), "PASSED", "DURATION") + BOLD[0]
+    results = BOLD[1] + "%s | %s | %s\n\n" % ("TEST".ljust(max_len_name), "STATUS ", "DURATION") + BOLD[0]
     for _ in range(len(test_list)):
-        (name, stdout, stderr, passed, duration) = job_queue.get_next()
-        all_passed = all_passed and passed
+        (name, stdout, stderr, status, duration) = job_queue.get_next()
+        all_passed = all_passed and status != "Failed"
         time_sum += duration
 
         print('\n' + BOLD[1] + name + BOLD[0] + ":")
-        print('' if passed else stdout + '\n', end='')
+        print('' if status == "Passed" else stdout + '\n', end='')
         print('' if stderr == '' else 'stderr:\n' + stderr + '\n', end='')
-        print("Pass: %s%s%s, Duration: %s s\n" % (BOLD[1], passed, BOLD[0], duration))
+        print("Status: %s%s%s, Duration: %s s\n" % (BOLD[1], status, BOLD[0], duration))
 
-        results += "%s | %s | %s s\n" % (name.ljust(max_len_name), str(passed).ljust(6), duration)
+        results += "%s | %s | %s s\n" % (name.ljust(max_len_name), status.ljust(7), duration)
 
-    results += BOLD[1] + "\n%s | %s | %s s (accumulated)" % ("ALL".ljust(max_len_name), str(all_passed).ljust(6), time_sum) + BOLD[0]
+    results += BOLD[1] + "\n%s | %s | %s s (accumulated)" % ("ALL".ljust(max_len_name), str(all_passed).ljust(7), time_sum) + BOLD[0]
     print(results)
     print("\nRuntime: %s s" % (int(time.time() - time0)))
 
@@ -315,10 +318,15 @@ def get_next(self):
                     log_out.seek(0), log_err.seek(0)
                     [stdout, stderr] = [l.read().decode('utf-8') for l in (log_out, log_err)]
                     log_out.close(), log_err.close()
-                    passed = stderr == "" and proc.returncode == 0
+                    if proc.returncode == TEST_EXIT_PASSED and stderr == "":
+                        status = "Passed"
+                    elif proc.returncode == TEST_EXIT_SKIPPED:
+                        status = "Skipped"
+                    else:
+                        status = "Failed"
                     self.num_running -= 1
                     self.jobs.remove(j)
-                    return name, stdout, stderr, passed, int(time.time() - time0)
+                    return name, stdout, stderr, status, int(time.time() - time0)
             print('.', end='', flush=True)