Merge #18166: ci: Run fuzz testing test cases (bitcoin-core/qa-assets) under valgrind to catch memory errors

f2472f6 tests: Improve test runner output in case of target errors (practicalswift)
733bbec tests: Add --exclude integer,parse_iso8601 (temporarily) to make Travis pass until uninitialized read issue in FormatISO8601DateTime is fixed (practicalswift)
5ea8144 tests: Add support for excluding fuzz targets using -x/--exclude (practicalswift)
555236f tests: Remove -detect_leaks=0 from test/fuzz/test_runner.py - no longer needed (practicalswift)
a3b539a ci: Run fuzz testing test cases under valgrind (practicalswift)

Pull request description:

  Run fuzz testing [test cases (bitcoin-core/qa-assets)](https://github.com/bitcoin-core/qa-assets) under `valgrind`.

  This would have caught `util: Avoid potential uninitialized read in FormatISO8601DateTime(int64_t) by checking gmtime_s/gmtime_r return value` (#18162) and similar cases.

ACKs for top commit:
  MarcoFalke:
    ACK f2472f6 👼

Tree-SHA512: bb0879d40167cf6906bc0ed31bed39db83c39c7beb46026f7b0ee53f28ff0526ad6fabc3f4cb3f5f18d3b8cafdcbf5f30105b35919f4e83697c71e838ed71493

Author: MarcoFalke

.travis.yml

@@ -134,6 +134,11 @@ jobs:
       env: >-
         FILE_ENV="./ci/test/00_setup_env_native_fuzz.sh"
 
+    - stage: test
+      name: 'x86_64 Linux  [GOAL: install]  [bionic]  [no depends, only system libs, fuzzers under valgrind]'
+      env: >-
+        FILE_ENV="./ci/test/00_setup_env_native_fuzz_with_valgrind.sh"
+
     - stage: test
       name: 'x86_64 Linux  [GOAL: install]  [bionic]  [no wallet]'
       env: >-

ci/test/00_setup_env_native_fuzz_with_valgrind.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+#
+# Copyright (c) 2019 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+export LC_ALL=C.UTF-8
+
+export CONTAINER_NAME=ci_native_fuzz_valgrind
+export PACKAGES="clang-8 llvm-8 python3 libevent-dev bsdmainutils libboost-system-dev libboost-filesystem-dev libboost-chrono-dev libboost-test-dev libboost-thread-dev valgrind"
+export NO_DEPENDS=1
+export RUN_UNIT_TESTS=false
+export RUN_FUNCTIONAL_TESTS=false
+export RUN_FUZZ_TESTS=true
+export FUZZ_TESTS_CONFIG="--exclude integer,parse_iso8601 --valgrind"
+export GOAL="install"
+export BITCOIN_CONFIG="--enable-fuzz --with-sanitizers=fuzzer CC=clang-8 CXX=clang++-8"
+# Use clang-8, instead of default clang on bionic, which is clang-6 and does not come with libfuzzer on aarch64

ci/test/06_script_b.sh

@@ -36,6 +36,6 @@ fi
 
 if [ "$RUN_FUZZ_TESTS" = "true" ]; then
   BEGIN_FOLD fuzz-tests
-  DOCKER_EXEC test/fuzz/test_runner.py -l DEBUG ${DIR_FUZZ_IN}
+  DOCKER_EXEC test/fuzz/test_runner.py ${FUZZ_TESTS_CONFIG} -l DEBUG ${DIR_FUZZ_IN}
   END_FOLD
 fi

test/fuzz/test_runner.py

@@ -47,6 +47,7 @@
     "tx_out",
 ]
 
+
 def main():
     parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
     parser.add_argument(
@@ -64,7 +65,12 @@ def main():
     parser.add_argument(
         '--valgrind',
         action='store_true',
-        help='If true, run fuzzing binaries under the valgrind memory error detector. Valgrind 3.14 or later required.',
+        help='If true, run fuzzing binaries under the valgrind memory error detector',
+    )
+    parser.add_argument(
+        '-x',
+        '--exclude',
+        help="A comma-separated list of targets to exclude",
     )
     parser.add_argument(
         'seed_dir',
@@ -100,7 +106,7 @@ def main():
         logging.error("No fuzz targets found")
         sys.exit(1)
 
-    logging.info("Fuzz targets found: {}".format(test_list_all))
+    logging.debug("{} fuzz target(s) found: {}".format(len(test_list_all), " ".join(sorted(test_list_all))))
 
     args.target = args.target or test_list_all  # By default run all
     test_list_error = list(set(args.target).difference(set(test_list_all)))
@@ -109,7 +115,15 @@ def main():
     test_list_selection = list(set(test_list_all).intersection(set(args.target)))
     if not test_list_selection:
         logging.error("No fuzz targets selected")
-    logging.info("Fuzz targets selected: {}".format(test_list_selection))
+    if args.exclude:
+        for excluded_target in args.exclude.split(","):
+            if excluded_target not in test_list_selection:
+                logging.error("Target \"{}\" not found in current target list.".format(excluded_target))
+                continue
+            test_list_selection.remove(excluded_target)
+    test_list_selection.sort()
+
+    logging.info("{} of {} detected fuzz target(s) selected: {}".format(len(test_list_selection), len(test_list_all), " ".join(test_list_selection)))
 
     try:
         help_output = subprocess.run(
@@ -146,16 +160,23 @@ def run_once(*, corpus, test_list, build_dir, export_coverage, use_valgrind):
         args = [
             os.path.join(build_dir, 'src', 'test', 'fuzz', t),
             '-runs=1',
-            '-detect_leaks=0',
             corpus_path,
         ]
         if use_valgrind:
-            args = ['valgrind', '--quiet', '--error-exitcode=1', '--exit-on-first-error=yes'] + args
+            args = ['valgrind', '--quiet', '--error-exitcode=1'] + args
         logging.debug('Run {} with args {}'.format(t, args))
         result = subprocess.run(args, stderr=subprocess.PIPE, universal_newlines=True)
         output = result.stderr
         logging.debug('Output: {}'.format(output))
-        result.check_returncode()
+        try:
+            result.check_returncode()
+        except subprocess.CalledProcessError as e:
+            if e.stdout:
+                logging.info(e.stdout)
+            if e.stderr:
+                logging.info(e.stderr)
+            logging.info("Target \"{}\" failed with exit code {}: {}".format(t, e.returncode, " ".join(args)))
+            sys.exit(1)
         if not export_coverage:
             continue
         for l in output.splitlines():