Merge #15295: fuzz: Add test/fuzz/test_runner.py and run it in travis

fa535af fuzz: test_runner: Better error message when built with afl (MarcoFalke)
fa7ca8e qa: Add test/fuzz/test_runner.py (MarcoFalke)

Pull request description:

  Can be run with `./test/fuzz/test_runner.py` after building as described in `doc/fuzzing.md`

Tree-SHA512: f6a3cd8165ec2de4b363be4fd0a936b4a60829cce923f93fe5d6a046b1bbd64c959cdf790440bf70c0e13b0bb1b956a746a24c6fd92bddeab15b837ed50ffad2

Author: MarcoFalke

.travis.yml

@@ -15,6 +15,7 @@ env:
     - MAKEJOBS=-j3
     - RUN_UNIT_TESTS=true
     - RUN_FUNCTIONAL_TESTS=true
+    - RUN_FUZZ_TESTS=false
     - DOCKER_NAME_TAG=ubuntu:18.04
     - BOOST_TEST_RANDOM=1$TRAVIS_BUILD_ID
     - CCACHE_SIZE=100M
@@ -100,7 +101,7 @@ jobs:
         PACKAGES="python3-zmq qtbase5-dev qttools5-dev-tools protobuf-compiler libdbus-1-dev libharfbuzz-dev libprotobuf-dev"
         DEP_OPTS="NO_QT=1 NO_UPNP=1 DEBUG=1 ALLOW_HOST_PACKAGES=1"
         GOAL="install"
-        BITCOIN_CONFIG="--enable-zmq --with-gui=qt5 --enable-fuzz --enable-glibc-back-compat --enable-reduce-exports --enable-debug CXXFLAGS=\"-g0 -O2\""
+        BITCOIN_CONFIG="--enable-zmq --with-gui=qt5 --enable-glibc-back-compat --enable-reduce-exports --enable-debug CXXFLAGS=\"-g0 -O2\""
 
     - stage: test
       name: 'x86_64 Linux  [GOAL: install]  [trusty]  [no functional tests, no depends, only system libs]'
@@ -132,6 +133,18 @@ jobs:
         GOAL="install"
         BITCOIN_CONFIG="--enable-zmq --with-incompatible-bdb --with-gui=qt5 CPPFLAGS=-DDEBUG_LOCKORDER --with-sanitizers=address,integer,undefined CC=clang CXX=clang++"
 
+    - stage: test
+      name: 'x86_64 Linux  [GOAL: install]  [bionic]  [no depends, only system libs, sanitizers: fuzzer,address]'
+      env: >-
+        HOST=x86_64-unknown-linux-gnu
+        PACKAGES="clang llvm python3 libssl1.0-dev libevent-dev bsdmainutils libboost-system-dev libboost-filesystem-dev libboost-chrono-dev libboost-test-dev libboost-thread-dev"
+        NO_DEPENDS=1
+        RUN_UNIT_TESTS=false
+        RUN_FUNCTIONAL_TESTS=false
+        RUN_FUZZ_TESTS=true
+        GOAL="install"
+        BITCOIN_CONFIG="--disable-wallet --disable-bench --with-utils=no --with-daemon=no --with-libs=no --with-gui=no --enable-fuzz --with-sanitizers=fuzzer,address CC=clang CXX=clang++"
+
     - stage: test
       name: 'x86_64 Linux  [GOAL: install]  [bionic]  [no wallet]'
       env: >-

.travis/test_04_install.sh

@@ -7,6 +7,11 @@
 export LC_ALL=C.UTF-8
 
 travis_retry docker pull "$DOCKER_NAME_TAG"
+
+export DIR_FUZZ_IN=${TRAVIS_BUILD_DIR}/qa-assets
+git clone https://github.com/bitcoin-core/qa-assets ${DIR_FUZZ_IN}
+export DIR_FUZZ_IN=${DIR_FUZZ_IN}/fuzz_seed_corpus/
+
 mkdir -p "${TRAVIS_BUILD_DIR}/sanitizer-output/"
 export ASAN_OPTIONS=""
 export LSAN_OPTIONS="suppressions=${TRAVIS_BUILD_DIR}/test/sanitizer_suppressions/lsan"

.travis/test_06_script_b.sh

@@ -19,3 +19,9 @@ if [ "$RUN_FUNCTIONAL_TESTS" = "true" ]; then
   DOCKER_EXEC test/functional/test_runner.py --ci --combinedlogslen=4000 --coverage --quiet --failfast
   END_FOLD
 fi
+
+if [ "$RUN_FUZZ_TESTS" = "true" ]; then
+  BEGIN_FOLD fuzz-tests
+  DOCKER_EXEC test/fuzz/test_runner.py -l DEBUG ${DIR_FUZZ_IN}
+  END_FOLD
+fi

Makefile.am

@@ -225,7 +225,11 @@ endif
 
 dist_noinst_SCRIPTS = autogen.sh
 
-EXTRA_DIST = $(DIST_SHARE) test/functional/test_runner.py test/functional $(DIST_CONTRIB) $(DIST_DOCS) $(WINDOWS_PACKAGING) $(OSX_PACKAGING) $(BIN_CHECKS)
+EXTRA_DIST = $(DIST_SHARE) $(DIST_CONTRIB) $(DIST_DOCS) $(WINDOWS_PACKAGING) $(OSX_PACKAGING) $(BIN_CHECKS)
+
+EXTRA_DIST += \
+    test/functional \
+    test/fuzz
 
 EXTRA_DIST += \
     test/util/bitcoin-util-test.py \

doc/fuzzing.md

@@ -5,6 +5,29 @@ A special test harness in `src/test/fuzz/` is provided for each fuzz target to
 provide an easy entry point for fuzzers and the like. In this document we'll
 describe how to use it with AFL and libFuzzer.
 
+## Preparing fuzzing
+
+AFL needs an input directory with examples, and an output directory where it
+will place examples that it found. These can be anywhere in the file system,
+we'll define environment variables to make it easy to reference them.
+
+libFuzzer will use the input directory as output directory.
+
+Extract the example seeds (or other starting inputs) into the inputs
+directory before starting fuzzing.
+
+```
+git clone https://github.com/bitcoin-core/qa-assets
+export DIR_FUZZ_IN=$PWD/qa-assets/fuzz_seed_corpus
+```
+
+Only for AFL:
+
+```
+mkdir outputs
+export AFLOUT=$PWD/outputs
+```
+
 ## AFL
 
 ### Building AFL
@@ -23,7 +46,7 @@ export AFLPATH=$PWD
 To build Bitcoin Core using AFL instrumentation (this assumes that the
 `AFLPATH` was set as above):
 ```
-./configure --disable-ccache --disable-shared --enable-tests --enable-fuzz CC=${AFLPATH}/afl-gcc CXX=${AFLPATH}/afl-g++
+./configure --disable-ccache --disable-shared --enable-tests --enable-fuzz --disable-wallet --disable-bench --with-utils=no --with-daemon=no --with-libs=no --with-gui=no CC=${AFLPATH}/afl-gcc CXX=${AFLPATH}/afl-g++
 export AFL_HARDEN=1
 cd src/
 make
@@ -39,31 +62,14 @@ binary will be instrumented in such a way that the AFL
 features "persistent mode" and "deferred forkserver" can be used. See
 https://github.com/mcarpenter/afl/tree/master/llvm_mode for details.
 
-### Preparing fuzzing
-
-AFL needs an input directory with examples, and an output directory where it
-will place examples that it found. These can be anywhere in the file system,
-we'll define environment variables to make it easy to reference them.
-
-```
-mkdir inputs
-AFLIN=$PWD/inputs
-mkdir outputs
-AFLOUT=$PWD/outputs
-```
-
-Example inputs are available from:
-
-- https://download.visucore.com/bitcoin/bitcoin_fuzzy_in.tar.xz
-- http://strateman.ninja/fuzzing.tar.xz
-
-Extract these (or other starting inputs) into the `inputs` directory before starting fuzzing.
-
 ### Fuzzing
 
 To start the actual fuzzing use:
+
 ```
-$AFLPATH/afl-fuzz -i ${AFLIN} -o ${AFLOUT} -m52 -- test/fuzz/fuzz_target_foo
+export FUZZ_TARGET=fuzz_target_foo  # Pick a fuzz_target
+mkdir ${AFLOUT}/${FUZZ_TARGET}
+$AFLPATH/afl-fuzz -i ${DIR_FUZZ_IN}/${FUZZ_TARGET} -o ${AFLOUT}/${FUZZ_TARGET} -m52 -- test/fuzz/${FUZZ_TARGET}
 ```
 
 You may have to change a few kernel parameters to test optimally - `afl-fuzz`
@@ -74,10 +80,10 @@ will print an error and suggestion if so.
 A recent version of `clang`, the address sanitizer and libFuzzer is needed (all
 found in the `compiler-rt` runtime libraries package).
 
-To build the `test/test_bitcoin_fuzzy` executable run
+To build all fuzz targets with libFuzzer, run
 
 ```
-./configure --disable-ccache --enable-fuzz --with-sanitizers=fuzzer,address CC=clang CXX=clang++
+./configure --disable-ccache --disable-wallet --disable-bench --with-utils=no --with-daemon=no --with-libs=no --with-gui=no --enable-fuzz --with-sanitizers=fuzzer,address CC=clang CXX=clang++
 make
 ```
 
@@ -86,3 +92,6 @@ interchangeably between libFuzzer and AFL.
 
 See https://llvm.org/docs/LibFuzzer.html#running on how to run the libFuzzer
 instrumented executable.
+
+Alternatively run the script in `./test/fuzz/test_runner.py` and provide it
+with the `${DIR_FUZZ_IN}` created earlier.

src/Makefile.test.include

@@ -2,7 +2,6 @@
 # Distributed under the MIT software license, see the accompanying
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
 
-bin_PROGRAMS += test/test_bitcoin
 
 FUZZ_TARGETS = \
   test/fuzz/address_deserialize \
@@ -28,6 +27,8 @@ FUZZ_TARGETS = \
 
 if ENABLE_FUZZ
 noinst_PROGRAMS += $(FUZZ_TARGETS:=)
+else
+bin_PROGRAMS += test/test_bitcoin
 endif
 
 TEST_SRCDIR = test

test/config.ini.in

@@ -16,4 +16,5 @@ RPCAUTH=@abs_top_srcdir@/share/rpcauth/rpcauth.py
 @ENABLE_WALLET_TRUE@ENABLE_WALLET=true
 @BUILD_BITCOIN_CLI_TRUE@ENABLE_CLI=true
 @BUILD_BITCOIND_TRUE@ENABLE_BITCOIND=true
+@ENABLE_FUZZ_TRUE@ENABLE_FUZZ=true
 @ENABLE_ZMQ_TRUE@ENABLE_ZMQ=true

test/fuzz/test_runner.py

@@ -0,0 +1,138 @@
+#!/usr/bin/env python3
+# Copyright (c) 2019 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+"""Run fuzz test targets.
+"""
+
+import argparse
+import configparser
+import os
+import sys
+import subprocess
+import logging
+
+
+def main():
+    parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+    parser.add_argument(
+        "-l",
+        "--loglevel",
+        dest="loglevel",
+        default="INFO",
+        help="log events at this level and higher to the console. Can be set to DEBUG, INFO, WARNING, ERROR or CRITICAL. Passing --loglevel DEBUG will output all logs to console.",
+    )
+    parser.add_argument(
+        '--export_coverage',
+        action='store_true',
+        help='If true, export coverage information to files in the seed corpus',
+    )
+    parser.add_argument(
+        'seed_dir',
+        help='The seed corpus to run on (must contain subfolders for each fuzz target).',
+    )
+    parser.add_argument(
+        'target',
+        nargs='*',
+        help='The target(s) to run. Default is to run all targets.',
+    )
+
+    args = parser.parse_args()
+
+    # Set up logging
+    logging.basicConfig(
+        format='%(message)s',
+        level=int(args.loglevel) if args.loglevel.isdigit() else args.loglevel.upper(),
+    )
+
+    # Read config generated by configure.
+    config = configparser.ConfigParser()
+    configfile = os.path.abspath(os.path.dirname(__file__)) + "/../config.ini"
+    config.read_file(open(configfile, encoding="utf8"))
+
+    if not config["components"].getboolean("ENABLE_FUZZ"):
+        logging.error("Must have fuzz targets built")
+        sys.exit(1)
+
+    # Build list of tests
+    test_list_all = parse_test_list(makefile=os.path.join(config["environment"]["SRCDIR"], 'src', 'Makefile.test.include'))
+
+    if not test_list_all:
+        logging.error("No fuzz targets found")
+        sys.exit(1)
+
+    logging.info("Fuzz targets found: {}".format(test_list_all))
+
+    args.target = args.target or test_list_all  # By default run all
+    test_list_error = list(set(args.target).difference(set(test_list_all)))
+    if test_list_error:
+        logging.error("Unknown fuzz targets selected: {}".format(test_list_error))
+    test_list_selection = list(set(test_list_all).intersection(set(args.target)))
+    if not test_list_selection:
+        logging.error("No fuzz targets selected")
+    logging.info("Fuzz targets selected: {}".format(test_list_selection))
+
+    try:
+        help_output = subprocess.run(
+            args=[
+                os.path.join(config["environment"]["BUILDDIR"], 'src', 'test', 'fuzz', test_list_selection[0]),
+                '-help=1',
+            ],
+            timeout=1,
+            check=True,
+            stderr=subprocess.PIPE,
+            universal_newlines=True,
+        ).stderr
+        if "libFuzzer" not in help_output:
+            logging.error("Must be built with libFuzzer")
+            sys.exit(1)
+    except subprocess.TimeoutExpired:
+        logging.error("subprocess timed out: Currently only libFuzzer is supported")
+        sys.exit(1)
+
+    run_once(
+        corpus=args.seed_dir,
+        test_list=test_list_selection,
+        build_dir=config["environment"]["BUILDDIR"],
+        export_coverage=args.export_coverage,
+    )
+
+
+def run_once(*, corpus, test_list, build_dir, export_coverage):
+    for t in test_list:
+        args = [
+            os.path.join(build_dir, 'src', 'test', 'fuzz', t),
+            '-runs=1',
+            os.path.join(corpus, t),
+        ]
+        logging.debug('Run {} with args {}'.format(t, args))
+        output = subprocess.run(args, check=True, stderr=subprocess.PIPE, universal_newlines=True).stderr
+        logging.debug('Output: {}'.format(output))
+        if not export_coverage:
+            continue
+        for l in output.splitlines():
+            if 'INITED' in l:
+                with open(os.path.join(corpus, t + '_coverage'), 'w', encoding='utf-8') as cov_file:
+                    cov_file.write(l)
+                    break
+
+
+def parse_test_list(makefile):
+    with open(makefile, encoding='utf-8') as makefile_test:
+        test_list_all = []
+        read_targets = False
+        for line in makefile_test.readlines():
+            line = line.strip().replace('test/fuzz/', '').replace(' \\', '')
+            if read_targets:
+                if not line:
+                    break
+                test_list_all.append(line)
+                continue
+
+            if line == 'FUZZ_TARGETS =':
+                read_targets = True
+    return test_list_all
+
+
+if __name__ == '__main__':
+    main()