diff --git a/analysis/benchmark_results.py b/analysis/benchmark_results.py index a9dd8f918..f71c0c778 100644 --- a/analysis/benchmark_results.py +++ b/analysis/benchmark_results.py @@ -52,8 +52,7 @@ def _get_full_path(self, filename): def get_coverage_report_path(self, fuzzer_name, benchmark_name): """Returns the filestore name of the |fuzzer_name|.""" - filestore_path = coverage_data_utils.get_coverage_report_filestore_path( - fuzzer_name, benchmark_name, self._benchmark_df) + filestore_path = coverage_data_utils.get_coverage_report_filestore_path(fuzzer_name, benchmark_name, self._benchmark_df) return filestore_utils.get_user_facing_path(filestore_path) @property diff --git a/docker/base-image/Dockerfile b/docker/base-image/Dockerfile index 3cf0b869d..f24cb09d6 100644 --- a/docker/base-image/Dockerfile +++ b/docker/base-image/Dockerfile @@ -30,7 +30,17 @@ RUN apt-get update && \ libffi-dev \ libsqlite3-dev \ libbz2-dev \ - liblzma-dev + liblzma-dev \ + lsb-release \ + wget \ + software-properties-common \ + gnupg + +ENV LLVM_VERSION=19 + +RUN wget https://apt.llvm.org/llvm.sh && \ + chmod +x llvm.sh && \ + ./llvm.sh ${LLVM_VERSION} all RUN cd /tmp/ && \ curl -O https://www.python.org/ftp/python/$PYTHON_VERSION/Python-$PYTHON_VERSION.tar.xz && \ diff --git a/docker/benchmark-builder/Dockerfile b/docker/benchmark-builder/Dockerfile index e58b6d022..c4247f14c 100755 --- a/docker/benchmark-builder/Dockerfile +++ b/docker/benchmark-builder/Dockerfile @@ -36,7 +36,17 @@ RUN apt-get update && \ xz-utils \ zlib1g-dev \ libssl-dev \ - libffi-dev + libffi-dev \ + lsb-release \ + wget \ + software-properties-common \ + gnupg + +ENV LLVM_VERSION=19 + +RUN wget https://apt.llvm.org/llvm.sh && \ + chmod +x llvm.sh && \ + ./llvm.sh ${LLVM_VERSION} all RUN rm -rf /usr/local/bin/python3.8* /usr/local/bin/pip3 /usr/local/lib/python3.8 \ /usr/local/include/python3.8 /usr/local/lib/python3.8/site-packages diff --git a/docker/benchmark-runner/Dockerfile b/docker/benchmark-runner/Dockerfile index 5dfb45b7d..a7ac69786 100644 --- a/docker/benchmark-runner/Dockerfile +++ b/docker/benchmark-runner/Dockerfile @@ -47,7 +47,17 @@ RUN apt-get update -y && \ libglib2.0-0 \ libxml2 \ libarchive13 \ - libgss3 + libgss3 \ + lsb-release \ + wget \ + software-properties-common \ + gnupg + +ENV LLVM_VERSION=19 + +RUN wget https://apt.llvm.org/llvm.sh && \ + chmod +x llvm.sh && \ + ./llvm.sh ${LLVM_VERSION} all # Set up the directory for the build artifacts. ENV OUT /out diff --git a/docker/dispatcher-image/Dockerfile b/docker/dispatcher-image/Dockerfile index af733ea9f..cba4e9908 100644 --- a/docker/dispatcher-image/Dockerfile +++ b/docker/dispatcher-image/Dockerfile @@ -26,7 +26,17 @@ RUN apt-get update -y && apt-get install -y \ libxml2 \ libarchive13 \ libgss3 \ - libgl1 + libgl1 \ + lsb-release \ + wget \ + software-properties-common \ + gnupg + +ENV LLVM_VERSION=19 + +RUN wget https://apt.llvm.org/llvm.sh && \ + chmod +x llvm.sh && \ + ./llvm.sh ${LLVM_VERSION} all # Install docker cli. RUN DOCKER_VERSION=18.09.7 && \ diff --git a/experiment/measurer/coverage_utils.py b/experiment/measurer/coverage_utils.py index 27ab69965..bc7570027 100644 --- a/experiment/measurer/coverage_utils.py +++ b/experiment/measurer/coverage_utils.py @@ -150,8 +150,9 @@ def generate_coverage_summary_json(self): def generate_coverage_report(self): """Generates the coverage report and stores in bucket.""" command = [ - 'llvm-cov', + 'llvm-cov-19', 'show', + '-show-mcdc-summary', '-format=html', f'-path-equivalence=/,{self.source_files_dir}', f'-output-dir={self.report_dir}', @@ -167,7 +168,6 @@ def generate_coverage_report(self): logger.error('Coverage report generation failed for ' f'fuzzer: {self.fuzzer},benchmark: {self.benchmark}.') return - src_dir = self.report_dir dst_dir = exp_path.filestore(self.report_dir) filestore_utils.cp(src_dir, dst_dir, recursive=True, parallel=True) @@ -224,7 +224,7 @@ def get_trial_ids(experiment: str, fuzzer: str, benchmark: str): def merge_profdata_files(src_files, dst_file): """Uses llvm-profdata to merge |src_files| to |dst_files|.""" - command = ['llvm-profdata', 'merge', '-sparse'] + command = ['llvm-profdata-19', 'merge', '-sparse'] command.extend(src_files) command.extend(['-o', dst_file]) result = new_process.execute(command, expect_zero=False) @@ -264,7 +264,7 @@ def generate_json_summary(coverage_binary, """Generates the json summary file from |coverage_binary| and |profdata_file|.""" command = [ - 'llvm-cov', + 'llvm-cov-19', 'export', '-format=text', '-num-threads=1', diff --git a/experiment/measurer/test_measure_manager.py b/experiment/measurer/test_measure_manager.py index 7b6521869..4a9e6c1ce 100644 --- a/experiment/measurer/test_measure_manager.py +++ b/experiment/measurer/test_measure_manager.py @@ -103,7 +103,7 @@ def test_generate_profdata_create(mocked_execute, experiment, fs): snapshot_measurer.generate_profdata(CYCLE) expected = [ - 'llvm-profdata', 'merge', '-sparse', '/work/reports/data-123.profraw', + 'llvm-profdata-19', 'merge', '-sparse', '/work/reports/data-123.profraw', '-o', '/work/reports/data.profdata' ] @@ -126,7 +126,7 @@ def test_generate_profdata_merge(mocked_execute, experiment, fs): snapshot_measurer.generate_profdata(CYCLE) expected = [ - 'llvm-profdata', 'merge', '-sparse', '/work/reports/data-123.profraw', + 'llvm-profdata-19', 'merge', '-sparse', '/work/reports/data-123.profraw', '/work/reports/data.profdata', '-o', '/work/reports/data.profdata' ] @@ -153,7 +153,7 @@ def test_generate_summary(mocked_get_coverage_binary, mocked_execute, snapshot_measurer.generate_summary(CYCLE) expected = [ - 'llvm-cov', 'export', '-format=text', '-num-threads=1', + 'llvm-cov-19', 'export', '-format=text', '-num-threads=1', '-region-coverage-gt=0', '-skip-expansions', '/work/coverage-binaries/benchmark-a/fuzz-target', '-instr-profile=/reports/data.profdata' diff --git a/fuzzers/coverage/builder.Dockerfile b/fuzzers/coverage/builder.Dockerfile index 470fb57b6..9eb9e767e 100644 --- a/fuzzers/coverage/builder.Dockerfile +++ b/fuzzers/coverage/builder.Dockerfile @@ -17,16 +17,10 @@ FROM $parent_image ENV LF_PATH /tmp/libfuzzer.zip -# Use a libFuzzer version that supports clang source-based coverage. -# This libfuzzer is 0b5e6b11c358e704384520dc036eddb5da1c68bf with -# https://github.com/google/fuzzbench/blob/cf86138081ec705a47ce0a4bab07b5737292e7e0/fuzzers/coverage/patch.diff -# applied. - -RUN wget https://storage.googleapis.com/fuzzbench-artifacts/libfuzzer-coverage.zip -O $LF_PATH && \ - echo "cc78179f6096cae4b799d0cc9436f000cc0be9b1fb59500d16b14b1585d46b61 $LF_PATH" | sha256sum --check --status && \ - mkdir /tmp/libfuzzer && \ +COPY libfuzzer-coverage.zip ${LF_PATH} +RUN mkdir /tmp/libfuzzer && \ cd /tmp/libfuzzer && \ - unzip $LF_PATH && \ + unzip $LF_PATH && \ bash build.sh && \ cp libFuzzer.a /usr/lib && \ - rm -rf /tmp/libfuzzer $LF_PATH \ No newline at end of file + rm -rf /tmp/libfuzzer $LF_PATH diff --git a/fuzzers/coverage/fuzzer.py b/fuzzers/coverage/fuzzer.py index bb2fad34e..f8fb05b34 100644 --- a/fuzzers/coverage/fuzzer.py +++ b/fuzzers/coverage/fuzzer.py @@ -14,6 +14,7 @@ """Integration code for clang source-based coverage builds.""" import os +import subprocess from fuzzers import utils @@ -21,13 +22,21 @@ def build(): """Build benchmark.""" cflags = [ - '-fprofile-instr-generate', '-fcoverage-mapping', '-gline-tables-only' + '-fprofile-instr-generate', '-fcoverage-mapping', '-gline-tables-only', '-fcoverage-mcdc' ] utils.append_flags('CFLAGS', cflags) utils.append_flags('CXXFLAGS', cflags) - os.environ['CC'] = 'clang' - os.environ['CXX'] = 'clang++' + os.environ['CC'] = 'clang-19' + os.environ['CXX'] = 'clang++-19' os.environ['FUZZER_LIB'] = '/usr/lib/libFuzzer.a' utils.build_benchmark() + +def fuzz(input_corpus, output_corpus, target_binary): + """Run fuzzer. Wrapper that uses the defaults when calling + run_fuzzer.""" + + command = [target_binary, input_corpus] + print('[run_fuzzer] Running command: ' + ' '.join(command)) + subprocess.check_call(command) \ No newline at end of file diff --git a/fuzzers/coverage/libfuzzer-coverage.zip b/fuzzers/coverage/libfuzzer-coverage.zip new file mode 100644 index 000000000..e04e1a5cd Binary files /dev/null and b/fuzzers/coverage/libfuzzer-coverage.zip differ