Added MC/DC

analysis/benchmark_results.py

@@ -52,8 +52,7 @@ def _get_full_path(self, filename):
 
     def get_coverage_report_path(self, fuzzer_name, benchmark_name):
         """Returns the filestore name of the |fuzzer_name|."""
-        filestore_path = coverage_data_utils.get_coverage_report_filestore_path(
-            fuzzer_name, benchmark_name, self._benchmark_df)
+        filestore_path = coverage_data_utils.get_coverage_report_filestore_path(fuzzer_name, benchmark_name, self._benchmark_df)        
         return filestore_utils.get_user_facing_path(filestore_path)
 
     @property

docker/base-image/Dockerfile

@@ -30,7 +30,17 @@ RUN apt-get update && \
     libffi-dev \
     libsqlite3-dev \
     libbz2-dev \
-    liblzma-dev
+    liblzma-dev \
+    lsb-release \
+    wget \
+    software-properties-common \
+    gnupg
+
+ENV LLVM_VERSION=19
+
+RUN wget https://apt.llvm.org/llvm.sh && \
+    chmod +x llvm.sh && \
+    ./llvm.sh ${LLVM_VERSION} all
 
 RUN cd /tmp/ && \
     curl -O https://www.python.org/ftp/python/$PYTHON_VERSION/Python-$PYTHON_VERSION.tar.xz && \

docker/benchmark-builder/Dockerfile

@@ -36,7 +36,17 @@ RUN apt-get update && \
     xz-utils \
     zlib1g-dev \
     libssl-dev \
-    libffi-dev
+    libffi-dev \
+    lsb-release \
+    wget \
+    software-properties-common \
+    gnupg
+
+ENV LLVM_VERSION=19
+
+RUN wget https://apt.llvm.org/llvm.sh && \
+    chmod +x llvm.sh && \
+    ./llvm.sh ${LLVM_VERSION} all
 
 RUN rm -rf /usr/local/bin/python3.8* /usr/local/bin/pip3 /usr/local/lib/python3.8 \
     /usr/local/include/python3.8 /usr/local/lib/python3.8/site-packages

docker/benchmark-runner/Dockerfile

@@ -47,7 +47,17 @@ RUN apt-get update -y && \
     libglib2.0-0 \
     libxml2 \
     libarchive13 \
-    libgss3
+    libgss3 \
+    lsb-release \
+    wget \
+    software-properties-common \
+    gnupg
+
+ENV LLVM_VERSION=19
+
+RUN wget https://apt.llvm.org/llvm.sh && \
+    chmod +x llvm.sh && \
+    ./llvm.sh ${LLVM_VERSION} all
 
 # Set up the directory for the build artifacts.
 ENV OUT /out

docker/dispatcher-image/Dockerfile

@@ -26,7 +26,17 @@ RUN apt-get update -y && apt-get install -y \
     libxml2 \
     libarchive13 \
     libgss3 \
-    libgl1
+    libgl1 \
+    lsb-release \
+    wget \
+    software-properties-common \
+    gnupg
+
+ENV LLVM_VERSION=19
+
+RUN wget https://apt.llvm.org/llvm.sh && \
+    chmod +x llvm.sh && \
+    ./llvm.sh ${LLVM_VERSION} all
 
 # Install docker cli.
 RUN DOCKER_VERSION=18.09.7 && \

experiment/measurer/coverage_utils.py

@@ -150,8 +150,9 @@ def generate_coverage_summary_json(self):
     def generate_coverage_report(self):
         """Generates the coverage report and stores in bucket."""
         command = [
-            'llvm-cov',
+            'llvm-cov-19',
             'show',
+            '-show-mcdc-summary',
             '-format=html',
             f'-path-equivalence=/,{self.source_files_dir}',
             f'-output-dir={self.report_dir}',
@@ -167,7 +168,6 @@ def generate_coverage_report(self):
             logger.error('Coverage report generation failed for '
                          f'fuzzer: {self.fuzzer},benchmark: {self.benchmark}.')
             return
-
         src_dir = self.report_dir
         dst_dir = exp_path.filestore(self.report_dir)
         filestore_utils.cp(src_dir, dst_dir, recursive=True, parallel=True)
@@ -224,7 +224,7 @@ def get_trial_ids(experiment: str, fuzzer: str, benchmark: str):
 
 def merge_profdata_files(src_files, dst_file):
     """Uses llvm-profdata to merge |src_files| to |dst_files|."""
-    command = ['llvm-profdata', 'merge', '-sparse']
+    command = ['llvm-profdata-19', 'merge', '-sparse']
     command.extend(src_files)
     command.extend(['-o', dst_file])
     result = new_process.execute(command, expect_zero=False)
@@ -264,7 +264,7 @@ def generate_json_summary(coverage_binary,
     """Generates the json summary file from |coverage_binary|
     and |profdata_file|."""
     command = [
-        'llvm-cov',
+        'llvm-cov-19',
         'export',
         '-format=text',
         '-num-threads=1',

experiment/measurer/test_measure_manager.py

@@ -103,7 +103,7 @@ def test_generate_profdata_create(mocked_execute, experiment, fs):
     snapshot_measurer.generate_profdata(CYCLE)
 
     expected = [
-        'llvm-profdata', 'merge', '-sparse', '/work/reports/data-123.profraw',
+        'llvm-profdata-19', 'merge', '-sparse', '/work/reports/data-123.profraw',
         '-o', '/work/reports/data.profdata'
     ]
 
@@ -126,7 +126,7 @@ def test_generate_profdata_merge(mocked_execute, experiment, fs):
     snapshot_measurer.generate_profdata(CYCLE)
 
     expected = [
-        'llvm-profdata', 'merge', '-sparse', '/work/reports/data-123.profraw',
+        'llvm-profdata-19', 'merge', '-sparse', '/work/reports/data-123.profraw',
         '/work/reports/data.profdata', '-o', '/work/reports/data.profdata'
     ]
 
@@ -153,7 +153,7 @@ def test_generate_summary(mocked_get_coverage_binary, mocked_execute,
     snapshot_measurer.generate_summary(CYCLE)
 
     expected = [
-        'llvm-cov', 'export', '-format=text', '-num-threads=1',
+        'llvm-cov-19', 'export', '-format=text', '-num-threads=1',
         '-region-coverage-gt=0', '-skip-expansions',
         '/work/coverage-binaries/benchmark-a/fuzz-target',
         '-instr-profile=/reports/data.profdata'

fuzzers/coverage/builder.Dockerfile

@@ -17,16 +17,10 @@ FROM $parent_image
 
 ENV LF_PATH /tmp/libfuzzer.zip
 
-# Use a libFuzzer version that supports clang source-based coverage.
-# This libfuzzer is 0b5e6b11c358e704384520dc036eddb5da1c68bf with
-# https://github.com/google/fuzzbench/blob/cf86138081ec705a47ce0a4bab07b5737292e7e0/fuzzers/coverage/patch.diff
-# applied.
-
-RUN wget https://storage.googleapis.com/fuzzbench-artifacts/libfuzzer-coverage.zip -O $LF_PATH && \
-    echo "cc78179f6096cae4b799d0cc9436f000cc0be9b1fb59500d16b14b1585d46b61 $LF_PATH" | sha256sum --check --status && \
-    mkdir /tmp/libfuzzer && \
+COPY libfuzzer-coverage.zip ${LF_PATH}
+RUN mkdir /tmp/libfuzzer && \
     cd /tmp/libfuzzer && \
-    unzip $LF_PATH  && \
+    unzip $LF_PATH && \
     bash build.sh && \
     cp libFuzzer.a /usr/lib && \
-    rm -rf /tmp/libfuzzer $LF_PATH
+    rm -rf /tmp/libfuzzer $LF_PATH

fuzzers/coverage/fuzzer.py

@@ -14,20 +14,29 @@
 """Integration code for clang source-based coverage builds."""
 
 import os
+import subprocess
 
 from fuzzers import utils
 
 
 def build():
     """Build benchmark."""
     cflags = [
-        '-fprofile-instr-generate', '-fcoverage-mapping', '-gline-tables-only'
+        '-fprofile-instr-generate', '-fcoverage-mapping', '-gline-tables-only', '-fcoverage-mcdc'
     ]
     utils.append_flags('CFLAGS', cflags)
     utils.append_flags('CXXFLAGS', cflags)
 
-    os.environ['CC'] = 'clang'
-    os.environ['CXX'] = 'clang++'
+    os.environ['CC'] = 'clang-19'
+    os.environ['CXX'] = 'clang++-19'
     os.environ['FUZZER_LIB'] = '/usr/lib/libFuzzer.a'
 
     utils.build_benchmark()
+
+def fuzz(input_corpus, output_corpus, target_binary):
+    """Run fuzzer. Wrapper that uses the defaults when calling
+    run_fuzzer."""
+
+    command = [target_binary, input_corpus]
+    print('[run_fuzzer] Running command: ' + ' '.join(command))
+    subprocess.check_call(command)