Exp repetition test for romain

fuzzers/afl_257/builder.Dockerfile

@@ -0,0 +1,33 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+ARG parent_image
+FROM $parent_image
+
+# Download and compile AFL v2.57b.
+# Set AFL_NO_X86 to skip flaky tests.
+RUN git clone \
+        --depth 1 \
+        --branch v2.57b \
+        https://github.com/google/AFL.git /afl && \
+    cd /afl && \
+    CFLAGS= CXXFLAGS= AFL_NO_X86=1 make
+
+# Use afl_driver.cpp from LLVM as our fuzzing library.
+RUN apt-get update && \
+    apt-get install wget -y && \
+    wget https://raw.githubusercontent.com/llvm/llvm-project/5feb80e748924606531ba28c97fe65145c65372e/compiler-rt/lib/fuzzer/afl/afl_driver.cpp -O /afl/afl_driver.cpp && \
+    clang -Wno-pointer-sign -c /afl/llvm_mode/afl-llvm-rt.o.c -I/afl && \
+    clang++ -stdlib=libc++ -std=c++11 -O2 -c /afl/afl_driver.cpp && \
+    ar r /libAFL.a *.o

fuzzers/afl_random_favored/fuzzer.py → fuzzers/afl_257/fuzzer.py

@@ -48,6 +48,9 @@ def get_stats(output_corpus, fuzzer_log):  # pylint: disable=unused-argument
     """Gets fuzzer stats for AFL."""
     # Get a dictionary containing the stats AFL reports.
     stats_file = os.path.join(output_corpus, 'fuzzer_stats')
+    if not os.path.exists(stats_file):
+        print('Can\'t find fuzzer_stats')
+        return '{}'
     with open(stats_file, encoding='utf-8') as file_handle:
         stats_file_lines = file_handle.read().splitlines()
     stats_file_dict = {}

fuzzers/afl_mb/builder.Dockerfile

@@ -0,0 +1,32 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+ARG parent_image
+FROM $parent_image
+
+# Download and compile AFL v2.57b.
+# Set AFL_NO_X86 to skip flaky tests.
+RUN git clone --depth 1 \
+    https://github.com/MagicHavoc/Havoc-Study
+RUN mv Havoc-Study/fuzzers/Havoc_DMA /afl
+RUN cd /afl && \
+    CFLAGS= CXXFLAGS= AFL_NO_X86=1 make
+
+# Use afl_driver.cpp from LLVM as our fuzzing library.
+RUN apt-get update && \
+    apt-get install wget -y && \
+    wget https://raw.githubusercontent.com/llvm/llvm-project/5feb80e748924606531ba28c97fe65145c65372e/compiler-rt/lib/fuzzer/afl/afl_driver.cpp -O /afl/afl_driver.cpp && \
+    clang -Wno-pointer-sign -c /afl/llvm_mode/afl-llvm-rt.o.c -I/afl && \
+    clang++ -stdlib=libc++ -std=c++11 -O2 -c /afl/afl_driver.cpp && \
+    ar r /libAFL.a *.o

fuzzers/afl_virginmap/fuzzer.py → fuzzers/afl_mb/fuzzer.py

@@ -48,6 +48,9 @@ def get_stats(output_corpus, fuzzer_log):  # pylint: disable=unused-argument
     """Gets fuzzer stats for AFL."""
     # Get a dictionary containing the stats AFL reports.
     stats_file = os.path.join(output_corpus, 'fuzzer_stats')
+    if not os.path.exists(stats_file):
+        print('Can\'t find fuzzer_stats')
+        return '{}'
     with open(stats_file, encoding='utf-8') as file_handle:
         stats_file_lines = file_handle.read().splitlines()
     stats_file_dict = {}

fuzzers/afl_t0/builder.Dockerfile

@@ -0,0 +1,33 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+ARG parent_image
+FROM $parent_image
+
+# Download and compile AFL v2.57b.
+# Set AFL_NO_X86 to skip flaky tests.
+RUN git clone \
+        --depth 1 \
+        --branch v2.57b \
+        https://github.com/google/AFL.git /afl && \
+    cd /afl && \
+    CFLAGS= CXXFLAGS= AFL_NO_X86=1 make
+
+# Use afl_driver.cpp from LLVM as our fuzzing library.
+RUN apt-get update && \
+    apt-get install wget -y && \
+    wget https://raw.githubusercontent.com/llvm/llvm-project/5feb80e748924606531ba28c97fe65145c65372e/compiler-rt/lib/fuzzer/afl/afl_driver.cpp -O /afl/afl_driver.cpp && \
+    clang -Wno-pointer-sign -c /afl/llvm_mode/afl-llvm-rt.o.c -I/afl && \
+    clang++ -stdlib=libc++ -std=c++11 -O2 -c /afl/afl_driver.cpp && \
+    ar r /libAFL.a *.o

fuzzers/afl_t0/fuzzer.py

@@ -0,0 +1,141 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Integration code for AFL fuzzer."""
+
+import json
+import os
+import shutil
+import subprocess
+
+from fuzzers import utils
+
+
+def prepare_build_environment():
+    """Set environment variables used to build targets for AFL-based
+    fuzzers."""
+    cflags = ['-fsanitize-coverage=trace-pc-guard']
+    utils.append_flags('CFLAGS', cflags)
+    utils.append_flags('CXXFLAGS', cflags)
+
+    os.environ['CC'] = 'clang'
+    os.environ['CXX'] = 'clang++'
+    os.environ['FUZZER_LIB'] = '/libAFL.a'
+
+
+def build():
+    """Build benchmark."""
+    prepare_build_environment()
+
+    utils.build_benchmark()
+
+    print('[post_build] Copying afl-fuzz to $OUT directory')
+    # Copy out the afl-fuzz binary as a build artifact.
+    shutil.copy('/afl/afl-fuzz', os.environ['OUT'])
+
+
+def get_stats(output_corpus, fuzzer_log):  # pylint: disable=unused-argument
+    """Gets fuzzer stats for AFL."""
+    # Get a dictionary containing the stats AFL reports.
+    stats_file = os.path.join(output_corpus, 'fuzzer_stats')
+    if not os.path.exists(stats_file):
+        print('Can\'t find fuzzer_stats')
+        return '{}'
+    with open(stats_file, encoding='utf-8') as file_handle:
+        stats_file_lines = file_handle.read().splitlines()
+    stats_file_dict = {}
+    for stats_line in stats_file_lines:
+        key, value = stats_line.split(': ')
+        stats_file_dict[key.strip()] = value.strip()
+
+    # Report to FuzzBench the stats it accepts.
+    stats = {'execs_per_sec': float(stats_file_dict['execs_per_sec'])}
+    return json.dumps(stats)
+
+
+def prepare_fuzz_environment(input_corpus):
+    """Prepare to fuzz with AFL or another AFL-based fuzzer."""
+    # Tell AFL to not use its terminal UI so we get usable logs.
+    os.environ['AFL_NO_UI'] = '1'
+    # Skip AFL's CPU frequency check (fails on Docker).
+    os.environ['AFL_SKIP_CPUFREQ'] = '1'
+    # No need to bind affinity to one core, Docker enforces 1 core usage.
+    os.environ['AFL_NO_AFFINITY'] = '1'
+    # AFL will abort on startup if the core pattern sends notifications to
+    # external programs. We don't care about this.
+    os.environ['AFL_I_DONT_CARE_ABOUT_MISSING_CRASHES'] = '1'
+    # Don't exit when crashes are found. This can happen when corpus from
+    # OSS-Fuzz is used.
+    os.environ['AFL_SKIP_CRASHES'] = '1'
+    # Shuffle the queue
+    os.environ['AFL_SHUFFLE_QUEUE'] = '1'
+
+    # AFL needs at least one non-empty seed to start.
+    utils.create_seed_file_for_empty_corpus(input_corpus)
+
+
+def check_skip_det_compatible(additional_flags):
+    """ Checks if additional flags are compatible with '-d' option"""
+    # AFL refuses to take in '-d' with '-M' or '-S' options for parallel mode.
+    # (cf. https://github.com/google/AFL/blob/8da80951/afl-fuzz.c#L7477)
+    if '-M' in additional_flags or '-S' in additional_flags:
+        return False
+    return True
+
+
+def run_afl_fuzz(input_corpus,
+                 output_corpus,
+                 target_binary,
+                 additional_flags=None,
+                 hide_output=False):
+    """Run afl-fuzz."""
+    # Spawn the afl fuzzing process.
+    print('[run_afl_fuzz] Running target with afl-fuzz')
+    command = [
+        './afl-fuzz',
+        '-i',
+        input_corpus,
+        '-o',
+        output_corpus,
+        # Use no memory limit as ASAN doesn't play nicely with one.
+        '-m',
+        'none',
+        '-t',
+        '1000+',  # Use same default 1 sec timeout, but add '+' to skip hangs.
+    ]
+    # Use '-d' to skip deterministic mode, as long as it it compatible with
+    # additional flags.
+    if not additional_flags or check_skip_det_compatible(additional_flags):
+        command.append('-d')
+    if additional_flags:
+        command.extend(additional_flags)
+    dictionary_path = utils.get_dictionary_path(target_binary)
+    if dictionary_path:
+        command.extend(['-x', dictionary_path])
+    command += [
+        '--',
+        target_binary,
+        # Pass INT_MAX to afl the maximize the number of persistent loops it
+        # performs.
+        '2147483647'
+    ]
+    print('[run_afl_fuzz] Running command: ' + ' '.join(command))
+    output_stream = subprocess.DEVNULL if hide_output else None
+    subprocess.check_call(command, stdout=output_stream, stderr=output_stream)
+
+
+def fuzz(input_corpus, output_corpus, target_binary):
+    """Run afl-fuzz on target."""
+    prepare_fuzz_environment(input_corpus)
+
+    run_afl_fuzz(input_corpus, output_corpus, target_binary)

fuzzers/symcc_afl/runner.Dockerfile → fuzzers/afl_t0/runner.Dockerfile

@@ -1,4 +1,4 @@
-# Copyright 2021 Google LLC
+# Copyright 2020 Google LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -13,5 +13,3 @@
 # limitations under the License.
 
 FROM gcr.io/fuzzbench/base-image
-
-ENV LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/out"

fuzzers/afl_t1/builder.Dockerfile

@@ -0,0 +1,33 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+ARG parent_image
+FROM $parent_image
+
+# Download and compile AFL v2.57b.
+# Set AFL_NO_X86 to skip flaky tests.
+RUN git clone \
+        --depth 1 \
+        --branch v2.57b \
+        https://github.com/google/AFL.git /afl && \
+    cd /afl && \
+    CFLAGS= CXXFLAGS= AFL_NO_X86=1 make
+
+# Use afl_driver.cpp from LLVM as our fuzzing library.
+RUN apt-get update && \
+    apt-get install wget -y && \
+    wget https://raw.githubusercontent.com/llvm/llvm-project/5feb80e748924606531ba28c97fe65145c65372e/compiler-rt/lib/fuzzer/afl/afl_driver.cpp -O /afl/afl_driver.cpp && \
+    clang -Wno-pointer-sign -c /afl/llvm_mode/afl-llvm-rt.o.c -I/afl && \
+    clang++ -stdlib=libc++ -std=c++11 -O2 -c /afl/afl_driver.cpp && \
+    ar r /libAFL.a *.o