diff --git a/infra/experimental/chronos/README.md b/infra/experimental/chronos/README.md index 3c29d16c8412..67c59f71bd61 100644 --- a/infra/experimental/chronos/README.md +++ b/infra/experimental/chronos/README.md @@ -1,5 +1,34 @@ # Chronos: rebuilding OSS-Fuzz harnesses using cached builds +## Pre-built images. + +Daily pre-built images are available at: + +- `us-central1-docker.pkg.dev/oss-fuzz/oss-fuzz-gen/-ofg-cached-address` +- `us-central1-docker.pkg.dev/oss-fuzz/oss-fuzz-gen/-ofg-cached-coverage` + +They can be used as drop-in replacements for the usual `gcr.io/oss-fuzz/` images. + +These images are generated in 2 ways: +- (Preferred) [Generate](https://github.com/google/oss-fuzz/blob/master/infra/base-images/base-builder/bash_parser.py) + a replay build script that can be re-run alongside existing build artifacts, + leveraging existing build system mechanisms to avoid rebuilding (e.g. running + `make` twice should not actually rebuild everything). This is error-prone, so + we validate the script works by running it. +- (Fallback, if the replay build script didn't work). We leverage + [ccache](https://ccache.dev/), to provide a compiler cache. This is often not + as fast as the replay build script, because some project builds spend + significant time doing non-compiler tasks (e.g. checking out submodules, + running configure scripts). + +Note: this mechanism does not work for every single OSS-Fuzz project today. The +resulting image may either: +- Not provide much performance improvement compared with a normal image, or +- Not exist at all (if neither approach worked). + +Stats from a recent run: +(Feb 3 2025). + ## Usage locally **Example 1: htslib** diff --git a/infra/experimental/chronos/build_cache_local.sh b/infra/experimental/chronos/build_cache_local.sh index 64c203381f45..9b201832aded 100755 --- a/infra/experimental/chronos/build_cache_local.sh +++ b/infra/experimental/chronos/build_cache_local.sh @@ -97,17 +97,16 @@ REPLAY_WORKED= # If this step is successful, then the process can exit as it's ready. if [[ "$executables_replay" == "$executables_vanilla" ]] then - echo "Replay worked" - echo "Vanilla compile time: ${B_TIME}" - echo "Replay compile time: ${R_TIME}" - REPLAY_WORKED=1 if [ -z "${RUN_ALL+1}" ]; then + echo "${_PROJECT}: Replay worked." + echo "${_PROJECT}: Compile times: Vanilla=${B_TIME}; Replay=${R_TIME};" exit 0 fi else - echo "Replay did not work" + echo "${_PROJECT}: Replay did not work" + R_TIME="N/A" fi # Step 8: prepare Dockerfile for ccache @@ -145,17 +144,14 @@ executables_ccache="$(find ./build/out/${_PROJECT}/ -executable -type f | sort)" # Step 12: validate the ccache builds are successful if [[ "$executables_ccache" == "$executables_vanilla" ]] then - echo "Vanilla compile time: ${B_TIME}" - if [[ "$executables_replay" == "$executables_vanilla" ]] - then - echo "Replay worked" - echo "Replay compile time: ${R_TIME}" - fi + echo "${_PROJECT}: Compile times: Vanilla=${B_TIME}; Replay=${R_TIME}; CCache=${A_TIME};" - echo "Ccache compile time: ${A_TIME}" + if [[ -z "${REPLAY_WORKED}" || ${R_TIME} -gt ${A_TIME} ]]; then + if [ ${R_TIME} -gt ${A_TIME} ]; then + echo "Replay was slower than ccache." + fi - if [ -z "${REPLAY_WORKED}" ]; then - # Replay didn't work, so make the default "cached" image use the ccache one. + # Replay didn't work or was slower, so make the default "cached" image use the ccache one. docker image tag \ $CCACHE_IMAGE_NAME \ $FINAL_IMAGE_NAME @@ -163,7 +159,7 @@ then exit 0 else - echo "Replay and ccaching did not work." + echo "${_PROJECT}: Replay and ccaching did not work." exit 1 fi diff --git a/projects/apache-logging-log4cxx/build.sh b/projects/apache-logging-log4cxx/build.sh index b40e179c4b69..881dbdf83a29 100644 --- a/projects/apache-logging-log4cxx/build.sh +++ b/projects/apache-logging-log4cxx/build.sh @@ -17,3 +17,6 @@ git clone --quiet --depth 1 --branch master --single-branch https://github.com/apache/logging-log4cxx ./logging-log4cxx/src/fuzzers/bash/oss-fuzz-build.sh "$OUT" + +# Add seed corpus +zip $OUT/DOMConfiguratorFuzzer_seed_corpus.zip $SRC/logging-log4cxx/src/test/resources/input/xml/*.xml diff --git a/projects/apache-logging-log4cxx/project.yaml b/projects/apache-logging-log4cxx/project.yaml index bcb8ac62cc63..49a393001ccf 100644 --- a/projects/apache-logging-log4cxx/project.yaml +++ b/projects/apache-logging-log4cxx/project.yaml @@ -3,10 +3,14 @@ main_repo: "https://github.com/apache/logging-log4cxx" language: c++ fuzzing_engines: + - afl + - honggfuzz - libfuzzer + - centipede sanitizers: - address + - undefined # Apache Logging Services PMC members[1] that contribute the fuzz tests. # We cannot share `security@logging.apache.org` here, since it must be associated with a Google account[2]. @@ -18,3 +22,5 @@ auto_ccs: - piotr.karwasz@gmail.com - osfan6313@gmail.com - adam@adalogics.com + - swebb2066@gmail.com + diff --git a/projects/cgif/build.sh b/projects/cgif/build.sh index c20f15513652..d641f75513c9 100644 --- a/projects/cgif/build.sh +++ b/projects/cgif/build.sh @@ -23,7 +23,11 @@ meson install -C build meson test -C build cp "build/fuzz/cgif_fuzzer_seed_corpus.zip" $OUT/. +cp "build/fuzz/cgif_file_fuzzer_seed_corpus.zip" $OUT/. -# build cgif's fuzz target +# build cgif's fuzz targets $CXX $CXXFLAGS -o "$OUT/cgif_fuzzer" -I"$WORK/include" \ $LIB_FUZZING_ENGINE fuzz/cgif_fuzzer.c "$WORK/lib/libcgif.a" + +$CXX $CXXFLAGS -o "$OUT/cgif_file_fuzzer" -I"$WORK/include" \ + $LIB_FUZZING_ENGINE fuzz/cgif_file_fuzzer.c "$WORK/lib/libcgif.a" diff --git a/projects/kimageformats/Dockerfile b/projects/kimageformats/Dockerfile index 72aa8d01d114..29d7771cf1f2 100644 --- a/projects/kimageformats/Dockerfile +++ b/projects/kimageformats/Dockerfile @@ -17,7 +17,7 @@ FROM gcr.io/oss-fuzz-base/base-builder RUN apt-get update && apt-get install --yes cmake make autoconf automake autopoint libtool wget po4a ninja-build pkgconf RUN git clone --depth 1 https://github.com/madler/zlib.git -RUN git clone --depth 1 https://github.com/facebook/zstd.git +RUN git clone --depth 1 -b v1.5.6 https://github.com/facebook/zstd.git RUN git clone --depth 1 https://github.com/nih-at/libzip.git RUN wget https://sourceware.org/pub/bzip2/bzip2-1.0.8.tar.gz RUN git clone https://github.com/tukaani-project/xz.git diff --git a/projects/numpy/Dockerfile b/projects/numpy/Dockerfile index 9bde369877e7..2961aeb70ef6 100644 --- a/projects/numpy/Dockerfile +++ b/projects/numpy/Dockerfile @@ -15,6 +15,16 @@ ################################################################################ FROM gcr.io/oss-fuzz-base/base-builder-python +# Numpy requires python3.11+ +RUN apt-get update && \ + apt-get install -y software-properties-common && \ + add-apt-repository ppa:deadsnakes/ppa && \ + apt-get install -y python3.11 python3.11-dev && \ + update-alternatives --install /usr/local/bin/python3 python $(which python3.11) 5 +# Install Python dependencies for python 3.11 +RUN curl -LO https://bootstrap.pypa.io/get-pip.py && \ + python3 get-pip.py && \ + python3 -m pip install --root-user-action=ignore atheris pyinstaller RUN git clone https://github.com/numpy/numpy && cd numpy && git submodule update --init WORKDIR $SRC COPY *.py build.sh $SRC/ diff --git a/projects/surrealdb/project.yaml b/projects/surrealdb/project.yaml index b4be29fd6c81..899b46fca0e7 100644 --- a/projects/surrealdb/project.yaml +++ b/projects/surrealdb/project.yaml @@ -4,15 +4,11 @@ primary_contact: "tobie@surrealdb.com" main_repo: "https://github.com/surrealdb/surrealdb" auto_ccs: - security@surrealdb.com - - finn.bear@surrealdb.com - - salvador.girones@surrealdb.com - - gerard.guillemas@surrealdb.com - mees.delzenne@surrealdb.com - emmanuel.keller@surrealdb.com - micha.de.vries@surrealdb.com - rushmore@surrealdb.com - rowan.baker@surrealdb.com - - nathaniel.brough@gmail.com sanitizers: - address diff --git a/projects/tarantool/build.sh b/projects/tarantool/build.sh index 509d98aeb41c..1d45fb8fc1f4 100755 --- a/projects/tarantool/build.sh +++ b/projects/tarantool/build.sh @@ -91,6 +91,7 @@ cmake --build build --target fuzzers --parallel --verbose # used in Lua C API tests [1]. # # 1. https://github.com/ligurio/lua-c-api-tests/ +cp test/static/*.dict test/static/*.options $OUT/ for f in $(find build/test/fuzz/ \( -name '*_fuzzer' -o -name '*_test' \) -type f); do name=$(basename $f); @@ -98,10 +99,6 @@ do corpus_dir="test/static/corpus/$module" echo "Copying for $module"; cp $f $OUT/ - dict_path="test/static/$module.dict" - if [ -e "$dict_path" ]; then - cp $dict_path $OUT/ - fi if [ -e "$corpus_dir" ]; then zip --quiet -j $OUT/"$name"_seed_corpus.zip $corpus_dir/* fi diff --git a/projects/unrar/project.yaml b/projects/unrar/project.yaml index 09c4340260fa..be7268cde469 100644 --- a/projects/unrar/project.yaml +++ b/projects/unrar/project.yaml @@ -3,7 +3,7 @@ language: c++ primary_contact: "roshal@rarlab.com" auto_ccs: - "vakh@chromium.org" - - "drubery@chromium.org" + - "chlily@chromium.org" sanitizers: - address - memory diff --git a/projects/vlc/project.yaml b/projects/vlc/project.yaml index 3aa97aa955c8..983ddb789271 100644 --- a/projects/vlc/project.yaml +++ b/projects/vlc/project.yaml @@ -1,9 +1,12 @@ homepage: "https://github.com/videolan/vlc" language: c -primary_contact: "ossfuzz@videolan.org" +primary_contact: "security@videolan.org" auto_ccs: - - "adam@adalogics.com" - - "david@adalogics.com" + - "le.businessman@gmail.com" + - "thomas.guillem@gmail.com" + - "alexandre.janniaux@gmail.com" + - "kempfjb@gmail.com" + - "courmisch@gmail.com" sanitizers: - address - undefined diff --git a/projects/wasmtime/build.sh b/projects/wasmtime/build.sh index d8d5a63deaf1..c562795c601d 100755 --- a/projects/wasmtime/build.sh +++ b/projects/wasmtime/build.sh @@ -18,7 +18,6 @@ # Commands migrated from Dockerfile to make CIFuzz work # REF: https://github.com/google/oss-fuzz/issues/6755 git submodule update --init --recursive -git clone --depth 1 https://github.com/bytecodealliance/wasmtime-libfuzzer-corpus wasmtime-libfuzzer-corpus # Note: This project creates Rust fuzz targets exclusively @@ -53,12 +52,6 @@ build() { dst_name=$fuzzer_prefix$src_name cp $FUZZ_TARGET_OUTPUT_DIR/$src_name $OUT/$dst_name - if [[ -d $SRC/wasmtime/wasmtime-libfuzzer-corpus/$dst_name/ ]]; then - zip -jr \ - $OUT/${dst_name}_seed_corpus.zip \ - $SRC/wasmtime/wasmtime-libfuzzer-corpus/$dst_name/ - fi - if [[ -f $SRC/$dst_name.options ]]; then cp $SRC/$dst_name.options $OUT/$dst_name.options else