Merge branch 'master' into swift_5_10

infra/experimental/chronos/README.md

@@ -1,5 +1,34 @@
 # Chronos: rebuilding OSS-Fuzz harnesses using cached builds
 
+## Pre-built images.
+
+Daily pre-built images are available at:
+
+- `us-central1-docker.pkg.dev/oss-fuzz/oss-fuzz-gen/<PROJECT>-ofg-cached-address`
+- `us-central1-docker.pkg.dev/oss-fuzz/oss-fuzz-gen/<PROJECT>-ofg-cached-coverage`
+
+They can be used as drop-in replacements for the usual `gcr.io/oss-fuzz/<PROJECT>` images.
+
+These images are generated in 2 ways:
+- (Preferred) [Generate](https://github.com/google/oss-fuzz/blob/master/infra/base-images/base-builder/bash_parser.py)
+  a replay build script that can be re-run alongside existing build artifacts,
+  leveraging existing build system mechanisms to avoid rebuilding (e.g. running
+  `make` twice should not actually rebuild everything). This is error-prone, so
+  we validate the script works by running it.
+- (Fallback, if the replay build script didn't work). We leverage
+  [ccache](https://ccache.dev/), to provide a compiler cache. This is often not
+  as fast as the replay build script, because some project builds spend
+  significant time doing non-compiler tasks (e.g. checking out submodules,
+  running configure scripts).
+
+Note: this mechanism does not work for every single OSS-Fuzz project today. The
+resulting image may either:
+- Not provide much performance improvement compared with a normal image, or
+- Not exist at all (if neither approach worked).
+
+Stats from a recent run: <https://gist.github.com/oliverchang/abaf3a1106a2b923c0ac3a577410aaaa>
+(Feb 3 2025).
+
 ## Usage locally
 
 **Example 1: htslib**

infra/experimental/chronos/build_cache_local.sh

@@ -97,17 +97,16 @@ REPLAY_WORKED=
 #         If this step is successful, then the process can exit as it's ready.
 if [[ "$executables_replay" == "$executables_vanilla" ]]
 then
-  echo "Replay worked"
-  echo "Vanilla compile time: ${B_TIME}"
-  echo "Replay compile time: ${R_TIME}"
-
   REPLAY_WORKED=1
 
   if [ -z "${RUN_ALL+1}" ]; then
+    echo "${_PROJECT}: Replay worked."
+    echo "${_PROJECT}: Compile times: Vanilla=${B_TIME}; Replay=${R_TIME};"
     exit 0
   fi
 else
-  echo "Replay did not work"
+  echo "${_PROJECT}: Replay did not work"
+  R_TIME="N/A"
 fi
 
 # Step 8: prepare Dockerfile for ccache
@@ -145,25 +144,22 @@ executables_ccache="$(find ./build/out/${_PROJECT}/ -executable -type f | sort)"
 # Step 12: validate the ccache builds are successful
 if [[ "$executables_ccache" == "$executables_vanilla" ]]
 then
-  echo "Vanilla compile time: ${B_TIME}"
-  if [[ "$executables_replay" == "$executables_vanilla" ]]
-  then
-    echo "Replay worked"
-    echo "Replay compile time: ${R_TIME}"
-  fi
+  echo "${_PROJECT}: Compile times: Vanilla=${B_TIME}; Replay=${R_TIME}; CCache=${A_TIME};"
 
-  echo "Ccache compile time: ${A_TIME}"
+  if [[ -z "${REPLAY_WORKED}" || ${R_TIME} -gt ${A_TIME} ]]; then
+    if [ ${R_TIME} -gt ${A_TIME} ]; then
+      echo "Replay was slower than ccache."
+    fi
 
-  if [ -z "${REPLAY_WORKED}" ]; then
-    # Replay didn't work, so make the default "cached" image use the ccache one.
+    # Replay didn't work or was slower, so make the default "cached" image use the ccache one.
     docker image tag \
       $CCACHE_IMAGE_NAME \
       $FINAL_IMAGE_NAME
   fi
 
   exit 0
 else
-  echo "Replay and ccaching did not work."
+  echo "${_PROJECT}: Replay and ccaching did not work."
   exit 1
 fi
 

projects/apache-logging-log4cxx/build.sh

@@ -17,3 +17,6 @@
 
 git clone --quiet --depth 1 --branch master --single-branch https://github.com/apache/logging-log4cxx
 ./logging-log4cxx/src/fuzzers/bash/oss-fuzz-build.sh "$OUT"
+
+# Add seed corpus
+zip $OUT/DOMConfiguratorFuzzer_seed_corpus.zip $SRC/logging-log4cxx/src/test/resources/input/xml/*.xml

projects/apache-logging-log4cxx/project.yaml

@@ -3,10 +3,14 @@ main_repo: "https://github.com/apache/logging-log4cxx"
 language: c++
 
 fuzzing_engines:
+  - afl
+  - honggfuzz
   - libfuzzer
+  - centipede
 
 sanitizers:
   - address
+  - undefined
 
 # Apache Logging Services PMC members[1] that contribute the fuzz tests.
 # We cannot share `[email protected]` here, since it must be associated with a Google account[2].
@@ -18,3 +22,5 @@ auto_ccs:
   - [email protected]
   - [email protected]
   - [email protected]
+  - [email protected]
+

projects/cgif/build.sh

@@ -23,7 +23,11 @@ meson install -C build
 meson test -C build
 
 cp "build/fuzz/cgif_fuzzer_seed_corpus.zip" $OUT/.
+cp "build/fuzz/cgif_file_fuzzer_seed_corpus.zip" $OUT/.
 
-# build cgif's fuzz target
+# build cgif's fuzz targets
 $CXX $CXXFLAGS -o "$OUT/cgif_fuzzer" -I"$WORK/include" \
   $LIB_FUZZING_ENGINE fuzz/cgif_fuzzer.c "$WORK/lib/libcgif.a"
+
+$CXX $CXXFLAGS -o "$OUT/cgif_file_fuzzer" -I"$WORK/include" \
+  $LIB_FUZZING_ENGINE fuzz/cgif_file_fuzzer.c "$WORK/lib/libcgif.a"

projects/kimageformats/Dockerfile

@@ -17,7 +17,7 @@
 FROM gcr.io/oss-fuzz-base/base-builder
 RUN apt-get update && apt-get install --yes cmake make autoconf automake autopoint libtool wget po4a ninja-build pkgconf
 RUN git clone --depth 1 https://github.com/madler/zlib.git
-RUN git clone --depth 1 https://github.com/facebook/zstd.git
+RUN git clone --depth 1 -b v1.5.6 https://github.com/facebook/zstd.git
 RUN git clone --depth 1 https://github.com/nih-at/libzip.git
 RUN wget https://sourceware.org/pub/bzip2/bzip2-1.0.8.tar.gz
 RUN git clone https://github.com/tukaani-project/xz.git

projects/numpy/Dockerfile

@@ -15,6 +15,16 @@
 ################################################################################
 
 FROM gcr.io/oss-fuzz-base/base-builder-python
+# Numpy requires python3.11+
+RUN apt-get update && \
+    apt-get install -y software-properties-common && \
+    add-apt-repository ppa:deadsnakes/ppa && \
+    apt-get install -y python3.11 python3.11-dev && \
+    update-alternatives --install /usr/local/bin/python3 python $(which python3.11) 5
+# Install Python dependencies for python 3.11
+RUN curl -LO https://bootstrap.pypa.io/get-pip.py && \
+    python3 get-pip.py && \
+    python3 -m pip install --root-user-action=ignore atheris pyinstaller
 RUN git clone https://github.com/numpy/numpy && cd numpy && git submodule update --init
 WORKDIR $SRC
 COPY *.py build.sh $SRC/

projects/surrealdb/project.yaml

@@ -4,15 +4,11 @@ primary_contact: "[email protected]"
 main_repo: "https://github.com/surrealdb/surrealdb"
 auto_ccs:
   - [email protected]
-  - [email protected]
-  - [email protected]
-  - [email protected]
   - [email protected]
   - [email protected]
   - [email protected]
   - [email protected]
   - [email protected]
-  - [email protected]
 
 sanitizers:
   - address

projects/tarantool/build.sh

@@ -91,17 +91,14 @@ cmake --build build --target fuzzers --parallel --verbose
 # used in Lua C API tests [1].
 #
 # 1. https://github.com/ligurio/lua-c-api-tests/
+cp test/static/*.dict test/static/*.options $OUT/
 for f in $(find build/test/fuzz/ \( -name '*_fuzzer' -o -name '*_test' \) -type f);
 do
   name=$(basename $f);
   module=$(echo $name | sed 's/_fuzzer//' | sed 's/_test//' )
   corpus_dir="test/static/corpus/$module"
   echo "Copying for $module";
   cp $f $OUT/
-  dict_path="test/static/$module.dict"
-  if [ -e "$dict_path" ]; then
-    cp $dict_path $OUT/
-  fi
   if [ -e "$corpus_dir" ]; then
     zip --quiet -j $OUT/"$name"_seed_corpus.zip $corpus_dir/*
   fi

projects/unrar/project.yaml

@@ -3,7 +3,7 @@ language: c++
 primary_contact: "[email protected]"
 auto_ccs:
   - "[email protected]"
-  - "drubery@chromium.org"
+  - "chlily@chromium.org"
 sanitizers:
   - address
   - memory

projects/vlc/project.yaml

@@ -1,9 +1,12 @@
 homepage: "https://github.com/videolan/vlc"
 language: c
-primary_contact: "ossfuzz@videolan.org"
+primary_contact: "security@videolan.org"
 auto_ccs:
-  - "[email protected]"
-  - "[email protected]"
+  - "[email protected]"
+  - "[email protected]"
+  - "[email protected]"
+  - "[email protected]"
+  - "[email protected]"
 sanitizers:
   - address
   - undefined

projects/wasmtime/build.sh

@@ -18,7 +18,6 @@
 # Commands migrated from Dockerfile to make CIFuzz work
 # REF: https://github.com/google/oss-fuzz/issues/6755
 git submodule update --init --recursive
-git clone --depth 1 https://github.com/bytecodealliance/wasmtime-libfuzzer-corpus wasmtime-libfuzzer-corpus
 
 # Note: This project creates Rust fuzz targets exclusively
 
@@ -53,12 +52,6 @@ build() {
       dst_name=$fuzzer_prefix$src_name
       cp $FUZZ_TARGET_OUTPUT_DIR/$src_name $OUT/$dst_name
 
-      if [[ -d $SRC/wasmtime/wasmtime-libfuzzer-corpus/$dst_name/ ]]; then
-          zip -jr \
-              $OUT/${dst_name}_seed_corpus.zip \
-              $SRC/wasmtime/wasmtime-libfuzzer-corpus/$dst_name/
-      fi
-
       if [[ -f $SRC/$dst_name.options ]]; then
         cp $SRC/$dst_name.options $OUT/$dst_name.options
       else