From 10f9bb78149029f2c74a8179f00c1a9cf6b540de Mon Sep 17 00:00:00 2001 From: David Korczynski Date: Sat, 25 Jan 2025 11:11:27 -0800 Subject: [PATCH 1/4] infra: update introspector and add new light version Adds a light version of FI that should work on all projects. This will only be used if the existing FI does not work, e.g. due to failures around LTO. Signed-off-by: David Korczynski --- infra/base-images/base-builder/compile | 74 ++++++++++++++++++------- infra/base-images/base-clang/Dockerfile | 2 +- 2 files changed, 56 insertions(+), 20 deletions(-) diff --git a/infra/base-images/base-builder/compile b/infra/base-images/base-builder/compile index 1c10d9e234e7..6684725c3fb6 100755 --- a/infra/base-images/base-builder/compile +++ b/infra/base-images/base-builder/compile @@ -229,7 +229,7 @@ if [ "$SANITIZER" = "introspector" ] || [ "$RUST_SANITIZER" = "introspector" ]; # Install Fuzz-Introspector pushd /fuzz-introspector/src - python3 -m pip install . + python3 -m pip install -e . popd if [ "$FUZZING_LANGUAGE" = "python" ]; then @@ -242,6 +242,33 @@ if [ "$SANITIZER" = "introspector" ] || [ "$RUST_SANITIZER" = "introspector" ]; python3 /fuzz-introspector/src/main.py light fi + # Make a copy of the light. This is needed because we run two versions of + # introspector: one based on pure statis analysis and one based on + # regular LTO. + cp -rf $SRC/inspector/ /tmp/inspector-saved + + + # Move coverage report. + if [ -d "$OUT/textcov_reports" ] + then + find $OUT/textcov_reports/ -name "*.covreport" -exec cp {} $SRC/inspector/ \; + find $OUT/textcov_reports/ -name "*.json" -exec cp {} $SRC/inspector/ \; + fi + + # Make fuzz-introspector HTML report using light approach. + REPORT_ARGS="--name=$PROJECT_NAME" + + # Only pass coverage_url when COVERAGE_URL is set (in cloud builds) + if [[ ! -z "${COVERAGE_URL+x}" ]]; then + REPORT_ARGS="$REPORT_ARGS --coverage-url=${COVERAGE_URL}" + fi + + # Run pure static analysis fuzz introspector + fuzz-introspector full --target-dir=$SRC \ + --language=${FUZZING_LANGUAGE} \ + --out-dir=$SRC/inspector \ + ${REPORT_ARGS} + rsync -avu --delete "$SRC/inspector/" "$OUT/inspector" fi @@ -313,28 +340,37 @@ if [ "$SANITIZER" = "introspector" ] || [ "$RUST_SANITIZER" = "introspector" ]; unset CFLAGS export G_ANALYTICS_TAG="G-8WTFM1Y62J" + # If we get to here, it means the e.g. LTO had no problems and succeeded. + # TO this end, we wlil restore the original light analysis and used the + # LTO processing itself. + rm -rf $SRC/inspector + cp -rf /tmp/inspector-saved $SRC/inspector + + cd /fuzz-introspector/src + python3 -m pip install -e . + cd /src/ + if [ "$FUZZING_LANGUAGE" = "jvm" ]; then echo "GOING jvm route" set -x # Output will be put in /out/ - python3 /fuzz-introspector/frontends/java/oss-fuzz-main.py + python3 -m fuzz_introspector.frontends.oss_fuzz --language jvm --target-dir $SRC --entrypoint fuzzerTestOneInput + # Move files temporarily to fit workflow of other languages. mkdir -p $SRC/my-fi-data - find $OUT/ -name *.data -exec mv {} $SRC/my-fi-data/ \; - find $OUT/ -name *.data.yaml -exec mv {} $SRC/my-fi-data/ \; + find ./ -name *.data -exec mv {} $SRC/my-fi-data/ \; + find ./ -name *.data.yaml -exec mv {} $SRC/my-fi-data/ \; elif [ "$FUZZING_LANGUAGE" = "rust" ]; then echo "GOING rust route" # Run the rust frontend - pushd /fuzz-introspector/frontends/rust/rust_function_analyser - cargo run -- $SRC + python3 -m fuzz_introspector.frontends.oss_fuzz --language rust --target-dir $SRC # Move files temporarily to fix workflow of other languages. mkdir -p $SRC/my-fi-data find ./ -name "*.data" -exec mv {} $SRC/my-fi-data/ \; find ./ -name "*.data.yaml" -exec mv {} $SRC/my-fi-data/ \; - popd # Restore the sanitizer flag for rust export SANITIZER="introspector" @@ -359,43 +395,43 @@ if [ "$SANITIZER" = "introspector" ] || [ "$RUST_SANITIZER" = "introspector" ]; REPORT_ARGS="--name=$PROJECT_NAME" # Only pass coverage_url when COVERAGE_URL is set (in cloud builds) if [[ ! -z "${COVERAGE_URL+x}" ]]; then - REPORT_ARGS="$REPORT_ARGS --coverage_url=${COVERAGE_URL}" + REPORT_ARGS="$REPORT_ARGS --coverage-url=${COVERAGE_URL}" fi # Do different things depending on languages if [ "$FUZZING_LANGUAGE" = "python" ]; then echo "GOING python route" set -x - REPORT_ARGS="$REPORT_ARGS --target_dir=$SRC/inspector" + REPORT_ARGS="$REPORT_ARGS --target-dir=$SRC/inspector" REPORT_ARGS="$REPORT_ARGS --language=python" - python3 /fuzz-introspector/src/main.py report $REPORT_ARGS + fuzz-introspector report $REPORT_ARGS rsync -avu --delete "$SRC/inspector/" "$OUT/inspector" elif [ "$FUZZING_LANGUAGE" = "jvm" ]; then echo "GOING jvm route" set -x find $OUT/ -name "jacoco.xml" -exec cp {} $SRC/inspector/ \; - REPORT_ARGS="$REPORT_ARGS --target_dir=$SRC/inspector" + REPORT_ARGS="$REPORT_ARGS --target-dir=$SRC/inspector" REPORT_ARGS="$REPORT_ARGS --language=jvm" - python3 /fuzz-introspector/src/main.py report $REPORT_ARGS + fuzz-introspector report $REPORT_ARGS rsync -avu --delete "$SRC/inspector/" "$OUT/inspector" elif [ "$FUZZING_LANGUAGE" = "rust" ]; then echo "GOING rust route" - REPORT_ARGS="$REPORT_ARGS --target_dir=$SRC/inspector" + REPORT_ARGS="$REPORT_ARGS --target-dir=$SRC/inspector" REPORT_ARGS="$REPORT_ARGS --language=rust" - python3 /fuzz-introspector/src/main.py report $REPORT_ARGS + fuzz-introspector report $REPORT_ARGS rsync -avu --delete "$SRC/inspector/" "$OUT/inspector" else # C/C++ - + mkdir -p $SRC/inspector # Correlate fuzzer binaries to fuzz-introspector's raw data - python3 /fuzz-introspector/src/main.py correlate --binaries_dir=$OUT/ + fuzz-introspector correlate --binaries-dir=$OUT/ # Generate fuzz-introspector HTML report, this generates # the file exe_to_fuzz_introspector_logs.yaml - REPORT_ARGS="$REPORT_ARGS --target_dir=$SRC/inspector" + REPORT_ARGS="$REPORT_ARGS --target-dir=$SRC/inspector" # Use the just-generated correlation file - REPORT_ARGS="$REPORT_ARGS --correlation_file=exe_to_fuzz_introspector_logs.yaml" - python3 /fuzz-introspector/src/main.py report $REPORT_ARGS + REPORT_ARGS="$REPORT_ARGS --correlation-file=exe_to_fuzz_introspector_logs.yaml" + fuzz-introspector report $REPORT_ARGS rsync -avu --delete "$SRC/inspector/" "$OUT/inspector" fi diff --git a/infra/base-images/base-clang/Dockerfile b/infra/base-images/base-clang/Dockerfile index d403bb07d876..3420ae4ed26d 100644 --- a/infra/base-images/base-clang/Dockerfile +++ b/infra/base-images/base-clang/Dockerfile @@ -36,7 +36,7 @@ RUN apt-get update && apt-get install -y wget sudo && \ RUN apt-get update && apt-get install -y git && \ git clone https://github.com/ossf/fuzz-introspector.git fuzz-introspector && \ cd fuzz-introspector && \ - git checkout 74917384c5a4e368d900862b4bd3d16ce3fe5dd8 && \ + git checkout 35a4edc82a912ba155ff0f8ee10eb7bb9a05c80c && \ git submodule init && \ git submodule update && \ apt-get autoremove --purge -y git && \ From 2d66f873b0c11569075a958858053ff1b44a728d Mon Sep 17 00:00:00 2001 From: David Korczynski Date: Sat, 25 Jan 2025 22:31:40 +0000 Subject: [PATCH 2/4] bump to next commit Signed-off-by: David Korczynski --- infra/base-images/base-clang/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infra/base-images/base-clang/Dockerfile b/infra/base-images/base-clang/Dockerfile index 3420ae4ed26d..7aa27348f380 100644 --- a/infra/base-images/base-clang/Dockerfile +++ b/infra/base-images/base-clang/Dockerfile @@ -36,7 +36,7 @@ RUN apt-get update && apt-get install -y wget sudo && \ RUN apt-get update && apt-get install -y git && \ git clone https://github.com/ossf/fuzz-introspector.git fuzz-introspector && \ cd fuzz-introspector && \ - git checkout 35a4edc82a912ba155ff0f8ee10eb7bb9a05c80c && \ + git checkout d0e433ca1cc8a1a04b48ff1dfdb3507d9752f81b && \ git submodule init && \ git submodule update && \ apt-get autoremove --purge -y git && \ From 56bdf30078b9301aebc7f9dc6c6b41addf48d27a Mon Sep 17 00:00:00 2001 From: David Korczynski Date: Mon, 27 Jan 2025 02:14:33 -0800 Subject: [PATCH 3/4] fix light Signed-off-by: David Korczynski --- infra/base-images/base-builder/compile | 47 +++++++++++++------------ infra/base-images/base-clang/Dockerfile | 2 +- 2 files changed, 26 insertions(+), 23 deletions(-) diff --git a/infra/base-images/base-builder/compile b/infra/base-images/base-builder/compile index 6684725c3fb6..e05d0e6ea251 100755 --- a/infra/base-images/base-builder/compile +++ b/infra/base-images/base-builder/compile @@ -234,40 +234,43 @@ if [ "$SANITIZER" = "introspector" ] || [ "$RUST_SANITIZER" = "introspector" ]; if [ "$FUZZING_LANGUAGE" = "python" ]; then python3 /fuzz-introspector/src/main.py light --language=python + cp -rf $SRC/inspector/ /tmp/inspector-saved elif [ "$FUZZING_LANGUAGE" = "jvm" ]; then python3 /fuzz-introspector/src/main.py light --language=jvm + cp -rf $SRC/inspector/ /tmp/inspector-saved elif [ "$FUZZING_LANGUAGE" = "rust" ]; then python3 /fuzz-introspector/src/main.py light --language=rust + cp -rf $SRC/inspector/ /tmp/inspector-saved else python3 /fuzz-introspector/src/main.py light - fi - # Make a copy of the light. This is needed because we run two versions of - # introspector: one based on pure statis analysis and one based on - # regular LTO. - cp -rf $SRC/inspector/ /tmp/inspector-saved + # Make a copy of the light. This is needed because we run two versions of + # introspector: one based on pure statis analysis and one based on + # regular LTO. + cp -rf $SRC/inspector/ /tmp/inspector-saved - # Move coverage report. - if [ -d "$OUT/textcov_reports" ] - then - find $OUT/textcov_reports/ -name "*.covreport" -exec cp {} $SRC/inspector/ \; - find $OUT/textcov_reports/ -name "*.json" -exec cp {} $SRC/inspector/ \; - fi + # Move coverage report. + if [ -d "$OUT/textcov_reports" ] + then + find $OUT/textcov_reports/ -name "*.covreport" -exec cp {} $SRC/inspector/ \; + find $OUT/textcov_reports/ -name "*.json" -exec cp {} $SRC/inspector/ \; + fi - # Make fuzz-introspector HTML report using light approach. - REPORT_ARGS="--name=$PROJECT_NAME" + # Make fuzz-introspector HTML report using light approach. + REPORT_ARGS="--name=$PROJECT_NAME" - # Only pass coverage_url when COVERAGE_URL is set (in cloud builds) - if [[ ! -z "${COVERAGE_URL+x}" ]]; then - REPORT_ARGS="$REPORT_ARGS --coverage-url=${COVERAGE_URL}" - fi + # Only pass coverage_url when COVERAGE_URL is set (in cloud builds) + if [[ ! -z "${COVERAGE_URL+x}" ]]; then + REPORT_ARGS="$REPORT_ARGS --coverage-url=${COVERAGE_URL}" + fi - # Run pure static analysis fuzz introspector - fuzz-introspector full --target-dir=$SRC \ - --language=${FUZZING_LANGUAGE} \ - --out-dir=$SRC/inspector \ - ${REPORT_ARGS} + # Run pure static analysis fuzz introspector + fuzz-introspector full --target-dir=$SRC \ + --language=${FUZZING_LANGUAGE} \ + --out-dir=$SRC/inspector \ + ${REPORT_ARGS} + fi rsync -avu --delete "$SRC/inspector/" "$OUT/inspector" fi diff --git a/infra/base-images/base-clang/Dockerfile b/infra/base-images/base-clang/Dockerfile index 7aa27348f380..f5c816e9366b 100644 --- a/infra/base-images/base-clang/Dockerfile +++ b/infra/base-images/base-clang/Dockerfile @@ -36,7 +36,7 @@ RUN apt-get update && apt-get install -y wget sudo && \ RUN apt-get update && apt-get install -y git && \ git clone https://github.com/ossf/fuzz-introspector.git fuzz-introspector && \ cd fuzz-introspector && \ - git checkout d0e433ca1cc8a1a04b48ff1dfdb3507d9752f81b && \ + git checkout b661b3d669cf56fbc3742e56b760af5e2b2ce1d2 && \ git submodule init && \ git submodule update && \ apt-get autoremove --purge -y git && \ From dbf9f8579b9557c418d5bd2dd8ff69d7922d5156 Mon Sep 17 00:00:00 2001 From: DavidKorczynski Date: Mon, 27 Jan 2025 20:23:15 +0000 Subject: [PATCH 4/4] infra: bump to next commit --- infra/base-images/base-clang/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infra/base-images/base-clang/Dockerfile b/infra/base-images/base-clang/Dockerfile index f5c816e9366b..296b1f7fb6ca 100644 --- a/infra/base-images/base-clang/Dockerfile +++ b/infra/base-images/base-clang/Dockerfile @@ -36,7 +36,7 @@ RUN apt-get update && apt-get install -y wget sudo && \ RUN apt-get update && apt-get install -y git && \ git clone https://github.com/ossf/fuzz-introspector.git fuzz-introspector && \ cd fuzz-introspector && \ - git checkout b661b3d669cf56fbc3742e56b760af5e2b2ce1d2 && \ + git checkout eab8da2f42d07fb63d28b30c1cc12a2bfcd8e648 && \ git submodule init && \ git submodule update && \ apt-get autoremove --purge -y git && \