From 375486dedb29c4f7c48249116a341dc197d80761 Mon Sep 17 00:00:00 2001 From: DavidKorczynski Date: Tue, 28 Jan 2025 10:25:33 +0000 Subject: [PATCH] infra: update introspector and add new light version (#12983) Adds a light version of FI that should work on all projects. This will only be used if the existing FI does not work, e.g. due to failures around LTO. Also starts using the CLI version of fuzz-introspector as opposed to calling `main.py`. --------- Signed-off-by: David Korczynski --- infra/base-images/base-builder/compile | 77 +++++++++++++++++++------ infra/base-images/base-clang/Dockerfile | 2 +- 2 files changed, 59 insertions(+), 20 deletions(-) diff --git a/infra/base-images/base-builder/compile b/infra/base-images/base-builder/compile index 1c10d9e234e7..e05d0e6ea251 100755 --- a/infra/base-images/base-builder/compile +++ b/infra/base-images/base-builder/compile @@ -229,17 +229,47 @@ if [ "$SANITIZER" = "introspector" ] || [ "$RUST_SANITIZER" = "introspector" ]; # Install Fuzz-Introspector pushd /fuzz-introspector/src - python3 -m pip install . + python3 -m pip install -e . popd if [ "$FUZZING_LANGUAGE" = "python" ]; then python3 /fuzz-introspector/src/main.py light --language=python + cp -rf $SRC/inspector/ /tmp/inspector-saved elif [ "$FUZZING_LANGUAGE" = "jvm" ]; then python3 /fuzz-introspector/src/main.py light --language=jvm + cp -rf $SRC/inspector/ /tmp/inspector-saved elif [ "$FUZZING_LANGUAGE" = "rust" ]; then python3 /fuzz-introspector/src/main.py light --language=rust + cp -rf $SRC/inspector/ /tmp/inspector-saved else python3 /fuzz-introspector/src/main.py light + + # Make a copy of the light. This is needed because we run two versions of + # introspector: one based on pure statis analysis and one based on + # regular LTO. + cp -rf $SRC/inspector/ /tmp/inspector-saved + + + # Move coverage report. + if [ -d "$OUT/textcov_reports" ] + then + find $OUT/textcov_reports/ -name "*.covreport" -exec cp {} $SRC/inspector/ \; + find $OUT/textcov_reports/ -name "*.json" -exec cp {} $SRC/inspector/ \; + fi + + # Make fuzz-introspector HTML report using light approach. + REPORT_ARGS="--name=$PROJECT_NAME" + + # Only pass coverage_url when COVERAGE_URL is set (in cloud builds) + if [[ ! -z "${COVERAGE_URL+x}" ]]; then + REPORT_ARGS="$REPORT_ARGS --coverage-url=${COVERAGE_URL}" + fi + + # Run pure static analysis fuzz introspector + fuzz-introspector full --target-dir=$SRC \ + --language=${FUZZING_LANGUAGE} \ + --out-dir=$SRC/inspector \ + ${REPORT_ARGS} fi rsync -avu --delete "$SRC/inspector/" "$OUT/inspector" @@ -313,28 +343,37 @@ if [ "$SANITIZER" = "introspector" ] || [ "$RUST_SANITIZER" = "introspector" ]; unset CFLAGS export G_ANALYTICS_TAG="G-8WTFM1Y62J" + # If we get to here, it means the e.g. LTO had no problems and succeeded. + # TO this end, we wlil restore the original light analysis and used the + # LTO processing itself. + rm -rf $SRC/inspector + cp -rf /tmp/inspector-saved $SRC/inspector + + cd /fuzz-introspector/src + python3 -m pip install -e . + cd /src/ + if [ "$FUZZING_LANGUAGE" = "jvm" ]; then echo "GOING jvm route" set -x # Output will be put in /out/ - python3 /fuzz-introspector/frontends/java/oss-fuzz-main.py + python3 -m fuzz_introspector.frontends.oss_fuzz --language jvm --target-dir $SRC --entrypoint fuzzerTestOneInput + # Move files temporarily to fit workflow of other languages. mkdir -p $SRC/my-fi-data - find $OUT/ -name *.data -exec mv {} $SRC/my-fi-data/ \; - find $OUT/ -name *.data.yaml -exec mv {} $SRC/my-fi-data/ \; + find ./ -name *.data -exec mv {} $SRC/my-fi-data/ \; + find ./ -name *.data.yaml -exec mv {} $SRC/my-fi-data/ \; elif [ "$FUZZING_LANGUAGE" = "rust" ]; then echo "GOING rust route" # Run the rust frontend - pushd /fuzz-introspector/frontends/rust/rust_function_analyser - cargo run -- $SRC + python3 -m fuzz_introspector.frontends.oss_fuzz --language rust --target-dir $SRC # Move files temporarily to fix workflow of other languages. mkdir -p $SRC/my-fi-data find ./ -name "*.data" -exec mv {} $SRC/my-fi-data/ \; find ./ -name "*.data.yaml" -exec mv {} $SRC/my-fi-data/ \; - popd # Restore the sanitizer flag for rust export SANITIZER="introspector" @@ -359,43 +398,43 @@ if [ "$SANITIZER" = "introspector" ] || [ "$RUST_SANITIZER" = "introspector" ]; REPORT_ARGS="--name=$PROJECT_NAME" # Only pass coverage_url when COVERAGE_URL is set (in cloud builds) if [[ ! -z "${COVERAGE_URL+x}" ]]; then - REPORT_ARGS="$REPORT_ARGS --coverage_url=${COVERAGE_URL}" + REPORT_ARGS="$REPORT_ARGS --coverage-url=${COVERAGE_URL}" fi # Do different things depending on languages if [ "$FUZZING_LANGUAGE" = "python" ]; then echo "GOING python route" set -x - REPORT_ARGS="$REPORT_ARGS --target_dir=$SRC/inspector" + REPORT_ARGS="$REPORT_ARGS --target-dir=$SRC/inspector" REPORT_ARGS="$REPORT_ARGS --language=python" - python3 /fuzz-introspector/src/main.py report $REPORT_ARGS + fuzz-introspector report $REPORT_ARGS rsync -avu --delete "$SRC/inspector/" "$OUT/inspector" elif [ "$FUZZING_LANGUAGE" = "jvm" ]; then echo "GOING jvm route" set -x find $OUT/ -name "jacoco.xml" -exec cp {} $SRC/inspector/ \; - REPORT_ARGS="$REPORT_ARGS --target_dir=$SRC/inspector" + REPORT_ARGS="$REPORT_ARGS --target-dir=$SRC/inspector" REPORT_ARGS="$REPORT_ARGS --language=jvm" - python3 /fuzz-introspector/src/main.py report $REPORT_ARGS + fuzz-introspector report $REPORT_ARGS rsync -avu --delete "$SRC/inspector/" "$OUT/inspector" elif [ "$FUZZING_LANGUAGE" = "rust" ]; then echo "GOING rust route" - REPORT_ARGS="$REPORT_ARGS --target_dir=$SRC/inspector" + REPORT_ARGS="$REPORT_ARGS --target-dir=$SRC/inspector" REPORT_ARGS="$REPORT_ARGS --language=rust" - python3 /fuzz-introspector/src/main.py report $REPORT_ARGS + fuzz-introspector report $REPORT_ARGS rsync -avu --delete "$SRC/inspector/" "$OUT/inspector" else # C/C++ - + mkdir -p $SRC/inspector # Correlate fuzzer binaries to fuzz-introspector's raw data - python3 /fuzz-introspector/src/main.py correlate --binaries_dir=$OUT/ + fuzz-introspector correlate --binaries-dir=$OUT/ # Generate fuzz-introspector HTML report, this generates # the file exe_to_fuzz_introspector_logs.yaml - REPORT_ARGS="$REPORT_ARGS --target_dir=$SRC/inspector" + REPORT_ARGS="$REPORT_ARGS --target-dir=$SRC/inspector" # Use the just-generated correlation file - REPORT_ARGS="$REPORT_ARGS --correlation_file=exe_to_fuzz_introspector_logs.yaml" - python3 /fuzz-introspector/src/main.py report $REPORT_ARGS + REPORT_ARGS="$REPORT_ARGS --correlation-file=exe_to_fuzz_introspector_logs.yaml" + fuzz-introspector report $REPORT_ARGS rsync -avu --delete "$SRC/inspector/" "$OUT/inspector" fi diff --git a/infra/base-images/base-clang/Dockerfile b/infra/base-images/base-clang/Dockerfile index d403bb07d876..296b1f7fb6ca 100644 --- a/infra/base-images/base-clang/Dockerfile +++ b/infra/base-images/base-clang/Dockerfile @@ -36,7 +36,7 @@ RUN apt-get update && apt-get install -y wget sudo && \ RUN apt-get update && apt-get install -y git && \ git clone https://github.com/ossf/fuzz-introspector.git fuzz-introspector && \ cd fuzz-introspector && \ - git checkout 74917384c5a4e368d900862b4bd3d16ce3fe5dd8 && \ + git checkout eab8da2f42d07fb63d28b30c1cc12a2bfcd8e648 && \ git submodule init && \ git submodule update && \ apt-get autoremove --purge -y git && \