Merge branch 'master' into master

.github/workflows/project_tests.yml

@@ -68,7 +68,7 @@ jobs:
           sudo swapoff -a
           sudo rm -f /swapfile
           sudo apt clean
-          docker rmi $(docker images -a -q)
+          docker rmi $(docker images -a -q) || true
           df -h
           echo "Remove large unused folders, inspired by https://github.com/apache/flink/blame/master/tools/azure-pipelines/free_disk_space.sh"
           sudo bash -c '(ionice -c 3 nice -n 19 rm -rf /usr/share/dotnet/ /usr/local/graalvm/ /usr/local/.ghcup/ /usr/local/share/powershell /usr/local/share/chromium /usr/local/lib/android /usr/local/lib/node_modules)&'

docs/Gemfile.lock

@@ -222,8 +222,7 @@ GEM
     rb-fsevent (0.11.1)
     rb-inotify (0.10.1)
       ffi (~> 1.0)
-    rexml (3.3.6)
-      strscan
+    rexml (3.3.9)
     rouge (3.26.0)
     ruby2_keywords (0.0.5)
     rubyzip (2.3.2)
@@ -238,7 +237,6 @@ GEM
       faraday (>= 0.17.3, < 3)
     simpleidn (0.2.1)
       unf (~> 0.1.4)
-    strscan (3.1.0)
     terminal-table (1.8.0)
       unicode-display_width (~> 1.1, >= 1.1.1)
     typhoeus (1.4.0)

infra/base-images/base-builder-jvm/Dockerfile

@@ -36,7 +36,7 @@ WORKDIR $SRC
 
 RUN git clone https://github.com/CodeIntelligenceTesting/jazzer && \
     cd jazzer && \
-    git checkout aa376ca3c0255b9cf04e1fc79fcf1fe5c63b2376
+    git checkout 11b42852df4344737df54a380c2f522025bb4e84
 
 WORKDIR $SRC/jazzer
 

infra/base-images/base-builder/bash_parser.py

@@ -47,6 +47,10 @@ def should_discard_command(ast_tree) -> bool:
   except:  # pylint: disable=bare-except
     return False
 
+  if ('cmake' in first_word and
+      any('--build' in part.word for part in ast_tree.parts)):
+    return False
+
   cmds_to_avoid_replaying = {
       'configure', 'autoheader', 'autoconf', 'autoreconf', 'cmake', 'autogen.sh'
   }
@@ -196,7 +200,7 @@ def parse_script(bash_script, all_scripts) -> str:
     build_script = f.read()
   try:
     parts = bashlex.parse(build_script)
-  except bashlex.error.ParsingError:
+  except bashlex.errors.ParsingError:
     return ''
   for part in parts:
     new_script += handle_node(part, all_scripts, build_script)

infra/base-images/base-builder/compile

@@ -21,6 +21,9 @@ sysctl -w vm.mmap_rnd_bits=28
 
 OSS_FUZZ_ON_DEMAND="${OSS_FUZZ_ON_DEMAND:-0}"
 
+# Used for Rust introspector builds
+RUST_SANITIZER=$SANITIZER
+
 if [ "$FUZZING_LANGUAGE" = "jvm" ]; then
   if [ "$FUZZING_ENGINE" != "libfuzzer" ] && [ "$FUZZING_ENGINE" != "wycheproof" ]; then
     echo "ERROR: JVM projects can be fuzzed with libFuzzer or tested with wycheproof engines only."
@@ -36,6 +39,15 @@ if [ "$FUZZING_LANGUAGE" = "jvm" ]; then
   fi
 fi
 
+if [ "$FUZZING_LANGUAGE" = "rust" ]; then
+  if [ "$SANITIZER" = "introspector" ]; then
+    # introspector sanitizer flag will cause cargo build to fail. Rremove it
+    # temporarily, RUST_SANITIZER will hold the original sanitizer.
+    export SANITIZER=address
+  fi
+fi
+
+
 if [ "$FUZZING_LANGUAGE" = "javascript" ]; then
   if [ "$FUZZING_ENGINE" != "libfuzzer" ]; then
     echo "ERROR: JavaScript projects can be fuzzed with libFuzzer engine only."
@@ -111,7 +123,9 @@ fi
 # use RUSTFLAGS.
 # FIXME: Support code coverage once support is in.
 # See https://github.com/rust-lang/rust/issues/34701.
-if [ "$SANITIZER" != "undefined" ] && [ "$SANITIZER" != "coverage" ] && [ "$SANITIZER" != "none" ] && [ "$ARCHITECTURE" != 'i386' ]; then
+if [ "$RUST_SANITIZER" == "introspector" ]; then
+  export RUSTFLAGS="-Cdebuginfo=2 -Cforce-frame-pointers"
+elif [ "$SANITIZER" != "undefined" ] && [ "$SANITIZER" != "coverage" ] && [ "$SANITIZER" != "none" ] && [ "$ARCHITECTURE" != 'i386' ]; then
   export RUSTFLAGS="--cfg fuzzing -Zsanitizer=${SANITIZER} -Cdebuginfo=1 -Cforce-frame-pointers"
 else
   export RUSTFLAGS="--cfg fuzzing -Cdebuginfo=1 -Cforce-frame-pointers"
@@ -188,7 +202,7 @@ EOF
   export CXXFLAGS="$CXXFLAGS -fno-sanitize=leak"
 fi
 
-if [ "$SANITIZER" = "introspector" ]; then
+if [ "$SANITIZER" = "introspector" ] || [ "$RUST_SANITIZER" = "introspector" ]; then
   export AR=llvm-ar
   export NM=llvm-nm
   export RANLIB=llvm-ranlib
@@ -210,10 +224,24 @@ if [ "$SANITIZER" = "introspector" ]; then
 
   apt-get install -y libjpeg-dev zlib1g-dev libyaml-dev
   python3 -m pip install --upgrade pip setuptools
-  python3 -m pip install cxxfilt pyyaml beautifulsoup4 lxml soupsieve
+  python3 -m pip install cxxfilt pyyaml beautifulsoup4 lxml soupsieve rust-demangler
   python3 -m pip install --prefer-binary matplotlib
 
-  python3 /fuzz-introspector/src/main.py light
+  # Install Fuzz-Introspector
+  pushd /fuzz-introspector/src
+  python3 -m pip install .
+  popd
+
+  if [ "$FUZZING_LANGUAGE" = "python" ]; then
+    python3 /fuzz-introspector/src/main.py light --language=python
+  elif [ "$FUZZING_LANGUAGE" = "jvm" ]; then
+    python3 /fuzz-introspector/src/main.py light --language=jvm
+  elif [ "$FUZZING_LANGUAGE" = "rust" ]; then
+    python3 /fuzz-introspector/src/main.py light --language=rust
+  else
+    python3 /fuzz-introspector/src/main.py light
+  fi
+
   rsync -avu --delete "$SRC/inspector/" "$OUT/inspector"
 fi
 
@@ -280,7 +308,7 @@ else
   fi
 fi
 
-if [ "$SANITIZER" = "introspector" ]; then
+if [ "$SANITIZER" = "introspector" ] || [ "$RUST_SANITIZER" = "introspector" ]; then
   unset CXXFLAGS
   unset CFLAGS
   export G_ANALYTICS_TAG="G-8WTFM1Y62J"
@@ -295,6 +323,21 @@ if [ "$SANITIZER" = "introspector" ]; then
     mkdir -p $SRC/my-fi-data
     find $OUT/ -name *.data -exec mv {} $SRC/my-fi-data/ \;
     find $OUT/ -name *.data.yaml -exec mv {} $SRC/my-fi-data/ \;
+  elif [ "$FUZZING_LANGUAGE" = "rust" ]; then
+    echo "GOING rust route"
+
+    # Run the rust frontend
+    pushd /fuzz-introspector/frontends/rust/rust_function_analyser
+    cargo run -- $SRC
+
+    # Move files temporarily to fix workflow of other languages.
+    mkdir -p $SRC/my-fi-data
+    find ./ -name "*.data" -exec mv {} $SRC/my-fi-data/ \;
+    find ./ -name "*.data.yaml" -exec mv {} $SRC/my-fi-data/ \;
+    popd
+
+    # Restore the sanitizer flag for rust
+    export SANITIZER="introspector"
   fi
 
   mkdir -p $SRC/inspector
@@ -335,6 +378,12 @@ if [ "$SANITIZER" = "introspector" ]; then
     REPORT_ARGS="$REPORT_ARGS --language=jvm"
     python3 /fuzz-introspector/src/main.py report $REPORT_ARGS
     rsync -avu --delete "$SRC/inspector/" "$OUT/inspector"
+  elif [ "$FUZZING_LANGUAGE" = "rust" ]; then
+    echo "GOING rust route"
+    REPORT_ARGS="$REPORT_ARGS --target_dir=$SRC/inspector"
+    REPORT_ARGS="$REPORT_ARGS --language=rust"
+    python3 /fuzz-introspector/src/main.py report $REPORT_ARGS
+    rsync -avu --delete "$SRC/inspector/" "$OUT/inspector"
   else
     # C/C++
 

infra/base-images/base-builder/install_go.sh

@@ -16,9 +16,11 @@
 ################################################################################
 
 cd /tmp
-wget https://go.dev/dl/go1.22.1.linux-amd64.tar.gz
+
+wget https://go.dev/dl/go1.23.4.linux-amd64.tar.gz
 mkdir temp-go
-tar -C temp-go/ -xzf go1.22.1.linux-amd64.tar.gz
+tar -C temp-go/ -xzf go1.23.4.linux-amd64.tar.gz
+
 mkdir /root/.go/
 mv temp-go/go/* /root/.go/
 rm -rf temp-go

infra/base-images/base-clang/Dockerfile

@@ -36,7 +36,7 @@ RUN apt-get update && apt-get install -y wget sudo && \
 RUN apt-get update && apt-get install -y git && \
     git clone https://github.com/ossf/fuzz-introspector.git fuzz-introspector && \
     cd fuzz-introspector && \
-    git checkout 5924aea8bcfe1fbdac9dc815adff91d3ee51f52b && \
+    git checkout 74917384c5a4e368d900862b4bd3d16ce3fe5dd8 && \
     git submodule init && \
     git submodule update && \
     apt-get autoremove --purge -y git && \

infra/build/functions/build_and_run_coverage.py

@@ -46,7 +46,7 @@
     'c', 'c++', 'go', 'jvm', 'rust', 'swift', 'python'
 ]
 
-LANGUAGES_WITH_INTROSPECTOR_SUPPORT = ['c', 'c++', 'python', 'jvm']
+LANGUAGES_WITH_INTROSPECTOR_SUPPORT = ['c', 'c++', 'python', 'jvm', 'rust']
 
 
 class Bucket:  # pylint: disable=too-few-public-methods

infra/build/functions/build_and_run_coverage_test.py

@@ -39,6 +39,9 @@ class TestRequestCoverageBuilds(fake_filesystem_unittest.TestCase):
   def setUp(self):
     self.maxDiff = None  # pylint: disable=invalid-name
     self.setUpPyfakefs()
+    self.patcher = mock.patch('build_lib.get_unique_build_step_image_id',
+                              return_value='UNIQUE_ID')
+    self.mock_function = self.patcher.start()
 
   @mock.patch('build_lib.get_signed_url', return_value='test_url')
   @mock.patch('build_lib.download_corpora_steps',

infra/build/functions/build_lib.py

@@ -275,11 +275,13 @@ def download_coverage_data_steps(project_name, latest, bucket_name, out_dir):
   bucket_url = f'gs://{bucket_name}/{project_name}/textcov_reports/{latest}/*'
   steps.append({
       'name': 'gcr.io/cloud-builders/gsutil',
-      'args': ['-m', 'cp', '-r', bucket_url, coverage_data_path]
+      'args': ['-m', 'cp', '-r', bucket_url, coverage_data_path],
+      'allowFailure': True
   })
   steps.append({
       'name': 'gcr.io/oss-fuzz-base/base-runner',
-      'args': ['bash', '-c', f'ls -lrt {out_dir}/textcov_reports']
+      'args': ['bash', '-c', f'ls -lrt {out_dir}/textcov_reports'],
+      'allowFailure': True
   })
 
   return steps
@@ -383,6 +385,10 @@ def _make_image_name_architecture_specific(image_name, architecture):
   return f'{image_name}-{architecture.lower()}'
 
 
+def get_unique_build_step_image_id():
+  return uuid.uuid4()
+
+
 def get_docker_build_step(image_names,
                           directory,
                           use_buildkit_cache=False,
@@ -408,13 +414,14 @@ def get_docker_build_step(image_names,
   if cache_image:
     args.extend(['--build-arg', f'CACHE_IMAGE={cache_image}'])
 
-  for image_name in image_names:
+  for image_name in sorted(image_names):
     args.extend(['--tag', image_name])
 
   step = {
       'name': DOCKER_TOOL_IMAGE,
       'args': args,
       'dir': directory,
+      'id': f'build-{get_unique_build_step_image_id()}',
   }
   # Handle buildkit args
   # Note that we mutate "args" after making it a value in step.
@@ -460,10 +467,11 @@ def get_project_image_steps(  # pylint: disable=too-many-arguments
     steps.extend(get_pull_test_images_steps(config.test_image_suffix))
   src_root = 'oss-fuzz' if not experiment else '.'
 
-  docker_build_step = get_docker_build_step([image],
-                                            os.path.join('projects', name),
-                                            src_root=src_root,
-                                            cache_image=cache_image)
+  docker_build_step = get_docker_build_step(
+      [image, _get_unsafe_name(name)],
+      os.path.join('projects', name),
+      src_root=src_root,
+      cache_image=cache_image)
   steps.append(docker_build_step)
   if srcmap:
     srcmap_step_id = get_srcmap_step_id()
@@ -475,7 +483,7 @@ def get_project_image_steps(  # pylint: disable=too-many-arguments
         ],
         'env': [
             'OSSFUZZ_REVISION=$REVISION_ID',
-            'FUZZING_LANGUAGE=%s' % language,
+            f'FUZZING_LANGUAGE={language}',
         ],
         'id': srcmap_step_id
     }])
@@ -496,15 +504,30 @@ def get_project_image_steps(  # pylint: disable=too-many-arguments
             'args': ['buildx', 'use', builder_name]
         },
     ])
-    docker_build_arm_step = get_docker_build_step([image],
-                                                  os.path.join(
-                                                      'projects', name),
-                                                  architecture=_ARM64)
+    docker_build_arm_step = get_docker_build_step(
+        [image, _get_unsafe_name(name)],
+        os.path.join('projects', name),
+        architecture=_ARM64)
     steps.append(docker_build_arm_step)
 
+  if (config.build_type == 'fuzzing' and language in ('c', 'c++')):
+    # Push so that historical bugs are reproducible.
+    push_step = {
+        'name': 'gcr.io/cloud-builders/docker',
+        'args': ['push', _get_unsafe_name(name)],
+        'id': 'push-image',
+        'waitFor': [docker_build_step['id']],
+        'allowFailure': True
+    }
+    steps.append(push_step)
+
   return steps
 
 
+def _get_unsafe_name(name):
+  return f'us-central1-docker.pkg.dev/oss-fuzz/unsafe/{name}'
+
+
 def get_logs_url(build_id):
   """Returns url that displays the build logs."""
   return (

infra/build/functions/build_project.py

@@ -19,8 +19,6 @@
 Usage: build_project.py <project_dir>
 """
 
-from __future__ import print_function
-
 import argparse
 from dataclasses import dataclass
 import datetime
@@ -77,6 +75,7 @@ class Config:
   experiment: bool = False
   # TODO(ochang): This should be different per engine+sanitizer combination.
   upload_build_logs: str = None
+  build_type: str = None
 
 
 WORKDIR_REGEX = re.compile(r'\s*WORKDIR\s*([^\s]+)')
@@ -476,7 +475,6 @@ def get_build_steps_for_project(project,
           upload_steps = get_upload_steps(project, build, timestamp,
                                           config.testing)
           build_steps.extend(upload_steps)
-
   return build_steps
 
 
@@ -629,15 +627,16 @@ def get_args(description):
   return parser.parse_args()
 
 
-def create_config_from_commandline(args):
+def create_config(args, build_type):
   """Create a Config object from parsed command line |args|."""
   upload = not args.experiment
   return Config(testing=args.testing,
                 test_image_suffix=args.test_image_suffix,
                 branch=args.branch,
                 parallel=args.parallel,
                 upload=upload,
-                experiment=args.experiment)
+                experiment=args.experiment,
+                build_type=build_type)
 
 
 def build_script_main(script_description, get_build_steps_func, build_type):
@@ -650,7 +649,7 @@ def build_script_main(script_description, get_build_steps_func, build_type):
 
   credentials = oauth2client.client.GoogleCredentials.get_application_default()
   error = False
-  config = create_config_from_commandline(args)
+  config = create_config(args, build_type)
   for project_name in args.projects:
     logging.info('Getting steps for: "%s".', project_name)
     try:

infra/build/functions/test_data/expected_build_steps.json

@@ -14,9 +14,12 @@
       "build",
       "--tag",
       "gcr.io/oss-fuzz/test-project",
+      "--tag",
+      "us-central1-docker.pkg.dev/oss-fuzz/unsafe/test-project",
       "."
     ],
-    "dir": "oss-fuzz/projects/test-project"
+    "dir": "oss-fuzz/projects/test-project",
+    "id": "build-UNIQUE_ID"
   },
   {
     "name": "gcr.io/oss-fuzz/test-project",
@@ -68,9 +71,12 @@
       "--load",
       "--tag",
       "gcr.io/oss-fuzz/test-project-aarch64",
+      "--tag",
+      "us-central1-docker.pkg.dev/oss-fuzz/unsafe/test-project-aarch64",
       "."
     ],
-    "dir": "oss-fuzz/projects/test-project"
+    "dir": "oss-fuzz/projects/test-project",
+    "id": "build-UNIQUE_ID"
   },
   {
     "name": "gcr.io/cloud-builders/docker",