Cirrus: Improve caching effectiveness

Prior to this commit, the cache maintained for the main branch has grown
to over 16gigabytes.  Uploading it is causing frequent branch-level CI
failures due to being larger than the maximum allowable size by Cirrus.
Attempt to fix this by introducing a cache-grooming script and altering
the cache-key being used.  Content for the grooming script comes mainly
from two places:

1. [Cargo
   documentation](https://doc.rust-lang.org/nightly/cargo/guide/build-cache.html)
2. The rust-cache github action [(typescript) source
   code](https://github.com/Swatinem/rust-cache/tree/master/src)

Signed-off-by: Chris Evich <cevich@redhat.com>

Author: cevich

.cirrus.yml

@@ -54,12 +54,11 @@ build_task:
     # Required to be set explicitly since fingerprint_key is also set
     reupload_on_changes: true
   targets_cache: &targets_cache
-    # Similar to cargo_cache, but holds the actual compiled artifacts. This must
-    # be scoped similar to bin_cache to avoid binary pollution across cache
-    # contexts.  For example, two PRs that happen to coincidentally change
-    # and use cache.  Adjust vX if cache schema changes.
+    # Similar to cargo_cache, but holds the actual compiled dependent artifacts.
+    # This should be scoped to a hash of the dependency-metadata lock file.
+    # Cirrus-CI will automatically use separate caches for PRs and branches.
     folder: "$CARGO_TARGET_DIR"
-    fingerprint_key: "targets_v2_${CIRRUS_TAG}${DEST_BRANCH}${CIRRUS_PR}_amd64" # Cache only within same tag, branch, or PR (branch will be 'pull/#')
+    fingerprint_script: echo "targets_v3_${CIRRUS_TAG}${DEST_BRANCH}${CIRRUS_PR}_amd64: $(<Cargo.lock) : $(<Cargo.toml)"
     reupload_on_changes: true
   bin_cache: &bin_cache
     # This simply prevents rebuilding bin/netavark for every subsequent task.
@@ -70,6 +69,7 @@ build_task:
     reupload_on_changes: true
   setup_script: &setup "$SCRIPT_BASE/setup.sh"
   main_script: &main "$SCRIPT_BASE/runner.sh $CIRRUS_TASK_NAME"
+  cache_grooming_script: &groom bash "$SCRIPT_BASE/cache_groom.sh"
   upload_caches: [ "cargo", "targets", "bin" ]
 
 
@@ -86,7 +86,7 @@ build_aarch64_task:
     reupload_on_changes: true
   targets_cache: &targets_cache_aarch64
     folder: "$CARGO_TARGET_DIR"
-    fingerprint_key: "targets_v2_${CIRRUS_TAG}${DEST_BRANCH}${CIRRUS_PR}_aarch64" # Cache only within same tag, branch, or PR (branch will be 'pull/#')
+    fingerprint_script: echo "targets_v3_${CIRRUS_TAG}${DEST_BRANCH}${CIRRUS_PR}_aarch64: $(<Cargo.lock) : $(<Cargo.toml)"
     reupload_on_changes: true
   bin_cache: &bin_cache_aarch64
     # This simply prevents rebuilding bin/netavark for every subsequent task.
@@ -95,6 +95,7 @@ build_aarch64_task:
     reupload_on_changes: true
   setup_script: *setup
   main_script: *main
+  cache_grooming_script: *groom
   upload_caches: [ "cargo", "targets", "bin" ]
   # Downstream CI needs the aarch64 binaries from this CI system.
   # However, we don't want to confuse architectures.

contrib/cirrus/cache_groom.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+#
+# This script is intended to be run from Cirrus-CI to prepare the
+# rust targets cache for re-use during subsequent runs.  This mainly
+# involves removing files and directories which change frequently
+# but are cheap/quick to regenerate - i.e. prevent "cache-flapping".
+# Any other use of this script is not supported and may cause harm.
+
+set -eo pipefail
+
+source $(dirname ${BASH_SOURCE[0]})/lib.sh
+
+if [[ "$CIRRUS_CI" != true ]]; then
+  die "Script is not intended for use outside of Cirrus-CI"
+fi
+
+req_env_vars CARGO_TARGET_DIR
+cd $CARGO_TARGET_DIR
+
+# Ref: https://doc.rust-lang.org/nightly/cargo/guide/build-cache.html
+for trgname in $(ls -1); do
+  echo "Grooming $CARGO_TARGET_DIR/$trgname..."
+  cd $CARGO_TARGET_DIR/$trgname
+  # Example targets
+  rm -rf ./target/debug/examples
+  # Documentation
+  rm -rf ./target/doc
+  # Internal to rust build process
+  rm -rf ./target/debug/deps ./target/debug/incremental ./target/debug/build
+done
+cd $CARGO_TARGET_DIR

contrib/cirrus/lib.sh

@@ -32,6 +32,12 @@ else  # set default values - see make_cienv() below
     # VM Images are built with this setup
     CARGO_HOME="${CARGO_HOME:-/var/cache/cargo}"
     source $CARGO_HOME/env
+
+    # Make caching more effective - disable incremental compilation,
+    # so that the Rust compiler doesn't waste time creating the
+    # additional artifacts required for incremental builds.
+    # Ref: https://github.com/marketplace/actions/rust-cache#cache-details
+    CARGO_INCREMENTAL=0
 fi
 
 # END Global export of all variables