test: Update code coverage to grcov

Jonathan Woollett-Light · pb8o · Jonathan Woollett-Light · commit 29786a59bb26 · 2023-01-09T17:39:59.000Z
Moved code coverage from kcov to grcov.

Updated `verify_dependencies.rs` as no longer proxying through
`cargo kcov` and the environment variable `CARGO_MANIFEST_DIR` is now
always set.

Signed-off-by: Jonathan Woollett-Light &lt;jcawl@amazon.co.uk&gt;
Co-authored-by: Pablo Barbáchano &lt;pablob@amazon.com&gt;
diff --git a/.gitignore b/.gitignore
@@ -9,3 +9,4 @@ __pycache__
 .vscode
 test_results/*
 *.core
+*.profraw
diff --git a/src/firecracker/tests/verify_dependencies.rs b/src/firecracker/tests/verify_dependencies.rs
@@ -12,12 +12,7 @@ fn test_no_comparison_requirements() {
     // HashMap mapping crate -> [(violating dependency, specified version)]
     let mut violating_dependencies = HashMap::new();
 
-    let src_firecracker_path = match std::env::var("CARGO_MANIFEST_DIR") {
-        Ok(path) => path,
-        Err(_) => return, /* when running under kcov, this variable is not set. As we do not
-                           * actually run any firecracker code, we can just skip this test
-                           * without affecting coverage */
-    };
+    let src_firecracker_path = std::env::var("CARGO_MANIFEST_DIR").unwrap();
     let src_path = format!("{}/..", src_firecracker_path);
 
     for fc_crate in std::fs::read_dir(src_path).unwrap() {
diff --git a/tests/integration_tests/build/test_coverage.py b/tests/integration_tests/build/test_coverage.py
@@ -2,15 +2,9 @@
 # SPDX-License-Identifier: Apache-2.0
 """Tests enforcing code coverage for production code."""
 
-
-import os
-import platform
-import re
-import shutil
 import pytest
 
 from framework import utils
-import host_tools.cargo_build as host  # pylint: disable=import-error
 from host_tools import proc
 
 # We have different coverages based on the host kernel version. This is
@@ -22,116 +16,99 @@
 # this contains the frequency while on AMD it does not.
 # Checkout the cpuid crate. In the future other
 # differences may appear.
-if utils.is_io_uring_supported():
-    COVERAGE_DICT = {"Intel": 82.90, "AMD": 82.23, "ARM": 82.50}
+if utils.is_io_uring_supported():a
+    COVERAGE_DICT = {"Intel": 83.81, "AMD": 82.35, "ARM": 83.29}
 else:
-    COVERAGE_DICT = {"Intel": 79.98, "AMD": 79.35, "ARM": 79.63}
+    COVERAGE_DICT = {"Intel": 80.09, "AMD": 78.62, "ARM": 79.32}
 
 PROC_MODEL = proc.proc_type()
 
-COVERAGE_MAX_DELTA = 0.05
-
-CARGO_KCOV_REL_PATH = os.path.join(host.CARGO_BUILD_REL_PATH, "kcov")
-
-KCOV_COVERAGE_FILE = "index.js"
-"""kcov will aggregate coverage data in this file."""
-
-KCOV_COVERED_LINES_REGEX = r'"covered_lines":"(\d+)"'
-"""Regex for extracting number of total covered lines found by kcov."""
+# Toolchain target architecture.
+if ("Intel" in PROC_MODEL) or ("AMD" in PROC_MODEL):
+    ARCH = "x86_64"
+elif "ARM" in PROC_MODEL:
+    ARCH = "aarch64"
+else:
+    raise Exception(f"Unsupported processor model ({PROC_MODEL})")
 
-KCOV_TOTAL_LINES_REGEX = r'"total_lines" : "(\d+)"'
-"""Regex for extracting number of total executable lines found by kcov."""
+# Toolchain target.
+# Currently profiling with `aarch64-unknown-linux-musl` is unsupported (see
+# https://github.com/rust-lang/rustup/issues/3095#issuecomment-1280705619) therefore we profile and
+# run coverage with the `gnu` toolchains and run unit tests with the `musl` toolchains.
+TARGET = f"{ARCH}-unknown-linux-gnu"
 
-SECCOMPILER_BUILD_DIR = "../build/seccompiler"
+# We allow coverage to have a max difference of `COVERAGE_MAX_DELTA` as percentage before failing
+# the test (currently 0.05%).
+COVERAGE_MAX_DELTA = 0.0005
 
 
 @pytest.mark.timeout(400)
-def test_coverage(test_fc_session_root_path, test_session_tmp_path, record_property):
-    """Test line coverage for rust tests is within bounds.
-
-    The result is extracted from the $KCOV_COVERAGE_FILE file created by kcov
-    after a coverage run.
+def test_coverage(monkeypatch, record_property):
+    """Test code coverage
 
     @type: build
     """
-    proc_model = [item for item in COVERAGE_DICT if item in PROC_MODEL]
-    assert len(proc_model) == 1, "Could not get processor model!"
-    coverage_target_pct = COVERAGE_DICT[proc_model[0]]
-    exclude_pattern = (
-        "${CARGO_HOME:-$HOME/.cargo/},"
-        "build/,"
-        "tests/,"
-        "usr/lib/gcc,"
-        "lib/x86_64-linux-gnu/,"
-        "test_utils.rs,"
-        # The following files/directories are auto-generated
-        "bootparam.rs,"
-        "elf.rs,"
-        "mpspec.rs,"
-        "msr_index.rs,"
-        "bindings.rs,"
-        "_gen"
+    # Get coverage target.
+    processor_model = [item for item in COVERAGE_DICT if item in PROC_MODEL]
+    assert len(processor_model) == 1, "Could not get processor model!"
+    coverage_target = COVERAGE_DICT[processor_model[0]]
+
+    # Re-direct to repository root.
+    monkeypatch.chdir("..")
+
+    # Generate test profiles.
+    utils.run_cmd(
+        f'\
+        RUSTFLAGS="-Cinstrument-coverage" \
+        LLVM_PROFILE_FILE="coverage-%p-%m.profraw" \
+        cargo test --all --target={TARGET} -- --test-threads=1 \
+    '
     )
-    exclude_region = "'mod tests {'"
-    target = "{}-unknown-linux-musl".format(platform.machine())
-
-    cmd = (
-        'CARGO_WRAPPER="kcov" RUSTFLAGS="{}" CARGO_TARGET_DIR={} '
-        "cargo kcov --all "
-        "--target {} --output {} -- "
-        "--exclude-pattern={} "
-        "--exclude-region={} --verify"
-    ).format(
-        host.get_rustflags(),
-        os.path.join(test_fc_session_root_path, CARGO_KCOV_REL_PATH),
-        target,
-        test_session_tmp_path,
-        exclude_pattern,
-        exclude_region,
-    )
-    # We remove the seccompiler custom build directory, created by the
-    # vmm-level `build.rs`.
-    # If we don't delete it before and after running the kcov command, we will
-    # run into linker errors.
-    shutil.rmtree(SECCOMPILER_BUILD_DIR, ignore_errors=True)
-    # By default, `cargo kcov` passes `--exclude-pattern=$CARGO_HOME --verify`
-    # to kcov. To pass others arguments, we need to include the defaults.
-    utils.run_cmd(cmd)
-
-    shutil.rmtree(SECCOMPILER_BUILD_DIR)
-
-    coverage_file = os.path.join(test_session_tmp_path, KCOV_COVERAGE_FILE)
-    with open(coverage_file, encoding="utf-8") as cov_output:
-        contents = cov_output.read()
-        covered_lines = int(re.findall(KCOV_COVERED_LINES_REGEX, contents)[0])
-        total_lines = int(re.findall(KCOV_TOTAL_LINES_REGEX, contents)[0])
-        coverage = covered_lines / total_lines * 100
-    print("Number of executable lines: {}".format(total_lines))
-    print("Number of covered lines: {}".format(covered_lines))
-    print("Thus, coverage is: {:.2f}%".format(coverage))
-
-    coverage_low_msg = (
-        "Current code coverage ({:.2f}%) is >{:.2f}% below the target ({}%).".format(
-            coverage, COVERAGE_MAX_DELTA, coverage_target_pct
-        )
-    )
-
-    assert coverage >= coverage_target_pct - COVERAGE_MAX_DELTA, coverage_low_msg
 
-    # Get the name of the variable that needs updating.
-    namespace = globals()
-    cov_target_name = [name for name in namespace if namespace[name] is COVERAGE_DICT][
-        0
-    ]
+    # Generate coverage report.
+    utils.run_cmd(
+        f"\
+        grcov . \
+            -s . \
+            --binary-path ./build/cargo_target/{TARGET}/debug/ \
+            --excl-start \"mod tests\" \
+            --ignore \"['build/*', 'tests/*']\" \
+            -t html \
+            --ignore-not-existing \
+            -o ./build/cargo_target/{TARGET}/debug/coverage \
+    "
+    )
 
-    coverage_high_msg = (
-        "Current code coverage ({:.2f}%) is >{:.2f}% above the target ({}%).\n"
-        "Please update the value of {}.".format(
-            coverage, COVERAGE_MAX_DELTA, coverage_target_pct, cov_target_name
-        )
+    # Extract coverage from html report.
+    #
+    # The line looks like `<abbr title="44724 / 49237">90.83 %</abbr></p>` and is the first
+    # occurrence of the `<abbr>` element in the file.
+    #
+    # When we update grcov to 0.8.* we can update this to pull the coverage from a generated .json
+    # file.
+    index = open(
+        f"./build/cargo_target/{TARGET}/debug/coverage/index.html", encoding="utf-8"
     )
+    index_contents = index.read()
+    end = index_contents.find(" %</abbr></p>")
+    start = index_contents[:end].rfind(">")
+    coverage_str = index_contents[start + 1 : end]
+    coverage = float(coverage_str)
+
+    # Compare coverage.
+    high = coverage_target * (1.0 + COVERAGE_MAX_DELTA)
+    low = coverage_target * (1.0 - COVERAGE_MAX_DELTA)
 
+    # Record coverage.
     record_property(
-        "coverage", f"{coverage}% {coverage_target_pct}% ±{COVERAGE_MAX_DELTA:.2}%"
+        "coverage", f"{coverage}% {coverage_target}% ±{COVERAGE_MAX_DELTA:.2}%"
     )
-    assert coverage <= coverage_target_pct + COVERAGE_MAX_DELTA, coverage_high_msg
+
+    # Assert coverage within delta.
+    coverage_max_delta_percent = 100 * COVERAGE_MAX_DELTA
+    assert (
+        coverage >= low
+    ), f"Current code coverage ({coverage:.2f}%) is more than {coverage_max_delta_percent:.2f}% below the target ({coverage_target:.2f}%)"
+    assert (
+        coverage <= high
+    ), f"Current code coverage ({coverage:.2f}%) is more than {coverage_max_delta_percent:.2f}% above the target ({coverage_target:.2f}%)"
diff --git a/tests/integration_tests/build/test_unittests.py b/tests/integration_tests/build/test_unittests.py
@@ -7,9 +7,10 @@
 import host_tools.cargo_build as host  # pylint:disable=import-error
 
 MACHINE = platform.machine()
-# No need to run unittests for musl since
-# we run coverage with musl for all platforms.
-TARGET = "{}-unknown-linux-gnu".format(MACHINE)
+# Currently profiling with `aarch64-unknown-linux-musl` is unsupported (see
+# https://github.com/rust-lang/rustup/issues/3095#issuecomment-1280705619) therefore we profile and
+# run coverage with the `gnu` toolchains and run unit tests with the `musl` toolchains.
+TARGET = "{}-unknown-linux-musl".format(MACHINE)
 
 
 def test_unittests(test_fc_session_root_path):
diff --git a/tools/devctr/Dockerfile.aarch64 b/tools/devctr/Dockerfile.aarch64
@@ -15,6 +15,9 @@ ARG CARGO_GIT_REGISTRY_DIR="$FIRECRACKER_BUILD_DIR/cargo_git_registry"
 ARG DEBIAN_FRONTEND=noninteractive
 # By default we don't provide a poetry.lock file
 ARG POETRY_LOCK_PATH="/dev/null/*"
+# grcov 0.8.* requires GLIBC >2.27, this is not present in ubuntu 18.04, when we update the docker
+# container with a newer version of ubuntu we can also update this.
+ARG GRCOV_VERSION="0.7.1"
 
 ENV CARGO_HOME=/usr/local/rust
 ENV RUSTUP_HOME=/usr/local/rust
@@ -101,20 +104,10 @@ RUN cd "$TMP_POETRY_DIR" \
     &&  poetry install --no-dev --no-interaction
 
 # Install the Rust toolchain
-#
-RUN mkdir "$TMP_BUILD_DIR" \
-    && curl https://sh.rustup.rs -sSf | sh -s -- -y --profile minimal --default-toolchain "$RUST_TOOLCHAIN" \
-        && rustup target add aarch64-unknown-linux-musl \
-        && rustup component add clippy \
-        && cd "$TMP_BUILD_DIR" \
-                    && cargo install cargo-kcov \
-                    && cargo kcov --print-install-kcov-sh | sh \
-        && rm -rf "$CARGO_HOME/registry" \
-        && ln -s "$CARGO_REGISTRY_DIR" "$CARGO_HOME/registry" \
-        && rm -rf "$CARGO_HOME/git" \
-        && ln -s "$CARGO_GIT_REGISTRY_DIR" "$CARGO_HOME/git" \
-    && cd / \
-    && rm -rf "$TMP_BUILD_DIR"
+RUN curl https://sh.rustup.rs -sSf | sh -s -- -y --default-toolchain "$RUST_TOOLCHAIN" \
+    && rustup target add aarch64-unknown-linux-musl \
+    && rustup component add llvm-tools-preview \
+    && cargo install --version $GRCOV_VERSION grcov
 
 RUN ln -s /usr/bin/musl-gcc /usr/bin/aarch64-linux-musl-gcc
 
diff --git a/tools/devctr/Dockerfile.x86_64 b/tools/devctr/Dockerfile.x86_64
@@ -15,6 +15,9 @@ ARG CARGO_GIT_REGISTRY_DIR="$FIRECRACKER_BUILD_DIR/cargo_git_registry"
 ARG DEBIAN_FRONTEND=noninteractive
 # By default we don't provide a poetry.lock file
 ARG POETRY_LOCK_PATH="/dev/null/*"
+# grcov 0.8.* requires GLIBC >2.27, this is not present in ubuntu 18.04, when we update the docker
+# container with a newer version of ubuntu we can also update this.
+ARG GRCOV_VERSION="0.7.1"
 
 ENV CARGO_HOME=/usr/local/rust
 ENV RUSTUP_HOME=/usr/local/rust
@@ -106,24 +109,12 @@ RUN (curl -sL https://deb.nodesource.com/setup_14.x | bash) \
     && rm -rf /var/lib/apt/lists/*
 
 # Install the Rust toolchain
-#
-RUN mkdir "$TMP_BUILD_DIR" \
-    && curl https://sh.rustup.rs -sSf | sh -s -- -y --profile minimal --default-toolchain "$RUST_TOOLCHAIN" \
-        && rustup target add x86_64-unknown-linux-musl \
-        && rustup component add rustfmt clippy clippy-preview \
-        && rustup install --profile minimal "stable" \
-        && cd "$TMP_BUILD_DIR" \
-            && cargo install cargo-kcov \
-            && cargo +"stable" install cargo-audit \
-            # Fix a version that does not require cargo edition 2021.
-            && cargo install --locked cargo-deny --version '^0.9.1' \
-            && cargo kcov --print-install-kcov-sh | sh \
-        && rm -rf "$CARGO_HOME/registry" \
-        && ln -s "$CARGO_REGISTRY_DIR" "$CARGO_HOME/registry" \
-        && rm -rf "$CARGO_HOME/git" \
-        && ln -s "$CARGO_GIT_REGISTRY_DIR" "$CARGO_HOME/git" \
-    && cd / \
-    && rm -rf "$TMP_BUILD_DIR"
+RUN curl -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain "$RUST_TOOLCHAIN" \
+    && rustup target add x86_64-unknown-linux-musl \
+    && rustup component add llvm-tools-preview \
+    && cargo install cargo-audit \
+    && cargo install --locked cargo-deny \
+    && cargo install --version $GRCOV_VERSION grcov
 
 # help musl-gcc find linux headers
 RUN cd /usr/include/x86_64-linux-musl \
diff --git a/tools/devctr/poetry.lock b/tools/devctr/poetry.lock
diff --git a/tools/devtool b/tools/devtool
