test: Update code coverage to grcov

Moved code coverage from kcov to grcov.

Updated `verify_dependencies.rs` as no longer proxying through
`cargo kcov` and the environment variable `CARGO_MANIFEST_DIR`
is now always set.

Signed-off-by: Jonathan Woollett-Light <[email protected]>
Co-authored-by: Pablo Barbáchano <[email protected]>

Author: Jonathan Woollett-Light

.gitignore

@@ -9,3 +9,4 @@ __pycache__
 .vscode
 test_results/*
 *.core
+*.profraw

src/firecracker/tests/verify_dependencies.rs

@@ -12,12 +12,7 @@ fn test_no_comparison_requirements() {
     // HashMap mapping crate -> [(violating dependency, specified version)]
     let mut violating_dependencies = HashMap::new();
 
-    let src_firecracker_path = match std::env::var("CARGO_MANIFEST_DIR") {
-        Ok(path) => path,
-        Err(_) => return, /* when running under kcov, this variable is not set. As we do not
-                           * actually run any firecracker code, we can just skip this test
-                           * without affecting coverage */
-    };
+    let src_firecracker_path = std::env::var("CARGO_MANIFEST_DIR").unwrap();
     let src_path = format!("{}/..", src_firecracker_path);
 
     for fc_crate in std::fs::read_dir(src_path).unwrap() {

tests/integration_tests/build/test_coverage.py

@@ -2,15 +2,9 @@
 # SPDX-License-Identifier: Apache-2.0
 """Tests enforcing code coverage for production code."""
 
-
-import os
-import platform
-import re
-import shutil
 import pytest
 
 from framework import utils
-import host_tools.cargo_build as host  # pylint: disable=import-error
 from host_tools import proc
 
 # We have different coverages based on the host kernel version. This is
@@ -29,109 +23,92 @@
 
 PROC_MODEL = proc.proc_type()
 
-COVERAGE_MAX_DELTA = 0.05
-
-CARGO_KCOV_REL_PATH = os.path.join(host.CARGO_BUILD_REL_PATH, "kcov")
-
-KCOV_COVERAGE_FILE = "index.js"
-"""kcov will aggregate coverage data in this file."""
+# Toolchain target architecture.
+if ("Intel" in PROC_MODEL) or ("AMD" in PROC_MODEL):
+    ARCH = "x86_64"
+elif "ARM" in PROC_MODEL:
+    ARCH = "aarch64"
+else:
+    raise Exception(f"Unsupported processor model ({PROC_MODEL})")
 
-KCOV_COVERED_LINES_REGEX = r'"covered_lines":"(\d+)"'
-"""Regex for extracting number of total covered lines found by kcov."""
+# Toolchain target.
+# Currently profiling with `aarch64-unknown-linux-musl` is unsupported (see
+# https://github.com/rust-lang/rustup/issues/3095#issuecomment-1280705619) therefore we profile and
+# run coverage with the `gnu` toolchains and run unit tests with the `musl` toolchains.
+TARGET = f"{ARCH}-unknown-linux-gnu"
 
-KCOV_TOTAL_LINES_REGEX = r'"total_lines" : "(\d+)"'
-"""Regex for extracting number of total executable lines found by kcov."""
+# We allow coverage to have a max difference of `COVERAGE_MAX_DELTA` as percentage before failing
+# the test.
+COVERAGE_MAX_DELTA = 0.05
 
-SECCOMPILER_BUILD_DIR = "../build/seccompiler"
+# grcov 0.8.* requires GLIBC >2.27, this is not present in ubuntu 18.04, when we update the docker
+# container with a newer version of ubuntu we can also update this.
+GRCOV_VERSION = "0.7.1"
 
 
 @pytest.mark.timeout(400)
-def test_coverage(test_fc_session_root_path, test_session_tmp_path, record_property):
-    """Test line coverage for rust tests is within bounds.
-
-    The result is extracted from the $KCOV_COVERAGE_FILE file created by kcov
-    after a coverage run.
+def test_coverage(monkeypatch):
+    """Test code coverage
 
     @type: build
     """
-    proc_model = [item for item in COVERAGE_DICT if item in PROC_MODEL]
-    assert len(proc_model) == 1, "Could not get processor model!"
-    coverage_target_pct = COVERAGE_DICT[proc_model[0]]
-    exclude_pattern = (
-        "${CARGO_HOME:-$HOME/.cargo/},"
-        "build/,"
-        "tests/,"
-        "usr/lib/gcc,"
-        "lib/x86_64-linux-gnu/,"
-        "test_utils.rs,"
-        # The following files/directories are auto-generated
-        "bootparam.rs,"
-        "elf.rs,"
-        "mpspec.rs,"
-        "msr_index.rs,"
-        "bindings.rs,"
-        "_gen"
-    )
-    exclude_region = "'mod tests {'"
-    target = "{}-unknown-linux-musl".format(platform.machine())
-
-    cmd = (
-        'CARGO_WRAPPER="kcov" RUSTFLAGS="{}" CARGO_TARGET_DIR={} '
-        "cargo kcov --all "
-        "--target {} --output {} -- "
-        "--exclude-pattern={} "
-        "--exclude-region={} --verify"
-    ).format(
-        host.get_rustflags(),
-        os.path.join(test_fc_session_root_path, CARGO_KCOV_REL_PATH),
-        target,
-        test_session_tmp_path,
-        exclude_pattern,
-        exclude_region,
-    )
-    # We remove the seccompiler custom build directory, created by the
-    # vmm-level `build.rs`.
-    # If we don't delete it before and after running the kcov command, we will
-    # run into linker errors.
-    shutil.rmtree(SECCOMPILER_BUILD_DIR, ignore_errors=True)
-    # By default, `cargo kcov` passes `--exclude-pattern=$CARGO_HOME --verify`
-    # to kcov. To pass others arguments, we need to include the defaults.
-    utils.run_cmd(cmd)
-
-    shutil.rmtree(SECCOMPILER_BUILD_DIR)
-
-    coverage_file = os.path.join(test_session_tmp_path, KCOV_COVERAGE_FILE)
-    with open(coverage_file, encoding="utf-8") as cov_output:
-        contents = cov_output.read()
-        covered_lines = int(re.findall(KCOV_COVERED_LINES_REGEX, contents)[0])
-        total_lines = int(re.findall(KCOV_TOTAL_LINES_REGEX, contents)[0])
-        coverage = covered_lines / total_lines * 100
-    print("Number of executable lines: {}".format(total_lines))
-    print("Number of covered lines: {}".format(covered_lines))
-    print("Thus, coverage is: {:.2f}%".format(coverage))
-
-    coverage_low_msg = (
-        "Current code coverage ({:.2f}%) is >{:.2f}% below the target ({}%).".format(
-            coverage, COVERAGE_MAX_DELTA, coverage_target_pct
-        )
+    # Get coverage target.
+    processor_model = [item for item in COVERAGE_DICT if item in PROC_MODEL]
+    assert len(processor_model) == 1, "Could not get processor model!"
+    coverage_target = COVERAGE_DICT[processor_model[0]]
+
+    # Re-direct to repository root.
+    monkeypatch.chdir("..")
+
+    # Generate test profiles.
+    utils.run_cmd(
+        f'\
+        env RUSTFLAGS="-Cinstrument-coverage" \
+        LLVM_PROFILE_FILE="coverage-%p-%m.profraw" \
+        cargo test --all --target={TARGET} -- --test-threads=1 \
+    '
     )
 
-    assert coverage >= coverage_target_pct - COVERAGE_MAX_DELTA, coverage_low_msg
-
-    # Get the name of the variable that needs updating.
-    namespace = globals()
-    cov_target_name = [name for name in namespace if namespace[name] is COVERAGE_DICT][
-        0
-    ]
-
-    coverage_high_msg = (
-        "Current code coverage ({:.2f}%) is >{:.2f}% above the target ({}%).\n"
-        "Please update the value of {}.".format(
-            coverage, COVERAGE_MAX_DELTA, coverage_target_pct, cov_target_name
-        )
+    # Generate coverage report.
+    utils.run_cmd(
+        f'\
+        cargo install --version {GRCOV_VERSION} grcov \
+        && grcov . \
+            -s . \
+            --binary-path ./build/cargo_target/{TARGET}/debug/ \
+            --excl-start "mod tests" \
+            --ignore "build/*" \
+            -t html \
+            --branch \
+            --ignore-not-existing \
+            -o ./build/cargo_target/{TARGET}/debug/coverage \
+    '
     )
 
-    record_property(
-        "coverage", f"{coverage}% {coverage_target_pct}% ±{COVERAGE_MAX_DELTA:.2}%"
+    # Extract coverage from html report.
+    #
+    # The line looks like `<abbr title="44724 / 49237">90.83 %</abbr></p>` and is the first
+    # occurrence of the `<abbr>` element in the file.
+    #
+    # When we update grcov to 0.8.* we can update this to pull the coverage from a generated .json
+    # file.
+    index = open(
+        f"./build/cargo_target/{TARGET}/debug/coverage/index.html", encoding="utf-8"
     )
-    assert coverage <= coverage_target_pct + COVERAGE_MAX_DELTA, coverage_high_msg
+    index_contents = index.read()
+    end = index_contents.find(" %</abbr></p>")
+    start = index_contents[:end].rfind(">")
+    coverage_str = index_contents[start + 1 : end]
+    coverage = float(coverage_str)
+
+    # Compare coverage.
+    high = coverage_target * (1.0 + COVERAGE_MAX_DELTA)
+    low = coverage_target * (1.0 - COVERAGE_MAX_DELTA)
+    assert (
+        coverage >= low
+    ), f"Current code coverage ({coverage:.2f}%) is more than {COVERAGE_MAX_DELTA:.2f}% below \
+            the target ({coverage_target:.2f}%)"
+    assert (
+        coverage <= high
+    ), f"Current code coverage ({coverage:.2f}%) is more than {COVERAGE_MAX_DELTA:.2f}% above \
+            the target ({coverage_target:.2f}%)"

tests/integration_tests/build/test_unittests.py

@@ -7,9 +7,10 @@
 import host_tools.cargo_build as host  # pylint:disable=import-error
 
 MACHINE = platform.machine()
-# No need to run unittests for musl since
-# we run coverage with musl for all platforms.
-TARGET = "{}-unknown-linux-gnu".format(MACHINE)
+# Currently profiling with `aarch64-unknown-linux-musl` is unsupported (see
+# https://github.com/rust-lang/rustup/issues/3095#issuecomment-1280705619) therefore we profile and
+# run coverage with the `gnu` toolchains and run unit tests with the `musl` toolchains.
+TARGET = "{}-unknown-linux-musl".format(MACHINE)
 
 
 def test_unittests(test_fc_session_root_path):

tools/devctr/Dockerfile.aarch64

@@ -105,10 +105,8 @@ RUN cd "$TMP_POETRY_DIR" \
 RUN mkdir "$TMP_BUILD_DIR" \
     && curl https://sh.rustup.rs -sSf | sh -s -- -y --profile minimal --default-toolchain "$RUST_TOOLCHAIN" \
         && rustup target add aarch64-unknown-linux-musl \
-        && rustup component add clippy \
+        && rustup component add clippy llvm-tools-preview \
         && cd "$TMP_BUILD_DIR" \
-                    && cargo install cargo-kcov \
-                    && cargo kcov --print-install-kcov-sh | sh \
         && rm -rf "$CARGO_HOME/registry" \
         && ln -s "$CARGO_REGISTRY_DIR" "$CARGO_HOME/registry" \
         && rm -rf "$CARGO_HOME/git" \

tools/devctr/Dockerfile.x86_64

@@ -110,14 +110,12 @@ RUN (curl -sL https://deb.nodesource.com/setup_14.x | bash) \
 RUN mkdir "$TMP_BUILD_DIR" \
     && curl https://sh.rustup.rs -sSf | sh -s -- -y --profile minimal --default-toolchain "$RUST_TOOLCHAIN" \
         && rustup target add x86_64-unknown-linux-musl \
-        && rustup component add rustfmt clippy clippy-preview \
+        && rustup component add rustfmt clippy clippy-preview llvm-tools-preview \
         && rustup install --profile minimal "stable" \
         && cd "$TMP_BUILD_DIR" \
-            && cargo install cargo-kcov \
             && cargo +"stable" install cargo-audit \
             # Fix a version that does not require cargo edition 2021.
             && cargo install --locked cargo-deny --version '^0.9.1' \
-            && cargo kcov --print-install-kcov-sh | sh \
         && rm -rf "$CARGO_HOME/registry" \
         && ln -s "$CARGO_REGISTRY_DIR" "$CARGO_HOME/registry" \
         && rm -rf "$CARGO_HOME/git" \

tools/devtool

@@ -72,7 +72,7 @@
 DEVCTR_IMAGE_NO_TAG="public.ecr.aws/firecracker/fcuvm"
 
 # Development container tag
-DEVCTR_IMAGE_TAG="v48"
+DEVCTR_IMAGE_TAG="v49"
 
 # Development container image (name:tag)
 # This should be updated whenever we upgrade the development container.