Implement coverage output

Author: Ari Breitkreuz

crates/cli/src/bin/wasm-bindgen-test-runner/deno.rs

@@ -1,7 +1,7 @@
-use std::ffi::OsString;
 use std::fs;
 use std::path::Path;
 use std::process::Command;
+use std::{ffi::OsString, path::PathBuf};
 
 use anyhow::{Context, Error};
 
@@ -12,12 +12,37 @@ pub fn execute(
     tmpdir: &Path,
     args: &[OsString],
     tests: &[String],
+    coverage: Option<PathBuf>,
 ) -> Result<(), Error> {
+    let (cov_fn, cov_dump) = if let Some(profraw_path) = coverage {
+        (
+            format!(
+                r#"
+            async function dumpCoverage() {{
+                const fs = require('node:fs');
+                const data = wasm.__wbgtest_cov_dump();
+                fs.writeFile("{}", data, err => {{
+                    if (err) {{
+                        console.error(err);
+                    }}
+                }})
+            }}
+            "#,
+                profraw_path.display()
+            ),
+            "await dumpCoverage();",
+        )
+    } else {
+        (String::new(), "")
+    };
+
     let mut js_to_execute = format!(
         r#"import * as wasm from "./{0}.js";
 
         {console_override}
 
+        {cov_fn}
+
         // global.__wbg_test_invoke = f => f();
 
         // Forward runtime arguments. These arguments are also arguments to the
@@ -27,6 +52,7 @@ pub fn execute(
         cx.args(Deno.args.slice(1));
 
         const ok = await cx.run(tests.map(n => wasm.__wasm[n]));
+        {cov_dump}
         if (!ok) Deno.exit(1);
 
         const tests = [];

crates/cli/src/bin/wasm-bindgen-test-runner/main.rs

@@ -15,6 +15,7 @@ use anyhow::{anyhow, bail, Context};
 use log::error;
 use std::env;
 use std::fs;
+use std::path::Path;
 use std::path::PathBuf;
 use std::thread;
 use wasm_bindgen_cli_support::Bindgen;
@@ -187,6 +188,8 @@ fn main() -> anyhow::Result<()> {
         b.split_linked_modules(true);
     }
 
+    let coverage = coverage_args(&tmpdir);
+
     b.debug(debug)
         .input_module(module, wasm)
         .keep_debug(false)
@@ -198,8 +201,8 @@ fn main() -> anyhow::Result<()> {
     let args: Vec<_> = args.collect();
 
     match test_mode {
-        TestMode::Node => node::execute(module, &tmpdir, &args, &tests)?,
-        TestMode::Deno => deno::execute(module, &tmpdir, &args, &tests)?,
+        TestMode::Node => node::execute(module, &tmpdir, &args, &tests, coverage)?,
+        TestMode::Deno => deno::execute(module, &tmpdir, &args, &tests, coverage)?,
         TestMode::Browser { no_modules } | TestMode::Worker { no_modules } => {
             let srv = server::spawn(
                 &if headless {
@@ -216,6 +219,7 @@ fn main() -> anyhow::Result<()> {
                 &tests,
                 no_modules,
                 matches!(test_mode, TestMode::Worker { no_modules: _ }),
+                coverage,
             )
             .context("failed to spawn server")?;
             let addr = srv.server_addr();
@@ -242,3 +246,29 @@ fn main() -> anyhow::Result<()> {
     }
     Ok(())
 }
+
+fn coverage_args(tmpdir: &Path) -> Option<PathBuf> {
+    fn generated(tmpdir: &Path) -> String {
+        format!(
+            "{}.profraw",
+            tmpdir.file_name().and_then(|s| s.to_str()).unwrap()
+        )
+    }
+
+    // Profraw path is ignored if coverage isn't enabled
+    env::var_os("WASM_BINDGEN_TEST_COVERAGE")?;
+    log::warn!("Coverage support is still considered highly experimental!");
+    // TODO coverage link to wasm-bindgen book documenting correct usage
+
+    match env::var_os("WASM_BINDGEN_TEST_PROFRAW_OUT") {
+        Some(s) => {
+            let mut buf = PathBuf::from(s);
+            if buf.is_dir() {
+                buf.push(generated(tmpdir));
+            }
+            buf
+        }
+        None => PathBuf::from(generated(tmpdir)),
+    }
+    .into()
+}

crates/cli/src/bin/wasm-bindgen-test-runner/node.rs

@@ -1,8 +1,8 @@
-use std::env;
 use std::ffi::OsString;
 use std::fs;
 use std::path::Path;
 use std::process::Command;
+use std::{env, path::PathBuf};
 
 use anyhow::{Context, Error};
 
@@ -43,7 +43,30 @@ pub fn execute(
     tmpdir: &Path,
     args: &[OsString],
     tests: &[String],
+    coverage: Option<PathBuf>,
 ) -> Result<(), Error> {
+    let (cov_fn, cov_dump) = if let Some(profraw_path) = coverage {
+        (
+            format!(
+                r#"
+            async function dumpCoverage() {{
+                const fs = require('node:fs');
+                const data = wasm.__wbgtest_cov_dump();
+                fs.writeFile("{}", data, err => {{
+                    if (err) {{
+                        console.error(err);
+                    }}
+                }})
+            }}
+            "#,
+                profraw_path.display()
+            ),
+            "await dumpCoverage();",
+        )
+    } else {
+        (String::new(), "")
+    };
+
     let mut js_to_execute = format!(
         r#"
         const {{ exit }} = require('process');
@@ -53,6 +76,8 @@ pub fn execute(
 
         global.__wbg_test_invoke = f => f();
 
+        {cov_fn}
+
         async function main(tests) {{
             // Forward runtime arguments. These arguments are also arguments to the
             // `wasm-bindgen-test-runner` which forwards them to node which we
@@ -61,6 +86,7 @@ pub fn execute(
             cx.args(process.argv.slice(2));
 
             const ok = await cx.run(tests.map(n => wasm.__wasm[n]));
+            {cov_dump}
             if (!ok)
                 exit(1);
         }}

crates/cli/src/bin/wasm-bindgen-test-runner/server.rs

@@ -1,8 +1,9 @@
 use std::borrow::Cow;
-use std::ffi::OsString;
 use std::fs;
+use std::io::{Read, Write};
 use std::net::SocketAddr;
 use std::path::Path;
+use std::{ffi::OsString, path::PathBuf};
 
 use anyhow::{anyhow, Context, Error};
 use rouille::{Request, Response, Server};
@@ -16,9 +17,30 @@ pub fn spawn(
     tests: &[String],
     no_module: bool,
     worker: bool,
+    coverage: Option<PathBuf>,
 ) -> Result<Server<impl Fn(&Request) -> Response + Send + Sync>, Error> {
     let mut js_to_execute = String::new();
 
+    let (cov_import, cov_dump) = if coverage.is_some() {
+        (
+            if no_module {
+                "let __wbgtest_cov_dump = wasm_bindgen.__wbgtest_cov_dump;"
+            } else {
+                "__wbgtest_cov_dump,"
+            },
+            r#"
+            // Dump the coverage data collected during the tests
+            const coverage = __wbgtest_cov_dump();
+            await fetch("/__coverage/dump", {
+                method: "POST",
+                body: coverage
+            });
+            "#,
+        )
+    } else {
+        ("", "")
+    };
+
     let wbg_import_script = if no_module {
         String::from(
             r#"
@@ -28,6 +50,7 @@ pub fn spawn(
             let __wbgtest_console_info = wasm_bindgen.__wbgtest_console_info;
             let __wbgtest_console_warn = wasm_bindgen.__wbgtest_console_warn;
             let __wbgtest_console_error = wasm_bindgen.__wbgtest_console_error;
+            {cov_import}
             let init = wasm_bindgen;
             "#,
         )
@@ -41,6 +64,7 @@ pub fn spawn(
                 __wbgtest_console_info,
                 __wbgtest_console_warn,
                 __wbgtest_console_error,
+                {cov_import}
                 default as init,
             }} from './{}';
             "#,
@@ -95,6 +119,7 @@ pub fn spawn(
 
                 cx.args({1:?});
                 await cx.run(tests.map(s => wasm[s]));
+                {cov_dump}
             }}
 
             onmessage = function(e) {{
@@ -175,6 +200,7 @@ pub fn spawn(
                 cx.args({1:?});
 
                 await cx.run(test.map(s => wasm[s]));
+                {cov_dump}
             }}
 
             const tests = [];
@@ -218,6 +244,25 @@ pub fn spawn(
                 )
             };
             return set_isolate_origin_headers(Response::from_data("text/html", s));
+        } else if request.url() == "/__coverage/dump" {
+            let profraw_path = coverage.as_ref().expect(
+                "Received coverage dump request but server wasn't set up to accept coverage",
+            );
+            // This is run after all tests are done and dumps the data received in the request
+            // into a single profraw file
+            let mut profraw =
+                std::fs::File::create(profraw_path).expect("Couldn't create .profraw for coverage");
+            let mut data = Vec::new();
+            request
+                .data()
+                .expect("Expected coverage data in body")
+                .read_to_end(&mut data)
+                .expect("Failed to read message body");
+
+            profraw
+                .write_all(&data)
+                .expect("Couldn't dump coverage data to profraw");
+            return Response::text("Coverage dumped");
         }
 
         // Otherwise we need to find the asset here. It may either be in our

crates/test/Cargo.toml

@@ -8,6 +8,12 @@ repository = "https://github.com/rustwasm/wasm-bindgen"
 edition = "2018"
 rust-version = "1.57"
 
+[features]
+default = []
+experimental = ["coverage"]
+
+coverage = ["minicov"]
+
 [dependencies]
 console_error_panic_hook = '0.1'
 js-sys = { path = '../js-sys', version = '0.3.67' }
@@ -16,6 +22,7 @@ wasm-bindgen = { path = '../..', version = '0.2.90' }
 wasm-bindgen-futures = { path = '../futures', version = '0.4.40' }
 wasm-bindgen-test-macro = { path = '../test-macro', version = '=0.3.40' }
 gg-alloc = { version = "1.0", optional = true }
+minicov = { version = "0.3", optional = true }
 
 [lib]
 test = false

crates/test/src/coverage.rs

@@ -0,0 +1,29 @@
+use wasm_bindgen::prelude::wasm_bindgen;
+
+#[cfg(feature = "coverage")]
+#[wasm_bindgen]
+pub fn __wbgtest_cov_dump() -> Vec<u8> {
+    let mut coverage = Vec::new();
+    unsafe {
+        minicov::capture_coverage(&mut coverage).unwrap();
+    }
+    if coverage.is_empty() {
+        console_error!(
+            "Empty coverage data received. Make sure you compile the tests with
+        RUSTFLAGS=\"-Cinstrument-coverage -Zno-profile-runtime --emit=llvm-ir\"",
+        );
+    }
+    coverage
+}
+
+/// Called when setting WASM_BINDGEN_TEST_COVERAGE but coverage feature is disabled.
+/// Currently not being used because of issues in the interpreter regarding profiling
+/// information which cause an error before we get here.
+#[cfg(not(feature = "coverage"))]
+#[wasm_bindgen]
+pub fn __wbgtest_cov_dump() -> Vec<u8> {
+    console_error!(
+        "Coverage was supposed to be dumped, but the \"coverage\" feature is disabled in wasm-bindgen-test",
+    );
+    Vec::new()
+}

crates/test/src/lib.rs

@@ -21,6 +21,24 @@ macro_rules! console_log {
     )
 }
 
+/// Helper macro which acts like `println!` only routes to `console.warn`
+/// instead.
+#[macro_export]
+macro_rules! console_warn {
+    ($($arg:tt)*) => (
+        $crate::__rt::log_warn(&format_args!($($arg)*))
+    )
+}
+
+/// Helper macro which acts like `println!` only routes to `console.error`
+/// instead.
+#[macro_export]
+macro_rules! console_error {
+    ($($arg:tt)*) => (
+        $crate::__rt::log_error(&format_args!($($arg)*))
+    )
+}
+
 /// A macro used to configured how this test is executed by the
 /// `wasm-bindgen-test-runner` harness.
 ///
@@ -59,3 +77,5 @@ macro_rules! wasm_bindgen_test_configure {
 
 #[path = "rt/mod.rs"]
 pub mod __rt;
+
+mod coverage;