test(rustfix): Use snapbox for snapshot testing (#15429)

### What does this PR try to resolve?

- separates each test into different test cases
- `snapbox` is used to test the snapshots
  - difference in `.json` file alone should never cause a test to fail
- `.json` files updated only if expected fix != actual fix &&
`SNAPSHOTS=overwrite`
- when `.json` files are updated, the json is pretty printed
- replaced environment variables `RUSTFIX_TEST_*` for overwriting test
snapshots with `SNAPSHOTS=overwrite`
- ❗ The `RUSTFIX_TEST_RECORD_FIXED_RUST` feature is removed (generate a
`*.fixed.rs` on demand`). We can add it back whenever needed.

Fixes #13891

### How should we test and review this PR?

Run tests with:
```sh
cargo test -p rustfix
```
All the test should run as different test cases
nightly tests run only when using nightly version of rustc is used

Author: weihanglo

Cargo.lock

@@ -1,6 +1,6 @@
 [package]
 name = "rustfix"
-version = "0.9.0"
+version = "0.9.1"
 authors = [
     "Pascal Hertleif <[email protected]>",
     "Oliver Schneider <[email protected]>",
@@ -29,6 +29,7 @@ proptest.workspace = true
 similar.workspace = true
 tempfile.workspace = true
 tracing-subscriber.workspace = true
+snapbox.workspace = true
 
 [lints]
 workspace = true

crates/rustfix/Cargo.toml

@@ -7,44 +7,30 @@
 //! compares the result with the corresponding `.fixed.rs` file. If they don't
 //! match, then the test fails.
 //!
-//! There are several debugging environment variables for this test that you can set:
+//! The files ending in `.nightly.rs` will run only on the nightly toolchain
 //!
-//! - `RUST_LOG=parse_and_replace=debug`: Print debug information.
-//! - `RUSTFIX_TEST_BLESS=test-name.rs`: When given the name of a test, this
-//!   will overwrite the `.json` and `.fixed.rs` files with the expected
-//!   values. This can be used when adding a new test.
-//! - `RUSTFIX_TEST_RECORD_JSON=1`:  Records the JSON output to
-//!   `*.recorded.json` files. You can then move that to `.json` or whatever
-//!   you need.
-//! - `RUSTFIX_TEST_RECORD_FIXED_RUST=1`: Records the fixed result to
-//!   `*.recorded.rs` files. You can then move that to `.rs` or whatever you
-//!   need.
+//! To override snapshots, run `SNAPSHOTS=overwrite cargo test`.
+//! See [`snapbox::assert::Action`] for different actions.
 
 #![allow(clippy::disallowed_methods, clippy::print_stdout, clippy::print_stderr)]
 
-use anyhow::{anyhow, ensure, Context, Error};
+use anyhow::{anyhow, Context, Error};
 use rustfix::apply_suggestions;
+use serde_json::Value;
+use snapbox::data::DataFormat;
+use snapbox::{Assert, Data};
 use std::collections::HashSet;
 use std::env;
 use std::ffi::OsString;
 use std::fs;
-use std::path::{Path, PathBuf};
+use std::path::Path;
 use std::process::{Command, Output};
 use tempfile::tempdir;
-use tracing::{debug, info, warn};
 
 mod fixmode {
     pub const EVERYTHING: &str = "yolo";
 }
 
-mod settings {
-    // can be set as env var to debug
-    pub const CHECK_JSON: &str = "RUSTFIX_TEST_CHECK_JSON";
-    pub const RECORD_JSON: &str = "RUSTFIX_TEST_RECORD_JSON";
-    pub const RECORD_FIXED_RUST: &str = "RUSTFIX_TEST_RECORD_FIXED_RUST";
-    pub const BLESS: &str = "RUSTFIX_TEST_BLESS";
-}
-
 static mut VERSION: (u32, bool) = (0, false);
 
 // Temporarily copy from `cargo_test_macro::version`.
@@ -108,170 +94,98 @@ fn compiles_without_errors(file: &Path) -> Result<(), Error> {
 
     match res.status.code() {
         Some(0) => Ok(()),
-        _ => {
-            info!(
-                "file {:?} failed to compile:\n{}",
-                file,
-                String::from_utf8(res.stderr)?
-            );
-            Err(anyhow!(
-                "failed with status {:?} (`env RUST_LOG=parse_and_replace=info` for more info)",
-                res.status.code(),
-            ))
-        }
-    }
-}
-
-fn diff(expected: &str, actual: &str) -> String {
-    use similar::{ChangeTag, TextDiff};
-    use std::fmt::Write;
-
-    let mut res = String::new();
-    let diff = TextDiff::from_lines(expected.trim(), actual.trim());
-
-    let mut different = false;
-    for op in diff.ops() {
-        for change in diff.iter_changes(op) {
-            let prefix = match change.tag() {
-                ChangeTag::Equal => continue,
-                ChangeTag::Insert => "+",
-                ChangeTag::Delete => "-",
-            };
-            if !different {
-                writeln!(&mut res, "differences found (+ == actual, - == expected):").unwrap();
-                different = true;
-            }
-            write!(&mut res, "{} {}", prefix, change.value()).unwrap();
-        }
-    }
-    if different {
-        write!(&mut res, "").unwrap();
+        _ => Err(anyhow!(
+            "file {:?} failed compile with status {:?}:\n {}",
+            file,
+            res.status.code(),
+            String::from_utf8(res.stderr)?
+        )),
     }
-
-    res
 }
 
-fn test_rustfix_with_file<P: AsRef<Path>>(file: P, mode: &str) -> Result<(), Error> {
+fn test_rustfix_with_file<P: AsRef<Path>>(file: P, mode: &str) {
     let file: &Path = file.as_ref();
     let json_file = file.with_extension("json");
-    let fixed_file = file.with_extension("fixed.rs");
+    let expected_fixed_file = file.with_extension("fixed.rs");
 
     let filter_suggestions = if mode == fixmode::EVERYTHING {
         rustfix::Filter::Everything
     } else {
         rustfix::Filter::MachineApplicableOnly
     };
 
-    debug!("next up: {:?}", file);
-    let code = fs::read_to_string(file)?;
-    let errors = compile_and_get_json_errors(file)
-        .with_context(|| format!("could not compile {}", file.display()))?;
-    let suggestions =
-        rustfix::get_suggestions_from_json(&errors, &HashSet::new(), filter_suggestions)
-            .context("could not load suggestions")?;
-
-    if std::env::var(settings::RECORD_JSON).is_ok() {
-        fs::write(file.with_extension("recorded.json"), &errors)?;
-    }
+    let code = fs::read_to_string(file).unwrap();
 
-    if std::env::var(settings::CHECK_JSON).is_ok() {
-        let expected_json = fs::read_to_string(&json_file)
-            .with_context(|| format!("could not load json fixtures for {}", file.display()))?;
-        let expected_suggestions =
-            rustfix::get_suggestions_from_json(&expected_json, &HashSet::new(), filter_suggestions)
-                .context("could not load expected suggestions")?;
+    let json = compile_and_get_json_errors(file)
+        .with_context(|| format!("could not compile {}", file.display()))
+        .unwrap();
 
-        ensure!(
-            expected_suggestions == suggestions,
-            "got unexpected suggestions from clippy:\n{}",
-            diff(
-                &format!("{:?}", expected_suggestions),
-                &format!("{:?}", suggestions)
-            )
-        );
-    }
+    let suggestions =
+        rustfix::get_suggestions_from_json(&json, &HashSet::new(), filter_suggestions)
+            .context("could not load suggestions")
+            .unwrap();
 
     let fixed = apply_suggestions(&code, &suggestions)
-        .with_context(|| format!("could not apply suggestions to {}", file.display()))?
+        .with_context(|| format!("could not apply suggestions to {}", file.display()))
+        .unwrap()
         .replace('\r', "");
 
-    if std::env::var(settings::RECORD_FIXED_RUST).is_ok() {
-        fs::write(file.with_extension("recorded.rs"), &fixed)?;
-    }
-
-    if let Some(bless_name) = std::env::var_os(settings::BLESS) {
-        if bless_name == file.file_name().unwrap() {
-            std::fs::write(&json_file, &errors)?;
-            std::fs::write(&fixed_file, &fixed)?;
-        }
-    }
-
-    let expected_fixed = fs::read_to_string(&fixed_file)
-        .with_context(|| format!("could read fixed file for {}", file.display()))?
-        .replace('\r', "");
-    ensure!(
-        fixed.trim() == expected_fixed.trim(),
-        "file {} doesn't look fixed:\n{}",
-        file.display(),
-        diff(fixed.trim(), expected_fixed.trim())
+    let assert = Assert::new().action_env(snapbox::assert::DEFAULT_ACTION_ENV);
+    let (actual_fix, expected_fix) = assert.normalize(
+        Data::text(&fixed),
+        Data::read_from(expected_fixed_file.as_path(), Some(DataFormat::Text)),
     );
 
-    compiles_without_errors(&fixed_file)?;
-
-    Ok(())
-}
+    if actual_fix != expected_fix {
+        let fixed_assert = assert.try_eq(Some(&"Current Fix"), actual_fix, expected_fix);
+        assert!(fixed_assert.is_ok(), "{}", fixed_assert.err().unwrap());
+
+        let expected_json = Data::read_from(json_file.as_path(), Some(DataFormat::Text));
+
+        let pretty_json = json
+            .split("\n")
+            .filter(|j| !j.is_empty())
+            .map(|j| {
+                serde_json::to_string_pretty(&serde_json::from_str::<Value>(j).unwrap()).unwrap()
+            })
+            .collect::<Vec<String>>()
+            .join("\n");
+
+        let json_assert = assert.try_eq(
+            Some(&"Compiler Error"),
+            Data::text(pretty_json),
+            expected_json,
+        );
+        assert!(json_assert.is_ok(), "{}", json_assert.err().unwrap());
+    }
 
-fn get_fixture_files(p: &str) -> Result<Vec<PathBuf>, Error> {
-    Ok(fs::read_dir(p)?
-        .map(|e| e.unwrap().path())
-        .filter(|p| p.is_file())
-        .filter(|p| {
-            let x = p.to_string_lossy();
-            x.ends_with(".rs") && !x.ends_with(".fixed.rs") && !x.ends_with(".recorded.rs")
-        })
-        .collect())
+    compiles_without_errors(&expected_fixed_file).unwrap();
 }
 
-fn assert_fixtures(dir: &str, mode: &str) {
-    let files = get_fixture_files(dir)
-        .with_context(|| format!("couldn't load dir `{dir}`"))
-        .unwrap();
-    let mut failures = 0;
-
-    let is_not_nightly = !version().1;
-
-    for file in &files {
-        if file
-            .file_stem()
-            .unwrap()
-            .to_str()
-            .unwrap()
-            .ends_with(".nightly")
-            && is_not_nightly
-        {
-            info!("skipped: {file:?}");
-            continue;
-        }
-        if let Err(err) = test_rustfix_with_file(file, mode) {
-            println!("failed: {}", file.display());
-            warn!("{:?}", err);
-            failures += 1;
+macro_rules! run_test {
+    ($name:ident, $file:expr) => {
+        #[test]
+        #[allow(non_snake_case)]
+        fn $name() {
+            let (_, nightly) = version();
+            if !$file.ends_with(".nightly.rs") || nightly {
+                let file = Path::new(concat!("./tests/everything/", $file));
+                assert!(file.is_file(), "could not load {}", $file);
+                test_rustfix_with_file(file, fixmode::EVERYTHING);
+            }
         }
-        info!("passed: {:?}", file);
-    }
-
-    if failures > 0 {
-        panic!(
-            "{} out of {} fixture asserts failed\n\
-             (run with `env RUST_LOG=parse_and_replace=info` to get more details)",
-            failures,
-            files.len(),
-        );
-    }
+    };
 }
 
-#[test]
-fn everything() {
-    tracing_subscriber::fmt::init();
-    assert_fixtures("./tests/everything", fixmode::EVERYTHING);
+run_test! {
+    closure_immutable_outer_variable,
+    "closure-immutable-outer-variable.rs"
 }
+run_test! {dedup_suggestions, "dedup-suggestions.rs"}
+run_test! {E0178, "E0178.rs"}
+run_test! {handle_insert_only, "handle-insert-only.rs"}
+run_test! {lt_generic_comp, "lt-generic-comp.rs"}
+run_test! {multiple_solutions, "multiple-solutions.rs"}
+run_test! {replace_only_one_char, "replace-only-one-char.rs"}
+run_test! {str_lit_type_mismatch, "str-lit-type-mismatch.rs"}
+run_test! {use_insert, "use-insert.rs"}