diff --git a/Project.toml b/Project.toml index f6f659d..f4b5a87 100644 --- a/Project.toml +++ b/Project.toml @@ -1,6 +1,6 @@ name = "ReferenceTests" uuid = "324d217c-45ce-50fc-942e-d289b448e8cf" -version = "0.10.6" +version = "0.10.7" authors = ["Christof Stocker ", "Frames White ", "Johnny Chen "] [deps] diff --git a/README.md b/README.md index 8992ab8..778b4be 100644 --- a/README.md +++ b/README.md @@ -75,7 +75,7 @@ using ReferenceTests, TestImages Proper image formats such as `.png` are also supported for full-res image testing. If your terminal [supports Sixel](https://www.arewesixelyet.com/), then ReferenceTests will display the full image -in your terminal using (Sixel.jl)[https://github.com/JuliaIO/Sixel.jl]: +in your terminal using [Sixel.jl](https://github.com/JuliaIO/Sixel.jl): ![sixel_demo](.github/images/sixel_demo.png) @@ -97,11 +97,15 @@ test suite. These tests are easy to run via `pkg> test` but the child process used within `pkg> test` is non-interactive, so the update prompt will not show if there are mismatches. -To update references within a package test suite, there are three options: +To update references within a package test suite, there are four options: - Set the environment variable `JULIA_REFERENCETESTS_UPDATE` to `"true"` and run `pkg> test`, which will force update any non-matches. You can then check changes to any git-tracked reference images before commit. +- Set the environment variable `JULIA_REFERENCETESTS_UPDATE_AND_ERROR` to + `"true"` and run `pkg> test`, which will also force update any non-matches, + but each non-matching file will result in a test failure. You can then + check changes to any git-tracked reference images before commit. - Delete any reference images you wish to update and run `pkg> test`, given that missing references are created automatically. - Run the `test/runtests.jl` interactively. This may be easier using diff --git a/src/test_reference.jl b/src/test_reference.jl index 98af86f..3985a19 100644 --- a/src/test_reference.jl +++ b/src/test_reference.jl @@ -171,14 +171,26 @@ function test_reference( if force_update() || input_bool("Replace reference with actual result?") mv(actual_path, reference_path; force=true) # overwrite old file it - @info "Please run the tests again for any changes to take effect" + if error_on_force_update() + error(""" + The reference file has been updated, but an error was thrown because the environment variable + `JULIA_REFERENCETEST_UPDATE_AND_ERROR` was set to `true`. + """) + else + @info "Please run the tests again for any changes to take effect" + end else @test false end end end -force_update() = tryparse(Bool, get(ENV, "JULIA_REFERENCETESTS_UPDATE", "false")) === true +function force_update() + update = tryparse(Bool, get(ENV, "JULIA_REFERENCETESTS_UPDATE", "false")) === true + update_and_err = tryparse(Bool, get(ENV, "JULIA_REFERENCETESTS_UPDATE_AND_ERROR", "false")) === true + update || update_and_err +end +error_on_force_update() = tryparse(Bool, get(ENV, "JULIA_REFERENCETESTS_UPDATE_AND_ERROR", "false")) === true """ mismatch_staging_dir() diff --git a/test/runtests.jl b/test/runtests.jl index 8734163..54e0663 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -205,4 +205,19 @@ end @test_reference file Dict(:ar=>arr_float) by=comp end +@testset "force update and error" begin + withenv("JULIA_REFERENCETESTS_UPDATE" => "true") do + @test begin + @test_reference "references/random_string.txt" rand(10) + @test_reference "references/random_string.txt" rand(10) + true + end + end + withenv("JULIA_REFERENCETESTS_UPDATE_AND_ERROR" => "true") do + @test_throws Exception begin + @test_reference "references/random_string.txt" rand(10) + end + end +end + end # top level testset