Fix testing

src/ClimatePref.jl

@@ -13,6 +13,9 @@ function __init__()
     end
 end
 
+# Path into package
+path(path...; dir::String = "test") = joinpath(@__DIR__, "..", dir, path...)
+
 module Units
 
 import Unitful

test/clean_JuliaFormatter.jl

@@ -0,0 +1,42 @@
+# SPDX-License-Identifier: BSD-2-Clause
+
+module CleanJuliaFormatter
+using Test
+using ClimatePref
+using Git
+using Logging
+using Pkg
+using JuliaFormatter
+
+function is_repo_clean(repo_path; ignore_untracked = true)
+    # Get the status of the repository
+    statuses = readlines(`$(Git.git()) status -s $repo_path`)
+
+    if ignore_untracked
+        # Repo must be clean except for untracked files
+        statuses = filter((!) ∘ contains("??"), statuses)
+    end
+
+    is_clean = isempty(statuses)
+
+    # If not clean then report on dirty files
+    is_clean || @error "\n" * join(statuses, "\n")
+
+    return is_clean
+end
+
+# Metadata crosswalk testing only works on Julia v1.8 and after due to Project.toml changes
+# Also does not currently work on Windows runners on GitHub due to file writing issues
+if VERSION ≥ VersionNumber("1.8.0") &&
+   (!haskey(ENV, "RUNNER_OS") || ENV["RUNNER_OS"] ≠ "Windows")
+    @testset "JuliaFormatter" begin
+        git_dir = readchomp(`$(Git.git()) rev-parse --show-toplevel`)
+        @test_nowarn format(EcoSISTEM)
+        @test is_repo_clean(git_dir)
+    end
+else
+    @test_broken VERSION ≥ VersionNumber("1.8.0") &&
+                 (!haskey(ENV, "RUNNER_OS") || ENV["RUNNER_OS"] ≠ "Windows")
+end
+
+end

test/clean_ResearchSoftwareMetadata.jl

@@ -0,0 +1,44 @@
+# SPDX-License-Identifier: BSD-2-Clause
+
+module CleanRSMD
+using Test
+using Phylo
+using Git
+using Logging
+using Pkg
+using ResearchSoftwareMetadata
+
+function is_repo_clean(repo_path; ignore_untracked = true)
+    # Get the status of the repository
+    statuses = readlines(`$(Git.git()) status -s $repo_path`)
+
+    if ignore_untracked
+        # Repo must be clean except for untracked files
+        statuses = filter((!) ∘ contains("??"), statuses)
+    end
+
+    is_clean = isempty(statuses)
+
+    # If not clean then report on dirty files
+    is_clean || @error "\n" * join(statuses, "\n")
+
+    return is_clean
+end
+
+# Metadata crosswalk testing only works on Julia v1.8 and after due to Project.toml changes
+# Also does not currently work on Windows runners on GitHub due to file writing issues
+if VERSION ≥ VersionNumber("1.8.0") &&
+   (!haskey(ENV, "RUNNER_OS") || ENV["RUNNER_OS"] ≠ "Windows")
+    @testset "RSMD" begin
+        git_dir = readchomp(`$(Git.git()) rev-parse --show-toplevel`)
+        @test isnothing(ResearchSoftwareMetadata.crosswalk())
+        global_logger(SimpleLogger(stderr, Logging.Warn))
+        @test_nowarn ResearchSoftwareMetadata.crosswalk()
+        @test is_repo_clean(git_dir)
+    end
+else
+    @test_broken VERSION ≥ VersionNumber("1.8.0") &&
+                 (!haskey(ENV, "RUNNER_OS") || ENV["RUNNER_OS"] ≠ "Windows")
+end
+
+end

test/runtests.jl

@@ -1,51 +1,185 @@
-# Identify files in test/ that are testing matching files in src/
-#  - src/Source.jl will be matched by test/test_Source.jl
+# SPDX-License-Identifier: BSD-2-Clause
 
+using Random
 using Test
+using ClimatePref
 using Pkg
 
-filebase = String[]
-for (root, dirs, files) in walkdir("../src")
-    append!(filebase,
-            map(file -> replace(file, r"(.*).jl" => s"\1"),
-                filter(file -> occursin(r".*\.jl", file), files)))
-end
+rsmd = get(ENV, "RSMD_CROSSWALK", "FALSE")
 
-testbase = map(file -> replace(file, r"test_(.*).jl" => s"\1"),
-                filter(str -> occursin(r"^test_.*\.jl$", str), readdir()))
+if rsmd == "FALSE"
+    # Normal testing
 
-@testset "ClimatePref.jl" begin
-    println()
-    @info "Running tests for files:"
-    for t in testbase
-        println("    = $t.jl")
+    # Identify files in test/ that are testing matching files in src/
+    #  - src/Source.jl will be matched by test/test_Source.jl
+    filebase = String[]
+    for (root, dirs, files) in walkdir("../src")
+        append!(filebase,
+                map(file -> replace(file, r"(.*).jl" => s"\1"),
+                    filter(file -> occursin(r".*\.jl", file), files)))
     end
-    println()
 
-    @info "Running tests..."
-    @testset for t in testbase
-        fn = "test_$t.jl"
-        println("    * Testing $t.jl ...")
-        include(fn)
+    testbase = map(file -> replace(file, r"test_(.*).jl" => s"\1"),
+                   filter(str -> occursin(r"^test_.*\.jl$", str), readdir()))
+
+    # Identify tests with no matching file
+    superfluous = filter(f -> f ∉ filebase, testbase)
+    if length(superfluous) > 0
+        println()
+        @info "Potentially superfluous tests:"
+        for f in superfluous
+            println("    + $f.jl")
+        end
+        println()
     end
-end
 
-# Identify tests with no matching file
-superfluous = filter(f -> f ∉ filebase, testbase)
-if length(superfluous) > 0
-    println()
-    @info "Potentially superfluous tests:"
-    for f in superfluous
-        println("    + $f.jl")
+    # Identify files with no matching test
+    notest = filter(f -> f ∉ testbase, filebase)
+    if length(notest) > 0
+        println()
+        @info "Potentially missing tests:"
+        for f in notest
+            println("    - $f.jl")
+        end
+        println()
+    end
+
+    # Identify files in test/ that are testing matching files in ext/
+    #  - ext/SourceExt.jl will be matched by test/ext_SourceExt.jl
+    filebase = String[]
+    if isdir("../ext")
+        for (root, dirs, files) in walkdir("../ext")
+            append!(filebase,
+                    map(file -> replace(file, r"(.*).jl" => s"\1"),
+                        filter(file -> occursin(r".*\.jl", file), files)))
+        end
+    end
+
+    extbase = map(file -> replace(file, r"ext_(.*).jl" => s"\1"),
+                  filter(str -> occursin(r"^ext_.*\.jl$", str), readdir()))
+
+    # Identify tests with no matching file
+    superfluous = filter(f -> f ∉ filebase, extbase)
+    if length(superfluous) > 0
+        println()
+        @info "Potentially superfluous extension tests:"
+        for f in superfluous
+            println("    + $f.jl")
+        end
+        println()
+    end
+
+    # Identify files with no matching test
+    notest = filter(f -> f ∉ extbase, filebase)
+    if length(notest) > 0
+        println()
+        @info "Potentially missing extension tests:"
+        for f in notest
+            println("    - $f.jl")
+        end
+        println()
+    end
+
+    # Seed RNG to make tests reproducible
+    Random.seed!(1234)
+
+    @testset "ClimatePref.jl" begin
+        @test isfile(ClimatePref.path("runtests.jl"))
+        println()
+        @info "Running tests for files:"
+        for t in testbase
+            println("    = $t.jl")
+        end
+        println()
+
+        @info "Running tests..."
+        @testset for t in testbase
+            fn = "test_$t.jl"
+            println("    * Testing $t.jl ...")
+            include(fn)
+        end
+
+        println()
+        @info "Running tests for extensions:"
+        for t in extbase
+            println("    = $t.jl")
+        end
+        println()
+
+        @info "Running extension tests..."
+        @testset for t in extbase
+            fn = "ext_$t.jl"
+            println("    * Testing $t.jl extension...")
+            include(fn)
+        end
+    end
+
+    # Identify files that are cross-validating results against other packages
+    # test/pkg_Package.jl should validate results against the Package package
+
+    pkgbase = map(file -> replace(file, r"pkg_(.*).jl$" => s"\1"),
+                  filter(str -> occursin(r"^pkg_.*\.jl$", str),
+                         readdir()))
+
+    if length(pkgbase) > 0
+        @info "Cross validation packages:"
+        @testset begin
+            for p in pkgbase
+                println("    = $p")
+            end
+            println()
+
+            @testset for p in pkgbase
+                fn = "pkg_$p.jl"
+                println("    * Validating $p.jl ...")
+                include(fn)
+            end
+        end
+    end
+
+    # Identify files that are cross-validating results against other packages
+    # test/pkg_Package.jl should validate results against the Package package
+
+    @testset "Examples folder" begin
+        println()
+        @info "Running from examples folder ..."
+        Pkg.activate(ClimatePref.path(dir = "examples"))
+        Pkg.rm("ClimatePref")
+        Pkg.develop(PackageSpec(path = ClimatePref.path(dir = "")))
+        Pkg.instantiate()
+        Pkg.update()
+        example_testbase = map(file -> replace(file, r"test_(.*).jl" => s"\1"),
+                               filter(str -> occursin(r"^test_.*\.jl$", str),
+                                      readdir("../examples/")))
+        for t in example_testbase
+            fn = "../examples/test_$t.jl"
+            println("    * Testing $t.jl ...")
+            @test_nowarn include(fn)
+        end
     end
 end
 
-# Identify files with no matching test
-notest = filter(f -> f ∉ testbase, filebase)
-if length(notest) > 0
-    println()
-    @info "Potentially missing tests:"
-    for f in notest
-        println("    - $f.jl")
+if rsmd == "TRUE" || !haskey(ENV, "RUNNER_OS") # Crosswalk runner or local testing
+    # Test RSMD crosswalk and other hygene issues
+
+    # Identify files that are checking package hygene
+    cleanbase = map(file -> replace(file, r"clean_(.*).jl$" => s"\1"),
+                    filter(str -> occursin(r"^clean_.*\.jl$", str),
+                           readdir()))
+
+    if length(cleanbase) > 0
+        @info "Crosswalk and clean testing:"
+        @testset begin
+            for c in cleanbase
+                println("    = $c")
+            end
+            println()
+
+            @testset for c in cleanbase
+                fn = "clean_$c.jl"
+                println("    * Verifying $c.jl ...")
+                include(fn)
+            end
+        end
     end
-end
+end

test/test_ClimateTypes.jl

@@ -1,3 +1,5 @@
+# SPDX-License-Identifier: BSD-2-Clause
+
 using ClimatePref
 using ClimatePref.Units
 using AxisArrays
@@ -6,6 +8,7 @@ using Test
 wc = AxisArray(fill(1.0, 100, 100, 12))
 @test_nowarn Worldclim(wc)
 
-era = AxisArray(fill(1.0, 100, 100, 12), Axis{:lat}(1:100), Axis{:lon}(1:100), Axis{:time}(1month:1month:12months))
+era = AxisArray(fill(1.0, 100, 100, 12), Axis{:lat}(1:100), Axis{:lon}(1:100),
+                Axis{:time}((1month):(1month):(12months)))
 @test_nowarn ERA(era)
-@test_nowarn CERA(era)
+@test_nowarn CERA(era)

test/test_DataCleaning.jl

@@ -1,3 +1,5 @@
+# SPDX-License-Identifier: BSD-2-Clause
+
 using ClimatePref
 using ClimatePref.Units
 using Unitful
@@ -6,4 +8,4 @@ using Unitful.DefaultSymbols
 @test_nowarn create_reference(0.5)
 ref = create_reference(0.5)
 @test size(ref.array) == (721, 361)
-@test ref.array[1:end] == collect(1:260281)
+@test ref.array[1:end] == collect(1:260281)