From 4c1c985a8efb46a2d2201e56c0fd68b9b8be33ec Mon Sep 17 00:00:00 2001 From: Nathanael Wong Date: Sat, 15 Feb 2025 19:37:55 -0500 Subject: [PATCH 01/11] Bumping to v0.5 and updating [compat] for packages in the GeoRegions Ecosystem --- Project.toml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Project.toml b/Project.toml index 0766abb..b1a8aaf 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "ERA5Reanalysis" uuid = "99c076f5-c00e-46eb-a0bd-7ba31777c146" authors = ["Nathanael Wong "] -version = "0.4.0" +version = "0.5.0" [deps] Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" @@ -23,16 +23,16 @@ UnitfulParsableString = "06c00241-927a-4d5b-bb5e-6b5a2ada3567" [compat] Dates = "1" DelimitedFiles = "1" -GeoRegions = "7" +GeoRegions = "^8.0.1" HTTP = "1" JSON3 = "1" -LandSea = "0.0.2" +LandSea = "0.1" Logging = "1" NCDatasets = "0.14" PrettyTables = "2" Printf = "1" Reexport = "1" -RegionGrids = "0.0.5" +RegionGrids = "0.1" Statistics = "1" Unitful = "1" UnitfulParsableString = "0.1" From 9fbb69586a96f95c277522394c2bde7db00ba165 Mon Sep 17 00:00:00 2001 From: Nathanael Wong Date: Sun, 16 Feb 2025 04:56:11 -0500 Subject: [PATCH 02/11] Updating .github workflows, hopefully faster? --- .github/dependabot.yml | 7 +++ .github/workflows/CI.yml | 68 +++++++++++++++++------------- .github/workflows/CompatHelper.yml | 5 +-- .github/workflows/TagBot.yml | 16 +++++++ 4 files changed, 64 insertions(+), 32 deletions(-) create mode 100644 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..700707c --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,7 @@ +# https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates +version: 2 +updates: + - package-ecosystem: "github-actions" + directory: "/" # Location of package manifests + schedule: + interval: "weekly" diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 11fb2ba..e0010ff 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -1,22 +1,30 @@ name: CI on: - pull_request: - branches: - - main push: branches: - main - tags: '*' + tags: ['*'] + pull_request: + workflow_dispatch: +concurrency: + # Skip intermediate builds: always. + # Cancel intermediate builds: only if it is a pull request build. + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }} jobs: test: name: Julia ${{ matrix.version }} - ${{ matrix.os }} - ${{ matrix.arch }} - ${{ github.event_name }} runs-on: ${{ matrix.os }} + timeout-minutes: 60 + permissions: # needed to allow julia-actions/cache to proactively delete old caches that it has created + actions: write + contents: read strategy: fail-fast: false matrix: version: - - '1.10' # Replace this with the minimum Julia version that your package supports. E.g. if your package requires Julia 1.5 or higher, change this to '1.5'. - - '1' # Leave this line unchanged. '1' will automatically expand to the latest stable 1.x release of Julia. + - '1' + - '1.10' os: - ubuntu-latest arch: @@ -27,41 +35,43 @@ jobs: with: version: ${{ matrix.version }} arch: ${{ matrix.arch }} - - uses: actions/cache@v4 - env: - cache-name: cache-artifacts - with: - path: ~/.julia/artifacts - key: ${{ runner.os }}-test-${{ env.cache-name }}-${{ hashFiles('**/Project.toml') }} - restore-keys: | - ${{ runner.os }}-test-${{ env.cache-name }}- - ${{ runner.os }}-test- - ${{ runner.os }}- + - uses: julia-actions/cache@v2 - uses: julia-actions/julia-buildpkg@v1 - uses: julia-actions/julia-runtest@v1 - uses: julia-actions/julia-processcoverage@v1 - uses: codecov/codecov-action@v5 with: - file: lcov.info + files: lcov.info + token: ${{ secrets.CODECOV_TOKEN }} + fail_ci_if_error: false docs: name: Documentation runs-on: ubuntu-latest + permissions: + actions: write # needed to allow julia-actions/cache to proactively delete old caches that it has created + contents: write + statuses: write steps: - uses: actions/checkout@v4 - uses: julia-actions/setup-julia@v2 with: - version: '1' - - run: | - julia --project=docs -e ' - using Pkg - Pkg.develop(PackageSpec(path=pwd())) - Pkg.instantiate()' - - run: | - julia --project=docs -e ' - using Documenter: doctest - using ERA5Reanalysis - doctest(ERA5Reanalysis)' - - run: julia --project=docs docs/make.jl + version: '1.10' + - uses: julia-actions/cache@v2 + - name: Configure doc environment + shell: julia --project=docs --color=yes {0} + run: | + using Pkg + Pkg.develop(PackageSpec(path=pwd())) + Pkg.instantiate() + - uses: julia-actions/julia-buildpkg@v1 + - uses: julia-actions/julia-docdeploy@v1 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} DOCUMENTER_KEY: ${{ secrets.DOCUMENTER_KEY }} + - name: Run doctests + shell: julia --project=docs --color=yes {0} + run: | + using Documenter: DocMeta, doctest + using ERA5Reanalysis + DocMeta.setdocmeta!(ERA5Reanalysis, :DocTestSetup, :(using ERA5Reanalysis); recursive=true) + doctest(ERA5Reanalysis) diff --git a/.github/workflows/CompatHelper.yml b/.github/workflows/CompatHelper.yml index b9fe40f..d48734a 100644 --- a/.github/workflows/CompatHelper.yml +++ b/.github/workflows/CompatHelper.yml @@ -1,17 +1,16 @@ name: CompatHelper - on: schedule: - cron: 0 0 1 * * - + workflow_dispatch: jobs: CompatHelper: runs-on: ubuntu-latest steps: - - uses: julia-actions/setup-julia@latest - name: Pkg.add("CompatHelper") run: julia -e 'using Pkg; Pkg.add("CompatHelper")' - name: CompatHelper.main() env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + COMPATHELPER_PRIV: ${{ secrets.DOCUMENTER_KEY }} run: julia -e 'using CompatHelper; CompatHelper.main()' diff --git a/.github/workflows/TagBot.yml b/.github/workflows/TagBot.yml index f49313b..0cd3114 100644 --- a/.github/workflows/TagBot.yml +++ b/.github/workflows/TagBot.yml @@ -4,6 +4,22 @@ on: types: - created workflow_dispatch: + inputs: + lookback: + default: "3" +permissions: + actions: read + checks: read + contents: write + deployments: read + issues: read + discussions: read + packages: read + pages: read + pull-requests: read + repository-projects: read + security-events: read + statuses: read jobs: TagBot: if: github.event_name == 'workflow_dispatch' || github.actor == 'JuliaTagBot' From 9ca33989961ddd40a3f560f418dedf01f3697384 Mon Sep 17 00:00:00 2001 From: Nathanael Wong Date: Tue, 18 Feb 2025 14:53:11 -0500 Subject: [PATCH 03/11] Changes to CI --- .github/workflows/CI.yml | 39 ++++------------------ .github/workflows/Documentation.yml | 50 +++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+), 33 deletions(-) create mode 100644 .github/workflows/Documentation.yml diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index e0010ff..7551115 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -4,7 +4,11 @@ on: branches: - main tags: ['*'] - pull_request: + paths: + - src/** + - test/** + - .github/workflows/CI.yml + - Project.toml workflow_dispatch: concurrency: # Skip intermediate builds: always. @@ -43,35 +47,4 @@ jobs: with: files: lcov.info token: ${{ secrets.CODECOV_TOKEN }} - fail_ci_if_error: false - docs: - name: Documentation - runs-on: ubuntu-latest - permissions: - actions: write # needed to allow julia-actions/cache to proactively delete old caches that it has created - contents: write - statuses: write - steps: - - uses: actions/checkout@v4 - - uses: julia-actions/setup-julia@v2 - with: - version: '1.10' - - uses: julia-actions/cache@v2 - - name: Configure doc environment - shell: julia --project=docs --color=yes {0} - run: | - using Pkg - Pkg.develop(PackageSpec(path=pwd())) - Pkg.instantiate() - - uses: julia-actions/julia-buildpkg@v1 - - uses: julia-actions/julia-docdeploy@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - DOCUMENTER_KEY: ${{ secrets.DOCUMENTER_KEY }} - - name: Run doctests - shell: julia --project=docs --color=yes {0} - run: | - using Documenter: DocMeta, doctest - using ERA5Reanalysis - DocMeta.setdocmeta!(ERA5Reanalysis, :DocTestSetup, :(using ERA5Reanalysis); recursive=true) - doctest(ERA5Reanalysis) + fail_ci_if_error: false \ No newline at end of file diff --git a/.github/workflows/Documentation.yml b/.github/workflows/Documentation.yml new file mode 100644 index 0000000..4c1ddb9 --- /dev/null +++ b/.github/workflows/Documentation.yml @@ -0,0 +1,50 @@ +name: Documentation +on: + push: + branches: + - main + tags: ['*'] + paths: + - docs/** + - src/** + - test/** + - .github/workflows/Documentation.yml + - Project.toml + workflow_dispatch: +concurrency: + # Skip intermediate builds: always. + # Cancel intermediate builds: only if it is a pull request build. + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }} +jobs: + docs: + name: Documentation + runs-on: ubuntu-latest + permissions: + actions: write # needed to allow julia-actions/cache to proactively delete old caches that it has created + contents: write + statuses: write + steps: + - uses: actions/checkout@v4 + - uses: julia-actions/setup-julia@v2 + with: + version: '1' + - uses: julia-actions/cache@v2 + - name: Configure doc environment + shell: julia --project=docs --color=yes {0} + run: | + using Pkg + Pkg.develop(PackageSpec(path=pwd())) + Pkg.instantiate() + - uses: julia-actions/julia-buildpkg@v1 + - uses: julia-actions/julia-docdeploy@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + DOCUMENTER_KEY: ${{ secrets.DOCUMENTER_KEY }} + - name: Run doctests + shell: julia --project=docs --color=yes {0} + run: | + using Documenter: DocMeta, doctest + using ERA5Reanalysis + DocMeta.setdocmeta!(ERA5Reanalysis, :DocTestSetup, :(using ERA5Reanalysis); recursive=true) + doctest(ERA5Reanalysis) From 101f539c769431f19824dd12ef3fad047e2b01a4 Mon Sep 17 00:00:00 2001 From: Nathanael Wong Date: Sun, 23 Feb 2025 21:37:47 -0500 Subject: [PATCH 04/11] For GeoRegionsv8, geo.bound is now the geo.N, geo.S, geo.E, geo.W --- src/downloads/cdsretrieve.jl | 2 +- src/region/region.jl | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/downloads/cdsretrieve.jl b/src/downloads/cdsretrieve.jl index e4b060c..8640740 100644 --- a/src/downloads/cdsretrieve.jl +++ b/src/downloads/cdsretrieve.jl @@ -273,7 +273,7 @@ function cdsretrieve_area!( if !(ereg.isglb) geo = ereg.geo - dkeys["area"] = geo.bound[1,4,2,3] + dkeys["area"] = [geo.N, geo.W, geo.S, geo.E] end return diff --git a/src/region/region.jl b/src/region/region.jl index 9a9bc6d..4754a30 100644 --- a/src/region/region.jl +++ b/src/region/region.jl @@ -48,7 +48,7 @@ function ERA5Region( @info "$(modulelog()) - Creating an ERA5Region based on the GeoRegion \"$(geo.ID)\"" resolution = regionstep(geo.ID,resolution) if geo.ID == "GLB"; isglb = true; else; isglb = false end - if mod(geo.bound[3],360) == mod(geo.bound[4],360); is360 = true; else; is360 = false end + if mod(geo.E,360) == mod(geo.W,360); is360 = true; else; is360 = false end return ERA5Region{ST,FT}( geo, geo.ID, resolution, @@ -133,7 +133,7 @@ function show(io::IO, ereg::ERA5Region) " Name (geo.name) : ", ereg.geo.name, '\n', " Resolution (resolution) : ", ereg.resolution, '\n', " Folder ID (string) : ", ereg.string, '\n', - " Bounds (geo.[N,S,E,W]) : ", geo.bound, '\n', + " Bounds (geo.[N,S,E,W]) : ", geo.N, ", ", geo.S, ", ", geo.E, ", ", geo.W, '\n', " Shape (geo.shape) : ", geo.shape, '\n', " (geo.[isglb,is360]) : ",(ereg.isglb,ereg.is360),"\n", ) @@ -145,7 +145,7 @@ function show(io::IO, ereg::ERA5Region) " Name (geo.name) : ", ereg.geo.name, '\n', " Resolution (resolution) : ", ereg.resolution, '\n', " Folder ID (string) : ", ereg.string, '\n', - " Bounds (geo.[N,S,E,W]) : ", geo.bound, '\n', + " Bounds (geo.[N,S,E,W]) : ", geo.N, ", ", geo.S, ", ", geo.E, ", ", geo.W, '\n', " (geo.[isglb,is360]) : ",(ereg.isglb,ereg.is360),"\n", ) end From 140aa8bf1eebd48063abcd00696fe9cdbf0264ce Mon Sep 17 00:00:00 2001 From: Nathanael Wong Date: Mon, 24 Feb 2025 10:56:07 -0500 Subject: [PATCH 05/11] Need to update the show() for the ERA5Region --- src/region/region.jl | 39 ++++++++++++++------------------------- 1 file changed, 14 insertions(+), 25 deletions(-) diff --git a/src/region/region.jl b/src/region/region.jl index 4754a30..a90dc3e 100644 --- a/src/region/region.jl +++ b/src/region/region.jl @@ -124,30 +124,19 @@ end function show(io::IO, ereg::ERA5Region) geo = ereg.geo - - if typeof(geo) <: PolyRegion - print( - io, - "The ERA5Region wrapper for the \"$(ereg.ID)\" GeoRegion has the following properties:\n", - " Region ID (ID) : ", ereg.ID, '\n', - " Name (geo.name) : ", ereg.geo.name, '\n', - " Resolution (resolution) : ", ereg.resolution, '\n', - " Folder ID (string) : ", ereg.string, '\n', - " Bounds (geo.[N,S,E,W]) : ", geo.N, ", ", geo.S, ", ", geo.E, ", ", geo.W, '\n', - " Shape (geo.shape) : ", geo.shape, '\n', - " (geo.[isglb,is360]) : ",(ereg.isglb,ereg.is360),"\n", - ) - else - print( - io, - "The ERA5Region wrapper for the \"$(ereg.ID)\" GeoRegion has the following properties:\n", - " Region ID (ID) : ", ereg.ID, '\n', - " Name (geo.name) : ", ereg.geo.name, '\n', - " Resolution (resolution) : ", ereg.resolution, '\n', - " Folder ID (string) : ", ereg.string, '\n', - " Bounds (geo.[N,S,E,W]) : ", geo.N, ", ", geo.S, ", ", geo.E, ", ", geo.W, '\n', - " (geo.[isglb,is360]) : ",(ereg.isglb,ereg.is360),"\n", - ) - end + shape = geo.geometry.shape + + print( + io, + "The ERA5Region wrapper for the \"$(ereg.ID)\" GeoRegion has the following properties:\n", + " Region ID (ID) : ", ereg.ID, '\n', + " Name (geo.name) : ", geo.name, '\n', + " Resolution (resolution) : ", ereg.resolution, '\n', + " Folder ID (string) : ", ereg.string, '\n', + " Bounds (geo.[N,S,E,W]) : ", geo.N, ", ", geo.S, ", ", geo.E, ", ", geo.W, '\n', + " Rotation (geo.θ) : ", geo.θ, '\n', + " File Path (geo.path) : ", geo.path, '\n', + " Shape (geo.geometry.shape) : ", typeof(shape), "($(length(shape)))", '\n', + ) end \ No newline at end of file From 1dae993f00785869ab0f96449b3e5a9623a14438 Mon Sep 17 00:00:00 2001 From: Nathanael Wong Date: Fri, 28 Feb 2025 00:22:52 -0500 Subject: [PATCH 06/11] Get rid of python printing, adapting to new GeoRegions code --- src/downloads/cdsretrieve.jl | 41 ++------------------------------ src/downloads/downloads.jl | 46 ++++++++---------------------------- 2 files changed, 12 insertions(+), 75 deletions(-) diff --git a/src/downloads/cdsretrieve.jl b/src/downloads/cdsretrieve.jl index 8640740..e4724fe 100644 --- a/src/downloads/cdsretrieve.jl +++ b/src/downloads/cdsretrieve.jl @@ -47,19 +47,6 @@ function cdsretrieve( if !isfile(fnc) || overwrite retrieve(cdsretrieve_dataset(evar,e5ds),e5dkey,fnc,ckeys) - # tryretrieve = 0 - # while isinteger(tryretrieve) && (tryretrieve < 20) - # try - # retrieve(cdsretrieve_dataset(evar,e5ds),e5dkey,fnc,ckeys) - # tryretrieve += 0.5 - # catch - # tryretrieve += 1 - # @info "$(modulelog()) - Failed to retrieve/request data from CDSAPI on Attempt $(tryretrieve) of 20" - # end - # end - # if tryretrieve == 20 - # @warn "$(modulelog()) - Failed to retrieve/request data, skipping to next set of requests" - # end end flush(stderr) @@ -176,19 +163,7 @@ function cdsretrievegrib( cdsretrieve_area!(e5dkey,ereg) if !isfile(fnc) || overwrite - tryretrieve = 0 - while isinteger(tryretrieve) && (tryretrieve < 20) - try - retrieve(cdsretrieve_dataset(evar,e5ds),e5dkey,fnc,ckeys) - tryretrieve += 0.5 - catch - tryretrieve += 1 - @info "$(modulelog()) - Failed to retrieve/request data from CDSAPI on Attempt $(tryretrieve) of 20" - end - end - if tryretrieve == 20 - @warn "$(modulelog()) - Failed to retrieve/request data, skipping to next set of requests" - end + retrieve(cdsretrieve_dataset(evar,e5ds),e5dkey,fnc,ckeys) end flush(stderr) @@ -237,19 +212,7 @@ function cdsretrieve( cdsretrieve_area!(e5dkey,ereg) if !isfile(inc) || overwrite - tryretrieve = 0 - while isinteger(tryretrieve) && (tryretrieve < 20) - try - retrieve(cdsretrieve_dataset(evar[1],e5ds),e5dkey,fnc,ckeys) - tryretrieve += 0.5 - catch - tryretrieve += 1 - @info "$(modulelog()) - Failed to retrieve/request data from CDSAPI on Attempt $(tryretrieve) of 20" - end - end - if tryretrieve == 20 - @warn "$(modulelog()) - Failed to retrieve/request data, skipping to next set of requests" - end + retrieve(cdsretrieve_dataset(evar,e5ds),e5dkey,fnc,ckeys) split(e5ds,evar,ereg,lsd,dtii,fnc,tmpd) end diff --git a/src/downloads/downloads.jl b/src/downloads/downloads.jl index 56c4b87..e88ad14 100644 --- a/src/downloads/downloads.jl +++ b/src/downloads/downloads.jl @@ -27,17 +27,11 @@ function download( e5ds :: ERA5CDStore, evar :: SingleVariable, ereg :: ERA5Region; - ispy :: Bool = false, grib :: Bool = false, overwrite :: Bool = false ) - downloadcheckereg(ereg) - - if ispy - pythonprint(e5ds,evar,ereg) - else; cdsretrieve(e5ds,evar,ereg,grib,overwrite) - end + cdsretrieve(e5ds,evar,ereg,grib,overwrite) end @@ -71,7 +65,6 @@ function download( overwrite :: Bool = false ) - downloadcheckereg(ereg) cdsretrieve(e5ds,evar,ereg,overwrite) end @@ -113,7 +106,6 @@ function download( e5ds :: ERA5CDStore, evar :: PressureVariable, ereg :: ERA5Region; - ispy :: Bool = false, pall :: Bool = false, ptop :: Int = 0, pbot :: Int = 0, @@ -122,23 +114,17 @@ function download( overwrite :: Bool = false ) - downloadcheckereg(ereg) - - if ispy - pythonprint(e5ds,evar,ereg) - else - if pvec == [0] || iszero(evar.hPa) - pvec = downloadcheckplvl(pall,ptop,pbot) - end - if pall - if !grib - cdsretrieve(e5ds,evar,ereg,pvec,overwrite) - else - cdsretrievegrib(e5ds,evar,ereg,pvec,overwrite) - end + if pvec == [0] || iszero(evar.hPa) + pvec = downloadcheckplvl(pall,ptop,pbot) + end + if pall + if !grib + cdsretrieve(e5ds,evar,ereg,pvec,overwrite) else - cdsretrieve(e5ds,evar,ereg,grib,overwrite) + cdsretrievegrib(e5ds,evar,ereg,pvec,overwrite) end + else + cdsretrieve(e5ds,evar,ereg,grib,overwrite) end end @@ -155,18 +141,6 @@ function downloadcheckhPa( end -function downloadcheckereg( - ereg :: ERA5Region -) - - if !(typeof(ereg.geo) <: RectRegion) - - error("$(modulelog()) - ERA5Reanalysis is not yet set up to download GeoRegions that are not RectRegions. Check back in a later update for more.") - - end - -end - function downloadcheckplvl( pall :: Bool, ptop :: Int, From 27df8f1ca2a8ad792ec39ee49fa10ed95103f4e3 Mon Sep 17 00:00:00 2001 From: Nathanael Wong Date: Fri, 28 Feb 2025 01:08:30 -0500 Subject: [PATCH 07/11] Debugging time dimension definition, and update types --- src/save/daily.jl | 6 +++--- src/save/define.jl | 4 ++-- src/save/extract.jl | 10 +++++----- src/save/hourly.jl | 6 +++--- src/save/monthly.jl | 6 +++--- 5 files changed, 16 insertions(+), 16 deletions(-) diff --git a/src/save/daily.jl b/src/save/daily.jl index 6e9ab0d..cc58e10 100644 --- a/src/save/daily.jl +++ b/src/save/daily.jl @@ -24,11 +24,11 @@ function save( ds.dim["longitude"] = length(lsd.lon) ds.dim["latitude"] = length(lsd.lat) - ds.dim["time"] = ndy + ds.dim["valid_time"] = ndy nclon,nclat = save_definelonlat!(ds) - nctime = defVar(ds,"time",Int32,("time",),attrib = Dict( + nctime = defVar(ds,"valid_time",Int64,("valid_time",),attrib = Dict( "units" => "days since $(dt) 00:00:00.0", "long_name" => "time", "calendar" => "gregorian", @@ -76,7 +76,7 @@ function save( nclon,nclat = save_definelonlat!(ds) - nctime = defVar(ds,"valid_time",Int32,("valid_time",),attrib = Dict( + nctime = defVar(ds,"valid_time",Int64,("valid_time",),attrib = Dict( "units" => "days since $(dt) 00:00:00.0", "long_name" => "time", "calendar" => "gregorian", diff --git a/src/save/define.jl b/src/save/define.jl index 0784547..95f4be6 100644 --- a/src/save/define.jl +++ b/src/save/define.jl @@ -41,12 +41,12 @@ end function save_definelonlat!(ds::NCDataset) - nclon = defVar(ds,"longitude",Float32,("longitude",),attrib = Dict( + nclon = defVar(ds,"longitude",Float64,("longitude",),attrib = Dict( "units" => "degrees_east", "long_name" => "longitude", )) - nclat = defVar(ds,"latitude",Float32,("latitude",),attrib = Dict( + nclat = defVar(ds,"latitude",Float64,("latitude",),attrib = Dict( "units" => "degrees_north", "long_name" => "latitude", )) diff --git a/src/save/extract.jl b/src/save/extract.jl index a97e3cf..1093302 100644 --- a/src/save/extract.jl +++ b/src/save/extract.jl @@ -24,11 +24,11 @@ function save( ds.dim["longitude"] = length(lsd.lon) ds.dim["latitude"] = length(lsd.lat) - ds.dim["time"] = nhr + ds.dim["valid_time"] = nhr nclon,nclat = save_definelonlat!(ds) - nctime = defVar(ds,"time",Int32,("time",),attrib = Dict( + nctime = defVar(ds,"valid_time",Int64,("valid_time",),attrib = Dict( "units" => "hours since $(dt) 00:00:00.0", "long_name" => "time", "calendar" => "gregorian", @@ -73,11 +73,11 @@ function save( ds.dim["longitude"] = length(lsd.lon) ds.dim["latitude"] = length(lsd.lat) - ds.dim["time"] = nhr + ds.dim["valid_time"] = nhr nclon,nclat = save_definelonlat!(ds) - nctime = defVar(ds,"time",Int32,("time",),attrib = Dict( + nctime = defVar(ds,"valid_time",Int64,("valid_time",),attrib = Dict( "units" => "days since $(dt) 00:00:00.0", "long_name" => "time", "calendar" => "gregorian", @@ -121,7 +121,7 @@ function save( nclon,nclat = save_definelonlat!(ds) - nctime = defVar(ds,"valid_time",Int32,("valid_time",),attrib = Dict( + nctime = defVar(ds,"valid_time",Int64,("valid_time",),attrib = Dict( "units" => "hours since $(dt) 00:00:00.0", "long_name" => "time", "calendar" => "gregorian", diff --git a/src/save/hourly.jl b/src/save/hourly.jl index 22df329..ea059a6 100644 --- a/src/save/hourly.jl +++ b/src/save/hourly.jl @@ -24,11 +24,11 @@ function save( ds.dim["longitude"] = length(lsd.lon) ds.dim["latitude"] = length(lsd.lat) - ds.dim["time"] = nhr + ds.dim["valid_time"] = nhr nclon,nclat = save_definelonlat!(ds) - nctime = defVar(ds,"time",Int32,("time",),attrib = Dict( + nctime = defVar(ds,"valid_time",Int64,("valid_time",),attrib = Dict( "units" => "hours since $(dt) 00:00:00.0", "long_name" => "time", "calendar" => "gregorian", @@ -76,7 +76,7 @@ function save( nclon,nclat = save_definelonlat!(ds) - nctime = defVar(ds,"valid_time",Int32,("valid_time",),attrib = Dict( + nctime = defVar(ds,"valid_time",Int64,("valid_time",),attrib = Dict( "units" => "hours since $(dt) 00:00:00.0", "long_name" => "time", "calendar" => "gregorian", diff --git a/src/save/monthly.jl b/src/save/monthly.jl index 7c9f5d1..19f4b48 100644 --- a/src/save/monthly.jl +++ b/src/save/monthly.jl @@ -23,11 +23,11 @@ function save( ds.dim["longitude"] = length(lsd.lon) ds.dim["latitude"] = length(lsd.lat) - ds.dim["time"] = ntimesteps(e5ds) + ds.dim["valid_time"] = ntimesteps(e5ds) nclon,nclat = save_definelonlat!(ds) - nctime = defVar(ds,"time",Int32,("time",),attrib = Dict( + nctime = defVar(ds,"valid_time",Int64,("valid_time",),attrib = Dict( "units" => "hours since $(dt) 00:00:00.0", "long_name" => "time", "calendar" => "gregorian", @@ -73,7 +73,7 @@ function save( nclon,nclat = save_definelonlat!(ds) - nctime = defVar(ds,"valid_time",Int32,("valid_time",),attrib = Dict( + nctime = defVar(ds,"valid_time",Int64,("valid_time",),attrib = Dict( "units" => "hours since $(dt) 00:00:00.0", "long_name" => "time", "calendar" => "gregorian", From b5837e630ad69828ea63834835bd9a0f5bfcfc56 Mon Sep 17 00:00:00 2001 From: Nathanael Wong Date: Fri, 28 Feb 2025 15:01:07 -0500 Subject: [PATCH 08/11] Debugging the tables --- src/variable/tables.jl | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/variable/tables.jl b/src/variable/tables.jl index 1cc91f7..d6862b8 100644 --- a/src/variable/tables.jl +++ b/src/variable/tables.jl @@ -63,7 +63,8 @@ function tableSingles(; path :: AbstractString = homedir(), predefined :: Bool = true, custom :: Bool = false, - crop :: Bool = false + warn :: Bool = true, + crop :: Bool = false ) fmat = [] @@ -121,7 +122,8 @@ function tablePressures(; path :: AbstractString = homedir(), predefined :: Bool = true, custom :: Bool = false, - crop :: Bool = false + warn :: Bool = true, + crop :: Bool = false ) fmat = [] From ce7b75b560eaa3abf553ae086204d049e662c221 Mon Sep 17 00:00:00 2001 From: Nathanael Wong Date: Fri, 28 Feb 2025 15:40:10 -0500 Subject: [PATCH 09/11] Debugging save for Monthly data --- src/save/monthly.jl | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/save/monthly.jl b/src/save/monthly.jl index 19f4b48..dbe32fa 100644 --- a/src/save/monthly.jl +++ b/src/save/monthly.jl @@ -19,8 +19,6 @@ function save( extractnc,smoothlon,smoothlat ) - scale,offset = ncoffsetscale(data) - ds.dim["longitude"] = length(lsd.lon) ds.dim["latitude"] = length(lsd.lat) ds.dim["valid_time"] = ntimesteps(e5ds) From 385aaca6c9f8305b5d4b8e96b496927f6d987d96 Mon Sep 17 00:00:00 2001 From: Nathanael Wong Date: Thu, 6 Mar 2025 15:43:59 -0500 Subject: [PATCH 10/11] Force save at Float32 precision --- src/save/daily.jl | 4 ++-- src/save/hourly.jl | 4 ++-- src/save/monthly.jl | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/save/daily.jl b/src/save/daily.jl index cc58e10..0c0b875 100644 --- a/src/save/daily.jl +++ b/src/save/daily.jl @@ -1,5 +1,5 @@ function save( - data :: AbstractArray{<:Real,3}, + data :: AbstractArray{Float32,3}, dt :: Date, e5ds :: ERA5Daily, evar :: ERA5Variable, @@ -48,7 +48,7 @@ function save( end function save( - data :: AbstractArray{<:Real,3}, + data :: AbstractArray{Float32,3}, dt :: Date, e5ds :: ERA5Daily, evar :: ERA5Variable, diff --git a/src/save/hourly.jl b/src/save/hourly.jl index ea059a6..dca6521 100644 --- a/src/save/hourly.jl +++ b/src/save/hourly.jl @@ -1,5 +1,5 @@ function save( - data :: AbstractArray{<:Real,3}, + data :: AbstractArray{Float32,3}, dt :: Date, e5ds :: ERA5Hourly, evar :: ERA5Variable, @@ -48,7 +48,7 @@ function save( end function save( - data :: AbstractArray{<:Real,3}, + data :: AbstractArray{Float32,3}, dt :: Date, e5ds :: ERA5Hourly, evar :: ERA5Variable, diff --git a/src/save/monthly.jl b/src/save/monthly.jl index dbe32fa..68fe90f 100644 --- a/src/save/monthly.jl +++ b/src/save/monthly.jl @@ -1,5 +1,5 @@ function save( - data :: AbstractArray{<:Real,3}, + data :: AbstractArray{Float32,3}, dt :: Date, e5ds :: ERA5Monthly, evar :: ERA5Variable, @@ -45,7 +45,7 @@ function save( end function save( - data :: AbstractArray{<:Real,3}, + data :: AbstractArray{Float32,3}, dt :: Date, e5ds :: ERA5Monthly, evar :: ERA5Variable, From 861dd5869a4b554374093df7ee7ebbf643df99fe Mon Sep 17 00:00:00 2001 From: Nathanael Wong Date: Thu, 6 Mar 2025 17:01:25 -0500 Subject: [PATCH 11/11] There's no need to specifically have save methods for extraction anymore --- src/ERA5Reanalysis.jl | 1 - src/save/extract.jl | 141 ------------------------------------------ src/save/monthly.jl | 14 +++-- 3 files changed, 8 insertions(+), 148 deletions(-) delete mode 100644 src/save/extract.jl diff --git a/src/ERA5Reanalysis.jl b/src/ERA5Reanalysis.jl index 14ddcfc..5ffa5fd 100644 --- a/src/ERA5Reanalysis.jl +++ b/src/ERA5Reanalysis.jl @@ -172,6 +172,5 @@ include("save/define.jl") include("save/hourly.jl") include("save/daily.jl") include("save/monthly.jl") -include("save/extract.jl") end diff --git a/src/save/extract.jl b/src/save/extract.jl deleted file mode 100644 index 1093302..0000000 --- a/src/save/extract.jl +++ /dev/null @@ -1,141 +0,0 @@ -function save( - data :: AbstractArray{Float32,3}, - dt :: Date, - e5ds :: ERA5Hourly, - evar :: ERA5Variable, - ereg :: ERA5Region, - lsd :: LandSeaTopo; - extract :: Bool = false, - smooth :: Bool = false, - extractnc :: AbstractString = "", - smoothlon :: Real = 0, - smoothlat :: Real = 0, - smoothtime :: Int = 0 -) - - @info "$(modulelog()) - Saving raw $(e5ds.name) $(evar.name) data in $(ereg.geo.name) (Horizontal Resolution: $(ereg.resolution)) for $(year(dt)) $(Dates.monthname(dt)) ..." - - ds,fnc = save_createds( - e5ds, evar, ereg, dt, extract, smooth, - extractnc, smoothlon, smoothlat, smoothtime - ) - - nhr = 24 * daysinmonth(dt) - - ds.dim["longitude"] = length(lsd.lon) - ds.dim["latitude"] = length(lsd.lat) - ds.dim["valid_time"] = nhr - - nclon,nclat = save_definelonlat!(ds) - - nctime = defVar(ds,"valid_time",Int64,("valid_time",),attrib = Dict( - "units" => "hours since $(dt) 00:00:00.0", - "long_name" => "time", - "calendar" => "gregorian", - )) - - ncvar = save_definevar!(ds,evar) - - nclon[:] = lsd.lon - nclat[:] = lsd.lat - nctime[:] = collect(1:nhr) .- 1 - ncvar.var[:,:,:] = data - - close(ds) - - @info "$(modulelog()) - Raw $(uppercase(e5ds.name)) $(evar.name) in $(ereg.geo.name) (Horizontal Resolution: $(ereg.resolution)) for $(year(dt)) $(Dates.monthname(dt)) has been saved into $(fnc)." - -end - -function save( - data :: AbstractArray{Float32,3}, - dt :: Date, - e5ds :: ERA5Daily, - evar :: ERA5Variable, - ereg :: ERA5Region, - lsd :: LandSeaTopo; - extract :: Bool = false, - smooth :: Bool = false, - extractnc :: AbstractString = "", - smoothlon :: Real = 0, - smoothlat :: Real = 0, - smoothtime :: Int = 0 -) - - @info "$(modulelog()) - Saving raw $(e5ds.name) $(evar.name) data in $(ereg.geo.name) (Horizontal Resolution: $(ereg.resolution)) for $(year(dt)) $(Dates.monthname(dt)) ..." - - ds,fnc = save_createds( - e5ds, evar, ereg, dt, extract, smooth, - extractnc, smoothlon, smoothlat, smoothtime - ) - - nhr = daysinmonth(dt) - - ds.dim["longitude"] = length(lsd.lon) - ds.dim["latitude"] = length(lsd.lat) - ds.dim["valid_time"] = nhr - - nclon,nclat = save_definelonlat!(ds) - - nctime = defVar(ds,"valid_time",Int64,("valid_time",),attrib = Dict( - "units" => "days since $(dt) 00:00:00.0", - "long_name" => "time", - "calendar" => "gregorian", - )) - - ncvar = save_definevar!(ds,evar) - - nclon[:] = lsd.lon - nclat[:] = lsd.lat - nctime[:] = collect(1:nhr) .- 1 - ncvar.var[:,:,:] = data - - close(ds) - - @info "$(modulelog()) - Raw $(uppercase(e5ds.name)) $(evar.name) in $(ereg.geo.name) (Horizontal Resolution: $(ereg.resolution)) for $(year(dt)) $(Dates.monthname(dt)) has been saved into $(fnc)." - -end - -function save( - data :: AbstractArray{Float32,3}, - dt :: Date, - e5ds :: ERA5Monthly, - evar :: ERA5Variable, - ereg :: ERA5Region, - lsd :: LandSeaTopo; - extract :: Bool = false, - smooth :: Bool = false, - extractnc :: AbstractString = "", - smoothlon :: Real = 0, - smoothlat :: Real = 0, - smoothtime :: Int = 0 -) - - @info "$(modulelog()) - Saving raw $(e5ds.name) $(evar.name) data in $(ereg.geo.name) (Horizontal Resolution: $(ereg.resolution)) for $(year(dt)) ..." - - ds,fnc = save_createds(e5ds,evar,ereg,dt,extract,smooth,extractnc,smoothlon,smoothlat) - - ds.dim["longitude"] = length(lsd.lon) - ds.dim["latitude"] = length(lsd.lat) - ds.dim["valid_time"] = ntimesteps(e5ds) - - nclon,nclat = save_definelonlat!(ds) - - nctime = defVar(ds,"valid_time",Int64,("valid_time",),attrib = Dict( - "units" => "hours since $(dt) 00:00:00.0", - "long_name" => "time", - "calendar" => "gregorian", - )) - - ncvar = save_definevar!(ds,evar) - - nclon[:] = lsd.lon - nclat[:] = lsd.lat - nctime[:] = save_definetimes(e5ds,dt) - ncvar.var[:,:,:] = data - - close(ds) - - @info "$(modulelog()) - Raw $(uppercase(e5ds.name)) $(evar.name) in $(ereg.geo.name) (Horizontal Resolution: $(ereg.resolution)) for $(year(dt)) has been saved into $(fnc)." - -end \ No newline at end of file diff --git a/src/save/monthly.jl b/src/save/monthly.jl index 68fe90f..0ffe63a 100644 --- a/src/save/monthly.jl +++ b/src/save/monthly.jl @@ -7,9 +7,10 @@ function save( lsd :: LandSeaTopo; extract :: Bool = false, smooth :: Bool = false, - extractnc :: AbstractString = "", - smoothlon :: Real = 0, - smoothlat :: Real = 0, + extractnc :: AbstractString = "", + smoothlon :: Real = 0, + smoothlat :: Real = 0, + smoothtime :: Int = 0 ) @info "$(modulelog()) - Saving raw $(e5ds.name) $(evar.name) data in $(ereg.geo.name) (Horizontal Resolution: $(ereg.resolution)) for $(year(dt)) ..." @@ -52,9 +53,10 @@ function save( ereg :: ERA5Region; extract :: Bool = false, smooth :: Bool = false, - extractnc :: AbstractString = "", - smoothlon :: Real = 0, - smoothlat :: Real = 0, + extractnc :: AbstractString = "", + smoothlon :: Real = 0, + smoothlat :: Real = 0, + smoothtime :: Int = 0, ) @info "$(modulelog()) - Saving raw $(e5ds.name) $(evar.name) data in $(ereg.geo.name) (Horizontal Resolution: $(ereg.resolution)) for $(year(dt)) ..."