Skip to content

Commit ff35e6e

Browse files
kandersolarkandersolar
committed
function, tests, whatsnew, docs
1 parent 28ea577 commit ff35e6e

File tree

6 files changed

+253
-1
lines changed

6 files changed

+253
-1
lines changed

‎docs/sphinx/source/reference/iotools.rst‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -237,6 +237,18 @@ lower quality.
237237
iotools.read_crn
238238

239239

240+
MERRA-2
241+
^^^^^^^
242+
243+
A global reanalysis dataset providing weather, aerosol, and solar irradiance
244+
data.
245+
246+
.. autosummary::
247+
:toctree: generated/
248+
249+
iotools.get_merra2
250+
251+
240252
Generic data file readers
241253
-------------------------
242254

‎docs/sphinx/source/whatsnew/v0.13.2.rst‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,8 @@ Enhancements
2727
:py:func:`~pvlib.singlediode.bishop88_mpp`,
2828
:py:func:`~pvlib.singlediode.bishop88_v_from_i`, and
2929
:py:func:`~pvlib.singlediode.bishop88_i_from_v`. (:issue:`2497`, :pull:`2498`)
30-
30+
* Add :py:func:`~pvlib.iotools.get_merra2`, a function for accessing
31+
MERRA-2 reanalysis data. (:pull:`2572`)
3132

3233

3334
Documentation
@@ -53,4 +54,5 @@ Maintenance
5354

5455
Contributors
5556
~~~~~~~~~~~~
57+
* Kevin Anderson (:ghuser:`kandersolar`)
5658

‎pvlib/iotools/__init__.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,3 +45,4 @@
4545
from pvlib.iotools.meteonorm import get_meteonorm_observation_training # noqa: F401, E501
4646
from pvlib.iotools.meteonorm import get_meteonorm_tmy # noqa: F401
4747
from pvlib.iotools.nasa_power import get_nasa_power # noqa: F401
48+
from pvlib.iotools.merra2 import get_merra2 # noqa: F401

‎pvlib/iotools/merra2.py‎

Lines changed: 154 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,154 @@
1+
import pandas as pd
2+
import requests
3+
from io import StringIO
4+
5+
6+
VARIABLE_MAP = {
7+
'SWGDN': 'ghi',
8+
'SWGDNCLR': 'ghi_clear',
9+
'ALBEDO': 'albedo',
10+
'T2M': 'temp_air',
11+
'T2MDEW': 'temp_dew',
12+
'PS': 'pressure',
13+
'TOTEXTTAU': 'aod550',
14+
}
15+
16+
def get_merra2(latitude, longitude, start, end, username, password, dataset,
17+
variables, map_variables=True):
18+
"""
19+
Retrieve MERRA-2 time-series irradiance and meteorological data from
20+
NASA's GESDISC data archive.
21+
22+
MERRA-2 [1]_ offers modeled data for many atmospheric quantities at hourly
23+
resolution on a 0.5° x 0.625° global grid.
24+
25+
Access must be granted to the GESDISC data archive before EarthData
26+
credentials will work. See [2]_ for instructions.
27+
28+
Parameters
29+
----------
30+
latitude : float
31+
In decimal degrees, north is positive (ISO 19115).
32+
longitude: float
33+
In decimal degrees, east is positive (ISO 19115).
34+
start : datetime like or str
35+
First timestamp of the requested period. If a timezone is not
36+
specified, UTC is assumed.
37+
end : datetime like or str
38+
Last timestamp of the requested period. If a timezone is not
39+
specified, UTC is assumed.
40+
username : str
41+
NASA EarthData username.
42+
password : str
43+
NASA EarthData password.
44+
dataset : str
45+
Dataset name (with version), e.g. "M2T1NXRAD.5.12.4".
46+
variables : list of str
47+
List of variable names to retrieve. See the documentation of the
48+
specific dataset you are accessing for options.
49+
map_variables : bool, default True
50+
When true, renames columns of the DataFrame to pvlib variable names
51+
where applicable. See variable :const:`VARIABLE_MAP`.
52+
53+
Raises
54+
------
55+
ValueError
56+
If ``start`` and ``end`` are in different years, when converted to UTC.
57+
58+
Returns
59+
-------
60+
data : pd.DataFrame
61+
Time series data. The index corresponds to the middle of the interval.
62+
meta : dict
63+
Metadata.
64+
65+
Notes
66+
-----
67+
The following datasets provide quantities useful for PV modeling:
68+
69+
- M2T1NXRAD.5.12.4: SWGDN, SWGDNCLR, ALBEDO
70+
- M2T1NXSLV.5.12.4: T2M, U10M, V10M, T2MDEW, PS
71+
- M2T1NXAER.5.12.4: TOTEXTTAU
72+
73+
Note that MERRA2 does not currently provide DNI or DHI.
74+
75+
References
76+
----------
77+
.. [1] https://gmao.gsfc.nasa.gov/gmao-products/merra-2/
78+
.. [2] https://disc.gsfc.nasa.gov/earthdata-login
79+
"""
80+
81+
# general API info here:
82+
# https://docs.unidata.ucar.edu/tds/5.0/userguide/netcdf_subset_service_ref.html # noqa: E501
83+
84+
def _to_utc_dt_notz(dt):
85+
dt = pd.to_datetime(dt)
86+
if dt.tzinfo is None: # convert everything to UTC
87+
dt = dt.tz_localize("UTC")
88+
else:
89+
dt = dt.tz_convert("UTC")
90+
return dt.tz_localize(None) # drop tz so that isoformat() is clean
91+
92+
start = _to_utc_dt_notz(start)
93+
end = _to_utc_dt_notz(end)
94+
95+
if (year := start.year) != end.year:
96+
raise ValueError("start and end must be in the same year (in UTC)")
97+
98+
url = (
99+
"https://goldsmr4.gesdisc.eosdis.nasa.gov/thredds/ncss/grid/"
100+
f"MERRA2_aggregation/{dataset}/{dataset}_Aggregation_{year}.ncml"
101+
)
102+
103+
parameters = {
104+
'var': ",".join(variables),
105+
'latitude': latitude,
106+
'longitude': longitude,
107+
'time_start': start.isoformat() + "Z",
108+
'time_end': end.isoformat() + "Z",
109+
'accept': 'csv',
110+
}
111+
112+
auth = (username, password)
113+
114+
with requests.Session() as session:
115+
session.auth = auth
116+
login = session.request('get', url, params=parameters)
117+
response = session.get(login.url, auth=auth, params=parameters)
118+
119+
response.raise_for_status()
120+
121+
content = response.content.decode('utf-8')
122+
buffer = StringIO(content)
123+
df = pd.read_csv(buffer)
124+
125+
df.index = pd.to_datetime(df['time'])
126+
127+
meta = {}
128+
meta['dataset'] = dataset
129+
meta['station'] = df['station'].values[0]
130+
meta['latitude'] = df['latitude[unit="degrees_north"]'].values[0]
131+
meta['longitude'] = df['longitude[unit="degrees_east"]'].values[0]
132+
133+
# drop the non-data columns
134+
dropcols = ['time', 'station', 'latitude[unit="degrees_north"]',
135+
'longitude[unit="degrees_east"]']
136+
df = df.drop(columns=dropcols)
137+
138+
# column names are like T2M[unit="K"] by default. extract the unit
139+
# for the metadata, then rename col to just T2M
140+
units = {}
141+
rename = {}
142+
for col in df.columns:
143+
name, _ = col.split("[", maxsplit=1)
144+
unit = col.split('"')[1]
145+
units[name] = unit
146+
rename[col] = name
147+
148+
meta['units'] = units
149+
df = df.rename(columns=rename)
150+
151+
if map_variables:
152+
df = df.rename(columns=VARIABLE_MAP)
153+
154+
return df, meta

‎tests/conftest.py‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -130,6 +130,20 @@ def nrel_api_key():
130130
reason='requires solaranywhere credentials')
131131

132132

133+
try:
134+
# Attempt to load NASA EarthData credentials used for testing
135+
# pvlib.iotools.get_merra2
136+
earthdata_username = os.environ["EARTHDATA_USERNAME"]
137+
earthdata_password = os.environ["EARTHDATA_PASSWORD"]
138+
has_earthdata_credentials = True
139+
except KeyError:
140+
has_earthdata_credentials = False
141+
142+
requires_earthdata_credentials = pytest.mark.skipif(
143+
not has_earthdata_credentials,
144+
reason='requires EarthData credentials')
145+
146+
133147
try:
134148
import statsmodels # noqa: F401
135149
has_statsmodels = True

‎tests/iotools/test_merra2.py‎

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
"""
2+
tests for pvlib/iotools/merra2.py
3+
"""
4+
5+
import pandas as pd
6+
import pytest
7+
import pvlib
8+
import os
9+
from tests.conftest import RERUNS, RERUNS_DELAY, requires_earthdata_credentials
10+
11+
12+
@pytest.fixture
13+
def params():
14+
earthdata_username = os.environ["EARTHDATA_USERNAME"]
15+
earthdata_password = os.environ["EARTHDATA_PASSWORD"]
16+
17+
return {'latitude': 40.01, 'longitude': -80.01,
18+
'start': '2020-06-01 15:00', 'end': '2020-06-01 20:00',
19+
'dataset': 'M2T1NXRAD.5.12.4', 'variables': ['ALBEDO', 'SWGDN'],
20+
'username': earthdata_username, 'password': earthdata_password,
21+
}
22+
23+
24+
@pytest.fixture
25+
def expected():
26+
index = pd.date_range("2020-06-01 15:30", "2020-06-01 20:30", freq="h",
27+
tz="UTC")
28+
index.name = 'time'
29+
albedo = [0.163931, 0.1609407, 0.1601474, 0.1612476, 0.164664, 0.1711341]
30+
ghi = [ 930., 1002.75, 1020.25, 981.25, 886.5, 743.5]
31+
df = pd.DataFrame({'albedo': albedo, 'ghi': ghi}, index=index)
32+
return df
33+
34+
35+
@pytest.fixture
36+
def expected_meta():
37+
return {
38+
'dataset': 'M2T1NXRAD.5.12.4',
39+
'station': 'GridPointRequestedAt[40.010N_80.010W]',
40+
'latitude': 40.0,
41+
'longitude': -80.0,
42+
'units': {'ALBEDO': '1', 'SWGDN': 'W m-2'}
43+
}
44+
45+
46+
@requires_earthdata_credentials
47+
@pytest.mark.remote_data
48+
@pytest.mark.flaky(reruns=RERUNS, reruns_delay=RERUNS_DELAY)
49+
def test_get_merra2(params, expected, expected_meta):
50+
df, meta = pvlib.iotools.get_merra2(**params)
51+
pd.testing.assert_frame_equal(df, expected, check_freq=False)
52+
assert meta == expected_meta
53+
54+
55+
@requires_earthdata_credentials
56+
@pytest.mark.remote_data
57+
@pytest.mark.flaky(reruns=RERUNS, reruns_delay=RERUNS_DELAY)
58+
def test_get_merra2_map_variables(params, expected, expected_meta):
59+
df, meta = pvlib.iotools.get_merra2(**params, map_variables=False)
60+
expected = expected.rename(columns={'albedo': 'ALBEDO', 'ghi': 'SWGDN'})
61+
pd.testing.assert_frame_equal(df, expected, check_freq=False)
62+
assert meta == expected_meta
63+
64+
65+
def test_get_merra2_error():
66+
with pytest.raises(ValueError, match='must be in the same year'):
67+
pvlib.iotools.get_merra2(40, -80, '2019-12-31', '2020-01-02',
68+
username='anything', password='anything',
69+
dataset='anything', variables=[])

0 commit comments

Comments
 (0)