Skip to content

Commit 77ccf56

Browse files
marqhmarqh
committed
recurse through groups
1 parent 4532a64 commit 77ccf56

File tree

4 files changed

+291
-21
lines changed

4 files changed

+291
-21
lines changed

‎lib/bald/__init__.py

Lines changed: 33 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -900,8 +900,28 @@ def _prefixes_and_aliases(fhandle, identity, alias_dict, cache):
900900
return prefixes, aliases, aliasgraph, prefix_var_name
901901

902902

903-
def _load_netcdf_group():
904-
pass
903+
def _load_netcdf_group(agroup, baseuri, gk, root_container, prefixes, prefix_group_name, aliases, aliasgraph, cache):
904+
gattrs = {}
905+
for k in agroup.ncattrs():
906+
gattrs[k] = getattr(agroup, k)
907+
908+
gidentity = baseuri + gk + '/'
909+
910+
gcontainer = Container(gidentity, '', gattrs, prefixes=prefixes,
911+
aliases=aliases, alias_graph=aliasgraph)
912+
913+
gcontainer.attrs['bald__contains'] = set()
914+
915+
_load_netcdf_group_vars(agroup, gcontainer, gidentity, gattrs, prefixes, prefix_group_name, aliases, aliasgraph, cache)
916+
if 'bald__contains' not in root_container.attrs:
917+
root_container.attrs['bald__contains'] = set()
918+
root_container.attrs['bald__contains'].add(gcontainer)
919+
for gk in agroup.groups:
920+
921+
_load_netcdf_group(agroup.groups[gk], baseuri, gk, gcontainer,
922+
prefixes, prefix_group_name, aliases, aliasgraph, cache)
923+
924+
905925

906926
def _load_netcdf_group_vars(fhandle, root_container, baseuri, attrs, prefixes,
907927
prefix_var_name, aliases, aliasgraph, cache):
@@ -923,14 +943,19 @@ def _load_netcdf_group_vars(fhandle, root_container, baseuri, attrs, prefixes,
923943

924944
if not isinstance(fhandle.variables[name][0], np.ma.core.MaskedConstant):
925945
sattrs['bald__first_value'] = fhandle.variables[name][0]
926-
if np.issubdtype(sattrs['bald__first_value'], np.integer):
946+
if isinstance(sattrs['bald__first_value'], str):
947+
pass
948+
949+
elif np.issubdtype(sattrs['bald__first_value'], np.integer):
927950
sattrs['bald__first_value'] = int(sattrs['bald__first_value'])
928951
elif np.issubdtype(sattrs['bald__first_value'], np.floating):
929952
sattrs['bald__first_value'] = float(sattrs['bald__first_value'])
930953
if (len(fhandle.variables[name]) > 1 and
931954
not isinstance(fhandle.variables[name][-1], np.ma.core.MaskedConstant)):
932955
sattrs['bald__last_value'] = fhandle.variables[name][-1]
933-
if np.issubdtype(sattrs['bald__last_value'], np.integer):
956+
if isinstance(sattrs['bald__last_value'], str):
957+
pass
958+
elif np.issubdtype(sattrs['bald__last_value'], np.integer):
934959
sattrs['bald__last_value'] = int(sattrs['bald__last_value'])
935960
elif np.issubdtype(sattrs['bald__last_value'], np.floating):
936961
sattrs['bald__last_value'] = float(sattrs['bald__last_value'])
@@ -999,7 +1024,6 @@ def _load_netcdf_group_vars(fhandle, root_container, baseuri, attrs, prefixes,
9991024

10001025
file_variables[name] = var
10011026

1002-
10031027
reference_prefixes = dict()
10041028
reference_graph = copy.copy(aliasgraph)
10051029

@@ -1149,23 +1173,11 @@ def load_netcdf(afilepath, baseuri=None, alias_dict=None, cache=None, file_locat
11491173
for gk in fhandle.groups:
11501174
if gk == prefix_group_name:
11511175
continue
1152-
gattrs = {}
1153-
for k in fhandle.groups[gk].ncattrs():
1154-
gattrs[k] = getattr(fhandle.groups[gk], k)
1155-
1156-
gidentity = baseuri + gk + '/'
1157-
1158-
gcontainer = Container(gidentity, '', gattrs, prefixes=prefixes,
1159-
aliases=aliases, alias_graph=aliasgraph)
1160-
1161-
gcontainer.attrs['bald__contains'] = set()
1162-
1163-
_load_netcdf_group_vars(fhandle.groups[gk], gcontainer, gidentity, attrs, prefixes, prefix_group_name, aliases, aliasgraph, cache)
1164-
1165-
root_container.attrs['bald__contains'].add(gcontainer)
1166-
11671176

1168-
_load_netcdf_group()
1177+
_load_netcdf_group(fhandle.groups[gk], baseuri, gk, root_container,
1178+
prefixes, prefix_group_name, aliases, aliasgraph, cache)
1179+
# _create_references(root_container,
1180+
# prefixes, prefix_group_name, aliases, aliasgraph, cache)
11691181

11701182
return root_container
11711183

‎lib/bald/tests/integration/CDL/group_array_geo.cdl

Lines changed: 113 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,113 @@
1+
netcdf tmpMwXy8U {
2+
dimensions:
3+
x_t = 144 ;
4+
y_t = 90 ;
5+
x_q = 144 ;
6+
y_q = 91 ;
7+
time = 4 ;
8+
variables:
9+
float x_t(x_t) ;
10+
x_t:cf__standard_name = "longitude" ;
11+
x_t:nc__units = "degrees" ;
12+
13+
14+
float y_t(y_t) ;
15+
y_t:cf__standard_name = "latitude" ;
16+
y_t:nc__units = "degrees" ;
17+
18+
19+
float x_q(x_q) ;
20+
x_q:cf__standard_name = "longitude" ;
21+
x_q:nc__units = "degrees" ;
22+
23+
24+
float y_q(y_q) ;
25+
y_q:cf__standard_name = "latitude" ;
26+
y_q:nc__units = "degrees" ;
27+
28+
int theta_points ;
29+
theta_points:georef__wkt_crs = "http://www.epsg-registry.org/export.htm?wkt=urn:ogc:def:crs:EPSG::4326" ;
30+
theta_points:georef__coord_tuple = "(y_t, x_t)" ;
31+
theta_points:rdfs__label = "Theta Points" ;
32+
theta_points:dct__description = "Arakawa C Theta points defined with respect to a 2D WGS84 geodetic surface." ;
33+
34+
int u_points ;
35+
u_points:cf__wkt_crs = "http://www.epsg-registry.org/export.htm?wkt=urn:ogc:def:crs:EPSG::4326" ;
36+
u_points:georef__coord_tuple = "(y_t, x_q)" ;
37+
u_points:rdfs__label = "U Points" ;
38+
u_points:dct__description = "Arakawa C U points defined with respect to a 2D WGS84 geodetic surface." ;
39+
40+
int v_points ;
41+
v_points:georef__wkt_crs = "http://www.epsg-registry.org/export.htm?wkt=urn:ogc:def:crs:EPSG::4326" ;
42+
v_points:georef__coord_tuple = "(y_q, x_t)" ;
43+
v_points:rdfs__label = "V Points" ;
44+
v_points:dct__description = "Arakawa C V points defined with respect to a 2D WGS84 geodetic surface." ;
45+
46+
string time(time) ;
47+
time:rdfs__label = "Time" ;
48+
49+
// global attributes:
50+
:bald__isPrefixedBy = "prefix_list" ;
51+
52+
data:
53+
54+
y_t = -90, -88, -86, -84, -82, -80, -78, -76, -74, -72, -70, -68, -66, -64, -62, -60, -58, -56, -54, -52, -50, -48, -46, -44, -42, -40, -38, -36, -34, -32, -30, -28, -26, -24, -22, -20, -18, -16, -14, -12, -10, -8, -6, -4, -2, 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, 32, 34, 36, 38, 40, 42, 44, 46, 48, 50, 52, 54, 56, 58, 60, 62, 64, 66, 68, 70, 72, 74, 76, 78, 80, 82, 84, 86, 88, 90 ;
55+
56+
x_t = 0.0, 2.5, 5.0, 7.5, 10.0, 12.5, 15.0, 17.5, 20.0, 22.5, 25.0, 27.5, 30.0, 32.5, 35.0, 37.5, 40.0, 42.5, 45.0, 47.5, 50.0, 52.5, 55.0, 57.5, 60.0, 62.5, 65.0, 67.5, 70.0, 72.5, 75.0, 77.5, 80.0, 82.5, 85.0, 87.5, 90.0, 92.5, 95.0, 97.5, 100.0, 102.5, 105.0, 107.5, 110.0, 112.5, 115.0, 117.5, 120.0, 122.5, 125.0, 127.5, 130.0, 132.5, 135.0, 137.5, 140.0, 142.5, 145.0, 147.5, 150.0, 152.5, 155.0, 157.5, 160.0, 162.5, 165.0, 167.5, 170.0, 172.5, 175.0, 177.5, 180.0, 182.5, 185.0, 187.5, 190.0, 192.5, 195.0, 197.5, 200.0, 202.5, 205.0, 207.5, 210.0, 212.5, 215.0, 217.5, 220.0, 222.5, 225.0, 227.5, 230.0, 232.5, 235.0, 237.5, 240.0, 242.5, 245.0, 247.5, 250.0, 252.5, 255.0, 257.5, 260.0, 262.5, 265.0, 267.5, 270.0, 272.5, 275.0, 277.5, 280.0, 282.5, 285.0, 287.5, 290.0, 292.5, 295.0, 297.5, 300.0, 302.5, 305.0, 307.5, 310.0, 312.5, 315.0, 317.5, 320.0, 322.5, 325.0, 327.5, 330.0, 332.5, 335.0, 337.5, 340.0, 342.5, 345.0, 347.5, 350.0, 352.5, 355.0, 357.5 ;
57+
58+
y_q = -89, -87, -85, -83, -81, -79, -77, -75, -73, -71, -69, -67, -65, -63, -61, -59, -57, -55, -53, -51, -49, -47, -45, -43, -41, -39, -37, -35, -33, -31, -29, -27, -25, -23, -21, -19, -17, -15, -13, -11, -9, -7, -5, -3, -1, 1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31, 33, 35, 37, 39, 41, 43, 45, 47, 49, 51, 53, 55, 57, 59, 61, 63, 65, 67, 69, 71, 73, 75, 77, 79, 81, 83, 85, 87, 89 ;
59+
60+
x_q = 1.25, 3.75, 6.25, 8.75, 11.25, 13.75, 16.25, 18.75, 21.25, 23.75, 26.25, 28.75, 31.25, 33.75, 36.25, 38.75, 41.25, 43.75, 46.25, 48.75, 51.25, 53.75, 56.25, 58.75, 61.25, 63.75, 66.25, 68.75, 71.25, 73.75, 76.25, 78.75, 81.25, 83.75, 86.25, 88.75, 91.25, 93.75, 96.25, 98.75, 101.25, 103.75, 106.25, 108.75, 111.25, 113.75, 116.25, 118.75, 121.25, 123.75, 126.25, 128.75, 131.25, 133.75, 136.25, 138.75, 141.25, 143.75, 146.25, 148.75, 151.25, 153.75, 156.25, 158.75, 161.25, 163.75, 166.25, 168.75, 171.25, 173.75, 176.25, 178.75, 181.25, 183.75, 186.25, 188.75, 191.25, 193.75, 196.25, 198.75, 201.25, 203.75, 206.25, 208.75, 211.25, 213.75, 216.25, 218.75, 221.25, 223.75, 226.25, 228.75, 231.25, 233.75, 236.25, 238.75, 241.25, 243.75, 246.25, 248.75, 251.25, 253.75, 256.25, 258.75, 261.25, 263.75, 266.25, 268.75, 271.25, 273.75, 276.25, 278.75, 281.25, 283.75, 286.25, 288.75, 291.25, 293.75, 296.25, 298.75, 301.25, 303.75, 306.25, 308.75, 311.25, 313.75, 316.25, 318.75, 321.25, 323.75, 326.25, 328.75, 331.25, 333.75, 336.25, 338.75, 341.25, 343.75, 346.25, 348.75, 351.25, 353.75, 356.25, 358.75 ;
61+
62+
time = "2020-05-05T00:00Z", "2020-05-05T06:00Z", "2020-05-05T12:00Z", "2020-05-05T18:00Z" ;
63+
64+
// prefix group
65+
group: prefix_list {
66+
:bald__ = "https://www.opengis.net/def/binary-array-ld/" ;
67+
:rdf__ = "http://www.w3.org/1999/02/22-rdf-syntax-ns#" ;
68+
:bald__ = "https://www.opengis.net/def/binary-array-ld/" ;
69+
:rdf__ = "http://www.w3.org/1999/02/22-rdf-syntax-ns#" ;
70+
:rdfs__ = "http://www.w3.org/2000/01/rdf-schema#" ;
71+
:cf__ = "http://def.scitools.org.uk/CFTerms/" ;
72+
:nc__ = "http://def.scitools.org.uk/NetCDF/" ;
73+
:geo__ = "http://www.opengis.net/ont/geosparql#" ;
74+
:nc__ = "http://def.scitools.org.uk/NetCDF/" ;
75+
:georef__ = "http://def.scitools.org.uk/referencing-by-coordinates/" ;
76+
:dct__ = "http://purl.org/dc/terms/" ;
77+
}
78+
79+
group: vector_quantities {
80+
:vq__vector_quantities = "wind" ;
81+
82+
83+
group: wind {
84+
variables:
85+
86+
float u_wind(time, y_t, x_q) ;
87+
u_wind:cf__standard_name = "x_wind" ;
88+
u_wind:nc__units = "m s-1" ;
89+
u_wind:georef__domain = "u_points" ;
90+
91+
float v_wind(time, y_q, x_t) ;
92+
v_wind:cf__standard_name = "y_wind" ;
93+
v_wind:nc__units = "m s-1" ;
94+
v_wind:georef__domain = "v_points" ;
95+
96+
:vq__i_component = "x_wind" ;
97+
:vq__j_component = "y_wind" ;
98+
:rdfs__label = "Wind Vector" ;
99+
}
100+
101+
}
102+
103+
group: scalar_quantities {
104+
variables:
105+
106+
float air_pressure(time, y_t, x_t) ;
107+
air_pressure:cf__standard_name = "air_pressure" ;
108+
air_pressure:nc__units = "Pa" ;
109+
air_pressure:geo__domain = "theta_points" ;
110+
111+
}
112+
113+
}

‎lib/bald/tests/integration/TTL/group_array_geo.ttl

Lines changed: 116 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,116 @@
1+
@prefix bald: <https://www.opengis.net/def/binary-array-ld/> .
2+
@prefix cf: <http://def.scitools.org.uk/CFTerms/> .
3+
@prefix dcat: <http://www.w3.org/ns/dcat#> .
4+
@prefix dct: <http://purl.org/dc/terms/> .
5+
@prefix geo: <http://www.opengis.net/ont/geosparql#> .
6+
@prefix georef: <http://def.scitools.org.uk/referencing-by-coordinates/> .
7+
@prefix nc: <http://def.scitools.org.uk/NetCDF/> .
8+
@prefix ns1: <file://CDL/group_array_geo.cdl/vector_quantities/> .
9+
@prefix ns2: <file://CDL/group_array_geo.cdl/wind/> .
10+
@prefix rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> .
11+
@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#> .
12+
@prefix this: <file://CDL/group_array_geo.cdl/> .
13+
@prefix xml: <http://www.w3.org/XML/1998/namespace> .
14+
@prefix xsd: <http://www.w3.org/2001/XMLSchema#> .
15+
16+
this: a bald:Container ;
17+
dct:format [ a dct:MediaType ;
18+
dct:identifier <http://vocab.nerc.ac.uk/collection/M01/current/NC/> ] ;
19+
dcat:distribution [ a dcat:Distribution ;
20+
dcat:mediaType [ a dcat:MediaType ;
21+
dct:identifier "application/x-netcdf" ] ] ;
22+
bald:contains <file://CDL/group_array_geo.cdl/scalar_quantities/>,
23+
this:theta_points,
24+
this:time,
25+
this:u_points,
26+
this:v_points,
27+
ns1:,
28+
this:x_q,
29+
this:x_t,
30+
this:y_q,
31+
this:y_t ;
32+
bald:isPrefixedBy "prefix_list" .
33+
34+
<file://CDL/group_array_geo.cdl/scalar_quantities/> a bald:Container ;
35+
bald:contains <file://CDL/group_array_geo.cdl/scalar_quantities/air_pressure> .
36+
37+
<file://CDL/group_array_geo.cdl/scalar_quantities/air_pressure> a bald:Array ;
38+
cf:standard_name "air_pressure" ;
39+
nc:units "Pa" ;
40+
geo:domain "theta_points" ;
41+
bald:shape ( 4 90 144 ) .
42+
43+
this:theta_points a bald:Resource ;
44+
rdfs:label "Theta Points" ;
45+
georef:coord_tuple "(y_t, x_t)" ;
46+
georef:wkt_crs <http://www.epsg-registry.org/export.htm?wkt=urn:ogc:def:crs:EPSG::4326> ;
47+
dct:description "Arakawa C Theta points defined with respect to a 2D WGS84 geodetic surface." .
48+
49+
this:time a bald:Array ;
50+
rdfs:label "Time" ;
51+
bald:first_value "2020-05-05T00:00Z" ;
52+
bald:last_value "2020-05-05T18:00Z" ;
53+
bald:shape ( 4 ) .
54+
55+
this:u_points a bald:Resource ;
56+
rdfs:label "U Points" ;
57+
cf:wkt_crs <http://www.epsg-registry.org/export.htm?wkt=urn:ogc:def:crs:EPSG::4326> ;
58+
georef:coord_tuple "(y_t, x_q)" ;
59+
dct:description "Arakawa C U points defined with respect to a 2D WGS84 geodetic surface." .
60+
61+
this:v_points a bald:Resource ;
62+
rdfs:label "V Points" ;
63+
georef:coord_tuple "(y_q, x_t)" ;
64+
georef:wkt_crs <http://www.epsg-registry.org/export.htm?wkt=urn:ogc:def:crs:EPSG::4326> ;
65+
dct:description "Arakawa C V points defined with respect to a 2D WGS84 geodetic surface." .
66+
67+
ns1: a bald:Container ;
68+
ns1:vq__vector_quantities "wind" ;
69+
bald:contains ns2: .
70+
71+
ns2: a bald:Container ;
72+
rdfs:label "Wind Vector" ;
73+
ns2:vq__i_component "x_wind" ;
74+
ns2:vq__j_component "y_wind" ;
75+
bald:contains ns2:u_wind,
76+
ns2:v_wind .
77+
78+
ns2:u_wind a bald:Array ;
79+
cf:standard_name "x_wind" ;
80+
nc:units "m s-1" ;
81+
georef:domain "u_points" ;
82+
bald:shape ( 4 90 144 ) .
83+
84+
ns2:v_wind a bald:Array ;
85+
cf:standard_name "y_wind" ;
86+
nc:units "m s-1" ;
87+
georef:domain "v_points" ;
88+
bald:shape ( 4 91 144 ) .
89+
90+
this:x_q a bald:Array ;
91+
cf:standard_name "longitude" ;
92+
nc:units "degrees" ;
93+
bald:first_value 1.25 ;
94+
bald:last_value 358.75 ;
95+
bald:shape ( 144 ) .
96+
97+
this:x_t a bald:Array ;
98+
cf:standard_name "longitude" ;
99+
nc:units "degrees" ;
100+
bald:first_value 0.0 ;
101+
bald:last_value 357.5 ;
102+
bald:shape ( 144 ) .
103+
104+
this:y_q a bald:Array ;
105+
cf:standard_name "latitude" ;
106+
nc:units "degrees" ;
107+
bald:first_value -89.0 ;
108+
bald:shape ( 91 ) .
109+
110+
this:y_t a bald:Array ;
111+
cf:standard_name "latitude" ;
112+
nc:units "degrees" ;
113+
bald:first_value -90.0 ;
114+
bald:last_value 88.0 ;
115+
bald:shape ( 90 ) .
116+

‎lib/bald/tests/integration/test_cdl_rdfgraph.py

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -241,3 +241,32 @@ def test_hgroups(self):
241241
expected_rdfgraph.parse(sf, format='n3')
242242
self.check_result(rdfgraph, expected_rdfgraph)
243243

244+
def test_group_array_geo(self):
245+
with self.temp_filename('.nc') as tfile:
246+
name = 'group_array_geo'
247+
# hgurl = 'https://www.unidata.ucar.edu/software/netcdf/examples/test_hgroups.cdl'
248+
# res = requests.get(hgurl)
249+
# if res.status_code != 200:
250+
# raise ValueError('{} failed to download: {}'.format(hgurl, res.status_code))
251+
# with self.temp_filename('.cdl.') as cdlfile:
252+
# with open(cdlfile, 'w') as fh:
253+
# fh.write(res.text)
254+
cdl_file = os.path.join(self.cdl_path, '{}.cdl'.format(name))
255+
subprocess.check_call(['ncgen', '-o', tfile, cdl_file])
256+
cdl_file_uri = 'file://CDL/{}.cdl'.format(name)
257+
alias_dict = {# 'NetCDF': 'http://def.scitools.org.uk/NetCDF',
258+
# 'CFTerms': 'http://def.scitools.org.uk/CFTerms',
259+
# 'cf_sname': 'http://vocab.nerc.ac.uk/standard_name/'
260+
}
261+
root_container = bald.load_netcdf(tfile, baseuri=cdl_file_uri,
262+
alias_dict=alias_dict, cache=self.acache)
263+
rdfgraph = root_container.rdfgraph()
264+
ttl = rdfgraph.serialize(format='n3').decode("utf-8")
265+
if os.environ.get('bald_update_results') is not None:
266+
with open(os.path.join(self.ttl_path, '{}.ttl'.format(name)), 'w') as sf:
267+
sf.write(ttl)
268+
with open(os.path.join(self.ttl_path, '{}.ttl'.format(name)), 'r') as sf:
269+
expected_rdfgraph = rdflib.Graph()
270+
expected_rdfgraph.parse(sf, format='n3')
271+
self.check_result(rdfgraph, expected_rdfgraph)
272+

0 commit comments

Comments
 (0)