Skip to content

Commit ba60ce2

Browse files
marqhmarqh
committed
prefix context and firstValue lastValue
1 parent bbb2f03 commit ba60ce2

8 files changed

+178
-114
lines changed

‎lib/bald/__init__.py

Lines changed: 69 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
import contextlib
33
import copy
44
from difflib import SequenceMatcher
5+
import json
56
import operator
67
import os
78
import re
@@ -803,7 +804,7 @@ def load(afilepath):
803804
except NameError:
804805
pass
805806

806-
def _prefixes_and_aliases(fhandle, identity, alias_dict, cache):
807+
def _prefixes_and_aliases(fhandle, identity, alias_dict, prefix_contexts, cache):
807808
# prefixes are defined as group attributes in a dedicated group, and/or
808809
# by external resources
809810
prefix_var_name = None
@@ -833,25 +834,52 @@ def _prefixes_and_aliases(fhandle, identity, alias_dict, cache):
833834
# if k.endswith('__'):
834835
# prefixes[k] = getattr(fhandle, k)
835836

836-
prefix_graph = rdflib.Graph()
837-
for prefix_url in prefix_urls:
838-
res = cache[prefix_url]
839-
try:
840-
prefix_graph.parse(data=res.text, format='xml')
841-
except Exception:
842-
print('Failed to parse: {} for prefixes.'.format(prefix_url))
843-
844-
qres = prefix_graph.query("select ?prefix ?uri where \n"
845-
"{\n"
846-
"?s <http://purl.org/vocab/vann/preferredNamespacePrefix> ?prefix ;\n"
847-
"<http://purl.org/vocab/vann/preferredNamespaceUri> ?uri . \n"
848-
"}")
849-
for res in qres:
850-
key, value = (str(res[0]), str(res[1]))
851-
if key in prefixes and value !=prefixes[key]:
852-
prefixes.pop(key)
837+
# prefix_graph = rdflib.Graph()
838+
# for prefix_url in prefix_urls:
839+
# res = cache[prefix_url]
840+
# try:
841+
# prefix_graph.parse(data=res.text, format='xml')
842+
# except Exception:
843+
# print('Failed to parse: {} for prefixes.'.format(prefix_url))
844+
845+
# qres = prefix_graph.query("select ?prefix ?uri where \n"
846+
# "{\n"
847+
# "?s <http://purl.org/vocab/vann/preferredNamespacePrefix> ?prefix ;\n"
848+
# "<http://purl.org/vocab/vann/preferredNamespaceUri> ?uri . \n"
849+
# "}")
850+
# for res in qres:
851+
# key, value = (str(res[0]), str(res[1]))
852+
# if key in prefixes and value !=prefixes[key]:
853+
# prefixes.pop(key)
854+
# else:
855+
# prefixes[key] = value
856+
857+
# # check that default set is handled, i.e. bald__ and rdf__
858+
# if 'bald__' not in prefixes:
859+
# prefixes['bald__'] = "https://www.opengis.net/def/binary-array-ld/"
860+
861+
# if 'rdf__' not in prefixes:
862+
# prefixes['rdf__'] = "http://www.w3.org/1999/02/22-rdf-syntax-ns#"
863+
864+
## query keep above
865+
context_prefixes = {}
866+
for prefix_context in prefix_contexts:
867+
if prefix_context.startswith('http://') or prefix_context.startswith('https://'):
868+
prefcon = json.loads(cache[prefix_context].text)
853869
else:
854-
prefixes[key] = value
870+
prefcon = json.loads(prefix_context)
871+
if '@context' in prefcon:
872+
for key in prefcon['@context']:
873+
pref = '{}__'.format(key)
874+
if pref in context_prefixes and context_prefixes[pref] != prefcon['@context'][key]:
875+
context_prefixes[pref] = None
876+
else:
877+
context_prefixes[pref] = prefcon['@context'][key]
878+
for akey in context_prefixes:
879+
if context_prefixes[akey] is None:
880+
context_prefixes.pop(akey)
881+
882+
precedence_update(prefixes, context_prefixes)
855883

856884
# check that default set is handled, i.e. bald__ and rdf__
857885
if 'bald__' not in prefixes:
@@ -960,18 +988,18 @@ def _load_netcdf_group_vars(fhandle, agroup, root_container, baseuri, identity_p
960988
if isinstance(sattrs['bald__first_value'], str):
961989
pass
962990

963-
elif np.issubdtype(sattrs['bald__first_value'], np.integer):
991+
elif np.issubdtype(sattrs['bald__first_value'].dtype, np.integer):
964992
sattrs['bald__first_value'] = int(sattrs['bald__first_value'])
965-
elif np.issubdtype(sattrs['bald__first_value'], np.floating):
993+
elif np.issubdtype(sattrs['bald__first_value'].dtype, np.floating):
966994
sattrs['bald__first_value'] = float(sattrs['bald__first_value'])
967995
if (len(agroup.variables[name]) > 1 and
968996
not isinstance(agroup.variables[name][-1], np.ma.core.MaskedConstant)):
969997
sattrs['bald__last_value'] = agroup.variables[name][-1]
970998
if isinstance(sattrs['bald__last_value'], str):
971999
pass
972-
elif np.issubdtype(sattrs['bald__last_value'], np.integer):
1000+
elif np.issubdtype(sattrs['bald__last_value'].dtype, np.integer):
9731001
sattrs['bald__last_value'] = int(sattrs['bald__last_value'])
974-
elif np.issubdtype(sattrs['bald__last_value'], np.floating):
1002+
elif np.issubdtype(sattrs['bald__last_value'].dtype, np.floating):
9751003
sattrs['bald__last_value'] = float(sattrs['bald__last_value'])
9761004

9771005
# datetime special case
@@ -1162,14 +1190,18 @@ def _load_netcdf_group_vars(fhandle, agroup, root_container, baseuri, identity_p
11621190
aliases, aliasgraph)
11631191

11641192

1165-
def load_netcdf(afilepath, baseuri=None, alias_dict=None, cache=None, file_locator=None):
1193+
def load_netcdf(afilepath, baseuri=None, alias_dict=None, prefix_contexts=None, cache=None, file_locator=None):
11661194
"""
11671195
Load a file with respect to binary-array-linked-data.
11681196
Returns a :class:`bald.Collection`
11691197
"""
11701198

1171-
if alias_dict == None:
1199+
if alias_dict is None:
11721200
alias_dict = {}
1201+
if isinstance(prefix_contexts, str):
1202+
prefix_contexts = [prefix_contexts]
1203+
elif prefix_contexts is None:
1204+
prefix_contexts = []
11731205
if cache is None:
11741206
cache = HttpCache()
11751207

@@ -1183,7 +1215,8 @@ def load_netcdf(afilepath, baseuri=None, alias_dict=None, cache=None, file_locat
11831215

11841216
identity = baseuri
11851217

1186-
prefixes, aliases, aliasgraph, prefix_group_name = _prefixes_and_aliases(fhandle, identity, alias_dict, cache)
1218+
prefixes, aliases, aliasgraph, prefix_group_name = _prefixes_and_aliases(fhandle, identity, alias_dict,
1219+
prefix_contexts, cache)
11871220

11881221
attrs = {}
11891222
for k in fhandle.ncattrs():
@@ -1337,6 +1370,16 @@ def careful_update(adict, bdict):
13371370
adict.update(bdict)
13381371
return adict
13391372

1373+
def precedence_update(maindict, updatingdict):
1374+
"""
1375+
Carefully updates a main dictionary with an updating dictionary,
1376+
only inputting new values, and never overwriting values.
1377+
1378+
"""
1379+
for akey in updatingdict:
1380+
if akey not in maindict:
1381+
maindict[akey] = updatingdict[akey]
1382+
13401383
def load_hdf5(afilepath, baseuri=None, alias_dict=None, cache=None):
13411384
if cache is None:
13421385
cache = HttpCache()

‎lib/bald/tests/integration/CDL/array_reference_external_prefix.cdl renamed to ‎lib/bald/tests/integration/CDL/array_reference_external_prefix_context.cdl

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,5 @@ variables:
1010
child_variable:rdf__type = "bald__Reference" ;
1111

1212
// global attributes:
13-
:_NCProperties = "version=1|netcdflibversion=4.4.1|hdf5libversion=1.8.17" ;
1413
:rdf__type = "bald__Container" ;
15-
:bald__isPrefixedBy = "http://def.binary-array-ld.net/prefixes" ;
1614
}

‎lib/bald/tests/integration/TTL/GEMS_CO2_Apr2006.ttl

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -33,9 +33,9 @@ this:co2 a bald:Array ;
3333
NetCDF:scale_factor 0.000981685145029486 ;
3434
NetCDF:units "kg kg**-1" ;
3535
bald:references [ a bald:Reference ;
36-
bald:target this:longitude ;
37-
bald:targetReshape ( 1 1 1 360 ) ;
38-
bald:targetShape ( 360 ) ],
36+
bald:target this:latitude ;
37+
bald:targetReshape ( 1 1 181 1 ) ;
38+
bald:targetShape ( 181 ) ],
3939
[ a bald:Reference ;
4040
bald:target this:time ;
4141
bald:targetReshape ( 1 1 1 1 ) ;
@@ -45,9 +45,9 @@ this:co2 a bald:Array ;
4545
bald:targetReshape ( 1 60 1 1 ) ;
4646
bald:targetShape ( 60 ) ],
4747
[ a bald:Reference ;
48-
bald:target this:latitude ;
49-
bald:targetReshape ( 1 1 181 1 ) ;
50-
bald:targetShape ( 181 ) ] ;
48+
bald:target this:longitude ;
49+
bald:targetReshape ( 1 1 1 360 ) ;
50+
bald:targetShape ( 360 ) ] ;
5151
bald:shape ( 1 60 181 360 ) .
5252

5353
this:lnsp a bald:Array ;
@@ -56,9 +56,9 @@ this:lnsp a bald:Array ;
5656
NetCDF:add_offset 11.2087164280841 ;
5757
NetCDF:long_name "Logarithm of surface pressure" ;
5858
bald:references [ a bald:Reference ;
59-
bald:target this:time ;
60-
bald:targetReshape ( 1 1 1 1 ) ;
61-
bald:targetShape ( 1 ) ],
59+
bald:target this:latitude ;
60+
bald:targetReshape ( 1 1 181 1 ) ;
61+
bald:targetShape ( 181 ) ],
6262
[ a bald:Reference ;
6363
bald:target this:longitude ;
6464
bald:targetReshape ( 1 1 1 360 ) ;
@@ -68,9 +68,9 @@ this:lnsp a bald:Array ;
6868
bald:targetReshape ( 1 60 1 1 ) ;
6969
bald:targetShape ( 60 ) ],
7070
[ a bald:Reference ;
71-
bald:target this:latitude ;
72-
bald:targetReshape ( 1 1 181 1 ) ;
73-
bald:targetShape ( 181 ) ] ;
71+
bald:target this:time ;
72+
bald:targetReshape ( 1 1 1 1 ) ;
73+
bald:targetShape ( 1 ) ] ;
7474
bald:shape ( 1 60 181 360 ) .
7575

7676
this:latitude a bald:Array ;

‎lib/bald/tests/integration/TTL/array_reference_external_prefix.ttl renamed to ‎lib/bald/tests/integration/TTL/array_reference_external_prefix_context.ttl

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
@prefix dct: <http://purl.org/dc/terms/> .
44
@prefix rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> .
55
@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#> .
6-
@prefix this: <file://CDL/array_reference_external_prefix.cdl/> .
6+
@prefix this: <file://CDL/array_reference_external_prefix_context.cdl/> .
77
@prefix xml: <http://www.w3.org/XML/1998/namespace> .
88
@prefix xsd: <http://www.w3.org/2001/XMLSchema#> .
99

@@ -14,8 +14,7 @@ this: a bald:Container ;
1414
dcat:mediaType [ a dcat:MediaType ;
1515
dct:identifier "application/x-netcdf" ] ] ;
1616
bald:contains this:child_variable,
17-
this:parent_variable ;
18-
bald:isPrefixedBy <http://def.binary-array-ld.net/prefixes> .
17+
this:parent_variable .
1918

2019
this:parent_variable a bald:Array ;
2120
bald:references this:child_variable ;

‎lib/bald/tests/integration/TTL/ereefs_gbr4_ncld.ttl

Lines changed: 31 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -90,19 +90,19 @@ this:salt a bald:Array ;
9090
NetCDF:long_name "Salinity" ;
9191
NetCDF:units "PSU" ;
9292
bald:references [ a bald:Reference ;
93-
bald:target this:time ;
94-
bald:targetReshape ( 5 1 1 1 ) ;
95-
bald:targetShape ( 5 ) ],
96-
[ a bald:Reference ;
97-
bald:target this:longitude ;
93+
bald:target this:latitude ;
9894
bald:targetReshape ( 1 1 180 600 ) ;
9995
bald:targetShape ( 180 600 ) ],
10096
[ a bald:Reference ;
10197
bald:target this:zc ;
10298
bald:targetReshape ( 1 47 1 1 ) ;
10399
bald:targetShape ( 47 ) ],
104100
[ a bald:Reference ;
105-
bald:target this:latitude ;
101+
bald:target this:time ;
102+
bald:targetReshape ( 5 1 1 1 ) ;
103+
bald:targetShape ( 5 ) ],
104+
[ a bald:Reference ;
105+
bald:target this:longitude ;
106106
bald:targetReshape ( 1 1 180 600 ) ;
107107
bald:targetShape ( 180 600 ) ] ;
108108
bald:shape ( 5 47 180 600 ) .
@@ -121,19 +121,19 @@ this:temp a bald:Array ;
121121
NetCDF:units "degrees C" ;
122122
NetCDF:valid_range ( -3.0 40.0 ) ;
123123
bald:references [ a bald:Reference ;
124-
bald:target this:zc ;
125-
bald:targetReshape ( 1 47 1 1 ) ;
126-
bald:targetShape ( 47 ) ],
127-
[ a bald:Reference ;
128124
bald:target this:time ;
129125
bald:targetReshape ( 5 1 1 1 ) ;
130126
bald:targetShape ( 5 ) ],
131127
[ a bald:Reference ;
132-
bald:target this:latitude ;
128+
bald:target this:zc ;
129+
bald:targetReshape ( 1 47 1 1 ) ;
130+
bald:targetShape ( 47 ) ],
131+
[ a bald:Reference ;
132+
bald:target this:longitude ;
133133
bald:targetReshape ( 1 1 180 600 ) ;
134134
bald:targetShape ( 180 600 ) ],
135135
[ a bald:Reference ;
136-
bald:target this:longitude ;
136+
bald:target this:latitude ;
137137
bald:targetReshape ( 1 1 180 600 ) ;
138138
bald:targetShape ( 180 600 ) ] ;
139139
bald:shape ( 5 47 180 600 ) .
@@ -155,9 +155,9 @@ this:u a bald:Array ;
155155
NetCDF:units "ms-1" ;
156156
NetCDF:valid_range ( -100.0 100.0 ) ;
157157
bald:references [ a bald:Reference ;
158-
bald:target this:zc ;
159-
bald:targetReshape ( 1 47 1 1 ) ;
160-
bald:targetShape ( 47 ) ],
158+
bald:target this:longitude ;
159+
bald:targetReshape ( 1 1 180 600 ) ;
160+
bald:targetShape ( 180 600 ) ],
161161
[ a bald:Reference ;
162162
bald:target this:latitude ;
163163
bald:targetReshape ( 1 1 180 600 ) ;
@@ -167,9 +167,9 @@ this:u a bald:Array ;
167167
bald:targetReshape ( 5 1 1 1 ) ;
168168
bald:targetShape ( 5 ) ],
169169
[ a bald:Reference ;
170-
bald:target this:longitude ;
171-
bald:targetReshape ( 1 1 180 600 ) ;
172-
bald:targetShape ( 180 600 ) ] ;
170+
bald:target this:zc ;
171+
bald:targetReshape ( 1 47 1 1 ) ;
172+
bald:targetShape ( 47 ) ] ;
173173
bald:shape ( 5 47 180 600 ) .
174174

175175
this:v a bald:Array ;
@@ -189,21 +189,21 @@ this:v a bald:Array ;
189189
NetCDF:units "ms-1" ;
190190
NetCDF:valid_range ( -100.0 100.0 ) ;
191191
bald:references [ a bald:Reference ;
192-
bald:target this:zc ;
193-
bald:targetReshape ( 1 47 1 1 ) ;
194-
bald:targetShape ( 47 ) ],
192+
bald:target this:time ;
193+
bald:targetReshape ( 5 1 1 1 ) ;
194+
bald:targetShape ( 5 ) ],
195195
[ a bald:Reference ;
196196
bald:target this:latitude ;
197197
bald:targetReshape ( 1 1 180 600 ) ;
198198
bald:targetShape ( 180 600 ) ],
199-
[ a bald:Reference ;
200-
bald:target this:time ;
201-
bald:targetReshape ( 5 1 1 1 ) ;
202-
bald:targetShape ( 5 ) ],
203199
[ a bald:Reference ;
204200
bald:target this:longitude ;
205201
bald:targetReshape ( 1 1 180 600 ) ;
206-
bald:targetShape ( 180 600 ) ] ;
202+
bald:targetShape ( 180 600 ) ],
203+
[ a bald:Reference ;
204+
bald:target this:zc ;
205+
bald:targetReshape ( 1 47 1 1 ) ;
206+
bald:targetShape ( 47 ) ] ;
207207
bald:shape ( 5 47 180 600 ) .
208208

209209
this:wspeed_u a bald:Array ;
@@ -237,17 +237,17 @@ this:wspeed_v a bald:Array ;
237237
NetCDF:units "ms-1" ;
238238
NetCDF:valid_range ( -1000.0 1000.0 ) ;
239239
bald:references [ a bald:Reference ;
240+
bald:target this:time ;
241+
bald:targetReshape ( 5 1 1 ) ;
242+
bald:targetShape ( 5 ) ],
243+
[ a bald:Reference ;
240244
bald:target this:latitude ;
241245
bald:targetReshape ( 1 180 600 ) ;
242246
bald:targetShape ( 180 600 ) ],
243247
[ a bald:Reference ;
244248
bald:target this:longitude ;
245249
bald:targetReshape ( 1 180 600 ) ;
246-
bald:targetShape ( 180 600 ) ],
247-
[ a bald:Reference ;
248-
bald:target this:time ;
249-
bald:targetReshape ( 5 1 1 ) ;
250-
bald:targetShape ( 5 ) ] ;
250+
bald:targetShape ( 180 600 ) ] ;
251251
bald:shape ( 5 180 600 ) .
252252

253253
this:zc a bald:Array ;

0 commit comments

Comments
 (0)