Skip to content

adding test case for ereefs #51

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 14 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
80 changes: 59 additions & 21 deletions lib/bald/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -240,6 +240,7 @@ def __getitem__(self, item):

def check_uri(self, uri):
result = False
#print("Checking uri: " + uri)
if self[uri].status_code == 200:
result = True
return result
Expand Down Expand Up @@ -414,6 +415,12 @@ def rdfnode(self, graph):
selfnode = rdflib.URIRef(self.identity)
for attr in self.attrs:
objs = self.attrs[attr]
if(isinstance(objs, np.ndarray)):
#print("Found np.ndarray")
#print(objs)
#print(attr)
#try to convert np.ndarray to a list
objs = objs.tolist()
if not (isinstance(objs, set) or isinstance(objs, list)):
objs = set([objs])
for obj in objs:
Expand All @@ -438,6 +445,14 @@ def rdfgraph(self):
"""
graph = rdflib.Graph()
graph.bind('bald', 'http://binary-array-ld.net/latest/')
for prefix_name in self._prefixes:
#strip the double underscore suffix
new_name = prefix_name[:-2]
#print(prefix_name)
#print(new_name)
#print(self._prefixes[prefix_name])

graph.bind(new_name, self._prefixes[prefix_name])
graph = self.rdfnode(graph)

return graph
Expand Down Expand Up @@ -530,35 +545,48 @@ def load(afilepath):
finally:
f.close()

def load_netcdf(afilepath, uri=None):
def load_netcdf(afilepath, uri=None, baseuri=None):
"""
Validate a file with respect to binary-array-linked-data.
Returns a :class:`bald.validation.Validation`
"""

with load(afilepath) as fhandle:
prefix_group = (fhandle[fhandle.bald__isPrefixedBy] if
prefix_var_name = None
if hasattr(fhandle, 'bald__isPrefixedBy'):
prefix_var_name = fhandle.bald__isPrefixedBy

prefix_var = (fhandle[fhandle.bald__isPrefixedBy] if
hasattr(fhandle, 'bald__isPrefixedBy') else {})
prefixes = {}

skipped_variables = []
if prefix_group != {}:
prefixes = (dict([(prefix, getattr(prefix_group, prefix)) for
prefix in prefix_group.ncattrs()]))
if isinstance(prefix_group, netCDF4._netCDF4.Variable):
skipped_variables.append(prefix_group.name)
if prefix_var != {} :
prefixes = (dict([(prefix, getattr(prefix_var, prefix)) for
prefix in prefix_var.ncattrs()]))
else:
for k in fhandle.ncattrs():
if k.endswith('__'):
prefixes[k] = getattr(fhandle, k)
alias_group = (fhandle[fhandle.bald__isAliasedBy]

# check that default set is handled, i.e. bald__ and rdf__
if 'bald__' not in prefixes:
prefixes['bald__'] = "http://binary-array-ld.net/latest/"

if 'rdf__' not in prefixes:
prefixes['rdf__'] = "http://www.w3.org/1999/02/22-rdf-syntax-ns#"

#print(prefixes)

alias_var_name = None
if hasattr(fhandle, 'bald__isAliasedBy'):
alias_var_name = fhandle.bald__isAliasedBy

alias_var = (fhandle[fhandle.bald__isAliasedBy]
if hasattr(fhandle, 'bald__isAliasedBy') else {})
aliases = {}
if alias_group != {}:
aliases = (dict([(alias, getattr(alias_group, alias))
for alias in alias_group.ncattrs()]))
if isinstance(alias_group, netCDF4._netCDF4.Variable):
skipped_variables.append(alias_group.name)
if alias_var != {}:
aliases = (dict([(alias, getattr(alias_var, alias))
for alias in alias_var.ncattrs()]))
#print(aliases)

attrs = {}
for k in fhandle.ncattrs():
Expand All @@ -573,32 +601,40 @@ def load_netcdf(afilepath, uri=None):
root_container.attrs['bald__contains'] = []
file_variables = {}
for name in fhandle.variables:
#print(name)
if name == prefix_var_name or name == alias_var_name:
#print("Skipping " + name)
continue

sattrs = fhandle.variables[name].__dict__.copy()
# inconsistent use of '/'; fix it
identity = name
if baseuri is not None:
identity = baseuri + "/" + name

# netCDF coordinate variable special case
if (len(fhandle.variables[name].dimensions) == 1 and
fhandle.variables[name].dimensions[0] == name):
sattrs['bald__array'] = name
#sattrs['bald__array'] = name
sattrs['bald__array'] = identity
sattrs['rdf__type'] = 'bald__Reference'

if fhandle.variables[name].shape:
sattrs['bald__shape'] = fhandle.variables[name].shape
var = Array(identity, sattrs, prefixes=prefixes, aliases=aliases)
else:
var = Subject(identity, sattrs, prefixes=prefixes, aliases=aliases)
if name not in skipped_variables:
# Don't include skipped variables, such as prefix or alias
# variables, within the containment relation.
root_container.attrs['bald__contains'].append(var)

root_container.attrs['bald__contains'].append(var)
file_variables[name] = var



# cycle again and find references
for name in fhandle.variables:
if name == prefix_var_name or name == alias_var_name:
#print("Skipping " + name)
continue

var = file_variables[name]
# reverse lookup based on type to be added
lookups = ['bald__references', 'bald__array']
Expand All @@ -624,6 +660,8 @@ def load_netcdf(afilepath, uri=None):
# Else, define a bald:childBroadcast
else:
identity = '{}_{}_ref'.format(name, dim)
if baseuri is not None:
identity = baseuri + '/' + '{}_{}_ref'.format(name, dim)
rattrs = {}
rattrs['rdf__type'] = 'bald__Reference'
reshape = [1 for adim in var_shape]
Expand Down
58 changes: 58 additions & 0 deletions lib/bald/tests/integration/CDL/array_alias_v2.cdl
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
netcdf tmpMwXy8U {
dimensions:
pdim0 = 11 ;
pdim1 = 17 ;
variables:
int prefix_list ;
prefix_list:bald__ = "http://binary-array-ld.net/latest/" ;
prefix_list:rdf__ = "http://www.w3.org/1999/02/22-rdf-syntax-ns#" ;

int alias_list ;
alias_list:SDN_ParameterDiscoveryCode = "http://vocab.nerc.ac.uk/isoCodelists/sdnCodelists/cdicsrCodeList.xml#SDN_ParameterDiscoveryCode" ;
alias_list:BactTaxaAbundSed = "http://vocab.nerc.ac.uk/collection/P02/current/BAUC/" ;
alias_list:standard_name = "http://def.scitools.org.uk/CFTerms/standard_name" ;
alias_list:air_temperature = "http://vocab.nerc.ac.uk/collection/P07/current/CFSN0023/" ;

int parent_variable(pdim0, pdim1) ;
parent_variable:rdf__type = "bald__Array" ;
parent_variable:SDN_ParameterDiscoveryCode = "BactTaxaAbundSed" ;
parent_variable:submursible_name = "Nautilus" ;

int temp(pdim0, pdim1) ;
temp:standard_name = "air_temperature" ;

// global attributes:
:_NCProperties = "version=1|netcdflibversion=4.4.1|hdf5libversion=1.8.17" ;
:rdf__type = "bald__Container" ;
:bald__isPrefixedBy = "prefix_list" ;
:bald__isAliasedBy = "alias_list" ;
data:

parent_variable =
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _ ;

temp =
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _ ;


}
45 changes: 45 additions & 0 deletions lib/bald/tests/integration/CDL/array_geo.cdl
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
netcdf tmpMwXy8U {
dimensions:
pdim0 = 11 ;
pdim1 = 17 ;
variables:
int prefix_list ;
prefix_list:bald__ = "http://binary-array-ld.net/latest/" ;
prefix_list:rdf__ = "http://www.w3.org/1999/02/22-rdf-syntax-ns#" ;
prefix_list:rdfs__ = "http://www.w3.org/2000/01/rdf-schema#" ;
prefix_list:cf__ = "http://def.scitools.org.uk/CFTerms/" ;
prefix_list:geo__ = "http://www.opengis.net/ont/geosparql#" ;

int temp(pdim0, pdim1) ;
temp:cf__standard_name = "air_temperature" ;
temp:cf__long_name = "Air temperature obs example at point" ;
temp:rdfs__label = "Air temperature obs example at point" ;
temp:geo__asWKT = "POINT(-77.03524 38.889468)" ;

int pressure(pdim0, pdim1) ;
pressure:cf__standard_name = "air_pressure" ;
pressure:cf__long_name = "Air pressure at UCAR Centre Green" ;
pressure:rdfs__label = "Air pressure at UCAR Centre Green" ;
pressure:geo__asWKT = "POINT(-105.24584700000003 40.0315278)" ;

// global attributes:
:_NCProperties = "version=1|netcdflibversion=4.4.1|hdf5libversion=1.8.17" ;
:rdf__type = "bald__Container" ;
:bald__isPrefixedBy = "prefix_list" ;
data:

temp =
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _ ;


}
55 changes: 55 additions & 0 deletions lib/bald/tests/integration/CDL/array_prefix_v2.cdl
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
netcdf tmpMwXy8U {
dimensions:
pdim0 = 11 ;
pdim1 = 17 ;
variables:
int prefix_list ;
prefix_list:bald__ = "http://binary-array-ld.net/latest/" ;
prefix_list:rdf__ = "http://www.w3.org/1999/02/22-rdf-syntax-ns#" ;
prefix_list:sdn__ = "http://vocab.nerc.ac.uk/isoCodelists/sdnCodelists/cdicsrCodeList.xml#" ;
prefix_list:sdn-vocab__= "http://vocab.nerc.ac.uk/collection/P02/current/" ;
prefix_list:cf__ = "http://def.scitools.org.uk/CFTerms/" ;
prefix_list:cfsn__ = "http://vocab.nerc.ac.uk/collection/P07/current/CFSN0023/" ;

int parent_variable(pdim0, pdim1) ;
parent_variable:rdf__type = "bald__Array" ;
parent_variable:sdn__SDN_ParameterDiscoveryCode = "sdn-vocab__BAUC" ;
parent_variable:submursible_name = "Nautilus" ;

int temp(pdim0, pdim1) ;
temp:standard_name = "air_temperature" ;

// global attributes:
:_NCProperties = "version=1|netcdflibversion=4.4.1|hdf5libversion=1.8.17" ;
:rdf__type = "bald__Container" ;
:bald__isPrefixedBy = "prefix_list" ;
data:

parent_variable =
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _ ;

temp =
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _ ;


}
57 changes: 57 additions & 0 deletions lib/bald/tests/integration/CDL/array_prefix_v2_full.cdl
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
netcdf tmpMwXy8U {
dimensions:
pdim0 = 11 ;
pdim1 = 17 ;
variables:
int prefix_list ;
prefix_list:bald__ = "http://binary-array-ld.net/latest/" ;
prefix_list:rdf__ = "http://www.w3.org/1999/02/22-rdf-syntax-ns#" ;
prefix_list:sdn__ = "http://vocab.nerc.ac.uk/isoCodelists/sdnCodelists/cdicsrCodeList.xml#" ;
prefix_list:sdn-vocab__= "http://vocab.nerc.ac.uk/collection/P02/current/" ;
prefix_list:cf__ = "http://def.scitools.org.uk/CFTerms/" ;
prefix_list:cfsn-mmi__ = "http://mmisw.org/ont/cf/parameter/" ;
prefix_list:cfsn-nerc__ = "http://vocab.nerc.ac.uk/collection/P07/current/";

int parent_variable(pdim0, pdim1) ;
parent_variable:rdf__type = "bald__Array" ;
parent_variable:sdn__SDN_ParameterDiscoveryCode = "sdn-vocab__BAUC" ;
parent_variable:submursible_name = "Nautilus" ;

int temp(pdim0, pdim1) ;
temp:cf__standard_name = "cfsn-mmi__air_temperature" ;
//temp:cf__standard_name = "cfsn-nerc__CFSN0023";

// global attributes:
:_NCProperties = "version=1|netcdflibversion=4.4.1|hdf5libversion=1.8.17" ;
:rdf__type = "bald__Container" ;
:bald__isPrefixedBy = "prefix_list" ;
data:

parent_variable =
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _ ;

temp =
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _ ;


}
12 changes: 5 additions & 7 deletions lib/bald/tests/integration/CDL/multi_array_reference.cdl
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,11 @@ dimensions:
pdim0 = 11 ;
pdim1 = 17 ;
variables:
int prefix_list(pdim0, pdim1) ;
prefix_list:bald__ = "http://binary-array-ld.net/latest/" ;
prefix_list:metce__ = "http://codes.wmo.int/common/observation-type/METCE/2013/" ;
prefix_list:rdf__ = "http://www.w3.org/1999/02/22-rdf-syntax-ns#" ;

int variable1(pdim0, pdim1) ;
variable1:bald__references = "location_variable" ;
variable1:long_name = "Gerald";
Expand Down Expand Up @@ -31,11 +36,4 @@ variables:
:_NCProperties = "version=1|netcdflibversion=4.4.1|hdf5libversion=1.8.17" ;
:bald__isPrefixedBy = "prefix_list" ;

group: prefix_list {

// group attributes:
:bald__ = "http://binary-array-ld.net/latest/" ;
:metce__ = "http://codes.wmo.int/common/observation-type/METCE/2013/" ;
:rdf__ = "http://www.w3.org/1999/02/22-rdf-syntax-ns#" ;
} // group bald__prefix_list
}
Loading