Skip to content

adding test case for ereefs #51

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 14 commits into
base: master
Choose a base branch
from
Open

adding test case for ereefs #51

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
26d3765
adding test case for ereefs
jyucsiro Aug 15, 2017
4cf1112
Refactoring alias and prefix handling to variable
jyucsiro Aug 30, 2017
eed0687
fixing bug in prefix handling
jyucsiro Aug 30, 2017
8a64095
fixing tests for cdl rdfgraph
jyucsiro Aug 30, 2017
2857d4f
updating alias tests and cdl
jyucsiro Aug 30, 2017
fcce72f
updating OISST test
jyucsiro Aug 30, 2017
847cee9
adding default prefixes in bald lib
jyucsiro Sep 4, 2017
1745701
implementing parameters to set graph uris and base uris
jyucsiro Sep 4, 2017
9d2f96c
fixing refs to alias_group and prefix_group to vars
jyucsiro Sep 6, 2017
ea895aa
Merge branch 'master' into nc2rdf.
jyucsiro Sep 6, 2017
cd88109
updating tests to reflect URL checks
jyucsiro Sep 6, 2017
e738360
Merge branch 'nc2rdf' into ereefscdl-rdf
jyucsiro Sep 6, 2017
a8c4a5b
removing bug
jyucsiro Sep 6, 2017
0999f88
fixing bald lib to convert np.ndarray to list
jyucsiro Sep 6, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
80 changes: 59 additions & 21 deletions lib/bald/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -240,6 +240,7 @@ def __getitem__(self, item):

def check_uri(self, uri):
result = False
#print("Checking uri: " + uri)
if self[uri].status_code == 200:
result = True
return result
Expand Down Expand Up @@ -414,6 +415,12 @@ def rdfnode(self, graph):
selfnode = rdflib.URIRef(self.identity)
for attr in self.attrs:
objs = self.attrs[attr]
if(isinstance(objs, np.ndarray)):
#print("Found np.ndarray")
#print(objs)
#print(attr)
#try to convert np.ndarray to a list
objs = objs.tolist()
if not (isinstance(objs, set) or isinstance(objs, list)):
objs = set([objs])
for obj in objs:
Expand All @@ -438,6 +445,14 @@ def rdfgraph(self):
"""
graph = rdflib.Graph()
graph.bind('bald', 'http://binary-array-ld.net/latest/')
for prefix_name in self._prefixes:
#strip the double underscore suffix
new_name = prefix_name[:-2]
#print(prefix_name)
#print(new_name)
#print(self._prefixes[prefix_name])

graph.bind(new_name, self._prefixes[prefix_name])
graph = self.rdfnode(graph)

return graph
Expand Down Expand Up @@ -530,35 +545,48 @@ def load(afilepath):
finally:
f.close()

def load_netcdf(afilepath, uri=None):
def load_netcdf(afilepath, uri=None, baseuri=None):
"""
Validate a file with respect to binary-array-linked-data.
Returns a :class:`bald.validation.Validation`
"""

with load(afilepath) as fhandle:
prefix_group = (fhandle[fhandle.bald__isPrefixedBy] if
prefix_var_name = None
if hasattr(fhandle, 'bald__isPrefixedBy'):
prefix_var_name = fhandle.bald__isPrefixedBy

prefix_var = (fhandle[fhandle.bald__isPrefixedBy] if
hasattr(fhandle, 'bald__isPrefixedBy') else {})
prefixes = {}

skipped_variables = []
if prefix_group != {}:
prefixes = (dict([(prefix, getattr(prefix_group, prefix)) for
prefix in prefix_group.ncattrs()]))
if isinstance(prefix_group, netCDF4._netCDF4.Variable):
skipped_variables.append(prefix_group.name)
if prefix_var != {} :
prefixes = (dict([(prefix, getattr(prefix_var, prefix)) for
prefix in prefix_var.ncattrs()]))
else:
for k in fhandle.ncattrs():
if k.endswith('__'):
prefixes[k] = getattr(fhandle, k)
alias_group = (fhandle[fhandle.bald__isAliasedBy]

# check that default set is handled, i.e. bald__ and rdf__
if 'bald__' not in prefixes:
prefixes['bald__'] = "http://binary-array-ld.net/latest/"

if 'rdf__' not in prefixes:
prefixes['rdf__'] = "http://www.w3.org/1999/02/22-rdf-syntax-ns#"

#print(prefixes)

alias_var_name = None
if hasattr(fhandle, 'bald__isAliasedBy'):
alias_var_name = fhandle.bald__isAliasedBy

alias_var = (fhandle[fhandle.bald__isAliasedBy]
if hasattr(fhandle, 'bald__isAliasedBy') else {})
aliases = {}
if alias_group != {}:
aliases = (dict([(alias, getattr(alias_group, alias))
for alias in alias_group.ncattrs()]))
if isinstance(alias_group, netCDF4._netCDF4.Variable):
skipped_variables.append(alias_group.name)
if alias_var != {}:
aliases = (dict([(alias, getattr(alias_var, alias))
for alias in alias_var.ncattrs()]))
#print(aliases)

attrs = {}
for k in fhandle.ncattrs():
Expand All @@ -573,32 +601,40 @@ def load_netcdf(afilepath, uri=None):
root_container.attrs['bald__contains'] = []
file_variables = {}
for name in fhandle.variables:
#print(name)
if name == prefix_var_name or name == alias_var_name:
#print("Skipping " + name)
continue

sattrs = fhandle.variables[name].__dict__.copy()
# inconsistent use of '/'; fix it
identity = name
if baseuri is not None:
identity = baseuri + "/" + name

# netCDF coordinate variable special case
if (len(fhandle.variables[name].dimensions) == 1 and
fhandle.variables[name].dimensions[0] == name):
sattrs['bald__array'] = name
#sattrs['bald__array'] = name
sattrs['bald__array'] = identity
sattrs['rdf__type'] = 'bald__Reference'

if fhandle.variables[name].shape:
sattrs['bald__shape'] = fhandle.variables[name].shape
var = Array(identity, sattrs, prefixes=prefixes, aliases=aliases)
else:
var = Subject(identity, sattrs, prefixes=prefixes, aliases=aliases)
if name not in skipped_variables:
# Don't include skipped variables, such as prefix or alias
# variables, within the containment relation.
root_container.attrs['bald__contains'].append(var)

root_container.attrs['bald__contains'].append(var)
file_variables[name] = var



# cycle again and find references
for name in fhandle.variables:
if name == prefix_var_name or name == alias_var_name:
#print("Skipping " + name)
continue

var = file_variables[name]
# reverse lookup based on type to be added
lookups = ['bald__references', 'bald__array']
Expand All @@ -624,6 +660,8 @@ def load_netcdf(afilepath, uri=None):
# Else, define a bald:childBroadcast
else:
identity = '{}_{}_ref'.format(name, dim)
if baseuri is not None:
identity = baseuri + '/' + '{}_{}_ref'.format(name, dim)
rattrs = {}
rattrs['rdf__type'] = 'bald__Reference'
reshape = [1 for adim in var_shape]
Expand Down
58 changes: 58 additions & 0 deletions lib/bald/tests/integration/CDL/array_alias_v2.cdl
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
netcdf tmpMwXy8U {
dimensions:
pdim0 = 11 ;
pdim1 = 17 ;
variables:
int prefix_list ;
prefix_list:bald__ = "http://binary-array-ld.net/latest/" ;
prefix_list:rdf__ = "http://www.w3.org/1999/02/22-rdf-syntax-ns#" ;

int alias_list ;
alias_list:SDN_ParameterDiscoveryCode = "http://vocab.nerc.ac.uk/isoCodelists/sdnCodelists/cdicsrCodeList.xml#SDN_ParameterDiscoveryCode" ;
alias_list:BactTaxaAbundSed = "http://vocab.nerc.ac.uk/collection/P02/current/BAUC/" ;
alias_list:standard_name = "http://def.scitools.org.uk/CFTerms/standard_name" ;
alias_list:air_temperature = "http://vocab.nerc.ac.uk/collection/P07/current/CFSN0023/" ;

int parent_variable(pdim0, pdim1) ;
parent_variable:rdf__type = "bald__Array" ;
parent_variable:SDN_ParameterDiscoveryCode = "BactTaxaAbundSed" ;
parent_variable:submursible_name = "Nautilus" ;

int temp(pdim0, pdim1) ;
temp:standard_name = "air_temperature" ;

// global attributes:
:_NCProperties = "version=1|netcdflibversion=4.4.1|hdf5libversion=1.8.17" ;
:rdf__type = "bald__Container" ;
:bald__isPrefixedBy = "prefix_list" ;
:bald__isAliasedBy = "alias_list" ;
data:

parent_variable =
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _ ;

temp =
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _ ;


}
45 changes: 45 additions & 0 deletions lib/bald/tests/integration/CDL/array_geo.cdl
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
netcdf tmpMwXy8U {
dimensions:
pdim0 = 11 ;
pdim1 = 17 ;
variables:
int prefix_list ;
prefix_list:bald__ = "http://binary-array-ld.net/latest/" ;
prefix_list:rdf__ = "http://www.w3.org/1999/02/22-rdf-syntax-ns#" ;
prefix_list:rdfs__ = "http://www.w3.org/2000/01/rdf-schema#" ;
prefix_list:cf__ = "http://def.scitools.org.uk/CFTerms/" ;
prefix_list:geo__ = "http://www.opengis.net/ont/geosparql#" ;

int temp(pdim0, pdim1) ;
temp:cf__standard_name = "air_temperature" ;
temp:cf__long_name = "Air temperature obs example at point" ;
temp:rdfs__label = "Air temperature obs example at point" ;
temp:geo__asWKT = "POINT(-77.03524 38.889468)" ;

int pressure(pdim0, pdim1) ;
pressure:cf__standard_name = "air_pressure" ;
pressure:cf__long_name = "Air pressure at UCAR Centre Green" ;
pressure:rdfs__label = "Air pressure at UCAR Centre Green" ;
pressure:geo__asWKT = "POINT(-105.24584700000003 40.0315278)" ;

// global attributes:
:_NCProperties = "version=1|netcdflibversion=4.4.1|hdf5libversion=1.8.17" ;
:rdf__type = "bald__Container" ;
:bald__isPrefixedBy = "prefix_list" ;
data:

temp =
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _ ;


}
55 changes: 55 additions & 0 deletions lib/bald/tests/integration/CDL/array_prefix_v2.cdl
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
netcdf tmpMwXy8U {
dimensions:
pdim0 = 11 ;
pdim1 = 17 ;
variables:
int prefix_list ;
prefix_list:bald__ = "http://binary-array-ld.net/latest/" ;
prefix_list:rdf__ = "http://www.w3.org/1999/02/22-rdf-syntax-ns#" ;
prefix_list:sdn__ = "http://vocab.nerc.ac.uk/isoCodelists/sdnCodelists/cdicsrCodeList.xml#" ;
prefix_list:sdn-vocab__= "http://vocab.nerc.ac.uk/collection/P02/current/" ;
prefix_list:cf__ = "http://def.scitools.org.uk/CFTerms/" ;
prefix_list:cfsn__ = "http://vocab.nerc.ac.uk/collection/P07/current/CFSN0023/" ;

int parent_variable(pdim0, pdim1) ;
parent_variable:rdf__type = "bald__Array" ;
parent_variable:sdn__SDN_ParameterDiscoveryCode = "sdn-vocab__BAUC" ;
parent_variable:submursible_name = "Nautilus" ;

int temp(pdim0, pdim1) ;
temp:standard_name = "air_temperature" ;

// global attributes:
:_NCProperties = "version=1|netcdflibversion=4.4.1|hdf5libversion=1.8.17" ;
:rdf__type = "bald__Container" ;
:bald__isPrefixedBy = "prefix_list" ;
data:

parent_variable =
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _ ;

temp =
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _ ;


}
57 changes: 57 additions & 0 deletions lib/bald/tests/integration/CDL/array_prefix_v2_full.cdl
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
netcdf tmpMwXy8U {
dimensions:
pdim0 = 11 ;
pdim1 = 17 ;
variables:
int prefix_list ;
prefix_list:bald__ = "http://binary-array-ld.net/latest/" ;
prefix_list:rdf__ = "http://www.w3.org/1999/02/22-rdf-syntax-ns#" ;
prefix_list:sdn__ = "http://vocab.nerc.ac.uk/isoCodelists/sdnCodelists/cdicsrCodeList.xml#" ;
prefix_list:sdn-vocab__= "http://vocab.nerc.ac.uk/collection/P02/current/" ;
prefix_list:cf__ = "http://def.scitools.org.uk/CFTerms/" ;
prefix_list:cfsn-mmi__ = "http://mmisw.org/ont/cf/parameter/" ;
prefix_list:cfsn-nerc__ = "http://vocab.nerc.ac.uk/collection/P07/current/";

int parent_variable(pdim0, pdim1) ;
parent_variable:rdf__type = "bald__Array" ;
parent_variable:sdn__SDN_ParameterDiscoveryCode = "sdn-vocab__BAUC" ;
parent_variable:submursible_name = "Nautilus" ;

int temp(pdim0, pdim1) ;
temp:cf__standard_name = "cfsn-mmi__air_temperature" ;
//temp:cf__standard_name = "cfsn-nerc__CFSN0023";

// global attributes:
:_NCProperties = "version=1|netcdflibversion=4.4.1|hdf5libversion=1.8.17" ;
:rdf__type = "bald__Container" ;
:bald__isPrefixedBy = "prefix_list" ;
data:

parent_variable =
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _ ;

temp =
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _ ;


}
12 changes: 5 additions & 7 deletions lib/bald/tests/integration/CDL/multi_array_reference.cdl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,11 @@ dimensions:
pdim0 = 11 ;
pdim1 = 17 ;
variables:
int prefix_list(pdim0, pdim1) ;
prefix_list:bald__ = "http://binary-array-ld.net/latest/" ;
prefix_list:metce__ = "http://codes.wmo.int/common/observation-type/METCE/2013/" ;
prefix_list:rdf__ = "http://www.w3.org/1999/02/22-rdf-syntax-ns#" ;

int variable1(pdim0, pdim1) ;
variable1:bald__references = "location_variable" ;
variable1:long_name = "Gerald";
Expand Down Expand Up @@ -31,11 +36,4 @@ variables:
:_NCProperties = "version=1|netcdflibversion=4.4.1|hdf5libversion=1.8.17" ;
:bald__isPrefixedBy = "prefix_list" ;

group: prefix_list {

// group attributes:
:bald__ = "http://binary-array-ld.net/latest/" ;
:metce__ = "http://codes.wmo.int/common/observation-type/METCE/2013/" ;
:rdf__ = "http://www.w3.org/1999/02/22-rdf-syntax-ns#" ;
} // group bald__prefix_list
}
Loading