Merge pull request #95 from marqh/prefixLocator

DCAT Distribution

Author: mo-marqh

.travis.yml

@@ -17,11 +17,6 @@ install:
   - conda install --quiet --file requirements.txt
   - conda list
   - conda info -a
-  - wget https://github.com/marqh/terra/archive/master.zip
-  - unzip master.zip
-  - cd terra-master
-  - python setup.py --quiet install
-  - cd ..
   - python setup.py --quiet install
 
 script:

lib/bald/__init__.py

@@ -14,18 +14,11 @@
 import pyparsing
 import rdflib
 import rdflib.collection
+import rdflib.namespace
 import requests
 import six
 
-# try:
-#     #import terra.datetime
-#     from terra import datetime
-#     terra_imp = True
-# except ImportError:
-#     terra_imp = False
-
 from bald import datetime
-terra_imp = True
 import bald.validation as bv
 
 __version__ = '0.3'
@@ -288,14 +281,19 @@ def check_uri(self, uri):
 
 class Resource(object):
     _rdftype = 'bald__Resource'
+    # def __init__(self, baseuri, relative_id, attrs=None, prefixes=None,
+    #              aliases=None, alias_graph=None):
     def __init__(self, baseuri, relative_id, attrs=None, prefixes=None,
-                 aliases=None, alias_graph=None):
+                 aliases=None, alias_graph=None, file_resource=False, file_locator=None):
+
         """
         A resource of metadata statements.
 
         attrs: an dictionary of key value pair attributes
         """
         self.baseuri = baseuri
+        self.file_locator = file_locator
+        self.is_file = file_resource
         self.relative_id = relative_id
 
         if attrs is None:
@@ -339,7 +337,8 @@ def __repr__(self):
     def __setattr__(self, attr, value):
         reserved_attrs = ['baseuri', 'relative_id', 'prefixes', '_prefixes',
                           '_prefix_suffix', '_http_uri_prefix', '_http_uri',
-                          'aliases', 'alias_graph', 'attrs', '_rdftype']
+                          'aliases', 'alias_graph', 'attrs', '_rdftype', 'file_locator',
+                          'is_file']
         if attr in reserved_attrs:
             object.__setattr__(self, attr, value)
         else:
@@ -535,6 +534,38 @@ def viewgraph(self):
 
         return html
 
+    def _dcat_location(self, graph, selfnode):
+        graph.bind('dcat', 'http://www.w3.org/ns/dcat#')
+        graph.bind('dct', 'http://purl.org/dc/terms/')
+        # template = ('dcat:distribution [
+	# 	a dcat:Distribution;
+	# 	dcat:downloadURL <{}>;
+	# 	dcat:mediaType [
+	# 		a dct:MediaType;
+	# 		dct:identifier "application/x-netcdf"
+	# 	];
+	# 	dct:format [
+	# 		a dct:MediaType;
+	# 		dct:identifier <http://vocab.nerc.ac.uk/collection/M01/current/NC/>
+	# 	]
+	#                 ].')
+        dcatnode = rdflib.BNode()
+        dcfnode = rdflib.BNode()
+        graph.add((selfnode, rdflib.URIRef('http://www.w3.org/ns/dcat#distribution'), dcatnode))
+        graph.add((dcatnode, rdflib.namespace.RDF.type, rdflib.URIRef('http://www.w3.org/ns/dcat#Distribution')))
+        if self.file_locator is not None:
+            graph.add((dcatnode, rdflib.URIRef('http://www.w3.org/ns/dcat#downloadURL'),  rdflib.URIRef(self.file_locator)))
+        dcatmednode = rdflib.BNode()
+        graph.add((dcatmednode, rdflib.namespace.RDF.type, rdflib.URIRef('http://www.w3.org/ns/dcat#MediaType')))
+        graph.add((dcatmednode, rdflib.URIRef('http://purl.org/dc/terms/identifier'), rdflib.Literal('application/x-netcdf')))
+        graph.add((dcatnode, rdflib.URIRef('http://www.w3.org/ns/dcat#mediaType'), dcatmednode))
+
+        graph.add((dcfnode, rdflib.namespace.RDF.type, rdflib.URIRef('http://purl.org/dc/terms/MediaType')))
+        graph.add((dcfnode, rdflib.URIRef('http://purl.org/dc/terms/identifier'),
+                   rdflib.URIRef('http://vocab.nerc.ac.uk/collection/M01/current/NC/')))
+        graph.add((selfnode, rdflib.URIRef('http://purl.org/dc/terms/format'), dcfnode))
+
+
     def rdfnode(self, graph):
         """
         Create an RDF Node,
@@ -568,10 +599,10 @@ def rdfnode(self, graph):
                     if is_http_uri(rdfobj):
 
                         rdfobj = rdflib.URIRef(rdfobj)
-                    elif terra_imp and isinstance(rdfobj, datetime.EpochDateTimes):
+                    elif isinstance(rdfobj, datetime.EpochDateTimes):
                         rdfobj = rdflib.Literal(str(rdfobj), datatype=rdflib.XSD.dateTime)
                     elif isinstance(rdfobj, float):
-                        rdfobj = rdflib.Literal(rdfobj, datatype=rdflib.XSD.decimal)
+                        rdfobj = rdflib.Literal(float(rdfobj), datatype=rdflib.XSD.decimal)
                     else:
                         rdfobj = rdflib.Literal(rdfobj)
                 rdfpred = rdflib.URIRef(rdfpred)
@@ -594,6 +625,9 @@ def rdfnode(self, graph):
                 col = rdflib.collection.Collection(graph, list_name, list_items)
                 graph.add((selfnode, rdfpred, list_name))
 
+        if self.is_file:
+            self._dcat_location(graph, selfnode)
+
         return selfnode
 
     def rdfgraph(self):
@@ -757,7 +791,7 @@ def load(afilepath):
         except NameError:
             pass
 
-def load_netcdf(afilepath, baseuri=None, alias_dict=None, cache=None):
+def load_netcdf(afilepath, baseuri=None, alias_dict=None, cache=None, file_locator=None):
     """
     Load a file with respect to binary-array-linked-data.
     Returns a :class:`bald.Collection`
@@ -773,6 +807,7 @@ def load_netcdf(afilepath, baseuri=None, alias_dict=None, cache=None):
             baseuri = 'file://{}/'.format(afilepath)
         elif type(baseuri) == str and not baseuri.endswith('/'):
             baseuri = '{}/'.format(baseuri)
+
         identity = baseuri
         prefix_var_name  = None
         if hasattr(fhandle, 'bald__isPrefixedBy'):
@@ -852,7 +887,8 @@ def load_netcdf(afilepath, baseuri=None, alias_dict=None, cache=None):
             #     raise ValueError('duplicate aliases')
             # aliases = careful_update(aliases, dict(new_aliases))
         root_container = Container(baseuri, '', attrs, prefixes=prefixes,
-                                   aliases=aliases, alias_graph=aliasgraph)
+                                   aliases=aliases, alias_graph=aliasgraph,
+                                   file_resource=True, file_locator=file_locator)
 
         root_container.attrs['bald__contains'] = set()
         file_variables = {}
@@ -886,7 +922,7 @@ def load_netcdf(afilepath, baseuri=None, alias_dict=None, cache=None):
                             sattrs['bald__last_value'] = float(sattrs['bald__last_value'])
 
                 # datetime special case
-                if 'units' in fhandle.variables[name].ncattrs() and terra_imp:
+                if 'units' in fhandle.variables[name].ncattrs():
                     ustr = fhandle.variables[name].getncattr('units')
                     pattern = '^([a-z]+) since ([0-9T:\\. -]+)'
 

lib/bald/tests/integration/TTL/GEMS_CO2_Apr2006.ttl

@@ -2,6 +2,8 @@
 @prefix NetCDF: <http://def.scitools.org.uk/NetCDF/> .
 @prefix bald: <https://www.opengis.net/def/binary-array-ld/> .
 @prefix cf_sname: <http://vocab.nerc.ac.uk/standard_name/> .
+@prefix dcat: <http://www.w3.org/ns/dcat#> .
+@prefix dct: <http://purl.org/dc/terms/> .
 @prefix rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> .
 @prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#> .
 @prefix this: <file://CDL/GEMS_CO2_Apr2006.cdl/> .
@@ -10,6 +12,11 @@
 
 this: a bald:Container ;
     NetCDF:Conventions "CF-1.0" ;
+    dct:format [ a dct:MediaType ;
+            dct:identifier <http://vocab.nerc.ac.uk/collection/M01/current/NC/> ] ;
+    dcat:distribution [ a dcat:Distribution ;
+            dcat:mediaType [ a dcat:MediaType ;
+                    dct:identifier "application/x-netcdf" ] ] ;
     bald:contains this:co2,
         this:latitude,
         this:levelist,
@@ -29,14 +36,14 @@ this:co2 a bald:Array ;
             bald:target this:levelist ;
             bald:targetReshape ( 1 60 1 1 ) ;
             bald:targetShape ( 60 ) ],
-        [ a bald:Reference ;
-            bald:target this:latitude ;
-            bald:targetReshape ( 1 1 181 1 ) ;
-            bald:targetShape ( 181 ) ],
         [ a bald:Reference ;
             bald:target this:time ;
             bald:targetReshape ( 1 1 1 1 ) ;
             bald:targetShape ( 1 ) ],
+        [ a bald:Reference ;
+            bald:target this:latitude ;
+            bald:targetReshape ( 1 1 181 1 ) ;
+            bald:targetShape ( 181 ) ],
         [ a bald:Reference ;
             bald:target this:longitude ;
             bald:targetReshape ( 1 1 1 360 ) ;
@@ -49,21 +56,21 @@ this:lnsp a bald:Array ;
     NetCDF:add_offset 11.2087164280841 ;
     NetCDF:long_name "Logarithm of surface pressure" ;
     bald:references [ a bald:Reference ;
-            bald:target this:longitude ;
-            bald:targetReshape ( 1 1 1 360 ) ;
-            bald:targetShape ( 360 ) ],
-        [ a bald:Reference ;
             bald:target this:time ;
             bald:targetReshape ( 1 1 1 1 ) ;
             bald:targetShape ( 1 ) ],
-        [ a bald:Reference ;
-            bald:target this:latitude ;
-            bald:targetReshape ( 1 1 181 1 ) ;
-            bald:targetShape ( 181 ) ],
         [ a bald:Reference ;
             bald:target this:levelist ;
             bald:targetReshape ( 1 60 1 1 ) ;
-            bald:targetShape ( 60 ) ] ;
+            bald:targetShape ( 60 ) ],
+        [ a bald:Reference ;
+            bald:target this:longitude ;
+            bald:targetReshape ( 1 1 1 360 ) ;
+            bald:targetShape ( 360 ) ],
+        [ a bald:Reference ;
+            bald:target this:latitude ;
+            bald:targetReshape ( 1 1 181 1 ) ;
+            bald:targetShape ( 181 ) ] ;
     bald:shape ( 1 60 181 360 ) .
 
 this:latitude a bald:Array ;

lib/bald/tests/integration/TTL/ProcessChain0300.ttl

@@ -3,8 +3,9 @@
 @prefix NWP: <https://codes.nws.noaa.gov/NumericalWeatherPrediction> .
 @prefix StatPP: <https://codes.nws.noaa.gov/StatisticalPostProcessing> .
 @prefix bald: <https://www.opengis.net/def/binary-array-ld/> .
-@prefix bald1: <https://www.opengis.net/def/binary-array-ld/x> .
 @prefix cf_sname: <http://vocab.nerc.ac.uk/standard_name/> .
+@prefix dcat: <http://www.w3.org/ns/dcat#> .
+@prefix dct: <http://purl.org/dc/terms/> .
 @prefix rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> .
 @prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#> .
 @prefix this: <file://CDL/ProcessChain0300.cdl/> .
@@ -13,6 +14,11 @@
 
 this: a bald:Container ;
     this:process_chain "gfsmos_process_chain" ;
+    dct:format [ a dct:MediaType ;
+            dct:identifier <http://vocab.nerc.ac.uk/collection/M01/current/NC/> ] ;
+    dcat:distribution [ a dcat:Distribution ;
+            dcat:mediaType [ a dcat:MediaType ;
+                    dct:identifier "application/x-netcdf" ] ] ;
     bald:contains this:gfsmos_process_chain,
         this:step1,
         this:step2 ;

lib/bald/tests/integration/TTL/array_reference.ttl

@@ -1,11 +1,18 @@
 @prefix bald: <https://www.opengis.net/def/binary-array-ld/> .
+@prefix dcat: <http://www.w3.org/ns/dcat#> .
+@prefix dct: <http://purl.org/dc/terms/> .
 @prefix rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> .
 @prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#> .
 @prefix this: <file://CDL/array_reference.cdl/> .
 @prefix xml: <http://www.w3.org/XML/1998/namespace> .
 @prefix xsd: <http://www.w3.org/2001/XMLSchema#> .
 
 this: a bald:Container ;
+    dct:format [ a dct:MediaType ;
+            dct:identifier <http://vocab.nerc.ac.uk/collection/M01/current/NC/> ] ;
+    dcat:distribution [ a dcat:Distribution ;
+            dcat:mediaType [ a dcat:MediaType ;
+                    dct:identifier "application/x-netcdf" ] ] ;
     bald:contains this:child_variable,
         this:parent_variable ;
     bald:isPrefixedBy "prefix_list" .

lib/bald/tests/integration/TTL/array_reference_withbase.ttl

@@ -1,11 +1,18 @@
 @prefix bald: <https://www.opengis.net/def/binary-array-ld/> .
+@prefix dcat: <http://www.w3.org/ns/dcat#> .
+@prefix dct: <http://purl.org/dc/terms/> .
 @prefix rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> .
 @prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#> .
 @prefix this: <http://example.org/base/> .
 @prefix xml: <http://www.w3.org/XML/1998/namespace> .
 @prefix xsd: <http://www.w3.org/2001/XMLSchema#> .
 
 this: a bald:Container ;
+    dct:format [ a dct:MediaType ;
+            dct:identifier <http://vocab.nerc.ac.uk/collection/M01/current/NC/> ] ;
+    dcat:distribution [ a dcat:Distribution ;
+            dcat:mediaType [ a dcat:MediaType ;
+                    dct:identifier "application/x-netcdf" ] ] ;
     bald:contains this:child_variable,
         this:parent_variable ;
     bald:isPrefixedBy "prefix_list" .

lib/bald/tests/integration/TTL/ereefs_gbr4_ncld.ttl

@@ -2,6 +2,8 @@
 @prefix NetCDF: <http://def.scitools.org.uk/NetCDF/> .
 @prefix bald: <https://www.opengis.net/def/binary-array-ld/> .
 @prefix cf_sname: <http://vocab.nerc.ac.uk/standard_name/> .
+@prefix dcat: <http://www.w3.org/ns/dcat#> .
+@prefix dct: <http://purl.org/dc/terms/> .
 @prefix rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> .
 @prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#> .
 @prefix this: <file://CDL/ereefs_gbr4_ncld.cdl/> .
@@ -17,6 +19,11 @@ this: a bald:Container ;
     this:shoc_version "v1.1 rev(5249)" ;
     NetCDF:Conventions "CF-1.0" ;
     NetCDF:title "GBR4 Hydro" ;
+    dct:format [ a dct:MediaType ;
+            dct:identifier <http://vocab.nerc.ac.uk/collection/M01/current/NC/> ] ;
+    dcat:distribution [ a dcat:Distribution ;
+            dcat:mediaType [ a dcat:MediaType ;
+                    dct:identifier "application/x-netcdf" ] ] ;
     bald:contains this:botz,
         this:eta,
         this:latitude,

lib/bald/tests/integration/TTL/hgroups.ttl

@@ -0,0 +1,26 @@
+@prefix CFTerms: <http://def.scitools.org.uk/CFTerms/> .
+@prefix NetCDF: <http://def.scitools.org.uk/NetCDF/> .
+@prefix bald: <https://www.opengis.net/def/binary-array-ld/> .
+@prefix cf_sname: <http://vocab.nerc.ac.uk/standard_name/> .
+@prefix dcat: <http://www.w3.org/ns/dcat#> .
+@prefix dct: <http://purl.org/dc/terms/> .
+@prefix rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> .
+@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#> .
+@prefix this: <file://CDL/hgroups.cdl/> .
+@prefix xml: <http://www.w3.org/XML/1998/namespace> .
+@prefix xsd: <http://www.w3.org/2001/XMLSchema#> .
+
+this: a bald:Container ;
+    dct:format [ a dct:MediaType ;
+            dct:identifier <http://vocab.nerc.ac.uk/collection/M01/current/NC/> ] ;
+    dcat:distribution [ a dcat:Distribution ;
+            dcat:downloadURL <https://www.unidata.ucar.edu/software/netcdf/examples/test_hgroups.cdl> ;
+            dcat:mediaType [ a dcat:MediaType ;
+                    dct:identifier "application/x-netcdf" ] ] ;
+    bald:contains this:UTC_time .
+
+this:UTC_time a bald:Array ;
+    this:name "time" ;
+    this:unit "none" ;
+    bald:shape ( 74 ) .
+

lib/bald/tests/integration/TTL/multi_array_reference.ttl

@@ -1,4 +1,6 @@
 @prefix bald: <https://www.opengis.net/def/binary-array-ld/> .
+@prefix dcat: <http://www.w3.org/ns/dcat#> .
+@prefix dct: <http://purl.org/dc/terms/> .
 @prefix metce: <http://codes.wmo.int/common/observation-type/METCE/2013/> .
 @prefix rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> .
 @prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#> .
@@ -7,6 +9,11 @@
 @prefix xsd: <http://www.w3.org/2001/XMLSchema#> .
 
 this: a bald:Container ;
+    dct:format [ a dct:MediaType ;
+            dct:identifier <http://vocab.nerc.ac.uk/collection/M01/current/NC/> ] ;
+    dcat:distribution [ a dcat:Distribution ;
+            dcat:mediaType [ a dcat:MediaType ;
+                    dct:identifier "application/x-netcdf" ] ] ;
     bald:contains this:data_variable1,
         this:data_variable2,
         this:list_collection,

lib/bald/tests/integration/TTL/point_template.ttl

@@ -2,6 +2,8 @@
 @prefix NetCDF: <http://def.scitools.org.uk/NetCDF/> .
 @prefix bald: <https://www.opengis.net/def/binary-array-ld/> .
 @prefix cf_sname: <http://vocab.nerc.ac.uk/standard_name/> .
+@prefix dcat: <http://www.w3.org/ns/dcat#> .
+@prefix dct: <http://purl.org/dc/terms/> .
 @prefix rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> .
 @prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#> .
 @prefix this: <file://CDL/point_template.cdl/> .
@@ -69,6 +71,11 @@ this: a bald:Container ;
     NetCDF:Conventions "CF-1.6, ACDD-1.3" ;
     NetCDF:history "This file was created on 2016-09-22T18:16:06.590413Z" ;
     NetCDF:title "Oceanographic and surface meteorological data collected from the cordell bank monitoring station by the National Centers for Environmental Information (NCEI) in the Cordell Bank National Marine Sanctuary from 2015-03-25 to 2015-03-25" ;
+    dct:format [ a dct:MediaType ;
+            dct:identifier <http://vocab.nerc.ac.uk/collection/M01/current/NC/> ] ;
+    dcat:distribution [ a dcat:Distribution ;
+            dcat:mediaType [ a dcat:MediaType ;
+                    dct:identifier "application/x-netcdf" ] ] ;
     bald:contains this:crs,
         this:instrument1,
         this:lat,