From b9ebd5238c7211846df54fd5b479662eda95387a Mon Sep 17 00:00:00 2001 From: Jonathan Yu Date: Fri, 30 Sep 2022 09:54:48 +1000 Subject: [PATCH 1/2] modifying affected tests --- lib/bald/tests/integration/test_aliases.py | 4 ++-- lib/bald/tests/integration/test_netcdf.py | 4 ++-- lib/bald/tests/integration/test_netcdf4_classic.py | 4 ++-- lib/bald/tests/integration/test_validation.py | 3 +-- 4 files changed, 7 insertions(+), 8 deletions(-) diff --git a/lib/bald/tests/integration/test_aliases.py b/lib/bald/tests/integration/test_aliases.py index 2ced71a..7a586f1 100644 --- a/lib/bald/tests/integration/test_aliases.py +++ b/lib/bald/tests/integration/test_aliases.py @@ -55,8 +55,8 @@ def test_invalid_uri(self): validation = bald.validate_hdf5(tfile, cache=self.acache, uris_resolve=True) exns = validation.exceptions() - expected = ['https://www.opengis.net/def/binary-array-ld/turtle is not resolving as a resource (404).', - 'https://www.opengis.net/def/binary-array-ld/walnut is not resolving as a resource (404).'] + expected = ['https://www.opengis.net/def/binary-array-ld/ is not resolving as a resource (404).', 'https://www.opengis.net/def/binary-array-ld/turtle is not resolving as a resource (404).', 'https://www.opengis.net/def/binary-array-ld/walnut is not resolving as a resource (404).', 'https://www.opengis.net/def/binary-array-ld/ is not resolving as a resource (404).', 'https://www.opengis.net/def/binary-array-ld/ is not resolving as a resource (404).'] + self.assertTrue((not validation.is_valid()) and exns == expected, msg='{} != {}'.format(exns, expected)) diff --git a/lib/bald/tests/integration/test_netcdf.py b/lib/bald/tests/integration/test_netcdf.py index 5c96ce0..17d9ab3 100644 --- a/lib/bald/tests/integration/test_netcdf.py +++ b/lib/bald/tests/integration/test_netcdf.py @@ -51,8 +51,8 @@ def test_invalid_uri(self): validation = bald.validate_netcdf(tfile, cache=self.acache, uris_resolve=True) exns = validation.exceptions() - expected = ['https://www.opengis.net/def/binary-array-ld/turtle is not resolving as a resource (404).', - 'https://www.opengis.net/def/binary-array-ld/walnut is not resolving as a resource (404).'] + expected = ['https://www.opengis.net/def/binary-array-ld/ is not resolving as a resource (404).', 'https://www.opengis.net/def/binary-array-ld/turtle is not resolving as a resource (404).', 'https://www.opengis.net/def/binary-array-ld/walnut is not resolving as a resource (404).', 'https://www.opengis.net/def/binary-array-ld/ is not resolving as a resource (404).', 'https://www.opengis.net/def/binary-array-ld/ is not resolving as a resource (404).'] + self.assertTrue((not validation.is_valid()) and exns == expected, msg='{} != {}'.format(exns, expected)) diff --git a/lib/bald/tests/integration/test_netcdf4_classic.py b/lib/bald/tests/integration/test_netcdf4_classic.py index 8fd7102..813391b 100644 --- a/lib/bald/tests/integration/test_netcdf4_classic.py +++ b/lib/bald/tests/integration/test_netcdf4_classic.py @@ -50,8 +50,8 @@ def test_invalid_uri(self): uris_resolve=True) exns = validation.exceptions() - expected = ['https://www.opengis.net/def/binary-array-ld/turtle is not resolving as a resource (404).', - 'https://www.opengis.net/def/binary-array-ld/walnut is not resolving as a resource (404).'] + expected = ['https://www.opengis.net/def/binary-array-ld/ is not resolving as a resource (404).', 'https://www.opengis.net/def/binary-array-ld/turtle is not resolving as a resource (404).', 'https://www.opengis.net/def/binary-array-ld/walnut is not resolving as a resource (404).', 'https://www.opengis.net/def/binary-array-ld/ is not resolving as a resource (404).', 'https://www.opengis.net/def/binary-array-ld/ is not resolving as a resource (404).'] + self.assertTrue((not validation.is_valid()) and exns == expected, msg='{} != {}'.format(exns, expected)) diff --git a/lib/bald/tests/integration/test_validation.py b/lib/bald/tests/integration/test_validation.py index 17ece3e..17349ac 100644 --- a/lib/bald/tests/integration/test_validation.py +++ b/lib/bald/tests/integration/test_validation.py @@ -47,8 +47,7 @@ def test_invalid_uri(self): validation = bald.validate_hdf5(tfile, cache=self.acache, uris_resolve=True) exns = validation.exceptions() - expected = ['https://www.opengis.net/def/binary-array-ld/turtle is not resolving as a resource (404).', - 'https://www.opengis.net/def/binary-array-ld/walnut is not resolving as a resource (404).'] + expected = ['https://www.opengis.net/def/binary-array-ld/ is not resolving as a resource (404).', 'https://www.opengis.net/def/binary-array-ld/turtle is not resolving as a resource (404).', 'https://www.opengis.net/def/binary-array-ld/walnut is not resolving as a resource (404).', 'https://www.opengis.net/def/binary-array-ld/ is not resolving as a resource (404).', 'https://www.opengis.net/def/binary-array-ld/ is not resolving as a resource (404).'] self.assertTrue((not validation.is_valid()) and exns == expected, msg='{} != {}'.format(exns, expected)) From ca8af9c673f08e463fa47e6c51c9abe063cbcb4d Mon Sep 17 00:00:00 2001 From: Jonathan Yu Date: Fri, 14 Oct 2022 17:30:00 +1100 Subject: [PATCH 2/2] fix utf decode exceptions --- lib/bald/tests/integration/test_aliases.py | 6 ++++- .../tests/integration/test_cdl_rdfgraph.py | 27 ++++++++++--------- .../integration/test_multi_array_reference.py | 2 +- lib/bald/tests/integration/test_netcdf.py | 6 +++-- .../tests/integration/test_netcdf4_classic.py | 7 +++-- 5 files changed, 29 insertions(+), 19 deletions(-) diff --git a/lib/bald/tests/integration/test_aliases.py b/lib/bald/tests/integration/test_aliases.py index 7a586f1..fa6a2b8 100644 --- a/lib/bald/tests/integration/test_aliases.py +++ b/lib/bald/tests/integration/test_aliases.py @@ -55,7 +55,11 @@ def test_invalid_uri(self): validation = bald.validate_hdf5(tfile, cache=self.acache, uris_resolve=True) exns = validation.exceptions() - expected = ['https://www.opengis.net/def/binary-array-ld/ is not resolving as a resource (404).', 'https://www.opengis.net/def/binary-array-ld/turtle is not resolving as a resource (404).', 'https://www.opengis.net/def/binary-array-ld/walnut is not resolving as a resource (404).', 'https://www.opengis.net/def/binary-array-ld/ is not resolving as a resource (404).', 'https://www.opengis.net/def/binary-array-ld/ is not resolving as a resource (404).'] + expected = ['https://www.opengis.net/def/binary-array-ld/ is not resolving as a resource (404).', + 'https://www.opengis.net/def/binary-array-ld/turtle is not resolving as a resource (404).', + 'https://www.opengis.net/def/binary-array-ld/walnut is not resolving as a resource (404).', + 'https://www.opengis.net/def/binary-array-ld/ is not resolving as a resource (404).' + ] self.assertTrue((not validation.is_valid()) and exns == expected, msg='{} != {}'.format(exns, expected)) diff --git a/lib/bald/tests/integration/test_cdl_rdfgraph.py b/lib/bald/tests/integration/test_cdl_rdfgraph.py index e006543..d699e13 100644 --- a/lib/bald/tests/integration/test_cdl_rdfgraph.py +++ b/lib/bald/tests/integration/test_cdl_rdfgraph.py @@ -26,7 +26,7 @@ def check_result(self, result, expected): lb = [lbb, lbr, lbe] self.assertTrue(rdflib.compare.isomorphic(result, expected), - ''.join(list(itertools.chain(*zip(lb, [g.serialize(format='n3').decode("utf-8") for g in + ''.join(list(itertools.chain(*zip(lb, [g.serialize(format='n3') for g in rdflib.compare.graph_diff(result, # expected)[1:]])) expected)]))))) @@ -45,7 +45,8 @@ def test_array_reference(self): cdl_file_uri = 'file://CDL/{}'.format(cdlname) root_container = bald.load_netcdf(tfile, baseuri=cdl_file_uri, cache=self.acache) rdfgraph = root_container.rdfgraph() - ttl = rdfgraph.serialize(format='n3').decode("utf-8") + ttl = rdfgraph.serialize(format='n3') + #ttl = rdfgraph.serialize(format='n3').decode("utf-8") if os.environ.get('bald_update_results') is not None: with open(os.path.join(self.ttl_path, 'array_reference.ttl'), 'w') as sf: sf.write(ttl) @@ -60,7 +61,7 @@ def test_array_reference_with_baseuri(self): subprocess.check_call(['ncgen', '-o', tfile, cdl_file]) root_container = bald.load_netcdf(tfile, baseuri='http://example.org/base', cache=self.acache) rdfgraph = root_container.rdfgraph() - ttl = rdfgraph.serialize(format='n3').decode("utf-8") + ttl = rdfgraph.serialize(format='n3') if os.environ.get('bald_update_results') is not None: with open(os.path.join(self.ttl_path, 'array_reference_withbase.ttl'), 'w') as sf: sf.write(ttl) @@ -81,7 +82,7 @@ def test_array_reference_external_prefix_context(self): root_container = bald.load_netcdf(tfile, baseuri=cdl_file_uri, prefix_contexts=prefix_context, cache=self.acache) rdfgraph = root_container.rdfgraph() - ttl = rdfgraph.serialize(format='n3').decode("utf-8") + ttl = rdfgraph.serialize(format='n3') if os.environ.get('bald_update_results') is not None: with open(os.path.join(self.ttl_path, 'array_reference_external_prefix_context.ttl'), 'w') as sf: sf.write(ttl) @@ -98,7 +99,7 @@ def test_multi_array_reference(self): cdl_file_uri = 'file://CDL/{}'.format(cdlname) root_container = bald.load_netcdf(tfile, baseuri=cdl_file_uri, cache=self.acache) rdfgraph = root_container.rdfgraph() - ttl = rdfgraph.serialize(format='n3').decode("utf-8") + ttl = rdfgraph.serialize(format='n3') if os.environ.get('bald_update_results') is not None: with open(os.path.join(self.ttl_path, 'multi_array_reference.ttl'), 'w') as sf: sf.write(ttl) @@ -120,7 +121,7 @@ def test_point_template(self): root_container = bald.load_netcdf(tfile, baseuri=cdl_file_uri, alias_dict=alias_dict, cache=self.acache) rdfgraph = root_container.rdfgraph() - ttl = rdfgraph.serialize(format='n3').decode("utf-8") + ttl = rdfgraph.serialize(format='n3') if os.environ.get('bald_update_results') is not None: with open(os.path.join(self.ttl_path, 'point_template.ttl'), 'w') as sf: sf.write(ttl) @@ -142,7 +143,7 @@ def test_gems_co2(self): root_container = bald.load_netcdf(tfile, baseuri=cdl_file_uri, alias_dict=alias_dict, cache=self.acache) rdfgraph = root_container.rdfgraph() - ttl = rdfgraph.serialize(format='n3').decode("utf-8") + ttl = rdfgraph.serialize(format='n3') if os.environ.get('bald_update_results') is not None: with open(os.path.join(self.ttl_path, 'GEMS_CO2_Apr2006.ttl'), 'w') as sf: sf.write(ttl) @@ -163,7 +164,7 @@ def test_ProcessChain0300(self): root_container = bald.load_netcdf(tfile, baseuri=cdl_file_uri, alias_dict=alias_dict, cache=self.acache) rdfgraph = root_container.rdfgraph() - ttl = rdfgraph.serialize(format='n3').decode("utf-8") + ttl = rdfgraph.serialize(format='n3') if os.environ.get('bald_update_results') is not None: with open(os.path.join(self.ttl_path, '{}.ttl'.format(name)), 'w') as sf: sf.write(ttl) @@ -185,7 +186,7 @@ def test_ereefs(self): root_container = bald.load_netcdf(tfile, baseuri=cdl_file_uri, alias_dict=alias_dict, cache=self.acache) rdfgraph = root_container.rdfgraph() - ttl = rdfgraph.serialize(format='n3').decode("utf-8") + ttl = rdfgraph.serialize(format='n3') if os.environ.get('bald_update_results') is not None: with open(os.path.join(self.ttl_path, '{}.ttl'.format(name)), 'w') as sf: sf.write(ttl) @@ -208,7 +209,7 @@ def test_votemper(self): root_container = bald.load_netcdf(tfile, baseuri=cdl_file_uri, alias_dict=alias_dict, cache=self.acache) rdfgraph = root_container.rdfgraph() - ttl = rdfgraph.serialize(format='n3').decode("utf-8") + ttl = rdfgraph.serialize(format='n3') if os.environ.get('bald_update_results') is not None: with open(os.path.join(self.ttl_path, '{}.ttl'.format(name)), 'w') as sf: sf.write(ttl) @@ -237,7 +238,7 @@ def test_hgroups(self): root_container = bald.load_netcdf(tfile, baseuri=cdl_file_uri, alias_dict=alias_dict, cache=self.acache, file_locator=hgurl) rdfgraph = root_container.rdfgraph() - ttl = rdfgraph.serialize(format='n3').decode("utf-8") + ttl = rdfgraph.serialize(format='n3') if os.environ.get('bald_update_results') is not None: with open(os.path.join(self.ttl_path, '{}.ttl'.format(name)), 'w') as sf: sf.write(ttl) @@ -270,7 +271,7 @@ def test_hgroups_schema_dot_org(self): rdfgraph = root_container.rdfgraph() schema_org_inst = bald.schemaOrg(rdfgraph,hgurl,baseuri).getSchemaOrgGraph() #rdfgraph = schema_org_inst.distribution(baseuri, rdfgraph, hgurl) - ttl = schema_org_inst.serialize(format='n3').decode("utf-8") + ttl = schema_org_inst.serialize(format='n3') if os.environ.get('bald_update_results') is not None: with open(os.path.join(self.ttl_path, '{}.ttl'.format(name)), 'w') as sf: sf.write(ttl) @@ -299,7 +300,7 @@ def test_group_array_geo(self): root_container = bald.load_netcdf(tfile, baseuri=cdl_file_uri, alias_dict=alias_dict, cache=self.acache) rdfgraph = root_container.rdfgraph() - ttl = rdfgraph.serialize(format='n3').decode("utf-8") + ttl = rdfgraph.serialize(format='n3') if os.environ.get('bald_update_results') is not None: with open(os.path.join(self.ttl_path, '{}.ttl'.format(name)), 'w') as sf: sf.write(ttl) diff --git a/lib/bald/tests/integration/test_multi_array_reference.py b/lib/bald/tests/integration/test_multi_array_reference.py index dd284b2..50ad9a0 100644 --- a/lib/bald/tests/integration/test_multi_array_reference.py +++ b/lib/bald/tests/integration/test_multi_array_reference.py @@ -45,7 +45,7 @@ def test_turtle(self): subprocess.check_call(['ncgen', '-o', tfile, cdl_file]) cdl_file_uri = 'file://CDL/{}'.format(cdlname) root_container = bald.load_netcdf(tfile, baseuri=cdl_file_uri, cache=self.acache) - ttl = root_container.rdfgraph().serialize(format='n3').decode("utf-8") + ttl = root_container.rdfgraph().serialize(format='n3') if os.environ.get('bald_update_results') is not None: with open(os.path.join(self.ttl_path, 'multi_array_reference.ttl'), 'w') as sf: sf.write(ttl) diff --git a/lib/bald/tests/integration/test_netcdf.py b/lib/bald/tests/integration/test_netcdf.py index 17d9ab3..5a9539f 100644 --- a/lib/bald/tests/integration/test_netcdf.py +++ b/lib/bald/tests/integration/test_netcdf.py @@ -51,9 +51,11 @@ def test_invalid_uri(self): validation = bald.validate_netcdf(tfile, cache=self.acache, uris_resolve=True) exns = validation.exceptions() - expected = ['https://www.opengis.net/def/binary-array-ld/ is not resolving as a resource (404).', 'https://www.opengis.net/def/binary-array-ld/turtle is not resolving as a resource (404).', 'https://www.opengis.net/def/binary-array-ld/walnut is not resolving as a resource (404).', 'https://www.opengis.net/def/binary-array-ld/ is not resolving as a resource (404).', 'https://www.opengis.net/def/binary-array-ld/ is not resolving as a resource (404).'] + expected = ['https://www.opengis.net/def/binary-array-ld/ is not resolving as a resource (404).', + 'https://www.opengis.net/def/binary-array-ld/turtle is not resolving as a resource (404).', + 'https://www.opengis.net/def/binary-array-ld/walnut is not resolving as a resource (404).'] - self.assertTrue((not validation.is_valid()) and exns == expected, + self.assertTrue((not validation.is_valid()) and list(set((exns)) == expected, msg='{} != {}'.format(exns, expected)) class TestArrayReference(BaldTestCase): diff --git a/lib/bald/tests/integration/test_netcdf4_classic.py b/lib/bald/tests/integration/test_netcdf4_classic.py index 813391b..a00842a 100644 --- a/lib/bald/tests/integration/test_netcdf4_classic.py +++ b/lib/bald/tests/integration/test_netcdf4_classic.py @@ -39,6 +39,7 @@ def test_valid_uri(self): exns = validation.exceptions() self.assertTrue(validation.is_valid(), msg='{} != []'.format(exns)) + def test_invalid_uri(self): with self.temp_filename('.nc') as tfile: f = netCDF4.Dataset(tfile, "w", format="NETCDF4_CLASSIC") @@ -50,9 +51,11 @@ def test_invalid_uri(self): uris_resolve=True) exns = validation.exceptions() - expected = ['https://www.opengis.net/def/binary-array-ld/ is not resolving as a resource (404).', 'https://www.opengis.net/def/binary-array-ld/turtle is not resolving as a resource (404).', 'https://www.opengis.net/def/binary-array-ld/walnut is not resolving as a resource (404).', 'https://www.opengis.net/def/binary-array-ld/ is not resolving as a resource (404).', 'https://www.opengis.net/def/binary-array-ld/ is not resolving as a resource (404).'] + expected = ['https://www.opengis.net/def/binary-array-ld/ is not resolving as a resource (404).', + 'https://www.opengis.net/def/binary-array-ld/turtle is not resolving as a resource (404).', + 'https://www.opengis.net/def/binary-array-ld/walnut is not resolving as a resource (404).'] - self.assertTrue((not validation.is_valid()) and exns == expected, + self.assertTrue((not validation.is_valid()) and list(set((exns))) == expected, msg='{} != {}'.format(exns, expected))