Adds validation results rendering

mfdz · Jan 16, 2021 · 7284452 · 7284452
1 parent 0ea36e6
commit 7284452
Show file tree

Hide file tree

Showing 9 changed files with 226 additions and 56 deletions.
diff --git a/config/schemas/DATEXII_MDM_Parkdaten.sch b/config/schemas/DATEXII_MDM_Parkdaten.sch
@@ -11,7 +11,7 @@
    </pattern>
    <pattern id="override_checks">
      <title>Override checks</title>
-     <rule context="dtx:parkingFacilityStatus">
+     <rule context="dtx:parkingFacilityStatus[totalParkingCapacityLongTermOverride|totalParkingCapacityLongTermOverride|totalParkingCapacityShortTermOverride]">
        <assert role="warn" test="(sum(dtx:totalParkingCapacityLongTermOverride) + sum(dtx:totalParkingCapacityShortTermOverride) + sum(dtx:totalParkingCapacityOverride)) > 0">Sum of all given totalParkingXXXCapacityOverride of a parkingFacility should be greater than 0 for # <value-of select="dtx:parkingFacilityReference/@id"/></assert>
        <assert role="error" test="(sum(dtx:totalParkingCapacityLongTermOverride) + sum(dtx:totalParkingCapacityShortTermOverride)) &lt;= 2 * sum(dtx:totalParkingCapacityOverride)">Sum of all totalParkingCapacityShortTermOverride (<value-of select="dtx:totalParkingCapacityShortTermOverride"/>) and totalParkingCapacityLongTermOverride (<value-of select="dtx:totalParkingCapacityLongTermOverride"/>) should not exceed totalParkingCapacityOverride (<value-of select="dtx:totalParkingCapacityOverride"/>) for # <value-of select="dtx:parkingFacilityReference/@id"/>
 		<value-of select="(dtx:totalParkingCapacityLongTermOverride + dtx:totalParkingCapacityShortTermOverride)"/>

diff --git a/config/templates/dataset_template.html b/config/templates/dataset_template.html
@@ -48,8 +48,8 @@
     </script>
 
     <title>MFDZ - DPC - {{ ld.name }}</title>
-	</head>
-	<body>
+  </head>
+  <body>
   {% include "includes/header.html" %}
 
 <main role="main">
@@ -63,36 +63,36 @@ <h1 class="title">{{ ld.name }}</h1>
         <div class="column is-8 content">
           <div class="field is-grouped is-grouped-multiline">
             {% include "includes/license_badge.html" %}
+            {% include "includes/validation_badge.html" %}
+            <!--% include "includes/size_badge.html" %-->
             <!-- TODO: last update -->
-            <!-- TODO: size -->
-            <!-- TODO: validation errors -->
           </div>
         </div>
         <div class="column is-8 content">
           <p>{{ ld.description }}<p>
-      		<p>Erstellt durch {{ld.creator.name or 'Unbekannt' }}</p>
-    		  <ul class="tag-list">{% for keyword in ld.keywords %}
+          <p>Erstellt durch {{ld.creator.name or 'Unbekannt' }}</p>
+          <ul class="tag-list">{% for keyword in ld.keywords %}
               <li class="tag is-rounded">{{ keyword }}</li>
             {% endfor %}</ul>
 
-    		  <h3>Download-Optionen</h3>
-    		  <div>
-      			<ul>{% for download in ld.distribution %}
-      			<li><a href="{{ download.contentUrl }}">Datensatz</a> ({{download.encodingFormat}})</li>
-      			{% endfor %}<li><a href="ld.json" title="JSON-LD Datenbeschreibung">Metadaten</a><img width="32" src="https://json-ld.org/images/json-ld-data-32.png" alt="JSON-LD-logo-32"/></li>
+          <h3>Download-Optionen</h3>
+          <div>
+            <ul>{% for download in ld.distribution %}
+            <li><a href="{{ download.contentUrl }}">Datensatz</a> ({{download.encodingFormat}})</li>
+            {% endfor %}<li><a href="ld.json" title="JSON-LD Datenbeschreibung">Metadaten</a><img width="32" src="https://json-ld.org/images/json-ld-data-32.png" alt="JSON-LD-logo-32"/></li>
             <li><a href="datapackage.json" title="Datapackage Datenbeschreibung">datapackage.json</a></li>
           </ul>
-    		  </div>
-      		
+          </div>
+          
           <h3>Nutzungsbedingungen</h3>
-      		<div>Lizenz: {{ ld.license or 'Unbekannt'}}<br/>
-      			Quellenvermerk: {{ ld.creditText or 'Keine Angabe' }}
-      		</div>
+          <div>Lizenz: {{ ld.license or 'Unbekannt'}}<br/>
+            Quellenvermerk: {{ ld.creditText or 'Keine Angabe' }}
+          </div>
         </div>
       </div>
     </div>
   </section>
 </main>
   {% include "includes/footer.html" %}
-	</body>
-</html>
+  </body>
+</html>
diff --git a/config/templates/includes/size_badge.html b/config/templates/includes/size_badge.html
@@ -0,0 +1,6 @@
+<div class="control">
+  <div class="tags has-addons">
+    <span class="tag is-dark">Größe</span>
+    <span class="tag is-info">{{size}}</span>
+  </div>
+</div>
diff --git a/config/templates/includes/validation_badge.html b/config/templates/includes/validation_badge.html
@@ -0,0 +1,10 @@
+<div class="control">
+  <div class="tags has-addons">
+    <span class="tag is-dark">Validierung</span>
+    {% if validation_ok %}
+    	<span class="tag is-success">erfolgreich</span>
+    {% else %}
+    	<span class="tag is-danger"><a style="border-bottom: 0; color: #fff" href="validation_results.html">Fehler</a></span>
+    {% endif %} 	
+  </div>
+</div>
diff --git a/config/templates/validation_results_template.html b/config/templates/validation_results_template.html
@@ -0,0 +1,78 @@
+<!DOCTYPE html>
+<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="de" lang="de">
+<head>
+    <meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1">
+    <meta name="description" content="{{ ld.description }}" />
+    <link rel="shortcut icon" type="image/x-icon" href="/favicon.ico"/>
+     <!-- OPENGRAPH -->
+    <meta content="MITFAHR|DE|ZENTRALE" property="og:title">
+    <meta content="https://data.mfdz.de/img/data.jpg" property="og:image">
+    <meta content="1280" property="og:image:width">
+    <meta content="904" property="og:image:height">
+    <meta content="MITFAHR|DE|ZENTRALE" property="og:site_name">
+    <meta content="MITFAHR|DE|ZENTRALE Datensatz {{ ld.name }}" property="og:description">
+    <meta content="MITFAHR|DE|ZENTRALE" property="article:author">
+
+    <!-- SEO -->
+    <meta content="MITFAHR|DE|ZENTRALE" name="author">
+    <meta content="MITFAHR|DE|ZENTRALE Datensatz {{ ld.name }}" name="description">
+    <meta content="index, follow, noodp" name="robots">
+    <link rel="apple-touch-icon" sizes="180x180" href="https://mfdz.de/media/site/3434498041-1603810793/apple-touch-icon-180-180.png">
+      <!-- TWITTER -->
+
+    <meta name="twitter:card" content="summary_large_image">
+    <meta name="twitter:site" content="@mfdz_de">
+    <meta name="twitter:creator" content="@mfdz_de">
+
+    <meta content="MITFAHR|DE|ZENTRALE" property="twitter:title">
+    <meta content="https://data.mfdz.de/img/data.jpg  " property="twitter:image">
+    <meta content="MITFAHR|DE|ZENTRALE Datensatz {{ ld.name }}" property="twitter:description">
+
+    <link href="https://mfdz.de/assets/css/bulma.min.css" rel="stylesheet">
+    <link href="https://maxcdn.bootstrapcdn.com/font-awesome/4.7.0/css/font-awesome.min.css" rel="stylesheet">
+    <link href="https://mfdz.de/assets/css/lity.min.css" rel="stylesheet">
+    <link href="https://mfdz.de/assets/css/slick.css" rel="stylesheet">
+    <link href="https://mfdz.de/assets/css/index.css" rel="stylesheet">
+    <link href="https://mfdz.de/assets/css/templates/default.css" rel="stylesheet">
+
+    <script type="application/ld+json">
+    {
+      "@context": "https://schema.org",
+      "@type": "Organization",
+      "url": "https://www.mfdz.de",
+      "logo": "https://www.mfdz.de/media/site/3434498041-1603810793/apple-touch-icon-180-180.png"
+    }
+    </script>
+
+    <title>MFDZ - DPC - Validierungsergebnisse {{ ld.name }}</title>
+  </head>
+  <body>
+  {% include "includes/header.html" %}
+
+<main role="main">
+  <section class="section module-text-section">
+    <div class="container">
+      <div class="columns is-multiline is-centered">
+        <div class="column is-8 content module-title has-text-centered">
+            <h1 class="title">Validierungsergebnisse Datensatz {{ ld.name }}</h1>
+        </div>
+
+
+        <div class="column is-8 content">
+          <table>
+            <thead><th>Validator</th><th>Beschreibung</th></thead>
+            <tbody>
+              {% for error in errors %}
+              <tr><td>{{error.domain_name}}</td><td>{{error.message}}</td></tr>
+              {% endfor %}
+            </tbody>
+          </table>
+        </div>
+      </div>
+    </div>
+  </section>
+</main>
+  {% include "includes/footer.html" %}
+  </body>
+</html>
diff --git a/dodo.py b/dodo.py
@@ -1,5 +1,5 @@
 import glob
-from dpc.tasks import render_index_page, download_links, dpc_files, validate_xml, validate_xml_via_schematron, render_index, datasets_with_schematron_validation
+import dpc.tasks as tasks
 
 DOIT_CONFIG = {
     'action_string_formatting': 'new'
@@ -19,33 +19,19 @@ def task_copy_static_files():
             'actions': ['mkdir -p {0} && cp -rf config/static/* {0}'.format(DPC_CONFIG['out_dir'])],
         }
 
-def task_render_landing_page():
-    '''render index.html page for datasets'''
-    for (dataset_name, dpc_file) in dpc_files(DPC_CONFIG['dataset_definitions_dir']+'**/dpc.json'):
-        dst_file = DPC_CONFIG['out_dir'] + dataset_name + '/index.html'
-        ld_file = DPC_CONFIG['out_dir'] + dataset_name + '/ld.json'
-        datapackage_file = DPC_CONFIG['out_dir'] + dataset_name + '/datapackage.json'
-
-        yield {
-            'name': dataset_name,
-            'file_dep': [dpc_file, DPC_CONFIG['dataset_template'], DPC_CONFIG['datapackage_template']],
-            'targets': [dst_file, ld_file, datapackage_file],
-            'actions': [(render_index_page, [dpc_file, dst_file, ld_file, datapackage_file, dataset_name, DPC_CONFIG])],
-        }
-
 def task_render_index():
     '''render index.html and sitemap.txt for datasets'''
     SITEMAP = DPC_CONFIG['out_dir'] + 'sitemap.txt'
     INDEX = DPC_CONFIG['out_dir'] + 'index.html'
 
     return {
         'targets': [SITEMAP, INDEX],
-        'actions': [(render_index, [DPC_CONFIG['dataset_definitions_dir'], DPC_CONFIG['host'], INDEX, SITEMAP])],
+        'actions': [(tasks.render_index, [DPC_CONFIG['dataset_definitions_dir'], DPC_CONFIG['host'], INDEX, SITEMAP])],
     }
 
 
 def task_download_mdm_dataset():
-    for (dataset_name, download_url, cert, file_type) in download_links(DPC_CONFIG['dataset_definitions_dir']+'**/dpc.json'):
+    for (dataset_name, download_url, cert, file_type) in tasks.download_links(DPC_CONFIG['dataset_definitions_dir']+'**/dpc.json'):
         dst_file = DPC_CONFIG['out_dir'] + dataset_name + '/body.' + file_type
 
         action = 'curl -fsS -z {targets} -o {targets} --create-dirs -R --compressed '
@@ -58,29 +44,69 @@ def task_download_mdm_dataset():
         }
 
 def task_validate_xml():
-    for (dataset_name, download_url, cert, file_type) in download_links(DPC_CONFIG['dataset_definitions_dir']+'**/dpc.json'):
+    for (dataset_name, download_url, cert, file_type) in tasks.download_links(DPC_CONFIG['dataset_definitions_dir']+'**/dpc.json'):
         if file_type != 'xml':
             continue
 
         body_file = DPC_CONFIG['out_dir'] + dataset_name + '/body.xml'
-        dst_file = DPC_CONFIG['out_dir'] + dataset_name + '/validation_results.txt'
+        dst_file = DPC_CONFIG['out_dir'] + dataset_name + '/validation_results.schema.jsonl'
 
         yield {
             'name': dataset_name,
             'file_dep': [body_file],
             'targets': [dst_file],
-            'actions': [(validate_xml, (body_file, dst_file))]
+            'actions': [(tasks.validate_xml, (body_file, dst_file))]
         }    
 
 def task_validate_xml_using_schematron():
-    for (dataset_name, schematron_file) in datasets_with_schematron_validation(DPC_CONFIG['dataset_definitions_dir']+'**/dpc.json'):
+    for (dataset_name, schematron_file) in tasks.datasets_with_schematron_validation(DPC_CONFIG['dataset_definitions_dir']+'**/dpc.json'):
 
         body_file = DPC_CONFIG['out_dir'] + dataset_name + '/body.xml'
-        dst_file = DPC_CONFIG['out_dir'] + dataset_name + '/validation_results_schematron.txt'
+        dst_file = DPC_CONFIG['out_dir'] + dataset_name + '/validation_results.schematron.jsonl'
 
         yield {
             'name': dataset_name,
             'file_dep': [body_file, schematron_file],
             'targets': [dst_file],
-            'actions': [(validate_xml_via_schematron, (body_file, schematron_file, dst_file))]
+            'actions': [(tasks.validate_xml_via_schematron, (body_file, schematron_file, dst_file))]
+        }
+
+def task_merge_validation_results():
+    for (dataset_name, validation_files) in tasks.datasets_out_files(DPC_CONFIG['dataset_definitions_dir']+'**/dpc.json', DPC_CONFIG['out_dir'], 'validation_results.*.jsonl'):
+        dst_file = DPC_CONFIG['out_dir'] + dataset_name + '/validation_results.jsonl'
+
+        yield {
+            'name': dataset_name,
+            'file_dep': validation_files,
+            'targets': [dst_file],
+            'actions': [(tasks.merge_validation_results, (validation_files, dst_file))]
+        }
+
+def task_render_validation_results():
+    for (dataset_name, dpc_file) in tasks.dpc_files(DPC_CONFIG['dataset_definitions_dir']+'**/dpc.json'):
+        validation_results_file = DPC_CONFIG['out_dir'] + dataset_name + '/validation_results.jsonl'
+        dst_file = DPC_CONFIG['out_dir'] + dataset_name + '/validation_results.html'
+
+        yield {
+            'name': dataset_name,
+            'file_dep': [validation_results_file],
+            'targets': [dst_file],
+            'actions': [(tasks.render_validation_results, (validation_results_file, dpc_file, dst_file))]
+        }
+
+def task_render_landing_page():
+    '''render index.html page for datasets'''
+    for (dataset_name, dpc_file) in tasks.dpc_files(DPC_CONFIG['dataset_definitions_dir']+'**/dpc.json'):
+        dst_file = DPC_CONFIG['out_dir'] + dataset_name + '/index.html'
+        ld_file = DPC_CONFIG['out_dir'] + dataset_name + '/ld.json'
+        validation_results_file = DPC_CONFIG['out_dir'] + dataset_name + '/validation_results.jsonl'
+
+        datapackage_file = DPC_CONFIG['out_dir'] + dataset_name + '/datapackage.json'
+
+        yield {
+            'name': dataset_name,
+            'file_dep': [dpc_file, validation_results_file, DPC_CONFIG['dataset_template'], DPC_CONFIG['datapackage_template']],
+            'targets': [dst_file, ld_file, datapackage_file],
+            'actions': [(tasks.render_landing_page, [dpc_file, dst_file, ld_file, datapackage_file, dataset_name, validation_results_file, DPC_CONFIG])],
         }
+
diff --git a/dpc/schemavalidator.py b/dpc/schemavalidator.py
@@ -27,15 +27,17 @@
 </xs:schema>"""
 
 
-def validate_XML(tree):
+def validate_XML(tree, namespaces = None):
     """Validate an XML file represented as tree. Follow all schemaLocations.
     :param tree: 
     :type tree: ElementTree
     """
 
     schema_tree = etree.XML(SCHEMA_TEMPLATE)
-    # Find all unique instances of 'xsi:schemaLocation="<namespace> <path-to-schema.xsd> ..."'
-    schema_locations = set(tree.xpath("//*/@xsi:schemaLocation", namespaces={'xsi': XSI}))
+
+    # Find all unique instances of 'xsi:schemaLocation="<namespace> <path-to-schema.xsd> ..."'    
+    schema_locations = namespaces if namespaces else set(tree.xpath("//*/@xsi:schemaLocation", namespaces={'xsi': XSI}))
+
     for schema_location in schema_locations:
         # Split namespaces and schema locations ; use strip to remove leading
         # and trailing whitespace.
@@ -46,7 +48,7 @@ def validate_XML(tree):
             xs_import.attrib['namespace'] = namespace
             xs_import.attrib['schemaLocation'] = location
             schema_tree.append(xs_import)
-    # Contstruct the schema
+    # Construct the schema
     schema = etree.XMLSchema(schema_tree)
     # Validate!
     schema.validate(tree)