provenance diff tools

Author: terrywbrady

src-colladmin/actions/storage_action.rb

@@ -228,6 +228,49 @@ def perform_action
 
     end
 
+    if @path == 'storage-get-provenance-yaml'
+      srvc = get_storage_service
+      endpoint = "/content/#{nodenum}/#{CGI.escape(ark)}/0/#{CGI.escape('system/provenance_manifest.xml')}"
+      return '<message>Storage service undefined</message>' if srvc.empty?
+      return '<message>Empty Ark</message>' if ark.empty?
+
+      begin
+        qxml = HttpGetXml.new(srvc, endpoint)
+        return "<message>Status #{qxml.status} for #{endpoint}</message>" unless qxml.status == 200
+        if qxml.body.length > 5_000_000
+          return { error: "Provenance Manifest is too large to download:  use curl: #{srvc}#{endpoint}" }.to_json
+        end
+
+        return ManifestToYaml.new.load_xml(qxml.body)
+      rescue StandardError => e
+        log(e.message)
+        log(e.backtrace)
+        return { error: "#{e.message} for #{endpoint}" }.to_json
+      end
+
+    end
+
+    if @path == 'storage-get-provenance-diff'
+      srvc = get_storage_service
+      currendpoint = "/manifest/#{nodenum}/#{CGI.escape(ark)}"
+      oldendpoint = "/content/#{nodenum}/#{CGI.escape(ark)}/0/#{CGI.escape('system/provenance_manifest.xml')}"
+      return '<message>Storage service undefined</message>' if srvc.empty?
+      return '<message>Empty Ark</message>' if ark.empty?
+
+      begin
+        curr = HttpGetXml.new(srvc, currendpoint)
+        old = HttpGetXml.new(srvc, oldendpoint)
+
+        diff = ManifestToYaml.new.load_xml_diff(old.body, curr.body)
+        return JSON.pretty_generate(diff)
+      rescue StandardError => e
+        log(e.message)
+        log(e.backtrace)
+        return { error: "#{e.message} for #{endpoint}" }.to_json
+      end
+
+    end
+
     if @path == 'storage-get-ingest-checkm'
       ver = @myparams.fetch('ver', '1').to_i
       srvc = get_storage_service

src-colladmin/config/actions.yml

@@ -433,6 +433,40 @@ storage-get-manifest-yaml:
     Convert the downloaded xml file to a user-friendly yaml format.
   documentation: |
     Storage: GET /manifest/{nodenum}/{ark}
+storage-get-provenance-yaml:
+  link-title: Get Storage Provenance for an Object from Cloud Storage as a Yaml file
+  class: StorageAction
+  category: Storage Manifest
+  sensitivity: readonly
+  testing: automated
+  format: xml
+  test_params:
+    ark: ark:/99999/fk4dv31c0b
+    nodenum: 9502
+  description: |
+    This will fail if the provenance file does not exist.
+
+    Download the XML *storage manifest* representation of a object on a specific storage node.  This request will be forwarded to the storage service.
+    Convert the downloaded xml file to a user-friendly yaml format.
+  documentation: |
+    Storage: GET /ark/{nodenum}/{ark}/0/system/provenance-metadata.xml
+storage-get-provenance-diff:
+  link-title: Get Storage Provenance for an Object from Cloud Storage as a Yaml file
+  class: StorageAction
+  category: Storage Manifest
+  sensitivity: readonly
+  testing: automated
+  format: xml
+  test_params:
+    ark: ark:/99999/fk4dv31c0b
+    nodenum: 9502
+  description: |
+    This will fail if the provenance file does not exist.
+
+    Download the XML *storage manifest* representation of a object on a specific storage node.  This request will be forwarded to the storage service.
+    Convert the downloaded xml file to a user-friendly yaml format.
+  documentation: |
+    Storage: GET /ark/{nodenum}/{ark}/0/system/provenance-metadata.xml
 storage-get-ingest-checkm:
   link-title: Generate Ingest Checkm Manifest Using Content from Cloud Storage
   class: StorageAction

src-colladmin/lib/manifest_to_yaml.rb

@@ -66,4 +66,43 @@ def load_xml(xmlbody)
     manifest = JSON.parse(manifest.to_json)
     "# See https://github.com/CDLUC3/merritt-tinker/tree/main/yaml-manifest for more info\n#{YAML.dump(manifest)}"
   end
+
+
+  def load_paths(xmlbody)
+    ark = ''
+    keys = {}
+    xml = Nokogiri::XML(xmlbody).remove_namespaces!
+    xml.xpath('/objectInfo/object').each do |doc|
+      ark = doc['id']
+    end
+    xml.xpath('/objectInfo/versions/version').each do |doc|
+      v = doc['id'].to_i
+      doc.xpath('manifest').each do |m|
+        m.xpath('file').each do |f|
+          keys["#{ark}/#{v}/#{f['id']}"] = f.xpath('digest').text
+        end
+      end
+    end
+    keys
+  end
+
+  def load_xml_diff(old, curr)
+    pathsold = load_paths(old)
+    pathscurr = load_paths(curr)
+
+    diff = {}
+    pathsold.each do |k, v|
+      next if pathsold[k] == pathscurr[k]
+
+      diff[v] = diff.fetch(v, {oldpath: [], newpath: []})
+      diff[v].fetch(:oldpath, []).append(k)
+    end
+    pathscurr.each do |k, v|
+      next if pathsold[k] == pathscurr[k]
+
+      diff[v] = diff.fetch(v, {oldpath: [], newpath: []})
+      diff[v].fetch(:newpath, []).append(k)
+    end
+    diff
+  end
 end

src-colladmin/template/storage.js

@@ -250,6 +250,32 @@ function init() {
     invoke_text(params, fname);
   });
 
+  $("button.storage-get-provenance-yaml").on("click", function(){
+    var ark = $(this).attr("data-ark");
+    var nodenum = $(this).attr("data-node-num");
+    params = {
+      path: 'storage-get-provenance-yaml',
+      ark: ark,
+      nodenum: nodenum
+    }
+    const RE=/[\/:]+/g;
+    fname = "provenance_manifest." + ark.replaceAll(RE, '_') + ".yml";
+    invoke_text(params, fname);
+  });
+
+  $("button.storage-get-provenance-diff").on("click", function(){
+    var ark = $(this).attr("data-ark");
+    var nodenum = $(this).attr("data-node-num");
+    params = {
+      path: 'storage-get-provenance-diff',
+      ark: ark,
+      nodenum: nodenum
+    }
+    const RE=/[\/:]+/g;
+    fname = "provenance_manifest_diff." + ark.replaceAll(RE, '_') + ".txt";
+    invoke_text(params, fname);
+  });
+
   $("button.storage-get-augmented-manifest").on("click", function(){
     var ark = $(this).attr("data-ark");
     var nodenum = $(this).attr("data-node-num");

src-colladmin/web/storeObjectNodes.html

@@ -166,6 +166,22 @@ <h2>Object Node Actions</h2>
               class="colladmin storage-get-manifest-yaml"
             >Get Storage Manifest Yaml</button>
 
+            <button 
+              data-ark="{{ark}}"
+              data-node-num="{{number}}"
+              data-version="{{version}}"
+              title="Get Manifest Yaml"
+              class="colladmin storage-get-provenance-yaml"
+            >Get Storage Provenance Yaml</button>
+
+            <button 
+              data-ark="{{ark}}"
+              data-node-num="{{number}}"
+              data-version="{{version}}"
+              title="Get Manifest Yaml"
+              class="colladmin storage-get-provenance-diff"
+            >Get Storage Provenance Diff</button>
+
             <button 
               data-ark="{{ark}}"
               data-node-num="{{number}}"