WIP: Feature/fcl workshop changes (#53)

* Rename all mentions of diagnostics --> instruments

* Create lens schema and remove from camera schema

* Run new jsonschema fmt --indentation 4

* Rename instruments to equipment

* Add lens to equipment list in equipment.schema.json

* Change all IDs to UUIDs explicitly and add configurationUUID to pulse.schema.json

* Add examples for camera and lens equipment

* Move 'comment' and 'pulseQuality' into 'pulseInformation'.

* move required fields to correct place.

* Add pre-pulse flag to signify fields that are set in the UI.

* Update heating to coil information and add required fields.

* Add back coolant-related fields

* Re-add fields

* Remove pre-pulse flag and add AC/DC logic to coilInformation.

* Make all coolInformation fields required.

* Refactor pulse schema: update date fields, rename coilInformation to heatingInformation, and adjust required properties.

* Add thermocouple summary fields.

* Add new JSON and JQ files for pulse and dataset mapping, and update schema definitions

* Refactor dataset and experiment schemas: remove unnecessary fields, update required properties, and add new schema for experiments.

* Remove superfluous additionalProperties from ukaea-dataset caught by the linter, and add an ID field

* Move ukaea-experiment placeholder to ukaea-schema

* Add sampleID to schema

* Fix dataset.jq to correctly reference pulseStart, comment, and experimentID; update sourceFolder and additional fields.

* First draft of mapping script.

* Move comment

* Fleshing out experiment bits.

* update urls

* This should (might) perform a one-to-many mapping so that a dataset gets produced for each diagnostic.

* Allow diagnostics to be a single instrument or a combination (expressed as an array).

* Fix typo in description of contactPerson property in experiment schema

* Fix pulse.schema.json required fields at the wrong level

* Fix typo in OutputCurrent, now passes linter and metaschema (minux x- prefix)

* Reformat the new files created by NC

---------

Co-authored-by: Nathan Cummings <nathan.cummings@hotmail.co.uk>

Author: adam-parker1

scicat-mapping/examples/README.md metacat-mapping/examples/README.mdscicat-mapping/examples/README.md renamed to metacat-mapping/examples/README.md

@@ -0,0 +1,53 @@
+import json
+from pathlib import Path
+
+import jq
+import requests
+from jsonschema import validate, ValidationError
+
+
+# Read in pulse json file
+with open("pulse.json") as file:
+    pulse_data = json.load(file)
+
+pulse_directory = Path(
+    f"/mnt/HIVE/{pulse_data['experimentID']}/{pulse_data['sampleID']}/{pulse_data['pulseID']}"
+)
+
+# Query metacat using experimentID to get contactEmail, ownerGroup and owner
+contact_email = "<contactEmail>"
+owner_group = "<ownerGroup>"
+owner = "<owner>"
+
+# Map data to ukaea-dataset schema using dataset.jq
+jq_url = "https://ukaea.github.io/ukaea-metadata/metacat-mapping/hive/dataset.jq"
+jq_spec = requests.get(jq_url).text
+
+hive_pulse = jq.compile(jq_spec).input(pulse_data).all()
+
+for dataset in hive_pulse:
+    dataset["contactEmail"] = contact_email
+    dataset["ownerGroup"] = owner_group
+    dataset["owner"] = owner
+
+    # Validate against ukaea-dataset.schema.json
+    schema_url = (
+        "https://ukaea.github.io/ukaea-metadata/ukaea-schema/ukaea-dataset.schema.json"
+    )
+    schema = requests.get(schema_url).json()
+
+    try:
+        validate(instance=dataset, schema=schema)
+    except ValidationError as e:
+        print(f"Validation error: {e.message}")
+
+    output_file = (
+        pulse_directory
+        / Path(f"{dataset['diagnosticID']}")
+        / f"{dataset['diagnosticID']}-metadata.json"
+    )
+
+    output_file.parent.mkdir(parents=True, exist_ok=True)
+
+    with open(output_file, "w") as file:
+        json.dump(dataset, file, indent=4)

scicat-mapping/examples/jq_spec.txt metacat-mapping/examples/jq_spec.txtscicat-mapping/examples/jq_spec.txt renamed to metacat-mapping/examples/jq_spec.txt

@@ -0,0 +1,11 @@
+. as $parent
+| .diagnostics[]
+| {
+      "creationTime" : $parent.pulseStart,
+      "type": "raw",
+      "sourceFolder": "/mnt/HIVE/\(.experimentID)/\(.sampleID)/\(.pulseID)",
+      "description": $parent.comment,
+      "experimentID": $parent.experimentID,
+      "diagnosticID": .diagnosticID,
+      "additional": (del($parent.comment, $parent.experimentID, $parent.diagnostics))
+  }

…at-mapping/examples/standard-mapped.json …at-mapping/examples/standard-mapped.jsonscicat-mapping/examples/standard-mapped.json renamed to metacat-mapping/examples/standard-mapped.json scicat-mapping/examples/standard-mapped.json renamed to metacat-mapping/examples/standard-mapped.json

@@ -0,0 +1,25 @@
+import json
+
+import jq
+import requests
+from jsonschema import validate, ValidationError
+
+
+# Read in experiment json file
+with open("experiment.json") as file:
+    experiment_data = json.load(file)
+
+# Map data to ukaea-dataset schema using dataset.jq
+jq_url = "https://ukaea.github.io/ukaea-metadata/metacat-mapping/hive/experiment.jq"
+jq_spec = requests.get(jq_url).text
+
+hive_experiment = jq.compile(jq_spec).input(experiment_data).first()
+
+# Validate against ukaea-dataset.schema.json
+schema_url = "https://ukaea.github.io/ukaea-metadata/ukaea-schema/ukaea-experiment.schema.json"
+schema = requests.get(schema_url).json()
+
+try:
+    validate(instance=hive_experiment, schema=schema)
+except ValidationError as e:
+    print(f"Validation error: {e.message}")

metacat-mapping/hive/dataset-mapping.py

@@ -0,0 +1,9 @@
+{
+    "facility": "HIVE",
+    "facilityExperimentID": .ID,
+    "startDate": .startDate,
+    "endDate": .endDate?,
+    "description": .description,
+    "leadInvestigator": .leadInvestigator,
+    "customer": .customer?
+}

metacat-mapping/hive/dataset.jq

@@ -1,11 +1,11 @@
 {
     "$schema": "https://json-schema.org/draft/2020-12/schema",
-    "$id": "https://w3id.org/fusion/schemas/1.0.0/diagnostics/camera",
-    "description": "Metadata to describe camera diagnostics",
+    "$id": "https://w3id.org/fusion/schemas/1.0.0/equipment/camera",
+    "description": "Metadata to describe camera equipment",
     "type": "object",
-    "required": [ "cameraInformation", "lensInformation" ],
+    "required": [ "deviceInformation" ],
     "properties": {
-        "cameraInformation": {
+        "deviceInformation": {
             "type": "object",
             "required": [ "make", "model", "resolution" ],
             "properties": {
@@ -39,49 +39,6 @@
                 }
             }
         },
-        "lensInformation": {
-            "type": "object",
-            "required": [ "make", "model", "aperture", "focalLength" ],
-            "properties": {
-                "make": {
-                    "type": "string",
-                    "description": "The manufacturer or brand of the lens (e.g., Canon, Nikon, Sigma, Tamron)."
-                },
-                "model": {
-                    "type": "string",
-                    "description": "The specific product designation given by the manufacturer, identifying the lens type or version (e.g., EF 24–70mm f/2.8L II USM)."
-                },
-                "serialNumber": {
-                    "type": "string",
-                    "description": "A unique identifier assigned by the manufacturer to distinguish an individual lens unit."
-                },
-                "aperture": {
-                    "type": "string",
-                    "description": "The maximum aperture size of the lens, usually expressed as an f-number (e.g., f/1.8, f/2.8)."
-                },
-                "focalLength": {
-                    "type": "number",
-                    "unit": "qudt:MilliM",
-                    "description": "The focal length or range of the lens in millimeters, determining its angle of view (e.g., 50)."
-                },
-                "fieldOfView": {
-                    "type": "object",
-                    "required": [ "x", "y" ],
-                    "properties": {
-                        "x": {
-                            "description": "The field of view in pixels, represented as width.",
-                            "type": "integer",
-                            "unit": "qudt:px"
-                        },
-                        "y": {
-                            "description": "The field of view in pixels, represented as height.",
-                            "type": "integer",
-                            "unit": "qudt:px"
-                        }
-                    }
-                }
-            }
-        },
         "captureSettings": {
             "type": "object",
             "required": [ "imageAcquisitionRate", "imageNoise" ],
@@ -103,4 +60,4 @@
             }
         }
     }
-}
+}