feat(shared-data): json protocol schema 8 (#13848)

Add JSON protocol schema 8, which is similar in semantic concept to json
protocol schema 7 but
- Requires user specification of the schema for the command List
This allows json protocols to switch which commands they're interning
without requiring further bumps to the protocol schema
- Does the same thing for command annotations, liquids, and labware
For the same reasons.

This should be a big step forward in flexibility for our protocol schemas.

Unfortunately, these new structures cannot be represented in jsonschema
because the schema to which components adhere is specified by the value
of another key in the document; jsonschema relations must be computable
at schema parse/verification time. We could solve this by using json hyper
schema but this seems a little much. Instead, there's a new js module for
parsing schema v8 that we'll introduce to the rest of the stack over time.
---------

Co-authored-by: Brian Cooper <[email protected]>

.github/workflows/robot-server-lint-test.yaml

@@ -52,7 +52,7 @@ defaults:
 jobs:
   lint-test:
     name: 'robot server package linting and tests'
-    timeout-minutes: 20
+    timeout-minutes: 40
     runs-on: 'ubuntu-22.04'
     strategy:
       matrix:

api/src/opentrons/protocol_reader/file_format_validator.py

@@ -10,7 +10,9 @@
 from opentrons_shared_data.protocol.models import (
     ProtocolSchemaV6 as JsonProtocolV6,
     ProtocolSchemaV7 as JsonProtocolV7,
+    ProtocolSchemaV8 as JsonProtocolV8,
 )
+from opentrons_shared_data.errors.exceptions import PythonException
 
 from opentrons.protocols.models import JsonProtocol as JsonProtocolUpToV5
 
@@ -51,7 +53,9 @@ def validate_sync() -> None:
             LabwareDefinition.parse_obj(info.unvalidated_json)
         except PydanticValidationError as e:
             raise FileFormatValidationError(
-                f"{info.original_file.name} could not be read as a labware definition."
+                message=f"{info.original_file.name} could not be read as a labware definition.",
+                detail={"kind": "bad-labware-definition"},
+                wrapping=[PythonException(e)],
             ) from e
 
     await anyio.to_thread.run_sync(validate_sync)
@@ -60,15 +64,19 @@ def validate_sync() -> None:
 async def _validate_json_protocol(info: IdentifiedJsonMain) -> None:
     def validate_sync() -> None:
         try:
-            if info.schema_version == 7:
+            if info.schema_version == 8:
+                JsonProtocolV8.parse_obj(info.unvalidated_json)
+            elif info.schema_version == 7:
                 JsonProtocolV7.parse_obj(info.unvalidated_json)
             elif info.schema_version == 6:
                 JsonProtocolV6.parse_obj(info.unvalidated_json)
             else:
                 JsonProtocolUpToV5.parse_obj(info.unvalidated_json)
         except PydanticValidationError as e:
             raise FileFormatValidationError(
-                f"{info.original_file.name} could not be read as a JSON protocol."
+                message=f"{info.original_file.name} could not be read as a JSON protocol.",
+                detail={"kind": "bad-json-protocol"},
+                wrapping=[PythonException(e)],
             ) from e
 
     await anyio.to_thread.run_sync(validate_sync)

api/src/opentrons/protocol_reader/file_identifier.py

@@ -2,11 +2,12 @@
 
 import json
 from dataclasses import dataclass
-from typing import Any, Dict, Sequence, Union
+from typing import Any, Dict, Sequence, Union, Optional
 
 import anyio
 
 from opentrons_shared_data.robot.dev_types import RobotType
+from opentrons_shared_data.errors.exceptions import EnumeratedError, PythonException
 
 from opentrons.protocols.api_support.definitions import MAX_SUPPORTED_VERSION
 from opentrons.protocols.api_support.types import APIVersion
@@ -96,6 +97,23 @@ class IdentifiedData:
 class FileIdentificationError(ProtocolFilesInvalidError):
     """Raised when FileIdentifier detects an invalid file."""
 
+    def __init__(
+        self,
+        message: str,
+        detail: Optional[Dict[str, Any]] = None,
+        wrapping: Optional[Sequence[EnumeratedError]] = None,
+        only_message: bool = False,
+    ) -> None:
+        super().__init__(message=message)
+        self._only_message = only_message
+
+    def __str__(self) -> str:
+        """Special stringifier to conform to expecations about python protocol errors."""
+        if self._only_message:
+            return self.message
+        else:
+            return super().__str__()
+
 
 class FileIdentifier:
     """File identifier interface."""
@@ -128,7 +146,8 @@ async def _identify(
         return IdentifiedData(original_file=file)
     else:
         raise FileIdentificationError(
-            f"{file.name} has an unrecognized file extension."
+            message=f"{file.name} has an unrecognized file extension.",
+            detail={"type": "bad-file-extension", "file": file.name},
         )
 
 
@@ -139,7 +158,9 @@ async def _analyze_json(
         json_contents = await anyio.to_thread.run_sync(json.loads, json_file.contents)
     except json.JSONDecodeError as e:
         raise FileIdentificationError(
-            f"{json_file.name} is not valid JSON. {str(e)}"
+            message=f"{json_file.name} is not valid JSON. {str(e)}",
+            detail={"type": "invalid-json", "file": json_file.name},
+            wrapping=[PythonException(e)],
         ) from e
 
     if _json_seems_like_labware(json_contents):
@@ -154,7 +175,8 @@ async def _analyze_json(
         )
     else:
         raise FileIdentificationError(
-            f"{json_file.name} is not a known Opentrons format."
+            message=f"{json_file.name} is not a known Opentrons format.",
+            detail={"type": "no-schema-match", "file": json_file.name},
         )
 
 
@@ -182,19 +204,34 @@ def _analyze_json_protocol(
         schema_version = json_contents["schemaVersion"]
         robot_type = json_contents["robot"]["model"]
     except KeyError as e:
-        raise FileIdentificationError(error_message) from e
+        raise FileIdentificationError(
+            message=error_message,
+            detail={"kind": "missing-json-metadata", "missing-key": str(e)},
+            wrapping=[PythonException(e)],
+        ) from e
 
     # todo(mm, 2022-12-22): A JSON protocol file's metadata is not quite just an
     # arbitrary dict: its fields are supposed to follow a schema. Should we validate
     # this metadata against that schema instead of doing this simple isinstance() check?
     if not isinstance(metadata, dict):
-        raise FileIdentificationError(error_message)
+        raise FileIdentificationError(
+            message=error_message, detail={"kind": "json-metadata-not-object"}
+        )
 
     if not isinstance(schema_version, int):
-        raise FileIdentificationError(error_message)
+        raise FileIdentificationError(
+            message=error_message,
+            detail={
+                "kind": "json-schema-version-not-int",
+                "schema-version": schema_version,
+            },
+        )
 
     if robot_type not in ("OT-2 Standard", "OT-3 Standard"):
-        raise FileIdentificationError(error_message)
+        raise FileIdentificationError(
+            message=error_message,
+            detail={"kind": "bad-json-protocol-robot-type", "robot-type": robot_type},
+        )
 
     return IdentifiedJsonMain(
         original_file=original_file,
@@ -216,7 +253,12 @@ def _analyze_python_protocol(
             python_parse_mode=python_parse_mode,
         )
     except MalformedPythonProtocolError as e:
-        raise FileIdentificationError(e.short_message) from e
+        raise FileIdentificationError(
+            message=e.short_message,
+            detail={"kind": "malformed-python-protocol"},
+            wrapping=[PythonException(e)],
+            only_message=True,
+        ) from e
 
     # We know this should never be a JsonProtocol. Help out the type-checker.
     assert isinstance(
@@ -225,9 +267,13 @@ def _analyze_python_protocol(
 
     if parsed.api_level > MAX_SUPPORTED_VERSION:
         raise FileIdentificationError(
-            f"API version {parsed.api_level} is not supported by this "
-            f"robot software. Please either reduce your requested API "
-            f"version or update your robot."
+            message=(
+                f"API version {parsed.api_level} is not supported by this "
+                f"robot software. Please either reduce your requested API "
+                f"version or update your robot."
+            ),
+            detail={"kind": "future-api-version", "api-version": str(parsed.api_level)},
+            only_message=True,
         )
 
     return IdentifiedPythonMain(

api/src/opentrons/protocol_reader/protocol_files_invalid_error.py

@@ -1,5 +1,6 @@
 # noqa: D100
+from opentrons_shared_data.errors.exceptions import InvalidProtocolData
 
 
-class ProtocolFilesInvalidError(ValueError):
+class ProtocolFilesInvalidError(InvalidProtocolData):
     """Raised when the input to a ProtocolReader is not a well-formed protocol."""

api/src/opentrons/protocol_reader/role_analyzer.py

@@ -40,7 +40,9 @@ def analyze(files: Sequence[IdentifiedFile]) -> RoleAnalysis:
         This validates that there is exactly one main protocol file.
         """
         if len(files) == 0:
-            raise RoleAnalysisError("No files were provided.")
+            raise RoleAnalysisError(
+                message="No files were provided.", detail={"kind": "no-files"}
+            )
 
         main_file_candidates: List[Union[IdentifiedJsonMain, IdentifiedPythonMain]] = []
         labware_files: List[IdentifiedLabwareDefinition] = []
@@ -57,17 +59,22 @@ def analyze(files: Sequence[IdentifiedFile]) -> RoleAnalysis:
         if len(main_file_candidates) == 0:
             if len(files) == 1:
                 raise RoleAnalysisError(
-                    f'"{files[0].original_file.name}" is not a valid protocol file.'
+                    message=f'"{files[0].original_file.name}" is not a valid protocol file.',
+                    detail={"kind": "protocol-does-not-have-main"},
                 )
             else:
                 file_list = ", ".join(f'"{f.original_file.name}"' for f in files)
-                raise RoleAnalysisError(f"No valid protocol file found in {file_list}.")
+                raise RoleAnalysisError(
+                    message=f"No valid protocol file found in {file_list}.",
+                    detail={"kind": "protocol-does-not-have-main"},
+                )
         elif len(main_file_candidates) > 1:
             file_list = ", ".join(
                 f'"{f.original_file.name}"' for f in main_file_candidates
             )
             raise RoleAnalysisError(
-                f"Could not pick single main file from {file_list}."
+                message=f"Could not pick single main file from {file_list}.",
+                detail={"kind": "multiple-main-candidates"},
             )
         else:
             main_file = main_file_candidates[0]

api/src/opentrons/protocol_runner/json_file_reader.py

@@ -3,7 +3,12 @@
 
 from opentrons_shared_data.protocol.models.protocol_schema_v6 import ProtocolSchemaV6
 from opentrons_shared_data.protocol.models.protocol_schema_v7 import ProtocolSchemaV7
-from opentrons.protocol_reader import ProtocolSource, JsonProtocolConfig
+from opentrons_shared_data.protocol.models.protocol_schema_v8 import ProtocolSchemaV8
+from opentrons.protocol_reader import (
+    ProtocolSource,
+    JsonProtocolConfig,
+    ProtocolFilesInvalidError,
+)
 
 
 class JsonFileReader:
@@ -12,12 +17,31 @@ class JsonFileReader:
     @staticmethod
     def read(
         protocol_source: ProtocolSource,
-    ) -> Union[ProtocolSchemaV6, ProtocolSchemaV7]:
+    ) -> Union[ProtocolSchemaV6, ProtocolSchemaV7, ProtocolSchemaV8]:
         """Read and parse file into a JsonProtocol model."""
-        if (
-            isinstance(protocol_source.config, JsonProtocolConfig)
-            and protocol_source.config.schema_version == 6
-        ):
+        name = protocol_source.metadata.get("name", protocol_source.main_file.name)
+        if not isinstance(protocol_source.config, JsonProtocolConfig):
+            raise ProtocolFilesInvalidError(
+                message=f"Cannot execute {name} as a JSON protocol",
+                detail={
+                    "kind": "non-json-file-in-json-file-reader",
+                    "metadata-name": protocol_source.metadata.get("name"),
+                    "file-name": protocol_source.main_file.name,
+                },
+            )
+        if protocol_source.config.schema_version == 6:
             return ProtocolSchemaV6.parse_file(protocol_source.main_file)
-        else:
+        elif protocol_source.config.schema_version == 7:
             return ProtocolSchemaV7.parse_file(protocol_source.main_file)
+        elif protocol_source.config.schema_version == 8:
+            return ProtocolSchemaV8.parse_file(protocol_source.main_file)
+        else:
+            raise ProtocolFilesInvalidError(
+                message=f"{name} is a JSON protocol v{protocol_source.config.schema_version} which this robot cannot execute",
+                detail={
+                    "kind": "schema-version-unknown",
+                    "requested-schema-version": protocol_source.config.schema_version,
+                    "minimum-handled-schema-version": "6",
+                    "maximum-handled-shcema-version": "8",
+                },
+            )

api/src/opentrons/protocol_runner/json_translator.py

@@ -8,7 +8,10 @@
     protocol_schema_v6,
     ProtocolSchemaV7,
     protocol_schema_v7,
+    ProtocolSchemaV8,
+    protocol_schema_v8,
 )
+from opentrons_shared_data import command as command_schema
 
 from opentrons.types import MountType
 from opentrons.protocol_engine import (
@@ -163,7 +166,11 @@ def _translate_v7_pipette_command(
 
 
 def _translate_simple_command(
-    command: Union[protocol_schema_v6.Command, protocol_schema_v7.Command]
+    command: Union[
+        protocol_schema_v6.Command,
+        protocol_schema_v7.Command,
+        protocol_schema_v8.Command,
+    ]
 ) -> pe_commands.CommandCreate:
     dict_command = command.dict(exclude_none=True)
 
@@ -208,13 +215,15 @@ def translate_liquids(
 
     def translate_commands(
         self,
-        protocol: Union[ProtocolSchemaV7, ProtocolSchemaV6],
+        protocol: Union[ProtocolSchemaV8, ProtocolSchemaV7, ProtocolSchemaV6],
     ) -> List[pe_commands.CommandCreate]:
         """Takes json protocol and translates commands->protocol engine commands."""
         if isinstance(protocol, ProtocolSchemaV6):
             return self._translate_v6_commands(protocol)
-        else:
+        elif isinstance(protocol, ProtocolSchemaV7):
             return self._translate_v7_commands(protocol)
+        else:
+            return self._translate_v8_commands(protocol)
 
     def _translate_v6_commands(
         self,
@@ -244,3 +253,18 @@ def _translate_v7_commands(
             translated_obj = _translate_simple_command(command)
             commands_list.append(translated_obj)
         return commands_list
+
+    def _translate_v8_commands(
+        self, protocol: ProtocolSchemaV8
+    ) -> List[pe_commands.CommandCreate]:
+        """Translate commands in json protocol schema v8, which might be of different command schemas."""
+        command_schema_ref = protocol.commandSchemaId
+        # these calls will raise if the command schema version is invalid or unknown
+        command_schema_version = command_schema.schema_version_from_ref(
+            command_schema_ref
+        )
+        command_schema_string = command_schema.load_schema_string(  # noqa: F841
+            command_schema_version
+        )
+
+        return [_translate_simple_command(command) for command in protocol.commands]