change each __type__ in catalog to full qualified name rather than snake. read and write to a registry that just maps qualified-class-name to itself. toward removal of registry

Signed-off-by: dafnapension <[email protected]>

Author: dafnapension

docs/catalog.py

@@ -50,8 +50,8 @@ def imports_to_syntax_highlighted_html(subtypes: List[str])-> str:
         return ""
     module_to_class_names = defaultdict(list)
     for subtype in subtypes:
-        subtype_class = Artifact._class_register.get(subtype)
-        module_to_class_names[subtype_class.__module__].append(subtype_class.__name__)
+        (module, class_name) = Artifact.get_module_class(subtype)
+        module_to_class_names[module].append(class_name)
 
     imports_txt = ""
     for modu in sorted(module_to_class_names.keys()):
@@ -148,19 +148,18 @@ def recursive_search(d):
 
 @lru_cache(maxsize=None)
 def artifact_type_to_link(artifact_type):
-    artifact_class = Artifact._class_register.get(artifact_type)
-    type_class_name = artifact_class.__name__
-    artifact_class_id = f"{artifact_class.__module__}.{type_class_name}"
-    return f'<a class="reference internal" href="../{artifact_class.__module__}.html#{artifact_class_id}" title="{artifact_class_id}"><code class="xref py py-class docutils literal notranslate"><span class="pre">{type_class_name}</span></code></a>'
+    artifact_module, artifact_class_name = Artifact.get_module_class(artifact_type)
+    return f'<a class="reference internal" href="../{artifact_module}.html#{artifact_module}.{artifact_class_name}" title="{artifact_module}.{artifact_class_name}"><code class="xref py py-class docutils literal notranslate"><span class="pre">{artifact_class_name}</span></code></a>'
 
 
 # flake8: noqa: C901
+
+
 def make_content(artifact, label, all_labels):
-    artifact_type = artifact["__type__"]
-    artifact_class = Artifact._class_register.get(artifact_type)
-    type_class_name = artifact_class.__name__
-    catalog_id = label.replace("catalog.", "")
+    artifact_type = artifact["__type__"]    #qualified class name
+    artifact_class = Artifact.get_class_from_artifact_type(artifact_type)
 
+    catalog_id = label.replace("catalog.", "")
     result = ""
 
     if "__description__" in artifact and artifact["__description__"] is not None:
@@ -235,7 +234,7 @@ def make_content(artifact, label, all_labels):
     result += "    " + html_for_element + "\n"
 
     if artifact_class.__doc__:
-        explanation_str = f"Explanation about `{type_class_name}`"
+        explanation_str = f"Explanation about `{artifact_class.__name__}`"
         result += f"\n{explanation_str}\n"
         result += "+" * len(explanation_str) + "\n\n"
         result += artifact_class.__doc__ + "\n"

src/unitxt/artifact.py

@@ -6,6 +6,7 @@
 import re
 import warnings
 from abc import abstractmethod
+from importlib import import_module
 from typing import Any, Dict, List, Optional, Tuple, Union, final
 
 from .dataclass import (
@@ -22,7 +23,7 @@
     separate_inside_and_outside_square_brackets,
 )
 from .settings_utils import get_constants, get_settings
-from .text_utils import camel_to_snake_case, is_camel_case
+from .text_utils import is_camel_case
 from .type_utils import isoftype, issubtype
 from .utils import (
     artifacts_json_cache,
@@ -143,7 +144,7 @@ def get_closest_artifact_type(type):
 
 class UnrecognizedArtifactTypeError(ValueError):
     def __init__(self, type) -> None:
-        maybe_class = "".join(word.capitalize() for word in type.split("_"))
+        maybe_class = type.split(".")[-1]
         message = f"'{type}' is not a recognized artifact 'type'. Make sure a the class defined this type (Probably called '{maybe_class}' or similar) is defined and/or imported anywhere in the code executed."
         closest_artifact_type = get_closest_artifact_type(type)
         if closest_artifact_type is not None:
@@ -200,12 +201,21 @@ def verify_artifact_dict(cls, d):
             )
         if "__type__" not in d:
             raise MissingArtifactTypeError(d)
-        if not cls.is_registered_type(d["__type__"]):
-            raise UnrecognizedArtifactTypeError(d["__type__"])
+        # if not cls.is_registered_type(d["__type__"]):
+        #     raise UnrecognizedArtifactTypeError(d["__type__"])
 
     @classmethod
     def get_artifact_type(cls):
-        return camel_to_snake_case(cls.__name__)
+        if hasattr(cls, "__qualname__") and "." in cls.__qualname__:
+            return cls.__module__+"/"+cls.__qualname__
+        return cls.__module__+"."+cls.__name__
+
+    @classmethod
+    def get_module_class(cls, artifact_type:str):
+        if "/" in artifact_type:
+            return artifact_type.split("/")
+        return artifact_type.rsplit(".", 1)
+
 
     @classmethod
     def register_class(cls, artifact_class):
@@ -216,18 +226,16 @@ def register_class(cls, artifact_class):
             artifact_class.__name__
         ), f"Artifact class name must be legal camel case, got '{artifact_class.__name__}'"
 
-        snake_case_key = camel_to_snake_case(artifact_class.__name__)
-
-        if cls.is_registered_type(snake_case_key):
+        if cls.is_registered_type(cls.get_artifact_type()):
             assert (
-                str(cls._class_register[snake_case_key]) == str(artifact_class)
-            ), f"Artifact class name must be unique, '{snake_case_key}' already exists for {cls._class_register[snake_case_key]}. Cannot be overridden by {artifact_class}."
+                str(cls._class_register[cls.get_artifact_type()]) == cls.get_artifact_type()
+            ), f"Artifact class name must be unique, '{cls.get_artifact_type()}' is already registered as {cls._class_register[cls.get_artifact_type()]}. Cannot be overridden by {artifact_class}."
 
-            return snake_case_key
+            return cls.get_artifact_type()
 
-        cls._class_register[snake_case_key] = artifact_class
+        cls._class_register[cls.get_artifact_type()] = cls.get_artifact_type()   # for now, still maintain the registry from qualified to qualified
 
-        return snake_case_key
+        return cls.get_artifact_type()
 
     def __init_subclass__(cls, **kwargs):
         super().__init_subclass__(**kwargs)
@@ -247,12 +255,25 @@ def is_registered_type(cls, type: str):
 
     @classmethod
     def is_registered_class_name(cls, class_name: str):
-        snake_case_key = camel_to_snake_case(class_name)
-        return cls.is_registered_type(snake_case_key)
+        for k in cls._class_register:
+            _, artifact_class_name = cls.get_module_class(k)
+            if artifact_class_name == class_name:
+                return True
+        return False
 
     @classmethod
-    def is_registered_class(cls, clz: object):
-        return clz in set(cls._class_register.values())
+    def get_class_from_artifact_type(cls, type:str):
+        module_path, class_name = cls.get_module_class(type)
+        module = import_module(module_path)
+        if "." not in class_name:
+            return getattr(module, class_name)
+        class_name_components = class_name.split(".")
+        klass = getattr(module, class_name_components[0])
+        for i in range (1, len(class_name_components)):
+            klass = getattr(klass, class_name_components[i])
+        return klass
+
+
 
     @classmethod
     def _recursive_load(cls, obj):
@@ -267,9 +288,12 @@ def _recursive_load(cls, obj):
             pass
         if cls.is_artifact_dict(obj):
             cls.verify_artifact_dict(obj)
-            artifact_class = cls._class_register[obj.pop("__type__")]
-            obj = artifact_class.process_data_after_load(obj)
-            return artifact_class(**obj)
+            try:
+                artifact_class = cls.get_class_from_artifact_type(obj.pop("__type__"))
+                obj = artifact_class.process_data_after_load(obj)
+                return artifact_class(**obj)
+            except (ImportError, AttributeError) as e:
+                raise ImportError(obj) from e
 
         return obj
 
@@ -283,7 +307,7 @@ def from_dict(cls, d, overwrite_args=None):
     @classmethod
     def load(cls, path, artifact_identifier=None, overwrite_args=None):
         d = artifacts_json_cache(path)
-        if "__type__" in d and d["__type__"] == "artifact_link":
+        if "__type__" in d and d["__type__"].endswith("ArtifactLink"):
             cls.from_dict(d)  # for verifications and warnings
             catalog, artifact_rep, _ = get_catalog_name_and_args(name=d["to"])
             return catalog.get_with_overwrite(
@@ -347,7 +371,7 @@ def __post_init__(self):
 
     def _to_raw_dict(self):
         return {
-            "__type__": self.__type__,
+            "__type__": self.__class__.get_artifact_type(),
             **self.process_data_before_dump(self._init_dict),
         }
 

src/unitxt/catalog.py

@@ -128,6 +128,8 @@ def add_to_catalog(
     catalog_path: Optional[str] = None,
     verbose=True,
 ):
+    # print(artifact.__class__.__name__)
+    # print(artifact.__module__)
     reset_artifacts_json_cache()
     if catalog is None:
         if catalog_path is None:

src/unitxt/catalog/augmentors/augment_whitespace_prefix_and_suffix_task_input.json

@@ -1,5 +1,5 @@
 {
-    "__type__": "artifact_link",
+    "__type__": "unitxt.artifact.ArtifactLink",
     "to": "augmentors.text.whitespace_prefix_suffix",
     "__deprecated_msg__": "Artifact 'augmentors.augment_whitespace_prefix_and_suffix_task_input' is deprecated. Artifact 'augmentors.text.whitespace_prefix_suffix' will be instantiated instead. In future uses, please reference artifact 'augmentors.text.whitespace_prefix_suffix' directly."
 }

src/unitxt/catalog/augmentors/augment_whitespace_task_input.json

@@ -1,5 +1,5 @@
 {
-    "__type__": "artifact_link",
+    "__type__": "unitxt.artifact.ArtifactLink",
     "to": "augmentors.text.whitespace_prefix_suffix",
     "__deprecated_msg__": "Artifact 'augmentors.augment_whitespace_task_input' is deprecated. Artifact 'augmentors.text.whitespace_prefix_suffix' will be instantiated instead. In future uses, please reference artifact 'augmentors.text.whitespace_prefix_suffix' directly."
 }

src/unitxt/catalog/augmentors/image/grey_scale.json

@@ -1,3 +1,3 @@
 {
-    "__type__": "gray_scale"
+    "__type__": "unitxt.image_operators.GrayScale"
 }

src/unitxt/catalog/augmentors/image/grid_lines.json

@@ -1,3 +1,3 @@
 {
-    "__type__": "grid_lines"
+    "__type__": "unitxt.image_operators.GridLines"
 }

src/unitxt/catalog/augmentors/image/oldify.json

@@ -1,3 +1,3 @@
 {
-    "__type__": "oldify"
+    "__type__": "unitxt.image_operators.Oldify"
 }

src/unitxt/catalog/augmentors/image/white_noise.json

@@ -1,3 +1,3 @@
 {
-    "__type__": "pixel_noise"
+    "__type__": "unitxt.image_operators.PixelNoise"
 }

src/unitxt/catalog/augmentors/no_augmentation.json

@@ -1,3 +1,3 @@
 {
-    "__type__": "null_augmentor"
+    "__type__": "unitxt.augmentors.NullAugmentor"
 }