change each __type__ in catalog to full qualified name rather than snake, and remove _class_register altogether

Signed-off-by: dafnapension <[email protected]>

Author: dafnapension

docs/catalog.py

@@ -50,8 +50,8 @@ def imports_to_syntax_highlighted_html(subtypes: List[str])-> str:
         return ""
     module_to_class_names = defaultdict(list)
     for subtype in subtypes:
-        subtype_class = Artifact._class_register.get(subtype)
-        module_to_class_names[subtype_class.__module__].append(subtype_class.__name__)
+        (module, class_name) = Artifact.get_module_class(subtype)
+        module_to_class_names[module].append(class_name)
 
     imports_txt = ""
     for modu in sorted(module_to_class_names.keys()):
@@ -101,31 +101,6 @@ def custom_walk(top):
             yield entry
 
 
-def all_subtypes_of_artifact(artifact):
-    if (
-        artifact is None
-        or isinstance(artifact, str)
-        or isinstance(artifact, bool)
-        or isinstance(artifact, int)
-        or isinstance(artifact, float)
-    ):
-        return []
-    if isinstance(artifact, list):
-        to_return = []
-        for art in artifact:
-            to_return.extend(all_subtypes_of_artifact(art))
-        return to_return
-    # artifact is a dict
-    to_return = []
-    for key, value in artifact.items():
-        if isinstance(value, str):
-            if key == "__type__":
-                to_return.append(value)
-        else:
-            to_return.extend(all_subtypes_of_artifact(value))
-    return to_return
-
-
 def get_all_type_elements(nested_dict):
     type_elements = set()
 
@@ -148,19 +123,18 @@ def recursive_search(d):
 
 @lru_cache(maxsize=None)
 def artifact_type_to_link(artifact_type):
-    artifact_class = Artifact._class_register.get(artifact_type)
-    type_class_name = artifact_class.__name__
-    artifact_class_id = f"{artifact_class.__module__}.{type_class_name}"
-    return f'<a class="reference internal" href="../{artifact_class.__module__}.html#{artifact_class_id}" title="{artifact_class_id}"><code class="xref py py-class docutils literal notranslate"><span class="pre">{type_class_name}</span></code></a>'
+    artifact_module, artifact_class_name = Artifact.get_module_class(artifact_type)
+    return f'<a class="reference internal" href="../{artifact_module}.html#{artifact_module}.{artifact_class_name}" title="{artifact_module}.{artifact_class_name}"><code class="xref py py-class docutils literal notranslate"><span class="pre">{artifact_class_name}</span></code></a>'
 
 
 # flake8: noqa: C901
+
+
 def make_content(artifact, label, all_labels):
-    artifact_type = artifact["__type__"]
-    artifact_class = Artifact._class_register.get(artifact_type)
-    type_class_name = artifact_class.__name__
-    catalog_id = label.replace("catalog.", "")
+    artifact_type = artifact["__type__"]    #qualified class name
+    artifact_class = Artifact.get_class_from_artifact_type(artifact_type)
 
+    catalog_id = label.replace("catalog.", "")
     result = ""
 
     if "__description__" in artifact and artifact["__description__"] is not None:
@@ -203,23 +177,16 @@ def make_content(artifact, label, all_labels):
         )
 
     for type_name in type_elements:
-        # source = f'<span class="nt">__type__</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">{type_name}</span>'
-        source = f'<span class="n">__type__{type_name}</span><span class="p">'
-        target = artifact_type_to_link(type_name)
-        html_for_dict = html_for_dict.replace(
-            source,
-            f'<span class="n" STYLE="font-size:108%">{target}</span><span class="p">'
-            # '<span class="nt">&quot;type&quot;</span><span class="p">:</span><span class="w"> </span>'
-            # + target,
-        )
-
-    pattern = r'(<span class="nt">)&quot;(.*?)&quot;(</span>)'
+        artifact_module, artifact_class_name = Artifact.get_module_class(type_name)
+        pattern = re.compile(f'<span class="n">__type__(.*?)<span class="n">{artifact_class_name}</span>')
+        repl = '<span class="n" STYLE="font-size:108%">'+artifact_type_to_link(type_name)+"</span>"
+        html_for_dict = pattern.sub(repl, html_for_dict)
 
+    # pattern = r'(<span class="nt">)&quot;(.*?)&quot;(</span>)'
     # Replacement function
-    html_for_dict = re.sub(pattern, r"\1\2\3", html_for_dict)
+    # html_for_dict = re.sub(pattern, r"\1\2\3", html_for_dict)
 
-    subtypes = all_subtypes_of_artifact(artifact)
-    subtypes = list(set(subtypes))
+    subtypes = type_elements
     subtypes.remove(artifact_type)  # this was already documented
     html_for_imports = imports_to_syntax_highlighted_html(subtypes)
 
@@ -235,13 +202,13 @@ def make_content(artifact, label, all_labels):
     result += "    " + html_for_element + "\n"
 
     if artifact_class.__doc__:
-        explanation_str = f"Explanation about `{type_class_name}`"
+        explanation_str = f"Explanation about `{artifact_class.__name__}`"
         result += f"\n{explanation_str}\n"
         result += "+" * len(explanation_str) + "\n\n"
         result += artifact_class.__doc__ + "\n"
 
     for subtype in subtypes:
-        subtype_class = Artifact._class_register.get(subtype)
+        subtype_class = Artifact.get_class_from_artifact_type(subtype)
         subtype_class_name = subtype_class.__name__
         if subtype_class.__doc__:
             explanation_str = f"Explanation about `{subtype_class_name}`"

docs/conf.py

@@ -116,7 +116,6 @@ def autodoc_skip_member(app, what, name, obj, would_skip, options):
         class_name = obj.__qualname__.split(".")[0]
         if (
             class_name
-            and Artifact.is_registered_class_name(class_name)
             and class_name != name
         ):
             return True

examples/evaluate_rag_using_binary_llm_as_judge.py

@@ -1,7 +1,7 @@
 from unitxt import get_logger
 from unitxt.api import evaluate, load_dataset
 from unitxt.blocks import TaskCard
-from unitxt.inference import WMLInferenceEngine
+from unitxt.inference import WMLInferenceEngineGeneration
 from unitxt.loaders import LoadFromDictionary
 from unitxt.templates import TemplatesDict
 
@@ -77,7 +77,7 @@
 
     # Infer using flan t5 xl using wml
     model_name = "google/flan-t5-xl"
-    model = WMLInferenceEngine(model_name=model_name, max_new_tokens=32)
+    model = WMLInferenceEngineGeneration(model_name=model_name, max_new_tokens=32)
     predictions = model(test_dataset)
 
     # Evaluate the generated predictions using the selected metrics

prepare/engines/ibm_wml/llama3.py

@@ -1,11 +1,11 @@
 from unitxt.catalog import add_to_catalog
-from unitxt.inference import WMLInferenceEngine
+from unitxt.inference import WMLInferenceEngineGeneration
 
 model_list = ["meta-llama/llama-3-70b-instruct"]
 
 for model in model_list:
     model_label = model.split("/")[1].replace("-", "_").replace(".", ",").lower()
-    inference_model = WMLInferenceEngine(
+    inference_model = WMLInferenceEngineGeneration(
         model_name=model, max_new_tokens=2048, random_seed=42
     )
     add_to_catalog(inference_model, f"engines.ibm_wml.{model_label}", overwrite=True)

prepare/metrics/custom_f1.py

@@ -433,4 +433,7 @@ class NERWithoutClassReporting(NER):
     global_target=global_target,
 )
 
-add_to_catalog(metric, "metrics.ner", overwrite=True)
+if __name__ == "__main__" or __name__ == "custom_f1":
+    # because a class is defined in this module, need to not add_to_catalog just for importing that module in order to retrieve the defined class
+    # and need to prepare for case when this module is run directly from python (__main__) or, for example, from test_preparation (custom_f1)
+    add_to_catalog(metric, "metrics.ner", overwrite=True)

prepare/metrics/llm_as_judge/conversation_groundedness.py

@@ -3,15 +3,15 @@
 from unitxt import add_to_catalog
 from unitxt.inference import (
     IbmGenAiInferenceEngine,
-    IbmGenAiInferenceEngineParams,
+    IbmGenAiInferenceEngineParamsMixin,
 )
 from unitxt.llm_as_judge import LLMAsJudge
 from unitxt.metrics import (
     RandomForestMetricsEnsemble,
 )
 
 platform = "ibm_gen_ai"
-gen_params = IbmGenAiInferenceEngineParams(max_new_tokens=256)
+gen_params = IbmGenAiInferenceEngineParamsMixin(max_new_tokens=256)
 
 config_filepath = "prepare/metrics/llm_as_judge/ensemble_grounded_v1.json"
 

prepare/metrics/llm_as_judge/conversation_idk.py

@@ -2,7 +2,7 @@
 from unitxt.inference import (
     GenericInferenceEngine,
     IbmGenAiInferenceEngine,
-    IbmGenAiInferenceEngineParams,
+    IbmGenAiInferenceEngineParamsMixin,
 )
 from unitxt.llm_as_judge import LLMAsJudge
 
@@ -13,7 +13,7 @@
         "model_name": "llama370binstruct",
         "inference_model": IbmGenAiInferenceEngine(
             model_name="meta-llama/llama-3-70b-instruct",
-            parameters=IbmGenAiInferenceEngineParams(max_new_tokens=256),
+            parameters=IbmGenAiInferenceEngineParamsMixin(max_new_tokens=256),
         ),
     },
     "generic_inference_engine": {

prepare/metrics/llm_as_judge/conversation_topicality.py

@@ -3,15 +3,15 @@
 from unitxt import add_to_catalog
 from unitxt.inference import (
     IbmGenAiInferenceEngine,
-    IbmGenAiInferenceEngineParams,
+    IbmGenAiInferenceEngineParamsMixin,
 )
 from unitxt.llm_as_judge import LLMAsJudge
 from unitxt.metrics import (
     RandomForestMetricsEnsemble,
 )
 
 platform = "ibm_gen_ai"
-gen_params = IbmGenAiInferenceEngineParams(max_new_tokens=256)
+gen_params = IbmGenAiInferenceEngineParamsMixin(max_new_tokens=256)
 
 config_filepath = "prepare/metrics/llm_as_judge/ensemble_topicality_v1.json"
 

prepare/metrics/llm_as_judge/pairwise_rating/llama_3_arena_hard_template.py

@@ -3,7 +3,7 @@
     CrossProviderInferenceEngine,
     GenericInferenceEngine,
     IbmGenAiInferenceEngine,
-    WMLInferenceEngine,
+    WMLInferenceEngineGeneration,
 )
 from unitxt.llm_as_judge import LLMAsJudge
 
@@ -15,7 +15,7 @@
 ]
 
 inference_engines = [
-    ("ibm_wml", WMLInferenceEngine),
+    ("ibm_wml", WMLInferenceEngineGeneration),
     ("ibm_genai", IbmGenAiInferenceEngine),
     ("generic_engine", GenericInferenceEngine),
 ]