Fix erroneous prompts in evaluation tasks (and clean some json-schema-wise) (#1920)

* cast None to str, to comply with json schema

Signed-off-by: dafnapension <[email protected]>

* fix template and tasks of evaluation

Signed-off-by: dafnapension <[email protected]>

---------

Signed-off-by: dafnapension <[email protected]>
Co-authored-by: Elron Bandel <[email protected]>

Author: dafnapension

prepare/cards/coedit.py

@@ -106,8 +106,8 @@
         Shuffle(page_size=sys.maxsize),
         "splitters.small_no_test",
         Split(field="src", by=": "),
+        Copy(field="src/0", to_field="instance_instruction"),
         Slice(field="src", start=1),
-        Copy(field="src/0", to_field="instruction"),
         Join(field="src", by=": "),
         ListFieldValues(
             fields=["tgt", "src"],

prepare/cards/hh_rlhf.py

@@ -29,7 +29,7 @@
             fields={
                 "input_type": "dialog",
                 "output_type": "response",
-                "instruction": "Respond the following dialog in an helpful and harmfull way.",
+                "instance_instruction": "Respond the following dialog in an helpful and harmless way.",
             }
         ),
         IndexOf(

prepare/cards/summarize_from_human_feedback.py

@@ -6,6 +6,7 @@
     TaskCard,
 )
 from unitxt.catalog import add_to_catalog
+from unitxt.operators import Cast
 from unitxt.test_utils.card import test_card
 
 card = TaskCard(
@@ -24,12 +25,13 @@
                 "summaries/*/text": "choices",
             }
         ),
+        Cast(field="input", to="str"),
         Rename(field_to_field={"choice": "output_choice"}),
         Set(
             fields={
                 "input_type": "post",
                 "output_type": "summary",
-                "instruction": "Summarize the following post",
+                "instance_instruction": "Summarize the following post",
             }
         ),
     ],

prepare/tasks/evaluation.py

@@ -10,7 +10,7 @@
             "input_type": str,
             "output_type": str,
             "choices": List[str],
-            "instruction": str,
+            "instance_instruction": str,
         },
         reference_fields={
             "choices": List[str],
@@ -19,7 +19,7 @@
         metrics=[
             "metrics.accuracy",
         ],
-        augmentable_inputs=["input", "instruction"],
+        augmentable_inputs=["input", "instance_instruction"],
     ),
     "tasks.evaluation.preference",
     overwrite=True,

prepare/templates/evaluation/preference.py

@@ -4,7 +4,7 @@
 add_to_catalog(
     MultipleChoiceTemplate(
         instruction="For any Instruction and {input_type} given to a model, assess which {output_type} written by the model aligns most closely with the given instruction (choose from {numerals}).",
-        input_format="Instruction:\n{instruction}\n{input_type}:\n{input}\nResponses:\n{choices}",
+        input_format="Instruction:\n{instance_instruction}\n{input_type}:\n{input}\nResponses:\n{choices}",
         target_prefix="{output_type}:\n",
         target_field="output_choice",
         choices_separator="\n",

src/unitxt/catalog/cards/coedit/preference.json

@@ -16,16 +16,16 @@
             "field": "src",
             "by": ": "
         },
+        {
+            "__type__": "copy",
+            "field": "src/0",
+            "to_field": "instance_instruction"
+        },
         {
             "__type__": "slice",
             "field": "src",
             "start": 1
         },
-        {
-            "__type__": "copy",
-            "field": "src/0",
-            "to_field": "instruction"
-        },
         {
             "__type__": "join",
             "field": "src",

src/unitxt/catalog/cards/hh_rlhf.json

@@ -66,7 +66,7 @@
             "fields": {
                 "input_type": "dialog",
                 "output_type": "response",
-                "instruction": "Respond the following dialog in an helpful and harmfull way."
+                "instance_instruction": "Respond the following dialog in an helpful and harmless way."
             }
         },
         {

src/unitxt/catalog/cards/summarize_from_human_feedback.json

@@ -22,6 +22,11 @@
                 "summaries/*/text": "choices"
             }
         },
+        {
+            "__type__": "cast",
+            "field": "input",
+            "to": "str"
+        },
         {
             "__type__": "rename",
             "field_to_field": {
@@ -33,7 +38,7 @@
             "fields": {
                 "input_type": "post",
                 "output_type": "summary",
-                "instruction": "Summarize the following post"
+                "instance_instruction": "Summarize the following post"
             }
         }
     ],

src/unitxt/catalog/tasks/evaluation/preference.json

@@ -5,7 +5,7 @@
         "input_type": "str",
         "output_type": "str",
         "choices": "List[str]",
-        "instruction": "str"
+        "instance_instruction": "str"
     },
     "reference_fields": {
         "choices": "List[str]",
@@ -16,6 +16,6 @@
     ],
     "augmentable_inputs": [
         "input",
-        "instruction"
+        "instance_instruction"
     ]
 }

src/unitxt/catalog/templates/evaluation/preference/default.json

@@ -1,7 +1,7 @@
 {
     "__type__": "multiple_choice_template",
     "instruction": "For any Instruction and {input_type} given to a model, assess which {output_type} written by the model aligns most closely with the given instruction (choose from {numerals}).",
-    "input_format": "Instruction:\n{instruction}\n{input_type}:\n{input}\nResponses:\n{choices}",
+    "input_format": "Instruction:\n{instance_instruction}\n{input_type}:\n{input}\nResponses:\n{choices}",
     "target_prefix": "{output_type}:\n",
     "target_field": "output_choice",
     "choices_separator": "\n",