Skip to content

Commit 2be0840

Browse files
dafnapensiondafnapension
committed
cook options when empty rather than filter out
Signed-off-by: dafnapension <[email protected]>
1 parent 1fc17f4 commit 2be0840

31 files changed

+430
-575
lines changed

‎prepare/cards/mmmu.py‎

Lines changed: 10 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,7 @@
22
from unitxt.catalog import add_to_catalog
33
from unitxt.collections_operators import Filter
44
from unitxt.operators import (
5-
FilterByCondition,
6-
FilterByExpression,
5+
ExecuteExpression,
76
ListFieldValues,
87
MapValues,
98
)
@@ -45,7 +44,7 @@
4544
"Sociology",
4645
]
4746

48-
mapping = {"A": 0, "B": 1, "C": 2, "D": 3, "E": 4, "?": None}
47+
mapping = {"A": 0, "B": 1, "C": 2, "D": 3, "E": 4, "F": 5, "G": 6, "H": 7, "I": 8}
4948

5049
for name in config_names:
5150
card = TaskCard(
@@ -58,6 +57,14 @@
5857
fields=[f"image_{i}" for i in range(1, 8)], to_field="media/images"
5958
),
6059
Filter(field="media/images", values=[None]),
60+
ExecuteExpression(
61+
expression="options if options != '[]' else '[\"'+answer+'\"]'",
62+
to_field="options",
63+
),
64+
ExecuteExpression(
65+
expression="'A' if options == '[\"'+answer+'\"]' else answer",
66+
to_field="answer",
67+
),
6168
MapReplace(
6269
field_to_field={"question": "question", "options": "choices"},
6370
mapping={
@@ -67,12 +74,10 @@
6774
),
6875
LiteralEval(field="choices"),
6976
Lower(field="subfield", to_field="topic"),
70-
FilterByCondition(values={"answer": list(mapping.keys())}, condition="in"),
7177
MapValues(
7278
field="answer",
7379
mapping=mapping,
7480
),
75-
FilterByExpression(expression="answer < len(choices)"),
7681
],
7782
task="tasks.qa.multiple_choice.with_topic",
7883
templates="templates.qa.multiple_choice.with_topic.all",

‎src/unitxt/catalog/cards/mmmu/accounting.json‎

Lines changed: 14 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,16 @@
3636
null
3737
]
3838
},
39+
{
40+
"__type__": "execute_expression",
41+
"expression": "options if options != '[]' else '[\"'+answer+'\"]'",
42+
"to_field": "options"
43+
},
44+
{
45+
"__type__": "execute_expression",
46+
"expression": "'A' if options == '[\"'+answer+'\"]' else answer",
47+
"to_field": "answer"
48+
},
3949
{
4050
"__type__": "map_replace",
4151
"field_to_field": {
@@ -61,20 +71,6 @@
6171
"field": "subfield",
6272
"to_field": "topic"
6373
},
64-
{
65-
"__type__": "filter_by_condition",
66-
"values": {
67-
"answer": [
68-
"A",
69-
"B",
70-
"C",
71-
"D",
72-
"E",
73-
"?"
74-
]
75-
},
76-
"condition": "in"
77-
},
7874
{
7975
"__type__": "map_values",
8076
"field": "answer",
@@ -84,12 +80,11 @@
8480
"C": 2,
8581
"D": 3,
8682
"E": 4,
87-
"?": null
83+
"F": 5,
84+
"G": 6,
85+
"H": 7,
86+
"I": 8
8887
}
89-
},
90-
{
91-
"__type__": "filter_by_expression",
92-
"expression": "answer < len(choices)"
9388
}
9489
],
9590
"task": "tasks.qa.multiple_choice.with_topic",

‎src/unitxt/catalog/cards/mmmu/agriculture.json‎

Lines changed: 14 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,16 @@
3636
null
3737
]
3838
},
39+
{
40+
"__type__": "execute_expression",
41+
"expression": "options if options != '[]' else '[\"'+answer+'\"]'",
42+
"to_field": "options"
43+
},
44+
{
45+
"__type__": "execute_expression",
46+
"expression": "'A' if options == '[\"'+answer+'\"]' else answer",
47+
"to_field": "answer"
48+
},
3949
{
4050
"__type__": "map_replace",
4151
"field_to_field": {
@@ -61,20 +71,6 @@
6171
"field": "subfield",
6272
"to_field": "topic"
6373
},
64-
{
65-
"__type__": "filter_by_condition",
66-
"values": {
67-
"answer": [
68-
"A",
69-
"B",
70-
"C",
71-
"D",
72-
"E",
73-
"?"
74-
]
75-
},
76-
"condition": "in"
77-
},
7874
{
7975
"__type__": "map_values",
8076
"field": "answer",
@@ -84,12 +80,11 @@
8480
"C": 2,
8581
"D": 3,
8682
"E": 4,
87-
"?": null
83+
"F": 5,
84+
"G": 6,
85+
"H": 7,
86+
"I": 8
8887
}
89-
},
90-
{
91-
"__type__": "filter_by_expression",
92-
"expression": "answer < len(choices)"
9388
}
9489
],
9590
"task": "tasks.qa.multiple_choice.with_topic",

‎src/unitxt/catalog/cards/mmmu/architecture_and_engineering.json‎

Lines changed: 14 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,16 @@
3636
null
3737
]
3838
},
39+
{
40+
"__type__": "execute_expression",
41+
"expression": "options if options != '[]' else '[\"'+answer+'\"]'",
42+
"to_field": "options"
43+
},
44+
{
45+
"__type__": "execute_expression",
46+
"expression": "'A' if options == '[\"'+answer+'\"]' else answer",
47+
"to_field": "answer"
48+
},
3949
{
4050
"__type__": "map_replace",
4151
"field_to_field": {
@@ -61,20 +71,6 @@
6171
"field": "subfield",
6272
"to_field": "topic"
6373
},
64-
{
65-
"__type__": "filter_by_condition",
66-
"values": {
67-
"answer": [
68-
"A",
69-
"B",
70-
"C",
71-
"D",
72-
"E",
73-
"?"
74-
]
75-
},
76-
"condition": "in"
77-
},
7874
{
7975
"__type__": "map_values",
8076
"field": "answer",
@@ -84,12 +80,11 @@
8480
"C": 2,
8581
"D": 3,
8682
"E": 4,
87-
"?": null
83+
"F": 5,
84+
"G": 6,
85+
"H": 7,
86+
"I": 8
8887
}
89-
},
90-
{
91-
"__type__": "filter_by_expression",
92-
"expression": "answer < len(choices)"
9388
}
9489
],
9590
"task": "tasks.qa.multiple_choice.with_topic",

‎src/unitxt/catalog/cards/mmmu/art.json‎

Lines changed: 14 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,16 @@
3636
null
3737
]
3838
},
39+
{
40+
"__type__": "execute_expression",
41+
"expression": "options if options != '[]' else '[\"'+answer+'\"]'",
42+
"to_field": "options"
43+
},
44+
{
45+
"__type__": "execute_expression",
46+
"expression": "'A' if options == '[\"'+answer+'\"]' else answer",
47+
"to_field": "answer"
48+
},
3949
{
4050
"__type__": "map_replace",
4151
"field_to_field": {
@@ -61,20 +71,6 @@
6171
"field": "subfield",
6272
"to_field": "topic"
6373
},
64-
{
65-
"__type__": "filter_by_condition",
66-
"values": {
67-
"answer": [
68-
"A",
69-
"B",
70-
"C",
71-
"D",
72-
"E",
73-
"?"
74-
]
75-
},
76-
"condition": "in"
77-
},
7874
{
7975
"__type__": "map_values",
8076
"field": "answer",
@@ -84,12 +80,11 @@
8480
"C": 2,
8581
"D": 3,
8682
"E": 4,
87-
"?": null
83+
"F": 5,
84+
"G": 6,
85+
"H": 7,
86+
"I": 8
8887
}
89-
},
90-
{
91-
"__type__": "filter_by_expression",
92-
"expression": "answer < len(choices)"
9388
}
9489
],
9590
"task": "tasks.qa.multiple_choice.with_topic",

‎src/unitxt/catalog/cards/mmmu/art_theory.json‎

Lines changed: 14 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,16 @@
3636
null
3737
]
3838
},
39+
{
40+
"__type__": "execute_expression",
41+
"expression": "options if options != '[]' else '[\"'+answer+'\"]'",
42+
"to_field": "options"
43+
},
44+
{
45+
"__type__": "execute_expression",
46+
"expression": "'A' if options == '[\"'+answer+'\"]' else answer",
47+
"to_field": "answer"
48+
},
3949
{
4050
"__type__": "map_replace",
4151
"field_to_field": {
@@ -61,20 +71,6 @@
6171
"field": "subfield",
6272
"to_field": "topic"
6373
},
64-
{
65-
"__type__": "filter_by_condition",
66-
"values": {
67-
"answer": [
68-
"A",
69-
"B",
70-
"C",
71-
"D",
72-
"E",
73-
"?"
74-
]
75-
},
76-
"condition": "in"
77-
},
7874
{
7975
"__type__": "map_values",
8076
"field": "answer",
@@ -84,12 +80,11 @@
8480
"C": 2,
8581
"D": 3,
8682
"E": 4,
87-
"?": null
83+
"F": 5,
84+
"G": 6,
85+
"H": 7,
86+
"I": 8
8887
}
89-
},
90-
{
91-
"__type__": "filter_by_expression",
92-
"expression": "answer < len(choices)"
9388
}
9489
],
9590
"task": "tasks.qa.multiple_choice.with_topic",

‎src/unitxt/catalog/cards/mmmu/basic_medical_science.json‎

Lines changed: 14 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,16 @@
3636
null
3737
]
3838
},
39+
{
40+
"__type__": "execute_expression",
41+
"expression": "options if options != '[]' else '[\"'+answer+'\"]'",
42+
"to_field": "options"
43+
},
44+
{
45+
"__type__": "execute_expression",
46+
"expression": "'A' if options == '[\"'+answer+'\"]' else answer",
47+
"to_field": "answer"
48+
},
3949
{
4050
"__type__": "map_replace",
4151
"field_to_field": {
@@ -61,20 +71,6 @@
6171
"field": "subfield",
6272
"to_field": "topic"
6373
},
64-
{
65-
"__type__": "filter_by_condition",
66-
"values": {
67-
"answer": [
68-
"A",
69-
"B",
70-
"C",
71-
"D",
72-
"E",
73-
"?"
74-
]
75-
},
76-
"condition": "in"
77-
},
7874
{
7975
"__type__": "map_values",
8076
"field": "answer",
@@ -84,12 +80,11 @@
8480
"C": 2,
8581
"D": 3,
8682
"E": 4,
87-
"?": null
83+
"F": 5,
84+
"G": 6,
85+
"H": 7,
86+
"I": 8
8887
}
89-
},
90-
{
91-
"__type__": "filter_by_expression",
92-
"expression": "answer < len(choices)"
9388
}
9489
],
9590
"task": "tasks.qa.multiple_choice.with_topic",

0 commit comments

Comments
 (0)