Skip to content

Fix bfcl so that instances do not explode in size, and all cards yield recipes that pass _source_to_dataset #1915

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 5 commits into
base: main
Choose a base branch
from
Open

Fix bfcl so that instances do not explode in size, and all cards yield recipes that pass _source_to_dataset #1915

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
859af00
remove ''-s from values of properties of tools
dafnapension Aug 12, 2025
6be87ad
ipynb
dafnapension Aug 12, 2025
1fc158a
complete small fixes to allow all cards to pass through _source_to_da…
dafnapension Aug 13, 2025
3bc266a
remove the generating example of exploding instance size
dafnapension Aug 13, 2025
e98faa7
use the same json schema fixer as xlam does, an operator entered main…
dafnapension Aug 14, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
103 changes: 55 additions & 48 deletions prepare/cards/bfcl.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,11 @@
from unitxt.operators import (
Copy,
ExecuteExpression,
FilterByExpression,
FixJsonSchemaOfParameterTypes,
Set,
)
from unitxt.stream_operators import JoinStreams
from unitxt.stream_operators import DeleteSplits, JoinStreams
from unitxt.test_utils.card import test_card

base_path = "https://raw.githubusercontent.com/ShishirPatil/gorilla/70b6a4a2144597b1f99d1f4d3185d35d7ee532a4/berkeley-function-call-leaderboard/data/"
Expand All @@ -31,9 +33,10 @@
on="id",
new_stream_name="test",
),
DeleteSplits(splits=["questions", "answers"]),
Copy(field="question/0/0/content", to_field="query"),
Copy(field="function", to_field="tools"),
"operators.fix_json_schema",
FixJsonSchemaOfParameterTypes(main_field="tools"),
# Process ground truth data in this dataset, which is a provided as a list of options per field,
# and convert it into a list of explicit tool calls
#
Expand Down Expand Up @@ -100,11 +103,15 @@
on="id",
new_stream_name="test",
),
DeleteSplits(splits=["questions", "answers"]),
Copy(field="question/*/0", to_field="dialog"),
Copy(field="function", to_field="tools"),
"operators.fix_json_schema",
FixJsonSchemaOfParameterTypes(main_field="tools"),
FilterByExpression(
expression="all(isinstance(v, dict) for d in ground_truth for k, v in d.items())"
),
ExecuteExpression(
expression='[{"name": k, "arguments": dict(zip(v.keys(), vals))} for d in ground_truth for k, v in d.items() for vals in itertools.product(*v.values())]',
expression='[{"name": k, "arguments": dict(zip(v.keys(), vals))} for d in ground_truth for k, v in d.items() for vals in itertools.product(*[[w for w in vval if w != ""] for vval in v.values()])]',
to_field="reference_calls",
imports_list=["itertools"],
),
Expand Down Expand Up @@ -138,52 +145,52 @@
test_card(card, strict=False)
add_to_catalog(card, f"cards.bfcl.multi_turn.{subset}_v3", overwrite=True)

for subset in [
"live_relevance",
"live_irrelevance",
]:
card = TaskCard(
loader=LoadJsonFile(
files={
"test": base_path + f"BFCL_v3_{subset}.json",
},
lines=True,
data_classification_policy=["public"],
),
preprocess_steps=[
Copy(field="question/*/0", to_field="dialog"),
Copy(field="function", to_field="tools"),
"operators.fix_json_schema",
Set(fields={"reference_calls": []}),
],
task="tasks.tool_calling.multi_turn",
templates=["templates.tool_calling.multi_turn"],
__description__=(
"""The Berkeley function calling leaderboard is a live leaderboard to evaluate the ability of different LLMs to call functions (also referred to as tools). We built this dataset from our learnings to be representative of most users' function calling use-cases, for example, in agents, as a part of enterprise workflows, etc. To this end, our evaluation dataset spans diverse categories, and across multiple languages."""
),
__title__=f"""Berkeley Function Calling Leaderboard (Multi Turn Setup) - {subset.replace("_", " ").title()} V3""",
__tags__={
"annotations_creators": "expert-generated",
"language": ["en"],
"license": "apache-2.0",
"size_categories": ["10K<n<100K"],
"task_categories": [
"question-answering",
"reading-comprehension",
"tool-calling",
"multi-turn-tool-calling",
],
"task_ids": [
"tool-calling",
"multi-turn-tool-calling",
"reading-comprehension",
],
for subset in [
"live_relevance",
"live_irrelevance",
]:
card = TaskCard(
loader=LoadJsonFile(
files={
"test": base_path + f"BFCL_v3_{subset}.json",
},
)
lines=True,
data_classification_policy=["public"],
),
preprocess_steps=[
Copy(field="question/*/0", to_field="dialog"),
Copy(field="function", to_field="tools"),
FixJsonSchemaOfParameterTypes(main_field="tools"),
Set(fields={"reference_calls": []}),
],
task="tasks.tool_calling.multi_turn",
templates=["templates.tool_calling.multi_turn"],
__description__=(
"""The Berkeley function calling leaderboard is a live leaderboard to evaluate the ability of different LLMs to call functions (also referred to as tools). We built this dataset from our learnings to be representative of most users' function calling use-cases, for example, in agents, as a part of enterprise workflows, etc. To this end, our evaluation dataset spans diverse categories, and across multiple languages."""
),
__title__=f"""Berkeley Function Calling Leaderboard (Multi Turn Setup) - {subset.replace("_", " ").title()} V3""",
__tags__={
"annotations_creators": "expert-generated",
"language": ["en"],
"license": "apache-2.0",
"size_categories": ["10K<n<100K"],
"task_categories": [
"question-answering",
"reading-comprehension",
"tool-calling",
"multi-turn-tool-calling",
],
"task_ids": [
"tool-calling",
"multi-turn-tool-calling",
"reading-comprehension",
],
},
)

# Test and add the card to the catalog
test_card(card, strict=False)
add_to_catalog(card, f"cards.bfcl.multi_turn.{subset}_v3", overwrite=True)
# Test and add the card to the catalog
test_card(card, strict=False)
add_to_catalog(card, f"cards.bfcl.multi_turn.{subset}_v3", overwrite=True)

# card = TaskCard(
# loader=LoadJsonFile(
Expand Down
1 change: 1 addition & 0 deletions prepare/operators/fix_json_schema.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
operator = RecursiveReplace(
key="type",
map_values={
"": "object",
"dict": "object",
"float": "number",
"tuple": "array",
Expand Down
18 changes: 16 additions & 2 deletions src/unitxt/catalog/cards/bfcl/multi_turn/java_v3.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,13 @@
"on": "id",
"new_stream_name": "test"
},
{
"__type__": "delete_splits",
"splits": [
"questions",
"answers"
]
},
{
"__type__": "copy",
"field": "question/*/0",
Expand All @@ -30,10 +37,17 @@
"field": "function",
"to_field": "tools"
},
"operators.fix_json_schema",
{
"__type__": "fix_json_schema_of_parameter_types",
"main_field": "tools"
},
{
"__type__": "filter_by_expression",
"expression": "all(isinstance(v, dict) for d in ground_truth for k, v in d.items())"
},
{
"__type__": "execute_expression",
"expression": "[{\"name\": k, \"arguments\": dict(zip(v.keys(), vals))} for d in ground_truth for k, v in d.items() for vals in itertools.product(*v.values())]",
"expression": "[{\"name\": k, \"arguments\": dict(zip(v.keys(), vals))} for d in ground_truth for k, v in d.items() for vals in itertools.product(*[[w for w in vval if w != \"\"] for vval in v.values()])]",
"to_field": "reference_calls",
"imports_list": [
"itertools"
Expand Down
18 changes: 16 additions & 2 deletions src/unitxt/catalog/cards/bfcl/multi_turn/javascript_v3.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,13 @@
"on": "id",
"new_stream_name": "test"
},
{
"__type__": "delete_splits",
"splits": [
"questions",
"answers"
]
},
{
"__type__": "copy",
"field": "question/*/0",
Expand All @@ -30,10 +37,17 @@
"field": "function",
"to_field": "tools"
},
"operators.fix_json_schema",
{
"__type__": "fix_json_schema_of_parameter_types",
"main_field": "tools"
},
{
"__type__": "filter_by_expression",
"expression": "all(isinstance(v, dict) for d in ground_truth for k, v in d.items())"
},
{
"__type__": "execute_expression",
"expression": "[{\"name\": k, \"arguments\": dict(zip(v.keys(), vals))} for d in ground_truth for k, v in d.items() for vals in itertools.product(*v.values())]",
"expression": "[{\"name\": k, \"arguments\": dict(zip(v.keys(), vals))} for d in ground_truth for k, v in d.items() for vals in itertools.product(*[[w for w in vval if w != \"\"] for vval in v.values()])]",
"to_field": "reference_calls",
"imports_list": [
"itertools"
Expand Down
5 changes: 4 additions & 1 deletion src/unitxt/catalog/cards/bfcl/multi_turn/live_irrelevance_v3.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,10 @@
"field": "function",
"to_field": "tools"
},
"operators.fix_json_schema",
{
"__type__": "fix_json_schema_of_parameter_types",
"main_field": "tools"
},
{
"__type__": "set",
"fields": {
Expand Down
18 changes: 16 additions & 2 deletions src/unitxt/catalog/cards/bfcl/multi_turn/live_multiple_v3.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,13 @@
"on": "id",
"new_stream_name": "test"
},
{
"__type__": "delete_splits",
"splits": [
"questions",
"answers"
]
},
{
"__type__": "copy",
"field": "question/*/0",
Expand All @@ -30,10 +37,17 @@
"field": "function",
"to_field": "tools"
},
"operators.fix_json_schema",
{
"__type__": "fix_json_schema_of_parameter_types",
"main_field": "tools"
},
{
"__type__": "filter_by_expression",
"expression": "all(isinstance(v, dict) for d in ground_truth for k, v in d.items())"
},
{
"__type__": "execute_expression",
"expression": "[{\"name\": k, \"arguments\": dict(zip(v.keys(), vals))} for d in ground_truth for k, v in d.items() for vals in itertools.product(*v.values())]",
"expression": "[{\"name\": k, \"arguments\": dict(zip(v.keys(), vals))} for d in ground_truth for k, v in d.items() for vals in itertools.product(*[[w for w in vval if w != \"\"] for vval in v.values()])]",
"to_field": "reference_calls",
"imports_list": [
"itertools"
Expand Down
18 changes: 16 additions & 2 deletions src/unitxt/catalog/cards/bfcl/multi_turn/live_parallel_multiple_v3.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,13 @@
"on": "id",
"new_stream_name": "test"
},
{
"__type__": "delete_splits",
"splits": [
"questions",
"answers"
]
},
{
"__type__": "copy",
"field": "question/*/0",
Expand All @@ -30,10 +37,17 @@
"field": "function",
"to_field": "tools"
},
"operators.fix_json_schema",
{
"__type__": "fix_json_schema_of_parameter_types",
"main_field": "tools"
},
{
"__type__": "filter_by_expression",
"expression": "all(isinstance(v, dict) for d in ground_truth for k, v in d.items())"
},
{
"__type__": "execute_expression",
"expression": "[{\"name\": k, \"arguments\": dict(zip(v.keys(), vals))} for d in ground_truth for k, v in d.items() for vals in itertools.product(*v.values())]",
"expression": "[{\"name\": k, \"arguments\": dict(zip(v.keys(), vals))} for d in ground_truth for k, v in d.items() for vals in itertools.product(*[[w for w in vval if w != \"\"] for vval in v.values()])]",
"to_field": "reference_calls",
"imports_list": [
"itertools"
Expand Down
18 changes: 16 additions & 2 deletions src/unitxt/catalog/cards/bfcl/multi_turn/live_parallel_v3.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,13 @@
"on": "id",
"new_stream_name": "test"
},
{
"__type__": "delete_splits",
"splits": [
"questions",
"answers"
]
},
{
"__type__": "copy",
"field": "question/*/0",
Expand All @@ -30,10 +37,17 @@
"field": "function",
"to_field": "tools"
},
"operators.fix_json_schema",
{
"__type__": "fix_json_schema_of_parameter_types",
"main_field": "tools"
},
{
"__type__": "filter_by_expression",
"expression": "all(isinstance(v, dict) for d in ground_truth for k, v in d.items())"
},
{
"__type__": "execute_expression",
"expression": "[{\"name\": k, \"arguments\": dict(zip(v.keys(), vals))} for d in ground_truth for k, v in d.items() for vals in itertools.product(*v.values())]",
"expression": "[{\"name\": k, \"arguments\": dict(zip(v.keys(), vals))} for d in ground_truth for k, v in d.items() for vals in itertools.product(*[[w for w in vval if w != \"\"] for vval in v.values()])]",
"to_field": "reference_calls",
"imports_list": [
"itertools"
Expand Down
5 changes: 4 additions & 1 deletion src/unitxt/catalog/cards/bfcl/multi_turn/live_relevance_v3.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,10 @@
"field": "function",
"to_field": "tools"
},
"operators.fix_json_schema",
{
"__type__": "fix_json_schema_of_parameter_types",
"main_field": "tools"
},
{
"__type__": "set",
"fields": {
Expand Down
18 changes: 16 additions & 2 deletions src/unitxt/catalog/cards/bfcl/multi_turn/live_simple_v3.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,13 @@
"on": "id",
"new_stream_name": "test"
},
{
"__type__": "delete_splits",
"splits": [
"questions",
"answers"
]
},
{
"__type__": "copy",
"field": "question/*/0",
Expand All @@ -30,10 +37,17 @@
"field": "function",
"to_field": "tools"
},
"operators.fix_json_schema",
{
"__type__": "fix_json_schema_of_parameter_types",
"main_field": "tools"
},
{
"__type__": "filter_by_expression",
"expression": "all(isinstance(v, dict) for d in ground_truth for k, v in d.items())"
},
{
"__type__": "execute_expression",
"expression": "[{\"name\": k, \"arguments\": dict(zip(v.keys(), vals))} for d in ground_truth for k, v in d.items() for vals in itertools.product(*v.values())]",
"expression": "[{\"name\": k, \"arguments\": dict(zip(v.keys(), vals))} for d in ground_truth for k, v in d.items() for vals in itertools.product(*[[w for w in vval if w != \"\"] for vval in v.values()])]",
"to_field": "reference_calls",
"imports_list": [
"itertools"
Expand Down
Loading