From 0605a39a159020244cdce77f6b8cab44fd3f1c85 Mon Sep 17 00:00:00 2001 From: "Klamkin, Michael" Date: Wed, 7 May 2025 02:52:39 -0400 Subject: [PATCH 1/6] add `--merge-pull-request` option for `convert_to_parquet` --- src/datasets/commands/convert_to_parquet.py | 26 +++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/src/datasets/commands/convert_to_parquet.py b/src/datasets/commands/convert_to_parquet.py index 43e82de802b..81776bd56df 100644 --- a/src/datasets/commands/convert_to_parquet.py +++ b/src/datasets/commands/convert_to_parquet.py @@ -1,6 +1,9 @@ from argparse import ArgumentParser from typing import Optional +from huggingface_hub import HfApi + +import datasets.config from datasets.commands import BaseDatasetsCLICommand from datasets.hub import convert_to_parquet @@ -26,6 +29,11 @@ def register_subcommand(parser): parser.add_argument( "--trust_remote_code", action="store_true", help="whether to trust the code execution of the load script" ) + parser.add_argument( + "--merge-pull-request", + action="store_true", + help="whether to automatically merge the pull request(s) after conversion", + ) parser.set_defaults(func=_command_factory) def __init__( @@ -39,8 +47,22 @@ def __init__( self._token = token self._revision = revision self._trust_remote_code = trust_remote_code + self._merge_pull_request = False def run(self) -> None: - _ = convert_to_parquet( - self._dataset_id, revision=self._revision, token=self._token, trust_remote_code=self._trust_remote_code + commit_info = convert_to_parquet( + self._dataset_id, + revision=self._revision, + token=self._token, + trust_remote_code=self._trust_remote_code, + merge_pull_request=self._merge_pull_request, ) + + if self._merge_pull_request: + api = HfApi(endpoint=datasets.config.HF_ENDPOINT, token=self._token) + api.merge_pull_request( + repo_id=self._dataset_id, + discussion_num=int(commit_info.pr_num), + token=self._token, + repo_type="dataset", + ) From 37373ac33544f6c59d766a83ba6cd427b7b28c1a Mon Sep 17 00:00:00 2001 From: "Klamkin, Michael" Date: Wed, 7 May 2025 02:56:58 -0400 Subject: [PATCH 2/6] remove old kwarg --- src/datasets/commands/convert_to_parquet.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/datasets/commands/convert_to_parquet.py b/src/datasets/commands/convert_to_parquet.py index 81776bd56df..24f10ec9b6e 100644 --- a/src/datasets/commands/convert_to_parquet.py +++ b/src/datasets/commands/convert_to_parquet.py @@ -55,7 +55,6 @@ def run(self) -> None: revision=self._revision, token=self._token, trust_remote_code=self._trust_remote_code, - merge_pull_request=self._merge_pull_request, ) if self._merge_pull_request: From bd434aceae16d99eab0ce43883487958ea461a7f Mon Sep 17 00:00:00 2001 From: "Klamkin, Michael" Date: Wed, 7 May 2025 02:57:53 -0400 Subject: [PATCH 3/6] keep old format --- src/datasets/commands/convert_to_parquet.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/datasets/commands/convert_to_parquet.py b/src/datasets/commands/convert_to_parquet.py index 24f10ec9b6e..815ea7ac86f 100644 --- a/src/datasets/commands/convert_to_parquet.py +++ b/src/datasets/commands/convert_to_parquet.py @@ -51,10 +51,7 @@ def __init__( def run(self) -> None: commit_info = convert_to_parquet( - self._dataset_id, - revision=self._revision, - token=self._token, - trust_remote_code=self._trust_remote_code, + self._dataset_id, revision=self._revision, token=self._token, trust_remote_code=self._trust_remote_code ) if self._merge_pull_request: From 153121d0e2e1ae93dfe40866eb07c5ba00882527 Mon Sep 17 00:00:00 2001 From: "Klamkin, Michael" Date: Wed, 7 May 2025 02:58:30 -0400 Subject: [PATCH 4/6] remove plural --- src/datasets/commands/convert_to_parquet.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/datasets/commands/convert_to_parquet.py b/src/datasets/commands/convert_to_parquet.py index 815ea7ac86f..f7cbe1fb20e 100644 --- a/src/datasets/commands/convert_to_parquet.py +++ b/src/datasets/commands/convert_to_parquet.py @@ -32,7 +32,7 @@ def register_subcommand(parser): parser.add_argument( "--merge-pull-request", action="store_true", - help="whether to automatically merge the pull request(s) after conversion", + help="whether to automatically merge the pull request after conversion", ) parser.set_defaults(func=_command_factory) From f4c18577f8b4a554ed5275b442d62ab53db980ac Mon Sep 17 00:00:00 2001 From: "Klamkin, Michael" Date: Wed, 7 May 2025 03:03:26 -0400 Subject: [PATCH 5/6] pass through arg --- src/datasets/commands/convert_to_parquet.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/datasets/commands/convert_to_parquet.py b/src/datasets/commands/convert_to_parquet.py index f7cbe1fb20e..5ae75c00712 100644 --- a/src/datasets/commands/convert_to_parquet.py +++ b/src/datasets/commands/convert_to_parquet.py @@ -42,12 +42,13 @@ def __init__( token: Optional[str], revision: Optional[str], trust_remote_code: bool, + merge_pull_request: bool = False, ): self._dataset_id = dataset_id self._token = token self._revision = revision self._trust_remote_code = trust_remote_code - self._merge_pull_request = False + self._merge_pull_request = merge_pull_request def run(self) -> None: commit_info = convert_to_parquet( From 8cc594d7edd81ccc01174ffcb5295c5df5a9c0f1 Mon Sep 17 00:00:00 2001 From: "Klamkin, Michael" Date: Wed, 7 May 2025 03:05:24 -0400 Subject: [PATCH 6/6] command factory --- src/datasets/commands/convert_to_parquet.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/datasets/commands/convert_to_parquet.py b/src/datasets/commands/convert_to_parquet.py index 5ae75c00712..b64f8bbfac4 100644 --- a/src/datasets/commands/convert_to_parquet.py +++ b/src/datasets/commands/convert_to_parquet.py @@ -14,6 +14,7 @@ def _command_factory(args): args.token, args.revision, args.trust_remote_code, + args.merge_pull_request, ) @@ -42,7 +43,7 @@ def __init__( token: Optional[str], revision: Optional[str], trust_remote_code: bool, - merge_pull_request: bool = False, + merge_pull_request: bool, ): self._dataset_id = dataset_id self._token = token