Fix duplicate kwargs in load_dataset_builder

Avoid TypeError when builder_kwargs and config_kwargs share keys like base_path.

Fixes #4910

Author: Fahad Alghanim

src/datasets/load.py

@@ -1149,6 +1149,14 @@ def load_dataset_builder(
     dataset_name = builder_kwargs.pop("dataset_name", None)
     info = dataset_module.dataset_infos.get(config_name) if dataset_module.dataset_infos else None
 
+    # Avoid passing duplicate keyword arguments to the builder.
+    # `builder_kwargs` can contain keys like `base_path`, and users may also pass them via `config_kwargs`.
+    # In that case, Python raises: "TypeError: got multiple values for keyword argument ...".
+    # Keep the user-provided values (config_kwargs) by dropping overlaps from builder_kwargs.
+    if config_kwargs:
+        for key in set(builder_kwargs).intersection(config_kwargs):
+            builder_kwargs.pop(key, None)
+
     if (
         path in _PACKAGED_DATASETS_MODULES
         and data_files is None

tests/test_load_dataset_builder_kwargs.py

@@ -0,0 +1,17 @@
+import pytest
+
+from datasets.load import load_dataset_builder
+
+
+def test_load_dataset_builder_does_not_fail_with_duplicate_builder_kwargs(tmp_path):
+    # Regression test for https://github.com/huggingface/datasets/issues/4910
+    # Some module factories provide `base_path` in `builder_kwargs`, and users can also pass `base_path`
+    # via `config_kwargs`, which used to raise:
+    #   TypeError: ... got multiple values for keyword argument 'base_path'
+    train_csv = tmp_path / "train.csv"
+    train_csv.write_text("col\n1\n", encoding="utf-8")
+
+    custom_base_path = str(tmp_path / "custom_base_path")
+    builder = load_dataset_builder("csv", data_files=str(train_csv), base_path=custom_base_path)
+    assert builder.base_path == custom_base_path
+