Dill 3.7 support (#6061)

* Dill 3.7 support

* Nit

* Update setup.py

Co-authored-by: Quentin Lhoest <[email protected]>

* Nit

---------

Co-authored-by: Quentin Lhoest <[email protected]>

Author: mariosasko

setup.py

@@ -113,7 +113,7 @@
     # Minimum 8.0.0 to be able to use .to_reader()
     "pyarrow>=8.0.0",
     # For smart caching dataset processing
-    "dill>=0.3.0,<0.3.7",  # tmp pin until next 0.3.7 release: see https://github.com/huggingface/datasets/pull/5166
+    "dill>=0.3.0,<0.3.8",  # tmp pin until dill has official support for determinism see https://github.com/uqfoundation/dill/issues/19
     # For performance gains with apache arrow
     "pandas",
     # for downloading datasets over HTTPS

src/datasets/utils/py_utils.py

@@ -606,7 +606,7 @@ def save(self, obj, save_persistent_id=True):
                 def dill_log(pickler, msg):
                     dill._dill.log.info(msg)
 
-            elif config.DILL_VERSION.release[:3] == version.parse("0.3.6").release:
+            elif config.DILL_VERSION.release[:3] in [version.parse("0.3.6").release, version.parse("0.3.7").release]:
 
                 def dill_log(pickler, msg):
                     dill._dill.logger.trace(pickler, msg)
@@ -826,7 +826,7 @@ def _save_code(pickler, obj):
         dill._dill.log.info("# Co")
         return
 
-elif config.DILL_VERSION.release[:3] == version.parse("0.3.6").release:
+elif config.DILL_VERSION.release[:3] in [version.parse("0.3.6").release, version.parse("0.3.7").release]:
     # From: https://github.com/uqfoundation/dill/blob/dill-0.3.6/dill/_dill.py#L1104
     @pklregister(CodeType)
     def save_code(pickler, obj):
@@ -1193,7 +1193,7 @@ def save_function(pickler, obj):
             dill._dill.log.info("# F2")
         return
 
-elif config.DILL_VERSION.release[:3] == version.parse("0.3.6").release:
+elif config.DILL_VERSION.release[:3] in [version.parse("0.3.6").release, version.parse("0.3.7").release]:
     # From: https://github.com/uqfoundation/dill/blob/dill-0.3.6/dill/_dill.py#L1739
     @pklregister(FunctionType)
     def save_function(pickler, obj):