Dill 3.7 support

Author: mariosasko

setup.py

@@ -113,7 +113,7 @@
     # Minimum 8.0.0 to be able to use .to_reader()
     "pyarrow>=8.0.0",
     # For smart caching dataset processing
-    "dill>=0.3.0,<0.3.7",  # tmp pin until next 0.3.7 release: see https://github.com/huggingface/datasets/pull/5166
+    "dill>=0.3.0",
     # For performance gains with apache arrow
     "pandas",
     # for downloading datasets over HTTPS

src/datasets/utils/py_utils.py

@@ -606,7 +606,7 @@ def save(self, obj, save_persistent_id=True):
                 def dill_log(pickler, msg):
                     dill._dill.log.info(msg)
 
-            elif config.DILL_VERSION.release[:3] == version.parse("0.3.6").release:
+            else:
 
                 def dill_log(pickler, msg):
                     dill._dill.logger.trace(pickler, msg)
@@ -826,7 +826,7 @@ def _save_code(pickler, obj):
         dill._dill.log.info("# Co")
         return
 
-elif config.DILL_VERSION.release[:3] == version.parse("0.3.6").release:
+else:  # config.DILL_VERSION >= version.parse("0.3.6")
     # From: https://github.com/uqfoundation/dill/blob/dill-0.3.6/dill/_dill.py#L1104
     @pklregister(CodeType)
     def save_code(pickler, obj):
@@ -1193,7 +1193,7 @@ def save_function(pickler, obj):
             dill._dill.log.info("# F2")
         return
 
-elif config.DILL_VERSION.release[:3] == version.parse("0.3.6").release:
+else:  # config.DILL_VERSION >= version.parse("0.3.6")
     # From: https://github.com/uqfoundation/dill/blob/dill-0.3.6/dill/_dill.py#L1739
     @pklregister(FunctionType)
     def save_function(pickler, obj):