DEPS: bump pyarrow minimum version from 10.0 to 12.0

ci/deps/actions-310-minimum_versions.yaml

@@ -41,7 +41,7 @@ dependencies:
   - qtpy=2.3.0
   - openpyxl=3.1.2
   - psycopg2=2.9.6
-  - pyarrow=10.0.1
+  - pyarrow=12.0.1
   - pyiceberg=0.7.1
   - pymysql=1.1.0
   - pyqt=5.15.9

ci/deps/actions-310.yaml

@@ -39,7 +39,7 @@ dependencies:
   - qtpy>=2.3.0
   - openpyxl>=3.1.2
   - psycopg2>=2.9.6
-  - pyarrow>=10.0.1
+  - pyarrow>=12.0.1
   - pyiceberg>=0.7.1
   - pymysql>=1.1.0
   - pyqt>=5.15.9

ci/deps/actions-311-downstream_compat.yaml

@@ -40,7 +40,7 @@ dependencies:
   - qtpy>=2.3.0
   - openpyxl>=3.1.2
   - psycopg2>=2.9.6
-  - pyarrow>=10.0.1
+  - pyarrow>=12.0.1
   - pyiceberg>=0.7.1
   - pymysql>=1.1.0
   - pyqt>=5.15.9

ci/deps/actions-311.yaml

@@ -40,7 +40,7 @@ dependencies:
   - pyqt>=5.15.9
   - openpyxl>=3.1.2
   - psycopg2>=2.9.6
-  - pyarrow>=10.0.1
+  - pyarrow>=12.0.1
   - pyiceberg>=0.7.1
   - pymysql>=1.1.0
   - pyreadstat>=1.2.6

ci/deps/actions-312.yaml

@@ -40,7 +40,7 @@ dependencies:
   - pyqt>=5.15.9
   - openpyxl>=3.1.2
   - psycopg2>=2.9.6
-  - pyarrow>=10.0.1
+  - pyarrow>=12.0.1
   - pyiceberg>=0.7.1
   - pymysql>=1.1.0
   - pyreadstat>=1.2.6

ci/deps/actions-313.yaml

@@ -41,7 +41,7 @@ dependencies:
   - pyqt>=5.15.9
   - openpyxl>=3.1.2
   - psycopg2>=2.9.6
-  - pyarrow>=10.0.1
+  - pyarrow>=12.0.1
   - pymysql>=1.1.0
   - pyreadstat>=1.2.6
   - pytables>=3.8.0

doc/source/getting_started/install.rst

@@ -307,7 +307,7 @@ Dependency                                             Minimum Version    pip ex
 `PyTables <https://github.com/PyTables/PyTables>`__    3.8.0              hdf5             HDF5-based reading / writing
 `zlib <https://github.com/madler/zlib>`__                                 hdf5             Compression for HDF5
 `fastparquet <https://github.com/dask/fastparquet>`__  2024.2.0           -                Parquet reading / writing (pyarrow is default)
-`pyarrow <https://github.com/apache/arrow>`__          10.0.1             parquet, feather Parquet, ORC, and feather reading / writing
+`pyarrow <https://github.com/apache/arrow>`__          12.0.1             parquet, feather Parquet, ORC, and feather reading / writing
 `PyIceberg <https://py.iceberg.apache.org/>`__         0.7.1              iceberg          Apache Iceberg reading / writing
 `pyreadstat <https://github.com/Roche/pyreadstat>`__   1.2.6              spss             SPSS files (.sav) reading
 `odfpy <https://github.com/eea/odfpy>`__               1.4.1              excel            Open document format (.odf, .ods, .odt) reading / writing

environment.yml

@@ -43,7 +43,7 @@ dependencies:
   - openpyxl>=3.1.2
   - odfpy>=1.4.1
   - psycopg2>=2.9.6
-  - pyarrow>=10.0.1
+  - pyarrow>=12.0.1
   - pyiceberg>=0.7.1
   - pymysql>=1.1.0
   - pyreadstat>=1.2.6

pandas/_testing/__init__.py

@@ -18,7 +18,7 @@
     set_locale,
 )
 
-from pandas.compat import pa_version_under10p1
+from pandas.compat import HAS_PYARROW
 
 import pandas as pd
 from pandas import (
@@ -183,7 +183,7 @@
     ]
 ]
 
-if not pa_version_under10p1:
+if HAS_PYARROW:
     import pyarrow as pa
 
     UNSIGNED_INT_PYARROW_DTYPES = [pa.uint8(), pa.uint16(), pa.uint32(), pa.uint64()]

pandas/compat/__init__.py

@@ -26,7 +26,6 @@
 from pandas.compat.numpy import is_numpy_dev
 from pandas.compat.pyarrow import (
     HAS_PYARROW,
-    pa_version_under10p1,
     pa_version_under11p0,
     pa_version_under13p0,
     pa_version_under14p0,
@@ -160,7 +159,6 @@ def is_ci_environment() -> bool:
     "PYPY",
     "WASM",
     "is_numpy_dev",
-    "pa_version_under10p1",
     "pa_version_under11p0",
     "pa_version_under13p0",
     "pa_version_under14p0",

pandas/compat/_optional.py

@@ -38,7 +38,7 @@
     "openpyxl": "3.1.2",
     "psycopg2": "2.9.6",  # (dt dec pq3 ext lo64)
     "pymysql": "1.1.0",
-    "pyarrow": "10.0.1",
+    "pyarrow": "12.0.1",
     "pyiceberg": "0.7.1",
     "pyreadstat": "1.2.6",
     "pytest": "7.3.2",

pandas/compat/pyarrow.py

@@ -8,9 +8,8 @@
     import pyarrow as pa
 
     _palv = Version(Version(pa.__version__).base_version)
-    pa_version_under10p1 = _palv < Version("10.0.1")
     pa_version_under11p0 = _palv < Version("11.0.0")
-    pa_version_under12p0 = _palv < Version("12.0.0")
+    pa_version_under12p1 = _palv < Version("12.0.1")
     pa_version_under13p0 = _palv < Version("13.0.0")
     pa_version_under14p0 = _palv < Version("14.0.0")
     pa_version_under14p1 = _palv < Version("14.0.1")
@@ -20,11 +19,10 @@
     pa_version_under18p0 = _palv < Version("18.0.0")
     pa_version_under19p0 = _palv < Version("19.0.0")
     pa_version_under20p0 = _palv < Version("20.0.0")
-    HAS_PYARROW = True
+    HAS_PYARROW = _palv >= Version("12.0.1")
 except ImportError:
-    pa_version_under10p1 = True
     pa_version_under11p0 = True
-    pa_version_under12p0 = True
+    pa_version_under12p1 = True
     pa_version_under13p0 = True
     pa_version_under14p0 = True
     pa_version_under14p1 = True

pandas/core/arrays/_arrow_string_mixins.py

@@ -12,13 +12,13 @@
 
 from pandas._libs import lib
 from pandas.compat import (
-    pa_version_under10p1,
+    HAS_PYARROW,
     pa_version_under11p0,
     pa_version_under13p0,
     pa_version_under17p0,
 )
 
-if not pa_version_under10p1:
+if HAS_PYARROW:
     import pyarrow as pa
     import pyarrow.compute as pc
 

pandas/core/arrays/arrow/accessors.py

@@ -12,13 +12,13 @@
 )
 
 from pandas.compat import (
-    pa_version_under10p1,
+    HAS_PYARROW,
     pa_version_under11p0,
 )
 
 from pandas.core.dtypes.common import is_list_like
 
-if not pa_version_under10p1:
+if HAS_PYARROW:
     import pyarrow as pa
     import pyarrow.compute as pc
 
@@ -46,7 +46,7 @@ def _is_valid_pyarrow_dtype(self, pyarrow_dtype) -> bool:
 
     def _validate(self, data) -> None:
         dtype = data.dtype
-        if pa_version_under10p1 or not isinstance(dtype, ArrowDtype):
+        if not HAS_PYARROW or not isinstance(dtype, ArrowDtype):
             # Raise AttributeError so that inspect can handle non-struct Series.
             raise AttributeError(self._validation_msg.format(dtype=dtype))
 

pandas/core/arrays/arrow/array.py

@@ -22,8 +22,9 @@
     timezones,
 )
 from pandas.compat import (
-    pa_version_under10p1,
+    HAS_PYARROW,
     pa_version_under11p0,
+    pa_version_under12p1,
     pa_version_under13p0,
 )
 from pandas.util._decorators import doc
@@ -74,7 +75,7 @@
 from pandas.io._util import _arrow_dtype_mapping
 from pandas.tseries.frequencies import to_offset
 
-if not pa_version_under10p1:
+if HAS_PYARROW:
     import pyarrow as pa
     import pyarrow.compute as pc
 
@@ -300,7 +301,7 @@ class ArrowExtensionArray(
     _dtype: ArrowDtype
 
     def __init__(self, values: pa.Array | pa.ChunkedArray) -> None:
-        if pa_version_under10p1:
+        if pa_version_under12p1:
             msg = "pyarrow>=10.0.1 is required for PyArrow backed ArrowExtensionArray."
             raise ImportError(msg)
         if isinstance(values, pa.Array):

pandas/core/arrays/string_.py

@@ -25,7 +25,7 @@
 from pandas._libs.lib import ensure_string_array
 from pandas.compat import (
     HAS_PYARROW,
-    pa_version_under10p1,
+    pa_version_under12p1,
 )
 from pandas.compat.numpy import function as nv
 from pandas.util._decorators import (
@@ -182,9 +182,9 @@ def __init__(
             raise ValueError(
                 f"Storage must be 'python' or 'pyarrow'. Got {storage} instead."
             )
-        if storage == "pyarrow" and pa_version_under10p1:
+        if storage == "pyarrow" and pa_version_under12p1:
             raise ImportError(
-                "pyarrow>=10.0.1 is required for PyArrow backed StringArray."
+                "pyarrow>=12.0.1 is required for PyArrow backed StringArray."
             )
 
         if isinstance(na_value, float) and np.isnan(na_value):

pandas/core/arrays/string_arrow.py

@@ -15,7 +15,8 @@
     missing as libmissing,
 )
 from pandas.compat import (
-    pa_version_under10p1,
+    HAS_PYARROW,
+    pa_version_under12p1,
     pa_version_under13p0,
     pa_version_under16p0,
 )
@@ -39,7 +40,7 @@
 )
 from pandas.core.strings.object_array import ObjectStringArrayMixin
 
-if not pa_version_under10p1:
+if HAS_PYARROW:
     import pyarrow as pa
     import pyarrow.compute as pc
 
@@ -67,8 +68,8 @@
 
 
 def _chk_pyarrow_available() -> None:
-    if pa_version_under10p1:
-        msg = "pyarrow>=10.0.1 is required for PyArrow backed ArrowExtensionArray."
+    if pa_version_under12p1:
+        msg = "pyarrow>=12.0.1 is required for PyArrow backed ArrowExtensionArray."
         raise ImportError(msg)
 
 

pandas/core/dtypes/dtypes.py

@@ -46,7 +46,10 @@
     abbrev_to_npy_unit,
 )
 from pandas._libs.tslibs.offsets import BDay
-from pandas.compat import pa_version_under10p1
+from pandas.compat import (
+    HAS_PYARROW,
+    pa_version_under12p1,
+)
 from pandas.errors import PerformanceWarning
 from pandas.util._decorators import set_module
 from pandas.util._exceptions import find_stack_level
@@ -66,7 +69,7 @@
     is_list_like,
 )
 
-if not pa_version_under10p1:
+if HAS_PYARROW:
     import pyarrow as pa
 
 if TYPE_CHECKING:
@@ -2193,8 +2196,8 @@ class ArrowDtype(StorageExtensionDtype):
 
     def __init__(self, pyarrow_dtype: pa.DataType) -> None:
         super().__init__("pyarrow")
-        if pa_version_under10p1:
-            raise ImportError("pyarrow>=10.0.1 is required for ArrowDtype")
+        if pa_version_under12p1:
+            raise ImportError("pyarrow>=12.0.1 is required for ArrowDtype")
         if not isinstance(pyarrow_dtype, pa.DataType):
             raise ValueError(
                 f"pyarrow_dtype ({pyarrow_dtype}) must be an instance "
@@ -2346,7 +2349,7 @@ def construct_from_string(cls, string: str) -> ArrowDtype:
         if string in ("string[pyarrow]", "str[pyarrow]"):
             # Ensure Registry.find skips ArrowDtype to use StringDtype instead
             raise TypeError("string[pyarrow] should be constructed by StringDtype")
-        if pa_version_under10p1:
+        if pa_version_under12p1:
             raise ImportError("pyarrow>=10.0.1 is required for ArrowDtype")
 
         base_type = string[:-9]  # get rid of "[pyarrow]"

pandas/tests/arrays/period/test_arrow_compat.py

@@ -1,7 +1,5 @@
 import pytest
 
-from pandas.compat.pyarrow import pa_version_under10p1
-
 from pandas.core.dtypes.dtypes import PeriodDtype
 
 import pandas as pd
@@ -33,7 +31,6 @@ def test_arrow_extension_type():
     assert hash(p1) != hash(p3)
 
 
-@pytest.mark.xfail(not pa_version_under10p1, reason="Wrong behavior with pyarrow 10")
 @pytest.mark.parametrize(
     "data, freq",
     [

pandas/tests/arrays/string_/test_string.py

@@ -12,7 +12,7 @@
 
 from pandas.compat import HAS_PYARROW
 from pandas.compat.pyarrow import (
-    pa_version_under12p0,
+    pa_version_under12p1,
     pa_version_under19p0,
 )
 import pandas.util._test_decorators as td
@@ -600,7 +600,7 @@ def test_arrow_array(dtype):
     data = pd.array(["a", "b", "c"], dtype=dtype)
     arr = pa.array(data)
     expected = pa.array(list(data), type=pa.large_string(), from_pandas=True)
-    if dtype.storage == "pyarrow" and pa_version_under12p0:
+    if dtype.storage == "pyarrow" and pa_version_under12p1:
         expected = pa.chunked_array(expected)
     if dtype.storage == "python":
         expected = pc.cast(expected, pa.string())

pandas/tests/copy_view/test_astype.py

@@ -4,7 +4,7 @@
 import pytest
 
 from pandas.compat import HAS_PYARROW
-from pandas.compat.pyarrow import pa_version_under12p0
+from pandas.compat.pyarrow import pa_version_under12p1
 
 from pandas import (
     DataFrame,
@@ -196,7 +196,7 @@ def test_astype_arrow_timestamp():
     )
     result = df.astype("timestamp[ns][pyarrow]")
     assert not result._mgr._has_no_reference(0)
-    if pa_version_under12p0:
+    if pa_version_under12p1:
         assert not np.shares_memory(
             get_array(df, "a"), get_array(result, "a")._pa_array
         )

pandas/tests/groupby/test_groupby_dropna.py

@@ -1,8 +1,6 @@
 import numpy as np
 import pytest
 
-from pandas.compat.pyarrow import pa_version_under10p1
-
 from pandas.core.dtypes.missing import na_value_for_dtype
 
 import pandas as pd
@@ -411,12 +409,7 @@ def test_groupby_drop_nan_with_multi_index():
         "Float64",
         "category",
         "string",
-        pytest.param(
-            "string[pyarrow]",
-            marks=pytest.mark.skipif(
-                pa_version_under10p1, reason="pyarrow is not installed"
-            ),
-        ),
+        "string[pyarrow]",
         "datetime64[ns]",
         "period[D]",
         "Sparse[float]",