Unpin spark requirements

.github/workflows/build_docs.yaml

@@ -16,7 +16,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python_version: ["3.8", "3.10"]
+        python_version: ["3.8","3.9", "3.10", "3.11"]
     steps:
       - name: Checkout repository
         uses: actions/checkout@v3

docs/source/api_reference.rst

@@ -28,17 +28,6 @@ Deep Feature Synthesis
     dfs
     get_valid_primitives
 
-Wrappers
-~~~~~~~~
-.. currentmodule:: featuretools
-
-scikit-learn (BETA)
--------------------
-.. autosummary::
-    :toctree: generated/
-
-    wrappers.DFSTransformer
-
 Timedelta
 ~~~~~~~~~
 .. currentmodule:: featuretools

docs/source/release_notes.rst

@@ -11,6 +11,7 @@ Future Release
         * Add workaround for pandas 2.2.0 bug with nunique and unpin pandas (:pr:`2657`)
     * Changes
         * Fix deprecation warnings with is_categorical_dtype (:pr:`2641`)
+        * Remove woodwork, pyarrow, numpy, and pandas pins for spark installation (:pr:`2661`)
     * Documentation Changes
         * Update Featuretools logo to display properly in dark mode (:pr:`2632`)
     * Testing Changes

featuretools/entityset/entityset.py

@@ -1269,11 +1269,15 @@ def add_last_time_indexes(self, updated_dataframes=None):
                         lti_df = pd.Series([], dtype="object")
                     else:
                         if lti_is_spark:
-                            lti_df["last_time"] = ps.to_datetime(lti_df["last_time"])
-                            lti_df["last_time_old"] = ps.to_datetime(
-                                lti_df["last_time_old"],
-                            )
                             # TODO: Figure out a workaround for fillna and replace
+                            if lti_df["last_time_old"].dtype != "datetime64[ns]":
+                                lti_df["last_time_old"] = ps.to_datetime(
+                                    lti_df["last_time_old"],
+                                )
+                            if lti_df["last_time"].dtype != "datetime64[ns]":
+                                lti_df["last_time"] = ps.to_datetime(
+                                    lti_df["last_time"],
+                                )
                             lti_df = lti_df.max(axis=1)
                         else:
                             lti_df["last_time"] = lti_df["last_time"].astype(

featuretools/primitives/standard/aggregation/num_unique.py

@@ -60,6 +60,6 @@ def finalize(s):
 
             return dd.Aggregation(self.name, chunk=chunk, agg=agg, finalize=finalize)
 
-        if self.use_string_for_pd_calc:
+        if self.use_string_for_pd_calc or agg_type == Library.SPARK:
             return "nunique"
         return pd.Series.nunique

featuretools/tests/conftest.py

@@ -173,6 +173,8 @@ def pd_latlong_df():
 
 @pytest.fixture
 def dask_latlong_df(pd_latlong_df):
+    dask = pytest.importorskip("dask", reason="Dask not installed, skipping")
+    dask.config.set({"dataframe.convert-string": False})
     dd = pytest.importorskip("dask.dataframe", reason="Dask not installed, skipping")
     return dd.from_pandas(pd_latlong_df.reset_index(drop=True), npartitions=4)
 

featuretools/tests/entityset_tests/test_ww_es.py

@@ -748,6 +748,8 @@ def latlong_df(request):
 
 
 def test_replace_dataframe_data_transformation(latlong_df):
+    dask = pytest.importorskip("dask", reason="Dask not installed, skipping")
+    dask.config.set({"dataframe.convert-string": False})
     initial_df = latlong_df.copy()
     initial_df.ww.init(
         name="latlongs",

featuretools/tests/primitive_tests/test_transform_features.py

@@ -215,6 +215,8 @@ def pd_simple_es():
 @pytest.fixture
 def dd_simple_es(pd_simple_es):
     dd = pytest.importorskip("dask.dataframe", reason="Dask not installed, skipping")
+    dask = pytest.importorskip("dask", reason="Dask not installed, skipping")
+    dask.config.set({"dataframe.convert-string": False})
     dataframes = {}
     for df in pd_simple_es.dataframes:
         dataframes[df.ww.name] = (

featuretools/tests/testing_utils/generate_fake_dataframe.py

@@ -2,6 +2,7 @@
 from datetime import datetime as dt
 
 import pandas as pd
+import pytest
 import woodwork.type_sys.type_system as ww_type_system
 from woodwork import logical_types
 
@@ -40,6 +41,9 @@ def generate_fake_dataframe(
     n_rows=10,
     df_name="df",
 ):
+    dask = pytest.importorskip("dask", reason="Dask not installed, skipping")
+    dask.config.set({"dataframe.convert-string": False})
+
     def randomize(values_):
         random.seed(10)
         values = values_.copy()

featuretools/tests/testing_utils/mock_ds.py

@@ -2,6 +2,7 @@
 
 import numpy as np
 import pandas as pd
+import pytest
 from woodwork.logical_types import (
     URL,
     Boolean,
@@ -36,6 +37,9 @@ def make_ecommerce_entityset(with_integer_time_index=False):
          \\ /   .
           L     Log
     """
+    dask = pytest.importorskip("dask", reason="Dask not installed, skipping")
+    dask.config.set({"dataframe.convert-string": False})
+
     dataframes = make_ecommerce_dataframes(
         with_integer_time_index=with_integer_time_index,
     )

pyproject.toml

@@ -56,7 +56,7 @@ test = [
     "graphviz >= 0.8.4",
     "moto[all] >= 5.0.0",
     "pip >= 21.3.1",
-    "pyarrow >= 3.0.0, <13.0.0",
+    "pyarrow >= 14.0.1",
     "pympler >= 0.8",
     "pytest >= 7.1.2",
     "pytest-cov >= 3.0.0",
@@ -71,11 +71,9 @@ dask = [
     "woodwork[dask] >= 0.23.0",
 ]
 spark = [
-    "woodwork[spark] >= 0.23.0, <0.28.0",
+    "woodwork[spark] >= 0.23.0",
     "pyspark >= 3.2.2",
-    "numpy >= 1.21.0, < 1.24.0",
-    "pandas >= 1.5.0, < 2.0.0",
-    "pyarrow >= 3.0.0, < 13.0.0",
+    "pyarrow >= 14.0.1",
 ]
 updater = [
     "alteryx-open-src-update-checker >= 2.1.0"
@@ -84,7 +82,7 @@ tsfresh = [
     "featuretools-tsfresh-primitives >= 1.0.0",
 ]
 premium = [
-    "premium_primitives @ git+https://github.com/alteryx/premium_primitives.git@main",
+    "premium_primitives @ git+https://github.com/alteryx/premium_primitives.git@99e42b78e163abb0519d65184027515923a94576",
 ]
 autonormalize = [
     "autonormalize >= 2.0.1",
@@ -109,7 +107,7 @@ docs = [
     "myst-parser == 0.18.0",
     "autonormalize >= 2.0.1",
     "click >= 7.0.0",
-    "featuretools[sklearn,dask,spark,test,premium]",
+    "featuretools[dask,spark,test,premium]",
 ]
 dev = [
     "ruff >= 0.1.6",