Merge branch 'main' into v0_6

dougbrn · web-flow · commit 715e7932b802 · 2025-10-02T13:20:33.000-07:00
diff --git a/.github/workflows/asv-main.yml b/.github/workflows/asv-main.yml
@@ -26,7 +26,7 @@ jobs:
         working-directory: ${{env.WORKING_DIR}}
     steps:
     - name: Set up Python ${{env.PYTHON_VERSION}}
-      uses: actions/setup-python@v5
+      uses: actions/setup-python@v6
       with:
         python-version: ${{env.PYTHON_VERSION}}
     - name: Checkout main branch of the repository
diff --git a/.github/workflows/asv-nightly.yml b/.github/workflows/asv-nightly.yml
@@ -22,7 +22,7 @@ jobs:
         working-directory: ${{env.WORKING_DIR}}
     steps:
     - name: Set up Python ${{env.PYTHON_VERSION}}
-      uses: actions/setup-python@v5
+      uses: actions/setup-python@v6
       with:
         python-version: ${{env.PYTHON_VERSION}}
     - name: Checkout main branch of the repository
diff --git a/.github/workflows/asv-pr.yml b/.github/workflows/asv-pr.yml
@@ -28,7 +28,7 @@ jobs:
         working-directory: ${{env.WORKING_DIR}}
     steps:
     - name: Set up Python ${{env.PYTHON_VERSION}}
-      uses: actions/setup-python@v5
+      uses: actions/setup-python@v6
       with:
         python-version: ${{env.PYTHON_VERSION}}
     - name: Checkout PR branch of the repository
diff --git a/.github/workflows/build-documentation.yml b/.github/workflows/build-documentation.yml
@@ -22,7 +22,7 @@ jobs:
     steps:
     - uses: actions/checkout@v5
     - name: Set up Python 3.11
-      uses: actions/setup-python@v5
+      uses: actions/setup-python@v6
       with:
         python-version: '3.11'
     - name: Install dependencies
diff --git a/.github/workflows/pre-commit-ci.yml b/.github/workflows/pre-commit-ci.yml
@@ -18,7 +18,7 @@ jobs:
       with:
         fetch-depth: 0 
     - name: Set up Python
-      uses: actions/setup-python@v5
+      uses: actions/setup-python@v6
       with:
         python-version: '3.11'
     - name: Install dependencies
diff --git a/.github/workflows/publish-to-pypi.yml b/.github/workflows/publish-to-pypi.yml
@@ -25,7 +25,7 @@ jobs:
     steps:
     - uses: actions/checkout@v5
     - name: Set up Python
-      uses: actions/setup-python@v5
+      uses: actions/setup-python@v6
       with:
         python-version: '3.11'
     - name: Install dependencies
diff --git a/.github/workflows/smoke-test.yml b/.github/workflows/smoke-test.yml
@@ -25,7 +25,7 @@ jobs:
     steps:
     - uses: actions/checkout@v5
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v5
+      uses: actions/setup-python@v6
       with:
         python-version: ${{ matrix.python-version }}
     - name: Install dependencies
diff --git a/.github/workflows/testing-and-coverage.yml b/.github/workflows/testing-and-coverage.yml
@@ -21,7 +21,7 @@ jobs:
     steps:
     - uses: actions/checkout@v5
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v5
+      uses: actions/setup-python@v6
       with:
         python-version: ${{ matrix.python-version }}
     - name: Install dependencies
@@ -42,7 +42,7 @@ jobs:
     steps:
     - uses: actions/checkout@v5
     - name: Set up Python 3.10
-      uses: actions/setup-python@v5
+      uses: actions/setup-python@v6
       with:
         python-version: '3.10'
     - name: Install dependencies
diff --git a/.github/workflows/testing-windows.yml b/.github/workflows/testing-windows.yml
@@ -21,7 +21,7 @@ jobs:
     steps:
     - uses: actions/checkout@v5
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v5
+      uses: actions/setup-python@v6
       with:
         python-version: ${{ matrix.python-version }}
     - name: Install dependencies
diff --git a/docs/tutorials/data_manipulation.ipynb b/docs/tutorials/data_manipulation.ipynb
@@ -310,6 +310,30 @@
     "ndf[\"nested.ab\"]"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Concatenating Nested Frames\n",
+    "\n",
+    "Users can concatenate two nested data frames using panda's `concat` function. Each row from each individual frame becomes a single row in the combined frame with the nested data correctly propagated."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from pandas import concat\n",
+    "\n",
+    "ndf_part1 = generate_data(5, 20, seed=1)\n",
+    "ndf_part2 = generate_data(5, 20, seed=2)\n",
+    "ndf_joined = concat([ndf_part1, ndf_part2], ignore_index=True)\n",
+    "\n",
+    "ndf_joined"
+   ]
+  },
   {
    "cell_type": "markdown",
    "metadata": {},
@@ -363,7 +387,7 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "lsdb",
+   "display_name": "nested",
    "language": "python",
    "name": "python3"
   },
@@ -377,7 +401,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.12.8"
+   "version": "3.10.4"
   }
  },
  "nbformat": 4,
diff --git a/pyproject.toml b/pyproject.toml
@@ -21,10 +21,12 @@ dependencies = [
     "numpy>=2",
     # We use internal pd._libs.missing and experimental ArrowExtensionArray
     "pandas>=2.2.3,<2.4",
-    "pyarrow>=16",  # remove struct_field_names when upgraded to 18+
-    "universal_pathlib>=0.2",
+    "pyarrow>=16",  # remove struct_field_names() and struct_fields() when upgraded to 18+
     "Deprecated>=1.2.0",
     "wrapt>=1.12.1",
+
+    # NOTE: package PINNED at <0.3.0, see https://github.com/astronomy-commons/lsdb/issues/1047
+    "universal_pathlib>=0.2,<0.3.0",
 ]
 
 [project.urls]
diff --git a/src/nested_pandas/series/_storage/struct_list_storage.py b/src/nested_pandas/series/_storage/struct_list_storage.py
@@ -6,9 +6,9 @@
 import pyarrow as pa
 
 from nested_pandas.series.utils import (
+    align_chunked_struct_list_offsets,
     table_to_struct_array,
     transpose_list_struct_chunked,
-    validate_struct_list_array_for_equal_lengths,
 )
 
 if TYPE_CHECKING:
@@ -25,7 +25,9 @@ class StructListStorage:
         Pyarrow struct-array with all fields to be list-arrays.
         All list-values must be "aligned", e.g., have the same length.
     validate : bool (default True)
-        Check that all the lists have the same lengths for each struct-value.
+        Check that all the lists have the same lengths for each struct-value,
+        and if all list offset arrays are the same. Fails for the first check,
+        and reallocates the data for the second check.
     """
 
     _data: pa.ChunkedArray
@@ -37,8 +39,7 @@ def __init__(self, array: pa.StructArray | pa.ChunkedArray, *, validate: bool =
             raise ValueError("array must be a StructArray or ChunkedArray")
 
         if validate:
-            for chunk in array.chunks:
-                validate_struct_list_array_for_equal_lengths(chunk)
+            array = align_chunked_struct_list_offsets(array)
 
         self._data = array
 
diff --git a/src/nested_pandas/series/_storage/table_storage.py b/src/nested_pandas/series/_storage/table_storage.py
@@ -5,9 +5,9 @@
 import pyarrow as pa
 
 from nested_pandas.series.utils import (
+    align_chunked_struct_list_offsets,
     table_from_struct_array,
     table_to_struct_array,
-    validate_struct_list_array_for_equal_lengths,
 )
 
 if TYPE_CHECKING:
@@ -30,8 +30,8 @@ class TableStorage:
     def __init__(self, table: pa.Table, validate: bool = True) -> None:
         if validate:
             struct_array = table_to_struct_array(table)
-            for chunk in struct_array.iterchunks():
-                validate_struct_list_array_for_equal_lengths(chunk)
+            aligned_struct_array = align_chunked_struct_list_offsets(struct_array)
+            table = table_from_struct_array(aligned_struct_array)
 
         self._data = table
 
diff --git a/src/nested_pandas/series/ext_array.py b/src/nested_pandas/series/ext_array.py
@@ -622,7 +622,7 @@ def dropna(self) -> Self:  # type: ignore[name-defined] # noqa: F821
 
     def __arrow_array__(self, type=None):
         """Convert the extension array to a PyArrow array."""
-        # struct_array is the default "external" representation
+        # list_array is the default "external" representation
         if type is None:
             return self.list_array
         if pa.types.is_struct(type):
@@ -1003,17 +1003,9 @@ def view_fields(self, fields: str | list[str]) -> Self:  # type: ignore[name-def
         if not set(fields).issubset(self.field_names):
             raise ValueError(f"Some fields are not found, given: {fields}, available: {self.field_names}")
 
-        chunks = []
-        for chunk in self.struct_array.iterchunks():
-            chunk = cast(pa.StructArray, chunk)
-            struct_dict = {}
-            for field in fields:
-                struct_dict[field] = chunk.field(field)
-            struct_array = pa.StructArray.from_arrays(struct_dict.values(), struct_dict.keys())
-            chunks.append(struct_array)
-        pa_array = pa.chunked_array(chunks)
-
-        return type(self)(pa_array, validate=False)
+        new = self.copy()
+        new.pa_table = new.pa_table.select(fields)
+        return new
 
     def set_flat_field(self, field: str, value: ArrayLike, *, keep_dtype: bool = False) -> None:
         """Set the field from flat-array of values
@@ -1171,16 +1163,9 @@ def pop_fields(self, fields: Iterable[str]):
         if not fields.issubset(self.field_names):
             raise ValueError(f"Some fields are not found, given: {fields}, available: {self.field_names}")
 
-        if len(self.field_names) - len(fields) == 0:
+        fields_to_keep = [field for field in self.field_names if field not in fields]
+
+        if len(fields_to_keep) == 0:
             raise ValueError("Cannot delete all fields")
 
-        chunks = []
-        for chunk in self.struct_array.iterchunks():
-            chunk = cast(pa.StructArray, chunk)
-            struct_dict = {}
-            for pa_field in chunk.type:
-                if pa_field.name not in fields:
-                    struct_dict[pa_field.name] = chunk.field(pa_field.name)
-            struct_array = pa.StructArray.from_arrays(struct_dict.values(), struct_dict.keys())
-            chunks.append(struct_array)
-        self.struct_array = pa.chunked_array(chunks)
+        self.pa_table = self.pa_table.select(fields_to_keep)
diff --git a/src/nested_pandas/series/utils.py b/src/nested_pandas/series/utils.py
diff --git a/tests/nested_pandas/series/test_ext_array.py b/tests/nested_pandas/series/test_ext_array.py
diff --git a/tests/nested_pandas/series/test_series_utils.py b/tests/nested_pandas/series/test_series_utils.py
