Merge remote-tracking branch 'upstream/master'

Author: ccharlesgb

README.md

@@ -233,3 +233,5 @@ You can also triage issues which may include reproducing bug reports, or asking
 Or maybe through using pandas you have an idea of your own or are looking for something in the documentation and thinking ‘this can be improved’...you can do something about it!
 
 Feel free to ask questions on the [mailing list](https://groups.google.com/forum/?fromgroups#!forum/pydata) or on [Gitter](https://gitter.im/pydata/pandas).
+
+As contributors and maintainers to this project, you are expected to abide by pandas' code of conduct. More information can be found at: [Contributor Code of Conduct](https://github.com/pandas-dev/pandas/blob/master/.github/CODE_OF_CONDUCT.md)

doc/source/whatsnew/v0.25.1.rst

@@ -84,7 +84,7 @@ Indexing
 
 - Bug in partial-string indexing returning a NumPy array rather than a ``Series`` when indexing with a scalar like ``.loc['2015']`` (:issue:`27516`)
 - Break reference cycle involving :class:`Index` to allow garbage collection of :class:`Index` objects without running the GC. (:issue:`27585`)
--
+- Fix regression in assigning values to a single column of a DataFrame with a ``MultiIndex`` columns (:issue:`27841`).
 -
 
 Missing
@@ -133,7 +133,7 @@ Reshaping
 
 Sparse
 ^^^^^^
-
+- Bug in reductions for :class:`Series` with Sparse dtypes (:issue:`27080`)
 -
 -
 -

doc/source/whatsnew/v1.0.0.rst

@@ -165,6 +165,7 @@ Plotting
 
 - Bug in :meth:`Series.plot` not able to plot boolean values (:issue:`23719`)
 -
+- Bug in :meth:`DataFrame.plot` when ``kind='box'`` and data contains datetime or timedelta data. These types are now automatically dropped (:issue:`22799`)
 
 Groupby/resample/rolling
 ^^^^^^^^^^^^^^^^^^^^^^^^

pandas/core/arrays/categorical.py

@@ -1,3 +1,4 @@
+import operator
 from shutil import get_terminal_size
 import textwrap
 from typing import Type, Union, cast
@@ -77,7 +78,9 @@
 )
 
 
-def _cat_compare_op(opname):
+def _cat_compare_op(op):
+    opname = "__{op}__".format(op=op.__name__)
+
     def f(self, other):
         # On python2, you can usually compare any type to any type, and
         # Categoricals can be seen as a custom type, but having different
@@ -88,6 +91,9 @@ def f(self, other):
             return NotImplemented
 
         other = lib.item_from_zerodim(other)
+        if is_list_like(other) and len(other) != len(self):
+            # TODO: Could this fail if the categories are listlike objects?
+            raise ValueError("Lengths must match.")
 
         if not self.ordered:
             if opname in ["__lt__", "__gt__", "__le__", "__ge__"]:
@@ -1240,12 +1246,12 @@ def map(self, mapper):
                 new_categories = new_categories.insert(len(new_categories), np.nan)
             return np.take(new_categories, self._codes)
 
-    __eq__ = _cat_compare_op("__eq__")
-    __ne__ = _cat_compare_op("__ne__")
-    __lt__ = _cat_compare_op("__lt__")
-    __gt__ = _cat_compare_op("__gt__")
-    __le__ = _cat_compare_op("__le__")
-    __ge__ = _cat_compare_op("__ge__")
+    __eq__ = _cat_compare_op(operator.eq)
+    __ne__ = _cat_compare_op(operator.ne)
+    __lt__ = _cat_compare_op(operator.lt)
+    __gt__ = _cat_compare_op(operator.gt)
+    __le__ = _cat_compare_op(operator.le)
+    __ge__ = _cat_compare_op(operator.ge)
 
     # for Series/ndarray like compat
     @property

pandas/core/arrays/datetimelike.py

@@ -22,7 +22,6 @@
     is_datetime64tz_dtype,
     is_datetime_or_timedelta_dtype,
     is_dtype_equal,
-    is_extension_array_dtype,
     is_float_dtype,
     is_integer_dtype,
     is_list_like,
@@ -1230,29 +1229,17 @@ def __add__(self, other):
             if not is_period_dtype(self):
                 maybe_integer_op_deprecated(self)
             result = self._addsub_int_array(other, operator.add)
-        elif is_float_dtype(other):
-            # Explicitly catch invalid dtypes
-            raise TypeError(
-                "cannot add {dtype}-dtype to {cls}".format(
-                    dtype=other.dtype, cls=type(self).__name__
-                )
-            )
-        elif is_period_dtype(other):
-            # if self is a TimedeltaArray and other is a PeriodArray with
-            #  a timedelta-like (i.e. Tick) freq, this operation is valid.
-            #  Defer to the PeriodArray implementation.
-            # In remaining cases, this will end up raising TypeError.
-            return NotImplemented
-        elif is_extension_array_dtype(other):
-            # Categorical op will raise; defer explicitly
-            return NotImplemented
-        else:  # pragma: no cover
+        else:
+            # Includes Categorical, other ExtensionArrays
+            # For PeriodDtype, if self is a TimedeltaArray and other is a
+            #  PeriodArray with  a timedelta-like (i.e. Tick) freq, this
+            #  operation is valid.  Defer to the PeriodArray implementation.
+            #  In remaining cases, this will end up raising TypeError.
             return NotImplemented
 
         if is_timedelta64_dtype(result) and isinstance(result, np.ndarray):
             from pandas.core.arrays import TimedeltaArray
 
-            # TODO: infer freq?
             return TimedeltaArray(result)
         return result
 
@@ -1302,29 +1289,13 @@ def __sub__(self, other):
             if not is_period_dtype(self):
                 maybe_integer_op_deprecated(self)
             result = self._addsub_int_array(other, operator.sub)
-        elif isinstance(other, ABCIndexClass):
-            raise TypeError(
-                "cannot subtract {cls} and {typ}".format(
-                    cls=type(self).__name__, typ=type(other).__name__
-                )
-            )
-        elif is_float_dtype(other):
-            # Explicitly catch invalid dtypes
-            raise TypeError(
-                "cannot subtract {dtype}-dtype from {cls}".format(
-                    dtype=other.dtype, cls=type(self).__name__
-                )
-            )
-        elif is_extension_array_dtype(other):
-            # Categorical op will raise; defer explicitly
-            return NotImplemented
-        else:  # pragma: no cover
+        else:
+            # Includes ExtensionArrays, float_dtype
             return NotImplemented
 
         if is_timedelta64_dtype(result) and isinstance(result, np.ndarray):
             from pandas.core.arrays import TimedeltaArray
 
-            # TODO: infer freq?
             return TimedeltaArray(result)
         return result
 

pandas/core/arrays/datetimes.py

@@ -223,8 +223,6 @@ def wrapper(self, other):
                 result = op(self.view("i8"), other.view("i8"))
                 o_mask = other._isnan
 
-            result = com.values_from_object(result)
-
             if o_mask.any():
                 result[o_mask] = nat_result
 

pandas/core/arrays/integer.py

@@ -21,7 +21,7 @@
     is_scalar,
 )
 from pandas.core.dtypes.dtypes import register_extension_dtype
-from pandas.core.dtypes.generic import ABCIndexClass, ABCSeries
+from pandas.core.dtypes.generic import ABCDataFrame, ABCIndexClass, ABCSeries
 from pandas.core.dtypes.missing import isna, notna
 
 from pandas.core import nanops, ops
@@ -592,25 +592,29 @@ def _values_for_argsort(self) -> np.ndarray:
 
     @classmethod
     def _create_comparison_method(cls, op):
-        def cmp_method(self, other):
+        op_name = op.__name__
 
-            op_name = op.__name__
-            mask = None
+        def cmp_method(self, other):
 
-            if isinstance(other, (ABCSeries, ABCIndexClass)):
+            if isinstance(other, (ABCDataFrame, ABCSeries, ABCIndexClass)):
                 # Rely on pandas to unbox and dispatch to us.
                 return NotImplemented
 
+            other = lib.item_from_zerodim(other)
+            mask = None
+
             if isinstance(other, IntegerArray):
                 other, mask = other._data, other._mask
 
             elif is_list_like(other):
                 other = np.asarray(other)
-                if other.ndim > 0 and len(self) != len(other):
+                if other.ndim > 1:
+                    raise NotImplementedError(
+                        "can only perform ops with 1-d structures"
+                    )
+                if len(self) != len(other):
                     raise ValueError("Lengths must match to compare")
 
-            other = lib.item_from_zerodim(other)
-
             # numpy will show a DeprecationWarning on invalid elementwise
             # comparisons, this will raise in the future
             with warnings.catch_warnings():
@@ -683,31 +687,31 @@ def _maybe_mask_result(self, result, mask, other, op_name):
 
     @classmethod
     def _create_arithmetic_method(cls, op):
-        def integer_arithmetic_method(self, other):
+        op_name = op.__name__
 
-            op_name = op.__name__
-            mask = None
+        def integer_arithmetic_method(self, other):
 
-            if isinstance(other, (ABCSeries, ABCIndexClass)):
+            if isinstance(other, (ABCDataFrame, ABCSeries, ABCIndexClass)):
                 # Rely on pandas to unbox and dispatch to us.
                 return NotImplemented
 
-            if getattr(other, "ndim", 0) > 1:
-                raise NotImplementedError("can only perform ops with 1-d structures")
+            other = lib.item_from_zerodim(other)
+            mask = None
 
             if isinstance(other, IntegerArray):
                 other, mask = other._data, other._mask
 
-            elif getattr(other, "ndim", None) == 0:
-                other = other.item()
-
             elif is_list_like(other):
                 other = np.asarray(other)
-                if not other.ndim:
-                    other = other.item()
-                elif other.ndim == 1:
-                    if not (is_float_dtype(other) or is_integer_dtype(other)):
-                        raise TypeError("can only perform ops with numeric values")
+                if other.ndim > 1:
+                    raise NotImplementedError(
+                        "can only perform ops with 1-d structures"
+                    )
+                if len(self) != len(other):
+                    raise ValueError("Lengths must match")
+                if not (is_float_dtype(other) or is_integer_dtype(other)):
+                    raise TypeError("can only perform ops with numeric values")
+
             else:
                 if not (is_float(other) or is_integer(other)):
                     raise TypeError("can only perform ops with numeric values")

pandas/core/arrays/sparse.py

@@ -1693,6 +1693,9 @@ def __array_ufunc__(self, ufunc, method, *inputs, **kwargs):
                     for sp_value, fv in zip(sp_values, fill_value)
                 )
                 return arrays
+            elif is_scalar(sp_values):
+                # e.g. reductions
+                return sp_values
 
             return self._simple_new(
                 sp_values, self.sp_index, SparseDtype(sp_values.dtype, fill_value)

pandas/core/frame.py

@@ -770,7 +770,7 @@ def style(self):
     _shared_docs[
         "items"
     ] = r"""
-        Iterator over (column name, Series) pairs.
+        Iterate over (column name, Series) pairs.
 
         Iterates over the DataFrame columns, returning a tuple with
         the column name and the content as a Series.
@@ -845,8 +845,8 @@ def iterrows(self):
 
         See Also
         --------
-        itertuples : Iterate over DataFrame rows as namedtuples of the values.
-        items : Iterate over (column name, Series) pairs.
+        DataFrame.itertuples : Iterate over DataFrame rows as namedtuples of the values.
+        DataFrame.items : Iterate over (column name, Series) pairs.
 
         Notes
         -----
@@ -3447,15 +3447,14 @@ def _get_info_slice(obj, indexer):
         if not is_list_like(exclude):
             exclude = (exclude,) if exclude is not None else ()
 
-        selection = tuple(map(frozenset, (include, exclude)))
+        selection = (frozenset(include), frozenset(exclude))
 
         if not any(selection):
             raise ValueError("at least one of include or exclude must be nonempty")
 
         # convert the myriad valid dtypes object to a single representation
-        include, exclude = map(
-            lambda x: frozenset(map(infer_dtype_from_object, x)), selection
-        )
+        include = frozenset(infer_dtype_from_object(x) for x in include)
+        exclude = frozenset(infer_dtype_from_object(x) for x in exclude)
         for dtypes in (include, exclude):
             invalidate_string_dtypes(dtypes)
 

pandas/core/generic.py

@@ -2179,6 +2179,12 @@ def _repr_data_resource_(self):
     ...     df1.to_excel(writer, sheet_name='Sheet_name_1')
     ...     df2.to_excel(writer, sheet_name='Sheet_name_2')
 
+    ExcelWriter can also be used to append to an existing Excel file:
+
+    >>> with pd.ExcelWriter('output.xlsx',
+    ...                     mode='a') as writer:  # doctest: +SKIP
+    ...     df.to_excel(writer, sheet_name='Sheet_name_3')
+
     To set the library that is used to write the Excel file,
     you can pass the `engine` keyword (the default engine is
     automatically chosen depending on the file extension):