Merge pull request #146 from pandas-dev/master

Sync Fork from Upstream Repo

Author: sthagen

.pre-commit-config.yaml

@@ -2,7 +2,7 @@ minimum_pre_commit_version: 2.9.2
 exclude: ^LICENSES/|\.(html|csv|svg)$
 repos:
 -   repo: https://github.com/MarcoGorelli/absolufy-imports
-    rev: v0.2.1
+    rev: v0.3.0
     hooks:
     -   id: absolufy-imports
         files: ^pandas/
@@ -33,7 +33,7 @@ repos:
         exclude: ^pandas/_libs/src/(klib|headers)/
         args: [--quiet, '--extensions=c,h', '--headers=h', --recursive, '--filter=-readability/casting,-runtime/int,-build/include_subdir']
 -   repo: https://gitlab.com/pycqa/flake8
-    rev: 3.8.4
+    rev: 3.9.0
     hooks:
     -   id: flake8
         additional_dependencies: [flake8-comprehensions>=3.1.0]

doc/source/user_guide/window.rst

@@ -168,7 +168,7 @@ parameter:
 =============  ====================
 Value          Behavior
 =============  ====================
-``right'``     close right endpoint
+``'right'``     close right endpoint
 ``'left'``     close left endpoint
 ``'both'``     close both endpoints
 ``'neither'``  open endpoints
@@ -214,7 +214,7 @@ ending indices of the windows. Additionally, ``num_values``, ``min_periods``, ``
 and will automatically be passed to ``get_window_bounds`` and the defined method must
 always accept these arguments.
 
-For example, if we have the following :class:``DataFrame``:
+For example, if we have the following :class:`DataFrame`
 
 .. ipython:: python
 
@@ -370,8 +370,8 @@ two :class:`Series` or any combination of :class:`DataFrame`/:class:`Series` or
   with the passed Series, thus returning a DataFrame.
 * :class:`DataFrame`/:class:`DataFrame`: by default compute the statistic for matching column
   names, returning a DataFrame. If the keyword argument ``pairwise=True`` is
-  passed then computes the statistic for each pair of columns, returning a
-  ``MultiIndexed DataFrame`` whose ``index`` are the dates in question (see :ref:`the next section
+  passed then computes the statistic for each pair of columns, returning a :class:`DataFrame` with a
+  :class:`MultiIndex` whose values are the dates in question (see :ref:`the next section
   <window.corr_pairwise>`).
 
 For example:

doc/source/whatsnew/v1.3.0.rst

@@ -300,7 +300,7 @@ Optional libraries below the lowest tested version may still work, but are not c
 +=================+=================+=========+
 | beautifulsoup4  | 4.6.0           |         |
 +-----------------+-----------------+---------+
-| fastparquet     | 0.3.2           |         |
+| fastparquet     | 0.4.0           |    X    |
 +-----------------+-----------------+---------+
 | fsspec          | 0.7.4           |         |
 +-----------------+-----------------+---------+
@@ -367,8 +367,9 @@ Deprecations
 - Deprecated :meth:`core.window.ewm.ExponentialMovingWindow.vol` (:issue:`39220`)
 - Using ``.astype`` to convert between ``datetime64[ns]`` dtype and :class:`DatetimeTZDtype` is deprecated and will raise in a future version, use ``obj.tz_localize`` or ``obj.dt.tz_localize`` instead (:issue:`38622`)
 - Deprecated casting ``datetime.date`` objects to ``datetime64`` when used as ``fill_value`` in :meth:`DataFrame.unstack`, :meth:`DataFrame.shift`, :meth:`Series.shift`, and :meth:`DataFrame.reindex`, pass ``pd.Timestamp(dateobj)`` instead (:issue:`39767`)
-- Deprecated :meth:`.Styler.set_na_rep` and :meth:`.Styler.set_precision` in favour of :meth:`.Styler.format` with ``na_rep`` and ``precision`` as existing and new input arguments respectively (:issue:`40134`)
+- Deprecated :meth:`.Styler.set_na_rep` and :meth:`.Styler.set_precision` in favour of :meth:`.Styler.format` with ``na_rep`` and ``precision`` as existing and new input arguments respectively (:issue:`40134`, :issue:`40425`)
 - Deprecated allowing partial failure in :meth:`Series.transform` and :meth:`DataFrame.transform` when ``func`` is list-like or dict-like; will raise if any function fails on a column in a future version (:issue:`40211`)
+- Deprecated support for ``np.ma.mrecords.MaskedRecords`` in the :class:`DataFrame` constructor, pass ``{name: data[name] for name in data.dtype.names}`` instead (:issue:`40363`)
 
 .. ---------------------------------------------------------------------------
 
@@ -455,7 +456,7 @@ Conversion
 - Bug in :meth:`Series.view` and :meth:`Index.view` when converting between datetime-like (``datetime64[ns]``, ``datetime64[ns, tz]``, ``timedelta64``, ``period``) dtypes (:issue:`39788`)
 - Bug in creating a :class:`DataFrame` from an empty ``np.recarray`` not retaining the original dtypes (:issue:`40121`)
 - Bug in :class:`DataFrame` failing to raise ``TypeError`` when constructing from a ``frozenset`` (:issue:`40163`)
--
+- Bug in :class:`Index` construction silently ignoring a passed ``dtype`` when the data cannot be cast to that dtype (:issue:`21311`)
 
 Strings
 ^^^^^^^
@@ -592,6 +593,7 @@ Reshaping
 - Bug in :meth:`DataFrame.pivot_table` returning a ``MultiIndex`` for a single value when operating on and empty ``DataFrame`` (:issue:`13483`)
 - Allow :class:`Index` to be passed to the :func:`numpy.all` function (:issue:`40180`)
 - Bug in :meth:`DataFrame.stack` not preserving ``CategoricalDtype`` in a ``MultiIndex`` (:issue:`36991`)
+- Bug in :func:`to_datetime` raising error when input sequence contains unhashable items (:issue:`39756`)
 
 Sparse
 ^^^^^^
@@ -625,6 +627,7 @@ Other
 - Bug in :class:`Styler` where multiple elements in CSS-selectors were not correctly added to ``table_styles`` (:issue:`39942`)
 - Bug in :meth:`DataFrame.equals`, :meth:`Series.equals`, :meth:`Index.equals` with object-dtype containing ``np.datetime64("NaT")`` or ``np.timedelta64("NaT")`` (:issue:`39650`)
 - Bug in :func:`pandas.util.show_versions` where console JSON output was not proper JSON (:issue:`39701`)
+- Bug in :meth:`DataFrame.convert_dtypes` incorrectly raised ValueError when called on an empty DataFrame (:issue:`40393`)
 
 
 .. ---------------------------------------------------------------------------

pandas/_libs/parsers.pyx

@@ -284,21 +284,10 @@ cdef extern from "parser/tokenizer.h":
 
 
 cdef extern from "parser/io.h":
-    void *new_mmap(char *fname)
-    int del_mmap(void *src)
-    void* buffer_mmap_bytes(void *source, size_t nbytes,
-                            size_t *bytes_read, int *status)
-
-    void *new_file_source(char *fname, size_t buffer_size) except NULL
-
     void *new_rd_source(object obj) except NULL
 
-    int del_file_source(void *src)
     int del_rd_source(void *src)
 
-    void* buffer_file_bytes(void *source, size_t nbytes,
-                            size_t *bytes_read, int *status)
-
     void* buffer_rd_bytes(void *source, size_t nbytes,
                           size_t *bytes_read, int *status, const char *encoding_errors)
 

pandas/_libs/src/parser/io.c

@@ -9,83 +9,10 @@ The full license is in the LICENSE file, distributed with this software.
 
 #include "io.h"
 
-#include <sys/stat.h>
-#include <fcntl.h>
-
-#ifndef O_BINARY
-#define O_BINARY 0
-#endif  // O_BINARY
-
-#ifdef _WIN32
-#define USE_WIN_UTF16
-#include <Windows.h>
-#endif
-
 /*
   On-disk FILE, uncompressed
 */
 
-void *new_file_source(char *fname, size_t buffer_size) {
-    file_source *fs = (file_source *)malloc(sizeof(file_source));
-    if (fs == NULL) {
-        PyErr_NoMemory();
-        return NULL;
-    }
-
-#ifdef USE_WIN_UTF16
-    // Fix gh-15086 properly - convert UTF8 to UTF16 that Windows widechar API
-    // accepts. This is needed because UTF8 might _not_ be convertible to MBCS
-    // for some conditions, as MBCS is locale-dependent, and not all unicode
-    // symbols can be expressed in it.
-    {
-        wchar_t* wname = NULL;
-        int required = MultiByteToWideChar(CP_UTF8, 0, fname, -1, NULL, 0);
-        if (required == 0) {
-            free(fs);
-            PyErr_SetFromWindowsErr(0);
-            return NULL;
-        }
-        wname = (wchar_t*)malloc(required * sizeof(wchar_t));
-        if (wname == NULL) {
-            free(fs);
-            PyErr_NoMemory();
-            return NULL;
-        }
-        if (MultiByteToWideChar(CP_UTF8, 0, fname, -1, wname, required) <
-                                                                required) {
-            free(wname);
-            free(fs);
-            PyErr_SetFromWindowsErr(0);
-            return NULL;
-        }
-        fs->fd = _wopen(wname, O_RDONLY | O_BINARY);
-        free(wname);
-    }
-#else
-    fs->fd = open(fname, O_RDONLY | O_BINARY);
-#endif
-    if (fs->fd == -1) {
-        free(fs);
-        PyErr_SetFromErrnoWithFilename(PyExc_OSError, fname);
-        return NULL;
-    }
-
-    // Only allocate this heap memory if we are not memory-mapping the file
-    fs->buffer = (char *)malloc((buffer_size + 1) * sizeof(char));
-
-    if (fs->buffer == NULL) {
-        close(fs->fd);
-        free(fs);
-        PyErr_NoMemory();
-        return NULL;
-    }
-
-    memset(fs->buffer, '\0', buffer_size + 1);
-    fs->size = buffer_size;
-
-    return (void *)fs;
-}
-
 void *new_rd_source(PyObject *obj) {
     rd_source *rds = (rd_source *)malloc(sizeof(rd_source));
 
@@ -108,17 +35,6 @@ void *new_rd_source(PyObject *obj) {
 
  */
 
-int del_file_source(void *ptr) {
-    file_source *fs = ptr;
-    if (fs == NULL) return 0;
-
-    free(fs->buffer);
-    close(fs->fd);
-    free(fs);
-
-    return 0;
-}
-
 int del_rd_source(void *rds) {
     Py_XDECREF(RDS(rds)->obj);
     Py_XDECREF(RDS(rds)->buffer);
@@ -133,35 +49,6 @@ int del_rd_source(void *rds) {
 
  */
 
-void *buffer_file_bytes(void *source, size_t nbytes, size_t *bytes_read,
-                        int *status) {
-    file_source *fs = FS(source);
-    ssize_t rv;
-
-    if (nbytes > fs->size) {
-        nbytes = fs->size;
-    }
-
-    rv = read(fs->fd, fs->buffer, nbytes);
-    switch (rv) {
-    case -1:
-        *status = CALLING_READ_FAILED;
-        *bytes_read = 0;
-        return NULL;
-    case 0:
-        *status = REACHED_EOF;
-        *bytes_read = 0;
-        return NULL;
-    default:
-        *status = 0;
-        *bytes_read = rv;
-        fs->buffer[rv] = '\0';
-        break;
-    }
-
-    return (void *)fs->buffer;
-}
-
 void *buffer_rd_bytes(void *source, size_t nbytes, size_t *bytes_read,
                       int *status, const char *encoding_errors) {
     PyGILState_STATE state;
@@ -218,98 +105,3 @@ void *buffer_rd_bytes(void *source, size_t nbytes, size_t *bytes_read,
 
     return retval;
 }
-
-#ifdef HAVE_MMAP
-
-#include <sys/mman.h>
-
-void *new_mmap(char *fname) {
-    memory_map *mm;
-    struct stat stat;
-    size_t filesize;
-
-    mm = (memory_map *)malloc(sizeof(memory_map));
-    if (mm == NULL) {
-        return NULL;
-    }
-    mm->fd = open(fname, O_RDONLY | O_BINARY);
-    if (mm->fd == -1) {
-        free(mm);
-        return NULL;
-    }
-
-    if (fstat(mm->fd, &stat) == -1) {
-        close(mm->fd);
-        free(mm);
-        return NULL;
-    }
-    filesize = stat.st_size; /* XXX This might be 32 bits. */
-
-    mm->memmap = mmap(NULL, filesize, PROT_READ, MAP_SHARED, mm->fd, 0);
-    if (mm->memmap == MAP_FAILED) {
-        close(mm->fd);
-        free(mm);
-        return NULL;
-    }
-
-    mm->size = (off_t)filesize;
-    mm->position = 0;
-
-    return mm;
-}
-
-int del_mmap(void *ptr) {
-    memory_map *mm = ptr;
-
-    if (mm == NULL) return 0;
-
-    munmap(mm->memmap, mm->size);
-    close(mm->fd);
-    free(mm);
-
-    return 0;
-}
-
-void *buffer_mmap_bytes(void *source, size_t nbytes, size_t *bytes_read,
-                        int *status) {
-    void *retval;
-    memory_map *src = source;
-    size_t remaining = src->size - src->position;
-
-    if (remaining == 0) {
-        *bytes_read = 0;
-        *status = REACHED_EOF;
-        return NULL;
-    }
-
-    if (nbytes > remaining) {
-        nbytes = remaining;
-    }
-
-    retval = src->memmap + src->position;
-
-    /* advance position in mmap data structure */
-    src->position += nbytes;
-
-    *bytes_read = nbytes;
-    *status = 0;
-
-    return retval;
-}
-
-#else
-
-/* kludgy */
-
-void *new_mmap(char *fname) { return NULL; }
-
-int del_mmap(void *src) { return 0; }
-
-/* don't use this! */
-
-void *buffer_mmap_bytes(void *source, size_t nbytes, size_t *bytes_read,
-                        int *status) {
-    return NULL;
-}
-
-#endif  // HAVE_MMAP