Merge pull request #11252 from jreback/isin

COMPAT/PERF: lib.ismember_int64 on older numpies/cython not comparing correctly #11232

Author: jreback

asv_bench/benchmarks/series_methods.py

@@ -5,22 +5,21 @@ class series_isin_int64(object):
     goal_time = 0.2
 
     def setup(self):
-        self.s1 = Series(np.random.randn(10000))
-        self.s2 = Series(np.random.randint(1, 10, 10000))
         self.s3 = Series(np.random.randint(1, 10, 100000)).astype('int64')
+        self.s4 = Series(np.random.randint(1, 100, 10000000)).astype('int64')
         self.values = [1, 2]
-        self.s4 = self.s3.astype('object')
 
     def time_series_isin_int64(self):
         self.s3.isin(self.values)
 
+    def time_series_isin_int64_large(self):
+        self.s4.isin(self.values)
+
 
 class series_isin_object(object):
     goal_time = 0.2
 
     def setup(self):
-        self.s1 = Series(np.random.randn(10000))
-        self.s2 = Series(np.random.randint(1, 10, 10000))
         self.s3 = Series(np.random.randint(1, 10, 100000)).astype('int64')
         self.values = [1, 2]
         self.s4 = self.s3.astype('object')
@@ -71,4 +70,4 @@ def setup(self):
 
     def time_series_nsmallest2(self):
         self.s2.nsmallest(3, take_last=True)
-        self.s2.nsmallest(3, take_last=False)
+        self.s2.nsmallest(3, take_last=False)

ci/requirements-2.6.build

@@ -1,4 +1,4 @@
-numpy=1.7.0
+numpy=1.7.1
 cython=0.19.1
 dateutil=1.5
 pytz=2013b

ci/requirements-2.6.run

@@ -1,14 +1,16 @@
-numpy=1.7.0
+numpy=1.7.1
 dateutil=1.5
 pytz=2013b
 scipy=0.11.0
 xlwt=0.7.5
 xlrd=0.9.2
 statsmodels=0.4.3
+bottleneck=0.8.0
+numexpr=2.2.2
+pytables=3.0.0
 html5lib=1.0b2
 beautiful-soup=4.2.0
 psycopg2=2.5.1
-numexpr=1.4.2
 pymysql=0.6.0
 sqlalchemy=0.7.8
 xlsxwriter=0.4.6

ci/requirements-2.7.build

@@ -1,4 +1,4 @@
 dateutil=2.1
 pytz=2013b
-numpy=1.7.1
+numpy
 cython=0.19.1

ci/requirements-2.7.run

@@ -1,18 +1,18 @@
 dateutil=2.1
 pytz=2013b
-numpy=1.7.1
+numpy
 xlwt=0.7.5
-numexpr=2.2.2
-pytables=3.0.0
-matplotlib=1.3.1
+numexpr
+pytables
+matplotlib
 openpyxl=1.6.2
 xlrd=0.9.2
 sqlalchemy=0.9.6
 lxml=3.2.1
 scipy
 xlsxwriter=0.4.6
 boto=2.36.0
-bottleneck=0.8.0
+bottleneck
 psycopg2=2.5.2
 patsy
 pymysql=0.6.3

ci/requirements-2.7_SLOW.build

@@ -1,4 +1,4 @@
 python-dateutil
 pytz
-numpy
+numpy=1.8.2
 cython

ci/requirements-2.7_SLOW.run

@@ -1,7 +1,7 @@
 python-dateutil
 pytz
-numpy
-matplotlib
+numpy=1.8.2
+matplotlib=1.3.1
 scipy
 patsy
 statsmodels

doc/source/install.rst

@@ -214,7 +214,7 @@ Dependencies
 ------------
 
 * `setuptools <http://pythonhosted.org/setuptools>`__
-* `NumPy <http://www.numpy.org>`__: 1.7.0 or higher
+* `NumPy <http://www.numpy.org>`__: 1.7.1 or higher
 * `python-dateutil <http://labix.org/python-dateutil>`__ 1.5 or higher
 * `pytz <http://pytz.sourceforge.net/>`__
    * Needed for time zone support

doc/source/whatsnew/v0.17.0.txt

@@ -1034,6 +1034,7 @@ Bug Fixes
 ~~~~~~~~~
 
 - Bug in incorrection computation of ``.mean()`` on ``timedelta64[ns]`` because of overflow (:issue:`9442`)
+- Bug in  ``.isin`` on older numpies (:issue: `11232`)
 - Bug in ``DataFrame.to_html(index=False)`` renders unnecessary ``name`` row (:issue:`10344`)
 - Bug in ``DataFrame.to_latex()`` the ``column_format`` argument could not be passed (:issue:`9402`)
 - Bug in ``DatetimeIndex`` when localizing with ``NaT`` (:issue:`10477`)

pandas/core/algorithms.py

@@ -6,6 +6,7 @@
 from warnings import warn
 import numpy as np
 
+from pandas import compat, lib, _np_version_under1p8
 import pandas.core.common as com
 import pandas.algos as algos
 import pandas.hashtable as htable
@@ -66,6 +67,54 @@ def unique(values):
     return _hashtable_algo(f, values.dtype)
 
 
+def isin(comps, values):
+    """
+    Compute the isin boolean array
+
+    Parameters
+    ----------
+    comps: array-like
+    values: array-like
+
+    Returns
+    -------
+    boolean array same length as comps
+    """
+
+    if not com.is_list_like(comps):
+        raise TypeError("only list-like objects are allowed to be passed"
+                        " to isin(), you passed a "
+                        "[{0}]".format(type(comps).__name__))
+    comps = np.asarray(comps)
+    if not com.is_list_like(values):
+        raise TypeError("only list-like objects are allowed to be passed"
+                        " to isin(), you passed a "
+                        "[{0}]".format(type(values).__name__))
+
+    # GH11232
+    # work-around for numpy < 1.8 and comparisions on py3
+    # faster for larger cases to use np.in1d
+    if (_np_version_under1p8 and compat.PY3) or len(comps) > 1000000:
+        f = lambda x, y: np.in1d(x,np.asarray(list(y)))
+    else:
+        f = lambda x, y: lib.ismember_int64(x,set(y))
+
+    # may need i8 conversion for proper membership testing
+    if com.is_datetime64_dtype(comps):
+        from pandas.tseries.tools import to_datetime
+        values = to_datetime(values)._values.view('i8')
+        comps = comps.view('i8')
+    elif com.is_timedelta64_dtype(comps):
+        from pandas.tseries.timedeltas import to_timedelta
+        values = to_timedelta(values)._values.view('i8')
+        comps = comps.view('i8')
+    elif com.is_int64_dtype(comps):
+        pass
+    else:
+        f = lambda x, y: lib.ismember(x, set(values))
+
+    return f(comps, values)
+
 def _hashtable_algo(f, dtype, return_dtype=None):
     """
     f(HashTable, type_caster) -> result