Apply Nugend's fixes

.gitignore

@@ -55,6 +55,7 @@ dist
 # coverage
 .coverage
 __conda_version__.txt
+.pytest_cache
 
 # OS generated files #
 ######################

CHANGELOG.txt

@@ -1,3 +1,11 @@
+------------------------------------------------------------------------------
+  qPython 2.0.0 [2018.08.31]
+------------------------------------------------------------------------------
+
+  - BREAKING CHANGE: Renamed QConnection.async to QConnection.send to
+    support Python 3.7 because of async keyword promotion.
+  - Fix DeprecationWarnings and FutureWarnings
+
 ------------------------------------------------------------------------------
   qPython 1.3.0 [2017.03.xx]
 ------------------------------------------------------------------------------

doc/source/connection.rst

@@ -49,7 +49,7 @@ to numpy `datetime64`/`timedelta64` representation.
 
 Conversion options can be also overwritten while executing 
 synchronous/asynchronous queries (:meth:`~qpython.qconnection.QConnection.sync`,
-:meth:`~qpython.qconnection.QConnection.async`) or retrieving data from q
+:meth:`~qpython.qconnection.QConnection.send`) or retrieving data from q
 (:meth:`~qpython.qconnection.QConnection.receive`).
 
 

doc/source/queries.rst

@@ -16,7 +16,7 @@ The `qPython` library provides following API methods in the
 
 - :func:`~qpython.qconnection.QConnection.sync` - executes a synchronous query 
   against the remote q service,
-- :func:`~qpython.qconnection.QConnection.async` - executes an asynchronous 
+- :func:`~qpython.qconnection.QConnection.send` - executes an asynchronous 
   query against the remote q service,
 - :func:`~qpython.qconnection.QConnection.query` - executes a query against the
   remote q service.
@@ -65,17 +65,17 @@ Asynchronous queries
 
 Calls a anonymous function with a single parameter:
 
-    >>> q.async('{til x}', 10)
+    >>> q.send('{til x}', 10)
 
 Executes a q expression:
 
-    >>> q.async('til 10')
+    >>> q.send('til 10')
 
 .. note:: The asynchronous query doesn't fetch the result. Query result has
           to be retrieved explicitly.
 
 In order to retrieve query result (for the 
-:func:`~qpython.qconnection.QConnection.async` or 
+:func:`~qpython.qconnection.QConnection.send` or 
 :func:`~qpython.qconnection.QConnection.query` methods), one has to call:
 
 - :func:`~qpython.qconnection.QConnection.receive` method, which reads next 
@@ -96,7 +96,7 @@ QMessage: message type: 2, data size: 13, is_compressed: False, data: 10
 10
 
 >>> q.sync('asynchMult:{[a;b] res:a*b; (neg .z.w)(res) }')
->>> q.async('asynchMult', 2, 3)
+>>> q.send('asynchMult', 2, 3)
 >>> print(q.receive())
 6
 
@@ -114,7 +114,7 @@ Type conversions configuration
 Type conversion options can be overwritten while:
 
 - executing synchronous query: :meth:`~qpython.qconnection.QConnection.sync`
-- executing asynchronous query: :meth:`~qpython.qconnection.QConnection.async`
+- executing asynchronous query: :meth:`~qpython.qconnection.QConnection.send`
 - retrieving data from q: :meth:`~qpython.qconnection.QConnection.receive`
 
 These methods accepts the `options` keywords arguments::

doc/source/usage-examples.rst

@@ -124,7 +124,7 @@ Following example presents how to execute simple, asynchronous query against a r
                 a = random.randint(1, 100)
                 b = random.randint(1, 100)
                 print('Asynchronous call with queryid=%s with arguments: %s, %s' % (x, a, b))
-                q.async('asynchMult', x, a, b);
+                q.send('asynchMult', x, a, b);
 
             time.sleep(1)
         finally:

qpython/_pandas.py

@@ -58,8 +58,8 @@ def _read_dictionary(self, qtype = QDICTIONARY):
 
                 return table
             else:
-                keys = keys if not isinstance(keys, pandas.Series) else keys.as_matrix()
-                values = values if not isinstance(values, pandas.Series) else values.as_matrix()
+                keys = keys if not isinstance(keys, pandas.Series) else keys.values
+                values = values if not isinstance(values, pandas.Series) else values.values
                 return QDictionary(keys, values)
         else:
             return QReader._read_dictionary(self, qtype = qtype)
@@ -168,19 +168,19 @@ def _write_pandas_series(self, data, qtype = None):
             raise QWriterException('Unable to serialize pandas series %s' % data)
 
         if qtype == QGENERAL_LIST:
-            self._write_generic_list(data.as_matrix())
+            self._write_generic_list(data.values)
         elif qtype == QCHAR:
-            self._write_string(data.replace(numpy.nan, ' ').as_matrix().astype(numpy.string_).tostring())
+            self._write_string(data.replace(numpy.nan, ' ').values.astype(numpy.string_).tostring())
         elif data.dtype.type not in (numpy.datetime64, numpy.timedelta64):
             data = data.fillna(QNULLMAP[-abs(qtype)][1])
-            data = data.as_matrix()
+            data = data.values
 
             if PY_TYPE[qtype] != data.dtype:
                 data = data.astype(PY_TYPE[qtype])
 
             self._write_list(data, qtype = qtype)
         else:
-            data = data.as_matrix()
+            data = data.values
             data = data.astype(TEMPORAL_Q_TYPE[qtype])
             self._write_list(data, qtype = qtype)
 

qpython/qconnection.py

@@ -306,10 +306,10 @@ def sync(self, query, *parameters, **options):
             return response.data
         else:
             self._writer.write(QException('nyi: qPython expected response message'), MessageType.ASYNC if response.type == MessageType.ASYNC else MessageType.RESPONSE)
-            raise QReaderException('Received message of type: %s where response was expected')
+            raise QReaderException('Received message of type: %s where response was expected' % response.type)
 
 
-    def async(self, query, *parameters, **options):
+    def send(self, query, *parameters, **options):
         '''Performs an asynchronous query and returns **without** retrieving of 
         the response.
 
@@ -319,11 +319,11 @@ def async(self, query, *parameters, **options):
 
         Calls a anonymous function with a single parameter:
 
-            >>> q.async('{til x}', 10)
+            >>> q.send('{til x}', 10)
 
         Executes a q expression:
 
-            >>> q.async('til 10')
+            >>> q.send('til 10')
 
         :Parameters:
          - `query` (`string`) - query to be executed

qpython/qreader.py

@@ -200,7 +200,7 @@ def read_data(self, message_size, is_compressed = False, **options):
             uncompressed_size = -8 + self._buffer.get_int()
             compressed_data = self._read_bytes(message_size - 12) if self._stream else self._buffer.raw(message_size - 12)
 
-            raw_data = numpy.fromstring(compressed_data, dtype = numpy.uint8)
+            raw_data = numpy.frombuffer(compressed_data, dtype = numpy.uint8)
             if  uncompressed_size <= 0:
                 raise QReaderException('Error while data decompression.')
 
@@ -296,7 +296,7 @@ def _read_list(self, qtype):
             return qlist(data, qtype = qtype, adjust_dtype = False)
         elif conversion:
             raw = self._buffer.raw(length * ATOM_SIZE[qtype])
-            data = numpy.fromstring(raw, dtype = conversion)
+            data = numpy.frombuffer(raw, dtype = conversion)
             if not self._is_native:
                 data.byteswap(True)
 

qpython/qtemporal.py

@@ -72,7 +72,8 @@ def __str__(self):
     def __eq__(self, other):
         return (isinstance(other, self.__class__)
             and self.meta.qtype == other.meta.qtype
-            and self._datetime == other._datetime)
+            and (numpy.isnat(self._datetime) and numpy.isnat(other._datetime))
+            or self._datetime == other._datetime)
 
     def __ne__(self, other):
         return not self.__eq__(other)
@@ -231,7 +232,7 @@ def _to_qmonth(dt):
     if t_dt == numpy.int32:
         return dt
     elif t_dt == numpy.datetime64:
-        return (dt - _EPOCH_QMONTH).astype(int) if not dt == _NUMPY_NULL[QMONTH] else _QMONTH_NULL
+        return (dt - _EPOCH_QMONTH).astype(int) if not (numpy.isnat(dt) or dt == _NUMPY_NULL[QMONTH]) else _QMONTH_NULL
     else:
         raise ValueError('Cannot convert %s of type %s to q value.' % (dt, type(dt)))
 
@@ -250,7 +251,7 @@ def _to_qdate(dt):
     if t_dt == numpy.int32:
         return dt
     elif t_dt == numpy.datetime64:
-        return (dt - _EPOCH_QDATE).astype(int) if not dt == _NUMPY_NULL[QDATE] else _QDATE_NULL
+        return (dt - _EPOCH_QDATE).astype(int) if not (numpy.isnat(dt) or dt == _NUMPY_NULL[QDATE]) else _QDATE_NULL
     else:
         raise ValueError('Cannot convert %s of type %s to q value.' % (dt, type(dt)))
 
@@ -269,7 +270,7 @@ def _to_qdatetime(dt):
     if t_dt == numpy.float64:
         return dt
     elif t_dt == numpy.datetime64:
-        return (dt - _EPOCH_QDATETIME).astype(float) / _MILLIS_PER_DAY if not dt == _NUMPY_NULL[QDATETIME] else _QDATETIME_NULL
+        return (dt - _EPOCH_QDATETIME).astype(float) / _MILLIS_PER_DAY if not (numpy.isnat(dt) or dt == _NUMPY_NULL[QDATETIME]) else _QDATETIME_NULL
     else:
         raise ValueError('Cannot convert %s of type %s to q value.' % (dt, type(dt)))
 
@@ -288,7 +289,7 @@ def _to_qminute(dt):
     if t_dt == numpy.int32:
         return dt
     elif t_dt == numpy.timedelta64:
-        return dt.astype(int) if not dt == _NUMPY_NULL[QMINUTE] else _QMINUTE_NULL
+        return dt.astype(int) if not (numpy.isnat(dt) or dt == _NUMPY_NULL[QMINUTE]) else _QMINUTE_NULL
     else:
         raise ValueError('Cannot convert %s of type %s to q value.' % (dt, type(dt)))
 
@@ -307,7 +308,7 @@ def _to_qsecond(dt):
     if t_dt == numpy.int32:
         return dt
     elif t_dt == numpy.timedelta64:
-        return dt.astype(int) if not dt == _NUMPY_NULL[QSECOND] else _QSECOND_NULL
+        return dt.astype(int) if not (numpy.isnat(dt) or dt == _NUMPY_NULL[QSECOND]) else _QSECOND_NULL
     else:
         raise ValueError('Cannot convert %s of type %s to q value.' % (dt, type(dt)))
 
@@ -326,7 +327,7 @@ def _to_qtime(dt):
     if t_dt == numpy.int32:
         return dt
     elif t_dt == numpy.timedelta64:
-        return dt.astype(int) if not dt == _NUMPY_NULL[QTIME] else _QTIME_NULL
+        return dt.astype(int) if not (numpy.isnat(dt) or dt == _NUMPY_NULL[QTIME]) else _QTIME_NULL
     else:
         raise ValueError('Cannot convert %s of type %s to q value.' % (dt, type(dt)))
 
@@ -345,7 +346,7 @@ def _to_qtimestamp(dt):
     if t_dt == numpy.int64:
         return dt
     elif t_dt == numpy.datetime64:
-        return (dt - _EPOCH_TIMESTAMP).astype(longlong) if not dt == _NUMPY_NULL[QTIMESTAMP] else _QTIMESTAMP_NULL
+        return (dt - _EPOCH_TIMESTAMP).astype(longlong) if not (numpy.isnat(dt) or dt == _NUMPY_NULL[QTIMESTAMP]) else _QTIMESTAMP_NULL
     else:
         raise ValueError('Cannot convert %s of type %s to q value.' % (dt, type(dt)))
 
@@ -364,7 +365,7 @@ def _to_qtimespan(dt):
     if t_dt == numpy.int64:
         return dt
     elif t_dt == numpy.timedelta64:
-        return dt.astype(longlong) if not dt == _NUMPY_NULL[QTIMESPAN] else _QTIMESTAMP_NULL
+        return dt.astype(longlong) if not (numpy.isnat(dt) or dt == _NUMPY_NULL[QTIMESPAN]) else _QTIMESTAMP_NULL
     else:
         raise ValueError('Cannot convert %s of type %s to q value.' % (dt, type(dt)))
 

qpython/qwriter.py

@@ -145,7 +145,10 @@ def _write_atom(self, data, qtype):
         try:
             self._buffer.write(struct.pack('b', qtype))
             fmt = STRUCT_MAP[qtype]
-            self._buffer.write(struct.pack(fmt, data))
+            if isinstance(data,numpy.bool_):
+                self._buffer.write(struct.pack(fmt,bool(data)))
+            else:
+                self._buffer.write(struct.pack(fmt, data))
         except KeyError:
             raise QWriterException('Unable to serialize type: %s' % data.__class__ if isinstance(data, object) else type(data))
 

samples/async_query.py

@@ -81,7 +81,7 @@ def run(self):
             a = random.randint(1, 100)
             b = random.randint(1, 100)
             print('Asynchronous call with queryid=%s with arguments: %s, %s' % (x, a, b))
-            q.async('asynchMult', x, a, b);
+            q.send('asynchMult', x, a, b);
 
         time.sleep(1)
     finally:

tests/qreader_test.py

@@ -28,7 +28,7 @@
 from collections import OrderedDict
 from qpython import qreader
 from qpython.qtype import *  # @UnusedWildImport
-from qpython.qcollection import qlist, QList, QTemporalList, QDictionary, qtable, QKeyedTable
+from qpython.qcollection import qlist, QList, QTemporalList, QDictionary, qtable, QKeyedTable, QTable
 from qpython.qtemporal import qtemporal, QTemporal
 
 
@@ -278,12 +278,20 @@ def arrays_equal(left, right):
 def compare(left, right):
     if type(left) in [float, numpy.float32, numpy.float64] and numpy.isnan(left):
         return numpy.isnan(right)
+    if type(left) in [numpy.datetime64, numpy.timedelta64] and numpy.isnat(left):
+        return numpy.isnat(right)
     if type(left) == QTemporal and isinstance(left.raw, float) and numpy.isnan(left.raw):
         return numpy.isnan(right.raw)
+    elif type(left) == QTemporal and isinstance(left.raw, numpy.datetime64) and numpy.isnat(left.raw):
+        return numpy.isnat(right.raw)
+    elif type(left) == QTemporal and isinstance(left.raw, numpy.timedelta64) and numpy.isnat(left.raw):
+        return numpy.isnat(right.raw)
     elif type(left) in [list, tuple, numpy.ndarray, QList, QTemporalList]:
         return arrays_equal(left, right)
     elif type(left) == QFunction:
         return type(right) == QFunction
+    elif type(left) == QTable:
+        return left.dtype == right.dtype and all(arrays_equal(left[n],right[n]) for n in left.dtype.names)
     else:
         return left == right
 

tests/qtypes_test.py

@@ -195,7 +195,7 @@ def test_array_to_raw_qtemporal():
         assert na[x] == x - 365
         x += 1
 
-    na_dt = numpy.arange('1999-01-01T00:00:00.000Z', '2001-01-04T05:36:57.600Z', 12345678, dtype='datetime64[ms]')
+    na_dt = numpy.arange('1999-01-01T00:00:00.000', '2001-01-04T05:36:57.600', 12345678, dtype='datetime64[ms]')
     na = array_to_raw_qtemporal(na_dt, qtype=QDATETIME_LIST)
     assert na.dtype == numpy.float64
 
@@ -208,7 +208,7 @@ def test_array_to_raw_qtemporal():
         ref = (x * step) - 365
         assert abs(na[x] - ref) < 0.1, '%s %s' %(na[x], ref)
 
-    na_dt = numpy.arange('1999-01-01T00:00:00.000Z', '2001-01-04T05:36:57.600Z', 1234567890000, dtype='datetime64[ns]')
+    na_dt = numpy.arange('1999-01-01T00:00:00.000', '2001-01-04T05:36:57.600', 1234567890000, dtype='datetime64[ns]')
     na = array_to_raw_qtemporal(na_dt, qtype=QTIMESTAMP_LIST)
     assert na.dtype == numpy.int64
 
@@ -249,7 +249,7 @@ def test_array_from_raw_qtemporal():
 
     assert str(na_dt.dtype).startswith('datetime64[M]')
     for x in range(len(na_dt)):
-        if na_dt[x] != numpy.datetime64('NaT', 'M'):
+        if not numpy.isnat(na_dt[x]):
             assert na_dt[x].astype(int) == raw[x] + 360
         else:
             assert raw[x] == qnull(QMONTH)
@@ -259,7 +259,7 @@ def test_array_from_raw_qtemporal():
 
     assert str(na_dt.dtype).startswith('datetime64[D]')
     for x in range(len(na_dt)):
-        if na_dt[x] != numpy.datetime64('NaT', 'D'):
+        if not numpy.isnat(na_dt[x]):
             assert na_dt[x].astype(int) == raw[x] + 10957
         else:
             assert raw[x] == qnull(QDATE)
@@ -269,7 +269,7 @@ def test_array_from_raw_qtemporal():
 
     assert str(na_dt.dtype).startswith('timedelta64[m]')
     for x in range(len(na_dt)):
-        if na_dt[x] != numpy.timedelta64('NaT', 'm'):
+        if not numpy.isnat(na_dt[x]):
             assert na_dt[x].astype(int) == raw[x]
         else:
             assert raw[x] == qnull(QMINUTE)
@@ -279,7 +279,7 @@ def test_array_from_raw_qtemporal():
 
     assert str(na_dt.dtype).startswith('timedelta64[s]')
     for x in range(len(na_dt)):
-        if na_dt[x] != numpy.timedelta64('NaT', 's'):
+        if not numpy.isnat(na_dt[x]):
             assert na_dt[x].astype(int) == raw[x]
         else:
             assert raw[x] == qnull(QSECOND)
@@ -289,7 +289,7 @@ def test_array_from_raw_qtemporal():
 
     assert str(na_dt.dtype).startswith('timedelta64[ms]')
     for x in range(len(na_dt)):
-        if na_dt[x] != numpy.timedelta64('NaT', 'ms'):
+        if not numpy.isnat(na_dt[x]):
             assert na_dt[x].astype(int) == raw[x]
         else:
             assert raw[x] == qnull(QTIME)
@@ -299,7 +299,7 @@ def test_array_from_raw_qtemporal():
 
     assert str(na_dt.dtype).startswith('timedelta64[ns]')
     for x in range(len(na_dt)):
-        if na_dt[x] != numpy.timedelta64('NaT', 'ns'):
+        if not numpy.isnat(na_dt[x]):
             assert na_dt[x].astype(numpy.int64) == raw[x]
         else:
             assert raw[x] == qnull(QTIMESPAN)
@@ -309,19 +309,22 @@ def test_array_from_raw_qtemporal():
 
     assert str(na_dt.dtype).startswith('datetime64[ns]')
     for x in range(len(na_dt)):
-        if na_dt[x] != numpy.datetime64('NaT', 'ns'):
-            assert na_dt[x].astype(numpy.int64) == raw[x] + numpy.datetime64('2000-01-01T00:00:00Z', 'ns').astype(numpy.int64)
+        if not numpy.isnat(na_dt[x]):
+            assert na_dt[x].astype(numpy.int64) == raw[x] + numpy.datetime64('2000-01-01T00:00:00', 'ns').astype(numpy.int64)
         else:
             assert raw[x] == qnull(QTIMESTAMP)
 
 
     raw = numpy.array([3.234, qnull(QDATETIME)])
     na_dt = array_from_raw_qtemporal(raw, qtype=QDATETIME)
-    ref = numpy.array([numpy.datetime64('2000-01-04T05:36:57.600Z', 'ms'), numpy.datetime64('nat', 'ms')])
+    ref = numpy.array([numpy.datetime64('2000-01-04T05:36:57.600', 'ms'), numpy.datetime64('nat', 'ms')])
 
     assert str(na_dt.dtype).startswith('datetime64[ms]')
     for x in range(len(na_dt)):
-        assert na_dt[x] == ref[x]
+        if not numpy.isnat(na_dt[x]):
+          assert na_dt[x] == ref[x]
+        else:
+          assert numpy.isnan(raw[x])
 
 
 test_is_null()