Skip to content

Apply Nugend's fixes #65

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 4 commits into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,7 @@ dist
# coverage
.coverage
__conda_version__.txt
.pytest_cache

# OS generated files #
######################
Expand Down
8 changes: 8 additions & 0 deletions CHANGELOG.txt
Original file line number Diff line number Diff line change
@@ -1,3 +1,11 @@
------------------------------------------------------------------------------
qPython 2.0.0 [2018.08.31]
------------------------------------------------------------------------------

- BREAKING CHANGE: Renamed QConnection.async to QConnection.send to
support Python 3.7 because of async keyword promotion.
- Fix DeprecationWarnings and FutureWarnings

------------------------------------------------------------------------------
qPython 1.3.0 [2017.03.xx]
------------------------------------------------------------------------------
Expand Down
2 changes: 1 addition & 1 deletion doc/source/connection.rst
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ to numpy `datetime64`/`timedelta64` representation.

Conversion options can be also overwritten while executing
synchronous/asynchronous queries (:meth:`~qpython.qconnection.QConnection.sync`,
:meth:`~qpython.qconnection.QConnection.async`) or retrieving data from q
:meth:`~qpython.qconnection.QConnection.send`) or retrieving data from q
(:meth:`~qpython.qconnection.QConnection.receive`).


Expand Down
12 changes: 6 additions & 6 deletions doc/source/queries.rst
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ The `qPython` library provides following API methods in the

- :func:`~qpython.qconnection.QConnection.sync` - executes a synchronous query
against the remote q service,
- :func:`~qpython.qconnection.QConnection.async` - executes an asynchronous
- :func:`~qpython.qconnection.QConnection.send` - executes an asynchronous
query against the remote q service,
- :func:`~qpython.qconnection.QConnection.query` - executes a query against the
remote q service.
Expand Down Expand Up @@ -65,17 +65,17 @@ Asynchronous queries

Calls a anonymous function with a single parameter:

>>> q.async('{til x}', 10)
>>> q.send('{til x}', 10)

Executes a q expression:

>>> q.async('til 10')
>>> q.send('til 10')

.. note:: The asynchronous query doesn't fetch the result. Query result has
to be retrieved explicitly.

In order to retrieve query result (for the
:func:`~qpython.qconnection.QConnection.async` or
:func:`~qpython.qconnection.QConnection.send` or
:func:`~qpython.qconnection.QConnection.query` methods), one has to call:

- :func:`~qpython.qconnection.QConnection.receive` method, which reads next
Expand All @@ -96,7 +96,7 @@ QMessage: message type: 2, data size: 13, is_compressed: False, data: 10
10

>>> q.sync('asynchMult:{[a;b] res:a*b; (neg .z.w)(res) }')
>>> q.async('asynchMult', 2, 3)
>>> q.send('asynchMult', 2, 3)
>>> print(q.receive())
6

Expand All @@ -114,7 +114,7 @@ Type conversions configuration
Type conversion options can be overwritten while:

- executing synchronous query: :meth:`~qpython.qconnection.QConnection.sync`
- executing asynchronous query: :meth:`~qpython.qconnection.QConnection.async`
- executing asynchronous query: :meth:`~qpython.qconnection.QConnection.send`
- retrieving data from q: :meth:`~qpython.qconnection.QConnection.receive`

These methods accepts the `options` keywords arguments::
Expand Down
2 changes: 1 addition & 1 deletion doc/source/usage-examples.rst
Original file line number Diff line number Diff line change
Expand Up @@ -124,7 +124,7 @@ Following example presents how to execute simple, asynchronous query against a r
a = random.randint(1, 100)
b = random.randint(1, 100)
print('Asynchronous call with queryid=%s with arguments: %s, %s' % (x, a, b))
q.async('asynchMult', x, a, b);
q.send('asynchMult', x, a, b);

time.sleep(1)
finally:
Expand Down
12 changes: 6 additions & 6 deletions qpython/_pandas.py
Original file line number Diff line number Diff line change
Expand Up @@ -58,8 +58,8 @@ def _read_dictionary(self, qtype = QDICTIONARY):

return table
else:
keys = keys if not isinstance(keys, pandas.Series) else keys.as_matrix()
values = values if not isinstance(values, pandas.Series) else values.as_matrix()
keys = keys if not isinstance(keys, pandas.Series) else keys.values
values = values if not isinstance(values, pandas.Series) else values.values
return QDictionary(keys, values)
else:
return QReader._read_dictionary(self, qtype = qtype)
Expand Down Expand Up @@ -168,19 +168,19 @@ def _write_pandas_series(self, data, qtype = None):
raise QWriterException('Unable to serialize pandas series %s' % data)

if qtype == QGENERAL_LIST:
self._write_generic_list(data.as_matrix())
self._write_generic_list(data.values)
elif qtype == QCHAR:
self._write_string(data.replace(numpy.nan, ' ').as_matrix().astype(numpy.string_).tostring())
self._write_string(data.replace(numpy.nan, ' ').values.astype(numpy.string_).tostring())
elif data.dtype.type not in (numpy.datetime64, numpy.timedelta64):
data = data.fillna(QNULLMAP[-abs(qtype)][1])
data = data.as_matrix()
data = data.values

if PY_TYPE[qtype] != data.dtype:
data = data.astype(PY_TYPE[qtype])

self._write_list(data, qtype = qtype)
else:
data = data.as_matrix()
data = data.values
data = data.astype(TEMPORAL_Q_TYPE[qtype])
self._write_list(data, qtype = qtype)

Expand Down
8 changes: 4 additions & 4 deletions qpython/qconnection.py
Original file line number Diff line number Diff line change
Expand Up @@ -306,10 +306,10 @@ def sync(self, query, *parameters, **options):
return response.data
else:
self._writer.write(QException('nyi: qPython expected response message'), MessageType.ASYNC if response.type == MessageType.ASYNC else MessageType.RESPONSE)
raise QReaderException('Received message of type: %s where response was expected')
raise QReaderException('Received message of type: %s where response was expected' % response.type)


def async(self, query, *parameters, **options):
def send(self, query, *parameters, **options):
'''Performs an asynchronous query and returns **without** retrieving of
the response.

Expand All @@ -319,11 +319,11 @@ def async(self, query, *parameters, **options):

Calls a anonymous function with a single parameter:

>>> q.async('{til x}', 10)
>>> q.send('{til x}', 10)

Executes a q expression:

>>> q.async('til 10')
>>> q.send('til 10')

:Parameters:
- `query` (`string`) - query to be executed
Expand Down
4 changes: 2 additions & 2 deletions qpython/qreader.py
Original file line number Diff line number Diff line change
Expand Up @@ -200,7 +200,7 @@ def read_data(self, message_size, is_compressed = False, **options):
uncompressed_size = -8 + self._buffer.get_int()
compressed_data = self._read_bytes(message_size - 12) if self._stream else self._buffer.raw(message_size - 12)

raw_data = numpy.fromstring(compressed_data, dtype = numpy.uint8)
raw_data = numpy.frombuffer(compressed_data, dtype = numpy.uint8)
if uncompressed_size <= 0:
raise QReaderException('Error while data decompression.')

Expand Down Expand Up @@ -296,7 +296,7 @@ def _read_list(self, qtype):
return qlist(data, qtype = qtype, adjust_dtype = False)
elif conversion:
raw = self._buffer.raw(length * ATOM_SIZE[qtype])
data = numpy.fromstring(raw, dtype = conversion)
data = numpy.frombuffer(raw, dtype = conversion)
if not self._is_native:
data.byteswap(True)

Expand Down
19 changes: 10 additions & 9 deletions qpython/qtemporal.py
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,8 @@ def __str__(self):
def __eq__(self, other):
return (isinstance(other, self.__class__)
and self.meta.qtype == other.meta.qtype
and self._datetime == other._datetime)
and (numpy.isnat(self._datetime) and numpy.isnat(other._datetime))
or self._datetime == other._datetime)

def __ne__(self, other):
return not self.__eq__(other)
Expand Down Expand Up @@ -231,7 +232,7 @@ def _to_qmonth(dt):
if t_dt == numpy.int32:
return dt
elif t_dt == numpy.datetime64:
return (dt - _EPOCH_QMONTH).astype(int) if not dt == _NUMPY_NULL[QMONTH] else _QMONTH_NULL
return (dt - _EPOCH_QMONTH).astype(int) if not (numpy.isnat(dt) or dt == _NUMPY_NULL[QMONTH]) else _QMONTH_NULL
else:
raise ValueError('Cannot convert %s of type %s to q value.' % (dt, type(dt)))

Expand All @@ -250,7 +251,7 @@ def _to_qdate(dt):
if t_dt == numpy.int32:
return dt
elif t_dt == numpy.datetime64:
return (dt - _EPOCH_QDATE).astype(int) if not dt == _NUMPY_NULL[QDATE] else _QDATE_NULL
return (dt - _EPOCH_QDATE).astype(int) if not (numpy.isnat(dt) or dt == _NUMPY_NULL[QDATE]) else _QDATE_NULL
else:
raise ValueError('Cannot convert %s of type %s to q value.' % (dt, type(dt)))

Expand All @@ -269,7 +270,7 @@ def _to_qdatetime(dt):
if t_dt == numpy.float64:
return dt
elif t_dt == numpy.datetime64:
return (dt - _EPOCH_QDATETIME).astype(float) / _MILLIS_PER_DAY if not dt == _NUMPY_NULL[QDATETIME] else _QDATETIME_NULL
return (dt - _EPOCH_QDATETIME).astype(float) / _MILLIS_PER_DAY if not (numpy.isnat(dt) or dt == _NUMPY_NULL[QDATETIME]) else _QDATETIME_NULL
else:
raise ValueError('Cannot convert %s of type %s to q value.' % (dt, type(dt)))

Expand All @@ -288,7 +289,7 @@ def _to_qminute(dt):
if t_dt == numpy.int32:
return dt
elif t_dt == numpy.timedelta64:
return dt.astype(int) if not dt == _NUMPY_NULL[QMINUTE] else _QMINUTE_NULL
return dt.astype(int) if not (numpy.isnat(dt) or dt == _NUMPY_NULL[QMINUTE]) else _QMINUTE_NULL
else:
raise ValueError('Cannot convert %s of type %s to q value.' % (dt, type(dt)))

Expand All @@ -307,7 +308,7 @@ def _to_qsecond(dt):
if t_dt == numpy.int32:
return dt
elif t_dt == numpy.timedelta64:
return dt.astype(int) if not dt == _NUMPY_NULL[QSECOND] else _QSECOND_NULL
return dt.astype(int) if not (numpy.isnat(dt) or dt == _NUMPY_NULL[QSECOND]) else _QSECOND_NULL
else:
raise ValueError('Cannot convert %s of type %s to q value.' % (dt, type(dt)))

Expand All @@ -326,7 +327,7 @@ def _to_qtime(dt):
if t_dt == numpy.int32:
return dt
elif t_dt == numpy.timedelta64:
return dt.astype(int) if not dt == _NUMPY_NULL[QTIME] else _QTIME_NULL
return dt.astype(int) if not (numpy.isnat(dt) or dt == _NUMPY_NULL[QTIME]) else _QTIME_NULL
else:
raise ValueError('Cannot convert %s of type %s to q value.' % (dt, type(dt)))

Expand All @@ -345,7 +346,7 @@ def _to_qtimestamp(dt):
if t_dt == numpy.int64:
return dt
elif t_dt == numpy.datetime64:
return (dt - _EPOCH_TIMESTAMP).astype(longlong) if not dt == _NUMPY_NULL[QTIMESTAMP] else _QTIMESTAMP_NULL
return (dt - _EPOCH_TIMESTAMP).astype(longlong) if not (numpy.isnat(dt) or dt == _NUMPY_NULL[QTIMESTAMP]) else _QTIMESTAMP_NULL
else:
raise ValueError('Cannot convert %s of type %s to q value.' % (dt, type(dt)))

Expand All @@ -364,7 +365,7 @@ def _to_qtimespan(dt):
if t_dt == numpy.int64:
return dt
elif t_dt == numpy.timedelta64:
return dt.astype(longlong) if not dt == _NUMPY_NULL[QTIMESPAN] else _QTIMESTAMP_NULL
return dt.astype(longlong) if not (numpy.isnat(dt) or dt == _NUMPY_NULL[QTIMESPAN]) else _QTIMESTAMP_NULL
else:
raise ValueError('Cannot convert %s of type %s to q value.' % (dt, type(dt)))

Expand Down
5 changes: 4 additions & 1 deletion qpython/qwriter.py
Original file line number Diff line number Diff line change
Expand Up @@ -145,7 +145,10 @@ def _write_atom(self, data, qtype):
try:
self._buffer.write(struct.pack('b', qtype))
fmt = STRUCT_MAP[qtype]
self._buffer.write(struct.pack(fmt, data))
if isinstance(data,numpy.bool_):
self._buffer.write(struct.pack(fmt,bool(data)))
else:
self._buffer.write(struct.pack(fmt, data))
except KeyError:
raise QWriterException('Unable to serialize type: %s' % data.__class__ if isinstance(data, object) else type(data))

Expand Down
2 changes: 1 addition & 1 deletion samples/async_query.py
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,7 @@ def run(self):
a = random.randint(1, 100)
b = random.randint(1, 100)
print('Asynchronous call with queryid=%s with arguments: %s, %s' % (x, a, b))
q.async('asynchMult', x, a, b);
q.send('asynchMult', x, a, b);

time.sleep(1)
finally:
Expand Down
10 changes: 9 additions & 1 deletion tests/qreader_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@
from collections import OrderedDict
from qpython import qreader
from qpython.qtype import * # @UnusedWildImport
from qpython.qcollection import qlist, QList, QTemporalList, QDictionary, qtable, QKeyedTable
from qpython.qcollection import qlist, QList, QTemporalList, QDictionary, qtable, QKeyedTable, QTable
from qpython.qtemporal import qtemporal, QTemporal


Expand Down Expand Up @@ -278,12 +278,20 @@ def arrays_equal(left, right):
def compare(left, right):
if type(left) in [float, numpy.float32, numpy.float64] and numpy.isnan(left):
return numpy.isnan(right)
if type(left) in [numpy.datetime64, numpy.timedelta64] and numpy.isnat(left):
return numpy.isnat(right)
if type(left) == QTemporal and isinstance(left.raw, float) and numpy.isnan(left.raw):
return numpy.isnan(right.raw)
elif type(left) == QTemporal and isinstance(left.raw, numpy.datetime64) and numpy.isnat(left.raw):
return numpy.isnat(right.raw)
elif type(left) == QTemporal and isinstance(left.raw, numpy.timedelta64) and numpy.isnat(left.raw):
return numpy.isnat(right.raw)
elif type(left) in [list, tuple, numpy.ndarray, QList, QTemporalList]:
return arrays_equal(left, right)
elif type(left) == QFunction:
return type(right) == QFunction
elif type(left) == QTable:
return left.dtype == right.dtype and all(arrays_equal(left[n],right[n]) for n in left.dtype.names)
else:
return left == right

Expand Down
27 changes: 15 additions & 12 deletions tests/qtypes_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -195,7 +195,7 @@ def test_array_to_raw_qtemporal():
assert na[x] == x - 365
x += 1

na_dt = numpy.arange('1999-01-01T00:00:00.000Z', '2001-01-04T05:36:57.600Z', 12345678, dtype='datetime64[ms]')
na_dt = numpy.arange('1999-01-01T00:00:00.000', '2001-01-04T05:36:57.600', 12345678, dtype='datetime64[ms]')
na = array_to_raw_qtemporal(na_dt, qtype=QDATETIME_LIST)
assert na.dtype == numpy.float64

Expand All @@ -208,7 +208,7 @@ def test_array_to_raw_qtemporal():
ref = (x * step) - 365
assert abs(na[x] - ref) < 0.1, '%s %s' %(na[x], ref)

na_dt = numpy.arange('1999-01-01T00:00:00.000Z', '2001-01-04T05:36:57.600Z', 1234567890000, dtype='datetime64[ns]')
na_dt = numpy.arange('1999-01-01T00:00:00.000', '2001-01-04T05:36:57.600', 1234567890000, dtype='datetime64[ns]')
na = array_to_raw_qtemporal(na_dt, qtype=QTIMESTAMP_LIST)
assert na.dtype == numpy.int64

Expand Down Expand Up @@ -249,7 +249,7 @@ def test_array_from_raw_qtemporal():

assert str(na_dt.dtype).startswith('datetime64[M]')
for x in range(len(na_dt)):
if na_dt[x] != numpy.datetime64('NaT', 'M'):
if not numpy.isnat(na_dt[x]):
assert na_dt[x].astype(int) == raw[x] + 360
else:
assert raw[x] == qnull(QMONTH)
Expand All @@ -259,7 +259,7 @@ def test_array_from_raw_qtemporal():

assert str(na_dt.dtype).startswith('datetime64[D]')
for x in range(len(na_dt)):
if na_dt[x] != numpy.datetime64('NaT', 'D'):
if not numpy.isnat(na_dt[x]):
assert na_dt[x].astype(int) == raw[x] + 10957
else:
assert raw[x] == qnull(QDATE)
Expand All @@ -269,7 +269,7 @@ def test_array_from_raw_qtemporal():

assert str(na_dt.dtype).startswith('timedelta64[m]')
for x in range(len(na_dt)):
if na_dt[x] != numpy.timedelta64('NaT', 'm'):
if not numpy.isnat(na_dt[x]):
assert na_dt[x].astype(int) == raw[x]
else:
assert raw[x] == qnull(QMINUTE)
Expand All @@ -279,7 +279,7 @@ def test_array_from_raw_qtemporal():

assert str(na_dt.dtype).startswith('timedelta64[s]')
for x in range(len(na_dt)):
if na_dt[x] != numpy.timedelta64('NaT', 's'):
if not numpy.isnat(na_dt[x]):
assert na_dt[x].astype(int) == raw[x]
else:
assert raw[x] == qnull(QSECOND)
Expand All @@ -289,7 +289,7 @@ def test_array_from_raw_qtemporal():

assert str(na_dt.dtype).startswith('timedelta64[ms]')
for x in range(len(na_dt)):
if na_dt[x] != numpy.timedelta64('NaT', 'ms'):
if not numpy.isnat(na_dt[x]):
assert na_dt[x].astype(int) == raw[x]
else:
assert raw[x] == qnull(QTIME)
Expand All @@ -299,7 +299,7 @@ def test_array_from_raw_qtemporal():

assert str(na_dt.dtype).startswith('timedelta64[ns]')
for x in range(len(na_dt)):
if na_dt[x] != numpy.timedelta64('NaT', 'ns'):
if not numpy.isnat(na_dt[x]):
assert na_dt[x].astype(numpy.int64) == raw[x]
else:
assert raw[x] == qnull(QTIMESPAN)
Expand All @@ -309,19 +309,22 @@ def test_array_from_raw_qtemporal():

assert str(na_dt.dtype).startswith('datetime64[ns]')
for x in range(len(na_dt)):
if na_dt[x] != numpy.datetime64('NaT', 'ns'):
assert na_dt[x].astype(numpy.int64) == raw[x] + numpy.datetime64('2000-01-01T00:00:00Z', 'ns').astype(numpy.int64)
if not numpy.isnat(na_dt[x]):
assert na_dt[x].astype(numpy.int64) == raw[x] + numpy.datetime64('2000-01-01T00:00:00', 'ns').astype(numpy.int64)
else:
assert raw[x] == qnull(QTIMESTAMP)


raw = numpy.array([3.234, qnull(QDATETIME)])
na_dt = array_from_raw_qtemporal(raw, qtype=QDATETIME)
ref = numpy.array([numpy.datetime64('2000-01-04T05:36:57.600Z', 'ms'), numpy.datetime64('nat', 'ms')])
ref = numpy.array([numpy.datetime64('2000-01-04T05:36:57.600', 'ms'), numpy.datetime64('nat', 'ms')])

assert str(na_dt.dtype).startswith('datetime64[ms]')
for x in range(len(na_dt)):
assert na_dt[x] == ref[x]
if not numpy.isnat(na_dt[x]):
assert na_dt[x] == ref[x]
else:
assert numpy.isnan(raw[x])


test_is_null()
Expand Down