Update heavydb to 6.2 and remove omniscidb 5.10 (#520)

* Update heavydb to 6.2 and remove omniscidb 5.10

* add clangxx to environment

* update test

* Fix test for heavydb 6.2

* flake8

* address reviewer comments

Author: guilhermeleobas

.github/workflows/rbc_test.yml

@@ -101,7 +101,7 @@ jobs:
         os: [ubuntu-latest]
         python-version: ['3.10', '3.9', '3.8']
         numba-version: ['0.56', '0.55']
-        heavydb-version: ['6.1', '6.0', '5.10']
+        heavydb-version: ['6.2', '6.1', '6.0']
         heavydb-from: [conda]
         # include:
         #   - os: ubuntu-latest
@@ -144,11 +144,7 @@ jobs:
         shell: bash -l {0}
         if: matrix.os == 'ubuntu-latest' && matrix.heavydb-from == 'conda'
         run: |
-          if [[ ${{ matrix.heavydb-version }} == "5.10" ]]; then
-            mamba create -n heavydb-env omniscidb=${{ matrix.heavydb-version }}*=*_cpu -c conda-forge
-          else
-            mamba create -n heavydb-env heavydb=${{ matrix.heavydb-version }}*=*_cpu -c conda-forge
-          fi
+          mamba create -n heavydb-env heavydb=${{ matrix.heavydb-version }}*=*_cpu -c conda-forge
 
       - name: Build heavydb docker image and run it [docker]
         shell: bash -l {0}
@@ -173,20 +169,12 @@ jobs:
         shell: bash -l {0}
         if: matrix.os == 'ubuntu-latest' && matrix.heavydb-from == 'conda'
         run: |
-          if [[ ${{ matrix.heavydb-version }} == "5.10" ]]; then
-            mkdir data
-            mamba run -n heavydb-env omnisci_initdb data -f
-            mamba run -n heavydb-env omnisci_server --version
-            RUN_FLAGS="--enable-runtime-udf --enable-table-functions --enable-dev-table-functions"
-            mamba run -n heavydb-env omnisci_server $RUN_FLAGS &> heavydb-output.txt &
-          else
-            mkdir storage
-            mamba run -n heavydb-env initheavy storage -f
-            mamba run -n heavydb-env heavydb --version
-            RUN_FLAGS="--enable-runtime-udfs --enable-table-functions --enable-dev-table-functions --enable-udf-registration-for-all-users"
-            echo ${RUN_FLAGS}
-            mamba run -n heavydb-env heavydb $RUN_FLAGS &> heavydb-output.txt &
-          fi
+          mkdir storage
+          mamba run -n heavydb-env initheavy storage -f
+          mamba run -n heavydb-env heavydb --version
+          RUN_FLAGS="--enable-runtime-udfs --enable-table-functions --enable-dev-table-functions --enable-udf-registration-for-all-users"
+          echo ${RUN_FLAGS}
+          mamba run -n heavydb-env heavydb $RUN_FLAGS &> heavydb-output.txt &
           sleep 10
 
       - name: Create rbc test environment v${{ matrix.python-version }}
@@ -243,15 +231,9 @@ jobs:
         shell: bash -l {0}
         if: failure() && matrix.os == 'ubuntu-latest' && matrix.heavydb-from == 'conda'
         run: |
-          if [[ ${{ matrix.heavydb-version }} == "5.10" ]]; then
-            mamba run -n heavydb-env cat data/mapd_log/omnisci_server.INFO
-            mamba run -n heavydb-env cat data/mapd_log/omnisci_server.WARNING
-            mamba run -n heavydb-env cat data/mapd_log/omnisci_server.ERROR
-          else
-            mamba run -n heavydb-env cat storage/mapd_log/heavydb.INFO
-            mamba run -n heavydb-env cat storage/mapd_log/heavydb.WARNING
-            mamba run -n heavydb-env cat storage/mapd_log/heavydb.ERROR
-          fi
+          mamba run -n heavydb-env cat storage/mapd_log/heavydb.INFO
+          mamba run -n heavydb-env cat storage/mapd_log/heavydb.WARNING
+          mamba run -n heavydb-env cat storage/mapd_log/heavydb.ERROR
 
       - name: Show Heavydb docker logs on failure [docker]
         shell: bash -l {0}
@@ -265,11 +247,7 @@ jobs:
         continue-on-error: true
         if: matrix.os == 'ubuntu-latest' && matrix.heavydb-from == 'conda'
         run: |
-          if [[ ${{ matrix.heavydb-version }} == "5.10" ]]; then
-            mamba run -n heavydb-env killall omnisci_server
-          else
-            mamba run -n heavydb-env killall heavydb
-          fi
+          mamba run -n heavydb-env killall heavydb
           sleep 5
 
       - name: Show Heavydb server output [conda]

environment.yml

@@ -15,6 +15,7 @@ dependencies:
   - numba>=0.55
   - llvmlite
   - clang==11.1
+  - clangxx==11.1
   - pytest
   - packaging
   - tblib

rbc/extension_functions.thrift

@@ -3,7 +3,7 @@ namespace java com.mapd.thrift.calciteserver
 /* See QueryEngine/ExtensionFunctionsWhitelist.h for required
 values. It is ok if the following definition is out-of-date as
 get_device_parameters will contain up-to-date mapping of type names
-and enum values [OmnisciDB >= 5.2]. */
+and enum values [heavydb >= 5.2]. */
 
 enum TExtArgumentType {
   Int8,

rbc/heavydb/remoteheavydb.py

@@ -29,7 +29,7 @@
 from rbc import ctools, typesystem
 
 
-__all__ = ['RemoteHeavyDB', 'RemoteOmnisci', 'RemoteCallCapsule', 'is_available',
+__all__ = ['RemoteHeavyDB', 'RemoteCallCapsule', 'is_available',
            'type_to_type_name', 'get_client_config', 'global_heavydb_singleton']
 
 
@@ -95,8 +95,7 @@ def _global_heavydb():
     """Implements singleton of a global RemoteHeavyDB instance.
     """
     config = get_client_config()
-    remotedb = dict(heavyai=RemoteHeavyDB,
-                    omnisci=RemoteHeavyDB)[config['dbname']](**config)
+    remotedb = RemoteHeavyDB(**config)
     while True:
         yield remotedb
 
@@ -105,7 +104,7 @@ def _global_heavydb():
 
 
 def is_available(_cache={}):
-    """Return version tuple and None if HeavyDB/OmnisciDB server is
+    """Return version tuple and None if HeavyDB server is
     accessible or recent enough. Otherwise return None and the reason
     about unavailability.
     """
@@ -130,25 +129,22 @@ def get_client_config(**config):
     """Retrieve the HeavyDB client configuration parameters from a client
     home directory.
 
-    Two HeavyDB brands (HEAVYDB_BRAND) are supported: heavyai and
-    omnisci.
-
     Note that here the client configurations parameters are those that
     are used to configure the client software such as rbc or pymapd.
     This is different from heavydb instance configuration described in
     https://docs.heavy.ai/installation-and-configuration/config-parameters
     that is used for configuring the heavydb server software.
 
     In Linux clients, the HeavyDB client configuration is read from
-    :code:`$HOME/.config/$HEAVYDB_BRAND/client.conf`
+    :code:`$HOME/.config/heavyai/client.conf`
 
     In Windows clients, the configuration is read from
-    :code:`%UserProfile/.config/%HEAVYDB_BRAND%/client.conf` or
-    :code:`%AllUsersProfile/.config/%HEAVYDB_BRAND%/client.conf`
+    :code:`%UserProfile/.config/heavyai/client.conf` or
+    :code:`%AllUsersProfile/.config/heavyai/client.conf`
 
-    When :code:`HEAVYDB_CLIENT_CONF` or :code:`OMNISCI_CLIENT_CONF`
-    environment variable is defined then the configuration is read
-    from the file specified in this variable.
+    When :code:`HEAVYDB_CLIENT_CONF` environment variable is defined
+    then the configuration is read from the file specified in
+    this variable.
 
     The configuration file must use configuration language similar to
     one used in MS Windows INI files. For HeavyDB client
@@ -161,7 +157,7 @@ def get_client_config(**config):
       [server]
       host: <HeavyDB server host name or IP, defaults to localhost>
       port: <HeavyDB server port, defaults to 6274>
-      dbname: <HeavyDB database name, defaults to heavyai or omnisci>
+      dbname: <HeavyDB database name, defaults to heavyai>
 
     Parameters
     ----------
@@ -182,8 +178,7 @@ def get_client_config(**config):
     config = _config
 
     conf_file = None
-    for brand, client_conf_env in [('heavyai', 'HEAVYDB_CLIENT_CONF'),
-                                   ('omnisci', 'OMNISCI_CLIENT_CONF')]:
+    for brand, client_conf_env in [('heavyai', 'HEAVYDB_CLIENT_CONF')]:
         conf_file = os.environ.get(client_conf_env, None)
         if conf_file is not None and not os.path.isfile(conf_file):
             print('rbc.heavydb.get_client_config:'
@@ -264,7 +259,7 @@ def is_sizer(t):
 
 
 def get_sizer_enum(t):
-    """Return sizer enum value as defined by the omniscidb server.
+    """Return sizer enum value as defined by the heavydb server.
     """
     sizer = t.annotation()['sizer']
     sizer = output_buffer_sizer_map.get(sizer or None, sizer)
@@ -1561,8 +1556,3 @@ def remote_call(self, func, ftype: typesystem.Type, arguments: tuple, hold=False
             return numpy.array(list(result), dtype).view(numpy.recarray)
         else:
             return dtype[0][1](list(result)[0][0])
-
-
-class RemoteOmnisci(RemoteHeavyDB):
-    """Omnisci - the previous brand of HeavyAI
-    """

rbc/omniscidb.py

@@ -12,7 +12,7 @@
 pytestmark = pytest.mark.skipif(not available_version, reason=reason)
 
 
-@pytest.mark.parametrize("heavydb_brand", ['omnisci', 'heavyai'])
+@pytest.mark.parametrize("heavydb_brand", ['heavyai'])
 def test_get_client_config(tmpdir, heavydb_brand):
     d = tmpdir.mkdir(heavydb_brand)
     fh = d.join("client.conf")
@@ -34,8 +34,7 @@ def test_get_client_config(tmpdir, heavydb_brand):
 """)
     conf_file = os.path.join(fh.dirname, fh.basename)
 
-    client_conf_env = dict(heavyai='HEAVYDB_CLIENT_CONF',
-                           omnisci='OMNISCI_CLIENT_CONF')[heavydb_brand]
+    client_conf_env = dict(heavyai='HEAVYDB_CLIENT_CONF')[heavydb_brand]
     old_conf = os.environ.get(client_conf_env)
     os.environ[client_conf_env] = conf_file
 

rbc/tests/heavydb/test_heavydb.py

@@ -106,6 +106,7 @@ def test_getstring_lst(lst, unique, m, y):
                     y[j] += len(col.string_dict_proxy.getString(j))
             return unique
 
+    if heavydb.version[:2] >= (6, 3):
         @heavydb('TextEncodingDict(RowFunctionManager, TextEncodingDict)', devices=['cpu'])
         def fn_copy(mgr, t):
             db_id = mgr.getDictDbId('fn_copy', 0)
@@ -499,8 +500,8 @@ def test_getString_lst(heavydb, col):
 
 @pytest.mark.parametrize('col', ['t1'])
 def test_udf_copy_dict_encoded_string(heavydb, col):
-    if heavydb.version[:2] < (6, 2):
-        pytest.skip('Requires HeavyDB version 6.2 or newer')
+    if heavydb.version[:2] < (6, 3):
+        pytest.skip('Requires HeavyDB version 6.3 or newer')
 
     table = f"{heavydb.base_name}text"
     query = f"SELECT fn_copy({col}) from {table}"
@@ -510,8 +511,8 @@ def test_udf_copy_dict_encoded_string(heavydb, col):
 
 
 def test_row_function_manager(heavydb):
-    if heavydb.version[:2] < (6, 2):
-        pytest.skip('Requires HeavyDB version 6.2 or newer')
+    if heavydb.version[:2] < (6, 3):
+        pytest.skip('Requires HeavyDB version 6.3 or newer')
 
     @heavydb('int32(int32, RowFunctionManager)')
     def invalid_fn(a, mgr):

rbc/tests/heavydb/test_text_encoding_dict.py

@@ -6,11 +6,11 @@
 #   1. Run omnscidb server with ssh port forwarding::
 #
 #      ssh -L 6274:127.0.0.1:16274  <remote host name or IP>
-#      bin/omnisci_server --enable-runtime-udf --enable-table-functions -p 16274 --http-port 16278 --calcite-port 16279
+#      bin/heavydb --enable-runtime-udf --enable-table-functions -p 16274 --http-port 16278 --calcite-port 16279
 #
-#   2. Relate the omniscidb server to client:
+#   2. Relate the heavydb server to client:
 #
-#      export OMNISCI_CLIENT_CONF=utils/client_ssh_tunnel.conf
+#      export HEAVYDB_CLIENT_CONF=utils/client_ssh_tunnel.conf
 #
 
 [server]

start_omniscidb_bg.sh

@@ -84,7 +84,7 @@ def process_folder(path):
 
 def main():
     return
-    rootdir = '/path/to/omniscidb-internal/'
+    rootdir = '/path/to/heavydb-internal/'
     rootdir = os.path.dirname(os.path.dirname(os.path.dirname(__file__)))
 
     symbols = set()