Fix binaries loading

ConservationInternational · Sep 18, 2021 · 088a976 · 088a976
1 parent 45c0de8
commit 088a976
Show file tree

Hide file tree

Showing 7 changed files with 54 additions and 47 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -1,5 +1,6 @@
 services:
 - docker
+
 install:
 - docker-compose up -d
 - sleep 10
@@ -16,11 +17,13 @@ install:
 - docker cp trends.earth_admin_user_credentials.json trendsearth_qgis_1:/LDMP/test/trends.earth_admin_user_credentials.json
 - docker exec -it trendsearth_qgis_1 sh -c "rm -f  /root/.local/share/QGIS/QGIS3/profiles/default/python/plugins/LDMP"
 - docker exec -it trendsearth_qgis_1 sh -c "ln -s /LDMP/ /root/.local/share/QGIS/QGIS3/profiles/default/python/plugins/LDMP"
+
 script:
 - export BRANCH=$(if [ "$TRAVIS_PULL_REQUEST" == "false" ]; then echo $TRAVIS_BRANCH;
   else echo $TRAVIS_PULL_REQUEST_BRANCH; fi)
 - docker exec -it trendsearth_qgis_1 sh -c "cd /LDMP && qgis_testrunner.sh LDMP.test.testplugin"
 #- docker exec -it trendsearth_qgis_1 sh -c "cd /LDMP && pytest --cov=LDMP test/"
+
 after_success:
 - |
   if [ "${TRAVIS_PULL_REQUEST}" = "false" ] && [ "${TRAVIS_BRANCH}" = "master" ]; then

diff --git a/LDMP/__init__.py b/LDMP/__init__.py
@@ -21,6 +21,7 @@
 import json
 import subprocess
 from tempfile import NamedTemporaryFile
+from pathlib import Path
 
 from PyQt5 import (
     QtCore,
@@ -53,14 +54,14 @@ def _add_at_front_of_path(d):
     "trends_earth/advanced/binaries_folder",
     None
 )
-# TODO: Fix this
-# if binaries_folder:
-#     logger.log('Adding {} to path for binaries.'.format(binaries_folder))
-#     _add_at_front_of_path(os.path.join(
-#         binaries_folder,
-#         'trends_earth_binaries_{}'.format(__version__.replace('.', '_')))
-#     )
-_add_at_front_of_path(os.path.join(plugin_dir, 'ext-libs'))
+logger.log(f'sys.path is: {sys.path}')
+if binaries_folder:
+    binaries_path = Path(binaries_folder) / f"trends_earth_binaries_{__version__.replace('.', '_')}"
+    _add_at_front_of_path(str(binaries_path))
+logger.log(f'sys.path is: {sys.path}')
+
+_add_at_front_of_path(str(Path(plugin_dir) / 'ext-libs'))
+logger.log(f'sys.path is: {sys.path}')
 
 
 def tr(message):
@@ -120,12 +121,12 @@ def binaries_available():
             logger.log("Numba-compiled version of summary_numba not available: {}".format(e))
         ret = False
     try:
-        from trends_earth_binaries import calculate_numba
+        from trends_earth_binaries import ldn_numba
         if debug_enabled:
-            logger.log("Numba-compiled version of calculate_numba available.")
+            logger.log("Numba-compiled version of ldn_numba available.")
     except (ModuleNotFoundError, ImportError, RuntimeError) as e:
         if debug_enabled:
-            logger.log("Numba-compiled version of calculate_numba not available: {}".format(e))
+            logger.log("Numba-compiled version of ldn_numba not available: {}".format(e))
         ret = False
     return ret
 

diff --git a/LDMP/localexecution/ldn.py b/LDMP/localexecution/ldn.py
@@ -34,20 +34,10 @@
     SchemaBase
 )
 
-
-# if settings_manager.get_value(Setting.BINARIES_ENABLED):
-#     try:
-#         from trends_earth_binaries.ldn_numba import *
-#         log("Using numba-compiled version of calculate_numba.")
-#     except (ModuleNotFoundError, ImportError) as e:
-#         from .calculate_numba import *
-#         log("Failed import of numba-compiled code, falling back to python version of calculate_numba.")
-# else:
-#     from .ldn_numba import *
-#     log("Using python version of ldn_numba.")
-#
-
-from .ldn_numba import *
+from ..conf import (
+    settings_manager,
+    Setting
+)
 
 from .. import (
     areaofinterest,
@@ -67,6 +57,18 @@
 )
 from ..logger import log
 
+if settings_manager.get_value(Setting.BINARIES_ENABLED):
+    try:
+        from trends_earth_binaries.ldn_numba import *
+        log("Using numba-compiled version of ldn_numba.")
+    except (ModuleNotFoundError, ImportError) as e:
+        from .ldn_numba import *
+        log(f"Failed import of numba-compiled code: {e}. "
+            "Falling back to python version of ldn_numba.")
+else:
+    from .ldn_numba import *
+    log("Using python version of ldn_numba.")
+
 import marshmallow_dataclass
 
 
@@ -1065,7 +1067,7 @@ def _process_block(
     # Make an array of the same size as the input arrays containing
     # the area of each cell (which is identical for all cells in a
     # given row - cell areas only vary among rows)
-    cell_areas = np.repeat(cell_areas_raw, mask.shape[1], axis=1).astype(np.float)
+    cell_areas = np.repeat(cell_areas_raw, mask.shape[1], axis=1).astype(np.float64)
 
     if params.prod_mode == 'Trends.Earth productivity':
         traj_array = in_array[params.in_df.array_row_for_name(TRAJ_BAND_NAME), :, :]
@@ -1075,6 +1077,7 @@ def _process_block(
         state_recode = recode_state(state_array)
 
         perf_array = in_array[params.in_df.array_row_for_name(PERF_BAND_NAME), :, :]
+
         deg_prod5 = calc_prod5(
             traj_recode,
             state_recode,
@@ -1360,7 +1363,7 @@ def work(self):
 
                 cell_areas = np.array(
                     [
-                        summary.calc_cell_area(
+                        calc_cell_area(
                             lat + pixel_height * n,
                             lat + pixel_height * (n + 1),
                             long_width

diff --git a/LDMP/localexecution/ldn_numba.py b/LDMP/localexecution/ldn_numba.py
@@ -7,13 +7,20 @@
     cc = CC('ldn_numba')
 except ImportError:
     # Will use these as regular Python functions if numba is not present.
-    class CCSubstitute(object):
+    class DecoratorSubstitute(object):
         # Make a cc.export that doesn't do anything
         def export(*args, **kwargs):
             def wrapper(func):
                 return func
             return wrapper
-    cc = CCSubstitute()
+
+        # Make a numba.jit that doesn't do anything
+        def jit(*args, **kwargs):
+            def wrapper(func):
+                return func
+            return wrapper
+    cc = DecoratorSubstitute()
+    numba = DecoratorSubstitute()
 
 
 # Ensure mask and nodata values are saved as 16 bit integers to keep numba 
@@ -92,7 +99,7 @@ def recode_state(x):
 
 
 @numba.jit(nopython=True)
-@cc.export('calc_prod_5', 'i2[:,:](i2[:,:], i2[:,:], i2[:,:])')
+@cc.export('calc_prod5', 'i2[:,:](i2[:,:], i2[:,:], i2[:,:])')
 def calc_prod5(traj, state, perf):
     # Coding of LPD (prod5)
     # 1: declining
@@ -132,7 +139,7 @@ def calc_prod5(traj, state, perf):
 
 
 @numba.jit(nopython=True)
-@cc.export('pro5_to_prod3', 'i2[:,:](i2[:,:])')
+@cc.export('prod5_to_prod3', 'i2[:,:](i2[:,:])')
 def prod5_to_prod3(prod5):
     shp = prod5.shape
     prod5 = prod5.ravel()
@@ -143,10 +150,7 @@ def prod5_to_prod3(prod5):
     return np.reshape(out, shp)
 
 
-@numba.jit(
-    nopython=True,
-    locals={'a_trans_bl_tg': numba.types.int16[::1]}
-)
+@numba.jit(nopython=True)
 @cc.export('calc_lc_trans', 'i2[:,:](i2[:,:], i2[:,:])')
 def calc_lc_trans(lc_bl, lc_tg):
     shp = lc_bl.shape
@@ -194,10 +198,7 @@ def calc_deg_soc(soc_bl, soc_tg, water):
     return np.reshape(out, shp)
 
 
-@numba.jit(
-    nopython=True,
-    locals={'trans': numba.types.int16[::1]}
-)
+@numba.jit(nopython=True)
 @cc.export('calc_deg_lc', 'i2[:,:](i2[:,:], i2[:,:], i2[:], i2[:])')
 def calc_deg_lc(lc_bl, lc_tg, trans_code, trans_meaning):
     '''calculate land cover degradation'''
@@ -249,7 +250,8 @@ def zonal_total(z, d, mask):
     # Carry over nodata values from data layer to z so that they aren't
     # included in the totals
     z[d == NODATA_VALUE] = NODATA_VALUE
-    totals = numba.typed.Dict.empty(numba.types.int64, numba.types.float64)
+    #totals = numba.typed.Dict.empty(numba.types.int64, numba.types.float64)
+    totals = dict()
     for i in range(z.shape[0]):
         if z[i] not in totals:
             totals[int(z[i])] = d[i]
@@ -269,7 +271,8 @@ def zonal_total_weighted(z, d, weights, mask):
     # Carry over nodata values from data layer to z so that they aren't
     # included in the totals
     z[d == NODATA_VALUE] = NODATA_VALUE
-    totals = numba.typed.Dict.empty(numba.types.int64, numba.types.float64)
+    #totals = numba.typed.Dict.empty(numba.types.int64, numba.types.float64)
+    totals = dict()
     for i in range(z.shape[0]):
         if z[i] not in totals:
             totals[int(z[i])] = d[i] * weights[i]
@@ -293,10 +296,6 @@ def bizonal_total(z1, z2, d, mask):
     # Carry over nodata values from data layer to z so that they aren't
     # included in the totals
     z2[d == NODATA_VALUE] = NODATA_VALUE
-    # tab = numba.typed.Dict.empty(
-    #     key_type=numba.types.UniTuple(numba.types.int64, 2),
-    #     value_type=numba.types.int64
-    # )
     tab = dict()
     for i in range(z1.shape[0]):
         if (z1[i], z2[i]) not in tab:

diff --git a/LDMP/summary.py b/LDMP/summary.py
@@ -15,6 +15,8 @@
 from . import conf
 from .logger import log
 
+import numpy as np
+
 if conf.settings_manager.get_value(conf.Setting.BINARIES_ENABLED):
     try:
         from trends_earth_binaries.summary_numba import *

diff --git a/requirements-dev.txt b/requirements-dev.txt
@@ -7,5 +7,5 @@ transifex-client>=0.13.6
 pylint
 pip>=10.0.0
 PyQt5
-numba==0.53.1
+numba==0.54.0
 requests==2.26.0
diff --git a/requirements.txt b/requirements.txt
@@ -4,7 +4,6 @@ pyqtgraph==0.11.0
 openpyxl==3.0.7
 marshmallow==3.13.0
 marshmallow_dataclass[enum, union]==8.4.2
-numba
 git+https://github.com/ConservationInternational/trends.earth-schemas.git@develop
 
 # test requirements