Fixed RLEC problem of loading redisai.so (#143)

* Fixed RLEC problem of loading redisai.so

Fixed RLEC problem of loading redisai.so by setting RUNPATH to /opt/redislabs/lib (via patchelf).

* Fixed RLEC problem of loading redisai.so #2

* Add libgomp1 as dependency in Dockerfile

* Moved redisai.so RUNPATH fix code into Makefile

* Moved redisai.so RUNPATH fix code into Makefile #2

* Moved redisai.so RUNPATH fix code into Makefile #3

* Tabs to spaces in .circleci/config.yml

* Fixed RLEC problem of loading redisai.so

Fixed RLEC problem of loading redisai.so by setting RUNPATH to /opt/redislabs/lib (via patchelf).

* Fixed RLEC problem of loading redisai.so #2

* Moved redisai.so RUNPATH fix code into Makefile

* Moved redisai.so RUNPATH fix code into Makefile #2

* Moved redisai.so RUNPATH fix code into Makefile #3

* Tabs to spaces in .circleci/config.yml

Author: rafie

.circleci/config.yml

@@ -18,7 +18,7 @@ jobs:
           name: install dependencies
           command: |
             curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | sudo bash
-            sudo apt-get install -y git-lfs
+            sudo apt-get install -y git-lfs patchelf
             git lfs install
             curl -o cmake-3.12.4-Linux-x86_64.sh https://cmake.org/files/v3.12/cmake-3.12.4-Linux-x86_64.sh
             mkdir /home/circleci/project/cmake-bin
@@ -43,6 +43,8 @@ jobs:
             cd build
             cmake -DDEPS_PATH=../deps/install ..
             make
+            cd ..
+            make rlec_runpath_fix
 
       - run:
           name: Test
@@ -59,7 +61,7 @@ jobs:
           name: Persist Artifacts
           command: |
             mkdir -p ~/workspace/build
-            cp build/$MODULE_ARTIFACT ~/workspace/
+            cp build/ramp/$MODULE_ARTIFACT ~/workspace/
             cp deps/install/lib/*.so*  ~/workspace/
             cp ramp.yml ~/workspace/
       - persist_to_workspace:

CMakeLists.txt

@@ -53,4 +53,4 @@ SET_TARGET_PROPERTIES(redisai PROPERTIES SUFFIX ".so")
 IF (APPLE)
     SET_TARGET_PROPERTIES(redisai PROPERTIES
         LINK_FLAGS "-undefined dynamic_lookup")
-ENDIF()
+ENDIF()

Dockerfile

@@ -1,6 +1,6 @@
 FROM redis AS builder
 
-ENV DEPS "build-essential git ca-certificates curl unzip wget libgomp1"
+ENV DEPS "build-essential git ca-certificates curl unzip wget libgomp1 patchelf"
 
 # install latest cmake
 ADD https://cmake.org/files/v3.12/cmake-3.12.4-Linux-x86_64.sh /cmake-3.12.4-Linux-x86_64.sh

Dockerfile-gpu

@@ -1,6 +1,6 @@
 FROM nvidia/cuda:9.0-cudnn7-devel-ubuntu16.04 AS builder
 
-ENV DEPS "build-essential git ca-certificates curl unzip wget libgomp1"
+ENV DEPS "build-essential git ca-certificates curl unzip wget libgomp1 patchelf"
 ENV NVIDIA_VISIBLE_DEVICES all
 ENV NVIDIA_DRIVER_CAPABILITIES compute,utility
 

Makefile

@@ -0,0 +1,67 @@
+
+ifeq ($(CUDA),1)
+DEPS_FLAGS=
+else
+DEPS_FLAGS=cpu
+endif
+
+export REDIS_ENT_LIB_PATH=/opt/redislabs/lib
+
+GIT_BRANCH:=$(shell git rev-parse --abbrev-ref HEAD)
+GIT_COMMIT:=$(shell git describe --always --abbrev=7 --dirty="+")
+
+ifeq ($(VERSION),)
+PACK_VER:=$(GIT_BRANCH)-$(GIT_COMMIT)
+else
+PACK_VER:=$(VERSION)
+endif
+
+BINDIR=$(PWD)/build
+
+.PHONY: all clean deps pack rlec_runpath_fix
+
+all:
+ifeq ($(wildcard build/.),)
+	mkdir -p build
+	cd build; \
+	cmake -DDEPS_PATH=../deps/install ..
+endif
+	$(MAKE) -C build
+
+clean:
+ifeq ($(ALL),1)
+	rm -rf build deps
+else
+	$(MAKE) -C build clean
+endif
+
+deps:
+	@echo Fetching dependencies...
+	@./get_deps.sh $(DEPS_FLAGS)
+
+# in pack: create ramp/redisai.so with RUNPATH set to /opt/redislabs/lib for RLEC compliance
+rlec_runpath_fix: 
+	@echo Fixing RLEC RUNPATH...
+	@mkdir -p $(BINDIR)/ramp
+	@cp -f $(BINDIR)/redisai.so $(BINDIR)/ramp/
+	@patchelf --set-rpath $(REDIS_ENT_LIB_PATH) $(BINDIR)/ramp/redisai.so
+
+pack: rlec_runpath_fix
+	@[ ! -z `command -v redis-server` ] || { echo "Cannot find redis-server - aborting."; exit 1; }
+	@[ ! -e $(REDIS_ENT_LIB_PATH) ] || { echo "$(REDIS_ENT_LIB_PATH) exists - aborting."; exit 1; }
+ifeq ($(wildcard build/pyenv/.),)
+	@virtualenv build/pyenv ;\
+	. ./build/pyenv/bin/activate ;\
+	pip install git+https://github.com/RedisLabs/RAMP
+endif
+	@echo "Building RAMP file ..."
+	@set -e ;\
+	. ./build/pyenv/bin/activate ;\
+	ln -fs $(PWD)/deps/install/lib/ $(REDIS_ENT_LIB_PATH) ;\
+	ramp pack -m $(PWD)/ramp.yml -o "build/redisai.{os}-{architecture}.${PACK_VER}.zip" $(BINDIR)/ramp/redisai.so 2>&1 > /dev/null ;\
+	rm $(REDIS_ENT_LIB_PATH)
+	@echo Done.
+	@echo "Building dependencies file redisai-dependencies.${PACK_VER}.tgz ..."
+	@cd deps/install/lib; \
+	tar pczf ../../../build/redisai-dependencies.${PACK_VER}.tgz *.so*
+	@echo Done.