From a22f1811e108ba56c2370ed2ab98ff8acdc60f4f Mon Sep 17 00:00:00 2001 From: "Chayim I. Kirshen" Date: Wed, 30 Jun 2021 15:07:09 +0300 Subject: [PATCH 1/9] first stage - adding jetson to getdeps --- opt/build/docker/Makefile | 35 +++++++++++++------------------- opt/build/docker/dockerfile.tmpl | 16 +++++++++++---- 2 files changed, 26 insertions(+), 25 deletions(-) diff --git a/opt/build/docker/Makefile b/opt/build/docker/Makefile index f0f1acaf2..b0fc4be78 100755 --- a/opt/build/docker/Makefile +++ b/opt/build/docker/Makefile @@ -1,26 +1,24 @@ # Feel free to change these PRODUCT=redisai DOCKER_ORG=redislabs -REDIS_VERSION=6.2.4 -REDIS_CUDA_VERSION=11.0-cudnn8 -REDISAI_LITE=0 +export REDIS_VERSION=6.2.4 +export REDIS_CUDA_VERSION=11.0-cudnn8 +export REDISAI_LITE=0 PACK=1 # to fetch the artifacts - -DOCKER_OPTS= # set, to pass custom options to docker, but remember to quote them OSNICK ?= bionic -# set to pass multiple directories as docker sources to include in templates -DOCKER_SOURCES="" - -# set, to generate a file named Dockerfile, i.e useful when you want to run things in parallel -DOCKER_SUFFIX= - -# Add space delimited docker tags, and they'll all be applied -DOCKER_TAGS= - DEFAULT_TAG=${DOCKER_ORG}/${PRODUCT}:${VERSION}-cpu-${OSNICK} +# jetson flavour +ifdef REDIS_JETSON +export REDIS_JETSON=1 +export REDIS_CUDA_VERSION=10.2-cudnn7 +DOCKER_SUFFIX=.jetson +DOCKER_ARGS=-g gpu_build=1 +DEFAULT_TAG=${DOCKER_ORG}/${PRODUCT}:${VERSION}-jetson +endif + # set gpu arguments ifeq ($(GPU),1) DOCKER_ARGS = gpu_build=1 @@ -30,17 +28,12 @@ endif # remap ubuntu versions because of nvidia cuda ifeq ($(OSNICK),xenial) -REDIS_CUDA_UBUNTUVERSION=16.04 +export REDIS_CUDA_UBUNTUVERSION=16.04 endif ifeq ($(OSNICK),bionic) -REDIS_CUDA_UBUNTUVERSION=18.04 +export REDIS_CUDA_UBUNTUVERSION=18.04 endif -DOCKERWRAPPER_EXTRA_VARS=\ - REDISAI_LITE=${REDISAI_LITE} \ - REDIS_CUDA_VERSION=${REDIS_CUDA_VERSION} \ - REDIS_CUDA_UBUNTUVERSION=${REDIS_CUDA_UBUNTUVERSION} - ### Defaults ### ROOT=../../.. READIES=${ROOT}/opt/readies diff --git a/opt/build/docker/dockerfile.tmpl b/opt/build/docker/dockerfile.tmpl index 261fd2863..218cd76e0 100755 --- a/opt/build/docker/dockerfile.tmpl +++ b/opt/build/docker/dockerfile.tmpl @@ -8,8 +8,11 @@ ARG TEST={{REDIS_TEST}} #---------------------------------------------------------------------------------------------- FROM redisfab/redis:{{REDIS_VERSION}}-{{REDIS_ARCH}}-{{REDIS_OSNICK}} AS redis -{% if gpu_build is defined %} + +{% if gpu_build is defined and REDIS_JETSON is not defined %} FROM nvidia/cuda:{{REDIS_CUDA_VERSION}}-devel-ubuntu{{REDIS_CUDA_UBUNTUVERSION}} AS builder +{% elif REDIS_JETSON is defined %} +FROM redisfab/jetpack:4.4.1-{{REDIS_ARCH}}-l4t as builder {% else %} FROM {{REDIS_OS}} AS builder {% endif %} @@ -36,11 +39,16 @@ RUN apt-get update -qq RUN apt-get upgrade -yqq RUN rm -rf /var/cache/apt -ARG DEPS_ARGS="" COPY ./get_deps.sh . -RUN if [ "$DEPS_ARGS" = "" ]; then ./get_deps.sh {% if gpu_build is defined %}gpu {% else %} cpu {% endif %}; else env $DEPS_ARGS ./get_deps.sh {% if gpu_build is defined %} gpu {% else %} cpu {% endif %}; fi +{% if REDIS_JETSON is defined %} +ENV DEPS_ARGS="GPU=1 JETSON=1 WITH_PT=1 WITH_TF=1 WITH_TFLITE=0 WITH_ORT=0" +ENV BUILD_ARGS="GPU=1 JETSON=1 WITH_TF=1 WITH_PT=1 WITH_TFLITE=0 WITH_ORT=0" +{% else %} +ENV DEPS_ARGS="" +ENV BUILD_ARGS="" +{% endif %} +RUN $DEPS_ARGS ./get_deps.sh {% if gpu_build is defined %} gpu {% else %} cpu {% endif %} -ARG BUILD_ARGS="" ADD ./ /build RUN bash -l -c "make -C opt build {% if gpu_build is defined %} GPU=1 {% else %} REDISAI_LITE={{REDISAI_LITE}} {% endif %}$BUILD_ARGS SHOW=1" From 389b7c3afdbcc5c67e4a3a476856f4e592bc2087 Mon Sep 17 00:00:00 2001 From: "Chayim I. Kirshen" Date: Sun, 4 Jul 2021 12:13:56 +0300 Subject: [PATCH 2/9] building on jetson - need to fix tensorflow missing header now --- Dockerfile.arm | 83 -------------------------------- get_deps.sh | 4 +- opt/build/docker/dockerfile.tmpl | 2 +- 3 files changed, 4 insertions(+), 85 deletions(-) delete mode 100755 Dockerfile.arm diff --git a/Dockerfile.arm b/Dockerfile.arm deleted file mode 100755 index 22e609e7f..000000000 --- a/Dockerfile.arm +++ /dev/null @@ -1,83 +0,0 @@ -# BUILD redisfab/redisai:${VERSION}-cpu-${ARCH}-${OSNICK} - -ARG REDIS_VER=6.2.4 - -# OSNICK=bionic|stretch|buster -ARG OSNICK=buster - -# OS=debian:buster-slim|debian:stretch-slim|ubuntu:bionic -ARG OS=debian:buster-slim - -# ARCH=arm64v8|arm32v7 -ARG ARCH=arm64v8 - -ARG PACK=0 -ARG REDISAI_LITE=0 -ARG TEST=0 - -#---------------------------------------------------------------------------------------------- -FROM redisfab/redis:${REDIS_VER}-${ARCH}-${OSNICK} AS redis -FROM redisfab/xbuild:${ARCH}-${OS} AS builder - -ARG OSNICK -ARG OS -ARG ARCH -ARG REDIS_VER - -RUN [ "cross-build-start" ] - -RUN echo "Building for ${OSNICK} (${OS}) for ${ARCH}" - -WORKDIR /build -COPY --from=redis /usr/local/ /usr/local/ - -COPY ./opt/ opt/ -COPY ./tests/flow/tests_setup/test_requirements.txt tests/flow - -RUN ./opt/readies/bin/getpy3 -RUN ./opt/system-setup.py - -ARG DEPS_ARGS="" -COPY ./get_deps.sh . -RUN if [ "$DEPS_ARGS" = "" ]; then ./get_deps.sh cpu; else env $DEPS_ARGS ./get_deps.sh cpu; fi - -ARG BUILD_ARGS="" -ADD ./ /build -RUN set -e ;\ - . ./opt/readies/bin/sourced ./profile.d ;\ - make -C opt build $BUILD_ARGS SHOW=1 - -ARG PACK -ARG TEST - -RUN if [ "$PACK" = "1" ]; then make -C opt pack; fi -RUN if [ "$TEST" = "1" ]; then TEST= make -C opt test $BUILD_ARGS NO_LFS=1; fi - -RUN [ "cross-build-end" ] - -#---------------------------------------------------------------------------------------------- -FROM redisfab/redis-xbuild:${REDIS_VER}-${ARCH}-${OSNICK} - -RUN [ "cross-build-start" ] - -ARG OSNICK -ARG OS -ARG ARCH -ARG REDIS_VER -ARG PACK - -RUN if [ ! -z $(command -v apt-get) ]; then apt-get -qq update; apt-get -q install -y libgomp1; fi -RUN if [ ! -z $(command -v yum) ]; then yum install -y libgomp; fi - -ENV REDIS_MODULES /usr/lib/redis/modules -ENV LD_LIBRARY_PATH $REDIS_MODULES - -RUN mkdir -p $REDIS_MODULES/ - -COPY --from=builder /build/install-cpu/ $REDIS_MODULES/ - -WORKDIR /data -EXPOSE 6379 -CMD ["--loadmodule", "/usr/lib/redis/modules/redisai.so"] - -RUN [ "cross-build-end" ] diff --git a/get_deps.sh b/get_deps.sh index 11e18f8ae..96233213e 100755 --- a/get_deps.sh +++ b/get_deps.sh @@ -5,6 +5,8 @@ error() { exit 1 } +set -x + trap error ERR HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" @@ -201,7 +203,7 @@ fi # WITH_TFLITE ###################################################################################### LIBTORCH -PT_VERSION="1.7.1" +PT_VERSION="1.7.0" if [[ $WITH_PT != 0 ]]; then [[ $FORCE == 1 ]] && rm -rf $LIBTORCH diff --git a/opt/build/docker/dockerfile.tmpl b/opt/build/docker/dockerfile.tmpl index 218cd76e0..ad94923b0 100755 --- a/opt/build/docker/dockerfile.tmpl +++ b/opt/build/docker/dockerfile.tmpl @@ -47,7 +47,7 @@ ENV BUILD_ARGS="GPU=1 JETSON=1 WITH_TF=1 WITH_PT=1 WITH_TFLITE=0 WITH_ORT=0" ENV DEPS_ARGS="" ENV BUILD_ARGS="" {% endif %} -RUN $DEPS_ARGS ./get_deps.sh {% if gpu_build is defined %} gpu {% else %} cpu {% endif %} +RUN bash -l -c "$DEPS_ARGS ./get_deps.sh {% if gpu_build is defined %} gpu {% else %} cpu {% endif %}" ADD ./ /build RUN bash -l -c "make -C opt build {% if gpu_build is defined %} GPU=1 {% else %} REDISAI_LITE={{REDISAI_LITE}} {% endif %}$BUILD_ARGS SHOW=1" From db19a51477334da699436cdfb94359c6cbd4abb1 Mon Sep 17 00:00:00 2001 From: "Chayim I. Kirshen" Date: Tue, 6 Jul 2021 09:43:33 +0300 Subject: [PATCH 3/9] getting the variables into the command via jinja --- opt/build/docker/dockerfile.tmpl | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/opt/build/docker/dockerfile.tmpl b/opt/build/docker/dockerfile.tmpl index ad94923b0..311c48fbb 100755 --- a/opt/build/docker/dockerfile.tmpl +++ b/opt/build/docker/dockerfile.tmpl @@ -41,16 +41,13 @@ RUN rm -rf /var/cache/apt COPY ./get_deps.sh . {% if REDIS_JETSON is defined %} -ENV DEPS_ARGS="GPU=1 JETSON=1 WITH_PT=1 WITH_TF=1 WITH_TFLITE=0 WITH_ORT=0" -ENV BUILD_ARGS="GPU=1 JETSON=1 WITH_TF=1 WITH_PT=1 WITH_TFLITE=0 WITH_ORT=0" -{% else %} -ENV DEPS_ARGS="" -ENV BUILD_ARGS="" +{% set DEPS_ARGS = "GPU=1 JETSON=1 WITH_PT=1 WITH_TF=0 WITH_TFLITE=0 WITH_ORT=0" %} +{% set BUILD_ARGS="GPU=1 JETSON=1 WITH_TF=0 WITH_PT=1 WITH_TFLITE=0 WITH_ORT=0" %} {% endif %} -RUN bash -l -c "$DEPS_ARGS ./get_deps.sh {% if gpu_build is defined %} gpu {% else %} cpu {% endif %}" +RUN bash -l -c "{{DEPS_ARGS}} ./get_deps.sh {% if gpu_build is defined %} gpu {% else %} cpu {% endif %}" ADD ./ /build -RUN bash -l -c "make -C opt build {% if gpu_build is defined %} GPU=1 {% else %} REDISAI_LITE={{REDISAI_LITE}} {% endif %}$BUILD_ARGS SHOW=1" +RUN bash -l -c "make -C opt build {% if gpu_build is defined %} GPU=1 {% else %} REDISAI_LITE={{REDISAI_LITE}} {% endif %}{{BUILD_ARGS}} SHOW=1" RUN mkdir -p bin/artifacts @@ -59,7 +56,7 @@ RUN set -e ;\ RUN set -e ;\ if [ "$TEST" = "1" ]; then \ - bash -l -c "TEST= make -C opt test {% if gpu_build is defined %}GPU=1{% endif %} REDISAI_LITE={{REDISAI_LITE}} $BUILD_ARGS NO_LFS=1" ;\ + bash -l -c "TEST= make -C opt test {% if gpu_build is defined %}GPU=1{% endif %} REDISAI_LITE={{REDISAI_LITE}} {{BUILD_ARGS}} NO_LFS=1" ;\ if [[ -d test/logs ]]; then \ {% if gpu_build is defined %} tar -C test/logs -czf bin/artifacts/test-logs-gpu.tgz . ;\ From dcb12abb346e48e728d90b176d5ac6ca3e5ce5a4 Mon Sep 17 00:00:00 2001 From: "Chayim I. Kirshen" Date: Tue, 6 Jul 2021 10:46:39 +0300 Subject: [PATCH 4/9] GPU args are part of the generated deps --- opt/build/docker/Makefile | 2 +- opt/build/docker/dockerfile.tmpl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/opt/build/docker/Makefile b/opt/build/docker/Makefile index b0fc4be78..763ee0cc2 100755 --- a/opt/build/docker/Makefile +++ b/opt/build/docker/Makefile @@ -22,7 +22,7 @@ endif # set gpu arguments ifeq ($(GPU),1) DOCKER_ARGS = gpu_build=1 -DOCKER_SUFFIX=gpu +DOCKER_SUFFIX=.gpu DEFAULT_TAG=${DOCKER_ORG}/${PRODUCT}:${VERSION}-gpu-${OSNICK} endif diff --git a/opt/build/docker/dockerfile.tmpl b/opt/build/docker/dockerfile.tmpl index 311c48fbb..81d42c4ec 100755 --- a/opt/build/docker/dockerfile.tmpl +++ b/opt/build/docker/dockerfile.tmpl @@ -44,7 +44,7 @@ COPY ./get_deps.sh . {% set DEPS_ARGS = "GPU=1 JETSON=1 WITH_PT=1 WITH_TF=0 WITH_TFLITE=0 WITH_ORT=0" %} {% set BUILD_ARGS="GPU=1 JETSON=1 WITH_TF=0 WITH_PT=1 WITH_TFLITE=0 WITH_ORT=0" %} {% endif %} -RUN bash -l -c "{{DEPS_ARGS}} ./get_deps.sh {% if gpu_build is defined %} gpu {% else %} cpu {% endif %}" +RUN bash -l -c "{{DEPS_ARGS}} ./get_deps.sh" ADD ./ /build RUN bash -l -c "make -C opt build {% if gpu_build is defined %} GPU=1 {% else %} REDISAI_LITE={{REDISAI_LITE}} {% endif %}{{BUILD_ARGS}} SHOW=1" From 838c9d352e044a9ae66ee4249d001c1325fc50b4 Mon Sep 17 00:00:00 2001 From: "Chayim I. Kirshen" Date: Tue, 6 Jul 2021 13:59:37 +0300 Subject: [PATCH 5/9] now, on jetson, need to revalidate on x64 --- .circleci/config.yml | 31 +++++++++- Dockerfile.jetson | 102 ------------------------------- README.md | 7 +++ opt/build/docker/Makefile | 23 +------ opt/build/docker/dockerfile.tmpl | 32 ++++------ opt/readies | 2 +- 6 files changed, 51 insertions(+), 146 deletions(-) delete mode 100644 Dockerfile.jetson diff --git a/.circleci/config.yml b/.circleci/config.yml index 0ad738b80..18f6843d6 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -456,6 +456,31 @@ jobs: name: Run QA Automation (AILite) command: MODULE_VERSION=$CIRCLE_TAG VERBOSE=1 TEST=release VARIANT=lite ./tests/qa/run + build-jetson-platform: + machine: + resource_class: arm.medium + image: ubuntu-2004:202101-01 + + steps: + - abort_for_docs + - abort_for_noci + - early_return_for_forked_pull_requests + - relocate-docker-storage + - checkout-all + - setup-automation + - run: + name: Build + command: | + pip3 install --user jinja2 + pushd opt/build/docker + make REDIS_JETSON=1 PACK=0 TEST=0 VERBOSE=1 # PUBLISH=1 + popd > /dev/null + logstar=bin/artifacts/tests-logs-gpu.tgz + logsdir=tests/logs/gpu + mkdir -p $logsdir + if [[ -e $logstar ]]; then tar -C $logsdir -xzf $logstar; fi + # (cd bin/artifacts; tar -cf snapshots.tar snapshots/) + no_output_timeout: 40m #nightly-automation: # docker: @@ -559,6 +584,10 @@ workflows: - build-and-test-gpu-for-forked-prs: <<: *on-any-branch <<: *after-linter + - build-jetson-platform: + context: common + <<: *after-build-and-test + <<: *on-master-version-tags-and-dockertests - platforms-build: context: common <<: *after-build-and-test @@ -569,7 +598,7 @@ workflows: - xenial - bionic lite: - - "REDISAI_LITE=0 PUBLISH=1" + - "REDISAI_LITE=0" ### for now, disabled PUBLISH=1" - "REDISAI_LITE=1" target: - "CPU=1" diff --git a/Dockerfile.jetson b/Dockerfile.jetson deleted file mode 100644 index 34c544b71..000000000 --- a/Dockerfile.jetson +++ /dev/null @@ -1,102 +0,0 @@ -# BUILD redisfab/redisai:${VERSION}-jetson-${ARCH}-${OSNICK} - -ARG REDIS_VER=6.2.4 - -# OSNICK=bionic|centos7|centos6 -ARG OSNICK=bionic - -# OS=ubuntu18.04|ubuntu16.04|centos7 -ARG OS=ubuntu18.04 - -# ARCH=arm64v8|arm32v7 -ARG ARCH=arm64v8 - -ARG CUDA_VER=10.2-cudnn7 - -ARG L4T_VER=r32.4.4 - -ARG PACK=0 -ARG REDISAI_LITE=0 -ARG TEST=0 - -#---------------------------------------------------------------------------------------------- -FROM redisfab/redis:${REDIS_VER}-${ARCH}-${OSNICK} AS redis -FROM redisfab/jetpack:4.4.1-arm64v8-l4t as builder - -ARG OSNICK -ARG OS -ARG ARCH -ARG REDIS_VER -ARG CUDA_VER -ARG L4T_VER - -RUN echo "Building for ${OSNICK} (${OS}) for ${ARCH} [with Redis ${REDIS_VER}]" - -ENV NVIDIA_VISIBLE_DEVICES all -ENV NVIDIA_DRIVER_CAPABILITIES compute,utility - -WORKDIR /build -COPY --from=redis /usr/local/ /usr/local/ - -COPY ./opt/ opt/ -COPY ./tests/flow/tests_setup/test_requirements.txt tests/flow/ - -RUN FORCE=1 ./opt/readies/bin/getpy3 -RUN ./opt/system-setup.py - -ARG DEPS_ARGS="GPU=1 JETSON=1 WITH_PT=1 WITH_TF=1 WITH_TFLITE=0 WITH_ORT=0" -COPY ./get_deps.sh . -# RUN if [ "$DEPS_ARGS" = "" ]; then ./get_deps.sh gpu; else env $DEPS_ARGS ./get_deps.sh gpu; fi -RUN set -e ;\ - env $DEPS_ARGS ./get_deps.sh - -ARG BUILD_ARGS="GPU=1 JETSON=1 WITH_TF=1 WITH_PT=1 WITH_TFLITE=0 WITH_ORT=0" - -ADD ./ /build -RUN bash -c "set -e ;\ - . ./opt/readies/bin/sourced ./profile.d ;\ - make -C opt build $BUILD_ARGS SHOW=1" - -ARG PACK -ARG REDISAI_LITE -ARG TEST - -RUN mkdir -p bin/artifacts -RUN set -e ;\ - if [ "$PACK" = "1" ]; then make -C opt pack GPU=1; fi - -RUN set -e ;\ - if [ "$TEST" = "1" ]; then \ - TEST= make -C opt test GPU=1 $BUILD_ARGS NO_LFS=1 ;\ - if [[ -d test/logs ]]; then \ - tar -C test/logs -czf bin/artifacts/test-logs-gpu.tgz . ;\ - fi ;\ - fi - -#---------------------------------------------------------------------------------------------- -# FROM nvidia/cuda:${CUDA_VER}-runtime-${OS} -# FROM nvidia/cuda-arm64:11.1-runtime-ubuntu18.04 -# FROM nvcr.io/nvidia/l4t-base:${L4T_VER} -FROM redisfab/jetpack:4.4.1-arm64v8-l4t - -ARG OS -ARG L4T_VER - -RUN if [ ! -z $(command -v apt-get) ]; then apt-get -qq update; apt-get -q install -y libgomp1; fi -RUN if [ ! -z $(command -v yum) ]; then yum install -y libgomp; fi - -ENV REDIS_MODULES /usr/lib/redis/modules -RUN mkdir -p $REDIS_MODULES/ - -COPY --from=redis /usr/local/ /usr/local/ -COPY --from=builder /build/install-gpu/ $REDIS_MODULES/ - -COPY --from=builder /build/bin/artifacts/ /var/opt/redislabs/artifacts - -# COPY --from=builder /usr/local/etc/redis/redis.conf /usr/local/etc/redis/ - -WORKDIR /data -EXPOSE 6379 -# CMD ["/usr/local/bin/redis-server", "/usr/local/etc/redis/redis.conf", "--loadmodule", "/usr/lib/redis/modules/redisai.so"] -RUN rm -rf /root/.cache /usr/local/cuda/lib64/*.a /usr/local/cuda/doc /usr/local/cuda/samples -CMD ["/usr/local/bin/redis-server", "--loadmodule", "/usr/lib/redis/modules/redisai.so"] diff --git a/README.md b/README.md index bfd4b54b5..357e86ab8 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,13 @@ For docker instance with GPU support, you can launch it from `tensorwerk/redisai ```sh docker run -p 6379:6379 --gpus all -it --rm redislabs/redisai:edge-gpu + + +``` + +For a docker instance on an arm device we currently support CUDA 10.2 and CUDNN 7 - built on the nvidia jetson and xavier devices. The docker images below, are specific to those devices, and contain only the pytorch backend. +```sh +docker run -p 6379:6379 --gpus all -it --rm redislabs/redisai:edge-gpu-jetson ``` But if you'd like to build the docker image, you need a machine that has Nvidia driver (CUDA 10.0), nvidia-container-toolkit and Docker 19.03+ installed. For detailed information, checkout [nvidia-docker documentation](https://github.com/NVIDIA/nvidia-docker) diff --git a/opt/build/docker/Makefile b/opt/build/docker/Makefile index 763ee0cc2..abed1ba76 100755 --- a/opt/build/docker/Makefile +++ b/opt/build/docker/Makefile @@ -15,8 +15,8 @@ ifdef REDIS_JETSON export REDIS_JETSON=1 export REDIS_CUDA_VERSION=10.2-cudnn7 DOCKER_SUFFIX=.jetson -DOCKER_ARGS=-g gpu_build=1 -DEFAULT_TAG=${DOCKER_ORG}/${PRODUCT}:${VERSION}-jetson +DOCKER_ARGS=gpu_build=1 +DEFAULT_TAG=${DOCKER_ORG}/${PRODUCT}:${VERSION}-gpu-jetson endif # set gpu arguments @@ -38,24 +38,5 @@ endif ROOT=../../.. READIES=${ROOT}/opt/readies -#------------------------------------------------------------------------------ -define HELP -make build # build and optionally publish the docker, from a template file - OSNICK=<> # base operating system - REDISAI_LITE=1 # enable RedisAI lite builds - DOCKER_SUFFIX=<> # optional suffix for the generated dockerfile - DOCKER_ARGS=FOO=BAR # key-value pairs of variables to pass into the docker build - PACK=1 # fetch generated artifacts - DOCKER_TAGS=a,b,c # tags to append and push to dockerhub - DEFAULT_TAG=redislabs/redisai # default docker tag to build, and push - TEST=1 # run tests, if specified - VERSION=x.y.z # set the docker version - NOP=1 # set to echo files in docker generation, and not run - DOCKER_SOURCES=/a/path # append paths to the template generator - PUBLISH=1 # if set, push to dockerhub (requires docker login) - DOCKER_OPTS=XXX # Options to pass to the docker build command -endef -#------------------------------------------------------------------------------ - # rules, for building include ${READIES}/mk/docker.rules diff --git a/opt/build/docker/dockerfile.tmpl b/opt/build/docker/dockerfile.tmpl index 81d42c4ec..ff60d90bc 100755 --- a/opt/build/docker/dockerfile.tmpl +++ b/opt/build/docker/dockerfile.tmpl @@ -1,10 +1,4 @@ -# BUILD redisfab/redisai:{{VERSION}}-cpu-{{REDIS_ARCH}}-{{REDIS_OSNICK}} - - -ARG PACK={{REDIS_PACK}} -ARG REDISAI_LITE={{REDISAI_LITE}} -ARG TEST={{REDIS_TEST}} - +# BUILD redisfab/redisai:{{REDIS_VERSION}}-cpu|gpu|jetson-{{REDIS_ARCH}}-{{REDIS_OSNICK}} #---------------------------------------------------------------------------------------------- FROM redisfab/redis:{{REDIS_VERSION}}-{{REDIS_ARCH}}-{{REDIS_OSNICK}} AS redis @@ -17,10 +11,6 @@ FROM redisfab/jetpack:4.4.1-{{REDIS_ARCH}}-l4t as builder FROM {{REDIS_OS}} AS builder {% endif %} -ARG REDISAI_LITE -ARG PACK -ARG TEST - RUN echo "Building for {{REDIS_OSNICK}} ({{REDIS_OS}}) for {{REDIS_ARCH}} [with Redis {{REDIS_VERSION}}]" {% if gpu_build is defined %} @@ -42,20 +32,21 @@ RUN rm -rf /var/cache/apt COPY ./get_deps.sh . {% if REDIS_JETSON is defined %} {% set DEPS_ARGS = "GPU=1 JETSON=1 WITH_PT=1 WITH_TF=0 WITH_TFLITE=0 WITH_ORT=0" %} -{% set BUILD_ARGS="GPU=1 JETSON=1 WITH_TF=0 WITH_PT=1 WITH_TFLITE=0 WITH_ORT=0" %} +{% set BUILD_ARGS="GPU=1 JETSON=1 WITH_TF=0 WITH_PT=1 WITH_TFLITE=0 WITH_ORT=0 WITH_UNIT_TESTS=0" %} {% endif %} RUN bash -l -c "{{DEPS_ARGS}} ./get_deps.sh" ADD ./ /build -RUN bash -l -c "make -C opt build {% if gpu_build is defined %} GPU=1 {% else %} REDISAI_LITE={{REDISAI_LITE}} {% endif %}{{BUILD_ARGS}} SHOW=1" +RUN bash -l -c "make -C opt build REDISAI_LITE={{REDISAI_LITE}} {{BUILD_ARGS}} SHOW=1" RUN mkdir -p bin/artifacts -RUN set -e ;\ - if [ "$PACK" = "1" ]; then bash -l -c "make -C opt pack {% if gpu_build is defined %} GPU=1 {% endif %} REDISAI_LITE={{REDISAI_LITE}}"; fi +{% if REDIS_PACK == 1 %} +RUN set -e bash -l -c "make -C opt pack {% if gpu_build is defined %} GPU=1 {% endif %} REDISAI_LITE={{REDISAI_LITE}}" +{% endif %} -RUN set -e ;\ - if [ "$TEST" = "1" ]; then \ +{% if REDIS_TEST == 1 %} +RUN set -d; bash -l -c "TEST= make -C opt test {% if gpu_build is defined %}GPU=1{% endif %} REDISAI_LITE={{REDISAI_LITE}} {{BUILD_ARGS}} NO_LFS=1" ;\ if [[ -d test/logs ]]; then \ {% if gpu_build is defined %} @@ -64,17 +55,16 @@ RUN set -e ;\ tar -C test/logs -czf bin/artifacts/test-logs-cpu.tgz . ;\ {% endif %} fi ;\ - fi + +{% endif %} #---------------------------------------------------------------------------------------------- -{% if gpu_build is defined %} +{% if gpu_build is defined and REDIS_JETSON is not defined %} FROM nvidia/cuda:{{REDIS_CUDA_VERSION}}-runtime-ubuntu{{REDIS_CUDA_UBUNTUVERSION}} {% else %} FROM redisfab/redis:{{REDIS_VERSION}}-{{REDIS_ARCH}}-{{REDIS_OSNICK}} {% endif %} -ARG PACK - RUN if [ ! -z $(command -v apt-get) ]; then apt-get -qq update; apt-get -q install -y libgomp1; fi RUN if [ ! -z $(command -v yum) ]; then yum install -y libgomp; fi diff --git a/opt/readies b/opt/readies index 4cb9aa263..9a481a4ac 160000 --- a/opt/readies +++ b/opt/readies @@ -1 +1 @@ -Subproject commit 4cb9aa2638aab8518f63a7ecb0b97c9318035845 +Subproject commit 9a481a4ac0fd90573ea4ff10129e94ed4c7a30a4 From de3608b2fb3ebc1aa65cb89492152d9354648304 Mon Sep 17 00:00:00 2001 From: "Chayim I. Kirshen" Date: Tue, 6 Jul 2021 14:24:27 +0300 Subject: [PATCH 6/9] GPU restore x64 --- opt/build/docker/dockerfile.tmpl | 4 ++++ opt/readies | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/opt/build/docker/dockerfile.tmpl b/opt/build/docker/dockerfile.tmpl index ff60d90bc..096196120 100755 --- a/opt/build/docker/dockerfile.tmpl +++ b/opt/build/docker/dockerfile.tmpl @@ -33,6 +33,10 @@ COPY ./get_deps.sh . {% if REDIS_JETSON is defined %} {% set DEPS_ARGS = "GPU=1 JETSON=1 WITH_PT=1 WITH_TF=0 WITH_TFLITE=0 WITH_ORT=0" %} {% set BUILD_ARGS="GPU=1 JETSON=1 WITH_TF=0 WITH_PT=1 WITH_TFLITE=0 WITH_ORT=0 WITH_UNIT_TESTS=0" %} +{% else %} +{% if gpu_build is defined %} +{% set DEPS_ARGS = "GPU=1" %} +{% endif %} {% endif %} RUN bash -l -c "{{DEPS_ARGS}} ./get_deps.sh" diff --git a/opt/readies b/opt/readies index 9a481a4ac..2fdf0569e 160000 --- a/opt/readies +++ b/opt/readies @@ -1 +1 @@ -Subproject commit 9a481a4ac0fd90573ea4ff10129e94ed4c7a30a4 +Subproject commit 2fdf0569e4d662bc7c402031f2da58d1a2d3f780 From adf694f9b4a4af573e394b68fbc6ed547a531e14 Mon Sep 17 00:00:00 2001 From: "Chayim I. Kirshen" Date: Wed, 7 Jul 2021 10:42:39 +0300 Subject: [PATCH 7/9] test and pack fix --- .circleci/config.yml | 10 ++-------- opt/build/docker/Makefile | 1 + opt/build/docker/dockerfile.tmpl | 18 ++++++++++-------- 3 files changed, 13 insertions(+), 16 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 149197b97..62e26e32d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -473,13 +473,7 @@ jobs: command: | pip3 install --user jinja2 pushd opt/build/docker - make REDIS_JETSON=1 PACK=0 TEST=0 VERBOSE=1 # PUBLISH=1 - popd > /dev/null - logstar=bin/artifacts/tests-logs-gpu.tgz - logsdir=tests/logs/gpu - mkdir -p $logsdir - if [[ -e $logstar ]]; then tar -C $logsdir -xzf $logstar; fi - # (cd bin/artifacts; tar -cf snapshots.tar snapshots/) + make REDIS_JETSON=1 PACK=0 TEST=0 VERBOSE=1 PUBLISH=1 no_output_timeout: 40m #nightly-automation: @@ -598,7 +592,7 @@ workflows: - xenial - bionic lite: - - "REDISAI_LITE=0" ### for now, disabled PUBLISH=1" + - "REDISAI_LITE=0 PUBLISH=1" - "REDISAI_LITE=1" target: - "CPU=1" diff --git a/opt/build/docker/Makefile b/opt/build/docker/Makefile index abed1ba76..ed3e48658 100755 --- a/opt/build/docker/Makefile +++ b/opt/build/docker/Makefile @@ -6,6 +6,7 @@ export REDIS_CUDA_VERSION=11.0-cudnn8 export REDISAI_LITE=0 PACK=1 # to fetch the artifacts +TEST=1 # because dockers OSNICK ?= bionic DEFAULT_TAG=${DOCKER_ORG}/${PRODUCT}:${VERSION}-cpu-${OSNICK} diff --git a/opt/build/docker/dockerfile.tmpl b/opt/build/docker/dockerfile.tmpl index 096196120..e0fa14c40 100755 --- a/opt/build/docker/dockerfile.tmpl +++ b/opt/build/docker/dockerfile.tmpl @@ -45,12 +45,12 @@ RUN bash -l -c "make -C opt build REDISAI_LITE={{REDISAI_LITE}} {{BUILD_ARGS}} S RUN mkdir -p bin/artifacts -{% if REDIS_PACK == 1 %} +{% if REDIS_PACK|int == 1 %} RUN set -e bash -l -c "make -C opt pack {% if gpu_build is defined %} GPU=1 {% endif %} REDISAI_LITE={{REDISAI_LITE}}" {% endif %} -{% if REDIS_TEST == 1 %} -RUN set -d; +{% if REDIS_TEST|int == 1 %} +RUN set -e;\ bash -l -c "TEST= make -C opt test {% if gpu_build is defined %}GPU=1{% endif %} REDISAI_LITE={{REDISAI_LITE}} {{BUILD_ARGS}} NO_LFS=1" ;\ if [[ -d test/logs ]]; then \ {% if gpu_build is defined %} @@ -72,18 +72,20 @@ FROM redisfab/redis:{{REDIS_VERSION}}-{{REDIS_ARCH}}-{{REDIS_OSNICK}} RUN if [ ! -z $(command -v apt-get) ]; then apt-get -qq update; apt-get -q install -y libgomp1; fi RUN if [ ! -z $(command -v yum) ]; then yum install -y libgomp; fi -ENV REDIS_MODULES /usr/lib/redis/modules -ENV LD_LIBRARY_PATH $REDIS_MODULES +{% set REDIS_MODULES = "/usr/lib/redis/modules" %} +ENV LD_LIBRARY_PATH {{REDIS_MODULES}} -RUN mkdir -p $REDIS_MODULES/ +RUN mkdir -p {{REDIS_MODULES}}/ {% if gpu_build is defined %} -COPY --from=builder /build/install-gpu/ $REDIS_MODULES/ +COPY --from=builder /build/install-gpu/ {{REDIS_MODULES}}/ {% else %} -COPY --from=builder /build/install-cpu/ $REDIS_MODULES/ +COPY --from=builder /build/install-cpu/ {{REDIS_MODULES}}/ {% endif %} +{% if REDIS_TEST|int == 1 %} COPY --from=builder /build/bin/artifacts/ /var/opt/redislabs/artifacts +{% endif %} WORKDIR /data EXPOSE 6379 From beb33817894f34e212c3f19f6c74ee5b6acb5f9d Mon Sep 17 00:00:00 2001 From: "Chayim I. Kirshen" Date: Thu, 8 Jul 2021 09:48:51 +0300 Subject: [PATCH 8/9] pytorch per platform --- get_deps.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/get_deps.sh b/get_deps.sh index 96233213e..fab45f890 100755 --- a/get_deps.sh +++ b/get_deps.sh @@ -203,7 +203,10 @@ fi # WITH_TFLITE ###################################################################################### LIBTORCH -PT_VERSION="1.7.0" +PT_VERSION="1.7.1" +if [[ $JETSON == 1 ]]; then + PT_VERSION="1.7.0" +fi if [[ $WITH_PT != 0 ]]; then [[ $FORCE == 1 ]] && rm -rf $LIBTORCH From ba3507111f478c457c9246aa7ff577223ace5ef7 Mon Sep 17 00:00:00 2001 From: "Chayim I. Kirshen" Date: Thu, 8 Jul 2021 09:50:24 +0300 Subject: [PATCH 9/9] docker login --- .circleci/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 62e26e32d..91c1009db 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -473,6 +473,7 @@ jobs: command: | pip3 install --user jinja2 pushd opt/build/docker + docker login -u redisfab -p $DOCKER_REDISFAB_PWD make REDIS_JETSON=1 PACK=0 TEST=0 VERBOSE=1 PUBLISH=1 no_output_timeout: 40m