Docker ci update (#1219)

* Reworked CI to use default docker actions and cache using GH cache * Refactor builder dockerfiles * base-rocks and base-go are going to be consolidated into base with multi-stage * Added FROM with arg so we can use a local registry in the CI * Added local registry for CI to save images between steps * moved api key and spid files to arg in base dockerfile and added localsecret defaults * Remove reference to query enclave * Changed Azure pipeline to use buildkit
scrtlabs · Oct 20, 2022 · 9f20a31 · 9f20a31
1 parent 0196311
commit 9f20a31
Show file tree

Hide file tree

Showing 23 changed files with 391 additions and 783 deletions.
diff --git a/.dockerignore b/.dockerignore
@@ -10,13 +10,15 @@ docs
 third_party/vendor
 *.bin
 
-cosmwasm-js
-
+.gitpod.yml
+.golangci.yml
+azure-pipelines.yml
+sn-logo.png
 README.md
 .gitignore
 
 .vscode
 .github
 .git
 .idea
-**/.idea
+**/.idea
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
@@ -53,71 +53,20 @@ jobs:
           RUSTC_WRAPPER="$HOME/sccache" make enclave-tests
           make clean-enclave
 
-  Build:
+  Build-Contracts:
     runs-on: ubuntu-20.04
     steps:
       - uses: actions/checkout@v2
-      - uses: actions/setup-go@v3
-        with:
-          go-version: 1.19 # The Go version to download (if necessary) and use.
-      - name: Install Intel's SGX SDK
-        run: |
-          mkdir -p "$HOME/.sgxsdk"
-          cd "$HOME/.sgxsdk"
-          SDK_BIN=sgx_linux_x64_sdk_2.9.101.2.bin
-          wget https://download.01.org/intel-sgx/sgx-linux/2.9.1/distro/ubuntu18.04-server/"$SDK_BIN"
-          chmod +x "$SDK_BIN"
-          echo yes | ./"$SDK_BIN"
-      - name: Cache cargo registry
-        uses: actions/cache@v3
-        with:
-          path: ~/.cargo/registry
-          key: ${{ runner.os }}-cargo-registry-${{ hashFiles('**/Cargo.lock') }}
-      - name: Cache xargo sysroot
-        uses: actions/cache@v3
-        with:
-          path: ~/.xargo
-          key: ${{ runner.os }}-xargo-sysroot
-      - name: Cache build artifacts
-        uses: actions/cache@v3
-        with:
-          path: ~/.cache/sccache
-          key: ${{ runner.os }}-sccache
-      - run: |
-          rustup component add rust-src clippy
-          cd cosmwasm/enclaves/execute/
-          rustup component add rust-src clippy
-      - name: Install xargo
-        run: |
-          cargo --version
-          rustc --version  
-          cargo +stable install xargo --version 0.3.25
-          xargo --version
       - name: Download sccache
         run: |
           wget https://github.com/mozilla/sccache/releases/download/0.2.13/sccache-0.2.13-x86_64-unknown-linux-musl.tar.gz
           tar xf ./sccache-*.tar.gz
           mv ./sccache*/sccache "$HOME/sccache"
-      - name: Create fake api keys
-        run: |
-          mkdir -p ias_keys/sw_dummy
-          mkdir -p ias_keys/develop
-          echo "not_a_key" > ias_keys/sw_dummy/spid.txt
-          echo "not_a_key" > ias_keys/develop/spid.txt
-          echo "not_a_key" > ias_keys/sw_dummy/api_key.txt
-          echo "not_a_key" > ias_keys/develop/api_key.txt
       - name: Install Requirements
         run: |
           rustup target add wasm32-unknown-unknown
-          go install github.com/jteeuwen/go-bindata/go-bindata@latest
-          go-bindata -version # check installation
           chmod +x ./scripts/install-wasm-tools.sh
           ./scripts/install-wasm-tools.sh
-      - name: Build Executable
-        run: |
-          source "$HOME/.sgxsdk/sgxsdk/environment"
-          make vendor
-          SGX_MODE=SW BUILD_PROFILE="minimal" RUSTC_WRAPPER="$HOME/sccache" make build-linux
       - name: Build Contracts
         run: |
           make build-test-contract
@@ -126,10 +75,6 @@ jobs:
         with:
           name: erc20.wasm
           path: erc20.wasm
-      - uses: actions/upload-artifact@v3
-        with:
-          name: secretd
-          path: secretd
       - uses: actions/upload-artifact@v3
         with:
           name: contract.wasm
@@ -142,14 +87,6 @@ jobs:
         with:
           name: ibc.wasm
           path: cosmwasm/contracts/v1/compute-tests/ibc-test-contract/ibc.wasm
-      - uses: actions/upload-artifact@v3
-        with:
-          name: enclave
-          path: ./go-cosmwasm/librust_cosmwasm_enclave.signed.so
-      - uses: actions/upload-artifact@v3
-        with:
-          name: libgo_cosmwasm
-          path: ./go-cosmwasm/api/libgo_cosmwasm.so
       - uses: actions/upload-artifact@v3
         with:
           name: contract_with_floats.wasm
@@ -165,7 +102,7 @@ jobs:
 
   Go-Tests:
     runs-on: ubuntu-20.04
-    needs: Build
+    needs: [Build-Contracts, Build-LocalSecret]
     steps:
       - uses: actions/checkout@v2
       - uses: actions/setup-go@v3
@@ -179,12 +116,18 @@ jobs:
           wget https://download.01.org/intel-sgx/sgx-linux/2.9.1/distro/ubuntu18.04-server/"$SDK_BIN"
           chmod +x "$SDK_BIN"
           echo yes | ./"$SDK_BIN"
-      - uses: actions/download-artifact@v3
+      - name: Download LocalSecret
+        uses: actions/download-artifact@v3
         with:
-          name: libgo_cosmwasm
-      - uses: actions/download-artifact@v3
-        with:
-          name: enclave
+          name: localsecret
+          path: /tmp
+      - name: Load images
+        run: |
+          docker load --input /tmp/localsecret.tar
+          docker run -v $PWD:/opt/mount --rm --entrypoint cp ghcr.io/scrtlabs/localsecret:v0.0.0 /usr/bin/secretd /opt/mount/secretd
+          docker run -v $PWD:/opt/mount --rm --entrypoint cp ghcr.io/scrtlabs/localsecret:v0.0.0 /usr/lib/libgo_cosmwasm.so /opt/mount/libgo_cosmwasm.so
+          docker run -v $PWD:/opt/mount --rm --entrypoint cp ghcr.io/scrtlabs/localsecret:v0.0.0 /usr/lib/librust_cosmwasm_enclave.signed.so /opt/mount/librust_cosmwasm_enclave.signed.so
+
       - uses: actions/download-artifact@v3
         with:
           name: contract.wasm
@@ -296,17 +239,103 @@ jobs:
       - name: Build MacOS CLI
         run: make build_macos_arm64_cli
 
+  Build-LocalSecret:
+    runs-on: ubuntu-20.04
+    services:
+      registry:
+        image: registry:2
+        ports:
+          - 5000:5000
+    steps:
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+        with:
+          driver-opts: network=host
+      - uses: actions/checkout@v2
+      - name: Build LocalSecret Base
+        uses: docker/build-push-action@v3
+        with:
+          file: deployment/dockerfiles/base.Dockerfile
+          context: .
+          build-args: |
+            FEATURES=debug-print
+            SGX_MODE=SW
+          push: true
+          secrets: |
+            API_KEY=00000000000000000000000000000000
+            SPID=FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+          tags: localhost:5000/scrt-enclave-secretd:latest
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+      - name: Build LocalSecret Release
+        uses: docker/build-push-action@v3
+        with:
+          file: deployment/dockerfiles/release.Dockerfile
+          context: .
+          push: true
+          tags: localhost:5000/release:latest
+          build-args: |
+            SECRET_NODE_TYPE=bootstrap
+            CHAIN_ID=secretdev-1
+            SCRT_BIN_IMAGE=localhost:5000/scrt-enclave-secretd:latest
+      - name: Build LocalSecret Final Dev Image
+        uses: docker/build-push-action@v3
+        with:
+          file: deployment/dockerfiles/dev-image.Dockerfile
+          context: .
+          load: true
+          tags: ghcr.io/scrtlabs/localsecret:v0.0.0
+          build-args: |
+            SCRT_BASE_IMAGE=localhost:5000/release:latest
+          outputs: type=docker,dest=/tmp/localsecret.tar
+      - name: Upload Image
+        uses: actions/upload-artifact@v3
+        with:
+          name: localsecret
+          path: /tmp/localsecret.tar
+
+  Build-Hermes:
+    runs-on: ubuntu-20.04
+    steps:
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+      - uses: actions/checkout@v2
+      - name: Build Hermes Image
+        uses: docker/build-push-action@v3
+        with:
+          file: deployment/dockerfiles/ibc/hermes.Dockerfile
+          context: deployment/dockerfiles/ibc
+          load: true
+          tags: hermes:v0.0.0
+          build-args: |
+            SECRET_NODE_TYPE=bootstrap
+            CHAIN_ID=secretdev-1
+          outputs: type=docker,dest=/tmp/hermes.tar
+      - name: Upload Image
+        uses: actions/upload-artifact@v3
+        with:
+          name: hermes
+          path: /tmp/hermes.tar
+
   Integration-Tests:
     runs-on: ubuntu-20.04
+    needs: [Build-LocalSecret, Build-Hermes]
     steps:
       - uses: actions/checkout@v2
-      - name: Build LocalSecret
-        run: |
-          echo not_a_key | tee {api_key,spid}.txt
-          DOCKER_TAG=v0.0.0 make build-localsecret
-      - name: Build Hermes
+      - name: Download Hermes
+        uses: actions/download-artifact@v3
+        with:
+          name: hermes
+          path: /tmp
+      - name: Download LocalSecret
+        uses: actions/download-artifact@v3
+        with:
+          name: localsecret
+          path: /tmp
+      - name: Load images
         run: |
-          DOCKER_TAG=v0.0.0 make build-ibc-hermes
+          docker load --input /tmp/localsecret.tar
+          docker load --input /tmp/hermes.tar
       - name: Run integration tests
         run: |
           cd integration-tests

diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
@@ -17,8 +17,8 @@ jobs:
         db_backend: [rocksdb, goleveldb]
     runs-on: ubuntu-20.04
     env: # Or as an environment variable
-      SPID_TESTNET: ${{ secrets.SPID_TESTNET }}
-      API_KEY_TESTNET: ${{ secrets.API_KEY_TESTNET }}
+      SPID: ${{ secrets.SPID_TESTNET }}
+      API_KEY: ${{ secrets.API_KEY_TESTNET }}
     steps:
       - uses: actions/checkout@v3
       - name: Declare Commit Variables
@@ -27,46 +27,11 @@ jobs:
         run: |
           echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})"
           echo "::set-output name=sha_short::$(git rev-parse --short HEAD)"
-      - uses: actions/setup-go@v3
-        with:
-          go-version: 1.19 # The Go version to download (if necessary) and use.
-      - name: Install Intel's SGX SDK
-        run: |
-          mkdir -p "$HOME/.sgxsdk"
-          cd "$HOME/.sgxsdk"
-          SDK_BIN=sgx_linux_x64_sdk_2.9.101.2.bin
-          wget https://download.01.org/intel-sgx/sgx-linux/2.9.1/distro/ubuntu18.04-server/"$SDK_BIN"
-          chmod +x "$SDK_BIN"
-          echo yes | ./"$SDK_BIN"
-      - name: Cache cargo registry
-        uses: actions/cache@v3
-        with:
-          path: ~/.cargo/registry
-          key: ${{ runner.os }}-cargo-registry-${{ hashFiles('**/Cargo.lock') }}
-      - name: Cache xargo sysroot
-        uses: actions/cache@v3
-        with:
-          path: ~/.xargo
-          key: ${{ runner.os }}-xargo-sysroot
       - name: Cache build artifacts
         uses: actions/cache@v3
         with:
           path: ~/.cache/sccache
           key: ${{ runner.os }}-sccache
-      - run: rustup component add rust-src clippy
-      - name: Install xargo
-        run: |
-          cargo +stable install xargo --version 0.3.25
-          xargo --version
-      - name: Download sccache
-        run: |
-          wget https://github.com/mozilla/sccache/releases/download/0.2.13/sccache-0.2.13-x86_64-unknown-linux-musl.tar.gz
-          tar xf ./sccache-*.tar.gz
-          mv ./sccache*/sccache "$HOME/sccache"
-      - name: Create api keys
-        run: |
-          echo $SPID_TESTNET > spid.txt
-          echo $API_KEY_TESTNET > api_key.txt
       - name: Get the version
         id: get_version
         run: echo ::set-output name=VERSION::${GITHUB_REF/refs\/tags\/v/}
@@ -87,8 +52,8 @@ jobs:
       matrix:
         db_backend: [goleveldb, rocksdb]
     env: # Or as an environment variable
-      SPID_MAINNET: ${{ secrets.SPID_MAINNET }}
-      API_KEY_MAINNET: ${{ secrets.API_KEY_MAINNET }}
+      SPID: ${{ secrets.SPID_MAINNET }}
+      API_KEY: ${{ secrets.API_KEY_MAINNET }}
       REGISTRY: ghcr.io
       IMAGE_NAME: scrtlabs/secret-network-node
     steps:
@@ -99,10 +64,6 @@ jobs:
       - uses: actions/setup-go@v3
         with:
           go-version: 1.19 # The Go version to download (if necessary) and use.
-      - name: Create api keys
-        run: |
-          echo $SPID_MAINNET > spid.txt
-          echo $API_KEY_MAINNET > api_key.txt
       - name: Build
         run: |
           VERSION=${{ steps.get_version.outputs.VERSION }} DB_BACKEND=${{ matrix.db_backend }} FEATURES=production FEATURES_U=production, make build-mainnet-upgrade
@@ -187,7 +148,7 @@ jobs:
       - name: Build dev docker image
         shell: bash
         run: |
-          DOCKER_TAG=${{ steps.get_version.outputs.VERSION }} make build-localsecret
+          DOCKER_TAG=${{ steps.get_version.outputs.VERSION }} make localsecret
       - name: Push docker image
         run: docker push ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.get_version.outputs.VERSION }}