Merge remote-tracking branch 'other/shiva/vector' into shiva/vector

.github/CODEOWNERS

@@ -1,4 +1,4 @@
 # CODEOWNERS info: https://help.github.com/en/articles/about-code-owners
 # Owners are automatically requested for review for PRs that changes code
 # that they own.
-* @dgraph-io/committers
+* @gohypermode/database

.github/workflows/cd-dgraph.yml

@@ -2,21 +2,37 @@ name: cd-dgraph
 on:
   workflow_dispatch:
     inputs:
-      latest:
-        type: boolean
-        default: false
-        description: release latest tag docker-images on dockerhub
+      environment:
+        description: 'environment'
+        required: true
+        default: 'stage'
+        type: choice
+        options:
+          - stage
+          - prod
       releasetag:
-        description: releasetag
+        description: 'releasetag'
         required: true
         type: string
+  schedule:
+    - cron: '0 0 * * *' # nightly
+permissions:
+  id-token: write
+  contents: read
+env:
+  GOPRIVATE: "github.com/dgraph-io/*,github.com/gohypermode/*"
 jobs:
-  dgraph-build-amd64:
+  dgraph-build-nightly:
+    if: ${{ github.event_name == 'schedule' }}
+    environment: stage
     runs-on: ubuntu-20.04
     steps:
+      - name: Configure git for private modules
+        env:
+          # auth token for dgraph-bot; needed to checkout private go module dependencies
+          TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
+        run: git config --global url."https://dgraph-bot:${TOKEN}@github.com".insteadOf "https://github.com"
       - uses: actions/checkout@v3
-        with:
-          ref: '${{ github.event.inputs.releasetag }}'
       - name: Get Go Version
         run: |
           #!/bin/bash
@@ -34,76 +50,43 @@ jobs:
           go mod tidy
           make regenerate
           git diff --exit-code -- .
-      - name: Set Badger Release Version
+      - name: Dgraph Git SHA
         run: |
-          #!/bin/bash
-          BADGER_RELEASE_VERSION=$(cat go.mod | grep -i "github.com/dgraph-io/badger" | awk '{print $2}')
-          echo "setting badger version "$BADGER_RELEASE_VERSION
-          echo "BADGER_RELEASE_VERSION=$BADGER_RELEASE_VERSION" >> $GITHUB_ENV
-      - name: Download Badger Artifacts
-        run: |
-          #!/bin/bash
-          mkdir badger
-          cd badger
-          wget https://github.com/dgraph-io/badger/releases/download/${{ env.BADGER_RELEASE_VERSION }}/badger-checksum-linux-amd64.sha256
-          wget https://github.com/dgraph-io/badger/releases/download/${{ env.BADGER_RELEASE_VERSION }}/badger-linux-amd64.tar.gz
-      - name: Set Dgraph Release Version
-        run: |
-          #!/bin/bash
-          GIT_TAG_NAME='${{ github.event.inputs.releasetag }}'
-          if [[ "$GIT_TAG_NAME" == "v"* ]]; 
-          then
-            echo "this is a release branch"
-          else
-            echo "this is NOT a release branch"
-            exit 1
-          fi
-          DGRAPH_RELEASE_VERSION='${{ github.event.inputs.releasetag }}'
-          echo "making a new release for dgraph "$DGRAPH_RELEASE_VERSION
-          echo "DGRAPH_RELEASE_VERSION=$DGRAPH_RELEASE_VERSION" >> $GITHUB_ENV
+          DGRAPH_GIT_SHA=$(git rev-parse --short HEAD)
+          echo "dgraph git sha "$DGRAPH_GIT_SHA
+          echo "DGRAPH_GIT_SHA=$DGRAPH_GIT_SHA" >> $GITHUB_ENV
+      - name: Fetch date
+        run: echo "NIGHTLY_BUILD_TIME=$(date +'%Y-%m-%d')" >> $GITHUB_ENV
       - name: Make Dgraph Linux Build
-        run: make dgraph DGRAPH_VERSION=${{ env.DGRAPH_RELEASE_VERSION }}
-      - name: Generate SHA for Dgraph Linux Build
-        run: cd dgraph && sha256sum dgraph | cut -c-64 > dgraph-checksum-linux-amd64.sha256
-      - name: Tar Archive for Dgraph Linux Build
-        run: cd dgraph && tar -zcvf dgraph-linux-amd64.tar.gz dgraph
-      - name: Upload Build Artifacts
-        uses: actions/upload-artifact@v3
-        with:
-          path: |
-            badger/badger-checksum-linux-amd64.sha256
-            badger/badger-linux-amd64.tar.gz
-            dgraph/dgraph-checksum-linux-amd64.sha256
-            dgraph/dgraph-linux-amd64.tar.gz
-      - name: Move Badger Binary into Linux Directory
-        run: |
-          tar -xzf badger/badger-linux-amd64.tar.gz --directory badger
-          [ -d "linux" ] || mkdir linux
-          # linux directory will be added to docker image in build step
-          cp badger/badger-linux-amd64 linux/badger
+        run: make dgraph DGRAPH_VERSION='nightly-${{ env.NIGHTLY_BUILD_TIME }}'
       - name: Make Dgraph Docker Image
         run: |
-          make docker-image DGRAPH_VERSION=${{ env.DGRAPH_RELEASE_VERSION }}-amd64
-          [[ "${{ inputs.latest }}" = true ]] && docker tag dgraph/dgraph:${{ env.DGRAPH_RELEASE_VERSION }}-amd64 dgraph/dgraph:latest-amd64 || true
-      - name: Make Dgraph Standalone Docker Image with Version
-        run: |
-          make docker-image-standalone DGRAPH_VERSION=${{ env.DGRAPH_RELEASE_VERSION }}-amd64
-          [[ "${{ inputs.latest }}" = true ]] && docker tag dgraph/standalone:${{ env.DGRAPH_RELEASE_VERSION }}-amd64 dgraph/standalone:latest-amd64 || true
-      - name: Login to DockerHub
-        uses: docker/login-action@v2
+          make docker-image DGRAPH_VERSION='nightly-${{ env.NIGHTLY_BUILD_TIME }}-${{ env.DGRAPH_GIT_SHA }}' IMAGE_REPOSITORY='hypermode/dgraph-private'
+          docker tag hypermode/dgraph-private:nightly-${{ env.NIGHTLY_BUILD_TIME }}-${{ env.DGRAPH_GIT_SHA }} ${{ vars.AWS_ACCOUNT_ID }}.dkr.ecr.${{ vars.AWS_REGION }}.amazonaws.com/hypermode/dgraph-private:nightly-${{ env.NIGHTLY_BUILD_TIME }}-${{ env.DGRAPH_GIT_SHA }}
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v1
         with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_PASSWORD_TOKEN }}
-      - name: Push Images to DockerHub
-        run: |
-          docker push dgraph/standalone:${{ env.DGRAPH_RELEASE_VERSION }}-amd64
-          docker push dgraph/dgraph:${{ env.DGRAPH_RELEASE_VERSION }}-amd64
-
-  dgraph-build-arm64:
-    runs-on: [self-hosted, arm64]
+          role-to-assume: '${{ vars.AWS_ROLE }}'
+          role-session-name: CD_DGRAPH_PRIVATE
+          aws-region: '${{ vars.AWS_REGION }}'
+      - name: Login to Amazon ECR
+        uses: aws-actions/amazon-ecr-login@v1
+      - name: Push Images to ECR
+        run: |
+          docker push ${{ vars.AWS_ACCOUNT_ID }}.dkr.ecr.${{ vars.AWS_REGION }}.amazonaws.com/hypermode/dgraph-private:nightly-${{ env.NIGHTLY_BUILD_TIME }}-${{ env.DGRAPH_GIT_SHA }}
+  dgraph-build-workflow-dispatch:
+    if: ${{ github.event_name == 'workflow_dispatch' }}
+    environment: '${{ github.event.inputs.environment }}'
+    runs-on: ubuntu-20.04
     steps:
+      - name: Configure git for private modules
+        env:
+          # auth token for dgraph-bot; needed to checkout private go module dependencies
+          TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
+        run: git config --global url."https://dgraph-bot:${TOKEN}@github.com".insteadOf "https://github.com"
       - uses: actions/checkout@v3
         with:
+          # since this is on dispatch, use specified release tag
           ref: '${{ github.event.inputs.releasetag }}'
       - name: Get Go Version
         run: |
@@ -122,106 +105,39 @@ jobs:
           go mod tidy
           make regenerate
           git diff --exit-code -- .
-      - name: Set Badger Release Version
-        run: |
-          #!/bin/bash
-          BADGER_RELEASE_VERSION=$(cat go.mod | grep -i "github.com/dgraph-io/badger" | awk '{print $2}')
-          echo "setting badger version "$BADGER_RELEASE_VERSION
-          echo "BADGER_RELEASE_VERSION=$BADGER_RELEASE_VERSION" >> $GITHUB_ENV
-      - name: Download Badger Artifacts
-        run: |
-          #!/bin/bash
-          mkdir badger
-          cd badger
-          wget https://github.com/dgraph-io/badger/releases/download/${{ env.BADGER_RELEASE_VERSION }}/badger-checksum-linux-arm64.sha256
-          wget https://github.com/dgraph-io/badger/releases/download/${{ env.BADGER_RELEASE_VERSION }}/badger-linux-arm64.tar.gz
       - name: Set Dgraph Release Version
         run: |
           #!/bin/bash
           GIT_TAG_NAME='${{ github.event.inputs.releasetag }}'
-          if [[ "$GIT_TAG_NAME" == "v"* ]]; 
+          if [[ "$GIT_TAG_NAME" == "hyp-v"* ]];
           then
-            echo "this is a release branch"
+            echo "this is a valid release tag"
           else
-            echo "this is NOT a release branch"
+            echo "this is not a valid release tag"
             exit 1
           fi
           DGRAPH_RELEASE_VERSION='${{ github.event.inputs.releasetag }}'
           echo "making a new release for dgraph "$DGRAPH_RELEASE_VERSION
           echo "DGRAPH_RELEASE_VERSION=$DGRAPH_RELEASE_VERSION" >> $GITHUB_ENV
+      - name: Dgraph Git SHA
+        run: |
+          DGRAPH_GIT_SHA=$(git rev-parse --short HEAD)
+          echo "dgraph git sha "$DGRAPH_GIT_SHA
+          echo "DGRAPH_GIT_SHA=$DGRAPH_GIT_SHA" >> $GITHUB_ENV
       - name: Make Dgraph Linux Build
         run: make dgraph DGRAPH_VERSION=${{ env.DGRAPH_RELEASE_VERSION }}
-      - name: Generate SHA for Dgraph Linux Build
-        run: cd dgraph && sha256sum dgraph | cut -c-64 > dgraph-checksum-linux-arm64.sha256
-      - name: Tar Archive for Dgraph Linux Build
-        run: cd dgraph && tar -zcvf dgraph-linux-arm64.tar.gz dgraph
-      - name: Upload Build Artifacts
-        uses: actions/upload-artifact@v3
-        with:
-          path: |
-            badger/badger-checksum-linux-arm64.sha256
-            badger/badger-linux-arm64.tar.gz
-            dgraph/dgraph-checksum-linux-arm64.sha256
-            dgraph/dgraph-linux-arm64.tar.gz
-      - name: Move Badger Binary into Linux Directory
-        run: |
-          tar -xzf badger/badger-linux-arm64.tar.gz --directory badger
-          [ -d "linux" ] || mkdir linux
-          # linux directory will be added to docker image in build step
-          cp badger/badger-linux-arm64 linux/badger
       - name: Make Dgraph Docker Image
         run: |
-          make docker-image DGRAPH_VERSION=${{ env.DGRAPH_RELEASE_VERSION }}-arm64
-          [[ "${{ inputs.latest }}" = true ]] && docker tag dgraph/dgraph:${{ env.DGRAPH_RELEASE_VERSION }}-arm64 dgraph/dgraph:latest-arm64 || true
-      - name: Make Dgraph Standalone Docker Image with Version
-        run: |
-          make docker-image-standalone DGRAPH_VERSION=${{ env.DGRAPH_RELEASE_VERSION }}-arm64
-          [[ "${{ inputs.latest }}" = true ]] && docker tag dgraph/standalone:${{ env.DGRAPH_RELEASE_VERSION }}-arm64 dgraph/standalone:latest-arm64 || true
-      - name: Login to DockerHub
-        uses: docker/login-action@v2
+          make docker-image DGRAPH_VERSION=${{ env.DGRAPH_RELEASE_VERSION }}-${{ env.DGRAPH_GIT_SHA }} IMAGE_REPOSITORY='hypermode/dgraph-private'
+          docker tag hypermode/dgraph-private:${{ env.DGRAPH_RELEASE_VERSION }}-${{ env.DGRAPH_GIT_SHA }} ${{ vars.AWS_ACCOUNT_ID }}.dkr.ecr.${{ vars.AWS_REGION }}.amazonaws.com/hypermode/dgraph-private:${{ env.DGRAPH_RELEASE_VERSION }}-${{ env.DGRAPH_GIT_SHA }}
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v1
         with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_PASSWORD_TOKEN }}
-      - name: Push Images to DockerHub
-        run: |
-          docker push dgraph/standalone:${{ env.DGRAPH_RELEASE_VERSION }}-arm64
-          docker push dgraph/dgraph:${{ env.DGRAPH_RELEASE_VERSION }}-arm64
-
-  dgraph-docker-manifest:
-    needs: [dgraph-build-amd64, dgraph-build-arm64]
-    runs-on: ubuntu-20.04
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          ref: '${{ github.event.inputs.releasetag }}'
-      - name: Set Dgraph Release Version
-        run: |
-          #!/bin/bash
-          GIT_TAG_NAME='${{ github.event.inputs.releasetag }}'
-          if [[ "$GIT_TAG_NAME" == "v"* ]]; 
-          then
-            echo "this is a release branch"
-          else
-            echo "this is NOT a release branch"
-            exit 1
-          fi
-          DGRAPH_RELEASE_VERSION='${{ github.event.inputs.releasetag }}'
-          echo "making a new release for dgraph "$DGRAPH_RELEASE_VERSION
-          echo "DGRAPH_RELEASE_VERSION=$DGRAPH_RELEASE_VERSION" >> $GITHUB_ENV
-      - name: Login to DockerHub
-        uses: docker/login-action@v2
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_PASSWORD_TOKEN }}
-      - name: Docker Manifest
-        run: |
-          # standalone
-          docker manifest create dgraph/standalone:${{ env.DGRAPH_RELEASE_VERSION }} --amend dgraph/standalone:${{ env.DGRAPH_RELEASE_VERSION }}-amd64 --amend dgraph/standalone:${{ env.DGRAPH_RELEASE_VERSION }}-arm64
-          docker manifest push dgraph/standalone:${{ env.DGRAPH_RELEASE_VERSION }}
-          [[ "${{ inputs.latest }}" = true ]] && docker manifest create dgraph/standalone:latest --amend dgraph/standalone:${{ env.DGRAPH_RELEASE_VERSION }}-amd64 --amend dgraph/standalone:${{ env.DGRAPH_RELEASE_VERSION }}-arm64
-          [[ "${{ inputs.latest }}" = true ]] && docker manifest push dgraph/standalone:latest || true
-          # dgraph
-          docker manifest create dgraph/dgraph:${{ env.DGRAPH_RELEASE_VERSION }} --amend dgraph/dgraph:${{ env.DGRAPH_RELEASE_VERSION }}-amd64 --amend dgraph/dgraph:${{ env.DGRAPH_RELEASE_VERSION }}-arm64
-          docker manifest push dgraph/dgraph:${{ env.DGRAPH_RELEASE_VERSION }}
-          [[ "${{ inputs.latest }}" = true ]] && docker manifest create dgraph/dgraph:latest --amend dgraph/dgraph:${{ env.DGRAPH_RELEASE_VERSION }}-amd64 --amend dgraph/dgraph:${{ env.DGRAPH_RELEASE_VERSION }}-arm64 || true
-          [[ "${{ inputs.latest }}" = true ]] && docker manifest push dgraph/dgraph:latest || true
+          role-to-assume: '${{ vars.AWS_ROLE }}'
+          role-session-name: CD_DGRAPH_PRIVATE
+          aws-region: '${{ vars.AWS_REGION }}'
+      - name: Login to Amazon ECR
+        uses: aws-actions/amazon-ecr-login@v1
+      - name: Push Images to ECR
+        run: |
+          docker push ${{ vars.AWS_ACCOUNT_ID }}.dkr.ecr.${{ vars.AWS_REGION }}.amazonaws.com/hypermode/dgraph-private:${{ env.DGRAPH_RELEASE_VERSION }}-${{ env.DGRAPH_GIT_SHA }}

.github/workflows/ci-aqua-security-trivy-tests.yml

@@ -1,44 +1,25 @@
 name: ci-aqua-security-trivy-tests
 on:
-  push:
-    paths-ignore:
-      - '.github/CODEOWNERS'
-      - '.vscode/**'
-      - 'compose/**'
-      - 'contrib/systemd/**'
-      - 'licenses/**'
-      - 'paper/**'
-      - 'present/**'
-      - 'RFC/**'
-      - 'static/**'
-      - 'wiki/**'
-      - '**/**.dockerignore'
-      - '**/**.gitignore'
-      - '**/**.md'
-      - '**/**.png'
-      - '**/**.jpg'
-      - '**/**.gif'
-      - '**/**.ini'
-    branches:
-      - main
-      - 'release/**'
   pull_request:
     types:
       - opened
       - reopened
       - synchronize
       - ready_for_review
     branches:
-      - main
-      - 'release/**'
-  schedule:
-    - cron: "1 * * * *"
+      - main-private
+env:
+  GOPRIVATE: "github.com/dgraph-io/*,github.com/gohypermode/*"
 jobs:
   build:
     name: trivy-tests
     if: github.event.pull_request.draft == false
     runs-on: ubuntu-20.04
     steps:
+      - name: Configure git for private modules
+        env:
+          TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
+        run: git config --global url."https://dgraph-bot:${TOKEN}@github.com".insteadOf "https://github.com"
       - name: Checkout code
         uses: actions/checkout@v3
       - name: Get Go Version
@@ -58,8 +39,11 @@ jobs:
         with:
           image-ref: 'dgraph/dgraph:local'
           format: 'sarif'
-          output: 'trivy-results.sarif'
-      - name: Upload Trivy scan results to GitHub Security tab
-        uses: github/codeql-action/upload-sarif@v2
-        with:
-          sarif_file: 'trivy-results.sarif'
+          output: 'trivy-results-dgraph.sarif'
+      - name: Install Rust
+        run: curl https://sh.rustup.rs -sSf | sh -s -- -y
+      - name: Install sarif-fmt
+        run: |
+          cargo install sarif-fmt
+      - name: Print Trivy scan results
+        run: sarif-fmt -i trivy-results-dgraph.sarif

.github/workflows/ci-dgraph-code-coverage.yml

@@ -1,31 +1,17 @@
 name: ci-dgraph-code-coverage
 on:
-  push:
-    paths-ignore:
-      - '.github/CODEOWNERS'
-      - '.vscode/**'
-      - 'compose/**'
-      - 'contrib/systemd/**'
-      - 'licenses/**'
-      - 'paper/**'
-      - 'present/**'
-      - 'RFC/**'
-      - 'static/**'
-      - 'wiki/**'
-      - '**/**.dockerignore'
-      - '**/**.gitignore'
-      - '**/**.md'
-      - '**/**.png'
-      - '**/**.jpg'
-      - '**/**.gif'
-      - '**/**.ini'
-    branches:
-      - main
-      - 'release/**'
+  schedule:
+    - cron: "0 0 * * 3,7" # Wednesday and Sunday
+env:
+  GOPRIVATE: "github.com/dgraph-io/*,github.com/gohypermode/*"
 jobs:
   dgraph-code-coverage:
-    runs-on: ubuntu-20.04-32gb
+    runs-on: buildjet-32vcpu-ubuntu-2004
     steps:
+      - name: Configure git for private modules
+        env:
+          TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
+        run: git config --global url."https://dgraph-bot:${TOKEN}@github.com".insteadOf "https://github.com"
       - uses: actions/checkout@v3 # defaults to SHA of event that triggered workflow
       - name: Get Go Version
         run: |