(wip) CI

Author: BourgoisMickael

.github/docker/docker-compose.yaml

@@ -8,6 +8,9 @@ services:
       - /tmp/ssl-kmip:/ssl-kmip
       - ${HOME}/.aws/credentials:/root/.aws/credentials
       - /tmp/artifacts/${JOB_NAME}:/artifacts
+      # using artesca container, with persistent volumes for sse migration
+      - ../../localData:/usr/src/app/localData
+      - ../../localMetadata:/usr/src/app/localMetadata
     environment:
       - CI=true
       - ENABLE_LOCAL_CACHE=true
@@ -42,6 +45,16 @@ services:
     extra_hosts:
       - "bucketwebsitetester.s3-website-us-east-1.amazonaws.com:127.0.0.1"
       - "pykmip.local:127.0.0.1"
+  cloudserver-sse-migration:
+    extends: cloudserver
+    profiles: [sse-migration]
+    volumes:
+      # using artesca container
+      - ../../tests/functional/sse-kms-migration/config.${SSE_CONF}.json:/conf/config.json
+      - ../../localData:/usr/src/app/localData
+      - ../../localMetadata:/usr/src/app/localMetadata
+    environment:
+      - S3_CONFIG_FILE=/conf/config.json
   redis:
     image: redis:alpine
     network_mode: "host"

.github/workflows/tests.yaml

@@ -218,6 +218,8 @@ jobs:
         source: /tmp/artifacts
       if: always()
 
+  # All tests use non federation images
+
   file-ft-tests:
     strategy:
       matrix:
@@ -345,3 +347,108 @@ jobs:
         password: ${{ secrets.ARTIFACTS_PASSWORD }}
         source: /tmp/artifacts
       if: always()
+
+  sse-kms-migration-tests:
+    strategy:
+      matrix:
+        include:
+          - kmsHideScalityArn: 'showArn'
+            job-name: sse-kms-migration-tests-show-arn
+          # To use this one, the test needs to stop expecting an Arn in rsponse headers
+          # But the check it should query the metadata directly to ensure the md has the arn
+          # - kmsHideScalityArn: 'hideArn'
+          #   job-name: sse-kms-migration-tests-hide-arn
+    name: ${{ matrix.job-name }}
+    runs-on: ubuntu-latest
+    needs: build
+    env:
+      S3BACKEND: file
+      S3VAULT: mem
+      CLOUDSERVER_IMAGE_BEFORE_SSE_MIGRATION: ghcr.io/${{ github.repository }}:7.70.21-11
+      CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}
+      MPU_TESTING: "yes"
+      JOB_NAME: ${{ matrix.job-name }}
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+    - uses: actions/setup-python@v5
+      with:
+        python-version: 3.9
+    - name: Setup CI environment
+      uses: ./.github/actions/setup-ci
+    - name: Setup matrix job artifacts directory
+      shell: bash
+      run: |
+        set -exu
+        mkdir -p /tmp/artifacts/${{ matrix.job-name }}/
+    - name: Setup CI services (with old cloudserver image before sse migration)
+      run: docker compose up -d
+      working-directory: .github/docker
+      env:
+        CLOUDSERVER_IMAGE: ${{ env.CLOUDSERVER_IMAGE_BEFORE_SSE_MIGRATION }}
+        SSE_CONF: before
+    - name: Run SSE before migration tests (setup buckets and objects)
+      env:
+        # yarn run does a cd into the test folder
+        S3_CONFIG_FILE: config.before.json
+      run: |-
+        set -o pipefail;
+        bash wait_for_local_port.bash 8000 40
+
+        echo Ensures the expected version of cloudserver is old one:
+        VERSION=$(docker compose -f .github/docker/docker-compose.yaml \
+            exec cloudserver cat package.json | jq -r .version)
+        if [[ "$VERSION" != "7.70.21-11" ]]; then
+          echo "bad version of container. Should be 7.70.21-11. Was $VERSION" >&2
+          exit 1
+        else
+          echo OK $VERSION
+        fi
+
+        yarn run ft_sse_before_migration | tee /tmp/artifacts/${{ matrix.job-name }}/beforeMigration.log
+    - name: Replace old cloudserver image with current one
+      run: |-
+        docker compose down cloudserver
+        mv /tmp/artifacts/${{ matrix.job-name }}/s3.log /tmp/artifacts/${{ matrix.job-name }}/s3.old.log
+        docker compose up -d cloudserver-sse-migration
+      working-directory: .github/docker
+      env:
+        SSE_CONF: sseMigration.${{ matrix.kmsHideScalityArn }}
+    - name: Run SSE migration tests
+      env:
+        # yarn run does a cd into the test folder
+        S3_CONFIG_FILE: config.sseMigration.${{ matrix.kmsHideScalityArn }}.json
+      run: |-
+        set -o pipefail;
+        bash wait_for_local_port.bash 8000 40
+
+        echo Ensures the expected version of cloudserver is NOT old one
+        VERSION=$(docker compose -f .github/docker/docker-compose.yaml \
+            exec cloudserver-sse-migration cat package.json | jq -r .version)
+        if [[ "$VERSION" == "7.70.21-11" ]]; then
+          echo "bad version of container. Should NOT be 7.70.21-11. Was $VERSION" >&2
+          exit 1
+        else
+          echo OK $VERSION
+        fi
+
+        yarn run ft_sse_migration | tee /tmp/artifacts/${{ matrix.job-name }}/migration.log
+    - name: Run SSE arnPrefix tests
+      env:
+        S3_CONFIG_FILE: config.sseMigration.${{ matrix.kmsHideScalityArn }}.json
+      run: |-
+        set -o pipefail;
+        yarn run ft_sse_arn | tee /tmp/artifacts/${{ matrix.job-name }}/arnPrefix.log
+    - name: Print docker compose logs
+      run: docker compose logs cloudserver cloudserver-sse-migration
+      working-directory: .github/docker
+      if: failure()
+    - name: Upload logs to artifacts
+      uses: scality/action-artifacts@v4
+      with:
+        method: upload
+        url: https://artifacts.scality.net
+        user: ${{ secrets.ARTIFACTS_USER }}
+        password: ${{ secrets.ARTIFACTS_PASSWORD }}
+        source: /tmp/artifacts
+      if: always()