Merge branch 'awarno/haproxy' of https://github.com/NVIDIA-NeMo/Evaluator into awarno/haproxy

Author: AWarno

.github/actions/test-template/action.yml

@@ -26,13 +26,45 @@ inputs:
     description: "Failure will cancel all other tests if set to true"
     required: false
     default: "false"
-  is_unit_test:
-    description: "Upload coverage as unit test"
+  type_of_test:
+    description: "Type of test to run"
     required: false
-    default: "false"
+    default: "unit_tests"
   package:
-    description: "Package to test"
-    required: true
+    description: "Package to test. Only required for unit and functional tests."
+    required: false
+  use-credentials:
+    description: "Use credentials for test"
+    required: false
+    default: "false"
+  ngc-api-user:
+    description: "NGC API user"
+    required: false
+    default: "false"
+  ngc-api-key:
+    description: "NGC API key"
+    required: false
+    default: "false"
+  cpu-only:
+    description: "CPU only"
+    required: false
+    default: "true"
+  has-azure-credentials:
+    description: "Has Azure credentials"
+    required: false
+    default: "false"
+  azure-client-id:
+    description: "Azure client ID"
+    required: false
+    default: "false"
+  azure-tenant-id:
+    description: "Azure tenant ID"
+    required: false
+    default: "false"
+  azure-subscription-id:
+    description: "Azure subscription ID"
+    required: false
+    default: "false"
 
 runs:
   using: "composite"
@@ -42,30 +74,139 @@ runs:
       with:
         path: Eval
 
-    - name: Build container
+    - name: Validate type of test
+      shell: bash
+      run: |
+        echo ::group::Validate type of test
+        if [[ "${{ inputs.type_of_test }}" != "unit_tests" && "${{ inputs.type_of_test }}" != "functional_tests" && "${{ inputs.type_of_test }}" != "integration_tests" ]]; then
+          echo "Invalid type of test: ${{ inputs.type_of_test }}"
+          exit 1
+        fi
+        echo ::endgroup::
+
+    - name: Use credentials
+      shell: bash
+      if: ${{ inputs.use-credentials == 'true' }}
+      run: |
+        echo ::group::Use credentials
+        echo ${{ inputs.ngc-api-key }} | docker login -u '${{ inputs.ngc-api-user }}' --password-stdin nvcr.io
+        echo ::endgroup::
+
+    - name: Build container for unit and functional tests
+      shell: bash
+      if: ${{ inputs.type_of_test != 'integration_tests' }}
+      run: |
+        echo ::group::Build test container
+        docker build -f docker/Dockerfile.ci \
+          --build-arg PACKAGE=${{ inputs.package }} \
+          -t eval .
+        echo ::endgroup::
+
+    - name: Build container for integration tests
       shell: bash
+      if: ${{ inputs.type_of_test == 'integration_tests' }}
       run: |
         echo ::group::Build test container
-        docker build -f docker/Dockerfile.ci --build-arg PACKAGE=${{ inputs.package }} -t eval .
+        docker build -f docker/Dockerfile.ci-integration \
+          -t eval .
         echo ::endgroup::
 
+    - name: Azure Login
+      if: ${{ inputs.has-azure-credentials == 'true' }}
+      uses: azure/login@v2
+      with:
+        client-id: ${{ inputs.azure-client-id }}
+        tenant-id: ${{ inputs.azure-tenant-id }}
+        subscription-id: ${{ inputs.azure-subscription-id }}
+
+    - name: Azure Fileshare
+      if: ${{ inputs.has-azure-credentials == 'true' && inputs.type_of_test != 'unit-tests' }}
+      shell: bash
+      id: azure-fileshare
+      run: |
+        echo "::group::Mount SMB drive"
+        sudo apt update
+        sudo apt install -y cifs-utils
+
+        RESOURCE_GROUP_NAME="azure-gpu-vm-runner_group"
+        STORAGE_ACCOUNT_NAME="nemocistorageaccount2"
+        FILE_SHARE_NAME="fileshare"
+
+        MNT_PATH="/mnt/datadrive"
+
+        sudo mkdir -p $MNT_PATH
+
+        # Create a folder to store the credentials for this storage account and
+        # any other that you might set up.
+        CREDENTIAL_ROOT="/etc/smbcredentials"
+        sudo mkdir -p "/etc/smbcredentials"
+
+        # Get the storage account key for the indicated storage account.
+        # You must be logged in with az login and your user identity must have
+        # permissions to list the storage account keys for this command to work.
+        STORAGE_ACCOUNT_KEY=$(az storage account keys list \
+            --resource-group $RESOURCE_GROUP_NAME \
+            --account-name $STORAGE_ACCOUNT_NAME \
+            --query "[0].value" --output tsv | tr -d '"')
+
+        # Create the credential file for this individual storage account
+        SMB_CREDENTIAL_FILE="$CREDENTIAL_ROOT/$STORAGE_ACCOUNT_NAME.cred"
+        if [ ! -f $SMB_CREDENTIAL_FILE ]; then
+            echo "username=$STORAGE_ACCOUNT_NAME" | sudo tee $SMB_CREDENTIAL_FILE > /dev/null
+            echo "password=$STORAGE_ACCOUNT_KEY" | sudo tee -a $SMB_CREDENTIAL_FILE > /dev/null
+        else
+            echo "The credential file $SMB_CREDENTIAL_FILE already exists, and was not modified."
+        fi
+
+        # Change permissions on the credential file so only root can read or modify the password file.
+        sudo chmod 600 $SMB_CREDENTIAL_FILE
+
+        # This command assumes you have logged in with az login
+        HTTP_ENDPOINT=$(az storage account show --resource-group $RESOURCE_GROUP_NAME --name $STORAGE_ACCOUNT_NAME --query "primaryEndpoints.file" --output tsv | tr -d '"')
+        SMB_PATH=$(echo $HTTP_ENDPOINT | cut -c7-${#HTTP_ENDPOINT})$FILE_SHARE_NAME
+
+        STORAGE_ACCOUNT_KEY=$(az storage account keys list --resource-group $RESOURCE_GROUP_NAME --account-name $STORAGE_ACCOUNT_NAME --query "[0].value" --output tsv | tr -d '"')
+
+        sudo mount -t cifs $SMB_PATH $MNT_PATH -o credentials=$SMB_CREDENTIAL_FILE,serverino,nosharesock,actimeo=30,mfsymlinks,fsc,cache=strict
+
+        ls -al $MNT_PATH/TestData
+        echo "::endgroup::"
+
     - name: Start container
       shell: bash
+      env:
+        MOUNT_FS: ${{ inputs.type_of_test == 'integration_tests' || inputs.type_of_test == 'functional_tests' }}
       run: |
-        echo ::group::Start test container
+        echo "::group::Start test container"
+        if [[ "$MOUNT_FS" == "true" ]]; then
+          ls -al /mnt/datadrive/TestData
+          VOLUME_ARGS="--volume /mnt/datadrive/TestData:/home/TestData"
+        else
+          VOLUME_ARGS=""
+        fi
+
+        ARG=("")
+        if [[ "${{ inputs.type_of_test }}" == "integration_tests" ]]; then
+          ARG=("--runtime=nvidia --gpus all")
+        fi
+
         cmd=$(cat <<RUN_TEST_EOF
         #!/bin/bash
         docker container rm -f nemo_container_${{ github.run_id }} || true
+
         docker run \
           --rm \
           -d \
           --name nemo_container_${{ github.run_id }} \
+          ${ARG[@]} \
           --shm-size=64g \
-          --env TRANSFORMERS_OFFLINE=0 \
+          --env TRANSFORMERS_OFFLINE=1 \
           --env HYDRA_FULL_ERROR=1 \
+          --env HF_HUB_CACHE_PATH=/home/TestData/HF_HOME/hub \
           --env HF_HOME=/home/TestData/HF_HOME \
           --env RUN_ID=${{ github.run_id }} \
           --volume $(pwd)/Eval:/workspace \
+          $VOLUME_ARGS \
           eval \
           bash -c "sleep $(( ${{ inputs.timeout }} * 60 + 60 ))"
         RUN_TEST_EOF
@@ -80,16 +221,20 @@ runs:
       shell: bash
       run: |
         echo ::group::Create run-script
-        COVERAGE_PREFIX=$([[ "${{ inputs.is_unit_test }}" == "true" ]] && echo "unit-test" || echo "e2e")
+        COVERAGE_PREFIX=$([[ "${{ inputs.type_of_test }}" == "unit_tests" ]] && echo "unit-test" || echo "e2e")
         echo "coverage-prefix=$COVERAGE_PREFIX" | tee -a "$GITHUB_OUTPUT"
 
         cmd=$(cat <<'RUN_TEST_EOF'
         #!/bin/bash
 
         docker exec -t nemo_container_${{ github.run_id }} bash -c '
           set -e
-          cd packages/${{ inputs.package }}
-          bash tests/${{ inputs.is_unit_test == 'true' && 'unit_tests' || 'functional_tests' }}/${{ inputs.script }}.sh
+          if [[ "${{ inputs.type_of_test }}" != "integration_tests" ]]; then
+            cd packages/${{ inputs.package }}
+            bash tests/${{ inputs.type_of_test == 'unit_tests' && 'unit_tests' || 'functional_tests' }}/${{ inputs.script }}.sh
+          else
+            bash tests/integration_tests/Launch_Integration_Tests.sh
+          fi
         '
 
         RUN_TEST_EOF
@@ -115,9 +260,14 @@ runs:
       shell: bash
       run: |
         echo ::group::Check result
-        docker cp nemo_container_${{ github.run_id }}:/workspace/packages/${{ inputs.package }}/.coverage .coverage.${{ inputs.package }}
 
-        coverage_report=coverage-${{ steps.create.outputs.coverage-prefix }}-${{ github.run_id }}-$(uuidgen)
+        if [[ "${{ inputs.type_of_test }}" != "integration_tests" ]]; then
+          docker cp nemo_container_${{ github.run_id }}:/workspace/packages/${{ inputs.package }}/.coverage .coverage.${{ inputs.package }}
+          coverage_report=coverage-${{ steps.create.outputs.coverage-prefix }}-${{ github.run_id }}-$(uuidgen)
+        else
+          coverage_report=none
+        fi
+
         echo "coverage_report=$coverage_report" >> "$GITHUB_OUTPUT"
 
         EXIT_CODE=${{ steps.run-main-script.outputs.exit_code }}
@@ -138,6 +288,7 @@ runs:
 
     - name: Test coverage
       shell: bash -x -e -u -o pipefail {0}
+      if: ${{ steps.check.outputs.coverage_report != 'none' }}
       run: |
         echo ::group::Test coverage
         docker exec -t nemo_container_${{ github.run_id }} bash -c 'cd /workspace/packages/${{ inputs.package }} && /opt/venv/bin/coverage report -i'

.github/config/requirements.txt

@@ -5,6 +5,6 @@ sphinx-copybutton   # Adds a copy button for code blocks
 nvidia-sphinx-theme # Our NVIDIA theme
 sphinxcontrib-mermaid # For mermaid diagrams
 myst-parser # For our markdown docs
-sphinx-design
-sphinxcontrib-mermaid
-swagger-plugin-for-sphinx
+sphinx-design # For our design elements
+sphinxcontrib-mermaid # For mermaid diagrams
+swagger-plugin-for-sphinx # For Swagger API documentation

.github/labeler.yml

@@ -0,0 +1,26 @@
+nemo-evaluator:
+- packages/nemo-evaluator/**/*
+- docs/libraries/nemo-evaluator/**/*
+
+nemo-evaluator-launcher:
+- packages/nemo-evaluator-launcher/**/*
+- docs/libraries/nemo-evaluator-launcher/**/*
+
+documentation:
+- docs/**/*
+- README.md
+- tutorials/**/*
+
+tests:
+- tests/**/*
+- packages/nemo-evaluator/tests/**/*
+- packages/nemo-evaluator-launcher/tests/**/*
+
+scripts:
+- scripts/**/*
+
+CI:
+- .github/**/*
+- Dockerfile
+- docker/**/*
+

.github/workflows/build-docs.yml

@@ -50,15 +50,17 @@ jobs:
           GH_TOKEN: ${{ github.token }}
           RUN_ID: ${{ github.run_id }}
           SKIPPING_IS_ALLOWED: ${{ needs.pre-flight.outputs.docs_only == 'true' || needs.pre-flight.outputs.is_deployment_workflow == 'true' }}
+          BUILD_DOCS_RESULT: ${{ needs.build-docs.result }}
         run: |
-          FAILED_JOBS=$(gh run view $GITHUB_RUN_ID --json jobs --jq '[.jobs[] | select(.status == "completed" and .conclusion != "success")] | length') || echo 0
-
-          if [ "${FAILED_JOBS:-0}" -eq 0 ] || [ "$SKIPPING_IS_ALLOWED" == "true" ]; then
-              echo "✅ All previous jobs completed successfully"
-              exit 0
+          echo "build-docs job result: ${BUILD_DOCS_RESULT}"
+          
+          if [ "${BUILD_DOCS_RESULT}" == "success" ]; then
+              echo "✅ Build docs completed successfully"
+          elif [ "${BUILD_DOCS_RESULT}" == "skipped" ] || [ "$SKIPPING_IS_ALLOWED" == "true" ]; then
+              echo "⏭️ Build docs was skipped"
           else
-              echo "❌ Found $FAILED_JOBS failed job(s)"
-              # Show which jobs failed
-              gh run view $GITHUB_RUN_ID --json jobs --jq '.jobs[] | select(.status == "completed" and .conclusion != "success") | .name'
-              exit 1
+              echo "⚠️ Build docs failed (allowed to fail)"
           fi
+          
+          # Always exit 0 - build-docs is allowed to fail
+          exit 0

.github/workflows/build-test-publish-wheel.yml

@@ -35,7 +35,7 @@ jobs:
     if: |
       !(needs.pre-flight.outputs.docs_only == 'true'
       || needs.pre-flight.outputs.is_deployment_workflow == 'true')
-    uses: NVIDIA-NeMo/FW-CI-templates/.github/workflows/_build_test_publish_wheel.yml@v0.60.0
+    uses: NVIDIA-NeMo/FW-CI-templates/.github/workflows/_build_test_publish_wheel.yml@v0.64.2
     with:
       dry-run: true
       python-package: nemo_evaluator
@@ -70,7 +70,7 @@ jobs:
     if: |
       !(needs.pre-flight.outputs.docs_only == 'true'
       || needs.pre-flight.outputs.is_deployment_workflow == 'true')
-    uses: NVIDIA-NeMo/FW-CI-templates/.github/workflows/_build_test_publish_wheel.yml@v0.60.0
+    uses: NVIDIA-NeMo/FW-CI-templates/.github/workflows/_build_test_publish_wheel.yml@v0.64.2
     with:
       dry-run: true
       python-package: nemo_evaluator_launcher