From 372974e5d62040980a51d775ab0da989ba8292b7 Mon Sep 17 00:00:00 2001 From: Jian Chen Date: Mon, 20 May 2024 15:25:14 -0700 Subject: [PATCH] Using CPU pool to build Linux GPU C API Package (#20648) ### Description ### Motivation and Context --- .../jobs/linux-gpu-tensorrt-packaging-job.yml | 108 ------------------ .../nuget-linux-cuda-packaging-stage.yml | 89 +++++++++++++-- .../github/linux/build_cuda_c_api_package.sh | 3 +- .../linux/build_tensorrt_c_api_package.sh | 3 +- 4 files changed, 82 insertions(+), 121 deletions(-) delete mode 100644 tools/ci_build/github/azure-pipelines/stages/jobs/linux-gpu-tensorrt-packaging-job.yml diff --git a/tools/ci_build/github/azure-pipelines/stages/jobs/linux-gpu-tensorrt-packaging-job.yml b/tools/ci_build/github/azure-pipelines/stages/jobs/linux-gpu-tensorrt-packaging-job.yml deleted file mode 100644 index 541ab1ac74874..0000000000000 --- a/tools/ci_build/github/azure-pipelines/stages/jobs/linux-gpu-tensorrt-packaging-job.yml +++ /dev/null @@ -1,108 +0,0 @@ -parameters: -- name: artifactName - type: string - default: 'onnxruntime-linux-x64-gpu-tensorrt-$(OnnxRuntimeVersion)' - -- name: artifactNameNoVersionString - type: string - default: 'onnxruntime-linux-x64-gpu-tensorrt' - -- name: buildJava - type: boolean - default: false - -- name: buildJavaOption - type: string - default: '' - -- name: buildNodejs - type: boolean - default: true - -- name: buildNodejsOption - type: string - default: '' - -- name: CudaVersion - displayName: CUDA version - type: string - default: '11.8' - values: - - 11.8 - - 12.2 - -jobs: -- job: Linux_C_API_Packaging_TensorRT - dependsOn: [] - workspace: - clean: all - timeoutInMinutes: 180 - pool: 'Onnxruntime-Linux-GPU' - variables: - - name: CUDA_VERSION_MAJOR - ${{ if eq(parameters.CudaVersion, '11.8') }}: - value: '11' - ${{ if eq(parameters.CudaVersion, '12.2') }}: - value: '12' - - name: CUDA_VERSION - value: ${{ parameters.CudaVersion }} - - name: linux_trt_version - ${{ if eq(parameters.CudaVersion, '11.8') }}: - value: 10.0.1.6-1.cuda11.8 - ${{ if eq(parameters.CudaVersion, '12.2') }}: - value: 10.0.1.6-1.cuda12.4 - - name: docker_base_image - ${{ if eq(parameters.CudaVersion, '11.8') }}: - value: nvidia/cuda:11.8.0-cudnn8-devel-ubi8 - ${{ if eq(parameters.CudaVersion, '12.2') }}: - value: nvidia/cuda:12.2.2-cudnn8-devel-ubi8 - steps: - - checkout: self - clean: true - submodules: recursive - - template: ../../templates/get-docker-image-steps.yml - parameters: - Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.manylinux2_28_cuda - Context: tools/ci_build/github/linux/docker - DockerBuildArgs: " - --network=host - --build-arg BASEIMAGE=${{ variables.docker_base_image }} - --build-arg TRT_VERSION=${{ variables.linux_trt_version }} - --build-arg BUILD_UID=$( id -u ) - " - Repository: onnxruntimecuda${{ variables.CUDA_VERSION_MAJOR }}xtrt86build - - template: ../../templates/set-version-number-variables-step.yml - - - script: $(Build.SourcesDirectory)/tools/ci_build/github/linux/build_tensorrt_c_api_package.sh - workingDirectory: $(Build.SourcesDirectory) - displayName: 'Build and Test' - - - ${{ if eq(parameters.buildJava, true) }}: - - template: ../../templates/java-api-artifacts-package-and-publish-steps-posix.yml - parameters: - arch: 'linux-x64' - buildConfig: 'Release' - artifactName: 'onnxruntime-java-linux-x64-tensorrt' - version: '$(OnnxRuntimeVersion)' - libraryName: 'libonnxruntime.so' - nativeLibraryName: 'libonnxruntime4j_jni.so' - - - ${{ if eq(parameters.buildNodejs, 'true') }}: - - template: ../../templates/nodejs-artifacts-package-and-publish-steps-posix.yml - parameters: - arch: 'x64' - os: 'linux' - artifactName: 'drop-onnxruntime-nodejs-linux-x64-tensorrt' - - - template: ../../templates/c-api-artifacts-package-and-publish-steps-posix.yml - parameters: - buildConfig: 'Release' - artifactName: ${{ parameters.artifactName }} - artifactNameNoVersionString: ${{ parameters.artifactNameNoVersionString }} - libraryName: 'libonnxruntime.so.$(OnnxRuntimeVersion)' - - - - template: ../../templates/component-governance-component-detection-steps.yml - parameters: - condition: 'succeeded' - - template: ../../templates/clean-agent-build-directory-step.yml diff --git a/tools/ci_build/github/azure-pipelines/stages/nuget-linux-cuda-packaging-stage.yml b/tools/ci_build/github/azure-pipelines/stages/nuget-linux-cuda-packaging-stage.yml index d42f89b26774a..63f8fadff39ba 100644 --- a/tools/ci_build/github/azure-pipelines/stages/nuget-linux-cuda-packaging-stage.yml +++ b/tools/ci_build/github/azure-pipelines/stages/nuget-linux-cuda-packaging-stage.yml @@ -21,7 +21,7 @@ stages: workspace: clean: all timeoutInMinutes: 150 - pool: 'Onnxruntime-Linux-GPU' + pool: 'onnxruntime-Ubuntu2204-AMD-CPU' variables: - name: CUDA_VERSION_MAJOR ${{ if eq(parameters.CudaVersion, '11.8') }}: @@ -49,7 +49,8 @@ stages: - script: $(Build.SourcesDirectory)/tools/ci_build/github/linux/build_cuda_c_api_package.sh workingDirectory: $(Build.SourcesDirectory) - displayName: 'Build and Test' + displayName: 'Build CUDA C API Package' + - template: ../templates/java-api-artifacts-package-and-publish-steps-posix.yml parameters: arch: 'linux-x64' @@ -70,15 +71,81 @@ stages: condition: 'succeeded' - template: ../templates/clean-agent-build-directory-step.yml # Linux CUDA with TensorRT Packaging - - template: jobs/linux-gpu-tensorrt-packaging-job.yml - parameters: - artifactName: 'onnxruntime-linux-x64-tensorrt-$(OnnxRuntimeVersion)' - artifactNameNoVersionString: 'onnxruntime-linux-x64-tensorrt' - buildJava: ${{ parameters.buildJava }} - buildJavaOption: '--build_java' - buildNodejs: ${{ parameters.buildNodejs }} - buildNodejsOption: '--build_nodejs' - CudaVersion: ${{ parameters.CudaVersion }} + - job: Linux_C_API_Packaging_TensorRT + dependsOn: [] + workspace: + clean: all + timeoutInMinutes: 180 + pool: 'onnxruntime-Ubuntu2204-AMD-CPU' + variables: + - name: CUDA_VERSION_MAJOR + ${{ if eq(parameters.CudaVersion, '11.8') }}: + value: '11' + ${{ if eq(parameters.CudaVersion, '12.2') }}: + value: '12' + - name: CUDA_VERSION + value: ${{ parameters.CudaVersion }} + + - name: linux_trt_version + ${{ if eq(parameters.CudaVersion, '11.8') }}: + value: 10.0.1.6-1.cuda11.8 + ${{ if eq(parameters.CudaVersion, '12.2') }}: + value: 10.0.1.6-1.cuda12.4 + - name: docker_base_image + ${{ if eq(parameters.CudaVersion, '11.8') }}: + value: nvidia/cuda:11.8.0-cudnn8-devel-ubi8 + ${{ if eq(parameters.CudaVersion, '12.2') }}: + value: nvidia/cuda:12.2.2-cudnn8-devel-ubi8 + steps: + - checkout: self + clean: true + submodules: recursive + - template: ../templates/get-docker-image-steps.yml + parameters: + Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.manylinux2_28_cuda + Context: tools/ci_build/github/linux/docker + DockerBuildArgs: " + --network=host + --build-arg BASEIMAGE=${{ variables.docker_base_image }} + --build-arg TRT_VERSION=${{ variables.linux_trt_version }} + --build-arg BUILD_UID=$( id -u ) + " + Repository: onnxruntimecuda${{ variables.CUDA_VERSION_MAJOR }}xtrt86build + - template: ../templates/set-version-number-variables-step.yml + + - script: $(Build.SourcesDirectory)/tools/ci_build/github/linux/build_tensorrt_c_api_package.sh + workingDirectory: $(Build.SourcesDirectory) + displayName: 'Build TensorRT C API Package' + + - ${{ if eq(parameters.buildJava, true) }}: + - template: ../templates/java-api-artifacts-package-and-publish-steps-posix.yml + parameters: + arch: 'linux-x64' + buildConfig: 'Release' + artifactName: 'onnxruntime-java-linux-x64-tensorrt' + version: '$(OnnxRuntimeVersion)' + libraryName: 'libonnxruntime.so' + nativeLibraryName: 'libonnxruntime4j_jni.so' + + - ${{ if eq(parameters.buildNodejs, 'true') }}: + - template: ../templates/nodejs-artifacts-package-and-publish-steps-posix.yml + parameters: + arch: 'x64' + os: 'linux' + artifactName: 'drop-onnxruntime-nodejs-linux-x64-tensorrt' + + - template: ../templates/c-api-artifacts-package-and-publish-steps-posix.yml + parameters: + buildConfig: 'Release' + artifactName: 'onnxruntime-linux-x64-tensorrt-$(OnnxRuntimeVersion)' + artifactNameNoVersionString: 'onnxruntime-linux-x64-tensorrt' + libraryName: 'libonnxruntime.so.$(OnnxRuntimeVersion)' + + + - template: ../templates/component-governance-component-detection-steps.yml + parameters: + condition: 'succeeded' + - template: ../templates/clean-agent-build-directory-step.yml # Linux CUDA Combined Testing and Publishing - job: Linux_Packaging_combined_CUDA dependsOn: diff --git a/tools/ci_build/github/linux/build_cuda_c_api_package.sh b/tools/ci_build/github/linux/build_cuda_c_api_package.sh index aec02f76693b4..04968aacdb255 100755 --- a/tools/ci_build/github/linux/build_cuda_c_api_package.sh +++ b/tools/ci_build/github/linux/build_cuda_c_api_package.sh @@ -1,9 +1,10 @@ #!/bin/bash set -e -x -docker run --gpus all -e NVIDIA_VISIBLE_DEVICES=all --rm --volume \ +docker run --rm --volume \ $BUILD_SOURCESDIRECTORY:/onnxruntime_src --volume $BUILD_BINARIESDIRECTORY:/build \ --volume /data/models:/build/models:ro --volume /data/onnx:/data/onnx:ro -e NIGHTLY_BUILD onnxruntimecuda${CUDA_VERSION_MAJOR}build \ /usr/bin/python3.9 /onnxruntime_src/tools/ci_build/build.py --enable_lto --build_java --build_nodejs --build_dir /build --config Release \ --skip_submodule_sync --parallel --use_binskim_compliant_compile_flags --build_shared_lib --use_cuda --cuda_version=$CUDA_VERSION \ --cuda_home=/usr/local/cuda-$CUDA_VERSION --cudnn_home=/usr/local/cuda-$CUDA_VERSION \ +--skip_tests \ --cmake_extra_defines 'CMAKE_CUDA_ARCHITECTURES=60;61;70;75;80' diff --git a/tools/ci_build/github/linux/build_tensorrt_c_api_package.sh b/tools/ci_build/github/linux/build_tensorrt_c_api_package.sh index 7d65a6f738a5e..666e9945df1b6 100755 --- a/tools/ci_build/github/linux/build_tensorrt_c_api_package.sh +++ b/tools/ci_build/github/linux/build_tensorrt_c_api_package.sh @@ -1,7 +1,8 @@ #!/bin/bash set -e -x mkdir -p $HOME/.onnx -docker run --gpus all -e CFLAGS -e CXXFLAGS -e NVIDIA_VISIBLE_DEVICES=all --rm --volume /data/onnx:/data/onnx:ro --volume $BUILD_SOURCESDIRECTORY:/onnxruntime_src --volume $BUILD_BINARIESDIRECTORY:/build \ +docker run -e CFLAGS -e CXXFLAGS --rm --volume /data/onnx:/data/onnx:ro --volume $BUILD_SOURCESDIRECTORY:/onnxruntime_src --volume $BUILD_BINARIESDIRECTORY:/build \ --volume /data/models:/build/models:ro --volume $HOME/.onnx:/home/onnxruntimedev/.onnx -e NIGHTLY_BUILD onnxruntimecuda${CUDA_VERSION_MAJOR}xtrt86build \ /opt/python/cp38-cp38/bin/python3 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release \ +--skip_tests \ --skip_submodule_sync --parallel --use_binskim_compliant_compile_flags --build_shared_lib --build_java --build_nodejs --use_tensorrt --cuda_version=$CUDA_VERSION --cuda_home=/usr/local/cuda-$CUDA_VERSION --cudnn_home=/usr --tensorrt_home=/usr --cmake_extra_defines 'CMAKE_CUDA_ARCHITECTURES=60;61;70;75;80'