Skip to content

Commit e23ea25

Browse files
tianleiwusnnn
authored and
fs-eire
committed
[CUDA] Upgrade build pipelines to use CUDA 12.8 + cuDNN 9.8 (#26267)
This upgrades CUDA 12.2 + cuDNN 9.5 to CUDA 12.8 + cuDNN 9.8 in CI pipelines, so that we can build 120-real to support Blackwell GPU. To speed up build, we also disable relocatable-device-code. MSVC is updated to latest for some windows build pipelines. #### Known issues Some onnx models (yolo v3, yolo v4, mobilenet v1) failed to run due to cudnn frontend failed to find engine plan. We will try upgrade cudnn frontend later. Related failed tests are disabled for now. --------- Co-authored-by: Changming Sun <[email protected]>
1 parent 5125ed2 commit e23ea25

File tree

51 files changed

+145
-150
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

51 files changed

+145
-150
lines changed

‎.github/workflows/publish-csharp-apidocs.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ permissions:
2020

2121
jobs:
2222
build:
23-
runs-on: ["self-hosted", "1ES.Pool=onnxruntime-github-vs2022-mms"]
23+
runs-on: ["self-hosted", "1ES.Pool=onnxruntime-github-vs2022-latest"]
2424
env:
2525
DOCFXVERSION: 2.62.2
2626
steps:

‎.github/workflows/windows_cuda.yml‎

Lines changed: 13 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ concurrency:
1919
jobs:
2020
build:
2121
name: Windows GPU CUDA CI Pipeline
22-
runs-on: ["self-hosted", "1ES.Pool=onnxruntime-github-vs2022-mms"]
22+
runs-on: ["self-hosted", "1ES.Pool=onnxruntime-github-vs2022-latest"]
2323
steps:
2424
- uses: actions/checkout@v5
2525
with:
@@ -41,20 +41,20 @@ jobs:
4141
working-directory: ${{ github.workspace }}
4242
shell: cmd
4343

44-
- name: Download CUDA SDK v12.2
44+
- name: Download CUDA SDK v12.8
4545
working-directory: ${{ runner.temp }}
4646
run: |
47-
azcopy.exe cp --recursive "https://lotusscus.blob.core.windows.net/models/cuda_sdk/v12.2" .
47+
azcopy.exe cp --recursive "https://lotusscus.blob.core.windows.net/models/cuda_sdk/v12.8" .
4848
dir
4949
shell: pwsh
5050

5151
- name: Add CUDA to PATH
5252
shell: powershell
5353
run: |
5454
Write-Host "Adding CUDA to PATH"
55-
Write-Host "CUDA Path: $env:RUNNER_TEMP\v12.2\bin"
56-
Add-Content -Path $env:GITHUB_PATH -Value "$env:RUNNER_TEMP\v12.2\bin"
57-
Add-Content -Path $env:GITHUB_PATH -Value "$env:RUNNER_TEMP\v12.2\extras\CUPTI\lib64"
55+
Write-Host "CUDA Path: $env:RUNNER_TEMP\v12.8\bin"
56+
Add-Content -Path $env:GITHUB_PATH -Value "$env:RUNNER_TEMP\v12.8\bin"
57+
Add-Content -Path $env:GITHUB_PATH -Value "$env:RUNNER_TEMP\v12.8\extras\CUPTI\lib64"
5858
5959
- uses: actions/setup-node@v5
6060
with:
@@ -111,7 +111,7 @@ jobs:
111111
exit $lastExitCode
112112
}
113113
# Execute the build process
114-
python.exe ${{ github.workspace }}\tools\ci_build\build.py --update --build --config RelWithDebInfo --build_dir build --skip_submodule_sync --build_csharp --parallel --use_binskim_compliant_compile_flags --cmake_generator "Visual Studio 17 2022" --build_shared_lib --build_wheel --build_java --use_cuda --cuda_home="$env:RUNNER_TEMP\v12.2" --enable_cuda_profiling --use_vcpkg --use_vcpkg_ms_internal_asset_cache --enable_transformers_tool_test --cmake_extra_defines CMAKE_CUDA_ARCHITECTURES=86 --cmake_extra_defines onnxruntime_ENABLE_CUDA_EP_INTERNAL_TESTS=ON
114+
python.exe ${{ github.workspace }}\tools\ci_build\build.py --update --build --config RelWithDebInfo --build_dir build --skip_submodule_sync --build_csharp --parallel --use_binskim_compliant_compile_flags --cmake_generator "Visual Studio 17 2022" --build_shared_lib --build_wheel --build_java --use_cuda --cuda_home="$env:RUNNER_TEMP\v12.8" --enable_cuda_profiling --use_vcpkg --use_vcpkg_ms_internal_asset_cache --enable_transformers_tool_test --cmake_extra_defines CMAKE_CUDA_ARCHITECTURES=86 --cmake_extra_defines onnxruntime_ENABLE_CUDA_EP_INTERNAL_TESTS=ON
115115
if ($lastExitCode -ne 0) {
116116
exit $lastExitCode
117117
}
@@ -188,20 +188,20 @@ jobs:
188188
working-directory: ${{ github.workspace }}
189189
shell: cmd
190190

191-
- name: Download CUDA SDK v12.2
191+
- name: Download CUDA SDK v12.8
192192
working-directory: ${{ runner.temp }}
193193
run: |
194-
azcopy.exe cp --recursive "https://lotusscus.blob.core.windows.net/models/cuda_sdk/v12.2" .
194+
azcopy.exe cp --recursive "https://lotusscus.blob.core.windows.net/models/cuda_sdk/v12.8" .
195195
dir
196196
shell: pwsh
197197

198198
- name: Add CUDA to PATH
199199
shell: powershell
200200
run: |
201201
Write-Host "Adding CUDA to PATH"
202-
Write-Host "CUDA Path: $env:RUNNER_TEMP\v12.2\bin"
203-
Add-Content -Path $env:GITHUB_PATH -Value "$env:RUNNER_TEMP\v12.2\bin"
204-
Add-Content -Path $env:GITHUB_PATH -Value "$env:RUNNER_TEMP\v12.2\extras\CUPTI\lib64"
202+
Write-Host "CUDA Path: $env:RUNNER_TEMP\v12.8\bin"
203+
Add-Content -Path $env:GITHUB_PATH -Value "$env:RUNNER_TEMP\v12.8\bin"
204+
Add-Content -Path $env:GITHUB_PATH -Value "$env:RUNNER_TEMP\v12.8\extras\CUPTI\lib64"
205205
206206
- name: Set OnnxRuntimeBuildDirectory
207207
shell: pwsh
@@ -227,7 +227,7 @@ jobs:
227227
exit $lastExitCode
228228
}
229229
230-
python.exe ${{ github.workspace }}\tools\ci_build\build.py --test --config RelWithDebInfo --build_dir build --skip_submodule_sync --build_csharp --parallel --use_binskim_compliant_compile_flags --cmake_generator "Visual Studio 17 2022" --build_shared_lib --build_wheel --build_java --use_cuda --cuda_home="$env:RUNNER_TEMP\v12.2" --enable_cuda_profiling --use_vcpkg --use_vcpkg_ms_internal_asset_cache --enable_transformers_tool_test --cmake_extra_defines CMAKE_CUDA_ARCHITECTURES=86 --cmake_extra_defines onnxruntime_ENABLE_CUDA_EP_INTERNAL_TESTS=ON
230+
python.exe ${{ github.workspace }}\tools\ci_build\build.py --test --config RelWithDebInfo --build_dir build --skip_submodule_sync --build_csharp --parallel --use_binskim_compliant_compile_flags --cmake_generator "Visual Studio 17 2022" --build_shared_lib --build_wheel --build_java --use_cuda --cuda_home="$env:RUNNER_TEMP\v12.8" --enable_cuda_profiling --use_vcpkg --use_vcpkg_ms_internal_asset_cache --enable_transformers_tool_test --cmake_extra_defines CMAKE_CUDA_ARCHITECTURES=86 --cmake_extra_defines onnxruntime_ENABLE_CUDA_EP_INTERNAL_TESTS=ON
231231
if ($lastExitCode -ne 0) {
232232
exit $lastExitCode
233233
}

‎.github/workflows/windows_openvino.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ concurrency:
1818
jobs:
1919
BUILD_OPENVINO_EP:
2020
name: Windows OpenVINO CI Pipeline
21-
runs-on: ["self-hosted", "1ES.Pool=onnxruntime-github-vs2022-mms"]
21+
runs-on: ["self-hosted", "1ES.Pool=onnxruntime-github-vs2022-latest"]
2222
timeout-minutes: 240
2323
env:
2424
AZCOPY_AUTO_LOGIN_TYPE: MSI

‎.github/workflows/windows_qnn_x64.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ concurrency:
1818
jobs:
1919
build_test_qnn_ep:
2020
name: Windows x64 QNN CI Pipeline (${{ matrix.QnnLibKind }})
21-
runs-on: ["self-hosted", "1ES.Pool=onnxruntime-github-vs2022-mms"]
21+
runs-on: ["self-hosted", "1ES.Pool=onnxruntime-github-vs2022-latest"]
2222
timeout-minutes: 120
2323
strategy:
2424
matrix:

‎.github/workflows/windows_tensorrt.yml‎

Lines changed: 13 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ concurrency:
1919
jobs:
2020
build:
2121
name: Windows GPU TensorRT CI Pipeline
22-
runs-on: ["self-hosted", "1ES.Pool=onnxruntime-github-vs2022-mms"]
22+
runs-on: ["self-hosted", "1ES.Pool=onnxruntime-github-vs2022-latest"]
2323
steps:
2424
- uses: actions/checkout@v5
2525
with:
@@ -41,10 +41,10 @@ jobs:
4141
working-directory: ${{ github.workspace }}
4242
shell: cmd
4343

44-
- name: Download CUDA SDK v12.2
44+
- name: Download CUDA SDK v12.8
4545
working-directory: ${{ runner.temp }}
4646
run: |
47-
azcopy.exe cp --recursive "https://lotusscus.blob.core.windows.net/models/cuda_sdk/v12.2" .
47+
azcopy.exe cp --recursive "https://lotusscus.blob.core.windows.net/models/cuda_sdk/v12.8" .
4848
dir
4949
shell: pwsh
5050

@@ -56,9 +56,9 @@ jobs:
5656
shell: powershell
5757
run: |
5858
Write-Host "Adding CUDA to PATH"
59-
Write-Host "CUDA Path: $env:RUNNER_TEMP\v12.2\bin"
60-
Add-Content -Path $env:GITHUB_PATH -Value "$env:RUNNER_TEMP\v12.2\bin"
61-
Add-Content -Path $env:GITHUB_PATH -Value "$env:RUNNER_TEMP\v12.2\extras\CUPTI\lib64"
59+
Write-Host "CUDA Path: $env:RUNNER_TEMP\v12.8\bin"
60+
Add-Content -Path $env:GITHUB_PATH -Value "$env:RUNNER_TEMP\v12.8\bin"
61+
Add-Content -Path $env:GITHUB_PATH -Value "$env:RUNNER_TEMP\v12.8\extras\CUPTI\lib64"
6262
Add-Content -Path $env:GITHUB_PATH -Value "$env:RUNNER_TEMP\TensorRT-10.9.0.34.Windows10.x86_64.cuda-12.8\lib"
6363
6464
- uses: actions/setup-node@v5
@@ -116,7 +116,7 @@ jobs:
116116
exit $lastExitCode
117117
}
118118
# Execute the build process
119-
python ${{ github.workspace }}\tools\ci_build\build.py --config RelWithDebInfo --parallel --use_binskim_compliant_compile_flags --build_dir build --skip_submodule_sync --build_shared_lib --build --update --cmake_generator "Visual Studio 17 2022" --build_wheel --enable_onnx_tests --use_tensorrt --tensorrt_home="${{ runner.temp }}\TensorRT-10.9.0.34.Windows10.x86_64.cuda-12.8" --cuda_home="${{ runner.temp }}\v12.2" --use_vcpkg --use_vcpkg_ms_internal_asset_cache --cmake_extra_defines CMAKE_CUDA_ARCHITECTURES=86
119+
python ${{ github.workspace }}\tools\ci_build\build.py --config RelWithDebInfo --parallel --use_binskim_compliant_compile_flags --build_dir build --skip_submodule_sync --build_shared_lib --build --update --cmake_generator "Visual Studio 17 2022" --build_wheel --enable_onnx_tests --use_tensorrt --tensorrt_home="${{ runner.temp }}\TensorRT-10.9.0.34.Windows10.x86_64.cuda-12.8" --cuda_home="${{ runner.temp }}\v12.8" --use_vcpkg --use_vcpkg_ms_internal_asset_cache --cmake_extra_defines CMAKE_CUDA_ARCHITECTURES=86
120120
if ($lastExitCode -ne 0) {
121121
exit $lastExitCode
122122
}
@@ -193,10 +193,10 @@ jobs:
193193
working-directory: ${{ github.workspace }}
194194
shell: cmd
195195

196-
- name: Download CUDA SDK v12.2
196+
- name: Download CUDA SDK v12.8
197197
working-directory: ${{ runner.temp }}
198198
run: |
199-
azcopy.exe cp --recursive "https://lotusscus.blob.core.windows.net/models/cuda_sdk/v12.2" .
199+
azcopy.exe cp --recursive "https://lotusscus.blob.core.windows.net/models/cuda_sdk/v12.8" .
200200
dir
201201
shell: pwsh
202202

@@ -208,9 +208,9 @@ jobs:
208208
shell: powershell
209209
run: |
210210
Write-Host "Adding CUDA to PATH"
211-
Write-Host "CUDA Path: $env:RUNNER_TEMP\v12.2\bin"
212-
Add-Content -Path $env:GITHUB_PATH -Value "$env:RUNNER_TEMP\v12.2\bin"
213-
Add-Content -Path $env:GITHUB_PATH -Value "$env:RUNNER_TEMP\v12.2\extras\CUPTI\lib64"
211+
Write-Host "CUDA Path: $env:RUNNER_TEMP\v12.8\bin"
212+
Add-Content -Path $env:GITHUB_PATH -Value "$env:RUNNER_TEMP\v12.8\bin"
213+
Add-Content -Path $env:GITHUB_PATH -Value "$env:RUNNER_TEMP\v12.8\extras\CUPTI\lib64"
214214
Add-Content -Path $env:GITHUB_PATH -Value "$env:RUNNER_TEMP\TensorRT-10.9.0.34.Windows10.x86_64.cuda-12.8\lib"
215215
216216
- name: Set OnnxRuntimeBuildDirectory
@@ -237,7 +237,7 @@ jobs:
237237
exit $lastExitCode
238238
}
239239
240-
python ${{ github.workspace }}\tools\ci_build\build.py --config RelWithDebInfo --parallel --use_binskim_compliant_compile_flags --build_dir build --skip_submodule_sync --build_shared_lib --test --cmake_generator "Visual Studio 17 2022" --build_wheel --enable_onnx_tests --use_tensorrt --tensorrt_home="${{ runner.temp }}\TensorRT-10.9.0.34.Windows10.x86_64.cuda-12.8" --cuda_home="${{ runner.temp }}\v12.2" --use_vcpkg --use_vcpkg_ms_internal_asset_cache --cmake_extra_defines CMAKE_CUDA_ARCHITECTURES=86
240+
python ${{ github.workspace }}\tools\ci_build\build.py --config RelWithDebInfo --parallel --use_binskim_compliant_compile_flags --build_dir build --skip_submodule_sync --build_shared_lib --test --cmake_generator "Visual Studio 17 2022" --build_wheel --enable_onnx_tests --use_tensorrt --tensorrt_home="${{ runner.temp }}\TensorRT-10.9.0.34.Windows10.x86_64.cuda-12.8" --cuda_home="${{ runner.temp }}\v12.8" --use_vcpkg --use_vcpkg_ms_internal_asset_cache --cmake_extra_defines CMAKE_CUDA_ARCHITECTURES=86
241241
if ($lastExitCode -ne 0) {
242242
exit $lastExitCode
243243
}

‎.github/workflows/windows_x64_debug_build_x64_debug.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ concurrency:
1313

1414
jobs:
1515
build_x64_debug:
16-
runs-on: ["self-hosted", "1ES.Pool=onnxruntime-github-vs2022-mms"]
16+
runs-on: ["self-hosted", "1ES.Pool=onnxruntime-github-vs2022-latest"]
1717
timeout-minutes: 300
1818

1919
steps:

‎.github/workflows/windows_x64_release_build_x64_release.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ concurrency:
1313

1414
jobs:
1515
build_x64_release:
16-
runs-on: ["self-hosted", "1ES.Pool=onnxruntime-github-vs2022-mms"]
16+
runs-on: ["self-hosted", "1ES.Pool=onnxruntime-github-vs2022-latest"]
1717
timeout-minutes: 300
1818

1919
steps:

‎.github/workflows/windows_x64_release_ep_generic_interface_build_x64_release_ep_generic_interface.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ concurrency:
1313

1414
jobs:
1515
build_x64_release_ep_generic_interface:
16-
runs-on: ["self-hosted", "1ES.Pool=onnxruntime-github-vs2022-mms"]
16+
runs-on: ["self-hosted", "1ES.Pool=onnxruntime-github-vs2022-latest"]
1717
timeout-minutes: 300
1818

1919
steps:

‎.github/workflows/windows_x64_release_vitisai_build_x64_release.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ concurrency:
1313

1414
jobs:
1515
build_x64_release_vitisai:
16-
runs-on: ["self-hosted", "1ES.Pool=onnxruntime-github-vs2022-mms"]
16+
runs-on: ["self-hosted", "1ES.Pool=onnxruntime-github-vs2022-latest"]
1717
timeout-minutes: 300
1818

1919
steps:

‎.github/workflows/windows_x64_release_xnnpack.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ concurrency:
1313

1414
jobs:
1515
build_x64_release_xnnpack:
16-
runs-on: ["self-hosted", "1ES.Pool=onnxruntime-github-vs2022-mms"]
16+
runs-on: ["self-hosted", "1ES.Pool=onnxruntime-github-vs2022-latest"]
1717
timeout-minutes: 300
1818

1919
steps:

0 commit comments

Comments
 (0)