Skip to content

Commit 89289f7

Browse files
lanluo-nvidialanluo-nvidia
committed
[CI] Add initial github CI
1 parent a405788 commit 89289f7

File tree

26 files changed

+6178
-217
lines changed

26 files changed

+6178
-217
lines changed

‎.github/workflows/mlir-tensorrt-build-test.yml‎

Lines changed: 155 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,155 @@
1+
name: MLIR-TensorRT Build and Test
2+
3+
on:
4+
workflow_call:
5+
inputs:
6+
channel:
7+
description: 'Channel, valid values are "nightly", "test", or "release"'
8+
default: "test"
9+
type: string
10+
build-matrix:
11+
description: 'Build matrix to utilize'
12+
default: ""
13+
type: string
14+
15+
defaults:
16+
run:
17+
shell: bash
18+
19+
jobs:
20+
mlir-tensorrt-build-test:
21+
name: build-test-${{ matrix.arch }}-cu${{ matrix.cuda }}-trt${{ matrix.trt }}
22+
strategy:
23+
fail-fast: false
24+
matrix: ${{ fromJSON(inputs.build-matrix) }}
25+
env:
26+
# eg. CUDA_VERSION: 12.9.1 or 13.0.0
27+
CUDA_VERSION: ${{ matrix.cuda }}
28+
# eg. TENSORRT_VERSION: 10.12 or 10.13
29+
TENSORRT_VERSION: ${{ matrix.trt }}
30+
# align caches with restored paths
31+
CCACHE_DIR: ${{ github.workspace }}/tensorrt-incubator/mlir-tensorrt/ccache
32+
CPM_SOURCE_CACHE: ${{ github.workspace }}/tensorrt-incubator/mlir-tensorrt/.cache.cpm
33+
# eg. CHANNEL: nightly, test or release
34+
CHANNEL: ${{ inputs.channel }}
35+
runs-on: ${{ matrix.runner }}
36+
container:
37+
image: ${{ matrix.docker_image }}
38+
options: '--gpus all'
39+
steps:
40+
# Checkout the repository
41+
- name: Checkout TensorRT-Incubator
42+
uses: actions/checkout@v4
43+
with:
44+
path: ${{ github.workspace }}/tensorrt-incubator
45+
46+
# Create cache folders
47+
- name: Create Cache Folders
48+
run: |
49+
set -euo pipefail
50+
set -x
51+
mkdir -p ${{ github.workspace }}/tensorrt-incubator/mlir-tensorrt/ccache
52+
mkdir -p ${{ github.workspace }}/tensorrt-incubator/mlir-tensorrt/.cache.cpm
53+
54+
# Restore cache, if exists.
55+
- name: Restore Cache
56+
uses: actions/cache/restore@v4
57+
with:
58+
key: ${{ runner.os }}-mlir-tensorrt-cache-${{ hashFiles('mlir-tensorrt/**/*.cpp', 'mlir-tensorrt/**/*.h', 'mlir-tensorrt/build_tools/**/*') }}
59+
restore-keys: |
60+
${{ runner.os }}-mlir-tensorrt-cache-
61+
path: |
62+
${{ github.workspace }}/tensorrt-incubator/mlir-tensorrt/ccache
63+
${{ github.workspace }}/tensorrt-incubator/mlir-tensorrt/.cache.cpm/*
64+
!${{ github.workspace }}/tensorrt-incubator/mlir-tensorrt/.cache.cpm/tensorrt
65+
66+
# Build the project
67+
- name: Build With CUDA:${{ matrix.cuda }} + TensorRT:${{ matrix.trt }}
68+
run: |
69+
set -euo pipefail
70+
set -x
71+
cd ${{ github.workspace }}/tensorrt-incubator/mlir-tensorrt
72+
# Build only, skip tests
73+
SKIP_TESTS=1 \
74+
./build_tools/scripts/cicd_build_test.sh
75+
76+
# Save cache
77+
- name: Save cache
78+
uses: actions/cache/save@v4
79+
with:
80+
key: ${{ runner.os }}-mlir-tensorrt-cache-${{ hashFiles('mlir-tensorrt/**/*.cpp', 'mlir-tensorrt/**/*.h', 'mlir-tensorrt/build_tools/**/*') }}
81+
path: |
82+
${{ github.workspace }}/tensorrt-incubator/mlir-tensorrt/ccache
83+
${{ github.workspace }}/tensorrt-incubator/mlir-tensorrt/.cache.cpm/*
84+
!${{ github.workspace }}/tensorrt-incubator/mlir-tensorrt/.cache.cpm/tensorrt
85+
86+
# Run tests
87+
- name: Run Basic Tests
88+
id: basic-tests
89+
run: |
90+
set -euo pipefail
91+
set -x
92+
cd ${{ github.workspace }}/tensorrt-incubator/mlir-tensorrt
93+
# run tests
94+
./build_tools/scripts/cicd_build_test.sh
95+
96+
# Run ASAN tests
97+
- name: Run ASAN Tests
98+
id: asan-tests
99+
if: ${{ always() && (inputs.channel == 'nightly' || (github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'ASAN_TEST'))) }}
100+
run: |
101+
set -euo pipefail
102+
set -x
103+
cd ${{ github.workspace }}/tensorrt-incubator/mlir-tensorrt
104+
CMAKE_PRESET=github-cicd-with-asan \
105+
./build_tools/scripts/cicd_build_test.sh
106+
107+
# Run Long tests
108+
- name: Run Long Tests
109+
id: long-tests
110+
if: ${{ always() && (inputs.channel == 'nightly' || (github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'LONG_TEST'))) }}
111+
run: |
112+
set -euo pipefail
113+
set -x
114+
cd ${{ github.workspace }}/tensorrt-incubator/mlir-tensorrt
115+
CMAKE_PRESET=github-cicd-with-long-tests \
116+
./build_tools/scripts/cicd_build_test.sh
117+
118+
# Run NCCL Long tests
119+
- name: Run NCCL Long Tests
120+
id: nccl-long-tests
121+
if: ${{ always() && (inputs.channel == 'nightly' || (github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'NCCL_LONG_TEST'))) }}
122+
run: |
123+
set -euo pipefail
124+
set -x
125+
cd ${{ github.workspace }}/tensorrt-incubator/mlir-tensorrt
126+
CMAKE_PRESET=github-cicd-with-nccl-long-tests \
127+
./build_tools/scripts/cicd_build_test.sh
128+
129+
# Build wheels
130+
- name: Build Wheels
131+
id: build-wheels
132+
if: ${{ always() && inputs.channel == 'nightly' }}
133+
run: |
134+
set -euo pipefail
135+
set -x
136+
cd ${{ github.workspace }}/tensorrt-incubator/mlir-tensorrt
137+
./build_tools/scripts/cicd_build_wheels.sh
138+
139+
# Smoke test the wheels built
140+
- name: Smoke Test Wheels
141+
id: smoke-test-wheels
142+
if: ${{ inputs.channel == 'nightly' }}
143+
run: |
144+
set -euo pipefail
145+
set -x
146+
cd ${{ github.workspace }}/tensorrt-incubator/mlir-tensorrt
147+
for whl in .wheels/*.whl; do
148+
uv pip install "$whl"
149+
done
150+
# TODO: add smoke test for the wheel
151+
152+
153+
concurrency:
154+
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref_name }}-mlir-tensorrt-${{ inputs.channel }}
155+
cancel-in-progress: true

‎.github/workflows/mlir-tensorrt-ci.yml‎

Lines changed: 64 additions & 94 deletions
Original file line numberDiff line numberDiff line change
@@ -1,123 +1,93 @@
11
name: MLIR-TensorRT CI
22

33
on:
4-
pull_request:
5-
branches:
6-
- main
7-
types: [synchronize, opened, reopened, ready_for_review]
8-
paths: ["mlir-tensorrt/**"]
94
push:
105
branches:
11-
- main
12-
paths: ["mlir-tensorrt/**"]
6+
- "pull-request/[0-9]+"
7+
tags:
8+
# release tag example: v0.4.2
9+
# release candidate tag example: v0.4.2-rc1
10+
- v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+
11+
workflow_dispatch:
12+
schedule:
13+
- cron: '0 0 * * *' # Runs at 00:00 UTC every day (minute hour day-of-month month-of-year day-of-week)
1314

1415
env:
15-
DEFAULT_IMAGE: ghcr.io/nvidia/tensorrt-incubator/mlir-tensorrt:cuda12.5-ubuntu-llvm17
16+
DEFAULT_IMAGE: ghcr.io/nvidia/tensorrt-incubator/mlir-tensorrt:cuda12.9-ubuntu-llvm17
1617
REGISTRY: ghcr.io
1718

1819
jobs:
19-
mlir-tensorrt-test-pr:
20-
if: github.event_name != 'pull_request' || github.event.pull_request.draft == false
21-
runs-on: tripy-self-hosted
22-
20+
format-check:
21+
name: Lint Check
22+
runs-on: ubuntu-latest
2323
steps:
24-
# Value of `github.workspace` is /home/runner/work/{repo_name}/{repo-name}
25-
# i.e. /home/runner/work/TensorRT-Incubator/TensorRT-Incubator in our case.
26-
# After this action, repo is cloned inside above path.
27-
- uses: actions/checkout@v4
24+
- name: checkout TensorRT-Incubator
25+
uses: actions/checkout@v4
2826
with:
2927
fetch-depth: 5
30-
31-
# Run initial format check
3228
- name: Run python format and clang check
3329
uses: addnab/docker-run-action@v3
34-
if: ${{ github.event_name == 'pull_request' }}
3530
with:
3631
image: ${{ env.DEFAULT_IMAGE }}
3732
options: -v ${{ github.workspace }}:/tensorrt-incubator
3833
registry: ${{ env.REGISTRY }}
3934
username: ${{ github.actor }}
4035
password: ${{ secrets.GITHUB_TOKEN }}
41-
# This step does two things
42-
# 1. Check if Python files follow black format
43-
# 2. Check if C++ files follow clang format
44-
# NOTE: We are placed at the root directory ('/') inside the container.
4536
run: |
46-
cd tensorrt-incubator
37+
set -euo pipefail
38+
set -x
39+
cd /tensorrt-incubator
4740
git config --global --add safe.directory /tensorrt-incubator
48-
cat > run_format_check.sh <<EOF
49-
#!/bin/bash
50-
set -e
51-
python3 -m black --check --extend-exclude='.*\.pyi' mlir-tensorrt/compiler/
52-
python3 -m black --check --extend-exclude='.*\.pyi' mlir-tensorrt/integrations/python/
41+
uvx -p 3.12 black --check --extend-exclude='.*\.pyi' \
42+
mlir-tensorrt/compiler/ mlir-tensorrt/integrations/python/
5343
git clang-format HEAD~1 --diff
54-
EOF
55-
56-
bash run_format_check.sh
5744
58-
# Create cache folders
59-
- name: Create cache folder
60-
run: |
61-
mkdir -p ${{ github.workspace }}/ccache
62-
mkdir -p ${{ github.workspace }}/.cache.cpm
63-
64-
# Restore cache, if exists.
65-
- name: Restore cache
66-
id: restore-cache
67-
uses: actions/cache/restore@v4
45+
generate-matrix:
46+
name: Generate Build Matrix
47+
runs-on: ubuntu-latest
48+
outputs:
49+
matrix: ${{ steps.generate.outputs.matrix }}
50+
channel: ${{ steps.generate.outputs.channel }}
51+
steps:
52+
- name: Checkout TensorRT-Incubator
53+
uses: actions/checkout@v4
6854
with:
69-
key: ${{ runner.os }}-mlir-tensorrt-cache-${{ hashFiles('mlir-tensorrt/**/*.cpp', 'mlir-tensorrt/**/*.h', 'mlir-tensorrt/build_tools/**/*') }}
70-
restore-keys: |
71-
${{ runner.os }}-mlir-tensorrt-cache-
72-
path: |
73-
${{ github.workspace }}/ccache
74-
${{ github.workspace }}/.cache.cpm/*
75-
!${{ github.workspace }}/.cache.cpm/tensorrt
55+
path: ${{ github.workspace }}/tensorrt-incubator
56+
- name: Generate Build Matrix
57+
id: generate
58+
run: |
59+
set -euo pipefail
60+
set -x
61+
CHANNEL="test"
62+
if [ "${GITHUB_EVENT_NAME:-}" = "schedule" ]; then
63+
CHANNEL="nightly"
64+
elif [ "${GITHUB_REF_TYPE:-}" = "tag" ]; then
65+
CHANNEL="release"
66+
fi
67+
MATRIX_BLOB="$(python3 ${{ github.workspace }}/tensorrt-incubator/.github/workflows/mlir-tensorrt/generate-matrix.py --channel "${CHANNEL}")"
68+
echo "${MATRIX_BLOB}"
69+
echo "${CHANNEL}"
70+
echo "matrix=${MATRIX_BLOB}" >> "${GITHUB_OUTPUT}"
71+
echo "channel=${CHANNEL}" >> "${GITHUB_OUTPUT}"
7672
77-
# TensorRT 10 tests
78-
- name: TensorRT 10 build
79-
uses: addnab/docker-run-action@v3
80-
with:
81-
image: ${{ env.DEFAULT_IMAGE }}
82-
options: -v ${{ github.workspace }}/mlir-tensorrt:/mlir-tensorrt -v ${{ github.workspace }}/ccache:/ccache -v ${{ github.workspace }}/.cache.cpm:/.cache.cpm --gpus all
83-
registry: ${{ env.REGISTRY }}
84-
username: ${{ github.actor }}
85-
password: ${{ secrets.GITHUB_TOKEN }}
86-
run: |
87-
cd mlir-tensorrt
88-
./build_tools/scripts/cicd_build.sh --build_only
73+
mlir-tensorrt-build-test:
74+
name: Build and Test
75+
needs: generate-matrix
76+
if: ${{ needs.generate-matrix.outputs.channel != 'release' }}
77+
uses: ./.github/workflows/mlir-tensorrt-build-test.yml
78+
with:
79+
build-matrix: ${{ needs.generate-matrix.outputs.matrix }}
80+
channel: ${{ needs.generate-matrix.outputs.channel }}
8981

90-
- name: Save cache
91-
id: save-cache
92-
uses: actions/cache/save@v4
93-
with:
94-
key: ${{ runner.os }}-mlir-tensorrt-cache-${{ hashFiles('mlir-tensorrt/**/*.cpp', 'mlir-tensorrt/**/*.h', 'mlir-tensorrt/build_tools/**/*') }}
95-
path: |
96-
${{ github.workspace }}/ccache
97-
${{ github.workspace }}/.cache.cpm/*
98-
!${{ github.workspace }}/.cache.cpm/tensorrt
82+
mlir-tensorrt-release:
83+
name: Release
84+
needs: generate-matrix
85+
if: ${{ needs.generate-matrix.outputs.channel == 'release' }}
86+
uses: ./.github/workflows/mlir-tensorrt-release.yml
87+
with:
88+
build-matrix: ${{ needs.generate-matrix.outputs.matrix }}
89+
channel: release
9990

100-
- name: TensorRT 10 test
101-
uses: addnab/docker-run-action@v3
102-
with:
103-
image: ${{ env.DEFAULT_IMAGE }}
104-
options: -v ${{ github.workspace }}/mlir-tensorrt:/mlir-tensorrt -v ${{ github.workspace }}/ccache:/ccache -v ${{ github.workspace }}/.cache.cpm:/.cache.cpm --gpus all
105-
registry: ${{ env.REGISTRY }}
106-
username: ${{ github.actor }}
107-
password: ${{ secrets.GITHUB_TOKEN }}
108-
run: |
109-
cd mlir-tensorrt
110-
./build_tools/scripts/cicd_build.sh
111-
112-
# TensorRT 10 & ASAN
113-
- name: TensorRT 10 ASAN test
114-
uses: addnab/docker-run-action@v3
115-
with:
116-
image: ${{ env.DEFAULT_IMAGE }}
117-
options: -v ${{ github.workspace }}/mlir-tensorrt:/mlir-tensorrt -v ${{ github.workspace }}/ccache:/ccache -v ${{ github.workspace }}/.cache.cpm:/.cache.cpm --gpus all
118-
registry: ${{ env.REGISTRY }}
119-
username: ${{ github.actor }}
120-
password: ${{ secrets.GITHUB_TOKEN }}
121-
run: |
122-
cd mlir-tensorrt
123-
ENABLE_ASAN=ON ./build_tools/scripts/cicd_build.sh
91+
concurrency:
92+
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref_name }}-mlir-tensorrt
93+
cancel-in-progress: true

0 commit comments

Comments
 (0)