Skip to content

Commit 9e5e627

Browse files
dhuangnmdhuangnmdhuangnm
authored
Add workflows to build and test nightly and release (#155)
* add workflow to build and test * fix indent * clean up * add project id * minor update * fix a bug * test release * add test * fix test reporting issue * add upload and use compressed-tensors bucket * use secret for bucket * not install * still uses k8s-util * clean up * try GCP_GHA_SA * fix path * use NM_PYPI_SA again * authenticate with GCP_GHA_SA and NM_PYPI_SA * fix id * seperate install and test steps * test more configs * fix error * fix a bug * fix again * add step-status * fix failure * test release * clean up * clean up * update * test nightly * test release * fix a bug * fix a bug * change default back to NIGHTLY * clean up --------- Co-authored-by: dhuangnm <[email protected]> Co-authored-by: dhuangnm <[email protected]>
1 parent 329d62b commit 9e5e627

File tree

9 files changed

+611
-117
lines changed

9 files changed

+611
-117
lines changed

‎.github/actions/test/action.yml‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
name: test compressed-tensors
2+
description: 'test compressed-tensors'
3+
inputs:
4+
venv:
5+
description: "path of virtualenv"
6+
required: true
7+
outputs:
8+
status:
9+
description: "final status from test"
10+
value: ${{ steps.test.outputs.status }}
11+
runs:
12+
using: composite
13+
steps:
14+
15+
- name: install wheel
16+
uses: neuralmagic/nm-actions/actions/[email protected]
17+
with:
18+
venv: ${{ inputs.venv }}
19+
name: compressed
20+
extra: "[dev,accelerate]"
21+
22+
- name: test
23+
id: test
24+
run: |
25+
source ${{ inputs.venv }}/bin/activate
26+
rm -rf src
27+
SUCCESS=0
28+
pytest tests --junitxml=test-results/report.xml || SUCCESS=$?
29+
echo "status=${SUCCESS}" >> "$GITHUB_OUTPUT"
30+
deactivate
31+
exit ${SUCCESS}
32+
shell: bash

‎.github/scripts/step-status‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
#!/bin/bash -e
2+
3+
# echo "green encased checkmark" if "${1} == 0"
4+
# echo "red X" if "${1} != 0"
5+
6+
STEP_STATUS=${1}
7+
8+
if [ "$STEP_STATUS" -eq 0 ]; then
9+
# green check
10+
echo -e "\xE2\x9C\x85"
11+
else
12+
# red x
13+
echo -e "\xE2\x9D\x8C"
14+
fi

‎.github/workflows/build-test-publish-nightly.yaml‎

Lines changed: 0 additions & 51 deletions
This file was deleted.

‎.github/workflows/build-test-release.yaml‎

Lines changed: 0 additions & 66 deletions
This file was deleted.

‎.github/workflows/build-test.yml‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
name: build-test
2+
on:
3+
4+
# makes workflow reusable
5+
workflow_call:
6+
inputs:
7+
wf_category:
8+
description: "workflow category: NIGHTLY, RELEASE"
9+
type: string
10+
default: NIGHTLY
11+
push_to_pypi:
12+
description: "When set to true, built whl and tar.gz will be pushed to public pypi if all tests pass"
13+
type: boolean
14+
default: false
15+
gitref:
16+
description: "git commit hash or tag name"
17+
type: string
18+
default: main
19+
20+
# build related parameters
21+
build_label:
22+
description: "requested runner label for build (specifies instance)"
23+
type: string
24+
default: ubuntu-20.04
25+
26+
# test related parameters
27+
test_configs:
28+
description: "python, label, timeout"
29+
type: string
30+
required: true
31+
32+
jobs:
33+
34+
BUILD:
35+
uses: ./.github/workflows/build.yml
36+
with:
37+
wf_category: ${{ inputs.wf_category }}
38+
build_label: ${{ inputs.build_label }}
39+
gitref: ${{ inputs.gitref }}
40+
timeout: 20
41+
secrets: inherit
42+
43+
TEST:
44+
needs: [BUILD]
45+
strategy:
46+
fail-fast: false
47+
matrix:
48+
test_config: ${{ fromJson(inputs.test_configs) }}
49+
uses: ./.github/workflows/test.yml
50+
with:
51+
gitref: ${{ inputs.gitref }}
52+
test_label: ${{ matrix.test_config.label }}
53+
python: ${{ matrix.test_config.python }}
54+
timeout: ${{ matrix.test_config.timeout }}
55+
whl: ${{ needs.BUILD.outputs.whl }}
56+
testmo_run_id: ${{ needs.BUILD.outputs.testmo_run_id }}
57+
secrets: inherit
58+
59+
UPLOAD:
60+
needs: [BUILD, TEST]
61+
uses: ./.github/workflows/upload.yml
62+
with:
63+
label: k8s-util
64+
timeout: 40
65+
run_id: ${{ github.run_id }}
66+
push_to_pypi: ${{ inputs.push_to_pypi }}
67+
testmo_run_id: ${{ needs.BUILD.outputs.testmo_run_id }}
68+
secrets: inherit

‎.github/workflows/build.yml‎

Lines changed: 161 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,161 @@
1+
name: build
2+
on:
3+
# makes workflow reusable
4+
workflow_call:
5+
inputs:
6+
wf_category:
7+
description: "categories: NIGHTLY, RELEASE"
8+
type: string
9+
default: NIGHTLY
10+
build_label:
11+
description: "requested runner label (specifies instance)"
12+
type: string
13+
required: true
14+
timeout:
15+
description: "time limit for run in minutes "
16+
type: string
17+
default: 20
18+
gitref:
19+
description: "git commit hash or branch name"
20+
type: string
21+
default: main
22+
outputs:
23+
whl:
24+
description: 'basename for generated whl'
25+
value: ${{ jobs.BUILD.outputs.whl }}
26+
testmo_run_id:
27+
description: 'testmo run id'
28+
value: ${{ jobs.BUILD.outputs.testmo_run_id }}
29+
30+
# makes workflow manually callable
31+
workflow_dispatch:
32+
inputs:
33+
wf_category:
34+
description: "categories: NIGHTLY, RELEASE"
35+
type: string
36+
default: NIGHTLY
37+
build_label:
38+
description: "requested runner label (specifies instance)"
39+
type: string
40+
required: true
41+
timeout:
42+
description: "time limit for run in minutes "
43+
type: string
44+
default: 20
45+
gitref:
46+
description: "git commit hash or branch name"
47+
type: string
48+
default: main
49+
50+
jobs:
51+
52+
BUILD:
53+
54+
runs-on: ${{ inputs.build_label }}
55+
timeout-minutes: ${{ fromJson(inputs.timeout) }}
56+
permissions:
57+
contents: 'read'
58+
id-token: 'write'
59+
60+
outputs:
61+
run_id: ${{ github.run_id }}
62+
whl: ${{ steps.build.outputs.whlname }}
63+
tarfile: ${{ steps.build.outputs.tarname }}
64+
testmo_run_id: ${{ steps.create_testmo_run.outputs.id }}
65+
66+
steps:
67+
68+
- name: set python
69+
uses: actions/setup-python@v4
70+
with:
71+
python-version: '3.10'
72+
73+
- name: checkout code
74+
id: checkout
75+
uses: actions/checkout@v4
76+
with:
77+
ref: ${{ inputs.gitref }}
78+
79+
- name: install testmo
80+
uses: neuralmagic/nm-actions/actions/[email protected]
81+
82+
- name: create testmo run
83+
id: create_testmo_run
84+
uses: neuralmagic/nm-actions/actions/[email protected]
85+
if: success()
86+
with:
87+
testmo_url: https://neuralmagic.testmo.net
88+
testmo_token: ${{ secrets.TESTMO_TEST_TOKEN }}
89+
source: 'build-test'
90+
project_id: 14
91+
run_name: compressedtensors-${{ inputs.wf_category }}-${{ inputs.gitref }}-${GITHUB_ACTOR}
92+
93+
- name: build
94+
id: build
95+
uses: neuralmagic/nm-actions/actions/[email protected]
96+
with:
97+
dev: false
98+
release: ${{ inputs.wf_category == 'RELEASE' }}
99+
100+
# GCP
101+
- name: 'Authenticate to Google Cloud'
102+
id: auth
103+
uses: google-github-actions/[email protected]
104+
with:
105+
project_id: ${{ secrets.GCP_PROJECT }}
106+
workload_identity_provider: ${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }}
107+
service_account: ${{ secrets.GCP_GHA_SA }}
108+
109+
- name: 'Set up Cloud SDK'
110+
uses: 'google-github-actions/setup-gcloud@v2'
111+
with:
112+
version: '>= 473.0.0'
113+
114+
- name: copy whl and source distribution
115+
run: |
116+
gcloud storage cp dist/${{ steps.build.outputs.whlname }} ${{ secrets.GCP_BUILD_ML_ASSETS2 }}/${{ github.run_id }}/${{ steps.build.outputs.whlname }}
117+
gcloud storage cp dist/${{ steps.build.outputs.tarname }} ${{ secrets.GCP_BUILD_ML_ASSETS2 }}/${{ github.run_id }}/${{ steps.build.outputs.tarname }}
118+
119+
- name: upload whl
120+
uses: actions/upload-artifact@v4
121+
if: success() || failure()
122+
with:
123+
name: ${{ steps.build.outputs.whlname }}
124+
path: dist/${{ steps.build.outputs.whlname }}
125+
retention-days: 5
126+
127+
- name: upload tar.gz
128+
uses: actions/upload-artifact@v4
129+
if: success() || failure()
130+
with:
131+
name: ${{ steps.build.outputs.tarname }}
132+
path: dist/${{ steps.build.outputs.tarname }}
133+
retention-days: 5
134+
135+
- name: summary
136+
uses: neuralmagic/nm-actions/actions/[email protected]
137+
if: success() || failure()
138+
with:
139+
label: ${{ inputs.build_label }}
140+
gitref: ${{ inputs.gitref }}
141+
whl_status: ${{ steps.build.outputs.status }}
142+
143+
- name: report build status to testmo
144+
id: report_build
145+
uses: neuralmagic/nm-actions/actions/[email protected]
146+
if: (success() || failure()) && ${{ inputs.testmo_run_id != '' }}
147+
with:
148+
testmo_url: https://neuralmagic.testmo.net
149+
testmo_token: ${{ secrets.TESTMO_TEST_TOKEN }}
150+
testmo_run_id: ${{ steps.create_testmo_run.outputs.id }}
151+
results: build-results
152+
step_status: ${{ steps.build.outputs.status }}
153+
154+
- name: run status
155+
id: run_status
156+
if: success() || failure()
157+
env:
158+
WHL_STATUS: ${{ steps.build.outputs.status }}
159+
run: |
160+
echo "build status: ${WHL_STATUS}"
161+
if [ -z "${WHL_STATUS}" ] || [ "${WHL_STATUS}" -ne "0" ]; then exit 1; fi

0 commit comments

Comments
 (0)