Skip to content

Commit d7f51b5

Browse files
DiegoTavaresDiegoTavares
authored
[CICD] Add rust/rqd to both packaging and release pipelines (#1874)
1 parent 9ec3444 commit d7f51b5

File tree

11 files changed

+364
-43
lines changed

11 files changed

+364
-43
lines changed

.github/workflows/docs.yml renamed to .github/workflows/docs-pipeline.yml

File renamed without changes.

‎.github/workflows/packaging-pipeline.yml‎

Lines changed: 108 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ name: OpenCue Packaging Pipeline
33
# Trigger this pipeline on new commits to master.
44
on:
55
push:
6-
branches: [ master ]
6+
branches: [master]
77

88
jobs:
99
build_opencue_packages:
@@ -42,6 +42,78 @@ jobs:
4242
echo "opencue_cueman_path=$(find ./packages -name 'opencue_cueman-*.whl' -print -quit)" >> $GITHUB_OUTPUT
4343
echo "opencue_cuegui_path=$(find ./packages -name 'opencue_cuegui-*.whl' -print -quit)" >> $GITHUB_OUTPUT
4444
45+
build_rust_binaries:
46+
name: Build Rust Binaries
47+
strategy:
48+
matrix:
49+
include:
50+
- target: x86_64-unknown-linux-gnu
51+
os: ubuntu-22.04
52+
- target: x86_64-unknown-linux-musl
53+
os: ubuntu-22.04
54+
- target: x86_64-apple-darwin
55+
os: macos-13
56+
- target: aarch64-apple-darwin
57+
os: macos-13
58+
runs-on: ${{ matrix.os }}
59+
outputs:
60+
rust_binaries_artifact: rust-binaries-${{ github.sha }}
61+
steps:
62+
- uses: actions/checkout@v4
63+
with:
64+
fetch-tags: true
65+
fetch-depth: 0
66+
67+
- name: Mark repository as safe
68+
run: git config --global --add safe.directory $GITHUB_WORKSPACE
69+
70+
- name: Install Rust
71+
uses: dtolnay/rust-toolchain@stable
72+
with:
73+
targets: ${{ matrix.target }}
74+
75+
- name: Install dependencies (Ubuntu)
76+
if: matrix.os == 'ubuntu-22.04'
77+
run: |
78+
sudo apt-get update
79+
sudo apt-get install -y libx11-dev protobuf-compiler
80+
if [ "${{ matrix.target }}" = "x86_64-unknown-linux-musl" ]; then
81+
sudo apt-get install -y musl-tools
82+
fi
83+
84+
- name: Install dependencies (macOS)
85+
if: matrix.os == 'macos-13'
86+
run: |
87+
brew install protobuf
88+
89+
- name: Set build ID
90+
run: |
91+
echo "BUILD_ID=$(ci/generate_version_number.py)" >> $GITHUB_ENV
92+
93+
- name: Build Rust binaries
94+
run: |
95+
cd rust
96+
cargo build --release --target ${{ matrix.target }} --no-default-features
97+
98+
- name: Create release directory
99+
run: mkdir -p release
100+
101+
- name: Copy binaries (Linux)
102+
if: matrix.os == 'ubuntu-22.04'
103+
run: |
104+
cp rust/target/${{ matrix.target }}/release/openrqd release/openrqd-${{ env.BUILD_ID }}-${{ matrix.target }}
105+
106+
- name: Copy binaries (macOS)
107+
if: matrix.os == 'macos-13'
108+
run: |
109+
cp rust/target/${{ matrix.target }}/release/openrqd release/openrqd-${{ env.BUILD_ID }}-${{ matrix.target }}
110+
111+
- name: Upload binaries
112+
uses: actions/upload-artifact@v4
113+
with:
114+
name: rust-binaries-${{ matrix.target }}
115+
path: release/
116+
45117
integration_test:
46118
needs: build_opencue_packages
47119
name: Run Integration Test
@@ -56,12 +128,19 @@ jobs:
56128
- name: Mark repository as safe (Fix for https://github.com/actions/checkout/issues/1048)
57129
run: git config --global --add safe.directory $GITHUB_WORKSPACE
58130

59-
- name: Download a single artifact
131+
- name: Download Python packages artifact
60132
uses: actions/download-artifact@v4
61133
with:
62134
name: opencue_packages
63135
path: packages
64136

137+
- name: Download Rust binaries artifacts
138+
uses: actions/download-artifact@v4
139+
with:
140+
pattern: rust-binaries-*
141+
path: rust-binaries
142+
merge-multiple: true
143+
65144
- name: Run test
66145
run: |
67146
export OPENCUE_PROTO_PACKAGE_PATH="${{ needs.build_opencue_packages.outputs.opencue_proto_path }}"
@@ -122,12 +201,19 @@ jobs:
122201
password: ${{ secrets.DOCKER_PASS }}
123202
continue-on-error: true
124203

125-
- name: Download a single artifact
204+
- name: Download Python packages artifact
126205
uses: actions/download-artifact@v4
127206
with:
128207
name: opencue_packages
129208
path: packages
130209

210+
- name: Download Rust binaries artifacts
211+
uses: actions/download-artifact@v4
212+
with:
213+
pattern: rust-binaries-*
214+
path: rust-binaries
215+
merge-multiple: true
216+
131217
- name: Set build ID
132218
run: |
133219
set -e
@@ -177,6 +263,7 @@ jobs:
177263
needs:
178264
- build_components
179265
- build_opencue_packages
266+
- build_rust_binaries
180267
runs-on: ubuntu-22.04
181268
continue-on-error: true
182269
steps:
@@ -199,10 +286,10 @@ jobs:
199286

200287
- name: Set build ID
201288
run: |
202-
set -e
203-
echo "Build ID: $(ci/generate_version_number.py)"
204-
echo "BUILD_ID=$(ci/generate_version_number.py)" >> ${GITHUB_ENV}
205-
echo "BUILD_ID=$(ci/generate_version_number.py)" >VERSION
289+
set -e
290+
echo "Build ID: $(ci/generate_version_number.py)"
291+
echo "BUILD_ID=$(ci/generate_version_number.py)" >> ${GITHUB_ENV}
292+
echo "BUILD_ID=$(ci/generate_version_number.py)" >VERSION
206293
207294
- name: Extract database schema
208295
run: |
@@ -214,12 +301,19 @@ jobs:
214301
mkdir -p "${GITHUB_WORKSPACE}/artifacts/"
215302
echo "{\"git_commit\": \"$(BUILD_SOURCEVERSION)\"}" | tee "${GITHUB_WORKSPACE}/artifacts/build_metadata.json"
216303
217-
- name: Download a single artifact
304+
- name: Download Python packages artifact
218305
uses: actions/download-artifact@v4
219306
with:
220307
name: opencue_packages
221308
path: packages
222309

310+
- name: Download Rust binaries artifacts
311+
uses: actions/download-artifact@v4
312+
with:
313+
pattern: rust-binaries-*
314+
path: rust-binaries
315+
merge-multiple: true
316+
223317
- name: Upload artifacts
224318
env:
225319
S3_BUCKET: ${{ secrets.S3_BUCKET }}
@@ -237,6 +331,12 @@ jobs:
237331
aws s3 cp "${GITHUB_WORKSPACE}/artifacts/schema-${BUILD_ID}.sql" s3://${S3_BUCKET}/opencue/${BUILD_ID}/
238332
aws s3 cp "${GITHUB_WORKSPACE}/artifacts/seed_data-${BUILD_ID}.sql" s3://${S3_BUCKET}/opencue/${BUILD_ID}/
239333
aws s3 cp "${GITHUB_WORKSPACE}/artifacts/build_metadata.json" s3://${S3_BUCKET}/opencue/${BUILD_ID}/
334+
# Upload Rust binaries
335+
for binary in rust-binaries/*; do
336+
if [ -f "$binary" ]; then
337+
aws s3 cp "$binary" s3://${S3_BUCKET}/opencue/${BUILD_ID}/
338+
fi
339+
done
240340
241341
- name: Display artifacts
242342
env:

‎.github/workflows/release-pipeline.yml‎

Lines changed: 142 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,100 @@ jobs:
4343
4444
- uses: ./.github/actions/build-python-packages
4545

46+
- name: Gather package paths
47+
id: package_outputs
48+
run: |
49+
echo "opencue_proto_path=$(find ./packages -name 'opencue_proto-*.whl' -print -quit)" >> $GITHUB_OUTPUT
50+
echo "opencue_rqd_path=$(find ./packages -name 'opencue_rqd-*.whl' -print -quit)" >> $GITHUB_OUTPUT
51+
echo "opencue_pycue_path=$(find ./packages -name 'opencue_pycue-*.whl' -print -quit)" >> $GITHUB_OUTPUT
52+
echo "opencue_pyoutline_path=$(find ./packages -name 'opencue_pyoutline-*.whl' -print -quit)" >> $GITHUB_OUTPUT
53+
echo "opencue_cuesubmit_path=$(find ./packages -name 'opencue_cuesubmit-*.whl' -print -quit)" >> $GITHUB_OUTPUT
54+
echo "opencue_cueadmin_path=$(find ./packages -name 'opencue_cueadmin-*.whl' -print -quit)" >> $GITHUB_OUTPUT
55+
echo "opencue_cueman_path=$(find ./packages -name 'opencue_cueman-*.whl' -print -quit)" >> $GITHUB_OUTPUT
56+
echo "opencue_cuegui_path=$(find ./packages -name 'opencue_cuegui-*.whl' -print -quit)" >> $GITHUB_OUTPUT
57+
58+
build_rust_binaries:
59+
name: Build Rust Binaries
60+
strategy:
61+
matrix:
62+
include:
63+
- target: x86_64-unknown-linux-gnu
64+
os: ubuntu-22.04
65+
- target: x86_64-unknown-linux-musl
66+
os: ubuntu-22.04
67+
- target: x86_64-apple-darwin
68+
os: macos-13
69+
- target: aarch64-apple-darwin
70+
os: macos-13
71+
runs-on: ${{ matrix.os }}
72+
steps:
73+
- uses: actions/checkout@v4
74+
with:
75+
fetch-tags: true
76+
fetch-depth: 0
77+
78+
- name: Mark repository as safe
79+
run: git config --global --add safe.directory $GITHUB_WORKSPACE
80+
81+
- name: Get current tag name
82+
run: echo "TAG_NAME=${GITHUB_REF/refs\/tags\//}" >> ${GITHUB_ENV}
83+
84+
- name: Verify tag name and version match
85+
run: |
86+
set -e
87+
if [ "v$(ci/generate_version_number.py)" != "${TAG_NAME}" ]; then
88+
echo "Version check failed: code version v$(ci/generate_version_number.py) does not match tag name ${TAG_NAME}"
89+
echo "Original GITHUB_REF: ${GITHUB_REF}"
90+
exit 1
91+
fi
92+
93+
- name: Install Rust
94+
uses: dtolnay/rust-toolchain@stable
95+
with:
96+
targets: ${{ matrix.target }}
97+
98+
- name: Install dependencies (Ubuntu)
99+
if: matrix.os == 'ubuntu-22.04'
100+
run: |
101+
sudo apt-get update
102+
sudo apt-get install -y libx11-dev protobuf-compiler
103+
if [ "${{ matrix.target }}" = "x86_64-unknown-linux-musl" ]; then
104+
sudo apt-get install -y musl-tools
105+
fi
106+
107+
- name: Install dependencies (macOS)
108+
if: matrix.os == 'macos-13'
109+
run: |
110+
brew install protobuf
111+
112+
- name: Set build ID
113+
run: |
114+
echo "BUILD_ID=$(ci/generate_version_number.py)" >> $GITHUB_ENV
115+
116+
- name: Build Rust binaries
117+
run: |
118+
cd rust
119+
cargo build --release --target ${{ matrix.target }} --no-default-features
120+
121+
- name: Create release directory
122+
run: mkdir -p release
123+
124+
- name: Copy binaries (Linux)
125+
if: matrix.os == 'ubuntu-22.04'
126+
run: |
127+
cp rust/target/${{ matrix.target }}/release/openrqd release/openrqd-${{ env.BUILD_ID }}-${{ matrix.target }}
128+
129+
- name: Copy binaries (macOS)
130+
if: matrix.os == 'macos-13'
131+
run: |
132+
cp rust/target/${{ matrix.target }}/release/openrqd release/openrqd-${{ env.BUILD_ID }}-${{ matrix.target }}
133+
134+
- name: Upload binaries
135+
uses: actions/upload-artifact@v4
136+
with:
137+
name: rust-binaries-${{ matrix.target }}
138+
path: release/
139+
46140
upload_python_packages_test:
47141
needs: build_opencue_packages
48142
runs-on: ubuntu-22.04
@@ -133,7 +227,7 @@ jobs:
133227
# readme-filepath: ./${{ matrix.component }}/README.md
134228

135229
create_release:
136-
# needs: preflight
230+
needs: build_rust_binaries
137231
name: Create Release
138232
environment: dockerhub
139233
runs-on: ubuntu-22.04
@@ -159,6 +253,13 @@ jobs:
159253
echo "Build ID: $(ci/generate_version_number.py)"
160254
echo "BUILD_ID=$(ci/generate_version_number.py)" >> ${GITHUB_ENV}
161255
256+
- name: Download Rust binaries artifacts
257+
uses: actions/download-artifact@v4
258+
with:
259+
pattern: rust-binaries-*
260+
path: rust-binaries
261+
merge-multiple: true
262+
162263
- name: Fetch artifacts
163264
id: fetch_artifacts
164265
env:
@@ -320,3 +421,43 @@ jobs:
320421
asset_path: ${{ github.workspace }}/artifacts/cueman-${{ env.BUILD_ID }}-py2.py3-none-any.whl
321422
asset_name: cueman-${{ env.BUILD_ID }}-py2.py3-none-any.whl
322423
asset_content_type: application/octet-stream
424+
425+
- name: Upload openrqd (Linux GNU)
426+
uses: actions/upload-release-asset@v1
427+
env:
428+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
429+
with:
430+
upload_url: ${{ steps.create_release.outputs.upload_url }}
431+
asset_path: rust-binaries/openrqd-${{ env.BUILD_ID }}-x86_64-unknown-linux-gnu
432+
asset_name: openrqd-${{ env.BUILD_ID }}-x86_64-unknown-linux-gnu
433+
asset_content_type: application/octet-stream
434+
435+
- name: Upload openrqd (Linux MUSL)
436+
uses: actions/upload-release-asset@v1
437+
env:
438+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
439+
with:
440+
upload_url: ${{ steps.create_release.outputs.upload_url }}
441+
asset_path: rust-binaries/openrqd-${{ env.BUILD_ID }}-x86_64-unknown-linux-musl
442+
asset_name: openrqd-${{ env.BUILD_ID }}-x86_64-unknown-linux-musl
443+
asset_content_type: application/octet-stream
444+
445+
- name: Upload openrqd (macOS Intel)
446+
uses: actions/upload-release-asset@v1
447+
env:
448+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
449+
with:
450+
upload_url: ${{ steps.create_release.outputs.upload_url }}
451+
asset_path: rust-binaries/openrqd-${{ env.BUILD_ID }}-x86_64-apple-darwin
452+
asset_name: openrqd-${{ env.BUILD_ID }}-x86_64-apple-darwin
453+
asset_content_type: application/octet-stream
454+
455+
- name: Upload openrqd (macOS Apple Silicon)
456+
uses: actions/upload-release-asset@v1
457+
env:
458+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
459+
with:
460+
upload_url: ${{ steps.create_release.outputs.upload_url }}
461+
asset_path: rust-binaries/openrqd-${{ env.BUILD_ID }}-aarch64-apple-darwin
462+
asset_name: openrqd-${{ env.BUILD_ID }}-aarch64-apple-darwin
463+
asset_content_type: application/octet-stream

.github/workflows/rust.yml renamed to .github/workflows/rust-pipeline.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
name: Rust
1+
name: OpenCue Rust Build Pipeline
22

33
on:
44
push:

0 commit comments

Comments
 (0)