Skip to content

Commit e6a9155

Browse files
daanzudaanzu
committed
Implement CI caching of expensive native build
1 parent 734df98 commit e6a9155

File tree

4 files changed

+185
-21
lines changed

4 files changed

+185
-21
lines changed

‎.github/workflows/build.yml‎

Lines changed: 155 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,9 @@ jobs:
1515
build-linux:
1616
runs-on: ubuntu-latest
1717
if: true
18+
env:
19+
MKL_URL: ""
20+
# MKL_URL: "https://registrationcenter-download.intel.com/akdlm/irc_nas/tec/16917/l_mkl_2020.4.304.tgz"
1821
steps:
1922
- name: Checkout repository
2023
uses: actions/checkout@v5
@@ -26,29 +29,56 @@ jobs:
2629
export TAG=$(git tag --points-at HEAD)
2730
echo "TAG: $TAG"
2831
if [[ $TAG ]]; then
32+
echo "KALDI_BRANCH: kag-$TAG"
2933
echo "KALDI_BRANCH=kag-$TAG" >> $GITHUB_ENV
3034
echo "KALDI_BRANCH=kag-$TAG" >> $GITHUB_OUTPUT
3135
else
36+
echo "KALDI_BRANCH: ${GITHUB_REF/refs\/heads\//}"
3237
echo "KALDI_BRANCH=${GITHUB_REF/refs\/heads\//}" >> $GITHUB_ENV
3338
echo "KALDI_BRANCH=${GITHUB_REF/refs\/heads\//}" >> $GITHUB_OUTPUT
3439
fi
3540
41+
- name: Get Kaldi commit hash
42+
id: get-kaldi-commit
43+
run: |
44+
KALDI_COMMIT=$(git ls-remote https://github.com/daanzu/kaldi-fork-active-grammar.git $KALDI_BRANCH | cut -f1)
45+
echo "KALDI_COMMIT: $KALDI_COMMIT"
46+
echo "KALDI_COMMIT=$KALDI_COMMIT" >> $GITHUB_OUTPUT
47+
48+
- name: Restore cached native binaries
49+
id: cache-native-binaries-restore
50+
uses: actions/cache/restore@v4
51+
with:
52+
key: native-${{ runner.os }}-${{ steps.get-kaldi-commit.outputs.KALDI_COMMIT }}-${{ env.MKL_URL }}-v1
53+
path: kaldi_active_grammar/exec/linux
54+
3655
- name: Setup just
3756
run: |
3857
ls -al $HOME
3958
mkdir $HOME/bin
4059
curl --proto '=https' --tlsv1.2 -sSf https://just.systems/install.sh | bash -s -- --to $HOME/bin/
4160
echo "$HOME/bin" >> $GITHUB_PATH
4261
43-
- name: Build with dockcross
62+
- name: Build with dockcross (native binaries & python wheel)
4463
run: |
4564
echo "KALDI_BRANCH: $KALDI_BRANCH"
4665
echo "MKL_URL: $MKL_URL"
47-
# Example MKL_URL: https://registrationcenter-download.intel.com/akdlm/irc_nas/tec/16917/l_mkl_2020.4.304.tgz
48-
just build-dockcross $KALDI_BRANCH $MKL_URL
49-
# cp dist/* wheelhouse/
66+
just build-dockcross ${{ steps.cache-native-binaries-restore.outputs.cache-hit == 'true' && '--skip-native' || '' }} $KALDI_BRANCH $MKL_URL
5067
ls -al wheelhouse/
5168
69+
- name: Save cached native binaries
70+
if: steps.cache-native-binaries-restore.outputs.cache-hit != 'true'
71+
uses: actions/cache/save@v4
72+
with:
73+
key: ${{ steps.cache-native-binaries-restore.outputs.cache-primary-key }}
74+
path: kaldi_active_grammar/exec/linux
75+
76+
- name: Upload native binaries to artifacts
77+
uses: actions/upload-artifact@v4
78+
with:
79+
name: native-linux
80+
path: kaldi_active_grammar/exec/linux
81+
5282
- name: Upload Linux wheels
5383
uses: actions/upload-artifact@v4
5484
with:
@@ -74,33 +104,54 @@ jobs:
74104
path: main
75105

76106
- name: Get KALDI_BRANCH (kag-$TAG tag if commit is tagged; current branch name if not)
77-
id: get_kaldi_branch
107+
id: get-kaldi-branch
108+
working-directory: main
78109
run: |
79110
# Fetch tags on the one fetched commit (shallow clone)
80111
git fetch --depth=1 origin "+refs/tags/*:refs/tags/*"
81112
export TAG=$(git tag --points-at HEAD)
113+
echo "TAG: $TAG"
82114
if [[ $TAG ]]; then
115+
echo "KALDI_BRANCH: kag-$TAG"
83116
echo "KALDI_BRANCH=kag-$TAG" >> $GITHUB_ENV
84117
echo "KALDI_BRANCH=kag-$TAG" >> $GITHUB_OUTPUT
85118
else
119+
echo "KALDI_BRANCH: ${GITHUB_REF/refs\/heads\//}"
86120
echo "KALDI_BRANCH=${GITHUB_REF/refs\/heads\//}" >> $GITHUB_ENV
87121
echo "KALDI_BRANCH=${GITHUB_REF/refs\/heads\//}" >> $GITHUB_OUTPUT
88122
fi
89123
124+
- name: Get Kaldi commit hash
125+
id: get-kaldi-commit
126+
run: |
127+
KALDI_COMMIT=$(git ls-remote https://github.com/daanzu/kaldi-fork-active-grammar.git $KALDI_BRANCH | cut -f1)
128+
echo "KALDI_COMMIT: $KALDI_COMMIT"
129+
echo "KALDI_COMMIT=$KALDI_COMMIT" >> $GITHUB_OUTPUT
130+
131+
- name: Restore cached native binaries
132+
id: cache-native-binaries-restore
133+
uses: actions/cache/restore@v4
134+
with:
135+
key: native-${{ runner.os }}-${{ steps.get-kaldi-commit.outputs.KALDI_COMMIT }}-${{ env.VS_VERSION }}-${{ env.PLATFORM_TOOLSET }}-${{ env.WINDOWS_TARGET_PLATFORM_VERSION }}-${{ env.MKL_VERSION }}-v1
136+
path: main/kaldi_active_grammar/exec/windows
137+
90138
- name: Checkout OpenFST repository
139+
if: steps.cache-native-binaries-restore.outputs.cache-hit != 'true'
91140
uses: actions/checkout@v5
92141
with:
93142
repository: daanzu/openfst
94143
path: openfst
95144

96145
- name: Checkout Kaldi repository
146+
if: steps.cache-native-binaries-restore.outputs.cache-hit != 'true'
97147
uses: actions/checkout@v5
98148
with:
99149
repository: daanzu/kaldi-fork-active-grammar
100150
path: kaldi
101-
ref: ${{ steps.get_kaldi_branch.outputs.KALDI_BRANCH }}
151+
ref: ${{ steps.get-kaldi-branch.outputs.KALDI_BRANCH }}
102152

103153
- name: Gather system information
154+
if: steps.cache-native-binaries-restore.outputs.cache-hit != 'true'
104155
run: |
105156
echo $GITHUB_WORKSPACE
106157
df -h
@@ -118,6 +169,7 @@ jobs:
118169
vswhere -find 'VC\Redist\**\VC_redist.x64.exe'
119170
120171
- name: Setup Kaldi build configuration
172+
if: steps.cache-native-binaries-restore.outputs.cache-hit != 'true'
121173
run: |
122174
cd kaldi/windows
123175
cp kaldiwin_mkl.props kaldiwin.props
@@ -136,28 +188,49 @@ jobs:
136188
perl get_version.pl
137189
138190
- name: Add msbuild to PATH
191+
if: steps.cache-native-binaries-restore.outputs.cache-hit != 'true'
139192
uses: microsoft/setup-msbuild@v2
140193

141194
- name: Install MKL
195+
if: steps.cache-native-binaries-restore.outputs.cache-hit != 'true'
142196
run: winget install --id=Intel.oneMKL -v "${MKL_VERSION}" -e --accept-package-agreements --accept-source-agreements --disable-interactivity
143197

144198
- name: Build OpenFST
199+
if: steps.cache-native-binaries-restore.outputs.cache-hit != 'true'
145200
shell: cmd
146201
run: msbuild -t:Build -p:Configuration=Release -p:Platform=x64 -p:PlatformToolset=%PLATFORM_TOOLSET% -maxCpuCount -verbosity:minimal openfst/openfst.sln
147202

148203
- name: Build Kaldi
204+
if: steps.cache-native-binaries-restore.outputs.cache-hit != 'true'
149205
shell: cmd
150206
run: msbuild -t:Build -p:Configuration=Release -p:Platform=x64 -p:PlatformToolset=%PLATFORM_TOOLSET% -p:WindowsTargetPlatformVersion=%WINDOWS_TARGET_PLATFORM_VERSION% -maxCpuCount -verbosity:minimal kaldi/kaldiwin_%VS_VERSION%_MKL/kaldiwin/kaldi-dragonfly/kaldi-dragonfly.vcxproj
151207

208+
- name: Copy native binaries
209+
if: steps.cache-native-binaries-restore.outputs.cache-hit != 'true'
210+
run: |
211+
mkdir -p main/kaldi_active_grammar/exec/windows
212+
cp kaldi/kaldiwin_${VS_VERSION}_MKL/kaldiwin/kaldi-dragonfly/x64/Release/kaldi-dragonfly.dll main/kaldi_active_grammar/exec/windows/
213+
214+
- name: Save cached native binaries
215+
if: steps.cache-native-binaries-restore.outputs.cache-hit != 'true'
216+
uses: actions/cache/save@v4
217+
with:
218+
key: ${{ steps.cache-native-binaries-restore.outputs.cache-primary-key }}
219+
path: main/kaldi_active_grammar/exec/windows
220+
221+
- name: Upload native binaries to artifacts
222+
uses: actions/upload-artifact@v4
223+
with:
224+
name: native-windows
225+
path: main/kaldi_active_grammar/exec/windows
226+
152227
- name: Build Python wheel
228+
working-directory: main
153229
run: |
154-
cd main
155230
python -m pip -V
156231
python -m pip install --upgrade setuptools wheel scikit-build cmake ninja
157232
# ls -alR ../
158-
mkdir -p kaldi_active_grammar/exec/windows
159-
cp ../kaldi/kaldiwin_${VS_VERSION}_MKL/kaldiwin/kaldi-dragonfly/x64/Release/kaldi-dragonfly.dll kaldi_active_grammar/exec/windows/
160-
env KALDIAG_SETUP_RAW=1 python setup.py bdist_wheel
233+
env KALDIAG_BUILD_SKIP_NATIVE=1 python setup.py bdist_wheel
161234
ls -al dist/
162235
163236
- name: Upload Windows wheels
@@ -188,27 +261,46 @@ jobs:
188261
uses: actions/checkout@v5
189262

190263
- name: Get KALDI_BRANCH (kag-$TAG tag if commit is tagged; current branch name if not)
264+
id: get-kaldi-branch
191265
run: |
192266
# Fetch tags on the one fetched commit (shallow clone)
193267
git fetch --depth=1 origin "+refs/tags/*:refs/tags/*"
194268
export TAG=$(git tag --points-at HEAD)
269+
echo "TAG: $TAG"
195270
if [[ $TAG ]]; then
271+
echo "KALDI_BRANCH: kag-$TAG"
196272
echo "KALDI_BRANCH=kag-$TAG" >> $GITHUB_ENV
197273
echo "KALDI_BRANCH=kag-$TAG" >> $GITHUB_OUTPUT
198274
else
275+
echo "KALDI_BRANCH: ${GITHUB_REF/refs\/heads\//}"
199276
echo "KALDI_BRANCH=${GITHUB_REF/refs\/heads\//}" >> $GITHUB_ENV
200277
echo "KALDI_BRANCH=${GITHUB_REF/refs\/heads\//}" >> $GITHUB_OUTPUT
201278
fi
202279
280+
- name: Get Kaldi commit hash
281+
id: get-kaldi-commit
282+
run: |
283+
KALDI_COMMIT=$(git ls-remote https://github.com/daanzu/kaldi-fork-active-grammar.git $KALDI_BRANCH | cut -f1)
284+
echo "KALDI_COMMIT: $KALDI_COMMIT"
285+
echo "KALDI_COMMIT=$KALDI_COMMIT" >> $GITHUB_OUTPUT
286+
287+
- name: Restore cached native binaries
288+
id: cache-native-binaries-restore
289+
uses: actions/cache/restore@v4
290+
with:
291+
key: native-${{ runner.os }}-arm-${{ steps.get-kaldi-commit.outputs.KALDI_COMMIT }}-${{ env.MACOSX_DEPLOYMENT_TARGET }}-${{ env.MKL_URL }}-v1
292+
path: kaldi_active_grammar/exec/macos
293+
203294
- name: Install dependencies
295+
if: steps.cache-native-binaries-restore.outputs.cache-hit != 'true'
204296
run: |
205297
python3 -m pip install --break-system-packages --user --upgrade setuptools wheel scikit-build cmake ninja
206298
brew install automake sox libtool
207299
brew reinstall gfortran # For openblas
208300
# brew install autoconf
209301
210302
- name: Install MKL (if enabled)
211-
if: ${{ env.MKL_URL != '' }}
303+
if: ${{ env.MKL_URL != '' && steps.cache-native-binaries-restore.outputs.cache-hit != 'true' }}
212304
run: |
213305
echo "Installing MKL from: $MKL_URL"
214306
export MKL_FILE=${MKL_URL##*/}
@@ -222,9 +314,23 @@ jobs:
222314
- name: Build Python wheel
223315
run: |
224316
echo "KALDI_BRANCH: $KALDI_BRANCH"
225-
python3 setup.py bdist_wheel
317+
echo "MKL_URL: $MKL_URL"
318+
${{ steps.cache-native-binaries-restore.outputs.cache-hit == 'true' && 'KALDIAG_BUILD_SKIP_NATIVE=1' || '' }} python3 setup.py bdist_wheel
226319
ls -al dist/
227320
321+
- name: Save cached native binaries
322+
if: steps.cache-native-binaries-restore.outputs.cache-hit != 'true'
323+
uses: actions/cache/save@v4
324+
with:
325+
key: ${{ steps.cache-native-binaries-restore.outputs.cache-primary-key }}
326+
path: kaldi_active_grammar/exec/macos
327+
328+
- name: Upload native binaries to artifacts
329+
uses: actions/upload-artifact@v4
330+
with:
331+
name: native-macos-arm
332+
path: kaldi_active_grammar/exec/macos
333+
228334
- name: Upload MacOS ARM wheels
229335
uses: actions/upload-artifact@v4
230336
with:
@@ -243,27 +349,46 @@ jobs:
243349
uses: actions/checkout@v5
244350

245351
- name: Get KALDI_BRANCH (kag-$TAG tag if commit is tagged; current branch name if not)
352+
id: get-kaldi-branch
246353
run: |
247354
# Fetch tags on the one fetched commit (shallow clone)
248355
git fetch --depth=1 origin "+refs/tags/*:refs/tags/*"
249356
export TAG=$(git tag --points-at HEAD)
357+
echo "TAG: $TAG"
250358
if [[ $TAG ]]; then
359+
echo "KALDI_BRANCH: kag-$TAG"
251360
echo "KALDI_BRANCH=kag-$TAG" >> $GITHUB_ENV
252361
echo "KALDI_BRANCH=kag-$TAG" >> $GITHUB_OUTPUT
253362
else
363+
echo "KALDI_BRANCH: ${GITHUB_REF/refs\/heads\//}"
254364
echo "KALDI_BRANCH=${GITHUB_REF/refs\/heads\//}" >> $GITHUB_ENV
255365
echo "KALDI_BRANCH=${GITHUB_REF/refs\/heads\//}" >> $GITHUB_OUTPUT
256366
fi
257367
368+
- name: Get Kaldi commit hash
369+
id: get-kaldi-commit
370+
run: |
371+
KALDI_COMMIT=$(git ls-remote https://github.com/daanzu/kaldi-fork-active-grammar.git $KALDI_BRANCH | cut -f1)
372+
echo "KALDI_COMMIT: $KALDI_COMMIT"
373+
echo "KALDI_COMMIT=$KALDI_COMMIT" >> $GITHUB_OUTPUT
374+
375+
- name: Restore cached native binaries
376+
id: cache-native-binaries-restore
377+
uses: actions/cache/restore@v4
378+
with:
379+
key: native-${{ runner.os }}-intel-${{ steps.get-kaldi-commit.outputs.KALDI_COMMIT }}-${{ env.MACOSX_DEPLOYMENT_TARGET }}-${{ env.MKL_URL }}-v1
380+
path: kaldi_active_grammar/exec/macos
381+
258382
- name: Install dependencies
383+
if: steps.cache-native-binaries-restore.outputs.cache-hit != 'true'
259384
run: |
260385
python3 -m pip install --break-system-packages --user --upgrade setuptools wheel scikit-build cmake ninja
261386
brew install automake sox
262387
brew reinstall gfortran # For openblas
263388
# brew install autoconf libtool
264389
265390
- name: Install MKL (if enabled)
266-
if: ${{ env.MKL_URL != '' }}
391+
if: ${{ env.MKL_URL != '' && steps.cache-native-binaries-restore.outputs.cache-hit != 'true' }}
267392
run: |
268393
echo "Installing MKL from: $MKL_URL"
269394
export MKL_FILE=${MKL_URL##*/}
@@ -276,14 +401,29 @@ jobs:
276401
277402
- name: Build Python wheel
278403
run: |
279-
python3 setup.py bdist_wheel
404+
echo "KALDI_BRANCH: $KALDI_BRANCH"
405+
echo "MKL_URL: $MKL_URL"
406+
${{ steps.cache-native-binaries-restore.outputs.cache-hit == 'true' && 'KALDIAG_BUILD_SKIP_NATIVE=1' || '' }} python3 setup.py bdist_wheel
280407
ls -al dist/
281408
409+
- name: Save cached native binaries
410+
if: steps.cache-native-binaries-restore.outputs.cache-hit != 'true'
411+
uses: actions/cache/save@v4
412+
with:
413+
key: ${{ steps.cache-native-binaries-restore.outputs.cache-primary-key }}
414+
path: kaldi_active_grammar/exec/macos
415+
416+
- name: Upload native binaries to artifacts
417+
uses: actions/upload-artifact@v4
418+
with:
419+
name: native-macos-intel
420+
path: kaldi_active_grammar/exec/macos
421+
282422
- name: Upload MacOS Intel wheels
283423
uses: actions/upload-artifact@v4
284424
with:
285425
name: wheels-macos-intel
286-
path: dist/*
426+
path: dist/*.whl
287427

288428
merge-wheels:
289429
runs-on: ubuntu-latest

‎Justfile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ setup-dockcross:
2323
@# [ ! -e building/dockcross-manylinux2010-x64 ] && docker run --rm dockcross/manylinux2010-x64 > building/dockcross-manylinux2010-x64 && chmod +x building/dockcross-manylinux2010-x64 || true
2424

2525
pip-install-develop:
26-
KALDIAG_SETUP_RAW=1 pip3 install --user -e .
26+
KALDIAG_BUILD_SKIP_NATIVE=1 pip3 install --user -e .
2727

2828
# Setup an editable development environment on linux
2929
setup-linux-develop kaldi_root_dir:

0 commit comments

Comments
 (0)