Skip to content

Commit 682c101

Browse files
daanzudaanzu
committed
Implement CI caching of expensive native build
1 parent 734df98 commit 682c101

File tree

4 files changed

+203
-31
lines changed

4 files changed

+203
-31
lines changed

‎.github/workflows/build.yml‎

Lines changed: 173 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,9 @@ jobs:
1515
build-linux:
1616
runs-on: ubuntu-latest
1717
if: true
18+
env:
19+
MKL_URL: ""
20+
# MKL_URL: "https://registrationcenter-download.intel.com/akdlm/irc_nas/tec/16917/l_mkl_2020.4.304.tgz"
1821
steps:
1922
- name: Checkout repository
2023
uses: actions/checkout@v5
@@ -26,29 +29,56 @@ jobs:
2629
export TAG=$(git tag --points-at HEAD)
2730
echo "TAG: $TAG"
2831
if [[ $TAG ]]; then
32+
echo "KALDI_BRANCH: kag-$TAG"
2933
echo "KALDI_BRANCH=kag-$TAG" >> $GITHUB_ENV
3034
echo "KALDI_BRANCH=kag-$TAG" >> $GITHUB_OUTPUT
3135
else
36+
echo "KALDI_BRANCH: ${GITHUB_REF/refs\/heads\//}"
3237
echo "KALDI_BRANCH=${GITHUB_REF/refs\/heads\//}" >> $GITHUB_ENV
3338
echo "KALDI_BRANCH=${GITHUB_REF/refs\/heads\//}" >> $GITHUB_OUTPUT
3439
fi
3540
41+
- name: Get Kaldi commit hash
42+
id: get-kaldi-commit
43+
run: |
44+
KALDI_COMMIT=$(git ls-remote https://github.com/daanzu/kaldi-fork-active-grammar.git $KALDI_BRANCH | cut -f1)
45+
echo "KALDI_COMMIT: $KALDI_COMMIT"
46+
echo "KALDI_COMMIT=$KALDI_COMMIT" >> $GITHUB_OUTPUT
47+
48+
- name: Restore cached native binaries
49+
id: cache-native-binaries-restore
50+
uses: actions/cache/restore@v4
51+
with:
52+
key: native-${{ runner.os }}-${{ steps.get-kaldi-commit.outputs.KALDI_COMMIT }}-${{ env.MKL_URL }}-v1
53+
path: kaldi_active_grammar/exec/linux
54+
3655
- name: Setup just
3756
run: |
3857
ls -al $HOME
3958
mkdir $HOME/bin
4059
curl --proto '=https' --tlsv1.2 -sSf https://just.systems/install.sh | bash -s -- --to $HOME/bin/
4160
echo "$HOME/bin" >> $GITHUB_PATH
4261
43-
- name: Build with dockcross
62+
- name: Build with dockcross (native binaries & python wheel)
4463
run: |
4564
echo "KALDI_BRANCH: $KALDI_BRANCH"
4665
echo "MKL_URL: $MKL_URL"
47-
# Example MKL_URL: https://registrationcenter-download.intel.com/akdlm/irc_nas/tec/16917/l_mkl_2020.4.304.tgz
48-
just build-dockcross $KALDI_BRANCH $MKL_URL
49-
# cp dist/* wheelhouse/
66+
just build-dockcross ${{ steps.cache-native-binaries-restore.outputs.cache-hit == 'true' && '--skip-native' || '' }} $KALDI_BRANCH $MKL_URL
5067
ls -al wheelhouse/
5168
69+
- name: Save cached native binaries
70+
if: steps.cache-native-binaries-restore.outputs.cache-hit != 'true'
71+
uses: actions/cache/save@v4
72+
with:
73+
key: ${{ steps.cache-native-binaries-restore.outputs.cache-primary-key }}
74+
path: kaldi_active_grammar/exec/linux
75+
76+
- name: Upload native binaries to artifacts
77+
uses: actions/upload-artifact@v4
78+
with:
79+
name: native-linux
80+
path: kaldi_active_grammar/exec/linux
81+
5282
- name: Upload Linux wheels
5383
uses: actions/upload-artifact@v4
5484
with:
@@ -74,33 +104,54 @@ jobs:
74104
path: main
75105

76106
- name: Get KALDI_BRANCH (kag-$TAG tag if commit is tagged; current branch name if not)
77-
id: get_kaldi_branch
107+
id: get-kaldi-branch
108+
working-directory: main
78109
run: |
79110
# Fetch tags on the one fetched commit (shallow clone)
80111
git fetch --depth=1 origin "+refs/tags/*:refs/tags/*"
81112
export TAG=$(git tag --points-at HEAD)
113+
echo "TAG: $TAG"
82114
if [[ $TAG ]]; then
115+
echo "KALDI_BRANCH: kag-$TAG"
83116
echo "KALDI_BRANCH=kag-$TAG" >> $GITHUB_ENV
84117
echo "KALDI_BRANCH=kag-$TAG" >> $GITHUB_OUTPUT
85118
else
119+
echo "KALDI_BRANCH: ${GITHUB_REF/refs\/heads\//}"
86120
echo "KALDI_BRANCH=${GITHUB_REF/refs\/heads\//}" >> $GITHUB_ENV
87121
echo "KALDI_BRANCH=${GITHUB_REF/refs\/heads\//}" >> $GITHUB_OUTPUT
88122
fi
89123
124+
- name: Get Kaldi commit hash
125+
id: get-kaldi-commit
126+
run: |
127+
KALDI_COMMIT=$(git ls-remote https://github.com/daanzu/kaldi-fork-active-grammar.git $KALDI_BRANCH | cut -f1)
128+
echo "KALDI_COMMIT: $KALDI_COMMIT"
129+
echo "KALDI_COMMIT=$KALDI_COMMIT" >> $GITHUB_OUTPUT
130+
131+
- name: Restore cached native binaries
132+
id: cache-native-binaries-restore
133+
uses: actions/cache/restore@v4
134+
with:
135+
key: native-${{ runner.os }}-${{ steps.get-kaldi-commit.outputs.KALDI_COMMIT }}-${{ env.VS_VERSION }}-${{ env.PLATFORM_TOOLSET }}-${{ env.WINDOWS_TARGET_PLATFORM_VERSION }}-${{ env.MKL_VERSION }}-v1
136+
path: main/kaldi_active_grammar/exec/windows
137+
90138
- name: Checkout OpenFST repository
139+
if: steps.cache-native-binaries-restore.outputs.cache-hit != 'true'
91140
uses: actions/checkout@v5
92141
with:
93142
repository: daanzu/openfst
94143
path: openfst
95144

96145
- name: Checkout Kaldi repository
146+
if: steps.cache-native-binaries-restore.outputs.cache-hit != 'true'
97147
uses: actions/checkout@v5
98148
with:
99149
repository: daanzu/kaldi-fork-active-grammar
100150
path: kaldi
101-
ref: ${{ steps.get_kaldi_branch.outputs.KALDI_BRANCH }}
151+
ref: ${{ steps.get-kaldi-branch.outputs.KALDI_BRANCH }}
102152

103153
- name: Gather system information
154+
if: steps.cache-native-binaries-restore.outputs.cache-hit != 'true'
104155
run: |
105156
echo $GITHUB_WORKSPACE
106157
df -h
@@ -118,6 +169,7 @@ jobs:
118169
vswhere -find 'VC\Redist\**\VC_redist.x64.exe'
119170
120171
- name: Setup Kaldi build configuration
172+
if: steps.cache-native-binaries-restore.outputs.cache-hit != 'true'
121173
run: |
122174
cd kaldi/windows
123175
cp kaldiwin_mkl.props kaldiwin.props
@@ -136,28 +188,49 @@ jobs:
136188
perl get_version.pl
137189
138190
- name: Add msbuild to PATH
191+
if: steps.cache-native-binaries-restore.outputs.cache-hit != 'true'
139192
uses: microsoft/setup-msbuild@v2
140193

141194
- name: Install MKL
195+
if: steps.cache-native-binaries-restore.outputs.cache-hit != 'true'
142196
run: winget install --id=Intel.oneMKL -v "${MKL_VERSION}" -e --accept-package-agreements --accept-source-agreements --disable-interactivity
143197

144198
- name: Build OpenFST
199+
if: steps.cache-native-binaries-restore.outputs.cache-hit != 'true'
145200
shell: cmd
146201
run: msbuild -t:Build -p:Configuration=Release -p:Platform=x64 -p:PlatformToolset=%PLATFORM_TOOLSET% -maxCpuCount -verbosity:minimal openfst/openfst.sln
147202

148203
- name: Build Kaldi
204+
if: steps.cache-native-binaries-restore.outputs.cache-hit != 'true'
149205
shell: cmd
150206
run: msbuild -t:Build -p:Configuration=Release -p:Platform=x64 -p:PlatformToolset=%PLATFORM_TOOLSET% -p:WindowsTargetPlatformVersion=%WINDOWS_TARGET_PLATFORM_VERSION% -maxCpuCount -verbosity:minimal kaldi/kaldiwin_%VS_VERSION%_MKL/kaldiwin/kaldi-dragonfly/kaldi-dragonfly.vcxproj
151207

208+
- name: Copy native binaries
209+
if: steps.cache-native-binaries-restore.outputs.cache-hit != 'true'
210+
run: |
211+
mkdir -p main/kaldi_active_grammar/exec/windows
212+
cp kaldi/kaldiwin_${VS_VERSION}_MKL/kaldiwin/kaldi-dragonfly/x64/Release/kaldi-dragonfly.dll main/kaldi_active_grammar/exec/windows/
213+
214+
- name: Save cached native binaries
215+
if: steps.cache-native-binaries-restore.outputs.cache-hit != 'true'
216+
uses: actions/cache/save@v4
217+
with:
218+
key: ${{ steps.cache-native-binaries-restore.outputs.cache-primary-key }}
219+
path: main/kaldi_active_grammar/exec/windows
220+
221+
- name: Upload native binaries to artifacts
222+
uses: actions/upload-artifact@v4
223+
with:
224+
name: native-windows
225+
path: main/kaldi_active_grammar/exec/windows
226+
152227
- name: Build Python wheel
228+
working-directory: main
153229
run: |
154-
cd main
155230
python -m pip -V
156231
python -m pip install --upgrade setuptools wheel scikit-build cmake ninja
157232
# ls -alR ../
158-
mkdir -p kaldi_active_grammar/exec/windows
159-
cp ../kaldi/kaldiwin_${VS_VERSION}_MKL/kaldiwin/kaldi-dragonfly/x64/Release/kaldi-dragonfly.dll kaldi_active_grammar/exec/windows/
160-
env KALDIAG_SETUP_RAW=1 python setup.py bdist_wheel
233+
env KALDIAG_BUILD_SKIP_NATIVE=1 python setup.py bdist_wheel
161234
ls -al dist/
162235
163236
- name: Upload Windows wheels
@@ -188,27 +261,38 @@ jobs:
188261
uses: actions/checkout@v5
189262

190263
- name: Get KALDI_BRANCH (kag-$TAG tag if commit is tagged; current branch name if not)
264+
id: get-kaldi-branch
191265
run: |
192266
# Fetch tags on the one fetched commit (shallow clone)
193267
git fetch --depth=1 origin "+refs/tags/*:refs/tags/*"
194268
export TAG=$(git tag --points-at HEAD)
269+
echo "TAG: $TAG"
195270
if [[ $TAG ]]; then
271+
echo "KALDI_BRANCH: kag-$TAG"
196272
echo "KALDI_BRANCH=kag-$TAG" >> $GITHUB_ENV
197273
echo "KALDI_BRANCH=kag-$TAG" >> $GITHUB_OUTPUT
198274
else
275+
echo "KALDI_BRANCH: ${GITHUB_REF/refs\/heads\//}"
199276
echo "KALDI_BRANCH=${GITHUB_REF/refs\/heads\//}" >> $GITHUB_ENV
200277
echo "KALDI_BRANCH=${GITHUB_REF/refs\/heads\//}" >> $GITHUB_OUTPUT
201278
fi
202279
203-
- name: Install dependencies
280+
- name: Get Kaldi commit hash
281+
id: get-kaldi-commit
204282
run: |
205-
python3 -m pip install --break-system-packages --user --upgrade setuptools wheel scikit-build cmake ninja
206-
brew install automake sox libtool
207-
brew reinstall gfortran # For openblas
208-
# brew install autoconf
283+
KALDI_COMMIT=$(git ls-remote https://github.com/daanzu/kaldi-fork-active-grammar.git $KALDI_BRANCH | cut -f1)
284+
echo "KALDI_COMMIT: $KALDI_COMMIT"
285+
echo "KALDI_COMMIT=$KALDI_COMMIT" >> $GITHUB_OUTPUT
286+
287+
- name: Restore cached native binaries
288+
id: cache-native-binaries-restore
289+
uses: actions/cache/restore@v4
290+
with:
291+
key: native-${{ runner.os }}-arm-${{ steps.get-kaldi-commit.outputs.KALDI_COMMIT }}-${{ env.MACOSX_DEPLOYMENT_TARGET }}-${{ env.MKL_URL }}-v1
292+
path: kaldi_active_grammar/exec/macos
209293

210294
- name: Install MKL (if enabled)
211-
if: ${{ env.MKL_URL != '' }}
295+
if: ${{ env.MKL_URL != '' && steps.cache-native-binaries-restore.outputs.cache-hit != 'true' }}
212296
run: |
213297
echo "Installing MKL from: $MKL_URL"
214298
export MKL_FILE=${MKL_URL##*/}
@@ -219,12 +303,38 @@ jobs:
219303
sed -i.bak -e 's/decline/accept/g' silent.cfg
220304
sudo /Volumes/${MKL_FILE}/${MKL_FILE}.app/Contents/MacOS/install.sh --silent silent.cfg
221305
306+
- name: Install dependencies for native build
307+
if: steps.cache-native-binaries-restore.outputs.cache-hit != 'true'
308+
run: |
309+
python3 -m pip install --break-system-packages --user --upgrade scikit-build cmake ninja
310+
brew install automake sox libtool
311+
brew reinstall gfortran # For openblas
312+
# brew install autoconf
313+
314+
- name: Install dependencies for python build
315+
run: |
316+
python3 -m pip install --break-system-packages --user --upgrade setuptools wheel
317+
222318
- name: Build Python wheel
223319
run: |
224320
echo "KALDI_BRANCH: $KALDI_BRANCH"
225-
python3 setup.py bdist_wheel
321+
echo "MKL_URL: $MKL_URL"
322+
${{ steps.cache-native-binaries-restore.outputs.cache-hit == 'true' && 'KALDIAG_BUILD_SKIP_NATIVE=1' || '' }} python3 setup.py bdist_wheel
226323
ls -al dist/
227324
325+
- name: Save cached native binaries
326+
if: steps.cache-native-binaries-restore.outputs.cache-hit != 'true'
327+
uses: actions/cache/save@v4
328+
with:
329+
key: ${{ steps.cache-native-binaries-restore.outputs.cache-primary-key }}
330+
path: kaldi_active_grammar/exec/macos
331+
332+
- name: Upload native binaries to artifacts
333+
uses: actions/upload-artifact@v4
334+
with:
335+
name: native-macos-arm
336+
path: kaldi_active_grammar/exec/macos
337+
228338
- name: Upload MacOS ARM wheels
229339
uses: actions/upload-artifact@v4
230340
with:
@@ -243,27 +353,38 @@ jobs:
243353
uses: actions/checkout@v5
244354

245355
- name: Get KALDI_BRANCH (kag-$TAG tag if commit is tagged; current branch name if not)
356+
id: get-kaldi-branch
246357
run: |
247358
# Fetch tags on the one fetched commit (shallow clone)
248359
git fetch --depth=1 origin "+refs/tags/*:refs/tags/*"
249360
export TAG=$(git tag --points-at HEAD)
361+
echo "TAG: $TAG"
250362
if [[ $TAG ]]; then
363+
echo "KALDI_BRANCH: kag-$TAG"
251364
echo "KALDI_BRANCH=kag-$TAG" >> $GITHUB_ENV
252365
echo "KALDI_BRANCH=kag-$TAG" >> $GITHUB_OUTPUT
253366
else
367+
echo "KALDI_BRANCH: ${GITHUB_REF/refs\/heads\//}"
254368
echo "KALDI_BRANCH=${GITHUB_REF/refs\/heads\//}" >> $GITHUB_ENV
255369
echo "KALDI_BRANCH=${GITHUB_REF/refs\/heads\//}" >> $GITHUB_OUTPUT
256370
fi
257371
258-
- name: Install dependencies
372+
- name: Get Kaldi commit hash
373+
id: get-kaldi-commit
259374
run: |
260-
python3 -m pip install --break-system-packages --user --upgrade setuptools wheel scikit-build cmake ninja
261-
brew install automake sox
262-
brew reinstall gfortran # For openblas
263-
# brew install autoconf libtool
375+
KALDI_COMMIT=$(git ls-remote https://github.com/daanzu/kaldi-fork-active-grammar.git $KALDI_BRANCH | cut -f1)
376+
echo "KALDI_COMMIT: $KALDI_COMMIT"
377+
echo "KALDI_COMMIT=$KALDI_COMMIT" >> $GITHUB_OUTPUT
378+
379+
- name: Restore cached native binaries
380+
id: cache-native-binaries-restore
381+
uses: actions/cache/restore@v4
382+
with:
383+
key: native-${{ runner.os }}-intel-${{ steps.get-kaldi-commit.outputs.KALDI_COMMIT }}-${{ env.MACOSX_DEPLOYMENT_TARGET }}-${{ env.MKL_URL }}-v1
384+
path: kaldi_active_grammar/exec/macos
264385

265386
- name: Install MKL (if enabled)
266-
if: ${{ env.MKL_URL != '' }}
387+
if: ${{ env.MKL_URL != '' && steps.cache-native-binaries-restore.outputs.cache-hit != 'true' }}
267388
run: |
268389
echo "Installing MKL from: $MKL_URL"
269390
export MKL_FILE=${MKL_URL##*/}
@@ -274,16 +395,43 @@ jobs:
274395
sed -i.bak -e 's/decline/accept/g' silent.cfg
275396
sudo /Volumes/${MKL_FILE}/${MKL_FILE}.app/Contents/MacOS/install.sh --silent silent.cfg
276397
398+
- name: Install dependencies for native build
399+
if: steps.cache-native-binaries-restore.outputs.cache-hit != 'true'
400+
run: |
401+
python3 -m pip install --break-system-packages --user --upgrade scikit-build cmake ninja
402+
brew install automake sox
403+
brew reinstall gfortran # For openblas
404+
# brew install autoconf libtool
405+
406+
- name: Install dependencies for python build
407+
run: |
408+
python3 -m pip install --break-system-packages --user --upgrade setuptools wheel
409+
277410
- name: Build Python wheel
278411
run: |
279-
python3 setup.py bdist_wheel
412+
echo "KALDI_BRANCH: $KALDI_BRANCH"
413+
echo "MKL_URL: $MKL_URL"
414+
${{ steps.cache-native-binaries-restore.outputs.cache-hit == 'true' && 'KALDIAG_BUILD_SKIP_NATIVE=1' || '' }} python3 setup.py bdist_wheel
280415
ls -al dist/
281416
417+
- name: Save cached native binaries
418+
if: steps.cache-native-binaries-restore.outputs.cache-hit != 'true'
419+
uses: actions/cache/save@v4
420+
with:
421+
key: ${{ steps.cache-native-binaries-restore.outputs.cache-primary-key }}
422+
path: kaldi_active_grammar/exec/macos
423+
424+
- name: Upload native binaries to artifacts
425+
uses: actions/upload-artifact@v4
426+
with:
427+
name: native-macos-intel
428+
path: kaldi_active_grammar/exec/macos
429+
282430
- name: Upload MacOS Intel wheels
283431
uses: actions/upload-artifact@v4
284432
with:
285433
name: wheels-macos-intel
286-
path: dist/*
434+
path: dist/*.whl
287435

288436
merge-wheels:
289437
runs-on: ubuntu-latest

‎Justfile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ setup-dockcross:
2323
@# [ ! -e building/dockcross-manylinux2010-x64 ] && docker run --rm dockcross/manylinux2010-x64 > building/dockcross-manylinux2010-x64 && chmod +x building/dockcross-manylinux2010-x64 || true
2424

2525
pip-install-develop:
26-
KALDIAG_SETUP_RAW=1 pip3 install --user -e .
26+
KALDIAG_BUILD_SKIP_NATIVE=1 pip3 install --user -e .
2727

2828
# Setup an editable development environment on linux
2929
setup-linux-develop kaldi_root_dir:

0 commit comments

Comments
 (0)