Skip to content

Commit e3d61f2

Browse files
PastaPastaPastaPastaPastaPasta
committed
Merge dashpay#7029: ci: better depends caching
6f7f799 fix: include depends/Makefile in sparse checkout (pasta) fd070cc fix: add depends/Makefile to PACKAGES hash (pasta) 36c143b fix: include ci-slim.Dockerfile (pasta) 5c10d85 ci: improve depends caching to be more compact, and bypass quickly if we have a cache hit (pasta) Pull request description: ## Issue being fixed or feature implemented depends on [another](dashpay#7027) ## What was done? ## How Has This Been Tested? ## Breaking Changes ## Checklist: _Go over all the following points, and put an `x` in all the boxes that apply._ - [ ] I have performed a self-review of my own code - [ ] I have commented my code, particularly in hard-to-understand areas - [ ] I have added or updated relevant unit/integration/functional/e2e tests - [ ] I have made corresponding changes to the documentation - [x] I have assigned this pull request to a milestone _(for repository code-owners and collaborators only)_ ACKs for top commit: UdjinM6: utACK 6f7f799 Tree-SHA512: b8018aa17d43dc426cba4403ef63800c3af49f2ced0fb0b5be89b0dfb153c337af93a00b55dd1be091dd40efa61513649b15fc2a7b506e0857456273dc4f3708
2 parents a1c3afb + 6f7f799 commit e3d61f2

File tree

3 files changed

+98
-36
lines changed

3 files changed

+98
-36
lines changed

‎.github/workflows/build-depends.yml‎

Lines changed: 61 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -14,80 +14,108 @@ on:
1414
outputs:
1515
key:
1616
description: "Key needed for restoring depends cache"
17-
value: ${{ jobs.build-depends.outputs.key }}
17+
value: ${{ jobs.check-cache.outputs.cache-key }}
18+
host:
19+
description: "Host triplet for this build target"
20+
value: ${{ jobs.check-cache.outputs.host }}
21+
dep-opts:
22+
description: "DEP_OPTS used to build depends"
23+
value: ${{ jobs.check-cache.outputs.dep-opts }}
1824

1925
jobs:
20-
build-depends:
21-
name: Build depends
22-
runs-on: ubuntu-24.04
26+
check-cache:
27+
name: Check cache
28+
runs-on: ubuntu-latest
2329
outputs:
24-
key: ${{ steps.restore.outputs.cache-primary-key }}
25-
container:
26-
image: ${{ inputs.container-path }}
27-
options: --user root
30+
cache-hit: ${{ steps.cache-check.outputs.cache-hit }}
31+
cache-key: ${{ steps.setup.outputs.cache-key }}
32+
host: ${{ steps.setup.outputs.HOST }}
33+
dep-opts: ${{ steps.setup.outputs.DEP_OPTS }}
2834
steps:
2935
- name: Checkout code
3036
uses: actions/checkout@v4
3137
with:
3238
ref: ${{ github.event.pull_request.head.sha }}
39+
sparse-checkout: |
40+
ci/dash
41+
ci/test
42+
depends/Makefile
43+
depends/packages
44+
depends/hosts
45+
contrib/containers/ci/ci.Dockerfile
46+
contrib/containers/ci/ci-slim.Dockerfile
3347
34-
- name: Initial setup
48+
- name: Compute cache key
3549
id: setup
3650
run: |
3751
BUILD_TARGET="${{ inputs.build-target }}"
3852
source ./ci/dash/matrix.sh
3953
echo "DEP_OPTS=${DEP_OPTS}" >> "${GITHUB_OUTPUT}"
4054
echo "HOST=${HOST}" >> "${GITHUB_OUTPUT}"
4155
DEP_HASH="$(echo -n "${BUILD_TARGET}" "${DEP_OPTS}" "${HOST}" | sha256sum | head -c 64)"
42-
echo "\"${BUILD_TARGET}\" has HOST=\"${HOST}\" and DEP_OPTS=\"${DEP_OPTS}\" with hash \"${DEP_HASH}\""
4356
echo "DEP_HASH=${DEP_HASH}" >> "${GITHUB_OUTPUT}"
44-
57+
DOCKERFILE_HASH="${{ hashFiles('contrib/containers/ci/ci.Dockerfile', 'contrib/containers/ci/ci-slim.Dockerfile') }}"
58+
PACKAGES_HASH="${{ hashFiles('depends/packages/*', 'depends/Makefile') }}"
59+
CACHE_KEY="depends-${DOCKERFILE_HASH}-${{ inputs.build-target }}-${DEP_HASH}-${PACKAGES_HASH}"
60+
echo "cache-key=${CACHE_KEY}" >> "${GITHUB_OUTPUT}"
61+
echo "Cache key: ${CACHE_KEY}"
4562
shell: bash
4663

64+
- name: Check for cached depends
65+
id: cache-check
66+
uses: actions/cache@v4
67+
with:
68+
path: depends/built/${{ steps.setup.outputs.HOST }}
69+
key: ${{ steps.setup.outputs.cache-key }}
70+
lookup-only: true
71+
72+
build:
73+
name: Build depends
74+
needs: [check-cache]
75+
if: needs.check-cache.outputs.cache-hit != 'true'
76+
runs-on: ubuntu-24.04
77+
container:
78+
image: ${{ inputs.container-path }}
79+
options: --user root
80+
steps:
81+
- name: Checkout code
82+
uses: actions/checkout@v4
83+
with:
84+
ref: ${{ github.event.pull_request.head.sha }}
85+
4786
- name: Restore depends sources
4887
uses: actions/cache/restore@v4
4988
with:
50-
path: |
51-
depends/sources
89+
path: depends/sources
5290
key: depends-sources-${{ hashFiles('depends/packages/*') }}
53-
restore-keys: |
54-
depends-sources-
91+
restore-keys: depends-sources-
5592

5693
- name: Cache SDKs
5794
uses: actions/cache@v4
5895
if: inputs.build-target == 'mac'
5996
with:
60-
path: |
61-
depends/SDKs
97+
path: depends/SDKs
6298
key: depends-sdks-${{ hashFiles('depends/hosts/darwin.mk') }}
63-
restore-keys: |
64-
depends-sdks-
99+
restore-keys: depends-sdks-
65100

66101
- name: Restore cached depends
67102
uses: actions/cache/restore@v4
68-
id: restore
69103
with:
70-
path: |
71-
depends/built
72-
depends/${{ steps.setup.outputs.HOST }}
73-
key: depends-${{ hashFiles('contrib/containers/ci/ci.Dockerfile') }}-${{ inputs.build-target }}-${{ steps.setup.outputs.DEP_HASH }}-${{ hashFiles('depends/packages/*') }}
104+
path: depends/built/${{ needs.check-cache.outputs.host }}
105+
key: ${{ needs.check-cache.outputs.cache-key }}
74106
restore-keys: |
75-
depends-${{ hashFiles('contrib/containers/ci/ci.Dockerfile') }}-${{ inputs.build-target }}-${{ steps.setup.outputs.DEP_HASH }}-
76-
depends-${{ hashFiles('contrib/containers/ci/ci.Dockerfile') }}-${{ inputs.build-target }}-
107+
depends-${{ hashFiles('contrib/containers/ci/ci.Dockerfile', 'contrib/containers/ci/ci-slim.Dockerfile') }}-${{ inputs.build-target }}-
77108
78109
- name: Build depends
79110
run: |
80-
export HOST="${{ steps.setup.outputs.HOST }}"
111+
export HOST="${{ needs.check-cache.outputs.host }}"
81112
if [ "${HOST}" = "x86_64-apple-darwin" ]; then
82113
./contrib/containers/guix/scripts/setup-sdk
83114
fi
84-
env ${{ steps.setup.outputs.DEP_OPTS }} make -j$(nproc) -C depends
115+
env ${{ needs.check-cache.outputs.dep-opts }} make -j$(nproc) -C depends
85116
86117
- name: Save depends cache
87118
uses: actions/cache/save@v4
88-
if: steps.restore.outputs.cache-hit != 'true'
89119
with:
90-
path: |
91-
depends/built
92-
depends/${{ steps.setup.outputs.HOST }}
93-
key: ${{ steps.restore.outputs.cache-primary-key }}
120+
path: depends/built/${{ needs.check-cache.outputs.host }}
121+
key: ${{ needs.check-cache.outputs.cache-key }}

‎.github/workflows/build-src.yml‎

Lines changed: 17 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,15 @@ on:
1515
description: "Key needed to access cached depends"
1616
required: true
1717
type: string
18+
depends-host:
19+
description: "Host triplet from depends build"
20+
required: true
21+
type: string
22+
depends-dep-opts:
23+
description: "DEP_OPTS used to build depends"
24+
required: false
25+
type: string
26+
default: ""
1827
outputs:
1928
key:
2029
description: "Key needed for restoring artifacts bundle"
@@ -59,12 +68,17 @@ jobs:
5968
- name: Restore depends cache
6069
uses: actions/cache/restore@v4
6170
with:
62-
path: |
63-
depends/built
64-
depends/${{ steps.setup.outputs.HOST }}
71+
path: depends/built/${{ inputs.depends-host }}
6572
key: ${{ inputs.depends-key }}
6673
fail-on-cache-miss: true
6774

75+
- name: Rebuild depends prefix
76+
run: |
77+
# Use the HOST and DEP_OPTS from the depends build, not this build-target
78+
# This ensures the build_id matches the cached packages
79+
make -j$(nproc) -C depends HOST="${{ inputs.depends-host }}" ${{ inputs.depends-dep-opts }}
80+
shell: bash
81+
6882
- name: Manage ccache
6983
uses: actions/cache@v4
7084
with:

‎.github/workflows/build.yml‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -138,6 +138,8 @@ jobs:
138138
build-target: arm-linux
139139
container-path: ${{ needs.container.outputs.path }}
140140
depends-key: ${{ needs.depends-arm-linux.outputs.key }}
141+
depends-host: ${{ needs.depends-arm-linux.outputs.host }}
142+
depends-dep-opts: ${{ needs.depends-arm-linux.outputs.dep-opts }}
141143

142144
src-linux64:
143145
name: linux64-build
@@ -148,6 +150,8 @@ jobs:
148150
build-target: linux64
149151
container-path: ${{ needs.container.outputs.path }}
150152
depends-key: ${{ needs.depends-linux64.outputs.key }}
153+
depends-host: ${{ needs.depends-linux64.outputs.host }}
154+
depends-dep-opts: ${{ needs.depends-linux64.outputs.dep-opts }}
151155

152156
src-linux64_fuzz:
153157
name: linux64_fuzz-build
@@ -158,6 +162,8 @@ jobs:
158162
build-target: linux64_fuzz
159163
container-path: ${{ needs.container.outputs.path }}
160164
depends-key: ${{ needs.depends-linux64.outputs.key }}
165+
depends-host: ${{ needs.depends-linux64.outputs.host }}
166+
depends-dep-opts: ${{ needs.depends-linux64.outputs.dep-opts }}
161167

162168
src-linux64_multiprocess:
163169
name: linux64_multiprocess-build
@@ -168,6 +174,8 @@ jobs:
168174
build-target: linux64_multiprocess
169175
container-path: ${{ needs.container.outputs.path }}
170176
depends-key: ${{ needs.depends-linux64_multiprocess.outputs.key }}
177+
depends-host: ${{ needs.depends-linux64_multiprocess.outputs.host }}
178+
depends-dep-opts: ${{ needs.depends-linux64_multiprocess.outputs.dep-opts }}
171179

172180
src-linux64_nowallet:
173181
name: linux64_nowallet-build
@@ -177,6 +185,8 @@ jobs:
177185
build-target: linux64_nowallet
178186
container-path: ${{ needs.container.outputs.path }}
179187
depends-key: ${{ needs.depends-linux64_nowallet.outputs.key }}
188+
depends-host: ${{ needs.depends-linux64_nowallet.outputs.host }}
189+
depends-dep-opts: ${{ needs.depends-linux64_nowallet.outputs.dep-opts }}
180190

181191
src-linux64_sqlite:
182192
name: linux64_sqlite-build
@@ -187,6 +197,8 @@ jobs:
187197
build-target: linux64_sqlite
188198
container-path: ${{ needs.container.outputs.path }}
189199
depends-key: ${{ needs.depends-linux64.outputs.key }}
200+
depends-host: ${{ needs.depends-linux64.outputs.host }}
201+
depends-dep-opts: ${{ needs.depends-linux64.outputs.dep-opts }}
190202

191203
src-linux64_tsan:
192204
name: linux64_tsan-build
@@ -197,6 +209,8 @@ jobs:
197209
build-target: linux64_tsan
198210
container-path: ${{ needs.container.outputs.path }}
199211
depends-key: ${{ needs.depends-linux64_multiprocess.outputs.key }}
212+
depends-host: ${{ needs.depends-linux64_multiprocess.outputs.host }}
213+
depends-dep-opts: ${{ needs.depends-linux64_multiprocess.outputs.dep-opts }}
200214

201215
src-linux64_ubsan:
202216
name: linux64_ubsan-build
@@ -207,6 +221,8 @@ jobs:
207221
build-target: linux64_ubsan
208222
container-path: ${{ needs.container.outputs.path }}
209223
depends-key: ${{ needs.depends-linux64.outputs.key }}
224+
depends-host: ${{ needs.depends-linux64.outputs.host }}
225+
depends-dep-opts: ${{ needs.depends-linux64.outputs.dep-opts }}
210226

211227
src-mac:
212228
name: mac-build
@@ -216,6 +232,8 @@ jobs:
216232
build-target: mac
217233
container-path: ${{ needs.container.outputs.path }}
218234
depends-key: ${{ needs.depends-mac.outputs.key }}
235+
depends-host: ${{ needs.depends-mac.outputs.host }}
236+
depends-dep-opts: ${{ needs.depends-mac.outputs.dep-opts }}
219237

220238
src-win64:
221239
name: win64-build
@@ -225,6 +243,8 @@ jobs:
225243
build-target: win64
226244
container-path: ${{ needs.container.outputs.path }}
227245
depends-key: ${{ needs.depends-win64.outputs.key }}
246+
depends-host: ${{ needs.depends-win64.outputs.host }}
247+
depends-dep-opts: ${{ needs.depends-win64.outputs.dep-opts }}
228248

229249
test-linux64:
230250
name: linux64-test

0 commit comments

Comments
 (0)