Skip to content

Commit 04a5047

Browse files
matt-codecovmatt-codecov
committed
build: rewrite CI to use matrices and cut down on workflow calls
1 parent 9806daa commit 04a5047

File tree

14 files changed

+664
-1024
lines changed

14 files changed

+664
-1024
lines changed

‎.github/workflows/_build-app.yml‎

Lines changed: 0 additions & 148 deletions
This file was deleted.

‎.github/workflows/_build-images.yml‎

Lines changed: 223 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,223 @@
1+
name: Build Images
2+
3+
on:
4+
workflow_call:
5+
inputs:
6+
# A stringified JSON object containing a list of projects
7+
# (worker, codecov-api, shared) for which CI should run.
8+
changes:
9+
type: string
10+
required: true
11+
12+
# Whether to build production images + test images
13+
build-prod:
14+
type: boolean
15+
default: true
16+
17+
# Whether to build self-hosted images
18+
build-self-hosted:
19+
type: boolean
20+
default: false
21+
22+
env:
23+
AR_REQS_REPO: ${{ vars.CODECOV_UMBRELLA_REQS_IMAGE || 'codecov/umbrella-reqs-fallback' }}
24+
25+
jobs:
26+
build:
27+
name: Build App
28+
runs-on: ubuntu-latest
29+
strategy:
30+
fail-fast: false
31+
matrix:
32+
include:
33+
- project: worker
34+
enabled: ${{ contains(fromJSON(inputs.changes), 'worker') }}
35+
repo: ${{ vars.CODECOV_WORKER_IMAGE_V2 || vars.CODECOV_WORKER_IMAGE_V2_SELF_HOSTED || 'codecov/self-hosted-worker' }}
36+
output_directory: apps/worker
37+
make_target_prefix: worker.
38+
39+
- project: codecov-api
40+
enabled: ${{ contains(fromJSON(inputs.changes), 'codecov-api') }}
41+
repo: ${{ vars.CODECOV_API_IMAGE_V2 || vars.CODECOV_API_IMAGE_V2_SELF_HOSTED || 'codecov/self-hosted-api' }}
42+
output_directory: apps/codecov-api
43+
make_target_prefix: api.
44+
45+
- project: shared
46+
enabled: ${{ contains(fromJSON(inputs.changes), 'shared') }}
47+
repo: codecov/dev-shared
48+
output_directory: libs/shared
49+
make_target_prefix: shared.
50+
51+
env:
52+
AR_REPO: ${{ matrix.repo }}
53+
steps:
54+
- name: Checkout
55+
if: ${{ matrix.enabled }}
56+
uses: actions/checkout@v4
57+
with:
58+
fetch-depth: 2
59+
60+
- id: "auth"
61+
if: ${{ matrix.enabled && !github.event.pull_request.head.repo.fork && github.repository_owner == 'codecov' }}
62+
name: "Authenticate to Google Cloud"
63+
uses: "google-github-actions/auth@v2.1.2"
64+
with:
65+
token_format: "access_token"
66+
workload_identity_provider: ${{ secrets.CODECOV_GCP_WIDP }}
67+
service_account: ${{ secrets.CODECOV_GCP_WIDSA }}
68+
69+
- name: Docker configuration
70+
if: ${{ matrix.enabled && !github.event.pull_request.head.repo.fork && github.repository_owner == 'codecov' }}
71+
run: |-
72+
echo ${{steps.auth.outputs.access_token}} | docker login -u oauth2accesstoken --password-stdin https://us-docker.pkg.dev
73+
74+
- name: Cache Requirements
75+
id: cache-requirements
76+
if: ${{ matrix.enabled }}
77+
uses: actions/cache@v4
78+
env:
79+
# Forks can't access the variable containing our actual image repository. We want to
80+
# use a separate cache to make sure they don't interfere with reqs images being pushed.
81+
cache-name: ${{ !github.event.pull_request.repo.fork && 'umbrella-requirements' || 'umbrella-requirements-fork' }}
82+
with:
83+
path: |
84+
./requirements.tar
85+
key: ${{ runner.os }}-${{ runner.arch }}-${{ env.cache-name }}-${{ hashFiles('uv.lock') }}-${{ hashFiles('docker/Dockerfile.requirements') }}-${{ hashFiles('libs/shared/**') }}
86+
87+
- name: Cache App
88+
id: cache-app
89+
if: ${{ matrix.enabled && inputs.build-prod }}
90+
uses: actions/cache@v4
91+
env:
92+
cache-name: ${{ matrix.repo }}-app
93+
with:
94+
path: |
95+
${{ matrix.output_directory }}/app.tar
96+
key: ${{ runner.os }}-${{ env.cache-name }}-${{ github.run_id }}
97+
98+
- name: Cache Self-Hosted
99+
id: cache-self-hosted
100+
if: ${{ matrix.enabled && inputs.build-self-hosted }}
101+
uses: actions/cache@v4
102+
env:
103+
cache-name: ${{ matrix.repo }}-self-hosted
104+
with:
105+
path: |
106+
${{ matrix.output_directory }}/self-hosted-runtime.tar
107+
${{ matrix.output_directory }}/self-hosted.tar
108+
key: ${{ runner.os }}-${{ env.cache-name }}-${{ github.run_id }}
109+
110+
- name: Load requirements from cache
111+
if: ${{ matrix.enabled && steps.cache-requirements.outputs.cache-hit == 'true' }}
112+
run: |
113+
make load.requirements
114+
115+
# This shouldn't happen; the _build-requirements.yml job should have run.
116+
- name: Build/pull requirements
117+
if: ${{ matrix.enabled && steps.cache-requirements.outputs.cache-hit != 'true' }}
118+
run: |
119+
echo "Warning: requirements image not in cache, building a new one"
120+
make build.requirements
121+
make save.requirements
122+
123+
- name: Build Prod
124+
if: ${{ matrix.enabled && inputs.build-prod }}
125+
run: |
126+
make ${{ matrix.make_target_prefix }}build.app
127+
make ${{ matrix.make_target_prefix }}save.app
128+
129+
- name: Build Self-Hosted
130+
if: ${{ matrix.enabled && inputs.build-self-hosted }}
131+
run: |
132+
make ${{ matrix.make_target_prefix }}build.self-hosted
133+
make ${{ matrix.make_target_prefix }}save.self-hosted
134+
135+
build-test:
136+
name: Build Test App
137+
if: ${{ inputs.build-prod }}
138+
runs-on: ubuntu-latest
139+
strategy:
140+
fail-fast: false
141+
matrix:
142+
include:
143+
- project: worker
144+
enabled: ${{ contains(fromJSON(inputs.changes), 'worker') }}
145+
repo: ${{ vars.CODECOV_WORKER_IMAGE_V2 || vars.CODECOV_WORKER_IMAGE_V2_SELF_HOSTED || 'codecov/self-hosted-worker' }}
146+
output_directory: apps/worker
147+
make_target_prefix: worker.
148+
149+
- project: codecov-api
150+
enabled: ${{ contains(fromJSON(inputs.changes), 'codecov-api') }}
151+
repo: ${{ vars.CODECOV_API_IMAGE_V2 || vars.CODECOV_API_IMAGE_V2_SELF_HOSTED || 'codecov/self-hosted-api' }}
152+
output_directory: apps/codecov-api
153+
make_target_prefix: api.
154+
155+
- project: shared
156+
enabled: ${{ contains(fromJSON(inputs.changes), 'shared') }}
157+
repo: codecov/dev-shared
158+
output_directory: libs/shared
159+
make_target_prefix: shared.
160+
161+
env:
162+
AR_REPO: ${{ matrix.repo }}
163+
steps:
164+
- name: Checkout
165+
if: ${{ matrix.enabled }}
166+
uses: actions/checkout@v4
167+
with:
168+
fetch-depth: 2
169+
170+
- id: "auth"
171+
if: ${{ matrix.enabled && !github.event.pull_request.head.repo.fork && github.repository_owner == 'codecov' }}
172+
name: "Authenticate to Google Cloud"
173+
uses: "google-github-actions/auth@v2.1.2"
174+
with:
175+
token_format: "access_token"
176+
workload_identity_provider: ${{ secrets.CODECOV_GCP_WIDP }}
177+
service_account: ${{ secrets.CODECOV_GCP_WIDSA }}
178+
179+
- name: Docker configuration
180+
if: ${{ matrix.enabled && !github.event.pull_request.head.repo.fork && github.repository_owner == 'codecov' }}
181+
run: |-
182+
echo ${{steps.auth.outputs.access_token}} | docker login -u oauth2accesstoken --password-stdin https://us-docker.pkg.dev
183+
184+
- name: Cache Test Requirements
185+
id: cache-test-requirements
186+
if: ${{ matrix.enabled }}
187+
uses: actions/cache@v4
188+
env:
189+
cache-name: umbrella-test-requirements
190+
with:
191+
path: |
192+
./test-requirements.tar
193+
key: ${{ runner.os }}-${{ runner.arch }}-${{ env.cache-name }}-${{ hashFiles('./uv.lock') }}-${{ hashFiles('docker/Dockerfile.requirements') }}-${{ hashFiles('docker/Dockerfile.test-requirements') }}-${{ hashFiles('libs/shared/**') }}
194+
195+
- name: Cache Test App
196+
id: cache-test-app
197+
if: ${{ matrix.enabled }}
198+
uses: actions/cache@v4
199+
env:
200+
cache-name: ${{ matrix.repo }}-test-app
201+
with:
202+
path: |
203+
${{ matrix.output_directory }}/test-app.tar
204+
key: ${{ runner.os }}-${{ env.cache-name }}-${{ github.run_id }}
205+
206+
- name: Load test requirements from cache
207+
if: ${{ matrix.enabled && steps.cache-test-requirements.outputs.cache-hit == 'true' }}
208+
run: |
209+
make load.test-requirements
210+
211+
# This shouldn't happen; the _build-requirements.yml job should have run.
212+
- name: Build/pull test requirements
213+
if: ${{ matrix.enabled && steps.cache-test-requirements.outputs.cache-hit != 'true' }}
214+
run: |
215+
echo "Warning: test requirements image not in cache, building a new one"
216+
make build.test-requirements
217+
make save.test-requirements
218+
219+
- name: Build Test App
220+
if: ${{ matrix.enabled }}
221+
run: |
222+
make ${{ matrix.make_target_prefix }}build.test-app
223+
make ${{ matrix.make_target_prefix }}save.test-app

0 commit comments

Comments
 (0)