Skip to content

Commit 1abb6df

Browse files
ewdurbinmikethemandi
authored
run our docker ci on gha runners directly (#17442)
* run our docker ci on gha runners directly * chore: pin hashes Signed-off-by: Mike Fiedler <[email protected]> * chore: ignore cache poisoning as the artifact is not reused Signed-off-by: Mike Fiedler <[email protected]> * nit: use non-legacy syntax Signed-off-by: Mike Fiedler <[email protected]> * run depot and GHA in parallel * don't change name of the base CI tasks, since that's how branch protection ids them * another way of naming --------- Signed-off-by: Mike Fiedler <[email protected]> Co-authored-by: Mike Fiedler <[email protected]> Co-authored-by: Dustin Ingram <[email protected]>
1 parent fca7c00 commit 1abb6df

File tree

4 files changed

+169
-49
lines changed

4 files changed

+169
-49
lines changed

‎.github/workflows/ci.yml‎ renamed to ‎.github/workflows/ci-base.yml‎

Lines changed: 36 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -1,54 +1,42 @@
1-
name: CI
1+
name: CI Base
22
on:
3-
push:
4-
branches:
5-
- main
6-
pull_request:
7-
merge_group:
8-
types: [checks_requested]
9-
workflow_dispatch: # generally only for the "combine-prs" workflow
10-
permissions:
11-
contents: read
12-
concurrency:
13-
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
14-
cancel-in-progress: true
3+
workflow_call:
4+
inputs:
5+
runner:
6+
required: true
7+
type: string
8+
description: 'Runner to use for jobs'
9+
runner_large:
10+
required: false
11+
type: string
12+
description: 'Large runner to use for heavy jobs'
13+
image_registry:
14+
required: true
15+
type: string
16+
description: 'Container registry for images'
17+
build_id:
18+
required: true
19+
type: string
20+
description: 'Build ID for the container image'
21+
use_depot:
22+
required: false
23+
type: boolean
24+
default: false
25+
description: 'Whether to use Depot for builds'
26+
1527
jobs:
16-
build:
17-
if: github.repository == 'pypi/warehouse'
18-
runs-on: depot-ubuntu-24.04-arm
19-
outputs:
20-
buildId: ${{ steps.build.outputs.build-id}}
21-
permissions:
22-
id-token: write
23-
steps:
24-
- name: Check out repository
25-
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
26-
with:
27-
persist-credentials: false
28-
- name: Set up Depot CLI
29-
uses: depot/setup-action@b0b1ea4f69e92ebf5dea3f8713a1b0c37b2126a5 # v1.6.0
30-
- name: Build image
31-
id: build
32-
uses: depot/build-push-action@9785b135c3c76c33db102e45be96a25ab55cd507 # v1.16.2
33-
with:
34-
save: true
35-
build-args: |
36-
DEVEL=yes
37-
CI=yes
38-
tags: pypi/warehouse:ci-${{ github.run_id }}
3928
test:
4029
# Time out if our test suite has gotten hung
4130
timeout-minutes: 15
42-
needs: build
4331
strategy:
4432
matrix:
4533
include:
4634
- name: Tests
4735
command: bin/tests --postgresql-host postgres
48-
runs_on: depot-ubuntu-24.04-arm-4
36+
use_large_runner: true
4937
- name: Lint
5038
command: bin/lint
51-
runs_on: depot-ubuntu-24.04-arm-4
39+
use_large_runner: true
5240
- name: User Documentation
5341
command: bin/user-docs
5442
- name: Developer Documentation
@@ -59,13 +47,14 @@ jobs:
5947
command: bin/licenses
6048
- name: Translations
6149
command: bin/translations
62-
runs-on: ${{ (matrix.runs_on != null) && matrix.runs_on || 'depot-ubuntu-24.04-arm' }}
50+
runs-on: ${{ (matrix.use_large_runner && inputs.runner_large != '') && inputs.runner_large || inputs.runner }}
6351
container:
64-
image: registry.depot.dev/rltf7cln5v:${{ needs.build.outputs.buildId }}
52+
image: ${{ inputs.image_registry }}:${{ inputs.build_id }}
6553
env:
6654
BILLING_BACKEND: warehouse.subscriptions.services.MockStripeBillingService api_base=http://stripe:12111 api_version=2020-08-27
6755
permissions:
6856
id-token: write
57+
packages: read
6958
services:
7059
postgres:
7160
image: ${{ (matrix.name == 'Tests') && 'postgres:17.5' || '' }}
@@ -84,7 +73,7 @@ jobs:
8473
image: ${{ (matrix.name == 'Tests') && 'stripe/stripe-mock:v0.162.0' || '' }}
8574
ports:
8675
- 12111:12111
87-
name: ${{ matrix.name }}
76+
name: ${{ matrix.name }}${{ !inputs.use_depot && ' (GHA)' || '' }}
8877
steps:
8978
- name: Check out repository
9079
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
@@ -101,14 +90,14 @@ jobs:
10190
run: ${{ matrix.command }}
10291

10392
check_db:
104-
name: Check Database Consistency
105-
needs: build
106-
runs-on: depot-ubuntu-24.04-arm
93+
name: Check Database Consistency${{ !inputs.use_depot && ' (GHA)' || '' }}
94+
runs-on: ${{ inputs.runner }}
10795
continue-on-error: true
10896
container:
109-
image: registry.depot.dev/rltf7cln5v:${{ needs.build.outputs.buildId }}
97+
image: ${{ inputs.image_registry }}:${{ inputs.build_id }}
11098
permissions:
11199
id-token: write
100+
packages: read
112101
services:
113102
postgres:
114103
image: postgres:17.5
@@ -139,4 +128,4 @@ jobs:
139128
run: bin/db-check
140129
env:
141130
# override the hostname set in `dev/environment`
142-
DATABASE_URL: 'postgresql+psycopg://postgres@postgres/warehouse'
131+
DATABASE_URL: 'postgresql+psycopg://postgres@postgres/warehouse'

‎.github/workflows/ci-depot.yml‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
name: CI
2+
on:
3+
push:
4+
branches:
5+
- main
6+
pull_request:
7+
merge_group:
8+
types: [checks_requested]
9+
workflow_dispatch: # generally only for the "combine-prs" workflow
10+
permissions:
11+
contents: read
12+
concurrency:
13+
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
14+
cancel-in-progress: true
15+
jobs:
16+
build:
17+
if: github.repository == 'pypi/warehouse'
18+
runs-on: depot-ubuntu-24.04-arm
19+
outputs:
20+
buildId: ${{ steps.build.outputs.build-id}}
21+
permissions:
22+
id-token: write
23+
steps:
24+
- name: Check out repository
25+
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
26+
with:
27+
persist-credentials: false
28+
- name: Set up Depot CLI
29+
uses: depot/setup-action@b0b1ea4f69e92ebf5dea3f8713a1b0c37b2126a5 # v1.6.0
30+
- name: Build image
31+
id: build
32+
uses: depot/build-push-action@9785b135c3c76c33db102e45be96a25ab55cd507 # v1.16.2
33+
with:
34+
save: true
35+
build-args: |
36+
DEVEL=yes
37+
CI=yes
38+
tags: pypi/warehouse:ci-${{ github.run_id }}
39+
40+
ci:
41+
if: github.repository == 'pypi/warehouse'
42+
needs: build
43+
uses: ./.github/workflows/ci-base.yml
44+
with:
45+
runner: depot-ubuntu-24.04-arm
46+
runner_large: depot-ubuntu-24.04-arm-4
47+
image_registry: registry.depot.dev/rltf7cln5v
48+
build_id: ${{ needs.build.outputs.buildId }}
49+
use_depot: true
50+
permissions:
51+
id-token: write
52+
packages: read

‎.github/workflows/ci-gha.yml‎

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
name: CI (GHA)
2+
on:
3+
push:
4+
branches:
5+
- main
6+
pull_request:
7+
merge_group:
8+
types: [checks_requested]
9+
workflow_dispatch: # generally only for the "combine-prs" workflow
10+
permissions:
11+
contents: read
12+
concurrency:
13+
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
14+
cancel-in-progress: true
15+
jobs:
16+
build:
17+
runs-on: ubuntu-24.04-arm
18+
outputs:
19+
buildId: ${{ github.run_id }}
20+
permissions:
21+
packages: write
22+
steps:
23+
- name: Check out repository
24+
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
25+
with:
26+
persist-credentials: false
27+
- name: Set up Docker Buildx
28+
uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # v3.10.0
29+
- name: Cache
30+
uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 # zizmor: ignore[cache-poisoning]
31+
id: cache
32+
with:
33+
path: |
34+
var-cache-apt
35+
var-lib-apt
36+
root-cache-pip
37+
root-npm
38+
key: cache-${{ hashFiles('Dockerfile') }}
39+
- name: inject cache into docker
40+
uses: reproducible-containers/buildkit-cache-dance@653a570f730e3b9460adc576db523788ba59a0d7 # v3.2.0
41+
with:
42+
cache-map: |
43+
{
44+
"var-cache-apt": "/var/cache/apt",
45+
"var-lib-apt": "/var/lib/apt",
46+
"root-cache-pip": "/root/.cache/pip",
47+
"root-npm": "/root/.npm"
48+
}
49+
skip-extraction: ${{ steps.cache.outputs.cache-hit }}
50+
- name: Login To GHCR
51+
uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0
52+
with:
53+
registry: ghcr.io
54+
username: ${{ github.actor }}
55+
password: ${{ secrets.GITHUB_TOKEN }}
56+
- name: Build and push
57+
uses: docker/build-push-action@14487ce63c7a62a4a324b0bfb37086795e31c6c1 # v6.16.0
58+
with:
59+
context: .
60+
cache-from: type=gha
61+
cache-to: type=gha,mode=max
62+
push: true
63+
build-args: |
64+
DEVEL=yes
65+
CI=yes
66+
tags: |
67+
ghcr.io/pypi/warehouse:ci-${{ github.run_id }}
68+
69+
ci:
70+
needs: build
71+
uses: ./.github/workflows/ci-base.yml
72+
with:
73+
runner: ubuntu-24.04-arm
74+
image_registry: ghcr.io/pypi/warehouse
75+
build_id: ci-${{ needs.build.outputs.buildId }}
76+
use_depot: false
77+
permissions:
78+
id-token: write
79+
packages: read

‎Dockerfile‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -184,8 +184,8 @@ RUN --mount=type=cache,target=/root/.cache/pip \
184184
FROM python:${PYTHON_IMAGE_VERSION}
185185

186186
# Setup some basic environment variables that are ~never going to change.
187-
ENV PYTHONUNBUFFERED 1
188-
ENV PYTHONPATH /opt/warehouse/src/
187+
ENV PYTHONUNBUFFERED=1
188+
ENV PYTHONPATH=/opt/warehouse/src/
189189
ENV PATH="/opt/warehouse/bin:${PATH}"
190190

191191
WORKDIR /opt/warehouse/src/

0 commit comments

Comments
 (0)