Skip to content

Commit e082220

Browse files
tstellartstellar
authored
[GitHub][CI] Factor out duplicate container building code into composite actions (#166663)
1 parent fce5889 commit e082220

File tree

4 files changed

+198
-145
lines changed

4 files changed

+198
-145
lines changed

‎.github/workflows/build-ci-container-tooling.yml‎

Lines changed: 28 additions & 69 deletions
Original file line numberDiff line numberDiff line change
@@ -12,17 +12,30 @@ on:
1212
- '.github/workflows/containers/github-action-ci-tooling/**'
1313
- llvm/utils/git/requirements_formatting.txt
1414
- llvm/utils/git/requirements_linting.txt
15+
- '.github/workflows/build-container/**'
16+
- '.github/workflows/push-container/**'
1517
pull_request:
1618
paths:
1719
- .github/workflows/build-ci-container-tooling.yml
1820
- '.github/workflows/containers/github-action-ci-tooling/**'
1921
- llvm/utils/git/requirements_formatting.txt
2022
- llvm/utils/git/requirements_linting.txt
23+
- '.github/workflows/build-container/**'
24+
- '.github/workflows/push-container/**'
2125

2226
jobs:
2327
build-ci-container-tooling:
28+
name: Build Container ${{ matrix.container-name }}
2429
if: github.repository_owner == 'llvm'
2530
runs-on: ubuntu-24.04
31+
strategy:
32+
fail-fast: false
33+
matrix:
34+
include:
35+
- container-name: code-format
36+
test-command: 'cd $HOME && clang-format --version | grep version && git-clang-format -h | grep usage && black --version | grep black'
37+
- container-name: code-lint
38+
test-command: 'cd $HOME && clang-tidy --version | grep version && clang-tidy-diff.py -h | grep usage'
2639
steps:
2740
- name: Checkout LLVM
2841
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
@@ -32,48 +45,15 @@ jobs:
3245
llvm/utils/git/requirements_formatting.txt
3346
llvm/utils/git/requirements_linting.txt
3447
clang-tools-extra/clang-tidy/tool/clang-tidy-diff.py
48+
.github/workflows/build-container
3549
36-
- name: Write Variables
37-
id: vars
38-
run: |
39-
tag=$(git rev-parse --short=12 HEAD)
40-
container_name="ghcr.io/$GITHUB_REPOSITORY_OWNER/amd64/ci-ubuntu-24.04"
41-
echo "container-name-format=$container_name-code-format" >> $GITHUB_OUTPUT
42-
echo "container-name-lint=$container_name-code-lint" >> $GITHUB_OUTPUT
43-
echo "container-name-format-tag=$container_name-format:$tag" >> $GITHUB_OUTPUT
44-
echo "container-name-lint-tag=$container_name-lint:$tag" >> $GITHUB_OUTPUT
45-
echo "container-format-filename=$(echo $container_name-format:$tag | sed -e 's/\//-/g' -e 's/:/-/g').tar" >> $GITHUB_OUTPUT
46-
echo "container-lint-filename=$(echo $container_name-lint:$tag | sed -e 's/\//-/g' -e 's/:/-/g').tar" >> $GITHUB_OUTPUT
47-
48-
- name: Build container
49-
run: |
50-
podman build --target ci-container-code-format \
51-
-f .github/workflows/containers/github-action-ci-tooling/Dockerfile \
52-
-t ${{ steps.vars.outputs.container-name-format-tag }} .
53-
podman build --target ci-container-code-lint \
54-
-f .github/workflows/containers/github-action-ci-tooling/Dockerfile \
55-
-t ${{ steps.vars.outputs.container-name-lint-tag }} .
56-
57-
# Save the container so we have it in case the push fails. This also
58-
# allows us to separate the push step into a different job so we can
59-
# maintain minimal permissions while building the container.
60-
- name: Save container image
61-
run: |
62-
podman save ${{ steps.vars.outputs.container-name-format-tag }} > ${{ steps.vars.outputs.container-format-filename }}
63-
podman save ${{ steps.vars.outputs.container-name-lint-tag }} > ${{ steps.vars.outputs.container-lint-filename }}
64-
65-
- name: Upload container image
66-
uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0
50+
- name: Build Container
51+
uses: ./.github/workflows/build-container
6752
with:
68-
name: container-amd64
69-
path: "*.tar"
70-
retention-days: 14
71-
72-
- name: Test Container
73-
run: |
74-
# Use --pull=never to ensure we are testing the just built image.
75-
podman run --pull=never --rm -it ${{ steps.vars.outputs.container-name-format-tag }} /usr/bin/bash -x -c 'cd $HOME && clang-format --version | grep version && git-clang-format -h | grep usage && black --version | grep black'
76-
podman run --pull=never --rm -it ${{ steps.vars.outputs.container-name-lint-tag }} /usr/bin/bash -x -c 'cd $HOME && clang-tidy --version | grep version && clang-tidy-diff.py -h | grep usage'
53+
container-name: ci-ubuntu-24.04-${{ matrix.container-name }}
54+
dockerfile: .github/workflows/containers/github-action-ci-tooling/Dockerfile
55+
target: ci-container-${{ matrix.container-name }}
56+
test-command: ${{ matrix.test-command }}
7757

7858
push-ci-container:
7959
if: github.event_name == 'push'
@@ -82,34 +62,13 @@ jobs:
8262
permissions:
8363
packages: write
8464
runs-on: ubuntu-24.04
85-
env:
86-
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
8765
steps:
88-
- name: Download container
89-
uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0
90-
91-
- name: Push Container
92-
run: |
93-
function push_container {
94-
image_name=$1
95-
latest_name=$(echo $image_name | sed 's/:[a-f0-9]\+$/:latest/g')
96-
podman tag $image_name $latest_name
97-
echo "Pushing $image_name ..."
98-
podman push $image_name
99-
echo "Pushing $latest_name ..."
100-
podman push $latest_name
101-
}
102-
103-
podman login -u ${{ github.actor }} -p $GITHUB_TOKEN ghcr.io
104-
for f in $(find . -iname '*.tar'); do
105-
image_name=$(podman load -q -i $f | sed 's/Loaded image: //g')
106-
push_container $image_name
66+
- name: Checkout LLVM
67+
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
68+
with:
69+
sparse-checkout: |
70+
.github/workflows/push-container
10771
108-
if echo $image_name | grep '/amd64/'; then
109-
# For amd64, create an alias with the arch component removed.
110-
# This matches the convention used on dockerhub.
111-
default_image_name=$(echo $(dirname $(dirname $image_name))/$(basename $image_name))
112-
podman tag $image_name $default_image_name
113-
push_container $default_image_name
114-
fi
115-
done
72+
- uses: ./.github/workflows/push-container
73+
with:
74+
token: ${{ secrets.GITHUB_TOKEN }}

‎.github/workflows/build-ci-container.yml‎

Lines changed: 31 additions & 76 deletions
Original file line numberDiff line numberDiff line change
@@ -10,72 +10,46 @@ on:
1010
paths:
1111
- .github/workflows/build-ci-container.yml
1212
- '.github/workflows/containers/github-action-ci/**'
13+
- '.github/workflows/build-container/**'
14+
- '.github/workflows/push-container/**'
1315
pull_request:
1416
paths:
1517
- .github/workflows/build-ci-container.yml
1618
- '.github/workflows/containers/github-action-ci/**'
19+
- '.github/workflows/build-container/**'
20+
- '.github/workflows/push-container/**'
1721

1822
jobs:
1923
build-ci-container:
24+
name: Build Container ${{ matrix.container-name }} ${{ (contains(matrix.runs-on, 'arm') && 'ARM64') || 'X64' }}
2025
if: github.repository_owner == 'llvm'
2126
runs-on: ${{ matrix.runs-on }}
2227
strategy:
2328
matrix:
24-
include:
25-
# The arch names should match the names used on dockerhub.
26-
# See https://github.com/docker-library/official-images#architectures-other-than-amd64
27-
- arch: amd64
28-
runs-on: depot-ubuntu-24.04-16
29-
- arch: arm64v8
30-
runs-on: depot-ubuntu-24.04-arm-16
29+
runs-on:
30+
- depot-ubuntu-24.04-16
31+
- depot-ubuntu-24.04-arm-16
32+
container-name:
33+
- ''
34+
- agent
35+
test-command:
36+
- cd $HOME && printf '#include <iostream>\nint main(int argc, char **argv) { std::cout << "Hello\\n"; }' | clang++ -x c++ - && ./a.out | grep Hello
3137
steps:
3238
- name: Checkout LLVM
3339
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
3440
with:
35-
sparse-checkout: .github/workflows/containers/github-action-ci/
36-
# podman is not installed by default on the ARM64 images.
37-
- name: Install Podman
38-
if: runner.arch == 'ARM64'
39-
run: |
40-
sudo apt-get install podman
41-
- name: Write Variables
42-
id: vars
43-
run: |
44-
tag=$(git rev-parse --short=12 HEAD)
45-
container_name="ghcr.io/$GITHUB_REPOSITORY_OWNER/${{ matrix.arch }}/ci-ubuntu-24.04"
46-
echo "container-name=$container_name" >> $GITHUB_OUTPUT
47-
echo "container-name-agent=$container_name-agent" >> $GITHUB_OUTPUT
48-
echo "container-name-tag=$container_name:$tag" >> $GITHUB_OUTPUT
49-
echo "container-name-agent-tag=$container_name-agent:$tag" >> $GITHUB_OUTPUT
50-
echo "container-filename=$(echo $container_name:$tag | sed -e 's/\//-/g' -e 's/:/-/g').tar" >> $GITHUB_OUTPUT
51-
echo "container-agent-filename=$(echo $container_name-agent:$tag | sed -e 's/\//-/g' -e 's/:/-/g').tar" >> $GITHUB_OUTPUT
52-
- name: Build container
53-
working-directory: ./.github/workflows/containers/github-action-ci/
54-
run: |
55-
podman build --target ci-container -t ${{ steps.vars.outputs.container-name-tag }} .
56-
podman build --target ci-container-agent -t ${{ steps.vars.outputs.container-name-agent-tag }} .
41+
sparse-checkout: |
42+
.github/workflows/containers/github-action-ci/
43+
.github/workflows/build-container
5744
58-
# Save the container so we have it in case the push fails. This also
59-
# allows us to separate the push step into a different job so we can
60-
# maintain minimal permissions while building the container.
61-
- name: Save container image
62-
run: |
63-
podman save ${{ steps.vars.outputs.container-name-tag }} > ${{ steps.vars.outputs.container-filename }}
64-
podman save ${{ steps.vars.outputs.container-name-agent-tag }} > ${{ steps.vars.outputs.container-agent-filename }}
65-
66-
- name: Upload container image
67-
uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0
45+
- name: Build Container
46+
uses: ./.github/workflows/build-container
6847
with:
69-
name: container-${{ matrix.arch }}
70-
path: "*.tar"
71-
retention-days: 14
72-
73-
- name: Test Container
74-
run: |
75-
for image in ${{ steps.vars.outputs.container-name-tag }}; do
76-
# Use --pull=never to ensure we are testing the just built image.
77-
podman run --pull=never --rm -it $image /usr/bin/bash -x -c 'cd $HOME && printf '\''#include <iostream>\nint main(int argc, char **argv) { std::cout << "Hello\\n"; }'\'' | clang++ -x c++ - && ./a.out | grep Hello'
78-
done
48+
container-name: ci-ubuntu-24.04${{ matrix.container-name && format('-{0}', matrix.container-name)}}
49+
context: .github/workflows/containers/github-action-ci/
50+
dockerfile: .github/workflows/containers/github-action-ci/Dockerfile
51+
target: ci-container${{ matrix.container-name && format('-{0}', matrix.container-name) }}
52+
test-command: ${{ matrix.test-command }}
7953

8054
push-ci-container:
8155
if: github.event_name == 'push'
@@ -87,31 +61,12 @@ jobs:
8761
env:
8862
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
8963
steps:
90-
- name: Download container
91-
uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0
92-
93-
- name: Push Container
94-
run: |
95-
function push_container {
96-
image_name=$1
97-
latest_name=$(echo $image_name | sed 's/:[a-f0-9]\+$/:latest/g')
98-
podman tag $image_name $latest_name
99-
echo "Pushing $image_name ..."
100-
podman push $image_name
101-
echo "Pushing $latest_name ..."
102-
podman push $latest_name
103-
}
104-
105-
podman login -u ${{ github.actor }} -p $GITHUB_TOKEN ghcr.io
106-
for f in $(find . -iname '*.tar'); do
107-
image_name=$(podman load -q -i $f | sed 's/Loaded image: //g')
108-
push_container $image_name
64+
- name: Checkout LLVM
65+
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
66+
with:
67+
sparse-checkout: |
68+
.github/workflows/push-container
10969
110-
if echo $image_name | grep '/amd64/'; then
111-
# For amd64, create an alias with the arch component removed.
112-
# This matches the convention used on dockerhub.
113-
default_image_name=$(echo $(dirname $(dirname $image_name))/$(basename $image_name))
114-
podman tag $image_name $default_image_name
115-
push_container $default_image_name
116-
fi
117-
done
70+
- uses: ./.github/workflows/push-container
71+
with:
72+
token: ${{ secrets.GITHUB_TOKEN }}

‎.github/workflows/build-container/action.yml‎

Lines changed: 95 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,95 @@
1+
name: Build Container
2+
description: >-
3+
Build and test a container using the standard llvm naming scheme for containers.
4+
5+
inputs:
6+
tag:
7+
description: >-
8+
The tag to use for this container.
9+
required: false
10+
container-name:
11+
description: >-
12+
The name for the container.
13+
required: true
14+
dockerfile:
15+
description: >-
16+
Path to docker file.
17+
required: false
18+
target:
19+
description: >-
20+
The container target to build 'passed to podman via ---target option'
21+
required: false
22+
context:
23+
description: >-
24+
Path to context for the container build.
25+
required: false
26+
test-command:
27+
description: >-
28+
Test command to run to ensure the container is working correctly.
29+
required: false
30+
31+
runs:
32+
using: "composite"
33+
steps:
34+
# podman is not installed by default on the ARM64 images.
35+
- name: Install Podman
36+
if: runner.arch == 'ARM64'
37+
shell: bash
38+
run: |
39+
sudo apt-get install podman
40+
41+
- name: Build Container
42+
shell: bash
43+
env:
44+
INPUT_TAG: ${{inputs.tag }}
45+
INPUT_CONTAINER_NAME: ${{ inputs.container-name }}
46+
INPUT_TARGET: ${{ inputs.target }}
47+
INPUT_DOCKERFILE: ${{ inputs.dockerfile }}
48+
INPUT_CONTEXT: ${{ inputs.context }}
49+
id: build
50+
run: |
51+
env
52+
tag="${INPUT_TAG:-$(git rev-parse --short=12 HEAD)}"
53+
54+
case "$RUNNER_ARCH" in
55+
ARM64)
56+
container_arch="arm64v8"
57+
;;
58+
*)
59+
container_arch="amd64"
60+
;;
61+
esac
62+
63+
container_name="ghcr.io/$GITHUB_REPOSITORY_OWNER/$container_arch/$INPUT_CONTAINER_NAME:$tag"
64+
container_filename="$(echo $container_name | sed -e 's/\//-/g' -e 's/:/-/g').tar"
65+
if [ -n "$INPUT_TARGET" ]; then
66+
podman_options="$podman_options --target $INPUT_TARGET"
67+
fi
68+
if [ -n "$INPUT_DOCKERFILE" ]; then
69+
podman_options="$podman_options -f $INPUT_DOCKERFILE"
70+
fi
71+
podman_options="$podman_options ${INPUT_CONTEXT:-.}"
72+
echo "Podman Options: $podman_options"
73+
74+
podman build -t $container_name $podman_options
75+
76+
podman save $container_name > $container_filename
77+
78+
echo "container-full-name=$container_name" >> $GITHUB_OUTPUT
79+
80+
- name: Create container artifact
81+
uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0
82+
with:
83+
name: ${{ inputs.container-name }}-${{ runner.arch }}
84+
path: "*.tar"
85+
retention-days: 14
86+
87+
- name: Test container
88+
shell: bash
89+
if: inputs.test-command
90+
env:
91+
INPUT_TEST_COMMAND: ${{ inputs.test-command }}
92+
CONTAINER_FULL_NAME: ${{ steps.build.outputs.container-full-name }}
93+
run: |
94+
podman run --pull=never --rm -it $CONTAINER_FULL_NAME /usr/bin/bash -x -c "$INPUT_TEST_COMMAND"
95+

0 commit comments

Comments
 (0)