From 9f93adcd507ba66b5283a73768dc6e5ed4fd0637 Mon Sep 17 00:00:00 2001 From: baranov-V-V Date: Sun, 28 Sep 2025 17:25:49 +0300 Subject: [PATCH 01/12] [Github][CI] Add separate container for code-format premerge job --- .../build-ci-container-code-format.yml | 105 ++++++++++++++++++ .../github-action-ci-code-format/Dockerfile | 77 +++++++++++++ 2 files changed, 182 insertions(+) create mode 100644 .github/workflows/build-ci-container-code-format.yml create mode 100644 .github/workflows/containers/github-action-ci-code-format/Dockerfile diff --git a/.github/workflows/build-ci-container-code-format.yml b/.github/workflows/build-ci-container-code-format.yml new file mode 100644 index 0000000000000..723ebd8a6e252 --- /dev/null +++ b/.github/workflows/build-ci-container-code-format.yml @@ -0,0 +1,105 @@ +name: Build CI Container + +permissions: + contents: read + +on: + push: + branches: + - main + paths: + - .github/workflows/build-ci-container-code-format.yml + - '.github/workflows/containers/github-action-ci-code-format/**' + - llvm/utils/git/code-format-helper.py + - llvm/utils/git/requirements_formatting.txt + - llvm/utils/git/requirements_formatting.txt.in + pull_request: + paths: + - .github/workflows/build-ci-container-code-format.yml + - '.github/workflows/containers/github-action-ci-code-format/**' + - llvm/utils/git/code-format-helper.py + - llvm/utils/git/requirements_formatting.txt + - llvm/utils/git/requirements_formatting.txt.in + +jobs: + build-ci-container-code-format: + if: github.repository_owner == 'llvm' + runs-on: depot-ubuntu-24.04-16 + steps: + - name: Checkout LLVM + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + with: + sparse-checkout: .github/workflows/containers/github-action-ci-code-format/ + - name: Write Variables + id: vars + run: | + tag=$(git rev-parse --short=12 HEAD) + container_name="ghcr.io/$GITHUB_REPOSITORY_OWNER/amd64/ci-ubuntu-24.04-code-format" + echo "container-name=$container_name" >> $GITHUB_OUTPUT + echo "container-name-tag=$container_name:$tag" >> $GITHUB_OUTPUT + echo "container-filename=$(echo $container_name:$tag | sed -e 's/\//-/g' -e 's/:/-/g').tar" >> $GITHUB_OUTPUT + - name: Build container + run: | + podman build --target ci-container-code-format \ + -f .github/workflows/containers/github-action-ci-code-format/Dockerfile \ + -t ${{ steps.vars.outputs.container-name-tag }} . + + # Save the container so we have it in case the push fails. This also + # allows us to separate the push step into a different job so we can + # maintain minimal permissions while building the container. + - name: Save container image + run: | + podman save ${{ steps.vars.outputs.container-name-tag }} > ${{ steps.vars.outputs.container-filename }} + + - name: Upload container image + uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0 + with: + name: container-amd64 + path: "*.tar" + retention-days: 14 + + - name: Test Container + run: | + for image in ${{ steps.vars.outputs.container-name-tag }}; do + # Use --pull=never to ensure we are testing the just built image. + podman run --pull=never --rm -it $image /usr/bin/bash -x -c 'cd $HOME && clang-format --version | grep version' + done + + push-ci-container: + if: github.event_name == 'push' + needs: + - build-ci-container-code-format + permissions: + packages: write + runs-on: ubuntu-24.04 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + steps: + - name: Download container + uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0 + + - name: Push Container + run: | + function push_container { + image_name=$1 + latest_name=$(echo $image_name | sed 's/:[a-f0-9]\+$/:latest/g') + podman tag $image_name $latest_name + echo "Pushing $image_name ..." + podman push $image_name + echo "Pushing $latest_name ..." + podman push $latest_name + } + + podman login -u ${{ github.actor }} -p $GITHUB_TOKEN ghcr.io + for f in $(find . -iname *.tar); do + image_name=$(podman load -q -i $f | sed 's/Loaded image: //g') + push_container $image_name + + if echo $image_name | grep '/amd64/'; then + # For amd64, create an alias with the arch component removed. + # This matches the convention used on dockerhub. + default_image_name=$(echo $(dirname $(dirname $image_name))/$(basename $image_name)) + podman tag $image_name $default_image_name + push_container $default_image_name + fi + done diff --git a/.github/workflows/containers/github-action-ci-code-format/Dockerfile b/.github/workflows/containers/github-action-ci-code-format/Dockerfile new file mode 100644 index 0000000000000..88d43618bd22e --- /dev/null +++ b/.github/workflows/containers/github-action-ci-code-format/Dockerfile @@ -0,0 +1,77 @@ +FROM docker.io/library/ubuntu:24.04 AS base +ENV LLVM_SYSROOT=/opt/llvm + +FROM base AS clang-format-toolchain +ENV LLVM_VERSION=21.1.1 + +RUN apt-get update && \ + apt-get install -y \ + wget \ + gcc \ + g++ \ + cmake \ + ninja-build \ + python3 \ + git \ + curl \ + zlib1g-dev && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +RUN curl -O -L https://github.com/llvm/llvm-project/archive/refs/tags/llvmorg-$LLVM_VERSION.tar.gz && \ + tar -xf llvmorg-$LLVM_VERSION.tar.gz && \ + rm -f llvmorg-$LLVM_VERSION.tar.gz + +WORKDIR /llvm-project-llvmorg-$LLVM_VERSION + +RUN cmake -B ./build -G Ninja ./llvm \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX="$LLVM_SYSROOT" \ + -DLLVM_ENABLE_PROJECTS="clang" \ + -DLLVM_DISTRIBUTION_COMPONENTS="clang-format" + +RUN ninja -C ./build install-distribution + +FROM base AS ci-container-code-format + +COPY --from=clang-format-toolchain $LLVM_SYSROOT $LLVM_SYSROOT + +# Need nodejs for some of the GitHub actions. +# Need git for git-clang-format. +RUN apt-get update && \ + DEBIAN_FRONTEND=noninteractive apt-get install -y \ + # binutils \ + git \ + nodejs \ + # python3-psutil \ + sudo \ + # These are needed by the premerge pipeline. Pip and venv are used to + # install dependent python packages. + # Having a symlink from python to python3 enables code sharing between + # the Linux and Windows pipelines. + python3-pip \ + python3-venv \ + python-is-python3 && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +ENV LLVM_SYSROOT=$LLVM_SYSROOT +ENV PATH=${LLVM_SYSROOT}/bin:${PATH} + +# Create a new user to avoid test failures related to a lack of expected +# permissions issues in some tests. Set the user id to 1001 as that is the +# user id that Github Actions uses to perform the checkout action. +RUN useradd gha -u 1001 -m -s /bin/bash + +# Also add the user to passwordless sudoers so that we can install software +# later on without having to rebuild the container. +RUN adduser gha sudo +RUN echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers + +USER gha +WORKDIR /home/gha + +COPY llvm/utils/git/requirements_formatting.txt /home/gha/requirements_formatting.txt +RUN python -m venv venv && \ + venv/bin/pip install -r /home/gha/requirements_formatting.txt && \ + rm /home/gha/requirements_formatting.txt From df0f99bcf672b9029de38d380d1e696510c0d2aa Mon Sep 17 00:00:00 2001 From: baranov-V-V Date: Sun, 28 Sep 2025 18:06:37 +0300 Subject: [PATCH 02/12] better --- .github/workflows/build-ci-container-code-format.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-ci-container-code-format.yml b/.github/workflows/build-ci-container-code-format.yml index 723ebd8a6e252..885a4b47b4a51 100644 --- a/.github/workflows/build-ci-container-code-format.yml +++ b/.github/workflows/build-ci-container-code-format.yml @@ -29,7 +29,10 @@ jobs: - name: Checkout LLVM uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 with: - sparse-checkout: .github/workflows/containers/github-action-ci-code-format/ + sparse-checkout: | + .github/workflows/containers/github-action-ci-code-format/ + llvm/utils/git/requirements_formatting.txt + - name: Write Variables id: vars run: | From d7ee503b6e3b8de4f720e0d149b2075c9cac152a Mon Sep 17 00:00:00 2001 From: baranov-V-V Date: Sun, 28 Sep 2025 18:44:08 +0300 Subject: [PATCH 03/12] better --- .../containers/github-action-ci-code-format/Dockerfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/containers/github-action-ci-code-format/Dockerfile b/.github/workflows/containers/github-action-ci-code-format/Dockerfile index 88d43618bd22e..3ff87bcf563c7 100644 --- a/.github/workflows/containers/github-action-ci-code-format/Dockerfile +++ b/.github/workflows/containers/github-action-ci-code-format/Dockerfile @@ -40,10 +40,8 @@ COPY --from=clang-format-toolchain $LLVM_SYSROOT $LLVM_SYSROOT # Need git for git-clang-format. RUN apt-get update && \ DEBIAN_FRONTEND=noninteractive apt-get install -y \ - # binutils \ git \ nodejs \ - # python3-psutil \ sudo \ # These are needed by the premerge pipeline. Pip and venv are used to # install dependent python packages. @@ -71,6 +69,7 @@ RUN echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers USER gha WORKDIR /home/gha +# Install dependencies for 'pr-code-format.yml' job COPY llvm/utils/git/requirements_formatting.txt /home/gha/requirements_formatting.txt RUN python -m venv venv && \ venv/bin/pip install -r /home/gha/requirements_formatting.txt && \ From 8496a1b4f1b340ccaeab6b18141b39d8230e06b1 Mon Sep 17 00:00:00 2001 From: Victor Baranov Date: Fri, 3 Oct 2025 01:31:31 +0300 Subject: [PATCH 04/12] add dockerfile with binary download --- ...mat.yml => build-ci-container-tooling.yml} | 42 +++++++----- .../Dockerfile | 67 +++++++++---------- 2 files changed, 59 insertions(+), 50 deletions(-) rename .github/workflows/{build-ci-container-code-format.yml => build-ci-container-tooling.yml} (72%) rename .github/workflows/containers/{github-action-ci-code-format => github-action-ci-tooling}/Dockerfile (56%) diff --git a/.github/workflows/build-ci-container-code-format.yml b/.github/workflows/build-ci-container-tooling.yml similarity index 72% rename from .github/workflows/build-ci-container-code-format.yml rename to .github/workflows/build-ci-container-tooling.yml index 885a4b47b4a51..9bf90aa2241bc 100644 --- a/.github/workflows/build-ci-container-code-format.yml +++ b/.github/workflows/build-ci-container-tooling.yml @@ -8,43 +8,53 @@ on: branches: - main paths: - - .github/workflows/build-ci-container-code-format.yml - - '.github/workflows/containers/github-action-ci-code-format/**' - - llvm/utils/git/code-format-helper.py + - .github/workflows/build-ci-container-tooling.yml + - '.github/workflows/containers/github-action-ci-tooling/**' - llvm/utils/git/requirements_formatting.txt - - llvm/utils/git/requirements_formatting.txt.in + - llvm/utils/git/requirements_linting.txt pull_request: paths: - - .github/workflows/build-ci-container-code-format.yml - - '.github/workflows/containers/github-action-ci-code-format/**' - - llvm/utils/git/code-format-helper.py + - .github/workflows/build-ci-container-tooling.yml + - '.github/workflows/containers/github-action-ci-tooling/**' - llvm/utils/git/requirements_formatting.txt - - llvm/utils/git/requirements_formatting.txt.in + - llvm/utils/git/requirements_linting.txt jobs: - build-ci-container-code-format: + build-ci-container-tooling: if: github.repository_owner == 'llvm' runs-on: depot-ubuntu-24.04-16 + strategy: + matrix: + target: [ci-container-code-format, ci-container-code-lint] + include: + - target: ci-container-code-format + container_name: code-format + check_line: clang-format --version | grep version + - target: ci-container-code-lint + container_name: code-lint + check_line: clang-tidy --version | grep version steps: - name: Checkout LLVM uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 with: sparse-checkout: | - .github/workflows/containers/github-action-ci-code-format/ + .github/workflows/containers/github-action-ci-tooling/ llvm/utils/git/requirements_formatting.txt + llvm/utils/git/requirements_linting.txt + clang-tools-extra/clang-tidy/tool/clang-tidy-diff.py - name: Write Variables id: vars run: | tag=$(git rev-parse --short=12 HEAD) - container_name="ghcr.io/$GITHUB_REPOSITORY_OWNER/amd64/ci-ubuntu-24.04-code-format" + container_name="ghcr.io/$GITHUB_REPOSITORY_OWNER/amd64/ci-ubuntu-24.04-${{ matrix.container_name }}" echo "container-name=$container_name" >> $GITHUB_OUTPUT echo "container-name-tag=$container_name:$tag" >> $GITHUB_OUTPUT echo "container-filename=$(echo $container_name:$tag | sed -e 's/\//-/g' -e 's/:/-/g').tar" >> $GITHUB_OUTPUT - name: Build container run: | - podman build --target ci-container-code-format \ - -f .github/workflows/containers/github-action-ci-code-format/Dockerfile \ + podman build --target ${{ matrix.target }} \ + -f .github/workflows/containers/github-action-ci-tooling/Dockerfile \ -t ${{ steps.vars.outputs.container-name-tag }} . # Save the container so we have it in case the push fails. This also @@ -57,7 +67,7 @@ jobs: - name: Upload container image uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0 with: - name: container-amd64 + name: container-amd64-${{ matrix.container_name }} path: "*.tar" retention-days: 14 @@ -65,13 +75,13 @@ jobs: run: | for image in ${{ steps.vars.outputs.container-name-tag }}; do # Use --pull=never to ensure we are testing the just built image. - podman run --pull=never --rm -it $image /usr/bin/bash -x -c 'cd $HOME && clang-format --version | grep version' + podman run --pull=never --rm -it $image /usr/bin/bash -x -c 'cd $HOME && ${{ matrix.check_line }}' done push-ci-container: if: github.event_name == 'push' needs: - - build-ci-container-code-format + - build-ci-container-tooling permissions: packages: write runs-on: ubuntu-24.04 diff --git a/.github/workflows/containers/github-action-ci-code-format/Dockerfile b/.github/workflows/containers/github-action-ci-tooling/Dockerfile similarity index 56% rename from .github/workflows/containers/github-action-ci-code-format/Dockerfile rename to .github/workflows/containers/github-action-ci-tooling/Dockerfile index 3ff87bcf563c7..b0b9b24def2cd 100644 --- a/.github/workflows/containers/github-action-ci-code-format/Dockerfile +++ b/.github/workflows/containers/github-action-ci-tooling/Dockerfile @@ -1,40 +1,18 @@ -FROM docker.io/library/ubuntu:24.04 AS base -ENV LLVM_SYSROOT=/opt/llvm +FROM docker.io/library/ubuntu:24.04 AS llvm-downloader -FROM base AS clang-format-toolchain -ENV LLVM_VERSION=21.1.1 +ENV LLVM_VERSION=21.1.2 RUN apt-get update && \ - apt-get install -y \ - wget \ - gcc \ - g++ \ - cmake \ - ninja-build \ - python3 \ - git \ - curl \ - zlib1g-dev && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* - -RUN curl -O -L https://github.com/llvm/llvm-project/archive/refs/tags/llvmorg-$LLVM_VERSION.tar.gz && \ - tar -xf llvmorg-$LLVM_VERSION.tar.gz && \ - rm -f llvmorg-$LLVM_VERSION.tar.gz - -WORKDIR /llvm-project-llvmorg-$LLVM_VERSION + apt-get install -y wget pixz && \ + wget --progress=bar:force -O llvm.tar.xz https://github.com/llvm/llvm-project/releases/download/llvmorg-${LLVM_VERSION}/LLVM-${LLVM_VERSION}-Linux-X64.tar.xz && \ + mkdir -p /llvm-extract && \ + pixz -d llvm.tar.xz && \ + tar -xvf llvm.tar -C /llvm-extract LLVM-${LLVM_VERSION}-Linux-X64/bin/ && \ + rm llvm.tar -RUN cmake -B ./build -G Ninja ./llvm \ - -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_INSTALL_PREFIX="$LLVM_SYSROOT" \ - -DLLVM_ENABLE_PROJECTS="clang" \ - -DLLVM_DISTRIBUTION_COMPONENTS="clang-format" - -RUN ninja -C ./build install-distribution - -FROM base AS ci-container-code-format +FROM docker.io/library/ubuntu:24.04 AS base -COPY --from=clang-format-toolchain $LLVM_SYSROOT $LLVM_SYSROOT +ENV LLVM_SYSROOT=/opt/llvm # Need nodejs for some of the GitHub actions. # Need git for git-clang-format. @@ -53,8 +31,6 @@ RUN apt-get update && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* -ENV LLVM_SYSROOT=$LLVM_SYSROOT -ENV PATH=${LLVM_SYSROOT}/bin:${PATH} # Create a new user to avoid test failures related to a lack of expected # permissions issues in some tests. Set the user id to 1001 as that is the @@ -69,8 +45,31 @@ RUN echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers USER gha WORKDIR /home/gha +FROM base AS ci-container-code-format + +ENV LLVM_VERSION=21.1.2 + +COPY --from=llvm-downloader /llvm-extract/LLVM-${LLVM_VERSION}-Linux-X64/bin/clang-format ${LLVM_SYSROOT}/bin/clang-format + +ENV PATH=${LLVM_SYSROOT}/bin:${PATH} + # Install dependencies for 'pr-code-format.yml' job COPY llvm/utils/git/requirements_formatting.txt /home/gha/requirements_formatting.txt RUN python -m venv venv && \ venv/bin/pip install -r /home/gha/requirements_formatting.txt && \ rm /home/gha/requirements_formatting.txt + +# Stage 4: clang-lint container +FROM base AS ci-container-code-lint + +ENV LLVM_VERSION=21.1.2 + +COPY --from=llvm-downloader /llvm-extract/LLVM-${LLVM_VERSION}-Linux-X64/bin/clang-tidy ${LLVM_SYSROOT}/bin/ +COPY clang-tools-extra/clang-tidy/tool/clang-tidy-diff.py ${LLVM_SYSROOT}/bin/clang-tidy-diff.py + +ENV PATH=${LLVM_SYSROOT}/bin:${PATH} + +COPY llvm/utils/git/requirements_linting.txt /home/gha/requirements_linting.txt +RUN python -m venv venv && \ + venv/bin/pip install -r /home/gha/requirements_linting.txt && \ + rm /home/gha/requirements_linting.txt From ab88038adf9a699b43a3fbe00fae2228c857347a Mon Sep 17 00:00:00 2001 From: Victor Baranov Date: Fri, 3 Oct 2025 21:07:02 +0300 Subject: [PATCH 05/12] better? --- .../github-action-ci-tooling/Dockerfile | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/.github/workflows/containers/github-action-ci-tooling/Dockerfile b/.github/workflows/containers/github-action-ci-tooling/Dockerfile index b0b9b24def2cd..54cffd4dbc906 100644 --- a/.github/workflows/containers/github-action-ci-tooling/Dockerfile +++ b/.github/workflows/containers/github-action-ci-tooling/Dockerfile @@ -3,16 +3,16 @@ FROM docker.io/library/ubuntu:24.04 AS llvm-downloader ENV LLVM_VERSION=21.1.2 RUN apt-get update && \ - apt-get install -y wget pixz && \ + apt-get install -y wget xz-utils && \ wget --progress=bar:force -O llvm.tar.xz https://github.com/llvm/llvm-project/releases/download/llvmorg-${LLVM_VERSION}/LLVM-${LLVM_VERSION}-Linux-X64.tar.xz && \ mkdir -p /llvm-extract && \ - pixz -d llvm.tar.xz && \ - tar -xvf llvm.tar -C /llvm-extract LLVM-${LLVM_VERSION}-Linux-X64/bin/ && \ - rm llvm.tar + tar -xvJf llvm.tar.xz -C /llvm-extract LLVM-${LLVM_VERSION}-Linux-X64/bin/ && \ + rm llvm.tar.xz FROM docker.io/library/ubuntu:24.04 AS base ENV LLVM_SYSROOT=/opt/llvm +ENV LLVM_VERSION=21.1.2 # Need nodejs for some of the GitHub actions. # Need git for git-clang-format. @@ -21,10 +21,8 @@ RUN apt-get update && \ git \ nodejs \ sudo \ - # These are needed by the premerge pipeline. Pip and venv are used to - # install dependent python packages. - # Having a symlink from python to python3 enables code sharing between - # the Linux and Windows pipelines. + # These are needed by the premerge pipeline. + # Pip and venv are used to install dependent python packages. python3-pip \ python3-venv \ python-is-python3 && \ @@ -47,8 +45,6 @@ WORKDIR /home/gha FROM base AS ci-container-code-format -ENV LLVM_VERSION=21.1.2 - COPY --from=llvm-downloader /llvm-extract/LLVM-${LLVM_VERSION}-Linux-X64/bin/clang-format ${LLVM_SYSROOT}/bin/clang-format ENV PATH=${LLVM_SYSROOT}/bin:${PATH} @@ -62,8 +58,6 @@ RUN python -m venv venv && \ # Stage 4: clang-lint container FROM base AS ci-container-code-lint -ENV LLVM_VERSION=21.1.2 - COPY --from=llvm-downloader /llvm-extract/LLVM-${LLVM_VERSION}-Linux-X64/bin/clang-tidy ${LLVM_SYSROOT}/bin/ COPY clang-tools-extra/clang-tidy/tool/clang-tidy-diff.py ${LLVM_SYSROOT}/bin/clang-tidy-diff.py From dcaf7ba8da2eb51b815d83a5ccd7a272cb56da58 Mon Sep 17 00:00:00 2001 From: Victor Baranov Date: Fri, 3 Oct 2025 21:14:37 +0300 Subject: [PATCH 06/12] better? --- .../containers/github-action-ci-tooling/Dockerfile | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/containers/github-action-ci-tooling/Dockerfile b/.github/workflows/containers/github-action-ci-tooling/Dockerfile index 54cffd4dbc906..d3e8c8af3b2c4 100644 --- a/.github/workflows/containers/github-action-ci-tooling/Dockerfile +++ b/.github/workflows/containers/github-action-ci-tooling/Dockerfile @@ -1,6 +1,6 @@ FROM docker.io/library/ubuntu:24.04 AS llvm-downloader -ENV LLVM_VERSION=21.1.2 +ENV LLVM_VERSION=21.1.1 RUN apt-get update && \ apt-get install -y wget xz-utils && \ @@ -9,10 +9,11 @@ RUN apt-get update && \ tar -xvJf llvm.tar.xz -C /llvm-extract LLVM-${LLVM_VERSION}-Linux-X64/bin/ && \ rm llvm.tar.xz + FROM docker.io/library/ubuntu:24.04 AS base ENV LLVM_SYSROOT=/opt/llvm -ENV LLVM_VERSION=21.1.2 +ENV LLVM_VERSION=21.1.1 # Need nodejs for some of the GitHub actions. # Need git for git-clang-format. @@ -43,6 +44,7 @@ RUN echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers USER gha WORKDIR /home/gha + FROM base AS ci-container-code-format COPY --from=llvm-downloader /llvm-extract/LLVM-${LLVM_VERSION}-Linux-X64/bin/clang-format ${LLVM_SYSROOT}/bin/clang-format @@ -55,7 +57,7 @@ RUN python -m venv venv && \ venv/bin/pip install -r /home/gha/requirements_formatting.txt && \ rm /home/gha/requirements_formatting.txt -# Stage 4: clang-lint container + FROM base AS ci-container-code-lint COPY --from=llvm-downloader /llvm-extract/LLVM-${LLVM_VERSION}-Linux-X64/bin/clang-tidy ${LLVM_SYSROOT}/bin/ @@ -63,6 +65,7 @@ COPY clang-tools-extra/clang-tidy/tool/clang-tidy-diff.py ${LLVM_SYSROOT}/bin/cl ENV PATH=${LLVM_SYSROOT}/bin:${PATH} +# Install dependencies for 'pr-code-lint.yml' job COPY llvm/utils/git/requirements_linting.txt /home/gha/requirements_linting.txt RUN python -m venv venv && \ venv/bin/pip install -r /home/gha/requirements_linting.txt && \ From feeb8634700d344f3342deb598fb85f6a8ed9ecd Mon Sep 17 00:00:00 2001 From: Victor Baranov Date: Sun, 5 Oct 2025 15:09:04 +0300 Subject: [PATCH 07/12] resolve PR comments --- .../workflows/build-ci-container-tooling.yml | 42 +++++++++---------- .../github-action-ci-tooling/Dockerfile | 38 ++++++----------- 2 files changed, 31 insertions(+), 49 deletions(-) diff --git a/.github/workflows/build-ci-container-tooling.yml b/.github/workflows/build-ci-container-tooling.yml index 9bf90aa2241bc..b50307d317702 100644 --- a/.github/workflows/build-ci-container-tooling.yml +++ b/.github/workflows/build-ci-container-tooling.yml @@ -22,17 +22,7 @@ on: jobs: build-ci-container-tooling: if: github.repository_owner == 'llvm' - runs-on: depot-ubuntu-24.04-16 - strategy: - matrix: - target: [ci-container-code-format, ci-container-code-lint] - include: - - target: ci-container-code-format - container_name: code-format - check_line: clang-format --version | grep version - - target: ci-container-code-lint - container_name: code-lint - check_line: clang-tidy --version | grep version + runs-on: ubuntu-24.04 steps: - name: Checkout LLVM uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 @@ -47,36 +37,42 @@ jobs: id: vars run: | tag=$(git rev-parse --short=12 HEAD) - container_name="ghcr.io/$GITHUB_REPOSITORY_OWNER/amd64/ci-ubuntu-24.04-${{ matrix.container_name }}" - echo "container-name=$container_name" >> $GITHUB_OUTPUT - echo "container-name-tag=$container_name:$tag" >> $GITHUB_OUTPUT - echo "container-filename=$(echo $container_name:$tag | sed -e 's/\//-/g' -e 's/:/-/g').tar" >> $GITHUB_OUTPUT + container_name="ghcr.io/$GITHUB_REPOSITORY_OWNER/amd64/ci-ubuntu-24.04" + echo "container-name-format=$container_name-code-format" >> $GITHUB_OUTPUT + echo "container-name-lint=$container_name-code-lint" >> $GITHUB_OUTPUT + echo "container-name-format-tag=$container_name-format:$tag" >> $GITHUB_OUTPUT + echo "container-name-lint-tag=$container_name-lint:$tag" >> $GITHUB_OUTPUT + echo "container-format-filename=$(echo $container_name-format:$tag | sed -e 's/\//-/g' -e 's/:/-/g').tar" >> $GITHUB_OUTPUT + echo "container-lint-filename=$(echo $container_name-lint:$tag | sed -e 's/\//-/g' -e 's/:/-/g').tar" >> $GITHUB_OUTPUT - name: Build container run: | - podman build --target ${{ matrix.target }} \ + podman build --target ci-container-code-format \ + -f .github/workflows/containers/github-action-ci-tooling/Dockerfile \ + -t ${{ steps.vars.outputs.container-name-format-tag }} . + podman build --target ci-container-code-lint \ -f .github/workflows/containers/github-action-ci-tooling/Dockerfile \ - -t ${{ steps.vars.outputs.container-name-tag }} . + -t ${{ steps.vars.outputs.container-name-lint-tag }} . # Save the container so we have it in case the push fails. This also # allows us to separate the push step into a different job so we can # maintain minimal permissions while building the container. - name: Save container image run: | - podman save ${{ steps.vars.outputs.container-name-tag }} > ${{ steps.vars.outputs.container-filename }} + podman save ${{ steps.vars.outputs.container-name-format-tag }} > ${{ steps.vars.outputs.container-format-filename }} + podman save ${{ steps.vars.outputs.container-name-lint-tag }} > ${{ steps.vars.outputs.container-lint-filename }} - name: Upload container image uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0 with: - name: container-amd64-${{ matrix.container_name }} + name: container-amd64 path: "*.tar" retention-days: 14 - name: Test Container run: | - for image in ${{ steps.vars.outputs.container-name-tag }}; do - # Use --pull=never to ensure we are testing the just built image. - podman run --pull=never --rm -it $image /usr/bin/bash -x -c 'cd $HOME && ${{ matrix.check_line }}' - done + # Use --pull=never to ensure we are testing the just built image. + podman run --pull=never --rm -it ${{ steps.vars.outputs.container-name-format-tag }} /usr/bin/bash -x -c 'cd $HOME && clang-format --version | grep version && black --version | grep black' + podman run --pull=never --rm -it ${{ steps.vars.outputs.container-name-lint-tag }} /usr/bin/bash -x -c 'cd $HOME && clang-tidy --version | grep version && clang-tidy-diff.py -h | grep usage' push-ci-container: if: github.event_name == 'push' diff --git a/.github/workflows/containers/github-action-ci-tooling/Dockerfile b/.github/workflows/containers/github-action-ci-tooling/Dockerfile index d3e8c8af3b2c4..ad32d19818442 100644 --- a/.github/workflows/containers/github-action-ci-tooling/Dockerfile +++ b/.github/workflows/containers/github-action-ci-tooling/Dockerfile @@ -1,6 +1,7 @@ -FROM docker.io/library/ubuntu:24.04 AS llvm-downloader +ARG LLVM_VERSION=21.1.0 -ENV LLVM_VERSION=21.1.1 +FROM ubuntu:24.04 AS llvm-downloader +ARG LLVM_VERSION RUN apt-get update && \ apt-get install -y wget xz-utils && \ @@ -11,9 +12,7 @@ RUN apt-get update && \ FROM docker.io/library/ubuntu:24.04 AS base - ENV LLVM_SYSROOT=/opt/llvm -ENV LLVM_VERSION=21.1.1 # Need nodejs for some of the GitHub actions. # Need git for git-clang-format. @@ -23,42 +22,30 @@ RUN apt-get update && \ nodejs \ sudo \ # These are needed by the premerge pipeline. - # Pip and venv are used to install dependent python packages. + # Pip is used to install dependent python packages. python3-pip \ - python3-venv \ python-is-python3 && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* - -# Create a new user to avoid test failures related to a lack of expected -# permissions issues in some tests. Set the user id to 1001 as that is the -# user id that Github Actions uses to perform the checkout action. -RUN useradd gha -u 1001 -m -s /bin/bash - -# Also add the user to passwordless sudoers so that we can install software -# later on without having to rebuild the container. -RUN adduser gha sudo -RUN echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers - -USER gha WORKDIR /home/gha FROM base AS ci-container-code-format +ARG LLVM_VERSION COPY --from=llvm-downloader /llvm-extract/LLVM-${LLVM_VERSION}-Linux-X64/bin/clang-format ${LLVM_SYSROOT}/bin/clang-format ENV PATH=${LLVM_SYSROOT}/bin:${PATH} # Install dependencies for 'pr-code-format.yml' job -COPY llvm/utils/git/requirements_formatting.txt /home/gha/requirements_formatting.txt -RUN python -m venv venv && \ - venv/bin/pip install -r /home/gha/requirements_formatting.txt && \ - rm /home/gha/requirements_formatting.txt +COPY llvm/utils/git/requirements_formatting.txt requirements_formatting.txt +RUN pip install -r requirements_formatting.txt -break-system-packages && \ + rm requirements_formatting.txt FROM base AS ci-container-code-lint +ARG LLVM_VERSION COPY --from=llvm-downloader /llvm-extract/LLVM-${LLVM_VERSION}-Linux-X64/bin/clang-tidy ${LLVM_SYSROOT}/bin/ COPY clang-tools-extra/clang-tidy/tool/clang-tidy-diff.py ${LLVM_SYSROOT}/bin/clang-tidy-diff.py @@ -66,7 +53,6 @@ COPY clang-tools-extra/clang-tidy/tool/clang-tidy-diff.py ${LLVM_SYSROOT}/bin/cl ENV PATH=${LLVM_SYSROOT}/bin:${PATH} # Install dependencies for 'pr-code-lint.yml' job -COPY llvm/utils/git/requirements_linting.txt /home/gha/requirements_linting.txt -RUN python -m venv venv && \ - venv/bin/pip install -r /home/gha/requirements_linting.txt && \ - rm /home/gha/requirements_linting.txt +COPY llvm/utils/git/requirements_linting.txt requirements_linting.txt +RUN pip install -r requirements_linting.txt --break-system-packages && \ + rm requirements_linting.txt From d1ad1e7f931c28234fce5867ffba480a348216a9 Mon Sep 17 00:00:00 2001 From: Baranov Victor Date: Sun, 5 Oct 2025 16:56:34 +0300 Subject: [PATCH 08/12] Add back depot runner --- .github/workflows/build-ci-container-tooling.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-ci-container-tooling.yml b/.github/workflows/build-ci-container-tooling.yml index b50307d317702..13a6bd3b23543 100644 --- a/.github/workflows/build-ci-container-tooling.yml +++ b/.github/workflows/build-ci-container-tooling.yml @@ -22,7 +22,7 @@ on: jobs: build-ci-container-tooling: if: github.repository_owner == 'llvm' - runs-on: ubuntu-24.04 + runs-on: depot-ubuntu-24.04-16 steps: - name: Checkout LLVM uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 From b42e0e1f4736ba296e117830f7bf00cd7109bc55 Mon Sep 17 00:00:00 2001 From: Baranov Victor Date: Sun, 5 Oct 2025 17:07:26 +0300 Subject: [PATCH 09/12] Fix break-system-packages --- .../workflows/containers/github-action-ci-tooling/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/containers/github-action-ci-tooling/Dockerfile b/.github/workflows/containers/github-action-ci-tooling/Dockerfile index ad32d19818442..7f351c0ed8ac5 100644 --- a/.github/workflows/containers/github-action-ci-tooling/Dockerfile +++ b/.github/workflows/containers/github-action-ci-tooling/Dockerfile @@ -40,7 +40,7 @@ ENV PATH=${LLVM_SYSROOT}/bin:${PATH} # Install dependencies for 'pr-code-format.yml' job COPY llvm/utils/git/requirements_formatting.txt requirements_formatting.txt -RUN pip install -r requirements_formatting.txt -break-system-packages && \ +RUN pip install -r requirements_formatting.txt --break-system-packages && \ rm requirements_formatting.txt From 84239dc0675c99dc8afb7f3688d2cd922326403e Mon Sep 17 00:00:00 2001 From: Victor Baranov Date: Mon, 6 Oct 2025 23:09:03 +0300 Subject: [PATCH 10/12] add back 'docker.io/library' --- .../workflows/containers/github-action-ci-tooling/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/containers/github-action-ci-tooling/Dockerfile b/.github/workflows/containers/github-action-ci-tooling/Dockerfile index 7f351c0ed8ac5..bf05030d9e9e4 100644 --- a/.github/workflows/containers/github-action-ci-tooling/Dockerfile +++ b/.github/workflows/containers/github-action-ci-tooling/Dockerfile @@ -1,6 +1,6 @@ ARG LLVM_VERSION=21.1.0 -FROM ubuntu:24.04 AS llvm-downloader +FROM docker.io/library/ubuntu:24.04 AS llvm-downloader ARG LLVM_VERSION RUN apt-get update && \ From aba8c0b81583b7206ba3d324fe28c9d4a7347ebb Mon Sep 17 00:00:00 2001 From: Victor Baranov Date: Tue, 7 Oct 2025 00:27:12 +0300 Subject: [PATCH 11/12] resolve PR comments --- .github/workflows/build-ci-container-tooling.yml | 11 ++++++++++- .../containers/github-action-ci-tooling/Dockerfile | 4 +--- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build-ci-container-tooling.yml b/.github/workflows/build-ci-container-tooling.yml index 13a6bd3b23543..0f3b756f2f3c3 100644 --- a/.github/workflows/build-ci-container-tooling.yml +++ b/.github/workflows/build-ci-container-tooling.yml @@ -22,7 +22,7 @@ on: jobs: build-ci-container-tooling: if: github.repository_owner == 'llvm' - runs-on: depot-ubuntu-24.04-16 + runs-on: ubuntu-24.04 steps: - name: Checkout LLVM uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 @@ -44,6 +44,15 @@ jobs: echo "container-name-lint-tag=$container_name-lint:$tag" >> $GITHUB_OUTPUT echo "container-format-filename=$(echo $container_name-format:$tag | sed -e 's/\//-/g' -e 's/:/-/g').tar" >> $GITHUB_OUTPUT echo "container-lint-filename=$(echo $container_name-lint:$tag | sed -e 's/\//-/g' -e 's/:/-/g').tar" >> $GITHUB_OUTPUT + + # The default Docker storage location for GitHub Actions doesn't have + # enough disk space, so change it to /mnt, which has more disk space. + - name: Change Docker storage location + run: | + sudo mkdir /mnt/docker + echo '{ "data-root": "/mnt/docker" }' | sudo tee /etc/docker/daemon.json + sudo systemctl restart docker + - name: Build container run: | podman build --target ci-container-code-format \ diff --git a/.github/workflows/containers/github-action-ci-tooling/Dockerfile b/.github/workflows/containers/github-action-ci-tooling/Dockerfile index bf05030d9e9e4..e8fa64f1b4526 100644 --- a/.github/workflows/containers/github-action-ci-tooling/Dockerfile +++ b/.github/workflows/containers/github-action-ci-tooling/Dockerfile @@ -5,7 +5,7 @@ ARG LLVM_VERSION RUN apt-get update && \ apt-get install -y wget xz-utils && \ - wget --progress=bar:force -O llvm.tar.xz https://github.com/llvm/llvm-project/releases/download/llvmorg-${LLVM_VERSION}/LLVM-${LLVM_VERSION}-Linux-X64.tar.xz && \ + wget -O llvm.tar.xz https://github.com/llvm/llvm-project/releases/download/llvmorg-${LLVM_VERSION}/LLVM-${LLVM_VERSION}-Linux-X64.tar.xz && \ mkdir -p /llvm-extract && \ tar -xvJf llvm.tar.xz -C /llvm-extract LLVM-${LLVM_VERSION}-Linux-X64/bin/ && \ rm llvm.tar.xz @@ -28,8 +28,6 @@ RUN apt-get update && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* -WORKDIR /home/gha - FROM base AS ci-container-code-format ARG LLVM_VERSION From 3bc0c988784a712ef13f25035b3c963a98395f02 Mon Sep 17 00:00:00 2001 From: Victor Baranov Date: Tue, 7 Oct 2025 00:44:21 +0300 Subject: [PATCH 12/12] only unpack clang-tidy/clang-format --- .github/workflows/build-ci-container-tooling.yml | 8 -------- .../containers/github-action-ci-tooling/Dockerfile | 5 ++++- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build-ci-container-tooling.yml b/.github/workflows/build-ci-container-tooling.yml index 0f3b756f2f3c3..8095a68cfda9e 100644 --- a/.github/workflows/build-ci-container-tooling.yml +++ b/.github/workflows/build-ci-container-tooling.yml @@ -45,14 +45,6 @@ jobs: echo "container-format-filename=$(echo $container_name-format:$tag | sed -e 's/\//-/g' -e 's/:/-/g').tar" >> $GITHUB_OUTPUT echo "container-lint-filename=$(echo $container_name-lint:$tag | sed -e 's/\//-/g' -e 's/:/-/g').tar" >> $GITHUB_OUTPUT - # The default Docker storage location for GitHub Actions doesn't have - # enough disk space, so change it to /mnt, which has more disk space. - - name: Change Docker storage location - run: | - sudo mkdir /mnt/docker - echo '{ "data-root": "/mnt/docker" }' | sudo tee /etc/docker/daemon.json - sudo systemctl restart docker - - name: Build container run: | podman build --target ci-container-code-format \ diff --git a/.github/workflows/containers/github-action-ci-tooling/Dockerfile b/.github/workflows/containers/github-action-ci-tooling/Dockerfile index e8fa64f1b4526..7a5d8a3be53fd 100644 --- a/.github/workflows/containers/github-action-ci-tooling/Dockerfile +++ b/.github/workflows/containers/github-action-ci-tooling/Dockerfile @@ -7,7 +7,10 @@ RUN apt-get update && \ apt-get install -y wget xz-utils && \ wget -O llvm.tar.xz https://github.com/llvm/llvm-project/releases/download/llvmorg-${LLVM_VERSION}/LLVM-${LLVM_VERSION}-Linux-X64.tar.xz && \ mkdir -p /llvm-extract && \ - tar -xvJf llvm.tar.xz -C /llvm-extract LLVM-${LLVM_VERSION}-Linux-X64/bin/ && \ + tar -xvJf llvm.tar.xz -C /llvm-extract \ + # Only unpack these tools to save space on Github runner. + LLVM-${LLVM_VERSION}-Linux-X64/bin/clang-tidy \ + LLVM-${LLVM_VERSION}-Linux-X64/bin/clang-format && \ rm llvm.tar.xz