fix(CI): Build the generator and release images on pushes (#12273)

All pushes to master and release branches will result in a tag of the
name of the branch. PR CI also verifies the images build properly prior
to merge.

Signed-off-by: mprahl <[email protected]>

Author: mprahl

.github/workflows/build-tools-images.yml

@@ -0,0 +1,62 @@
+name: Build tools images
+
+on:
+  push:
+    branches:
+      - master
+      - 'release-*'
+  pull_request:
+    branches:
+      - master
+      - 'release-*'
+  workflow_dispatch: {}
+
+permissions:
+  contents: read
+  packages: write
+
+env:
+  IMAGE_REGISTRY: ghcr.io
+  IMAGE_ORG: ${{ github.repository_owner }}
+  IMAGE_TAG: ${{ github.event_name == 'pull_request' && github.base_ref || github.ref_name }}
+
+concurrency:
+  group: build-tools-images-${{ github.ref_name }}
+  cancel-in-progress: false
+
+jobs:
+  build-tools:
+    name: Build tools images
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v5
+
+      - name: Log in to the Container registry
+        uses: docker/login-action@v3
+        if: github.event_name != 'pull_request'
+        with:
+          registry: ${{ env.IMAGE_REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build and push api-generator
+        id: build_api
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          file: backend/api/Dockerfile
+          push: ${{ github.event_name != 'pull_request' }}
+          tags: |
+            ${{ env.IMAGE_REGISTRY }}/${{ env.IMAGE_ORG }}/kfp-api-generator:${{ env.IMAGE_TAG }}
+
+      - name: Build and push release tools
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          file: test/release/Dockerfile.release
+          push: ${{ github.event_name != 'pull_request' }}
+          build-args: |
+            BASE_IMAGE=${{ env.IMAGE_REGISTRY }}/${{ env.IMAGE_ORG }}/kfp-api-generator:${{ env.IMAGE_TAG }}
+          tags: |
+            ${{ env.IMAGE_REGISTRY }}/${{ env.IMAGE_ORG }}/kfp-release:${{ env.IMAGE_TAG }}

api/Makefile

@@ -13,7 +13,7 @@
 # limitations under the License.
 
 # Contact one of chensun, HumairAK if this remote image needs an update.
-PREBUILT_REMOTE_IMAGE=ghcr.io/kubeflow/kfp-api-generator:1.2
+PREBUILT_REMOTE_IMAGE=ghcr.io/kubeflow/kfp-api-generator:master
 
 .PHONY: all
 all: golang python

backend/api/Dockerfile

@@ -13,15 +13,15 @@
 # limitations under the License.
 
 # Generate client code (go & json) from API protocol buffers
-FROM golang:1.24 as generator
-ENV GRPC_GATEWAY_VERSION v2.27.1
-ENV GO_SWAGGER_VERSION v0.32.3
-ENV GRPC_VERSION v1.73.0
-ENV PROTOC_VERSION 31.1
+FROM golang:1.24 AS generator
+ENV GRPC_GATEWAY_VERSION=v2.27.1
+ENV GO_SWAGGER_VERSION=v0.32.3
+ENV GRPC_VERSION=v1.73.0
+ENV PROTOC_VERSION=31.1
 ENV GOBIN=/go/bin
 # The googleapis repo doesn't use GitHub releases or version tags,
 # so we pin a specific commit to make the clone reproducible.
-ENV GOOGLEAPIS_COMMIT 68d5196a529174df97c28c70622ffc1c3721815f
+ENV GOOGLEAPIS_COMMIT=68d5196a529174df97c28c70622ffc1c3721815f
 
 # **Note** that protoc-gen-go-grpc is packaged with grpc-go but is versioned
 # separately. You can find the releases for protoc-gen-go-grpc here:
@@ -37,7 +37,7 @@ ENV GOOGLEAPIS_COMMIT 68d5196a529174df97c28c70622ffc1c3721815f
 # **Note** That BOTH PROTOC_GEN_GO_GRPC & PROTOBUF_GO here are used for
 # Generating GO Code. These versions should be identical to the
 # runtime Go packages (in the project go.mod)
-ENV PROTOC_GEN_GO_GRPC v1.5.1
+ENV PROTOC_GEN_GO_GRPC=v1.5.1
 ENV PROTOBUF_GO=v1.36.6
 
 # Install protoc.
@@ -49,8 +49,8 @@ RUN unzip -o protoc.zip -d /tmp/protoc && \
 RUN unzip -o protoc.zip 'include/*' -d /tmp/protoc && \
     mv /tmp/protoc/include/* /usr/include
 RUN rm -f protoc.zip
-ENV PROTOCCOMPILER /usr/bin/protoc
-ENV PROTOCINCLUDE /usr/include/google/protobuf
+ENV PROTOCCOMPILER=/usr/bin/protoc
+ENV PROTOCINCLUDE=/usr/include/google/protobuf
 
 # Need grpc-gateway source code for -I in protoc command.
 # Install protoc-gen-rpc-gateway && protoc-gen-swagger.
@@ -77,10 +77,10 @@ RUN go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@${PROTOC_GEN_GO_GRP
 
 # Needed for buildling python packages requiring protoc
 RUN apt-get update && apt-get install -y python3-pip
-RUN pip3 install --upgrade pip setuptools wheel --break-system-packages
+RUN apt-get -y upgrade python3-pip && pip3 install --upgrade setuptools wheel --break-system-packages
 
 # WORKAROUND: https://github.com/docker-library/golang/issues/225#issuecomment-403170792
-ENV XDG_CACHE_HOME /tmp/.cache
+ENV XDG_CACHE_HOME=/tmp/.cache
 # Make all files accessible to non-root users.
 RUN chmod -R 777 /usr/bin/
 RUN chmod -R 777 /usr/include/google

backend/api/Makefile

@@ -21,8 +21,8 @@ REMOTE_IMAGE=ghcr.io/kubeflow/kfp-api-generator
 API_VERSION ?= v2beta1
 
 # Keep in sync with the version used in test/release/Dockerfile.release
-PREBUILT_REMOTE_IMAGE=ghcr.io/kubeflow/kfp-api-generator:1.2
-RELEASE_IMAGE=ghcr.io/kubeflow/kfp-release:1.2.1
+PREBUILT_REMOTE_IMAGE=ghcr.io/kubeflow/kfp-api-generator:master
+RELEASE_IMAGE=ghcr.io/kubeflow/kfp-release:master
 CONTAINER_ENGINE ?= docker
 
 # Generate clients using a pre-built api-generator image.

backend/api/README.md

@@ -77,13 +77,16 @@ API definitions in this folder are used to generate [`v1beta1`](https://www.kube
 
 4. Create a PR with the changes in [kubeflow.org website repository](https://github.com/kubeflow/website). See an example [here](https://github.com/kubeflow/website/pull/3444).
 
-## Updating API generator image
+## Updating API generator image (Manual)
+
+This is now automatic on pushes to GitHub branches, but the instructions are kept here in case you need to do it
+manually.
 
 API generator image is defined in [Dockerfile](`./Dockerfile`). If you need to update the container, follow these steps:
 
 1. Login to GHCR container registry: `echo "<PAT>" | docker login ghcr.io -u <USERNAME> --password-stdin` 
    * Replace `<PAT>` with a GitHub Personal Access Token (PAT) with the write:packages and `read:packages` scopes, as well as `delete:packages` if needed. 
-1. Update the [Dockerfile](`./Dockerfile`) and build the image by running `docker build -t ghcr.io/kubeflow/kfp-api-generator:$VERSION .`
-1. Push the new container by running `docker push ghcr.io/kubeflow/kfp-api-generator:$VERSION`.
+1. Update the [Dockerfile](`./Dockerfile`) and build the image by running `docker build -t ghcr.io/kubeflow/kfp-api-generator:$BRANCH .`
+1. Push the new container by running `docker push ghcr.io/kubeflow/kfp-api-generator:$BRANCH`.
 1. Update the `PREBUILT_REMOTE_IMAGE` variable in the [Makefile](./Makefile) to point to your new image.
-1. Similarly, push a new version of the release tools image to `ghcr.io/kubeflow/kfp-release:$VERSION` and run `make push` in [test/release/Makefile](../../test/release/Makefile).
+1. Similarly, push a new version of the release tools image to `ghcr.io/kubeflow/kfp-release:$BRANCH` and run `make push` in [test/release/Makefile](../../test/release/Makefile).

kubernetes_platform/Makefile

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-PREBUILT_REMOTE_IMAGE=ghcr.io/kubeflow/kfp-api-generator:1.2
+PREBUILT_REMOTE_IMAGE=ghcr.io/kubeflow/kfp-api-generator:master
 
 # Set USE_FIND_LINKS=true to use local sdk and pipeline_spec for buildling python builds rather than pulling from pypi
 # applies to `make python` and make pythone-dev` commands.

sdk/Makefile

@@ -13,7 +13,7 @@
 # limitations under the License.
 
 # Contact one of chensun, HumairAK if this remote image needs an update.
-PREBUILT_REMOTE_IMAGE=ghcr.io/kubeflow/kfp-api-generator:1.2
+PREBUILT_REMOTE_IMAGE=ghcr.io/kubeflow/kfp-api-generator:master
 
 .PHONY: python
 python: python

test/release/Dockerfile.release

@@ -14,21 +14,23 @@
 
 # Based on KFP backend api client generator dockerfile
 # Keep in sync with the version used in backend/api/Makefile
-FROM ghcr.io/kubeflow/kfp-api-generator:1.2
+# Allow overriding the base image tag at build time
+ARG BASE_IMAGE=ghcr.io/kubeflow/kfp-api-generator:master
+FROM ${BASE_IMAGE}
 
 # install nvm & node 12
 # Reference: https://stackoverflow.com/a/28390848
-ENV NODE_VERSION 12.21.0
+ENV NODE_VERSION=12.21.0
 ENV NVM_DIR=/usr/local/nvm
 RUN mkdir -p $NVM_DIR && \
     curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.37.2/install.sh | bash && \
     . $NVM_DIR/nvm.sh && \
     nvm install $NODE_VERSION && \
     nvm alias default $NODE_VERSION && \
     nvm use default
-ENV NODE_PATH $NVM_DIR/versions/node/v$NODE_VERSION/lib/node_modules
-ENV PATH $NVM_DIR/versions/node/v$NODE_VERSION/bin:$PATH
-ENV GIT_CLIFF_VERSION 2.10.0
+ENV NODE_PATH=$NVM_DIR/versions/node/v$NODE_VERSION/lib/node_modules
+ENV PATH=$NVM_DIR/versions/node/v$NODE_VERSION/bin:$PATH
+ENV GIT_CLIFF_VERSION=2.10.0
 
 # install java==11 python==3
 RUN apt-get update \

test/release/Makefile

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-REMOTE=ghcr.io/kubeflow/kfp-release:1.2.1
+REMOTE=ghcr.io/kubeflow/kfp-release:master
 
 CONTAINER_ENGINE ?= docker
 

test/release/bump-version-docker.sh

@@ -31,7 +31,7 @@ if [[ -z "$TAG_NAME" ]]; then
 fi
 
 pushd "${REPO_ROOT}"
-RELEASE_IMAGE=${RELEASE_IMAGE:-ghcr.io/kubeflow/kfp-release:1.2.1}
+RELEASE_IMAGE=${RELEASE_IMAGE:-ghcr.io/kubeflow/kfp-release:master}
 docker run -it --rm \
   --user $(id -u):$(id -g) \
   --mount type=bind,source="$(pwd)",target=/go/src/github.com/kubeflow/pipelines \