ci(images): Add sbom to docker builds for licensing reasons (#7068)

* Add sbom to docker builds

* Fix makefile buildx commands, tested manually

* add `docker-build-and-push-prod` command for `alibi-explain-server`

* add `docker-build-and-push-prod` for `alibi-detect-server`

* add commands for xgboostserver to build and push image with SBOM

* Enable `images` workflow to test from non-master branch

* Fix pushing python wrapper image with SBOM

* Attempt to fix a digest error when pulling the wrapper image in the prepackaged servers job

* fix jq command for fetching the wrapper digest

* Add step `Force docker to use the docker-container driver` to the `prepackaged-components` job

* Fix building alibi-detect-server (sdk wasn't copied)

* Try to use `docker/setup-buildx-action` with `driver` arg to use docker-container

* Optimising more space for alibi-detect-server image

* Add necessary commands to attach SBOM for mlflowserver, sklearnserver and tfserving_proxy

* Move models images push to the separate job

* Revert "Move models images push to the separate job"

This reverts commit 74b3787.

* Ensure models are pushed as paart of the prepackaged-components job

* Revert original triggers for the `images` workflow

---------

Co-authored-by: Antanina Vertsinskaya <[email protected]>

Author: vtaskow

.github/workflows/images.yml

@@ -25,6 +25,9 @@ jobs:
 
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
+        with:
+          driver: docker-container
+          cleanup: true
 
       - name: Login to DockerHub
         uses: docker/login-action@v3
@@ -38,20 +41,22 @@ jobs:
           USER_INPUT="${{ github.event.inputs.docker-tag }}"
           echo "value=${USER_INPUT:-latest}" >> $GITHUB_OUTPUT
 
-
       - name: Build and push
         working-directory: ./operator/
         env:
           VERSION: ${{ steps.docker-tag.outputs.value }}
         run: |
-          make docker-build docker-push
+          make docker-build-and-push-prod
 
   executor:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
+        with:
+          driver: docker-container
+          cleanup: true
 
       - name: Login to DockerHub
         uses: docker/login-action@v3
@@ -70,7 +75,7 @@ jobs:
         env:
           VERSION: ${{ steps.docker-tag.outputs.value }}
         run: |
-          make docker-build docker-push
+          make docker-build-and-push-prod
 
   rclone-storage-initializer:
     runs-on: ubuntu-latest
@@ -90,12 +95,18 @@ jobs:
           USER_INPUT="${{ github.event.inputs.docker-tag }}"
           echo "value=${USER_INPUT:-latest}" >> $GITHUB_OUTPUT
 
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+        with:
+          driver: docker-container
+          cleanup: true
+
       - name: Build and push (Rclone Storage Initializer)
         working-directory: ./components/rclone-storage-initializer
         env:
           VERSION: ${{ steps.docker-tag.outputs.value }}
         run: |
-          make docker-build docker-push
+          make docker-build-and-push-prod
 
   s2i-wrapper:
     runs-on: ubuntu-latest
@@ -105,6 +116,9 @@ jobs:
 
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
+        with:
+          driver: docker-container
+          cleanup: true
 
       - name: Login to DockerHub
         uses: docker/login-action@v3
@@ -123,22 +137,25 @@ jobs:
         env:
           VERSION: ${{ steps.docker-tag.outputs.value }}
         run: |
-          make docker-build-conda-base docker-push-conda-base
+          make docker-build-and-push-prod-conda-base
 
       - name: Build and push (Base Wrapper)
         working-directory: ./wrappers/s2i/python
         env:
           VERSION: ${{ steps.docker-tag.outputs.value }}
         run: |
-          make docker-build docker-push PYTHON_VERSION=3.12.12
-          make docker-tag-base-python docker-push-base-python PYTHON_VERSION=3.12.12
-          docker save -o /tmp/base-wrapper.tar seldonio/seldon-core-s2i-python312:${VERSION}
+          make docker-build-and-push-prod PYTHON_VERSION=3.12.12
+          make docker-tag-and-push-prod-default
+          
+          docker buildx imagetools inspect \
+            seldonio/seldon-core-s2i-python312:${VERSION} \
+            --format '{{json .}}' > /tmp/base-wrapper.json
 
       - name: Upload artifact
         uses: actions/upload-artifact@v4
         with:
           name: base-wrapper
-          path: /tmp/base-wrapper.tar
+          path: /tmp/base-wrapper.json
 
   prepackaged-components:
     runs-on: ubuntu-latest
@@ -159,11 +176,14 @@ jobs:
       - name: Checkout Git Commit
         uses: actions/checkout@v4
 
-      - name: Free up disk space (android, haskell, dotnet)
+      - name: Free up disk space (android, haskell, dotnet, toolchains, caches)
         run: |
           sudo rm -rf /usr/local/lib/android || true
           sudo rm -rf /opt/ghc || true
           sudo rm -rf /usr/share/dotnet || true
+          sudo rm -rf "$AGENT_TOOLSDIRECTORY" || true
+          sudo rm -rf /opt/hostedtoolcache || true
+          sudo rm -rf /home/runner/.cache/pip || true
           df -h
       
       - name: Login to DockerHub
@@ -191,16 +211,28 @@ jobs:
           name: base-wrapper
           path: /tmp
 
-      - name: Load image
+      - name: Extract image digest and pull
         run: |
-          docker load --input /tmp/base-wrapper.tar
+          DIGEST=$(jq -r '.manifest.digest' /tmp/base-wrapper.json)
+          IMAGE_REF="seldonio/seldon-core-s2i-python312@${DIGEST}"
+  
+          docker pull "${IMAGE_REF}"
 
       - name: Remove tarball
-        run: rm -f /tmp/base-wrapper.tar
+        run: rm -f /tmp/base-wrapper.json
+
+      - name: Remove base-wrapper artifact directory
+        run: rm -rf /tmp/base-wrapper || true
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+        with:
+          driver: docker-container
+          cleanup: true
 
       - name: Build and push (Prepackaged Server Image)
         working-directory: ./${{ matrix.server }}/
         env:
           VERSION: ${{ steps.docker-tag.outputs.value }}
         run: |
-          make docker-build docker-push
+          make docker-build-and-push-prod

components/alibi-detect-server/Makefile

@@ -95,6 +95,9 @@ curl-metrics-server-get:
 docker-build: get_local_repo
 	docker build --platform linux/amd64 -f Dockerfile --build-arg BASE_IMAGE=${BASE_IMAGE} --build-arg VERSION=${VERSION} -t ${IMAGE}:${VERSION} .
 
+docker-build-and-push-prod: get_local_repo
+	docker buildx build --provenance=true --attest type=sbom,generator=docker/scout-sbom-indexer:latest --platform linux/amd64 -f Dockerfile --build-arg BASE_IMAGE=${BASE_IMAGE} --build-arg VERSION=${VERSION} -t ${IMAGE}:${VERSION} . --push
+
 docker-push:
 	docker push ${IMAGE}:${VERSION}
 

components/alibi-explain-server/Makefile

@@ -51,6 +51,9 @@ docker-build:
 docker-push:
 	docker push ${IMAGE}:${VERSION}
 
+docker-build-and-push-prod:
+	docker buildx build --provenance=true --attest type=sbom,generator=docker/scout-sbom-indexer:latest --platform linux/amd64 -f Dockerfile --build-arg BASE_IMAGE=${BASE_IMAGE} --build-arg VERSION=${VERSION} -t ${IMAGE}:${VERSION} . --push
+
 kind_load: docker-build
 	kind load docker-image ${IMAGE}:${VERSION} --name ${KIND_NAME}
 

components/rclone-storage-initializer/Makefile

@@ -10,6 +10,9 @@ IMAGE_TAG = ${DOCKER_REGISTRY}/${IMAGE}:${VERSION}
 docker-build:
 	docker build --file=Dockerfile --force-rm=true -t ${IMAGE_TAG} .
 
+docker-build-and-push-prod:
+	docker buildx build --file=Dockerfile --force-rm=true -t ${IMAGE_TAG} . --provenance=true --attest type=sbom,generator=docker/scout-sbom-indexer:latest --push
+
 docker-push:
 	docker push ${DOCKER_REGISTRY}/${IMAGE}:${VERSION}
 

components/routers/epsilon-greedy/Makefile

@@ -8,5 +8,7 @@ docker-build:
 docker-push:
 	docker push $(IMAGE_NAME):$(VERSION)
 
+docker-build-and-push-prod: docker-build docker-push
+
 kind_load: docker-build
 	kind load -v 3 docker-image ${IMAGE_NAME}:${VERSION} --name ${KIND_NAME}

examples/models/mean_classifier/Makefile

@@ -12,6 +12,8 @@ docker-build:
 docker-push:
 	docker push ${IMAGE_BASE}:${VERSION}
 
+docker-build-and-push-prod: docker-build docker-push
+
 run_local:
 	export PREDICTIVE_UNIT_HTTP_SERVICE_PORT=9001 && export PREDICTIVE_UNIT_GRPC_SERVICE_PORT=5001 && export TRACING=1 && export JAEGER_AGENT_HOST=localhost && export JAEGER_AGENT_PORT=6831 && export JAEGER_SAMPLER_TYPE=const && export JAEGER_SAMPLER_PARAM=1 && export SELDON_DEBUG=0 && seldon-core-microservice --service-type MODEL MeanClassifier
 

executor/Makefile

@@ -97,6 +97,9 @@ docker-build: copy_operator copy_openapi_resources
 docker-push:
 	docker push ${SELDON_EXECUTOR_IMG}
 
+docker-build-and-push-prod: copy_operator copy_openapi_resources
+	docker buildx build -f Dockerfile.executor -t ${SELDON_EXECUTOR_IMG} . --provenance=true --attest type=sbom,generator=docker/scout-sbom-indexer:latest --push
+
 
 # Red Hat Related
 

operator/Makefile

@@ -177,6 +177,9 @@ docker-build: generate-resources
 docker-push:
 	docker push ${SELDON_OPERATOR_IMG}
 
+docker-build-and-push-prod: generate-resources
+	docker buildx build . -t ${SELDON_OPERATOR_IMG} --provenance=true --attest type=sbom,generator=docker/scout-sbom-indexer:latest --push
+
 # Installing Images in Kind CLusters
 kind-image-install: docker-build
 	kind load -v 3 docker-image ${SELDON_OPERATOR_IMG} --name ${KIND_NAME}

servers/mlflowserver/Dockerfile.s2i

@@ -0,0 +1,3 @@
+ARG BASE_IMAGE
+ARG BASE_TAG
+FROM ${BASE_IMAGE}:${BASE_TAG}

servers/mlflowserver/Makefile

@@ -5,6 +5,7 @@ DOCKER_REGISTRY ?= seldonio
 
 IMAGE_NAME_BASE = mlflowserver
 IMAGE_NAME = ${DOCKER_REGISTRY}/${IMAGE_NAME_BASE}
+IMAGE_NAME_PROD_BASE=${IMAGE_NAME}-prod-base
 KIND_NAME ?= kind
 
 BASE_IMAGE = ${DOCKER_REGISTRY}/seldon-core-s2i-python312:${VERSION}
@@ -19,6 +20,15 @@ docker-build:
 docker-push:
 	docker push ${IMAGE_NAME}:${VERSION}
 
+# Build and push s2i base image to the private repository in dockerhub
+# After it's used as a base image for building the final image with SBOM attached
+docker-push-prod-base: docker-build
+	docker tag ${IMAGE_NAME}:${VERSION} ${IMAGE_NAME_PROD_BASE}:${VERSION}
+	docker push ${IMAGE_NAME_PROD_BASE}:${VERSION}
+
+docker-build-and-push-prod: docker-push-prod-base
+	docker buildx build --provenance=true --attest type=sbom,generator=docker/scout-sbom-indexer:latest -f Dockerfile.s2i --build-arg BASE_IMAGE=${IMAGE_NAME_PROD_BASE} --build-arg BASE_TAG=${VERSION} -t ${IMAGE_NAME}:${VERSION} . --push
+
 kind-load: docker-build
 	kind load -v 3 docker-image ${IMAGE_NAME}:${VERSION} --name ${KIND_NAME}