Add image exist checks to tag builds.

HumairAK · HumairAK · commit 09f3d02261e4 · 2023-07-31T13:28:56.000-04:00
Signed-off-by: Humair Khan &lt;HumairAK@users.noreply.github.com&gt;
diff --git a/.github/actions/build/action.yaml b/.github/actions/build/action.yaml
@@ -0,0 +1,63 @@
+name: "Set up KinD"
+description: "Step to start and configure KinD cluster"
+inputs:
+  IMAGE_REPO:
+    description: "Quay image repo name."
+    required: true
+  DOCKERFILE:
+    description: "Path to Dockerfile."
+    required: true
+  GH_REPO:
+    description: "GH org/repo that contains the dockerfile to source."
+    required: true
+  OVERWRITE:
+    default: "false"
+    description: "GH org/repo that contains the dockerfile to source."
+    required: true
+runs:
+  using: "composite"
+  steps:
+    - uses: actions/checkout@v3
+      with:
+        repository: ${{ inputs.GH_REPO }}
+        ref: ${{ env.SOURCE_BRANCH }}
+        path: build
+    - name: Login to Quay.io
+      uses: redhat-actions/podman-login@v1
+      with:
+        username: ${{ env.QUAY_ID }}
+        password: ${{ env.QUAY_TOKEN }}
+        registry: quay.io
+    # Tags in quay stick around as objects in api, when deleted quay adds "end_ts" time stamp on the tag.
+    # To determine a tag is deleted, we need to check for the presence of this tag.
+    # Also note there can be multiple tags created/deleted, thus we have 4 cases:
+    # Case 1: Only 1 tag was ever created "tags: [{name:..},]"   -- no end_ts field
+    # Case 2: No tag was ever created "tags: []"
+    # Case 3: >1 tags were created, but they were all deleted at some point [{name:..., end_ts,..},....]  -- note they all have "end_ts" field.
+    # Case 4: >1 tags were created, but the most recent one was never deleted (same as case 3, but the latest tag does not have "end_ts".
+    - name: Check if Image already exists
+      shell: bash
+      if: ${{ inputs.OVERWRITE }} == "false"
+      env:
+        IMAGE: quay.io/${{ env.QUAY_ORG }}/${{ inputs.IMAGE_REPO }}:${{ env.TARGET_IMAGE_TAG }}
+      run: |
+        tags=$(curl --request GET 'https://quay.io/api/v1/repository/${{ env.QUAY_ORG }}/${{ inputs.IMAGE_REPO }}/tag/?specificTag=${{ env.TARGET_IMAGE_TAG }}')
+        echo $tags | yq .tags - | yq 'sort_by(.start_ts) | reverse' - -P | yq .[0].end_ts -
+        latest_tag_has_end_ts=$(echo $tags | yq .tags - | yq 'sort_by(.start_ts) | reverse' - -P | yq .[0].end_ts -)
+        echo $latest_tag_has_end_ts
+        notempty=$(echo ${tags} | yq .tags - | yq any)
+
+        # Image only exists if there is a tag that does not have "end_ts" (i.e. it is still present).
+        if [[ "$notempty" == "true" && $latest_tag_has_end_ts == "null" ]]; then
+            echo "::error::The image ${{ env.IMAGE }} already exists"
+            exit 1
+        else
+            echo "Image does not exist...proceeding with build & push."
+        fi
+    - name: Build image
+      shell: bash
+      working-directory: build
+      env:
+        IMAGE: quay.io/${{ env.QUAY_ORG }}/${{ inputs.IMAGE_REPO }}:${{ env.TARGET_IMAGE_TAG }}
+      run: |
+          podman build . -f ${{ inputs.DOCKERFILE }} -t ${{ env.IMAGE }} && podman push ${{ env.IMAGE }}
diff --git a/.github/workflows/tag-and-build.yml b/.github/workflows/tag-and-build.yml
@@ -20,248 +20,151 @@ on:
         description: 'DSP org/repo'
         required: true
 env:
-  IMAGE_DSPO: data-science-pipelines-operator
-  IMAGE_SERVER: ds-pipelines-api-server
-  IMAGE_UI: ds-pipelines-frontend
-  IMAGE_CACHE: ds-pipelines-cacheserver
-  IMAGE_PA: ds-pipelines-persistenceagent
-  IMAGE_SWF: ds-pipelines-scheduledworkflow
-  IMAGE_VC: ds-pipelines-viewercontroller
-  IMAGE_ARTIFACT: ds-pipelines-artifact-manager
-  IMAGE_MLMD_WRITER: ds-pipelines-metadata-writer
-  IMAGE_MLMD_ENVOY: ds-pipelines-metadata-envoy
-  IMAGE_MLMD_GRPC: ds-pipelines-metadata-grpc
+  IMAGE_REPO_DSPO: data-science-pipelines-operator
+  IMAGE_REPO_SERVER: ds-pipelines-api-server
+  IMAGE_REPO_UI: ds-pipelines-frontend
+  IMAGE_REPO_CACHE: ds-pipelines-cacheserver
+  IMAGE_REPO_PA: ds-pipelines-persistenceagent
+  IMAGE_REPO_SWF: ds-pipelines-scheduledworkflow
+  IMAGE_REPO_VC: ds-pipelines-viewercontroller
+  IMAGE_REPO_ARTIFACT: ds-pipelines-artifact-manager
+  IMAGE_REPO_MLMD_WRITER: ds-pipelines-metadata-writer
+  IMAGE_REPO_MLMD_ENVOY: ds-pipelines-metadata-envoy
+  IMAGE_REPO_MLMD_GRPC: ds-pipelines-metadata-grpc
+  SOURCE_BRANCH: ${{ inputs.src_branch }}
+  QUAY_ORG: ${{ inputs.quay_org }}
+  QUAY_ID: ${{ secrets.QUAY_ID }}
+  QUAY_TOKEN: ${{ secrets.QUAY_TOKEN }}
+  TARGET_IMAGE_TAG: ${{ inputs.target_tag }}
 jobs:
   dspo-build:
     runs-on: ubuntu-latest
     permissions:
       contents: read
     steps:
-    - uses: actions/checkout@v3
-      with:
-        ref: ${{ inputs.src_branch }}
-
-    - name: Login to Quay.io
-      uses: redhat-actions/podman-login@v1
-      with:
-        username: ${{ secrets.QUAY_ID }}
-        password: ${{ secrets.QUAY_TOKEN }}
-        registry: quay.io
-
-    - name: Image Build and Push
-      run: |
-        make podman-build podman-push -e IMG=quay.io/${{ inputs.quay_org }}/${IMAGE_DSPO}:${{ inputs.target_tag }}
+      - uses: actions/checkout@v3
+      - uses: ./.github/actions/build
+        with:
+          IMAGE_REPO: ${{ env.IMAGE_REPO_DSPO }}
+          DOCKERFILE: Dockerfile
+          GH_REPO: ${{ github.repository }}
 
   server-build:
     runs-on: ubuntu-latest
     permissions:
       contents: read
     steps:
       - uses: actions/checkout@v3
+      - uses: ./.github/actions/build
         with:
-          repository: ${{ inputs.dsp_org_repo }}
-          ref: ${{ inputs.src_branch }}
-      - name: Login to Quay.io
-        uses: redhat-actions/podman-login@v1
-        with:
-          username: ${{ secrets.QUAY_ID }}
-          password: ${{ secrets.QUAY_TOKEN }}
-          registry: quay.io
-      - name: Buid image
-        env:
-          IMAGE: quay.io/${{ inputs.quay_org }}/${IMAGE_SERVER}:${{ inputs.target_tag }}
+          IMAGE_REPO: ${{ env.IMAGE_REPO_SERVER }}
           DOCKERFILE: backend/Dockerfile
-        run: |
-          RESULT=$(podman image exists ${{ env.IMAGE }})
-          if [ $RESULT -eq 1 ]; then echo "Image already exists" && exit 1; fi
-          podman build . -f ${{ env.DOCKERFILE }} -t ${{ env.IMAGE }} && podman push ${{ env.IMAGE }}
+          GH_REPO: ${{ inputs.dsp_org_repo }}
 
   ui-build:
     runs-on: ubuntu-latest
     permissions:
       contents: read
     steps:
       - uses: actions/checkout@v3
+      - uses: ./.github/actions/build
         with:
-          repository: ${{ inputs.dsp_org_repo }}
-          ref: ${{ inputs.src_branch }}
-      - name: Login to Quay.io
-        uses: redhat-actions/podman-login@v1
-        with:
-          username: ${{ secrets.QUAY_ID }}
-          password: ${{ secrets.QUAY_TOKEN }}
-          registry: quay.io
-      - name: Build image
-        env:
-          IMAGE: quay.io/${{ inputs.quay_org }}/${IMAGE_UI}:${{ inputs.target_tag }}
+          IMAGE_REPO: ${{ env.IMAGE_REPO_UI }}
           DOCKERFILE: frontend/Dockerfile
-        run: |
-          podman build . -f frontend/Dockerfile -t ${{ env.UI }} && podman push ${{ env.UI }}
+          GH_REPO: ${{ inputs.dsp_org_repo }}
 
   cache-build:
     runs-on: ubuntu-latest
     permissions:
       contents: read
     steps:
       - uses: actions/checkout@v3
+      - uses: ./.github/actions/build
         with:
-          repository: ${{ inputs.dsp_org_repo }}
-          ref: ${{ inputs.src_branch }}
-      - name: Login to Quay.io
-        uses: redhat-actions/podman-login@v1
-        with:
-          username: ${{ secrets.QUAY_ID }}
-          password: ${{ secrets.QUAY_TOKEN }}
-          registry: quay.io
-      - name: Build image
-        env:
-          IMAGE: quay.io/${{ inputs.quay_org }}/${IMAGE_CACHE}:${{ inputs.target_tag }}
-        run: |
-          podman build . -f backend/Dockerfile.cacheserver -t ${{ env.CACHE }} && podman push ${{ env.CACHE }}
+          IMAGE_REPO: ${{ env.IMAGE_REPO_CACHE }}
+          DOCKERFILE: backend/Dockerfile.cacheserver
+          GH_REPO: ${{ inputs.dsp_org_repo }}
 
   PA-build:
     runs-on: ubuntu-latest
     permissions:
       contents: read
     steps:
       - uses: actions/checkout@v3
+      - uses: ./.github/actions/build
         with:
-          repository: ${{ inputs.dsp_org_repo }}
-          ref: ${{ inputs.src_branch }}
-      - name: Login to Quay.io
-        uses: redhat-actions/podman-login@v1
-        with:
-          username: ${{ secrets.QUAY_ID }}
-          password: ${{ secrets.QUAY_TOKEN }}
-          registry: quay.io
-      - name: Build image
-        env:
-          IMAGE: quay.io/${{ inputs.quay_org }}/${IMAGE_PA}:${{ inputs.target_tag }}
-        run: |
-          podman build . -f backend/Dockerfile.persistenceagent -t ${{ env.PA }} && podman push ${{ env.PA }}
+          IMAGE_REPO: ${{ env.IMAGE_REPO_PA }}
+          DOCKERFILE: backend/Dockerfile.persistenceagent
+          GH_REPO: ${{ inputs.dsp_org_repo }}
 
   SWF-build:
     runs-on: ubuntu-latest
     permissions:
       contents: read
     steps:
       - uses: actions/checkout@v3
+      - uses: ./.github/actions/build
         with:
-          repository: ${{ inputs.dsp_org_repo }}
-          ref: ${{ inputs.src_branch }}
-      - name: Login to Quay.io
-        uses: redhat-actions/podman-login@v1
-        with:
-          username: ${{ secrets.QUAY_ID }}
-          password: ${{ secrets.QUAY_TOKEN }}
-          registry: quay.io
-      - name: Build image
-        env:
-          IMAGE: quay.io/${{ inputs.quay_org }}/${IMAGE_SWF}:${{ inputs.target_tag }}
-        run: |
-          podman build . -f backend/Dockerfile.scheduledworkflow -t ${{ env.SWF }} && podman push ${{ env.SWF }}
+          IMAGE_REPO: ${{ env.IMAGE_REPO_SWF }}
+          DOCKERFILE: backend/Dockerfile.scheduledworkflow
+          GH_REPO: ${{ inputs.dsp_org_repo }}
 
   VC-build:
     runs-on: ubuntu-latest
     permissions:
       contents: read
     steps:
       - uses: actions/checkout@v3
+      - uses: ./.github/actions/build
         with:
-          repository: ${{ inputs.dsp_org_repo }}
-          ref: ${{ inputs.src_branch }}
-      - name: Login to Quay.io
-        uses: redhat-actions/podman-login@v1
-        with:
-          username: ${{ secrets.QUAY_ID }}
-          password: ${{ secrets.QUAY_TOKEN }}
-          registry: quay.io
-      - name: Build image
-        env:
-          IMAGE: quay.io/${{ inputs.quay_org }}/${IMAGE_VC}:${{ inputs.target_tag }}
-        run: |
-          podman build . -f backend/Dockerfile.viewercontroller -t ${{ env.VC }} && podman push ${{ env.VC }}
+          IMAGE_REPO: ${{ env.IMAGE_REPO_VC }}
+          DOCKERFILE: backend/Dockerfile.viewercontroller
+          GH_REPO: ${{ inputs.dsp_org_repo }}
 
   ARTIFACT-build:
     runs-on: ubuntu-latest
     permissions:
       contents: read
     steps:
       - uses: actions/checkout@v3
+      - uses: ./.github/actions/build
         with:
-          repository: ${{ inputs.dsp_org_repo }}
-          ref: ${{ inputs.src_branch }}
-      - name: Login to Quay.io
-        uses: redhat-actions/podman-login@v1
-        with:
-          username: ${{ secrets.QUAY_ID }}
-          password: ${{ secrets.QUAY_TOKEN }}
-          registry: quay.io
-      - name: Build image
-        env:
-          IMAGE: quay.io/${{ inputs.quay_org }}/${IMAGE_ARTIFACT}:${{ inputs.target_tag }}
-        run: |
-          podman build . -f backend/artifact_manager/Dockerfile -t ${{ env.ARTIFACT }} && podman push ${{ env.ARTIFACT }}
+          IMAGE_REPO: ${{ env.IMAGE_REPO_ARTIFACT }}
+          DOCKERFILE: backend/artifact_manager/Dockerfile
+          GH_REPO: ${{ inputs.dsp_org_repo }}
 
   MLMD_WRITER-build:
     runs-on: ubuntu-latest
     permissions:
       contents: read
     steps:
       - uses: actions/checkout@v3
+      - uses: ./.github/actions/build
         with:
-          repository: ${{ inputs.dsp_org_repo }}
-          ref: ${{ inputs.src_branch }}
-      - name: Login to Quay.io
-        uses: redhat-actions/podman-login@v1
-        with:
-          username: ${{ secrets.QUAY_ID }}
-          password: ${{ secrets.QUAY_TOKEN }}
-          registry: quay.io
-      - name: Build image
-        env:
-          IMAGE: quay.io/${{ inputs.quay_org }}/${IMAGE_MLMD_WRITER}:${{ inputs.target_tag }}
-        run: |
-          podman build . -f backend/metadata_writer/Dockerfile -t ${{ env.MLMD_WRITER }} && podman push ${{ env.MLMD_WRITER }}
+          IMAGE_REPO: ${{ env.IMAGE_REPO_MLMD_WRITER }}
+          DOCKERFILE: backend/metadata_writer/Dockerfile
+          GH_REPO: ${{ inputs.dsp_org_repo }}
 
   MLMD_ENVOY-build:
     runs-on: ubuntu-latest
     permissions:
       contents: read
     steps:
       - uses: actions/checkout@v3
+      - uses: ./.github/actions/build
         with:
-          repository: ${{ inputs.dsp_org_repo }}
-          ref: ${{ inputs.src_branch }}
-      - name: Login to Quay.io
-        uses: redhat-actions/podman-login@v1
-        with:
-          username: ${{ secrets.QUAY_ID }}
-          password: ${{ secrets.QUAY_TOKEN }}
-          registry: quay.io
-      - name: Build image
-        env:
-          IMAGE: quay.io/${{ inputs.quay_org }}/${IMAGE_MLMD_ENVOY}:${{ inputs.target_tag }}
-        run: |
-          podman build . -f third-party/metadata_envoy/Dockerfile -t ${{ env.MLMD_ENVOY }} && podman push ${{ env.MLMD_ENVOY }}
+          IMAGE_REPO: ${{ env.IMAGE_REPO_MLMD_ENVOY }}
+          DOCKERFILE: third-party/metadata_envoy/Dockerfile
+          GH_REPO: ${{ inputs.dsp_org_repo }}
 
   MLMD_GRPC-build:
     runs-on: ubuntu-latest
     permissions:
       contents: read
     steps:
       - uses: actions/checkout@v3
+      - uses: ./.github/actions/build
         with:
-          repository: ${{ inputs.dsp_org_repo }}
-          ref: ${{ inputs.src_branch }}
-      - name: Login to Quay.io
-        uses: redhat-actions/podman-login@v1
-        with:
-          username: ${{ secrets.QUAY_ID }}
-          password: ${{ secrets.QUAY_TOKEN }}
-          registry: quay.io
-      - name: Build image
-        env:
-          IMAGE: quay.io/${{ inputs.quay_org }}/${IMAGE_MLMD_GRPC}:${{ inputs.target_tag }}
-        run: |
-          podman build . -f third-party/ml-metadata/Dockerfile -t ${{ env.MLMD_GRPC }} && podman push ${{ env.MLMD_GRPC }}
+          IMAGE_REPO: ${{ env.IMAGE_REPO_MLMD_GRPC }}
+          DOCKERFILE: third-party/ml-metadata/Dockerfile
+          GH_REPO: ${{ inputs.dsp_org_repo }}
