Fix: Check for container signature during release

Previously, container releases checked for the container on the
remote server, and skip the release process if it is found. However,
a failure in signing could occur after the push to remote, and if
this step failed, it would never re-run because the entire release
step would be skipped.

This change adds a step to also verify that the signature is also on
the remote if the image is found, and then attempts to sign the image
if the signature is not present. To do this, the following changes
were made:
  * Move installation of cosign binary to before image check
  * Add public key credential for verification
  * Run only cosign command if image is found on remote but not sig

Issue: IT-29095
Change-Id: I025b1662238df38d558e2a31c96f4fa223d8ca3f
Signed-off-by: Eric Ball <eball@linuxfoundation.org>

Author: eb-oss

jjb/lf-release-jobs.yaml

@@ -197,6 +197,7 @@
     build-timeout: 15
     cosign-password-id: cosign-password
     cosign-private-key-id: cosign-private-key
+    cosign-public-key-id: cosign-public-key
     disable-job: false
     git-url: "$GIT_URL/$PROJECT"
     stream: master
@@ -278,6 +279,9 @@
           - text:
               credential-id: "{cosign-password-id}"
               variable: COSIGN_PASSWORD
+          - file:
+              credential-id: "{cosign-public-key-id}"
+              variable: COSIGN_PUBLIC_KEY
 
     scm:
       - lf-infra-gerrit-scm:
@@ -329,6 +333,9 @@
           - text:
               credential-id: "{cosign-password-id}"
               variable: COSIGN_PASSWORD
+          - file:
+              credential-id: "{cosign-public-key-id}"
+              variable: COSIGN_PUBLIC_KEY
 
     scm:
       - lf-infra-github-scm:

releasenotes/notes/cosign-verify-8600a1dd35b1e901.yaml

@@ -0,0 +1,8 @@
+---
+fixes:
+  - |
+    When a container release is checked, the job will also check for a cosign
+    signature. This fixes an issue where a docker image could be successfully
+    pushed, but cosign would fail. This would lead to a failed job, and upon
+    re-running the job, it would pass when the container was found on the server,
+    without ever checking the status of the signature.

shell/release-job.sh

@@ -432,9 +432,26 @@ container_release_file(){
         echo "$name"
         echo "$version"
         echo "INFO: Merge will release $name $version as $VERSION"
+        curl -O -L "https://github.com/sigstore/cosign/releases/latest/download/cosign-linux-amd64"
+        sudo mv cosign-linux-amd64 /usr/local/bin/cosign
+        sudo chmod +x /usr/local/bin/cosign
         # Attempt to pull from releases registry to see if the image has been released.
         if docker pull "$CONTAINER_PUSH_REGISTRY"/"$lfn_umbrella"/"$name":"$VERSION"; then
-            echo "INFO: $VERSION is already released for image $name, Continuing..."
+            echo "INFO: $VERSION is already released for image $name, checking signature..."
+            image_digest=$(docker inspect --format='{{index .RepoDigests 0}}' \
+                    "$CONTAINER_PUSH_REGISTRY"/"$lfn_umbrella"/"$name":"$VERSION")
+            cosign verify --key "$COSIGN_PUBLIC_KEY" "$image_digest"
+            cosign_verified=$?
+            if [ "$cosign_verified" -eq 0 ]; then
+                echo "INFO: $name:$VERSION is already signed, continuing..."
+            elif [ "$cosign_verified" -eq 10 ] && [[ "$JOB_NAME" =~ "merge" ]]; then
+                # Exit code 10 indicates the package was found without signature
+                echo "INFO: No signature found for $name:$VERSION. Attempting to sign..."
+                export COSIGN_PASSWORD
+                cosign sign -y --key "$COSIGN_PRIVATE_KEY" "$image_digest"
+            else
+                echo "INFO: Could not verify signature, cosign exited with code $cosign_verified."
+            fi
         else
             echo "INFO: $VERSION not found in releases, release will be prepared. Continuing..."
             docker pull "$CONTAINER_PULL_REGISTRY"/"$lfn_umbrella"/"$name":"$version"
@@ -443,14 +460,11 @@ container_release_file(){
             echo "docker tag $container_image_id $CONTAINER_PUSH_REGISTRY/$lfn_umbrella/$name:$VERSION"
             echo "docker push $CONTAINER_PUSH_REGISTRY/$lfn_umbrella/$name:$VERSION"
             if [[ "$JOB_NAME" =~ "merge" ]]; then
-                curl -O -L "https://github.com/sigstore/cosign/releases/latest/download/cosign-linux-amd64"
-                sudo mv cosign-linux-amd64 /usr/local/bin/cosign
-                sudo chmod +x /usr/local/bin/cosign
-                export COSIGN_PASSWORD
                 docker tag "$container_image_id" "$CONTAINER_PUSH_REGISTRY"/"$lfn_umbrella"/"$name":"$VERSION"
                 docker push "$CONTAINER_PUSH_REGISTRY"/"$lfn_umbrella"/"$name":"$VERSION"
                 image_digest=$(docker inspect --format='{{index .RepoDigests 0}}' \
                         "$CONTAINER_PUSH_REGISTRY"/"$lfn_umbrella"/"$name":"$VERSION")
+                export COSIGN_PASSWORD
                 cosign sign -y --key "$COSIGN_PRIVATE_KEY" "$image_digest"
             fi
             echo "#########################"