Merge pull request #861 from Altinity/antalya_fix_get_pr_number

Antalya smarter pr number fetching for grype and report

Author: MyroTk

.github/actions/create_workflow_report/action.yml

@@ -10,7 +10,6 @@ runs:
   steps:
     - name: Create and upload workflow report
       env:
-        PR_NUMBER: ${{ github.event.pull_request.number || 0 }}
         COMMIT_SHA: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
         ACTIONS_RUN_URL: ${{ github.event.repository.html_url }}/actions/runs/${{ github.run_id }}
         FINAL: ${{ inputs.final }}
@@ -19,9 +18,9 @@ runs:
         pip install clickhouse-driver==0.2.8 numpy==1.26.4 pandas==2.0.3 jinja2==3.1.5
 
         CMD="python3 .github/actions/create_workflow_report/create_workflow_report.py"
-        ARGS="--pr-number $PR_NUMBER --commit-sha $COMMIT_SHA --actions-run-url $ACTIONS_RUN_URL --known-fails tests/broken_tests.json --cves"
+        ARGS="--commit-sha $COMMIT_SHA --actions-run-url $ACTIONS_RUN_URL --known-fails tests/broken_tests.json --cves"
 
-        set +e
+        set +e -x
         if [[ "$FINAL" == "false" ]]; then
           REPORT_LINK=$($CMD $ARGS --mark-preview)
         else

.github/actions/create_workflow_report/create_workflow_report.py

@@ -537,9 +537,7 @@ def main():
 
     # get_cves returns ... in the case where no Grype result files were found.
     # This might occur when run in preview mode.
-    cves_not_checked = not args.cves or (
-        args.mark_preview and fail_results["docker_images_cves"] is ...
-    )
+    cves_not_checked = not args.cves or fail_results["docker_images_cves"] is ...
 
     if args.known_fails:
         if not os.path.exists(args.known_fails):

.github/workflows/grype_scan.yml

@@ -15,6 +15,16 @@ on:
                 description: 'Docker image. If no tag, it will be determined by version_helper.py'
                 required: true
                 type: string
+            version:
+                description: 'Version tag. If no version, it will be determined by version_helper.py'
+                required: false
+                type: string
+                default: ""
+            tag-suffix:
+                description: 'Tag suffix. To be appended the version from version_helper.py'
+                required: false
+                type: string
+                default: ""
 env:
   PYTHONUNBUFFERED: 1
   AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
@@ -40,17 +50,25 @@ jobs:
             sudo apt-get install -y python3-pip python3-venv
             python3 -m venv venv
             source venv/bin/activate
-            pip install --upgrade requests chardet urllib3
+            pip install --upgrade requests chardet urllib3 unidiff boto3 PyGithub
             pip install testflows==$TESTFLOWS_VERSION awscli==1.33.28
             echo PATH=$PATH >>$GITHUB_ENV
 
         - name: Set image tag if not given
           if: ${{ !contains(inputs.docker_image, ':') }}
           id: set_version
+          env:
+            TAG_SUFFIX: ${{ inputs.tag-suffix }}
+            SPECIFIED_VERSION: ${{ inputs.version }}
           run: |
             python3 ./tests/ci/version_helper.py | tee /tmp/version_info
             source /tmp/version_info
-            echo "docker_image=${{ inputs.docker_image }}:${{ github.event.pull_request.number || 0 }}-$CLICKHOUSE_VERSION_STRING" >> $GITHUB_OUTPUT
+            if [ -z "$SPECIFIED_VERSION" ]; then
+                VERSION=$CLICKHOUSE_VERSION_STRING
+            else
+                VERSION=$SPECIFIED_VERSION
+            fi
+            echo "docker_image=${{ inputs.docker_image }}:$PR_NUMBER-$VERSION$TAG_SUFFIX" >> $GITHUB_OUTPUT
             echo "commit_sha=$CLICKHOUSE_VERSION_GITHASH" >> $GITHUB_OUTPUT
 
         - name: Run Grype Scan
@@ -68,9 +86,10 @@ jobs:
           env:
             S3_BUCKET: "altinity-build-artifacts"
             COMMIT_SHA: ${{ steps.set_version.outputs.commit_sha || github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
-            PR_NUMBER: ${{ github.event.pull_request.number || 0 }}
+            PR_NUMBER: ${{ env.PR_NUMBER || github.event.pull_request.number || 0 }}
             DOCKER_IMAGE: ${{ steps.set_version.outputs.docker_image || inputs.docker_image }}
           run: |
+            echo "PR_NUMBER=$PR_NUMBER"
             ./.github/grype/transform_and_upload_results_s3.sh
 
         - name: Create step summary

.github/workflows/release_branches.yml

@@ -201,7 +201,9 @@ jobs:
     uses: ./.github/workflows/grype_scan.yml
     secrets: inherit
     with:
-      docker_image: altinityinfra/clickhouse-${{ matrix.image }}:${{ github.event.pull_request.number || 0 }}-${{ fromJson(needs.RunConfig.outputs.data).version }}${{ matrix.suffix }}
+      docker_image: altinityinfra/clickhouse-${{ matrix.image }}
+      version: ${{ fromJson(needs.RunConfig.outputs.data).version }}
+      tag-suffix: ${{ matrix.suffix }}
 ############################################################################################
 ##################################### BUILD REPORTER #######################################
 ############################################################################################

tests/ci/version_helper.py

@@ -4,6 +4,8 @@
 from pathlib import Path
 from typing import Any, Dict, Iterable, List, Literal, Optional, Set, Tuple, Union
 
+from pr_info import PRInfo  # grype scan needs to know the PR number
+
 from git_helper import TWEAK, Git, get_tags, git_runner, removeprefix, VersionType
 
 FILE_WITH_VERSION_PATH = "cmake/autogenerated_versions.txt"
@@ -532,6 +534,12 @@ def main():
     if args.update_part or args.update_cmake:
         update_cmake_version(version)
 
+    # grype scan needs to know the PR number
+    pr_info = PRInfo()
+    print(f"PR_NUMBER={pr_info.number}")
+    if args.export:
+        print(f"export PR_NUMBER")
+
     for k, v in version.as_dict().items():
         name = f"CLICKHOUSE_VERSION_{k.upper()}"
         print(f"{name}='{v}'")