release cicd: update info extraction job

Author: MyroTk

.github/workflows/sign_and_release.yml

@@ -37,166 +37,43 @@ env:
 jobs:
   extract-package-info:
     runs-on: [altinity-style-checker-aarch64, altinity-on-demand]
+    env:
+      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+      FOLDER_TIME: ${{ inputs.folder_time }}
+      GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
     outputs:
       docker_version: ${{ env.DOCKER_VERSION }}-${{ env.PACKAGE_VERSION }}
       commit_hash: ${{ env.COMMIT_HASH }}
       folder_time: ${{ env.FOLDER_TIME }}
       needs_binary_processing: ${{ env.NEEDS_BINARY_PROCESSING }}
-      package_version: ${{ env.PACKAGE_VERSION }}
-      src_dir: ${{ env.SRC_DIR }}
       test_results_src: ${{ env.TEST_RESULTS_SRC }}
-      altinity_build_feature: ${{ env.ALTINITY_BUILD_FEATURE }}
       repo_prefix: ${{ env.REPO_PREFIX }}
       src_url: ${{ env.SRC_URL }}
       dest_url: ${{ env.DEST_URL }}
       major_version: ${{ env.MAJOR_VERSION }}
     steps:
-      - name: Validate inputs
-        run: |
-          if [ -z "${{ inputs.workflow_url }}" ]; then
-            echo "Error: workflow_url is required"
-            exit 1
-          fi
-          if [ -z "${{ inputs.package_version }}" ]; then
-            echo "Error: package_version is required"
-            exit 1
-          fi
-          if [ "${{ inputs.release_environment }}" != "staging" ] && [ "${{ inputs.release_environment }}" != "production" ]; then
-            echo "Error: release_environment must be either 'staging' or 'production'"
-            exit 1
-          fi
-
-      - name: Download artifact "${{ env.ARTIFACT_NAME }}"
-        run: |
-          run_id=$(echo "${{ inputs.workflow_url }}" | grep -oE '[0-9]+$')
-
-          # Get artifact ID
-          artifact_id=$(curl -s "https://api.github.com/repos/Altinity/ClickHouse/actions/runs/$run_id/artifacts" \
-            | jq '.artifacts[] | select(.name == "'"${{ env.ARTIFACT_NAME }}"'") | .id')
-
-          # Download artifact
-          curl -L -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" \
-            -o "${{ env.ARTIFACT_NAME }}" \
-            "https://api.github.com/repos/Altinity/ClickHouse/actions/artifacts/$artifact_id/zip"
-
-      - name: Unzip Artifact
-        run: |
-          unzip -o "${{ env.ARTIFACT_NAME }}" -d "artifact"
-
-      - name: Extract and Parse JSON File
-        run: |
-          cd artifact
-          JSON_FILE=$(ls | grep -E "report.*release.*\.json" | head -n 1)
-          if [ -z "$JSON_FILE" ]; then
-            echo "Error: No JSON file matching the pattern was found"
-            exit 1
-          fi
-          echo "Found JSON file: ${JSON_FILE}"
-
-          # Extract client URL
-          CLIENT_URL=$(jq -r '.build_urls[] | select(test("clickhouse-client-.*-amd64.tgz$"))' "$JSON_FILE")
-          if [ -z "$CLIENT_URL" ]; then
-            echo "Error: No matching client URL found in JSON"
-            exit 1
-          fi
-          echo "Found client URL: ${CLIENT_URL}"
-          echo "CLIENT_URL=$CLIENT_URL" >> $GITHUB_ENV
+      - name: Checkout repository
+        uses: actions/checkout@v4
 
-      - name: Extract and Validate Package Information
+      - name: Run script
         run: |
-          # Define regex patterns
-          PR_REGEX="PRs/([^/]+)/([^/]+)/([^/]+)/clickhouse-client-([^-]+)-amd64.tgz"
-          NONPR_REGEX="s3.amazonaws.com/([^/]+)/([^/]+)/([^/]+)/([^/]+)/clickhouse-client-([^-]+)-amd64.tgz"
-
-          # Extract information based on URL pattern
-          if [[ "$CLIENT_URL" =~ $PR_REGEX ]]; then
-            echo "Matched PR pattern"
-            PR_NUMBER="${BASH_REMATCH[1]}"
-            COMMIT_HASH="${BASH_REMATCH[2]}"
-            PACKAGE_TYPE="${BASH_REMATCH[3]}"
-            PACKAGE_VERSION="${BASH_REMATCH[4]}"
-            DOCKER_VERSION="${PR_NUMBER}"
-            TEST_RESULTS_SRC="${PR_NUMBER}"
-            SRC_DIR="PRs/${PR_NUMBER}"
-          elif [[ "$CLIENT_URL" =~ $NONPR_REGEX ]]; then
-            echo "Matched non-PR pattern"
-            BRANCH="${BASH_REMATCH[2]}"
-            COMMIT_HASH="${BASH_REMATCH[3]}"
-            PACKAGE_TYPE="${BASH_REMATCH[4]}"
-            PACKAGE_VERSION="${BASH_REMATCH[5]}"
-            DOCKER_VERSION="0"
-            TEST_RESULTS_SRC="0"
-            SRC_DIR="${BRANCH}"
-          else
-            echo "Error: The client URL did not match any expected pattern"
-            exit 1
-          fi
-
-          # Verify package version
-          if [ "$PACKAGE_VERSION" != "${{ inputs.package_version }}" ]; then
-            echo "Error: Extracted package version ($PACKAGE_VERSION) does not match input package version (${{ inputs.package_version }})"
-            exit 1
-          fi
-
-          # Extract major version and determine binary processing need
-          MAJOR_VERSION=$(echo "$PACKAGE_VERSION" | cut -d. -f1)
-          NEEDS_BINARY_PROCESSING=$([ "$MAJOR_VERSION" -ge 24 ] && echo "true" || echo "false")
-
-          #Tweak s3 path is major version is 25 or newer
-          if [ "$MAJOR_VERSION" -ge 25 ]; then
-            SRC_URL=s3://${SRC_BUCKET}/REFs/${SRC_DIR}/${COMMIT_HASH}
-          else
-            SRC_URL=s3://${SRC_BUCKET}/${SRC_DIR}/${COMMIT_HASH}
-          fi
-          
-          # Extract feature and set repo prefix
-          ALTINITY_BUILD_FEATURE=$(echo "$PACKAGE_VERSION" | rev | cut -d. -f1 | rev)
-          case "$ALTINITY_BUILD_FEATURE" in
-            "altinityhotfix") REPO_PREFIX="hotfix-" ;;
-            "altinityfips") REPO_PREFIX="fips-" ;;
-            "altinityantalya") REPO_PREFIX="antalya-" ;;
-            "altinitystable"|"altinitytest") REPO_PREFIX="" ;;
-            *)
-              echo "Error: Build feature not supported: ${ALTINITY_BUILD_FEATURE}"
-              exit 1
-              ;;
-          esac
-
-          # Generate folder time if not provided
-          if [ -z "${{ inputs.folder_time }}" ]; then
-            FOLDER_TIME=$(date -u +"%Y-%m-%dT%H-%M-%S.%3N")
-          else
-            FOLDER_TIME="${{ inputs.folder_time }}"
-          fi
-
-          # Set all environment variables at once
-          {
-            echo "COMMIT_HASH=${COMMIT_HASH}"
-            echo "DOCKER_VERSION=${DOCKER_VERSION}"
-            echo "FOLDER_TIME=${FOLDER_TIME}"
-            echo "NEEDS_BINARY_PROCESSING=${NEEDS_BINARY_PROCESSING}"
-            echo "PACKAGE_VERSION=${PACKAGE_VERSION}"
-            echo "SRC_DIR=${SRC_DIR}"
-            echo "TEST_RESULTS_SRC=${TEST_RESULTS_SRC}"
-            echo "ALTINITY_BUILD_FEATURE=${ALTINITY_BUILD_FEATURE}"
-            echo "REPO_PREFIX=${REPO_PREFIX}"
-            echo "SRC_URL=${SRC_URL}"
-            echo "DEST_URL=s3://${S3_STORAGE_BUCKET}/builds/stable/v${PACKAGE_VERSION}/${FOLDER_TIME}"
-            echo "MAJOR_VERSION=${MAJOR_VERSION}"
-          } >> $GITHUB_ENV
+          python3 tests/ci/release/extract-package-info.py \
+            --artifact-dest-bucket $S3_STORAGE_BUCKET \
+            --artifact-src-bucket $SRC_BUCKET \
+            --package-version ${{ inputs.package_version }} \
+            --workflow-url ${{ inputs.workflow_url }}
 
       - name: Display Extracted Information
         run: |
           echo "Extracted information:"
-          echo "altinity_build_feature: ${ALTINITY_BUILD_FEATURE}"
           echo "commit_hash: ${COMMIT_HASH}"
           echo "docker_version: ${DOCKER_VERSION}"
           echo "folder_time: ${FOLDER_TIME}"
           echo "needs_binary_processing: ${NEEDS_BINARY_PROCESSING}"
           echo "package_version: ${PACKAGE_VERSION}"
           echo "repo_prefix: ${REPO_PREFIX}"
           echo "src_bucket: ${SRC_BUCKET}"
-          echo "src_dir: ${SRC_DIR}"
           echo "test_results_src: ${TEST_RESULTS_SRC}"
           echo "src_url: ${SRC_URL}"
           echo "dest_url: ${DEST_URL}"
@@ -211,9 +88,6 @@ jobs:
 
       - name: Process ARM binary
         if: ${{ env.NEEDS_BINARY_PROCESSING == 'true' }}
-        env:
-         AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-         AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
         run: |
           ARM_PATH=$([ "$MAJOR_VERSION" -ge 25 ] && echo "build_arm_release" || echo "package_aarch64")
 
@@ -223,13 +97,6 @@ jobs:
             exit 1
           fi
 
-          if [ "$MAJOR_VERSION" -ge 25 ]; then
-            echo "Strip the non-extracted binary..."
-            strip clickhouse -o clickhouse-stripped
-            aws s3 cp clickhouse-stripped "${SRC_URL}/${ARM_PATH}/"
-            rm clickhouse-stripped
-          fi
-          
           chmod +x clickhouse
 
           echo "Running clickhouse binary..."
@@ -294,6 +161,7 @@ jobs:
       - name: Separate AMD binary
         run: |
           aws s3 mv "${DEST_URL}/packages/AMD_PACKAGES/clickhouse" "${DEST_URL}/packages/AMD_PACKAGES/amd64-bin/clickhouse"
+          aws s3 mv "${DEST_URL}/packages/AMD_PACKAGES/clickhouse-stripped" "${DEST_URL}/packages/AMD_PACKAGES/amd64-bin/clickhouse-stripped"
 
       - name: Process AMD binary
         if: ${{ env.NEEDS_BINARY_PROCESSING == 'true' }}
@@ -304,15 +172,6 @@ jobs:
             exit 1
           fi
 
-          if [ "$MAJOR_VERSION" -ge 25 ]; then
-            echo "Strip the non-extracted binary..."
-            strip clickhouse -o clickhouse-stripped
-            aws s3 cp clickhouse-stripped "${DEST_URL}/packages/AMD_PACKAGES/amd64-bin/"
-            rm clickhouse-stripped
-          else
-            aws s3 mv "${DEST_URL}/packages/AMD_PACKAGES/clickhouse-stripped" "${DEST_URL}/packages/AMD_PACKAGES/amd64-bin/clickhouse-stripped"
-          fi
-          
           chmod +x clickhouse
 
           echo "Running clickhouse binary..."
@@ -524,12 +383,12 @@ jobs:
           echo "Copying to released directory..."
           echo "Source: ${DEST_URL}/"
           echo "Destination: s3://${S3_STORAGE_BUCKET}/builds/released/v${PACKAGE_VERSION}/"
-          
+
           if ! aws s3 sync "${DEST_URL}/" "s3://${S3_STORAGE_BUCKET}/builds/released/v${PACKAGE_VERSION}/" --no-progress; then
             echo "Failed to copy to released directory"
             exit 1
           fi
-          
+
           echo "Verifying copy operation..."
           if ! aws s3 ls "s3://${S3_STORAGE_BUCKET}/builds/released/v${PACKAGE_VERSION}/" | grep -q "packages"; then
             echo "Error: Packages directory not found in destination"