Merge pull request AlmaLinux#270 from yuravk/refactor-aws-copy-ami-workflow-to-merge-md-csv

andrewlukoshko · web-flow · commit 333b7ab0374e · 2025-09-24T15:32:15.000+02:00
refactor(CI: copy-ami.yml): Upload both x86_64 and arm64 AMIs and generate MD and CSV data for the Wiki
diff --git a/.github/workflows/copy-ami.yml b/.github/workflows/copy-ami.yml
@@ -4,8 +4,13 @@ on:
   workflow_dispatch:
       inputs:
 
-        ami_id:
-          description: "AMI ID to copy and make public"
+        x86_64_ami_id:
+          description: "AMI ID (x86_64)"
+          required: true
+          default: ''
+
+        aarch64_ami_id:
+          description: "AMI ID (aarch64)"
           required: true
           default: ''
 
@@ -15,16 +20,35 @@ on:
           type: boolean
           default: true
 
+        draft:
+          description: "Pull Request to Wiki as Draft"
+          required: true
+          type: boolean
+          default: true
+
         notify_mattermost:
           description: "Send notification to Mattermost"
           required: true
           type: boolean
           default: true
+env:
+  # AlmaLinux Wiki repository almalinux/wiki
+  wiki_repo: almalinux/wiki
+
+permissions:
+  id-token: write
+  contents: read
 
 jobs:
-  build-images:
-    name: "Copy ${{ inputs.ami_id }} AMI to all available AWS regions and make public"
+  copy-ami:
+    name: "Copy ${{ matrix.ami_id }} AMI to all available AWS regions and make public"
     runs-on: ubuntu-24.04
+    strategy:
+      fail-fast: false
+      matrix:
+        ami_id: ${{ fromJSON(format('["{0}", "{1}"]', inputs.x86_64_ami_id, inputs.aarch64_ami_id)) }}
+        exclude:
+          - ami_id: ''
 
     steps:
     - uses: actions/checkout@v4
@@ -38,44 +62,43 @@ jobs:
 
     - name: Get AMI Name
       run: |
-        AMI_NAME=$(aws ec2 describe-images --filters "Name=image-id,Values=${{ inputs.ami_id }}" --query 'Images[0].Name' --output text)
+        AMI_NAME=$(aws ec2 describe-images --filters "Name=image-id,Values=${{ matrix.ami_id }}" --query 'Images[0].Name' --output text)
         if [[ "${AMI_NAME}" == "" || "${AMI_NAME}" == "None" ]]; then
           exit 1
         else
           echo "[Debug] AMI Name: '${AMI_NAME}'"
         fi
         echo "AMI_NAME=${AMI_NAME}" >> $GITHUB_ENV
 
-    - name: Print AMI summary
-      uses: actions/github-script@v7
-      with:
-        result-encoding: string
-        script: |
-          core.summary
-              .addHeading('${{ env.AMI_NAME }}', '4')
-              .addHeading('AMI ID: ${{ inputs.ami_id }}', '5')
-              .addHeading('Public: ${{ inputs.make_public && '✅' || '❌'}}', '5')
-              .write()
-
     - name: Copy an AMI to all available AWS regions and make it public
       if: inputs.make_public
       run: |
         # Lowercase AMI Name, replace spaces and dots with underscores
         wiki_data_file="${{ env.AMI_NAME }}"
-        wiki_data_file="${wiki_data_file,,}"
         wiki_data_file="${wiki_data_file// /_}"
         wiki_data_file="${wiki_data_file//./_}"
-        echo "wiki_data_file=${wiki_data_file}" >> $GITHUB_ENV
+        echo "wiki_data_file=${wiki_data_file,,}" >> $GITHUB_ENV
 
         pip3 install markdown_table
 
         tools/aws_ami_mirror.py \
           --csv-output ${{ github.workspace }}/${wiki_data_file}.csv \
           --md-output ${{ github.workspace }}/${wiki_data_file}.md \
-          --ami ${{ inputs.ami_id }}
+          --ami ${{ matrix.ami_id }}
+
+    - name: Print AMI summary
+      uses: actions/github-script@v7
+      with:
+        result-encoding: string
+        script: |
+          core.summary
+              .addHeading('${{ env.AMI_NAME }}', '4')
+              .addHeading('AMI ID: ${{ matrix.ami_id }}', '5')
+              .addHeading('Copied and public: ${{ inputs.make_public && '✅' || '❌'}}', '5')
+              .write()
 
-    - uses: actions/upload-artifact@v4
-      name: Store Wiki's CSV and Markdown data as artifact
+    - name: Store Wiki's CSV and Markdown data as artifact
+      uses: actions/upload-artifact@v4
       if: inputs.make_public
       with:
         compression-level: 6
@@ -84,16 +107,165 @@ jobs:
           almalinux*.md
         name: ${{ env.wiki_data_file }}_wiki_data
 
-    - name: Send notification to Mattermost
-      uses: mattermost/action-mattermost-notify@master
-      if: inputs.notify_mattermost
-      with:
-        MATTERMOST_WEBHOOK_URL: ${{ secrets.MATTERMOST_WEBHOOK_URL }}
-        MATTERMOST_CHANNEL: ${{ vars.MATTERMOST_CHANNEL }}
-        MATTERMOST_USERNAME: ${{ github.triggering_actor }}
-        TEXT: |
-          **${{ env.AMI_NAME }}** AWS AMI, generated by the GitHub [Action](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})
+  prepare-data-for-wiki:
+    name: Prepare MD and CSV data for Wiki
+    runs-on: ubuntu-24.04
+    needs: [copy-ami]
+    if: inputs.make_public
+
+    steps:
+      - name: Checkout ${{ env.wiki_repo }}, branch 'master'
+        uses: actions/checkout@v4
+        with:
+          repository: ${{ env.wiki_repo }}
+          path: wiki
+          token:  ${{ secrets.GIT_HUB_TOKEN }}
+
+      - name: Download Wiki's CSV and Markdown data artifacts
+        uses: actions/download-artifact@v4
+
+      - name: Merge MD and CSV data
+        run: |
+          # Install prettier for Markdown formatting
+          sudo apt install npm
+
+          # Artifacts are downloaded into *x86_64_wiki_data/ and *aarch64_wiki_data/ directories
+          # They are like:
+          #  almalinux_os_kitten_10_20250904_0_x86_64_wiki_data/almalinux_os_kitten_10_20250904_0_x86_64_wiki_data.md
+          #  almalinux_os_10_0_20250904_aarch64_wiki_data/almalinux_os_10_0_20250904_aarch64_wiki_data.csv
+
+          # From x86_64 artifact extract string like:
+          #  almalinux_os_kitten_10_20250904_0_wiki_data
+          #  almalinux_os_kitten_10_0_20250904_wiki_data
+          RESULT=$( find . -name \*x86_64_wiki_data -printf '%P\n' | sed 's/x86_64_//g' )
+          [[ -z "$RESULT" ]] && { echo "[Error] Could not find *x86_64_wiki_data directory"; exit 1; }
+
+          # Extract full release, release major version, and date stamp
+          # Remove the prefix "almalinux_os_" and suffix "_wiki_data"
+          middle="${RESULT#almalinux_os_}"
+          middle="${middle%_wiki_data}"
+          # Find the date stamp (8 digits followed by optional _number)
+          if [[ $middle =~ ([0-9]{8}(_[0-9]+)?)$ ]]; then
+              datestamp="${BASH_REMATCH[1]}"
+              # Remove the datestamp and trailing underscore to get full release
+              release_full="${middle%_${datestamp}}"
+
+              # Prepare full release and release major version
+              if [[ $release_full == "kitten"* ]]; then
+                  release=$release_full
+                  release_full="${release_full//_/ }"
+                  release_full="${release_full^}"
+              else
+                  release=${release_full%_*}
+                  release_full="${release_full//_/.}"
+              fi
+
+              datestamp="${datestamp//_/.}"
+          else
+              echo "[Error] Could not parse: $input"
+              exit 1
+          fi
+
+          # Locate source .md and .csv files
+          file_x86_64_md=$( ls -1 *x86_64_wiki_data/*x86_64.md )
+          file_aarch64_md=$( ls -1 *aarch64_wiki_data/*aarch64.md )
+          file_x86_64_csv=$( ls -1 *x86_64_wiki_data/*x86_64.csv )
+          file_aarch64_csv=$( ls -1 *aarch64_wiki_data/*aarch64.csv )
+
+          # Set target .md document path and name
+          md_documents_path="docs/cloud"
+          [[ $release == *"kitten"* ]] && md_documents_path="docs/development"
+          md_document="${md_documents_path}/AWS_AMIS_${release}.md"
+
+          # Set target .csv document path and name
+          csv_documents_path="docs/.vuepress/public/ci-data"
+          csv_document="${csv_documents_path}/aws_amis_${release}.csv"
+
+          # Print header
+          head -n 2 "$file_x86_64_md" > wiki/${md_document}
+          # Merge .md files: sort primarily by Region (3rd column)
+          # and secondarily by Arch (5th column) reverse
+          (tail -n +3 "$file_x86_64_md"; tail -n +3 "$file_aarch64_md") | sort -t'|' -k3,3 -k5,5r >> wiki/${md_document}
+          # Make the resulted MD file pretty
+          npx prettier --parser markdown --write wiki/${md_document}
+
+          # Merge .csv files: sort by Region (3rd column) and Version (5th column, reverse)
+          (cat "$file_x86_64_csv"; cat "$file_aarch64_csv") | sort -t',' -k3,3 -k5,5r > wiki/${csv_document}
+
+          echo "RESULT=${RESULT}" >> $GITHUB_ENV
+          echo "RELEASE_STR=AlmaLinux OS ${release_full} ${datestamp}" >> $GITHUB_ENV
+          echo "RESULT_MD=wiki/${md_document}" >> $GITHUB_ENV
+          echo "RESULT_CSV=wiki/${csv_document}" >> $GITHUB_ENV
+
+      - name: Store merged CSV and Markdown data as artifact
+        uses: actions/upload-artifact@v4
+        with:
+          compression-level: 6
+          include-hidden-files: true
+          path: |
+            ${{ env.RESULT_MD }}
+            ${{ env.RESULT_CSV }}
+          name: ${{ env.RESULT }}
+
+      - name: Commit and push MD and CSV to ${{ env.wiki_repo }}, branch '${{ env.RESULT }}'
+        uses: EndBug/add-and-commit@v9
+        with:
+          cwd: wiki
+          new_branch: ${{ env.RESULT }}
+          default_author: user_info
+          message: "Update ${{ env.RELEASE_STR }} AWS AMI IDs"
+          push: true
+
+      - name: Create Pull Request for ${{ env.wiki_repo }}, branch 'master'
+        run: |
+          # create pull request with 'gh pr create'
+          gh_opts=''
+          [ "${{ inputs.draft }}" = "true" ] && gh_opts='--draft'
+          title="Update ${{ env.RELEASE_STR }} AWS AMI IDs"
+          body="This is an auto-generated PR. Any concern or issues, please contact Cloud SIG."
+
+          cd wiki
+          gh auth login --with-token < <(echo ${{ secrets.GIT_HUB_TOKEN }})
+
+          gh pr create \
+            --title "${title}" \
+            --body "${body}" \
+            --repo ${{ env.wiki_repo }} \
+            --base master \
+            ${gh_opts} \
+            |& tee gh_pr_create.log
+
+          exit_code=${PIPESTATUS[0]}
+          if [[ "${exit_code}" != "0" ]]; then
+            exit 1
+          fi
+
+          echo "WIKI_PR_URL=$( grep '${{ env.wiki_repo }}/pull/' gh_pr_create.log )" >> $GITHUB_ENV
+
+      - name: Print PR summary
+        uses: actions/github-script@v7
+        if: ${{ env.WIKI_PR_URL != '' }}
+        with:
+          result-encoding: string
+          script: |
+            core.summary
+                .addHeading('${{ env.AMI_NAME }}', '4')
+                .addLink('${{ env.WIKI_PR_URL }}', '${{ env.WIKI_PR_URL }}')
+                .addHeading('The PR is draft: ${{ inputs.draft && '✅' || '❌'}}', '5')
+                .write()
+
+      - name: Send notification to Mattermost
+        uses: mattermost/action-mattermost-notify@master
+        if: inputs.notify_mattermost
+        with:
+          MATTERMOST_WEBHOOK_URL: ${{ secrets.MATTERMOST_WEBHOOK_URL }}
+          MATTERMOST_CHANNEL: ${{ vars.MATTERMOST_CHANNEL }}
+          MATTERMOST_USERNAME: ${{ github.triggering_actor }}
+          TEXT: |
+            :almalinux: **${{ env.RELEASE_STR }}** AWS AMIs copied over regions and made public, by the GitHub [Action](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})
 
-          :almalinux: **AMI ID**: `${{ inputs.ami_id }}`
+            - **AMI ID (x86_64)**: `${{ inputs.x86_64_ami_id }}`
+            - **AMI ID (aarch64)**: `${{ inputs.aarch64_ami_id }}`
 
-          Public: ${{ inputs.make_public && '✅' || '❌'}}
+            Copied and public: ${{ inputs.make_public && '✅' || '❌'}}
+            ${{ env.WIKI_PR_URL != '' && format('Wiki PR: [{0}]({0})', env.WIKI_PR_URL) || '' }}
