ci(build): add linux release automation and release automation validation

Signed-off-by: Swapnanil Gupta <[email protected]>

Author: Swapnanil-Gupta

.github/workflows/build-and-test-msi.yaml

@@ -5,14 +5,19 @@ on:
   workflow_dispatch:
     inputs:
       ref_name:
-        description: "name of git ref for which to build installer"
+        description: "the ref (tag/branch) to use to extract tag/version"
         required: true
         type: string
   workflow_call:
     inputs:
       ref_name:
+        description: "the ref (tag/branch) to use to extract tag/version"
         required: true
         type: string
+      version:
+        description: "override for version, will be used instead of ref if set, used for testing"
+        required: false
+        type: string
   schedule:
     - cron: '0 9 * * *'
 env:
@@ -28,43 +33,11 @@ permissions:
 jobs:
   get-tag-name:
     name: Get tag name
-    runs-on: ubuntu-latest
-    timeout-minutes: 2
-    outputs:
-      tag: ${{ steps.check-tag.outputs.tag }}
-      version: ${{ steps.check-tag.outputs.version }}
-      commit: ${{ steps.export-commit.outputs.commit }}
-    steps:
-      - name: Check tag from workflow input and github ref
-        id: check-tag
-        run: |
-          if [ -n "${{ inputs.ref_name }}" ]; then
-            tag=${{ inputs.ref_name }}
-          else
-            tag=${{ github.ref_name }}
-          fi
-          echo "tag=$tag" >> ${GITHUB_OUTPUT}
-
-          version=${tag#v}
-          if [[ $version =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
-            echo "Version matches format: $version"
-          else
-            echo "Version $version doesn't match format. Using test version: 0.0.1"
-            version="0.0.1"
-          fi
-          echo "version=$version" >> ${GITHUB_OUTPUT}
-      - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
-        with:
-          ref: ${{ steps.check-tag.outputs.tag }}
-          fetch-depth: 0
-          persist-credentials: false
-          submodules: true
-      - name: Export commit hash
-        id: export-commit
-        run: |
-          commit=$(git rev-parse HEAD)
-          echo "commit=$commit" >> ${GITHUB_OUTPUT}
-
+    uses: ./.github/workflows/get-version-and-tag-for-ref.yaml
+    with:
+      ref_name: ${{ inputs.ref_name }}
+      version: ${{ inputs.version }}
+  
   windows-msi-build:
     needs: get-tag-name
     runs-on: [self-hosted, windows, amd64, release]
@@ -91,7 +64,7 @@ jobs:
           cache: false
       - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
         with:
-          ref: ${{ needs.get-tag-name.outputs.commit }}
+          ref: ${{ needs.get-tag-name.outputs.tag }}
           fetch-depth: 0
           persist-credentials: false
           submodules: recursive
@@ -114,20 +87,22 @@ jobs:
           cd deps/finch-core && make clean
       - name: Build project
         run: |
+          $dirty_tag = git describe --match 'v[0-9]*' --dirty='.modified' --always --tags
+          Write-Host "dirty tag: $dirty_tag"
           make FINCH_OS_IMAGE_LOCATION_ROOT=__INSTALLFOLDER__
       - name: generate and download signed msi
         run: |
           $version="${{ needs.get-tag-name.outputs.version }}"
           $tag="${{ needs.get-tag-name.outputs.tag }}"
           powershell .\msi-builder\BuildFinchMSI.ps1 -Version $version
           $timestamp=[math]::truncate((Get-Date (Get-Date).ToUniversalTime() -UFormat "%s"))
-          $unsignedMSI="Finch-$tag-$timestamp.msi"
+          $unsignedMSI="Finch-$version-$timestamp.msi"
           Write-Host "Upload unsigned MSI: $unsignedMSI"
-
+          
           aws s3 cp "./msi-builder/build/Finch-$version.msi" "${{ secrets.WINDOWS_UNSIGNED_BUCKET }}$unsignedMSI" --acl bucket-owner-full-control --no-progress
           New-Item -Path "./msi-builder/build/signed/" -ItemType Directory -Force
 
-          Write-Host "Attemp to download signed MSI"
+          Write-Host "Attempt to download signed MSI"
           $retryCount = 0
           $maxRetries = 20
           $delay = 5
@@ -137,7 +112,7 @@ jobs:
               $signedMSI = aws s3 ls ${{ secrets.WINDOWS_SIGNED_BUCKET }} 2>&1 | Where-Object { $_ -match "$unsignedMSI" } | Sort-Object -Descending | Select-Object -First 1 | ForEach-Object { ($_ -split '\s+')[-1] }
               if ($signedMSI -and ($signedMSI -notlike "*An error occurred (404) when calling the HeadObject operation*")) {
                   try {
-                      aws s3 cp "${{ secrets.WINDOWS_SIGNED_BUCKET }}$signedMSI" "./msi-builder/build/signed/Finch-$tag.msi"
+                      aws s3 cp "${{ secrets.WINDOWS_SIGNED_BUCKET }}$signedMSI" "./msi-builder/build/signed/Finch-$version.msi"
                       break
                   } catch {
                       Write-Host "Error during copy: $_"
@@ -159,8 +134,8 @@ jobs:
           aws-region: ${{ secrets.REGION }}
       - name: upload signed MSI to S3
         run: |
-          $tag="${{ needs.get-tag-name.outputs.tag }}"
-          aws s3 cp "./msi-builder/build/signed/Finch-$tag.msi" "s3://${{ secrets.INSTALLER_PRIVATE_BUCKET_NAME }}/Finch-$tag.msi" --no-progress
+          $version="${{ needs.get-tag-name.outputs.version }}"
+          aws s3 cp "./msi-builder/build/signed/Finch-$version.msi" "s3://${{ secrets.INSTALLER_PRIVATE_BUCKET_NAME }}/Finch-$version.msi" --no-progress
       - name: Remove Finch VM and Clean Up Previous Environment
         if: ${{ always() }}
         timeout-minutes: 5
@@ -235,8 +210,8 @@ jobs:
           }
       - name: Download MSI from S3
         run: |
-          $tag="${{ needs.get-tag-name.outputs.tag }}"
-          aws s3 cp "s3://${{ secrets.INSTALLER_PRIVATE_BUCKET_NAME }}/Finch-$tag.msi" ./Finch.msi
+          $version="${{ needs.get-tag-name.outputs.version }}"
+          aws s3 cp "s3://${{ secrets.INSTALLER_PRIVATE_BUCKET_NAME }}/Finch-$version.msi" ./Finch.msi
       - name: Install MSI silently
         run: |
           Start-Process 'Finch.msi' -ArgumentList '/quiet' -Wait

.github/workflows/build-and-test-pkg.yaml

@@ -5,14 +5,19 @@ on:
   workflow_dispatch:
     inputs:
       ref_name:
-        description: "name of git ref for which to build installer"
+        description: "the ref (tag/branch) to use to extract tag/version"
         required: true
         type: string
   workflow_call:
     inputs:
       ref_name:
+        description: "the ref (tag/branch) to use to extract tag/version"
         required: true
         type: string
+      version:
+        description: "override for version, will be used instead of ref if set, used for testing"
+        required: false
+        type: string
   schedule:
     - cron: '0 9 * * *'
 env:
@@ -28,36 +33,10 @@ permissions:
 jobs:
   get-tag-name:
     name: Get tag name
-    runs-on: ubuntu-latest
-    permissions:
-      contents: read
-    timeout-minutes: 2
-    outputs:
-      tag: ${{ steps.check-tag.outputs.tag }}
-      commit: ${{ steps.export-commit.outputs.commit }}
-    steps:
-      - name: Check tag from workflow input and github ref
-        id: check-tag
-        run: |
-          if [ -n "${{ inputs.ref_name }}" ]; then
-            tag=${{ inputs.ref_name }}
-          else
-            tag=${{ github.ref_name }}
-          fi
-          echo "using tag=${tag}"
-          echo "tag=$tag" >> ${GITHUB_OUTPUT}
-      - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
-        with:
-          ref: ${{ steps.check-tag.outputs.tag }}
-          fetch-depth: 0
-          persist-credentials: false
-          submodules: true
-      - name: Export commit hash
-        id: export-commit
-        run: |
-          commit=$(git rev-parse HEAD)
-          echo "using commit=${commit}"
-          echo "commit=$commit" >> ${GITHUB_OUTPUT}
+    uses: ./.github/workflows/get-version-and-tag-for-ref.yaml
+    with:
+      ref_name: ${{ inputs.ref_name }}
+      version: ${{ inputs.version }}
 
   macos-aarch64-pkg-build:
     needs: get-tag-name
@@ -75,7 +54,7 @@ jobs:
       output_arch: aarch64
       version: 14
       tag: ${{ needs.get-tag-name.outputs.tag }}
-      commit: ${{ needs.get-tag-name.outputs.commit }}
+      finch_version: ${{ needs.get-tag-name.outputs.version }}
 
   macos-x86-64-pkg-build:
     needs: get-tag-name
@@ -93,7 +72,7 @@ jobs:
       output_arch: x86_64
       version: 14
       tag: ${{ needs.get-tag-name.outputs.tag }}
-      commit: ${{ needs.get-tag-name.outputs.commit }}
+      finch_version: ${{ needs.get-tag-name.outputs.version }}
 
   macos-aarch64-pkg-test:
     strategy:
@@ -117,7 +96,7 @@ jobs:
       output_arch: aarch64
       version: ${{ matrix.version }}
       tag: ${{ needs.get-tag-name.outputs.tag }}
-      commit: ${{ needs.get-tag-name.outputs.commit }}
+      finch_version: ${{ needs.get-tag-name.outputs.version }}
 
   macos-x86-64-pkg-test:
     strategy:
@@ -141,4 +120,4 @@ jobs:
       output_arch: x86_64
       version: ${{ matrix.version }}
       tag: ${{ needs.get-tag-name.outputs.tag }}
-      commit: ${{ needs.get-tag-name.outputs.commit }}
+      finch_version: ${{ needs.get-tag-name.outputs.version }}

.github/workflows/build-linux.yaml

@@ -0,0 +1,138 @@
+name: Build Static Linux Binaries
+
+on:
+  workflow_dispatch:
+    inputs:
+      ref_name:
+        description: "the ref (tag/branch) to use to extract tag/version"
+        required: true
+        type: string
+  workflow_call:
+    inputs:
+      ref_name:
+        description: "the ref (tag/branch) to use to extract tag/version"
+        required: true
+        type: string
+      version:
+        description: "override for version, will be used instead of ref if set, used for testing"
+        required: false
+        type: string
+  schedule:
+    - cron: '0 9 * * *'
+
+permissions:
+    # This is required for configure-aws-credentials to request an OIDC JWT ID token to access AWS resources later on.
+    # More info: https://docs.github.com/en/actions/deployment/security-hardening-your-deployments/about-security-hardening-with-openid-connect#adding-permissions-settings
+    id-token: write
+    contents: read # this is required for actions/checkout
+
+jobs:
+  get-tag-name:
+    name: Get tag name
+    uses: ./.github/workflows/get-version-and-tag-for-ref.yaml
+    with:
+      ref_name: ${{ inputs.ref_name }}
+      version: ${{ inputs.version }}
+  
+  generate-artifacts:
+    needs: get-tag-name
+    runs-on: ubuntu-latest
+    env:
+      # Set during setup.
+      RELEASE_VERSION: ${{ needs.get-tag-name.outputs.version }}
+    steps:
+      - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
+        with:
+          ref: ${{ inputs.tag }}
+          fetch-depth: 0
+          persist-credentials: false
+          submodules: recursive
+      - uses: actions/setup-go@4dc6199c7b1a012772edbd06daecab0f50c9053c # v6.1.0
+        with:
+          go-version-file: go.mod
+          cache: true
+      - name: "Echo RELEASE_VERSION ENV"
+        run: echo ${{ env.RELEASE_VERSION }}
+      - name: Build
+        id: build
+        run: |
+          RELEASE_VERSION="${{ env.RELEASE_VERSION }}"
+          sudo make check-licenses download-licenses
+          
+          # static amd64
+          export STATIC_AMD64_BINARY_NAME="finch-${RELEASE_VERSION}-linux-amd64-static.tar.gz"
+          sudo GOARCH=amd64 STATIC=1 make
+          pushd _output/
+          sudo touch "${STATIC_AMD64_BINARY_NAME}"
+          sudo tar --exclude "*.tar.gz" --exclude "*.tar.gz.sha256sum" -cvzf "${STATIC_AMD64_BINARY_NAME}" .
+          echo "STATIC_AMD64_BINARY_NAME=${STATIC_AMD64_BINARY_NAME}" >> ${GITHUB_OUTPUT}
+          popd
+          sudo rm -rf ./_output/bin/
+          
+          # static arm64
+          export STATIC_ARM64_BINARY_NAME="finch-${RELEASE_VERSION}-linux-arm64-static.tar.gz"
+          sudo GOARCH=arm64 STATIC=1 make
+          pushd _output/
+          sudo touch "${STATIC_ARM64_BINARY_NAME}"
+          sudo tar --exclude "*.tar.gz" --exclude "*.tar.gz.sha256sum" -cvzf "${STATIC_ARM64_BINARY_NAME}" .
+          echo "STATIC_ARM64_BINARY_NAME=${STATIC_ARM64_BINARY_NAME}" >> ${GITHUB_OUTPUT}
+          popd
+          sudo rm -rf ./_output/bin/
+          
+          pushd _output/
+          sudo sh -c "sha256sum '${STATIC_AMD64_BINARY_NAME}' > '${STATIC_AMD64_BINARY_NAME}.sha256sum'"
+          sudo sh -c "sha256sum '${STATIC_ARM64_BINARY_NAME}' > '${STATIC_ARM64_BINARY_NAME}.sha256sum'"
+          popd
+      - name: Verify release versions
+        run: |
+          ARCH=$(uname -m)
+          if [ "$ARCH" = "x86_64" ]; then
+            GOARCH="amd64"
+            BINARY_NAME="${{ steps.build.outputs.STATIC_AMD64_BINARY_NAME }}"
+          elif [ "$ARCH" = "aarch64" ]; then
+            GOARCH="arm64"
+            BINARY_NAME="${{ steps.build.outputs.STATIC_ARM64_BINARY_NAME }}"
+          else
+            echo "Unsupported architecture: $ARCH"
+            exit 1
+          fi
+          
+          sudo mkdir -p ./_output/${{ env.RELEASE_VERSION }}/static/$GOARCH
+          sudo tar -xzf ./_output/$BINARY_NAME -C ./_output/${{ env.RELEASE_VERSION }}/static/$GOARCH
+          finch_version=$(sudo ./_output/${{ env.RELEASE_VERSION }}/static/$GOARCH/bin/finch --version)
+          BINARY_VERSION=$(echo $finch_version | sed -n 's/finch version //p')
+          export RELEASE_VERSION=${{ env.RELEASE_VERSION }}
+          if [ "$BINARY_VERSION" != "$RELEASE_VERSION" ]; then
+            echo "Version mismatch for $GOARCH binary"
+            echo "RELEASE_VERSION = ${RELEASE_VERSION}"
+            echo "finch_version = ${finch_version}"
+            exit 1
+          fi
+      - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
+        with:
+          name: artifacts
+          path: |
+            _output/${{ steps.build.outputs.STATIC_AMD64_BINARY_NAME }}
+            _output/${{ steps.build.outputs.STATIC_AMD64_BINARY_NAME }}.sha256sum
+            _output/${{ steps.build.outputs.STATIC_ARM64_BINARY_NAME }}
+            _output/${{ steps.build.outputs.STATIC_ARM64_BINARY_NAME }}.sha256sum
+          if-no-files-found: error
+    outputs:
+      static_amd64_binary_name: ${{ steps.build.outputs.STATIC_AMD64_BINARY_NAME }}
+      static_arm64_binary_name: ${{ steps.build.outputs.STATIC_ARM64_BINARY_NAME }}
+  
+  # TODO: do we need to sign them or upload them to GitHub releases?
+  upload-artifacts:
+    needs:
+    - generate-artifacts
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
+        with:
+          name: artifacts
+      - name: upload to S3
+        run: |
+          aws s3 cp --no-progress ${{ needs.generate-artifacts.outputs.static_amd64_binary_name }} s3://${{ secrets.INSTALLER_PRIVATE_BUCKET_NAME }}/
+          aws s3 cp --no-progress ${{ needs.generate-artifacts.outputs.static_amd64_binary_name }}.sha256sum s3://${{ secrets.INSTALLER_PRIVATE_BUCKET_NAME }}/
+          aws s3 cp --no-progress ${{ needs.generate-artifacts.outputs.static_arm64_binary_name }} s3://${{ secrets.INSTALLER_PRIVATE_BUCKET_NAME }}/
+          aws s3 cp --no-progress ${{ needs.generate-artifacts.outputs.static_arm64_binary_name }}.sha256sum s3://${{ secrets.INSTALLER_PRIVATE_BUCKET_NAME }}/