Simplify

* Use GitHub Release instead of GitHub Actions' artifacts
* Build directory structure in package generator not binary-task.rb

Author: kou

.github/workflows/package_linux.yml

@@ -33,7 +33,7 @@ on:
       - 'format/Flight.proto'
       - 'testing'
     tags:
-      - '**'
+      - "apache-arrow-*-rc*"
   pull_request:
     paths:
       - '.dockerignore'
@@ -52,23 +52,14 @@ on:
       - synchronize
   schedule:
     - cron: "0 2 * * *"
-  workflow_dispatch:
-    inputs:
-      version:
-        description: "The Arrow version"
-        type: string
-        required: true
-      no_rc_version:
-        description: "The Arrow version without RC"
-        type: string
-        required: true
 
 concurrency:
   group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}
   cancel-in-progress: true
 
 permissions:
-  contents: read
+  # Upload to GitHub Release
+  contents: write
 
 jobs:
   check-labels:
@@ -113,14 +104,20 @@ jobs:
           - ubuntu-noble-arm64
     env:
       DOCKER_VOLUME_PREFIX: ".docker/"
-      ARROW_VERSION: ${{ inputs.version || ''}}
-      NO_RC_VERSION: ${{ inputs.no_rc_version  || ''}}
     steps:
+      - name: Checkout Arrow
+        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
+        with:
+          fetch-depth: 0
+          submodules: recursive
       - name: Prepare environment variables
         env:
           ID: ${{ matrix.id }}
         run: |
-          set -ex
+          set -x
+          # Example: almalinux-8-amd64 -> almalinux
+          distribution="${ID%-*-*}"
+          echo "DISTRIBUTION=${distribution}" >> "${GITHUB_ENV}"
           # Example: almalinux-8-amd64 -> amd64
           architecture="${ID##*-}"
           echo "ARCHITECTURE=${architecture}" >> "${GITHUB_ENV}"
@@ -129,40 +126,33 @@ jobs:
           case "${target}" in
             almalinux-*|amazon-linux-*|centos-*)
               echo "TASK_NAMESPACE=yum" >> "${GITHUB_ENV}"
-              echo "UPLOAD_EXTENSIONS=rpm" >> "${GITHUB_ENV}"
-              if [[ "${architecture}" == "arm64" ]]; then
+              if [ "${architecture}" = "arm64" ]; then
                 # Example: almalinux-8 -> almalinux-8-aarch64
                 target="${target}-aarch64"
               fi
               echo "YUM_TARGETS=${target}" >> "${GITHUB_ENV}"
               ;;
             *)
               echo "TASK_NAMESPACE=apt" >> "${GITHUB_ENV}"
-              upload_extensions=(ddeb deb debian.tar.xz .dsc .orig.tar.gz)
-              echo "UPLOAD_EXTENSIONS=${upload_extensions[*]}" >> "${GITHUB_ENV}"
-              if [[ "${architecture}" == "arm64" ]]; then
+              if [ "${architecture}" = "arm64" ]; then
                 # Example: ubuntu-noble -> ubuntu-noble-arm64
                 target="${target}-arm64"
               fi
               echo "APT_TARGETS=${target}" >> "${GITHUB_ENV}"
               ;;
           esac
           echo "TARGET=${target}" >> "${GITHUB_ENV}"
-      - name: Checkout Arrow
-        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
-        with:
-          fetch-depth: 0
-          submodules: recursive
-      - name: Free up disk space
-        if: runner.os == 'Linux' && runner.arch == 'X64'
-        shell: bash
-        run: |
-          ci/scripts/util_free_space.sh
+
+          if [ "${GITHUB_REF_TYPE}" = "tag" ]; then
+            # Example: apache-arrow-21.0.0-rc0 -> 21.0.0-rc0
+            version="${GITHUB_REF_NAME#apache-arrow-}"
+            echo "ARROW_VERSION=${version}" >> "${GITHUB_ENV}"
+          fi
       - name: Cache Docker Volumes
         uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
         with:
           path: .docker
-          key:  package-linux-${{ matrix.id }}-${{ hashFiles('cpp/**') }}
+          key:  package-linux-${{ matrix.id }}-${{ hashFiles('cpp/**', 'c_glib/**') }}
           restore-keys: package-linux-${{ matrix.id }}-
       - name: Set up Ruby
         run: |
@@ -172,91 +162,106 @@ jobs:
             ruby \
             ruby-dev
       - name: Prepare apache-arrow-apt-source for arm64
-        if: ${{ env.ARCHITECTURE == 'arm64' }}
+        if: env.ARCHITECTURE == 'arm64'
         run: |
           pushd dev/tasks/linux-packages/apache-arrow-apt-source/apt
           for target in *-*; do
             cp -a ${target} ${target}-arm64
           done
           popd
       - name: Prepare apache-arrow-release for arm64
-        if: ${{ env.ARCHITECTURE == 'arm64' }}
+        if: env.ARCHITECTURE == 'arm64'
         run: |
           pushd dev/tasks/linux-packages/apache-arrow-release/yum
           for target in *-*; do
             cp -a ${target} ${target}-aarch64
           done
           popd
-      - name: Get Arrow Version
-        id: get-arrow-version
-        run: |
-          # In case of dev (no workflow_dispatch inputs) compute ARROW_VERSION and
-          # use the same version for NO_RC_VERSION.
-          if [ -z "${ARROW_VERSION}" ]; then
-            ARROW_VERSION=$(git describe --tags --abbrev=0 --match "apache-arrow-[0-9]*.*" | sed 's/^apache-arrow-//')
-            ARROW_VERSION="${ARROW_VERSION}$(date +%Y%m%d)"
-            echo "ARROW_VERSION=${ARROW_VERSION}" >> $GITHUB_ENV
-            echo "NO_RC_VERSION=${ARROW_VERSION}" >> $GITHUB_ENV
-          fi
-      - name: Build
+      - name: Update version
+        if: github.ref_type != 'tag'
         run: |
-          set -e
           pushd dev/tasks/linux-packages
-          rake version:update ARROW_RELEASE_TIME="$(date --iso-8601=seconds)"
-          rake docker:pull || :
-          rake --trace ${TASK_NAMESPACE}:build BUILD_DIR=build
+          rake version:update
           popd
-        env:
-          REPO: ghcr.io/${{ github.repository }}-package-linux
-      - name: Login to Dockerhub
-        if: >-
-          success() &&
-          github.event_name == 'push' &&
-          github.repository == 'apache/arrow' &&
-          github.ref_name == 'main'
+      - name: Login to GitHub Container registry
         uses: docker/login-action@184bdaa0721073962dff0199f1fb9940f07167d1 # v3.5.0
         with:
           registry: ghcr.io
           username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN}}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Wait for creating GitHub Release
+        if: github.ref_type == 'tag'
+        run: |
+          dev/release/utils-watch-gh-workflow.sh \
+            ${GITHUB_REF_NAME} \
+            release_candidate.yml
+      - name: Build
+        run: |
+          pushd dev/tasks/linux-packages
+          rake docker:pull || :
+          rake --trace ${TASK_NAMESPACE}:build BUILD_DIR=build
+          popd
       - name: Docker Push
         continue-on-error: true
         if: >-
           success() &&
           github.event_name == 'push' &&
-          github.repository == 'apache/arrow' &&
           github.ref_name == 'main'
-        shell: bash
         run: |
           pushd dev/tasks/linux-packages
           rake docker:push
           popd
-        env:
-          REPO: ${{ secrets.REPO }}
       - name: Build artifact tarball
-        shell: bash
         run: |
-          set -ex
-          pushd dev/tasks/linux-packages
-          tar cvzf ${{ matrix.id }}.tar.gz */${TASK_NAMESPACE}/repositories
-          popd
+          mkdir -p "${DISTRIBUTION}"
+          cp -a \
+            dev/tasks/linux-packages/*/${TASK_NAMESPACE}/repositories/${DISTRIBUTION}/* \
+            "${DISTRIBUTION}/"
+          # We use latest .deb/.rpm of
+          # apache-arrow-apt-source/apache-arrow-release built for
+          # amd64 because they are architecture independent.
+          if [ "${ARCHITECTURE}" = "amd64" ]; then
+            if [ "${APT_TASK_NAMESPACE}" = "apt" ]; then
+              # Example: debian-bookworm-amd64 -> debian-bookworm
+              code_name="${ID%-*}"
+              # Example: debian-bookworm -> bookworm
+              code_name="${code_name#*-}"
+              # Create
+              # https://packages.apache.org/artifactory/arrow/${DISTRIBUTION}/apache-arrow-apt-source-latest-${code_name}.deb
+              # for easy to install.
+              cp -a \
+                "${DISTRIBUTION}/pool/${code_name}/a/apache-arrow-apt-source/*.deb \
+                "${DISTRIBUTION}/apache-arrow-apt-source-latest-${code_name}.deb"
+            else
+              # Example: amazon-linux-2023-amd64 -> amazon-linux-2023
+              version="${ID%-*}"
+              # Example: amazon-linux-2023 -> 2023
+              version="${version##*-}"
+              # Create
+              # https://packages.apache.org/artifactory/arrow/${DISTRIBUTION}/${version}/apache-arrow-release-latest.rpm
+              # for easy to install.
+              cp -a \
+                "${DISTRIBUTION}/${version}/x86_64/Packages/apache-arrow-release-*.rpm \
+                "${DISTRIBUTION}/${version}/apache-arrow-release-latest.rpm"
+            fi
+          fi
+          tar cvzf ${{ matrix.id }}.tar.gz ${DISTRIBUTION}
+          dev/release/utils-generate-checksum.sh ${{ matrix.id }}.tar.gz
       - name: Upload the artifacts to the job
         uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
         with:
           name: ${{ matrix.id }}
-          path: dev/tasks/linux-packages/${{ matrix.id }}.tar.gz
-      - name: Download Artifacts
-        uses: actions/download-artifact@v4
-        with:
-          path: artifacts-downloaded
-          name: ${{ matrix.id }}
+          path: ${{ matrix.id }}.tar.gz*
+      - name: Upload the artifacts to GitHub Release
+        if: github.ref_type == 'tag'
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          gh release upload ${GITHUB_REF_NAME} \
+            --clobber \
+            ${{ matrix.id }}.tar.gz*
       - name: Set up test
         run: |
-          mv artifacts-downloaded/* dev/tasks/linux-packages
-          pushd dev/tasks/linux-packages
-          rm -rf */${TASK_NAMESPACE}/repositories # Remove artifacts
-          tar xf ${{ matrix.id }}.tar.gz # Use uploaded artifacts
-          popd
           sudo apt install -y \
             apt-utils \
             cpio \
@@ -266,27 +271,25 @@ jobs:
             rpm \
             rsync
           gem install --user-install apt-dists-merge
-          (echo "Key-Type: RSA"; \
-           echo "Key-Length: 4096"; \
-           echo "Name-Real: Test"; \
-           echo "Name-Email: [email protected]"; \
-           echo "%no-protection") | \
-            gpg --full-generate-key --batch
+          {
+            echo "Key-Type: RSA"
+            echo "Key-Length: 4096"
+            echo "Name-Real: Test"
+            echo "Name-Email: [email protected]"
+            echo "%no-protection"
+          } | gpg --full-generate-key --batch
           GPG_KEY_ID=$(gpg --list-keys --with-colon [email protected] | grep fpr | cut -d: -f10)
           echo "GPG_KEY_ID=${GPG_KEY_ID}" >> ${GITHUB_ENV}
-          case "${{ matrix.id }}" in
-            almalinux-*|amazon-linux-*|centos-*)
-              repositories_dir=dev/tasks/linux-packages/apache-arrow-release/yum/repositories
-              rpm2cpio ${repositories_dir}/*/*/*/Packages/apache-arrow-release-*.rpm | \
-                cpio -id
-              mv etc/pki/rpm-gpg/RPM-GPG-KEY-Apache-Arrow \
-                dev/tasks/linux-packages/KEYS
-              ;;
-          esac
+          if [ "${TASK_NAMESPACE}" = "yum" ]; then
+            repositories_dir=dev/tasks/linux-packages/apache-arrow-release/yum/repositories
+            rpm2cpio ${repositories_dir}/*/*/*/Packages/apache-arrow-release-*.rpm |
+              cpio -id
+            mv etc/pki/rpm-gpg/RPM-GPG-KEY-Apache-Arrow \
+              dev/tasks/linux-packages/KEYS
+          fi
           gpg --export --armor [email protected] >> dev/tasks/linux-packages/KEYS
       - name: Test
         run: |
-          set -e
           pushd dev/tasks/linux-packages
           rake --trace ${TASK_NAMESPACE}:test
           popd

.github/workflows/release.yml

@@ -20,9 +20,8 @@ name: Release
 on:
   push:
     tags:
-      # Trigger workflow when a tag whose name matches the pattern
-      # pattern "apache-arrow-{MAJOR}.{MINOR}.{PATCH}" is pushed.
-      - "apache-arrow-[0-9]+.[0-9]+.[0-9]+"
+      - "apache-arrow-*"
+      - "!apache-arrow-*-rc*"
 
 permissions:
   contents: write

.github/workflows/release_candidate.yml

@@ -23,9 +23,7 @@ on:
       - '**'
       - '!dependabot/**'
     tags:
-      # Trigger workflow when a tag whose name matches the pattern
-      # "apache-arrow-{MAJOR}.{MINOR}.{PATCH}-rc{RC_NUM}" is pushed.
-      - "apache-arrow-[0-9]+.[0-9]+.[0-9]+-rc[0-9]+"
+      - "apache-arrow-*-rc*"
     paths:
       - ".github/workflows/release_candidate.sh"
       - "dev/release/utils-create-release-tarball.sh"

dev/release/03-binary-submit.sh

@@ -51,12 +51,6 @@ archery crossbow submit \
     --arrow-sha ${ARROW_SHA} \
     --group packaging
 
-# Submit package_linux.yml workflow to build linux artifacts
-gh workflow run package_linux.yml \
-                --ref ${ARROW_BRANCH} \
-                -f version=${version_with_rc} \
-                -f no_rc_version=${version}
-
 # archery will add a comment to the automatically generated PR to track
 # the submitted jobs
 job_name=$(archery crossbow latest-prefix --no-fetch ${crossbow_job_prefix})

dev/release/04-binary-download.sh

@@ -33,7 +33,7 @@ rc_number=$1
 shift
 version_with_rc="${version}-rc${rc_number}"
 crossbow_job_prefix="release-${version_with_rc}"
-release_tag="apache-arrow-${version_with_rc}"
+tag="apache-arrow-${version_with_rc}"
 
 # archery will submit a job with id: "${crossbow_job_prefix}-0" unless there
 # are jobs submitted with the same prefix (the integer at the end is auto
@@ -43,21 +43,13 @@ release_tag="apache-arrow-${version_with_rc}"
 
 archery crossbow download-artifacts --no-fetch ${CROSSBOW_JOB_ID} "$@"
 
-# Wait for the GitHub Workflow that creates the Linux packages
-# to finish before downloading the artifacts.
-. "${SOURCE_DIR}/utils-watch-gh-workflow.sh" "${release_tag}" "package_linux.yml"
-
-RUN_ID=$(get_run_id)
-# Download the artifacts created by the package_linux.yml workflow
-download_artifacts "${SOURCE_DIR}/../../packages/${CROSSBOW_JOB_ID}"
-
-# Find and extract all .tar.gz files in their own artifact directory
-find "${SOURCE_DIR}/../../packages/${CROSSBOW_JOB_ID}" -name "*.tar.gz" -type f | while read -r tarfile; do
-    echo "Extracting: ${tarfile}"
-    tarfile_dir=$(dirname "${tarfile}")
-
-    # Extract to the same directory as the tar.gz file
-    tar -xzf "${tarfile}" -C "${tarfile_dir}"
-    # Should we remove the tar.gz file after extraction?
-    # rm "${tarfile}"
-done
+# Download Linux packages.
+gh release download "${tag}" \
+  --dir "${crossbow_job_prefix}" \
+  --pattern "almalinux-*.tar.gz" \
+  --pattern "amazon-linux-*.tar.gz" \
+  --pattern "centos-*.tar.gz" \
+  --pattern "debian-*.tar.gz" \
+  --pattern "ubuntu-*.tar.gz" \
+  --repo "${REPOSITORY:-apache/arrow}" \
+  --skip-existing