ARROW-15483: [Release] Exercise source verification builds on a nightly basis

dev/release/verify-release-candidate.bat

@@ -27,15 +27,25 @@ if not exist "C:\tmp\arrow-verify-release" mkdir C:\tmp\arrow-verify-release
 set _VERIFICATION_DIR=C:\tmp\arrow-verify-release
 set _VERIFICATION_DIR_UNIX=C:/tmp/arrow-verify-release
 set _VERIFICATION_CONDA_ENV=%_VERIFICATION_DIR%\conda-env
-set _DIST_URL=https://dist.apache.org/repos/dist/dev/arrow
-set _TARBALL=apache-arrow-%1.tar.gz
 set ARROW_SOURCE=%_VERIFICATION_DIR%\apache-arrow-%1
 set INSTALL_DIR=%_VERIFICATION_DIR%\install
 
-@rem Requires GNU Wget for Windows
-wget --no-check-certificate -O %_TARBALL% %_DIST_URL%/apache-arrow-%1-rc%2/%_TARBALL% || exit /B 1
-
-tar xf %_TARBALL% -C %_VERIFICATION_DIR_UNIX%
+set VERSION=%1
+set RC_NUMBER=%2
+
+if "%RC_NUMBER%"=="" (
+    @rem verify a specific git revision
+    if "%SOURCE_REPOSITORY%"=="" set SOURCE_REPOSITORY="https://github.com/apache/arrow.git"
+    git clone --recurse-submodules %SOURCE_REPOSITORY% %ARROW_SOURCE%
+    git -C %ARROW_SOURCE% checkout %VERSION%
+) else (
+    @rem verify a release candidate tarball
+    @rem Requires GNU Wget for Windows
+    set TARBALL_NAME=apache-arrow-%VERSION%.tar.gz
+    set TARBALL_URL=https://dist.apache.org/repos/dist/dev/arrow/apache-arrow-%VERSION%-rc%RC_NUMBER%/%TARBALL_NAME%
+    wget --no-check-certificate -O %TARBALL_NAME% %TARBALL_URL% || exit /B 1
+    tar xf %TARBALL_NAME% -C %_VERIFICATION_DIR_UNIX%
+)
 
 set PYTHON=3.8
 

dev/release/verify-release-candidate.sh

@@ -35,9 +35,20 @@
 # directory is not cleaned up automatically.
 
 case $# in
+  2) ARTIFACT="$1"
+     VERSION="$2"
+     SOURCE_KIND="git"
+     case $ARTIFACT in
+       source) ;;
+       *) echo "Invalid argument: '${ARTIFACT}', only valid option is 'source'"
+          exit 1
+          ;;
+     esac
+     ;;
   3) ARTIFACT="$1"
      VERSION="$2"
      RC_NUMBER="$3"
+     SOURCE_KIND="tarball"
      case $ARTIFACT in
        source|binaries|wheels|jars) ;;
        *) echo "Invalid argument: '${ARTIFACT}', valid options are \
@@ -366,9 +377,14 @@ test_csharp() {
   fi
 
   dotnet test
-  mv dummy.git ../.git
-  dotnet pack -c Release
-  mv ../.git dummy.git
+
+  if [ "${SOURCE_KIND}" = "git" ]; then
+    dotnet pack -c Release
+  else
+    mv dummy.git ../.git
+    dotnet pack -c Release
+    mv ../.git dummy.git
+  fi
 
   if ! which sourcelink > /dev/null 2>&1; then
     dotnet tool install --tool-path ${csharp_bin} sourcelink
@@ -544,27 +560,33 @@ test_integration() {
 
 ensure_source_directory() {
   dist_name="apache-arrow-${VERSION}"
-  if [ $((${TEST_SOURCE} + ${TEST_WHEELS})) -gt 0 ]; then
-    import_gpg_keys
-    if [ ! -d "${dist_name}" ]; then
-      fetch_archive ${dist_name}
-      tar xf ${dist_name}.tar.gz
-    fi
+  if [ "${SOURCE_KIND}" = "git" ]; then
+    git clone --recurse-submodules ${SOURCE_REPOSITORY:-"https://github.com/apache/arrow.git"} arrow
+    pushd arrow
+    git checkout ${VERSION}
   else
-    mkdir -p ${dist_name}
-    if [ ! -f ${TEST_ARCHIVE} ]; then
-      echo "${TEST_ARCHIVE} not found"
-      exit 1
+    if [ $((${TEST_SOURCE} + ${TEST_WHEELS})) -gt 0 ]; then
+      import_gpg_keys
+      if [ ! -d "${dist_name}" ]; then
+        fetch_archive ${dist_name}
+        tar xf ${dist_name}.tar.gz
+      fi
+    else
+      mkdir -p ${dist_name}
+      if [ ! -f ${TEST_ARCHIVE} ]; then
+        echo "${TEST_ARCHIVE} not found"
+        exit 1
+      fi
+      tar xf ${TEST_ARCHIVE} -C ${dist_name} --strip-components=1
+    fi
+    # clone testing repositories
+    pushd ${dist_name}
+    if [ ! -d "testing/data" ]; then
+      git clone https://github.com/apache/arrow-testing.git testing
+    fi
+    if [ ! -d "cpp/submodules/parquet-testing/data" ]; then
+      git clone https://github.com/apache/parquet-testing.git cpp/submodules/parquet-testing
     fi
-    tar xf ${TEST_ARCHIVE} -C ${dist_name} --strip-components=1
-  fi
-  # clone testing repositories
-  pushd ${dist_name}
-  if [ ! -d "testing/data" ]; then
-    git clone https://github.com/apache/arrow-testing.git testing
-  fi
-  if [ ! -d "cpp/submodules/parquet-testing/data" ]; then
-    git clone https://github.com/apache/parquet-testing.git cpp/submodules/parquet-testing
   fi
   export ARROW_DIR=$PWD
   export ARROW_TEST_DATA=$PWD/testing/data

dev/tasks/docker-tests/github.linux.yml

@@ -39,8 +39,6 @@ jobs:
         run: |
           archery docker run \
             -e SETUPTOOLS_SCM_PRETEND_VERSION="{{ arrow.no_rc_version }}" \
-            -e VERIFY_VERSION="{{ release|default("") }}" \
-            -e VERIFY_RC="{{ rc|default("") }}" \
             {{ flags|default("") }} \
             {{ image }} \
             {{ command|default("") }}

dev/tasks/tasks.yml

@@ -131,6 +131,7 @@ groups:
 {######################## Tasks to run regularly #############################}
 
   nightly:
+    - verify-rc-source-*
     - almalinux-*
     - amazon-linux-*
     - debian-*
@@ -876,12 +877,10 @@ tasks:
 
   verify-rc-source-{{ target }}-linux-{{ distribution }}-{{ version }}-amd64:
     ci: github
-    template: docker-tests/github.linux.yml
+    template: verify-rc/github.linux.amd64.docker.yml
     params:
-      flags: >-
-        -e TEST_DEFAULT=0
-        -e TEST_{{ target|upper }}=1
-      image: {{ distribution }}-verify-rc-source
+      target: {{ target }}
+      distro: {{ distribution }}
   {% endfor %}
 {% endfor %}
 

dev/tasks/verify-rc/github.linux.amd64.docker.yml

@@ -0,0 +1,39 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+{% import 'macros.jinja' as macros with context %}
+
+{{ macros.github_header() }}
+
+jobs:
+  test:
+    name: "Verify release candidate {{ distro }} source"
+    runs-on: ubuntu-latest
+    steps:
+      {{ macros.github_checkout_arrow(fetch_depth=fetch_depth if fetch_depth is defined else 1)|indent }}
+      {{ macros.github_install_archery()|indent }}
+
+      - name: Execute Docker Build
+        shell: bash
+        run: |
+          archery docker run \
+            -e SOURCE_REPOSITORY="{{ arrow.remote }}" \
+            -e VERIFY_VERSION="{{ release|default(arrow.head) }}" \
+            -e VERIFY_RC="{{ rc|default("") }}" \
+            -e TEST_DEFAULT=0 \
+            -e TEST_{{ target|upper }}=1 \
+            {{ distro }}-verify-rc-source

dev/tasks/verify-rc/github.linux.amd64.yml

@@ -71,7 +71,9 @@ jobs:
           node-version: '16'
       - name: Run verification
         shell: bash
+        env:
+          SOURCE_REPOSITORY: {{ arrow.remote }}
         run: |
           arrow/dev/release/verify-release-candidate.sh \
             {{ artifact }} \
-            {{ release|default("1.0.0") }} {{ rc|default("0") }}
+            {{ release|default(arrow.head) }} {{ rc|default("") }}

dev/tasks/verify-rc/github.macos.amd64.yml

@@ -44,7 +44,9 @@ jobs:
           node-version: '16'
       - name: Run verification
         shell: bash
+        env:
+          SOURCE_REPOSITORY: {{ arrow.remote }}
         run: |
           arrow/dev/release/verify-release-candidate.sh \
             {{ artifact }} \
-            {{ release|default("1.0.0") }} {{ rc|default("0") }}
+            {{ release|default(arrow.head) }} {{ rc|default("") }}

dev/tasks/verify-rc/github.macos.arm64.yml

@@ -39,10 +39,12 @@ jobs:
 
       - name: Run verification
         shell: bash
+        env:
+          SOURCE_REPOSITORY: {{ arrow.remote }}
         run: |
           export PATH="$(brew --prefix node@16)/bin:$PATH"
           export PATH="$(brew --prefix ruby)/bin:$PATH"
           export PKG_CONFIG_PATH="$(brew --prefix ruby)/lib/pkgconfig"
           arch -{{ arch_emulation|default("arm64") }} arrow/dev/release/verify-release-candidate.sh \
             {{ artifact }} \
-            {{ release|default("1.0.0") }} {{ rc|default("0") }}
+            {{ release|default(arrow.head) }} {{ rc|default("") }}

dev/tasks/verify-rc/github.win.yml

@@ -40,6 +40,6 @@ jobs:
           choco install wget
       - name: Run verification
         shell: cmd
-        run: |
-          cd arrow
-          dev/release/{{ script }} {{ release|default("1.0.0") }} {{ rc|default("0") }}
+        env:
+          SOURCE_REPOSITORY: {{ arrow.remote }}
+        run: arrow/dev/release/{{ script }} {{ release|default(arrow.head) }} {{ rc|default("") }}