diff --git a/dev/release/verify-release-candidate.bat b/dev/release/verify-release-candidate.bat index 387eb25e1db..d984ea85785 100644 --- a/dev/release/verify-release-candidate.bat +++ b/dev/release/verify-release-candidate.bat @@ -27,15 +27,25 @@ if not exist "C:\tmp\arrow-verify-release" mkdir C:\tmp\arrow-verify-release set _VERIFICATION_DIR=C:\tmp\arrow-verify-release set _VERIFICATION_DIR_UNIX=C:/tmp/arrow-verify-release set _VERIFICATION_CONDA_ENV=%_VERIFICATION_DIR%\conda-env -set _DIST_URL=https://dist.apache.org/repos/dist/dev/arrow -set _TARBALL=apache-arrow-%1.tar.gz set ARROW_SOURCE=%_VERIFICATION_DIR%\apache-arrow-%1 set INSTALL_DIR=%_VERIFICATION_DIR%\install -@rem Requires GNU Wget for Windows -wget --no-check-certificate -O %_TARBALL% %_DIST_URL%/apache-arrow-%1-rc%2/%_TARBALL% || exit /B 1 - -tar xf %_TARBALL% -C %_VERIFICATION_DIR_UNIX% +set VERSION=%1 +set RC_NUMBER=%2 + +if "%RC_NUMBER%"=="" ( + @rem verify a specific git revision + if "%SOURCE_REPOSITORY%"=="" set SOURCE_REPOSITORY="https://github.com/apache/arrow.git" + git clone --recurse-submodules %SOURCE_REPOSITORY% %ARROW_SOURCE% + git -C %ARROW_SOURCE% checkout %VERSION% +) else ( + @rem verify a release candidate tarball + @rem Requires GNU Wget for Windows + set TARBALL_NAME=apache-arrow-%VERSION%.tar.gz + set TARBALL_URL=https://dist.apache.org/repos/dist/dev/arrow/apache-arrow-%VERSION%-rc%RC_NUMBER%/%TARBALL_NAME% + wget --no-check-certificate -O %TARBALL_NAME% %TARBALL_URL% || exit /B 1 + tar xf %TARBALL_NAME% -C %_VERIFICATION_DIR_UNIX% +) set PYTHON=3.8 diff --git a/dev/release/verify-release-candidate.sh b/dev/release/verify-release-candidate.sh index b554e59e9e1..fe1aacba10f 100755 --- a/dev/release/verify-release-candidate.sh +++ b/dev/release/verify-release-candidate.sh @@ -35,9 +35,20 @@ # directory is not cleaned up automatically. case $# in + 2) ARTIFACT="$1" + VERSION="$2" + SOURCE_KIND="git" + case $ARTIFACT in + source) ;; + *) echo "Invalid argument: '${ARTIFACT}', only valid option is 'source'" + exit 1 + ;; + esac + ;; 3) ARTIFACT="$1" VERSION="$2" RC_NUMBER="$3" + SOURCE_KIND="tarball" case $ARTIFACT in source|binaries|wheels|jars) ;; *) echo "Invalid argument: '${ARTIFACT}', valid options are \ @@ -366,9 +377,14 @@ test_csharp() { fi dotnet test - mv dummy.git ../.git - dotnet pack -c Release - mv ../.git dummy.git + + if [ "${SOURCE_KIND}" = "git" ]; then + dotnet pack -c Release + else + mv dummy.git ../.git + dotnet pack -c Release + mv ../.git dummy.git + fi if ! which sourcelink > /dev/null 2>&1; then dotnet tool install --tool-path ${csharp_bin} sourcelink @@ -544,27 +560,33 @@ test_integration() { ensure_source_directory() { dist_name="apache-arrow-${VERSION}" - if [ $((${TEST_SOURCE} + ${TEST_WHEELS})) -gt 0 ]; then - import_gpg_keys - if [ ! -d "${dist_name}" ]; then - fetch_archive ${dist_name} - tar xf ${dist_name}.tar.gz - fi + if [ "${SOURCE_KIND}" = "git" ]; then + git clone --recurse-submodules ${SOURCE_REPOSITORY:-"https://github.com/apache/arrow.git"} arrow + pushd arrow + git checkout ${VERSION} else - mkdir -p ${dist_name} - if [ ! -f ${TEST_ARCHIVE} ]; then - echo "${TEST_ARCHIVE} not found" - exit 1 + if [ $((${TEST_SOURCE} + ${TEST_WHEELS})) -gt 0 ]; then + import_gpg_keys + if [ ! -d "${dist_name}" ]; then + fetch_archive ${dist_name} + tar xf ${dist_name}.tar.gz + fi + else + mkdir -p ${dist_name} + if [ ! -f ${TEST_ARCHIVE} ]; then + echo "${TEST_ARCHIVE} not found" + exit 1 + fi + tar xf ${TEST_ARCHIVE} -C ${dist_name} --strip-components=1 + fi + # clone testing repositories + pushd ${dist_name} + if [ ! -d "testing/data" ]; then + git clone https://github.com/apache/arrow-testing.git testing + fi + if [ ! -d "cpp/submodules/parquet-testing/data" ]; then + git clone https://github.com/apache/parquet-testing.git cpp/submodules/parquet-testing fi - tar xf ${TEST_ARCHIVE} -C ${dist_name} --strip-components=1 - fi - # clone testing repositories - pushd ${dist_name} - if [ ! -d "testing/data" ]; then - git clone https://github.com/apache/arrow-testing.git testing - fi - if [ ! -d "cpp/submodules/parquet-testing/data" ]; then - git clone https://github.com/apache/parquet-testing.git cpp/submodules/parquet-testing fi export ARROW_DIR=$PWD export ARROW_TEST_DATA=$PWD/testing/data diff --git a/dev/tasks/docker-tests/github.linux.yml b/dev/tasks/docker-tests/github.linux.yml index f7b7f2317a2..952dc87d111 100644 --- a/dev/tasks/docker-tests/github.linux.yml +++ b/dev/tasks/docker-tests/github.linux.yml @@ -39,8 +39,6 @@ jobs: run: | archery docker run \ -e SETUPTOOLS_SCM_PRETEND_VERSION="{{ arrow.no_rc_version }}" \ - -e VERIFY_VERSION="{{ release|default("") }}" \ - -e VERIFY_RC="{{ rc|default("") }}" \ {{ flags|default("") }} \ {{ image }} \ {{ command|default("") }} diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index 5537004fba3..e7e2912424a 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -131,6 +131,7 @@ groups: {######################## Tasks to run regularly #############################} nightly: + - verify-rc-source-* - almalinux-* - amazon-linux-* - debian-* @@ -876,12 +877,10 @@ tasks: verify-rc-source-{{ target }}-linux-{{ distribution }}-{{ version }}-amd64: ci: github - template: docker-tests/github.linux.yml + template: verify-rc/github.linux.amd64.docker.yml params: - flags: >- - -e TEST_DEFAULT=0 - -e TEST_{{ target|upper }}=1 - image: {{ distribution }}-verify-rc-source + target: {{ target }} + distro: {{ distribution }} {% endfor %} {% endfor %} diff --git a/dev/tasks/verify-rc/github.linux.amd64.docker.yml b/dev/tasks/verify-rc/github.linux.amd64.docker.yml new file mode 100644 index 00000000000..26d6cff68a4 --- /dev/null +++ b/dev/tasks/verify-rc/github.linux.amd64.docker.yml @@ -0,0 +1,39 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +{% import 'macros.jinja' as macros with context %} + +{{ macros.github_header() }} + +jobs: + test: + name: "Verify release candidate {{ distro }} source" + runs-on: ubuntu-latest + steps: + {{ macros.github_checkout_arrow(fetch_depth=fetch_depth if fetch_depth is defined else 1)|indent }} + {{ macros.github_install_archery()|indent }} + + - name: Execute Docker Build + shell: bash + run: | + archery docker run \ + -e SOURCE_REPOSITORY="{{ arrow.remote }}" \ + -e VERIFY_VERSION="{{ release|default(arrow.head) }}" \ + -e VERIFY_RC="{{ rc|default("") }}" \ + -e TEST_DEFAULT=0 \ + -e TEST_{{ target|upper }}=1 \ + {{ distro }}-verify-rc-source diff --git a/dev/tasks/verify-rc/github.linux.amd64.yml b/dev/tasks/verify-rc/github.linux.amd64.yml index 116a0c5714b..49069fe6b9b 100644 --- a/dev/tasks/verify-rc/github.linux.amd64.yml +++ b/dev/tasks/verify-rc/github.linux.amd64.yml @@ -71,7 +71,9 @@ jobs: node-version: '16' - name: Run verification shell: bash + env: + SOURCE_REPOSITORY: {{ arrow.remote }} run: | arrow/dev/release/verify-release-candidate.sh \ {{ artifact }} \ - {{ release|default("1.0.0") }} {{ rc|default("0") }} + {{ release|default(arrow.head) }} {{ rc|default("") }} diff --git a/dev/tasks/verify-rc/github.macos.amd64.yml b/dev/tasks/verify-rc/github.macos.amd64.yml index b884df8b787..335fc7af5e0 100644 --- a/dev/tasks/verify-rc/github.macos.amd64.yml +++ b/dev/tasks/verify-rc/github.macos.amd64.yml @@ -44,7 +44,9 @@ jobs: node-version: '16' - name: Run verification shell: bash + env: + SOURCE_REPOSITORY: {{ arrow.remote }} run: | arrow/dev/release/verify-release-candidate.sh \ {{ artifact }} \ - {{ release|default("1.0.0") }} {{ rc|default("0") }} + {{ release|default(arrow.head) }} {{ rc|default("") }} diff --git a/dev/tasks/verify-rc/github.macos.arm64.yml b/dev/tasks/verify-rc/github.macos.arm64.yml index a2f19f543f2..90414e22a76 100644 --- a/dev/tasks/verify-rc/github.macos.arm64.yml +++ b/dev/tasks/verify-rc/github.macos.arm64.yml @@ -39,10 +39,12 @@ jobs: - name: Run verification shell: bash + env: + SOURCE_REPOSITORY: {{ arrow.remote }} run: | export PATH="$(brew --prefix node@16)/bin:$PATH" export PATH="$(brew --prefix ruby)/bin:$PATH" export PKG_CONFIG_PATH="$(brew --prefix ruby)/lib/pkgconfig" arch -{{ arch_emulation|default("arm64") }} arrow/dev/release/verify-release-candidate.sh \ {{ artifact }} \ - {{ release|default("1.0.0") }} {{ rc|default("0") }} + {{ release|default(arrow.head) }} {{ rc|default("") }} diff --git a/dev/tasks/verify-rc/github.win.yml b/dev/tasks/verify-rc/github.win.yml index 5406327e874..233b5343e6c 100644 --- a/dev/tasks/verify-rc/github.win.yml +++ b/dev/tasks/verify-rc/github.win.yml @@ -40,6 +40,6 @@ jobs: choco install wget - name: Run verification shell: cmd - run: | - cd arrow - dev/release/{{ script }} {{ release|default("1.0.0") }} {{ rc|default("0") }} + env: + SOURCE_REPOSITORY: {{ arrow.remote }} + run: arrow/dev/release/{{ script }} {{ release|default(arrow.head) }} {{ rc|default("") }}