Skip to content

Commit 19e3f90

Browse files
koukou
authored
GH-45921: [Release][R] Use GitHub Release not apache.jfrog.io (#45964)
### Rationale for this change We want to stop using apache.jfrog.io as much as possible. See also: #40760 We can use https://github.com/apache/arrow/releases as alternative. ### What changes are included in this PR? If we use GitHub Release, we can automate uploading by GitHub Actions. But GitHub Release doesn't support directory structure. So we need to put all artifacts to https://github.com/apache/arrow/releases/download/apache-arrow-X.Y.Z/ . For example: apache.jfrog.io: ```text libarrow/bin/darwin-arm64-openssl-1.1/arrow-X.Y.Z.zip libarrow/bin/darwin-arm64-openssl-3.0/arrow-X.Y.Z.zip libarrow/bin/darwin-x86_64-openssl-1.1/arrow-X.Y.Z.zip libarrow/bin/darwin-x86_64-openssl-3.0/arrow-X.Y.Z.zip libarrow/bin/linux-openssl-1.0/arrow-X.Y.Z.zip libarrow/bin/linux-openssl-1.1/arrow-X.Y.Z.zip libarrow/bin/linux-openssl-3.0/arrow-X.Y.Z.zip libarrow/bin/windows/arrow-X.Y.Z.zip ``` GitHub Actions: ```text r-libarrow-darwin-arm64-openssl-1.1--X.Y.Z.zip r-libarrow-darwin-arm64-openssl-3.0-X.Y.Z.zip r-libarrow-darwin-x86_64-openssl-1.1-X.Y.Z.zip r-libarrow-darwin-x86_64-openssl-3.0-X.Y.Z.zip r-libarrow-linux-x86_64-openssl-1.0-X.Y.Z.zip r-libarrow-linux-x86_64-openssl-1.1-X.Y.Z.zip r-libarrow-linux-x86_64-openssl-3.0-X.Y.Z.zip r-libarrow-windows-x86_64-X.Y.Z.zip ``` ### Are these changes tested? No. ### Are there any user-facing changes? Yes. Custom `arrow_repo` doesn't work because URL is changed. * GitHub Issue: #45921 Lead-authored-by: Sutou Kouhei <kou@clear-code.com> Co-authored-by: Sutou Kouhei <kou@cozmixng.org> Signed-off-by: Sutou Kouhei <kou@clear-code.com>
1 parent b59bdef commit 19e3f90

File tree

11 files changed

+117
-217
lines changed

11 files changed

+117
-217
lines changed

‎ci/scripts/r_windows_build.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ printenv
3535
makepkg-mingw --noconfirm --noprogressbar --skippgpcheck --nocheck --syncdeps --cleanbuild
3636

3737
VERSION=$(grep Version $ARROW_HOME/r/DESCRIPTION | cut -d " " -f 2)
38-
DST_DIR="arrow-$VERSION"
38+
DST_DIR="r-libarrow-windows-x86_64-$VERSION"
3939

4040
# Collect the build artifacts and make the shape of zip file that rwinlib expects
4141
ls

‎dev/release/05-binary-upload.sh‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -109,6 +109,10 @@ if [ "${UPLOAD_PYTHON}" -gt 0 ]; then
109109
upload_to_github_release python \
110110
"${ARROW_ARTIFACTS_DIR}"/{python-sdist,wheel-*}/*
111111
fi
112+
if [ "${UPLOAD_R}" -gt 0 ]; then
113+
upload_to_github_release r \
114+
"${ARROW_ARTIFACTS_DIR}"/r-binary-packages/r-lib*
115+
fi
112116

113117
rake_tasks=()
114118
apt_targets=()
@@ -129,9 +133,6 @@ if [ "${UPLOAD_DEBIAN}" -gt 0 ]; then
129133
rake_tasks+=(apt:rc)
130134
apt_targets+=(debian)
131135
fi
132-
if [ "${UPLOAD_R}" -gt 0 ]; then
133-
rake_tasks+=(r:rc)
134-
fi
135136
if [ "${UPLOAD_UBUNTU}" -gt 0 ]; then
136137
rake_tasks+=(apt:rc)
137138
apt_targets+=(ubuntu)

‎dev/release/binary-task.rb‎

Lines changed: 0 additions & 136 deletions
Original file line numberDiff line numberDiff line change
@@ -2232,142 +2232,6 @@ def define_yum_tasks
22322232
define_yum_release_tasks
22332233
end
22342234

2235-
def define_generic_data_rc_tasks(label,
2236-
id,
2237-
rc_dir,
2238-
target_files_glob)
2239-
directory rc_dir
2240-
2241-
namespace id do
2242-
namespace :rc do
2243-
desc "Copy #{label} packages"
2244-
task :copy => rc_dir do
2245-
progress_label = "Copying: #{label}"
2246-
progress_reporter = ProgressReporter.new(progress_label)
2247-
2248-
Pathname(artifacts_dir).glob(target_files_glob) do |path|
2249-
next if path.directory?
2250-
destination_path = [
2251-
rc_dir,
2252-
path.basename.to_s,
2253-
].join("/")
2254-
copy_artifact(path, destination_path, progress_reporter)
2255-
end
2256-
2257-
progress_reporter.finish
2258-
end
2259-
2260-
desc "Sign #{label} packages"
2261-
task :sign => rc_dir do
2262-
sign_dir(label, rc_dir)
2263-
end
2264-
2265-
desc "Upload #{label} packages"
2266-
task :upload do
2267-
uploader =
2268-
ArtifactoryUploader.new(api_key: artifactory_api_key,
2269-
destination_prefix: full_version,
2270-
distribution: id.to_s,
2271-
rc: rc,
2272-
source: rc_dir,
2273-
staging: staging?)
2274-
uploader.upload
2275-
end
2276-
end
2277-
2278-
desc "Release RC #{label} packages"
2279-
rc_tasks = [
2280-
"#{id}:rc:copy",
2281-
"#{id}:rc:sign",
2282-
"#{id}:rc:upload",
2283-
]
2284-
task :rc => rc_tasks
2285-
end
2286-
end
2287-
2288-
def define_generic_data_release_tasks(label, id, release_dir)
2289-
directory release_dir
2290-
2291-
namespace id do
2292-
desc "Release #{label} packages"
2293-
task :release do
2294-
release_distribution(id.to_s,
2295-
rc_prefix: full_version,
2296-
release_prefix: version)
2297-
end
2298-
end
2299-
end
2300-
2301-
def define_generic_data_tasks(label,
2302-
id,
2303-
rc_dir,
2304-
release_dir,
2305-
target_files_glob)
2306-
define_generic_data_rc_tasks(label, id, rc_dir, target_files_glob)
2307-
define_generic_data_release_tasks(label, id, release_dir)
2308-
end
2309-
2310-
def define_r_rc_tasks(label, id, rc_dir)
2311-
directory rc_dir
2312-
2313-
namespace id do
2314-
namespace :rc do
2315-
desc "Prepare #{label} packages"
2316-
task :prepare => rc_dir do
2317-
progress_label = "Preparing #{label}"
2318-
progress_reporter = ProgressReporter.new(progress_label)
2319-
2320-
pattern = "r-binary-packages/r-lib*.{zip,tgz}"
2321-
Pathname(artifacts_dir).glob(pattern) do |path|
2322-
destination_path = [
2323-
rc_dir,
2324-
# r-lib__libarrow__bin__centos-7__arrow-8.0.0.zip
2325-
# --> libarrow/bin/centos-7/arrow-8.0.0.zip
2326-
path.basename.to_s.gsub(/\Ar-lib__/, "").gsub(/__/, "/"),
2327-
].join("/")
2328-
copy_artifact(path, destination_path, progress_reporter)
2329-
end
2330-
2331-
progress_reporter.finish
2332-
end
2333-
2334-
desc "Sign #{label} packages"
2335-
task :sign => rc_dir do
2336-
sign_dir(label, rc_dir)
2337-
end
2338-
2339-
desc "Upload #{label} packages"
2340-
task :upload do
2341-
uploader =
2342-
ArtifactoryUploader.new(api_key: artifactory_api_key,
2343-
destination_prefix: full_version,
2344-
distribution: id.to_s,
2345-
rc: rc,
2346-
source: rc_dir,
2347-
staging: staging?)
2348-
uploader.upload
2349-
end
2350-
end
2351-
2352-
desc "Release RC #{label} packages"
2353-
rc_tasks = [
2354-
"#{id}:rc:prepare",
2355-
"#{id}:rc:sign",
2356-
"#{id}:rc:upload",
2357-
]
2358-
task :rc => rc_tasks
2359-
end
2360-
end
2361-
2362-
def define_r_tasks
2363-
label = "R"
2364-
id = :r
2365-
r_rc_dir = "#{rc_dir}/r/#{full_version}"
2366-
r_release_dir = "#{release_dir}/r/#{full_version}"
2367-
define_r_rc_tasks(label, id, r_rc_dir)
2368-
define_generic_data_release_tasks(label, id, r_release_dir)
2369-
end
2370-
23712235
def define_summary_tasks
23722236
namespace :summary do
23732237
desc "Show RC summary"

‎dev/release/post-03-binary.sh‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,6 @@ cd "${SOURCE_DIR}"
4343
: "${DEPLOY_AMAZON_LINUX:=${DEPLOY_DEFAULT}}"
4444
: "${DEPLOY_CENTOS:=${DEPLOY_DEFAULT}}"
4545
: "${DEPLOY_DEBIAN:=${DEPLOY_DEFAULT}}"
46-
: "${DEPLOY_R:=${DEPLOY_DEFAULT}}"
4746
: "${DEPLOY_UBUNTU:=${DEPLOY_DEFAULT}}"
4847

4948
rake_tasks=()
@@ -65,9 +64,6 @@ if [ "${DEPLOY_DEBIAN}" -gt 0 ]; then
6564
rake_tasks+=(apt:release)
6665
apt_targets+=(debian)
6766
fi
68-
if [ "${DEPLOY_R}" -gt 0 ]; then
69-
rake_tasks+=(r:release)
70-
fi
7167
if [ "${DEPLOY_UBUNTU}" -gt 0 ]; then
7268
rake_tasks+=(apt:release)
7369
apt_targets+=(ubuntu)

‎dev/tasks/macros.jinja‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -243,16 +243,16 @@ env:
243243
- name: Get windows binary
244244
uses: actions/download-artifact@v4
245245
with:
246-
name: r-lib__libarrow__bin__windows
247-
path: repo/libarrow/bin/windows
246+
name: r-libarrow-windows-x86_64
247+
path: repo/libarrow
248248
{% endif %}
249249
{% if get_nix %}
250250
{% for openssl_version in ["1.0", "1.1", "3.0"] %}
251251
- name: Get Linux OpenSSL {{ openssl_version }} binary
252252
uses: actions/download-artifact@v4
253253
with:
254-
name: r-lib__libarrow__bin__linux-openssl-{{ openssl_version }}
255-
path: repo/libarrow/bin/linux-openssl-{{ openssl_version }}
254+
name: r-libarrow-linux-x86_64-openssl-{{ openssl_version }}
255+
path: repo/libarrow
256256
{% endfor %}
257257
{% endif %}
258258
{% if get_mac %}
@@ -261,8 +261,8 @@ env:
261261
- name: Get macOS {{ arch }} OpenSSL {{ openssl_version }} binary
262262
uses: actions/download-artifact@v4
263263
with:
264-
name: r-lib__libarrow__bin__darwin-{{arch}}-openssl-{{ openssl_version }}
265-
path: repo/libarrow/bin/darwin-{{ arch }}-openssl-{{ openssl_version }}
264+
name: r-libarrow-darwin-{{ arch }}-openssl-{{ openssl_version }}
265+
path: repo/libarrow
266266
{% endfor %}
267267
{% endfor %}
268268
{% endif %}
@@ -294,7 +294,7 @@ env:
294294
write(str, file = profile_path, append = TRUE)
295295
# Set envvar for later steps by appending to $GITHUB_ENV
296296
write(paste0("R_PROFILE_USER=", profile_path), file = Sys.getenv("GITHUB_ENV"), append = TRUE)
297-
{% endmacro %}
297+
{% endmacro %}
298298

299299
{# Detect if we are using a fork or the upstream repo #}
300300
{% set is_upstream_b = arrow.github_repo == 'apache/arrow' %}

‎dev/tasks/r/github.packages.yml‎

Lines changed: 33 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -58,9 +58,7 @@ jobs:
5858

5959
macos-cpp:
6060
name: C++ Binary macOS OpenSSL {{ '${{ matrix.openssl }}' }} {{ '${{ matrix.platform.arch }}' }}
61-
6261
runs-on: {{ '${{ matrix.platform.runs_on }}' }}
63-
6462
needs: source
6563
strategy:
6664
fail-fast: false
@@ -69,7 +67,9 @@ jobs:
6967
- { runs_on: macos-13, arch: "x86_64" }
7068
- { runs_on: macos-14, arch: "arm64" }
7169
openssl: ['3.0', '1.1']
72-
70+
env:
71+
PKG_ID: r-libarrow-darwin-{{ '${{ matrix.platform.arch }}' }}-openssl-{{ '${{ matrix.openssl }}' }}
72+
PKG_FILE: r-libarrow-darwin-{{ '${{ matrix.platform.arch }}' }}-openssl-{{ '${{ matrix.openssl }}' }}-{{ '${{ needs.source.outputs.pkg_version }}' }}.zip
7373
steps:
7474
{{ macros.github_checkout_arrow(action_v="3")|indent }}
7575
{{ macros.github_change_r_pkg_version(is_fork, '${{ needs.source.outputs.pkg_version }}')|indent }}
@@ -94,22 +94,22 @@ jobs:
9494
r/inst/build_arrow_static.sh
9595
- name: Bundle libarrow
9696
shell: bash
97-
env:
98-
PKG_FILE: arrow-{{ '${{ needs.source.outputs.pkg_version }}' }}.zip
99-
VERSION: {{ '${{ needs.source.outputs.pkg_version }}' }}
10097
run: |
10198
cd arrow/r/libarrow/dist
10299
zip -r $PKG_FILE lib/ include/
103100
- name: Create Checksum
104101
shell: bash
105102
run: |
106103
cd arrow/r/libarrow/dist
107-
shasum -a 512 arrow-*.zip > arrow-{{ '${{ needs.source.outputs.pkg_version }}' }}.zip.sha512
104+
shasum -a 512 ${PKG_FILE} > ${PKG_FILE}.sha512
105+
mv ${PKG_FILE}{,.sha512} ../../../../
108106
- name: Upload binary artifact
109107
uses: actions/upload-artifact@v4
110108
with:
111-
name: r-lib__libarrow__bin__darwin-{{ '${{ matrix.platform.arch }}' }}-openssl-{{ '${{ matrix.openssl }}' }}
112-
path: arrow/r/libarrow/dist/arrow-*.zip*
109+
name: {{ '${{ env.PKG_ID }}' }}
110+
path: |
111+
{{ '${{ env.PKG_FILE }}' }}
112+
{{ '${{ env.PKG_FILE }}' }}.sha512
113113
114114
linux-cpp:
115115
name: C++ Binary Linux OpenSSL {{ '${{ matrix.openssl }}' }}
@@ -129,11 +129,13 @@ jobs:
129129
os: centos
130130
- openssl: "1.0"
131131
os: centos
132+
env:
133+
PKG_ID: r-libarrow-linux-x86_64-openssl-{{ '${{ matrix.openssl }}' }}
134+
PKG_FILE: r-libarrow-linux-x86_64-openssl-{{ '${{ matrix.openssl }}' }}-{{ '${{ needs.source.outputs.pkg_version }}' }}.zip
132135
steps:
133136
{{ macros.github_checkout_arrow()|indent }}
134137
{{ macros.github_change_r_pkg_version(is_fork, '${{ needs.source.outputs.pkg_version }}')|indent }}
135138
{{ macros.github_install_archery()|indent }}
136-
137139
- name: Build libarrow
138140
shell: bash
139141
env:
@@ -146,9 +148,6 @@ jobs:
146148
{{ '${{ matrix.os }}' }}-cpp-static
147149
- name: Bundle libarrow
148150
shell: bash
149-
env:
150-
PKG_FILE: arrow-{{ '${{ needs.source.outputs.pkg_version }}' }}.zip
151-
VERSION: {{ '${{ needs.source.outputs.pkg_version }}' }}
152151
run: |
153152
# These files were created by the docker user so we have to chown them
154153
sudo chown -R $USER:$USER arrow/r/libarrow
@@ -159,17 +158,23 @@ jobs:
159158
shell: bash
160159
run: |
161160
cd arrow/r/libarrow/dist
162-
shasum -a 512 arrow-*.zip > arrow-{{ '${{ needs.source.outputs.pkg_version }}' }}.zip.sha512
161+
shasum -a 512 ${PKG_FILE} > ${PKG_FILE}.sha512
162+
mv ${PKG_FILE}{,.sha512} ../../../../
163163
- name: Upload binary artifact
164164
uses: actions/upload-artifact@v4
165165
with:
166-
name: r-lib__libarrow__bin__linux-openssl-{{ '${{ matrix.openssl }}' }}
167-
path: arrow/r/libarrow/dist/arrow-*.zip*
166+
name: {{ '${{ env.PKG_ID }}' }}
167+
path: |
168+
{{ '${{ env.PKG_FILE }}' }}
169+
{{ '${{ env.PKG_FILE }}' }}.sha512
168170
169171
windows-cpp:
170172
name: C++ Binary Windows RTools (40 only)
171173
needs: source
172174
runs-on: windows-latest
175+
env:
176+
PKG_ID: r-libarrow-windows-x86_64
177+
PKG_FILE: r-libarrow-windows-x86_64-{{ '${{ needs.source.outputs.pkg_version }}' }}.zip
173178
steps:
174179
- run: git config --global core.autocrlf false
175180
{{ macros.github_checkout_arrow()|indent }}
@@ -192,12 +197,15 @@ jobs:
192197
shell: bash
193198
run: |
194199
cd build
195-
sha512sum arrow-*.zip > arrow-{{ '${{ needs.source.outputs.pkg_version }}' }}.zip.sha512
200+
sha512sum ${PKG_FILE} > ${PKG_FILE}.sha512
201+
mv ${PKG_FILE}{,.sha512} ../
196202
- name: Upload binary artifact
197203
uses: actions/upload-artifact@v4
198204
with:
199-
name: r-lib__libarrow__bin__windows
200-
path: build/arrow-*.zip*
205+
name: {{ '${{ env.PKG_ID }}' }}
206+
path: |
207+
{{ '${{ env.PKG_FILE }}' }}
208+
{{ '${{ env.PKG_FILE }}' }}.sha512
201209
202210
r-packages:
203211
needs: [source, windows-cpp, macos-cpp]
@@ -249,7 +257,7 @@ jobs:
249257
LIBARROW_BINARY: "true" # has to be set as long as allowlist not updated
250258
LIBARROW_BUILD: "false"
251259
ARROW_R_ENFORCE_CHECKSUM: "true"
252-
ARROW_R_CHECKSUM_PATH: "{{ '${{ github.workspace }}' }}/repo/libarrow/bin"
260+
ARROW_R_CHECKSUM_PATH: "{{ '${{ github.workspace }}' }}/repo/libarrow"
253261
run: |
254262
on_windows <- tolower(Sys.info()[["sysname"]]) == "windows"
255263
@@ -338,7 +346,7 @@ jobs:
338346
LIBARROW_BUILD: "FALSE"
339347
LIBARROW_BINARY: {{ '${{ matrix.config.libarrow_binary }}' }}
340348
ARROW_R_ENFORCE_CHECKSUM: "true"
341-
ARROW_R_CHECKSUM_PATH: "{{ '${{ github.workspace }}' }}/repo/libarrow/bin"
349+
ARROW_R_CHECKSUM_PATH: "{{ '${{ github.workspace }}' }}/repo/libarrow"
342350
shell: bash
343351
run: |
344352
Rscript -e '
@@ -447,6 +455,10 @@ jobs:
447455
uses: r-lib/actions/setup-r@v2
448456
with:
449457
install-r: true
458+
- name: Move libarrow artifacts
459+
run: |
460+
mkdir -p binaries/
461+
mv artifacts/r-libarrow-*/* binaries/
450462
- name: Rename artifacts
451463
shell: Rscript {0}
452464
run: |

0 commit comments

Comments
 (0)