Skip to content

Commit 5e58dcc

Browse files
committed
Improve staging release scripts
1 parent bcb5eaa commit 5e58dcc

File tree

3 files changed

+91
-10
lines changed

3 files changed

+91
-10
lines changed

scripts/release/deploy_staging_jars.sh

Lines changed: 8 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -46,27 +46,26 @@ declare -a ALL_VERSION_OPTS=(
4646
# hudi-utilities-bundle_2.13
4747
# hudi-utilities-slim-bundle_2.13
4848
# hudi-cli-bundle_2.13
49-
"-Dscala-2.13 -Dspark3.5 -pl hudi-spark-datasource/hudi-spark-common,hudi-spark-datasource/hudi-spark3.5.x,hudi-spark-datasource/hudi-spark,hudi-utilities,packaging/hudi-spark-bundle,packaging/hudi-utilities-bundle,packaging/hudi-utilities-slim-bundle,packaging/hudi-cli-bundle -am"
49+
"-T 1C -Dscala-2.13 -Dspark3.5 -pl hudi-spark-datasource/hudi-spark-common,hudi-spark-datasource/hudi-spark3.5.x,hudi-spark-datasource/hudi-spark,hudi-utilities,packaging/hudi-spark-bundle,packaging/hudi-utilities-bundle,packaging/hudi-utilities-slim-bundle,packaging/hudi-cli-bundle -am"
5050
# For Spark 3.3, Scala 2.12:
5151
# hudi-spark3.3.x_2.12
5252
# hudi-spark3.3-bundle_2.12
53-
"-Dscala-2.12 -Dspark3.3 -pl hudi-spark-datasource/hudi-spark3.3.x,packaging/hudi-spark-bundle -am"
53+
"-T 1C -Dscala-2.12 -Dspark3.3 -pl hudi-spark-datasource/hudi-spark3.3.x,packaging/hudi-spark-bundle -am"
5454
# For Spark 3.4, Scala 2.12:
5555
# hudi-spark3.4.x_2.12
5656
# hudi-spark3.4-bundle_2.12
57-
"-Dscala-2.12 -Dspark3.4 -pl hudi-spark-datasource/hudi-spark3.4.x,packaging/hudi-spark-bundle -am"
57+
"-T 1C -Dscala-2.12 -Dspark3.4 -pl hudi-spark-datasource/hudi-spark3.4.x,packaging/hudi-spark-bundle -am"
5858
# For all modules spark3.5
5959
"-Dscala-2.12 -Dspark3.5"
6060

6161
# Upload legacy Spark bundles (not overwriting previous uploads as these jar names are unique)
62-
"-Dscala-2.12 -Dspark3 -pl packaging/hudi-spark-bundle -am" # for legacy bundle name hudi-spark3-bundle_2.12
62+
"-T 1C -Dscala-2.12 -Dspark3 -pl packaging/hudi-spark-bundle -am" # for legacy bundle name hudi-spark3-bundle_2.12
6363

6464
# Upload Flink bundles (overwriting previous uploads)
65-
"-Dscala-2.12 -Dflink1.17 -Davro.version=1.11.1 -pl packaging/hudi-flink-bundle -am"
66-
"-Dscala-2.12 -Dflink1.18 -Davro.version=1.11.1 -pl packaging/hudi-flink-bundle -am"
67-
"-Dscala-2.12 -Dflink1.19 -Davro.version=1.11.1 -pl packaging/hudi-flink-bundle -am"
68-
"-Dscala-2.12 -Dflink1.20 -Davro.version=1.11.3 -pl packaging/hudi-flink-bundle -am"
69-
"-Dscala-2.12 -Dflink2.0 -Davro.version=1.11.4 -pl packaging/hudi-flink-bundle -am"
65+
"-T 1C -Dscala-2.12 -Dflink1.17 -Davro.version=1.11.4 -Dparquet.version=1.12.3 -pl packaging/hudi-flink-bundle -am"
66+
"-T 1C -Dscala-2.12 -Dflink1.18 -Davro.version=1.11.4 -Dparquet.version=1.13.1 -pl packaging/hudi-flink-bundle -am"
67+
"-T 1C -Dscala-2.12 -Dflink1.19 -Davro.version=1.11.4 -Dparquet.version=1.13.1 -pl packaging/hudi-flink-bundle -am"
68+
"-T 1C -Dscala-2.12 -Dflink1.20 -Davro.version=1.11.4 -Dparquet.version=1.13.1 -pl packaging/hudi-flink-bundle -am"
7069
)
7170
printf -v joined "'%s'\n" "${ALL_VERSION_OPTS[@]}"
7271

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
#!/bin/bash
2+
3+
#
4+
# Licensed to the Apache Software Foundation (ASF) under one or more
5+
# contributor license agreements. See the NOTICE file distributed with
6+
# this work for additional information regarding copyright ownership.
7+
# The ASF licenses this file to You under the Apache License, Version 2.0
8+
# (the "License"); you may not use this file except in compliance with
9+
# the License. You may obtain a copy of the License at
10+
#
11+
# http://www.apache.org/licenses/LICENSE-2.0
12+
#
13+
# Unless required by applicable law or agreed to in writing, software
14+
# distributed under the License is distributed on an "AS IS" BASIS,
15+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16+
# See the License for the specific language governing permissions and
17+
# limitations under the License.
18+
#
19+
20+
##
21+
## Variables with defaults (if not overwritten by environment)
22+
##
23+
MVN=${MVN:-mvn}
24+
# fail immediately
25+
set -o errexit
26+
set -o nounset
27+
28+
CURR_DIR=$(pwd)
29+
if [ ! -d "$CURR_DIR/packaging" ] ; then
30+
echo "You have to call the script from the repository root dir that contains 'packaging/'"
31+
exit 1
32+
fi
33+
34+
if [ "$#" -gt "1" ]; then
35+
echo "Only accept 0 or 1 argument. Use -h to see examples."
36+
exit 1
37+
fi
38+
39+
declare -a ALL_VERSION_OPTS=(
40+
"-T 1C -Dscala-2.12 -Dflink2.0 -Davro.version=1.11.4 -Dparquet.version=1.14.4 -pl packaging/hudi-flink-bundle -am"
41+
)
42+
printf -v joined "'%s'\n" "${ALL_VERSION_OPTS[@]}"
43+
44+
if [ "${1:-}" == "-h" ]; then
45+
echo "
46+
Usage: $(basename "$0") [OPTIONS]
47+
48+
Options:
49+
<version option> One of the version options below
50+
${joined}
51+
-h, --help
52+
"
53+
exit 0
54+
fi
55+
56+
VERSION_OPT=${1:-}
57+
valid_version_opt=false
58+
for v in "${ALL_VERSION_OPTS[@]}"; do
59+
[[ $VERSION_OPT == "$v" ]] && valid_version_opt=true
60+
done
61+
62+
if [ "$valid_version_opt" = true ]; then
63+
# run deploy for only specified version option
64+
ALL_VERSION_OPTS=("$VERSION_OPT")
65+
elif [ "$#" == "1" ]; then
66+
echo "Version option $VERSION_OPT is invalid. Use -h to see examples."
67+
exit 1
68+
fi
69+
70+
COMMON_OPTIONS="-DdeployArtifacts=true -DskipTests -DretryFailedDeploymentCount=10"
71+
for v in "${ALL_VERSION_OPTS[@]}"
72+
do
73+
# TODO: consider cleaning all modules by listing directories instead of specifying profile
74+
echo "Cleaning everything before any deployment"
75+
$MVN clean $COMMON_OPTIONS ${v}
76+
echo "Building with options ${v}"
77+
$MVN install $COMMON_OPTIONS ${v}
78+
79+
echo "Deploying to repository.apache.org with version options ${v%-am}"
80+
# remove `-am` option to only deploy intended modules
81+
$MVN deploy $COMMON_OPTIONS ${v%-am}
82+
done

scripts/release/deploy_staging_jars_java17.sh

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ declare -a ALL_VERSION_OPTS=(
4141
# hudi-spark4-common
4242
# hudi-spark4.0.x_2.13
4343
# hudi-spark4.0-bundle_2.13
44-
"-Djava17 -Djava.version=17 -Dscala-2.13 -Dspark4.0 -pl hudi-spark-datasource/hudi-spark4-common,hudi-spark-datasource/hudi-spark4.0.x,packaging/hudi-spark-bundle -am"
44+
"-T 1C -Djava17 -Djava.version=17 -Dscala-2.13 -Dspark4.0 -pl hudi-spark-datasource/hudi-spark4-common,hudi-spark-datasource/hudi-spark4.0.x,packaging/hudi-spark-bundle -am"
4545
)
4646
printf -v joined "'%s'\n" "${ALL_VERSION_OPTS[@]}"
4747

0 commit comments

Comments
 (0)