Skip to content

Commit 31470ec

Browse files
pan3793pan3793
committed
[KYUUBI #7205] Use Apache closer.lua syntax for kyuubi-download to obtain engine archives
Speed up download Spark, Flink, and Hive tarballs, for the latest versions. https://infra.apache.org/release-download-pages.html#download-scripts Observe GHA. No. Closes #7205 from pan3793/closer-lua. Closes #7205 10ffa34 [Cheng Pan] Use Apache closer.lua syntax for kyuubi-download to obtain engine archives Authored-by: Cheng Pan <chengpan@apache.org> Signed-off-by: Cheng Pan <chengpan@apache.org> (cherry picked from commit 28bba27) Signed-off-by: Cheng Pan <chengpan@apache.org>
1 parent 874e2de commit 31470ec

File tree

4 files changed

+23
-14
lines changed

4 files changed

+23
-14
lines changed

‎.github/workflows/master.yml‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -62,22 +62,22 @@ jobs:
6262
comment: 'normal'
6363
- java: 8
6464
spark: '3.5'
65-
spark-archive: '-Dspark.archive.mirror=https://archive.apache.org/dist/spark/spark-3.2.4 -Dspark.archive.name=spark-3.2.4-bin-hadoop3.2.tgz -Pzookeeper-3.6'
65+
spark-archive: '-Dspark.archive.mirror=https://www.apache.org/dyn/closer.lua/spark/spark-3.2.4 -Dspark.archive.name=spark-3.2.4-bin-hadoop3.2.tgz -Pzookeeper-3.6'
6666
exclude-tags: '-Dmaven.plugin.scalatest.exclude.tags=org.scalatest.tags.Slow,org.apache.kyuubi.tags.DeltaTest,org.apache.kyuubi.tags.IcebergTest,org.apache.kyuubi.tags.PaimonTest,org.apache.kyuubi.tags.HudiTest,org.apache.kyuubi.tags.SparkLocalClusterTest'
6767
comment: 'verify-on-spark-3.2-binary'
6868
- java: 8
6969
spark: '3.5'
70-
spark-archive: '-Dspark.archive.mirror=https://archive.apache.org/dist/spark/spark-3.3.3 -Dspark.archive.name=spark-3.3.3-bin-hadoop3.tgz -Pzookeeper-3.6'
70+
spark-archive: '-Dspark.archive.mirror=https://www.apache.org/dyn/closer.lua/spark/spark-3.3.3 -Dspark.archive.name=spark-3.3.3-bin-hadoop3.tgz -Pzookeeper-3.6'
7171
exclude-tags: '-Dmaven.plugin.scalatest.exclude.tags=org.scalatest.tags.Slow,org.apache.kyuubi.tags.DeltaTest,org.apache.kyuubi.tags.IcebergTest,org.apache.kyuubi.tags.PaimonTest,org.apache.kyuubi.tags.HudiTest,org.apache.kyuubi.tags.SparkLocalClusterTest'
7272
comment: 'verify-on-spark-3.3-binary'
7373
- java: 8
7474
spark: '3.5'
75-
spark-archive: '-Dspark.archive.mirror=https://archive.apache.org/dist/spark/spark-3.4.3 -Dspark.archive.name=spark-3.4.3-bin-hadoop3.tgz -Pzookeeper-3.6'
75+
spark-archive: '-Dspark.archive.mirror=https://www.apache.org/dyn/closer.lua/spark/spark-3.4.3 -Dspark.archive.name=spark-3.4.3-bin-hadoop3.tgz -Pzookeeper-3.6'
7676
exclude-tags: '-Dmaven.plugin.scalatest.exclude.tags=org.scalatest.tags.Slow,org.apache.kyuubi.tags.DeltaTest,org.apache.kyuubi.tags.IcebergTest,org.apache.kyuubi.tags.PaimonTest,org.apache.kyuubi.tags.SparkLocalClusterTest'
7777
comment: 'verify-on-spark-3.4-binary'
7878
- java: 17
7979
spark: '3.5'
80-
spark-archive: '-Pscala-2.13 -Dspark.archive.mirror=https://archive.apache.org/dist/spark/spark-4.0.1 -Dspark.archive.name=spark-4.0.1-bin-hadoop3.tgz'
80+
spark-archive: '-Pscala-2.13 -Dspark.archive.mirror=https://www.apache.org/dyn/closer.lua/spark/spark-4.0.1 -Dspark.archive.name=spark-4.0.1-bin-hadoop3.tgz'
8181
exclude-tags: '-Dmaven.plugin.scalatest.exclude.tags=org.scalatest.tags.Slow,org.apache.kyuubi.tags.DeltaTest,org.apache.kyuubi.tags.IcebergTest,org.apache.kyuubi.tags.PaimonTest,org.apache.kyuubi.tags.SparkLocalClusterTest'
8282
comment: 'verify-on-spark-4.0-binary'
8383
env:
@@ -254,15 +254,15 @@ jobs:
254254
include:
255255
- java: 8
256256
flink: '1.20'
257-
flink-archive: '-Dflink.archive.mirror=https://archive.apache.org/dist/flink/flink-1.17.2 -Dflink.archive.name=flink-1.17.2-bin-scala_2.12.tgz'
257+
flink-archive: '-Dflink.archive.mirror=https://www.apache.org/dyn/closer.lua/flink/flink-1.17.2 -Dflink.archive.name=flink-1.17.2-bin-scala_2.12.tgz'
258258
comment: 'verify-on-flink-1.17-binary'
259259
- java: 8
260260
flink: '1.20'
261-
flink-archive: '-Dflink.archive.mirror=https://archive.apache.org/dist/flink/flink-1.18.1 -Dflink.archive.name=flink-1.18.1-bin-scala_2.12.tgz'
261+
flink-archive: '-Dflink.archive.mirror=https://www.apache.org/dyn/closer.lua/flink/flink-1.18.1 -Dflink.archive.name=flink-1.18.1-bin-scala_2.12.tgz'
262262
comment: 'verify-on-flink-1.18-binary'
263263
- java: 8
264264
flink: '1.20'
265-
flink-archive: '-Dflink.archive.mirror=https://archive.apache.org/dist/flink/flink-1.19.1 -Dflink.archive.name=flink-1.19.1-bin-scala_2.12.tgz'
265+
flink-archive: '-Dflink.archive.mirror=https://www.apache.org/dyn/closer.lua/flink/flink-1.19.1 -Dflink.archive.name=flink-1.19.1-bin-scala_2.12.tgz'
266266
comment: 'verify-on-flink-1.19-binary'
267267
steps:
268268
- uses: actions/checkout@v4
@@ -317,10 +317,10 @@ jobs:
317317
comment: [ "normal" ]
318318
include:
319319
- java: 8
320-
hive-archive: '-Dhive.archive.mirror=https://archive.apache.org/dist/hive/hive-2.3.10 -Dhive.archive.name=apache-hive-2.3.10-bin.tar.gz'
320+
hive-archive: '-Dhive.archive.mirror=https://www.apache.org/dyn/closer.lua/hive/hive-2.3.10 -Dhive.archive.name=apache-hive-2.3.10-bin.tar.gz'
321321
comment: 'verify-on-hive-2.3-binary'
322322
- java: 8
323-
hive-archive: '-Dhive.archive.mirror=https://github.com/pan3793/cdh-hive/releases/download/cdh6.3.2-release -Dhive.archive.name=apache-hive-2.1.1-cdh6.3.2-bin.tar.gz'
323+
hive-archive: '-Dhive.archive.mirror=https://github.com/pan3793/cdh-hive/releases/download/cdh6.3.2-release -Dhive.archive.name=apache-hive-2.1.1-cdh6.3.2-bin.tar.gz -Dhive.archive.query='
324324
comment: 'verify-on-hive-2.1-cdh6-binary'
325325
steps:
326326
- uses: actions/checkout@v4

‎docs/contributing/code/building.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -82,7 +82,7 @@ For the Scala version for Spark engines, the server will look up the `SPARK_SCAL
8282

8383
## Building With Apache dlcdn Site
8484

85-
By default, we use `https://archive.apache.org/dist/` to download the built-in release packages of engines,
85+
By default, we use [`closer.lua`](https://infra.apache.org/release-download-pages.html#download-scripts) to download the built-in release packages of engines,
8686
such as Spark or Flink.
8787
But sometimes, you may find it hard to reach, or the download speed is too slow,
8888
then you can define the `apache.archive.dist` by `-Pmirror-cdn` to accelerate to download speed.

‎externals/kyuubi-download/pom.xml‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,8 @@
5151
<phase>compile</phase>
5252
<configuration>
5353
<skip>${spark.archive.download.skip}</skip>
54-
<url>${spark.archive.mirror}/${spark.archive.name}</url>
54+
<url>${spark.archive.mirror}/${spark.archive.name}${spark.archive.query}</url>
55+
<outputFileName>${spark.archive.name}</outputFileName>
5556
</configuration>
5657
</execution>
5758
<execution>
@@ -62,7 +63,8 @@
6263
<phase>compile</phase>
6364
<configuration>
6465
<skip>${flink.archive.download.skip}</skip>
65-
<url>${flink.archive.mirror}/${flink.archive.name}</url>
66+
<url>${flink.archive.mirror}/${flink.archive.name}${flink.archive.query}</url>
67+
<outputFileName>${flink.archive.name}</outputFileName>
6668
</configuration>
6769
</execution>
6870
<execution>
@@ -73,7 +75,8 @@
7375
<phase>compile</phase>
7476
<configuration>
7577
<skip>${hive.archive.download.skip}</skip>
76-
<url>${hive.archive.mirror}/${hive.archive.name}</url>
78+
<url>${hive.archive.mirror}/${hive.archive.name}${hive.archive.query}</url>
79+
<outputFileName>${hive.archive.name}</outputFileName>
7780
</configuration>
7881
</execution>
7982
</executions>

‎pom.xml‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -125,7 +125,7 @@
125125
<!-- Please don't upgrade the version to 4.10+, it depends on JDK 11 -->
126126
<antlr4.version>4.9.3</antlr4.version>
127127
<antlr.st4.version>4.3.4</antlr.st4.version>
128-
<apache.archive.dist>https://archive.apache.org/dist</apache.archive.dist>
128+
<apache.archive.dist>https://www.apache.org/dyn/closer.lua</apache.archive.dist>
129129
<atlas.version>2.3.0</atlas.version>
130130
<byte-buddy.version>1.14.15</byte-buddy.version>
131131
<bouncycastle.version>1.78</bouncycastle.version>
@@ -143,6 +143,7 @@
143143
<flink.version>1.20.0</flink.version>
144144
<flink.archive.name>flink-${flink.version}-bin-scala_2.12.tgz</flink.archive.name>
145145
<flink.archive.mirror>${apache.archive.dist}/flink/flink-${flink.version}</flink.archive.mirror>
146+
<flink.archive.query>?action=download</flink.archive.query>
146147
<flink.archive.download.skip>false</flink.archive.download.skip>
147148
<google.jsr305.version>3.0.2</google.jsr305.version>
148149
<grpc.version>1.65.1</grpc.version>
@@ -154,6 +155,7 @@
154155
<hive.version>3.1.3</hive.version>
155156
<hive.archive.name>apache-hive-${hive.version}-bin.tar.gz</hive.archive.name>
156157
<hive.archive.mirror>${apache.archive.dist}/hive/hive-${hive.version}</hive.archive.mirror>
158+
<hive.archive.query>?action=download</hive.archive.query>
157159
<hive.archive.download.skip>false</hive.archive.download.skip>
158160
<httpclient.version>4.5.14</httpclient.version>
159161
<httpcore.version>4.4.16</httpcore.version>
@@ -204,6 +206,7 @@
204206
<spark.archive.scala.suffix></spark.archive.scala.suffix>
205207
<spark.archive.name>spark-${spark.version}-bin-hadoop3${spark.archive.scala.suffix}.tgz</spark.archive.name>
206208
<spark.archive.mirror>${apache.archive.dist}/spark/spark-${spark.version}</spark.archive.mirror>
209+
<spark.archive.query>?action=download</spark.archive.query>
207210
<spark.archive.download.skip>false</spark.archive.download.skip>
208211
<sqlite.version>3.46.1.3</sqlite.version>
209212
<supercsv.version>2.2.0</supercsv.version>
@@ -1905,6 +1908,9 @@
19051908
<properties>
19061909
<!-- the apache cdn mirror works only for latest apache releases -->
19071910
<apache.archive.dist>https://dlcdn.apache.org</apache.archive.dist>
1911+
<spark.archive.query></spark.archive.query>
1912+
<flink.archive.query></flink.archive.query>
1913+
<hive.archive.query></hive.archive.query>
19081914
<nodeDownloadRoot>https://npmmirror.com/mirrors/node/</nodeDownloadRoot>
19091915
<pnpmDownloadRoot>https://registry.npmmirror.com/pnpm/-/</pnpmDownloadRoot>
19101916
</properties>

0 commit comments

Comments
 (0)