Skip to content

Commit 2427d17

Browse files
dansanduleacRobert Kruszewski
dansanduleac
authored and
Robert Kruszewski
committed
Separate spark-dist_2.11-hadoop-palantir-bom as dependencyManagement-only parent of the dist (apache-spark-on-k8s#467)
1 parent 096a4ef commit 2427d17

File tree

4 files changed

+159
-122
lines changed

4 files changed

+159
-122
lines changed

‎dists/hadoop-palantir-bom/pom.xml

Lines changed: 154 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,154 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<!--
3+
~ Licensed to the Apache Software Foundation (ASF) under one or more
4+
~ contributor license agreements. See the NOTICE file distributed with
5+
~ this work for additional information regarding copyright ownership.
6+
~ The ASF licenses this file to You under the Apache License, Version 2.0
7+
~ (the "License"); you may not use this file except in compliance with
8+
~ the License. You may obtain a copy of the License at
9+
~
10+
~ http://www.apache.org/licenses/LICENSE-2.0
11+
~
12+
~ Unless required by applicable law or agreed to in writing, software
13+
~ distributed under the License is distributed on an "AS IS" BASIS,
14+
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
~ See the License for the specific language governing permissions and
16+
~ limitations under the License.
17+
-->
18+
19+
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
20+
<modelVersion>4.0.0</modelVersion>
21+
<parent>
22+
<groupId>org.apache.spark</groupId>
23+
<artifactId>spark-parent_2.11</artifactId>
24+
<version>3.0.0-SNAPSHOT</version>
25+
<relativePath>../../pom.xml</relativePath>
26+
</parent>
27+
28+
<artifactId>spark-dist_2.11-hadoop-palantir-bom</artifactId>
29+
<name>Spark Project Dist Palantir Hadoop (BOM)</name>
30+
<url>http://spark.apache.org/</url>
31+
<packaging>pom</packaging>
32+
33+
<properties>
34+
<sbt.project.name>spark-dist-hadoop-palantir-bom</sbt.project.name>
35+
<build.testJarPhase>none</build.testJarPhase>
36+
<build.copyDependenciesPhase>none</build.copyDependenciesPhase>
37+
<hadoop.version>2.9.1-palantir.11</hadoop.version>
38+
<curator.version>2.7.1</curator.version>
39+
</properties>
40+
41+
<dependencyManagement>
42+
<dependencies>
43+
<dependency>
44+
<groupId>org.apache.spark</groupId>
45+
<artifactId>spark-core_${scala.binary.version}</artifactId>
46+
<version>${project.version}</version>
47+
</dependency>
48+
<dependency>
49+
<groupId>org.apache.spark</groupId>
50+
<artifactId>spark-mllib_${scala.binary.version}</artifactId>
51+
<version>${project.version}</version>
52+
</dependency>
53+
<dependency>
54+
<groupId>org.apache.spark</groupId>
55+
<artifactId>spark-streaming_${scala.binary.version}</artifactId>
56+
<version>${project.version}</version>
57+
</dependency>
58+
<dependency>
59+
<groupId>org.apache.spark</groupId>
60+
<artifactId>spark-graphx_${scala.binary.version}</artifactId>
61+
<version>${project.version}</version>
62+
</dependency>
63+
<dependency>
64+
<groupId>org.apache.spark</groupId>
65+
<artifactId>spark-sql_${scala.binary.version}</artifactId>
66+
<version>${project.version}</version>
67+
</dependency>
68+
<dependency>
69+
<groupId>org.apache.spark</groupId>
70+
<artifactId>spark-hadoop-cloud_${scala.binary.version}</artifactId>
71+
<version>${project.version}</version>
72+
</dependency>
73+
<dependency>
74+
<groupId>org.apache.spark</groupId>
75+
<artifactId>spark-repl_${scala.binary.version}</artifactId>
76+
<version>${project.version}</version>
77+
</dependency>
78+
<dependency>
79+
<groupId>org.apache.spark</groupId>
80+
<artifactId>spark-yarn_${scala.binary.version}</artifactId>
81+
<version>${project.version}</version>
82+
</dependency>
83+
<dependency>
84+
<groupId>org.apache.spark</groupId>
85+
<artifactId>spark-mesos_${scala.binary.version}</artifactId>
86+
<version>${project.version}</version>
87+
</dependency>
88+
<dependency>
89+
<groupId>org.apache.spark</groupId>
90+
<artifactId>spark-kubernetes_${scala.binary.version}</artifactId>
91+
<version>${project.version}</version>
92+
</dependency>
93+
<dependency>
94+
<groupId>org.apache.spark</groupId>
95+
<artifactId>spark-hive_${scala.binary.version}</artifactId>
96+
<version>${project.version}</version>
97+
</dependency>
98+
<dependency>
99+
<groupId>org.apache.spark</groupId>
100+
<artifactId>spark-hive-thriftserver_${scala.binary.version}</artifactId>
101+
<version>${project.version}</version>
102+
</dependency>
103+
<dependency>
104+
<groupId>org.apache.spark</groupId>
105+
<artifactId>spark-ganglia-lgpl_${scala.binary.version}</artifactId>
106+
<version>${project.version}</version>
107+
</dependency>
108+
<dependency>
109+
<groupId>org.apache.spark</groupId>
110+
<artifactId>spark-avro_2.11</artifactId>
111+
<version>${project.version}</version>
112+
</dependency>
113+
<dependency>
114+
<groupId>org.apache.spark</groupId>
115+
<artifactId>spark-dist_${scala.binary.version}-hadoop-palantir</artifactId>
116+
<version>${project.version}</version>
117+
</dependency>
118+
<!--
119+
Propagate these from spark-parent hadoop-palantir profile
120+
-->
121+
<dependency>
122+
<groupId>org.apache.hadoop</groupId>
123+
<artifactId>hadoop-mapreduce-client-core</artifactId>
124+
<version>${hadoop.version}</version>
125+
</dependency>
126+
<dependency>
127+
<groupId>org.apache.hadoop</groupId>
128+
<artifactId>hadoop-azure-datalake</artifactId>
129+
<version>${hadoop.version}</version>
130+
<scope>${hadoop.deps.scope}</scope>
131+
</dependency>
132+
<dependency>
133+
<groupId>org.apache.hadoop</groupId>
134+
<artifactId>hadoop-azure</artifactId>
135+
<version>${hadoop.version}</version>
136+
<scope>${hadoop.deps.scope}</scope>
137+
<exclusions>
138+
<exclusion>
139+
<groupId>org.apache.hadoop</groupId>
140+
<artifactId>hadoop-common</artifactId>
141+
</exclusion>
142+
<exclusion>
143+
<groupId>org.codehaus.jackson</groupId>
144+
<artifactId>jackson-mapper-asl</artifactId>
145+
</exclusion>
146+
<exclusion>
147+
<groupId>com.fasterxml.jackson.core</groupId>
148+
<artifactId>jackson-core</artifactId>
149+
</exclusion>
150+
</exclusions>
151+
</dependency>
152+
</dependencies>
153+
</dependencyManagement>
154+
</project>

‎dists/hadoop-palantir/pom.xml

Lines changed: 2 additions & 120 deletions
Original file line numberDiff line numberDiff line change
@@ -20,9 +20,9 @@
2020
<modelVersion>4.0.0</modelVersion>
2121
<parent>
2222
<groupId>org.apache.spark</groupId>
23-
<artifactId>spark-parent_2.11</artifactId>
23+
<artifactId>spark-dist_2.11-hadoop-palantir-bom</artifactId>
2424
<version>3.0.0-SNAPSHOT</version>
25-
<relativePath>../../pom.xml</relativePath>
25+
<relativePath>../hadoop-palantir-bom/pom.xml</relativePath>
2626
</parent>
2727

2828
<artifactId>spark-dist_2.11-hadoop-palantir</artifactId>
@@ -32,126 +32,8 @@
3232

3333
<properties>
3434
<sbt.project.name>spark-dist-hadoop-palantir</sbt.project.name>
35-
<build.testJarPhase>none</build.testJarPhase>
36-
<build.copyDependenciesPhase>none</build.copyDependenciesPhase>
37-
<hadoop.version>2.9.1-palantir.11</hadoop.version>
38-
<curator.version>2.7.1</curator.version>
3935
</properties>
4036

41-
<dependencyManagement>
42-
<dependencies>
43-
<dependency>
44-
<groupId>org.apache.spark</groupId>
45-
<artifactId>spark-core_${scala.binary.version}</artifactId>
46-
<version>${project.version}</version>
47-
</dependency>
48-
<dependency>
49-
<groupId>org.apache.spark</groupId>
50-
<artifactId>spark-mllib_${scala.binary.version}</artifactId>
51-
<version>${project.version}</version>
52-
</dependency>
53-
<dependency>
54-
<groupId>org.apache.spark</groupId>
55-
<artifactId>spark-streaming_${scala.binary.version}</artifactId>
56-
<version>${project.version}</version>
57-
</dependency>
58-
<dependency>
59-
<groupId>org.apache.spark</groupId>
60-
<artifactId>spark-graphx_${scala.binary.version}</artifactId>
61-
<version>${project.version}</version>
62-
</dependency>
63-
<dependency>
64-
<groupId>org.apache.spark</groupId>
65-
<artifactId>spark-sql_${scala.binary.version}</artifactId>
66-
<version>${project.version}</version>
67-
</dependency>
68-
<dependency>
69-
<groupId>org.apache.spark</groupId>
70-
<artifactId>spark-hadoop-cloud_${scala.binary.version}</artifactId>
71-
<version>${project.version}</version>
72-
</dependency>
73-
<dependency>
74-
<groupId>org.apache.spark</groupId>
75-
<artifactId>spark-repl_${scala.binary.version}</artifactId>
76-
<version>${project.version}</version>
77-
</dependency>
78-
<dependency>
79-
<groupId>org.apache.spark</groupId>
80-
<artifactId>spark-yarn_${scala.binary.version}</artifactId>
81-
<version>${project.version}</version>
82-
</dependency>
83-
<dependency>
84-
<groupId>org.apache.spark</groupId>
85-
<artifactId>spark-mesos_${scala.binary.version}</artifactId>
86-
<version>${project.version}</version>
87-
</dependency>
88-
<dependency>
89-
<groupId>org.apache.spark</groupId>
90-
<artifactId>spark-kubernetes_${scala.binary.version}</artifactId>
91-
<version>${project.version}</version>
92-
</dependency>
93-
<dependency>
94-
<groupId>org.apache.spark</groupId>
95-
<artifactId>spark-hive_${scala.binary.version}</artifactId>
96-
<version>${project.version}</version>
97-
</dependency>
98-
<dependency>
99-
<groupId>org.apache.spark</groupId>
100-
<artifactId>spark-hive-thriftserver_${scala.binary.version}</artifactId>
101-
<version>${project.version}</version>
102-
</dependency>
103-
<dependency>
104-
<groupId>org.apache.spark</groupId>
105-
<artifactId>spark-ganglia-lgpl_${scala.binary.version}</artifactId>
106-
<version>${project.version}</version>
107-
</dependency>
108-
<dependency>
109-
<groupId>org.apache.spark</groupId>
110-
<artifactId>spark-avro_2.11</artifactId>
111-
<version>${project.version}</version>
112-
</dependency>
113-
<dependency>
114-
<groupId>org.apache.spark</groupId>
115-
<artifactId>spark-dist_${scala.binary.version}-hadoop-palantir</artifactId>
116-
<version>${project.version}</version>
117-
</dependency>
118-
<!--
119-
Propagate these from spark-parent hadoop-palantir profile
120-
-->
121-
<dependency>
122-
<groupId>org.apache.hadoop</groupId>
123-
<artifactId>hadoop-mapreduce-client-core</artifactId>
124-
<version>${hadoop.version}</version>
125-
</dependency>
126-
<dependency>
127-
<groupId>org.apache.hadoop</groupId>
128-
<artifactId>hadoop-azure-datalake</artifactId>
129-
<version>${hadoop.version}</version>
130-
<scope>${hadoop.deps.scope}</scope>
131-
</dependency>
132-
<dependency>
133-
<groupId>org.apache.hadoop</groupId>
134-
<artifactId>hadoop-azure</artifactId>
135-
<version>${hadoop.version}</version>
136-
<scope>${hadoop.deps.scope}</scope>
137-
<exclusions>
138-
<exclusion>
139-
<groupId>org.apache.hadoop</groupId>
140-
<artifactId>hadoop-common</artifactId>
141-
</exclusion>
142-
<exclusion>
143-
<groupId>org.codehaus.jackson</groupId>
144-
<artifactId>jackson-mapper-asl</artifactId>
145-
</exclusion>
146-
<exclusion>
147-
<groupId>com.fasterxml.jackson.core</groupId>
148-
<artifactId>jackson-core</artifactId>
149-
</exclusion>
150-
</exclusions>
151-
</dependency>
152-
</dependencies>
153-
</dependencyManagement>
154-
15537
<dependencies>
15638
<dependency>
15739
<groupId>org.apache.spark</groupId>

‎pom.xml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -105,6 +105,7 @@
105105
<module>external/kafka-0-10-assembly</module>
106106
<module>external/kafka-0-10-sql</module>
107107
<module>external/avro</module>
108+
<module>dists/hadoop-palantir-bom</module>
108109
<module>dists/hadoop-palantir</module>
109110
<!-- See additional modules enabled by profiles below -->
110111
</modules>

‎project/SparkBuild.scala

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -56,11 +56,11 @@ object BuildCommons {
5656

5757
val optionallyEnabledProjects@Seq(kubernetes, mesos, yarn,
5858
sparkGangliaLgpl, streamingKinesisAsl,
59-
dockerIntegrationTests, hadoopCloud, kubernetesIntegrationTests, sparkDist) =
59+
dockerIntegrationTests, hadoopCloud, kubernetesIntegrationTests, sparkDist, sparkDistBom) =
6060
Seq("kubernetes", "mesos", "yarn",
6161
"ganglia-lgpl", "streaming-kinesis-asl",
6262
"docker-integration-tests", "hadoop-cloud", "kubernetes-integration-tests",
63-
"spark-dist-hadoop-palantir").map(ProjectRef(buildLocation, _))
63+
"spark-dist-hadoop-palantir", "spark-dist-hadoop-palantir-bom").map(ProjectRef(buildLocation, _))
6464

6565
val assemblyProjects@Seq(networkYarn, streamingKafka010Assembly, streamingKinesisAslAssembly) =
6666
Seq("network-yarn", "streaming-kafka-0-10-assembly", "streaming-kinesis-asl-assembly")

0 commit comments

Comments
 (0)