Skip to content

Commit 33c1522

Browse files
author
Robert Kruszewski
committed
Merge branch 'palantir-master' into rk/merge-again
2 parents c46d56d + 5f203cd commit 33c1522

18 files changed

+115
-125
lines changed

‎.baseline/checkstyle/checkstyle.xml

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,6 @@
1919
<property name="lineSeparator" value="lf"/>
2020
</module>
2121
<module name="Header">
22-
<!--<property name="header" value="^/\*$\n^ \* Licensed to the Apache Software Foundation (ASF) under one or more$\n"/>-->
2322
<property name="headerFile" value="${config_loc}/header.txt"/>
2423
<property name="fileExtensions" value=".java,.ts"/>
2524
</module>
@@ -41,13 +40,18 @@
4140
<property name="message" value="Whitespace at end-of-line"/>
4241
</module>
4342
<module name="RegexpMultiline"> <!-- Java Style Guide: Vertical Whitespace -->
44-
<property name="fileExtensions" value="java"/>
43+
<property name="fileExtensions" value="java"/>
4544
<property name="format" value="^\n\n$"/>
4645
<property name="message" value="Two consecutive blank lines are not permitted."/>
4746
</module>
4847
<module name="SuppressionFilter"> <!-- baseline-gradle: README.md -->
4948
<property name="file" value="${config_loc}/checkstyle-suppressions.xml"/>
5049
</module>
50+
<module name="SuppressionFilter"> <!-- baseline-gradle: README.md -->
51+
<!-- custom-suppressions.xml allows users to specify suppresions that will not be overriden by baselineUpdateConfig -->
52+
<property name="file" value="${config_loc}/custom-suppressions.xml"/>
53+
<property name="optional" value="true"/>
54+
</module>
5155
<module name="SuppressWarningsFilter"/> <!-- baseline-gradle: README.md -->
5256
<module name="TreeWalker">
5357
<module name="SuppressionCommentFilter"/> <!-- baseline-gradle: README.md -->
@@ -128,6 +132,7 @@
128132
<message key="import.illegal" value="Must not import repackaged classes."/>
129133
</module>
130134
<module name="IllegalImport"> <!-- Java Coding Guidelines: Import the canonical package -->
135+
<property name="id" value="BanShadedClasses"/>
131136
<property name="illegalPkgs" value=".*\.(repackaged|shaded|thirdparty)"/>
132137
<property name="regexp" value="true" />
133138
<message key="import.illegal" value="Must not import repackaged classes."/>

‎.baseline/idea/intellij-java-palantir-style.xml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -116,12 +116,14 @@
116116
<option name="OPTIMIZE_IMPORTS_ON_THE_FLY" value="true" />
117117
<option name="PARENT_SETTINGS_INSTALLED" value="true" />
118118
<option name="RESOURCE_LIST_WRAP" value="5" />
119+
<option name="RIGHT_MARGIN" value="120" />
119120
<option name="SPACE_BEFORE_ARRAY_INITIALIZER_LBRACE" value="true" />
120121
<option name="TERNARY_OPERATION_SIGNS_ON_NEXT_LINE" value="true" />
121122
<option name="TERNARY_OPERATION_WRAP" value="5" />
122123
<option name="THROWS_KEYWORD_WRAP" value="1" />
123124
<option name="THROWS_LIST_WRAP" value="1" />
124125
<option name="WHILE_BRACE_FORCE" value="3" />
126+
<option name="WRAP_ON_TYPING" value="1" />
125127
<arrangement>
126128
<rules>
127129
<section>

‎.circleci/config.yml

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -106,11 +106,13 @@ step_templates:
106106
if [ -n "$CIRCLE_TAG" ]
107107
then
108108
git fetch origin "+refs/tags/${CIRCLE_TAG}:refs/tags/${CIRCLE_TAG}"
109+
elif [ -n "$CIRCLE_PR_NUMBER" ]
110+
then
111+
git fetch origin "+${CIRCLE_BRANCH}/head:remotes/origin/${CIRCLE_BRANCH}"
109112
else
110113
git fetch origin "+${CIRCLE_BRANCH}:remotes/origin/${CIRCLE_BRANCH}"
111114
fi
112115
113-
114116
if [ -n "$CIRCLE_TAG" ]
115117
then
116118
git reset --hard "$CIRCLE_SHA1"
@@ -147,7 +149,7 @@ jobs:
147149
keys:
148150
- build-binaries-{{ checksum "build/mvn" }}-{{ checksum "build/sbt" }}
149151
- build-binaries-
150-
- run: ./build/mvn -DskipTests -Phadoop-cloud -Phadoop-palantir -Pkinesis-asl -Pkubernetes -Pyarn -Psparkr install
152+
- run: ./build/mvn -DskipTests -Psparkr install
151153
# Get sbt to run trivially, ensures its launcher is downloaded under build/
152154
- run: ./build/sbt -h || true
153155
- save_cache:

‎assembly/pom.xml

Lines changed: 23 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -79,7 +79,29 @@
7979
<artifactId>spark-avro_${scala.binary.version}</artifactId>
8080
<version>${project.version}</version>
8181
</dependency>
82-
82+
<dependency>
83+
<groupId>org.apache.spark</groupId>
84+
<artifactId>spark-hadoop-cloud_${scala.binary.version}</artifactId>
85+
<version>${project.version}</version>
86+
</dependency>
87+
<!--
88+
Redeclare this dependency to force it into the distribution.
89+
-->
90+
<dependency>
91+
<groupId>org.eclipse.jetty</groupId>
92+
<artifactId>jetty-util</artifactId>
93+
<scope>${hadoop.deps.scope}</scope>
94+
</dependency>
95+
<dependency>
96+
<groupId>org.apache.spark</groupId>
97+
<artifactId>spark-yarn_${scala.binary.version}</artifactId>
98+
<version>${project.version}</version>
99+
</dependency>
100+
<dependency>
101+
<groupId>org.apache.spark</groupId>
102+
<artifactId>spark-kubernetes_${scala.binary.version}</artifactId>
103+
<version>${project.version}</version>
104+
</dependency>
83105
<!--
84106
Because we don't shade dependencies anymore, we need to restore Guava to compile scope so
85107
that the libraries Spark depend on have it available. We'll package the version that Spark

‎build.gradle

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,10 +23,10 @@ buildscript {
2323
}
2424

2525
dependencies {
26-
classpath 'com.palantir.baseline:gradle-baseline-java:0.41.0'
26+
classpath 'com.palantir.baseline:gradle-baseline-java:0.49.1'
2727
classpath 'com.palantir.gradle.gitversion:gradle-git-version:0.11.0'
2828
classpath 'com.jfrog.bintray.gradle:gradle-bintray-plugin:1.8.4'
29-
classpath 'com.netflix.nebula:nebula-publishing-plugin:9.2.0'
29+
classpath 'com.netflix.nebula:nebula-publishing-plugin:9.5.0'
3030
}
3131
}
3232

‎dev/deps/spark-deps-hadoop-palantir

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ avro-1.8.2.jar
1919
avro-ipc-1.8.2.jar
2020
avro-mapred-1.8.2-hadoop2.jar
2121
aws-java-sdk-bundle-1.11.201.jar
22+
azure-data-lake-store-sdk-2.2.3.jar
2223
azure-keyvault-core-0.8.0.jar
2324
azure-storage-5.4.0.jar
2425
breeze-macros_2.12-0.13.2.jar
@@ -66,6 +67,7 @@ hadoop-annotations-2.9.1-palantir.11.jar
6667
hadoop-auth-2.9.1-palantir.11.jar
6768
hadoop-aws-2.9.1-palantir.11.jar
6869
hadoop-azure-2.9.1-palantir.11.jar
70+
hadoop-azure-datalake-2.9.1-palantir.11.jar
6971
hadoop-client-2.9.1-palantir.11.jar
7072
hadoop-common-2.9.1-palantir.11.jar
7173
hadoop-hdfs-client-2.9.1-palantir.11.jar
@@ -171,12 +173,12 @@ orc-shims-1.5.4.jar
171173
oro-2.0.8.jar
172174
osgi-resource-locator-1.0.1.jar
173175
paranamer-2.8.jar
174-
parquet-column-1.12.0-palantir.1.jar
175-
parquet-common-1.12.0-palantir.1.jar
176-
parquet-encoding-1.12.0-palantir.1.jar
177-
parquet-format-structures-1.12.0-palantir.1.jar
178-
parquet-hadoop-1.12.0-palantir.1.jar
179-
parquet-jackson-1.12.0-palantir.1.jar
176+
parquet-column-1.12.0-palantir.4.jar
177+
parquet-common-1.12.0-palantir.4.jar
178+
parquet-encoding-1.12.0-palantir.4.jar
179+
parquet-format-structures-1.12.0-palantir.4.jar
180+
parquet-hadoop-1.12.0-palantir.4.jar
181+
parquet-jackson-1.12.0-palantir.4.jar
180182
protobuf-java-2.5.0.jar
181183
py4j-0.10.8.1.jar
182184
pyrolite-4.13.jar

‎dev/make-distribution.sh

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -153,7 +153,7 @@ fi
153153
echo "Spark version is $VERSION"
154154
VERSION_SET=$("$MVN" versions:set -DnewVersion=$VERSION | tail -n 1)
155155
if [ "$MAKE_TGZ" == "true" ]; then
156-
echo "Making spark-dist_2.11-$NAME-$VERSION.tgz"
156+
echo "Making spark-dist-$NAME-$VERSION.tgz"
157157
else
158158
echo "Making distribution for Spark $VERSION in '$DISTDIR'..."
159159
fi

‎dev/mima

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ set -e
2424
FWDIR="$(cd "`dirname "$0"`"/..; pwd)"
2525
cd "$FWDIR"
2626

27-
SPARK_PROFILES="-Phadoop-palantir -Pmesos -Phadoop-cloud -Pkubernetes -Pyarn -Pspark-ganglia-lgpl -Pkinesis-asl -Phive-thriftserver -Phive"
27+
SPARK_PROFILES="-Pmesos -Pkubernetes -Pyarn -Pspark-ganglia-lgpl -Pkinesis-asl -Phive-thriftserver -Phive"
2828
TOOLS_CLASSPATH="$(build/sbt -DcopyDependencies=false "export tools/fullClasspath" | tail -n1)"
2929
OLD_DEPS_CLASSPATH="$(build/sbt -DcopyDependencies=false $SPARK_PROFILES "export oldDeps/fullClasspath" | tail -n1)"
3030

‎dev/publish-local.sh

Lines changed: 6 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -1,27 +1,14 @@
11
#!/usr/bin/env bash
22

33
set -euo pipefail
4-
version=$(git describe --tags --first-parent)
5-
6-
PALANTIR_FLAGS=(-Phadoop-cloud -Phadoop-palantir -Pkinesis-asl -Pkubernetes -Pyarn -Psparkr)
74

5+
FWDIR="$(cd "`dirname "${BASH_SOURCE[0]}"`"; pwd)"
86
MVN_LOCAL=~/.m2/repository
97

10-
publish_artifacts() {
11-
./build/mvn versions:set -DnewVersion=$version
12-
./build/mvn -DskipTests "${PALANTIR_FLAGS[@]}" install
13-
}
8+
source "$FWDIR/publish_functions.sh"
149

15-
make_dist() {
16-
build_flags="$1"
17-
shift 1
18-
hadoop_name="hadoop-palantir"
19-
artifact_name="spark-dist_2.11-${hadoop_name}"
20-
file_name="spark-dist-${version}-${hadoop_name}.tgz"
21-
./dev/make-distribution.sh --name "hadoop-palantir" --tgz "$@" $build_flags
22-
mkdir -p $MVN_LOCAL/org/apache/spark/${artifact_name}/${version} && \
23-
cp $file_name $MVN_LOCAL/org/apache/spark/${artifact_name}/${version}/${artifact_name}-${version}.tgz
24-
}
10+
set_version_and_install
11+
DONT_BUILD=true make_dist
12+
mkdir -p $MVN_LOCAL/org/apache/spark/${artifact_name}/${version}
13+
cp $file_name $MVN_LOCAL/org/apache/spark/${artifact_name}/${version}/${artifact_name}-${version}.tgz
2514

26-
publish_artifacts
27-
make_dist "${PALANTIR_FLAGS[*]}"

‎dev/publish.sh

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,4 +7,4 @@ FWDIR="$(cd "`dirname "${BASH_SOURCE[0]}"`"; pwd)"
77
source "$FWDIR/publish_functions.sh"
88

99
publish_artifacts | tee -a "/tmp/publish_artifacts.log"
10-
DONT_BUILD=true make_dist "${PALANTIR_FLAGS[*]}" | tee -a "/tmp/make-dist.log"
10+
DONT_BUILD=true make_dist_and_deploy | tee -a "/tmp/make-dist.log"

0 commit comments

Comments
 (0)