Skip to content

Commit 1372eb2

Browse files
dansanduleacdansanduleac
authored
Prepackage deploy step (apache-spark-on-k8s#331)
1 parent fc092cb commit 1372eb2

File tree

5 files changed

+91
-34
lines changed

5 files changed

+91
-34
lines changed

‎.circleci/config.yml

Lines changed: 40 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,13 @@ all-branches-and-tags: &all-branches-and-tags
2121
tags:
2222
only: /.*/
2323

24+
deployable-branches-and-tags: &deployable-branches-and-tags
25+
filters:
26+
tags:
27+
only: /[0-9]+(?:\.[0-9]+){2,}-palantir\.[0-9]+(?:\.[0-9]+)*/
28+
branches:
29+
only: master
30+
2431

2532
jobs:
2633
build-maven:
@@ -323,28 +330,50 @@ jobs:
323330
- store_artifacts:
324331
path: /tmp/yarn-tests
325332

326-
deploy:
333+
build-maven-versioned:
327334
<<: *defaults
328335
# Some part of the maven setup fails if there's no R, so we need to use the R image here
329336
docker:
330337
- image: palantirtechnologies/circle-spark-r
331338
steps:
332339
- checkout
333-
- attach_workspace:
334-
at: .
335340
- restore_cache:
336341
key: maven-dependency-cache-{{ checksum "pom.xml" }}
337342
- restore_cache:
338343
key: build-binaries-{{ checksum "build/mvn" }}-{{ checksum "build/sbt" }}
344+
- run:
345+
command: dev/set_version_and_package.sh
346+
# This is potentially costly but we can't use the workspace as it would conflict with
347+
# compilation results from build-sbt
348+
- save_cache:
349+
key: v1-maven-build-with-version-{{ .Branch }}-{{ .BuildNum }}
350+
paths:
351+
- .
352+
353+
deploy:
354+
<<: *defaults
355+
# Some part of the maven setup fails if there's no R, so we need to use the R image here
356+
docker:
357+
- image: palantirtechnologies/circle-spark-r
358+
steps:
359+
# This cache contains the whole project after version was set and mvn package was called
360+
# Restoring first (and instead of checkout) as mvn versions:set mutates real source code...
361+
- restore_cache:
362+
key: v1-maven-build-with-version-{{ .Branch }}-{{ .BuildNum }}
363+
- restore_cache:
364+
key: maven-dependency-cache-{{ checksum "pom.xml" }}
365+
- restore_cache:
366+
key: build-binaries-{{ checksum "build/mvn" }}-{{ checksum "build/sbt" }}
367+
339368
- run: echo "user=$BINTRAY_USERNAME" > .credentials
340369
- run: echo "password=$BINTRAY_PASSWORD" >> .credentials
341370
- run: echo "realm=Bintray API Realm" >> .credentials
342371
- run: echo "host=api.bintray.com" >> .credentials
343372
- deploy:
344373
command: dev/publish.sh
345374
- store_artifacts:
346-
path: /tmp/make-distribution.log
347-
destination: make-distribution.log
375+
path: /tmp/make-dist.log
376+
destination: make-dist.log
348377
- store_artifacts:
349378
path: /tmp/publish_artifacts.log
350379
destination: publish_artifacts.log
@@ -383,6 +412,10 @@ workflows:
383412
requires:
384413
- build-sbt
385414
<<: *all-branches-and-tags
415+
- build-maven-versioned:
416+
requires:
417+
- build-maven
418+
<<: *deployable-branches-and-tags
386419
- deploy:
387420
requires:
388421
- build-maven
@@ -393,8 +426,5 @@ workflows:
393426
- run-scala-tests
394427
- run-python-tests
395428
- run-r-tests
396-
filters:
397-
tags:
398-
only: /[0-9]+(?:\.[0-9]+){2,}-palantir\.[0-9]+(?:\.[0-9]+)*/
399-
branches:
400-
only: master
429+
- build-maven-versioned
430+
<<: *deployable-branches-and-tags

‎dev/.rat-excludes

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -107,6 +107,8 @@ circle.yml
107107
.credentials
108108
publish.sh
109109
publish-local.sh
110+
publish_functions.sh
111+
set_version_and_package.sh
110112
structured-streaming/*
111113
kafka-source-initial-offset-version-2.1.0.bin
112114
kafka-source-initial-offset-future-version.bin

‎dev/publish.sh

Lines changed: 3 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -1,31 +1,10 @@
11
#!/usr/bin/env bash
22

33
set -euo pipefail
4-
version=$(git describe --tags --first-parent)
54

6-
PALANTIR_FLAGS=(-Phadoop-cloud -Phadoop-palantir -Pkinesis-asl -Pkubernetes -Phive -Pyarn -Psparkr)
5+
FWDIR="$(cd "`dirname "${BASH_SOURCE[0]}"`"; pwd)"
76

8-
publish_artifacts() {
9-
tmp_settings="tmp-settings.xml"
10-
echo "<settings><servers><server>" > $tmp_settings
11-
echo "<id>bintray-palantir-release</id><username>$BINTRAY_USERNAME</username>" >> $tmp_settings
12-
echo "<password>$BINTRAY_PASSWORD</password>" >> $tmp_settings
13-
echo "</server></servers></settings>" >> $tmp_settings
14-
15-
./build/mvn versions:set -DnewVersion=$version
16-
./build/mvn --settings $tmp_settings -DskipTests "${PALANTIR_FLAGS[@]}" deploy
17-
}
18-
19-
make_dist() {
20-
build_flags="$1"
21-
shift 1
22-
hadoop_name="hadoop-palantir"
23-
artifact_name="spark-dist_2.11-${hadoop_name}"
24-
file_name="spark-dist-${version}-${hadoop_name}.tgz"
25-
./dev/make-distribution.sh --name "hadoop-palantir" --tgz "$@" $build_flags | tee -a "/tmp/make-distribution.log"
26-
curl -u $BINTRAY_USERNAME:$BINTRAY_PASSWORD -T $file_name "https://api.bintray.com/content/palantir/releases/spark/${version}/org/apache/spark/${artifact_name}/${version}/${artifact_name}-${version}.tgz"
27-
curl -u $BINTRAY_USERNAME:$BINTRAY_PASSWORD -X POST "https://api.bintray.com/content/palantir/releases/spark/${version}/publish"
28-
}
7+
source "$FWDIR/publish_functions.sh"
298

309
publish_artifacts | tee -a "/tmp/publish_artifacts.log"
31-
make_dist "${PALANTIR_FLAGS[*]}" --clean
10+
make_dist "${PALANTIR_FLAGS[*]}" | tee -a "/tmp/make-dist.log"

‎dev/publish_functions.sh

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
#!/usr/bin/env bash
2+
3+
set -euo pipefail
4+
5+
PALANTIR_FLAGS=(-Phadoop-cloud -Phadoop-palantir -Pkinesis-asl -Pkubernetes -Phive -Pyarn -Psparkr)
6+
7+
get_version() {
8+
git describe --tags --first-parent
9+
}
10+
11+
set_version_and_package() {
12+
version=$(get_version)
13+
./build/mvn versions:set -DnewVersion="$version"
14+
./build/mvn -DskipTests "${PALANTIR_FLAGS[@]}" package
15+
}
16+
17+
publish_artifacts() {
18+
tmp_settings="tmp-settings.xml"
19+
echo "<settings><servers><server>" > $tmp_settings
20+
echo "<id>bintray-palantir-release</id><username>$BINTRAY_USERNAME</username>" >> $tmp_settings
21+
echo "<password>$BINTRAY_PASSWORD</password>" >> $tmp_settings
22+
echo "</server></servers></settings>" >> $tmp_settings
23+
24+
./build/mvn --settings $tmp_settings -DskipTests "${PALANTIR_FLAGS[@]}" deploy
25+
}
26+
27+
make_dist() {
28+
build_flags="$1"
29+
shift 1
30+
version=$(get_version)
31+
hadoop_name="hadoop-palantir"
32+
artifact_name="spark-dist_2.11-${hadoop_name}"
33+
file_name="spark-dist-${version}-${hadoop_name}.tgz"
34+
./dev/make-distribution.sh --name "hadoop-palantir" --tgz "$@" $build_flags
35+
curl -u $BINTRAY_USERNAME:$BINTRAY_PASSWORD -T "$file_name" "https://api.bintray.com/content/palantir/releases/spark/${version}/org/apache/spark/${artifact_name}/${version}/${artifact_name}-${version}.tgz"
36+
curl -u $BINTRAY_USERNAME:$BINTRAY_PASSWORD -X POST "https://api.bintray.com/content/palantir/releases/spark/${version}/publish"
37+
}

‎dev/set_version_and_package.sh

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
#!/usr/bin/env bash
2+
3+
set -euo pipefail
4+
5+
FWDIR="$(cd "`dirname "${BASH_SOURCE[0]}"`"; pwd)"
6+
7+
source "$FWDIR/publish_functions.sh"
8+
9+
set_version_and_package

0 commit comments

Comments
 (0)