Skip to content

Commit d46c920

Browse files
dansanduleacdansanduleac
authored
Merge pull request apache-spark-on-k8s#326 from palantir/hy/circle-2.0
Use circle 2.0
2 parents 9a55368 + 3e3e74d commit d46c920

33 files changed

+1433
-258
lines changed

‎.circleci/config.yml

Lines changed: 365 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,365 @@
1+
version: 2
2+
3+
defaults: &defaults
4+
docker:
5+
- image: palantirtechnologies/circle-spark-base
6+
resource_class: xlarge
7+
environment:
8+
TERM: dumb
9+
10+
11+
test-defaults: &test-defaults
12+
<<: *defaults
13+
environment:
14+
CIRCLE_TEST_REPORTS: /tmp/circle-test-reports
15+
16+
17+
all-branches-and-tags: &all-branches-and-tags
18+
filters:
19+
# run on all branches and tags
20+
tags:
21+
only: /.*/
22+
23+
24+
jobs:
25+
build-maven:
26+
<<: *defaults
27+
# Some part of the maven setup fails if there's no R, so we need to use the R image here
28+
docker:
29+
- image: palantirtechnologies/circle-spark-r
30+
steps:
31+
# Saves us from recompiling every time...
32+
- restore_cache:
33+
keys:
34+
- build-maven-{{ .Branch }}-{{ .BuildNum }}
35+
- build-maven-{{ .Branch }}-
36+
- build-maven-master-
37+
- checkout
38+
- restore_cache:
39+
keys:
40+
- maven-dependency-cache-{{ checksum "pom.xml" }}
41+
# Fallback - see https://circleci.com/docs/2.0/configuration-reference/#example-2
42+
- maven-dependency-cache-
43+
# Given the build-maven cache, this is superfluous, but leave it in in case we will want to remove the former
44+
- restore_cache:
45+
keys:
46+
- build-binaries-{{ checksum "build/mvn" }}-{{ checksum "build/sbt" }}
47+
- build-binaries-
48+
- run: |
49+
./build/mvn -T1C -DskipTests -Phadoop-cloud -Phadoop-palantir -Pkinesis-asl -Pkubernetes -Pyarn -Phive -Psparkr install \
50+
| tee -a "/tmp/mvn-install.log"
51+
- store_artifacts:
52+
path: /tmp/mvn-install.log
53+
destination: mvn-install.log
54+
# Get sbt to run trivially, ensures its launcher is downloaded under build/
55+
- run: ./build/sbt -h || true
56+
- save_cache:
57+
key: build-binaries-{{ checksum "build/mvn" }}-{{ checksum "build/sbt" }}
58+
paths:
59+
- "build"
60+
- save_cache:
61+
key: maven-dependency-cache-{{ checksum "pom.xml" }}
62+
paths:
63+
- "~/.m2"
64+
# And finally save the whole project directory
65+
- save_cache:
66+
key: build-maven-{{ .Branch }}-{{ .BuildNum }}
67+
paths: .
68+
69+
run-style-tests:
70+
# depends only on build-maven
71+
<<: *test-defaults
72+
resource_class: small
73+
steps:
74+
- checkout
75+
- restore_cache:
76+
key: build-maven-{{ .Branch }}-{{ .BuildNum }}
77+
# Need maven dependency cache, otherwise checkstyle tests fail as such:
78+
# Failed to execute goal on project spark-assembly_2.11: Could not resolve dependencies for project org.apache.spark:spark-assembly_2.11:pom:2.4.0-SNAPSHOT
79+
- restore_cache:
80+
key: maven-dependency-cache-{{ checksum "pom.xml" }}
81+
- restore_cache:
82+
key: build-binaries-{{ checksum "build/mvn" }}-{{ checksum "build/sbt" }}
83+
- run: dev/run-style-tests.py | tee /tmp/run-style-tests.log
84+
- store_artifacts:
85+
path: /tmp/run-style-tests.log
86+
destination: run-style-tests.log
87+
88+
run-build-tests:
89+
# depends only on build-maven
90+
<<: *test-defaults
91+
resource_class: small
92+
steps:
93+
- checkout
94+
- restore_cache:
95+
key: build-maven-{{ .Branch }}-{{ .BuildNum }}
96+
- restore_cache:
97+
key: maven-dependency-cache-{{ checksum "pom.xml" }}
98+
- restore_cache:
99+
key: build-binaries-{{ checksum "build/mvn" }}-{{ checksum "build/sbt" }}
100+
- run: |
101+
dev/run-build-tests.py | tee /tmp/run-build-tests.log
102+
- store_artifacts:
103+
path: /tmp/run-build-tests.log
104+
destination: run-build-tests.log
105+
106+
build-sbt:
107+
<<: *defaults
108+
environment:
109+
BUILD_SBT_CACHE: "/home/circleci/build-sbt-cache"
110+
steps:
111+
# Saves us from recompiling every time...
112+
- restore_cache:
113+
keys:
114+
- v1-build-sbt-{{ .Branch }}-{{ .BuildNum }}
115+
- v1-build-sbt-{{ .Branch }}-
116+
- v1-build-sbt-master-
117+
- checkout
118+
- run:
119+
name: Hard link cache contents into current build directory
120+
command: |
121+
if [[ -d "$BUILD_SBT_CACHE" ]]; then
122+
rsync --info=stats2,misc1,flist0 -a --link-dest="$BUILD_SBT_CACHE" "$BUILD_SBT_CACHE/" .
123+
fi
124+
- restore_cache:
125+
keys:
126+
- v5-ivy-dependency-cache-{{ checksum "pom.xml" }}
127+
# if cache for exact version of `pom.xml` is not present then load any most recent one
128+
- v5-ivy-dependency-cache-
129+
- restore_cache:
130+
key: maven-dependency-cache-{{ checksum "pom.xml" }}
131+
- restore_cache:
132+
keys:
133+
- v2-home-sbt-{{ checksum "build/sbt" }}-{{ checksum "project/target/streams/$global/update/$global/streams/update_cache_2.10/inputs" }}
134+
# Given the build-sbt cache, this is superfluous, but leave it in in case we will want to remove the former
135+
- restore_cache:
136+
key: build-binaries-{{ checksum "build/mvn" }}-{{ checksum "build/sbt" }}
137+
- run:
138+
name: Download all external dependencies for the test configuration (which extends compile) and ensure we update first
139+
command: dev/sbt test:externalDependencyClasspath oldDeps/test:externalDependencyClasspath
140+
- run: |
141+
dev/build-apache-spark.py | tee /tmp/build-apache-spark.log
142+
- store_artifacts:
143+
path: /tmp/heap.bin
144+
- save_cache:
145+
key: v5-ivy-dependency-cache-{{ checksum "pom.xml" }}
146+
paths:
147+
- "~/.ivy2"
148+
- store_artifacts:
149+
path: /tmp/build-apache-spark.log
150+
destination: build-apache-spark.log
151+
- save_cache:
152+
key: v2-home-sbt-{{ checksum "build/sbt" }}-{{ checksum "project/target/streams/$global/update/$global/streams/update_cache_2.10/inputs" }}
153+
paths: ~/.sbt
154+
# Also hard link all the things so we can save it as a cache and restore it in future builds
155+
- run:
156+
name: "Hard link all the files under ***/target directories to $BUILD_SBT_CACHE, excluding jars"
157+
command: >
158+
rsync --info=stats2,misc1,flist0 -a --link-dest=$PWD --delete-excluded --prune-empty-dirs
159+
--exclude '***/*.jar' --include 'target/***'
160+
--include '**/' --exclude '*' . "$BUILD_SBT_CACHE/"
161+
- save_cache:
162+
key: v1-build-sbt-{{ .Branch }}-{{ .BuildNum }}
163+
paths:
164+
- "~/build-sbt-cache"
165+
# Also save all the target directories to the workspace - need assembly jars for spark submitting
166+
- persist_to_workspace:
167+
root: .
168+
paths:
169+
- "target"
170+
- "*/target"
171+
- "common/*/target"
172+
- "dists/*/target"
173+
- "external/*/target"
174+
- "resource-managers/*/target"
175+
- "resource-managers/*/*/target"
176+
- "sql/*/target"
177+
- "project/project/target"
178+
179+
run-backcompat-tests:
180+
# depends on build-sbt
181+
<<: *defaults
182+
steps:
183+
- checkout
184+
- attach_workspace:
185+
at: .
186+
- restore_cache:
187+
key: v5-ivy-dependency-cache-{{ checksum "pom.xml" }}
188+
# TODO(dsanduleac): do we need the maven cache?
189+
- restore_cache:
190+
key: maven-dependency-cache-{{ checksum "pom.xml" }}
191+
- restore_cache:
192+
key: build-binaries-{{ checksum "build/mvn" }}-{{ checksum "build/sbt" }}
193+
- restore_cache:
194+
keys:
195+
- v2-home-sbt-{{ checksum "build/sbt" }}-{{ checksum "project/target/streams/$global/update/$global/streams/update_cache_2.10/inputs" }}
196+
- run: |
197+
dev/run-backcompat-tests.py | tee /tmp/run-backcompat-tests.log
198+
- store_artifacts:
199+
path: /tmp/run-backcompat-tests.log
200+
destination: run-backcompat-tests.log
201+
202+
203+
run-python-tests:
204+
# depends on build-sbt, so we're restoring the build-sbt cache
205+
<<: *defaults
206+
docker:
207+
- image: palantirtechnologies/circle-spark-python
208+
parallelism: 2
209+
steps:
210+
- checkout
211+
# Python tests need assembly files build by the `build-sbt` job
212+
# e.g. external/kafka-0-8-assembly/target/scala-2.11/spark-streaming-kafka-0-8-assembly-2.4.0-SNAPSHOT.jar
213+
- attach_workspace:
214+
at: .
215+
- run: dev/run-python-tests.py
216+
- store_test_results:
217+
path: target/test-reports
218+
- store_artifacts:
219+
path: python/unit-tests.log
220+
221+
222+
run-r-tests:
223+
# depends on build-sbt, so we're restoring the build-sbt cache
224+
<<: *defaults
225+
docker:
226+
- image: palantirtechnologies/circle-spark-r
227+
steps:
228+
- checkout
229+
- attach_workspace:
230+
at: .
231+
- run:
232+
name: Install SparkR
233+
command: R/install-dev.sh
234+
- run: dev/run-r-tests.py
235+
- store_test_results:
236+
path: target/R
237+
238+
239+
run-scala-tests:
240+
<<: *test-defaults
241+
# project/CirclePlugin.scala does its own test splitting in SBT based on CIRCLE_NODE_INDEX, CIRCLE_NODE_TOTAL
242+
parallelism: 6
243+
# Spark runs a lot of tests in parallel, we need 16 GB of RAM for this
244+
resource_class: xlarge
245+
steps:
246+
- run:
247+
name: Before running tests, ensure we created the CIRCLE_TEST_REPORTS directory
248+
command: mkdir -p $CIRCLE_TEST_REPORTS
249+
- checkout
250+
- attach_workspace:
251+
at: .
252+
- restore_cache:
253+
key: maven-dependency-cache-{{ checksum "pom.xml" }}
254+
- restore_cache:
255+
key: v5-ivy-dependency-cache-{{ checksum "pom.xml" }}
256+
- restore_cache:
257+
key: build-binaries-{{ checksum "build/mvn" }}-{{ checksum "build/sbt" }}
258+
- restore_cache:
259+
keys:
260+
- v2-home-sbt-{{ checksum "build/sbt" }}-{{ checksum "project/target/streams/$global/update/$global/streams/update_cache_2.10/inputs" }}
261+
- run:
262+
name: Run all tests
263+
command: ./dev/run-scala-tests.py \
264+
| tee -a "/tmp/run-scala-tests.log"
265+
no_output_timeout: 15m
266+
- store_artifacts:
267+
path: /tmp/run-scala-tests.log
268+
destination: run-scala-tests.log
269+
- run:
270+
name: Collect unit tests
271+
command: mkdir -p /tmp/unit-tests && find . -name unit-tests.log -exec rsync -R {} /tmp/unit-tests/ \;
272+
when: always
273+
- store_artifacts:
274+
path: /tmp/unit-tests
275+
- store_artifacts:
276+
path: target/tests-by-bucket.json
277+
destination: tests-by-bucket.json
278+
- store_test_results:
279+
# TODO(dsanduleac): can we use $CIRCLE_TEST_RESULTS here?
280+
path: /tmp/circle-test-reports
281+
- run:
282+
name: Collect yarn integration test logs
283+
command: |
284+
shopt -s nullglob
285+
files=(resource-managers/yarn/target/./org.apache.spark.deploy.yarn.*/*-logDir-*)
286+
mkdir -p /tmp/yarn-tests
287+
if [[ ${#files[@]} != 0 ]]; then
288+
rsync -Rrm "${files[@]}" /tmp/yarn-tests/
289+
fi
290+
when: always
291+
- store_artifacts:
292+
path: /tmp/yarn-tests
293+
294+
deploy:
295+
<<: *defaults
296+
steps:
297+
- checkout
298+
- attach_workspace:
299+
at: .
300+
- restore_cache:
301+
key: maven-dependency-cache-{{ checksum "pom.xml" }}
302+
- restore_cache:
303+
key: build-binaries-{{ checksum "build/mvn" }}-{{ checksum "build/sbt" }}
304+
- run: echo "user=$BINTRAY_USERNAME" > .credentials
305+
- run: echo "password=$BINTRAY_PASSWORD" >> .credentials
306+
- run: echo "realm=Bintray API Realm" >> .credentials
307+
- run: echo "host=api.bintray.com" >> .credentials
308+
- deploy: dev/publish.sh
309+
- store_artifacts:
310+
path: /tmp/make-distribution.log
311+
destination: make-distribution.log
312+
- store_artifacts:
313+
path: /tmp/publish_artifacts.log
314+
destination: publish_artifacts.log
315+
- deploy: curl -u $BINTRAY_USERNAME:$BINTRAY_PASSWORD -X POST https://api.bintray.com/content/palantir/releases/spark/$(git describe --tags)/publish
316+
317+
workflows:
318+
version: 2
319+
build-test-deploy:
320+
jobs:
321+
- build-maven:
322+
<<: *all-branches-and-tags
323+
- run-style-tests:
324+
requires:
325+
- build-maven
326+
<<: *all-branches-and-tags
327+
- run-build-tests:
328+
requires:
329+
- build-maven
330+
<<: *all-branches-and-tags
331+
- build-sbt:
332+
requires:
333+
- build-maven
334+
<<: *all-branches-and-tags
335+
- run-backcompat-tests:
336+
requires:
337+
- build-sbt
338+
<<: *all-branches-and-tags
339+
- run-scala-tests:
340+
requires:
341+
- build-sbt
342+
<<: *all-branches-and-tags
343+
- run-python-tests:
344+
requires:
345+
- build-sbt
346+
<<: *all-branches-and-tags
347+
- run-r-tests:
348+
requires:
349+
- build-sbt
350+
<<: *all-branches-and-tags
351+
- deploy:
352+
requires:
353+
- build-maven
354+
- build-sbt
355+
# Tests
356+
- run-build-tests
357+
- run-backcompat-tests
358+
- run-scala-tests
359+
- run-python-tests
360+
- run-r-tests
361+
filters:
362+
tags:
363+
only: /[0-9]+(?:\.[0-9]+){2,}-palantir\.[0-9]+(?:\.[0-9]+)*/
364+
branches:
365+
only: master

0 commit comments

Comments
 (0)