Skip to content

Commit 8e95951

Browse files
leewyangleewyang
authored
Merge pull request #339 from yahoo/leewyang_travis_maven
Automate scala build w/ Travis
2 parents 133f0af + f6253da commit 8e95951

File tree

5 files changed

+99
-51
lines changed

5 files changed

+99
-51
lines changed

‎.travis.settings.xml‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
<settings xmlns="http://maven.apache.org/SETTINGS/1.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.0.0 http://maven.apache.org/xsd/settings-1.0.0.xsd">
2+
<servers>
3+
<server>
4+
<id>bintray-tensorflowonspark-repo</id>
5+
<username>${env.BINTRAY_USER}</username>
6+
<password>${env.BINTRAY_API_KEY}</password>
7+
</server>
8+
</servers>
9+
</settings>

‎.travis.yml‎

Lines changed: 62 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -1,39 +1,65 @@
1-
language: python
2-
python:
3-
- 2.7
4-
- 3.6
5-
cache: pip
6-
before_install:
7-
- curl -LO http://www-us.apache.org/dist/spark/spark-2.3.1/spark-2.3.1-bin-hadoop2.7.tgz
8-
- export SPARK_HOME=./spark
9-
- mkdir $SPARK_HOME
10-
- tar -xf spark-2.3.1-bin-hadoop2.7.tgz -C $SPARK_HOME --strip-components=1
11-
- export PATH=$SPARK_HOME/bin:$PATH
12-
- export SPARK_LOCAL_IP=127.0.0.1
13-
- export SPARK_CLASSPATH=./lib/tensorflow-hadoop-1.0-SNAPSHOT.jar
14-
- export PYTHONPATH=$(pwd)
15-
install:
16-
- pip install -r requirements.txt
17-
script:
18-
- sphinx-build -b html docs/source docs/build/html
19-
- test/run_tests.sh
1+
matrix:
2+
include:
3+
- language: python
4+
python: 2.7
5+
before_install:
6+
- curl -LO http://www-us.apache.org/dist/spark/spark-2.3.1/spark-2.3.1-bin-hadoop2.7.tgz
7+
- export SPARK_HOME=./spark
8+
- mkdir $SPARK_HOME
9+
- tar -xf spark-2.3.1-bin-hadoop2.7.tgz -C $SPARK_HOME --strip-components=1
10+
- export PATH=$SPARK_HOME/bin:$PATH
11+
- export SPARK_LOCAL_IP=127.0.0.1
12+
- export SPARK_CLASSPATH=./lib/tensorflow-hadoop-1.0-SNAPSHOT.jar
13+
- export PYTHONPATH=$(pwd)
14+
install:
15+
- pip install -r requirements.txt
16+
script:
17+
- test/run_tests.sh
18+
- language: python
19+
python: 3.6
20+
before_install:
21+
- curl -LO http://www-us.apache.org/dist/spark/spark-2.3.1/spark-2.3.1-bin-hadoop2.7.tgz
22+
- export SPARK_HOME=./spark
23+
- mkdir $SPARK_HOME
24+
- tar -xf spark-2.3.1-bin-hadoop2.7.tgz -C $SPARK_HOME --strip-components=1
25+
- export PATH=$SPARK_HOME/bin:$PATH
26+
- export SPARK_LOCAL_IP=127.0.0.1
27+
- export SPARK_CLASSPATH=./lib/tensorflow-hadoop-1.0-SNAPSHOT.jar
28+
- export PYTHONPATH=$(pwd)
29+
install:
30+
- pip install -r requirements.txt
31+
script:
32+
- sphinx-build -b html docs/source docs/build/html
33+
- test/run_tests.sh
34+
- language: java
35+
jdk: oraclejdk8
2036
notifications:
2137
email: false
2238
deploy:
23-
- provider: pages
24-
skip_cleanup: true
25-
github_token: $GITHUB_TOKEN
26-
local_dir: docs/build/html
27-
on:
28-
branch: master
29-
python: 3.6
30-
tags: true
31-
- provider: pypi
32-
user: leewyang
33-
password:
34-
secure: T2Q8VM6SgcMtJDO2kJbaELE/5ICR5mx8pkM6TyNAJZ2Mr3fLIy6iDfPKunBAYVljl+SDEWmuoPTWqJdqMyo47LBKPKtBHbGzATqGSRTLvxLOYNSXUX+uCpPtr7CMp1eP3xpZ3YbAJZvoEFlWnBQKeBtX/PjNCpmKdp7ir+46CvR/pR1tcM5cFnSgU+uCPAMUt8KTZIxeRo+oJtaE0DM2RxLJ9nGnaRNz9fdXxwhViNj/bMnDRUI0G6k+Iy4sO2669si8nhTDr+Oq66ONUcJtAQymNUM/hzBTCkrJvuIq1TqTlKkA39UrtD5/wCkCqPUbCLVuIfNwkYfW2C8AlXcbphBKN4PhwaoL5XECr3/AOsgNpnPWhCF1Z1uLi58FhIlSyp+5c/x2wVJLZi2IE+c996An7UO3t16ZFpFEgzS6m9PVbi6Qil6Tl4AhV5QLKb0Qn0hLe2l0WixzK9KLMHfkqX8h5ZGC7i0TvCNcU2uIFjY8we91GORZKZhwUVDKbPqiUZIKn64Qq8EwJIsk/S344OrUTzm7z0lFCqtPphg1duU42QOFmaYWi6hgsbtDxN6+CubLw23G3PtKjOpNt8hHnrjZsz9H1MKbSAoYQ4fo+Iwb3owTjXnSTBr94StW7qysggWH6xQimFDh/SKOE9MfroMGt5YTXfduTbqyeameYqE=
35-
distributions: sdist bdist_wheel
36-
on:
37-
branch: master
38-
python: 3.6
39-
tags: true
39+
- provider: pages
40+
skip_cleanup: true
41+
github_token: "$GITHUB_TOKEN"
42+
local_dir: docs/build/html
43+
on:
44+
branch: master
45+
python: 3.6
46+
tags: true
47+
condition: "$TRAVIS_TAG =~ ^v.*$"
48+
- provider: pypi
49+
user: leewyang
50+
password:
51+
secure: T2Q8VM6SgcMtJDO2kJbaELE/5ICR5mx8pkM6TyNAJZ2Mr3fLIy6iDfPKunBAYVljl+SDEWmuoPTWqJdqMyo47LBKPKtBHbGzATqGSRTLvxLOYNSXUX+uCpPtr7CMp1eP3xpZ3YbAJZvoEFlWnBQKeBtX/PjNCpmKdp7ir+46CvR/pR1tcM5cFnSgU+uCPAMUt8KTZIxeRo+oJtaE0DM2RxLJ9nGnaRNz9fdXxwhViNj/bMnDRUI0G6k+Iy4sO2669si8nhTDr+Oq66ONUcJtAQymNUM/hzBTCkrJvuIq1TqTlKkA39UrtD5/wCkCqPUbCLVuIfNwkYfW2C8AlXcbphBKN4PhwaoL5XECr3/AOsgNpnPWhCF1Z1uLi58FhIlSyp+5c/x2wVJLZi2IE+c996An7UO3t16ZFpFEgzS6m9PVbi6Qil6Tl4AhV5QLKb0Qn0hLe2l0WixzK9KLMHfkqX8h5ZGC7i0TvCNcU2uIFjY8we91GORZKZhwUVDKbPqiUZIKn64Qq8EwJIsk/S344OrUTzm7z0lFCqtPphg1duU42QOFmaYWi6hgsbtDxN6+CubLw23G3PtKjOpNt8hHnrjZsz9H1MKbSAoYQ4fo+Iwb3owTjXnSTBr94StW7qysggWH6xQimFDh/SKOE9MfroMGt5YTXfduTbqyeameYqE=
52+
distributions: sdist bdist_wheel
53+
on:
54+
branch: master
55+
python: 3.6
56+
tags: true
57+
condition: "$TRAVIS_TAG =~ ^v.*$"
58+
- provider: script
59+
script: mvn deploy -DskipTests --settings .travis.settings.xml
60+
skip_cleanup: true
61+
on:
62+
branch: master
63+
jdk: oraclejdk8
64+
tags: true
65+
condition: "$TRAVIS_TAG =~ ^scala_.*$"

‎examples/mnist/spark/mnist_dist.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -138,6 +138,7 @@ def feed_dict(batch):
138138
with tf.train.MonitoredTrainingSession(master=server.target,
139139
is_chief=(task_index == 0),
140140
checkpoint_dir=logdir,
141+
save_checkpoint_secs=10,
141142
hooks=[tf.train.StopAtStepHook(last_step=args.steps)],
142143
chief_only_hooks=[ExportHook(ctx.absolute_path(args.export_dir), x, prediction)]) as mon_sess:
143144
step = 0

‎examples/mnist/spark/mnist_spark.py‎

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@
2626
parser.add_argument("--batch_size", help="number of records per batch", type=int, default=100)
2727
parser.add_argument("--epochs", help="number of epochs", type=int, default=1)
2828
parser.add_argument("--export_dir", help="HDFS path to export saved_model", default="mnist_export")
29-
parser.add_argument("--format", help="example format: (csv|pickle|tfr)", choices=["csv", "pickle", "tfr"], default="csv")
29+
parser.add_argument("--format", help="example format: (csv|tfr)", choices=["csv", "tfr"], default="csv")
3030
parser.add_argument("--images", help="HDFS path to MNIST images in parallelized format")
3131
parser.add_argument("--labels", help="HDFS path to MNIST labels in parallelized format")
3232
parser.add_argument("--model", help="HDFS path to save/load model during train/inference", default="mnist_model")
@@ -56,22 +56,22 @@ def toNumpy(bytestr):
5656
return (image, label)
5757

5858
dataRDD = images.map(lambda x: toNumpy(bytes(x[0])))
59-
else:
60-
if args.format == "csv":
61-
images = sc.textFile(args.images).map(lambda ln: [int(x) for x in ln.split(',')])
62-
labels = sc.textFile(args.labels).map(lambda ln: [float(x) for x in ln.split(',')])
63-
else: # args.format == "pickle":
64-
images = sc.pickleFile(args.images)
65-
labels = sc.pickleFile(args.labels)
59+
else: # "csv"
6660
print("zipping images and labels")
61+
# If partitions of images/labels don't match, you can use the following code:
62+
# images = sc.textFile(args.images).map(lambda ln: [int(x) for x in ln.split(',')]).zipWithIndex().map(lambda x: (x[1], x[0]))
63+
# labels = sc.textFile(args.labels).map(lambda ln: [float(x) for x in ln.split(',')]).zipWithIndex().map(lambda x: (x[1], x[0]))
64+
# dataRDD = images.join(labels).map(lambda x: (x[1][0], x[1][1]))
65+
images = sc.textFile(args.images).map(lambda ln: [int(x) for x in ln.split(',')])
66+
labels = sc.textFile(args.labels).map(lambda ln: [float(x) for x in ln.split(',')])
6767
dataRDD = images.zip(labels)
6868

6969
cluster = TFCluster.run(sc, mnist_dist.map_fun, args, args.cluster_size, num_ps, args.tensorboard, TFCluster.InputMode.SPARK, log_dir=args.model)
7070
if args.mode == "train":
7171
cluster.train(dataRDD, args.epochs)
72-
else:
73-
labelRDD = cluster.inference(dataRDD)
74-
labelRDD.saveAsTextFile(args.output)
72+
else: # inference
73+
predRDD = cluster.inference(dataRDD)
74+
predRDD.saveAsTextFile(args.output)
7575

7676
cluster.shutdown(grace_secs=30)
7777

‎pom.xml‎

Lines changed: 16 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -5,11 +5,18 @@
55
<modelVersion>4.0.0</modelVersion>
66
<groupId>com.yahoo.ml</groupId>
77
<artifactId>tensorflowonspark</artifactId>
8-
<version>1.0-SNAPSHOT</version>
8+
<version>1.0</version>
99
<packaging>jar</packaging>
1010
<name>tensorflowonspark</name>
1111
<description>Spark Scala inferencing for TensorFlowOnSpark</description>
1212

13+
<distributionManagement>
14+
<repository>
15+
<id>bintray-tensorflowonspark-repo</id>
16+
<url>https://api.bintray.com/maven/yahoo/maven/tensorflowonspark</url>
17+
</repository>
18+
</distributionManagement>
19+
1320
<properties>
1421
<maven.compiler.source>1.8</maven.compiler.source>
1522
<maven.compiler.target>1.8</maven.compiler.target>
@@ -22,11 +29,11 @@
2229
<scala.version>2.11.8</scala.version>
2330
<scala-maven-plugin.version>3.2.1</scala-maven-plugin.version>
2431
<scala-parser-combinators.version>1.1.0</scala-parser-combinators.version>
25-
<scalatest.version>3.0.3</scalatest.version>
32+
<scalatest.version>3.0.5</scalatest.version>
2633
<scalatest-maven-plugin.version>1.0</scalatest-maven-plugin.version>
2734
<scopt.version>3.7.0</scopt.version>
28-
<tensorflow.version>1.8.0</tensorflow.version>
29-
<tensorflow-hadoop.version>1.0-SNAPSHOT</tensorflow-hadoop.version>
35+
<tensorflow.version>1.9.0</tensorflow.version>
36+
<tensorflow-hadoop.version>1.9.0</tensorflow-hadoop.version>
3037
</properties>
3138
<dependencies>
3239
<dependency>
@@ -67,6 +74,11 @@
6774
<artifactId>hadoop</artifactId>
6875
<version>${tensorflow-hadoop.version}</version>
6976
</dependency>
77+
<dependency>
78+
<groupId>com.google.protobuf</groupId>
79+
<artifactId>protobuf-java</artifactId>
80+
<version>3.5.1</version>
81+
</dependency>
7082
<dependency>
7183
<groupId>org.scalatest</groupId>
7284
<artifactId>scalatest_2.11</artifactId>

0 commit comments

Comments
 (0)