migrate build from travis to screwdriver

Author: Lee Yang

.travis.yml

@@ -28,9 +28,9 @@
 author = 'Yahoo Inc'
 
 # The short X.Y version
-version = '2.2.1'
+version = '2.2.2'
 # The full version, including alpha/beta/rc tags
-release = '2.2.1'
+release = '2.2.2'
 
 
 # -- General configuration ---------------------------------------------------

docs/source/conf.py

@@ -5,7 +5,7 @@
     <modelVersion>4.0.0</modelVersion>
     <groupId>com.yahoo.ml</groupId>
     <artifactId>tensorflowonspark</artifactId>
-    <version>2.2.0-SNAPSHOT</version>
+    <version>2.2.2-SNAPSHOT</version>
     <packaging>jar</packaging>
     <name>tensorflowonspark</name>
     <description>Spark Scala inferencing for TensorFlowOnSpark</description>

pom.xml

@@ -2,7 +2,7 @@ h5py>=2.9.0
 numpy>=1.14.0
 packaging
 py4j==0.10.7
-pyspark==2.4.5
+pyspark==2.4.7
 scipy
 setuptools>=41.0.0
 sphinx

requirements.txt

@@ -0,0 +1,53 @@
+# Copyright 2017, Verizon Inc.
+# Licensed under the terms of the apache license.  See the LICENSE file in the project root for terms
+
+version: 4
+shared:
+  environment:
+    PACKAGE_DIRECTORY: tensorflowonspark
+    SPARK_HOME: ${SD_ROOT_DIR}/spark
+    TOX_ARGS: '--verbose'
+    TOX_ENVLIST: py37
+  annotations:
+    screwdriver.cd/cpu: HIGH
+    screwdriver.cd/ram: HIGH
+
+jobs:
+  validate_test:
+    template: python/validate_unittest
+    requires: [~commit, ~pr]
+    steps:
+      - prevalidate_code: |
+          source scripts/install_spark.sh
+
+  validate_lint:
+    template: python/validate_lint
+    requires: [~commit, ~pr]
+
+  validate_codestyle:
+    template: python/validate_codestyle
+    requires: [~commit, ~pr]
+
+  validate_safetydb:
+    template: python/validate_safety
+    requires: [~commit, ~pr]
+
+#  validate_security:
+#    template: python/validate_security
+#    requires: [~commit, ~pr]
+
+  publish_test_pypi:
+    template: python/package_python
+    environment:
+      PUBLISH: True
+      TWINE_REPOSITORY_URL: https://test.pypi.org/legacy/
+    requires: [validate_test, validate_lint, validate_codestyle, validate_safetydb, generate_version]
+    steps:
+      - update_version: |
+          echo 'using version from setup.cfg'
+
+#  publish_pypi:
+#    template: python/package_python
+#    environment:
+#      PUBLISH: True
+#    requires: [verify_test_package]

screwdriver.yaml

@@ -0,0 +1,11 @@
+#!/bin/bash -x
+
+# Install JDK8
+yum install -y java-1.8.0-openjdk
+export JAVA_HOME=/usr/lib/jvm/jre-1.8.0
+
+# Install Spark
+export SPARK_VERSION=2.4.7
+curl -LO http://www-us.apache.org/dist/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop2.7.tgz
+mkdir $SPARK_HOME
+tar -xf spark-${SPARK_VERSION}-bin-hadoop2.7.tgz -C $SPARK_HOME --strip-components=1

scripts/install_spark.sh

@@ -0,0 +1,12 @@
+#!/bin/bash -x
+#export SPARK_HOME=/opt/spark
+#export SPARK_LOCAL_IP=127.0.0.1
+#export PATH=$SPARK_HOME/bin:$PATH
+#
+## Start Spark Standalone Cluster
+#export SPARK_CLASSPATH=./lib/tensorflow-hadoop-1.0-SNAPSHOT.jar
+#export MASTER=spark://$(hostname):7077
+#export SPARK_WORKER_INSTANCES=2; export CORES_PER_WORKER=1
+#export TOTAL_CORES=$((${CORES_PER_WORKER}*${SPARK_WORKER_INSTANCES}))
+
+${SPARK_HOME}/sbin/start-master.sh; ${SPARK_HOME}/sbin/start-slave.sh -c ${CORES_PER_WORKER} -m 1G ${MASTER}

scripts/start_spark.sh

@@ -0,0 +1,3 @@
+#!/bin/bash -x
+
+${SPARK_HOME}/sbin/stop-slave.sh; ${SPARK_HOME}/sbin/stop-master.sh

scripts/stop_spark.sh

@@ -0,0 +1,4 @@
+version: 1
+push:
+  - screwdriver:6384
+  - screwdriver:6384pr