Merge branch 'master' into spark-2.0-preview

Author: noorul

.github/PULL_REQUEST_TEMPLATE

@@ -0,0 +1,21 @@
+**Pull Request checklist**
+
+- [ ] The commit(s) message(s) follows the contribution [guidelines](doc/contribution-guidelines.md#commit-message-format) ?
+- [ ] Tests for the changes have been added (for bug fixes / features) ?
+- [ ] Docs have been added / updated (for bug fixes / features) ?
+
+**Current behavior :** (link exiting issues here : https://help.github.com/articles/basic-writing-and-formatting-syntax/#referencing-issues-and-pull-requests)
+
+
+
+**New behavior :**
+
+
+
+**BREAKING CHANGES**
+
+If this PR contains a breaking change, please describe the impact and migration
+path for existing applications.
+If not please remove this section.
+
+**Other information**:

README.md

@@ -116,7 +116,7 @@ Spark Job Server is now included in Datastax Enterprise 4.8!
 | 0.7.0       | 1.6.2         |
 | spark-2.0-preview | 2.1.0    |
 
-For release notes, look in the `notes/` directory.  They should also be up on [notes.implicit.ly](http://notes.implicit.ly/search/spark-jobserver).
+For release notes, look in the `notes/` directory.
 
 If you need non-released jars, please visit [Jitpack](https://jitpack.io) - they provide non-release jar builds for any Git repo.  :)
 
@@ -759,7 +759,7 @@ This can be used to quickly develop python applications that can interact with
 Spark Jobserver programmatically.
 
 ## Contribution and Development
-Contributions via Github Pull Request are welcome.  See the TODO for some ideas.
+Contributions via Github Pull Request are welcome. Please start by taking a look at the [contribution guidelines](doc/contribution-guidelines.md) and check the TODO for some contribution ideas.
 
 - If you need to build with a specific scala version use ++x.xx.x followed by the regular command,
 for instance: `sbt ++2.11.6 job-server/compile`

akka-app/src/test/scala/spark/jobserver/common/akka/ActorMetricsSpec.scala

@@ -1,6 +1,5 @@
 package spark.jobserver.common.akka
 
-import org.scalatest.matchers.ShouldMatchers
 import org.scalatest.{Matchers, FunSpec}
 import akka.testkit.TestActorRef
 
@@ -19,4 +18,4 @@ class ActorMetricsSpec extends FunSpec with Matchers {
       actor.metricReceiveTimer.count should equal (1)
     }
   }
-}
+}

akka-app/src/test/scala/spark/jobserver/common/akka/ActorStackSpec.scala

@@ -1,6 +1,5 @@
 package spark.jobserver.common.akka
 
-import org.scalatest.matchers.ShouldMatchers
 import org.scalatest.{Matchers, FunSpec}
 import akka.testkit.TestActorRef
 

build.sbt

@@ -2,7 +2,7 @@
 import Dependencies._
 import JobServerRelease._
 
-transitiveClassifiers in Global := Seq()
+transitiveClassifiers in Global := Seq(Artifact.SourceClassifier)
 lazy val dirSettings = Seq()
 
 lazy val akkaApp = Project(id = "akka-app", base = file("akka-app"))
@@ -120,7 +120,8 @@ lazy val jobServerTestJarSettings = Seq(
 
 lazy val noPublishSettings = Seq(
   publishTo := Some(Resolver.file("Unused repo", file("target/unusedrepo"))),
-  publishArtifact := false
+  publishArtifact := false,
+  publish := {}
 )
 
 lazy val dockerSettings = Seq(
@@ -150,6 +151,16 @@ lazy val dockerSettings = Seq(
                 apt-get -y install mesos=${MESOS_VERSION} && \
                 apt-get clean
              """)
+      env("MAVEN_VERSION","3.3.9")
+      runRaw(
+        """mkdir -p /usr/share/maven /usr/share/maven/ref \
+          && curl -fsSL http://apache.osuosl.org/maven/maven-3/$MAVEN_VERSION/binaries/apache-maven-$MAVEN_VERSION-bin.tar.gz \
+          | tar -xzC /usr/share/maven --strip-components=1 \
+          && ln -s /usr/share/maven/bin/mvn /usr/bin/mvn
+        """)
+      env("MAVEN_HOME","/usr/share/maven")
+      env("MAVEN_CONFIG", "/.m2")
+      
       copy(artifact, artifactTargetPath)
       copy(baseDirectory(_ / "bin" / "server_start.sh").value, file("app/server_start.sh"))
       copy(baseDirectory(_ / "bin" / "server_stop.sh").value, file("app/server_stop.sh"))

doc/contribution-guidelines.md

@@ -0,0 +1,97 @@
+# Contribution guidelines
+
+- [Pull Requests](#pullrequest)
+- [Commit Message Guidelines](#commit)
+
+## <a name="pullrequest"></a> Pull Requests Guidelines
+Pull requests are the only means by which you can contribute to this project, please follow the following steps when submitting pull requests :
+
+1. Create your pull request. (PR name must match [commit name](#commit))
+2. Fill the pull request template with all of the required elements
+3. CI tests kick in and report back any issues, you should fix these issues before continuing.
+4. The reviewer(s) uses Github review system so you will know if the reviewer requested any changes, approved the Pull Request or simply added comments.
+5. If any changes are requested please fix them and then once you are ready ask for a new review
+
+## <a name="commit"></a> Git Commit Guidelines 
+
+### Commit Message Format
+A commit message is made up of a **header**, a **body** and a **footer**.  The header has a special
+format that includes a **type**, a **scope** and a **subject**:
+
+```
+<type>(<scope>): <subject>
+<BLANK LINE>
+<body>
+<BLANK LINE>
+<footer>
+```
+
+* The **header** and **scope** are mandatory
+* Lines shouldn't exceed 100 characters
+
+#### Type
+Must be one of the following:
+
+* **feat**: A new feature
+* **fix**: A bug fix
+* **refactor**: Refactoring change
+* **docs**: Documentation only changes
+* **style**: Style changes (white-space, formatting, missing semi-colons, etc)
+* **test**: Adding missing tests
+* **misc**: Changes to the build process or auxiliary tools and libraries
+
+#### Scope
+The scope sets the scope of the commit, for example `webapi`, `akka-app`, `doa`, etc...
+
+#### Subject
+Brief commit description
+
+* use the imperative, present tense: "change" not "changed" nor "changes"
+* don't capitalize first letter
+* no dot (.) at the end
+
+#### Body
+Extended commit description and just as in the **subject**, use the imperative, present tense: "change" not "changed" nor "changes".
+The body should include the motivation for the change and contrast this with previous behaviour if any.
+
+#### Footer
+
+* any information about **Breaking Changes**
+
+**Breaking Changes** should start with the word `BREAKING CHANGES:` followed by a new line and a list of breaking changes.
+
+#### Examples
+
+```
+feat(akka-app): change message serialization format
+
+* Add new serialization format
+* Add serialization tests
+* Document the changes
+
+BEARKING CHANGES :
+* Not compatible with 0.7.0
+```
+
+---
+
+```
+doc(webapi): add new routes documentation
+
+* Document all new routes
+* Add exmaple snippets
+* Remove obselete links
+
+```
+
+---
+
+```
+fix(webapi): fix webapi timeout
+```
+
+---
+
+```
+misc(ci): add new CI tests
+```

doc/docker.md

@@ -4,7 +4,7 @@ Spark-jobserver is available as a Docker container!  This might be the easiest w
 
 To get started:
 
-    docker run -d -p 8090:8090 velvia/spark-jobserver:0.6.2.mesos-0.28.1.spark-1.6.1
+    docker run -d -p 8090:8090 sparkjobserver/spark-jobserver:0.7.0.mesos-0.25.0.spark-1.6.2 
 
 This will start job server on port 8090 in a container, with H2 database and Mesos support, and expose that port to the host on which you run the container.
 
@@ -16,13 +16,13 @@ By default, the container has an embedded Spark distro and runs using Spark loca
 
 To change the spark master the container runs against, set SPARK_MASTER when you start the container:
 
-    docker run -d -p 8090:8090 -e SPARK_MASTER=mesos://zk://mesos.master:5050 velvia/spark-jobserver:0.6.2.mesos-0.28.1.spark-1.6.1
+    docker run -d -p 8090:8090 -e SPARK_MASTER=mesos://zk://mesos.master:5050 sparkjobserver/spark-jobserver:0.7.0.mesos-0.25.0.spark-1.6.2
 
 You can easily change the amount of memory job server uses with `JOBSERVER_MEMORY`, or replace the entire config job server uses at startup with `JOBSERVER_CONFIG`.
 
 The standard way to replace the config is to derive a custom Docker image from the job server one by overwriting the default config at `app/docker.conf`.  The Dockerfile would look like this:
 
-    from velvia/spark-jobserver:0.6.2.mesos-0.28.1.spark-1.6.1
+    from sparkjobserver/spark-jobserver:0.7.0.mesos-0.25.0.spark-1.6.2
     add /path/to/my/jobserver.conf /app/docker.conf
 
 Similarly, to change the logging configuration, inherit from this container and overwrite `/app/log4j-server.properties`.
@@ -31,15 +31,15 @@ Similarly, to change the logging configuration, inherit from this container and
 
 Any `spark-submit` arguments can be passed to the tail of the `docker run` command.  A very common use of this is to add custom jars to your Spark job environment.  For example, to add the Datastax Spark-Cassandra Connector to your job:
 
-    docker run -d -p 8090:8090 velvia/spark-jobserver:0.6.2.mesos-0.28.1.spark-1.6.1 --packages com.datastax.spark:spark-cassandra-connector_2.10:1.6.1
+    docker run -d -p 8090:8090 sparkjobserver/spark-jobserver:0.7.0.mesos-0.25.0.spark-1.6.2 --packages com.datastax.spark:spark-cassandra-connector_2.10:1.6.1
 
 ## Database, Persistence, Logs
 
 Docker containers are usually stateless, but it wouldn't be very useful to have the jars and job config reset every time you had to kill and restart a container.
 
 The job server docker image is configured to use H2 database by default and to write the database to a Docker volume at `/database`, which will be persisted between container restarts, and can even be shared amongst multiple job server containers on the same host. Note that in order to persist them to new containers, you need to create a local directory, something like this:
 
-    docker run -d -p 8090:8090 -v /opt/job-server-db:/database velvia/spark-jobserver:0.6.2.mesos-0.28.1.spark-1.6.1
+    docker run -d -p 8090:8090 -v /opt/job-server-db:/database sparkjobserver/spark-jobserver:0.7.0.mesos-0.25.0.spark-1.6.2
 
 See the [Docker Volumes Guide](http://docs-stage.docker.com/userguide/dockervolumes/#volume) for more info.
 
@@ -61,7 +61,7 @@ Example Marathon config, thanks to @peterklipfel:
   "container": {
     "type": "DOCKER",
     "docker": {
-      "image": "velvia/spark-jobserver:0.6.2.mesos-0.28.1.spark-1.6.1",
+      "image": "sparkjobserver/spark-jobserver:0.7.0.mesos-0.25.0.spark-1.6.2",
       "network": "BRIDGE",
       "portMappings": [{
         "containerPort": 8090,

doc/troubleshooting.md

@@ -39,8 +39,9 @@ If your job returns a large job result, it may exceed Akka's maximum network mes
 
 On jobs with large results or many concurrent jobs, the REST API at `/job/abc..` might return status `FINISHED` but does not contains any result. This might happen in two cases:
 
-1. the job finished right now and results are in transfer.
-2. the job finished some time ago and results are remove from results cache already. See `spark.jobserver.job-result-cache-size` to increase the cache.
+1. The job finished right now and results are in transfer.
+2. The job finished some time ago and results are remove from results cache already. See `spark.jobserver.job-result-cache-size` to increase the cache.
+3. The spark jobserver instance restarted which clears the results cache.
 
 
 ## AskTimeout when starting job server or contexts

doc/yarn.md

@@ -85,7 +85,7 @@ Now that we have a docker.conf that should work we can create our dockerfile to
 
 dockerfile:
 
-    FROM velvia/spark-jobserver:0.6.1
+    FROM sparkjobserver/spark-jobserver:0.7.0.mesos-0.25.0.spark-1.6.2
     EXPOSE 32456-32472                                    # Expose driver port range (spark.driver.port + 16)
     ADD /path/to/your/docker.conf /app/docker.conf        # Add the docker.conf to the container
     ADD /path/to/your/cluster-config /app/cluster-config  # Add the yarn-site.xml and hfds-site.xml to the container
@@ -99,8 +99,8 @@ Your dockercontainer is now ready to be build:
 Output should look like this:
 
     Sending build context to Docker daemon  21.5 kB
-    Step 0 : FROM velvia/spark-jobserver:0.5.2
-     ---> a41dbd362a7d
+    Step 0 : FROM sparkjobserver/spark-jobserver:0.7.0.mesos-0.25.0.spark-1.6.2
+     ---> 7a188f2d0dff
     Step 1 : EXPOSE 32456-32472
      ---> f1c91bbaa2d8
     Step 2 : ADD ./docker.conf /app/docker.conf

job-server/config/docker.conf

@@ -16,6 +16,12 @@ spark {
 
     context-per-jvm = true
 
+    # Default client mode will start up a new JobManager int local machine
+    # You can use mesos-cluster mode with REMOTE_JOBSERVER_DIR and MESOS_SPARK_DISPATCHER
+    # environment value set in xxxx.sh file to launch JobManager in remote node
+    # Mesos will take responsibility to offer resource to the JobManager process
+    driver-mode = client
+
     sqldao {
       # Directory where default H2 driver stores its data. Only needed for H2.
       rootdir = /database