Merge branch 'release/0.7.0'

Author: Joe Nievelt

.travis.yml

@@ -1,6 +1,73 @@
+#test execution script.
+
 language: scala
+sudo: false
+before_install: umask 0022
 scala:
-  - 2.10.4
+  - 2.10.5
 script:
-  - "sbt -Dlog4j.configuration=file://$TRAVIS_BUILD_DIR/project/travis-log4j.properties ++$TRAVIS_SCALA_VERSION test"
-  - "sbt -Dlog4j.configuration=file://$TRAVIS_BUILD_DIR/project/travis-log4j.properties ++$TRAVIS_SCALA_VERSION publish-local"
+- "echo no op"
+
+matrix:
+  include:
+#BASE TESTS
+    - scala: 2.10.5
+      env: BUILD="base" TEST_TARGET="summingbird-batch"
+      script: "scripts/run_tests.sh"
+
+    - scala: 2.10.5
+      env: BUILD="base" TEST_TARGET="summingbird-batch-hadoop"
+      script: "scripts/run_tests.sh"
+
+    - scala: 2.10.5
+      env: BUILD="base" TEST_TARGET="summingbird-builder"
+      script: "scripts/run_tests.sh"
+
+    - scala: 2.10.5
+      env: BUILD="base" TEST_TARGET="summingbird-chill"
+      script: "scripts/run_tests.sh"
+
+    - scala: 2.10.5
+      env: BUILD="base" TEST_TARGET="summingbird-core"
+      script: "scripts/run_tests.sh"
+
+    - scala: 2.10.5
+      env: BUILD="base" TEST_TARGET="summingbird-core-java"
+      script: "scripts/run_tests.sh"
+
+#    - scala: 2.10.5
+#      env: BUILD="base" TEST_TARGET="summingbird-core-test"
+#      script: "scripts/run_tests.sh"
+
+    - scala: 2.10.5
+      env: BUILD="base" TEST_TARGET="summingbird-example"
+      script: "scripts/run_tests.sh"
+
+    - scala: 2.10.5
+      env: BUILD="base" TEST_TARGET="summingbird-online"
+      script: "scripts/run_tests.sh"
+
+    - scala: 2.10.5
+      env: BUILD="base" TEST_TARGET="summingbird-scalding"
+      script: "scripts/run_tests.sh"
+
+    - scala: 2.10.5
+      env: BUILD="base" TEST_TARGET="summingbird-scalding-test"
+      script: "scripts/run_tests.sh"
+
+    - scala: 2.10.5
+      env: BUILD="base" TEST_TARGET="summingbird-spark"
+      script: "scripts/run_tests.sh"
+
+    - scala: 2.10.5
+      env: BUILD="base" TEST_TARGET="summingbird-storm"
+      script: "scripts/run_tests.sh"
+
+    - scala: 2.10.5
+      env: BUILD="base" TEST_TARGET="summingbird-storm-java"
+      script: "scripts/run_tests.sh"
+
+    - scala: 2.10.5
+      env: BUILD="base" TEST_TARGET="summingbird-storm-test"
+      script: "scripts/run_tests.sh"
+

CHANGES.md

@@ -1,5 +1,22 @@
 # Summingbird #
 
+## 0.7.0
+* Registering summingbird counters with tormenta spouts #553
+* Add counters to default summers #571
+* Fixes the types off the summingbird java storm api #601
+* Adding explicit hadoop deps; dfs-datastores to 1.3.6 #592
+* Replace deprecated ClassManifest and erasure #608
+* Spark monad #607
+* replace deprecated erasure by runtimeClass #609
+* Revert "Remove StripNameNodes" #610
+* refactored core-tests into separate sub-project #593
+* Fix checking delta batch in readDeltaTimestamps in batched store when merging #612
+* ensure at least one batch before merging #613
+* MergeableStoreFactory extends java.io.Serializable #616
+* Spout storm metrics foreach #617
+* upgrade scalacheck #615
+* Use latest bijection, algebird, scalding, storehaus, chill, and tormenta
+
 ## 0.6.0
 * Use latest bijection, algebird, scalding, storehaus, chill, and tormenta https://github.com/twitter/summingbird/pull/565
 * Remove scala 2.9 support https://github.com/twitter/summingbird/pull/565

README.md

@@ -108,7 +108,7 @@ Please feel free to use the beautiful [Summingbird logo](https://drive.google.co
 
 ## Maven
 
-Summingbird modules are published on maven central. The current groupid and version for all modules is, respectively, `"com.twitter"` and  `0.4.2`.
+Summingbird modules are published on maven central. The current groupid and version for all modules is, respectively, `"com.twitter"` and  `0.7.0`.
 
 Current published artifacts are
 

project/Build.scala

@@ -19,36 +19,38 @@ object SummingbirdBuild extends Build {
 
   def isScala210x(scalaVersion: String) = scalaBinaryVersion(scalaVersion) == "2.10"
 
-  val scalaTestVersion = "2.2.2"
-  val scalaCheckVersion = "1.11.5"
+  val scalaCheckVersion = "1.12.2"
   val hadoopVersion = "1.2.1"
-  val algebirdVersion = "0.9.0"
-  val bijectionVersion = "0.7.2"
-  val chillVersion = "0.5.2"
+  val algebirdVersion = "0.10.0"
+  val bijectionVersion = "0.8.0"
+  val chillVersion = "0.6.0"
   val slf4jVersion = "1.6.6"
   val parquetVersion = "1.6.0rc4"
 
-  val dfsDatastoresVersion = "1.3.4"
-  val scaldingVersion = "0.13.1"
-  val storehausVersion = "0.10.0"
+  val dfsDatastoresVersion = "1.3.6"
+  val scaldingVersion = "0.14.0"
+  val storehausVersion = "0.11.0"
   val utilVersion = "6.3.8"
 
   val finagleVersion = "6.12.2"
-  val tormentaVersion = "0.9.0"
+  val tormentaVersion = "0.10.0"
   val junitVersion = "4.11"
   val log4jVersion = "1.2.16"
   val stormVersion = "0.9.0-wip15"
   val commonsLangVersion = "2.6"
   val novocodeJunitVersion = "0.10"
   val specs2Version = "1.13"
 
+  val sparkCoreVersion ="1.2.0"
+  val commonsHttpClientVersion = "3.1"
+
   val extraSettings = Project.defaultSettings ++ mimaDefaultSettings ++ scalariformSettings
 
   val sharedSettings = extraSettings ++ Seq(
     organization := "com.twitter",
-    version := "0.6.0",
-    scalaVersion := "2.10.4",
-    crossScalaVersions := Seq("2.10.4"),
+    version := "0.7.0",
+    scalaVersion := "2.10.5",
+    crossScalaVersions := Seq("2.10.5"),
     // To support hadoop 1.x
     javacOptions ++= Seq("-source", "1.6", "-target", "1.6"),
 
@@ -66,7 +68,7 @@ object SummingbirdBuild extends Build {
       // These satisify's scaldings log4j needs when in test mode
       "log4j" % "log4j" % log4jVersion % "test",
       "com.novocode" % "junit-interface" % novocodeJunitVersion % "test",
-      "org.specs2" %% "specs2" % "1.13" % "test"
+      "org.specs2" %% "specs2" % specs2Version % "test"
     ),
 
     resolvers ++= Seq(
@@ -166,10 +168,10 @@ object SummingbirdBuild extends Build {
     summingbirdSpark,
     summingbirdBuilder,
     summingbirdChill,
-    summingbirdExample
+    summingbirdExample,
+    summingbirdCoreTest
   )
 
-
   /**
     * This returns the youngest jar we released that is compatible with
     * the current.
@@ -179,7 +181,7 @@ object SummingbirdBuild extends Build {
   def youngestForwardCompatible(subProj: String) =
     Some(subProj)
       .filterNot(unreleasedModules.contains(_))
-      .map { s => "com.twitter" % ("summingbird-" + s + "_2.10") % "0.6.0" }
+      .map { s => "com.twitter" % ("summingbird-" + s + "_2.10") % "0.7.0" }
 
   def module(name: String) = {
     val id = "summingbird-%s".format(name)
@@ -241,6 +243,7 @@ object SummingbirdBuild extends Build {
     )
   ).dependsOn(
     summingbirdCore % "test->test;compile->compile",
+    summingbirdCoreTest % "test->test",
     summingbirdBatch,
     summingbirdClient
   )
@@ -280,6 +283,7 @@ object SummingbirdBuild extends Build {
     )
   ).dependsOn(
     summingbirdCore % "test->test;compile->compile",
+    summingbirdCoreTest % "test->test",
     summingbirdStorm
   )
 
@@ -305,7 +309,8 @@ object SummingbirdBuild extends Build {
       "com.twitter" %% "chill-bijection" % chillVersion,
       "commons-lang" % "commons-lang" % commonsLangVersion,
       "com.twitter" %% "scalding-core" % scaldingVersion,
-      "com.twitter" %% "scalding-commons" % scaldingVersion
+      "com.twitter" %% "scalding-commons" % scaldingVersion,
+      "org.apache.hadoop" % "hadoop-core" % hadoopVersion % "provided"
     )
   ).dependsOn(
     summingbirdCore % "test->test;compile->compile",
@@ -317,10 +322,12 @@ object SummingbirdBuild extends Build {
   lazy val summingbirdScaldingTest = module("scalding-test").settings(
     libraryDependencies ++= Seq(
       "org.scalacheck" %% "scalacheck" % scalaCheckVersion,
-      "org.slf4j" % "slf4j-log4j12" % slf4jVersion % "test"
+      "org.slf4j" % "slf4j-log4j12" % slf4jVersion % "test",
+      "org.apache.hadoop" % "hadoop-core" % hadoopVersion % "provided"
     )
   ).dependsOn(
     summingbirdCore % "test->test;compile->compile",
+    summingbirdCoreTest % "test->test",
     summingbirdChill,
     summingbirdBatchHadoop,
     summingbirdScalding
@@ -332,7 +339,8 @@ object SummingbirdBuild extends Build {
       "com.twitter" %% "algebird-core" % algebirdVersion,
       "com.twitter" %% "bijection-json" % bijectionVersion,
       "com.twitter" %% "scalding-date" % scaldingVersion,
-      "org.slf4j" % "slf4j-log4j12" % slf4jVersion % "test"
+      "org.slf4j" % "slf4j-log4j12" % slf4jVersion % "test",
+      "org.apache.hadoop" % "hadoop-core" % hadoopVersion % "provided"
     )
   ).dependsOn(
     summingbirdCore % "test->test;compile->compile",
@@ -341,7 +349,8 @@ object SummingbirdBuild extends Build {
 
   lazy val summingbirdBuilder = module("builder").settings(
     libraryDependencies ++= Seq(
-      "storm" % "storm" % stormVersion % "provided"
+      "storm" % "storm" % stormVersion % "provided",
+      "org.apache.hadoop" % "hadoop-core" % hadoopVersion % "provided"
     )
   ).dependsOn(
     summingbirdCore,
@@ -356,7 +365,7 @@ object SummingbirdBuild extends Build {
       "storm" % "storm" % stormVersion exclude("org.slf4j", "log4j-over-slf4j") exclude("ch.qos.logback", "logback-classic"),
       "com.twitter" %% "bijection-netty" % bijectionVersion,
       "com.twitter" %% "tormenta-twitter" % tormentaVersion,
-      "com.twitter" %% "storehaus-memcache" % storehausVersion,
+      "com.twitter" %% "storehaus-memcache" % storehausVersion exclude("com.twitter.common", "dynamic-host-set") exclude("com.twitter.common", "service-thrift"),
       "org.slf4j" % "slf4j-log4j12" % slf4jVersion % "test"
     )
   ).dependsOn(summingbirdCore, summingbirdCoreJava, summingbirdStorm, summingbirdStormJava)
@@ -377,6 +386,7 @@ object SummingbirdBuild extends Build {
     }
   }
 
+
   val sparkDeps = Seq(
     "com.twitter" %% "algebird-core" % algebirdVersion,
     "com.twitter" %% "algebird-util" % algebirdVersion,
@@ -385,9 +395,9 @@ object SummingbirdBuild extends Build {
     "com.twitter" %% "chill" % chillVersion,
     "com.twitter" % "chill-hadoop" % chillVersion,
     "com.twitter" %% "chill-bijection" % chillVersion,
-    "commons-lang" % "commons-lang" % "2.6",
-    "commons-httpclient" % "commons-httpclient" % "3.1",
-    "org.apache.spark" %% "spark-core" % "0.9.0-incubating" % "provided"
+    "commons-lang" % "commons-lang" % commonsLangVersion,
+    "commons-httpclient" % "commons-httpclient" % commonsHttpClientVersion,
+    "org.apache.spark" %% "spark-core" % sparkCoreVersion % "provided"
   )
 
   def buildSparkDeps(scalaVersion: String) = if (isScala210x(scalaVersion)) sparkDeps else Seq()
@@ -403,6 +413,19 @@ object SummingbirdBuild extends Build {
   .settings(sparkAssemblyMergeSettings:_*)
   .dependsOn(
     summingbirdCore % "test->test;compile->compile",
+    summingbirdCoreTest % "test->test",
     summingbirdChill
   )
+
+  lazy val summingbirdCoreTest = module("core-test").settings(
+    parallelExecution in Test := false,
+    libraryDependencies ++=Seq(
+      "junit" % "junit" % junitVersion % "provided",
+      "org.slf4j" % "slf4j-api" % slf4jVersion % "provided",
+      "org.scalacheck" %% "scalacheck" % scalaCheckVersion % "provided",
+      "org.specs2" %% "specs2" % specs2Version % "provided")
+
+  ).dependsOn(
+      summingbirdCore % "test->test;compile->compile"
+    )
 }

sbt

@@ -126,8 +126,8 @@ declare -r default_jvm_opts="-Dfile.encoding=UTF8 -XX:MaxPermSize=384m -Xms512m
 declare -r noshare_opts="-Dsbt.global.base=project/.sbtboot -Dsbt.boot.directory=project/.boot -Dsbt.ivy.home=project/.ivy"
 declare -r latest_28="2.8.2"
 declare -r latest_29="2.9.3"
-declare -r latest_210="2.10.3"
-declare -r latest_211="2.11.0-M5"
+declare -r latest_210="2.10.5"
+declare -r latest_211="2.11.5"
 
 declare -r script_path=$(get_script_path "$BASH_SOURCE")
 declare -r script_dir="$(dirname $script_path)"

scripts/run_tests.sh

@@ -0,0 +1,11 @@
+BASE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )"/.. && pwd )"
+cd $BASE_DIR
+echo "$BASE_DIR"
+bash -c "while true; do echo -n .; sleep 5; done" &
+PROGRESS_REPORTER_PID=$!
+time ./sbt ++$TRAVIS_SCALA_VERSION $TEST_TARGET/compile $TEST_TARGET/test:compile &> /dev/null
+kill -9 $PROGRESS_REPORTER_PID
+
+export JVM_OPTS="-XX:+CMSClassUnloadingEnabled -XX:+UseConcMarkSweepGC -XX:ReservedCodeCacheSize=96m -XX:+TieredCompilation -XX:MaxPermSize=128m -Xms512m -Xmx2048m -Xss2m"
+
+./sbt -Dlog4j.configuration=file://$TRAVIS_BUILD_DIR/project/travis-log4j.properties ++$TRAVIS_SCALA_VERSION $TEST_TARGET/test

summingbird-batch/src/test/scala/com/twitter/summingbird/batch/BatchLaws.scala

@@ -16,7 +16,7 @@ limitations under the License.
 
 package com.twitter.summingbird.batch
 
-import org.scalacheck.{ Arbitrary, Properties }
+import org.scalacheck.{ Arbitrary, Gen, Properties }
 import org.scalacheck.Prop._
 
 import java.util.concurrent.TimeUnit
@@ -51,8 +51,9 @@ object BatchLaws extends Properties("BatchID") {
     }
 
   property("range, toInterval and toIterable should be equivalent") =
-    forAll { (b1: BatchID, diff: SmallLong) =>
-      val b2 = b1 + (diff.get)
+    forAll(Arbitrary.arbitrary[BatchID], Gen.choose(0L, 1000L)) { (b1: BatchID, diff: Long) =>
+      // We can't enumerate too much:
+      val b2 = b1 + diff
       val interval = Interval.leftClosedRightOpen(b1, b2.next) match {
         case Left(i) => i
         case Right(i) => i

summingbird-builder/src/main/scala/com/twitter/summingbird/Env.scala

@@ -51,7 +51,7 @@ object Env {
   }
 
   def main(inargs: Array[String]) {
-    Env(inargs).run
+    Env(inargs).run()
   }
 }
 
@@ -64,5 +64,5 @@ abstract class Env(val jobName: String) extends java.io.Serializable {
 
   // This is where the builder is actually populated.
   protected def abstractJob: AbstractJob = AbstractJob(jobName, this)
-  def run: Unit
+  def run(): Unit
 }

summingbird-builder/src/main/scala/com/twitter/summingbird/builder/CompletedBuilder.scala

@@ -36,10 +36,10 @@ import java.io.Serializable
 
 object CompletedBuilder {
   def injectionRegistrar[T: Manifest](injection: Injection[T, Array[Byte]]) =
-    InjectionRegistrar(manifest[T].erasure.asInstanceOf[Class[T]], injection)
+    InjectionRegistrar(manifest[T].runtimeClass.asInstanceOf[Class[T]], injection)
 
   def injectionDefaultRegistrar[T: Manifest](injection: Injection[T, Array[Byte]]) =
-    InjectionDefaultRegistrar(manifest[T].erasure.asInstanceOf[Class[T]], injection)
+    InjectionDefaultRegistrar(manifest[T].runtimeClass.asInstanceOf[Class[T]], injection)
 }
 
 case class CompletedBuilder[P <: Platform[P], K, V](

summingbird-builder/src/main/scala/com/twitter/summingbird/scalding/ScaldingEnv.scala

@@ -137,7 +137,7 @@ case class ScaldingEnv(override val jobName: String, inargs: Array[String])
     Built(scald, toRun, stateFn)
   }
 
-  def run = run(build)
+  def run() = run(build)
 
   def run(b: Built) {
     val Built(scald, toRun, stateFn) = b