diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index b8805208..00000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,76 +0,0 @@ -aliases: - - &restore_sbt_cache - key: sbt-cache-{{ checksum "/tmp/scala_version" }} - - - &save_sbt_cache - key: sbt-cache-{{ checksum "/tmp/scala_version" }}-{{ epoch }} - paths: - - "~/.ivy2/cache" - - "~/.sbt" - - "~/.cache/coursier" - - - &run_cibuild - - checkout - - run: echo "${SCALA_VERSION}" > /tmp/scala_version - - restore_cache: *restore_sbt_cache - - run: - name: Executing cibuild - command: ./scripts/cibuild - - save_cache: *save_sbt_cache - - - &run_cipublish - - checkout - - run: echo "${SCALA_VERSION}" > /tmp/scala_version - - restore_cache: *restore_sbt_cache - - run: - name: "Import signing key" - command: | - gpg --keyserver keyserver.ubuntu.com \ - --recv-keys 0x13E9AA1D8153E95E && \ - echo "${GPG_KEY}" | base64 -d > signing_key.asc && \ - gpg --import signing_key.asc - - run: - name: Executing cipublish - command: ./scripts/cipublish - - # Build environments - - &machine-openjdk8-scala2_11_12-environment - machine: - image: ubuntu-1604:201903-01 - environment: - SCALA_VERSION: 2.11.12 - - - &openjdk8-scala2_11_12-environment - docker: - - image: circleci/openjdk:8-jdk - environment: - SCALA_VERSION: 2.11.12 - -version: 2 -workflows: - version: 2 - build: - jobs: - - "openjdk8-scala2.11.12": - filters: # required since `openjdk8-scala2.11.12_deploy` has tag filters AND requires `openjdk8-scala2.11.12` - tags: - only: - - /^(.*)$/ - - "openjdk8-scala2.11.12_deploy": - requires: - - "openjdk8-scala2.11.12" - filters: - tags: - only: - - /^(.*)$/ - -jobs: - # Execute cibuild in machine executor so we can use our existing - # docker-compose test setup - "openjdk8-scala2.11.12": - <<: *machine-openjdk8-scala2_11_12-environment - steps: *run_cibuild - - "openjdk8-scala2.11.12_deploy": - <<: *openjdk8-scala2_11_12-environment - steps: *run_cipublish diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c311ec6d..a8ae593b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -33,3 +33,32 @@ jobs: - name: run tests run: docker compose -f .github/docker-compose.yml up test --abort-on-container-exit --exit-code-from test + + publish: + name: Publish Artifacts + needs: [build] + if: github.event_name != 'pull_request' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v')) + strategy: + matrix: + os: [ubuntu-latest] + java: [8] + distribution: [temurin] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + - uses: coursier/cache-action@v6 + - uses: actions/setup-java@v2 + with: + distribution: ${{ matrix.distribution }} + java-version: ${{ matrix.java }} + + - name: Release + run: sbt ci-release + env: + PGP_PASSPHRASE: ${{ secrets.PGP_PASSPHRASE }} + PGP_SECRET: ${{ secrets.PGP_SECRET }} + SONATYPE_PASSWORD: ${{ secrets.SONATYPE_PASSWORD }} + SONATYPE_USERNAME: ${{ secrets.SONATYPE_USERNAME }} + if: ${{ env.SONATYPE_PASSWORD != '' && env.SONATYPE_USERNAME != '' }} diff --git a/CHANGELOG.md b/CHANGELOG.md index d73b78c0..17175712 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - GitHub actions config +- Upgrade to Scala 2.12; GT 3.7.0-SNAPSHOT; Geomesa Spark JTS 3.2.2 (this standardizes on JTS 1.17.0) ### Changed diff --git a/build.sbt b/build.sbt index 22e0e66e..125aab88 100644 --- a/build.sbt +++ b/build.sbt @@ -2,22 +2,23 @@ import xerial.sbt.Sonatype._ import Dependencies._ lazy val commonSettings = Seq( - // We are overriding the default behavior of sbt-git which, by default, - // only appends the `-SNAPSHOT` suffix if there are uncommitted - // changes in the workspace. - version := { - // Avoid Cyclic reference involving error - if (git.gitCurrentTags.value.isEmpty || git.gitUncommittedChanges.value) - git.gitDescribedVersion.value.get + "-SNAPSHOT" - else - git.gitDescribedVersion.value.get - }, - - cancelable in Global := true, + scalaVersion := Version.scala.head, + crossScalaVersions := Version.scala, - scalaVersion in ThisBuild := Version.scala2_11, + description := "Import OSM data and output to VectorTiles with GeoTrellis.", + organization := "com.azavea.geotrellis", + organizationName := "GeoTrellis", + organizationHomepage := Some(new URL("https://geotrellis.io/")), + homepage := Some(url("https://github.com/geotrellis/vectorpipe")), + versionScheme := Some("semver-spec"), - crossScalaVersions := Seq(Version.scala2_11, Version.scala2_12), + developers := List( + Developer(id = "jpolchlo", name = "Justin Polchlopek", email = "jpolchlopek@azavea.com", url = url("https://github.com/jpolchlo")), + Developer(id = "mojodna", name = "Seth Fitzsimmons", email = "seth@mojodna.net", url = url("https://github.com/mojodna")) + ), + licenses := Seq("Apache-2.0" -> url("http://www.apache.org/licenses/LICENSE-2.0.txt")), + cancelable in Global := true, + //publishArtifact in Test := false, scalacOptions := Seq( "-deprecation", @@ -36,29 +37,19 @@ lazy val commonSettings = Seq( "-Ywarn-unused-import" ), - scalacOptions in (Compile, doc) += "-groups", - scalacOptions in (Compile, console) ~= { _.filterNot(Set("-Ywarn-unused-import", "-Ywarn-unused:imports")) }, + Compile / doc / scalacOptions += "-groups", + Compile / console / scalacOptions ~= { _.filterNot(Set("-Ywarn-unused-import", "-Ywarn-unused:imports")) }, /* For Monocle's Lens auto-generation */ - addCompilerPlugin("org.scalamacros" %% "paradise" % "2.1.0" cross CrossVersion.full), - - resolvers ++= Seq( - Resolver.sonatypeRepo("releases"), - Resolver.bintrayRepo("lonelyplanet", "maven"), - Resolver.bintrayRepo("kwark", "maven"), // Required for Slick 3.1.1.2, see https://github.com/azavea/raster-foundry/pull/1576 - Resolver.bintrayRepo("bkirwi", "maven"), // Required for `decline` dependency - "eclipse-releases" at "https://repo.eclipse.org/content/groups/releases", - "eclipse-snapshots" at "https://repo.eclipse.org/content/groups/snapshots", - "geosolutions" at "http://maven.geo-solutions.it/", - "osgeo-releases" at "https://repo.osgeo.org/repository/release/", - "apache.commons.io" at "https://mvnrepository.com/artifact/commons-io/commons-io" - ), + addCompilerPlugin("org.scalamacros" %% "paradise" % "2.1.1" cross CrossVersion.full), + + externalResolvers := Repositories.all, updateOptions := updateOptions.value.withGigahorse(false), shellPrompt := { s => Project.extract(s).currentProject.id + " > " }, - assemblyMergeStrategy in assembly := { + assembly / assemblyMergeStrategy := { case "reference.conf" | "application.conf" => MergeStrategy.concat case PathList("META-INF", xs@_*) => xs match { @@ -82,41 +73,6 @@ lazy val commonSettings = Seq( } ) -lazy val publishSettings = Seq( - organization := "com.azavea.geotrellis", - organizationName := "GeoTrellis", - organizationHomepage := Some(new URL("https://geotrellis.io/")), - description := "Import OSM data and output to VectorTiles with GeoTrellis.", - publishArtifact in Test := false -) ++ sonatypeSettings ++ credentialSettings - -lazy val sonatypeSettings = Seq( - publishMavenStyle := true, - - sonatypeProfileName := "com.azavea", - sonatypeProjectHosting := Some(GitHubHosting(user="geotrellis", repository="vectorpipe", email="systems@azavea.com")), - developers := List( - Developer(id = "jpolchlo", name = "Justin Polchlopek", email = "jpolchlopek@azavea.com", url = url("https://github.com/jpolchlo")), - Developer(id = "mojodna", name = "Seth Fitzsimmons", email = "seth@mojodna.net", url = url("https://github.com/mojodna")) - ), - licenses := Seq("Apache-2.0" -> url("http://www.apache.org/licenses/LICENSE-2.0.txt")), - - publishTo := sonatypePublishTo.value -) - -lazy val credentialSettings = Seq( - credentials ++= List( - for { - id <- sys.env.get("GPG_KEY_ID") - } yield Credentials("GnuPG Key ID", "gpg", id, "ignored") - , - for { - user <- sys.env.get("SONATYPE_USERNAME") - pass <- sys.env.get("SONATYPE_PASSWORD") - } yield Credentials("Sonatype Nexus Repository Manager", "oss.sonatype.org", user, pass) - ).flatten -) - val vpExtraSettings = Seq( libraryDependencies ++= Seq( // gtGeomesa exclude("com.google.protobuf", "protobuf-java") exclude("org.locationtech.geomesa", @@ -132,7 +88,7 @@ val vpExtraSettings = Seq( gtSpark exclude ("com.google.protobuf", "protobuf-java"), gtVectorTile exclude ("com.google.protobuf", "protobuf-java"), decline, - jaiCore from "http://download.osgeo.org/webdav/geotools/javax/media/jai_core/1.1.3/jai_core-1.1.3.jar", + //jaiCore from "http://download.osgeo.org/webdav/geotools/javax/media/jai_core/1.1.3/jai_core-1.1.3.jar", gtVector, cats, scalactic, @@ -161,8 +117,8 @@ val vpExtraSettings = Seq( } }, - test in assembly := {}, - assemblyJarName in assembly := "vectorpipe.jar", + assembly / test := {}, + assembly / assemblyJarName := "vectorpipe.jar", Test / fork := true, Test / baseDirectory := (baseDirectory.value).getParentFile, @@ -195,7 +151,7 @@ val vpExtraSettings = Seq( /* Main project */ lazy val vectorpipe = project .in(file(".")) - .settings(moduleName := "vectorpipe", commonSettings, publishSettings, vpExtraSettings/*, release*/) + .settings(moduleName := "vectorpipe", commonSettings, vpExtraSettings) /* Benchmarking suite. * Benchmarks can be executed by first switching to the `bench` project and then by running: diff --git a/project/Repositories.scala b/project/Repositories.scala new file mode 100644 index 00000000..8c51418a --- /dev/null +++ b/project/Repositories.scala @@ -0,0 +1,16 @@ +import sbt._ + +object Repositories { + val apacheSnapshots = "apache-snapshots" at "https://repository.apache.org/content/repositories/snapshots/" + val eclipseReleases = "eclipse-releases" at "https://repo.eclipse.org/content/groups/releases" + val eclipseSnapshots = "eclipse-snapshots" at "https://repo.eclipse.org/content/groups/snapshots" + val osgeoReleases = "osgeo-releases" at "https://repo.osgeo.org/repository/release/" + val geosolutions = "geosolutions" at "https://maven.geo-solutions.it/" + val jitpack = "jitpack" at "https://jitpack.io" // for https://github.com/everit-org/json-schema + val ivy2Local = Resolver.file("local", file(Path.userHome.absolutePath + "/.ivy2/local"))(Resolver.ivyStylePatterns) + val mavenLocal = Resolver.mavenLocal + val maven = DefaultMavenRepository + val local = Seq(ivy2Local, mavenLocal) + val external = Seq(osgeoReleases, maven, eclipseReleases, geosolutions, jitpack, apacheSnapshots, eclipseSnapshots) + val all = external ++ local +} diff --git a/project/Version.scala b/project/Version.scala index 7f94cbd1..1baad608 100644 --- a/project/Version.scala +++ b/project/Version.scala @@ -1,14 +1,13 @@ object Version { val awscala = "0.8.1" - val geotrellis = "3.5.1" - val scala2_11 = "2.11.12" - val scala2_12 = "2.12.12" - val geomesa = "2.2.1" - val decline = "0.6.1" - val cats = "1.6.1" + val geotrellis = "3.7.0-SNAPSHOT" + val scala= Seq("2.12.12") + val geomesa = "3.2.2" + val decline = "1.3.0" + val cats = "2.1.1" val scalactic = "3.0.6" val scalatest = "3.0.3" - val spark = "2.4.4" + val spark = "2.4.7" val kryo = "4.0.2" val circe = "0.11.0" val scalaLogging = "3.9.2" diff --git a/project/build.properties b/project/build.properties index c0bab049..10fd9eee 100644 --- a/project/build.properties +++ b/project/build.properties @@ -1 +1 @@ -sbt.version=1.2.8 +sbt.version=1.5.5 diff --git a/project/plugins.sbt b/project/plugins.sbt index 2767f287..37a92170 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -4,10 +4,12 @@ addSbtPlugin("pl.project13.scala" % "sbt-jmh" % "0.2.27") addSbtPlugin("net.virtual-void" % "sbt-dependency-graph" % "0.9.0") -addCompilerPlugin("org.scalamacros" % "paradise" % "2.1.0" cross CrossVersion.full) +//addCompilerPlugin("org.scalamacros" % "paradise" % "2.1.1" cross CrossVersion.full) addSbtPlugin("org.xerial.sbt" % "sbt-sonatype" % "2.5") addSbtPlugin("io.crashbox" % "sbt-gpg" % "0.2.0") addSbtPlugin("com.typesafe.sbt" % "sbt-git" % "1.0.0") + +addSbtPlugin("com.github.sbt" % "sbt-ci-release" % "1.5.9") diff --git a/scripts/cipublish b/scripts/cipublish index b6debb7d..0d92581d 100755 --- a/scripts/cipublish +++ b/scripts/cipublish @@ -17,12 +17,12 @@ if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then if [[ "${1:-}" == "--help" ]]; then usage else - if [[ -n "${CIRCLE_TAG}" ]]; then + if [[ -n "${BUILD_NUMBER}" ]]; then echo "Publishing artifacts to Sonatype" - ./sbt ";++${SCALA_VERSION:-2.11.12};sonatypeOpen ${CIRCLE_BUILD_NUM};publish;sonatypeRelease" + ./sbt ";++${SCALA_VERSION:-2.12.7};sonatypeOpen ${BUILD_NUMBER};publish;sonatypeRelease" else echo "Publishing artifacts to default location" - ./sbt "++${SCALA_VERSION:-2.11.12}" publish + ./sbt "++${SCALA_VERSION:-2.12.7}" publish fi fi fi diff --git a/src/main/scala/vectorpipe/sources/ReplicationStreamBatchReader.scala b/src/main/scala/vectorpipe/sources/ReplicationStreamBatchReader.scala index 252b2b4a..61fdccd3 100644 --- a/src/main/scala/vectorpipe/sources/ReplicationStreamBatchReader.scala +++ b/src/main/scala/vectorpipe/sources/ReplicationStreamBatchReader.scala @@ -8,7 +8,7 @@ import org.apache.spark.sql.catalyst.encoders.{ExpressionEncoder, RowEncoder} import org.apache.spark.sql.sources.v2.reader.InputPartitionReader import scala.collection.parallel.ForkJoinTaskSupport -import scala.concurrent.forkjoin.ForkJoinPool +import java.util.concurrent.ForkJoinPool import scala.reflect.runtime.universe.TypeTag abstract class ReplicationStreamBatchReader[T <: Product: TypeTag](baseURI: URI, diff --git a/src/test/scala/vectorpipe/MultiPolygonRelationReconstructionSpec.scala b/src/test/scala/vectorpipe/MultiPolygonRelationReconstructionSpec.scala index 3ce6a2d0..ac596fdc 100644 --- a/src/test/scala/vectorpipe/MultiPolygonRelationReconstructionSpec.scala +++ b/src/test/scala/vectorpipe/MultiPolygonRelationReconstructionSpec.scala @@ -122,7 +122,7 @@ class MultiPolygonRelationReconstructionSpec extends PropSpec with TableDrivenPr val expected = fixture.wkt.map(wktReader.read) try { - actual should ===(expected) + actual.zip(expected).forall{ case (act, exp) => if (act.isEmpty) exp.isEmpty else exp.buffer(1e-6).contains(act) } should be (true) } catch { case e: Throwable => println(s"${fixture.id} actual:")