PDAL 2.6.x (#73)

pomadchin · web-flow · commit 8752273177f2 · 2024-01-07T23:52:41.000-05:00
* Add ubuntu-2.6.0 dockerfile

* Add build-essential

* Bump dylib minor version up

* Use Conda for Mac PDAL releases

* Use Conda for both linux and macos builds
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -10,41 +10,17 @@ on:
     # types: [published]
 
 jobs:
-  linux:
+  build:
     strategy:
       matrix:
-        os: [ubuntu-latest]
-        image: ['daunnc/pdal-ubuntu:2.5.1']
-    runs-on: ${{ matrix.os }}
-    container:
-      image: ${{ matrix.image }}
-    
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
-      # https://github.com/actions/runner/issues/2033
-      - name: Set ownership
-        run: |
-          # this is to fix GIT not liking owner of the checkout dir
-          chown -R $(id -u):$(id -g) $PWD
-
-      - uses: coursier/cache-action@v6
-
-      - name: Check formatting
-        run: ./sbt scalafmtCheckAll
-
-      - name: Build project
-        run: ./sbt +test
-
-  macos:
-    strategy:
-      matrix:
-        os: [macos-latest]
+        os: [ubuntu-latest, macos-latest]
         java: [8]
         distribution: [temurin]
-        pdal: [2.5.6]
+        pdal: [2.6.2]
     runs-on: ${{ matrix.os }}
+    defaults:
+      run:
+        shell: bash -el {0}
 
     steps:
       - uses: actions/checkout@v3
@@ -56,20 +32,16 @@ jobs:
           distribution: ${{ matrix.distribution }}
           java-version: ${{ matrix.java }}
 
-      - name: Cache Homebrew
-        uses: actions/cache@v2
+      - uses: conda-incubator/setup-miniconda@v3
         with:
-          path: ~/Library/Caches/Homebrew
-          key: ${{ runner.os }}-homebrew
-
-      - name: Homebrew update & tap PDAL ${{ matrix.pdal }}
-        run: |
-          brew update && \
-          brew tap-new $USER/local-pdal && \
-          brew extract --version=${{ matrix.pdal }} pdal $USER/local-pdal
+          activate-environment: pdal-java
+          channels: conda-forge
 
       - name: Install PDAL
-        run: brew install pdal@${{ matrix.pdal }} || true
+        run: conda install pdal=${{ matrix.pdal }}
+
+      - name: Set LD_LIBRARY_PATH 
+        run: echo "LD_LIBRARY_PATH=$CONDA/envs/pdal-java/lib:/usr/local/lib:$LD_LIBRARY_PATH" >> "$GITHUB_ENV"
 
       - name: Check formatting
         run: sbt scalafmtCheckAll
@@ -78,6 +50,7 @@ jobs:
         run: sbt +test
 
       - uses: actions/upload-artifact@v3
+        if: ${{ startsWith(matrix.os, 'macos') }}
         with:
           name: macos
           path: native/target/native/x86_64-darwin/bin
@@ -86,12 +59,15 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest]
-        image: ['daunnc/pdal-ubuntu:2.5.1']
+        java: [8]
+        distribution: [temurin]
+        pdal: [2.6.2]
     runs-on: ${{ matrix.os }}
-    needs: [linux, macos]
-    container:
-      image: ${{ matrix.image }}
     if: github.event_name != 'pull_request'
+    needs: [build]
+    defaults:
+      run:
+        shell: bash -el {0}
     env: 
       PDAL_DEPEND_ON_NATIVE: "false"
       CI_CLEAN: ""
@@ -100,21 +76,30 @@ jobs:
       - uses: actions/checkout@v3
         with:
           fetch-depth: 0
-      # https://github.com/actions/runner/issues/2033
-      - name: Set ownership
-        run: |
-          # this is to fix GIT not liking owner of the checkout dir
-          chown -R $(id -u):$(id -g) $PWD
-
       - uses: coursier/cache-action@v6
+      - uses: actions/setup-java@v3
+        with:
+          distribution: ${{ matrix.distribution }}
+          java-version: ${{ matrix.java }}
+
+      - uses: conda-incubator/setup-miniconda@v3
+        with:
+          activate-environment: pdal-java
+          channels: conda-forge
+
+      - name: Install PDAL
+        run: conda install pdal=${{ matrix.pdal }}
+
+      - name: Set LD_LIBRARY_PATH 
+        run: echo "LD_LIBRARY_PATH=$CONDA/envs/pdal-java/lib:/usr/local/lib:$LD_LIBRARY_PATH" >> "$GITHUB_ENV"
 
       - uses: actions/download-artifact@v3
         with:
           name: macos
           path: native/target/native/x86_64-darwin/bin
 
       - name: Release
-        run: ./sbt ci-release
+        run: sbt ci-release
         env:
           PGP_PASSPHRASE: ${{ secrets.PGP_PASSPHRASE }}
           PGP_SECRET: ${{ secrets.PGP_SECRET }}
diff --git a/.scalafmt.conf b/.scalafmt.conf
@@ -1,4 +1,4 @@
-version=3.7.2
+version=3.7.17
 runner.dialect = scala3
 align.openParenCallSite = true
 align.openParenDefnSite = true
diff --git a/build.sbt b/build.sbt
@@ -71,10 +71,12 @@ lazy val core = project
   .settings(name := "pdal")
   .settings(javah / target := (native / nativeCompile / sourceDirectory).value / "include")
   .settings(sbtJniCoreScope := Compile)
-  .settings(libraryDependencies ++= Seq(
-    Dependencies.scalaTest % Test,
-    Dependencies.circe("parser") % Test
-  ))
+  .settings(
+    libraryDependencies ++= Seq(
+      Dependencies.scalaTest % Test,
+      Dependencies.circe("parser") % Test
+    )
+  )
   .dependsOn(Environment.dependOnNative(native % Runtime): _*)
 
 lazy val native = project
@@ -84,9 +86,9 @@ lazy val native = project
   .settings(nativeCompile / sourceDirectory := sourceDirectory.value)
   .settings(
     Compile / unmanagedPlatformDependentNativeDirectories := Seq(
-      "x86_64-linux"  -> target.value / "native/x86_64-linux/bin/",
+      "x86_64-linux" -> target.value / "native/x86_64-linux/bin/",
       "x86_64-darwin" -> target.value / "native/x86_64-darwin/bin/",
-      "arm64-darwin"  -> target.value / "native/arm64-darwin/bin/"
+      "arm64-darwin" -> target.value / "native/arm64-darwin/bin/"
     )
   )
   .settings(artifactName := { (sv: ScalaVersion, module: ModuleID, artifact: Artifact) =>
diff --git a/core/src/main/scala/io/pdal/Pipeline.scala b/core/src/main/scala/io/pdal/Pipeline.scala
@@ -44,7 +44,7 @@ class Pipeline(val json: String, val logLevel: Int) extends Native {
   @native def getLog(): String
 }
 
-object Pipeline extends NativeLoader("pdaljni.2.5") {
+object Pipeline extends NativeLoader("pdaljni.2.6") {
   def apply(json: String, logLevel: Int = LogLevel.Error): Pipeline = {
     val p = new Pipeline(json, logLevel); p.initialize(); p
   }
diff --git a/core/src/test/resources/quick-info-with-metadata-mac.json b/core/src/test/resources/quick-info-with-metadata-mac.json
@@ -1,19 +1,16 @@
 {
-  "readers.las":
-  {
-    "bounds":
-    {
+  "readers.las": {
+    "bounds": {
       "maxx": 638982.55,
       "maxy": 853535.43,
       "maxz": 586.38,
       "minx": 635619.85,
       "miny": 848899.7,
       "minz": 406.59
     },
-    "dimensions": "X, Y, Z, Intensity, ReturnNumber, NumberOfReturns, ScanDirectionFlag, EdgeOfFlightLine, Classification, ScanAngleRank, UserData, PointSourceId, GpsTime, Red, Green, Blue",
+    "dimensions": "X, Y, Z, Intensity, ReturnNumber, NumberOfReturns, ScanDirectionFlag, EdgeOfFlightLine, Classification, Synthetic, KeyPoint, Withheld, Overlap, ScanAngleRank, UserData, PointSourceId, GpsTime, Red, Green, Blue",
     "num_points": 1065,
-    "srs":
-    {
+    "srs": {
       "compoundwkt": "PROJCS[\"NAD83(HARN) / Oregon LCC (m)\",GEOGCS[\"NAD83(HARN)\",DATUM[\"NAD83_High_Accuracy_Reference_Network\",SPHEROID[\"GRS 1980\",6378137,298.257222101,AUTHORITY[\"EPSG\",\"7019\"]],AUTHORITY[\"EPSG\",\"6152\"]],PRIMEM[\"Greenwich\",0,AUTHORITY[\"EPSG\",\"8901\"]],UNIT[\"degree\",0.0174532925199433,AUTHORITY[\"EPSG\",\"9122\"]],AUTHORITY[\"EPSG\",\"4152\"]],PROJECTION[\"Lambert_Conformal_Conic_2SP\"],PARAMETER[\"latitude_of_origin\",41.75],PARAMETER[\"central_meridian\",-120.5],PARAMETER[\"standard_parallel_1\",43],PARAMETER[\"standard_parallel_2\",45.5],PARAMETER[\"false_easting\",400000],PARAMETER[\"false_northing\",0],UNIT[\"metre\",1,AUTHORITY[\"EPSG\",\"9001\"]],AXIS[\"Easting\",EAST],AXIS[\"Northing\",NORTH],AUTHORITY[\"EPSG\",\"2993\"]]",
       "horizontal": "PROJCS[\"NAD83(HARN) / Oregon LCC (m)\",GEOGCS[\"NAD83(HARN)\",DATUM[\"NAD83_High_Accuracy_Reference_Network\",SPHEROID[\"GRS 1980\",6378137,298.257222101,AUTHORITY[\"EPSG\",\"7019\"]],AUTHORITY[\"EPSG\",\"6152\"]],PRIMEM[\"Greenwich\",0,AUTHORITY[\"EPSG\",\"8901\"]],UNIT[\"degree\",0.0174532925199433,AUTHORITY[\"EPSG\",\"9122\"]],AUTHORITY[\"EPSG\",\"4152\"]],PROJECTION[\"Lambert_Conformal_Conic_2SP\"],PARAMETER[\"latitude_of_origin\",41.75],PARAMETER[\"central_meridian\",-120.5],PARAMETER[\"standard_parallel_1\",43],PARAMETER[\"standard_parallel_2\",45.5],PARAMETER[\"false_easting\",400000],PARAMETER[\"false_northing\",0],UNIT[\"metre\",1,AUTHORITY[\"EPSG\",\"9001\"]],AXIS[\"Easting\",EAST],AXIS[\"Northing\",NORTH],AUTHORITY[\"EPSG\",\"2993\"]]",
       "isgeocentric": false,
@@ -145,8 +142,7 @@
       "prettycompoundwkt": "PROJCS[\"NAD83(HARN) / Oregon LCC (m)\",\n    GEOGCS[\"NAD83(HARN)\",\n        DATUM[\"NAD83_High_Accuracy_Reference_Network\",\n            SPHEROID[\"GRS 1980\",6378137,298.257222101,\n                AUTHORITY[\"EPSG\",\"7019\"]],\n            AUTHORITY[\"EPSG\",\"6152\"]],\n        PRIMEM[\"Greenwich\",0,\n            AUTHORITY[\"EPSG\",\"8901\"]],\n        UNIT[\"degree\",0.0174532925199433,\n            AUTHORITY[\"EPSG\",\"9122\"]],\n        AUTHORITY[\"EPSG\",\"4152\"]],\n    PROJECTION[\"Lambert_Conformal_Conic_2SP\"],\n    PARAMETER[\"latitude_of_origin\",41.75],\n    PARAMETER[\"central_meridian\",-120.5],\n    PARAMETER[\"standard_parallel_1\",43],\n    PARAMETER[\"standard_parallel_2\",45.5],\n    PARAMETER[\"false_easting\",400000],\n    PARAMETER[\"false_northing\",0],\n    UNIT[\"metre\",1,\n        AUTHORITY[\"EPSG\",\"9001\"]],\n    AXIS[\"Easting\",EAST],\n    AXIS[\"Northing\",NORTH],\n    AUTHORITY[\"EPSG\",\"2993\"]]",
       "prettywkt": "PROJCS[\"NAD83(HARN) / Oregon LCC (m)\",\n    GEOGCS[\"NAD83(HARN)\",\n        DATUM[\"NAD83_High_Accuracy_Reference_Network\",\n            SPHEROID[\"GRS 1980\",6378137,298.257222101,\n                AUTHORITY[\"EPSG\",\"7019\"]],\n            AUTHORITY[\"EPSG\",\"6152\"]],\n        PRIMEM[\"Greenwich\",0,\n            AUTHORITY[\"EPSG\",\"8901\"]],\n        UNIT[\"degree\",0.0174532925199433,\n            AUTHORITY[\"EPSG\",\"9122\"]],\n        AUTHORITY[\"EPSG\",\"4152\"]],\n    PROJECTION[\"Lambert_Conformal_Conic_2SP\"],\n    PARAMETER[\"latitude_of_origin\",41.75],\n    PARAMETER[\"central_meridian\",-120.5],\n    PARAMETER[\"standard_parallel_1\",43],\n    PARAMETER[\"standard_parallel_2\",45.5],\n    PARAMETER[\"false_easting\",400000],\n    PARAMETER[\"false_northing\",0],\n    UNIT[\"metre\",1,\n        AUTHORITY[\"EPSG\",\"9001\"]],\n    AXIS[\"Easting\",EAST],\n    AXIS[\"Northing\",NORTH],\n    AUTHORITY[\"EPSG\",\"2993\"]]",
       "proj4": "+proj=lcc +lat_0=41.75 +lon_0=-120.5 +lat_1=43 +lat_2=45.5 +x_0=400000 +y_0=0 +ellps=GRS80 +units=m +no_defs",
-      "units":
-      {
+      "units": {
         "horizontal": "metre",
         "vertical": ""
       },
diff --git a/core/src/test/resources/quick-info-with-metadata.json b/core/src/test/resources/quick-info-with-metadata.json
@@ -10,7 +10,7 @@
       "miny": 848899.7,
       "minz": 406.59
     },
-    "dimensions": "X, Y, Z, Intensity, ReturnNumber, NumberOfReturns, ScanDirectionFlag, EdgeOfFlightLine, Classification, ScanAngleRank, UserData, PointSourceId, GpsTime, Red, Green, Blue",
+    "dimensions": "X, Y, Z, Intensity, ReturnNumber, NumberOfReturns, ScanDirectionFlag, EdgeOfFlightLine, Classification, Synthetic, KeyPoint, Withheld, Overlap, ScanAngleRank, UserData, PointSourceId, GpsTime, Red, Green, Blue",
     "num_points": 1065,
     "srs":
     {
diff --git a/core/src/test/resources/quick-info.json b/core/src/test/resources/quick-info.json
@@ -10,7 +10,7 @@
       "miny": 848899.7,
       "minz": 406.59
     },
-    "dimensions": "X, Y, Z, Intensity, ReturnNumber, NumberOfReturns, ScanDirectionFlag, EdgeOfFlightLine, Classification, ScanAngleRank, UserData, PointSourceId, GpsTime, Red, Green, Blue",
+    "dimensions": "X, Y, Z, Intensity, ReturnNumber, NumberOfReturns, ScanDirectionFlag, EdgeOfFlightLine, Classification, Synthetic, KeyPoint, Withheld, Overlap, ScanAngleRank, UserData, PointSourceId, GpsTime, Red, Green, Blue",
     "num_points": 1065
   }
 }
diff --git a/core/src/test/resources/schema.json b/core/src/test/resources/schema.json
@@ -48,6 +48,26 @@
         "size": 1,
         "type": "unsigned"
       },
+      {
+        "name": "Synthetic",
+        "size": 1,
+        "type": "unsigned"
+      },
+      {
+        "name": "KeyPoint",
+        "size": 1,
+        "type": "unsigned"
+      },
+      {
+        "name": "Withheld",
+        "size": 1,
+        "type": "unsigned"
+      },
+      {
+        "name": "Overlap",
+        "size": 1,
+        "type": "unsigned"
+      },
       {
         "name": "ScanAngleRank",
         "size": 4,
diff --git a/core/src/test/scala/io/pdal/PipelineSpec.scala b/core/src/test/scala/io/pdal/PipelineSpec.scala
@@ -125,7 +125,7 @@ class PipelineSpec extends TestEnvironmentSpec {
     it("layout should have a valid number of dims") {
       val pvi = pipeline.getPointViews()
       val pv = pvi.next()
-      pv.layout().dimTypes().length should be(16)
+      pv.layout().dimTypes().length should be(20)
       pv.close()
       pvi.close()
     }
diff --git a/native/src/CMakeLists.txt b/native/src/CMakeLists.txt
@@ -9,7 +9,7 @@ set(MAKE_COLOR_MAKEFILE ON)
 
 project (pdaljni)
 set(PROJECT_VERSION_MAJOR 2)
-set(PROJECT_VERSION_MINOR 5)
+set(PROJECT_VERSION_MINOR 6)
 set(PROJECT_VERSION_PATCH 0)
 
 set(PDAL_LIB_NAME pdalcpp)
diff --git a/project/Dependencies.scala b/project/Dependencies.scala
@@ -7,7 +7,8 @@ import de.heikoseeberger.sbtheader.HeaderPlugin.autoImport.{headerLicense, heade
 object Version {
   val jts = "1.19.0"
   val scalaTest = "3.2.17"
-  val circe = "0.14.3"
+  val circe = "0.14.6"
+  val circeExtras = "0.14.3"
 }
 
 object Dependencies {
@@ -41,7 +42,10 @@ object Dependencies {
     )
   )
 
-  def circe(module: String) = "io.circe" %% s"circe-$module" % Version.circe
+  def circe(module: String) = module match {
+    case "generic-extras" => "io.circe" %% s"circe-$module" % Version.circeExtras
+    case _                => "io.circe" %% s"circe-$module" % Version.circe
+  }
 
   val jtsCore = "org.locationtech.jts" % "jts-core" % Version.jts
   val scalaTest = "org.scalatest" %% "scalatest" % Version.scalaTest
diff --git a/project/build.properties b/project/build.properties
@@ -1 +1 @@
-sbt.version=1.9.6
+sbt.version=1.9.7
diff --git a/scripts/crosscompile-linux.sh b/scripts/crosscompile-linux.sh
@@ -6,4 +6,4 @@ docker run -it --rm \
   -v $HOME/.ivy2:/root/.ivy2 \
   -v $HOME/.sbt:/root/.sbt \
   -v $HOME/.coursier/cache:/root/.cache/coursier \
- daunnc/pdal-ubuntu:2.5.1 bash -c "cd ./pdal-java; ./sbt native/compile"
+ daunnc/pdal-ubuntu:2.6.0 bash -c "cd ./pdal-java; ./sbt native/compile"
diff --git a/scripts/docker/2.6.0/ubuntu/Dockerfile b/scripts/docker/2.6.0/ubuntu/Dockerfile
@@ -0,0 +1,23 @@
+FROM pdal/pdal:2.6.0
+LABEL Grigory Pomadchin <daunnc@gmail.com>
+
+ENV DEBIAN_FRONTEND noninteractive
+
+RUN set -ex && \
+    apt update -y && \
+    apt install --no-install-recommends -y openjdk-8-jdk
+
+ENV JAVA_HOME /usr/lib/jvm/java-1.8.0-openjdk-amd6
+RUN update-alternatives --set java `update-alternatives --list java | grep java-8`
+
+RUN apt-get -y install bash gcc g++ cmake wget unzip gpg software-properties-common
+
+RUN ln -s /opt/conda/envs/pdal/include/pdal /usr/include/pdal && \
+    ln -s /usr/include /usr/lib/include && \
+    ln -s /opt/conda/envs/pdal/lib /usr/lib/lib && \
+    ln -s /opt/conda/envs/pdal/share/* /usr/share/* || true && \
+    ln -s /opt/conda/envs/pdal/lib/* /usr/lib/* || true
+
+# GLIBCXX_3.4.30
+# strings /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | grep GLIBCXX
+RUN apt install -y build-essential

Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,4 @@`
`1`		`-version=3.7.2`
	`1`	`+version=3.7.17`
`2`	`2`	`runner.dialect = scala3`
`3`	`3`	`align.openParenCallSite = true`
`4`	`4`	`align.openParenDefnSite = true`
Original file line number	Diff line number	Diff line change
`@@ -44,7 +44,7 @@ class Pipeline(val json: String, val logLevel: Int) extends Native {`
`44`	`44`	`@native def getLog(): String`
`45`	`45`	`}`
`46`	`46`
`47`		`-object Pipeline extends NativeLoader("pdaljni.2.5") {`
	`47`	`+object Pipeline extends NativeLoader("pdaljni.2.6") {`
`48`	`48`	`def apply(json: String, logLevel: Int = LogLevel.Error): Pipeline = {`
`49`	`49`	`val p = new Pipeline(json, logLevel); p.initialize(); p`
`50`	`50`	`}`
Original file line number	Diff line number	Diff line change
`@@ -10,7 +10,7 @@`
`10`	`10`	`"miny": 848899.7,`
`11`	`11`	`"minz": 406.59`
`12`	`12`	`},`
`13`		`- "dimensions": "X, Y, Z, Intensity, ReturnNumber, NumberOfReturns, ScanDirectionFlag, EdgeOfFlightLine, Classification, ScanAngleRank, UserData, PointSourceId, GpsTime, Red, Green, Blue",`
	`13`	`+ "dimensions": "X, Y, Z, Intensity, ReturnNumber, NumberOfReturns, ScanDirectionFlag, EdgeOfFlightLine, Classification, Synthetic, KeyPoint, Withheld, Overlap, ScanAngleRank, UserData, PointSourceId, GpsTime, Red, Green, Blue",`
`14`	`14`	`"num_points": 1065,`
`15`	`15`	`"srs":`
`16`	`16`	`{`
Original file line number	Diff line number	Diff line change
`@@ -125,7 +125,7 @@ class PipelineSpec extends TestEnvironmentSpec {`
`125`	`125`	`it("layout should have a valid number of dims") {`
`126`	`126`	`val pvi = pipeline.getPointViews()`
`127`	`127`	`val pv = pvi.next()`
`128`		`- pv.layout().dimTypes().length should be(16)`
	`128`	`+ pv.layout().dimTypes().length should be(20)`
`129`	`129`	`pv.close()`
`130`	`130`	`pvi.close()`
`131`	`131`	`}`