Merge pull request #562 from s22s/feature/pyspark-3.1

Python binding updates for PySpark 3.1

Author: metasim

.circleci/.dockerignore

@@ -0,0 +1,3 @@
+*
+!requirements-conda.txt
+!fix-permissions

.circleci/Dockerfile

@@ -1,63 +1,35 @@
-FROM circleci/openjdk:8-jdk
-
-ENV MINICONDA_VERSION=4.8.2 \
-    MINICONDA_MD5=87e77f097f6ebb5127c77662dfc3165e \
-    CONDA_VERSION=4.8.2 \
-    CONDA_DIR=/opt/conda \
-    PYTHON_VERSION=3.7.7
+FROM circleci/openjdk:11-jdk
+#LABEL org.opencontainers.image.source=https://github.com/locationtech/rasterframes
 
 USER root
 
-ENV PATH=$CONDA_DIR/bin:$PATH
-
-# circleci is 3434
-COPY --chown=3434:3434 fix-permissions /tmp
-
+# See: https://docs.conda.io/projects/conda/en/latest/user-guide/install/rpm-debian.html
 RUN \
-    apt-get update && \
-    apt-get install -yq --no-install-recommends \
-        sudo \
-        wget \
-        bzip2 \
-        file \
-        libtinfo5 \
-        ca-certificates \
-        gettext-base \
-        locales && \
-    apt-get clean && \
-    rm -rf /var/lib/apt/lists/*
+  curl -s https://repo.anaconda.com/pkgs/misc/gpgkeys/anaconda.asc | gpg --dearmor > conda.gpg  && \
+  install -o root -g root -m 644 conda.gpg /usr/share/keyrings/conda-archive-keyring.gpg && \
+  gpg --keyring /usr/share/keyrings/conda-archive-keyring.gpg --no-default-keyring --fingerprint 34161F5BF5EB1D4BFBBB8F0A8AEB4F8B29D82806 && \
+  echo "deb [arch=amd64 signed-by=/usr/share/keyrings/conda-archive-keyring.gpg] https://repo.anaconda.com/pkgs/misc/debrepo/conda stable main" > /etc/apt/sources.list.d/conda.list
 
 RUN \
-    cd /tmp && \
-    mkdir -p $CONDA_DIR && \
-    wget --quiet https://repo.continuum.io/miniconda/Miniconda3-py37_${MINICONDA_VERSION}-Linux-x86_64.sh && \
-    echo "${MINICONDA_MD5} *Miniconda3-py37_${MINICONDA_VERSION}-Linux-x86_64.sh" | md5sum -c - && \
-    /bin/bash Miniconda3-py37_${MINICONDA_VERSION}-Linux-x86_64.sh -f -b -p $CONDA_DIR && \
-    rm Miniconda3-py37_${MINICONDA_VERSION}-Linux-x86_64.sh && \
-    conda config --system --set auto_update_conda false && \
-    conda config --system --set show_channel_urls true && \
-    conda config --system --set channel_priority strict && \
-    if [ ! $PYTHON_VERSION = 'default' ]; then conda install --yes python=$PYTHON_VERSION; fi && \
-    conda list python | grep '^python ' | tr -s ' ' | cut -d '.' -f 1,2 | sed 's/$/.*/' >> $CONDA_DIR/conda-meta/pinned && \
-    conda install --quiet --yes conda && \
-    conda install --quiet --yes pip && \
-    pip config set global.progress_bar off && \
-    echo "$CONDA_DIR/lib" > /etc/ld.so.conf.d/conda.conf && \
-    conda clean --all --force-pkgs-dirs --yes --quiet && \
-    sh /tmp/fix-permissions $CONDA_DIR 2> /dev/null
+  apt-get update && \
+  apt-get install -yq --no-install-recommends conda && \
+  apt-get clean && \
+  rm -rf /var/lib/apt/lists/*
 
-COPY requirements-conda.txt /tmp/
+ENV CONDA_DIR=/opt/conda
+ENV PATH=$CONDA_DIR/bin:$PATH
 
+COPY requirements-conda.txt fix-permissions /tmp
 RUN \
-    conda install --channel conda-forge --no-channel-priority --freeze-installed \
-        --file /tmp/requirements-conda.txt && \
+    conda install --quiet --yes --channel=conda-forge --file=/tmp/requirements-conda.txt && \
+    echo "$CONDA_DIR/lib" > /etc/ld.so.conf.d/conda.conf && \
+    ldconfig && \
     conda clean --all --force-pkgs-dirs --yes --quiet && \
-    sh /tmp/fix-permissions $CONDA_DIR 2> /dev/null && \
-    ldconfig 2> /dev/null
+    sh /tmp/fix-permissions $CONDA_DIR
+
 
 # Work-around for pyproj issue https://github.com/pyproj4/pyproj/issues/415
 ENV PROJ_LIB=/opt/conda/share/proj
 
 USER 3434
-
 WORKDIR /home/circleci

.circleci/Makefile

@@ -1,19 +1,27 @@
-IMAGE_NAME=miniconda-gdal
-VERSION=latest
-HOST=docker.pkg.github.com
-REPO=${HOST}/locationtech/rasterframes
-FULL_NAME=${REPO}/${IMAGE_NAME}:${VERSION}
+IMAGE_NAME=circleci-openjdk-conda-gdal
+SHA=$(shell git log -n1 --format=format:"%H" | cut -c 1-7)
+VERSION?=$(SHA)
+HOST=docker.io
+REPO=$(HOST)/s22s
+FULL_NAME=$(REPO)/$(IMAGE_NAME):$(VERSION)
 
-all: build login push
+.DEFAULT_GOAL := help
+help:
+# http://marmelab.com/blog/2016/02/29/auto-documented-makefile.html
+	@echo "Usage: make [target]"
+	@echo "Targets: "
+	@grep -E '^[a-zA-Z0-9_%/-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\t\033[36m%-20s\033[0m %s\n", $$1, $$2}'
 
-build:
+all: build push ## Build and then push image
+
+build: ## Build the docker image
 	docker build . -t ${FULL_NAME}
 
-login:
-	docker login ${HOST}
+login: ## Login to the docker registry
+	docker login
 
-push:
+push: login ## Push docker image to registry
 	docker push ${FULL_NAME}
 
-shell: build
+run: build ## Build image and launch shell
 	docker run --rm -it ${FULL_NAME} bash

.circleci/config.yml

@@ -6,11 +6,10 @@ orbs:
     executors:
       default:
         docker:
-          - image: s22s/miniconda-gdal:latest
+          - image: s22s/circleci-openjdk-conda-gdal:b8e30ee
         working_directory: ~/repo
         environment:
-          SBT_VERSION: 1.3.8
-          SBT_OPTS: -Xmx768m
+          SBT_OPTS: "-Xms64m -Xmx1536m -Djava.awt.headless=true -Dsun.io.serialization.extendedDebugInfo=true"
     commands:
       setup:
         description: Setup for sbt build
@@ -24,8 +23,7 @@ orbs:
         steps:
           - run:
               name: "Compile Scala via sbt"
-              command: |-
-                sbt -v -batch compile test:compile it:compile
+              command: sbt -v -batch compile test:compile it:compile
 
   python:
     commands:
@@ -60,6 +58,7 @@ orbs:
                 mkdir -p /tmp/core_dumps
                 ls -lh /tmp
                 cp core.* *.hs /tmp/core_dumps 2> /dev/null || true
+                cp core/* /tmp/core_dumps/ 2> /dev/null || true
                 cp -r /tmp/hsperfdata* /tmp/*.hprof /tmp/core_dumps 2> /dev/null || true
               when: on_fail
 
@@ -125,24 +124,23 @@ jobs:
 
       - run:
           name: "Scala Tests: core"
-          command: sbt -batch core/test
+          command: sbt -v -batch core/test
 
       - run:
           name: "Scala Tests: datasource"
-          command: sbt -batch datasource/test
+          command: sbt -v -batch datasource/test
 
       - run:
           name: "Scala Tests: experimental"
-          command: sbt -batch experimental/test
+          command: sbt -v -batch experimental/test
 
       - run:
           name: "Create PyRasterFrames package"
-          command: |-
-            sbt -v -batch pyrasterframes/package
+          command: sbt -v -batch pyrasterframes/package
 
       - run:
           name: "Python Tests"
-          command: sbt -batch pyrasterframes/test
+          command: sbt -v -batch pyrasterframes/test
 
       - rasterframes/save-artifacts
       - rasterframes/save-cache
@@ -249,4 +247,4 @@ workflows:
       - test
       - it
       - it-no-gdal
-      - docs
+      - docs

.circleci/fix-permissions

@@ -1,19 +1,4 @@
 #!/usr/bin/env bash
-# set permissions on a directory
-# after any installation, if a directory needs to be (human) user-writable,
-# run this script on it.
-# It will make everything in the directory owned by the group $NB_GID
-# and writable by that group.
-# Deployments that want to set a specific user id can preserve permissions
-# by adding the `--group-add users` line to `docker run`.
-
-# uses find to avoid touching files that already have the right permissions,
-# which would cause massive image explosion
-
-# right permissions are:
-# group=$NB_GID
-# AND permissions include group rwX (directory-execute)
-# AND directories have setuid,setgid bits set
 
 set -e
 

.circleci/requirements-conda.txt

@@ -1,3 +1,5 @@
-gdal==2.4.4
+python==3.8
+gdal==3.1.2
 libspatialindex
+rasterio[s3]
 rtree

.sbtopts

@@ -1,4 +1,4 @@
-version = 3.0.3
+version = 3.0.4
 runner.dialect = scala212
 indent.main = 2
 indent.significant = 2

.scalafmt.conf

@@ -19,6 +19,9 @@
  *
  */
 
+// Leave me an my custom keys alone!
+Global / lintUnusedKeysOnLoad := false
+
 addCommandAlias("makeSite", "docs/makeSite")
 addCommandAlias("previewSite", "docs/previewSite")
 addCommandAlias("ghpagesPushSite", "docs/ghpagesPushSite")
@@ -52,6 +55,10 @@ lazy val core = project
     libraryDependencies ++= Seq(
       `slf4j-api`,
       shapeless,
+      circe("core").value,
+      circe("generic").value,
+      circe("parser").value,
+      circe("generic-extras").value,
       frameless excludeAll ExclusionRule("com.github.mpilquist", "simulacrum"),
       `jts-core`,
       `spray-json`,
@@ -152,14 +159,14 @@ lazy val docs = project
   .dependsOn(core, datasource, pyrasterframes)
   .enablePlugins(SiteScaladocPlugin, ParadoxPlugin, ParadoxMaterialThemePlugin, GhpagesPlugin, ScalaUnidocPlugin)
   .settings(
-    apiURL := Some(url("http://rasterframes.io/latest/api")),
+    apiURL := Some(url("https://rasterframes.io/latest/api")),
     autoAPIMappings := true,
     ghpagesNoJekyll := true,
     ScalaUnidoc / siteSubdirName := "latest/api",
     paradox / siteSubdirName := ".",
     paradoxProperties ++= Map(
       "version" -> version.value,
-      "scaladoc.org.apache.spark.sql.rf" -> "http://rasterframes.io/latest",
+      "scaladoc.org.apache.spark.sql.rf" -> "https://rasterframes.io/latest",
       "github.base_url" -> ""
     ),
     paradoxNavigationExpandDepth := Some(3),

build.sbt

@@ -45,7 +45,7 @@ class TileUDT extends UserDefinedType[Tile] {
   def userClass: Class[Tile] = classOf[Tile]
 
   def sqlType: StructType = StructType(Seq(
-    StructField("cell_type", StringType, false),
+    StructField("cellType", StringType, false),
     StructField("cols", IntegerType, false),
     StructField("rows", IntegerType, false),
     StructField("cells", BinaryType, true),