Merge pull request #484 from s22s/feature/ci-it-rework

Updates to run IT tests in "medium" compute environment.

Author: metasim

.circleci/Dockerfile

@@ -10,6 +10,7 @@ USER root
 
 ENV PATH=$CONDA_DIR/bin:$PATH
 
+# circleci is 3434
 COPY --chown=3434:3434 fix-permissions /tmp
 
 RUN \
@@ -40,6 +41,7 @@ RUN \
     conda list python | grep '^python ' | tr -s ' ' | cut -d '.' -f 1,2 | sed 's/$/.*/' >> $CONDA_DIR/conda-meta/pinned && \
     conda install --quiet --yes conda && \
     conda install --quiet --yes pip && \
+    pip config set global.progress_bar off && \
     echo "$CONDA_DIR/lib" > /etc/ld.so.conf.d/conda.conf && \
     conda clean --all --force-pkgs-dirs --yes --quiet && \
     sh /tmp/fix-permissions $CONDA_DIR 2> /dev/null

.circleci/config.yml

@@ -14,7 +14,7 @@ orbs:
         working_directory: ~/repo
         environment:
           SBT_VERSION: 1.3.8
-          SBT_OPTS: -Xmx512m
+          SBT_OPTS: -Xmx768m
     commands:
       setup:
         description: Setup for sbt build
@@ -23,6 +23,14 @@ orbs:
               name: Setup sbt
               command: 'true' # NOOP
 
+      compile:
+        description: Do just the compilation stage to minimize sbt memory footprint
+        steps:
+          - run:
+              name: "Compile Scala via sbt"
+              command: |-
+                sbt -v -batch compile test:compile it:compile
+
   python:
     commands:
       setup:
@@ -31,9 +39,6 @@ orbs:
           - run:
               name: Install Python and PIP
               command: |-
-                sudo apt-get update -q -y
-                sudo apt-get install python3 python3-pip
-                sudo update-alternatives --install /usr/bin/python python /usr/bin/python3.7 1
                 python -m pip install --user 'setuptools>=45.2'
 
       requirements:
@@ -118,10 +123,7 @@ jobs:
       - python/setup
       - rasterframes/setup
       - rasterframes/restore-cache
-
-      - run:
-          name: "Compile Scala"
-          command: sbt -v -batch compile
+      - sbt/compile
 
       - run:
           name: "Scala Tests: core"
@@ -157,42 +159,52 @@ jobs:
       - python/requirements
       - rasterframes/setup
       - rasterframes/restore-cache
+      - sbt/compile
 
       - run:
           name: Build documentation
-          command: cat /dev/null | sbt makeSite
-          no-output-timeout: 30m
+          command: sbt makeSite
+          no_output_timeout: 30m
 
       - rasterframes/save-doc-artifacts
       - rasterframes/save-cache
 
   it:
     executor: sbt/default
+    resource_class: large
     steps:
       - checkout
       - sbt/setup
       - rasterframes/setup
       - rasterframes/restore-cache
+      - sbt/compile
 
       - run:
           name: Integration tests
-          command: cat /dev/null | sbt it:test
+          command: sbt it:test
           no_output_timeout: 30m
 
       - rasterframes/save-artifacts
       - rasterframes/save-cache
 
-  itWithoutGdal:
+  it-no-gdal:
     executor: sbt/default
+    resource_class: large
     steps:
       - checkout
       - sbt/setup
       - rasterframes/setup
       - rasterframes/restore-cache
 
+      - run:
+          name: Uninstall GDAL
+          command: conda remove gdal -q -y --offline
+
+      - sbt/compile
+
       - run:
           name: Integration tests
-          command: cat /dev/null | sbt it:test
+          command: sbt it:test
           no_output_timeout: 30m
 
       - rasterframes/save-artifacts
@@ -204,18 +216,27 @@ workflows:
     jobs:
       - test:
           context: rasterframes
+
       - it:
           context: rasterframes
+#          requires:
+#            - test
           filters:
             branches:
               only:
-                - /feature\/.*-its/
-      - itWithoutGdal:
+                - /feature\/.*-it.*/
+                - /it\/.*/
+
+      - it-no-gdal:
           context: rasterframes
+#          requires:
+#            - test
           filters:
             branches:
               only:
-                - /feature\/.*-its/
+                - /feature\/.*-it.*/
+                - /it\/.*/
+
       - docs:
           context: rasterframes
           filters:
@@ -236,5 +257,5 @@ workflows:
     jobs:
       - test
       - it
-      - itWithoutGdal
+      - it-no-gdal
       - docs

.sbtopts

@@ -1,5 +1 @@
--J-XX:+HeapDumpOnOutOfMemoryError
--J-XX:HeapDumpPath=/tmp
--J-XX:+CMSClassUnloadingEnabled
--J-XX:MaxMetaspaceSize=256m
--J-XX:ReservedCodeCacheSize=128m
+

.travis.yml

@@ -113,11 +113,13 @@ lazy val datasource = project
       spark("mllib").value % Provided,
       spark("sql").value % Provided
     ),
-    initialCommands in console := (initialCommands in console).value +
+    console / initialCommands := (console / initialCommands).value +
       """
         |import org.locationtech.rasterframes.datasource.geotrellis._
         |import org.locationtech.rasterframes.datasource.geotiff._
-        |""".stripMargin
+        |""".stripMargin,
+    IntegrationTest / fork := true,
+    IntegrationTest / javaOptions := Seq("-Xmx3g")
   )
 
 lazy val experimental = project
@@ -133,8 +135,8 @@ lazy val experimental = project
       spark("mllib").value % Provided,
       spark("sql").value % Provided
     ),
-    fork in IntegrationTest := true,
-    //javaOptions in IntegrationTest := Seq("-Xmx2G")
+    IntegrationTest / fork := true,
+    IntegrationTest / javaOptions := (datasource / IntegrationTest / javaOptions).value
   )
 
 lazy val docs = project

build.sbt

@@ -40,6 +40,8 @@ log4j.logger.org.apache.spark.repl.SparkILoop$SparkILoopInterpreter=INFO
 log4j.logger.org.locationtech.rasterframes=WARN
 log4j.logger.org.locationtech.rasterframes.ref=WARN
 log4j.logger.org.apache.parquet.hadoop.ParquetRecordReader=OFF
+log4j.logger.geotrellis.spark=INFO
+log4j.logger.geotrellis.raster.gdal=ERROR
 
 # SPARK-9183: Settings to avoid annoying messages when looking up nonexistent UDFs in SparkSQL with Hive support
 log4j.logger.org.apache.hadoop.hive.metastore.RetryingHMSHandler=FATAL

core/src/it/resources/log4j.properties

@@ -23,14 +23,15 @@
   - Revisit use of `Tile` equality since [it's more strict](https://github.com/locationtech/geotrellis/pull/2991)
   - Update `reference.conf` to use `geotrellis.raster.gdal` namespace.
   - Replace all uses of `TileDimensions` with `geotrellis.raster.Dimensions[Int]`.
+* Upgraded to `gdal-warp-bindings` 1.0.0.  
+* Upgraded to Spark 2.4.5
 * Formally abandoned support for Python 2. Python 2 is dead. Long live Python 2.
 * Introduction of type hints in Python API. 
 * Add functions for changing cell values based on either conditions or to achieve a distribution of values. ([#449](https://github.com/locationtech/rasterframes/pull/449))
     * Add `rf_local_min`, `rf_local_max`, and `rf_local_clip` functions.
     * Add cell value scaling functions `rf_rescale` and `rf_standardize`.
     * Add `rf_where` function, similar in spirit to numpy's `where`, or a cell-wise version of Spark SQL's `when` and `otherwise`.
 * Add `rf_sqrt` function to compute cell-wise square root.
-* Upgraded to Spark 2.4.5
 
 ## 0.8.x
 

docs/src/main/paradox/release-notes.md

@@ -20,6 +20,25 @@
 
 from pweave import PwebPandocFormatter
 
+# Setuptools/easy_install doesn't properly set the execute bit on the Spark scripts,
+# So this preemptively attempts to do it.
+def _chmodit():
+    try:
+        from importlib.util import find_spec
+        import os
+        module_home = find_spec("pyspark").origin
+        print(module_home)
+        bin_dir = os.path.join(os.path.dirname(module_home), 'bin')
+        for filename in os.listdir(bin_dir):
+            try:
+                os.chmod(os.path.join(bin_dir, filename), mode=0o555, follow_symlinks=True)
+            except OSError:
+                pass
+    except ImportError:
+        pass
+
+_chmodit()
+
 
 class PegdownMarkdownFormatter(PwebPandocFormatter):
     def __init__(self, *args, **kwargs):

pyrasterframes/src/main/python/docs/__init__.py

@@ -68,6 +68,12 @@ def find_pyrasterframes_assembly() -> Union[bytes, str]:
     return jarpath[0]
 
 
+def quiet_logs(sc):
+    logger = sc._jvm.org.apache.log4j
+    logger.LogManager.getLogger("geotrellis.raster.gdal").setLevel(logger.Level.ERROR)
+    logger.LogManager.getLogger("akka").setLevel(logger.Level.ERROR)
+
+
 def create_rf_spark_session(master="local[*]", **kwargs: str) -> SparkSession:
     """ Create a SparkSession with pyrasterframes enabled and configured. """
     jar_path = find_pyrasterframes_assembly()
@@ -86,6 +92,8 @@ def create_rf_spark_session(master="local[*]", **kwargs: str) -> SparkSession:
              .config(conf=conf)  # user can override the defaults
              .getOrCreate())
 
+    quiet_logs(spark)
+
     try:
         spark.withRasterFrames()
         return spark