Merge pull request #195 from s22s/feature/travis-set-python3

Another attempt at getting travis to use correct python version.

Author: metasim

.travis.yml

@@ -1,6 +1,9 @@
 sudo: false
 dist: xenial
-language: scala
+language: python
+
+python:
+  - "3.7"
 
 cache:
   directories:
@@ -11,30 +14,24 @@ cache:
 scala:
   - 2.11.11
 
-jdk:
-  - openjdk8
-
-python:
-  - "3.7"
+env:
+  - COURSIER_VERBOSITY=-1 JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64
 
 addons:
   apt:
     packages:
+      - openjdk-8-jdk
       - pandoc
-      - python-pip
 
 install:
-  - pip install setuptools
-
-sbt_args: -no-colors
+  - pip install rasterio shapely pandas numpy
+  - wget -O - https://piccolo.link/sbt-1.2.8.tgz | tar xzf -
 
 script:
-  - sbt test
-  - sbt it:test
+  - sbt/bin/sbt -java-home $JAVA_HOME -batch test
+  - sbt/bin/sbt -java-home $JAVA_HOME -batch it:test
   #  - sbt -Dfile.encoding=UTF8 clean coverage test coverageReport
   # Tricks to avoid unnecessary cache updates
   - find $HOME/.sbt -name "*.lock" | xargs rm
   - find $HOME/.ivy2 -name "ivydata-*.properties" | xargs rm
 
-#after_success:
-#  - bash <(curl -s https://codecov.io/bash)

project/RFDependenciesPlugin.scala

@@ -59,7 +59,7 @@ object RFDependenciesPlugin extends AutoPlugin {
     ),
 
     // NB: Make sure to update the Spark version in pyrasterframes/python/setup.py
-    rfSparkVersion := "2.3.2",
+    rfSparkVersion := "2.3.3",
     rfGeoTrellisVersion := "2.2.0",
     rfGeoMesaVersion := "2.2.1",
     dependencyOverrides += "com.azavea.gdal" % "gdal-warp-bindings" % "33.58d4965"

pyrasterframes/src/main/python/pyrasterframes/rf_types.py

@@ -298,7 +298,8 @@ def __init__(self, cells, cell_type=None):
 
     def __eq__(self, other):
         if type(other) is type(self):
-            return self.cell_type == other.cell_type and np.ma.allequal(self.cells, other.cells)
+            return self.cell_type == other.cell_type and \
+                   np.ma.allequal(self.cells, other.cells, fill_value=True)
         else:
             return False
 

pyrasterframes/src/main/python/tests/PyRasterFramesTests.py

@@ -228,28 +228,23 @@ def test_aggregations(self):
         self.assertEqual(row['rf_agg_stats(tile)'].data_cells, row['rf_agg_data_cells(tile)'])
 
     def test_sql(self):
-        self.rf.createOrReplaceTempView("rf")
-
-        dims = self.rf.withColumn('dims', rf_dimensions('tile')).first().dims
-        dims_str = """{}, {}""".format(dims.cols, dims.rows)
-
-        self.spark.sql("""SELECT tile, rf_make_constant_tile(1, {}, 'uint16') AS One, 
-                            rf_make_constant_tile(2, {}, 'uint16') AS Two FROM rf""".format(dims_str, dims_str)) \
-            .createOrReplaceTempView("r3")
-
-        ops = self.spark.sql("""SELECT tile, rf_local_add(tile, One) AS AndOne, 
-                                    rf_local_subtract(tile, One) AS LessOne, 
-                                    rf_local_multiply(tile, Two) AS TimesTwo, 
-                                    rf_local_divide(tile, Two) AS OverTwo 
-                                FROM r3""")
-
-        # ops.printSchema
-        statsRow = ops.select(rf_tile_mean('tile').alias('base'),
-                              rf_tile_mean("AndOne").alias('plus_one'),
-                              rf_tile_mean("LessOne").alias('minus_one'),
-                              rf_tile_mean("TimesTwo").alias('double'),
-                              rf_tile_mean("OverTwo").alias('half')) \
-            .first()
+        self.rf.createOrReplaceTempView("rf_test_sql")
+
+        self.spark.sql("""SELECT tile, 
+                            rf_local_add(tile, 1) AS and_one, 
+                            rf_local_subtract(tile, 1) AS less_one, 
+                            rf_local_multiply(tile, 2) AS times_two, 
+                            rf_local_divide(tile, 2) AS over_two 
+                        FROM rf_test_sql""").createOrReplaceTempView('rf_test_sql_1')
+
+        statsRow = self.spark.sql("""
+        SELECT rf_tile_mean(tile) as base,
+            rf_tile_mean(and_one) as plus_one,
+            rf_tile_mean(less_one) as minus_one,
+            rf_tile_mean(times_two) as double,
+            rf_tile_mean(over_two) as half
+        FROM rf_test_sql_1
+        """).first()
 
         self.assertTrue(self.rounded_compare(statsRow.base, statsRow.plus_one - 1))
         self.assertTrue(self.rounded_compare(statsRow.base, statsRow.minus_one + 1))
@@ -532,8 +527,6 @@ def less_pi(t):
 
 class TileOps(TestEnvironment):
 
-    from pyrasterframes.rf_types import Tile
-
     def setUp(self):
         # convenience so we can assert around Tile() == Tile()
         self.t1 = Tile(np.array([[1, 2],
@@ -589,9 +582,11 @@ def test_matmul(self):
         #     r1 = self.t1 @ self.t2
         r1 = self.t1.__matmul__(self.t2)
 
-        nd = r1.cell_type.no_data_value()
-        e1 = Tile(np.ma.masked_equal(np.array([[nd, 10],
-                                               [nd, nd]], dtype=r1.cell_type.to_numpy_dtype()), nd))
+        # The behavior of np.matmul with masked arrays is not well documented
+        # it seems to treat the 2nd arg as if not a MaskedArray
+        e1 = Tile(np.matmul(self.t1.cells, self.t2.cells), r1.cell_type)
+
+        self.assertTrue(r1 == e1, "{} was not equal to {}".format(r1, e1))
         self.assertEqual(r1, e1)
 
 
@@ -714,7 +709,7 @@ def test_strict_eval(self):
         # again for strict
         df_strict = self.spark.read.raster(self.img_uri, lazy_tiles=False)
         show_str_strict = df_strict.select('proj_raster')._jdf.showString(1, -1, False)
-        self.assertTrue('RasterRef' not in show_str_lazy)
+        self.assertTrue('RasterRef' not in show_str_strict)
 
 
     def test_prt_functions(self):