Skip to content

Commit 8da8bd7

Browse files
echeipeshecheipesh
committed
Bring in the Kryo setup
GT settings bring in jackson classes and with shading it was getting weird
1 parent a92ee4e commit 8da8bd7

File tree

1 file changed

+207
-3
lines changed

1 file changed

+207
-3
lines changed

‎core/src/main/scala/org/locationtech/rasterframes/util/RFKryoRegistrator.scala‎

Lines changed: 207 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -21,11 +21,12 @@
2121

2222
package org.locationtech.rasterframes.util
2323

24-
import org.locationtech.rasterframes.ref.{DelegatingRasterSource, RasterRef, RFRasterSource}
24+
import org.locationtech.rasterframes.ref.{DelegatingRasterSource, RFRasterSource, RasterRef}
2525
import org.locationtech.rasterframes.ref._
2626
import com.esotericsoftware.kryo.Kryo
2727
import geotrellis.raster.io.geotiff.reader.GeoTiffInfo
28-
import geotrellis.spark.store.kryo.KryoRegistrator
28+
import geotrellis.spark.store.kryo.{GeometrySerializer}
29+
import org.apache.spark.serializer.KryoRegistrator
2930

3031
/**
3132
*
@@ -35,7 +36,210 @@ import geotrellis.spark.store.kryo.KryoRegistrator
3536
*/
3637
class RFKryoRegistrator extends KryoRegistrator {
3738
override def registerClasses(kryo: Kryo): Unit = {
38-
super.registerClasses(kryo)
39+
40+
// TreeMap serializaiton has a bug; we fix it here as we're stuck on low
41+
// Kryo versions due to Spark. Hack-tastic.
42+
//kryo.register(classOf[util.TreeMap[_, _]], (new XTreeMapSerializer).asInstanceOf[com.esotericsoftware.kryo.Serializer[TreeMap[_, _]]])
43+
44+
kryo.register(classOf[(_,_)])
45+
kryo.register(classOf[::[_]])
46+
kryo.register(classOf[geotrellis.raster.ByteArrayFiller])
47+
48+
// CellTypes
49+
kryo.register(geotrellis.raster.BitCellType.getClass) // Bit
50+
kryo.register(geotrellis.raster.ByteCellType.getClass) // Byte
51+
kryo.register(geotrellis.raster.ByteConstantNoDataCellType.getClass)
52+
kryo.register(classOf[geotrellis.raster.ByteUserDefinedNoDataCellType])
53+
kryo.register(geotrellis.raster.UByteCellType.getClass) // UByte
54+
kryo.register(geotrellis.raster.UByteConstantNoDataCellType.getClass)
55+
kryo.register(classOf[geotrellis.raster.UByteUserDefinedNoDataCellType])
56+
kryo.register(geotrellis.raster.ShortCellType.getClass) // Short
57+
kryo.register(geotrellis.raster.ShortConstantNoDataCellType.getClass)
58+
kryo.register(classOf[geotrellis.raster.ShortUserDefinedNoDataCellType])
59+
kryo.register(geotrellis.raster.UShortCellType.getClass) // UShort
60+
kryo.register(geotrellis.raster.UShortConstantNoDataCellType.getClass)
61+
kryo.register(classOf[geotrellis.raster.UShortUserDefinedNoDataCellType])
62+
kryo.register(geotrellis.raster.IntCellType.getClass) // Int
63+
kryo.register(geotrellis.raster.IntConstantNoDataCellType.getClass)
64+
kryo.register(classOf[geotrellis.raster.IntUserDefinedNoDataCellType])
65+
kryo.register(geotrellis.raster.FloatCellType.getClass) // Float
66+
kryo.register(geotrellis.raster.FloatConstantNoDataCellType.getClass)
67+
kryo.register(classOf[geotrellis.raster.FloatUserDefinedNoDataCellType])
68+
kryo.register(geotrellis.raster.DoubleCellType.getClass) // Double
69+
kryo.register(geotrellis.raster.DoubleConstantNoDataCellType.getClass)
70+
kryo.register(classOf[geotrellis.raster.DoubleUserDefinedNoDataCellType])
71+
72+
// ArrayTiles
73+
kryo.register(classOf[geotrellis.raster.BitArrayTile]) // Bit
74+
kryo.register(classOf[geotrellis.raster.ByteArrayTile]) // Byte
75+
kryo.register(classOf[geotrellis.raster.ByteRawArrayTile])
76+
kryo.register(classOf[geotrellis.raster.ByteConstantNoDataArrayTile])
77+
kryo.register(classOf[geotrellis.raster.ByteUserDefinedNoDataArrayTile])
78+
kryo.register(classOf[geotrellis.raster.UByteArrayTile]) // UByte
79+
kryo.register(classOf[geotrellis.raster.UByteRawArrayTile])
80+
kryo.register(classOf[geotrellis.raster.UByteConstantNoDataArrayTile])
81+
kryo.register(classOf[geotrellis.raster.UByteUserDefinedNoDataArrayTile])
82+
kryo.register(classOf[geotrellis.raster.ShortArrayTile]) // Short
83+
kryo.register(classOf[geotrellis.raster.ShortRawArrayTile])
84+
kryo.register(classOf[geotrellis.raster.ShortConstantNoDataArrayTile])
85+
kryo.register(classOf[geotrellis.raster.ShortUserDefinedNoDataArrayTile])
86+
kryo.register(classOf[geotrellis.raster.UShortArrayTile]) // UShort
87+
kryo.register(classOf[geotrellis.raster.UShortRawArrayTile])
88+
kryo.register(classOf[geotrellis.raster.UShortConstantNoDataArrayTile])
89+
kryo.register(classOf[geotrellis.raster.UShortUserDefinedNoDataArrayTile])
90+
kryo.register(classOf[geotrellis.raster.IntArrayTile]) // Int
91+
kryo.register(classOf[geotrellis.raster.IntRawArrayTile])
92+
kryo.register(classOf[geotrellis.raster.IntConstantNoDataArrayTile])
93+
kryo.register(classOf[geotrellis.raster.IntUserDefinedNoDataArrayTile])
94+
kryo.register(classOf[geotrellis.raster.FloatArrayTile]) // Float
95+
kryo.register(classOf[geotrellis.raster.FloatRawArrayTile])
96+
kryo.register(classOf[geotrellis.raster.FloatConstantNoDataArrayTile])
97+
kryo.register(classOf[geotrellis.raster.FloatUserDefinedNoDataArrayTile])
98+
kryo.register(classOf[geotrellis.raster.DoubleArrayTile]) // Double
99+
kryo.register(classOf[geotrellis.raster.DoubleRawArrayTile])
100+
kryo.register(classOf[geotrellis.raster.DoubleConstantNoDataArrayTile])
101+
kryo.register(classOf[geotrellis.raster.DoubleUserDefinedNoDataArrayTile])
102+
103+
kryo.register(classOf[Array[geotrellis.raster.Tile]])
104+
kryo.register(classOf[Array[geotrellis.raster.TileFeature[_,_]]])
105+
kryo.register(classOf[geotrellis.raster.Tile])
106+
kryo.register(classOf[geotrellis.raster.TileFeature[_,_]])
107+
108+
kryo.register(classOf[geotrellis.raster.ArrayMultibandTile])
109+
kryo.register(classOf[geotrellis.raster.CompositeTile])
110+
kryo.register(classOf[geotrellis.raster.ConstantTile])
111+
kryo.register(classOf[geotrellis.raster.CroppedTile])
112+
kryo.register(classOf[geotrellis.raster.Raster[_]])
113+
kryo.register(classOf[geotrellis.raster.RasterExtent])
114+
kryo.register(classOf[geotrellis.raster.CellGrid[_]])
115+
kryo.register(classOf[geotrellis.raster.CellSize])
116+
kryo.register(classOf[geotrellis.raster.GridBounds[_]])
117+
kryo.register(classOf[geotrellis.raster.GridExtent[_]])
118+
kryo.register(classOf[geotrellis.raster.mapalgebra.focal.TargetCell])
119+
kryo.register(classOf[geotrellis.raster.summary.GridVisitor[_, _]])
120+
kryo.register(geotrellis.raster.mapalgebra.focal.TargetCell.All.getClass)
121+
kryo.register(geotrellis.raster.mapalgebra.focal.TargetCell.Data.getClass)
122+
kryo.register(geotrellis.raster.mapalgebra.focal.TargetCell.NoData.getClass)
123+
124+
kryo.register(classOf[geotrellis.layer.SpatialKey])
125+
kryo.register(classOf[geotrellis.layer.SpaceTimeKey])
126+
kryo.register(classOf[geotrellis.store.index.rowmajor.RowMajorSpatialKeyIndex])
127+
kryo.register(classOf[geotrellis.store.index.zcurve.ZSpatialKeyIndex])
128+
kryo.register(classOf[geotrellis.store.index.zcurve.ZSpaceTimeKeyIndex])
129+
kryo.register(classOf[geotrellis.store.index.hilbert.HilbertSpatialKeyIndex])
130+
kryo.register(classOf[geotrellis.store.index.hilbert.HilbertSpaceTimeKeyIndex])
131+
kryo.register(classOf[geotrellis.vector.ProjectedExtent])
132+
kryo.register(classOf[geotrellis.vector.Extent])
133+
kryo.register(classOf[geotrellis.proj4.CRS])
134+
135+
// UnmodifiableCollectionsSerializer.registerSerializers(kryo)
136+
kryo.register(geotrellis.raster.buffer.Direction.Center.getClass)
137+
kryo.register(geotrellis.raster.buffer.Direction.Top.getClass)
138+
kryo.register(geotrellis.raster.buffer.Direction.Bottom.getClass)
139+
kryo.register(geotrellis.raster.buffer.Direction.Left.getClass)
140+
kryo.register(geotrellis.raster.buffer.Direction.Right.getClass)
141+
kryo.register(geotrellis.raster.buffer.Direction.TopLeft.getClass)
142+
kryo.register(geotrellis.raster.buffer.Direction.TopRight.getClass)
143+
kryo.register(geotrellis.raster.buffer.Direction.BottomLeft.getClass)
144+
kryo.register(geotrellis.raster.buffer.Direction.BottomRight.getClass)
145+
146+
/* Exhaustive Registration */
147+
kryo.register(classOf[Array[Double]])
148+
kryo.register(classOf[Array[Float]])
149+
kryo.register(classOf[Array[Int]])
150+
kryo.register(classOf[Array[String]])
151+
kryo.register(classOf[Array[org.locationtech.jts.geom.Coordinate]])
152+
kryo.register(classOf[Array[org.locationtech.jts.geom.LinearRing]])
153+
kryo.register(classOf[Array[org.locationtech.jts.geom.Polygon]])
154+
kryo.register(classOf[Array[geotrellis.store.avro.AvroRecordCodec[Any]]])
155+
kryo.register(classOf[Array[geotrellis.layer.SpaceTimeKey]])
156+
kryo.register(classOf[Array[geotrellis.layer.SpatialKey]])
157+
kryo.register(classOf[Array[geotrellis.vector.Feature[_, Any]]])
158+
kryo.register(classOf[Array[geotrellis.vector.MultiPolygon]])
159+
kryo.register(classOf[Array[geotrellis.vector.Point]])
160+
kryo.register(classOf[Array[geotrellis.vector.Polygon]])
161+
kryo.register(classOf[Array[scala.collection.Seq[Any]]])
162+
kryo.register(classOf[Array[(Any, Any)]])
163+
kryo.register(classOf[Array[(Any, Any, Any)]])
164+
kryo.register(classOf[org.locationtech.jts.geom.Coordinate])
165+
kryo.register(classOf[org.locationtech.jts.geom.Envelope])
166+
kryo.register(classOf[org.locationtech.jts.geom.GeometryFactory])
167+
kryo.register(classOf[org.locationtech.jts.geom.impl.CoordinateArraySequence])
168+
kryo.register(classOf[org.locationtech.jts.geom.impl.CoordinateArraySequenceFactory])
169+
kryo.register(classOf[org.locationtech.jts.geom.LinearRing])
170+
kryo.register(classOf[org.locationtech.jts.geom.MultiPolygon])
171+
kryo.register(classOf[org.locationtech.jts.geom.Point])
172+
kryo.register(classOf[org.locationtech.jts.geom.Polygon])
173+
kryo.register(classOf[org.locationtech.jts.geom.PrecisionModel])
174+
kryo.register(classOf[org.locationtech.jts.geom.PrecisionModel.Type])
175+
kryo.register(classOf[geotrellis.raster.histogram.FastMapHistogram])
176+
kryo.register(classOf[geotrellis.raster.histogram.Histogram[AnyVal]])
177+
kryo.register(classOf[geotrellis.raster.histogram.MutableHistogram[AnyVal]])
178+
kryo.register(classOf[geotrellis.raster.histogram.StreamingHistogram])
179+
kryo.register(classOf[geotrellis.raster.histogram.StreamingHistogram.DeltaCompare])
180+
kryo.register(classOf[geotrellis.raster.histogram.StreamingHistogram.Delta])
181+
kryo.register(classOf[geotrellis.raster.histogram.StreamingHistogram.Bucket])
182+
kryo.register(classOf[geotrellis.raster.density.KernelStamper])
183+
kryo.register(classOf[geotrellis.raster.ProjectedRaster[_]])
184+
kryo.register(classOf[geotrellis.raster.TileLayout])
185+
kryo.register(classOf[geotrellis.layer.TemporalProjectedExtent])
186+
kryo.register(classOf[geotrellis.raster.buffer.BufferSizes])
187+
kryo.register(classOf[geotrellis.store.avro.AvroRecordCodec[Any]])
188+
kryo.register(classOf[geotrellis.store.avro.AvroUnionCodec[Any]])
189+
kryo.register(classOf[geotrellis.store.avro.codecs.KeyValueRecordCodec[Any, Any]])
190+
kryo.register(classOf[geotrellis.store.avro.codecs.TupleCodec[Any, Any]])
191+
kryo.register(classOf[geotrellis.layer.KeyBounds[Any]])
192+
kryo.register(classOf[geotrellis.spark.knn.KNearestRDD.Ord[Any]])
193+
kryo.register(classOf[geotrellis.vector.Feature[_, Any]])
194+
kryo.register(classOf[geotrellis.vector.Geometry], new GeometrySerializer[geotrellis.vector.Geometry])
195+
kryo.register(classOf[geotrellis.vector.GeometryCollection])
196+
kryo.register(classOf[geotrellis.vector.LineString], new GeometrySerializer[geotrellis.vector.LineString])
197+
kryo.register(classOf[geotrellis.vector.MultiLineString], new GeometrySerializer[geotrellis.vector.MultiLineString])
198+
kryo.register(classOf[geotrellis.vector.MultiPoint], new GeometrySerializer[geotrellis.vector.MultiPoint])
199+
kryo.register(classOf[geotrellis.vector.MultiPolygon], new GeometrySerializer[geotrellis.vector.MultiPolygon])
200+
kryo.register(classOf[geotrellis.vector.Point])
201+
kryo.register(classOf[geotrellis.vector.Polygon], new GeometrySerializer[geotrellis.vector.Polygon])
202+
kryo.register(classOf[geotrellis.vector.SpatialIndex[Any]])
203+
kryo.register(classOf[java.lang.Class[Any]])
204+
kryo.register(classOf[java.util.TreeMap[Any, Any]])
205+
kryo.register(classOf[java.util.HashMap[Any, Any]])
206+
kryo.register(classOf[java.util.HashSet[Any]])
207+
kryo.register(classOf[java.util.LinkedHashMap[Any, Any]])
208+
kryo.register(classOf[java.util.LinkedHashSet[Any]])
209+
kryo.register(classOf[org.apache.hadoop.io.BytesWritable])
210+
kryo.register(classOf[org.apache.hadoop.io.BigIntWritable])
211+
kryo.register(classOf[Array[org.apache.hadoop.io.BigIntWritable]])
212+
kryo.register(classOf[Array[org.apache.hadoop.io.BytesWritable]])
213+
kryo.register(classOf[org.locationtech.proj4j.CoordinateReferenceSystem])
214+
kryo.register(classOf[org.locationtech.proj4j.datum.AxisOrder])
215+
kryo.register(classOf[org.locationtech.proj4j.datum.AxisOrder.Axis])
216+
kryo.register(classOf[org.locationtech.proj4j.datum.Datum])
217+
kryo.register(classOf[org.locationtech.proj4j.datum.Ellipsoid])
218+
kryo.register(classOf[org.locationtech.proj4j.datum.Grid])
219+
kryo.register(classOf[org.locationtech.proj4j.datum.Grid.ConversionTable])
220+
kryo.register(classOf[org.locationtech.proj4j.util.PolarCoordinate])
221+
kryo.register(classOf[org.locationtech.proj4j.util.FloatPolarCoordinate])
222+
kryo.register(classOf[org.locationtech.proj4j.util.IntPolarCoordinate])
223+
kryo.register(classOf[Array[org.locationtech.proj4j.util.FloatPolarCoordinate]])
224+
kryo.register(classOf[org.locationtech.proj4j.datum.PrimeMeridian])
225+
kryo.register(classOf[org.locationtech.proj4j.proj.LambertConformalConicProjection])
226+
kryo.register(classOf[org.locationtech.proj4j.proj.LongLatProjection])
227+
kryo.register(classOf[org.locationtech.proj4j.proj.TransverseMercatorProjection])
228+
kryo.register(classOf[org.locationtech.proj4j.proj.MercatorProjection])
229+
kryo.register(classOf[org.locationtech.proj4j.units.DegreeUnit])
230+
kryo.register(classOf[org.locationtech.proj4j.units.Unit])
231+
kryo.register(classOf[scala.collection.mutable.WrappedArray.ofInt])
232+
kryo.register(classOf[scala.collection.mutable.WrappedArray.ofRef[AnyRef]])
233+
kryo.register(classOf[scala.collection.Seq[Any]])
234+
kryo.register(classOf[(Any, Any, Any)])
235+
kryo.register(geotrellis.proj4.LatLng.getClass)
236+
kryo.register(geotrellis.layer.EmptyBounds.getClass)
237+
kryo.register(scala.collection.immutable.Nil.getClass)
238+
kryo.register(scala.math.Ordering.Double.getClass)
239+
kryo.register(scala.math.Ordering.Float.getClass)
240+
kryo.register(scala.math.Ordering.Int.getClass)
241+
kryo.register(scala.math.Ordering.Long.getClass)
242+
kryo.register(scala.None.getClass)
39243
kryo.register(classOf[RFRasterSource])
40244
kryo.register(classOf[RasterRef])
41245
kryo.register(classOf[DelegatingRasterSource])

0 commit comments

Comments
 (0)