Skip to content

Commit f0967bb

Browse files
AndreiKingsleyAndreiKingsley
committed
GeoDataFrame init
1 parent cc8ae15 commit f0967bb

File tree

15 files changed

+610
-2
lines changed

15 files changed

+610
-2
lines changed

‎dataframe-geo/build.gradle.kts

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
import org.jetbrains.kotlin.gradle.tasks.BaseKotlinCompile
2+
import org.jetbrains.kotlin.gradle.tasks.KotlinCompile
3+
4+
plugins {
5+
with(libs.plugins) {
6+
alias(kotlin.jvm)
7+
alias(publisher)
8+
alias(jupyter.api)
9+
}
10+
}
11+
12+
group = "org.jetbrains.kotlinx"
13+
14+
repositories {
15+
// geo repositories should come before Maven Central
16+
maven("https://maven.geotoolkit.org")
17+
maven("https://repo.osgeo.org/repository/release")
18+
mavenCentral()
19+
}
20+
21+
// https://stackoverflow.com/questions/26993105/i-get-an-error-downloading-javax-media-jai-core1-1-3-from-maven-central
22+
// jai core dependency should be excluded from geotools dependencies and added separately
23+
fun ExternalModuleDependency.excludeJaiCore() = exclude("javax.media", "jai_core")
24+
25+
26+
dependencies {
27+
api(project(":core"))
28+
29+
implementation(libs.geotools.main) { excludeJaiCore() }
30+
implementation(libs.geotools.shapefile) { excludeJaiCore() }
31+
implementation(libs.geotools.geojson) { excludeJaiCore() }
32+
implementation(libs.geotools.referencing) { excludeJaiCore() }
33+
implementation(libs.geotools.epsg.hsql) { excludeJaiCore() }
34+
35+
implementation(libs.jai.core)
36+
37+
implementation(libs.jts.core)
38+
implementation(libs.jts.io.common)
39+
40+
implementation(libs.ktor.client.core)
41+
implementation(libs.ktor.client.cio)
42+
implementation(libs.ktor.client.content.negotiation)
43+
implementation(libs.ktor.serialization.kotlinx.json)
44+
45+
}
46+
47+
tasks.withType<KotlinCompile>().configureEach {
48+
val friendModule = project(":core")
49+
val jarTask = friendModule.tasks.getByName("jar") as Jar
50+
val jarPath = jarTask.archiveFile.get().asFile.absolutePath
51+
(this as BaseKotlinCompile).friendPaths.from(jarPath)
52+
}
53+
54+
tasks.processJupyterApiResources {
55+
libraryProducers = listOf("org.jetbrains.kotlinx.dataframe.jupyter.IntegrationGeo")
56+
}
57+
58+
59+
tasks.test {
60+
useJUnitPlatform()
61+
}
62+
kotlin {
63+
jvmToolchain(11)
64+
}

‎dataframe-geo/src/main/kotlin/org/jetbrains/kotlinx/dataframe/geo/GeoDataFrame.kt

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
package org.jetbrains.kotlinx.dataframe.geo
2+
3+
import org.geotools.api.referencing.crs.CoordinateReferenceSystem
4+
import org.geotools.geometry.jts.JTS
5+
import org.geotools.referencing.CRS
6+
import org.jetbrains.kotlinx.dataframe.DataFrame
7+
import org.jetbrains.kotlinx.dataframe.api.update
8+
import org.jetbrains.kotlinx.dataframe.api.with
9+
10+
class GeoDataFrame<T : GeoFrame>(val df: DataFrame<T>, val crs: CoordinateReferenceSystem?) {
11+
fun update(updateBlock: DataFrame<T>.() -> DataFrame<T>): GeoDataFrame<T> {
12+
return GeoDataFrame(df.updateBlock(), crs)
13+
}
14+
15+
fun applyCRS(targetCRS: CoordinateReferenceSystem? = null): GeoDataFrame<T> {
16+
if (targetCRS == this.crs) return this
17+
// Use WGS 84 by default TODO
18+
val sourceCRS: CoordinateReferenceSystem = this.crs ?: DEFAULT_CRS
19+
val transform = CRS.findMathTransform(sourceCRS, targetCRS, true)
20+
return GeoDataFrame(
21+
df.update { geometry }.with { JTS.transform(it, transform) },
22+
targetCRS
23+
)
24+
}
25+
26+
companion object {
27+
val DEFAULT_CRS = CRS.decode("EPSG:4326", true)
28+
}
29+
}

‎dataframe-geo/src/main/kotlin/org/jetbrains/kotlinx/dataframe/geo/GeoFrame.kt

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
package org.jetbrains.kotlinx.dataframe.geo
2+
3+
import org.jetbrains.kotlinx.dataframe.ColumnsContainer
4+
import org.jetbrains.kotlinx.dataframe.DataColumn
5+
import org.jetbrains.kotlinx.dataframe.annotations.DataSchema
6+
import org.locationtech.jts.geom.Geometry
7+
import org.locationtech.jts.geom.MultiPolygon
8+
import org.locationtech.jts.geom.Polygon
9+
10+
@DataSchema
11+
interface GeoFrame {
12+
val geometry: Geometry
13+
}
14+
15+
@DataSchema
16+
interface PolygonGeoFrame : GeoFrame {
17+
override val geometry: Polygon
18+
}
19+
20+
@DataSchema
21+
interface MultiPolygonGeoFrame : GeoFrame {
22+
override val geometry: MultiPolygon
23+
}
24+
25+
@get:JvmName("geometry")
26+
val <T : GeoFrame> ColumnsContainer<T>.geometry: DataColumn<Geometry>
27+
get() = get("geometry") as DataColumn<Geometry>
28+
29+
@get:JvmName("geometryPolygon")
30+
val <T : PolygonGeoFrame> ColumnsContainer<T>.geometry: DataColumn<Polygon>
31+
get() = get("geometry") as DataColumn<Polygon>
32+
33+
@get:JvmName("geometryMultiPolygon")
34+
val <T : MultiPolygonGeoFrame> ColumnsContainer<T>.geometry: DataColumn<MultiPolygon>
35+
get() = get("geometry") as DataColumn<MultiPolygon>

‎dataframe-geo/src/main/kotlin/org/jetbrains/kotlinx/dataframe/geo/bounds.kt

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
package org.jetbrains.kotlinx.dataframe.geo
2+
3+
import org.geotools.geometry.jts.ReferencedEnvelope
4+
import org.jetbrains.kotlinx.dataframe.api.asIterable
5+
import org.jetbrains.kotlinx.dataframe.geo.jts.computeBounds
6+
7+
fun GeoDataFrame<*>.bounds(): ReferencedEnvelope {
8+
return ReferencedEnvelope(df.geometry.asIterable().computeBounds(), crs)
9+
}

‎dataframe-geo/src/main/kotlin/org/jetbrains/kotlinx/dataframe/geo/geocode/Geocoder.kt

Lines changed: 121 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,121 @@
1+
package org.jetbrains.kotlinx.dataframe.geo.geocode
2+
3+
import io.ktor.client.HttpClient
4+
import io.ktor.client.engine.cio.CIO
5+
import io.ktor.client.plugins.contentnegotiation.ContentNegotiation
6+
import io.ktor.client.request.post
7+
import io.ktor.client.request.setBody
8+
import io.ktor.client.statement.bodyAsText
9+
import io.ktor.http.ContentType
10+
import io.ktor.http.contentType
11+
import io.ktor.serialization.kotlinx.json.json
12+
import kotlinx.coroutines.runBlocking
13+
import kotlinx.serialization.json.Json
14+
import kotlinx.serialization.json.jsonArray
15+
import kotlinx.serialization.json.jsonObject
16+
import kotlinx.serialization.json.jsonPrimitive
17+
import org.jetbrains.kotlinx.dataframe.api.dataFrameOf
18+
import org.jetbrains.kotlinx.dataframe.geo.GeoDataFrame
19+
import org.jetbrains.kotlinx.dataframe.geo.toGeo
20+
import org.locationtech.jts.geom.Geometry
21+
import org.locationtech.jts.geom.GeometryFactory
22+
import org.locationtech.jts.io.geojson.GeoJsonReader
23+
24+
25+
object Geocoder {
26+
27+
private val url = "https://geo2.datalore.jetbrains.com/map_data/geocoding"
28+
29+
private fun countryQuery(country: String) = """ {
30+
"region_query_names" : [ "$country" ],
31+
"region_query_countries" : null,
32+
"region_query_states" : null,
33+
"region_query_counties" : null,
34+
"ambiguity_resolver" : {
35+
"ambiguity_resolver_ignoring_strategy" : null,
36+
"ambiguity_resolver_box" : null,
37+
"ambiguity_resolver_closest_coord" : null
38+
}
39+
}
40+
""".trimIndent()
41+
42+
private fun geocodeQuery(countries: List<String>) = """
43+
{
44+
"version" : 3,
45+
"mode" : "by_geocoding",
46+
"feature_options" : [ "limit", "position", "centroid" ],
47+
"resolution" : null,
48+
"view_box" : null,
49+
"fetched_ids" : null,
50+
"region_queries" : [
51+
${countries.joinToString(",\n") { countryQuery(it) }}
52+
],
53+
"scope" : [ ],
54+
"level" : "country",
55+
"namesake_example_limit" : 10,
56+
"allow_ambiguous" : false
57+
}
58+
""".trimIndent()
59+
60+
private fun idsQuery(ids: List<String>) = """
61+
{"version": 3,
62+
"mode": "by_id",
63+
"feature_options": ["boundary"],
64+
"resolution": 5,
65+
"view_box": null,
66+
"fetched_ids": null,
67+
"ids": [${ids.joinToString(", ") { "\"" + it + "\"" }}]}
68+
""".trimIndent()
69+
70+
private val client = HttpClient(CIO) {
71+
install(ContentNegotiation) {
72+
json(Json {
73+
prettyPrint = true
74+
isLenient = true
75+
})
76+
}
77+
}
78+
79+
fun geocodeCountries(countries: List<String>): GeoDataFrame<*> {
80+
81+
val query = geocodeQuery(countries)
82+
val foundNames = mutableListOf<String>()
83+
val geometries = mutableListOf<Geometry>()
84+
runBlocking {
85+
val responseString = client.post(url) {
86+
contentType(ContentType.Application.Json)
87+
// headers[HttpHeaders.AcceptEncoding] = "gzip"
88+
setBody(query)
89+
}.bodyAsText()
90+
val ids = mutableListOf<String>()
91+
92+
Json.parseToJsonElement(responseString).jsonObject["data"]!!.jsonObject["answers"]!!.jsonArray.forEach {
93+
it.jsonObject["features"]!!.jsonArray.single().jsonObject.also {
94+
foundNames.add(it["name"]!!.jsonPrimitive.content)
95+
ids.add(it["id"]!!.jsonPrimitive.content)
96+
}
97+
}
98+
val idsQuery = idsQuery(ids)
99+
100+
val responseStringGeometries = client.post(url) {
101+
contentType(ContentType.Application.Json)
102+
// headers[HttpHeaders.AcceptEncoding] = "gzip"
103+
setBody(idsQuery)
104+
}.bodyAsText()
105+
106+
val geoJsonReader = GeoJsonReader(GeometryFactory())
107+
Json.parseToJsonElement(responseStringGeometries).jsonObject["data"]!!.jsonObject["answers"]!!.jsonArray.forEach {
108+
it.jsonObject["features"]!!.jsonArray.single().jsonObject.also {
109+
val boundary = it["boundary"]!!.jsonPrimitive.content
110+
geometries.add(geoJsonReader.read(boundary))
111+
}
112+
}
113+
114+
}
115+
return dataFrameOf(
116+
"country" to countries,
117+
"foundName" to foundNames,
118+
"geometry" to geometries,
119+
).toGeo()
120+
}
121+
}

‎dataframe-geo/src/main/kotlin/org/jetbrains/kotlinx/dataframe/geo/geotools/ToGeoDataFrame.kt

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
package org.jetbrains.kotlinx.dataframe.geo.geotools
2+
3+
import org.geotools.api.feature.simple.SimpleFeature
4+
import org.geotools.api.feature.simple.SimpleFeatureType
5+
import org.geotools.api.feature.type.GeometryDescriptor
6+
import org.geotools.api.referencing.crs.CoordinateReferenceSystem
7+
import org.geotools.data.simple.SimpleFeatureCollection
8+
import org.jetbrains.kotlinx.dataframe.DataColumn
9+
import org.jetbrains.kotlinx.dataframe.DataFrame
10+
import org.jetbrains.kotlinx.dataframe.api.Infer
11+
import org.jetbrains.kotlinx.dataframe.api.toDataFrame
12+
import org.jetbrains.kotlinx.dataframe.geo.GeoDataFrame
13+
import org.jetbrains.kotlinx.dataframe.geo.GeoFrame
14+
import org.locationtech.jts.geom.Geometry
15+
16+
fun SimpleFeatureCollection.toGeoDataFrame(): GeoDataFrame<*> {
17+
18+
require(schema is SimpleFeatureType) {
19+
"GeoTools: SimpleFeatureType expected but was: ${schema::class.simpleName}"
20+
}
21+
val attributeDescriptors = (schema as SimpleFeatureType).attributeDescriptors
22+
23+
val dataAttributes = attributeDescriptors?.filter { it !is GeometryDescriptor }?.map { it!! } ?: emptyList()
24+
val geometryAttribute = attributeDescriptors?.find { it is GeometryDescriptor }
25+
?: throw IllegalArgumentException("No geometry attribute")
26+
27+
// In GeoJSON the crs attribute is optional
28+
val crs: CoordinateReferenceSystem? = (geometryAttribute as GeometryDescriptor).coordinateReferenceSystem
29+
30+
val data = dataAttributes.associate { it.localName to ArrayList<Any?>() }
31+
val geometries = ArrayList<Geometry>()
32+
33+
features().use {
34+
while (it.hasNext()) {
35+
val feature = it.next()
36+
require(feature is SimpleFeature) {
37+
"GeoTools: SimpleFeature expected but was: ${feature::class.simpleName}"
38+
}
39+
val featureGeometry = feature.getAttribute(geometryAttribute.name)
40+
41+
require(featureGeometry is Geometry) {
42+
"Not a geometry: [${geometryAttribute.name}] = ${featureGeometry?.javaClass?.simpleName} (feature id: ${feature.id})"
43+
}
44+
// TODO require(featureGeometry.isValid) { "Invalid geometry, feature id: ${feature.id}" }
45+
46+
for (dataAttribute in dataAttributes) {
47+
data[dataAttribute.localName]?.add(feature.getAttribute(dataAttribute.name))
48+
}
49+
geometries.add(featureGeometry)
50+
}
51+
}
52+
53+
val geometryColumn = DataColumn.create("geometry", geometries, Infer.Type)
54+
55+
return GeoDataFrame((data.toDataFrame() + geometryColumn) as DataFrame<GeoFrame>, crs)
56+
}

‎dataframe-geo/src/main/kotlin/org/jetbrains/kotlinx/dataframe/geo/geotools/toSimpleFeatureCollection.kt

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
package org.jetbrains.kotlinx.dataframe.geo.geotools
2+
3+
import org.geotools.api.feature.simple.SimpleFeature
4+
import org.geotools.data.collection.ListFeatureCollection
5+
import org.geotools.data.simple.SimpleFeatureCollection
6+
import org.geotools.feature.simple.SimpleFeatureBuilder
7+
import org.geotools.feature.simple.SimpleFeatureTypeBuilder
8+
import org.jetbrains.kotlinx.dataframe.api.forEach
9+
import org.jetbrains.kotlinx.dataframe.api.map
10+
import org.jetbrains.kotlinx.dataframe.api.single
11+
import org.jetbrains.kotlinx.dataframe.geo.GeoDataFrame
12+
import org.locationtech.jts.geom.Geometry
13+
14+
fun GeoDataFrame<*>.toSimpleFeatureCollection(
15+
name: String? = null,
16+
singleGeometryType: Boolean = false
17+
): SimpleFeatureCollection {
18+
val typeBuilder = SimpleFeatureTypeBuilder()
19+
typeBuilder.name = name ?: "geodata"
20+
typeBuilder.setCRS(crs)
21+
val geometryClass = if (singleGeometryType) {
22+
// todo singleOrNull() ?: error()
23+
df["geometry"].map { it!!::class.java }.distinct().single()
24+
} else Geometry::class.java
25+
typeBuilder.add("the_geom", geometryClass)
26+
df.columnNames().filter { it != "geometry" }.forEach { colName ->
27+
typeBuilder.add(colName, String::class.java)
28+
}
29+
val featureType = typeBuilder.buildFeatureType()
30+
31+
val featureCollection = ListFeatureCollection(featureType)
32+
33+
val featureBuilder = SimpleFeatureBuilder(featureType)
34+
35+
df.forEach { row ->
36+
val geometry = row["geometry"]
37+
featureBuilder.add(geometry)
38+
df.columnNames().filter { it != "geometry" }.forEach { colName ->
39+
featureBuilder.add(row[colName])
40+
}
41+
val feature: SimpleFeature = featureBuilder.buildFeature(null)
42+
featureCollection.add(feature)
43+
}
44+
45+
return featureCollection
46+
}

0 commit comments

Comments
 (0)