Merge pull request #50 from JuliaGeo/better_columnnames

adapt name search from JSONTables.jl

Author: visr

Project.toml

@@ -23,8 +23,9 @@ julia = "1.6"
 
 [extras]
 Aqua = "4c88cf16-eb10-579e-8560-4a9242c79595"
+DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0"
 Plots = "91a5bcdd-55d7-5caf-9e0b-520d859cae80"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 
 [targets]
-test = ["Aqua", "Plots", "Test"]
+test = ["Aqua", "DataFrames", "Plots", "Test"]

src/GeoJSON.jl

@@ -6,7 +6,7 @@ module GeoJSON
     Base.read(path, String)
 end GeoJSON
 
-import JSON3, Tables, GeoFormatTypes, Extents, GeoInterfaceRecipes
+import Extents, GeoFormatTypes, GeoInterfaceRecipes, JSON3, Tables 
 import GeoInterface as GI
 
 include("geometries.jl")

src/features.jl

@@ -5,27 +5,40 @@ A feature wrapping the JSON object.
 """
 struct Feature{T}
     object::T
+    parent_properties::Vector{Symbol}
 end
 
-Feature{T}(f::Feature{T}) where {T} = f
-Feature(; geometry::Geometry, kwargs...) =
+Feature(f::Feature) = f
+Feature(f::Feature, names::Vector{Symbol}) = Feature(object(f), names)
+Feature{T}(f::Feature, names::Vector{Symbol}) where T = Feature(object(f), names)
+Feature{T}(f::Feature) where {T} = f
+Feature(object) = Feature(object, Symbol[])
+Feature(; geometry::Union{Geometry,Missing,Nothing}, kwargs...) =
     Feature(merge((type = "Feature", geometry), kwargs))
-Feature(geometry::Geometry; kwargs...) =
+Feature(geometry::Union{Geometry,Missing,Nothing}; kwargs...) =
     Feature(merge((type = "Feature", geometry), kwargs))
 
 """
     properties(f::Union{Feature,FeatureCollection})
 
 Access the properties JSON object of a Feature
 """
+properties(obj::JSON3.Object) = obj.properties
 properties(f::Feature) = object(f).properties
 
 """
     geometry(f::Feature)
 
 Access the JSON object that represents the Feature's geometry
 """
-geometry(f::Feature) = geometry(object(f).geometry)
+function geometry(f::Feature{<:JSON3.Object}) 
+    geom = geometry(object(f).geometry)
+    return ismissing(geom) ? nothing : geom
+end
+function geometry(f::Feature{<:NamedTuple}) 
+    geom = object(f).geometry
+    return ismissing(geom) ? nothing : geom
+end
 
 coordinates(f::Feature) = coordinates(geometry(object(f).geometry))
 
@@ -43,13 +56,23 @@ function Base.getproperty(f::Feature, nm::Symbol)
         geometry(f)
     else
         props = properties(f)
-        getproperty(props, nm)
+        if hasproperty(props, nm)
+            getproperty(props, nm)
+        else
+            if nm in getfield(f, :parent_properties)
+                missing
+            else
+                error("property `$nm` is not part of Feature or parent FeatureCollection")
+            end
+        end
     end
     return ifelse(x === nothing, missing, x)
 end
 
 Base.:(==)(f1::Feature, f2::Feature) = object(f1) == object(f2)
 
+
+
 """
     FeatureCollection <: AbstractVector{Feature}
 
@@ -61,25 +84,39 @@ and similarly the GeoInterface.jl interface.
 struct FeatureCollection{T,O,A} <: AbstractVector{T}
     object::O
     features::A
+    names::Vector{Symbol}
+    types::Dict{Symbol,Type}
 end
-
-function FeatureCollection(object::O) where {O}
+function FeatureCollection(object::O) where O
     features = object.features
-    T = isempty(features) ? Feature{Any} : typeof(Feature(first(features)))
-    return FeatureCollection{T,O,typeof(features)}(object, features)
+    if isempty(features)
+        names = Symbol[:geometry]
+        types = Dict{Symbol,Type}(:geometry => Union{Missing,Geometry})
+        T = Feature{Any}
+    else
+        names, types = property_schema(features)
+        insert!(names, 1, :geometry)
+        types[:geometry] = Union{Missing,Geometry}
+        f1 = first(features)
+        T = if f1 isa JSON3.Object
+            typeof(Feature(f1, names))
+        elseif f1 isa NamedTuple && isconcretetype(eltype(features))
+            typeof(Feature(f1, names))
+        else
+            T = Feature{Any}
+        end
+    end
+    return FeatureCollection{T,O,typeof(features)}(object, features, names, types)
 end
-
 function FeatureCollection(; features::AbstractVector{T}, kwargs...) where {T}
-    FT = ifelse(T <: Feature, T, Feature{T})
     object = merge((type = "FeatureCollection", features), kwargs)
-    return FeatureCollection{FT,typeof(object),typeof(features)}(object, features)
+    return FeatureCollection(object)
 end
+FeatureCollection(features::AbstractVector; kwargs...) =
+    FeatureCollection(; features, kwargs...)
 
-function FeatureCollection(features::AbstractVector{T}; kwargs...) where {T}
-    FT = ifelse(T <: Feature, T, Feature{T})
-    object = merge((type = "FeatureCollection", features), kwargs)
-    return FeatureCollection{FT,typeof(object),typeof(features)}(object, features)
-end
+names(fc::FeatureCollection) = getfield(fc, :names)
+types(fc::FeatureCollection) = getfield(fc, :types)
 
 """
     features(fc::FeatureCollection)
@@ -90,16 +127,7 @@ features(fc::FeatureCollection) = getfield(fc, :features)
 
 # Base methods
 
-function Base.propertynames(fc::FeatureCollection)
-    # get the propertynames from the first feature, if it exists
-    return if isempty(fc)
-        (:geometry,)
-    else
-        f = first(fc)
-        propertynames(f)
-    end
-end
-
+Base.propertynames(fc::FeatureCollection) = getfield(fc, :names)
 Base.getproperty(fc::FeatureCollection, nm::Symbol) = getproperty.(fc, nm)
 
 Base.IteratorSize(::Type{<:FeatureCollection}) = Base.HasLength()
@@ -109,31 +137,38 @@ Base.IteratorEltype(::Type{<:FeatureCollection}) = Base.HasEltype()
 # read only AbstractVector
 Base.size(fc::FeatureCollection) = size(features(fc))
 Base.eltype(::FeatureCollection{T}) where {T<:Feature} = T
-Base.getindex(fc::FeatureCollection{T}, i) where {T<:Feature} = T(features(fc)[i])
 Base.IndexStyle(::Type{<:FeatureCollection}) = IndexLinear()
+function Base.getindex(fc::FeatureCollection{T}, i) where {T<:Feature}
+    f = features(fc)[i]
+    if f isa Feature
+        return f
+    else
+        return T(f, names(fc))
+    end
+end
 
 function Base.iterate(fc::FeatureCollection{T}) where {T<:Feature}
     x = iterate(features(fc))
     x === nothing && return nothing
     val, state = x
-    return T(val), state
+    return T(val, names(fc)), state
 end
 
 function Base.iterate(fc::FeatureCollection{T}, state) where {T<:Feature}
     x = iterate(features(fc), state)
     x === nothing && return nothing
     val, state = x
-    return T(val), state
+    return T(val, names(fc)), state
 end
 
-Base.show(io::IO, fc::FeatureCollection) =
+Base.show(io::IO, ::MIME"text/plain", fc::FeatureCollection) =
     println(io, "FeatureCollection with $(length(fc)) Features")
 
-function Base.show(io::IO, f::Feature)
+function Base.show(io::IO, ::MIME"text/plain", f::Feature)
     geom = geometry(f)
     propnames = propertynames(f)
     n = length(propnames)
-    if geom === nothing
+    if ismissing(geom) || isnothing(geom)
         print(io, "Feature with null geometry")
     else
         print(io, "Feature with a ", type(geom))
@@ -145,6 +180,8 @@ end
 Tables.istable(::Type{<:FeatureCollection}) = true
 Tables.rowaccess(::Type{<:FeatureCollection}) = true
 Tables.rows(fc::FeatureCollection) = fc
+Tables.schema(fc::FeatureCollection) =
+    Tables.Schema(getfield(fc, :names), [getfield(fc, :types)[nm] for nm in getfield(fc, :names)])
 
 # methods that apply to all GeoJSON Objects
 const GeoJSONObject = Union{Geometry,Feature,FeatureCollection}
@@ -164,3 +201,57 @@ type(x) = String(x.type)
 bbox(x::GeoJSONObject) = get(object(x), :bbox, nothing)
 
 Base.show(io::IO, ::MIME"text/plain", x::GeoJSONObject) = show(io, x)
+
+# Adapted from JSONTables.jl jsontable method
+# We cannot simply use their method as we need the key/value pairs
+# of the properties field, rather than the main object
+function property_schema(features)
+    # Short cut for concrete eltypes of NamedTuple
+    if features isa AbstractArray{<:NamedTuple} && isconcretetype(eltype(features))
+        f1 = first(features)
+        props = properties(f1)
+        names = [keys(props)...]
+        types = Dict{Symbol,Type}(map((k, v) -> k => typeof(v), keys(props), props)...)
+        return names, types
+    end
+    # Otherwise find the shared names
+    names = Symbol[]
+    seen = Set{Symbol}()
+    types = Dict{Symbol, Type}()
+    for feature in features
+        props = properties(feature)
+        isnothing(props) && continue
+        if isempty(names)
+            for k in propertynames(props)
+                k === :geometry && continue
+                push!(names, k)
+                types[k] = missT(typeof(props[k]))
+            end
+            seen = Set(names)
+        else
+            for nm in names
+                if hasproperty(props, nm)
+                    T = types[nm]
+                    v = props[nm]
+                    if !(missT(typeof(v)) <: T)
+                        types[nm] = Union{T, missT(typeof(v))}
+                    end
+                else
+                    types[nm] = Union{Missing, types[nm]}
+                end
+            end
+            for (k, v) in pairs(props)
+                k === :geometry && continue
+                if !(k in seen)
+                    push!(seen, k)
+                    push!(names, k)
+                    types[k] = Union{Missing, missT(typeof(v))}
+                end
+            end
+        end
+    end
+    return names, types
+end
+
+missT(::Type{Nothing}) = Missing
+missT(::Type{T}) where {T} = T

src/json.jl

@@ -91,7 +91,7 @@ end
 Convert a GeoJSON geometry from JSON style object to a struct specific
 to that geometry type.
 """
-function geometry(g)
+function geometry(g::JSON3.Object)
     t = type(g)
     if t == "Point"
         Point(g)

test/geojson_samples.jl

@@ -348,7 +348,7 @@ table_not_present = """{
     "type":"FeatureCollection",
     "features":[
         {"type":"Feature","properties":{"a":1,"b":null,"c":"only-here"},"geometry":null},
-        {"type":"Feature","properties":{"a":null,"b":null},"geometry":null},
+        {"type":"Feature","properties":{"a":null,"b":null,"d":"appears-later"},"geometry":null},
         {"type":"Feature","properties":{"a":3,"b":null},"geometry":null}
 ]}"""
 

test/runtests.jl

@@ -7,6 +7,7 @@ using JSON3
 using Tables
 using Test
 using Plots
+using DataFrames
 
 # samples and collections thereof defined under module T
 include("geojson_samples.jl")
@@ -113,18 +114,22 @@ include("geojson_samples.jl")
         # FeatureCollection
         features = [f]
         fc = GeoJSON.FeatureCollection(features)
-        @test GeoJSON.features(fc) === features
-        @test propertynames(fc) === Tables.columnnames(fc) === (:geometry, :a, :b)
+        @test GeoJSON.features(fc) == features
+        @test propertynames(fc) == Tables.columnnames(fc) == [:geometry, :a, :b]
         @test GeoJSON.geometry.(fc) == [p]
         @test iterate(p) === (1.1, 2)
         @test iterate(p, 2) === (2.2, 3)
         @test iterate(p, 3) === nothing
 
         # other constructors
-        GeoJSON.Feature(geometry = p, properties = (a = 1, geometry = "g", b = 2))
-        GeoJSON.Feature((geometry = p, properties = (a = 1, geometry = "g", b = 2)))
-        GeoJSON.FeatureCollection(; features)
-        GeoJSON.FeatureCollection((type = "FeatureCollection", features = [f]))
+        @test DataFrame([GeoJSON.Feature(geometry = p, properties = (a = 1, geometry = "g", b = 2))]) ==
+              DataFrame([GeoJSON.Feature((geometry = p, properties = (a = 1, geometry = "g", b = 2)))]) ==
+              DataFrame(GeoJSON.FeatureCollection((type="FeatureCollection", features=[f]))) ==
+              DataFrame(GeoJSON.FeatureCollection(; features))
+
+        # Mixed name vector
+        f2 = GeoJSON.Feature(p; properties = (a = 1, geometry = "g", b = 2, c = 3))
+        GeoJSON.FeatureCollection((type = "FeatureCollection", features = [f, f2]))
     end
 
     @testset "extent" begin
@@ -178,7 +183,7 @@ include("geojson_samples.jl")
             <:JSON3.Object,
             <:JSON3.Array,
         }
-        @test propertynames(t) == (:geometry, :park, :cartodb_id, :addr1, :addr2)
+        @test propertynames(t) == [:geometry, :cartodb_id, :addr1, :addr2, :park]
         @test Tables.rowtable(t) isa Vector{<:NamedTuple}
         @test Tables.columntable(t) isa NamedTuple
         @inferred first(t)
@@ -204,6 +209,16 @@ include("geojson_samples.jl")
         @test geom[1][1][3] == [-117.912919, 33.96445]
         @test geom[1][1][4] == [-117.913883, 33.96657]
 
+        @testset "With NamedTuple feature" begin
+            nt_feature = GeoJSON.Feature(; 
+                geometry=t[1].geometry, 
+                properties=(cartodb_id=t[1].cartodb_id, addr1=t[1].addr1, addr2=t[1].addr2, park=t[1].park),
+            )
+            fc = GeoJSON.FeatureCollection([nt_feature])
+            @test fc isa GeoJSON.FeatureCollection
+            @test occursin("(:geometry, :cartodb_id, :addr1, :addr2, :park)", sprint(show, MIME"text/plain"(), fc[1]))
+        end
+
         @testset "write to disk" begin
             fc = t
             GeoJSON.write("test.json", fc)
@@ -232,6 +247,9 @@ include("geojson_samples.jl")
     @testset "Tables with missings" begin
         t = GeoJSON.read(T.tablenull)
         @test t[1] isa GeoJSON.Feature
+        @test occursin("(:geometry, :a, :b)", sprint(show, MIME"text/plain"(), t[1]))
+        @test ismissing(t[1].geometry)
+        GeoJSON.geometry(t[1])
         @test t.geometry isa Vector{Union{T,Missing}} where {T<:GeoJSON.Point}
         @test ismissing(t.geometry[3])
         @test t.a isa Vector{Union{Float64,Missing}}
@@ -240,15 +258,26 @@ include("geojson_samples.jl")
         @test Tables.columntable(t) isa NamedTuple
 
         t = GeoJSON.read(T.table_not_present)
-        @test propertynames(t) === propertynames(t[1]) === (:geometry, :a, :b, :c)
-        @test propertynames(t[2]) === (:geometry, :a, :b)
-        # "c" is only present in the properyies of the first row
-        # We don't support automatically setting these to missing in the tables interface.
-        # They have to be explicitly set to null.
-        # We could support it by having getproperty(f::Feature, :not_present) return missing
-        # if needed, but then you always get missing instead of KeyError.
-        @test_throws KeyError t.c
-        @test_throws KeyError Tables.columntable(t)
+        @test occursin("(:geometry, :a, :b, :c)", sprint(show, MIME"text/plain"(), t[1]))
+        @test propertynames(t) == [:geometry, :a, :b, :c, :d]
+        @test propertynames(t[1]) == (:geometry, :a, :b, :c)
+        @test propertynames(t[2]) == (:geometry, :a, :b, :d)
+        # "c" and "d" are only present in the properties of a single row
+        @test all(t.c .=== ["only-here", missing, missing])
+        @test all(t.d .=== [missing, "appears-later", missing])
+        @testset "With NamedTuple feature" begin
+            nt_props = [
+                (a=t[1].a, b=t[1].b, c=t[1].c),
+                (a=t[2].a, b=t[2].b, d=t[2].d),
+                (a=t[3].a, b=t[3].b),
+            ]
+            features = map(t.geometry, nt_props) do geometry, properties
+                GeoJSON.Feature(; geometry, properties)
+            end
+            fc =  GeoJSON.FeatureCollection(features)
+            @test fc isa GeoJSON.FeatureCollection
+            @test occursin("(:geometry, :a, :b, :c)", sprint(show, MIME"text/plain"(), fc[1]))
+        end
     end
 
     @testset "FeatureCollection of one GeometryCollection" begin