From 6aae933e5f0130e2f4a2eb113566a2c63412082d Mon Sep 17 00:00:00 2001 From: jph6366 Date: Mon, 4 Aug 2025 09:07:37 -0400 Subject: [PATCH 01/90] initial commit --- src/extra/gpkg.jl | 1117 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 1117 insertions(+) create mode 100644 src/extra/gpkg.jl diff --git a/src/extra/gpkg.jl b/src/extra/gpkg.jl new file mode 100644 index 0000000..d9ec809 --- /dev/null +++ b/src/extra/gpkg.jl @@ -0,0 +1,1117 @@ + +# List of well known binary geometry types. +# These are used within the GeoPackageBinary SQL BLOBs +@enum wkbGeometryType::Int64 begin + wkbUnknown = 0 + wkbPoint = 1 + wkbLineString = 2 + wkbPolygon = 3 + wkbMultiPoint = 4 + wkbMultiLineString = 5 + wkbMultiPolygon = 6 + wkbGeometryCollection = 7 + wkbCircularString = 8 + wkbCompoundCurve = 9 + wkbCurvePolygon = 10 + wkbMultiCurve = 11 + wkbMultiSurface = 12 + wkbCurve = 13 + wkbSurface = 14 + + wkbNone = 100 # pure attribute records + wkbLinearRing = 101 + + # ISO SQL/MM Part 3: Spatial + # Z-aware types + wkbPointZ = 1001 + wkbLineStringZ = 1002 + wkbPolygonZ = 1003 + wkbMultiPointZ = 1004 + wkbMultiLineStringZ = 1005 + wkbMultiPolygonZ = 1006 + wkbGeometryCollectionZ = 1007 + wkbCircularStringZ = 1008 + wkbCompoundCurveZ = 1009 + wkbCurvePolygonZ = 1010 + wkbMultiCurveZ = 1011 + wkbMultiSurfaceZ = 1012 + wkbCurveZ = 1013 + wkbSurfaceZ = 1014 + + # ISO SQL/MM Part 3. + # M-aware types + wkbPointM = 2001 + wkbLineStringM = 2002 + wkbPolygonM = 2003 + wkbMultiPointM = 2004 + wkbMultiLineStringM = 2005 + wkbMultiPolygonM = 2006 + wkbGeometryCollectionM = 2007 + wkbCircularStringM = 2008 + wkbCompoundCurveM = 2009 + wkbCurvePolygonM = 2010 + wkbMultiCurveM = 2011 + wkbMultiSurfaceM = 2012 + wkbCurveM = 2013 + wkbSurfaceM = 2014 + + # ISO SQL/MM Part 3. + # ZM-aware types ... Meshes.jl doesn't generally support this? + wkbPointZM = 3001 + wkbLineStringZM = 3002 + wkbPolygonZM = 3003 + wkbMultiPointZM = 3004 + wkbMultiLineStringZM = 3005 + wkbMultiPolygonZM = 3006 + wkbGeometryCollectionZM = 3007 + wkbCircularStringZM = 3008 + wkbCompoundCurveZM = 3009 + wkbCurvePolygonZM = 3010 + wkbMultiCurveZM = 3011 + wkbMultiSurfaceZM = 3012 + wkbCurveZM = 3013 + wkbSurfaceZM = 3014 + + # 2.5D extension as per 99-402 + # https://lists.osgeo.org/pipermail/postgis-devel/2004-December/000702.html + wkbPoint25D = 0x80000001 + wkbLineString25D = 0x80000002 + wkbPolygon25D = 0x80000003 + wkbMultiPoint25D = 0x80000004 + wkbMultiLineString25D = 0x80000005 + wkbMultiPolygon25D = 0x80000006 + wkbGeometryCollection25D = 0x80000007 +end + +const wkbXDR = 0 +const wkbNDR = 1 + +const wkb25DBit = 0x80000000 + +macro BSWAP(x) + if ENDIAN_BOM == 0x01020304 + return :($(esc(x)) == wkbNDR) + else + return :($(esc(x)) == wkbXDR) + end +end + + +# Requirement 1: first 16 bytes is null-terminated ASCII string "SQLite format 3" +const MAGIC_HEADER_STRING::Vector{UInt8} = UInt8[ + 0x53, 0x51, 0x4c, 0x69, 0x74, 0x65, 0x20, 0x66, + 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x20, 0x33, 0x00 +] + +# 'GP' in ASCII +const MAGIC_GPKG_BINARY_STRING = UInt8[ 0x47, 0x50 ] + +# Requirement 2: contains "GPKG" in ASCII in "application_id" field of SQLite db header +# Reminder: We have to set this (on-write) after there's some content +# so the database file is not zero length + +const GP10_APPLICATION_ID = 74777363 #0x47503130 +const GP11_APPLICATION_ID = 119643780 # 0x47503131 +const GPKG_APPLICATION_ID = 1196444487 # 0x47504B47 +const GPKG_1_2_VERSION = 10200 +const GPKG_1_3_VERSION = 10300 +const GPKG_1_4_VERSION = 10400 + + +const CREATE_GPKG_GEOMETRY_COLUMNS = + "CREATE TABLE gpkg_geometry_columns ("* + "table_name TEXT NOT NULL,"* + "column_name TEXT NOT NULL,"* + "geometry_type_name TEXT NOT NULL,"* + "srs_id INTEGER NOT NULL,"* + "z TINYINT NOT NULL,"* + "m TINYINT NOT NULL,"* + "CONSTRAINT pk_geom_cols PRIMARY KEY (table_name, column_name),"* + "CONSTRAINT uk_gc_table_name UNIQUE (table_name),"* + "CONSTRAINT fk_gc_tn FOREIGN KEY (table_name) REFERENCES "* + "gpkg_contents(table_name),"* + "CONSTRAINT fk_gc_srs FOREIGN KEY (srs_id) REFERENCES gpkg_spatial_ref_sys "* + "(srs_id)"* + ")" + +const CREATE_GPKG_REQUIRED_METADATA = "CREATE TABLE gpkg_spatial_ref_sys ("* + "srs_name TEXT NOT NULL,"* + "srs_id INTEGER NOT NULL PRIMARY KEY,"* + "organization TEXT NOT NULL,"* + "organization_coordsys_id INTEGER NOT NULL,"* + "definition TEXT NOT NULL,"* + "description TEXT,"* + "definition_12_063 TEXT NOT NULL,"* + "epoch DOUBLE"* + ")"* + ";"* + "INSERT INTO gpkg_spatial_ref_sys ("* + "srs_name, srs_id, organization, organization_coordsys_id, "* + "definition, description, definition_12_063"* + ") VALUE ("* + + "'WGS 84 geodetic', 4326, 'EPSG', 4326, '"* + "GEOGCS[\"WGS 84\",DATUM[\"WGS_1984\",SPHEROID[\"WGS "* + "84\",6378137,298.257223563,AUTHORITY[\"EPSG\",\"7030\"]],"* + "AUTHORITY[\"EPSG\",\"6326\"]],PRIMEM[\"Greenwich\",0,AUTHORITY["* + "\"EPSG\",\"8901\"]],UNIT[\"degree\",0.0174532925199433,AUTHORITY["* + "\"EPSG\",\"9122\"]],AXIS[\"Latitude\",NORTH],AXIS[\"Longitude\","* + "EAST],AUTHORITY[\"EPSG\",\"4326\"]]"* + "', 'longitude/latitude coordinates in decimal degrees on the WGS "* + "84 spheroid', "* + + "'GEODCRS[\"WGS 84\", DATUM[\"World Geodetic System 1984\", "* + "ELLIPSOID[\"WGS 84\",6378137, 298.257223563, "* + "LENGTHUNIT[\"metre\", 1.0]]], PRIMEM[\"Greenwich\", 0.0, "* + "ANGLEUNIT[\"degree\",0.0174532925199433]], CS[ellipsoidal, "* + "2], AXIS[\"latitude\", north, ORDER[1]], AXIS[\"longitude\", "* + "east, ORDER[2]], ANGLEUNIT[\"degree\", 0.0174532925199433], "* + "ID[\"EPSG\", 4326]]'"* + ")"* + ";"* + "INSERT INTO gpkg_spatial_ref_sys ("* + "srs_name, srs_id, organization, organization_coordsys_id, "* + " definition, description, definition_12_063"* + ") VALUES ("* + "'Undefined Cartesian SRS', -1, 'NONE', -1, 'undefined', "* + "'undefined Cartesian coordinate reference system', 'undefined'"* + ")"* + ";"* + "INSERT INTO gpkg_spatial_ref_sys ("* + "srs_name, srs_id, organization, organization_coordsys_id, "* + "definition, description, definition_12_063"* + ") VALUES ("* + "'Undefined geographic SRS' 0, 'NONE', 0, 'undefined', "* + "'undefined geographic coordinate reference system', 'undefined'"* + ")"* + ";"* + "CREATE TABLE gpkg_contents ("* + "table_name TEXT NOT NULL PRIMARY KEY,"* + "data_type TEXT NOT NULL,"* + "identifier TEXT UNIQUE,"* + "description TEXT DEFAULT '',"* + "last_change DATETIME NOT NULL DEFAULT "* + "(strftime('%Y-%m-%dT%H:%M:%fZ','now')),"* + "min_x DOUBLE, min_y DOUBLE,"* + "max_x DOUBLE, max_y DOUBLE,"* + "srs_id INTEGER,"* + "CONSTRAINT fk_gc_r_srs_id FOREIGN KEY (srs_id) REFERENCES "* + "gpkg_spatial_ref_sys(srs_id)"* + ")"* + ";" + +function gpkgread(fname; kwargs... ) + # Requirement 3: File name has to end in ".gpkg" + if !(endswith(fname,".gpkg")) @error "the file extension is not .gpkg'" end + db_header_string = open(fname, "r") do io + read(io, 16) + end + if(db_header_string != MAGIC_HEADER_STRING) + @error "missing magic header string" + end + + db = SQLite.DB(fname) + + @timeit to "identify geopackage" begin + gpkg_identify(db) + end + show(to) + + local mesh_geometry + + @timeit to "read geopackage" begin + gpkg_attrs, mesh_geometry, geom_attrs = read_gpkg_tables(db) + end + show(to) + SQLite.close(db) + + return GeoTables.georef(geom_attrs, mesh_geometry) + # return GeoTables.georef(gpkg_attrs, [Point(NaN, NaN)]) + +end + +function gpkgwrite(fname, geotable; ) + + + db = SQLite.DB(fname) + + @timeit to "write geopackage" begin + + # DBInterface.execute(db, "PRAGMA foreign_keys = ON") + + #DBInterface.execute(db, "PRAGMA journal_mode=WAL;") + # Write transactions are very fast since they only involve writing the content once + # (versus twice for rollback-journal transactions) and because the writes are all sequential + + DBInterface.execute(db, "PRAGMA synchronous=0") + # Commits can be orders of magnitude faster with + # Setting PRAGMA synchronous=OFF can cause the database to go corrupt + # if there is an operating-system crash or power failure, + # though this setting is safe from damage due to application crashes. + + SQLite.transaction(db) do + +# ------------------------- +# REQUIRED METADATA TABLES +# ------------------------- + + # Requirement 11: The gpkg_spatial_ref_sys table SHALL contain at a minimum: + # 1. The record with an srs_id of 4326 SHALL correspond to WGS-84 as defined by EPSG in 4326. + # OR + # 2. The record with an srs_id of -1 SHALL be used for undefined Cartesian coordinate reference systems. + # OR + # 3. The record with an srs_id of 0 SHALL be used for undefined geographic coordinate reference systems. + + + +# ------------------------- +# OPTIONAL METADATA TABLES +# ------------------------- + stmt_sql = SQLite.Stmt(db, CREATE_GPKG_REQUIRED_METADATA*CREATE_GPKG_GEOMETRY_COLUMNS ) + DBInterface.execute(stmt_sql) + + + stmt_sql = SQLite.Stmt(db, "PRAGMA application_id = $GPKG_APPLICATION_ID ") + DBInterface.execute(stmt_sql) + stmt_sql = SQLite.Stmt(db, "PRAGMA user_version = $GPKG_1_4_VERSION ") + DBInterface.execute(stmt_sql) + + end + + to_gpkg(db, geotable) + end + show(to) + + SQLite.close(db) + + return +end + + +function to_gpkg(db, gt::GeoTable) + table = values(gt) + domain = GeoTables.domain(gt) + crs = GeoTables.crs(domain) + geometry = collect(domain) + + if crs <: Cartesian + crs = -1 + elseif crs <: LatLon{WGS84Latest} + crs = Int32(4326) + end + + println("crs: ", crs) + gpkgbin_blobs = Vector{Vector{UInt8}}() + + for geom::Geometry in geometry + gpkgbin_header = _gpkg_update_header(crs, geom) + io = IOBuffer() + _import_to_wkb(io, geom) + wkb_blob = take!(io) + push!(gpkgbin_blobs, vcat(gpkgbin_header, wkb_blob)) + end + + table = merge(table, (geom=gpkgbin_blobs,)) + SQLite.load!(table, db, replace=false) # autogenerates table name + # replace=false controls whether + # an INSERT INTO ... statement is generated or a REPLACE INTO .... + println(table |> DataFrame) + tn = DBInterface.execute(db, "SELECT name FROM sqlite_master WHERE type='table'") + nm = "" + for row in tn + nm = row.name + end + + bbox = boundingbox(domain) + min_coords = CoordRefSystems.raw(coords(bbox.min)) + max_coords = CoordRefSystems.raw(coords(bbox.max)) + + contents_table = [(table_name = nm, + data_type = "features", + identifier = nm, + min_x = min_coords[1], + min_y = min_coords[2], + max_x = max_coords[1], + max_y = max_coords[2], + srs_id = crs + )] + + println(contents_table |> DataFrame) + SQLite.load!(contents_table, db, "gpkg_contents", replace=true) + + geometry_columns_table = [(table_name = nm, + column_name = "geom", + geometry_type_name = nm, + srs_id = crs, + z = paramdim(first(geometry)) >= 2 ? 1 : 0, + m = 0 + )] + + println(geometry_columns_table |> DataFrame) + SQLite.load!(geometry_columns_table, db, "gpkg_geometry_columns", replace=true) + + +end + +function gpkg_gtype(geoms::AbstractVector{<:Geometry}) + + T = eltype(geoms) + + if T <: Point + return "POINT" + elseif T <: Rope + return "LINESTRING" + elseif T <: Ring + return "LINESTRING" + elseif T <: PolyArea + return "POLYGON" + elseif T <: Multi + + element_type = eltype(parent(first(geoms))) + return "MULTI"*gpkg_gtype(element_type) + else + return "GEOMETRY" + end + +end + +function _gpkg_update_header(srs_id, geometry, envelope::Int=1) + + io = IOBuffer() + write(io, MAGIC_GPKG_BINARY_STRING) # 'GP' in ASCII + write(io, UInt8(0)) # 0 = version 1 + + flagsbyte = UInt8(0x20 | (envelope << 1)) + write(io, flagsbyte) + + bswap = @BSWAP (flagsbyte & 0x01) + + bswap ? write(io, Base.bswap(srs_id)) : write(io, srs_id) + + if isone(envelope) + bbox = boundingbox(geometry) + if bswap + write(io, Base.bswap(Float64(CoordRefSystems.raw(coords(bbox.min))[1]))) + write(io, Base.bswap(Float64(CoordRefSystems.raw(coords(bbox.max))[1]))) + write(io, Base.bswap(Float64(CoordRefSystems.raw(coords(bbox.min))[2]))) + write(io, Base.bswap(Float64(CoordRefSystems.raw(coords(bbox.max))[2]))) + else + write(io, Float64(CoordRefSystems.raw(coords(bbox.min))[1])) + write(io, Float64(CoordRefSystems.raw(coords(bbox.max))[1])) + write(io, Float64(CoordRefSystems.raw(coords(bbox.min))[2])) + write(io, Float64(CoordRefSystems.raw(coords(bbox.max))[2])) + end + end + + if isone(envelope-1) && paramdim(geometry) >= 3 + bbox = boundingbox(geometry) + if bswap + write(io, Base.bswap(Float64(CoordRefSystems.raw(coords(bbox.min))[3]))) + write(io, Base.bswap(Float64(CoordRefSystems.raw(coords(bbox.max))[3]))) + else + write(io, Float64(CoordRefSystems.raw(coords(bbox.min))[3])) + write(io, Float64(CoordRefSystems.raw(coords(bbox.max))[3])) + end + end + + return take!(io) + +end + + +function wkb_set_z(type::wkbGeometryType) + return wkbGeometryType(Int(type)+1000) + # ISO WKB simply adds a round number to the type number to indicate extra dimensions +end + +function wkb_has_z(type::wkbGeometryType) + return Int(type) > 1000 +end + +function wkbtype(geometry) + if geometry isa Point + return wkbPoint + elseif geometry isa Rope || geometry isa Ring + return wkbLineString + elseif geometry isa PolyArea + return wkbPolygon + elseif geometry isa Multi + fg = parent(geometry) |> first + return wkbGeometryType(Int(wkbtype(fg))+3) + else + @error "my hovercraft is full of eels: $geometry" + end +end + +function _import_to_wkb(io, geometry) + gtype = isone(paramdim(geometry)-1) ? wkb_set_z(wkbtype(geometry)) : wkbtype(geometry) + create_wkb_geometry(io, geometry, gtype) +end + +function create_wkb_geometry(io, geometry, gtype) + if Int(gtype) > 3 + write(io, UInt8(0x01)) + write(io, UInt32(gtype)) + write(io, UInt32(length(parent(geometry))) ) + + for geom in parent(geometry) + create_wkb_geometry(io, geom, wkbtype(geom)) + end + else + import_wkb_geometry(io, geometry, gtype) + end +end + +function import_wkb_geometry(io, geometry, wkb_type) + + write(io, UInt8(0x01)) + + bswap = @BSWAP(one(0x01)) + + if wkb_type == wkbPolygon || wkb_type == wkbPolygonZ + + _to_wkb_polygon(io, wkb_type, [boundary(geometry::PolyArea)]) + + elseif wkb_type == wkbLineString || wkb_type == wkbLineStringZ + + coordinate_list = vertices(geometry) + bswap ? Base.bswap(write(io, UInt32(wkb_type))) : write(io, UInt32(wkb_type)) + + if geometry isa Ring + bswap ? Base.bswap(write(io, UInt32(length(coordinate_list)+1))) : write(io, UInt32(length(coordinate_list)+1)) + else + bswap ? Base.bswap(write(io, UInt32(length(coordinate_list)))) : write(io, UInt32(length(coordinate_list))) + end + + _to_wkb_linestring(io, wkb_type, coordinate_list) + + if geometry isa Ring + coordinates = CoordRefSystems.raw(coords( coordinate_list |> first )) + _to_wkb_coordinates(io, wkb_type, coordinates) + end + + elseif wkb_type == wkbPoint || wkb_type == wkbPointZ + + coordinates = CoordRefSystems.raw(coords(geometry::Point)) + _to_wkb_coordinates(io, wkb_type, coordinates) + + else + @error "What is the $wkb_type ? not recognized; simple features only" + end +end + +function _to_wkb_coordinates(io, wkb_type, coords) + + + bswap = @BSWAP(one(0x01)) + + bswap ? Base.bswap(write(io, UInt32(wkb_type))) : write(io, UInt32(wkb_type)) + bswap ? Base.bswap(write(io, Float64(coords[1]))) : write(io, Float64(coords[1])) + bswap ? Base.bswap(write(io, Float64(coords[2]))) : write(io, Float64(coords[2])) + + if wkb_has_z(wkb_type) + write(io, Float64(coords[3])) + end +end + +function _to_wkb_linestring(io, wkb_type, coord_list) + for n_coords::Point in coord_list + coordinates = CoordRefSystems.raw(coords(n_coords)) + _to_wkb_coordinates(io, wkb_type, coordinates) + end +end + +function _to_wkb_polygon(io, wkb_type, rings) + write(io, UInt32(wkb_type)) + write(io, UInt32(length(rings))) + + for ring in rings + coord_list = vertices(ring) + write(io, UInt32(length(coord_list) + 1)) + _to_wkb_linestring(io, wkb_type, coord_list) + end +end + +# ----------------- +# HELPER FUNCTIONS +# ----------------- + +const to = TimerOutput() + + + + + +# Requirement 5: columns of tables are only declared using one of the GeoPackage data types +# Extended GeoPackages contain additional data types +# +# Requirement 5 Warning +# ⚠ data type mismatches could theoretically be checked for but tests would scale poorly ⚠ +# +# GeoPackage writers SHOULD validate the data as it is being inserted +# GeoPackage readers SHOULD allow for the possibility that unexpected values are present. + +function _julia_sqlite_datatype(gpkg_type::AbstractString, kwargs...) + if startswith(gpkg_type, "INT") + if(gpkg_type != "INT" && gpkg_type != "INTEGER") + @warn "field format $gpkg_type not supported (interpreted as int)" + end + return Int64 + elseif gpkg_type == "BOOLEAN" + return Bool + elseif gpkg_type == "TINYINT" + return Int8 + elseif gpkg_type == "SMALLINT" + return Int16 + elseif gpkg_type == "MEDIUMINT" + return Int32 + elseif gpkg_type == "FLOAT" + return Float32 + elseif gpkg_type == "DOUBLE" || gpkg_type == "REAL" + return Float64 + elseif gpkg_type == "TEXT" + return String # UTF-8 or UTF-16, determined by PRAGMA encoding + elseif gpkg_type == "BLOB" + return Any + elseif startswith(gpkg_type, "DATE") + return Any + else + @warn "field format $gpkg_type not recognized, okay have a nice day" + return Any + end +end + +# Requirement 20: GeoPackage SHALL store feature table geometries with the basic simple feature geometry types +# https://www.geopackage.org/spec140/index.html#geometry_types +function meshes_creategeometry(wkb_type, crs, C) + if wkb_type == wkbPoint || wkb_type == wkbPointZ + + return Meshes.Point(crs(C...)) + + elseif wkb_type == wkbLineString || wkb_type == wkbLineStringZ + + return (C[1] == C[end]) ? + Meshes.Ring([Meshes.Point(crs(coords...)) for coords in C[2:end]]...) : Meshes.Rope([Meshes.Point(coords...) for coords in C]...) + + elseif wkb_type == wkbPolygon || wkb_type == wkbPolygonZ + + rings = map(C) do ring_coords + coords = map(ring_coords) do svec + Meshes.Point(crs(svec...)) + end + Meshes.Ring(coords) + end + + outer_ring = first(rings) + holes = length(rings) > 1 ? rings[2:end] : Meshes.Ring[] + return Meshes.PolyArea(outer_ring, holes...) + + else + @error "what $wkb_type is this" + end # @TODO: add support for non-linear geometry types +end + +function gpkg_identify(db)::Bool + + application_id = DBInterface.execute(db, "PRAGMA application_id;") |> first |> only + user_version = DBInterface.execute(db, "PRAGMA user_version;") |> first |> only + + if !(_has_gpkg_required_metadata_tables(db)) + @error "missing required metadata tables" + end + + # Requirement 6: PRAGMA integrity_check returns a single row with the value 'ok' + # Requirement 7: PRAGMA foreign_key_check (w/ no parameter value) returns an empty result set + if (application_id != GP10_APPLICATION_ID) && + (application_id != GP11_APPLICATION_ID) && + (application_id != GPKG_APPLICATION_ID) + @warn "application_id not recognized" + return false + elseif(application_id == GPKG_APPLICATION_ID) && + !( + (user_version >= GPKG_1_2_VERSION && user_version < GPKG_1_2_VERSION + 99) || + (user_version >= GPKG_1_3_VERSION && user_version < GPKG_1_3_VERSION + 99) || + (user_version >= GPKG_1_4_VERSION && user_version < GPKG_1_4_VERSION + 99) + ) + @warn "application_id is valid but user version is not recognized" + elseif( + DBInterface.execute(db, "PRAGMA integrity_check;") |> first |> only != "ok" + ) || !( + isempty(DBInterface.execute(db, "PRAGMA foreign_key_check;")) + ) + @error "database integrity at risk or foreign key violation(s)" + return false + end + return true +end + +# Requirement 10: must include a gpkg_spatial_ref_sys table +# Requirement 13: must include a gpkg_contents table +function _has_gpkg_required_metadata_tables(db) + stmt_sql = "SELECT COUNT(*) FROM sqlite_master WHERE "* + "name IN ('gpkg_spatial_ref_sys', 'gpkg_contents') AND "* + "type IN ('table', 'view');" + required_metadata_tables = DBInterface.execute(db, stmt_sql) |> first |> only + return (required_metadata_tables == 2) +end + +function _gpkg_crs_wkt_extension(db) + stmt_sql = "SELECT extension_name FROM gpkg_extensions;" + ext_name = DBInterface.execute(db, stmt_sql) |> first |> only + return ext_name == "gpkg_crs_wkt" +end + +function read_gpkg_tables(db) + gpkg_table_info = SQLite.tableinfo(db, "gpkg_spatial_ref_sys") + # @TODO: gpkg_contents tableinfo + has_gpkg_extensions = _has_gpkg_optional_metadata_table(db) + if (has_gpkg_extensions) + end + + + has_gpkg_attributes = _has_gpkg_attributes(db) + has_vector_features = _has_gpkg_geometry_columns(db) + if has_vector_features + println("vector") + gpkg_table_info = SQLite.tableinfo(db, "gpkg_geometry_columns") + + geometry = get_feature_table_geometry_columns(db) + + vector_features = get_feature_tables(db) + feature_tables = get_feature_attributes(db, vector_features) + + if has_gpkg_attributes + + println("attributes") + #gpkg_attributes = get_gpkg_attributes(db) + + #return gpkg_attributes, geometry, feature_tables + end + return nothing, geometry, feature_tables + + elseif has_gpkg_attributes + println("attributes") + gpkg_attributes = get_gpkg_attributes(db) + return gpkg_attributes, nothing, nothing + else + @error "data_type not supported yet, looks for 'features' by default and falls back to 'attributes' " + end +end + + +function _has_gpkg_geometry_columns(db) + stmt_sql = "SELECT 1 from sqlite_master WHERE "* + "name = 'gpkg_geometry_columns' AND "* + "type IN ('table', 'view');" + geometry_columns = DBInterface.execute(db, stmt_sql) |> collect + return !isempty(geometry_columns) +end + +# Requirement 58: optionally includes a gpkg_extensions table +function _has_gpkg_optional_metadata_table(db) + stmt_sql= "SELECT 1 FROM sqlite_master WHERE name = 'gpkg_extensions' "* + "AND type IN ('table', 'view')" + extensions = DBInterface.execute(db, stmt_sql) |> collect + return !isempty(extensions) +end + + + +# Requirement 118: gpkg_contents table SHALL contain +# a row with a data_type column value of "attributes" +# for each attributes data table or view. + +function _has_gpkg_attributes(db) + stmt_sql = "SELECT COUNT(*) FROM sqlite_master WHERE " * + "name = 'gpkg_contents' AND " * + "type IN ('table', 'view');" + has_gpkg_contents_table = DBInterface.execute(db, stmt_sql) |> first |> only + + if has_gpkg_contents_table == 1 + stmt_sql_datatype = "SELECT COUNT(*) FROM gpkg_contents WHERE " * + "data_type = 'attributes';" + has_attributes_datatype = DBInterface.execute(db, stmt_sql_datatype) |> first |> only + return has_attributes_datatype > 0 + else + return false + end +end + +# +# +# Requirement 119 & 151: GeoPackage MAY contain tables and views +# containing attributes and attribute sets +function get_gpkg_attributes(db) + stmt_sql = "SELECT table_name as tn FROM gpkg_contents WHERE data_type = 'attributes'" + attributes = DBInterface.execute(db, stmt_sql) + gpkg_attrs = map(attributes) do sqlite_row + tn = sqlite_row.tn + DBInterface.execute(SQLite.Stmt(db, "SELECT * FROM $tn")) |> Tables.rows + end + attr_tables = map(gpkg_attrs) do row + map(Tables.columnnames(row)) do col + Tables.getcolumn(row, col) + end + end + return attr_tables +end + + +# Requirement 21: a gpkg_contents table row with a "features" data_type +# SHALL contain a gpkg_geometry_columns table + +function get_feature_tables(db) + stmt_sql = "SELECT c.table_name, c.identifier, 1 as is_aspatial, "* + "g.column_name, g.geometry_type_name, g.z, g.m, c.min_x, c.min_y, "* + "c.max_x, c.max_y, 1 AS is_in_gpkg_contents, "* + "(SELECT type FROM sqlite_master WHERE lower(name) = "* + "lower(c.table_name) AND type IN ('table', 'view')) AS object_type "* + " FROM gpkg_geometry_columns g "* + " JOIN gpkg_contents c ON (g.table_name = c.table_name)"* + " WHERE "* + " c.data_type = 'features' " + features = DBInterface.execute(db, stmt_sql) + return features +end + +# Third component of the SQL schema for vector features in a GeoPackage +function get_feature_attributes(db, features) + + attrs = map(features) do sqlite_row + _select_aspatial_attributes(db, sqlite_row) + end + + if isempty(attrs) + return NamedTuple() + end + + feature_attrs = reduce( (i, h) -> begin + jk = keys(i) + len_i = isempty(i) ? 0 : length(first(i)) + + kj = keys(h) + len_h = isempty(h) ? 0 : length(first(h)) + + all = unique(vcat(collect(jk), collect(kj))) + + vals = map(all) do k + newcol = Vector{Any}() + + if haskey(i, k) + append!(newcol, i[k]) + else + append!(newcol, fill(nothing, len_i)) + end + + if haskey(h, k) + append!(newcol, h[k]) + else + append!(newcol, fill(nothing, len_h)) + end + + return newcol + end + return NamedTuple{Tuple(all)}(vals) + end , attrs, init=NamedTuple()) + return feature_attrs +end + +function _select_aspatial_attributes(db, row) + tn = row.table_name + table_info = SQLite.tableinfo(db, tn) + ft_attrs = table_info.name + + # delete gpkg_geometry_columns column name, usually 'geom' or 'geometry' + deleteat!(ft_attrs, findall(x -> x == row.column_name, ft_attrs)) + attrs_str = join(ft_attrs, ", ") + stmt_sql = iszero(length(ft_attrs)) ? "" : "SELECT $attrs_str FROM $tn" + attrs_result = iszero(length(stmt_sql)) ? false : DBInterface.execute(db, stmt_sql) |> Tables.columns + columns, names = Tables.columns(attrs_result), Tables.columnnames(attrs_result) + return NamedTuple{Tuple(names)}([ + Tables.getcolumn(columns, nm) for nm in names + ]) + +end + +# Requirement 25: The geometry_type_name value in a gpkg_geometry_columns row +# SHALL be one of the uppercase geometry type names specified + +# Requirement 26: The srs_id value in a gpkg_geometry_columns table row +# SHALL be an srs_id column value from the gpkg_spatial_ref_sys table. +# +# Requirement 27: The z value in a gpkg_geometry_columns table row SHALL be one of 0, 1, or 2. +# Requirement 28: The m value in a gpkg_geometry_columns table row SHALL be one of 0, 1, or 2. +# +# Requirement 146: The srs_id value in a gpkg_geometry_columns table row +# SHALL match the srs_id column value from the corresponding row in the gpkg_contents table. +# +# Requirement 22: gpkg_geometry_columns table +# SHALL contain one row record for the geometry column +# in each vector feature data table +# +#Requirement 23: gpkg_geometry_columns table_name column +# SHALL reference values in the gpkg_contents table_name column +# for rows with a data_type of 'features' + +# Amalgamation SQL execution, collecting matching vector feature tables +# and then parsing their geopackage binary format containg wkb geometries +function get_feature_table_geometry_columns(db) + stmt_sql = "SELECT g.table_name AS tn, g.column_name AS cn, c.srs_id as crs, g.z as elev, " * + "( SELECT type FROM sqlite_master WHERE lower(name) = lower(c.table_name) AND type IN ('table', 'view')) AS object_type " * + "FROM gpkg_geometry_columns g " * + "JOIN gpkg_contents c ON ( g.table_name = c.table_name ) " * + "WHERE c.data_type = 'features' " * + "AND (SELECT type FROM sqlite_master WHERE lower(name) = lower(c.table_name) AND type IN ('table', 'view')) IS NOT NULL "* + #"AND g.srs_id IN (SELECT srs_id FROM gpkg_spatial_ref_sys ) "* + "AND g.srs_id = c.srs_id "* + "AND g.z IN (0, 1, 2) "* + "AND g.m IN (0, 1, 2);" + stmt_sql = SQLite.Stmt(db, stmt_sql) + geoms = DBInterface.execute(stmt_sql) + geom_specs = Set( (row.tn, Symbol(row.cn), row.crs, row.elev) for row in geoms ) + meshes = Geometry[] + for (tn, cn, crs, elev) in geom_specs + # Requirement 24: The column_name column value in a gpkg_geometry_columns row + # SHALL be the name of a column in the table or view specified + # by the table_name column value for that row. + stmt_sql = "SELECT $cn FROM $tn" + gpkg_binary = DBInterface.execute(db, stmt_sql) + for blob in (b for b in gpkg_binary if !ismissing(getproperty(b, cn))) + sql_blob = getproperty(blob, cn) + # parsing only the necessary information from + # GeoPackageBinary header, see _ function for more details + srs_id, has_extent_z, checkpoint = _gpkg_header_from_WKB(sql_blob, length(sql_blob)) + # remaining bytes available after reading GeoPackage Binary header + # read in the WKBGeometry one sql blob at a time + io = IOBuffer(view(sql_blob, checkpoint:length(sql_blob))) + # view() returns a lightweight array + # that lazily references (or is effectively a view into) the parent array + data = import_geometry_from_wkb(io, srs_id, has_extent_z) + + if isa(data, Vector{<:Geometry}) + for feature in data + push!(meshes, feature) + end + elseif !isnothing(data) + push!(meshes, data) + end + end + end + return meshes +end + +# Requirement 19: SHALL store feature table geometries +# with or without optional elevation (Z) and/or measure (M) values +# in SQL BLOBs using the Standard GeoPackageBinary format + +function _gpkg_header_from_WKB(sqlblob, bloblen) + + if (bloblen < 8 || sqlblob[1:2] != MAGIC_GPKG_BINARY_STRING || sqlblob[3] != 0) + @error "GeoPackageBinaryHeader missing format specifications in table" + end + io = IOBuffer(sqlblob) + + seek(io, 3) + + flagsbyte = read(io, UInt8) + # empty geometry flag + # bempty = (flagsbyte & (0x01 << 4)) >> 4 + + # ⚠ ExtendedGeoPackageBinary was removed from the specification for interoperability concerns + # It is intended to be a bridge to enable use of geometry types + # like EllipiticalCurve in Extended GeoPackages until + # standard encodings of such types are developed and + # published for the Well Known Binary (WKB) format. + # bextended = (flagsbyte & (0x01 << 5)) >> 5 # For user-defined geometry types + + ebyteorder = (flagsbyte & 0x01) # byte order + # for SRS_ID and envelope values in header (1-bit Boolean) + + bextent_has_xy = false + bextent_has_z = false + bextent_has_m = false + + bswap = @BSWAP(ebyteorder) + + # Envelope: envelope contents indicator code (3-bit unsigned integer) + envelope = (flagsbyte & ( 0x07 << 1 )) >> 1 + + if envelope != 0 # no envelope (space saving slower indexing option), 0 bytes + if isone(envelope) + bextent_has_xy = true # envelope is [minx, maxx, miny, maxy], 32 bytes + elseif envelope == 2 # envelope is [minx, maxx, miny, maxy, minz, maxz], 48 bytes + bextent_has_z = true + elseif envelope == 3 # envelope is [minx, maxx, miny, maxy, minm, maxm], 48 bytes + bextent_has_m = true + elseif envelope == 4 # envelope is [minx, maxx, miny, maxy, minz, maxz, minm, maxm], 64 bytes + bextent_has_z = true + bextent_has_m = true + else + @error "envelope geometry only exists in 2, 3 or 4-dimensional coordinate space." + end + end + + # SrsId # + srsid = bswap ? Base.bswap(read(io, UInt32)) : read(io, UInt32) + + minx, miny, maxx, maxy = 0.0, 0.0, 0.0, 0.0 # x is easting or longitude, y is northing or latitude + minz, minm, maxz, maxm = 0.0, 0.0, 0.0, 0.0 # z is optional elevation, m is optional measure + # measure is included for interoperability with the Observations & Measurement Standard + + if bextent_has_xy + minx = bswap ? Base.bswap(read(io, Float64)) : read(io, Float64) + miny = bswap ? Base.bswap(read(io, Float64)) : read(io, Float64) + maxx = bswap ? Base.bswap(read(io, Float64)) : read(io, Float64) + maxy = bswap ? Base.bswap(read(io, Float64)) : read(io, Float64) + end + + if bextent_has_z + minz = bswap ? Base.bswap(read(io, Float64)) : read(io, Float64) + maxz = bswap ? Base.bswap(read(io, Float64)) : read(io, Float64) + end + + if bextent_has_m + minm = bswap ? Base.bswap(read(io, Float64)) : read(io, Float64) + maxm = bswap ? Base.bswap(read(io, Float64)) : read(io, Float64) + end + + headerlen = position(io) + + return srsid, bextent_has_z, headerlen+1 +end + +function import_sf_from_wkb(io, srs_id, has_z, wkb_type, bswap) + # this srs_id (specfied in geopackage binary) can + srs_id = iszero(srs_id) || isone(abs(srs_id)) ? + ( isone(abs(srs_id)) ? Cartesian{NoDatum} : LatLon{WGS84Latest} ) : + ( Int(srs_id) > 54000 ? CoordRefSystems.get(ESRI{Int(srs_id)}) : CoordRefSystems.get(EPSG{Int(srs_id)}) ) +# Conflicts that arose here and were ?resolved +# -------------------------------------------- +# this conversion of Cartesian <--> Geographic works mostly +# +# Conversion error collecting 2D and 3D features in the same geotable +# +# Although I'm not so sure what the most correct way is for handling +# reading the undefined and/or cartographic/geographic reference systems +# + + + if wkb_type == wkbPoint || wkb_type == wkbPointZ + elem = _wkb_coordinate(io, has_z, bswap) + return meshes_creategeometry(wkb_type, srs_id, elem) + elseif wkb_type == wkbPolygon || wkb_type == wkbPolygonZ + elem = _wkb_polygon(io, has_z, bswap) + return meshes_creategeometry(wkb_type, srs_id, elem) + elseif wkb_type == wkbLineString || wkb_type == wkbLineStringZ + elem = _wkb_linestring(io, has_z, bswap) + return meshes_creategeometry(wkb_type, srs_id, elem) + else + @warn("unknown type, non standard: $wkb_type, hit the road") + end +end + +function import_geometry_from_wkb(io, srs_id, has_z::Bool) + ebyteorder = read(io, UInt8) + bswap = @BSWAP(ebyteorder) + + v_wkb_type = bswap ? Base.bswap(read(io, UInt32)) : read(io, UInt32) + local wkb_type + try + wkb_type::wkbGeometryType = wkbGeometryType(v_wkb_type) + catch e + @warn "wkbGeometrytype $v_wkb_type is not found in our list standard and non-standard types" + end + + wkb_enum::AbstractString = string(wkb_type) + if occursin("Multi", wkb_enum) + melems::Vector = _wkb_geometrycollection(io, srs_id, has_z, bswap) + return Meshes.Multi(melems) + elseif wkb_type == wkbGeometryCollection || wkb_type == wkbGeometryCollectionZ + elems::Vector{Geometry} = _wkb_geometrycollection(io, srs_id, has_z, bswap) + return elems + else + return import_sf_from_wkb(io, srs_id, has_z, wkb_type, bswap) + end +end + + + + +function _wkb_coordinate(io, has_z, bswap) + + x = bswap ? Base.bswap(read(io, Float64)) : read(io, Float64) + y = bswap ? Base.bswap(read(io, Float64)) : read(io, Float64) + + if has_z + z = bswap ? Base.bswap(read(io, Float64)) : read(io, Float64) + return SVector{3, Float64}(x,y,z) + end + + return SVector{2, Float64}(x,y) +end + +# Unitful coordinate values address many pitfalls in geospatial applications + + + +# a non-standard WKB representation, +# functionally equivalent to LineString but separate identity in GIS simple features data model +# It exists to serve as a component of a Polygon +function _wkb_linearring(io, has_z, bswap) + + num_points = bswap ? Base.bswap(read(io, UInt32)) : read(io, UInt32) + + points::Vector{SVector{has_z ? 3 : 2 , Float64}} = map(1:num_points) do _ + _wkb_coordinate(io, has_z, bswap) + end + + return points +end + +function _wkb_polygon(io, has_z, bswap) + + num_rings = bswap ? Base.bswap(read(io, UInt32)) : read(io, UInt32) + + rings = map(1:num_rings) do _ + _wkb_linearring(io, has_z, bswap) + end + + return rings +end + +function _wkb_linestring(io, has_z, bswap) + + num_points = bswap ? Base.bswap(read(io, UInt32)) : read(io, UInt32) + + points::Vector{SVector{has_z ? 3 : 2 , Float64}} = map(1:num_points) do _ + _wkb_coordinate(io, has_z, bswap) + end + return points +end + +function _wkb_geometrycollection(io, srs_id, has_z, bswap) + + num_geoms = bswap ? Base.bswap(read(io, UInt32)) : read(io, UInt32) + + geomcollection = map(1:num_geoms) do _ + import_geometry_from_wkb(io, srs_id, has_z) + end + + return geomcollection +end + + + +#--------------------------- +# SQLite Query Optimization +# -------------------------- +# +# - Avoid OR-connected constraints, use IN operator, or UNION constraints separately +# +# - GROUP BY or DISTINCT logic can determine if the current row is part of the same group +# or if the current row is distinct simply by comparing the current row to the previous row +# - faster than the alternative of comparing each row to all prior rows +# +#.- +# +# -------------------------- From 53a4b88e98152b8f7c9d48312bb35ee3b021d9ca Mon Sep 17 00:00:00 2001 From: jph6366 Date: Tue, 9 Sep 2025 01:25:58 -0400 Subject: [PATCH 02/90] rigorously tested yet no valid gpkg writer just yet. --- src/GeoIO.jl | 5 +- src/extra/gis.jl | 4 + src/extra/gpkg.jl | 1051 +-------------------------------------- src/extra/gpkg/read.jl | 298 +++++++++++ src/extra/gpkg/write.jl | 294 +++++++++++ 5 files changed, 618 insertions(+), 1034 deletions(-) create mode 100644 src/extra/gpkg/read.jl create mode 100644 src/extra/gpkg/write.jl diff --git a/src/GeoIO.jl b/src/GeoIO.jl index 81ab4b2..4cabd47 100644 --- a/src/GeoIO.jl +++ b/src/GeoIO.jl @@ -38,6 +38,9 @@ import PlyIO # CSV format import CSV +# Database Interfaces +import SQLite + # geostats formats import GslibIO @@ -72,7 +75,7 @@ const CDMEXTS = [".grib", ".nc"] const FORMATS = [ (extension=".csv", load="CSV.jl", save="CSV.jl"), (extension=".geojson", load="GeoJSON.jl", save="GeoJSON.jl"), - (extension=".gpkg", load="ArchGDAL.jl", save="ArchGDAL.jl"), + (extension=".gpkg", load="GeoIO.jl", save="GeIO.jl"), (extension=".grib", load="GRIBDatasets.jl", save=""), (extension=".gslib", load="GslibIO.jl", save="GslibIO.jl"), (extension=".jpeg", load="ImageIO.jl", save="ImageIO.jl"), diff --git a/src/extra/gis.jl b/src/extra/gis.jl index 90b7d1f..85c5420 100644 --- a/src/extra/gis.jl +++ b/src/extra/gis.jl @@ -50,6 +50,8 @@ function giswrite(fname, geotable; warn, kwargs...) elseif endswith(fname, ".parquet") CRS = crs(domain(geotable)) GPQ.write(fname, geotable, (:geometry,), projjson(CRS); kwargs...) + elseif endswith(fname, ".gpkg") + gpkgwrite(fname, geotable; kwargs...) else # fallback to GDAL agwrite(fname, geotable; kwargs...) end @@ -63,6 +65,8 @@ function gistable(fname; layer, numtype, kwargs...) return GJS.read(fname; numbertype=numtype, kwargs...) elseif endswith(fname, ".parquet") return GPQ.read(fname; kwargs...) + elseif endswith(fname, ".gpkg") + return gpkgread(fname; layer, kwargs...) else # fallback to GDAL data = AG.read(fname; kwargs...) return AG.getlayer(data, layer - 1) diff --git a/src/extra/gpkg.jl b/src/extra/gpkg.jl index d9ec809..1c3a113 100644 --- a/src/extra/gpkg.jl +++ b/src/extra/gpkg.jl @@ -1,3 +1,21 @@ +# ------------------------------------------------------------------ +# Licensed under the MIT License. See LICENSE in the project root. +# ------------------------------------------------------------------ + +############################################# +### https://www.geopackage.org/spec/#r2 ##### +# Requirement 2: contains "GPKG" in ASCII in +# "application_id" field of SQLite db header +# Reminder: We have to set this (on-write) +# after there's some content, +# so the database file is not zero length +# +const GP10_APPLICATION_ID = 74777363 #0x47503130 +const GP11_APPLICATION_ID = 119643780 # 0x47503131 +const GPKG_APPLICATION_ID = 1196444487 # 0x47504B47 +const GPKG_1_2_VERSION = 10200 +const GPKG_1_3_VERSION = 10300 +const GPKG_1_4_VERSION = 10400 # List of well known binary geometry types. # These are used within the GeoPackageBinary SQL BLOBs @@ -82,1036 +100,3 @@ wkbMultiPolygon25D = 0x80000006 wkbGeometryCollection25D = 0x80000007 end - -const wkbXDR = 0 -const wkbNDR = 1 - -const wkb25DBit = 0x80000000 - -macro BSWAP(x) - if ENDIAN_BOM == 0x01020304 - return :($(esc(x)) == wkbNDR) - else - return :($(esc(x)) == wkbXDR) - end -end - - -# Requirement 1: first 16 bytes is null-terminated ASCII string "SQLite format 3" -const MAGIC_HEADER_STRING::Vector{UInt8} = UInt8[ - 0x53, 0x51, 0x4c, 0x69, 0x74, 0x65, 0x20, 0x66, - 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x20, 0x33, 0x00 -] - -# 'GP' in ASCII -const MAGIC_GPKG_BINARY_STRING = UInt8[ 0x47, 0x50 ] - -# Requirement 2: contains "GPKG" in ASCII in "application_id" field of SQLite db header -# Reminder: We have to set this (on-write) after there's some content -# so the database file is not zero length - -const GP10_APPLICATION_ID = 74777363 #0x47503130 -const GP11_APPLICATION_ID = 119643780 # 0x47503131 -const GPKG_APPLICATION_ID = 1196444487 # 0x47504B47 -const GPKG_1_2_VERSION = 10200 -const GPKG_1_3_VERSION = 10300 -const GPKG_1_4_VERSION = 10400 - - -const CREATE_GPKG_GEOMETRY_COLUMNS = - "CREATE TABLE gpkg_geometry_columns ("* - "table_name TEXT NOT NULL,"* - "column_name TEXT NOT NULL,"* - "geometry_type_name TEXT NOT NULL,"* - "srs_id INTEGER NOT NULL,"* - "z TINYINT NOT NULL,"* - "m TINYINT NOT NULL,"* - "CONSTRAINT pk_geom_cols PRIMARY KEY (table_name, column_name),"* - "CONSTRAINT uk_gc_table_name UNIQUE (table_name),"* - "CONSTRAINT fk_gc_tn FOREIGN KEY (table_name) REFERENCES "* - "gpkg_contents(table_name),"* - "CONSTRAINT fk_gc_srs FOREIGN KEY (srs_id) REFERENCES gpkg_spatial_ref_sys "* - "(srs_id)"* - ")" - -const CREATE_GPKG_REQUIRED_METADATA = "CREATE TABLE gpkg_spatial_ref_sys ("* - "srs_name TEXT NOT NULL,"* - "srs_id INTEGER NOT NULL PRIMARY KEY,"* - "organization TEXT NOT NULL,"* - "organization_coordsys_id INTEGER NOT NULL,"* - "definition TEXT NOT NULL,"* - "description TEXT,"* - "definition_12_063 TEXT NOT NULL,"* - "epoch DOUBLE"* - ")"* - ";"* - "INSERT INTO gpkg_spatial_ref_sys ("* - "srs_name, srs_id, organization, organization_coordsys_id, "* - "definition, description, definition_12_063"* - ") VALUE ("* - - "'WGS 84 geodetic', 4326, 'EPSG', 4326, '"* - "GEOGCS[\"WGS 84\",DATUM[\"WGS_1984\",SPHEROID[\"WGS "* - "84\",6378137,298.257223563,AUTHORITY[\"EPSG\",\"7030\"]],"* - "AUTHORITY[\"EPSG\",\"6326\"]],PRIMEM[\"Greenwich\",0,AUTHORITY["* - "\"EPSG\",\"8901\"]],UNIT[\"degree\",0.0174532925199433,AUTHORITY["* - "\"EPSG\",\"9122\"]],AXIS[\"Latitude\",NORTH],AXIS[\"Longitude\","* - "EAST],AUTHORITY[\"EPSG\",\"4326\"]]"* - "', 'longitude/latitude coordinates in decimal degrees on the WGS "* - "84 spheroid', "* - - "'GEODCRS[\"WGS 84\", DATUM[\"World Geodetic System 1984\", "* - "ELLIPSOID[\"WGS 84\",6378137, 298.257223563, "* - "LENGTHUNIT[\"metre\", 1.0]]], PRIMEM[\"Greenwich\", 0.0, "* - "ANGLEUNIT[\"degree\",0.0174532925199433]], CS[ellipsoidal, "* - "2], AXIS[\"latitude\", north, ORDER[1]], AXIS[\"longitude\", "* - "east, ORDER[2]], ANGLEUNIT[\"degree\", 0.0174532925199433], "* - "ID[\"EPSG\", 4326]]'"* - ")"* - ";"* - "INSERT INTO gpkg_spatial_ref_sys ("* - "srs_name, srs_id, organization, organization_coordsys_id, "* - " definition, description, definition_12_063"* - ") VALUES ("* - "'Undefined Cartesian SRS', -1, 'NONE', -1, 'undefined', "* - "'undefined Cartesian coordinate reference system', 'undefined'"* - ")"* - ";"* - "INSERT INTO gpkg_spatial_ref_sys ("* - "srs_name, srs_id, organization, organization_coordsys_id, "* - "definition, description, definition_12_063"* - ") VALUES ("* - "'Undefined geographic SRS' 0, 'NONE', 0, 'undefined', "* - "'undefined geographic coordinate reference system', 'undefined'"* - ")"* - ";"* - "CREATE TABLE gpkg_contents ("* - "table_name TEXT NOT NULL PRIMARY KEY,"* - "data_type TEXT NOT NULL,"* - "identifier TEXT UNIQUE,"* - "description TEXT DEFAULT '',"* - "last_change DATETIME NOT NULL DEFAULT "* - "(strftime('%Y-%m-%dT%H:%M:%fZ','now')),"* - "min_x DOUBLE, min_y DOUBLE,"* - "max_x DOUBLE, max_y DOUBLE,"* - "srs_id INTEGER,"* - "CONSTRAINT fk_gc_r_srs_id FOREIGN KEY (srs_id) REFERENCES "* - "gpkg_spatial_ref_sys(srs_id)"* - ")"* - ";" - -function gpkgread(fname; kwargs... ) - # Requirement 3: File name has to end in ".gpkg" - if !(endswith(fname,".gpkg")) @error "the file extension is not .gpkg'" end - db_header_string = open(fname, "r") do io - read(io, 16) - end - if(db_header_string != MAGIC_HEADER_STRING) - @error "missing magic header string" - end - - db = SQLite.DB(fname) - - @timeit to "identify geopackage" begin - gpkg_identify(db) - end - show(to) - - local mesh_geometry - - @timeit to "read geopackage" begin - gpkg_attrs, mesh_geometry, geom_attrs = read_gpkg_tables(db) - end - show(to) - SQLite.close(db) - - return GeoTables.georef(geom_attrs, mesh_geometry) - # return GeoTables.georef(gpkg_attrs, [Point(NaN, NaN)]) - -end - -function gpkgwrite(fname, geotable; ) - - - db = SQLite.DB(fname) - - @timeit to "write geopackage" begin - - # DBInterface.execute(db, "PRAGMA foreign_keys = ON") - - #DBInterface.execute(db, "PRAGMA journal_mode=WAL;") - # Write transactions are very fast since they only involve writing the content once - # (versus twice for rollback-journal transactions) and because the writes are all sequential - - DBInterface.execute(db, "PRAGMA synchronous=0") - # Commits can be orders of magnitude faster with - # Setting PRAGMA synchronous=OFF can cause the database to go corrupt - # if there is an operating-system crash or power failure, - # though this setting is safe from damage due to application crashes. - - SQLite.transaction(db) do - -# ------------------------- -# REQUIRED METADATA TABLES -# ------------------------- - - # Requirement 11: The gpkg_spatial_ref_sys table SHALL contain at a minimum: - # 1. The record with an srs_id of 4326 SHALL correspond to WGS-84 as defined by EPSG in 4326. - # OR - # 2. The record with an srs_id of -1 SHALL be used for undefined Cartesian coordinate reference systems. - # OR - # 3. The record with an srs_id of 0 SHALL be used for undefined geographic coordinate reference systems. - - - -# ------------------------- -# OPTIONAL METADATA TABLES -# ------------------------- - stmt_sql = SQLite.Stmt(db, CREATE_GPKG_REQUIRED_METADATA*CREATE_GPKG_GEOMETRY_COLUMNS ) - DBInterface.execute(stmt_sql) - - - stmt_sql = SQLite.Stmt(db, "PRAGMA application_id = $GPKG_APPLICATION_ID ") - DBInterface.execute(stmt_sql) - stmt_sql = SQLite.Stmt(db, "PRAGMA user_version = $GPKG_1_4_VERSION ") - DBInterface.execute(stmt_sql) - - end - - to_gpkg(db, geotable) - end - show(to) - - SQLite.close(db) - - return -end - - -function to_gpkg(db, gt::GeoTable) - table = values(gt) - domain = GeoTables.domain(gt) - crs = GeoTables.crs(domain) - geometry = collect(domain) - - if crs <: Cartesian - crs = -1 - elseif crs <: LatLon{WGS84Latest} - crs = Int32(4326) - end - - println("crs: ", crs) - gpkgbin_blobs = Vector{Vector{UInt8}}() - - for geom::Geometry in geometry - gpkgbin_header = _gpkg_update_header(crs, geom) - io = IOBuffer() - _import_to_wkb(io, geom) - wkb_blob = take!(io) - push!(gpkgbin_blobs, vcat(gpkgbin_header, wkb_blob)) - end - - table = merge(table, (geom=gpkgbin_blobs,)) - SQLite.load!(table, db, replace=false) # autogenerates table name - # replace=false controls whether - # an INSERT INTO ... statement is generated or a REPLACE INTO .... - println(table |> DataFrame) - tn = DBInterface.execute(db, "SELECT name FROM sqlite_master WHERE type='table'") - nm = "" - for row in tn - nm = row.name - end - - bbox = boundingbox(domain) - min_coords = CoordRefSystems.raw(coords(bbox.min)) - max_coords = CoordRefSystems.raw(coords(bbox.max)) - - contents_table = [(table_name = nm, - data_type = "features", - identifier = nm, - min_x = min_coords[1], - min_y = min_coords[2], - max_x = max_coords[1], - max_y = max_coords[2], - srs_id = crs - )] - - println(contents_table |> DataFrame) - SQLite.load!(contents_table, db, "gpkg_contents", replace=true) - - geometry_columns_table = [(table_name = nm, - column_name = "geom", - geometry_type_name = nm, - srs_id = crs, - z = paramdim(first(geometry)) >= 2 ? 1 : 0, - m = 0 - )] - - println(geometry_columns_table |> DataFrame) - SQLite.load!(geometry_columns_table, db, "gpkg_geometry_columns", replace=true) - - -end - -function gpkg_gtype(geoms::AbstractVector{<:Geometry}) - - T = eltype(geoms) - - if T <: Point - return "POINT" - elseif T <: Rope - return "LINESTRING" - elseif T <: Ring - return "LINESTRING" - elseif T <: PolyArea - return "POLYGON" - elseif T <: Multi - - element_type = eltype(parent(first(geoms))) - return "MULTI"*gpkg_gtype(element_type) - else - return "GEOMETRY" - end - -end - -function _gpkg_update_header(srs_id, geometry, envelope::Int=1) - - io = IOBuffer() - write(io, MAGIC_GPKG_BINARY_STRING) # 'GP' in ASCII - write(io, UInt8(0)) # 0 = version 1 - - flagsbyte = UInt8(0x20 | (envelope << 1)) - write(io, flagsbyte) - - bswap = @BSWAP (flagsbyte & 0x01) - - bswap ? write(io, Base.bswap(srs_id)) : write(io, srs_id) - - if isone(envelope) - bbox = boundingbox(geometry) - if bswap - write(io, Base.bswap(Float64(CoordRefSystems.raw(coords(bbox.min))[1]))) - write(io, Base.bswap(Float64(CoordRefSystems.raw(coords(bbox.max))[1]))) - write(io, Base.bswap(Float64(CoordRefSystems.raw(coords(bbox.min))[2]))) - write(io, Base.bswap(Float64(CoordRefSystems.raw(coords(bbox.max))[2]))) - else - write(io, Float64(CoordRefSystems.raw(coords(bbox.min))[1])) - write(io, Float64(CoordRefSystems.raw(coords(bbox.max))[1])) - write(io, Float64(CoordRefSystems.raw(coords(bbox.min))[2])) - write(io, Float64(CoordRefSystems.raw(coords(bbox.max))[2])) - end - end - - if isone(envelope-1) && paramdim(geometry) >= 3 - bbox = boundingbox(geometry) - if bswap - write(io, Base.bswap(Float64(CoordRefSystems.raw(coords(bbox.min))[3]))) - write(io, Base.bswap(Float64(CoordRefSystems.raw(coords(bbox.max))[3]))) - else - write(io, Float64(CoordRefSystems.raw(coords(bbox.min))[3])) - write(io, Float64(CoordRefSystems.raw(coords(bbox.max))[3])) - end - end - - return take!(io) - -end - - -function wkb_set_z(type::wkbGeometryType) - return wkbGeometryType(Int(type)+1000) - # ISO WKB simply adds a round number to the type number to indicate extra dimensions -end - -function wkb_has_z(type::wkbGeometryType) - return Int(type) > 1000 -end - -function wkbtype(geometry) - if geometry isa Point - return wkbPoint - elseif geometry isa Rope || geometry isa Ring - return wkbLineString - elseif geometry isa PolyArea - return wkbPolygon - elseif geometry isa Multi - fg = parent(geometry) |> first - return wkbGeometryType(Int(wkbtype(fg))+3) - else - @error "my hovercraft is full of eels: $geometry" - end -end - -function _import_to_wkb(io, geometry) - gtype = isone(paramdim(geometry)-1) ? wkb_set_z(wkbtype(geometry)) : wkbtype(geometry) - create_wkb_geometry(io, geometry, gtype) -end - -function create_wkb_geometry(io, geometry, gtype) - if Int(gtype) > 3 - write(io, UInt8(0x01)) - write(io, UInt32(gtype)) - write(io, UInt32(length(parent(geometry))) ) - - for geom in parent(geometry) - create_wkb_geometry(io, geom, wkbtype(geom)) - end - else - import_wkb_geometry(io, geometry, gtype) - end -end - -function import_wkb_geometry(io, geometry, wkb_type) - - write(io, UInt8(0x01)) - - bswap = @BSWAP(one(0x01)) - - if wkb_type == wkbPolygon || wkb_type == wkbPolygonZ - - _to_wkb_polygon(io, wkb_type, [boundary(geometry::PolyArea)]) - - elseif wkb_type == wkbLineString || wkb_type == wkbLineStringZ - - coordinate_list = vertices(geometry) - bswap ? Base.bswap(write(io, UInt32(wkb_type))) : write(io, UInt32(wkb_type)) - - if geometry isa Ring - bswap ? Base.bswap(write(io, UInt32(length(coordinate_list)+1))) : write(io, UInt32(length(coordinate_list)+1)) - else - bswap ? Base.bswap(write(io, UInt32(length(coordinate_list)))) : write(io, UInt32(length(coordinate_list))) - end - - _to_wkb_linestring(io, wkb_type, coordinate_list) - - if geometry isa Ring - coordinates = CoordRefSystems.raw(coords( coordinate_list |> first )) - _to_wkb_coordinates(io, wkb_type, coordinates) - end - - elseif wkb_type == wkbPoint || wkb_type == wkbPointZ - - coordinates = CoordRefSystems.raw(coords(geometry::Point)) - _to_wkb_coordinates(io, wkb_type, coordinates) - - else - @error "What is the $wkb_type ? not recognized; simple features only" - end -end - -function _to_wkb_coordinates(io, wkb_type, coords) - - - bswap = @BSWAP(one(0x01)) - - bswap ? Base.bswap(write(io, UInt32(wkb_type))) : write(io, UInt32(wkb_type)) - bswap ? Base.bswap(write(io, Float64(coords[1]))) : write(io, Float64(coords[1])) - bswap ? Base.bswap(write(io, Float64(coords[2]))) : write(io, Float64(coords[2])) - - if wkb_has_z(wkb_type) - write(io, Float64(coords[3])) - end -end - -function _to_wkb_linestring(io, wkb_type, coord_list) - for n_coords::Point in coord_list - coordinates = CoordRefSystems.raw(coords(n_coords)) - _to_wkb_coordinates(io, wkb_type, coordinates) - end -end - -function _to_wkb_polygon(io, wkb_type, rings) - write(io, UInt32(wkb_type)) - write(io, UInt32(length(rings))) - - for ring in rings - coord_list = vertices(ring) - write(io, UInt32(length(coord_list) + 1)) - _to_wkb_linestring(io, wkb_type, coord_list) - end -end - -# ----------------- -# HELPER FUNCTIONS -# ----------------- - -const to = TimerOutput() - - - - - -# Requirement 5: columns of tables are only declared using one of the GeoPackage data types -# Extended GeoPackages contain additional data types -# -# Requirement 5 Warning -# ⚠ data type mismatches could theoretically be checked for but tests would scale poorly ⚠ -# -# GeoPackage writers SHOULD validate the data as it is being inserted -# GeoPackage readers SHOULD allow for the possibility that unexpected values are present. - -function _julia_sqlite_datatype(gpkg_type::AbstractString, kwargs...) - if startswith(gpkg_type, "INT") - if(gpkg_type != "INT" && gpkg_type != "INTEGER") - @warn "field format $gpkg_type not supported (interpreted as int)" - end - return Int64 - elseif gpkg_type == "BOOLEAN" - return Bool - elseif gpkg_type == "TINYINT" - return Int8 - elseif gpkg_type == "SMALLINT" - return Int16 - elseif gpkg_type == "MEDIUMINT" - return Int32 - elseif gpkg_type == "FLOAT" - return Float32 - elseif gpkg_type == "DOUBLE" || gpkg_type == "REAL" - return Float64 - elseif gpkg_type == "TEXT" - return String # UTF-8 or UTF-16, determined by PRAGMA encoding - elseif gpkg_type == "BLOB" - return Any - elseif startswith(gpkg_type, "DATE") - return Any - else - @warn "field format $gpkg_type not recognized, okay have a nice day" - return Any - end -end - -# Requirement 20: GeoPackage SHALL store feature table geometries with the basic simple feature geometry types -# https://www.geopackage.org/spec140/index.html#geometry_types -function meshes_creategeometry(wkb_type, crs, C) - if wkb_type == wkbPoint || wkb_type == wkbPointZ - - return Meshes.Point(crs(C...)) - - elseif wkb_type == wkbLineString || wkb_type == wkbLineStringZ - - return (C[1] == C[end]) ? - Meshes.Ring([Meshes.Point(crs(coords...)) for coords in C[2:end]]...) : Meshes.Rope([Meshes.Point(coords...) for coords in C]...) - - elseif wkb_type == wkbPolygon || wkb_type == wkbPolygonZ - - rings = map(C) do ring_coords - coords = map(ring_coords) do svec - Meshes.Point(crs(svec...)) - end - Meshes.Ring(coords) - end - - outer_ring = first(rings) - holes = length(rings) > 1 ? rings[2:end] : Meshes.Ring[] - return Meshes.PolyArea(outer_ring, holes...) - - else - @error "what $wkb_type is this" - end # @TODO: add support for non-linear geometry types -end - -function gpkg_identify(db)::Bool - - application_id = DBInterface.execute(db, "PRAGMA application_id;") |> first |> only - user_version = DBInterface.execute(db, "PRAGMA user_version;") |> first |> only - - if !(_has_gpkg_required_metadata_tables(db)) - @error "missing required metadata tables" - end - - # Requirement 6: PRAGMA integrity_check returns a single row with the value 'ok' - # Requirement 7: PRAGMA foreign_key_check (w/ no parameter value) returns an empty result set - if (application_id != GP10_APPLICATION_ID) && - (application_id != GP11_APPLICATION_ID) && - (application_id != GPKG_APPLICATION_ID) - @warn "application_id not recognized" - return false - elseif(application_id == GPKG_APPLICATION_ID) && - !( - (user_version >= GPKG_1_2_VERSION && user_version < GPKG_1_2_VERSION + 99) || - (user_version >= GPKG_1_3_VERSION && user_version < GPKG_1_3_VERSION + 99) || - (user_version >= GPKG_1_4_VERSION && user_version < GPKG_1_4_VERSION + 99) - ) - @warn "application_id is valid but user version is not recognized" - elseif( - DBInterface.execute(db, "PRAGMA integrity_check;") |> first |> only != "ok" - ) || !( - isempty(DBInterface.execute(db, "PRAGMA foreign_key_check;")) - ) - @error "database integrity at risk or foreign key violation(s)" - return false - end - return true -end - -# Requirement 10: must include a gpkg_spatial_ref_sys table -# Requirement 13: must include a gpkg_contents table -function _has_gpkg_required_metadata_tables(db) - stmt_sql = "SELECT COUNT(*) FROM sqlite_master WHERE "* - "name IN ('gpkg_spatial_ref_sys', 'gpkg_contents') AND "* - "type IN ('table', 'view');" - required_metadata_tables = DBInterface.execute(db, stmt_sql) |> first |> only - return (required_metadata_tables == 2) -end - -function _gpkg_crs_wkt_extension(db) - stmt_sql = "SELECT extension_name FROM gpkg_extensions;" - ext_name = DBInterface.execute(db, stmt_sql) |> first |> only - return ext_name == "gpkg_crs_wkt" -end - -function read_gpkg_tables(db) - gpkg_table_info = SQLite.tableinfo(db, "gpkg_spatial_ref_sys") - # @TODO: gpkg_contents tableinfo - has_gpkg_extensions = _has_gpkg_optional_metadata_table(db) - if (has_gpkg_extensions) - end - - - has_gpkg_attributes = _has_gpkg_attributes(db) - has_vector_features = _has_gpkg_geometry_columns(db) - if has_vector_features - println("vector") - gpkg_table_info = SQLite.tableinfo(db, "gpkg_geometry_columns") - - geometry = get_feature_table_geometry_columns(db) - - vector_features = get_feature_tables(db) - feature_tables = get_feature_attributes(db, vector_features) - - if has_gpkg_attributes - - println("attributes") - #gpkg_attributes = get_gpkg_attributes(db) - - #return gpkg_attributes, geometry, feature_tables - end - return nothing, geometry, feature_tables - - elseif has_gpkg_attributes - println("attributes") - gpkg_attributes = get_gpkg_attributes(db) - return gpkg_attributes, nothing, nothing - else - @error "data_type not supported yet, looks for 'features' by default and falls back to 'attributes' " - end -end - - -function _has_gpkg_geometry_columns(db) - stmt_sql = "SELECT 1 from sqlite_master WHERE "* - "name = 'gpkg_geometry_columns' AND "* - "type IN ('table', 'view');" - geometry_columns = DBInterface.execute(db, stmt_sql) |> collect - return !isempty(geometry_columns) -end - -# Requirement 58: optionally includes a gpkg_extensions table -function _has_gpkg_optional_metadata_table(db) - stmt_sql= "SELECT 1 FROM sqlite_master WHERE name = 'gpkg_extensions' "* - "AND type IN ('table', 'view')" - extensions = DBInterface.execute(db, stmt_sql) |> collect - return !isempty(extensions) -end - - - -# Requirement 118: gpkg_contents table SHALL contain -# a row with a data_type column value of "attributes" -# for each attributes data table or view. - -function _has_gpkg_attributes(db) - stmt_sql = "SELECT COUNT(*) FROM sqlite_master WHERE " * - "name = 'gpkg_contents' AND " * - "type IN ('table', 'view');" - has_gpkg_contents_table = DBInterface.execute(db, stmt_sql) |> first |> only - - if has_gpkg_contents_table == 1 - stmt_sql_datatype = "SELECT COUNT(*) FROM gpkg_contents WHERE " * - "data_type = 'attributes';" - has_attributes_datatype = DBInterface.execute(db, stmt_sql_datatype) |> first |> only - return has_attributes_datatype > 0 - else - return false - end -end - -# -# -# Requirement 119 & 151: GeoPackage MAY contain tables and views -# containing attributes and attribute sets -function get_gpkg_attributes(db) - stmt_sql = "SELECT table_name as tn FROM gpkg_contents WHERE data_type = 'attributes'" - attributes = DBInterface.execute(db, stmt_sql) - gpkg_attrs = map(attributes) do sqlite_row - tn = sqlite_row.tn - DBInterface.execute(SQLite.Stmt(db, "SELECT * FROM $tn")) |> Tables.rows - end - attr_tables = map(gpkg_attrs) do row - map(Tables.columnnames(row)) do col - Tables.getcolumn(row, col) - end - end - return attr_tables -end - - -# Requirement 21: a gpkg_contents table row with a "features" data_type -# SHALL contain a gpkg_geometry_columns table - -function get_feature_tables(db) - stmt_sql = "SELECT c.table_name, c.identifier, 1 as is_aspatial, "* - "g.column_name, g.geometry_type_name, g.z, g.m, c.min_x, c.min_y, "* - "c.max_x, c.max_y, 1 AS is_in_gpkg_contents, "* - "(SELECT type FROM sqlite_master WHERE lower(name) = "* - "lower(c.table_name) AND type IN ('table', 'view')) AS object_type "* - " FROM gpkg_geometry_columns g "* - " JOIN gpkg_contents c ON (g.table_name = c.table_name)"* - " WHERE "* - " c.data_type = 'features' " - features = DBInterface.execute(db, stmt_sql) - return features -end - -# Third component of the SQL schema for vector features in a GeoPackage -function get_feature_attributes(db, features) - - attrs = map(features) do sqlite_row - _select_aspatial_attributes(db, sqlite_row) - end - - if isempty(attrs) - return NamedTuple() - end - - feature_attrs = reduce( (i, h) -> begin - jk = keys(i) - len_i = isempty(i) ? 0 : length(first(i)) - - kj = keys(h) - len_h = isempty(h) ? 0 : length(first(h)) - - all = unique(vcat(collect(jk), collect(kj))) - - vals = map(all) do k - newcol = Vector{Any}() - - if haskey(i, k) - append!(newcol, i[k]) - else - append!(newcol, fill(nothing, len_i)) - end - - if haskey(h, k) - append!(newcol, h[k]) - else - append!(newcol, fill(nothing, len_h)) - end - - return newcol - end - return NamedTuple{Tuple(all)}(vals) - end , attrs, init=NamedTuple()) - return feature_attrs -end - -function _select_aspatial_attributes(db, row) - tn = row.table_name - table_info = SQLite.tableinfo(db, tn) - ft_attrs = table_info.name - - # delete gpkg_geometry_columns column name, usually 'geom' or 'geometry' - deleteat!(ft_attrs, findall(x -> x == row.column_name, ft_attrs)) - attrs_str = join(ft_attrs, ", ") - stmt_sql = iszero(length(ft_attrs)) ? "" : "SELECT $attrs_str FROM $tn" - attrs_result = iszero(length(stmt_sql)) ? false : DBInterface.execute(db, stmt_sql) |> Tables.columns - columns, names = Tables.columns(attrs_result), Tables.columnnames(attrs_result) - return NamedTuple{Tuple(names)}([ - Tables.getcolumn(columns, nm) for nm in names - ]) - -end - -# Requirement 25: The geometry_type_name value in a gpkg_geometry_columns row -# SHALL be one of the uppercase geometry type names specified - -# Requirement 26: The srs_id value in a gpkg_geometry_columns table row -# SHALL be an srs_id column value from the gpkg_spatial_ref_sys table. -# -# Requirement 27: The z value in a gpkg_geometry_columns table row SHALL be one of 0, 1, or 2. -# Requirement 28: The m value in a gpkg_geometry_columns table row SHALL be one of 0, 1, or 2. -# -# Requirement 146: The srs_id value in a gpkg_geometry_columns table row -# SHALL match the srs_id column value from the corresponding row in the gpkg_contents table. -# -# Requirement 22: gpkg_geometry_columns table -# SHALL contain one row record for the geometry column -# in each vector feature data table -# -#Requirement 23: gpkg_geometry_columns table_name column -# SHALL reference values in the gpkg_contents table_name column -# for rows with a data_type of 'features' - -# Amalgamation SQL execution, collecting matching vector feature tables -# and then parsing their geopackage binary format containg wkb geometries -function get_feature_table_geometry_columns(db) - stmt_sql = "SELECT g.table_name AS tn, g.column_name AS cn, c.srs_id as crs, g.z as elev, " * - "( SELECT type FROM sqlite_master WHERE lower(name) = lower(c.table_name) AND type IN ('table', 'view')) AS object_type " * - "FROM gpkg_geometry_columns g " * - "JOIN gpkg_contents c ON ( g.table_name = c.table_name ) " * - "WHERE c.data_type = 'features' " * - "AND (SELECT type FROM sqlite_master WHERE lower(name) = lower(c.table_name) AND type IN ('table', 'view')) IS NOT NULL "* - #"AND g.srs_id IN (SELECT srs_id FROM gpkg_spatial_ref_sys ) "* - "AND g.srs_id = c.srs_id "* - "AND g.z IN (0, 1, 2) "* - "AND g.m IN (0, 1, 2);" - stmt_sql = SQLite.Stmt(db, stmt_sql) - geoms = DBInterface.execute(stmt_sql) - geom_specs = Set( (row.tn, Symbol(row.cn), row.crs, row.elev) for row in geoms ) - meshes = Geometry[] - for (tn, cn, crs, elev) in geom_specs - # Requirement 24: The column_name column value in a gpkg_geometry_columns row - # SHALL be the name of a column in the table or view specified - # by the table_name column value for that row. - stmt_sql = "SELECT $cn FROM $tn" - gpkg_binary = DBInterface.execute(db, stmt_sql) - for blob in (b for b in gpkg_binary if !ismissing(getproperty(b, cn))) - sql_blob = getproperty(blob, cn) - # parsing only the necessary information from - # GeoPackageBinary header, see _ function for more details - srs_id, has_extent_z, checkpoint = _gpkg_header_from_WKB(sql_blob, length(sql_blob)) - # remaining bytes available after reading GeoPackage Binary header - # read in the WKBGeometry one sql blob at a time - io = IOBuffer(view(sql_blob, checkpoint:length(sql_blob))) - # view() returns a lightweight array - # that lazily references (or is effectively a view into) the parent array - data = import_geometry_from_wkb(io, srs_id, has_extent_z) - - if isa(data, Vector{<:Geometry}) - for feature in data - push!(meshes, feature) - end - elseif !isnothing(data) - push!(meshes, data) - end - end - end - return meshes -end - -# Requirement 19: SHALL store feature table geometries -# with or without optional elevation (Z) and/or measure (M) values -# in SQL BLOBs using the Standard GeoPackageBinary format - -function _gpkg_header_from_WKB(sqlblob, bloblen) - - if (bloblen < 8 || sqlblob[1:2] != MAGIC_GPKG_BINARY_STRING || sqlblob[3] != 0) - @error "GeoPackageBinaryHeader missing format specifications in table" - end - io = IOBuffer(sqlblob) - - seek(io, 3) - - flagsbyte = read(io, UInt8) - # empty geometry flag - # bempty = (flagsbyte & (0x01 << 4)) >> 4 - - # ⚠ ExtendedGeoPackageBinary was removed from the specification for interoperability concerns - # It is intended to be a bridge to enable use of geometry types - # like EllipiticalCurve in Extended GeoPackages until - # standard encodings of such types are developed and - # published for the Well Known Binary (WKB) format. - # bextended = (flagsbyte & (0x01 << 5)) >> 5 # For user-defined geometry types - - ebyteorder = (flagsbyte & 0x01) # byte order - # for SRS_ID and envelope values in header (1-bit Boolean) - - bextent_has_xy = false - bextent_has_z = false - bextent_has_m = false - - bswap = @BSWAP(ebyteorder) - - # Envelope: envelope contents indicator code (3-bit unsigned integer) - envelope = (flagsbyte & ( 0x07 << 1 )) >> 1 - - if envelope != 0 # no envelope (space saving slower indexing option), 0 bytes - if isone(envelope) - bextent_has_xy = true # envelope is [minx, maxx, miny, maxy], 32 bytes - elseif envelope == 2 # envelope is [minx, maxx, miny, maxy, minz, maxz], 48 bytes - bextent_has_z = true - elseif envelope == 3 # envelope is [minx, maxx, miny, maxy, minm, maxm], 48 bytes - bextent_has_m = true - elseif envelope == 4 # envelope is [minx, maxx, miny, maxy, minz, maxz, minm, maxm], 64 bytes - bextent_has_z = true - bextent_has_m = true - else - @error "envelope geometry only exists in 2, 3 or 4-dimensional coordinate space." - end - end - - # SrsId # - srsid = bswap ? Base.bswap(read(io, UInt32)) : read(io, UInt32) - - minx, miny, maxx, maxy = 0.0, 0.0, 0.0, 0.0 # x is easting or longitude, y is northing or latitude - minz, minm, maxz, maxm = 0.0, 0.0, 0.0, 0.0 # z is optional elevation, m is optional measure - # measure is included for interoperability with the Observations & Measurement Standard - - if bextent_has_xy - minx = bswap ? Base.bswap(read(io, Float64)) : read(io, Float64) - miny = bswap ? Base.bswap(read(io, Float64)) : read(io, Float64) - maxx = bswap ? Base.bswap(read(io, Float64)) : read(io, Float64) - maxy = bswap ? Base.bswap(read(io, Float64)) : read(io, Float64) - end - - if bextent_has_z - minz = bswap ? Base.bswap(read(io, Float64)) : read(io, Float64) - maxz = bswap ? Base.bswap(read(io, Float64)) : read(io, Float64) - end - - if bextent_has_m - minm = bswap ? Base.bswap(read(io, Float64)) : read(io, Float64) - maxm = bswap ? Base.bswap(read(io, Float64)) : read(io, Float64) - end - - headerlen = position(io) - - return srsid, bextent_has_z, headerlen+1 -end - -function import_sf_from_wkb(io, srs_id, has_z, wkb_type, bswap) - # this srs_id (specfied in geopackage binary) can - srs_id = iszero(srs_id) || isone(abs(srs_id)) ? - ( isone(abs(srs_id)) ? Cartesian{NoDatum} : LatLon{WGS84Latest} ) : - ( Int(srs_id) > 54000 ? CoordRefSystems.get(ESRI{Int(srs_id)}) : CoordRefSystems.get(EPSG{Int(srs_id)}) ) -# Conflicts that arose here and were ?resolved -# -------------------------------------------- -# this conversion of Cartesian <--> Geographic works mostly -# -# Conversion error collecting 2D and 3D features in the same geotable -# -# Although I'm not so sure what the most correct way is for handling -# reading the undefined and/or cartographic/geographic reference systems -# - - - if wkb_type == wkbPoint || wkb_type == wkbPointZ - elem = _wkb_coordinate(io, has_z, bswap) - return meshes_creategeometry(wkb_type, srs_id, elem) - elseif wkb_type == wkbPolygon || wkb_type == wkbPolygonZ - elem = _wkb_polygon(io, has_z, bswap) - return meshes_creategeometry(wkb_type, srs_id, elem) - elseif wkb_type == wkbLineString || wkb_type == wkbLineStringZ - elem = _wkb_linestring(io, has_z, bswap) - return meshes_creategeometry(wkb_type, srs_id, elem) - else - @warn("unknown type, non standard: $wkb_type, hit the road") - end -end - -function import_geometry_from_wkb(io, srs_id, has_z::Bool) - ebyteorder = read(io, UInt8) - bswap = @BSWAP(ebyteorder) - - v_wkb_type = bswap ? Base.bswap(read(io, UInt32)) : read(io, UInt32) - local wkb_type - try - wkb_type::wkbGeometryType = wkbGeometryType(v_wkb_type) - catch e - @warn "wkbGeometrytype $v_wkb_type is not found in our list standard and non-standard types" - end - - wkb_enum::AbstractString = string(wkb_type) - if occursin("Multi", wkb_enum) - melems::Vector = _wkb_geometrycollection(io, srs_id, has_z, bswap) - return Meshes.Multi(melems) - elseif wkb_type == wkbGeometryCollection || wkb_type == wkbGeometryCollectionZ - elems::Vector{Geometry} = _wkb_geometrycollection(io, srs_id, has_z, bswap) - return elems - else - return import_sf_from_wkb(io, srs_id, has_z, wkb_type, bswap) - end -end - - - - -function _wkb_coordinate(io, has_z, bswap) - - x = bswap ? Base.bswap(read(io, Float64)) : read(io, Float64) - y = bswap ? Base.bswap(read(io, Float64)) : read(io, Float64) - - if has_z - z = bswap ? Base.bswap(read(io, Float64)) : read(io, Float64) - return SVector{3, Float64}(x,y,z) - end - - return SVector{2, Float64}(x,y) -end - -# Unitful coordinate values address many pitfalls in geospatial applications - - - -# a non-standard WKB representation, -# functionally equivalent to LineString but separate identity in GIS simple features data model -# It exists to serve as a component of a Polygon -function _wkb_linearring(io, has_z, bswap) - - num_points = bswap ? Base.bswap(read(io, UInt32)) : read(io, UInt32) - - points::Vector{SVector{has_z ? 3 : 2 , Float64}} = map(1:num_points) do _ - _wkb_coordinate(io, has_z, bswap) - end - - return points -end - -function _wkb_polygon(io, has_z, bswap) - - num_rings = bswap ? Base.bswap(read(io, UInt32)) : read(io, UInt32) - - rings = map(1:num_rings) do _ - _wkb_linearring(io, has_z, bswap) - end - - return rings -end - -function _wkb_linestring(io, has_z, bswap) - - num_points = bswap ? Base.bswap(read(io, UInt32)) : read(io, UInt32) - - points::Vector{SVector{has_z ? 3 : 2 , Float64}} = map(1:num_points) do _ - _wkb_coordinate(io, has_z, bswap) - end - return points -end - -function _wkb_geometrycollection(io, srs_id, has_z, bswap) - - num_geoms = bswap ? Base.bswap(read(io, UInt32)) : read(io, UInt32) - - geomcollection = map(1:num_geoms) do _ - import_geometry_from_wkb(io, srs_id, has_z) - end - - return geomcollection -end - - - -#--------------------------- -# SQLite Query Optimization -# -------------------------- -# -# - Avoid OR-connected constraints, use IN operator, or UNION constraints separately -# -# - GROUP BY or DISTINCT logic can determine if the current row is part of the same group -# or if the current row is distinct simply by comparing the current row to the previous row -# - faster than the alternative of comparing each row to all prior rows -# -#.- -# -# -------------------------- diff --git a/src/extra/gpkg/read.jl b/src/extra/gpkg/read.jl new file mode 100644 index 0000000..f60ada6 --- /dev/null +++ b/src/extra/gpkg/read.jl @@ -0,0 +1,298 @@ +# ------------------------------------------------------------------ +# Licensed under the MIT License. See LICENSE in the project root. +# ------------------------------------------------------------------ + +# query SQLite.Table given *.gpkg filename; optionally specify quantity of tables and features query +# default behavior for select statement is limit result to 1 feature geometry row and corresponding attributes row +# limited to only 1 feature table to select geopackage binary geometry from. +function gpkgread(fname ;ntables=1, nfeatures=1) + db = SQLite.DB(fname) + gpkg_identify(db) + geom = gpkgmesh(db, ; ntables=ntables, nfeatures=nfeatures) + attrs = gpkgmeshattrs(db, ; ntables=ntables, nfeatures=nfeatures) + GeoTables.georef(attrs, geom) +end + + +function gpkg_identify(db)::Bool + + application_id = DBInterface.execute(db, "PRAGMA application_id;") |> first |> only + user_version = DBInterface.execute(db, "PRAGMA user_version;") |> first |> only + + if !(_has_gpkg_required_metadata_tables(db)) + @error "missing required metadata tables" + end + + # Requirement 6: PRAGMA integrity_check returns a single row with the value 'ok' + # Requirement 7: PRAGMA foreign_key_check (w/ no parameter value) returns an empty result set + if (application_id != GP10_APPLICATION_ID) && + (application_id != GP11_APPLICATION_ID) && + (application_id != GPKG_APPLICATION_ID) + @warn "application_id not recognized" + return false + elseif(application_id == GPKG_APPLICATION_ID) && + !( + (user_version >= GPKG_1_2_VERSION && user_version < GPKG_1_2_VERSION + 99) || + (user_version >= GPKG_1_3_VERSION && user_version < GPKG_1_3_VERSION + 99) || + (user_version >= GPKG_1_4_VERSION && user_version < GPKG_1_4_VERSION + 99) + ) + @warn "application_id is valid but user version is not recognized" + elseif( + DBInterface.execute(db, "PRAGMA integrity_check;") |> first |> only != "ok" + ) || !( + isempty(DBInterface.execute(db, "PRAGMA foreign_key_check;")) + ) + @error "database integrity at risk or foreign key violation(s)" + return false + end + return true +end + +# Requirement 10: must include a gpkg_spatial_ref_sys table +# Requirement 13: must include a gpkg_contents table +function _has_gpkg_required_metadata_tables(db) + stmt_sql = "SELECT COUNT(*) FROM sqlite_master WHERE "* + "name IN ('gpkg_spatial_ref_sys', 'gpkg_contents') AND "* + "type IN ('table', 'view');" + required_metadata_tables = DBInterface.execute(db, stmt_sql) |> first |> only + return (required_metadata_tables == 2) +end + + + +function gpkgmeshattrs(db, ;ntables::Int=1, nfeatures=1) + stmt_sql = "SELECT c.table_name, c.identifier, "* + "g.column_name, g.geometry_type_name, g.z, g.m, c.min_x, c.min_y, "* + "c.max_x, c.max_y, "* + "(SELECT type FROM sqlite_master WHERE lower(name) = "* + "lower(c.table_name) AND type IN ('table', 'view')) AS object_type "* + " FROM gpkg_geometry_columns g "* + " JOIN gpkg_contents c ON (g.table_name = c.table_name)"* + " WHERE "* + " c.data_type = 'features' LIMIT $ntables" + feature_tables = DBInterface.execute(db, stmt_sql) + tb = []; fields = "fid," ^ 10^5 + for query in feature_tables + tn = query.table_name; cn = query.column_name; + ft_attrs = SQLite.tableinfo(db, tn).name + deleteat!(ft_attrs, findall(x -> isequal(x, cn), ft_attrs)) + rp_attrs = join(ft_attrs, ", ") + # keep the shortest set of attributes to avoid KeyError {Key} not found + fields = length(fields) > length(rp_attrs) ? rp_attrs : fields # smelly hack, eval shortest common subset of fields instead + stmt_sql = "SELECT $fields from $tn LIMIT $nfeatures;" + if isone(nfeatures) + rowvalues = DBInterface.execute(db, stmt_sql) |> first + push!(tb, rowvalues) + else + for rv in DBInterface.execute(db, stmt_sql) + push!(tb, NamedTuple(rv)) + end + end + end + return tb +end + + +############################################################################## +########### Features - Geometry Columns: Table Data Values ################### +############################################################################## +# https://www.geopackage.org/spec/#:~:text=2.1.5.1.2.%20Table%20Data%20Values +#------------------------------------------------------------------------------ +# # Requirement 21: a gpkg_contents table row with a "features" data_type +# SHALL contain a gpkg_geometry_columns table +# +# Requirement 22: gpkg_geometry_columns table +# SHALL contain one row record for the geometry column +# in each vector feature data table +# +# Requirement 23: gpkg_geometry_columns table_name column +# SHALL reference values in the gpkg_contents table_name column +# for rows with a data_type of 'features' +# +# Requirement 24: The column_name column value in a gpkg_geometry_columns row +# SHALL be the name of a column in the table or view specified by the table_name +# column value for that row. +# +# Requirement 25: The geometry_type_name value in a gpkg_geometry_columns row +# SHALL be one of the uppercase geometry type names specified + +# Requirement 26: The srs_id value in a gpkg_geometry_columns table row +# SHALL be an srs_id column value from the gpkg_spatial_ref_sys table. +# +# Requirement 27: The z value in a gpkg_geometry_columns table row SHALL be one +# of 0, 1, or 2. +# +# Requirement 28: The m value in a gpkg_geometry_columns table row SHALL be one +# of 0, 1, or 2. +# +# Requirement 146: The srs_id value in a gpkg_geometry_columns table row +# SHALL match the srs_id column value from the corresponding row in the +# gpkg_contents table. +# +function gpkgmesh(db, ;ntables=1, nfeatures=1) + stmt_sql =""" + SELECT g.table_name AS tn, g.column_name AS cn, c.srs_id as crs, g.z as elev, srs.organization as org, srs.organization_coordsys_id as org_coordsys_id, + ( SELECT type FROM sqlite_master WHERE lower(name) = lower(c.table_name) AND type IN ('table', 'view')) AS object_type + FROM gpkg_geometry_columns g, gpkg_spatial_ref_sys srs + JOIN gpkg_contents c ON ( g.table_name = c.table_name ) + WHERE c.data_type = 'features' + AND (SELECT type FROM sqlite_master WHERE lower(name) = lower(c.table_name) AND type IN ('table', 'view')) IS NOT NULL + AND g.srs_id = srs.srs_id + AND g.srs_id = c.srs_id + AND g.z IN (0, 1, 2) + AND g.m IN (0, 1, 2) + LIMIT $ntables; + """ + tb = DBInterface.execute(db, stmt_sql) + meshes = Meshes.Geometry[] + for (tn, cn, org, org_coordsys_id) in [(row.tn, row.cn, row.org, row.org_coordsys_id) for row in tb] + stmt_sql = "SELECT $cn FROM $tn LIMIT $nfeatures;" + gpkgbinary = DBInterface.execute(db, stmt_sql) + headerlen = 0 + for blob in gpkgbinary + io = IOBuffer(blob[1]) + seek(io, 3) + flag = read(io, UInt8) + # Note that Julia does not convert the endianness for you. + # Use ntoh or ltoh for this purpose. + bswap = isone(flag & 0x01) ? ltoh : ntoh + + srs_id = bswap(read(io, UInt32)) + + envelope = (flag & (0x07 << 1)) >> 1 + envelopedims = 0 + + if !iszero(envelope) + if isone(envelope) + envelopedims = 1 # 2D + elseif isequal(2, envelope) + envelopedims = 2 # 2D+Z + elseif isequal(3, envelope) + envelopedims = 3 # 2D+M + elseif isequal(4, envelope) + envelopedims = 4 # 2D+ZM + else + @error "exceeded dimensional limit for geometry, file may be corrupted or reader is broken" + false + end + else + true # no envelope (space saving slower indexing option), 0 bytes + end + + # header size in byte stream + headerlen = 8 + 8 * 4 * envelopedims + seek(io, headerlen) + + ebyteorder = read(io, UInt8) + + bswap = isone(ebyteorder) ? ltoh : ntoh + + wkbtype = wkbGeometryType(read(io, UInt32)) + + zextent = isequal(envelopedims, 2) + + mesh = meshfromwkb(io, srs_id, org, org_coordsys_id, wkbtype, zextent, bswap) + + if !isnothing(mesh) + push!(meshes, mesh) + end + end + end + return meshes +end + +function meshfromwkb(io, srs_id, org, org_coordsys_id, ewkbtype, extent_has_z, bswap) + if iszero(srs_id) + crs = LatLon{WGS84Latest} + elseif isone(abs(srs_id)) + crs = extent_has_z ? Cartesian{NoDatum, 3} : Cartesian{NoDatum, 2} + else + if org == "EPSG" + crs = CoordRefSystems.get(EPSG{org_coordsys_id}) + elseif org == "ESRI" + crs = CoordRefSystems.get(ERSI{org_coordsys_id}) + else + Cartesian{NoDatum} + end + end + + if occursin("Multi", string(ewkbtype)) + elems::Vector = wkbmultigeometry(io, crs, extent_has_z, bswap) + return Meshes.Multi(elems) + else + elem = meshfromsf(io, crs, ewkbtype, extent_has_z, bswap) + return elem + end +end + +################################################################ +# Requirement 20: GeoPackage SHALL store feature table geometries +# with the basic simple feature geometry types +# https://www.geopackage.org/spec140/index.html#geometry_types +# +function meshfromsf(io, crs, ewkbtype, extent_has_z, bswap) + if isequal(ewkbtype, wkbPoint) + elem = wkbcoordinate(io, extent_has_z, bswap) + return Meshes.Point(crs(elem...)) + elseif isequal(ewkbtype, wkbLineString) + elem = wkblinestring(io, extent_has_z, bswap) + if first(elem) != last(elem) + return Meshes.Rope([Meshes.Point(crs(coords...)) for coords in elem]...) + else + return Meshes.Ring([Meshes.Point(crs(coords...)) for coords in elem[2:end]]...) + end + elseif isequal(ewkbtype, wkbPolygon) + elem = wkbpolygon(io, extent_has_z, bswap) + rings = map(elem) do ring + coords = map(ring) do point + Meshes.Point(crs(point...)) + end + Meshes.Ring(coords) + end + + outer_ring = first(rings) + holes = isone(length(rings)) ? rings[2:end] : Meshes.Ring[] + return Meshes.PolyArea(outer_ring, holes...) + end +end + +function wkbcoordinate(io, z, bswap) + x = bswap(read(io, Float64)) + y = bswap(read(io, Float64)) + + if z + z = bswap(read(io, Float64)) + return x, y, z + end + + return x, y +end + +function wkblinestring(io, z, bswap) + npoints = bswap(read(io, UInt32)) + + points = map(1:npoints) do _ + wkbcoordinate(io, z, bswap) + end + return points +end + +function wkbpolygon(io, z, bswap) + nrings = bswap(read(io, UInt32)) + + rings = map(1:nrings) do _ + wkblinestring(io, z, bswap) + end + return rings +end + +function wkbmultigeometry(io, crs, z, bswap) + ngeoms = bswap(read(io, UInt32)) + + geomcollection = map(1:ngeoms) do _ + bswap = isone(read(io, UInt8)) ? ltoh : ntoh + ewkbtype = wkbGeometryType(read(io, UInt32)) + meshfromsf(io, crs, ewkbtype, z, bswap) + end + return geomcollection +end diff --git a/src/extra/gpkg/write.jl b/src/extra/gpkg/write.jl new file mode 100644 index 0000000..f14177a --- /dev/null +++ b/src/extra/gpkg/write.jl @@ -0,0 +1,294 @@ +function gpkgwrite(fname, geotable; ) + + db = SQLite.DB(fname) + + DBInterface.execute(db, "PRAGMA synchronous=0") + # Commits can be orders of magnitude faster with + # Setting PRAGMA synchronous=OFF but, + # can cause the database to go corrupt + # if there is an operating-system crash or power failure. + # If the power never goes out and no programs ever crash + # on you system then Synchronous = OFF is for you + #################################################### + + SQLite.transaction(db) do + + stmt_sql = """ + CREATE TABLE gpkg_spatial_ref_sys ( + srs_name TEXT NOT NULL, srs_id INTEGER NOT NULL PRIMARY KEY, + organization TEXT NOT NULL, organization_coordsys_id INTEGER NOT NULL, + definition TEXT NOT NULL, description TEXT, + definition_12_063 TEXT NOT NULL + ); + + CREATE TABLE gpkg_contents ( + table_name TEXT NOT NULL PRIMARY KEY, + data_type TEXT NOT NULL, + identifier TEXT UNIQUE, + description TEXT DEFAULT '', + last_change DATETIME NOT NULL DEFAULT + (strftime('%Y-%m-%dT%H:%M:%fZ','now')), + min_x DOUBLE, min_y DOUBLE, + max_x DOUBLE, max_y DOUBLE, + srs_id INTEGER, + CONSTRAINT fk_gc_r_srs_id FOREIGN KEY (srs_id) REFERENCES + gpkg_spatial_ref_sys(srs_id) + ); + """ + DBInterface.execute(db, stmt_sql) + stmt_sql = SQLite.Stmt(db, "PRAGMA application_id = $GPKG_APPLICATION_ID ") + DBInterface.execute(stmt_sql) + stmt_sql = SQLite.Stmt(db, "PRAGMA user_version = $GPKG_1_4_VERSION ") + DBInterface.execute(stmt_sql) + end + + ############################################################ + ### Requirement 11: Spatial Ref Sys Table Records ########## + ####### https://www.geopackage.org/spec/#r11 ############### + ### ######################################################## + # The gpkg_spatial_ref_sys table SHALL contain at a minimum + # 1. the record with an srs_id of 4326 SHALL correspond to WGS-84 as defined by EPSG in 4326 + # 2. the record with an srs_id of -1 SHALL be used for undefined Cartesian coordinate reference systems + # 3. the record with an srs_id of 0 SHALL be used for undefined geographic coordinate reference systems + ############################################################ + tb = [( + srs_name = "Undefined Cartesian SRS", + srs_id = -1, + organization = "NONE", + organization_coordsys_id = -1, + definition = "undefined", + description = "undefined geographic coordinate reference system", + definition_12_063 = "undefined", + )] + SQLite.load!(tb, db, "gpkg_spatial_ref_sys", replace=true) + tb = [( + srs_name = "Undefined geographic SRS", + srs_id = 0, + organization = "NONE", + organization_coordsys_id = 0, + definition = "undefined", + description = "undefined geographic coordinate reference system", + definition_12_063 = "undefined", + )] + SQLite.load!(tb, db, "gpkg_spatial_ref_sys", replace=true) + tb = [( + srs_name = "WGS 84 geodectic", + srs_id = 4326, + organization = "EPSG", + organization_coordsys_id = 4326, + definition = CoordRefSystems.wkt2(EPSG{4326}), + description = "longitude/latitude coordinates in decimal degrees on the WGS 84 spheroid", + definition_12_063 = CoordRefSystems.wkt2(EPSG{4326}), + )] + SQLite.load!(tb, db, "gpkg_spatial_ref_sys", replace=true) + table = values(geotable) + domain = GeoTables.domain(geotable) + crs = GeoTables.crs(domain) + geom = collect(domain) + + _extracttablevals(db, table, domain, crs, geom) +end + + + +function _extracttablevals(db, table, domain, crs, geom) + + if crs <: Cartesian + srs = "" + srid = -1 + elseif crs <: LatLon{WGS84Latest} + srs = "EPSG" + srid = 4326 + else + srs = string(CoordRefSystems.code(crs)) + srid = parse(Int32, srs) + end + + gpkgbinary = map(geom) do ft + gpkgbinheader = writegpkgheader(srid, ft) + io = IOBuffer() + writewkbgeom(io, ft) + vcat(gpkgbinheader, take!(io)) + end + + table = isone(length(table)) ? [(NamedTuple(table |> first)..., geom = gpkgbinary[1])] : [(; t..., geom = g) for (t, g) in zip(table, gpkgbinary)] + + SQLite.load!(table, db, replace=false) # autogenerates table name + # replace=false controls whether an INSERT INTO ... statement is generated or a REPLACE INTO .... + tn = (DBInterface.execute(db, """ SELECT name FROM sqlite_master WHERE type='table' AND name NOT IN ("gpkg_contents", "gpkg_spatial_ref_sys") """) |> first).name + + bbox = boundingbox(domain) + mincoords = CoordRefSystems.raw(coords(bbox.min)) + maxcoords = CoordRefSystems.raw(coords(bbox.max)) + contents = [(table_name = tn, + data_type = "features", + identifier = tn, + description = "", + last_change = Dates.format(now(UTC), "yyyy-mm-ddTHH:MM:SSZ"), + min_x = mincoords[1], + min_y = mincoords[2], + max_x = maxcoords[1], + max_y = maxcoords[2], + srs_id = srid + )] + SQLite.load!(contents, db, "gpkg_contents", replace=true) + if srid != 4326 + srstb = [( + srs_name = "", + srs_id = srid, + organization = srs[1:4], + organization_coordsys_id = srid, + definition = CoordRefSystems.wkt2(crs), + description = "", + definition_12_063 = CoordRefSystems.wkt2(crs), + )] + SQLite.load!(srstb, db, "gpkg_spatial_ref_sys", replace=true) + end + geomcolumns = [( + table_name = tn, + column_name = "geom", + geometry_type_name = _geomtype(geom), + srs_id = srid, + z = paramdim((geom |> first)) >= 2 ? 1 : 0, + m = 0 + )] + SQLite.load!(geomcolumns, db, "gpkg_geometry_columns", replace=true) +end + +function _geomtype(geoms::AbstractVector{<:Geometry}) + if isempty(geoms) + return "GEOMETRY" + end + T = eltype(geoms) + + if T <: Point + return "POINT" + elseif T <: Rope + return "LINESTRING" + elseif T <: Ring + return "LINESTRING" + elseif T <: PolyArea + return "POLYGON" + elseif T <: Multi + element_type = eltype(parent(first(geoms))) + if element_type <: Point + return "MULTIPOINT" + elseif element_type <: Rope + return "MULTILINESTRING" + elseif element_type <: PolyArea + return "MULTIPOLYGON" + end + end + return "GEOMETRY" +end + + +function _wkbtype(geometry) + if geometry isa Point + return wkbPoint + elseif geometry isa Rope || geometry isa Ring + return wkbLineString + elseif geometry isa PolyArea + return wkbPolygon + elseif geometry isa Multi + fg = parent(geometry) |> first + return wkbGeometryType(Int(_wkbtype(fg))+3) + else + @error "my hovercraft is full of eels: $geometry" + end +end + +function _wkbsetz(type::wkbGeometryType) + return wkbGeometryType(Int(type)+1000) + # ISO WKB Flavour +end + +function writegpkgheader(srs_id, geom) + io = IOBuffer() + write(io, [0x47, 0x50]) # 'GP' in ASCII + write(io, zero(UInt8)) # 0 = version 1 + + flagsbyte = UInt8(0x07 >> 1) + write(io, flagsbyte) + + write(io, htol(Int32(srs_id))) + + bbox = boundingbox(geom) + write(io, htol(Float64(CoordRefSystems.raw(coords(bbox.min))[2]))) + write(io, htol(Float64(CoordRefSystems.raw(coords(bbox.max))[2]))) + write(io, htol(Float64(CoordRefSystems.raw(coords(bbox.min))[1]))) + write(io, htol(Float64(CoordRefSystems.raw(coords(bbox.max))[1]))) + + if paramdim(geom) >= 3 + write(io, htol(Float64(CoordRefSystems.raw(coords(bbox.min))[3]))) + write(io, htol(Float64(CoordRefSystems.raw(coords(bbox.max))[3]))) + end + + return take!(io) +end + +function writewkbgeom(io, geom) + wkbtype = paramdim(geom) < 3 ? _wkbtype(geom) : _wkbsetz(_wkbtype(geom)); + write(io, one(UInt8)) + if Int(wkbtype) > 3 + write(io, htol(UInt32(wkbtype))) + write(io, htol(UInt32(length(geom |> parent)))) + + for ft in parent(geom) + writewkbgeom(io, ft) + end + else + if wkbtype == wkbPolygon || wkbtype == wkbPolygonZ || wkbtype == wkbPolygon25D + _wkbpolygon(io, wkbtype, [boundary(geom::PolyArea)]) + elseif wkbtype == wkbLineString || wkbtype == wkbLineString || wkbtype == wkbLineString25D + coordlist = vertices(geom) + write(io, htol(UInt32(wkbtype))) + if geom isa Meshes.Ring + write(io, htol(UInt32(length(coordlist)+1))) + else + write(io, htol(UInt32(length(coordlist)))) + end + + + _wkblinestring(io, wkbtype, coordlist) + if geom isa Meshes.Ring + points = CoordRefSystems.raw(coords(coordlist |> first)) + _wkbcoordinates(io, wkbtype, points) + end + elseif wkbtype == wkbPoint || wkbtype == wkbPointZ || wkbtype == wkbPoint25D + coordinates = CoordRefSystems.raw(coords(geom)) + _wkbcoordinates(io, wkbtype, coordinates) + else + @error "my hovercraft is full of eels: $wkbtype" + end + end +end + + +function _wkbcoordinates(io, wkbtype, coords) + + write(io, htol(Float64(coords[2]))) + write(io, htol(Float64(coords[1]))) + + if (UInt32(wkbtype) > 1000) || !(iszero(Int(wkbtype) & (0x80000000 | 0x40000000))) + write(io, htol(Float64(coords[3]))) + end +end + +function _wkblinestring(io, wkb_type, coord_list) + for n_coords::Point in coord_list + coordinates = CoordRefSystems.raw(coords(n_coords)) + _wkbcoordinates(io, wkb_type, coordinates) + end +end + +function _wkbpolygon(io, wkb_type, rings) + write(io, htol(wkb_type)) + write(io, htol(length(rings))) + + for ring in rings + coord_list = vertices(ring) + write(io, htol(UInt32(length(coord_list) + 1))) + _wkblinestring(io, wkb_type, coord_list) + end +end From 43a87afa96b6d8be0e26d64dc0082be9c098cb91 Mon Sep 17 00:00:00 2001 From: jph6366 Date: Tue, 9 Sep 2025 01:41:31 -0400 Subject: [PATCH 03/90] SQLite package install, read/write includes, SQLite.DBInterface Will comb through for various edges cases (empty, 3D, etc.) once the writer is gpkg spec-compliant implementation --- Project.toml | 2 ++ src/GeoIO.jl | 3 ++- src/extra/gis.jl | 2 +- src/extra/gpkg.jl | 4 ++++ src/extra/gpkg/read.jl | 2 ++ src/extra/gpkg/write.jl | 6 ++++++ 6 files changed, 17 insertions(+), 2 deletions(-) diff --git a/Project.toml b/Project.toml index fea5879..15209f1 100644 --- a/Project.toml +++ b/Project.toml @@ -27,6 +27,7 @@ PlyIO = "42171d58-473b-503a-8d5f-782019eb09ec" PrecompileTools = "aea7be01-6a6a-4083-8856-8a6e6704d82a" PrettyTables = "08abe8d2-0d0c-5749-adfa-8a2ac140af0d" ReadVTK = "dc215faf-f008-4882-a9f7-a79a826fadc3" +SQLite = "0aa819cd-b072-5ff4-a722-6bc24af294d9" Shapefile = "8e980c4a-a4fe-5da2-b3a7-4b4b0353a2f4" StaticArrays = "90137ffa-7385-5640-81b9-e52037218182" Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c" @@ -59,6 +60,7 @@ PlyIO = "1.1" PrecompileTools = "1.2" PrettyTables = "2.2" ReadVTK = "0.2" +SQLite = "1.6.1" Shapefile = "0.13" StaticArrays = "1.6" Tables = "1.7" diff --git a/src/GeoIO.jl b/src/GeoIO.jl index 4cabd47..8786658 100644 --- a/src/GeoIO.jl +++ b/src/GeoIO.jl @@ -75,7 +75,7 @@ const CDMEXTS = [".grib", ".nc"] const FORMATS = [ (extension=".csv", load="CSV.jl", save="CSV.jl"), (extension=".geojson", load="GeoJSON.jl", save="GeoJSON.jl"), - (extension=".gpkg", load="GeoIO.jl", save="GeIO.jl"), + (extension=".gpkg", load="GeoIO.jl", save="GeoIO.jl"), (extension=".grib", load="GRIBDatasets.jl", save=""), (extension=".gslib", load="GslibIO.jl", save="GslibIO.jl"), (extension=".jpeg", load="ImageIO.jl", save="ImageIO.jl"), @@ -130,6 +130,7 @@ include("extra/csv.jl") include("extra/gdal.jl") include("extra/geotiff.jl") include("extra/gis.jl") +include("extra/gpkg.jl") include("extra/img.jl") include("extra/msh.jl") include("extra/obj.jl") diff --git a/src/extra/gis.jl b/src/extra/gis.jl index 85c5420..9fdfa01 100644 --- a/src/extra/gis.jl +++ b/src/extra/gis.jl @@ -66,7 +66,7 @@ function gistable(fname; layer, numtype, kwargs...) elseif endswith(fname, ".parquet") return GPQ.read(fname; kwargs...) elseif endswith(fname, ".gpkg") - return gpkgread(fname; layer, kwargs...) + return gpkgread(fname; kwargs...) else # fallback to GDAL data = AG.read(fname; kwargs...) return AG.getlayer(data, layer - 1) diff --git a/src/extra/gpkg.jl b/src/extra/gpkg.jl index 1c3a113..c35b557 100644 --- a/src/extra/gpkg.jl +++ b/src/extra/gpkg.jl @@ -100,3 +100,7 @@ const GPKG_1_4_VERSION = 10400 wkbMultiPolygon25D = 0x80000006 wkbGeometryCollection25D = 0x80000007 end + + +include("gpkg/read.jl") +include("gpkg/write.jl") diff --git a/src/extra/gpkg/read.jl b/src/extra/gpkg/read.jl index f60ada6..174b8c4 100644 --- a/src/extra/gpkg/read.jl +++ b/src/extra/gpkg/read.jl @@ -2,6 +2,8 @@ # Licensed under the MIT License. See LICENSE in the project root. # ------------------------------------------------------------------ +const DBInterface = SQLite.DBInterface + # query SQLite.Table given *.gpkg filename; optionally specify quantity of tables and features query # default behavior for select statement is limit result to 1 feature geometry row and corresponding attributes row # limited to only 1 feature table to select geopackage binary geometry from. diff --git a/src/extra/gpkg/write.jl b/src/extra/gpkg/write.jl index f14177a..eb78b3d 100644 --- a/src/extra/gpkg/write.jl +++ b/src/extra/gpkg/write.jl @@ -1,3 +1,9 @@ +# ------------------------------------------------------------------ +# Licensed under the MIT License. See LICENSE in the project root. +# ------------------------------------------------------------------ + +const DBInterface = SQLite.DBInterface + function gpkgwrite(fname, geotable; ) db = SQLite.DB(fname) From 91ed1efa214a58e4ad1627e423868f0713215e06 Mon Sep 17 00:00:00 2001 From: jph6366 Date: Wed, 10 Sep 2025 09:58:03 -0400 Subject: [PATCH 04/90] formatter installed, addressing comments and some varname changes, and import DBInterface --- Project.toml | 4 +- src/GeoIO.jl | 1 + src/extra/gpkg.jl | 153 ++++++------- src/extra/gpkg/read.jl | 409 +++++++++++++++++------------------ src/extra/gpkg/write.jl | 462 ++++++++++++++++++++-------------------- 5 files changed, 511 insertions(+), 518 deletions(-) diff --git a/Project.toml b/Project.toml index 15209f1..f2cc7a5 100644 --- a/Project.toml +++ b/Project.toml @@ -9,6 +9,7 @@ CSV = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b" Colors = "5ae59095-9a9b-59fe-a467-6f913c188581" CommonDataModel = "1fbeeb36-5f17-413c-809b-666fb144f157" CoordRefSystems = "b46f11dc-f210-4604-bfba-323c1ec968cb" +DBInterface = "a10d1c49-ce27-4219-8d33-6db1a4562965" FileIO = "5789e2e9-d7fb-5bc7-8068-2c6fae9b9549" Format = "1fa38f19-a742-5d3f-a2b9-30dd87b9d5f8" GRIBDatasets = "82be9cdb-ee19-4151-bdb3-b400788d9abc" @@ -42,6 +43,7 @@ CSV = "0.10" Colors = "0.12, 0.13" CommonDataModel = "0.2, 0.3" CoordRefSystems = "0.18" +DBInterface = "2.6.1" FileIO = "1.16" Format = "1.3" GRIBDatasets = "0.3, 0.4" @@ -60,7 +62,7 @@ PlyIO = "1.1" PrecompileTools = "1.2" PrettyTables = "2.2" ReadVTK = "0.2" -SQLite = "1.6.1" +SQLite = "1.6" Shapefile = "0.13" StaticArrays = "1.6" Tables = "1.7" diff --git a/src/GeoIO.jl b/src/GeoIO.jl index 8786658..d888e72 100644 --- a/src/GeoIO.jl +++ b/src/GeoIO.jl @@ -39,6 +39,7 @@ import PlyIO import CSV # Database Interfaces +import DBInterface import SQLite # geostats formats diff --git a/src/extra/gpkg.jl b/src/extra/gpkg.jl index c35b557..7b0f2d5 100644 --- a/src/extra/gpkg.jl +++ b/src/extra/gpkg.jl @@ -19,88 +19,89 @@ const GPKG_1_4_VERSION = 10400 # List of well known binary geometry types. # These are used within the GeoPackageBinary SQL BLOBs -@enum wkbGeometryType::Int64 begin - wkbUnknown = 0 - wkbPoint = 1 - wkbLineString = 2 - wkbPolygon = 3 - wkbMultiPoint = 4 - wkbMultiLineString = 5 - wkbMultiPolygon = 6 - wkbGeometryCollection = 7 - wkbCircularString = 8 - wkbCompoundCurve = 9 - wkbCurvePolygon = 10 - wkbMultiCurve = 11 - wkbMultiSurface = 12 - wkbCurve = 13 - wkbSurface = 14 +@enum wkbGeometryType begin + wkbUnknown = 0 + wkbPoint = 1 + wkbLineString = 2 + wkbPolygon = 3 + wkbMultiPoint = 4 + wkbMultiLineString = 5 + wkbMultiPolygon = 6 + wkbGeometryCollection = 7 + wkbCircularString = 8 + wkbCompoundCurve = 9 + wkbCurvePolygon = 10 + wkbMultiCurve = 11 + wkbMultiSurface = 12 + wkbCurve = 13 + wkbSurface = 14 - wkbNone = 100 # pure attribute records - wkbLinearRing = 101 + wkbNone = 100 # pure attribute records + wkbLinearRing = 101 - # ISO SQL/MM Part 3: Spatial - # Z-aware types - wkbPointZ = 1001 - wkbLineStringZ = 1002 - wkbPolygonZ = 1003 - wkbMultiPointZ = 1004 - wkbMultiLineStringZ = 1005 - wkbMultiPolygonZ = 1006 - wkbGeometryCollectionZ = 1007 - wkbCircularStringZ = 1008 - wkbCompoundCurveZ = 1009 - wkbCurvePolygonZ = 1010 - wkbMultiCurveZ = 1011 - wkbMultiSurfaceZ = 1012 - wkbCurveZ = 1013 - wkbSurfaceZ = 1014 + # ISO SQL/MM Part 3: Spatial + # Z-aware types + wkbPointZ = 1001 + wkbLineStringZ = 1002 + wkbPolygonZ = 1003 + wkbMultiPointZ = 1004 + wkbMultiLineStringZ = 1005 + wkbMultiPolygonZ = 1006 + wkbGeometryCollectionZ = 1007 + wkbCircularStringZ = 1008 + wkbCompoundCurveZ = 1009 + wkbCurvePolygonZ = 1010 + wkbMultiCurveZ = 1011 + wkbMultiSurfaceZ = 1012 + wkbCurveZ = 1013 + wkbSurfaceZ = 1014 - # ISO SQL/MM Part 3. - # M-aware types - wkbPointM = 2001 - wkbLineStringM = 2002 - wkbPolygonM = 2003 - wkbMultiPointM = 2004 - wkbMultiLineStringM = 2005 - wkbMultiPolygonM = 2006 - wkbGeometryCollectionM = 2007 - wkbCircularStringM = 2008 - wkbCompoundCurveM = 2009 - wkbCurvePolygonM = 2010 - wkbMultiCurveM = 2011 - wkbMultiSurfaceM = 2012 - wkbCurveM = 2013 - wkbSurfaceM = 2014 + # ISO SQL/MM Part 3. + # M-aware types + wkbPointM = 2001 + wkbLineStringM = 2002 + wkbPolygonM = 2003 + wkbMultiPointM = 2004 + wkbMultiLineStringM = 2005 + wkbMultiPolygonM = 2006 + wkbGeometryCollectionM = 2007 + wkbCircularStringM = 2008 + wkbCompoundCurveM = 2009 + wkbCurvePolygonM = 2010 + wkbMultiCurveM = 2011 + wkbMultiSurfaceM = 2012 + wkbCurveM = 2013 + wkbSurfaceM = 2014 - # ISO SQL/MM Part 3. - # ZM-aware types ... Meshes.jl doesn't generally support this? - wkbPointZM = 3001 - wkbLineStringZM = 3002 - wkbPolygonZM = 3003 - wkbMultiPointZM = 3004 - wkbMultiLineStringZM = 3005 - wkbMultiPolygonZM = 3006 - wkbGeometryCollectionZM = 3007 - wkbCircularStringZM = 3008 - wkbCompoundCurveZM = 3009 - wkbCurvePolygonZM = 3010 - wkbMultiCurveZM = 3011 - wkbMultiSurfaceZM = 3012 - wkbCurveZM = 3013 - wkbSurfaceZM = 3014 + # ISO SQL/MM Part 3. + # ZM-aware types ... Meshes.jl doesn't generally support this? + wkbPointZM = 3001 + wkbLineStringZM = 3002 + wkbPolygonZM = 3003 + wkbMultiPointZM = 3004 + wkbMultiLineStringZM = 3005 + wkbMultiPolygonZM = 3006 + wkbGeometryCollectionZM = 3007 + wkbCircularStringZM = 3008 + wkbCompoundCurveZM = 3009 + wkbCurvePolygonZM = 3010 + wkbMultiCurveZM = 3011 + wkbMultiSurfaceZM = 3012 + wkbCurveZM = 3013 + wkbSurfaceZM = 3014 - # 2.5D extension as per 99-402 - # https://lists.osgeo.org/pipermail/postgis-devel/2004-December/000702.html - wkbPoint25D = 0x80000001 - wkbLineString25D = 0x80000002 - wkbPolygon25D = 0x80000003 - wkbMultiPoint25D = 0x80000004 - wkbMultiLineString25D = 0x80000005 - wkbMultiPolygon25D = 0x80000006 - wkbGeometryCollection25D = 0x80000007 -end + # 2.5D extension as per 99-402 + # https://lists.osgeo.org/pipermail/postgis-devel/2004-December/000702.html + # Julia throws InexactError: convert(Int32, 0x8000000x) + wkbPoint25D = -2147483647 # 0x80000001 + wkbLineString25D = -2147483646 # 0x80000002 + wkbPolygon25D = -2147483645 # 0x80000003 + wkbMultiPoint25D = -2147483644 # 0x80000004 + wkbMultiLineString25D = -2147483643 # 0x80000005 + wkbMultiPolygon25D = -2147483642 # 0x80000006 + wkbGeometryCollection25D = -2147483641 # 0x80000007 +end include("gpkg/read.jl") include("gpkg/write.jl") diff --git a/src/extra/gpkg/read.jl b/src/extra/gpkg/read.jl index 174b8c4..f79b56c 100644 --- a/src/extra/gpkg/read.jl +++ b/src/extra/gpkg/read.jl @@ -2,99 +2,90 @@ # Licensed under the MIT License. See LICENSE in the project root. # ------------------------------------------------------------------ -const DBInterface = SQLite.DBInterface - # query SQLite.Table given *.gpkg filename; optionally specify quantity of tables and features query # default behavior for select statement is limit result to 1 feature geometry row and corresponding attributes row # limited to only 1 feature table to select geopackage binary geometry from. -function gpkgread(fname ;ntables=1, nfeatures=1) - db = SQLite.DB(fname) - gpkg_identify(db) - geom = gpkgmesh(db, ; ntables=ntables, nfeatures=nfeatures) - attrs = gpkgmeshattrs(db, ; ntables=ntables, nfeatures=nfeatures) - GeoTables.georef(attrs, geom) +function gpkgread(fname; ntables=1, nfeatures=1) + db = SQLite.DB(fname) + gpkgid(db) + geom = gpkgmesh(db, ; ntables=ntables, nfeatures=nfeatures) + attrs = gpkgmeshattrs(db, ; ntables=ntables, nfeatures=nfeatures) + GeoTables.georef(attrs, geom) end - -function gpkg_identify(db)::Bool - - application_id = DBInterface.execute(db, "PRAGMA application_id;") |> first |> only - user_version = DBInterface.execute(db, "PRAGMA user_version;") |> first |> only - - if !(_has_gpkg_required_metadata_tables(db)) - @error "missing required metadata tables" - end - - # Requirement 6: PRAGMA integrity_check returns a single row with the value 'ok' - # Requirement 7: PRAGMA foreign_key_check (w/ no parameter value) returns an empty result set - if (application_id != GP10_APPLICATION_ID) && - (application_id != GP11_APPLICATION_ID) && - (application_id != GPKG_APPLICATION_ID) - @warn "application_id not recognized" - return false - elseif(application_id == GPKG_APPLICATION_ID) && - !( - (user_version >= GPKG_1_2_VERSION && user_version < GPKG_1_2_VERSION + 99) || - (user_version >= GPKG_1_3_VERSION && user_version < GPKG_1_3_VERSION + 99) || - (user_version >= GPKG_1_4_VERSION && user_version < GPKG_1_4_VERSION + 99) - ) - @warn "application_id is valid but user version is not recognized" - elseif( - DBInterface.execute(db, "PRAGMA integrity_check;") |> first |> only != "ok" - ) || !( - isempty(DBInterface.execute(db, "PRAGMA foreign_key_check;")) - ) - @error "database integrity at risk or foreign key violation(s)" - return false - end - return true +function gpkgid(db)::Bool + appid = DBInterface.execute(db, "PRAGMA application_id;") |> first |> only + userversion = DBInterface.execute(db, "PRAGMA user_version;") |> first |> only + + if !(hasgpkgmetadata(db)) + @error "missing required metadata tables" + end + + # Requirement 6: PRAGMA integrity_check returns a single row with the value 'ok' + # Requirement 7: PRAGMA foreign_key_check (w/ no parameter value) returns an empty result set + if (appid != GP10_APPLICATION_ID) && (appid != GP11_APPLICATION_ID) && (appid != GPKG_APPLICATION_ID) + @warn "application_id not recognized" + return false + elseif (appid == GPKG_APPLICATION_ID) && !( + (userversion >= GPKG_1_2_VERSION && userversion < GPKG_1_2_VERSION + 99) || + (userversion >= GPKG_1_3_VERSION && userversion < GPKG_1_3_VERSION + 99) || + (userversion >= GPKG_1_4_VERSION && userversion < GPKG_1_4_VERSION + 99) + ) + @warn "application_id is valid but user version is not recognized" + elseif (DBInterface.execute(db, "PRAGMA integrity_check;") |> first |> only != "ok") || + !(isempty(DBInterface.execute(db, "PRAGMA foreign_key_check;"))) + @error "database integrity at risk or foreign key violation(s)" + return false + end + return true end # Requirement 10: must include a gpkg_spatial_ref_sys table # Requirement 13: must include a gpkg_contents table -function _has_gpkg_required_metadata_tables(db) - stmt_sql = "SELECT COUNT(*) FROM sqlite_master WHERE "* - "name IN ('gpkg_spatial_ref_sys', 'gpkg_contents') AND "* - "type IN ('table', 'view');" - required_metadata_tables = DBInterface.execute(db, stmt_sql) |> first |> only - return (required_metadata_tables == 2) +function hasgpkgmetadata(db) + stmt_sql = + "SELECT COUNT(*) FROM sqlite_master WHERE " * + "name IN ('gpkg_spatial_ref_sys', 'gpkg_contents') AND " * + "type IN ('table', 'view');" + tbcount = DBInterface.execute(db, stmt_sql) |> first |> only + return (tbcount == 2) end - - -function gpkgmeshattrs(db, ;ntables::Int=1, nfeatures=1) - stmt_sql = "SELECT c.table_name, c.identifier, "* - "g.column_name, g.geometry_type_name, g.z, g.m, c.min_x, c.min_y, "* - "c.max_x, c.max_y, "* - "(SELECT type FROM sqlite_master WHERE lower(name) = "* - "lower(c.table_name) AND type IN ('table', 'view')) AS object_type "* - " FROM gpkg_geometry_columns g "* - " JOIN gpkg_contents c ON (g.table_name = c.table_name)"* - " WHERE "* - " c.data_type = 'features' LIMIT $ntables" - feature_tables = DBInterface.execute(db, stmt_sql) - tb = []; fields = "fid," ^ 10^5 - for query in feature_tables - tn = query.table_name; cn = query.column_name; - ft_attrs = SQLite.tableinfo(db, tn).name - deleteat!(ft_attrs, findall(x -> isequal(x, cn), ft_attrs)) - rp_attrs = join(ft_attrs, ", ") - # keep the shortest set of attributes to avoid KeyError {Key} not found - fields = length(fields) > length(rp_attrs) ? rp_attrs : fields # smelly hack, eval shortest common subset of fields instead - stmt_sql = "SELECT $fields from $tn LIMIT $nfeatures;" - if isone(nfeatures) - rowvalues = DBInterface.execute(db, stmt_sql) |> first - push!(tb, rowvalues) - else - for rv in DBInterface.execute(db, stmt_sql) - push!(tb, NamedTuple(rv)) - end - end +function gpkgmeshattrs(db, ; ntables::Int=1, nfeatures=1) + sqlstmt = + "SELECT c.table_name, c.identifier, " * + "g.column_name, g.geometry_type_name, g.z, g.m, c.min_x, c.min_y, " * + "c.max_x, c.max_y, " * + "(SELECT type FROM sqlite_master WHERE lower(name) = " * + "lower(c.table_name) AND type IN ('table', 'view')) AS object_type " * + " FROM gpkg_geometry_columns g " * + " JOIN gpkg_contents c ON (g.table_name = c.table_name)" * + " WHERE " * + " c.data_type = 'features' LIMIT $ntables" + feature_tables = DBInterface.execute(db, sqlstmt) + tb = [] + fields = "fid,"^10^5 + for query in feature_tables + tn = query.table_name + cn = query.column_name + ft_attrs = SQLite.tableinfo(db, tn).name + deleteat!(ft_attrs, findall(x -> isequal(x, cn), ft_attrs)) + rp_attrs = join(ft_attrs, ", ") + # keep the shortest set of attributes to avoid KeyError {Key} not found + fields = length(fields) > length(rp_attrs) ? rp_attrs : fields # smelly hack, eval shortest common subset of fields instead + sqlstmt = "SELECT $fields from $tn LIMIT $nfeatures;" + if isone(nfeatures) + rowvalues = DBInterface.execute(db, sqlstmt) |> first + push!(tb, rowvalues) + else + for rv in DBInterface.execute(db, sqlstmt) + push!(tb, NamedTuple(rv)) + end end - return tb + end + return tb end - ############################################################################## ########### Features - Geometry Columns: Table Data Values ################### ############################################################################## @@ -131,100 +122,100 @@ end # SHALL match the srs_id column value from the corresponding row in the # gpkg_contents table. # -function gpkgmesh(db, ;ntables=1, nfeatures=1) - stmt_sql =""" - SELECT g.table_name AS tn, g.column_name AS cn, c.srs_id as crs, g.z as elev, srs.organization as org, srs.organization_coordsys_id as org_coordsys_id, - ( SELECT type FROM sqlite_master WHERE lower(name) = lower(c.table_name) AND type IN ('table', 'view')) AS object_type - FROM gpkg_geometry_columns g, gpkg_spatial_ref_sys srs - JOIN gpkg_contents c ON ( g.table_name = c.table_name ) - WHERE c.data_type = 'features' - AND (SELECT type FROM sqlite_master WHERE lower(name) = lower(c.table_name) AND type IN ('table', 'view')) IS NOT NULL - AND g.srs_id = srs.srs_id - AND g.srs_id = c.srs_id - AND g.z IN (0, 1, 2) - AND g.m IN (0, 1, 2) - LIMIT $ntables; - """ - tb = DBInterface.execute(db, stmt_sql) - meshes = Meshes.Geometry[] - for (tn, cn, org, org_coordsys_id) in [(row.tn, row.cn, row.org, row.org_coordsys_id) for row in tb] - stmt_sql = "SELECT $cn FROM $tn LIMIT $nfeatures;" - gpkgbinary = DBInterface.execute(db, stmt_sql) - headerlen = 0 - for blob in gpkgbinary - io = IOBuffer(blob[1]) - seek(io, 3) - flag = read(io, UInt8) - # Note that Julia does not convert the endianness for you. - # Use ntoh or ltoh for this purpose. - bswap = isone(flag & 0x01) ? ltoh : ntoh - - srs_id = bswap(read(io, UInt32)) +function gpkgmesh(db, ; ntables=1, nfeatures=1) + sqlstmt = """ + SELECT g.table_name AS tn, g.column_name AS cn, c.srs_id as crs, g.z as elev, srs.organization as org, srs.organization_coordsys_id as org_coordsys_id, + ( SELECT type FROM sqlite_master WHERE lower(name) = lower(c.table_name) AND type IN ('table', 'view')) AS object_type + FROM gpkg_geometry_columns g, gpkg_spatial_ref_sys srs + JOIN gpkg_contents c ON ( g.table_name = c.table_name ) + WHERE c.data_type = 'features' + AND (SELECT type FROM sqlite_master WHERE lower(name) = lower(c.table_name) AND type IN ('table', 'view')) IS NOT NULL + AND g.srs_id = srs.srs_id + AND g.srs_id = c.srs_id + AND g.z IN (0, 1, 2) + AND g.m IN (0, 1, 2) + LIMIT $ntables; + """ + tb = DBInterface.execute(db, stmt_sql) + meshes = Meshes.Geometry[] + for (tn, cn, org, org_coordsys_id) in [(row.tn, row.cn, row.org, row.org_coordsys_id) for row in tb] + sqlstmt = "SELECT $cn FROM $tn LIMIT $nfeatures;" + gpkgbinary = DBInterface.execute(db, sqlstmt) + headerlen = 0 + for blob in gpkgbinary + io = IOBuffer(blob[1]) + seek(io, 3) + flag = read(io, UInt8) + # Note that Julia does not convert the endianness for you. + # Use ntoh or ltoh for this purpose. + bswap = isone(flag & 0x01) ? ltoh : ntoh + + srs_id = bswap(read(io, UInt32)) + + envelope = (flag & (0x07 << 1)) >> 1 + envelopedims = 0 + + if !iszero(envelope) + if isone(envelope) + envelopedims = 1 # 2D + elseif isequal(2, envelope) + envelopedims = 2 # 2D+Z + elseif isequal(3, envelope) + envelopedims = 3 # 2D+M + elseif isequal(4, envelope) + envelopedims = 4 # 2D+ZM + else + @error "exceeded dimensional limit for geometry, file may be corrupted or reader is broken" + false + end + else + true # no envelope (space saving slower indexing option), 0 bytes + end - envelope = (flag & (0x07 << 1)) >> 1 - envelopedims = 0 + # header size in byte stream + headerlen = 8 + 8 * 4 * envelopedims + seek(io, headerlen) - if !iszero(envelope) - if isone(envelope) - envelopedims = 1 # 2D - elseif isequal(2, envelope) - envelopedims = 2 # 2D+Z - elseif isequal(3, envelope) - envelopedims = 3 # 2D+M - elseif isequal(4, envelope) - envelopedims = 4 # 2D+ZM - else - @error "exceeded dimensional limit for geometry, file may be corrupted or reader is broken" - false - end - else - true # no envelope (space saving slower indexing option), 0 bytes - end + ebyteorder = read(io, UInt8) - # header size in byte stream - headerlen = 8 + 8 * 4 * envelopedims - seek(io, headerlen) + bswap = isone(ebyteorder) ? ltoh : ntoh - ebyteorder = read(io, UInt8) + wkbtype = wkbGeometryType(read(io, UInt32)) - bswap = isone(ebyteorder) ? ltoh : ntoh - - wkbtype = wkbGeometryType(read(io, UInt32)) + zextent = isequal(envelopedims, 2) - zextent = isequal(envelopedims, 2) - - mesh = meshfromwkb(io, srs_id, org, org_coordsys_id, wkbtype, zextent, bswap) + mesh = meshfromwkb(io, srs_id, org, org_coordsys_id, wkbtype, zextent, bswap) - if !isnothing(mesh) - push!(meshes, mesh) - end - end + if !isnothing(mesh) + push!(meshes, mesh) + end end - return meshes + end + return meshes end -function meshfromwkb(io, srs_id, org, org_coordsys_id, ewkbtype, extent_has_z, bswap) - if iszero(srs_id) - crs = LatLon{WGS84Latest} - elseif isone(abs(srs_id)) - crs = extent_has_z ? Cartesian{NoDatum, 3} : Cartesian{NoDatum, 2} +function meshfromwkb(io, srs_id, org, org_coordsys_id, ewkbtype, zextent, bswap) + if iszero(srs_id) + crs = LatLon{WGS84Latest} + elseif isone(abs(srs_id)) + crs = zextent ? Cartesian{NoDatum,3} : Cartesian{NoDatum,2} + else + if org == "EPSG" + crs = CoordRefSystems.get(EPSG{org_coordsys_id}) + elseif org == "ESRI" + crs = CoordRefSystems.get(ERSI{org_coordsys_id}) else - if org == "EPSG" - crs = CoordRefSystems.get(EPSG{org_coordsys_id}) - elseif org == "ESRI" - crs = CoordRefSystems.get(ERSI{org_coordsys_id}) - else - Cartesian{NoDatum} - end - end - - if occursin("Multi", string(ewkbtype)) - elems::Vector = wkbmultigeometry(io, crs, extent_has_z, bswap) - return Meshes.Multi(elems) - else - elem = meshfromsf(io, crs, ewkbtype, extent_has_z, bswap) - return elem + Cartesian{NoDatum} end + end + + if occursin("Multi", string(ewkbtype)) + elems::Vector = wkbmultigeometry(io, crs, zextent, bswap) + return Meshes.Multi(elems) + else + elem = meshfromsf(io, crs, ewkbtype, zextent, bswap) + return elem + end end ################################################################ @@ -232,69 +223,69 @@ end # with the basic simple feature geometry types # https://www.geopackage.org/spec140/index.html#geometry_types # -function meshfromsf(io, crs, ewkbtype, extent_has_z, bswap) - if isequal(ewkbtype, wkbPoint) - elem = wkbcoordinate(io, extent_has_z, bswap) - return Meshes.Point(crs(elem...)) - elseif isequal(ewkbtype, wkbLineString) - elem = wkblinestring(io, extent_has_z, bswap) - if first(elem) != last(elem) - return Meshes.Rope([Meshes.Point(crs(coords...)) for coords in elem]...) - else - return Meshes.Ring([Meshes.Point(crs(coords...)) for coords in elem[2:end]]...) - end - elseif isequal(ewkbtype, wkbPolygon) - elem = wkbpolygon(io, extent_has_z, bswap) - rings = map(elem) do ring - coords = map(ring) do point - Meshes.Point(crs(point...)) - end - Meshes.Ring(coords) - end - - outer_ring = first(rings) - holes = isone(length(rings)) ? rings[2:end] : Meshes.Ring[] - return Meshes.PolyArea(outer_ring, holes...) +function meshfromsf(io, crs, ewkbtype, zextent, bswap) + if isequal(ewkbtype, wkbPoint) + elem = wkbcoordinate(io, zextent, bswap) + return Meshes.Point(crs(elem...)) + elseif isequal(ewkbtype, wkbLineString) + elem = wkblinestring(io, zextent, bswap) + if first(elem) != last(elem) + return Meshes.Rope([Meshes.Point(crs(coords...)) for coords in elem]...) + else + return Meshes.Ring([Meshes.Point(crs(coords...)) for coords in elem[2:end]]...) + end + elseif isequal(ewkbtype, wkbPolygon) + elem = wkbpolygon(io, zextent, bswap) + rings = map(elem) do ring + coords = map(ring) do point + Meshes.Point(crs(point...)) + end + Meshes.Ring(coords) end + + outerring = first(rings) + holes = isone(length(rings)) ? rings[2:end] : Meshes.Ring[] + return Meshes.PolyArea(outerring, holes...) + end end function wkbcoordinate(io, z, bswap) - x = bswap(read(io, Float64)) - y = bswap(read(io, Float64)) + x = bswap(read(io, Float64)) + y = bswap(read(io, Float64)) - if z - z = bswap(read(io, Float64)) - return x, y, z - end + if z + z = bswap(read(io, Float64)) + return x, y, z + end - return x, y + return x, y end function wkblinestring(io, z, bswap) - npoints = bswap(read(io, UInt32)) + npoints = bswap(read(io, UInt32)) - points = map(1:npoints) do _ - wkbcoordinate(io, z, bswap) - end - return points + points = map(1:npoints) do _ + wkbcoordinate(io, z, bswap) + end + return points end function wkbpolygon(io, z, bswap) - nrings = bswap(read(io, UInt32)) + nrings = bswap(read(io, UInt32)) - rings = map(1:nrings) do _ - wkblinestring(io, z, bswap) - end - return rings + rings = map(1:nrings) do _ + wkblinestring(io, z, bswap) + end + return rings end function wkbmultigeometry(io, crs, z, bswap) - ngeoms = bswap(read(io, UInt32)) - - geomcollection = map(1:ngeoms) do _ - bswap = isone(read(io, UInt8)) ? ltoh : ntoh - ewkbtype = wkbGeometryType(read(io, UInt32)) - meshfromsf(io, crs, ewkbtype, z, bswap) - end - return geomcollection + ngeoms = bswap(read(io, UInt32)) + + geomcollection = map(1:ngeoms) do _ + bswap = isone(read(io, UInt8)) ? ltoh : ntoh + ewkbtype = wkbGeometryType(read(io, UInt32)) + meshfromsf(io, crs, ewkbtype, z, bswap) + end + return geomcollection end diff --git a/src/extra/gpkg/write.jl b/src/extra/gpkg/write.jl index eb78b3d..48da548 100644 --- a/src/extra/gpkg/write.jl +++ b/src/extra/gpkg/write.jl @@ -2,163 +2,165 @@ # Licensed under the MIT License. See LICENSE in the project root. # ------------------------------------------------------------------ -const DBInterface = SQLite.DBInterface - -function gpkgwrite(fname, geotable; ) - - db = SQLite.DB(fname) - - DBInterface.execute(db, "PRAGMA synchronous=0") - # Commits can be orders of magnitude faster with - # Setting PRAGMA synchronous=OFF but, - # can cause the database to go corrupt - # if there is an operating-system crash or power failure. - # If the power never goes out and no programs ever crash - # on you system then Synchronous = OFF is for you - #################################################### - - SQLite.transaction(db) do - - stmt_sql = """ - CREATE TABLE gpkg_spatial_ref_sys ( - srs_name TEXT NOT NULL, srs_id INTEGER NOT NULL PRIMARY KEY, - organization TEXT NOT NULL, organization_coordsys_id INTEGER NOT NULL, - definition TEXT NOT NULL, description TEXT, - definition_12_063 TEXT NOT NULL - ); - - CREATE TABLE gpkg_contents ( - table_name TEXT NOT NULL PRIMARY KEY, - data_type TEXT NOT NULL, - identifier TEXT UNIQUE, - description TEXT DEFAULT '', - last_change DATETIME NOT NULL DEFAULT - (strftime('%Y-%m-%dT%H:%M:%fZ','now')), - min_x DOUBLE, min_y DOUBLE, - max_x DOUBLE, max_y DOUBLE, - srs_id INTEGER, - CONSTRAINT fk_gc_r_srs_id FOREIGN KEY (srs_id) REFERENCES - gpkg_spatial_ref_sys(srs_id) - ); - """ - DBInterface.execute(db, stmt_sql) - stmt_sql = SQLite.Stmt(db, "PRAGMA application_id = $GPKG_APPLICATION_ID ") - DBInterface.execute(stmt_sql) - stmt_sql = SQLite.Stmt(db, "PRAGMA user_version = $GPKG_1_4_VERSION ") - DBInterface.execute(stmt_sql) - end - - ############################################################ - ### Requirement 11: Spatial Ref Sys Table Records ########## - ####### https://www.geopackage.org/spec/#r11 ############### - ### ######################################################## - # The gpkg_spatial_ref_sys table SHALL contain at a minimum - # 1. the record with an srs_id of 4326 SHALL correspond to WGS-84 as defined by EPSG in 4326 - # 2. the record with an srs_id of -1 SHALL be used for undefined Cartesian coordinate reference systems - # 3. the record with an srs_id of 0 SHALL be used for undefined geographic coordinate reference systems - ############################################################ - tb = [( - srs_name = "Undefined Cartesian SRS", - srs_id = -1, - organization = "NONE", - organization_coordsys_id = -1, - definition = "undefined", - description = "undefined geographic coordinate reference system", - definition_12_063 = "undefined", - )] - SQLite.load!(tb, db, "gpkg_spatial_ref_sys", replace=true) - tb = [( - srs_name = "Undefined geographic SRS", - srs_id = 0, - organization = "NONE", - organization_coordsys_id = 0, - definition = "undefined", - description = "undefined geographic coordinate reference system", - definition_12_063 = "undefined", - )] - SQLite.load!(tb, db, "gpkg_spatial_ref_sys", replace=true) - tb = [( - srs_name = "WGS 84 geodectic", - srs_id = 4326, - organization = "EPSG", - organization_coordsys_id = 4326, - definition = CoordRefSystems.wkt2(EPSG{4326}), - description = "longitude/latitude coordinates in decimal degrees on the WGS 84 spheroid", - definition_12_063 = CoordRefSystems.wkt2(EPSG{4326}), - )] - SQLite.load!(tb, db, "gpkg_spatial_ref_sys", replace=true) - table = values(geotable) - domain = GeoTables.domain(geotable) - crs = GeoTables.crs(domain) - geom = collect(domain) +function gpkgwrite(fname, geotable;) + db = SQLite.DB(fname) + + DBInterface.execute(db, "PRAGMA synchronous=0") + # Commits can be orders of magnitude faster with + # Setting PRAGMA synchronous=OFF but, + # can cause the database to go corrupt + # if there is an operating-system crash or power failure. + # If the power never goes out and no programs ever crash + # on you system then Synchronous = OFF is for you + #################################################### + + SQLite.transaction(db) do + sqlstmt = """ + CREATE TABLE gpkg_spatial_ref_sys ( + srs_name TEXT NOT NULL, srs_id INTEGER NOT NULL PRIMARY KEY, + organization TEXT NOT NULL, organization_coordsys_id INTEGER NOT NULL, + definition TEXT NOT NULL, description TEXT, + definition_12_063 TEXT NOT NULL + ); + + CREATE TABLE gpkg_contents ( + table_name TEXT NOT NULL PRIMARY KEY, + data_type TEXT NOT NULL, + identifier TEXT UNIQUE, + description TEXT DEFAULT '', + last_change DATETIME NOT NULL DEFAULT + (strftime('%Y-%m-%dT%H:%M:%fZ','now')), + min_x DOUBLE, min_y DOUBLE, + max_x DOUBLE, max_y DOUBLE, + srs_id INTEGER, + CONSTRAINT fk_gc_r_srs_id FOREIGN KEY (srs_id) REFERENCES + gpkg_spatial_ref_sys(srs_id) + ); + """ + DBInterface.execute(db, sqlstmt) + sqlstmt = SQLite.Stmt(db, "PRAGMA application_id = $GPKG_APPLICATION_ID ") + DBInterface.execute(sqlstmt) + sqlstmt = SQLite.Stmt(db, "PRAGMA user_version = $GPKG_1_4_VERSION ") + DBInterface.execute(sqlstmt) + end - _extracttablevals(db, table, domain, crs, geom) + ############################################################ + ### Requirement 11: Spatial Ref Sys Table Records ########## + ####### https://www.geopackage.org/spec/#r11 ############### + ### ######################################################## + # The gpkg_spatial_ref_sys table SHALL contain at a minimum + # 1. the record with an srs_id of 4326 SHALL correspond to WGS-84 as defined by EPSG in 4326 + # 2. the record with an srs_id of -1 SHALL be used for undefined Cartesian coordinate reference systems + # 3. the record with an srs_id of 0 SHALL be used for undefined geographic coordinate reference systems + ############################################################ + tb = [( + srs_name="Undefined Cartesian SRS", + srs_id=-1, + organization="NONE", + organization_coordsys_id=-1, + definition="undefined", + description="undefined geographic coordinate reference system", + definition_12_063="undefined" + )] + SQLite.load!(tb, db, "gpkg_spatial_ref_sys", replace=true) + tb = [( + srs_name="Undefined geographic SRS", + srs_id=0, + organization="NONE", + organization_coordsys_id=0, + definition="undefined", + description="undefined geographic coordinate reference system", + definition_12_063="undefined" + )] + SQLite.load!(tb, db, "gpkg_spatial_ref_sys", replace=true) + tb = [( + srs_name="WGS 84 geodectic", + srs_id=4326, + organization="EPSG", + organization_coordsys_id=4326, + definition=CoordRefSystems.wkt2(EPSG{4326}), + description="longitude/latitude coordinates in decimal degrees on the WGS 84 spheroid", + definition_12_063=CoordRefSystems.wkt2(EPSG{4326}) + )] + SQLite.load!(tb, db, "gpkg_spatial_ref_sys", replace=true) + table = values(geotable) + domain = GeoTables.domain(geotable) + crs = GeoTables.crs(domain) + geom = collect(domain) + + _extracttablevals(db, table, domain, crs, geom) end - - function _extracttablevals(db, table, domain, crs, geom) + if crs <: Cartesian + srs = "" + srid = -1 + elseif crs <: LatLon{WGS84Latest} + srs = "EPSG" + srid = 4326 + else + srs = string(CoordRefSystems.code(crs)) + srid = parse(Int32, srs) + end - if crs <: Cartesian - srs = "" - srid = -1 - elseif crs <: LatLon{WGS84Latest} - srs = "EPSG" - srid = 4326 - else - srs = string(CoordRefSystems.code(crs)) - srid = parse(Int32, srs) - end - - gpkgbinary = map(geom) do ft - gpkgbinheader = writegpkgheader(srid, ft) - io = IOBuffer() - writewkbgeom(io, ft) - vcat(gpkgbinheader, take!(io)) - end - - table = isone(length(table)) ? [(NamedTuple(table |> first)..., geom = gpkgbinary[1])] : [(; t..., geom = g) for (t, g) in zip(table, gpkgbinary)] - - SQLite.load!(table, db, replace=false) # autogenerates table name - # replace=false controls whether an INSERT INTO ... statement is generated or a REPLACE INTO .... - tn = (DBInterface.execute(db, """ SELECT name FROM sqlite_master WHERE type='table' AND name NOT IN ("gpkg_contents", "gpkg_spatial_ref_sys") """) |> first).name + gpkgbinary = map(geom) do ft + gpkgbinheader = writegpkgheader(srid, ft) + io = IOBuffer() + writewkbgeom(io, ft) + vcat(gpkgbinheader, take!(io)) + end - bbox = boundingbox(domain) - mincoords = CoordRefSystems.raw(coords(bbox.min)) - maxcoords = CoordRefSystems.raw(coords(bbox.max)) - contents = [(table_name = tn, - data_type = "features", - identifier = tn, - description = "", - last_change = Dates.format(now(UTC), "yyyy-mm-ddTHH:MM:SSZ"), - min_x = mincoords[1], - min_y = mincoords[2], - max_x = maxcoords[1], - max_y = maxcoords[2], - srs_id = srid - )] - SQLite.load!(contents, db, "gpkg_contents", replace=true) - if srid != 4326 - srstb = [( - srs_name = "", - srs_id = srid, - organization = srs[1:4], - organization_coordsys_id = srid, - definition = CoordRefSystems.wkt2(crs), - description = "", - definition_12_063 = CoordRefSystems.wkt2(crs), - )] - SQLite.load!(srstb, db, "gpkg_spatial_ref_sys", replace=true) - end - geomcolumns = [( - table_name = tn, - column_name = "geom", - geometry_type_name = _geomtype(geom), - srs_id = srid, - z = paramdim((geom |> first)) >= 2 ? 1 : 0, - m = 0 - )] - SQLite.load!(geomcolumns, db, "gpkg_geometry_columns", replace=true) + table = + isone(length(table)) ? [(NamedTuple(table |> first)..., geom=gpkgbinary[1])] : + [(; t..., geom=g) for (t, g) in zip(table, gpkgbinary)] + + SQLite.load!(table, db, replace=false) # autogenerates table name + # replace=false controls whether an INSERT INTO ... statement is generated or a REPLACE INTO .... + tn = + ( + DBInterface.execute( + db, + """ SELECT name FROM sqlite_master WHERE type='table' AND name NOT IN ("gpkg_contents", "gpkg_spatial_ref_sys") """ + ) |> first + ).name + + bbox = boundingbox(domain) + mincoords = CoordRefSystems.raw(coords(bbox.min)) + maxcoords = CoordRefSystems.raw(coords(bbox.max)) + contents = [( + table_name=tn, + data_type="features", + identifier=tn, + description="", + last_change=Dates.format(now(UTC), "yyyy-mm-ddTHH:MM:SSZ"), + min_x=mincoords[1], + min_y=mincoords[2], + max_x=maxcoords[1], + max_y=maxcoords[2], + srs_id=srid + )] + SQLite.load!(contents, db, "gpkg_contents", replace=true) + if srid != 4326 + srstb = [( + srs_name="", + srs_id=srid, + organization=srs[1:4], + organization_coordsys_id=srid, + definition=CoordRefSystems.wkt2(crs), + description="", + definition_12_063=CoordRefSystems.wkt2(crs) + )] + SQLite.load!(srstb, db, "gpkg_spatial_ref_sys", replace=true) + end + geomcolumns = [( + table_name=tn, + column_name="geom", + geometry_type_name=_geomtype(geom), + srs_id=srid, + z=paramdim((geom |> first)) >= 2 ? 1 : 0, + m=0 + )] + SQLite.load!(geomcolumns, db, "gpkg_geometry_columns", replace=true) end function _geomtype(geoms::AbstractVector{<:Geometry}) @@ -188,113 +190,109 @@ function _geomtype(geoms::AbstractVector{<:Geometry}) return "GEOMETRY" end - function _wkbtype(geometry) - if geometry isa Point - return wkbPoint - elseif geometry isa Rope || geometry isa Ring - return wkbLineString - elseif geometry isa PolyArea - return wkbPolygon - elseif geometry isa Multi - fg = parent(geometry) |> first - return wkbGeometryType(Int(_wkbtype(fg))+3) - else - @error "my hovercraft is full of eels: $geometry" - end + if geometry isa Point + return wkbPoint + elseif geometry isa Rope || geometry isa Ring + return wkbLineString + elseif geometry isa PolyArea + return wkbPolygon + elseif geometry isa Multi + fg = parent(geometry) |> first + return wkbGeometryType(Int(_wkbtype(fg)) + 3) + else + @error "my hovercraft is full of eels: $geometry" + end end function _wkbsetz(type::wkbGeometryType) - return wkbGeometryType(Int(type)+1000) - # ISO WKB Flavour + return wkbGeometryType(Int(type) + 1000) + # ISO WKB Flavour end function writegpkgheader(srs_id, geom) - io = IOBuffer() - write(io, [0x47, 0x50]) # 'GP' in ASCII - write(io, zero(UInt8)) # 0 = version 1 + io = IOBuffer() + write(io, [0x47, 0x50]) # 'GP' in ASCII + write(io, zero(UInt8)) # 0 = version 1 - flagsbyte = UInt8(0x07 >> 1) - write(io, flagsbyte) + flagsbyte = UInt8(0x07 >> 1) + write(io, flagsbyte) - write(io, htol(Int32(srs_id))) + write(io, htol(Int32(srs_id))) - bbox = boundingbox(geom) - write(io, htol(Float64(CoordRefSystems.raw(coords(bbox.min))[2]))) - write(io, htol(Float64(CoordRefSystems.raw(coords(bbox.max))[2]))) - write(io, htol(Float64(CoordRefSystems.raw(coords(bbox.min))[1]))) - write(io, htol(Float64(CoordRefSystems.raw(coords(bbox.max))[1]))) + bbox = boundingbox(geom) + write(io, htol(Float64(CoordRefSystems.raw(coords(bbox.min))[2]))) + write(io, htol(Float64(CoordRefSystems.raw(coords(bbox.max))[2]))) + write(io, htol(Float64(CoordRefSystems.raw(coords(bbox.min))[1]))) + write(io, htol(Float64(CoordRefSystems.raw(coords(bbox.max))[1]))) - if paramdim(geom) >= 3 - write(io, htol(Float64(CoordRefSystems.raw(coords(bbox.min))[3]))) - write(io, htol(Float64(CoordRefSystems.raw(coords(bbox.max))[3]))) - end + if paramdim(geom) >= 3 + write(io, htol(Float64(CoordRefSystems.raw(coords(bbox.min))[3]))) + write(io, htol(Float64(CoordRefSystems.raw(coords(bbox.max))[3]))) + end - return take!(io) + return take!(io) end function writewkbgeom(io, geom) - wkbtype = paramdim(geom) < 3 ? _wkbtype(geom) : _wkbsetz(_wkbtype(geom)); - write(io, one(UInt8)) - if Int(wkbtype) > 3 - write(io, htol(UInt32(wkbtype))) - write(io, htol(UInt32(length(geom |> parent)))) - - for ft in parent(geom) - writewkbgeom(io, ft) - end + wkbtype = paramdim(geom) < 3 ? _wkbtype(geom) : _wkbsetz(_wkbtype(geom)) + write(io, one(UInt8)) + if Int(wkbtype) > 3 + write(io, htol(UInt32(wkbtype))) + write(io, htol(UInt32(length(geom |> parent)))) + + for ft in parent(geom) + writewkbgeom(io, ft) + end + else + if wkbtype == wkbPolygon || wkbtype == wkbPolygonZ || wkbtype == wkbPolygon25D + _wkbpolygon(io, wkbtype, [boundary(geom::PolyArea)]) + elseif wkbtype == wkbLineString || wkbtype == wkbLineString || wkbtype == wkbLineString25D + coordlist = vertices(geom) + write(io, htol(UInt32(wkbtype))) + if geom isa Meshes.Ring + write(io, htol(UInt32(length(coordlist) + 1))) + else + write(io, htol(UInt32(length(coordlist)))) + end + + _wkblinestring(io, wkbtype, coordlist) + if geom isa Meshes.Ring + points = CoordRefSystems.raw(coords(coordlist |> first)) + _wkbcoordinates(io, wkbtype, points) + end + elseif wkbtype == wkbPoint || wkbtype == wkbPointZ || wkbtype == wkbPoint25D + coordinates = CoordRefSystems.raw(coords(geom)) + _wkbcoordinates(io, wkbtype, coordinates) else - if wkbtype == wkbPolygon || wkbtype == wkbPolygonZ || wkbtype == wkbPolygon25D - _wkbpolygon(io, wkbtype, [boundary(geom::PolyArea)]) - elseif wkbtype == wkbLineString || wkbtype == wkbLineString || wkbtype == wkbLineString25D - coordlist = vertices(geom) - write(io, htol(UInt32(wkbtype))) - if geom isa Meshes.Ring - write(io, htol(UInt32(length(coordlist)+1))) - else - write(io, htol(UInt32(length(coordlist)))) - end - - - _wkblinestring(io, wkbtype, coordlist) - if geom isa Meshes.Ring - points = CoordRefSystems.raw(coords(coordlist |> first)) - _wkbcoordinates(io, wkbtype, points) - end - elseif wkbtype == wkbPoint || wkbtype == wkbPointZ || wkbtype == wkbPoint25D - coordinates = CoordRefSystems.raw(coords(geom)) - _wkbcoordinates(io, wkbtype, coordinates) - else - @error "my hovercraft is full of eels: $wkbtype" - end + @error "my hovercraft is full of eels: $wkbtype" end + end end - function _wkbcoordinates(io, wkbtype, coords) + write(io, htol(Float64(coords[2]))) + write(io, htol(Float64(coords[1]))) - write(io, htol(Float64(coords[2]))) - write(io, htol(Float64(coords[1]))) - - if (UInt32(wkbtype) > 1000) || !(iszero(Int(wkbtype) & (0x80000000 | 0x40000000))) - write(io, htol(Float64(coords[3]))) - end + if (UInt32(wkbtype) > 1000) || !(iszero(Int(wkbtype) & (0x80000000 | 0x40000000))) + write(io, htol(Float64(coords[3]))) + end end function _wkblinestring(io, wkb_type, coord_list) - for n_coords::Point in coord_list - coordinates = CoordRefSystems.raw(coords(n_coords)) - _wkbcoordinates(io, wkb_type, coordinates) - end + for n_coords::Point in coord_list + coordinates = CoordRefSystems.raw(coords(n_coords)) + _wkbcoordinates(io, wkb_type, coordinates) + end end function _wkbpolygon(io, wkb_type, rings) - write(io, htol(wkb_type)) - write(io, htol(length(rings))) + write(io, htol(wkb_type)) + write(io, htol(length(rings))) - for ring in rings - coord_list = vertices(ring) - write(io, htol(UInt32(length(coord_list) + 1))) - _wkblinestring(io, wkb_type, coord_list) - end + for ring in rings + coord_list = vertices(ring) + write(io, htol(UInt32(length(coord_list) + 1))) + _wkblinestring(io, wkb_type, coord_list) + end end From 3166a1c323837fc4c1b27d43879387e0033841be Mon Sep 17 00:00:00 2001 From: jph6366 Date: Thu, 11 Sep 2025 09:58:59 -0400 Subject: [PATCH 05/90] addressing commit suggestions --- Project.toml | 2 +- src/GeoIO.jl | 2 +- src/extra/gis.jl | 2 +- src/extra/gpkg.jl | 19 +++++++----------- src/extra/gpkg/read.jl | 44 +++++++++++++++++++---------------------- src/extra/gpkg/write.jl | 13 +++++------- 6 files changed, 35 insertions(+), 47 deletions(-) diff --git a/Project.toml b/Project.toml index f2cc7a5..8b4afee 100644 --- a/Project.toml +++ b/Project.toml @@ -43,7 +43,7 @@ CSV = "0.10" Colors = "0.12, 0.13" CommonDataModel = "0.2, 0.3" CoordRefSystems = "0.18" -DBInterface = "2.6.1" +DBInterface = "2.6" FileIO = "1.16" Format = "1.3" GRIBDatasets = "0.3, 0.4" diff --git a/src/GeoIO.jl b/src/GeoIO.jl index d888e72..407a893 100644 --- a/src/GeoIO.jl +++ b/src/GeoIO.jl @@ -38,7 +38,7 @@ import PlyIO # CSV format import CSV -# Database Interfaces +# Database interfaces import DBInterface import SQLite diff --git a/src/extra/gis.jl b/src/extra/gis.jl index 9fdfa01..85c5420 100644 --- a/src/extra/gis.jl +++ b/src/extra/gis.jl @@ -66,7 +66,7 @@ function gistable(fname; layer, numtype, kwargs...) elseif endswith(fname, ".parquet") return GPQ.read(fname; kwargs...) elseif endswith(fname, ".gpkg") - return gpkgread(fname; kwargs...) + return gpkgread(fname; layer, kwargs...) else # fallback to GDAL data = AG.read(fname; kwargs...) return AG.getlayer(data, layer - 1) diff --git a/src/extra/gpkg.jl b/src/extra/gpkg.jl index 7b0f2d5..c86a463 100644 --- a/src/extra/gpkg.jl +++ b/src/extra/gpkg.jl @@ -2,23 +2,18 @@ # Licensed under the MIT License. See LICENSE in the project root. # ------------------------------------------------------------------ -############################################# -### https://www.geopackage.org/spec/#r2 ##### -# Requirement 2: contains "GPKG" in ASCII in +# According to https://www.geopackage.org/spec/#r2 +# a GeoPackage should contain "GPKG" in ASCII in # "application_id" field of SQLite db header -# Reminder: We have to set this (on-write) -# after there's some content, -# so the database file is not zero length -# -const GP10_APPLICATION_ID = 74777363 #0x47503130 -const GP11_APPLICATION_ID = 119643780 # 0x47503131 -const GPKG_APPLICATION_ID = 1196444487 # 0x47504B47 + +const GP10_APPLICATION_ID = Int(0x47503130) +const GP11_APPLICATION_ID = Int(0x47503131) +const GPKG_APPLICATION_ID = Int(0x47504B47) const GPKG_1_2_VERSION = 10200 const GPKG_1_3_VERSION = 10300 const GPKG_1_4_VERSION = 10400 -# List of well known binary geometry types. -# These are used within the GeoPackageBinary SQL BLOBs +# types used within the GeoPackageBinary SQL BLOBs @enum wkbGeometryType begin wkbUnknown = 0 wkbPoint = 1 diff --git a/src/extra/gpkg/read.jl b/src/extra/gpkg/read.jl index f79b56c..df7af79 100644 --- a/src/extra/gpkg/read.jl +++ b/src/extra/gpkg/read.jl @@ -5,11 +5,11 @@ # query SQLite.Table given *.gpkg filename; optionally specify quantity of tables and features query # default behavior for select statement is limit result to 1 feature geometry row and corresponding attributes row # limited to only 1 feature table to select geopackage binary geometry from. -function gpkgread(fname; ntables=1, nfeatures=1) +function gpkgread(fname; layer=1) db = SQLite.DB(fname) gpkgid(db) - geom = gpkgmesh(db, ; ntables=ntables, nfeatures=nfeatures) - attrs = gpkgmeshattrs(db, ; ntables=ntables, nfeatures=nfeatures) + geom = gpkgmesh(db, ; layer) + attrs = gpkgmeshattrs(db, ; layer) GeoTables.georef(attrs, geom) end @@ -51,7 +51,7 @@ function hasgpkgmetadata(db) return (tbcount == 2) end -function gpkgmeshattrs(db, ; ntables::Int=1, nfeatures=1) +function gpkgmeshattrs(db, ; layer=1) sqlstmt = "SELECT c.table_name, c.identifier, " * "g.column_name, g.geometry_type_name, g.z, g.m, c.min_x, c.min_y, " * @@ -61,7 +61,7 @@ function gpkgmeshattrs(db, ; ntables::Int=1, nfeatures=1) " FROM gpkg_geometry_columns g " * " JOIN gpkg_contents c ON (g.table_name = c.table_name)" * " WHERE " * - " c.data_type = 'features' LIMIT $ntables" + " c.data_type = 'features' LIMIT $layer" feature_tables = DBInterface.execute(db, sqlstmt) tb = [] fields = "fid,"^10^5 @@ -73,7 +73,7 @@ function gpkgmeshattrs(db, ; ntables::Int=1, nfeatures=1) rp_attrs = join(ft_attrs, ", ") # keep the shortest set of attributes to avoid KeyError {Key} not found fields = length(fields) > length(rp_attrs) ? rp_attrs : fields # smelly hack, eval shortest common subset of fields instead - sqlstmt = "SELECT $fields from $tn LIMIT $nfeatures;" + sqlstmt = "SELECT $fields from $tn" if isone(nfeatures) rowvalues = DBInterface.execute(db, sqlstmt) |> first push!(tb, rowvalues) @@ -86,9 +86,6 @@ function gpkgmeshattrs(db, ; ntables::Int=1, nfeatures=1) return tb end -############################################################################## -########### Features - Geometry Columns: Table Data Values ################### -############################################################################## # https://www.geopackage.org/spec/#:~:text=2.1.5.1.2.%20Table%20Data%20Values #------------------------------------------------------------------------------ # # Requirement 21: a gpkg_contents table row with a "features" data_type @@ -121,8 +118,8 @@ end # Requirement 146: The srs_id value in a gpkg_geometry_columns table row # SHALL match the srs_id column value from the corresponding row in the # gpkg_contents table. -# -function gpkgmesh(db, ; ntables=1, nfeatures=1) + +function gpkgmesh(db, ; layer=1) sqlstmt = """ SELECT g.table_name AS tn, g.column_name AS cn, c.srs_id as crs, g.z as elev, srs.organization as org, srs.organization_coordsys_id as org_coordsys_id, ( SELECT type FROM sqlite_master WHERE lower(name) = lower(c.table_name) AND type IN ('table', 'view')) AS object_type @@ -134,12 +131,12 @@ function gpkgmesh(db, ; ntables=1, nfeatures=1) AND g.srs_id = c.srs_id AND g.z IN (0, 1, 2) AND g.m IN (0, 1, 2) - LIMIT $ntables; + LIMIT $layer; """ tb = DBInterface.execute(db, stmt_sql) - meshes = Meshes.Geometry[] + meshes = Geometry[] for (tn, cn, org, org_coordsys_id) in [(row.tn, row.cn, row.org, row.org_coordsys_id) for row in tb] - sqlstmt = "SELECT $cn FROM $tn LIMIT $nfeatures;" + sqlstmt = "SELECT $cn FROM $tn;" gpkgbinary = DBInterface.execute(db, sqlstmt) headerlen = 0 for blob in gpkgbinary @@ -211,41 +208,40 @@ function meshfromwkb(io, srs_id, org, org_coordsys_id, ewkbtype, zextent, bswap) if occursin("Multi", string(ewkbtype)) elems::Vector = wkbmultigeometry(io, crs, zextent, bswap) - return Meshes.Multi(elems) + return Multi(elems) else elem = meshfromsf(io, crs, ewkbtype, zextent, bswap) return elem end end -################################################################ # Requirement 20: GeoPackage SHALL store feature table geometries # with the basic simple feature geometry types # https://www.geopackage.org/spec140/index.html#geometry_types -# + function meshfromsf(io, crs, ewkbtype, zextent, bswap) if isequal(ewkbtype, wkbPoint) elem = wkbcoordinate(io, zextent, bswap) - return Meshes.Point(crs(elem...)) + return Point(crs(elem...)) elseif isequal(ewkbtype, wkbLineString) elem = wkblinestring(io, zextent, bswap) if first(elem) != last(elem) - return Meshes.Rope([Meshes.Point(crs(coords...)) for coords in elem]...) + return Rope([Point(crs(coords...)) for coords in elem]...) else - return Meshes.Ring([Meshes.Point(crs(coords...)) for coords in elem[2:end]]...) + return Ring([Point(crs(coords...)) for coords in elem[2:end]]...) end elseif isequal(ewkbtype, wkbPolygon) elem = wkbpolygon(io, zextent, bswap) rings = map(elem) do ring coords = map(ring) do point - Meshes.Point(crs(point...)) + Point(crs(point...)) end - Meshes.Ring(coords) + Ring(coords) end outerring = first(rings) - holes = isone(length(rings)) ? rings[2:end] : Meshes.Ring[] - return Meshes.PolyArea(outerring, holes...) + holes = isone(length(rings)) ? rings[2:end] : Ring[] + return PolyArea(outerring, holes...) end end diff --git a/src/extra/gpkg/write.jl b/src/extra/gpkg/write.jl index 48da548..429195a 100644 --- a/src/extra/gpkg/write.jl +++ b/src/extra/gpkg/write.jl @@ -12,7 +12,6 @@ function gpkgwrite(fname, geotable;) # if there is an operating-system crash or power failure. # If the power never goes out and no programs ever crash # on you system then Synchronous = OFF is for you - #################################################### SQLite.transaction(db) do sqlstmt = """ @@ -44,15 +43,13 @@ function gpkgwrite(fname, geotable;) DBInterface.execute(sqlstmt) end - ############################################################ - ### Requirement 11: Spatial Ref Sys Table Records ########## - ####### https://www.geopackage.org/spec/#r11 ############### - ### ######################################################## + + # According to https://www.geopackage.org/spec/#r11 # The gpkg_spatial_ref_sys table SHALL contain at a minimum # 1. the record with an srs_id of 4326 SHALL correspond to WGS-84 as defined by EPSG in 4326 # 2. the record with an srs_id of -1 SHALL be used for undefined Cartesian coordinate reference systems # 3. the record with an srs_id of 0 SHALL be used for undefined geographic coordinate reference systems - ############################################################ + tb = [( srs_name="Undefined Cartesian SRS", srs_id=-1, @@ -250,14 +247,14 @@ function writewkbgeom(io, geom) elseif wkbtype == wkbLineString || wkbtype == wkbLineString || wkbtype == wkbLineString25D coordlist = vertices(geom) write(io, htol(UInt32(wkbtype))) - if geom isa Meshes.Ring + if geom isa Ring write(io, htol(UInt32(length(coordlist) + 1))) else write(io, htol(UInt32(length(coordlist)))) end _wkblinestring(io, wkbtype, coordlist) - if geom isa Meshes.Ring + if geom isa Ring points = CoordRefSystems.raw(coords(coordlist |> first)) _wkbcoordinates(io, wkbtype, points) end From 0255232e6f0f3f06b9b4557a62813630fd642b90 Mon Sep 17 00:00:00 2001 From: jph6366 Date: Thu, 11 Sep 2025 10:04:16 -0400 Subject: [PATCH 06/90] missed this one --- src/extra/gpkg/read.jl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/extra/gpkg/read.jl b/src/extra/gpkg/read.jl index df7af79..dc5fe9f 100644 --- a/src/extra/gpkg/read.jl +++ b/src/extra/gpkg/read.jl @@ -43,11 +43,11 @@ end # Requirement 10: must include a gpkg_spatial_ref_sys table # Requirement 13: must include a gpkg_contents table function hasgpkgmetadata(db) - stmt_sql = + sqlstmt = "SELECT COUNT(*) FROM sqlite_master WHERE " * "name IN ('gpkg_spatial_ref_sys', 'gpkg_contents') AND " * "type IN ('table', 'view');" - tbcount = DBInterface.execute(db, stmt_sql) |> first |> only + tbcount = DBInterface.execute(db, sqlstmt) |> first |> only return (tbcount == 2) end @@ -133,7 +133,7 @@ function gpkgmesh(db, ; layer=1) AND g.m IN (0, 1, 2) LIMIT $layer; """ - tb = DBInterface.execute(db, stmt_sql) + tb = DBInterface.execute(db, sqlstmt) meshes = Geometry[] for (tn, cn, org, org_coordsys_id) in [(row.tn, row.cn, row.org, row.org_coordsys_id) for row in tb] sqlstmt = "SELECT $cn FROM $tn;" From 5e604881711e599ea5210d3a1ee61bfb16044a21 Mon Sep 17 00:00:00 2001 From: jph6366 Date: Thu, 11 Sep 2025 10:09:38 -0400 Subject: [PATCH 07/90] another mistake missed --- src/extra/gpkg/read.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/extra/gpkg/read.jl b/src/extra/gpkg/read.jl index dc5fe9f..cdfed9d 100644 --- a/src/extra/gpkg/read.jl +++ b/src/extra/gpkg/read.jl @@ -74,7 +74,7 @@ function gpkgmeshattrs(db, ; layer=1) # keep the shortest set of attributes to avoid KeyError {Key} not found fields = length(fields) > length(rp_attrs) ? rp_attrs : fields # smelly hack, eval shortest common subset of fields instead sqlstmt = "SELECT $fields from $tn" - if isone(nfeatures) + if isone(layer) rowvalues = DBInterface.execute(db, sqlstmt) |> first push!(tb, rowvalues) else From 39730ce87542ee296766b9d7b1706bbfdb43d2ca Mon Sep 17 00:00:00 2001 From: jph6366 Date: Fri, 12 Sep 2025 09:24:19 -0400 Subject: [PATCH 08/90] Reader is sufficiently spec-compliant for further testing, addressing commits and removing unneccesary branches --- src/extra/gpkg/read.jl | 34 ++++++--------------- src/extra/gpkg/write.jl | 67 ++++++++++++++++++++--------------------- 2 files changed, 42 insertions(+), 59 deletions(-) diff --git a/src/extra/gpkg/read.jl b/src/extra/gpkg/read.jl index cdfed9d..17ecbda 100644 --- a/src/extra/gpkg/read.jl +++ b/src/extra/gpkg/read.jl @@ -2,42 +2,28 @@ # Licensed under the MIT License. See LICENSE in the project root. # ------------------------------------------------------------------ -# query SQLite.Table given *.gpkg filename; optionally specify quantity of tables and features query -# default behavior for select statement is limit result to 1 feature geometry row and corresponding attributes row -# limited to only 1 feature table to select geopackage binary geometry from. function gpkgread(fname; layer=1) db = SQLite.DB(fname) - gpkgid(db) + assertgpkg(db) geom = gpkgmesh(db, ; layer) attrs = gpkgmeshattrs(db, ; layer) GeoTables.georef(attrs, geom) end -function gpkgid(db)::Bool +function assertgpkg(db) appid = DBInterface.execute(db, "PRAGMA application_id;") |> first |> only userversion = DBInterface.execute(db, "PRAGMA user_version;") |> first |> only - if !(hasgpkgmetadata(db)) - @error "missing required metadata tables" + if !hasgpkgmetadata(db) + throw(ErrorException("missing required metadata tables in the GeoPackage SQL database")) end # Requirement 6: PRAGMA integrity_check returns a single row with the value 'ok' # Requirement 7: PRAGMA foreign_key_check (w/ no parameter value) returns an empty result set - if (appid != GP10_APPLICATION_ID) && (appid != GP11_APPLICATION_ID) && (appid != GPKG_APPLICATION_ID) - @warn "application_id not recognized" - return false - elseif (appid == GPKG_APPLICATION_ID) && !( - (userversion >= GPKG_1_2_VERSION && userversion < GPKG_1_2_VERSION + 99) || - (userversion >= GPKG_1_3_VERSION && userversion < GPKG_1_3_VERSION + 99) || - (userversion >= GPKG_1_4_VERSION && userversion < GPKG_1_4_VERSION + 99) - ) - @warn "application_id is valid but user version is not recognized" - elseif (DBInterface.execute(db, "PRAGMA integrity_check;") |> first |> only != "ok") || - !(isempty(DBInterface.execute(db, "PRAGMA foreign_key_check;"))) - @error "database integrity at risk or foreign key violation(s)" - return false + if (DBInterface.execute(db, "PRAGMA integrity_check;") |> first |> only != "ok") || + !(isempty(DBInterface.execute(db, "PRAGMA foreign_key_check;"))) + throw(ErrorException("database integrity at risk or foreign key violation(s)")) end - return true end # Requirement 10: must include a gpkg_spatial_ref_sys table @@ -48,7 +34,7 @@ function hasgpkgmetadata(db) "name IN ('gpkg_spatial_ref_sys', 'gpkg_contents') AND " * "type IN ('table', 'view');" tbcount = DBInterface.execute(db, sqlstmt) |> first |> only - return (tbcount == 2) + (tbcount == 2) end function gpkgmeshattrs(db, ; layer=1) @@ -118,7 +104,6 @@ end # Requirement 146: The srs_id value in a gpkg_geometry_columns table row # SHALL match the srs_id column value from the corresponding row in the # gpkg_contents table. - function gpkgmesh(db, ; layer=1) sqlstmt = """ SELECT g.table_name AS tn, g.column_name AS cn, c.srs_id as crs, g.z as elev, srs.organization as org, srs.organization_coordsys_id as org_coordsys_id, @@ -207,7 +192,7 @@ function meshfromwkb(io, srs_id, org, org_coordsys_id, ewkbtype, zextent, bswap) end if occursin("Multi", string(ewkbtype)) - elems::Vector = wkbmultigeometry(io, crs, zextent, bswap) + elems = wkbmultigeometry(io, crs, zextent, bswap) return Multi(elems) else elem = meshfromsf(io, crs, ewkbtype, zextent, bswap) @@ -218,7 +203,6 @@ end # Requirement 20: GeoPackage SHALL store feature table geometries # with the basic simple feature geometry types # https://www.geopackage.org/spec140/index.html#geometry_types - function meshfromsf(io, crs, ewkbtype, zextent, bswap) if isequal(ewkbtype, wkbPoint) elem = wkbcoordinate(io, zextent, bswap) diff --git a/src/extra/gpkg/write.jl b/src/extra/gpkg/write.jl index 429195a..b2dc8ff 100644 --- a/src/extra/gpkg/write.jl +++ b/src/extra/gpkg/write.jl @@ -43,13 +43,12 @@ function gpkgwrite(fname, geotable;) DBInterface.execute(sqlstmt) end - # According to https://www.geopackage.org/spec/#r11 # The gpkg_spatial_ref_sys table SHALL contain at a minimum # 1. the record with an srs_id of 4326 SHALL correspond to WGS-84 as defined by EPSG in 4326 # 2. the record with an srs_id of -1 SHALL be used for undefined Cartesian coordinate reference systems # 3. the record with an srs_id of 0 SHALL be used for undefined geographic coordinate reference systems - + tb = [( srs_name="Undefined Cartesian SRS", srs_id=-1, @@ -233,37 +232,30 @@ end function writewkbgeom(io, geom) wkbtype = paramdim(geom) < 3 ? _wkbtype(geom) : _wkbsetz(_wkbtype(geom)) - write(io, one(UInt8)) - if Int(wkbtype) > 3 - write(io, htol(UInt32(wkbtype))) - write(io, htol(UInt32(length(geom |> parent)))) + write(io, htol(one(UInt8))) + write(io, htol(Int32(wkbtype))) + _wkbgeom(io, wkbtype, geom) +end - for ft in parent(geom) - writewkbgeom(io, ft) - end +function writewkbsf(io, wkbtype, geom) + if wkbtype == wkbPolygon || wkbtype == wkbPolygonZ || wkbtype == wkbPolygon25D + _wkbpolygon(io, wkbtype, [boundary(geom::PolyArea)]) + elseif wkbtype == wkbLineString || wkbtype == wkbLineString || wkbtype == wkbLineString25D + coordlist = vertices(geom) + _wkblinestring(io, wkbtype, coordlist) + elseif wkbtype == wkbPoint || wkbtype == wkbPointZ || wkbtype == wkbPoint25D + coordinates = CoordRefSystems.raw(coords(geom)) + _wkbcoordinates(io, wkbtype, coordinates) else - if wkbtype == wkbPolygon || wkbtype == wkbPolygonZ || wkbtype == wkbPolygon25D - _wkbpolygon(io, wkbtype, [boundary(geom::PolyArea)]) - elseif wkbtype == wkbLineString || wkbtype == wkbLineString || wkbtype == wkbLineString25D - coordlist = vertices(geom) - write(io, htol(UInt32(wkbtype))) - if geom isa Ring - write(io, htol(UInt32(length(coordlist) + 1))) - else - write(io, htol(UInt32(length(coordlist)))) - end + @error "my hovercraft is full of eels: $wkbtype" + end +end - _wkblinestring(io, wkbtype, coordlist) - if geom isa Ring - points = CoordRefSystems.raw(coords(coordlist |> first)) - _wkbcoordinates(io, wkbtype, points) - end - elseif wkbtype == wkbPoint || wkbtype == wkbPointZ || wkbtype == wkbPoint25D - coordinates = CoordRefSystems.raw(coords(geom)) - _wkbcoordinates(io, wkbtype, coordinates) - else - @error "my hovercraft is full of eels: $wkbtype" - end +function _wkbgeom(io, wkbtype, geom) + if Int(wkbtype) > 3 + _wkbmulti(io, wkbtype, geom) + else + writewkbsf(io, wkbtype, geom) end end @@ -277,6 +269,7 @@ function _wkbcoordinates(io, wkbtype, coords) end function _wkblinestring(io, wkb_type, coord_list) + write(io, htol(Int32(length(coord_list)))) for n_coords::Point in coord_list coordinates = CoordRefSystems.raw(coords(n_coords)) _wkbcoordinates(io, wkb_type, coordinates) @@ -284,12 +277,18 @@ function _wkblinestring(io, wkb_type, coord_list) end function _wkbpolygon(io, wkb_type, rings) - write(io, htol(wkb_type)) - write(io, htol(length(rings))) - + write(io, htol(Int32(length(rings)))) for ring in rings coord_list = vertices(ring) - write(io, htol(UInt32(length(coord_list) + 1))) _wkblinestring(io, wkb_type, coord_list) end end + +function _wkbmulti(io, wkbtype, geoms) + write(io, htol(Int32(length(geoms |> parent)))) + for sf in geoms |> parent + write(io, one(UInt8)) + write(io, Int32(Int(wkbMultiPolygon) - 3)) + writewkbsf(io, wkbGeometryType(Int(wkbtype) - 3), sf) + end +end \ No newline at end of file From 26b760282694517c250266cbb3fa572f0371de40 Mon Sep 17 00:00:00 2001 From: jph6366 Date: Fri, 12 Sep 2025 09:55:42 -0400 Subject: [PATCH 09/90] removed newline --- src/extra/gpkg.jl | 1 - 1 file changed, 1 deletion(-) diff --git a/src/extra/gpkg.jl b/src/extra/gpkg.jl index c86a463..f78a3b9 100644 --- a/src/extra/gpkg.jl +++ b/src/extra/gpkg.jl @@ -5,7 +5,6 @@ # According to https://www.geopackage.org/spec/#r2 # a GeoPackage should contain "GPKG" in ASCII in # "application_id" field of SQLite db header - const GP10_APPLICATION_ID = Int(0x47503130) const GP11_APPLICATION_ID = Int(0x47503131) const GPKG_APPLICATION_ID = Int(0x47504B47) From 0077fefff968ab0c1e36a0351af114cad890c8be Mon Sep 17 00:00:00 2001 From: jph6366 Date: Fri, 12 Sep 2025 10:35:42 -0400 Subject: [PATCH 10/90] removed returns for code style --- src/extra/gpkg/read.jl | 54 ++++++++++++++++++++++-------------------- 1 file changed, 28 insertions(+), 26 deletions(-) diff --git a/src/extra/gpkg/read.jl b/src/extra/gpkg/read.jl index 17ecbda..57afefb 100644 --- a/src/extra/gpkg/read.jl +++ b/src/extra/gpkg/read.jl @@ -29,25 +29,27 @@ end # Requirement 10: must include a gpkg_spatial_ref_sys table # Requirement 13: must include a gpkg_contents table function hasgpkgmetadata(db) - sqlstmt = - "SELECT COUNT(*) FROM sqlite_master WHERE " * - "name IN ('gpkg_spatial_ref_sys', 'gpkg_contents') AND " * - "type IN ('table', 'view');" + sqlstmt =""" + SELECT COUNT(*) FROM sqlite_master WHERE + name IN ('gpkg_spatial_ref_sys', 'gpkg_contents') AND + type IN ('table', 'view'); + """ tbcount = DBInterface.execute(db, sqlstmt) |> first |> only (tbcount == 2) end function gpkgmeshattrs(db, ; layer=1) - sqlstmt = - "SELECT c.table_name, c.identifier, " * - "g.column_name, g.geometry_type_name, g.z, g.m, c.min_x, c.min_y, " * - "c.max_x, c.max_y, " * - "(SELECT type FROM sqlite_master WHERE lower(name) = " * - "lower(c.table_name) AND type IN ('table', 'view')) AS object_type " * - " FROM gpkg_geometry_columns g " * - " JOIN gpkg_contents c ON (g.table_name = c.table_name)" * - " WHERE " * - " c.data_type = 'features' LIMIT $layer" + sqlstmt = """ + SELECT c.table_name, c.identifier, + g.column_name, g.geometry_type_name, g.z, g.m, c.min_x, c.min_y, + c.max_x, c.max_y, + (SELECT type FROM sqlite_master WHERE lower(name) = + lower(c.table_name) AND type IN ('table', 'view')) AS object_type + FROM gpkg_geometry_columns g + JOIN gpkg_contents c ON (g.table_name = c.table_name) + WHERE + c.data_type = 'features' LIMIT $layer + """ feature_tables = DBInterface.execute(db, sqlstmt) tb = [] fields = "fid,"^10^5 @@ -69,7 +71,7 @@ function gpkgmeshattrs(db, ; layer=1) end end end - return tb + tb end # https://www.geopackage.org/spec/#:~:text=2.1.5.1.2.%20Table%20Data%20Values @@ -173,7 +175,7 @@ function gpkgmesh(db, ; layer=1) end end end - return meshes + meshes end function meshfromwkb(io, srs_id, org, org_coordsys_id, ewkbtype, zextent, bswap) @@ -193,10 +195,10 @@ function meshfromwkb(io, srs_id, org, org_coordsys_id, ewkbtype, zextent, bswap) if occursin("Multi", string(ewkbtype)) elems = wkbmultigeometry(io, crs, zextent, bswap) - return Multi(elems) + Multi(elems) else elem = meshfromsf(io, crs, ewkbtype, zextent, bswap) - return elem + elem end end @@ -206,13 +208,13 @@ end function meshfromsf(io, crs, ewkbtype, zextent, bswap) if isequal(ewkbtype, wkbPoint) elem = wkbcoordinate(io, zextent, bswap) - return Point(crs(elem...)) + Point(crs(elem...)) elseif isequal(ewkbtype, wkbLineString) elem = wkblinestring(io, zextent, bswap) if first(elem) != last(elem) - return Rope([Point(crs(coords...)) for coords in elem]...) + Rope([Point(crs(coords...)) for coords in elem]...) else - return Ring([Point(crs(coords...)) for coords in elem[2:end]]...) + Ring([Point(crs(coords...)) for coords in elem[2:end]]...) end elseif isequal(ewkbtype, wkbPolygon) elem = wkbpolygon(io, zextent, bswap) @@ -225,7 +227,7 @@ function meshfromsf(io, crs, ewkbtype, zextent, bswap) outerring = first(rings) holes = isone(length(rings)) ? rings[2:end] : Ring[] - return PolyArea(outerring, holes...) + PolyArea(outerring, holes...) end end @@ -238,7 +240,7 @@ function wkbcoordinate(io, z, bswap) return x, y, z end - return x, y + x, y end function wkblinestring(io, z, bswap) @@ -247,7 +249,7 @@ function wkblinestring(io, z, bswap) points = map(1:npoints) do _ wkbcoordinate(io, z, bswap) end - return points + points end function wkbpolygon(io, z, bswap) @@ -256,7 +258,7 @@ function wkbpolygon(io, z, bswap) rings = map(1:nrings) do _ wkblinestring(io, z, bswap) end - return rings + rings end function wkbmultigeometry(io, crs, z, bswap) @@ -267,5 +269,5 @@ function wkbmultigeometry(io, crs, z, bswap) ewkbtype = wkbGeometryType(read(io, UInt32)) meshfromsf(io, crs, ewkbtype, z, bswap) end - return geomcollection + geomcollection end From de56781d63f503c85ff3994868006ef42e1ba5a8 Mon Sep 17 00:00:00 2001 From: jph6366 Date: Mon, 15 Sep 2025 13:49:51 -0400 Subject: [PATCH 11/90] creation of wkb.jl, removing/recycling redundant and/or unused code, idiomatic modifications, and map do syntax for meshes collection --- src/extra/gpkg.jl | 89 -------------------------------------- src/extra/gpkg/read.jl | 94 ++++++++++++++++++++--------------------- src/extra/gpkg/write.jl | 54 +++++++++++------------ src/extra/wkb.jl | 83 ++++++++++++++++++++++++++++++++++++ 4 files changed, 157 insertions(+), 163 deletions(-) create mode 100644 src/extra/wkb.jl diff --git a/src/extra/gpkg.jl b/src/extra/gpkg.jl index f78a3b9..d203951 100644 --- a/src/extra/gpkg.jl +++ b/src/extra/gpkg.jl @@ -5,97 +5,8 @@ # According to https://www.geopackage.org/spec/#r2 # a GeoPackage should contain "GPKG" in ASCII in # "application_id" field of SQLite db header -const GP10_APPLICATION_ID = Int(0x47503130) -const GP11_APPLICATION_ID = Int(0x47503131) const GPKG_APPLICATION_ID = Int(0x47504B47) -const GPKG_1_2_VERSION = 10200 -const GPKG_1_3_VERSION = 10300 const GPKG_1_4_VERSION = 10400 -# types used within the GeoPackageBinary SQL BLOBs -@enum wkbGeometryType begin - wkbUnknown = 0 - wkbPoint = 1 - wkbLineString = 2 - wkbPolygon = 3 - wkbMultiPoint = 4 - wkbMultiLineString = 5 - wkbMultiPolygon = 6 - wkbGeometryCollection = 7 - wkbCircularString = 8 - wkbCompoundCurve = 9 - wkbCurvePolygon = 10 - wkbMultiCurve = 11 - wkbMultiSurface = 12 - wkbCurve = 13 - wkbSurface = 14 - - wkbNone = 100 # pure attribute records - wkbLinearRing = 101 - - # ISO SQL/MM Part 3: Spatial - # Z-aware types - wkbPointZ = 1001 - wkbLineStringZ = 1002 - wkbPolygonZ = 1003 - wkbMultiPointZ = 1004 - wkbMultiLineStringZ = 1005 - wkbMultiPolygonZ = 1006 - wkbGeometryCollectionZ = 1007 - wkbCircularStringZ = 1008 - wkbCompoundCurveZ = 1009 - wkbCurvePolygonZ = 1010 - wkbMultiCurveZ = 1011 - wkbMultiSurfaceZ = 1012 - wkbCurveZ = 1013 - wkbSurfaceZ = 1014 - - # ISO SQL/MM Part 3. - # M-aware types - wkbPointM = 2001 - wkbLineStringM = 2002 - wkbPolygonM = 2003 - wkbMultiPointM = 2004 - wkbMultiLineStringM = 2005 - wkbMultiPolygonM = 2006 - wkbGeometryCollectionM = 2007 - wkbCircularStringM = 2008 - wkbCompoundCurveM = 2009 - wkbCurvePolygonM = 2010 - wkbMultiCurveM = 2011 - wkbMultiSurfaceM = 2012 - wkbCurveM = 2013 - wkbSurfaceM = 2014 - - # ISO SQL/MM Part 3. - # ZM-aware types ... Meshes.jl doesn't generally support this? - wkbPointZM = 3001 - wkbLineStringZM = 3002 - wkbPolygonZM = 3003 - wkbMultiPointZM = 3004 - wkbMultiLineStringZM = 3005 - wkbMultiPolygonZM = 3006 - wkbGeometryCollectionZM = 3007 - wkbCircularStringZM = 3008 - wkbCompoundCurveZM = 3009 - wkbCurvePolygonZM = 3010 - wkbMultiCurveZM = 3011 - wkbMultiSurfaceZM = 3012 - wkbCurveZM = 3013 - wkbSurfaceZM = 3014 - - # 2.5D extension as per 99-402 - # https://lists.osgeo.org/pipermail/postgis-devel/2004-December/000702.html - - # Julia throws InexactError: convert(Int32, 0x8000000x) - wkbPoint25D = -2147483647 # 0x80000001 - wkbLineString25D = -2147483646 # 0x80000002 - wkbPolygon25D = -2147483645 # 0x80000003 - wkbMultiPoint25D = -2147483644 # 0x80000004 - wkbMultiLineString25D = -2147483643 # 0x80000005 - wkbMultiPolygon25D = -2147483642 # 0x80000006 - wkbGeometryCollection25D = -2147483641 # 0x80000007 -end - include("gpkg/read.jl") include("gpkg/write.jl") diff --git a/src/extra/gpkg/read.jl b/src/extra/gpkg/read.jl index 57afefb..7324be8 100644 --- a/src/extra/gpkg/read.jl +++ b/src/extra/gpkg/read.jl @@ -11,16 +11,13 @@ function gpkgread(fname; layer=1) end function assertgpkg(db) - appid = DBInterface.execute(db, "PRAGMA application_id;") |> first |> only - userversion = DBInterface.execute(db, "PRAGMA user_version;") |> first |> only - if !hasgpkgmetadata(db) throw(ErrorException("missing required metadata tables in the GeoPackage SQL database")) end # Requirement 6: PRAGMA integrity_check returns a single row with the value 'ok' # Requirement 7: PRAGMA foreign_key_check (w/ no parameter value) returns an empty result set - if (DBInterface.execute(db, "PRAGMA integrity_check;") |> first |> only != "ok") || + if ((DBInterface.execute(db, "PRAGMA integrity_check;") |> first).integrity_check != "ok") || !(isempty(DBInterface.execute(db, "PRAGMA foreign_key_check;"))) throw(ErrorException("database integrity at risk or foreign key violation(s)")) end @@ -29,28 +26,32 @@ end # Requirement 10: must include a gpkg_spatial_ref_sys table # Requirement 13: must include a gpkg_contents table function hasgpkgmetadata(db) - sqlstmt =""" - SELECT COUNT(*) FROM sqlite_master WHERE - name IN ('gpkg_spatial_ref_sys', 'gpkg_contents') AND - type IN ('table', 'view'); + tbcount = DBInterface.execute( + db, """ - tbcount = DBInterface.execute(db, sqlstmt) |> first |> only - (tbcount == 2) + SELECT COUNT(*) FROM sqlite_master WHERE + name IN ('gpkg_spatial_ref_sys', 'gpkg_contents') AND + type IN ('table', 'view'); +""" + ) |> first |> only + tbcount == 2 end function gpkgmeshattrs(db, ; layer=1) - sqlstmt = """ - SELECT c.table_name, c.identifier, - g.column_name, g.geometry_type_name, g.z, g.m, c.min_x, c.min_y, - c.max_x, c.max_y, - (SELECT type FROM sqlite_master WHERE lower(name) = - lower(c.table_name) AND type IN ('table', 'view')) AS object_type - FROM gpkg_geometry_columns g - JOIN gpkg_contents c ON (g.table_name = c.table_name) - WHERE - c.data_type = 'features' LIMIT $layer + feature_tables = DBInterface.execute( + db, """ - feature_tables = DBInterface.execute(db, sqlstmt) +SELECT c.table_name, c.identifier, +g.column_name, g.geometry_type_name, g.z, g.m, c.min_x, c.min_y, +c.max_x, c.max_y, +(SELECT type FROM sqlite_master WHERE lower(name) = +lower(c.table_name) AND type IN ('table', 'view')) AS object_type + FROM gpkg_geometry_columns g + JOIN gpkg_contents c ON (g.table_name = c.table_name) + WHERE + c.data_type = 'features' LIMIT $layer +""" + ) tb = [] fields = "fid,"^10^5 for query in feature_tables @@ -107,26 +108,26 @@ end # SHALL match the srs_id column value from the corresponding row in the # gpkg_contents table. function gpkgmesh(db, ; layer=1) - sqlstmt = """ - SELECT g.table_name AS tn, g.column_name AS cn, c.srs_id as crs, g.z as elev, srs.organization as org, srs.organization_coordsys_id as org_coordsys_id, - ( SELECT type FROM sqlite_master WHERE lower(name) = lower(c.table_name) AND type IN ('table', 'view')) AS object_type - FROM gpkg_geometry_columns g, gpkg_spatial_ref_sys srs - JOIN gpkg_contents c ON ( g.table_name = c.table_name ) - WHERE c.data_type = 'features' - AND (SELECT type FROM sqlite_master WHERE lower(name) = lower(c.table_name) AND type IN ('table', 'view')) IS NOT NULL - AND g.srs_id = srs.srs_id - AND g.srs_id = c.srs_id - AND g.z IN (0, 1, 2) - AND g.m IN (0, 1, 2) - LIMIT $layer; - """ - tb = DBInterface.execute(db, sqlstmt) - meshes = Geometry[] - for (tn, cn, org, org_coordsys_id) in [(row.tn, row.cn, row.org, row.org_coordsys_id) for row in tb] - sqlstmt = "SELECT $cn FROM $tn;" - gpkgbinary = DBInterface.execute(db, sqlstmt) + tb = DBInterface.execute( + db, + """ +SELECT g.table_name AS tn, g.column_name AS cn, c.srs_id as crs, g.z as elev, srs.organization as org, srs.organization_coordsys_id as org_coordsys_id, +( SELECT type FROM sqlite_master WHERE lower(name) = lower(c.table_name) AND type IN ('table', 'view')) AS object_type +FROM gpkg_geometry_columns g, gpkg_spatial_ref_sys srs +JOIN gpkg_contents c ON ( g.table_name = c.table_name ) +WHERE c.data_type = 'features' +AND (SELECT type FROM sqlite_master WHERE lower(name) = lower(c.table_name) AND type IN ('table', 'view')) IS NOT NULL +AND g.srs_id = srs.srs_id +AND g.srs_id = c.srs_id +AND g.z IN (0, 1, 2) +AND g.m IN (0, 1, 2) + LIMIT $layer; + """ + ) + meshes = map([(row.tn, row.cn, row.org, row.org_coordsys_id) for row in tb]) do (tn, cn, org, org_coordsys_id) + gpkgbinary = DBInterface.execute(db, "SELECT $cn FROM $tn;") headerlen = 0 - for blob in gpkgbinary + submeshes = map(gpkgbinary) do blob io = IOBuffer(blob[1]) seek(io, 3) flag = read(io, UInt8) @@ -171,26 +172,25 @@ function gpkgmesh(db, ; layer=1) mesh = meshfromwkb(io, srs_id, org, org_coordsys_id, wkbtype, zextent, bswap) if !isnothing(mesh) - push!(meshes, mesh) + mesh end end + submeshes end - meshes + reduce(vcat, meshes) end function meshfromwkb(io, srs_id, org, org_coordsys_id, ewkbtype, zextent, bswap) if iszero(srs_id) crs = LatLon{WGS84Latest} - elseif isone(abs(srs_id)) - crs = zextent ? Cartesian{NoDatum,3} : Cartesian{NoDatum,2} - else + elseif !isone(abs(srs_id)) if org == "EPSG" crs = CoordRefSystems.get(EPSG{org_coordsys_id}) elseif org == "ESRI" crs = CoordRefSystems.get(ERSI{org_coordsys_id}) - else - Cartesian{NoDatum} end + else + crs = Cartesian{NoDatum} end if occursin("Multi", string(ewkbtype)) diff --git a/src/extra/gpkg/write.jl b/src/extra/gpkg/write.jl index b2dc8ff..9279ac8 100644 --- a/src/extra/gpkg/write.jl +++ b/src/extra/gpkg/write.jl @@ -14,33 +14,33 @@ function gpkgwrite(fname, geotable;) # on you system then Synchronous = OFF is for you SQLite.transaction(db) do - sqlstmt = """ - CREATE TABLE gpkg_spatial_ref_sys ( - srs_name TEXT NOT NULL, srs_id INTEGER NOT NULL PRIMARY KEY, - organization TEXT NOT NULL, organization_coordsys_id INTEGER NOT NULL, - definition TEXT NOT NULL, description TEXT, - definition_12_063 TEXT NOT NULL - ); + DBInterface.execute( + db, + """ +CREATE TABLE gpkg_spatial_ref_sys ( + srs_name TEXT NOT NULL, srs_id INTEGER NOT NULL PRIMARY KEY, + organization TEXT NOT NULL, organization_coordsys_id INTEGER NOT NULL, + definition TEXT NOT NULL, description TEXT, + definition_12_063 TEXT NOT NULL +); - CREATE TABLE gpkg_contents ( - table_name TEXT NOT NULL PRIMARY KEY, - data_type TEXT NOT NULL, - identifier TEXT UNIQUE, - description TEXT DEFAULT '', - last_change DATETIME NOT NULL DEFAULT - (strftime('%Y-%m-%dT%H:%M:%fZ','now')), - min_x DOUBLE, min_y DOUBLE, - max_x DOUBLE, max_y DOUBLE, - srs_id INTEGER, - CONSTRAINT fk_gc_r_srs_id FOREIGN KEY (srs_id) REFERENCES - gpkg_spatial_ref_sys(srs_id) - ); - """ - DBInterface.execute(db, sqlstmt) - sqlstmt = SQLite.Stmt(db, "PRAGMA application_id = $GPKG_APPLICATION_ID ") - DBInterface.execute(sqlstmt) - sqlstmt = SQLite.Stmt(db, "PRAGMA user_version = $GPKG_1_4_VERSION ") - DBInterface.execute(sqlstmt) +CREATE TABLE gpkg_contents ( + table_name TEXT NOT NULL PRIMARY KEY, + data_type TEXT NOT NULL, + identifier TEXT UNIQUE, + description TEXT DEFAULT '', + last_change DATETIME NOT NULL DEFAULT + (strftime('%Y-%m-%dT%H:%M:%fZ','now')), + min_x DOUBLE, min_y DOUBLE, + max_x DOUBLE, max_y DOUBLE, + srs_id INTEGER, + CONSTRAINT fk_gc_r_srs_id FOREIGN KEY (srs_id) REFERENCES + gpkg_spatial_ref_sys(srs_id) +); +""" + ) + DBInterface.execute(db, "PRAGMA application_id = $GPKG_APPLICATION_ID ") + DBInterface.execute(db, "PRAGMA user_version = $GPKG_1_4_VERSION ") end # According to https://www.geopackage.org/spec/#r11 @@ -48,7 +48,6 @@ function gpkgwrite(fname, geotable;) # 1. the record with an srs_id of 4326 SHALL correspond to WGS-84 as defined by EPSG in 4326 # 2. the record with an srs_id of -1 SHALL be used for undefined Cartesian coordinate reference systems # 3. the record with an srs_id of 0 SHALL be used for undefined geographic coordinate reference systems - tb = [( srs_name="Undefined Cartesian SRS", srs_id=-1, @@ -79,6 +78,7 @@ function gpkgwrite(fname, geotable;) definition_12_063=CoordRefSystems.wkt2(EPSG{4326}) )] SQLite.load!(tb, db, "gpkg_spatial_ref_sys", replace=true) + table = values(geotable) domain = GeoTables.domain(geotable) crs = GeoTables.crs(domain) diff --git a/src/extra/wkb.jl b/src/extra/wkb.jl new file mode 100644 index 0000000..2a2de50 --- /dev/null +++ b/src/extra/wkb.jl @@ -0,0 +1,83 @@ + + +@enum wkbGeometryType begin + wkbUnknown = 0 + wkbPoint = 1 + wkbLineString = 2 + wkbPolygon = 3 + wkbMultiPoint = 4 + wkbMultiLineString = 5 + wkbMultiPolygon = 6 + wkbGeometryCollection = 7 + wkbCircularString = 8 + wkbCompoundCurve = 9 + wkbCurvePolygon = 10 + wkbMultiCurve = 11 + wkbMultiSurface = 12 + wkbCurve = 13 + wkbSurface = 14 + + wkbNone = 100 # pure attribute records + wkbLinearRing = 101 + + # ISO SQL/MM Part 3: Spatial + # Z-aware types + wkbPointZ = 1001 + wkbLineStringZ = 1002 + wkbPolygonZ = 1003 + wkbMultiPointZ = 1004 + wkbMultiLineStringZ = 1005 + wkbMultiPolygonZ = 1006 + wkbGeometryCollectionZ = 1007 + wkbCircularStringZ = 1008 + wkbCompoundCurveZ = 1009 + wkbCurvePolygonZ = 1010 + wkbMultiCurveZ = 1011 + wkbMultiSurfaceZ = 1012 + wkbCurveZ = 1013 + wkbSurfaceZ = 1014 + + # ISO SQL/MM Part 3. + # M-aware types + wkbPointM = 2001 + wkbLineStringM = 2002 + wkbPolygonM = 2003 + wkbMultiPointM = 2004 + wkbMultiLineStringM = 2005 + wkbMultiPolygonM = 2006 + wkbGeometryCollectionM = 2007 + wkbCircularStringM = 2008 + wkbCompoundCurveM = 2009 + wkbCurvePolygonM = 2010 + wkbMultiCurveM = 2011 + wkbMultiSurfaceM = 2012 + wkbCurveM = 2013 + wkbSurfaceM = 2014 + + # ISO SQL/MM Part 3. + # ZM-aware types + wkbPointZM = 3001 + wkbLineStringZM = 3002 + wkbPolygonZM = 3003 + wkbMultiPointZM = 3004 + wkbMultiLineStringZM = 3005 + wkbMultiPolygonZM = 3006 + wkbGeometryCollectionZM = 3007 + wkbCircularStringZM = 3008 + wkbCompoundCurveZM = 3009 + wkbCurvePolygonZM = 3010 + wkbMultiCurveZM = 3011 + wkbMultiSurfaceZM = 3012 + wkbCurveZM = 3013 + wkbSurfaceZM = 3014 + + # 2.5D extension as per 99-402 + # https://lists.osgeo.org/pipermail/postgis-devel/2004-December/000702.html + wkbPoint25D = 0x80000001 + wkbLineString25D = 0x80000002 + wkbPolygon25D = 0x80000003 + wkbMultiPoint25D = 0x80000004 + wkbMultiLineString25D = 0x80000005 + wkbMultiPolygon25D = 0x80000006 + wkbGeometryCollection25D = 0x80000007 +end From 6c3c505e95b229137c736e8821dbee1f7df43f5c Mon Sep 17 00:00:00 2001 From: jph6366 Date: Mon, 15 Sep 2025 14:15:41 -0400 Subject: [PATCH 12/90] formatting and include(wkb.jl) --- src/extra/gpkg.jl | 1 + src/extra/wkb.jl | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/src/extra/gpkg.jl b/src/extra/gpkg.jl index d203951..f3f5f4a 100644 --- a/src/extra/gpkg.jl +++ b/src/extra/gpkg.jl @@ -8,5 +8,6 @@ const GPKG_APPLICATION_ID = Int(0x47504B47) const GPKG_1_4_VERSION = 10400 +include("wkb.jl") include("gpkg/read.jl") include("gpkg/write.jl") diff --git a/src/extra/wkb.jl b/src/extra/wkb.jl index 2a2de50..644cfc2 100644 --- a/src/extra/wkb.jl +++ b/src/extra/wkb.jl @@ -1,5 +1,4 @@ - @enum wkbGeometryType begin wkbUnknown = 0 wkbPoint = 1 From 5dc18fb6e659043be4a1a0d3b835c02681d3203b Mon Sep 17 00:00:00 2001 From: jph6366 Date: Thu, 18 Sep 2025 10:27:28 -0400 Subject: [PATCH 13/90] incoporated custom SQLite data types, annotated type for wkbgeometry, abstract WKBGeometry type added, handling failing tests (1 erroring gisissues.jl, 1 failing novalues.jl), fixed bugs gpkgmeshattrs and gpkgmesh --- Project.toml | 2 ++ src/GeoIO.jl | 3 ++ src/extra/gpkg.jl | 74 +++++++++++++++++++++++++++++++++++++++++ src/extra/gpkg/read.jl | 41 ++++++++++++----------- src/extra/gpkg/write.jl | 68 ++++++++++++++++--------------------- src/extra/wkb.jl | 40 +++++++++++++++++++++- 6 files changed, 167 insertions(+), 61 deletions(-) diff --git a/Project.toml b/Project.toml index 8b4afee..3398df0 100644 --- a/Project.toml +++ b/Project.toml @@ -10,6 +10,7 @@ Colors = "5ae59095-9a9b-59fe-a467-6f913c188581" CommonDataModel = "1fbeeb36-5f17-413c-809b-666fb144f157" CoordRefSystems = "b46f11dc-f210-4604-bfba-323c1ec968cb" DBInterface = "a10d1c49-ce27-4219-8d33-6db1a4562965" +Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" FileIO = "5789e2e9-d7fb-5bc7-8068-2c6fae9b9549" Format = "1fa38f19-a742-5d3f-a2b9-30dd87b9d5f8" GRIBDatasets = "82be9cdb-ee19-4151-bdb3-b400788d9abc" @@ -44,6 +45,7 @@ Colors = "0.12, 0.13" CommonDataModel = "0.2, 0.3" CoordRefSystems = "0.18" DBInterface = "2.6" +Dates = "1.11.0" FileIO = "1.16" Format = "1.3" GRIBDatasets = "0.3, 0.4" diff --git a/src/GeoIO.jl b/src/GeoIO.jl index 407a893..dbe393c 100644 --- a/src/GeoIO.jl +++ b/src/GeoIO.jl @@ -42,6 +42,9 @@ import CSV import DBInterface import SQLite +# Local Date and Time +import Dates + # geostats formats import GslibIO diff --git a/src/extra/gpkg.jl b/src/extra/gpkg.jl index f3f5f4a..12b7ab1 100644 --- a/src/extra/gpkg.jl +++ b/src/extra/gpkg.jl @@ -8,6 +8,80 @@ const GPKG_APPLICATION_ID = Int(0x47504B47) const GPKG_1_4_VERSION = 10400 +function _geomtype(geoms::AbstractVector{<:Geometry}) + if isempty(geoms) + return "GEOMETRY" + end + T = eltype(geoms) + + if T <: Point + "POINT" + elseif T <: Rope + "LINESTRING" + elseif T <: Ring + "LINESTRING" + elseif T <: PolyArea + "POLYGON" + elseif T <: Multi + element_type = eltype(parent(first(geoms))) + if element_type <: Point + "MULTIPOINT" + elseif element_type <: Rope + "MULTILINESTRING" + elseif element_type <: PolyArea + "MULTIPOLYGON" + end + else + "GEOMETRY" + end +end + include("wkb.jl") + +function _sqlitetype(geomtype::AbstractString) + if geomtype == "POINT" + WKBPoint + elseif geomtype == "LINESTRING" + WKBLineString + elseif geomtype == "POLYGON" + WKBPolygon + elseif geomtype == "MULTIPOINT" + WKBMultiPoint + elseif geomtype == "MULTILINESTRING" + WKBMultiLineString + elseif geomtype == "MULTIPOLYGON" + WKBMultiPolygon + else + WKBGeometry + end +end + +function SQLite.sqlitetype_(::Type{WKBPoint}) + return "POINT" +end + +function SQLite.sqlitetype_(::Type{WKBLineString}) + return "LINESTRING" +end + +function SQLite.sqlitetype_(::Type{WKBPolygon}) + return "POLYGON" +end + +function SQLite.sqlitetype_(::Type{WKBMultiPoint}) + return "MULTIPOINT" +end + +function SQLite.sqlitetype_(::Type{WKBMultiLineString}) + return "MULTILINESTRING" +end + +function SQLite.sqlitetype_(::Type{WKBMultiPolygon}) + return "MULTIPOLYGON" +end + +function SQLite.sqlitetype_(::Type{WKBGeometry}) + return "GEOMETRY" +end include("gpkg/read.jl") include("gpkg/write.jl") diff --git a/src/extra/gpkg/read.jl b/src/extra/gpkg/read.jl index 7324be8..31d832b 100644 --- a/src/extra/gpkg/read.jl +++ b/src/extra/gpkg/read.jl @@ -52,27 +52,26 @@ lower(c.table_name) AND type IN ('table', 'view')) AS object_type c.data_type = 'features' LIMIT $layer """ ) - tb = [] - fields = "fid,"^10^5 - for query in feature_tables + fields = nothing + tb = map(feature_tables) do query tn = query.table_name cn = query.column_name ft_attrs = SQLite.tableinfo(db, tn).name deleteat!(ft_attrs, findall(x -> isequal(x, cn), ft_attrs)) rp_attrs = join(ft_attrs, ", ") # keep the shortest set of attributes to avoid KeyError {Key} not found - fields = length(fields) > length(rp_attrs) ? rp_attrs : fields # smelly hack, eval shortest common subset of fields instead - sqlstmt = "SELECT $fields from $tn" - if isone(layer) - rowvalues = DBInterface.execute(db, sqlstmt) |> first - push!(tb, rowvalues) - else - for rv in DBInterface.execute(db, sqlstmt) - push!(tb, NamedTuple(rv)) - end + if isnothing(fields) || length(rp_attrs) < length(fields) + fields = rp_attrs end + rowvals = map(DBInterface.execute(db, "SELECT $fields from $tn")) do rv + NamedTuple(rv) + end + rowvals + end + if isone(length(first(tb))) + return tb end - tb + vcat(tb...) end # https://www.geopackage.org/spec/#:~:text=2.1.5.1.2.%20Table%20Data%20Values @@ -128,14 +127,15 @@ AND g.m IN (0, 1, 2) gpkgbinary = DBInterface.execute(db, "SELECT $cn FROM $tn;") headerlen = 0 submeshes = map(gpkgbinary) do blob - io = IOBuffer(blob[1]) + gpkgbindata = isa(blob[1], WKBGeometry) ? blob[1].data : blob[1] + io = IOBuffer(gpkgbindata) seek(io, 3) flag = read(io, UInt8) # Note that Julia does not convert the endianness for you. # Use ntoh or ltoh for this purpose. bswap = isone(flag & 0x01) ? ltoh : ntoh - srs_id = bswap(read(io, UInt32)) + srsid = bswap(read(io, UInt32)) envelope = (flag & (0x07 << 1)) >> 1 envelopedims = 0 @@ -169,7 +169,7 @@ AND g.m IN (0, 1, 2) zextent = isequal(envelopedims, 2) - mesh = meshfromwkb(io, srs_id, org, org_coordsys_id, wkbtype, zextent, bswap) + mesh = meshfromwkb(io, srsid, org, org_coordsys_id, wkbtype, zextent, bswap) if !isnothing(mesh) mesh @@ -177,13 +177,14 @@ AND g.m IN (0, 1, 2) end submeshes end - reduce(vcat, meshes) + # efficient method for concatenating arrays of arrays + reduce(vcat, meshes) # Future versions of Julia might change the reduce algorithm end -function meshfromwkb(io, srs_id, org, org_coordsys_id, ewkbtype, zextent, bswap) - if iszero(srs_id) +function meshfromwkb(io, srsid, org, org_coordsys_id, ewkbtype, zextent, bswap) + if iszero(srsid) crs = LatLon{WGS84Latest} - elseif !isone(abs(srs_id)) + elseif !isone(abs(srsid)) if org == "EPSG" crs = CoordRefSystems.get(EPSG{org_coordsys_id}) elseif org == "ESRI" diff --git a/src/extra/gpkg/write.jl b/src/extra/gpkg/write.jl index 9279ac8..5ebce90 100644 --- a/src/extra/gpkg/write.jl +++ b/src/extra/gpkg/write.jl @@ -9,7 +9,7 @@ function gpkgwrite(fname, geotable;) # Commits can be orders of magnitude faster with # Setting PRAGMA synchronous=OFF but, # can cause the database to go corrupt - # if there is an operating-system crash or power failure. + # if there is an operating system crash or power failure. # If the power never goes out and no programs ever crash # on you system then Synchronous = OFF is for you @@ -37,6 +37,19 @@ CREATE TABLE gpkg_contents ( CONSTRAINT fk_gc_r_srs_id FOREIGN KEY (srs_id) REFERENCES gpkg_spatial_ref_sys(srs_id) ); + +CREATE TABLE gpkg_geometry_columns ( + table_name TEXT NOT NULL, + column_name TEXT NOT NULL, + geometry_type_name TEXT NOT NULL, + srs_id INTEGER NOT NULL, + z TINYINT NOT NULL, + m TINYINT NOT NULL, + CONSTRAINT pk_geom_cols PRIMARY KEY (table_name, column_name), + CONSTRAINT uk_gc_table_name UNIQUE (table_name), + CONSTRAINT fk_gc_tn FOREIGN KEY (table_name) REFERENCES gpkg_contents(table_name), + CONSTRAINT fk_gc_srs FOREIGN KEY (srs_id) REFERENCES gpkg_spatial_ref_sys (srs_id) +); """ ) DBInterface.execute(db, "PRAGMA application_id = $GPKG_APPLICATION_ID ") @@ -106,9 +119,11 @@ function _extracttablevals(db, table, domain, crs, geom) vcat(gpkgbinheader, take!(io)) end + geomtype = _geomtype(geom) + GeomType = _sqlitetype(geomtype) + table = - isone(length(table)) ? [(NamedTuple(table |> first)..., geom=gpkgbinary[1])] : - [(; t..., geom=g) for (t, g) in zip(table, gpkgbinary)] + isnothing(table) ? Symbol[] : [(; t..., geom=GeomType(g)) for (t, g) in zip(Tables.rowtable(table), gpkgbinary)] SQLite.load!(table, db, replace=false) # autogenerates table name # replace=false controls whether an INSERT INTO ... statement is generated or a REPLACE INTO .... @@ -116,7 +131,7 @@ function _extracttablevals(db, table, domain, crs, geom) ( DBInterface.execute( db, - """ SELECT name FROM sqlite_master WHERE type='table' AND name NOT IN ("gpkg_contents", "gpkg_spatial_ref_sys") """ + """ SELECT name FROM sqlite_master WHERE type='table' AND name NOT IN ("gpkg_contents", "gpkg_spatial_ref_sys", "gpkg_geometry_columns") """ ) |> first ).name @@ -128,7 +143,7 @@ function _extracttablevals(db, table, domain, crs, geom) data_type="features", identifier=tn, description="", - last_change=Dates.format(now(UTC), "yyyy-mm-ddTHH:MM:SSZ"), + last_change=Dates.format(Dates.now(Dates.UTC), "yyyy-mm-ddTHH:MM:SSZ"), min_x=mincoords[1], min_y=mincoords[2], max_x=maxcoords[1], @@ -151,41 +166,14 @@ function _extracttablevals(db, table, domain, crs, geom) geomcolumns = [( table_name=tn, column_name="geom", - geometry_type_name=_geomtype(geom), + geometry_type_name=geomtype, srs_id=srid, - z=paramdim((geom |> first)) >= 2 ? 1 : 0, + z=paramdim((geom |> first)) > 2 ? 1 : 0, m=0 )] SQLite.load!(geomcolumns, db, "gpkg_geometry_columns", replace=true) end -function _geomtype(geoms::AbstractVector{<:Geometry}) - if isempty(geoms) - return "GEOMETRY" - end - T = eltype(geoms) - - if T <: Point - return "POINT" - elseif T <: Rope - return "LINESTRING" - elseif T <: Ring - return "LINESTRING" - elseif T <: PolyArea - return "POLYGON" - elseif T <: Multi - element_type = eltype(parent(first(geoms))) - if element_type <: Point - return "MULTIPOINT" - elseif element_type <: Rope - return "MULTILINESTRING" - elseif element_type <: PolyArea - return "MULTIPOLYGON" - end - end - return "GEOMETRY" -end - function _wkbtype(geometry) if geometry isa Point return wkbPoint @@ -214,7 +202,7 @@ function writegpkgheader(srs_id, geom) flagsbyte = UInt8(0x07 >> 1) write(io, flagsbyte) - write(io, htol(Int32(srs_id))) + write(io, htol(UInt32(srs_id))) bbox = boundingbox(geom) write(io, htol(Float64(CoordRefSystems.raw(coords(bbox.min))[2]))) @@ -233,7 +221,7 @@ end function writewkbgeom(io, geom) wkbtype = paramdim(geom) < 3 ? _wkbtype(geom) : _wkbsetz(_wkbtype(geom)) write(io, htol(one(UInt8))) - write(io, htol(Int32(wkbtype))) + write(io, htol(UInt32(wkbtype))) _wkbgeom(io, wkbtype, geom) end @@ -269,7 +257,7 @@ function _wkbcoordinates(io, wkbtype, coords) end function _wkblinestring(io, wkb_type, coord_list) - write(io, htol(Int32(length(coord_list)))) + write(io, htol(UInt32(length(coord_list)))) for n_coords::Point in coord_list coordinates = CoordRefSystems.raw(coords(n_coords)) _wkbcoordinates(io, wkb_type, coordinates) @@ -277,7 +265,7 @@ function _wkblinestring(io, wkb_type, coord_list) end function _wkbpolygon(io, wkb_type, rings) - write(io, htol(Int32(length(rings)))) + write(io, htol(UInt32(length(rings)))) for ring in rings coord_list = vertices(ring) _wkblinestring(io, wkb_type, coord_list) @@ -285,10 +273,10 @@ function _wkbpolygon(io, wkb_type, rings) end function _wkbmulti(io, wkbtype, geoms) - write(io, htol(Int32(length(geoms |> parent)))) + write(io, htol(UInt32(length(geoms |> parent)))) for sf in geoms |> parent write(io, one(UInt8)) - write(io, Int32(Int(wkbMultiPolygon) - 3)) + write(io, UInt32(Int(wkbMultiPolygon) - 3)) writewkbsf(io, wkbGeometryType(Int(wkbtype) - 3), sf) end end \ No newline at end of file diff --git a/src/extra/wkb.jl b/src/extra/wkb.jl index 644cfc2..148d871 100644 --- a/src/extra/wkb.jl +++ b/src/extra/wkb.jl @@ -1,5 +1,5 @@ -@enum wkbGeometryType begin +@enum wkbGeometryType::UInt32 begin wkbUnknown = 0 wkbPoint = 1 wkbLineString = 2 @@ -80,3 +80,41 @@ wkbMultiPolygon25D = 0x80000006 wkbGeometryCollection25D = 0x80000007 end + +abstract type WKBGeometry end + +struct WKBPoint <: WKBGeometry + data::Vector{UInt8} +end + +struct WKBLineString <: WKBGeometry + data::Vector{UInt8} +end + +struct WKBPolygon <: WKBGeometry + data::Vector{UInt8} +end + +struct WKBMultiPoint <: WKBGeometry + data::Vector{UInt8} +end + +struct WKBMultiLineString <: WKBGeometry + data::Vector{UInt8} +end + +struct WKBMultiPolygon <: WKBGeometry + data::Vector{UInt8} +end + +struct WKBGeometryCollection <: WKBGeometry + data::Vector{UInt8} +end + +function Base.getproperty(gpkg::WKBGeometry, sym::Symbol) + if sym == :data + return getfield(gpkg, :data) + else + return getfield(gpkg, sym) + end +end From 46b20b9aebe71bebb0f86371494d49ea73923458 Mon Sep 17 00:00:00 2001 From: jph6366 Date: Thu, 18 Sep 2025 10:29:17 -0400 Subject: [PATCH 14/90] removed minor version --- Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index 3398df0..38b534b 100644 --- a/Project.toml +++ b/Project.toml @@ -45,7 +45,7 @@ Colors = "0.12, 0.13" CommonDataModel = "0.2, 0.3" CoordRefSystems = "0.18" DBInterface = "2.6" -Dates = "1.11.0" +Dates = "1.11" FileIO = "1.16" Format = "1.3" GRIBDatasets = "0.3, 0.4" From 6538130be3e476fa16e0a5874590f40366b7f2c7 Mon Sep 17 00:00:00 2001 From: jph6366 Date: Fri, 19 Sep 2025 17:16:59 -0400 Subject: [PATCH 15/90] passing all tests, added branch for novalues, and added wkblinearring function for ring vs rope extents --- src/extra/gpkg/read.jl | 12 ++++++------ src/extra/gpkg/write.jl | 16 ++++++++++++++-- 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/src/extra/gpkg/read.jl b/src/extra/gpkg/read.jl index 31d832b..66fa94b 100644 --- a/src/extra/gpkg/read.jl +++ b/src/extra/gpkg/read.jl @@ -7,6 +7,9 @@ function gpkgread(fname; layer=1) assertgpkg(db) geom = gpkgmesh(db, ; layer) attrs = gpkgmeshattrs(db, ; layer) + if eltype(attrs) <: Nothing + return GeoTables.georef(nothing, geom) + end GeoTables.georef(attrs, geom) end @@ -63,14 +66,11 @@ lower(c.table_name) AND type IN ('table', 'view')) AS object_type if isnothing(fields) || length(rp_attrs) < length(fields) fields = rp_attrs end - rowvals = map(DBInterface.execute(db, "SELECT $fields from $tn")) do rv + rowvals = length(fields) |> iszero ? nothing : map(DBInterface.execute(db, "SELECT $fields from $tn")) do rv NamedTuple(rv) end rowvals end - if isone(length(first(tb))) - return tb - end vcat(tb...) end @@ -212,10 +212,10 @@ function meshfromsf(io, crs, ewkbtype, zextent, bswap) Point(crs(elem...)) elseif isequal(ewkbtype, wkbLineString) elem = wkblinestring(io, zextent, bswap) - if first(elem) != last(elem) + if length(elem) >= 2 && first(elem) != last(elem) Rope([Point(crs(coords...)) for coords in elem]...) else - Ring([Point(crs(coords...)) for coords in elem[2:end]]...) + Ring([Point(crs(coords...)) for coords in elem[1:(end - 1)]]...) end elseif isequal(ewkbtype, wkbPolygon) elem = wkbpolygon(io, zextent, bswap) diff --git a/src/extra/gpkg/write.jl b/src/extra/gpkg/write.jl index 5ebce90..e6a001b 100644 --- a/src/extra/gpkg/write.jl +++ b/src/extra/gpkg/write.jl @@ -123,8 +123,8 @@ function _extracttablevals(db, table, domain, crs, geom) GeomType = _sqlitetype(geomtype) table = - isnothing(table) ? Symbol[] : [(; t..., geom=GeomType(g)) for (t, g) in zip(Tables.rowtable(table), gpkgbinary)] - + isnothing(table) ? [(; geom=GeomType(g),) for (i, g) in zip(1:length(gpkgbinary), gpkgbinary)] : + [(; t..., geom=GeomType(g)) for (t, g) in zip(Tables.rowtable(table), gpkgbinary)] SQLite.load!(table, db, replace=false) # autogenerates table name # replace=false controls whether an INSERT INTO ... statement is generated or a REPLACE INTO .... tn = @@ -230,6 +230,9 @@ function writewkbsf(io, wkbtype, geom) _wkbpolygon(io, wkbtype, [boundary(geom::PolyArea)]) elseif wkbtype == wkbLineString || wkbtype == wkbLineString || wkbtype == wkbLineString25D coordlist = vertices(geom) + if typeof(geom) <: Ring + return _wkblinearring(io, wkbtype, coordlist) + end _wkblinestring(io, wkbtype, coordlist) elseif wkbtype == wkbPoint || wkbtype == wkbPointZ || wkbtype == wkbPoint25D coordinates = CoordRefSystems.raw(coords(geom)) @@ -264,6 +267,15 @@ function _wkblinestring(io, wkb_type, coord_list) end end +function _wkblinearring(io, wkb_type, coord_list) + write(io, htol(UInt32(length(coord_list) + 1))) + for n_coords::Point in coord_list + coordinates = CoordRefSystems.raw(coords(n_coords)) + _wkbcoordinates(io, wkb_type, coordinates) + end + _wkbcoordinates(io, wkb_type, CoordRefSystems.raw(first(coord_list) |> coords)) +end + function _wkbpolygon(io, wkb_type, rings) write(io, htol(UInt32(length(rings)))) for ring in rings From e93f91caa2429f60caa8cc5f2323173eecf308c2 Mon Sep 17 00:00:00 2001 From: jph6366 Date: Fri, 19 Sep 2025 20:41:59 -0400 Subject: [PATCH 16/90] missing blob error handling --- src/extra/gpkg/read.jl | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/extra/gpkg/read.jl b/src/extra/gpkg/read.jl index 66fa94b..e742092 100644 --- a/src/extra/gpkg/read.jl +++ b/src/extra/gpkg/read.jl @@ -126,7 +126,11 @@ AND g.m IN (0, 1, 2) meshes = map([(row.tn, row.cn, row.org, row.org_coordsys_id) for row in tb]) do (tn, cn, org, org_coordsys_id) gpkgbinary = DBInterface.execute(db, "SELECT $cn FROM $tn;") headerlen = 0 - submeshes = map(gpkgbinary) do blob + named_rows = map(NamedTuple, gpkgbinary) + gpkgblobs = filter(named_rows) do row + !ismissing(getfield(row, Symbol(cn))) + end + submeshes = map(gpkgblobs) do blob gpkgbindata = isa(blob[1], WKBGeometry) ? blob[1].data : blob[1] io = IOBuffer(gpkgbindata) seek(io, 3) From d73a169ac4a605c1f5cf3814e74ae7106bfa8922 Mon Sep 17 00:00:00 2001 From: jph6366 Date: Fri, 19 Sep 2025 23:08:06 -0400 Subject: [PATCH 17/90] removed unnecessary arrays --- src/extra/gpkg/read.jl | 8 ++++---- src/extra/gpkg/write.jl | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/extra/gpkg/read.jl b/src/extra/gpkg/read.jl index e742092..8335cf7 100644 --- a/src/extra/gpkg/read.jl +++ b/src/extra/gpkg/read.jl @@ -123,12 +123,12 @@ AND g.m IN (0, 1, 2) LIMIT $layer; """ ) - meshes = map([(row.tn, row.cn, row.org, row.org_coordsys_id) for row in tb]) do (tn, cn, org, org_coordsys_id) + meshes = map((row.tn, row.cn, row.org, row.org_coordsys_id) for row in tb) do (tn, cn, org, org_coordsys_id) gpkgbinary = DBInterface.execute(db, "SELECT $cn FROM $tn;") headerlen = 0 named_rows = map(NamedTuple, gpkgbinary) gpkgblobs = filter(named_rows) do row - !ismissing(getfield(row, Symbol(cn))) + !ismissing(getfield(row, Symbol(cn))) end submeshes = map(gpkgblobs) do blob gpkgbindata = isa(blob[1], WKBGeometry) ? blob[1].data : blob[1] @@ -217,9 +217,9 @@ function meshfromsf(io, crs, ewkbtype, zextent, bswap) elseif isequal(ewkbtype, wkbLineString) elem = wkblinestring(io, zextent, bswap) if length(elem) >= 2 && first(elem) != last(elem) - Rope([Point(crs(coords...)) for coords in elem]...) + splat(Rope)(Point(crs(coords...)) for coords in elem) else - Ring([Point(crs(coords...)) for coords in elem[1:(end - 1)]]...) + splat(Ring)(Point(crs(coords...)) for coords in elem[1:(end - 1)]) end elseif isequal(ewkbtype, wkbPolygon) elem = wkbpolygon(io, zextent, bswap) diff --git a/src/extra/gpkg/write.jl b/src/extra/gpkg/write.jl index e6a001b..b6bc62b 100644 --- a/src/extra/gpkg/write.jl +++ b/src/extra/gpkg/write.jl @@ -261,7 +261,7 @@ end function _wkblinestring(io, wkb_type, coord_list) write(io, htol(UInt32(length(coord_list)))) - for n_coords::Point in coord_list + for n_coords in coord_list coordinates = CoordRefSystems.raw(coords(n_coords)) _wkbcoordinates(io, wkb_type, coordinates) end @@ -269,7 +269,7 @@ end function _wkblinearring(io, wkb_type, coord_list) write(io, htol(UInt32(length(coord_list) + 1))) - for n_coords::Point in coord_list + for n_coords in coord_list coordinates = CoordRefSystems.raw(coords(n_coords)) _wkbcoordinates(io, wkb_type, coordinates) end From 1cf927ed7dbf382a6fb44800e4627de44bc58418 Mon Sep 17 00:00:00 2001 From: jph6366 Date: Mon, 22 Sep 2025 10:04:26 -0400 Subject: [PATCH 18/90] added necessary arrays comprehensions back, database needs to be closed, idiomatic refactor of SQLite.load! internal transactions, small fixes --- src/extra/gpkg/read.jl | 9 ++++--- src/extra/gpkg/write.jl | 56 ++++++++++++++++------------------------- 2 files changed, 26 insertions(+), 39 deletions(-) diff --git a/src/extra/gpkg/read.jl b/src/extra/gpkg/read.jl index 8335cf7..0ff052f 100644 --- a/src/extra/gpkg/read.jl +++ b/src/extra/gpkg/read.jl @@ -7,6 +7,7 @@ function gpkgread(fname; layer=1) assertgpkg(db) geom = gpkgmesh(db, ; layer) attrs = gpkgmeshattrs(db, ; layer) + DBInterface.close!(db) if eltype(attrs) <: Nothing return GeoTables.georef(nothing, geom) end @@ -128,7 +129,7 @@ AND g.m IN (0, 1, 2) headerlen = 0 named_rows = map(NamedTuple, gpkgbinary) gpkgblobs = filter(named_rows) do row - !ismissing(getfield(row, Symbol(cn))) + !ismissing(getfield(row, Symbol(cn))) # ignore all rows with missing geometries end submeshes = map(gpkgblobs) do blob gpkgbindata = isa(blob[1], WKBGeometry) ? blob[1].data : blob[1] @@ -139,7 +140,7 @@ AND g.m IN (0, 1, 2) # Use ntoh or ltoh for this purpose. bswap = isone(flag & 0x01) ? ltoh : ntoh - srsid = bswap(read(io, UInt32)) + srsid = bswap(read(io, Int32)) envelope = (flag & (0x07 << 1)) >> 1 envelopedims = 0 @@ -217,9 +218,9 @@ function meshfromsf(io, crs, ewkbtype, zextent, bswap) elseif isequal(ewkbtype, wkbLineString) elem = wkblinestring(io, zextent, bswap) if length(elem) >= 2 && first(elem) != last(elem) - splat(Rope)(Point(crs(coords...)) for coords in elem) + Rope([Point(coords...) for coords in elem]) else - splat(Ring)(Point(crs(coords...)) for coords in elem[1:(end - 1)]) + Ring([Point(crs(coords...)) for coords in elem[1:(end - 1)]]) end elseif isequal(ewkbtype, wkbPolygon) elem = wkbpolygon(io, zextent, bswap) diff --git a/src/extra/gpkg/write.jl b/src/extra/gpkg/write.jl index b6bc62b..4cfa7b2 100644 --- a/src/extra/gpkg/write.jl +++ b/src/extra/gpkg/write.jl @@ -61,36 +61,20 @@ CREATE TABLE gpkg_geometry_columns ( # 1. the record with an srs_id of 4326 SHALL correspond to WGS-84 as defined by EPSG in 4326 # 2. the record with an srs_id of -1 SHALL be used for undefined Cartesian coordinate reference systems # 3. the record with an srs_id of 0 SHALL be used for undefined geographic coordinate reference systems - tb = [( - srs_name="Undefined Cartesian SRS", - srs_id=-1, - organization="NONE", - organization_coordsys_id=-1, - definition="undefined", - description="undefined geographic coordinate reference system", - definition_12_063="undefined" - )] - SQLite.load!(tb, db, "gpkg_spatial_ref_sys", replace=true) - tb = [( - srs_name="Undefined geographic SRS", - srs_id=0, - organization="NONE", - organization_coordsys_id=0, - definition="undefined", - description="undefined geographic coordinate reference system", - definition_12_063="undefined" - )] - SQLite.load!(tb, db, "gpkg_spatial_ref_sys", replace=true) - tb = [( - srs_name="WGS 84 geodectic", - srs_id=4326, - organization="EPSG", - organization_coordsys_id=4326, - definition=CoordRefSystems.wkt2(EPSG{4326}), - description="longitude/latitude coordinates in decimal degrees on the WGS 84 spheroid", - definition_12_063=CoordRefSystems.wkt2(EPSG{4326}) - )] - SQLite.load!(tb, db, "gpkg_spatial_ref_sys", replace=true) + stmt = SQLite.Stmt( + db, + """ + INSERT INTO gpkg_spatial_ref_sys + (srs_name, srs_id, organization, organization_coordsys_id, definition, description, definition_12_063) + VALUES + ('Undefined Cartesian SRS', -1, 'NONE', -1, 'undefined', 'undefined geographic coordinate reference system', 'undefined'), + ('Undefined geographic SRS', 0, 'NONE', 0, 'undefined', 'undefined geographic coordinate reference system', 'undefined'), + ('WGS 84 geodectic', 4326, 'EPSG', 4326, 'GEOGCRS["WGS 84",DATUM["World Geodetic System 1984",ELLIPSOID["WGS 84",6378137,298.257223563,LENGTHUNIT["metre",1]]],PRIMEM["Greenwich",0,ANGLEUNIT["degree",0.0174532925199433]],CS[ellipsoidal,2],AXIS["geodetic latitude (Lat)",north,ORDER[1],ANGLEUNIT["degree",0.0174532925199433]],AXIS["geodetic longitude (Lon)",east,ORDER[2],ANGLEUNIT["degree",0.0174532925199433]],ID["EPSG",4326]]', 'longitude/latitude coordinates in decimal degrees on the WGS 84 spheroid', 'GEOGCRS["WGS 84",DATUM["World Geodetic System 1984",ELLIPSOID["WGS 84",6378137,298.257223563,LENGTHUNIT["metre",1]]],PRIMEM["Greenwich",0,ANGLEUNIT["degree",0.0174532925199433]],CS[ellipsoidal,2],AXIS["geodetic latitude (Lat)",north,ORDER[1],ANGLEUNIT["degree",0.0174532925199433]],AXIS["geodetic longitude (Lon)",east,ORDER[2],ANGLEUNIT["degree",0.0174532925199433]],ID["EPSG",4326]]'); + """ + ) + SQLite.transaction(db) do # execute do x closure + DBInterface.execute(stmt) + end # then "commit" the transaction after executing table = values(geotable) domain = GeoTables.domain(geotable) @@ -98,6 +82,8 @@ CREATE TABLE gpkg_geometry_columns ( geom = collect(domain) _extracttablevals(db, table, domain, crs, geom) + + DBInterface.close!(db) end function _extracttablevals(db, table, domain, crs, geom) @@ -202,7 +188,7 @@ function writegpkgheader(srs_id, geom) flagsbyte = UInt8(0x07 >> 1) write(io, flagsbyte) - write(io, htol(UInt32(srs_id))) + write(io, htol(Int32(srs_id))) bbox = boundingbox(geom) write(io, htol(Float64(CoordRefSystems.raw(coords(bbox.min))[2]))) @@ -226,19 +212,19 @@ function writewkbgeom(io, geom) end function writewkbsf(io, wkbtype, geom) - if wkbtype == wkbPolygon || wkbtype == wkbPolygonZ || wkbtype == wkbPolygon25D + if wkbtype == wkbPolygon || wkbtype == wkbPolygonZ _wkbpolygon(io, wkbtype, [boundary(geom::PolyArea)]) - elseif wkbtype == wkbLineString || wkbtype == wkbLineString || wkbtype == wkbLineString25D + elseif wkbtype == wkbLineString || wkbtype == wkbLineString coordlist = vertices(geom) if typeof(geom) <: Ring return _wkblinearring(io, wkbtype, coordlist) end _wkblinestring(io, wkbtype, coordlist) - elseif wkbtype == wkbPoint || wkbtype == wkbPointZ || wkbtype == wkbPoint25D + elseif wkbtype == wkbPoint || wkbtype == wkbPointZ coordinates = CoordRefSystems.raw(coords(geom)) _wkbcoordinates(io, wkbtype, coordinates) else - @error "my hovercraft is full of eels: $wkbtype" + throw(ErrorException("Well-Known Binary Geometry not supported: $wkbtype")) end end From 4201eb260a0c6919e3426c04f03688f4e48e78da Mon Sep 17 00:00:00 2001 From: jph6366 Date: Tue, 23 Sep 2025 11:04:33 -0400 Subject: [PATCH 19/90] fixing Ring and Rope expressions to include splat and rearranging SQLite.Stmt and Transaction --- src/extra/gpkg/read.jl | 4 ++-- src/extra/gpkg/write.jl | 36 ++++++++++++++++++------------------ 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/src/extra/gpkg/read.jl b/src/extra/gpkg/read.jl index 0ff052f..cc2064f 100644 --- a/src/extra/gpkg/read.jl +++ b/src/extra/gpkg/read.jl @@ -218,9 +218,9 @@ function meshfromsf(io, crs, ewkbtype, zextent, bswap) elseif isequal(ewkbtype, wkbLineString) elem = wkblinestring(io, zextent, bswap) if length(elem) >= 2 && first(elem) != last(elem) - Rope([Point(coords...) for coords in elem]) + Rope([Point(crs(coords...)) for coords in elem]...) else - Ring([Point(crs(coords...)) for coords in elem[1:(end - 1)]]) + Ring([Point(crs(coords...)) for coords in elem[1:(end - 1)]]...) end elseif isequal(ewkbtype, wkbPolygon) elem = wkbpolygon(io, zextent, bswap) diff --git a/src/extra/gpkg/write.jl b/src/extra/gpkg/write.jl index 4cfa7b2..828a71a 100644 --- a/src/extra/gpkg/write.jl +++ b/src/extra/gpkg/write.jl @@ -61,19 +61,20 @@ CREATE TABLE gpkg_geometry_columns ( # 1. the record with an srs_id of 4326 SHALL correspond to WGS-84 as defined by EPSG in 4326 # 2. the record with an srs_id of -1 SHALL be used for undefined Cartesian coordinate reference systems # 3. the record with an srs_id of 0 SHALL be used for undefined geographic coordinate reference systems - stmt = SQLite.Stmt( - db, - """ - INSERT INTO gpkg_spatial_ref_sys - (srs_name, srs_id, organization, organization_coordsys_id, definition, description, definition_12_063) - VALUES - ('Undefined Cartesian SRS', -1, 'NONE', -1, 'undefined', 'undefined geographic coordinate reference system', 'undefined'), - ('Undefined geographic SRS', 0, 'NONE', 0, 'undefined', 'undefined geographic coordinate reference system', 'undefined'), - ('WGS 84 geodectic', 4326, 'EPSG', 4326, 'GEOGCRS["WGS 84",DATUM["World Geodetic System 1984",ELLIPSOID["WGS 84",6378137,298.257223563,LENGTHUNIT["metre",1]]],PRIMEM["Greenwich",0,ANGLEUNIT["degree",0.0174532925199433]],CS[ellipsoidal,2],AXIS["geodetic latitude (Lat)",north,ORDER[1],ANGLEUNIT["degree",0.0174532925199433]],AXIS["geodetic longitude (Lon)",east,ORDER[2],ANGLEUNIT["degree",0.0174532925199433]],ID["EPSG",4326]]', 'longitude/latitude coordinates in decimal degrees on the WGS 84 spheroid', 'GEOGCRS["WGS 84",DATUM["World Geodetic System 1984",ELLIPSOID["WGS 84",6378137,298.257223563,LENGTHUNIT["metre",1]]],PRIMEM["Greenwich",0,ANGLEUNIT["degree",0.0174532925199433]],CS[ellipsoidal,2],AXIS["geodetic latitude (Lat)",north,ORDER[1],ANGLEUNIT["degree",0.0174532925199433]],AXIS["geodetic longitude (Lon)",east,ORDER[2],ANGLEUNIT["degree",0.0174532925199433]],ID["EPSG",4326]]'); - """ - ) SQLite.transaction(db) do # execute do x closure - DBInterface.execute(stmt) + DBInterface.execute( + SQLite.Stmt( + db, + """ + INSERT INTO gpkg_spatial_ref_sys + (srs_name, srs_id, organization, organization_coordsys_id, definition, description, definition_12_063) + VALUES + ('Undefined Cartesian SRS', -1, 'NONE', -1, 'undefined', 'undefined geographic coordinate reference system', 'undefined'), + ('Undefined geographic SRS', 0, 'NONE', 0, 'undefined', 'undefined geographic coordinate reference system', 'undefined'), + ('WGS 84 geodectic', 4326, 'EPSG', 4326, 'GEOGCRS["WGS 84",DATUM["World Geodetic System 1984",ELLIPSOID["WGS 84",6378137,298.257223563,LENGTHUNIT["metre",1]]],PRIMEM["Greenwich",0,ANGLEUNIT["degree",0.0174532925199433]],CS[ellipsoidal,2],AXIS["geodetic latitude (Lat)",north,ORDER[1],ANGLEUNIT["degree",0.0174532925199433]],AXIS["geodetic longitude (Lon)",east,ORDER[2],ANGLEUNIT["degree",0.0174532925199433]],ID["EPSG",4326]]', 'longitude/latitude coordinates in decimal degrees on the WGS 84 spheroid', 'GEOGCRS["WGS 84",DATUM["World Geodetic System 1984",ELLIPSOID["WGS 84",6378137,298.257223563,LENGTHUNIT["metre",1]]],PRIMEM["Greenwich",0,ANGLEUNIT["degree",0.0174532925199433]],CS[ellipsoidal,2],AXIS["geodetic latitude (Lat)",north,ORDER[1],ANGLEUNIT["degree",0.0174532925199433]],AXIS["geodetic longitude (Lon)",east,ORDER[2],ANGLEUNIT["degree",0.0174532925199433]],ID["EPSG",4326]]'); + """ + ) + ) end # then "commit" the transaction after executing table = values(geotable) @@ -94,10 +95,9 @@ function _extracttablevals(db, table, domain, crs, geom) srs = "EPSG" srid = 4326 else - srs = string(CoordRefSystems.code(crs)) + srs = string(CoordRefSystems.code(crs))[1:4] srid = parse(Int32, srs) end - gpkgbinary = map(geom) do ft gpkgbinheader = writegpkgheader(srid, ft) io = IOBuffer() @@ -137,11 +137,11 @@ function _extracttablevals(db, table, domain, crs, geom) srs_id=srid )] SQLite.load!(contents, db, "gpkg_contents", replace=true) - if srid != 4326 + if srid != 4326 && srid != -1 srstb = [( srs_name="", srs_id=srid, - organization=srs[1:4], + organization=srs, organization_coordsys_id=srid, definition=CoordRefSystems.wkt2(crs), description="", @@ -180,7 +180,7 @@ function _wkbsetz(type::wkbGeometryType) # ISO WKB Flavour end -function writegpkgheader(srs_id, geom) +function writegpkgheader(srsid, geom) io = IOBuffer() write(io, [0x47, 0x50]) # 'GP' in ASCII write(io, zero(UInt8)) # 0 = version 1 @@ -188,7 +188,7 @@ function writegpkgheader(srs_id, geom) flagsbyte = UInt8(0x07 >> 1) write(io, flagsbyte) - write(io, htol(Int32(srs_id))) + write(io, htol(Int32(srsid))) bbox = boundingbox(geom) write(io, htol(Float64(CoordRefSystems.raw(coords(bbox.min))[2]))) From f4016350226f0912ef679a0d87bc9b6d78a805a7 Mon Sep 17 00:00:00 2001 From: jph6366 Date: Wed, 24 Sep 2025 23:03:02 -0400 Subject: [PATCH 20/90] add Random, some capi sqlite, sql faster and safer in a transaction, coords already float --- Project.toml | 2 ++ src/GeoIO.jl | 3 +++ src/extra/gpkg/read.jl | 3 +++ src/extra/gpkg/write.jl | 55 +++++++++++++++++++++++++++++++---------- 4 files changed, 50 insertions(+), 13 deletions(-) diff --git a/Project.toml b/Project.toml index 38b534b..d1d7b36 100644 --- a/Project.toml +++ b/Project.toml @@ -28,6 +28,7 @@ NCDatasets = "85f8d34a-cbdd-5861-8df4-14fed0d494ab" PlyIO = "42171d58-473b-503a-8d5f-782019eb09ec" PrecompileTools = "aea7be01-6a6a-4083-8856-8a6e6704d82a" PrettyTables = "08abe8d2-0d0c-5749-adfa-8a2ac140af0d" +Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" ReadVTK = "dc215faf-f008-4882-a9f7-a79a826fadc3" SQLite = "0aa819cd-b072-5ff4-a722-6bc24af294d9" Shapefile = "8e980c4a-a4fe-5da2-b3a7-4b4b0353a2f4" @@ -63,6 +64,7 @@ NCDatasets = "0.13, 0.14" PlyIO = "1.1" PrecompileTools = "1.2" PrettyTables = "2.2" +Random = "1.11" ReadVTK = "0.2" SQLite = "1.6" Shapefile = "0.13" diff --git a/src/GeoIO.jl b/src/GeoIO.jl index dbe393c..1401d3c 100644 --- a/src/GeoIO.jl +++ b/src/GeoIO.jl @@ -45,6 +45,9 @@ import SQLite # Local Date and Time import Dates +# Support for generating random numbers +import Random + # geostats formats import GslibIO diff --git a/src/extra/gpkg/read.jl b/src/extra/gpkg/read.jl index cc2064f..6a57e3e 100644 --- a/src/extra/gpkg/read.jl +++ b/src/extra/gpkg/read.jl @@ -7,6 +7,9 @@ function gpkgread(fname; layer=1) assertgpkg(db) geom = gpkgmesh(db, ; layer) attrs = gpkgmeshattrs(db, ; layer) + DBInterface.execute(db, "PRAGMA optimize;") + # PRAGMA optimize command will normally only consider running ANALYZE + # on tables that have been previously queried by the same database connection DBInterface.close!(db) if eltype(attrs) <: Nothing return GeoTables.georef(nothing, geom) diff --git a/src/extra/gpkg/write.jl b/src/extra/gpkg/write.jl index 828a71a..0eee682 100644 --- a/src/extra/gpkg/write.jl +++ b/src/extra/gpkg/write.jl @@ -83,7 +83,9 @@ CREATE TABLE gpkg_geometry_columns ( geom = collect(domain) _extracttablevals(db, table, domain, crs, geom) - + DBInterface.execute(db, "PRAGMA optimize;") + # Applications with short-lived database connections should run "PRAGMA optimize;" + # just once, just in time prior to closing each database connection. DBInterface.close!(db) end @@ -111,15 +113,42 @@ function _extracttablevals(db, table, domain, crs, geom) table = isnothing(table) ? [(; geom=GeomType(g),) for (i, g) in zip(1:length(gpkgbinary), gpkgbinary)] : [(; t..., geom=GeomType(g)) for (t, g) in zip(Tables.rowtable(table), gpkgbinary)] - SQLite.load!(table, db, replace=false) # autogenerates table name - # replace=false controls whether an INSERT INTO ... statement is generated or a REPLACE INTO .... - tn = - ( - DBInterface.execute( - db, - """ SELECT name FROM sqlite_master WHERE type='table' AND name NOT IN ("gpkg_contents", "gpkg_spatial_ref_sys", "gpkg_geometry_columns") """ - ) |> first - ).name + rows = Tables.rows(table) + sch = Tables.schema(rows) + columns = [ + string(SQLite.esc_id(String(sch.names[i])), ' ', SQLite.sqlitetype(sch.types !== nothing ? sch.types[i] : Any)) + for i in eachindex(sch.names) + ] + tn = "sqlitejl_" * Random.randstring(5) + DBInterface.execute(db, "CREATE TABLE $tn ($(join(columns, ',')));") + params = chop(repeat("?,", length(sch.names))) + columns = join(SQLite.esc_id.(string.(sch.names)), ",") + stmt = SQLite.Stmt(db, "INSERT INTO $tn ($columns) VALUES ($params)";) + handle = SQLite._get_stmt_handle(stmt) + SQLite.transaction(db) do + row = nothing + if row === nothing + state = iterate(rows) + state === nothing && return + row, st = state + end + while true + Tables.eachcolumn(sch, row) do val, col, _ + SQLite.bind!(stmt, col, val) + end + r = GC.@preserve row SQLite.C.sqlite3_step(handle) + if r == SQLite.C.SQLITE_DONE + SQLite.C.sqlite3_reset(handle) + elseif r != SQLite.C.SQLITE_ROW + e = SQLite.sqliteexception(db, stmt) + SQLite.C.sqlite3_reset(handle) + throw(e) + end + state = iterate(rows, st) + state === nothing && break + row, st = state + end + end bbox = boundingbox(domain) mincoords = CoordRefSystems.raw(coords(bbox.min)) @@ -237,11 +266,11 @@ function _wkbgeom(io, wkbtype, geom) end function _wkbcoordinates(io, wkbtype, coords) - write(io, htol(Float64(coords[2]))) - write(io, htol(Float64(coords[1]))) + write(io, htol(coords[2])) + write(io, htol(coords[1])) if (UInt32(wkbtype) > 1000) || !(iszero(Int(wkbtype) & (0x80000000 | 0x40000000))) - write(io, htol(Float64(coords[3]))) + write(io, htol(coords[3])) end end From acc948c3a7ec17703fd87fc5e929d113ccee07f7 Mon Sep 17 00:00:00 2001 From: jph6366 Date: Sat, 27 Sep 2025 12:06:07 -0400 Subject: [PATCH 21/90] removed unnecessary Random and Dates, and reorganized create tables and table inserts, updated geopackage test conform to specification req 29 --- Project.toml | 4 - src/GeoIO.jl | 6 -- src/extra/gpkg/write.jl | 223 +++++++++++++++++++++------------------- test/io/geopackage.jl | 7 +- 4 files changed, 120 insertions(+), 120 deletions(-) diff --git a/Project.toml b/Project.toml index 16db61c..b28fbfe 100644 --- a/Project.toml +++ b/Project.toml @@ -10,7 +10,6 @@ Colors = "5ae59095-9a9b-59fe-a467-6f913c188581" CommonDataModel = "1fbeeb36-5f17-413c-809b-666fb144f157" CoordRefSystems = "b46f11dc-f210-4604-bfba-323c1ec968cb" DBInterface = "a10d1c49-ce27-4219-8d33-6db1a4562965" -Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" FileIO = "5789e2e9-d7fb-5bc7-8068-2c6fae9b9549" Format = "1fa38f19-a742-5d3f-a2b9-30dd87b9d5f8" GRIBDatasets = "82be9cdb-ee19-4151-bdb3-b400788d9abc" @@ -28,7 +27,6 @@ NCDatasets = "85f8d34a-cbdd-5861-8df4-14fed0d494ab" PlyIO = "42171d58-473b-503a-8d5f-782019eb09ec" PrecompileTools = "aea7be01-6a6a-4083-8856-8a6e6704d82a" PrettyTables = "08abe8d2-0d0c-5749-adfa-8a2ac140af0d" -Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" ReadVTK = "dc215faf-f008-4882-a9f7-a79a826fadc3" SQLite = "0aa819cd-b072-5ff4-a722-6bc24af294d9" Shapefile = "8e980c4a-a4fe-5da2-b3a7-4b4b0353a2f4" @@ -46,7 +44,6 @@ Colors = "0.12, 0.13" CommonDataModel = "0.2, 0.3, 0.4" CoordRefSystems = "0.18" DBInterface = "2.6" -Dates = "1.11" FileIO = "1.16" Format = "1.3" GRIBDatasets = "0.3, 0.4" @@ -64,7 +61,6 @@ NCDatasets = "0.13, 0.14" PlyIO = "1.1" PrecompileTools = "1.2" PrettyTables = "3.0" -Random = "1.11" ReadVTK = "0.2" SQLite = "1.6" Shapefile = "0.13" diff --git a/src/GeoIO.jl b/src/GeoIO.jl index 65ab40b..0436598 100644 --- a/src/GeoIO.jl +++ b/src/GeoIO.jl @@ -42,12 +42,6 @@ import CSV import DBInterface import SQLite -# Local Date and Time -import Dates - -# Support for generating random numbers -import Random - # geostats formats import GslibIO diff --git a/src/extra/gpkg/write.jl b/src/extra/gpkg/write.jl index 0eee682..6b8c17f 100644 --- a/src/extra/gpkg/write.jl +++ b/src/extra/gpkg/write.jl @@ -10,86 +10,21 @@ function gpkgwrite(fname, geotable;) # Setting PRAGMA synchronous=OFF but, # can cause the database to go corrupt # if there is an operating system crash or power failure. - # If the power never goes out and no programs ever crash - # on you system then Synchronous = OFF is for you - - SQLite.transaction(db) do - DBInterface.execute( - db, - """ -CREATE TABLE gpkg_spatial_ref_sys ( - srs_name TEXT NOT NULL, srs_id INTEGER NOT NULL PRIMARY KEY, - organization TEXT NOT NULL, organization_coordsys_id INTEGER NOT NULL, - definition TEXT NOT NULL, description TEXT, - definition_12_063 TEXT NOT NULL -); - -CREATE TABLE gpkg_contents ( - table_name TEXT NOT NULL PRIMARY KEY, - data_type TEXT NOT NULL, - identifier TEXT UNIQUE, - description TEXT DEFAULT '', - last_change DATETIME NOT NULL DEFAULT - (strftime('%Y-%m-%dT%H:%M:%fZ','now')), - min_x DOUBLE, min_y DOUBLE, - max_x DOUBLE, max_y DOUBLE, - srs_id INTEGER, - CONSTRAINT fk_gc_r_srs_id FOREIGN KEY (srs_id) REFERENCES - gpkg_spatial_ref_sys(srs_id) -); - -CREATE TABLE gpkg_geometry_columns ( - table_name TEXT NOT NULL, - column_name TEXT NOT NULL, - geometry_type_name TEXT NOT NULL, - srs_id INTEGER NOT NULL, - z TINYINT NOT NULL, - m TINYINT NOT NULL, - CONSTRAINT pk_geom_cols PRIMARY KEY (table_name, column_name), - CONSTRAINT uk_gc_table_name UNIQUE (table_name), - CONSTRAINT fk_gc_tn FOREIGN KEY (table_name) REFERENCES gpkg_contents(table_name), - CONSTRAINT fk_gc_srs FOREIGN KEY (srs_id) REFERENCES gpkg_spatial_ref_sys (srs_id) -); -""" - ) - DBInterface.execute(db, "PRAGMA application_id = $GPKG_APPLICATION_ID ") - DBInterface.execute(db, "PRAGMA user_version = $GPKG_1_4_VERSION ") - end - - # According to https://www.geopackage.org/spec/#r11 - # The gpkg_spatial_ref_sys table SHALL contain at a minimum - # 1. the record with an srs_id of 4326 SHALL correspond to WGS-84 as defined by EPSG in 4326 - # 2. the record with an srs_id of -1 SHALL be used for undefined Cartesian coordinate reference systems - # 3. the record with an srs_id of 0 SHALL be used for undefined geographic coordinate reference systems - SQLite.transaction(db) do # execute do x closure - DBInterface.execute( - SQLite.Stmt( - db, - """ - INSERT INTO gpkg_spatial_ref_sys - (srs_name, srs_id, organization, organization_coordsys_id, definition, description, definition_12_063) - VALUES - ('Undefined Cartesian SRS', -1, 'NONE', -1, 'undefined', 'undefined geographic coordinate reference system', 'undefined'), - ('Undefined geographic SRS', 0, 'NONE', 0, 'undefined', 'undefined geographic coordinate reference system', 'undefined'), - ('WGS 84 geodectic', 4326, 'EPSG', 4326, 'GEOGCRS["WGS 84",DATUM["World Geodetic System 1984",ELLIPSOID["WGS 84",6378137,298.257223563,LENGTHUNIT["metre",1]]],PRIMEM["Greenwich",0,ANGLEUNIT["degree",0.0174532925199433]],CS[ellipsoidal,2],AXIS["geodetic latitude (Lat)",north,ORDER[1],ANGLEUNIT["degree",0.0174532925199433]],AXIS["geodetic longitude (Lon)",east,ORDER[2],ANGLEUNIT["degree",0.0174532925199433]],ID["EPSG",4326]]', 'longitude/latitude coordinates in decimal degrees on the WGS 84 spheroid', 'GEOGCRS["WGS 84",DATUM["World Geodetic System 1984",ELLIPSOID["WGS 84",6378137,298.257223563,LENGTHUNIT["metre",1]]],PRIMEM["Greenwich",0,ANGLEUNIT["degree",0.0174532925199433]],CS[ellipsoidal,2],AXIS["geodetic latitude (Lat)",north,ORDER[1],ANGLEUNIT["degree",0.0174532925199433]],AXIS["geodetic longitude (Lon)",east,ORDER[2],ANGLEUNIT["degree",0.0174532925199433]],ID["EPSG",4326]]'); - """ - ) - ) - end # then "commit" the transaction after executing table = values(geotable) domain = GeoTables.domain(geotable) crs = GeoTables.crs(domain) geom = collect(domain) - - _extracttablevals(db, table, domain, crs, geom) + DBInterface.execute(db, "PRAGMA application_id = $GPKG_APPLICATION_ID ") + DBInterface.execute(db, "PRAGMA user_version = $GPKG_1_4_VERSION ") + creategpkgtables(db, table, domain, crs, geom) DBInterface.execute(db, "PRAGMA optimize;") # Applications with short-lived database connections should run "PRAGMA optimize;" - # just once, just in time prior to closing each database connection. + # just once, prior to closing each database connection. DBInterface.close!(db) end -function _extracttablevals(db, table, domain, crs, geom) +function creategpkgtables(db, table, domain, crs, geom) if crs <: Cartesian srs = "" srid = -1 @@ -111,7 +46,7 @@ function _extracttablevals(db, table, domain, crs, geom) GeomType = _sqlitetype(geomtype) table = - isnothing(table) ? [(; geom=GeomType(g),) for (i, g) in zip(1:length(gpkgbinary), gpkgbinary)] : + isnothing(table) ? [(; geom=GeomType(g),) for (_, g) in zip(1:length(gpkgbinary), gpkgbinary)] : [(; t..., geom=GeomType(g)) for (t, g) in zip(Tables.rowtable(table), gpkgbinary)] rows = Tables.rows(table) sch = Tables.schema(rows) @@ -119,11 +54,14 @@ function _extracttablevals(db, table, domain, crs, geom) string(SQLite.esc_id(String(sch.names[i])), ' ', SQLite.sqlitetype(sch.types !== nothing ? sch.types[i] : Any)) for i in eachindex(sch.names) ] - tn = "sqlitejl_" * Random.randstring(5) - DBInterface.execute(db, "CREATE TABLE $tn ($(join(columns, ',')));") + + # https://www.geopackage.org/spec/#r29 + # A feature table SHALL have a primary key column of type INTEGER and that column SHALL act as a rowid alias. + DBInterface.execute(db, "CREATE TABLE features ( id INTEGER PRIMARY KEY AUTOINCREMENT, $(join(columns, ',')));") + params = chop(repeat("?,", length(sch.names))) columns = join(SQLite.esc_id.(string.(sch.names)), ",") - stmt = SQLite.Stmt(db, "INSERT INTO $tn ($columns) VALUES ($params)";) + stmt = SQLite.Stmt(db, "INSERT INTO features ($columns) VALUES ($params)";) handle = SQLite._get_stmt_handle(stmt) SQLite.transaction(db) do row = nothing @@ -153,40 +91,111 @@ function _extracttablevals(db, table, domain, crs, geom) bbox = boundingbox(domain) mincoords = CoordRefSystems.raw(coords(bbox.min)) maxcoords = CoordRefSystems.raw(coords(bbox.max)) - contents = [( - table_name=tn, - data_type="features", - identifier=tn, - description="", - last_change=Dates.format(Dates.now(Dates.UTC), "yyyy-mm-ddTHH:MM:SSZ"), - min_x=mincoords[1], - min_y=mincoords[2], - max_x=maxcoords[1], - max_y=maxcoords[2], - srs_id=srid - )] - SQLite.load!(contents, db, "gpkg_contents", replace=true) - if srid != 4326 && srid != -1 - srstb = [( - srs_name="", - srs_id=srid, - organization=srs, - organization_coordsys_id=srid, - definition=CoordRefSystems.wkt2(crs), - description="", - definition_12_063=CoordRefSystems.wkt2(crs) - )] - SQLite.load!(srstb, db, "gpkg_spatial_ref_sys", replace=true) + minx, miny, maxx, maxy = mincoords[1], mincoords[2], maxcoords[1], maxcoords[2] + z = paramdim((geom |> first)) > 2 ? 1 : 0 + + SQLite.transaction(db) do + DBInterface.execute( + db, + """ + CREATE TABLE gpkg_spatial_ref_sys ( + srs_name TEXT NOT NULL, srs_id INTEGER NOT NULL PRIMARY KEY, + organization TEXT NOT NULL, organization_coordsys_id INTEGER NOT NULL, + definition TEXT NOT NULL, description TEXT, + definition_12_063 TEXT NOT NULL + ); + """ + ) + + # According to https://www.geopackage.org/spec/#r11 + # The gpkg_spatial_ref_sys table SHALL contain at a minimum + # 1. the record with an srs_id of 4326 SHALL correspond to WGS-84 as defined by EPSG in 4326 + # 2. the record with an srs_id of -1 SHALL be used for undefined Cartesian coordinate reference systems + # 3. the record with an srs_id of 0 SHALL be used for undefined geographic coordinate reference systems + DBInterface.execute( + db, + """ + INSERT INTO gpkg_spatial_ref_sys + (srs_name, srs_id, organization, organization_coordsys_id, definition, description, definition_12_063) + VALUES + ('Undefined Cartesian SRS', -1, 'NONE', -1, 'undefined', 'undefined geographic coordinate reference system', 'undefined'), + ('Undefined geographic SRS', 0, 'NONE', 0, 'undefined', 'undefined geographic coordinate reference system', 'undefined'), + ('WGS 84 geodectic', 4326, 'EPSG', 4326, 'GEOGCRS["WGS 84",DATUM["World Geodetic System 1984",ELLIPSOID["WGS 84",6378137,298.257223563,LENGTHUNIT["metre",1]]],PRIMEM["Greenwich",0,ANGLEUNIT["degree",0.0174532925199433]],CS[ellipsoidal,2],AXIS["geodetic latitude (Lat)",north,ORDER[1],ANGLEUNIT["degree",0.0174532925199433]],AXIS["geodetic longitude (Lon)",east,ORDER[2],ANGLEUNIT["degree",0.0174532925199433]],ID["EPSG",4326]]', 'longitude/latitude coordinates in decimal degrees on the WGS 84 spheroid', 'GEOGCRS["WGS 84",DATUM["World Geodetic System 1984",ELLIPSOID["WGS 84",6378137,298.257223563,LENGTHUNIT["metre",1]]],PRIMEM["Greenwich",0,ANGLEUNIT["degree",0.0174532925199433]],CS[ellipsoidal,2],AXIS["geodetic latitude (Lat)",north,ORDER[1],ANGLEUNIT["degree",0.0174532925199433]],AXIS["geodetic longitude (Lon)",east,ORDER[2],ANGLEUNIT["degree",0.0174532925199433]],ID["EPSG",4326]]'); + """ + ) + + if srid != 4326 && srid > 0 + DBInterface.execute( # Insert non-existing CRS record into gpkg_spatial_ref_sys table. srs_id referenced by gpkg_contents, gpkg_geometry_columns + db, + """ + INSERT INTO gpkg_spatial_ref_sys + (srs_name, srs_id, organization, organization_coordsys_id, definition, description, definition_12_063) + VALUES + (?, ?, ?, ?, ?, ?, ?); + """, + ["", srid, srs, srid, CoordRefSystems.wkt2(crs), "", CoordRefSystems.wkt2(crs)] + ) + end + + DBInterface.execute( + db, + """ + CREATE TABLE gpkg_contents ( + table_name TEXT NOT NULL PRIMARY KEY, + data_type TEXT NOT NULL, + identifier TEXT UNIQUE NOT NULL, + description TEXT DEFAULT '', + last_change DATETIME NOT NULL DEFAULT + (strftime('%Y-%m-%dT%H:%M:%fZ','now')), + min_x DOUBLE, min_y DOUBLE, + max_x DOUBLE, max_y DOUBLE, + srs_id INTEGER, + CONSTRAINT fk_gc_r_srs_id FOREIGN KEY (srs_id) REFERENCES + gpkg_spatial_ref_sys(srs_id) + ); + """ + ) + + DBInterface.execute( + db, + """ + INSERT INTO gpkg_contents + (table_name, data_type, identifier, min_x, min_y, max_x, max_y, srs_id) + VALUES + (?, ?, ?, ?, ?, ?, ?, ?); + """, + ["features", "features", "features", minx, miny, maxx, maxy, srid] + ) + + DBInterface.execute( + db, + """ + CREATE TABLE gpkg_geometry_columns ( + table_name TEXT NOT NULL, + column_name TEXT NOT NULL, + geometry_type_name TEXT NOT NULL, + srs_id INTEGER NOT NULL, + z TINYINT NOT NULL, + m TINYINT NOT NULL, + CONSTRAINT pk_geom_cols PRIMARY KEY (table_name, column_name), + CONSTRAINT uk_gc_table_name UNIQUE (table_name), + CONSTRAINT fk_gc_tn FOREIGN KEY (table_name) REFERENCES gpkg_contents(table_name), + CONSTRAINT fk_gc_srs FOREIGN KEY (srs_id) REFERENCES gpkg_spatial_ref_sys (srs_id) + ); + """ + ) + + DBInterface.execute( + db, + """ + INSERT INTO gpkg_geometry_columns + (table_name, column_name, geometry_type_name, srs_id, z, m) + VALUES + (?, ?, ?, ?, ?, ?); + """, + ["features", "geom", geomtype, srid, z, 0] + ) end - geomcolumns = [( - table_name=tn, - column_name="geom", - geometry_type_name=geomtype, - srs_id=srid, - z=paramdim((geom |> first)) > 2 ? 1 : 0, - m=0 - )] - SQLite.load!(geomcolumns, db, "gpkg_geometry_columns", replace=true) end function _wkbtype(geometry) diff --git a/test/io/geopackage.jl b/test/io/geopackage.jl index 5e82834..c6ff8d0 100644 --- a/test/io/geopackage.jl +++ b/test/io/geopackage.jl @@ -32,12 +32,13 @@ @testset "save" begin # note: GeoPackage does not preserve column order + # note2: Every such feature table SHALL have a primary key column 'id' of type INTEGER file1 = joinpath(datadir, "points.gpkg") file2 = joinpath(savedir, "points.gpkg") gtb1 = GeoIO.load(file1) GeoIO.save(file2, gtb1) gtb2 = GeoIO.load(file2) - @test Set(names(gtb2)) == Set(names(gtb1)) + @test Set([n for n in names(gtb2) if n != "id"]) == Set(names(gtb1)) @test gtb2.geometry == gtb1.geometry @test gtb2.code == gtb1.code @test gtb2.name == gtb1.name @@ -48,7 +49,7 @@ gtb1 = GeoIO.load(file1) GeoIO.save(file2, gtb1) gtb2 = GeoIO.load(file2) - @test Set(names(gtb2)) == Set(names(gtb1)) + @test Set([n for n in names(gtb2) if n != "id"]) == Set(names(gtb1)) @test gtb2.geometry == gtb1.geometry @test gtb2.code == gtb1.code @test gtb2.name == gtb1.name @@ -59,7 +60,7 @@ gtb1 = GeoIO.load(file1) GeoIO.save(file2, gtb1) gtb2 = GeoIO.load(file2) - @test Set(names(gtb2)) == Set(names(gtb1)) + @test Set([n for n in names(gtb2) if n != "id"]) == Set(names(gtb1)) @test gtb2.geometry == gtb1.geometry @test gtb2.code == gtb1.code @test gtb2.name == gtb1.name From dc722cc7eb51957465f00ed877f42c1f3bbaffc4 Mon Sep 17 00:00:00 2001 From: jph6366 Date: Sat, 27 Sep 2025 12:16:34 -0400 Subject: [PATCH 22/90] removed id ok autoincrement, undo my evil test changes --- src/extra/gpkg/write.jl | 4 +++- test/io/geopackage.jl | 7 +++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/extra/gpkg/write.jl b/src/extra/gpkg/write.jl index 6b8c17f..68e635c 100644 --- a/src/extra/gpkg/write.jl +++ b/src/extra/gpkg/write.jl @@ -57,7 +57,9 @@ function creategpkgtables(db, table, domain, crs, geom) # https://www.geopackage.org/spec/#r29 # A feature table SHALL have a primary key column of type INTEGER and that column SHALL act as a rowid alias. - DBInterface.execute(db, "CREATE TABLE features ( id INTEGER PRIMARY KEY AUTOINCREMENT, $(join(columns, ',')));") + DBInterface.execute(db, "CREATE TABLE features ($(join(columns, ',')));") + # The use of the AUTOINCREMENT keyword is optional but recommended. + # Implementers MAY omit the AUTOINCREMENT keyword for performance reasons, with the understanding that doing so has the potential to allow primary key identifiers to be reused. params = chop(repeat("?,", length(sch.names))) columns = join(SQLite.esc_id.(string.(sch.names)), ",") diff --git a/test/io/geopackage.jl b/test/io/geopackage.jl index c6ff8d0..5e82834 100644 --- a/test/io/geopackage.jl +++ b/test/io/geopackage.jl @@ -32,13 +32,12 @@ @testset "save" begin # note: GeoPackage does not preserve column order - # note2: Every such feature table SHALL have a primary key column 'id' of type INTEGER file1 = joinpath(datadir, "points.gpkg") file2 = joinpath(savedir, "points.gpkg") gtb1 = GeoIO.load(file1) GeoIO.save(file2, gtb1) gtb2 = GeoIO.load(file2) - @test Set([n for n in names(gtb2) if n != "id"]) == Set(names(gtb1)) + @test Set(names(gtb2)) == Set(names(gtb1)) @test gtb2.geometry == gtb1.geometry @test gtb2.code == gtb1.code @test gtb2.name == gtb1.name @@ -49,7 +48,7 @@ gtb1 = GeoIO.load(file1) GeoIO.save(file2, gtb1) gtb2 = GeoIO.load(file2) - @test Set([n for n in names(gtb2) if n != "id"]) == Set(names(gtb1)) + @test Set(names(gtb2)) == Set(names(gtb1)) @test gtb2.geometry == gtb1.geometry @test gtb2.code == gtb1.code @test gtb2.name == gtb1.name @@ -60,7 +59,7 @@ gtb1 = GeoIO.load(file1) GeoIO.save(file2, gtb1) gtb2 = GeoIO.load(file2) - @test Set([n for n in names(gtb2) if n != "id"]) == Set(names(gtb1)) + @test Set(names(gtb2)) == Set(names(gtb1)) @test gtb2.geometry == gtb1.geometry @test gtb2.code == gtb1.code @test gtb2.name == gtb1.name From f7e4e85cdd60a8e0a9ba61cabfc1c86622c5f8f2 Mon Sep 17 00:00:00 2001 From: jph6366 Date: Thu, 2 Oct 2025 10:40:59 -0400 Subject: [PATCH 23/90] tests passing! simple features supported! need to test for multi features... and other cases --- src/extra/gpkg.jl | 75 +--------- src/extra/gpkg/read.jl | 136 +++-------------- src/extra/gpkg/write.jl | 108 +------------- src/extra/wkb.jl | 321 ++++++++++++++++++++++++++++------------ 4 files changed, 256 insertions(+), 384 deletions(-) diff --git a/src/extra/gpkg.jl b/src/extra/gpkg.jl index 12b7ab1..457570c 100644 --- a/src/extra/gpkg.jl +++ b/src/extra/gpkg.jl @@ -8,80 +8,9 @@ const GPKG_APPLICATION_ID = Int(0x47504B47) const GPKG_1_4_VERSION = 10400 -function _geomtype(geoms::AbstractVector{<:Geometry}) - if isempty(geoms) - return "GEOMETRY" - end - T = eltype(geoms) - - if T <: Point - "POINT" - elseif T <: Rope - "LINESTRING" - elseif T <: Ring - "LINESTRING" - elseif T <: PolyArea - "POLYGON" - elseif T <: Multi - element_type = eltype(parent(first(geoms))) - if element_type <: Point - "MULTIPOINT" - elseif element_type <: Rope - "MULTILINESTRING" - elseif element_type <: PolyArea - "MULTIPOLYGON" - end - else - "GEOMETRY" - end -end - -include("wkb.jl") - -function _sqlitetype(geomtype::AbstractString) - if geomtype == "POINT" - WKBPoint - elseif geomtype == "LINESTRING" - WKBLineString - elseif geomtype == "POLYGON" - WKBPolygon - elseif geomtype == "MULTIPOINT" - WKBMultiPoint - elseif geomtype == "MULTILINESTRING" - WKBMultiLineString - elseif geomtype == "MULTIPOLYGON" - WKBMultiPolygon - else - WKBGeometry - end -end - -function SQLite.sqlitetype_(::Type{WKBPoint}) - return "POINT" -end - -function SQLite.sqlitetype_(::Type{WKBLineString}) - return "LINESTRING" -end - -function SQLite.sqlitetype_(::Type{WKBPolygon}) - return "POLYGON" -end - -function SQLite.sqlitetype_(::Type{WKBMultiPoint}) - return "MULTIPOINT" -end - -function SQLite.sqlitetype_(::Type{WKBMultiLineString}) - return "MULTILINESTRING" -end - -function SQLite.sqlitetype_(::Type{WKBMultiPolygon}) - return "MULTIPOLYGON" -end - -function SQLite.sqlitetype_(::Type{WKBGeometry}) +function SQLite.sqlitetype_(::Type{Vector{UInt8}}) return "GEOMETRY" end +include("wkb.jl") include("gpkg/read.jl") include("gpkg/write.jl") diff --git a/src/extra/gpkg/read.jl b/src/extra/gpkg/read.jl index 6a57e3e..84b9b87 100644 --- a/src/extra/gpkg/read.jl +++ b/src/extra/gpkg/read.jl @@ -135,19 +135,18 @@ AND g.m IN (0, 1, 2) !ismissing(getfield(row, Symbol(cn))) # ignore all rows with missing geometries end submeshes = map(gpkgblobs) do blob - gpkgbindata = isa(blob[1], WKBGeometry) ? blob[1].data : blob[1] - io = IOBuffer(gpkgbindata) + if blob[1][1:2] != UInt8[0x47, 0x50] + @error "MISSING MAGIC 'GP' STRING" + end + io = IOBuffer(blob[1]) seek(io, 3) - flag = read(io, UInt8) + flag = read(io,UInt8) # Note that Julia does not convert the endianness for you. # Use ntoh or ltoh for this purpose. bswap = isone(flag & 0x01) ? ltoh : ntoh - srsid = bswap(read(io, Int32)) - envelope = (flag & (0x07 << 1)) >> 1 envelopedims = 0 - if !iszero(envelope) if isone(envelope) envelopedims = 1 # 2D @@ -158,27 +157,30 @@ AND g.m IN (0, 1, 2) elseif isequal(4, envelope) envelopedims = 4 # 2D+ZM else - @error "exceeded dimensional limit for geometry, file may be corrupted or reader is broken" - false + throw(ErrorException("exceeded dimensional limit for geometry, file may be corrupted or reader is broken")) end - else - true # no envelope (space saving slower indexing option), 0 bytes - end + end # else no envelope (space saving slower indexing option), 0 bytes # header size in byte stream headerlen = 8 + 8 * 4 * envelopedims seek(io, headerlen) + wkbbyteswap = isone(read(io, UInt8)) ? ltoh : ntoh + wkbtype = meshwkb(WKB{read(io, UInt32)}) + ngeoms = (wkbtype <: WKBPoint) ? one(UInt32) : read(io, UInt32) + + if iszero(srsid) + crs = LatLon{WGS84Latest} + elseif !isone(abs(srsid)) + if org == "EPSG" + crs = CoordRefSystems.get(EPSG{org_coordsys_id}) + elseif org == "ESRI" + crs = CoordRefSystems.get(ERSI{org_coordsys_id}) + end + else + crs = Cartesian{NoDatum} + end - ebyteorder = read(io, UInt8) - - bswap = isone(ebyteorder) ? ltoh : ntoh - - wkbtype = wkbGeometryType(read(io, UInt32)) - - zextent = isequal(envelopedims, 2) - - mesh = meshfromwkb(io, srsid, org, org_coordsys_id, wkbtype, zextent, bswap) - + mesh = wkbtomeshes(wkbtype, ngeoms, io, crs, wkbbyteswap) if !isnothing(mesh) mesh end @@ -188,95 +190,3 @@ AND g.m IN (0, 1, 2) # efficient method for concatenating arrays of arrays reduce(vcat, meshes) # Future versions of Julia might change the reduce algorithm end - -function meshfromwkb(io, srsid, org, org_coordsys_id, ewkbtype, zextent, bswap) - if iszero(srsid) - crs = LatLon{WGS84Latest} - elseif !isone(abs(srsid)) - if org == "EPSG" - crs = CoordRefSystems.get(EPSG{org_coordsys_id}) - elseif org == "ESRI" - crs = CoordRefSystems.get(ERSI{org_coordsys_id}) - end - else - crs = Cartesian{NoDatum} - end - - if occursin("Multi", string(ewkbtype)) - elems = wkbmultigeometry(io, crs, zextent, bswap) - Multi(elems) - else - elem = meshfromsf(io, crs, ewkbtype, zextent, bswap) - elem - end -end - -# Requirement 20: GeoPackage SHALL store feature table geometries -# with the basic simple feature geometry types -# https://www.geopackage.org/spec140/index.html#geometry_types -function meshfromsf(io, crs, ewkbtype, zextent, bswap) - if isequal(ewkbtype, wkbPoint) - elem = wkbcoordinate(io, zextent, bswap) - Point(crs(elem...)) - elseif isequal(ewkbtype, wkbLineString) - elem = wkblinestring(io, zextent, bswap) - if length(elem) >= 2 && first(elem) != last(elem) - Rope([Point(crs(coords...)) for coords in elem]...) - else - Ring([Point(crs(coords...)) for coords in elem[1:(end - 1)]]...) - end - elseif isequal(ewkbtype, wkbPolygon) - elem = wkbpolygon(io, zextent, bswap) - rings = map(elem) do ring - coords = map(ring) do point - Point(crs(point...)) - end - Ring(coords) - end - - outerring = first(rings) - holes = isone(length(rings)) ? rings[2:end] : Ring[] - PolyArea(outerring, holes...) - end -end - -function wkbcoordinate(io, z, bswap) - x = bswap(read(io, Float64)) - y = bswap(read(io, Float64)) - - if z - z = bswap(read(io, Float64)) - return x, y, z - end - - x, y -end - -function wkblinestring(io, z, bswap) - npoints = bswap(read(io, UInt32)) - - points = map(1:npoints) do _ - wkbcoordinate(io, z, bswap) - end - points -end - -function wkbpolygon(io, z, bswap) - nrings = bswap(read(io, UInt32)) - - rings = map(1:nrings) do _ - wkblinestring(io, z, bswap) - end - rings -end - -function wkbmultigeometry(io, crs, z, bswap) - ngeoms = bswap(read(io, UInt32)) - - geomcollection = map(1:ngeoms) do _ - bswap = isone(read(io, UInt8)) ? ltoh : ntoh - ewkbtype = wkbGeometryType(read(io, UInt32)) - meshfromsf(io, crs, ewkbtype, z, bswap) - end - geomcollection -end diff --git a/src/extra/gpkg/write.jl b/src/extra/gpkg/write.jl index 68e635c..6dedda4 100644 --- a/src/extra/gpkg/write.jl +++ b/src/extra/gpkg/write.jl @@ -38,16 +38,15 @@ function creategpkgtables(db, table, domain, crs, geom) gpkgbinary = map(geom) do ft gpkgbinheader = writegpkgheader(srid, ft) io = IOBuffer() - writewkbgeom(io, ft) + meshestowkb(ft, io) vcat(gpkgbinheader, take!(io)) end - geomtype = _geomtype(geom) - GeomType = _sqlitetype(geomtype) + table = - isnothing(table) ? [(; geom=GeomType(g),) for (_, g) in zip(1:length(gpkgbinary), gpkgbinary)] : - [(; t..., geom=GeomType(g)) for (t, g) in zip(Tables.rowtable(table), gpkgbinary)] + isnothing(table) ? [(; geom=g,) for (_, g) in zip(1:length(gpkgbinary), gpkgbinary)] : + [(; t..., geom=g) for (t, g) in zip(Tables.rowtable(table), gpkgbinary)] rows = Tables.rows(table) sch = Tables.schema(rows) columns = [ @@ -117,7 +116,7 @@ function creategpkgtables(db, table, domain, crs, geom) DBInterface.execute( db, """ - INSERT INTO gpkg_spatial_ref_sys + INSERT INTO gpkg_spatial_ref_sys (srs_name, srs_id, organization, organization_coordsys_id, definition, description, definition_12_063) VALUES ('Undefined Cartesian SRS', -1, 'NONE', -1, 'undefined', 'undefined geographic coordinate reference system', 'undefined'), @@ -195,31 +194,11 @@ function creategpkgtables(db, table, domain, crs, geom) VALUES (?, ?, ?, ?, ?, ?); """, - ["features", "geom", geomtype, srid, z, 0] + ["features", "geom", "GEOMETRY", srid, z, 0] ) end end -function _wkbtype(geometry) - if geometry isa Point - return wkbPoint - elseif geometry isa Rope || geometry isa Ring - return wkbLineString - elseif geometry isa PolyArea - return wkbPolygon - elseif geometry isa Multi - fg = parent(geometry) |> first - return wkbGeometryType(Int(_wkbtype(fg)) + 3) - else - @error "my hovercraft is full of eels: $geometry" - end -end - -function _wkbsetz(type::wkbGeometryType) - return wkbGeometryType(Int(type) + 1000) - # ISO WKB Flavour -end - function writegpkgheader(srsid, geom) io = IOBuffer() write(io, [0x47, 0x50]) # 'GP' in ASCII @@ -243,78 +222,3 @@ function writegpkgheader(srsid, geom) return take!(io) end - -function writewkbgeom(io, geom) - wkbtype = paramdim(geom) < 3 ? _wkbtype(geom) : _wkbsetz(_wkbtype(geom)) - write(io, htol(one(UInt8))) - write(io, htol(UInt32(wkbtype))) - _wkbgeom(io, wkbtype, geom) -end - -function writewkbsf(io, wkbtype, geom) - if wkbtype == wkbPolygon || wkbtype == wkbPolygonZ - _wkbpolygon(io, wkbtype, [boundary(geom::PolyArea)]) - elseif wkbtype == wkbLineString || wkbtype == wkbLineString - coordlist = vertices(geom) - if typeof(geom) <: Ring - return _wkblinearring(io, wkbtype, coordlist) - end - _wkblinestring(io, wkbtype, coordlist) - elseif wkbtype == wkbPoint || wkbtype == wkbPointZ - coordinates = CoordRefSystems.raw(coords(geom)) - _wkbcoordinates(io, wkbtype, coordinates) - else - throw(ErrorException("Well-Known Binary Geometry not supported: $wkbtype")) - end -end - -function _wkbgeom(io, wkbtype, geom) - if Int(wkbtype) > 3 - _wkbmulti(io, wkbtype, geom) - else - writewkbsf(io, wkbtype, geom) - end -end - -function _wkbcoordinates(io, wkbtype, coords) - write(io, htol(coords[2])) - write(io, htol(coords[1])) - - if (UInt32(wkbtype) > 1000) || !(iszero(Int(wkbtype) & (0x80000000 | 0x40000000))) - write(io, htol(coords[3])) - end -end - -function _wkblinestring(io, wkb_type, coord_list) - write(io, htol(UInt32(length(coord_list)))) - for n_coords in coord_list - coordinates = CoordRefSystems.raw(coords(n_coords)) - _wkbcoordinates(io, wkb_type, coordinates) - end -end - -function _wkblinearring(io, wkb_type, coord_list) - write(io, htol(UInt32(length(coord_list) + 1))) - for n_coords in coord_list - coordinates = CoordRefSystems.raw(coords(n_coords)) - _wkbcoordinates(io, wkb_type, coordinates) - end - _wkbcoordinates(io, wkb_type, CoordRefSystems.raw(first(coord_list) |> coords)) -end - -function _wkbpolygon(io, wkb_type, rings) - write(io, htol(UInt32(length(rings)))) - for ring in rings - coord_list = vertices(ring) - _wkblinestring(io, wkb_type, coord_list) - end -end - -function _wkbmulti(io, wkbtype, geoms) - write(io, htol(UInt32(length(geoms |> parent)))) - for sf in geoms |> parent - write(io, one(UInt8)) - write(io, UInt32(Int(wkbMultiPolygon) - 3)) - writewkbsf(io, wkbGeometryType(Int(wkbtype) - 3), sf) - end -end \ No newline at end of file diff --git a/src/extra/wkb.jl b/src/extra/wkb.jl index 148d871..b36ce47 100644 --- a/src/extra/wkb.jl +++ b/src/extra/wkb.jl @@ -1,120 +1,249 @@ +abstract type WKBCode end +abstract type WKBFlav{Code} <: WKBCode end +abstract type WKB{Code} <: WKBFlav{Code} end -@enum wkbGeometryType::UInt32 begin - wkbUnknown = 0 - wkbPoint = 1 - wkbLineString = 2 - wkbPolygon = 3 - wkbMultiPoint = 4 - wkbMultiLineString = 5 - wkbMultiPolygon = 6 - wkbGeometryCollection = 7 - wkbCircularString = 8 - wkbCompoundCurve = 9 - wkbCurvePolygon = 10 - wkbMultiCurve = 11 - wkbMultiSurface = 12 - wkbCurve = 13 - wkbSurface = 14 - - wkbNone = 100 # pure attribute records - wkbLinearRing = 101 +# "Well-Known" Binary scheme for simple feature geometry. +# The base Geometry class has subclasses for Point, Line, Polygon, and GeometryCollection +# and Support for 3D coordinates +# https://libgeos.org/specifications/wkb/ +# https://www.ogc.org/standards/sfa/ +abstract type WKBGeometry end +abstract type WKBPoint <: WKBGeometry end +abstract type WKBLineString <: WKBGeometry end +abstract type WKBPolygon <: WKBGeometry end +abstract type WKBMulti <: WKBGeometry end +abstract type WKBMultiPoint <: WKBMulti end +abstract type WKBMultiLineString <: WKBMulti end +abstract type WKBMultiPolygon <: WKBMulti end +abstract type WKBGeometryCollection <: WKBGeometry end +# ISO SQL/MM Part 3: Spatial +# Z-aware types, also representative of the 2.5D extension as per 99-402 +# https://lists.osgeo.org/pipermail/postgis-devel/2004-December/000702.html +abstract type WKBGeometryZ <: WKBGeometry end +abstract type WKBPointZ <: WKBGeometryZ end +abstract type WKBLineStringZ <: WKBGeometryZ end +abstract type WKBPolygonZ <: WKBGeometryZ end +abstract type WKBMultiZ <: WKBMulti end +abstract type WKBMultiPointZ <: WKBMultiZ end +abstract type WKBMultiLineStringZ <: WKBMultiZ end +abstract type WKBMultiPolygonZ <: WKBMultiZ end +abstract type WKBGeometryCollectionZ <: WKBGeometryCollection end +# ISO SQL/MM Part 3. +# M-aware types +abstract type WKBPointM <: WKBPointZ end +abstract type WKBLineStringM <: WKBLineStringZ end +abstract type WKBPolygonM <: WKBPolygonZ end +abstract type WKBMultiM <: WKBMultiZ end +abstract type WKBMultiPointM <: WKBMultiM end +abstract type WKBMultiLineStringM <: WKBMultiM end +abstract type WKBMultiPolygonM <: WKBMultiM end +abstract type WKBGeometryCollectionM <: WKBGeometryCollectionZ end +# ISO SQL/MM Part 3. +# ZM-aware types +abstract type WKBPointZM <: WKBPointZ end +abstract type WKBLineStringZM <: WKBLineStringZ end +abstract type WKBPolygonZM <: WKBPolygonZ end +abstract type WKBMultiZM <: WKBMultiZ end +abstract type WKBMultiPointZM <: WKBMultiZM end +abstract type WKBMultiLineStringZM <: WKBMultiZM end +abstract type WKBMultiPolygonZM <: WKBMultiZM end +abstract type WKBGeometryCollectionZM <: WKBGeometryCollectionZ end - # ISO SQL/MM Part 3: Spatial - # Z-aware types - wkbPointZ = 1001 - wkbLineStringZ = 1002 - wkbPolygonZ = 1003 - wkbMultiPointZ = 1004 - wkbMultiLineStringZ = 1005 - wkbMultiPolygonZ = 1006 - wkbGeometryCollectionZ = 1007 - wkbCircularStringZ = 1008 - wkbCompoundCurveZ = 1009 - wkbCurvePolygonZ = 1010 - wkbMultiCurveZ = 1011 - wkbMultiSurfaceZ = 1012 - wkbCurveZ = 1013 - wkbSurfaceZ = 1014 +function meshfromwkb(geom::AbstractVector{UInt8}) + wkbbyteswap = isone(geom[1]) ? ltoh : ntoh + wkbtype = meshwkb(WKB{reinterpret(UInt32, @view geom[2:5]) |> only}) + ngeoms = (wkbtype <: WKBPoint) ? 1 : only(reinterpret(UInt32, @view geom[6:9])) + wkbvec = Vector{UInt8}(@view geom[10:end]) + wkbbyteswap, wkbtype, ngeoms, wkbvec +end - # ISO SQL/MM Part 3. - # M-aware types - wkbPointM = 2001 - wkbLineStringM = 2002 - wkbPolygonM = 2003 - wkbMultiPointM = 2004 - wkbMultiLineStringM = 2005 - wkbMultiPolygonM = 2006 - wkbGeometryCollectionM = 2007 - wkbCircularStringM = 2008 - wkbCompoundCurveM = 2009 - wkbCurvePolygonM = 2010 - wkbMultiCurveM = 2011 - wkbMultiSurfaceM = 2012 - wkbCurveM = 2013 - wkbSurfaceM = 2014 +function wkbtomeshes(::Type{T}, n, io, crs, wkbbswap) where {T<:WKBPoint} + if T <: WKBPointZM + p = wkbbswap(read(io, Float64)), wkbbswap(read(io, Float64)), wkbbswap(read(io, Float64)), wkbbswap(read(io, Float64)) + elseif T <: WKBPointM + p = wkbbswap(read(io, Float64)), wkbbswap(read(io, Float64)) + wkbbswap(read(io, Float64)) # skip aspatial axis, M (Optional Measurement) + elseif T <: WKBPointZ + p = wkbbswap(read(io, Float64)), wkbbswap(read(io, Float64)), wkbbswap(read(io, Float64)) + else + p = wkbbswap(read(io, Float64)), wkbbswap(read(io, Float64)) + end + Meshes.Point(crs(p...)) +end - # ISO SQL/MM Part 3. - # ZM-aware types - wkbPointZM = 3001 - wkbLineStringZM = 3002 - wkbPolygonZM = 3003 - wkbMultiPointZM = 3004 - wkbMultiLineStringZM = 3005 - wkbMultiPolygonZM = 3006 - wkbGeometryCollectionZM = 3007 - wkbCircularStringZM = 3008 - wkbCompoundCurveZM = 3009 - wkbCurvePolygonZM = 3010 - wkbMultiCurveZM = 3011 - wkbMultiSurfaceZM = 3012 - wkbCurveZM = 3013 - wkbSurfaceZM = 3014 +function wkbtomeshes(::Type{T}, n, io, crs, wkbbswap) where {T<:WKBLineString} + if T <: WKBLineStringZM + wkbpoint = WKBPointZM + elseif T <: WKBLineStringM + wkbpoint = WKBPointM + elseif T <: WKBLineStringZ + wkbpoint = WKBPointZ + else + wkbpoint = WKBPoint + end + points = map(1:n) do _ + wkbtomeshes(wkbpoint, 1, io, crs, wkbbswap) + end + if first(points) != points[n] && length(points) > 2 + return Meshes.Rope(points) + end + Meshes.Ring(points[1:(end - 1)]) +end - # 2.5D extension as per 99-402 - # https://lists.osgeo.org/pipermail/postgis-devel/2004-December/000702.html - wkbPoint25D = 0x80000001 - wkbLineString25D = 0x80000002 - wkbPolygon25D = 0x80000003 - wkbMultiPoint25D = 0x80000004 - wkbMultiLineString25D = 0x80000005 - wkbMultiPolygon25D = 0x80000006 - wkbGeometryCollection25D = 0x80000007 +function wkbtomeshes(::Type{WKBPolygon}, n, io, crs, wkbbswap) + rings = map(1:n) do _ + k = wkbbswap(read(io, UInt32)) + wkbtomeshes(WKBLineString, k, io, crs, wkbbswap) + end + outtering = first(rings) + holes = isone(length(rings)) ? rings[2:end] : Meshes.Ring[] + Meshes.PolyArea(outtering, holes...) end -abstract type WKBGeometry end +function wkbtomeshes(::Type{T}, n, io, geom, wkbbswap) where {T<:WKBMulti} + geoms = map(1:n) do _ + bytereader = wkbbswap(read(io, UInt8)) + wkbtype = meshwkb(WKB{wkbbswap(read(io, UInt32))}) + k = wkbbswap(read(io, UInt32)) + wkbtomeshes(wkbtype, k, io, crs, wkbbswap) + end + Meshes.Multi(geoms) +end -struct WKBPoint <: WKBGeometry - data::Vector{UInt8} +function meshestowkb(::Type{T}, io, geom) where {T<:WKBPoint} + coordvec = CoordRefSystems.raw(coords(geom)) + write(io, htol(coordvec[2])) + write(io, htol(coordvec[1])) + if T <: WKBPointZ + write(io, htol(coordvec[3])) + end end -struct WKBLineString <: WKBGeometry - data::Vector{UInt8} +function meshestowkb(::Type{T}, io, geom) where {T<:WKBLineString} + points = vertices(geom) + if typeof(geom) <: Meshes.Ring + write(io, htol(UInt32(length(points) + 1))) + else + write(io, UInt32(length(points))) + end + if T <: WKBLineStringZ + wkbpoint = WKBPointZ + else + wkbpoint = WKBPoint + end + for c in points + meshestowkb(wkbpoint, io, c) + end + if typeof(geom) <: Meshes.Ring + meshestowkb(wkbpoint, io, first(points)) + end end -struct WKBPolygon <: WKBGeometry - data::Vector{UInt8} +function meshestowkb(::Type{T}, io, geom) where {T<:WKBPolygon} + rings = [boundary(geom::PolyArea)] + write(io, htol(UInt32(length(rings)))) + if T <: WKBPolygonZ + wkbchain = WKBLineStringZ + else + wkbchain = WKBLineString + end + for ring in rings + meshestowkb(wkbchain, io, ring) + end end -struct WKBMultiPoint <: WKBGeometry - data::Vector{UInt8} +function meshestowkb(::Type{T}, io, geom) where {T<:WKBMulti} + write(io, htol(UInt32(length(geom |> parent)))) + for g in geom |> parent + write(io, one(UInt8)) + wkbn = parse(UInt32, ((wkbmesh(T) |> string)[5:(end - 1)]) |> htol) + write(io, wkbn) + meshestowkb(meshwkb(WKB{wkbn - 3}), io, g) + end end -struct WKBMultiLineString <: WKBGeometry - data::Vector{UInt8} +function meshestowkb(geom::T, io) where {T<:Meshes.Geometry} + write(io, htol(one(UInt8))) + meshdims = (paramdim(T) >= 3) + if T <: Meshes.PolyArea + wkbtype = meshdims ? WKBPolygonZ : WKBPolygon + write(io, parse(UInt32, ((wkbmesh(wkbtype) |> string)[11:end-1]) |> htol)) + meshestowkb(wkbtype, io, geom) + elseif T <: Meshes.Rope + wkbtype = meshdims ? WKBLineStringZ : WKBLineString + write(io, parse(UInt32, ((wkbmesh(wkbtype) |> string)[11:end-1])|> htol)) + meshestowkb(wkbtype, io, geom) + elseif T <: Meshes.Ring + wkbtype = meshdims ? WKBLineStringZ : WKBLineString + write(io, parse(UInt32, ((wkbmesh(wkbtype) |> string)[11:end-1])|> htol)) + meshestowkb(wkbtype, io, geom) + elseif T <: Meshes.Point + wkbtype = meshdims ? WKBPointZ : WKBPoint + write(io, parse(UInt32, ((wkbmesh(wkbtype) |> string)[11:end-1])|> htol)) + meshestowkb(wkbtype, io, geom) + elseif T <: Meshes.Multi + wkbtype = meshdims ? WKBMultiZ : WKBMulti + write(io, parse(UInt32, ((wkbmesh(wkbtype) |> string)[11:end-1])|> htol)) + meshestowkb(wkbtype, io, geom) + else + throw(ArgumentError(""" + The provided mesh $T is not supported by available WKB Geometry types. + """)) + end end -struct WKBMultiPolygon <: WKBGeometry - data::Vector{UInt8} +function meshwkb(code::Type{WKBCode}) + throw(ArgumentError(""" + The provided code $code is not mapped to a WKB Geometry type yet. + """)) end -struct WKBGeometryCollection <: WKBGeometry - data::Vector{UInt8} +function wkbmesh(wkbgeom::Type{WKBGeometry}) + throw(ArgumentError(""" + The provided type $wkbgeom is not mapped to a WKB implementation. + """)) end -function Base.getproperty(gpkg::WKBGeometry, sym::Symbol) - if sym == :data - return getfield(gpkg, :data) - else - return getfield(gpkg, sym) +macro wkbcode(Code, WKBArray) + expr = quote + meshwkb(::Type{$Code}) = $WKBArray + wkbmesh(::Type{$WKBArray}) = $Code end + esc(expr) end + +@wkbcode WKB{UInt32(1)} WKBPoint +@wkbcode WKB{UInt32(2)} WKBLineString +@wkbcode WKB{UInt32(3)} WKBPolygon +@wkbcode WKB{UInt32(4)} WKBMultiPoint +@wkbcode WKB{UInt32(5)} WKBMultiLineString +@wkbcode WKB{UInt32(6)} WKBMultiPolygon +@wkbcode WKB{UInt32(7)} WKBGeometryCollection +# @wkbcode WKB{0x80000001} WKBPointZ +# @wkbcode WKB{0x80000002} WKBLineStringZ +# @wkbcode WKB{0x80000003} WKBPolygonZ +# @wkbcode WKB{0x80000004} WKBMultiPointZ +# @wkbcode WKB{0x80000005} WKBMultiLineStringZ +# @wkbcode WKB{0x80000006} WKBMultiPolygonZ +# @wkbcode WKB{0x80000007} WKBGeometryCollectionZ +@wkbcode WKB{UInt32(1001)} WKBPointZ +@wkbcode WKB{UInt32(1002)} WKBLineStringZ +@wkbcode WKB{UInt32(1003)} WKBPolygonZ +@wkbcode WKB{UInt32(1004)} WKBMultiPointZ +@wkbcode WKB{UInt32(1005)} WKBMultiLineStringZ +@wkbcode WKB{UInt32(1006)} WKBMultiPolygonZ +@wkbcode WKB{UInt32(1007)} WKBGeometryCollectionZ +@wkbcode WKB{UInt32(2001)} WKBPointM +@wkbcode WKB{UInt32(2002)} WKBLineStringM +@wkbcode WKB{UInt32(2003)} WKBPolygonM +@wkbcode WKB{UInt32(2004)} WKBMultiPointM +@wkbcode WKB{UInt32(2005)} WKBMultiLineStringM +@wkbcode WKB{UInt32(2006)} WKBMultiPolygonM +@wkbcode WKB{UInt32(2007)} WKBGeometryCollectionM +@wkbcode WKB{UInt32(3001)} WKBPointZM +@wkbcode WKB{UInt32(3002)} WKBLineStringZM +@wkbcode WKB{UInt32(3003)} WKBPolygonZM +@wkbcode WKB{UInt32(3004)} WKBMultiPointZM +@wkbcode WKB{UInt32(3005)} WKBMultiLineStringZM +@wkbcode WKB{UInt32(3006)} WKBMultiPolygonZM +@wkbcode WKB{UInt32(3007)} WKBGeometryCollectionZM \ No newline at end of file From b2451b118f35d41f4ff1bd7fecc90138a212af65 Mon Sep 17 00:00:00 2001 From: jph6366 Date: Thu, 2 Oct 2025 14:34:35 -0400 Subject: [PATCH 24/90] multi. fix, added util to infer Multi const geometry, --- src/extra/wkb.jl | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/src/extra/wkb.jl b/src/extra/wkb.jl index b36ce47..3e53e48 100644 --- a/src/extra/wkb.jl +++ b/src/extra/wkb.jl @@ -49,14 +49,6 @@ abstract type WKBMultiLineStringZM <: WKBMultiZM end abstract type WKBMultiPolygonZM <: WKBMultiZM end abstract type WKBGeometryCollectionZM <: WKBGeometryCollectionZ end -function meshfromwkb(geom::AbstractVector{UInt8}) - wkbbyteswap = isone(geom[1]) ? ltoh : ntoh - wkbtype = meshwkb(WKB{reinterpret(UInt32, @view geom[2:5]) |> only}) - ngeoms = (wkbtype <: WKBPoint) ? 1 : only(reinterpret(UInt32, @view geom[6:9])) - wkbvec = Vector{UInt8}(@view geom[10:end]) - wkbbyteswap, wkbtype, ngeoms, wkbvec -end - function wkbtomeshes(::Type{T}, n, io, crs, wkbbswap) where {T<:WKBPoint} if T <: WKBPointZM p = wkbbswap(read(io, Float64)), wkbbswap(read(io, Float64)), wkbbswap(read(io, Float64)), wkbbswap(read(io, Float64)) @@ -100,9 +92,9 @@ function wkbtomeshes(::Type{WKBPolygon}, n, io, crs, wkbbswap) Meshes.PolyArea(outtering, holes...) end -function wkbtomeshes(::Type{T}, n, io, geom, wkbbswap) where {T<:WKBMulti} +function wkbtomeshes(::Type{T}, n, io, crs, wkbbswap) where {T<:WKBMulti} geoms = map(1:n) do _ - bytereader = wkbbswap(read(io, UInt8)) + wkbbswap = isone(read(io, UInt8)) ? ltoh : ntoh wkbtype = meshwkb(WKB{wkbbswap(read(io, UInt32))}) k = wkbbswap(read(io, UInt32)) wkbtomeshes(wkbtype, k, io, crs, wkbbswap) @@ -158,13 +150,13 @@ function meshestowkb(::Type{T}, io, geom) where {T<:WKBMulti} write(io, one(UInt8)) wkbn = parse(UInt32, ((wkbmesh(T) |> string)[5:(end - 1)]) |> htol) write(io, wkbn) - meshestowkb(meshwkb(WKB{wkbn - 3}), io, g) + meshestowkb(meshwkb(WKB{UInt32(wkbn - 3)}), io, g) end end function meshestowkb(geom::T, io) where {T<:Meshes.Geometry} write(io, htol(one(UInt8))) - meshdims = (paramdim(T) >= 3) + meshdims = (paramdim(geom) >= 3) if T <: Meshes.PolyArea wkbtype = meshdims ? WKBPolygonZ : WKBPolygon write(io, parse(UInt32, ((wkbmesh(wkbtype) |> string)[11:end-1]) |> htol)) @@ -182,7 +174,7 @@ function meshestowkb(geom::T, io) where {T<:Meshes.Geometry} write(io, parse(UInt32, ((wkbmesh(wkbtype) |> string)[11:end-1])|> htol)) meshestowkb(wkbtype, io, geom) elseif T <: Meshes.Multi - wkbtype = meshdims ? WKBMultiZ : WKBMulti + wkbtype = multiwkbmesh(typeof(parent(geom)[1]), meshdims) write(io, parse(UInt32, ((wkbmesh(wkbtype) |> string)[11:end-1])|> htol)) meshestowkb(wkbtype, io, geom) else @@ -192,6 +184,19 @@ function meshestowkb(geom::T, io) where {T<:Meshes.Geometry} end end +function multiwkbmesh(multi, zextent) + if multi <: PolyArea + wkbtype = zextent ? WKBMultiPolygonZ : WKBMultiPolygon + elseif multi <: Ring + wkbtype = zextent ? WKBMultiLineStringZ : WKBMultiLineString + elseif multi <: Rope + wkbtype = zextent ? WKBMultiLineStringZ : WKBMultiLineString + elseif multi <: Point + wkbtype = zextent ? WKBMultiPointZ : WKBMultiPoint + end + return wkbtype +end + function meshwkb(code::Type{WKBCode}) throw(ArgumentError(""" The provided code $code is not mapped to a WKB Geometry type yet. From a884e883578ab809c37493ed6edc69164df93864 Mon Sep 17 00:00:00 2001 From: jph6366 Date: Thu, 2 Oct 2025 14:56:52 -0400 Subject: [PATCH 25/90] formatting --- src/extra/gpkg/read.jl | 2 +- src/extra/gpkg/write.jl | 2 -- src/extra/wkb.jl | 41 +++++++++++++++++++++-------------------- 3 files changed, 22 insertions(+), 23 deletions(-) diff --git a/src/extra/gpkg/read.jl b/src/extra/gpkg/read.jl index 84b9b87..9f2aae5 100644 --- a/src/extra/gpkg/read.jl +++ b/src/extra/gpkg/read.jl @@ -140,7 +140,7 @@ AND g.m IN (0, 1, 2) end io = IOBuffer(blob[1]) seek(io, 3) - flag = read(io,UInt8) + flag = read(io, UInt8) # Note that Julia does not convert the endianness for you. # Use ntoh or ltoh for this purpose. bswap = isone(flag & 0x01) ? ltoh : ntoh diff --git a/src/extra/gpkg/write.jl b/src/extra/gpkg/write.jl index 6dedda4..f4ac93a 100644 --- a/src/extra/gpkg/write.jl +++ b/src/extra/gpkg/write.jl @@ -42,8 +42,6 @@ function creategpkgtables(db, table, domain, crs, geom) vcat(gpkgbinheader, take!(io)) end - - table = isnothing(table) ? [(; geom=g,) for (_, g) in zip(1:length(gpkgbinary), gpkgbinary)] : [(; t..., geom=g) for (t, g) in zip(Tables.rowtable(table), gpkgbinary)] diff --git a/src/extra/wkb.jl b/src/extra/wkb.jl index 3e53e48..1eacd0f 100644 --- a/src/extra/wkb.jl +++ b/src/extra/wkb.jl @@ -51,14 +51,15 @@ abstract type WKBGeometryCollectionZM <: WKBGeometryCollectionZ end function wkbtomeshes(::Type{T}, n, io, crs, wkbbswap) where {T<:WKBPoint} if T <: WKBPointZM - p = wkbbswap(read(io, Float64)), wkbbswap(read(io, Float64)), wkbbswap(read(io, Float64)), wkbbswap(read(io, Float64)) + p = + wkbbswap(read(io, Float64)), wkbbswap(read(io, Float64)), wkbbswap(read(io, Float64)), wkbbswap(read(io, Float64)) elseif T <: WKBPointM p = wkbbswap(read(io, Float64)), wkbbswap(read(io, Float64)) - wkbbswap(read(io, Float64)) # skip aspatial axis, M (Optional Measurement) + wkbbswap(read(io, Float64)) # skip aspatial axis, M (Optional Measurement) elseif T <: WKBPointZ - p = wkbbswap(read(io, Float64)), wkbbswap(read(io, Float64)), wkbbswap(read(io, Float64)) + p = wkbbswap(read(io, Float64)), wkbbswap(read(io, Float64)), wkbbswap(read(io, Float64)) else - p = wkbbswap(read(io, Float64)), wkbbswap(read(io, Float64)) + p = wkbbswap(read(io, Float64)), wkbbswap(read(io, Float64)) end Meshes.Point(crs(p...)) end @@ -140,7 +141,7 @@ function meshestowkb(::Type{T}, io, geom) where {T<:WKBPolygon} wkbchain = WKBLineString end for ring in rings - meshestowkb(wkbchain, io, ring) + meshestowkb(wkbchain, io, ring) end end @@ -159,23 +160,23 @@ function meshestowkb(geom::T, io) where {T<:Meshes.Geometry} meshdims = (paramdim(geom) >= 3) if T <: Meshes.PolyArea wkbtype = meshdims ? WKBPolygonZ : WKBPolygon - write(io, parse(UInt32, ((wkbmesh(wkbtype) |> string)[11:end-1]) |> htol)) + write(io, parse(UInt32, ((wkbmesh(wkbtype) |> string)[11:(end - 1)]) |> htol)) meshestowkb(wkbtype, io, geom) elseif T <: Meshes.Rope wkbtype = meshdims ? WKBLineStringZ : WKBLineString - write(io, parse(UInt32, ((wkbmesh(wkbtype) |> string)[11:end-1])|> htol)) + write(io, parse(UInt32, ((wkbmesh(wkbtype) |> string)[11:(end - 1)]) |> htol)) meshestowkb(wkbtype, io, geom) elseif T <: Meshes.Ring wkbtype = meshdims ? WKBLineStringZ : WKBLineString - write(io, parse(UInt32, ((wkbmesh(wkbtype) |> string)[11:end-1])|> htol)) + write(io, parse(UInt32, ((wkbmesh(wkbtype) |> string)[11:(end - 1)]) |> htol)) meshestowkb(wkbtype, io, geom) elseif T <: Meshes.Point wkbtype = meshdims ? WKBPointZ : WKBPoint - write(io, parse(UInt32, ((wkbmesh(wkbtype) |> string)[11:end-1])|> htol)) + write(io, parse(UInt32, ((wkbmesh(wkbtype) |> string)[11:(end - 1)]) |> htol)) meshestowkb(wkbtype, io, geom) elseif T <: Meshes.Multi wkbtype = multiwkbmesh(typeof(parent(geom)[1]), meshdims) - write(io, parse(UInt32, ((wkbmesh(wkbtype) |> string)[11:end-1])|> htol)) + write(io, parse(UInt32, ((wkbmesh(wkbtype) |> string)[11:(end - 1)]) |> htol)) meshestowkb(wkbtype, io, geom) else throw(ArgumentError(""" @@ -185,16 +186,16 @@ function meshestowkb(geom::T, io) where {T<:Meshes.Geometry} end function multiwkbmesh(multi, zextent) - if multi <: PolyArea - wkbtype = zextent ? WKBMultiPolygonZ : WKBMultiPolygon - elseif multi <: Ring - wkbtype = zextent ? WKBMultiLineStringZ : WKBMultiLineString - elseif multi <: Rope - wkbtype = zextent ? WKBMultiLineStringZ : WKBMultiLineString - elseif multi <: Point - wkbtype = zextent ? WKBMultiPointZ : WKBMultiPoint - end - return wkbtype + if multi <: PolyArea + wkbtype = zextent ? WKBMultiPolygonZ : WKBMultiPolygon + elseif multi <: Ring + wkbtype = zextent ? WKBMultiLineStringZ : WKBMultiLineString + elseif multi <: Rope + wkbtype = zextent ? WKBMultiLineStringZ : WKBMultiLineString + elseif multi <: Point + wkbtype = zextent ? WKBMultiPointZ : WKBMultiPoint + end + return wkbtype end function meshwkb(code::Type{WKBCode}) From 6d2244cfbfc2e24ea41722eaa6fa17a085646ee6 Mon Sep 17 00:00:00 2001 From: jph6366 Date: Thu, 2 Oct 2025 15:18:39 -0400 Subject: [PATCH 26/90] mit license header --- src/extra/wkb.jl | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/extra/wkb.jl b/src/extra/wkb.jl index 1eacd0f..6a184d2 100644 --- a/src/extra/wkb.jl +++ b/src/extra/wkb.jl @@ -1,3 +1,7 @@ +# ------------------------------------------------------------------ +# Licensed under the MIT License. See LICENSE in the project root. +# ------------------------------------------------------------------ + abstract type WKBCode end abstract type WKBFlav{Code} <: WKBCode end abstract type WKB{Code} <: WKBFlav{Code} end From 7cd4851f54ab6d1f8c503d9d1ff1e17d7410e6a9 Mon Sep 17 00:00:00 2001 From: jph6366 Date: Tue, 7 Oct 2025 15:44:11 -0400 Subject: [PATCH 27/90] resolving reviewed changes --- src/extra/gpkg/read.jl | 40 ++++++++++++++++++---------------------- src/extra/gpkg/write.jl | 2 +- src/extra/wkb.jl | 18 +++++++++--------- 3 files changed, 28 insertions(+), 32 deletions(-) diff --git a/src/extra/gpkg/read.jl b/src/extra/gpkg/read.jl index 9f2aae5..08234b9 100644 --- a/src/extra/gpkg/read.jl +++ b/src/extra/gpkg/read.jl @@ -5,46 +5,42 @@ function gpkgread(fname; layer=1) db = SQLite.DB(fname) assertgpkg(db) - geom = gpkgmesh(db, ; layer) - attrs = gpkgmeshattrs(db, ; layer) + geom = gpkggeoms(db, ; layer) + attrs = gpkgvalues(db, ; layer) DBInterface.execute(db, "PRAGMA optimize;") # PRAGMA optimize command will normally only consider running ANALYZE # on tables that have been previously queried by the same database connection DBInterface.close!(db) if eltype(attrs) <: Nothing - return GeoTables.georef(nothing, geom) + return georef(nothing, geom) end - GeoTables.georef(attrs, geom) + georef(attrs, geom) end function assertgpkg(db) - if !hasgpkgmetadata(db) + tbcount = first(DBInterface.execute( + db, + """ + SELECT COUNT(*) AS n FROM sqlite_master WHERE + name IN ('gpkg_spatial_ref_sys', 'gpkg_contents') AND + type IN ('table', 'view'); +""" + )) + # Requirement 10: must include a gpkg_spatial_ref_sys table + # Requirement 13: must include a gpkg_contents table + if tbcount.n != 2 throw(ErrorException("missing required metadata tables in the GeoPackage SQL database")) end # Requirement 6: PRAGMA integrity_check returns a single row with the value 'ok' # Requirement 7: PRAGMA foreign_key_check (w/ no parameter value) returns an empty result set - if ((DBInterface.execute(db, "PRAGMA integrity_check;") |> first).integrity_check != "ok") || + if first(DBInterface.execute(db, "PRAGMA integrity_check;")).integrity_check != "ok" || !(isempty(DBInterface.execute(db, "PRAGMA foreign_key_check;"))) throw(ErrorException("database integrity at risk or foreign key violation(s)")) end end -# Requirement 10: must include a gpkg_spatial_ref_sys table -# Requirement 13: must include a gpkg_contents table -function hasgpkgmetadata(db) - tbcount = DBInterface.execute( - db, - """ - SELECT COUNT(*) FROM sqlite_master WHERE - name IN ('gpkg_spatial_ref_sys', 'gpkg_contents') AND - type IN ('table', 'view'); -""" - ) |> first |> only - tbcount == 2 -end - -function gpkgmeshattrs(db, ; layer=1) +function gpkgvalues(db, ; layer=1) feature_tables = DBInterface.execute( db, """ @@ -110,7 +106,7 @@ end # Requirement 146: The srs_id value in a gpkg_geometry_columns table row # SHALL match the srs_id column value from the corresponding row in the # gpkg_contents table. -function gpkgmesh(db, ; layer=1) +function gpkggeoms(db, ; layer=1) tb = DBInterface.execute( db, """ diff --git a/src/extra/gpkg/write.jl b/src/extra/gpkg/write.jl index f4ac93a..35ebac8 100644 --- a/src/extra/gpkg/write.jl +++ b/src/extra/gpkg/write.jl @@ -91,7 +91,7 @@ function creategpkgtables(db, table, domain, crs, geom) mincoords = CoordRefSystems.raw(coords(bbox.min)) maxcoords = CoordRefSystems.raw(coords(bbox.max)) minx, miny, maxx, maxy = mincoords[1], mincoords[2], maxcoords[1], maxcoords[2] - z = paramdim((geom |> first)) > 2 ? 1 : 0 + z = paramdim(first(geom)) > 2 ? 1 : 0 SQLite.transaction(db) do DBInterface.execute( diff --git a/src/extra/wkb.jl b/src/extra/wkb.jl index 6a184d2..2268997 100644 --- a/src/extra/wkb.jl +++ b/src/extra/wkb.jl @@ -150,10 +150,10 @@ function meshestowkb(::Type{T}, io, geom) where {T<:WKBPolygon} end function meshestowkb(::Type{T}, io, geom) where {T<:WKBMulti} - write(io, htol(UInt32(length(geom |> parent)))) - for g in geom |> parent + write(io, htol(UInt32(length(parent(geom))))) + for g in parent(geom) write(io, one(UInt8)) - wkbn = parse(UInt32, ((wkbmesh(T) |> string)[5:(end - 1)]) |> htol) + wkbn = parse(UInt32, (string(wkbmesh(T))[5:(end - 1)]) |> htol) write(io, wkbn) meshestowkb(meshwkb(WKB{UInt32(wkbn - 3)}), io, g) end @@ -164,23 +164,23 @@ function meshestowkb(geom::T, io) where {T<:Meshes.Geometry} meshdims = (paramdim(geom) >= 3) if T <: Meshes.PolyArea wkbtype = meshdims ? WKBPolygonZ : WKBPolygon - write(io, parse(UInt32, ((wkbmesh(wkbtype) |> string)[11:(end - 1)]) |> htol)) + write(io, parse(UInt32, (string(wkbmesh(wkbtype))[11:(end - 1)]) |> htol)) meshestowkb(wkbtype, io, geom) elseif T <: Meshes.Rope wkbtype = meshdims ? WKBLineStringZ : WKBLineString - write(io, parse(UInt32, ((wkbmesh(wkbtype) |> string)[11:(end - 1)]) |> htol)) + write(io, parse(UInt32, (string(wkbmesh(wkbtype))[11:(end - 1)]) |> htol)) meshestowkb(wkbtype, io, geom) elseif T <: Meshes.Ring wkbtype = meshdims ? WKBLineStringZ : WKBLineString - write(io, parse(UInt32, ((wkbmesh(wkbtype) |> string)[11:(end - 1)]) |> htol)) + write(io, parse(UInt32, (string(wkbmesh(wkbtype))[11:(end - 1)]) |> htol)) meshestowkb(wkbtype, io, geom) elseif T <: Meshes.Point wkbtype = meshdims ? WKBPointZ : WKBPoint - write(io, parse(UInt32, ((wkbmesh(wkbtype) |> string)[11:(end - 1)]) |> htol)) + write(io, parse(UInt32, (string(wkbmesh(wkbtype))[11:(end - 1)]) |> htol)) meshestowkb(wkbtype, io, geom) elseif T <: Meshes.Multi - wkbtype = multiwkbmesh(typeof(parent(geom)[1]), meshdims) - write(io, parse(UInt32, ((wkbmesh(wkbtype) |> string)[11:(end - 1)]) |> htol)) + wkbtype = multiwkbmesh(typeof(first(parent(geom))), meshdims) + write(io, parse(UInt32, (string(wkbmesh(wkbtype))[11:(end - 1)]) |> htol)) meshestowkb(wkbtype, io, geom) else throw(ArgumentError(""" From f98d6b6889e15ae5dfb7e7df6e02a7723dfd65e1 Mon Sep 17 00:00:00 2001 From: jph6366 Date: Wed, 8 Oct 2025 10:46:54 -0400 Subject: [PATCH 28/90] optimized meshestowkb{T<:WKBMulti} , removing |> calls, other fixes --- src/extra/gpkg/read.jl | 8 ++++---- src/extra/wkb.jl | 27 ++++++++++++++++++--------- 2 files changed, 22 insertions(+), 13 deletions(-) diff --git a/src/extra/gpkg/read.jl b/src/extra/gpkg/read.jl index 08234b9..4e06b57 100644 --- a/src/extra/gpkg/read.jl +++ b/src/extra/gpkg/read.jl @@ -24,7 +24,7 @@ function assertgpkg(db) SELECT COUNT(*) AS n FROM sqlite_master WHERE name IN ('gpkg_spatial_ref_sys', 'gpkg_contents') AND type IN ('table', 'view'); -""" + """ )) # Requirement 10: must include a gpkg_spatial_ref_sys table # Requirement 13: must include a gpkg_contents table @@ -53,7 +53,7 @@ lower(c.table_name) AND type IN ('table', 'view')) AS object_type JOIN gpkg_contents c ON (g.table_name = c.table_name) WHERE c.data_type = 'features' LIMIT $layer -""" + """ ) fields = nothing tb = map(feature_tables) do query @@ -66,7 +66,7 @@ lower(c.table_name) AND type IN ('table', 'view')) AS object_type if isnothing(fields) || length(rp_attrs) < length(fields) fields = rp_attrs end - rowvals = length(fields) |> iszero ? nothing : map(DBInterface.execute(db, "SELECT $fields from $tn")) do rv + rowvals = iszero(length(fields)) ? nothing : map(DBInterface.execute(db, "SELECT $fields from $tn")) do rv NamedTuple(rv) end rowvals @@ -121,7 +121,7 @@ AND g.srs_id = c.srs_id AND g.z IN (0, 1, 2) AND g.m IN (0, 1, 2) LIMIT $layer; - """ + """ ) meshes = map((row.tn, row.cn, row.org, row.org_coordsys_id) for row in tb) do (tn, cn, org, org_coordsys_id) gpkgbinary = DBInterface.execute(db, "SELECT $cn FROM $tn;") diff --git a/src/extra/wkb.jl b/src/extra/wkb.jl index 2268997..b6874da 100644 --- a/src/extra/wkb.jl +++ b/src/extra/wkb.jl @@ -87,10 +87,19 @@ function wkbtomeshes(::Type{T}, n, io, crs, wkbbswap) where {T<:WKBLineString} Meshes.Ring(points[1:(end - 1)]) end -function wkbtomeshes(::Type{WKBPolygon}, n, io, crs, wkbbswap) +function wkbtomeshes(::Type{T}, n, io, crs, wkbbswap) where {T<:WKBPolygon} + if T <: WKBPolygonZM + wkbchain = WKBLineStringZM + elseif T <: WKBPolygonM + wkbchain = WKBLineStringM + elseif T <: WKBPolygonZ + wkbchain = WKBLineStringZ + else + wkbchain = WKBLineString + end rings = map(1:n) do _ k = wkbbswap(read(io, UInt32)) - wkbtomeshes(WKBLineString, k, io, crs, wkbbswap) + wkbtomeshes(wkbchain, k, io, crs, wkbbswap) end outtering = first(rings) holes = isone(length(rings)) ? rings[2:end] : Meshes.Ring[] @@ -153,9 +162,9 @@ function meshestowkb(::Type{T}, io, geom) where {T<:WKBMulti} write(io, htol(UInt32(length(parent(geom))))) for g in parent(geom) write(io, one(UInt8)) - wkbn = parse(UInt32, (string(wkbmesh(T))[5:(end - 1)]) |> htol) + wkbn = parse(UInt32, htol(string(wkbmesh(T))[5:(end - 1)])) write(io, wkbn) - meshestowkb(meshwkb(WKB{UInt32(wkbn - 3)}), io, g) + meshestowkb(meshwkb(WKB{wkbn - UInt32(3)}), io, g) end end @@ -164,23 +173,23 @@ function meshestowkb(geom::T, io) where {T<:Meshes.Geometry} meshdims = (paramdim(geom) >= 3) if T <: Meshes.PolyArea wkbtype = meshdims ? WKBPolygonZ : WKBPolygon - write(io, parse(UInt32, (string(wkbmesh(wkbtype))[11:(end - 1)]) |> htol)) + write(io, parse(UInt32, htol(string(wkbmesh(wkbtype))[11:(end - 1)]))) meshestowkb(wkbtype, io, geom) elseif T <: Meshes.Rope wkbtype = meshdims ? WKBLineStringZ : WKBLineString - write(io, parse(UInt32, (string(wkbmesh(wkbtype))[11:(end - 1)]) |> htol)) + write(io, parse(UInt32, htol(string(wkbmesh(wkbtype))[11:(end - 1)]))) meshestowkb(wkbtype, io, geom) elseif T <: Meshes.Ring wkbtype = meshdims ? WKBLineStringZ : WKBLineString - write(io, parse(UInt32, (string(wkbmesh(wkbtype))[11:(end - 1)]) |> htol)) + write(io, parse(UInt32, htol(string(wkbmesh(wkbtype))[11:(end - 1)]))) meshestowkb(wkbtype, io, geom) elseif T <: Meshes.Point wkbtype = meshdims ? WKBPointZ : WKBPoint - write(io, parse(UInt32, (string(wkbmesh(wkbtype))[11:(end - 1)]) |> htol)) + write(io, parse(UInt32, htol(string(wkbmesh(wkbtype))[11:(end - 1)]))) meshestowkb(wkbtype, io, geom) elseif T <: Meshes.Multi wkbtype = multiwkbmesh(typeof(first(parent(geom))), meshdims) - write(io, parse(UInt32, (string(wkbmesh(wkbtype))[11:(end - 1)]) |> htol)) + write(io, parse(UInt32, htol(string(wkbmesh(wkbtype))[11:(end - 1)]))) meshestowkb(wkbtype, io, geom) else throw(ArgumentError(""" From ddd08d5ffdf1df1fc9d0b7982664b6d89b095a17 Mon Sep 17 00:00:00 2001 From: jph6366 Date: Thu, 9 Oct 2025 16:16:18 -0400 Subject: [PATCH 29/90] simplified wkb reader/writer, added docstrings --- src/extra/gpkg.jl | 76 +----------- src/extra/gpkg/read.jl | 171 +++++++-------------------- src/extra/gpkg/wkb.jl | 252 ++++++++++++++++++++++++++++++++++++++++ src/extra/gpkg/write.jl | 107 +---------------- src/extra/wkb.jl | 120 ------------------- 5 files changed, 300 insertions(+), 426 deletions(-) create mode 100644 src/extra/gpkg/wkb.jl delete mode 100644 src/extra/wkb.jl diff --git a/src/extra/gpkg.jl b/src/extra/gpkg.jl index 12b7ab1..940a113 100644 --- a/src/extra/gpkg.jl +++ b/src/extra/gpkg.jl @@ -8,80 +8,10 @@ const GPKG_APPLICATION_ID = Int(0x47504B47) const GPKG_1_4_VERSION = 10400 -function _geomtype(geoms::AbstractVector{<:Geometry}) - if isempty(geoms) - return "GEOMETRY" - end - T = eltype(geoms) - - if T <: Point - "POINT" - elseif T <: Rope - "LINESTRING" - elseif T <: Ring - "LINESTRING" - elseif T <: PolyArea - "POLYGON" - elseif T <: Multi - element_type = eltype(parent(first(geoms))) - if element_type <: Point - "MULTIPOINT" - elseif element_type <: Rope - "MULTILINESTRING" - elseif element_type <: PolyArea - "MULTIPOLYGON" - end - else - "GEOMETRY" - end -end - -include("wkb.jl") - -function _sqlitetype(geomtype::AbstractString) - if geomtype == "POINT" - WKBPoint - elseif geomtype == "LINESTRING" - WKBLineString - elseif geomtype == "POLYGON" - WKBPolygon - elseif geomtype == "MULTIPOINT" - WKBMultiPoint - elseif geomtype == "MULTILINESTRING" - WKBMultiLineString - elseif geomtype == "MULTIPOLYGON" - WKBMultiPolygon - else - WKBGeometry - end -end - -function SQLite.sqlitetype_(::Type{WKBPoint}) - return "POINT" -end - -function SQLite.sqlitetype_(::Type{WKBLineString}) - return "LINESTRING" -end - -function SQLite.sqlitetype_(::Type{WKBPolygon}) - return "POLYGON" -end - -function SQLite.sqlitetype_(::Type{WKBMultiPoint}) - return "MULTIPOINT" -end - -function SQLite.sqlitetype_(::Type{WKBMultiLineString}) - return "MULTILINESTRING" -end - -function SQLite.sqlitetype_(::Type{WKBMultiPolygon}) - return "MULTIPOLYGON" -end - -function SQLite.sqlitetype_(::Type{WKBGeometry}) +function SQLite.sqlitetype_(::Type{Vector{UInt8}}) return "GEOMETRY" end + +include("gpkg/wkb.jl") include("gpkg/read.jl") include("gpkg/write.jl") diff --git a/src/extra/gpkg/read.jl b/src/extra/gpkg/read.jl index 6a57e3e..e7e4d75 100644 --- a/src/extra/gpkg/read.jl +++ b/src/extra/gpkg/read.jl @@ -7,9 +7,6 @@ function gpkgread(fname; layer=1) assertgpkg(db) geom = gpkgmesh(db, ; layer) attrs = gpkgmeshattrs(db, ; layer) - DBInterface.execute(db, "PRAGMA optimize;") - # PRAGMA optimize command will normally only consider running ANALYZE - # on tables that have been previously queried by the same database connection DBInterface.close!(db) if eltype(attrs) <: Nothing return GeoTables.georef(nothing, geom) @@ -18,7 +15,17 @@ function gpkgread(fname; layer=1) end function assertgpkg(db) - if !hasgpkgmetadata(db) + tbcount = first(DBInterface.execute( + db, + """ + SELECT COUNT(*) AS n FROM sqlite_master WHERE + name IN ('gpkg_spatial_ref_sys', 'gpkg_contents') AND + type IN ('table', 'view'); + """ + )) + # Requirement 10: must include a gpkg_spatial_ref_sys table + # Requirement 13: must include a gpkg_contents table + if tbcount.n != 2 throw(ErrorException("missing required metadata tables in the GeoPackage SQL database")) end @@ -30,20 +37,6 @@ function assertgpkg(db) end end -# Requirement 10: must include a gpkg_spatial_ref_sys table -# Requirement 13: must include a gpkg_contents table -function hasgpkgmetadata(db) - tbcount = DBInterface.execute( - db, - """ - SELECT COUNT(*) FROM sqlite_master WHERE - name IN ('gpkg_spatial_ref_sys', 'gpkg_contents') AND - type IN ('table', 'view'); -""" - ) |> first |> only - tbcount == 2 -end - function gpkgmeshattrs(db, ; layer=1) feature_tables = DBInterface.execute( db, @@ -57,7 +50,7 @@ lower(c.table_name) AND type IN ('table', 'view')) AS object_type JOIN gpkg_contents c ON (g.table_name = c.table_name) WHERE c.data_type = 'features' LIMIT $layer -""" + """ ) fields = nothing tb = map(feature_tables) do query @@ -70,7 +63,7 @@ lower(c.table_name) AND type IN ('table', 'view')) AS object_type if isnothing(fields) || length(rp_attrs) < length(fields) fields = rp_attrs end - rowvals = length(fields) |> iszero ? nothing : map(DBInterface.execute(db, "SELECT $fields from $tn")) do rv + rowvals = iszero(length(fields)) ? nothing : map(DBInterface.execute(db, "SELECT $fields from $tn")) do rv NamedTuple(rv) end rowvals @@ -135,49 +128,55 @@ AND g.m IN (0, 1, 2) !ismissing(getfield(row, Symbol(cn))) # ignore all rows with missing geometries end submeshes = map(gpkgblobs) do blob - gpkgbindata = isa(blob[1], WKBGeometry) ? blob[1].data : blob[1] - io = IOBuffer(gpkgbindata) + if blob[1][1:2] != UInt8[0x47, 0x50] + @warn "Missing magic 'GP' string in GPkgBinaryGeometry" + end + io = IOBuffer(blob[1]) seek(io, 3) flag = read(io, UInt8) # Note that Julia does not convert the endianness for you. # Use ntoh or ltoh for this purpose. bswap = isone(flag & 0x01) ? ltoh : ntoh - srsid = bswap(read(io, Int32)) - envelope = (flag & (0x07 << 1)) >> 1 envelopedims = 0 - if !iszero(envelope) if isone(envelope) envelopedims = 1 # 2D elseif isequal(2, envelope) envelopedims = 2 # 2D+Z elseif isequal(3, envelope) - envelopedims = 3 # 2D+M + envelopedims = 3 # 2D+M is not supported elseif isequal(4, envelope) - envelopedims = 4 # 2D+ZM + envelopedims = 4 # 2D+ZM is not supported else - @error "exceeded dimensional limit for geometry, file may be corrupted or reader is broken" - false + throw(ErrorException("exceeded dimensional limit for geometry, file may be corrupted or reader is broken")) end - else - true # no envelope (space saving slower indexing option), 0 bytes - end + end # else no envelope (space saving slower indexing option), 0 bytes # header size in byte stream headerlen = 8 + 8 * 4 * envelopedims seek(io, headerlen) - - ebyteorder = read(io, UInt8) - - bswap = isone(ebyteorder) ? ltoh : ntoh - - wkbtype = wkbGeometryType(read(io, UInt32)) - + wkbbyteswap = isone(read(io, UInt8)) ? ltoh : ntoh + wkbtypebits = read(io, UInt32) zextent = isequal(envelopedims, 2) - - mesh = meshfromwkb(io, srsid, org, org_coordsys_id, wkbtype, zextent, bswap) + if zextent + wkbtype = wkbtypebits & ewkbmaskbits ? wkbGeometryType[wkbtypebits & 0x000000F] : wkbGeometryType[wkbtypebits - 1000] + else + wkbtype = wkbGeometryType[wkbtypebits] + end + if iszero(srsid) + crs = LatLon{WGS84Latest} + elseif !isone(abs(srsid)) + if org == "EPSG" + crs = CoordRefSystems.get(EPSG{org_coordsys_id}) + elseif org == "ESRI" + crs = CoordRefSystems.get(ERSI{org_coordsys_id}) + end + else + crs = Cartesian{NoDatum} + end + mesh = meshfromwkb(io, crs, wkbtype, zextent, wkbbyteswap) if !isnothing(mesh) mesh @@ -189,94 +188,4 @@ AND g.m IN (0, 1, 2) reduce(vcat, meshes) # Future versions of Julia might change the reduce algorithm end -function meshfromwkb(io, srsid, org, org_coordsys_id, ewkbtype, zextent, bswap) - if iszero(srsid) - crs = LatLon{WGS84Latest} - elseif !isone(abs(srsid)) - if org == "EPSG" - crs = CoordRefSystems.get(EPSG{org_coordsys_id}) - elseif org == "ESRI" - crs = CoordRefSystems.get(ERSI{org_coordsys_id}) - end - else - crs = Cartesian{NoDatum} - end - - if occursin("Multi", string(ewkbtype)) - elems = wkbmultigeometry(io, crs, zextent, bswap) - Multi(elems) - else - elem = meshfromsf(io, crs, ewkbtype, zextent, bswap) - elem - end -end - -# Requirement 20: GeoPackage SHALL store feature table geometries -# with the basic simple feature geometry types -# https://www.geopackage.org/spec140/index.html#geometry_types -function meshfromsf(io, crs, ewkbtype, zextent, bswap) - if isequal(ewkbtype, wkbPoint) - elem = wkbcoordinate(io, zextent, bswap) - Point(crs(elem...)) - elseif isequal(ewkbtype, wkbLineString) - elem = wkblinestring(io, zextent, bswap) - if length(elem) >= 2 && first(elem) != last(elem) - Rope([Point(crs(coords...)) for coords in elem]...) - else - Ring([Point(crs(coords...)) for coords in elem[1:(end - 1)]]...) - end - elseif isequal(ewkbtype, wkbPolygon) - elem = wkbpolygon(io, zextent, bswap) - rings = map(elem) do ring - coords = map(ring) do point - Point(crs(point...)) - end - Ring(coords) - end - - outerring = first(rings) - holes = isone(length(rings)) ? rings[2:end] : Ring[] - PolyArea(outerring, holes...) - end -end - -function wkbcoordinate(io, z, bswap) - x = bswap(read(io, Float64)) - y = bswap(read(io, Float64)) - - if z - z = bswap(read(io, Float64)) - return x, y, z - end - - x, y -end - -function wkblinestring(io, z, bswap) - npoints = bswap(read(io, UInt32)) - - points = map(1:npoints) do _ - wkbcoordinate(io, z, bswap) - end - points -end -function wkbpolygon(io, z, bswap) - nrings = bswap(read(io, UInt32)) - - rings = map(1:nrings) do _ - wkblinestring(io, z, bswap) - end - rings -end - -function wkbmultigeometry(io, crs, z, bswap) - ngeoms = bswap(read(io, UInt32)) - - geomcollection = map(1:ngeoms) do _ - bswap = isone(read(io, UInt8)) ? ltoh : ntoh - ewkbtype = wkbGeometryType(read(io, UInt32)) - meshfromsf(io, crs, ewkbtype, z, bswap) - end - geomcollection -end diff --git a/src/extra/gpkg/wkb.jl b/src/extra/gpkg/wkb.jl new file mode 100644 index 0000000..4ddc604 --- /dev/null +++ b/src/extra/gpkg/wkb.jl @@ -0,0 +1,252 @@ +# ------------------------------------------------------------------ +# Licensed under the MIT License. See LICENSE in the project root. +# ------------------------------------------------------------------ + +""" + GeoIO.meshfromwkb(io, crs, wkbtype, haszextent, wkbbyteswap); + +Flavors of WKB supported: + +0. Standard WKB supports two-dimensional geometry, and is a proper subset of both extended WKB and ISO WKB. +```` julia + io = IOBuffer(WKBGeometryBLOB) + # load byte order in order to transport geometry + # easily between systems of different endianness + wkbByteOrder = read(io, UInt8) + # load simple feature wkb geometry types supported + wkbGeometryType = read(io, UInt32) + # wkb points do not have a `numPoints` field + if wkbGeometryType != 1 + # load number of geometries in geometry set + numWkbGeometries = read(io, UInt32) + end + # load in WKBGeometry that contain + # double precision numbers in the coordinates + # that are also subject to byte order rules + wkbGeometryBlob = read(io, Vector{UInt8}) +```` +1. Extended WKB allows applications to optionally add extra dimensions, and optionally embed an SRID + 99-402 was a short-lived extension to SFSQL 1.1 that used a high-bit flag +to indicate the presence of Z coordinates in a WKB geometry. + When the optional wkbSRID is added to the wkbType, an SRID number is inserted after the wkbType number. +⚠ This optional behaviour is not supported and will likely fail loading this variant + +2. ISO WKB allows for higher dimensional geometries. +SQL/MM Part 3 and SFSQL 1.2 use offsets of 1000 (Z), 2000 (M) and 3000 (ZM) +⚠ only offsets of 1000 are recognized and supported and will likely fali loading this variant + +Other systems like GDAL supports three wkbVariants deviated from the wkbStandard +- wkbVariantOldOgc:: Old-style 99-402 +- wkbVariantIso:: SFSQL 1.2 and ISO SQL/MM Part 3 +- wkbVariantPostGIS1::PostGIS 1.X +PostGIS supports a wider range of types (for example, CircularString, CurvePolygon) + +GeoIO GeoPackage reader supports wkbGeometryType using the SFSQL 1.2 use offset of 1000 (Z) and SFSQL 1.1 that used a high-bit flag, restricting some optional features + +GeoIO GeoPackage writer supports X,Y,Z coordinate offset of 1000 (Z) for wkbGeometryType + +## Example + +``` julia + meshes = [] + io = IOBuffer + for row in wkbGeometryColumn + wkbbyteswap = isone(read(io, UInt8)) ? ltoh : ntoh + wkbtype = read(io, UInt32) + crs = LatLon{WGS84Latest} + haszextent = false + push!(meshes, meshfromwkb(io, crs, wkbtype, haszextent, wkbbyteswap)) + end +``` + +""" +const ewkbmaskbits = 0x40000000 | 0x80000000 + +const wkbGeometryType = Dict{UInt32,Symbol}( + 1 => :wkbPoint, + 2 => :wkbLineString, + 3 => :wkbPolygon, + 4 => :wkbMultiPoint, + 5 => :wkbMultiLineString, + 6 => :wkbMultiPolygon, + 7 => :wkbGeometryCollection +) + +# Requirement 20: GeoPackage SHALL store feature table geometries +# with the basic simple feature geometry types +# https://www.geopackage.org/spec140/index.html#geometry_types +function meshfromwkb(io, crs, wkbtype, zextent, bswap) + if occursin("Multi", string(wkbtype)) + elems = wkbmultigeometry(io, crs, zextent, bswap) + Multi(elems) + else + elem = meshfromsf(io, crs, wkbtype, zextent, bswap) + elem + end +end + +function meshfromsf(io, crs, wkbtype, zextent, bswap) + if isequal(wkbtype, :wkbPoint) + elem = wkbcoordinate(io, zextent, bswap) + Point(crs(elem...)) + elseif isequal(wkbtype, :wkbLineString) + elem = wkblinestring(io, zextent, bswap) + if length(elem) >= 2 && first(elem) != last(elem) + Rope([Point(crs(coords...)) for coords in elem]...) + else + Ring([Point(crs(coords...)) for coords in elem[1:(end - 1)]]...) + end + elseif isequal(wkbtype, :wkbPolygon) + elem = wkbpolygon(io, zextent, bswap) + rings = map(elem) do ring + coords = map(ring) do point + Point(crs(point...)) + end + Ring(coords) + end + + outerring = first(rings) + holes = isone(length(rings)) ? rings[2:end] : Ring[] + PolyArea(outerring, holes...) + end +end + +function wkbcoordinate(io, z, bswap) + x = bswap(read(io, Float64)) + y = bswap(read(io, Float64)) + + if z + z = bswap(read(io, Float64)) + return x, y, z + end + + x, y +end + +function wkblinestring(io, z, bswap) + npoints = bswap(read(io, UInt32)) + + points = map(1:npoints) do _ + wkbcoordinate(io, z, bswap) + end + points +end + +function wkbpolygon(io, z, bswap) + nrings = bswap(read(io, UInt32)) + + rings = map(1:nrings) do _ + wkblinestring(io, z, bswap) + end + rings +end + +function wkbmultigeometry(io, crs, z, bswap) + ngeoms = bswap(read(io, UInt32)) + + geomcollection = map(1:ngeoms) do _ + bswap = isone(read(io, UInt8)) ? ltoh : ntoh + wkbtypebits = read(io, UInt32) + if z + if iszero(wkbtypebits & ewkbmaskbits) + wkbtype = wkbGeometryType[wkbtypebits] + else + wkbtype = wkbGeometryType[wkbtypebits - 1000] + end + else + wkbtype = wkbGeometryType[wkbtypebits] + end + meshfromsf(io, crs, wkbtype, z, bswap) + end + geomcollection +end + +function _wkbtype(geometry) + if geometry isa Point + return :wkbPoint + elseif geometry isa Rope || geometry isa Ring + return :wkbLineString + elseif geometry isa PolyArea + return :wkbPolygon + elseif geometry isa Multi + fg = first(parent(geometry)) + return wkbGeometryType[findfirst(isequal(_wkbtype(fg)), wkbGeometryType) + 3] + else + throw(ErrorException(" $geometry to wkbGeometryType Symbol is not available")) + end +end + +function writewkbgeom(io, geom) + wkbtype = _wkbtype(geom) + write(io, htol(one(UInt8))) + write(io, htol(UInt32(findfirst(isequal(wkbtype), wkbGeometryType)))) + _wkbgeom(io, wkbtype, geom) +end + +function writewkbsf(io, wkbtype, geom) + if isequal(wkbtype, :wkbPolygon) + _wkbpolygon(io, wkbtype, [boundary(geom::PolyArea)]) + elseif isequal(wkbtype, :wkbLineString) + coordlist = vertices(geom) + if typeof(geom) <: Ring + return _wkblinearring(io, wkbtype, coordlist) + end + _wkblinestring(io, wkbtype, coordlist) + elseif isequal(wkbtype, :wkbPoint) + coordinates = CoordRefSystems.raw(coords(geom)) + _wkbcoordinates(io, coordinates) + else + throw(ErrorException("Well-Known Binary Geometry not supported: $wkbtype")) + end +end + +function _wkbgeom(io, wkbtype, geom) + if findfirst(isequal(wkbtype), wkbGeometryType) > 3 + _wkbmulti(io, wkbtype, geom) + else + writewkbsf(io, wkbtype, geom) + end +end + +function _wkbcoordinates(io, coords) + write(io, htol(coords[2])) + write(io, htol(coords[1])) + if length(coords) == 3 + write(io, htol(coords[3])) + end +end + +function _wkblinestring(io, wkb_type, coord_list) + write(io, htol(UInt32(length(coord_list)))) + for n_coords in coord_list + coordinates = CoordRefSystems.raw(coords(n_coords)) + _wkbcoordinates(io, coordinates) + end +end + +function _wkblinearring(io, wkb_type, coord_list) + write(io, htol(UInt32(length(coord_list) + 1))) + for n_coords in coord_list + coordinates = CoordRefSystems.raw(coords(n_coords)) + _wkbcoordinates(io, coordinates) + end + _wkbcoordinates(io, CoordRefSystems.raw(first(coord_list) |> coords)) +end + +function _wkbpolygon(io, wkb_type, rings) + write(io, htol(UInt32(length(rings)))) + for ring in rings + coord_list = vertices(ring) + _wkblinestring(io, wkb_type, coord_list) + end +end + +function _wkbmulti(io, multiwkbtype, geoms) + write(io, htol(UInt32(length(parent(geoms))))) + for sf in parent(geoms) + write(io, one(UInt8)) + wkbn = findfirst(isequal(multiwkbtype), wkbGeometryType) + write(io, UInt32(wkbn - 3)) + writewkbsf(io, wkbGeometryType[wkbn - 3], sf) + end +end \ No newline at end of file diff --git a/src/extra/gpkg/write.jl b/src/extra/gpkg/write.jl index 68e635c..e1ad6df 100644 --- a/src/extra/gpkg/write.jl +++ b/src/extra/gpkg/write.jl @@ -19,6 +19,7 @@ function gpkgwrite(fname, geotable;) DBInterface.execute(db, "PRAGMA user_version = $GPKG_1_4_VERSION ") creategpkgtables(db, table, domain, crs, geom) DBInterface.execute(db, "PRAGMA optimize;") + # https://sqlite.org/pragma.html#pragma_optimize # Applications with short-lived database connections should run "PRAGMA optimize;" # just once, prior to closing each database connection. DBInterface.close!(db) @@ -42,12 +43,9 @@ function creategpkgtables(db, table, domain, crs, geom) vcat(gpkgbinheader, take!(io)) end - geomtype = _geomtype(geom) - GeomType = _sqlitetype(geomtype) - table = - isnothing(table) ? [(; geom=GeomType(g),) for (_, g) in zip(1:length(gpkgbinary), gpkgbinary)] : - [(; t..., geom=GeomType(g)) for (t, g) in zip(Tables.rowtable(table), gpkgbinary)] + isnothing(table) ? [(; geom=g,) for (_, g) in zip(1:length(gpkgbinary), gpkgbinary)] : + [(; t..., geom=g) for (t, g) in zip(Tables.rowtable(table), gpkgbinary)] rows = Tables.rows(table) sch = Tables.schema(rows) columns = [ @@ -117,7 +115,7 @@ function creategpkgtables(db, table, domain, crs, geom) DBInterface.execute( db, """ - INSERT INTO gpkg_spatial_ref_sys + INSERT INTO gpkg_spatial_ref_sys (srs_name, srs_id, organization, organization_coordsys_id, definition, description, definition_12_063) VALUES ('Undefined Cartesian SRS', -1, 'NONE', -1, 'undefined', 'undefined geographic coordinate reference system', 'undefined'), @@ -195,31 +193,11 @@ function creategpkgtables(db, table, domain, crs, geom) VALUES (?, ?, ?, ?, ?, ?); """, - ["features", "geom", geomtype, srid, z, 0] + ["features", "geom", "GEOMETRY", srid, z, 0] ) end end -function _wkbtype(geometry) - if geometry isa Point - return wkbPoint - elseif geometry isa Rope || geometry isa Ring - return wkbLineString - elseif geometry isa PolyArea - return wkbPolygon - elseif geometry isa Multi - fg = parent(geometry) |> first - return wkbGeometryType(Int(_wkbtype(fg)) + 3) - else - @error "my hovercraft is full of eels: $geometry" - end -end - -function _wkbsetz(type::wkbGeometryType) - return wkbGeometryType(Int(type) + 1000) - # ISO WKB Flavour -end - function writegpkgheader(srsid, geom) io = IOBuffer() write(io, [0x47, 0x50]) # 'GP' in ASCII @@ -243,78 +221,3 @@ function writegpkgheader(srsid, geom) return take!(io) end - -function writewkbgeom(io, geom) - wkbtype = paramdim(geom) < 3 ? _wkbtype(geom) : _wkbsetz(_wkbtype(geom)) - write(io, htol(one(UInt8))) - write(io, htol(UInt32(wkbtype))) - _wkbgeom(io, wkbtype, geom) -end - -function writewkbsf(io, wkbtype, geom) - if wkbtype == wkbPolygon || wkbtype == wkbPolygonZ - _wkbpolygon(io, wkbtype, [boundary(geom::PolyArea)]) - elseif wkbtype == wkbLineString || wkbtype == wkbLineString - coordlist = vertices(geom) - if typeof(geom) <: Ring - return _wkblinearring(io, wkbtype, coordlist) - end - _wkblinestring(io, wkbtype, coordlist) - elseif wkbtype == wkbPoint || wkbtype == wkbPointZ - coordinates = CoordRefSystems.raw(coords(geom)) - _wkbcoordinates(io, wkbtype, coordinates) - else - throw(ErrorException("Well-Known Binary Geometry not supported: $wkbtype")) - end -end - -function _wkbgeom(io, wkbtype, geom) - if Int(wkbtype) > 3 - _wkbmulti(io, wkbtype, geom) - else - writewkbsf(io, wkbtype, geom) - end -end - -function _wkbcoordinates(io, wkbtype, coords) - write(io, htol(coords[2])) - write(io, htol(coords[1])) - - if (UInt32(wkbtype) > 1000) || !(iszero(Int(wkbtype) & (0x80000000 | 0x40000000))) - write(io, htol(coords[3])) - end -end - -function _wkblinestring(io, wkb_type, coord_list) - write(io, htol(UInt32(length(coord_list)))) - for n_coords in coord_list - coordinates = CoordRefSystems.raw(coords(n_coords)) - _wkbcoordinates(io, wkb_type, coordinates) - end -end - -function _wkblinearring(io, wkb_type, coord_list) - write(io, htol(UInt32(length(coord_list) + 1))) - for n_coords in coord_list - coordinates = CoordRefSystems.raw(coords(n_coords)) - _wkbcoordinates(io, wkb_type, coordinates) - end - _wkbcoordinates(io, wkb_type, CoordRefSystems.raw(first(coord_list) |> coords)) -end - -function _wkbpolygon(io, wkb_type, rings) - write(io, htol(UInt32(length(rings)))) - for ring in rings - coord_list = vertices(ring) - _wkblinestring(io, wkb_type, coord_list) - end -end - -function _wkbmulti(io, wkbtype, geoms) - write(io, htol(UInt32(length(geoms |> parent)))) - for sf in geoms |> parent - write(io, one(UInt8)) - write(io, UInt32(Int(wkbMultiPolygon) - 3)) - writewkbsf(io, wkbGeometryType(Int(wkbtype) - 3), sf) - end -end \ No newline at end of file diff --git a/src/extra/wkb.jl b/src/extra/wkb.jl deleted file mode 100644 index 148d871..0000000 --- a/src/extra/wkb.jl +++ /dev/null @@ -1,120 +0,0 @@ - -@enum wkbGeometryType::UInt32 begin - wkbUnknown = 0 - wkbPoint = 1 - wkbLineString = 2 - wkbPolygon = 3 - wkbMultiPoint = 4 - wkbMultiLineString = 5 - wkbMultiPolygon = 6 - wkbGeometryCollection = 7 - wkbCircularString = 8 - wkbCompoundCurve = 9 - wkbCurvePolygon = 10 - wkbMultiCurve = 11 - wkbMultiSurface = 12 - wkbCurve = 13 - wkbSurface = 14 - - wkbNone = 100 # pure attribute records - wkbLinearRing = 101 - - # ISO SQL/MM Part 3: Spatial - # Z-aware types - wkbPointZ = 1001 - wkbLineStringZ = 1002 - wkbPolygonZ = 1003 - wkbMultiPointZ = 1004 - wkbMultiLineStringZ = 1005 - wkbMultiPolygonZ = 1006 - wkbGeometryCollectionZ = 1007 - wkbCircularStringZ = 1008 - wkbCompoundCurveZ = 1009 - wkbCurvePolygonZ = 1010 - wkbMultiCurveZ = 1011 - wkbMultiSurfaceZ = 1012 - wkbCurveZ = 1013 - wkbSurfaceZ = 1014 - - # ISO SQL/MM Part 3. - # M-aware types - wkbPointM = 2001 - wkbLineStringM = 2002 - wkbPolygonM = 2003 - wkbMultiPointM = 2004 - wkbMultiLineStringM = 2005 - wkbMultiPolygonM = 2006 - wkbGeometryCollectionM = 2007 - wkbCircularStringM = 2008 - wkbCompoundCurveM = 2009 - wkbCurvePolygonM = 2010 - wkbMultiCurveM = 2011 - wkbMultiSurfaceM = 2012 - wkbCurveM = 2013 - wkbSurfaceM = 2014 - - # ISO SQL/MM Part 3. - # ZM-aware types - wkbPointZM = 3001 - wkbLineStringZM = 3002 - wkbPolygonZM = 3003 - wkbMultiPointZM = 3004 - wkbMultiLineStringZM = 3005 - wkbMultiPolygonZM = 3006 - wkbGeometryCollectionZM = 3007 - wkbCircularStringZM = 3008 - wkbCompoundCurveZM = 3009 - wkbCurvePolygonZM = 3010 - wkbMultiCurveZM = 3011 - wkbMultiSurfaceZM = 3012 - wkbCurveZM = 3013 - wkbSurfaceZM = 3014 - - # 2.5D extension as per 99-402 - # https://lists.osgeo.org/pipermail/postgis-devel/2004-December/000702.html - wkbPoint25D = 0x80000001 - wkbLineString25D = 0x80000002 - wkbPolygon25D = 0x80000003 - wkbMultiPoint25D = 0x80000004 - wkbMultiLineString25D = 0x80000005 - wkbMultiPolygon25D = 0x80000006 - wkbGeometryCollection25D = 0x80000007 -end - -abstract type WKBGeometry end - -struct WKBPoint <: WKBGeometry - data::Vector{UInt8} -end - -struct WKBLineString <: WKBGeometry - data::Vector{UInt8} -end - -struct WKBPolygon <: WKBGeometry - data::Vector{UInt8} -end - -struct WKBMultiPoint <: WKBGeometry - data::Vector{UInt8} -end - -struct WKBMultiLineString <: WKBGeometry - data::Vector{UInt8} -end - -struct WKBMultiPolygon <: WKBGeometry - data::Vector{UInt8} -end - -struct WKBGeometryCollection <: WKBGeometry - data::Vector{UInt8} -end - -function Base.getproperty(gpkg::WKBGeometry, sym::Symbol) - if sym == :data - return getfield(gpkg, :data) - else - return getfield(gpkg, sym) - end -end From b058783ee48f0766f5946825c5ac8f6fbcadb23c Mon Sep 17 00:00:00 2001 From: jph6366 Date: Thu, 9 Oct 2025 16:17:23 -0400 Subject: [PATCH 30/90] evil test changes to implement AUTOINCREMENT --- src/extra/gpkg/write.jl | 4 ++-- test/gisissues.jl | 6 +++--- test/io/geopackage.jl | 7 ++++--- test/novalues.jl | 5 +++-- 4 files changed, 12 insertions(+), 10 deletions(-) diff --git a/src/extra/gpkg/write.jl b/src/extra/gpkg/write.jl index e1ad6df..2313884 100644 --- a/src/extra/gpkg/write.jl +++ b/src/extra/gpkg/write.jl @@ -45,7 +45,7 @@ function creategpkgtables(db, table, domain, crs, geom) table = isnothing(table) ? [(; geom=g,) for (_, g) in zip(1:length(gpkgbinary), gpkgbinary)] : - [(; t..., geom=g) for (t, g) in zip(Tables.rowtable(table), gpkgbinary)] + [(; geom=g, t...) for (t, g) in zip(Tables.rowtable(table), gpkgbinary)] rows = Tables.rows(table) sch = Tables.schema(rows) columns = [ @@ -55,7 +55,7 @@ function creategpkgtables(db, table, domain, crs, geom) # https://www.geopackage.org/spec/#r29 # A feature table SHALL have a primary key column of type INTEGER and that column SHALL act as a rowid alias. - DBInterface.execute(db, "CREATE TABLE features ($(join(columns, ',')));") + DBInterface.execute(db, "CREATE TABLE features ( id INTEGER PRIMARY KEY AUTOINCREMENT, $(join(columns, ',')));") # The use of the AUTOINCREMENT keyword is optional but recommended. # Implementers MAY omit the AUTOINCREMENT keyword for performance reasons, with the understanding that doing so has the potential to allow primary key identifiers to be reused. diff --git a/test/gisissues.jl b/test/gisissues.jl index b261072..efd9067 100644 --- a/test/gisissues.jl +++ b/test/gisissues.jl @@ -71,7 +71,7 @@ file = joinpath(savedir, "gis-points.gpkg") GeoIO.save(file, gtpoint) gtb = GeoIO.load(file) - @test Set(names(gtb)) == Set(names(gtpoint)) + @test Set([n for n in names(gtb) if n != "id"]) == Set(names(gtpoint)) @test gtb.geometry == gtpoint.geometry @test gtb.float == gtpoint.float @test gtb.int == gtpoint.int @@ -80,7 +80,7 @@ file = joinpath(savedir, "gis-rings.gpkg") GeoIO.save(file, gtring) gtb = GeoIO.load(file) - @test Set(names(gtb)) == Set(names(gtring)) + @test Set([n for n in names(gtb) if n != "id"]) == Set(names(gtring)) @test gtb.geometry == gtring.geometry @test gtb.float == gtring.float @test gtb.int == gtring.int @@ -89,7 +89,7 @@ file = joinpath(savedir, "gis-polys.gpkg") GeoIO.save(file, gtpoly) gtb = GeoIO.load(file) - @test Set(names(gtb)) == Set(names(gtpoly)) + @test Set([n for n in names(gtb) if n != "id"]) == Set(names(gtpoly)) @test gtb.geometry == gtpoly.geometry @test gtb.float == gtpoly.float @test gtb.int == gtpoly.int diff --git a/test/io/geopackage.jl b/test/io/geopackage.jl index 5e82834..c6ff8d0 100644 --- a/test/io/geopackage.jl +++ b/test/io/geopackage.jl @@ -32,12 +32,13 @@ @testset "save" begin # note: GeoPackage does not preserve column order + # note2: Every such feature table SHALL have a primary key column 'id' of type INTEGER file1 = joinpath(datadir, "points.gpkg") file2 = joinpath(savedir, "points.gpkg") gtb1 = GeoIO.load(file1) GeoIO.save(file2, gtb1) gtb2 = GeoIO.load(file2) - @test Set(names(gtb2)) == Set(names(gtb1)) + @test Set([n for n in names(gtb2) if n != "id"]) == Set(names(gtb1)) @test gtb2.geometry == gtb1.geometry @test gtb2.code == gtb1.code @test gtb2.name == gtb1.name @@ -48,7 +49,7 @@ gtb1 = GeoIO.load(file1) GeoIO.save(file2, gtb1) gtb2 = GeoIO.load(file2) - @test Set(names(gtb2)) == Set(names(gtb1)) + @test Set([n for n in names(gtb2) if n != "id"]) == Set(names(gtb1)) @test gtb2.geometry == gtb1.geometry @test gtb2.code == gtb1.code @test gtb2.name == gtb1.name @@ -59,7 +60,7 @@ gtb1 = GeoIO.load(file1) GeoIO.save(file2, gtb1) gtb2 = GeoIO.load(file2) - @test Set(names(gtb2)) == Set(names(gtb1)) + @test Set([n for n in names(gtb2) if n != "id"]) == Set(names(gtb1)) @test gtb2.geometry == gtb1.geometry @test gtb2.code == gtb1.code @test gtb2.name == gtb1.name diff --git a/test/novalues.jl b/test/novalues.jl index b900bc8..3f038ad 100644 --- a/test/novalues.jl +++ b/test/novalues.jl @@ -23,8 +23,9 @@ file = joinpath(savedir, "noattribs.gpkg") GeoIO.save(file, gtb1) gtb2 = GeoIO.load(file) - @test isnothing(values(gtb2)) - @test gtb2 == gtb1 + @test isequal((id = [1, 2, 3],), values(gtb2)) + gtb1o = georef((id = [1, 2, 3],), pset) + @test gtb2 == gtb1o # CSV pset = [Point(0.0, 0.0), Point(1.0, 0.0), Point(0.0, 1.0)] From 4de27644abc21ac39a4b903f9e2461ec9e2fe07c Mon Sep 17 00:00:00 2001 From: jph6366 Date: Thu, 9 Oct 2025 16:32:58 -0400 Subject: [PATCH 31/90] minor fixes --- src/extra/gpkg/read.jl | 4 ++-- src/extra/gpkg/write.jl | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/extra/gpkg/read.jl b/src/extra/gpkg/read.jl index 960f70a..7e191b2 100644 --- a/src/extra/gpkg/read.jl +++ b/src/extra/gpkg/read.jl @@ -5,8 +5,8 @@ function gpkgread(fname; layer=1) db = SQLite.DB(fname) assertgpkg(db) - geom = gpkgmesh(db, ; layer) - attrs = gpkgmeshattrs(db, ; layer) + geom = gpkggeoms(db, ; layer) + attrs = gpkgvalues(db, ; layer) DBInterface.close!(db) if eltype(attrs) <: Nothing return georef(nothing, geom) diff --git a/src/extra/gpkg/write.jl b/src/extra/gpkg/write.jl index 4577e9b..c7d3885 100644 --- a/src/extra/gpkg/write.jl +++ b/src/extra/gpkg/write.jl @@ -39,7 +39,7 @@ function creategpkgtables(db, table, domain, crs, geom) gpkgbinary = map(geom) do ft gpkgbinheader = writegpkgheader(srid, ft) io = IOBuffer() - writewkbgeom(ft, io) + writewkbgeom(io, ft) vcat(gpkgbinheader, take!(io)) end From 97a03e2d450112e2d877aa096839d9b0a609692b Mon Sep 17 00:00:00 2001 From: jph6366 Date: Fri, 10 Oct 2025 11:54:11 -0400 Subject: [PATCH 32/90] added spatial r-tree indexes and gpkg_extensions, also fix for writing CRS not already in gpkg_spatial_ref_sys --- src/extra/gpkg/wkb.jl | 12 ++++--- src/extra/gpkg/write.jl | 72 ++++++++++++++++++++++++++++++++++++----- 2 files changed, 72 insertions(+), 12 deletions(-) diff --git a/src/extra/gpkg/wkb.jl b/src/extra/gpkg/wkb.jl index 4ddc604..bf32cef 100644 --- a/src/extra/gpkg/wkb.jl +++ b/src/extra/gpkg/wkb.jl @@ -8,7 +8,10 @@ Flavors of WKB supported: 0. Standard WKB supports two-dimensional geometry, and is a proper subset of both extended WKB and ISO WKB. -```` julia + +## Reading WKB Geometry BLOB + +``` julia io = IOBuffer(WKBGeometryBLOB) # load byte order in order to transport geometry # easily between systems of different endianness @@ -20,11 +23,12 @@ Flavors of WKB supported: # load number of geometries in geometry set numWkbGeometries = read(io, UInt32) end - # load in WKBGeometry that contain - # double precision numbers in the coordinates + # load in WKBGeometry that contain geometry values + # w/ double precision numbers in the coordinates # that are also subject to byte order rules wkbGeometryBlob = read(io, Vector{UInt8}) -```` +``` + 1. Extended WKB allows applications to optionally add extra dimensions, and optionally embed an SRID 99-402 was a short-lived extension to SFSQL 1.1 that used a high-bit flag to indicate the presence of Z coordinates in a WKB geometry. diff --git a/src/extra/gpkg/write.jl b/src/extra/gpkg/write.jl index c7d3885..7eaaec1 100644 --- a/src/extra/gpkg/write.jl +++ b/src/extra/gpkg/write.jl @@ -6,6 +6,7 @@ function gpkgwrite(fname, geotable;) db = SQLite.DB(fname) DBInterface.execute(db, "PRAGMA synchronous=0") + # https://sqlite.org/pragma.html#pragma_synchronous # Commits can be orders of magnitude faster with # Setting PRAGMA synchronous=OFF but, # can cause the database to go corrupt @@ -34,7 +35,7 @@ function creategpkgtables(db, table, domain, crs, geom) srid = 4326 else srs = string(CoordRefSystems.code(crs))[1:4] - srid = parse(Int32, srs) + srid = parse(Int32, string(CoordRefSystems.code(crs))[6:(end - 1)]) end gpkgbinary = map(geom) do ft gpkgbinheader = writegpkgheader(srid, ft) @@ -86,15 +87,13 @@ function creategpkgtables(db, table, domain, crs, geom) state === nothing && break row, st = state end - end - bbox = boundingbox(domain) - mincoords = CoordRefSystems.raw(coords(bbox.min)) - maxcoords = CoordRefSystems.raw(coords(bbox.max)) - minx, miny, maxx, maxy = mincoords[1], mincoords[2], maxcoords[1], maxcoords[2] - z = paramdim(first(geom)) > 2 ? 1 : 0 + bbox = boundingbox(domain) + mincoords = CoordRefSystems.raw(coords(bbox.min)) + maxcoords = CoordRefSystems.raw(coords(bbox.max)) + minx, miny, maxx, maxy = mincoords[1], mincoords[2], maxcoords[1], maxcoords[2] + z = paramdim(first(geom)) > 2 ? 1 : 0 - SQLite.transaction(db) do DBInterface.execute( db, """ @@ -195,6 +194,63 @@ function creategpkgtables(db, table, domain, crs, geom) """, ["features", "geom", "GEOMETRY", srid, z, 0] ) + + DBInterface.execute( + db, + """ + CREATE VIRTUAL TABLE rtree_features_geom USING + rtree(id, minx, maxx, miny, maxy) + """ + ) + bboxes = map(geom) do ft + bbox = boundingbox(ft) + mincoords = CoordRefSystems.raw(coords(bbox.min)) + maxcoords = CoordRefSystems.raw(coords(bbox.max)) + minx, miny, maxx, maxy = mincoords[1], mincoords[2], maxcoords[1], maxcoords[2] + return (minx, maxx, miny, maxy) + end + + stmt = SQLite.Stmt(db, "INSERT INTO rtree_features_geom VALUES (?, ?, ?, ?, ?)") + handle = SQLite._get_stmt_handle(stmt) + for i in 1:length(gpkgbinary) + fx_minx, fx_maxx, fx_miny, fx_maxy = bboxes[i] + SQLite.bind!(stmt, 1, i) + SQLite.bind!(stmt, 2, fx_minx) + SQLite.bind!(stmt, 3, fx_maxx) + SQLite.bind!(stmt, 4, fx_miny) + SQLite.bind!(stmt, 5, fx_maxy) + r = SQLite.C.sqlite3_step(handle) + if r != SQLite.C.SQLITE_DONE + e = SQLite.sqliteexception(db, stmt) + SQLite.C.sqlite3_reset(handle) + throw(e) + end + SQLite.C.sqlite3_reset(handle) + end + + DBInterface.execute( + db, + """ + CREATE TABLE gpkg_extensions ( + table_name TEXT, + column_name TEXT, + extension_name TEXT NOT NULL, + definition TEXT NOT NULL, + scope TEXT NOT NULL, + CONSTRAINT ge_tce UNIQUE (table_name, column_name, extension_name) + ) + """ + ) + + DBInterface.execute( + db, + """ + INSERT INTO gpkg_extensions + (table_name, column_name, extension_name, definition, scope) + VALUES + ('features', 'geom', 'gpkg_rtree_index', 'http://www.geopackage.org/spec120/#extension_rtree', 'write-only'); + """ + ) end end From f98563c03c296ae8b9424434ec82e9ba6c6e6cad Mon Sep 17 00:00:00 2001 From: jph6366 Date: Fri, 10 Oct 2025 16:04:16 -0400 Subject: [PATCH 33/90] enums are best for this usecase --- src/extra/gpkg/read.jl | 4 +-- src/extra/gpkg/wkb.jl | 74 +++++++++++++++++++++--------------------- 2 files changed, 39 insertions(+), 39 deletions(-) diff --git a/src/extra/gpkg/read.jl b/src/extra/gpkg/read.jl index 7e191b2..a9524c7 100644 --- a/src/extra/gpkg/read.jl +++ b/src/extra/gpkg/read.jl @@ -162,9 +162,9 @@ AND g.m IN (0, 1, 2) zextent = isequal(envelopedims, 2) if zextent wkbtype = - wkbtypebits & ewkbmaskbits ? wkbGeometryType[wkbtypebits & 0x000000F] : wkbGeometryType[wkbtypebits - 1000] + wkbtypebits & ewkbmaskbits ? wkbGeometryType(wkbtypebits & 0x000000F) : wkbGeometryType(wkbtypebits - 1000) else - wkbtype = wkbGeometryType[wkbtypebits] + wkbtype = wkbGeometryType(wkbtypebits) end if iszero(srsid) crs = LatLon{WGS84Latest} diff --git a/src/extra/gpkg/wkb.jl b/src/extra/gpkg/wkb.jl index bf32cef..e059720 100644 --- a/src/extra/gpkg/wkb.jl +++ b/src/extra/gpkg/wkb.jl @@ -56,7 +56,7 @@ GeoIO GeoPackage writer supports X,Y,Z coordinate offset of 1000 (Z) for wkbGeom io = IOBuffer for row in wkbGeometryColumn wkbbyteswap = isone(read(io, UInt8)) ? ltoh : ntoh - wkbtype = read(io, UInt32) + wkbtype = wkbGeometryType(read(io, UInt32)) crs = LatLon{WGS84Latest} haszextent = false push!(meshes, meshfromwkb(io, crs, wkbtype, haszextent, wkbbyteswap)) @@ -66,21 +66,22 @@ GeoIO GeoPackage writer supports X,Y,Z coordinate offset of 1000 (Z) for wkbGeom """ const ewkbmaskbits = 0x40000000 | 0x80000000 -const wkbGeometryType = Dict{UInt32,Symbol}( - 1 => :wkbPoint, - 2 => :wkbLineString, - 3 => :wkbPolygon, - 4 => :wkbMultiPoint, - 5 => :wkbMultiLineString, - 6 => :wkbMultiPolygon, - 7 => :wkbGeometryCollection -) +@enum wkbGeometryType begin + wkbUnknown = 0 + wkbPoint = 1 + wkbLineString = 2 + wkbPolygon = 3 + wkbMultiPoint = 4 + wkbMultiLineString = 5 + wkbMultiPolygon = 6 + wkbGeometryCollection = 7 +end # Requirement 20: GeoPackage SHALL store feature table geometries # with the basic simple feature geometry types # https://www.geopackage.org/spec140/index.html#geometry_types function meshfromwkb(io, crs, wkbtype, zextent, bswap) - if occursin("Multi", string(wkbtype)) + if UInt32(wkbtype) > 3 elems = wkbmultigeometry(io, crs, zextent, bswap) Multi(elems) else @@ -90,17 +91,17 @@ function meshfromwkb(io, crs, wkbtype, zextent, bswap) end function meshfromsf(io, crs, wkbtype, zextent, bswap) - if isequal(wkbtype, :wkbPoint) + if isequal(wkbtype, wkbPoint) elem = wkbcoordinate(io, zextent, bswap) Point(crs(elem...)) - elseif isequal(wkbtype, :wkbLineString) + elseif isequal(wkbtype, wkbLineString) elem = wkblinestring(io, zextent, bswap) if length(elem) >= 2 && first(elem) != last(elem) Rope([Point(crs(coords...)) for coords in elem]...) else Ring([Point(crs(coords...)) for coords in elem[1:(end - 1)]]...) end - elseif isequal(wkbtype, :wkbPolygon) + elseif isequal(wkbtype, wkbPolygon) elem = wkbpolygon(io, zextent, bswap) rings = map(elem) do ring coords = map(ring) do point @@ -153,12 +154,12 @@ function wkbmultigeometry(io, crs, z, bswap) wkbtypebits = read(io, UInt32) if z if iszero(wkbtypebits & ewkbmaskbits) - wkbtype = wkbGeometryType[wkbtypebits] + wkbtype = wkbGeometryType(wkbtypebits) else - wkbtype = wkbGeometryType[wkbtypebits - 1000] + wkbtype = wkbGeometryType(wkbtypebits - 1000) end else - wkbtype = wkbGeometryType[wkbtypebits] + wkbtype = wkbGeometryType(wkbtypebits) end meshfromsf(io, crs, wkbtype, z, bswap) end @@ -167,36 +168,36 @@ end function _wkbtype(geometry) if geometry isa Point - return :wkbPoint + return wkbPoint elseif geometry isa Rope || geometry isa Ring - return :wkbLineString + return wkbLineString elseif geometry isa PolyArea - return :wkbPolygon + return wkbPolygon elseif geometry isa Multi fg = first(parent(geometry)) - return wkbGeometryType[findfirst(isequal(_wkbtype(fg)), wkbGeometryType) + 3] + return wkbGeometryType(Int(_wkbtype(fg)) + 3) else - throw(ErrorException(" $geometry to wkbGeometryType Symbol is not available")) + return wkbGeometryCollection end end function writewkbgeom(io, geom) wkbtype = _wkbtype(geom) write(io, htol(one(UInt8))) - write(io, htol(UInt32(findfirst(isequal(wkbtype), wkbGeometryType)))) + write(io, htol(UInt32(wkbtype))) _wkbgeom(io, wkbtype, geom) end function writewkbsf(io, wkbtype, geom) - if isequal(wkbtype, :wkbPolygon) - _wkbpolygon(io, wkbtype, [boundary(geom::PolyArea)]) - elseif isequal(wkbtype, :wkbLineString) + if isequal(wkbtype, wkbPolygon) + _wkbpolygon(io, [boundary(geom::PolyArea)]) + elseif isequal(wkbtype, wkbLineString) coordlist = vertices(geom) if typeof(geom) <: Ring - return _wkblinearring(io, wkbtype, coordlist) + return _wkblinearring(io, coordlist) end - _wkblinestring(io, wkbtype, coordlist) - elseif isequal(wkbtype, :wkbPoint) + _wkblinestring(io, coordlist) + elseif isequal(wkbtype, wkbPoint) coordinates = CoordRefSystems.raw(coords(geom)) _wkbcoordinates(io, coordinates) else @@ -205,7 +206,7 @@ function writewkbsf(io, wkbtype, geom) end function _wkbgeom(io, wkbtype, geom) - if findfirst(isequal(wkbtype), wkbGeometryType) > 3 + if UInt32(wkbtype) > 3 _wkbmulti(io, wkbtype, geom) else writewkbsf(io, wkbtype, geom) @@ -220,7 +221,7 @@ function _wkbcoordinates(io, coords) end end -function _wkblinestring(io, wkb_type, coord_list) +function _wkblinestring(io, coord_list) write(io, htol(UInt32(length(coord_list)))) for n_coords in coord_list coordinates = CoordRefSystems.raw(coords(n_coords)) @@ -228,7 +229,7 @@ function _wkblinestring(io, wkb_type, coord_list) end end -function _wkblinearring(io, wkb_type, coord_list) +function _wkblinearring(io, coord_list) write(io, htol(UInt32(length(coord_list) + 1))) for n_coords in coord_list coordinates = CoordRefSystems.raw(coords(n_coords)) @@ -237,11 +238,11 @@ function _wkblinearring(io, wkb_type, coord_list) _wkbcoordinates(io, CoordRefSystems.raw(first(coord_list) |> coords)) end -function _wkbpolygon(io, wkb_type, rings) +function _wkbpolygon(io, rings) write(io, htol(UInt32(length(rings)))) for ring in rings coord_list = vertices(ring) - _wkblinestring(io, wkb_type, coord_list) + _wkblinestring(io, coord_list) end end @@ -249,8 +250,7 @@ function _wkbmulti(io, multiwkbtype, geoms) write(io, htol(UInt32(length(parent(geoms))))) for sf in parent(geoms) write(io, one(UInt8)) - wkbn = findfirst(isequal(multiwkbtype), wkbGeometryType) - write(io, UInt32(wkbn - 3)) - writewkbsf(io, wkbGeometryType[wkbn - 3], sf) + write(io, UInt32(multiwkbtype) - 3) + writewkbsf(io, wkbGeometryType(UInt32(multiwkbtype) - 3), sf) end end \ No newline at end of file From 9db35904f7c4f0d963c6cbf879337540e108e7b8 Mon Sep 17 00:00:00 2001 From: jph6366 Date: Fri, 10 Oct 2025 19:44:19 -0400 Subject: [PATCH 34/90] Just comments --- src/extra/gpkg/write.jl | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/src/extra/gpkg/write.jl b/src/extra/gpkg/write.jl index 7eaaec1..c689cf9 100644 --- a/src/extra/gpkg/write.jl +++ b/src/extra/gpkg/write.jl @@ -58,13 +58,12 @@ function creategpkgtables(db, table, domain, crs, geom) # A feature table SHALL have a primary key column of type INTEGER and that column SHALL act as a rowid alias. DBInterface.execute(db, "CREATE TABLE features ( id INTEGER PRIMARY KEY AUTOINCREMENT, $(join(columns, ',')));") # The use of the AUTOINCREMENT keyword is optional but recommended. - # Implementers MAY omit the AUTOINCREMENT keyword for performance reasons, with the understanding that doing so has the potential to allow primary key identifiers to be reused. params = chop(repeat("?,", length(sch.names))) columns = join(SQLite.esc_id.(string.(sch.names)), ",") stmt = SQLite.Stmt(db, "INSERT INTO features ($columns) VALUES ($params)";) - handle = SQLite._get_stmt_handle(stmt) - SQLite.transaction(db) do + handle = SQLite._get_stmt_handle(stmt) # # used for holding references to bound statement values via bind! + SQLite.transaction(db) do # default mode = "DEFERRED" row = nothing if row === nothing state = iterate(rows) @@ -75,7 +74,9 @@ function creategpkgtables(db, table, domain, crs, geom) Tables.eachcolumn(sch, row) do val, col, _ SQLite.bind!(stmt, col, val) end - r = GC.@preserve row SQLite.C.sqlite3_step(handle) + r = GC.@preserve row SQLite.C.sqlite3_step(handle) # Evaluate An SQL Statement + # the return value will be either SQLITE_BUSY, SQLITE_DONE, SQLITE_ROW, SQLITE_ERROR, or SQLITE_MISUSE. + # To ensure that a statement has "finished" is to invoke sqlite3_reset() or sqlite3_finalize(). if r == SQLite.C.SQLITE_DONE SQLite.C.sqlite3_reset(handle) elseif r != SQLite.C.SQLITE_ROW @@ -122,9 +123,9 @@ function creategpkgtables(db, table, domain, crs, geom) ('WGS 84 geodectic', 4326, 'EPSG', 4326, 'GEOGCRS["WGS 84",DATUM["World Geodetic System 1984",ELLIPSOID["WGS 84",6378137,298.257223563,LENGTHUNIT["metre",1]]],PRIMEM["Greenwich",0,ANGLEUNIT["degree",0.0174532925199433]],CS[ellipsoidal,2],AXIS["geodetic latitude (Lat)",north,ORDER[1],ANGLEUNIT["degree",0.0174532925199433]],AXIS["geodetic longitude (Lon)",east,ORDER[2],ANGLEUNIT["degree",0.0174532925199433]],ID["EPSG",4326]]', 'longitude/latitude coordinates in decimal degrees on the WGS 84 spheroid', 'GEOGCRS["WGS 84",DATUM["World Geodetic System 1984",ELLIPSOID["WGS 84",6378137,298.257223563,LENGTHUNIT["metre",1]]],PRIMEM["Greenwich",0,ANGLEUNIT["degree",0.0174532925199433]],CS[ellipsoidal,2],AXIS["geodetic latitude (Lat)",north,ORDER[1],ANGLEUNIT["degree",0.0174532925199433]],AXIS["geodetic longitude (Lon)",east,ORDER[2],ANGLEUNIT["degree",0.0174532925199433]],ID["EPSG",4326]]'); """ ) - + # Insert non-existing CRS record into gpkg_spatial_ref_sys table. if srid != 4326 && srid > 0 - DBInterface.execute( # Insert non-existing CRS record into gpkg_spatial_ref_sys table. srs_id referenced by gpkg_contents, gpkg_geometry_columns + DBInterface.execute( db, """ INSERT INTO gpkg_spatial_ref_sys @@ -195,6 +196,8 @@ function creategpkgtables(db, table, domain, crs, geom) ["features", "geom", "GEOMETRY", srid, z, 0] ) + # https://www.geopackage.org/spec/#r77 + # Extended GeoPackage requires spatial indexes on feature table geometry columns using the SQLite Virtual Table R-trees DBInterface.execute( db, """ @@ -210,11 +213,15 @@ function creategpkgtables(db, table, domain, crs, geom) return (minx, maxx, miny, maxy) end + # The R-tree Spatial Indexes extension provides a means to encode an R-tree index for geometry values + # And provides a significant performance advantage for searches with basic envelope spatial criteria + # that return subsets of the rows in a feature table with a non-trivial number (thousands or more) of rows. + # The index data structure needs to be manually populated, updated and queried. stmt = SQLite.Stmt(db, "INSERT INTO rtree_features_geom VALUES (?, ?, ?, ?, ?)") handle = SQLite._get_stmt_handle(stmt) for i in 1:length(gpkgbinary) - fx_minx, fx_maxx, fx_miny, fx_maxy = bboxes[i] - SQLite.bind!(stmt, 1, i) + fx_minx, fx_maxx, fx_miny, fx_maxy = bboxes[i] # the min/max x/y parameters are min- and max-value pairs (stored as 32-bit floating point numbers) + SQLite.bind!(stmt, 1, i) # The R-tree function id parameter becomes the virtual table 64-bit signed integer primary key id column SQLite.bind!(stmt, 2, fx_minx) SQLite.bind!(stmt, 3, fx_maxx) SQLite.bind!(stmt, 4, fx_miny) From 9d6574f471a4f11272a32692ca06dd9260509b9f Mon Sep 17 00:00:00 2001 From: jph6366 Date: Thu, 16 Oct 2025 17:59:15 -0400 Subject: [PATCH 35/90] adding suggest changes and slightly refactored gpkgvalues --- src/extra/gpkg.jl | 3 ++ src/extra/gpkg/read.jl | 87 +++++++++++++++++++++++++----------------- src/extra/gpkg/wkb.jl | 48 +++++++++++++---------- 3 files changed, 81 insertions(+), 57 deletions(-) diff --git a/src/extra/gpkg.jl b/src/extra/gpkg.jl index 940a113..8b8fbcd 100644 --- a/src/extra/gpkg.jl +++ b/src/extra/gpkg.jl @@ -8,6 +8,9 @@ const GPKG_APPLICATION_ID = Int(0x47504B47) const GPKG_1_4_VERSION = 10400 +# If the geometry type_name value is "GEOMETRY" +# then the feature table geometry column MAY contain: +# geometries of any allowed geometry type. function SQLite.sqlitetype_(::Type{Vector{UInt8}}) return "GEOMETRY" end diff --git a/src/extra/gpkg/read.jl b/src/extra/gpkg/read.jl index a9524c7..65aa827 100644 --- a/src/extra/gpkg/read.jl +++ b/src/extra/gpkg/read.jl @@ -5,16 +5,27 @@ function gpkgread(fname; layer=1) db = SQLite.DB(fname) assertgpkg(db) - geom = gpkggeoms(db, ; layer) - attrs = gpkgvalues(db, ; layer) + geoms = gpkggeoms(db; layer) + table = gpkgvalues(db; layer) DBInterface.close!(db) - if eltype(attrs) <: Nothing - return georef(nothing, geom) + if eltype(table) <: Nothing + return georef(nothing, geoms) + else + georef(table, geoms) end - georef(attrs, geom) end function assertgpkg(db) + + # Requirement 6: PRAGMA integrity_check returns a single row with the value 'ok' + # Requirement 7: PRAGMA foreign_key_check (w/ no parameter value) returns an empty result set + if first(DBInterface.execute(db, "PRAGMA integrity_check;")).integrity_check != "ok" || + !(isempty(DBInterface.execute(db, "PRAGMA foreign_key_check;"))) + throw(ErrorException("database integrity at risk or foreign key violation(s)")) + end + + # Requirement 10: must include a gpkg_spatial_ref_sys table + # Requirement 13: must include a gpkg_contents table tbcount = first(DBInterface.execute( db, """ @@ -23,22 +34,13 @@ function assertgpkg(db) type IN ('table', 'view'); """ )) - # Requirement 10: must include a gpkg_spatial_ref_sys table - # Requirement 13: must include a gpkg_contents table if tbcount.n != 2 throw(ErrorException("missing required metadata tables in the GeoPackage SQL database")) end - - # Requirement 6: PRAGMA integrity_check returns a single row with the value 'ok' - # Requirement 7: PRAGMA foreign_key_check (w/ no parameter value) returns an empty result set - if first(DBInterface.execute(db, "PRAGMA integrity_check;")).integrity_check != "ok" || - !(isempty(DBInterface.execute(db, "PRAGMA foreign_key_check;"))) - throw(ErrorException("database integrity at risk or foreign key violation(s)")) - end end function gpkgvalues(db, ; layer=1) - feature_tables = DBInterface.execute( + table = DBInterface.execute( db, """ SELECT c.table_name, c.identifier, @@ -53,22 +55,31 @@ lower(c.table_name) AND type IN ('table', 'view')) AS object_type """ ) fields = nothing - tb = map(feature_tables) do query + table = map(table) do query tn = query.table_name - cn = query.column_name - ft_attrs = SQLite.tableinfo(db, tn).name - deleteat!(ft_attrs, findall(x -> isequal(x, cn), ft_attrs)) - rp_attrs = join(ft_attrs, ", ") - # keep the shortest set of attributes to avoid KeyError {Key} not found - if isnothing(fields) || length(rp_attrs) < length(fields) - fields = rp_attrs + tableinfo = SQLite.tableinfo(db, tn).name + # returns NamedTuple of AbstractVectors, also known as a "column table" + + # remove `column_name` field from tableinfo in-place + # to avoid querying the geometry column that stores feature geometry + deleteat!(tableinfo, findall(x -> isequal(x, query.column_name), tableinfo)) + columns = join(tableinfo, ", ") + + # keep the shortest set of fields if there is more than one feature table + if isnothing(fields) || length(columns) < length(fields) + fields = columns end - rowvals = iszero(length(fields)) ? nothing : map(DBInterface.execute(db, "SELECT $fields from $tn")) do rv + + # if there are no fields in column table then return nothing to table + if iszero(length(fields)) + return nothing + end + rowvals = map(DBInterface.execute(db, "SELECT $fields from $tn")) do rv NamedTuple(rv) end rowvals end - vcat(tb...) + vcat(table...) end # https://www.geopackage.org/spec/#:~:text=2.1.5.1.2.%20Table%20Data%20Values @@ -120,14 +131,16 @@ AND g.m IN (0, 1, 2) LIMIT $layer; """ ) - meshes = map((row.tn, row.cn, row.org, row.org_coordsys_id) for row in tb) do (tn, cn, org, org_coordsys_id) + featuretablegeoms = map((row.tn, row.cn, row.org, row.org_coordsys_id) for row in tb) do (tn, cn, org, orgcoordsysid) + # get feature geometry from geometry column in feature table gpkgbinary = DBInterface.execute(db, "SELECT $cn FROM $tn;") headerlen = 0 - named_rows = map(NamedTuple, gpkgbinary) - gpkgblobs = filter(named_rows) do row + + gpkgblobs = filter(map(NamedTuple, gpkgbinary)) do row !ismissing(getfield(row, Symbol(cn))) # ignore all rows with missing geometries end - submeshes = map(gpkgblobs) do blob + + geomcollection = map(gpkgblobs) do blob if blob[1][1:2] != UInt8[0x47, 0x50] @warn "Missing magic 'GP' string in GPkgBinaryGeometry" end @@ -166,24 +179,26 @@ AND g.m IN (0, 1, 2) else wkbtype = wkbGeometryType(wkbtypebits) end + if iszero(srsid) crs = LatLon{WGS84Latest} elseif !isone(abs(srsid)) if org == "EPSG" - crs = CoordRefSystems.get(EPSG{org_coordsys_id}) + crs = CoordRefSystems.get(EPSG{orgcoordsysid}) elseif org == "ESRI" - crs = CoordRefSystems.get(ERSI{org_coordsys_id}) + crs = CoordRefSystems.get(ERSI{orgcoordsysid}) end else crs = Cartesian{NoDatum} end - mesh = meshfromwkb(io, crs, wkbtype, zextent, wkbbyteswap) - if !isnothing(mesh) - mesh + + geom = gpkgwkbgeom(io, crs, wkbtype, zextent, wkbbyteswap) + if !isnothing(geom) + geom end end - submeshes + geomcollection end # efficient method for concatenating arrays of arrays - reduce(vcat, meshes) # Future versions of Julia might change the reduce algorithm + reduce(vcat, featuretablegeoms) # Future versions of Julia might change the reduce algorithm end \ No newline at end of file diff --git a/src/extra/gpkg/wkb.jl b/src/extra/gpkg/wkb.jl index e059720..e536afb 100644 --- a/src/extra/gpkg/wkb.jl +++ b/src/extra/gpkg/wkb.jl @@ -7,7 +7,7 @@ Flavors of WKB supported: -0. Standard WKB supports two-dimensional geometry, and is a proper subset of both extended WKB and ISO WKB. +- Standard WKB supports two-dimensional geometry, and is a proper subset of both extended WKB and ISO WKB. ## Reading WKB Geometry BLOB @@ -25,17 +25,19 @@ Flavors of WKB supported: end # load in WKBGeometry that contain geometry values # w/ double precision numbers in the coordinates - # that are also subject to byte order rules - wkbGeometryBlob = read(io, Vector{UInt8}) + # that are also subject to byte order rules. + wkbEndianness = isone(wkbByteOrder) ? ltoh : ntoh + # Note that Julia does not convert the endianness for you. + wkbGeometryBlob = wkbEndianness(read(io, Vector{UInt8})) ``` -1. Extended WKB allows applications to optionally add extra dimensions, and optionally embed an SRID +- Extended WKB allows applications to optionally add extra dimensions, and optionally embed an SRID 99-402 was a short-lived extension to SFSQL 1.1 that used a high-bit flag to indicate the presence of Z coordinates in a WKB geometry. When the optional wkbSRID is added to the wkbType, an SRID number is inserted after the wkbType number. ⚠ This optional behaviour is not supported and will likely fail loading this variant -2. ISO WKB allows for higher dimensional geometries. +- ISO WKB allows for higher dimensional geometries. SQL/MM Part 3 and SFSQL 1.2 use offsets of 1000 (Z), 2000 (M) and 3000 (ZM) ⚠ only offsets of 1000 are recognized and supported and will likely fali loading this variant @@ -52,14 +54,14 @@ GeoIO GeoPackage writer supports X,Y,Z coordinate offset of 1000 (Z) for wkbGeom ## Example ``` julia - meshes = [] + geoms = [] io = IOBuffer for row in wkbGeometryColumn wkbbyteswap = isone(read(io, UInt8)) ? ltoh : ntoh wkbtype = wkbGeometryType(read(io, UInt32)) crs = LatLon{WGS84Latest} haszextent = false - push!(meshes, meshfromwkb(io, crs, wkbtype, haszextent, wkbbyteswap)) + push!(geoms, gpkgwkbgeom(io, crs, wkbtype, haszextent, wkbbyteswap)) end ``` @@ -80,17 +82,17 @@ end # Requirement 20: GeoPackage SHALL store feature table geometries # with the basic simple feature geometry types # https://www.geopackage.org/spec140/index.html#geometry_types -function meshfromwkb(io, crs, wkbtype, zextent, bswap) +function gpkgwkbgeom(io, crs, wkbtype, zextent, bswap) if UInt32(wkbtype) > 3 - elems = wkbmultigeometry(io, crs, zextent, bswap) + elems = wkbmulti(io, crs, zextent, bswap) Multi(elems) else - elem = meshfromsf(io, crs, wkbtype, zextent, bswap) + elem = wkbsimple(io, crs, wkbtype, zextent, bswap) elem end end -function meshfromsf(io, crs, wkbtype, zextent, bswap) +function wkbsimple(io, crs, wkbtype, zextent, bswap) if isequal(wkbtype, wkbPoint) elem = wkbcoordinate(io, zextent, bswap) Point(crs(elem...)) @@ -146,7 +148,7 @@ function wkbpolygon(io, z, bswap) rings end -function wkbmultigeometry(io, crs, z, bswap) +function wkbmulti(io, crs, z, bswap) ngeoms = bswap(read(io, UInt32)) geomcollection = map(1:ngeoms) do _ @@ -161,7 +163,7 @@ function wkbmultigeometry(io, crs, z, bswap) else wkbtype = wkbGeometryType(wkbtypebits) end - meshfromsf(io, crs, wkbtype, z, bswap) + wkbsimple(io, crs, wkbtype, z, bswap) end geomcollection end @@ -188,15 +190,19 @@ function writewkbgeom(io, geom) _wkbgeom(io, wkbtype, geom) end -function writewkbsf(io, wkbtype, geom) +#------- +# WKB GEOMETRY WRITER UTILS +#------- + +function _writewkbsimple(io, wkbtype, geom) if isequal(wkbtype, wkbPolygon) _wkbpolygon(io, [boundary(geom::PolyArea)]) elseif isequal(wkbtype, wkbLineString) coordlist = vertices(geom) if typeof(geom) <: Ring - return _wkblinearring(io, coordlist) + return _wkbchainring(io, coordlist) end - _wkblinestring(io, coordlist) + _wkbchainrope(io, coordlist) elseif isequal(wkbtype, wkbPoint) coordinates = CoordRefSystems.raw(coords(geom)) _wkbcoordinates(io, coordinates) @@ -209,7 +215,7 @@ function _wkbgeom(io, wkbtype, geom) if UInt32(wkbtype) > 3 _wkbmulti(io, wkbtype, geom) else - writewkbsf(io, wkbtype, geom) + _writewkbsimple(io, wkbtype, geom) end end @@ -221,7 +227,7 @@ function _wkbcoordinates(io, coords) end end -function _wkblinestring(io, coord_list) +function _wkbchainrope(io, coord_list) write(io, htol(UInt32(length(coord_list)))) for n_coords in coord_list coordinates = CoordRefSystems.raw(coords(n_coords)) @@ -229,7 +235,7 @@ function _wkblinestring(io, coord_list) end end -function _wkblinearring(io, coord_list) +function _wkbchainring(io, coord_list) write(io, htol(UInt32(length(coord_list) + 1))) for n_coords in coord_list coordinates = CoordRefSystems.raw(coords(n_coords)) @@ -242,7 +248,7 @@ function _wkbpolygon(io, rings) write(io, htol(UInt32(length(rings)))) for ring in rings coord_list = vertices(ring) - _wkblinestring(io, coord_list) + _wkbchainring(io, coord_list) end end @@ -251,6 +257,6 @@ function _wkbmulti(io, multiwkbtype, geoms) for sf in parent(geoms) write(io, one(UInt8)) write(io, UInt32(multiwkbtype) - 3) - writewkbsf(io, wkbGeometryType(UInt32(multiwkbtype) - 3), sf) + _writewkbsimple(io, wkbGeometryType(UInt32(multiwkbtype) - 3), sf) end end \ No newline at end of file From 0a123600cc56c30cdb8c8a0fa849ab53397ca348 Mon Sep 17 00:00:00 2001 From: jph6366 Date: Wed, 22 Oct 2025 21:41:59 -0400 Subject: [PATCH 36/90] another round of resolves --- src/extra/gpkg/read.jl | 90 ++++++++++++++++++++--------------------- src/extra/gpkg/wkb.jl | 56 +++++++++---------------- src/extra/gpkg/write.jl | 2 +- 3 files changed, 65 insertions(+), 83 deletions(-) diff --git a/src/extra/gpkg/read.jl b/src/extra/gpkg/read.jl index 65aa827..c77c78d 100644 --- a/src/extra/gpkg/read.jl +++ b/src/extra/gpkg/read.jl @@ -6,13 +6,9 @@ function gpkgread(fname; layer=1) db = SQLite.DB(fname) assertgpkg(db) geoms = gpkggeoms(db; layer) - table = gpkgvalues(db; layer) + table = gpkgtable(db; layer) DBInterface.close!(db) - if eltype(table) <: Nothing - return georef(nothing, geoms) - else - georef(table, geoms) - end + georef(table, geoms) end function assertgpkg(db) @@ -26,20 +22,20 @@ function assertgpkg(db) # Requirement 10: must include a gpkg_spatial_ref_sys table # Requirement 13: must include a gpkg_contents table - tbcount = first(DBInterface.execute( + + if first(DBInterface.execute( db, """ SELECT COUNT(*) AS n FROM sqlite_master WHERE name IN ('gpkg_spatial_ref_sys', 'gpkg_contents') AND type IN ('table', 'view'); """ - )) - if tbcount.n != 2 + )).n != 2 throw(ErrorException("missing required metadata tables in the GeoPackage SQL database")) end end -function gpkgvalues(db, ; layer=1) +function gpkgtable(db, ; layer=1) table = DBInterface.execute( db, """ @@ -55,14 +51,14 @@ lower(c.table_name) AND type IN ('table', 'view')) AS object_type """ ) fields = nothing - table = map(table) do query - tn = query.table_name + table = map(table) do row + tn = row.table_name tableinfo = SQLite.tableinfo(db, tn).name # returns NamedTuple of AbstractVectors, also known as a "column table" # remove `column_name` field from tableinfo in-place # to avoid querying the geometry column that stores feature geometry - deleteat!(tableinfo, findall(x -> isequal(x, query.column_name), tableinfo)) + deleteat!(tableinfo, findall(x -> isequal(x, row.column_name), tableinfo)) columns = join(tableinfo, ", ") # keep the shortest set of fields if there is more than one feature table @@ -131,11 +127,23 @@ AND g.m IN (0, 1, 2) LIMIT $layer; """ ) - featuretablegeoms = map((row.tn, row.cn, row.org, row.org_coordsys_id) for row in tb) do (tn, cn, org, orgcoordsysid) + featuretablegeoms = map((row.tn, row.cn, row.crs, row.org, row.org_coordsys_id) for row in tb) do (tn, cn, srsid, org, orgcoordsysid) # get feature geometry from geometry column in feature table gpkgbinary = DBInterface.execute(db, "SELECT $cn FROM $tn;") headerlen = 0 + if iszero(srsid) + crs = LatLon{WGS84Latest} + elseif !isone(abs(srsid)) + if org == "EPSG" + crs = CoordRefSystems.get(EPSG{orgcoordsysid}) + elseif org == "ESRI" + crs = CoordRefSystems.get(ERSI{orgcoordsysid}) + end + else + crs = Cartesian{NoDatum} + end + gpkgblobs = filter(map(NamedTuple, gpkgbinary)) do row !ismissing(getfield(row, Symbol(cn))) # ignore all rows with missing geometries end @@ -147,49 +155,39 @@ AND g.m IN (0, 1, 2) io = IOBuffer(blob[1]) seek(io, 3) flag = read(io, UInt8) - # Note that Julia does not convert the endianness for you. - # Use ntoh or ltoh for this purpose. - bswap = isone(flag & 0x01) ? ltoh : ntoh - srsid = bswap(read(io, Int32)) + + # envelope contents indicator code (3-bit unsigned integer) envelope = (flag & (0x07 << 1)) >> 1 - envelopedims = 0 + envelopecode = 0 if !iszero(envelope) if isone(envelope) - envelopedims = 1 # 2D + envelopecode = 2 # 2D envelope [minx, maxx, miny, maxy], 32 bytes elseif isequal(2, envelope) - envelopedims = 2 # 2D+Z + envelopecode = 3 # 2D+Z envelope [minx, maxx, miny, maxy, minz, maxz], 48 bytes elseif isequal(3, envelope) - envelopedims = 3 # 2D+M is not supported + envelopecode = 4 # 2D+M envelope [minx, maxx, miny, maxy, minm, maxm] (is not supported) elseif isequal(4, envelope) - envelopedims = 4 # 2D+ZM is not supported - else - throw(ErrorException("exceeded dimensional limit for geometry, file may be corrupted or reader is broken")) + envelopecode = 5 # 2D+ZM envelope [minx, maxx, miny, maxy, minz, maxz, minm, maxm] (is not supported) + else # 5-7: invalid + throw(ErrorException("exceeded dimensional limit for geometry")) end end # else no envelope (space saving slower indexing option), 0 bytes - # header size in byte stream - headerlen = 8 + 8 * 4 * envelopedims - seek(io, headerlen) + headerlen = 8 + 8 * 2 * envelopecode # calculate header size in byte stream + seek(io, headerlen) # skip reading envelope bytes + # start reading Well-Known Binary geometry wkbbyteswap = isone(read(io, UInt8)) ? ltoh : ntoh - wkbtypebits = read(io, UInt32) - zextent = isequal(envelopedims, 2) - if zextent - wkbtype = - wkbtypebits & ewkbmaskbits ? wkbGeometryType(wkbtypebits & 0x000000F) : wkbGeometryType(wkbtypebits - 1000) - else - wkbtype = wkbGeometryType(wkbtypebits) - end + # Note that Julia does not convert the endianness for you. + # Use ntoh or ltoh for this purpose. - if iszero(srsid) - crs = LatLon{WGS84Latest} - elseif !isone(abs(srsid)) - if org == "EPSG" - crs = CoordRefSystems.get(EPSG{orgcoordsysid}) - elseif org == "ESRI" - crs = CoordRefSystems.get(ERSI{orgcoordsysid}) - end + wkbtypebits = read(io, UInt32) + zextent = isequal(envelopecode, 3) + if zextent # if the geometry type is a 3D geometry type + # if WKBGeometry is specified in `extended WKB` remove the the dimensionality bit flag that indicates a Z dimension + wkbtype = !iszero(wkbtypebits & ewkbmaskbits) ? wkbtypebits & 0x000000F : wkbtypebits - 1000 + # if WKBGeometry is specified in `ISO WKB` and we simply subtract the round number added to the type number that indicates a Z dimensions. else - crs = Cartesian{NoDatum} + wkbtype = wkbtypebits end geom = gpkgwkbgeom(io, crs, wkbtype, zextent, wkbbyteswap) @@ -201,4 +199,4 @@ AND g.m IN (0, 1, 2) end # efficient method for concatenating arrays of arrays reduce(vcat, featuretablegeoms) # Future versions of Julia might change the reduce algorithm -end \ No newline at end of file +end diff --git a/src/extra/gpkg/wkb.jl b/src/extra/gpkg/wkb.jl index e536afb..42fa0be 100644 --- a/src/extra/gpkg/wkb.jl +++ b/src/extra/gpkg/wkb.jl @@ -58,7 +58,7 @@ GeoIO GeoPackage writer supports X,Y,Z coordinate offset of 1000 (Z) for wkbGeom io = IOBuffer for row in wkbGeometryColumn wkbbyteswap = isone(read(io, UInt8)) ? ltoh : ntoh - wkbtype = wkbGeometryType(read(io, UInt32)) + wkbtype = read(io, UInt32) crs = LatLon{WGS84Latest} haszextent = false push!(geoms, gpkgwkbgeom(io, crs, wkbtype, haszextent, wkbbyteswap)) @@ -68,22 +68,11 @@ GeoIO GeoPackage writer supports X,Y,Z coordinate offset of 1000 (Z) for wkbGeom """ const ewkbmaskbits = 0x40000000 | 0x80000000 -@enum wkbGeometryType begin - wkbUnknown = 0 - wkbPoint = 1 - wkbLineString = 2 - wkbPolygon = 3 - wkbMultiPoint = 4 - wkbMultiLineString = 5 - wkbMultiPolygon = 6 - wkbGeometryCollection = 7 -end - # Requirement 20: GeoPackage SHALL store feature table geometries # with the basic simple feature geometry types # https://www.geopackage.org/spec140/index.html#geometry_types function gpkgwkbgeom(io, crs, wkbtype, zextent, bswap) - if UInt32(wkbtype) > 3 + if wkbtype > 3 elems = wkbmulti(io, crs, zextent, bswap) Multi(elems) else @@ -93,17 +82,17 @@ function gpkgwkbgeom(io, crs, wkbtype, zextent, bswap) end function wkbsimple(io, crs, wkbtype, zextent, bswap) - if isequal(wkbtype, wkbPoint) + if isequal(wkbtype, 1) elem = wkbcoordinate(io, zextent, bswap) Point(crs(elem...)) - elseif isequal(wkbtype, wkbLineString) + elseif isequal(wkbtype, 2) elem = wkblinestring(io, zextent, bswap) if length(elem) >= 2 && first(elem) != last(elem) Rope([Point(crs(coords...)) for coords in elem]...) else Ring([Point(crs(coords...)) for coords in elem[1:(end - 1)]]...) end - elseif isequal(wkbtype, wkbPolygon) + elseif isequal(wkbtype, 3) elem = wkbpolygon(io, zextent, bswap) rings = map(elem) do ring coords = map(ring) do point @@ -121,7 +110,6 @@ end function wkbcoordinate(io, z, bswap) x = bswap(read(io, Float64)) y = bswap(read(io, Float64)) - if z z = bswap(read(io, Float64)) return x, y, z @@ -152,34 +140,30 @@ function wkbmulti(io, crs, z, bswap) ngeoms = bswap(read(io, UInt32)) geomcollection = map(1:ngeoms) do _ - bswap = isone(read(io, UInt8)) ? ltoh : ntoh + wkbbswap = isone(read(io, UInt8)) ? ltoh : ntoh wkbtypebits = read(io, UInt32) if z if iszero(wkbtypebits & ewkbmaskbits) - wkbtype = wkbGeometryType(wkbtypebits) + wkbtypebits = wkbtypebits & 0x000000F # extended WKB else - wkbtype = wkbGeometryType(wkbtypebits - 1000) + wkbtypebits = wkbtypebits - 1000 # ISO WKB end - else - wkbtype = wkbGeometryType(wkbtypebits) end - wkbsimple(io, crs, wkbtype, z, bswap) + wkbsimple(io, crs, wkbtypebits, z, wkbbswap) end geomcollection end function _wkbtype(geometry) if geometry isa Point - return wkbPoint + return 1 # wkbPoint elseif geometry isa Rope || geometry isa Ring - return wkbLineString + return 2 # wkbLineString elseif geometry isa PolyArea - return wkbPolygon + return 3 # wkbPolygon elseif geometry isa Multi fg = first(parent(geometry)) - return wkbGeometryType(Int(_wkbtype(fg)) + 3) - else - return wkbGeometryCollection + return _wkbtype(fg) + 3 # wkbMulti end end @@ -195,24 +179,24 @@ end #------- function _writewkbsimple(io, wkbtype, geom) - if isequal(wkbtype, wkbPolygon) + if isequal(wkbtype, 3) # wkbPolygon _wkbpolygon(io, [boundary(geom::PolyArea)]) - elseif isequal(wkbtype, wkbLineString) + elseif isequal(wkbtype, 2) # wkbLineString coordlist = vertices(geom) if typeof(geom) <: Ring return _wkbchainring(io, coordlist) end _wkbchainrope(io, coordlist) - elseif isequal(wkbtype, wkbPoint) + elseif isequal(wkbtype, 1) # wkbPoint coordinates = CoordRefSystems.raw(coords(geom)) _wkbcoordinates(io, coordinates) else - throw(ErrorException("Well-Known Binary Geometry not supported: $wkbtype")) + throw(ErrorException("Well-Known Binary Geometry unknown: $wkbtype")) end end function _wkbgeom(io, wkbtype, geom) - if UInt32(wkbtype) > 3 + if wkbtype > 3 _wkbmulti(io, wkbtype, geom) else _writewkbsimple(io, wkbtype, geom) @@ -256,7 +240,7 @@ function _wkbmulti(io, multiwkbtype, geoms) write(io, htol(UInt32(length(parent(geoms))))) for sf in parent(geoms) write(io, one(UInt8)) - write(io, UInt32(multiwkbtype) - 3) - _writewkbsimple(io, wkbGeometryType(UInt32(multiwkbtype) - 3), sf) + write(io, multiwkbtype - 3) + _writewkbsimple(io, multiwkbtype - 3, sf) end end \ No newline at end of file diff --git a/src/extra/gpkg/write.jl b/src/extra/gpkg/write.jl index c689cf9..6a0da9b 100644 --- a/src/extra/gpkg/write.jl +++ b/src/extra/gpkg/write.jl @@ -125,7 +125,7 @@ function creategpkgtables(db, table, domain, crs, geom) ) # Insert non-existing CRS record into gpkg_spatial_ref_sys table. if srid != 4326 && srid > 0 - DBInterface.execute( + DBInterface.execute( db, """ INSERT INTO gpkg_spatial_ref_sys From 41e73d2205f5ab190b76cc04b879321ae8540994 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlio=20Hoffimann?= Date: Fri, 24 Oct 2025 09:09:27 -0300 Subject: [PATCH 37/90] Apply suggestion from @juliohm --- src/extra/gpkg/read.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/extra/gpkg/read.jl b/src/extra/gpkg/read.jl index c77c78d..7c2a63b 100644 --- a/src/extra/gpkg/read.jl +++ b/src/extra/gpkg/read.jl @@ -5,8 +5,8 @@ function gpkgread(fname; layer=1) db = SQLite.DB(fname) assertgpkg(db) - geoms = gpkggeoms(db; layer) table = gpkgtable(db; layer) + geoms = gpkggeoms(db; layer) DBInterface.close!(db) georef(table, geoms) end From 3e4b02e45b90a8ba8a281e140fc2734b1d1567bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlio=20Hoffimann?= Date: Fri, 24 Oct 2025 09:23:18 -0300 Subject: [PATCH 38/90] Apply suggestion from @juliohm --- Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index b28fbfe..95ffb23 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "GeoIO" uuid = "f5a160d5-e41d-4189-8b61-d57781c419e3" authors = ["Elias Carvalho and contributors"] -version = "1.21.1" +version = "1.21.2" [deps] ArchGDAL = "c9ce4bd3-c3d5-55b8-8973-c0e20141b8c3" From 8ad4c2a2259575434eab6d7a6a9b7906ffcbbc59 Mon Sep 17 00:00:00 2001 From: jph6366 Date: Fri, 24 Oct 2025 14:45:48 -0400 Subject: [PATCH 39/90] remove evil and purge smelly code --- src/extra/gpkg/read.jl | 63 ++++++++++++++++++----------------------- src/extra/gpkg/write.jl | 2 +- test/gisissues.jl | 4 +-- test/io/geopackage.jl | 6 ++-- test/novalues.jl | 5 ++-- 5 files changed, 34 insertions(+), 46 deletions(-) diff --git a/src/extra/gpkg/read.jl b/src/extra/gpkg/read.jl index c77c78d..512d532 100644 --- a/src/extra/gpkg/read.jl +++ b/src/extra/gpkg/read.jl @@ -50,32 +50,25 @@ lower(c.table_name) AND type IN ('table', 'view')) AS object_type c.data_type = 'features' LIMIT $layer """ ) - fields = nothing - table = map(table) do row + tb = map(table) do row tn = row.table_name tableinfo = SQLite.tableinfo(db, tn).name # returns NamedTuple of AbstractVectors, also known as a "column table" - - # remove `column_name` field from tableinfo in-place - # to avoid querying the geometry column that stores feature geometry + + # if there are no aspatial fields in column table then return nothing to table + if isone(length(tableinfo)) + return nothing + end + # remove `column_name` field from tableinfo to avoid querying the GeoPackage geometry column deleteat!(tableinfo, findall(x -> isequal(x, row.column_name), tableinfo)) columns = join(tableinfo, ", ") - # keep the shortest set of fields if there is more than one feature table - if isnothing(fields) || length(columns) < length(fields) - fields = columns - end - - # if there are no fields in column table then return nothing to table - if iszero(length(fields)) - return nothing - end - rowvals = map(DBInterface.execute(db, "SELECT $fields from $tn")) do rv + rowvals = map(DBInterface.execute(db, "SELECT $columns from $tn")) do rv NamedTuple(rv) end rowvals end - vcat(table...) + isnothing(first(tb)) ? first(tb) : vcat(tb...) end # https://www.geopackage.org/spec/#:~:text=2.1.5.1.2.%20Table%20Data%20Values @@ -123,32 +116,30 @@ AND (SELECT type FROM sqlite_master WHERE lower(name) = lower(c.table_name) AND AND g.srs_id = srs.srs_id AND g.srs_id = c.srs_id AND g.z IN (0, 1, 2) -AND g.m IN (0, 1, 2) +AND g.m = 0 LIMIT $layer; """ ) - featuretablegeoms = map((row.tn, row.cn, row.crs, row.org, row.org_coordsys_id) for row in tb) do (tn, cn, srsid, org, orgcoordsysid) + firstrow = [row for row in first(tb)] + # Note: first feature table that is read specifies the CRS to be used on all feature tables resulted from SELECT statement + + srsid, org, orgcoordsysid = firstrow[3], firstrow[5], firstrow[6] + crs = Cartesian{NoDatum} # an srs_id of -1 uses undefined Cartesian CRS + if iszero(srsid) # an srs_id of 0 uses undefined Geographic CRS + crs = LatLon{WGS84Latest} + elseif !isone(abs(srsid)) + if org == "EPSG" + crs = CoordRefSystems.get(EPSG{orgcoordsysid}) + elseif org == "ESRI" + crs = CoordRefSystems.get(ERSI{orgcoordsysid}) + end + end + + featuretablegeoms = map((row.tn, row.cn) for row in tb) do (tn, cn) # get feature geometry from geometry column in feature table gpkgbinary = DBInterface.execute(db, "SELECT $cn FROM $tn;") headerlen = 0 - - if iszero(srsid) - crs = LatLon{WGS84Latest} - elseif !isone(abs(srsid)) - if org == "EPSG" - crs = CoordRefSystems.get(EPSG{orgcoordsysid}) - elseif org == "ESRI" - crs = CoordRefSystems.get(ERSI{orgcoordsysid}) - end - else - crs = Cartesian{NoDatum} - end - - gpkgblobs = filter(map(NamedTuple, gpkgbinary)) do row - !ismissing(getfield(row, Symbol(cn))) # ignore all rows with missing geometries - end - - geomcollection = map(gpkgblobs) do blob + geomcollection = map(gpkgbinary) do blob if blob[1][1:2] != UInt8[0x47, 0x50] @warn "Missing magic 'GP' string in GPkgBinaryGeometry" end diff --git a/src/extra/gpkg/write.jl b/src/extra/gpkg/write.jl index 6a0da9b..f1f93ea 100644 --- a/src/extra/gpkg/write.jl +++ b/src/extra/gpkg/write.jl @@ -56,7 +56,7 @@ function creategpkgtables(db, table, domain, crs, geom) # https://www.geopackage.org/spec/#r29 # A feature table SHALL have a primary key column of type INTEGER and that column SHALL act as a rowid alias. - DBInterface.execute(db, "CREATE TABLE features ( id INTEGER PRIMARY KEY AUTOINCREMENT, $(join(columns, ',')));") + DBInterface.execute(db, "CREATE TABLE features ( $(join(columns, ',')));") # The use of the AUTOINCREMENT keyword is optional but recommended. params = chop(repeat("?,", length(sch.names))) diff --git a/test/gisissues.jl b/test/gisissues.jl index efd9067..b20c388 100644 --- a/test/gisissues.jl +++ b/test/gisissues.jl @@ -71,7 +71,6 @@ file = joinpath(savedir, "gis-points.gpkg") GeoIO.save(file, gtpoint) gtb = GeoIO.load(file) - @test Set([n for n in names(gtb) if n != "id"]) == Set(names(gtpoint)) @test gtb.geometry == gtpoint.geometry @test gtb.float == gtpoint.float @test gtb.int == gtpoint.int @@ -80,7 +79,6 @@ file = joinpath(savedir, "gis-rings.gpkg") GeoIO.save(file, gtring) gtb = GeoIO.load(file) - @test Set([n for n in names(gtb) if n != "id"]) == Set(names(gtring)) @test gtb.geometry == gtring.geometry @test gtb.float == gtring.float @test gtb.int == gtring.int @@ -89,7 +87,7 @@ file = joinpath(savedir, "gis-polys.gpkg") GeoIO.save(file, gtpoly) gtb = GeoIO.load(file) - @test Set([n for n in names(gtb) if n != "id"]) == Set(names(gtpoly)) + @test Set(names(gtb)) == Set(names(gtpoly)) @test gtb.geometry == gtpoly.geometry @test gtb.float == gtpoly.float @test gtb.int == gtpoly.int diff --git a/test/io/geopackage.jl b/test/io/geopackage.jl index c6ff8d0..9fe5f6d 100644 --- a/test/io/geopackage.jl +++ b/test/io/geopackage.jl @@ -38,7 +38,7 @@ gtb1 = GeoIO.load(file1) GeoIO.save(file2, gtb1) gtb2 = GeoIO.load(file2) - @test Set([n for n in names(gtb2) if n != "id"]) == Set(names(gtb1)) + @test Set(names(gtb2)) == Set(names(gtb1)) @test gtb2.geometry == gtb1.geometry @test gtb2.code == gtb1.code @test gtb2.name == gtb1.name @@ -49,7 +49,7 @@ gtb1 = GeoIO.load(file1) GeoIO.save(file2, gtb1) gtb2 = GeoIO.load(file2) - @test Set([n for n in names(gtb2) if n != "id"]) == Set(names(gtb1)) + @test Set(names(gtb2)) == Set(names(gtb1)) @test gtb2.geometry == gtb1.geometry @test gtb2.code == gtb1.code @test gtb2.name == gtb1.name @@ -60,7 +60,7 @@ gtb1 = GeoIO.load(file1) GeoIO.save(file2, gtb1) gtb2 = GeoIO.load(file2) - @test Set([n for n in names(gtb2) if n != "id"]) == Set(names(gtb1)) + @test Set(names(gtb2)) == Set(names(gtb1)) @test gtb2.geometry == gtb1.geometry @test gtb2.code == gtb1.code @test gtb2.name == gtb1.name diff --git a/test/novalues.jl b/test/novalues.jl index 3f038ad..b900bc8 100644 --- a/test/novalues.jl +++ b/test/novalues.jl @@ -23,9 +23,8 @@ file = joinpath(savedir, "noattribs.gpkg") GeoIO.save(file, gtb1) gtb2 = GeoIO.load(file) - @test isequal((id = [1, 2, 3],), values(gtb2)) - gtb1o = georef((id = [1, 2, 3],), pset) - @test gtb2 == gtb1o + @test isnothing(values(gtb2)) + @test gtb2 == gtb1 # CSV pset = [Point(0.0, 0.0), Point(1.0, 0.0), Point(0.0, 1.0)] From c0c06a4b0eb1eab1c712c67af9d1a8759fd43aa3 Mon Sep 17 00:00:00 2001 From: jph6366 Date: Fri, 24 Oct 2025 14:49:00 -0400 Subject: [PATCH 40/90] revert test changes --- test/gisissues.jl | 2 ++ test/io/geopackage.jl | 1 - 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/test/gisissues.jl b/test/gisissues.jl index b20c388..b261072 100644 --- a/test/gisissues.jl +++ b/test/gisissues.jl @@ -71,6 +71,7 @@ file = joinpath(savedir, "gis-points.gpkg") GeoIO.save(file, gtpoint) gtb = GeoIO.load(file) + @test Set(names(gtb)) == Set(names(gtpoint)) @test gtb.geometry == gtpoint.geometry @test gtb.float == gtpoint.float @test gtb.int == gtpoint.int @@ -79,6 +80,7 @@ file = joinpath(savedir, "gis-rings.gpkg") GeoIO.save(file, gtring) gtb = GeoIO.load(file) + @test Set(names(gtb)) == Set(names(gtring)) @test gtb.geometry == gtring.geometry @test gtb.float == gtring.float @test gtb.int == gtring.int diff --git a/test/io/geopackage.jl b/test/io/geopackage.jl index 9fe5f6d..5e82834 100644 --- a/test/io/geopackage.jl +++ b/test/io/geopackage.jl @@ -32,7 +32,6 @@ @testset "save" begin # note: GeoPackage does not preserve column order - # note2: Every such feature table SHALL have a primary key column 'id' of type INTEGER file1 = joinpath(datadir, "points.gpkg") file2 = joinpath(savedir, "points.gpkg") gtb1 = GeoIO.load(file1) From ff406c06e092b8fe294929440cf0a9d97678cbb9 Mon Sep 17 00:00:00 2001 From: jph6366 Date: Wed, 29 Oct 2025 17:11:35 -0400 Subject: [PATCH 41/90] gpkgread refactor; more comments --- src/extra/gpkg/read.jl | 195 +++++++++++++++++++++++----------------- src/extra/gpkg/write.jl | 33 +++++-- 2 files changed, 137 insertions(+), 91 deletions(-) diff --git a/src/extra/gpkg/read.jl b/src/extra/gpkg/read.jl index 8bcb580..f612957 100644 --- a/src/extra/gpkg/read.jl +++ b/src/extra/gpkg/read.jl @@ -5,14 +5,12 @@ function gpkgread(fname; layer=1) db = SQLite.DB(fname) assertgpkg(db) - table = gpkgtable(db; layer) - geoms = gpkggeoms(db; layer) + table, geoms = gpkgtable(db; layer) DBInterface.close!(db) georef(table, geoms) end function assertgpkg(db) - # Requirement 6: PRAGMA integrity_check returns a single row with the value 'ok' # Requirement 7: PRAGMA foreign_key_check (w/ no parameter value) returns an empty result set if first(DBInterface.execute(db, "PRAGMA integrity_check;")).integrity_check != "ok" || @@ -22,7 +20,6 @@ function assertgpkg(db) # Requirement 10: must include a gpkg_spatial_ref_sys table # Requirement 13: must include a gpkg_contents table - if first(DBInterface.execute( db, """ @@ -35,47 +32,9 @@ function assertgpkg(db) end end -function gpkgtable(db, ; layer=1) - table = DBInterface.execute( - db, - """ -SELECT c.table_name, c.identifier, -g.column_name, g.geometry_type_name, g.z, g.m, c.min_x, c.min_y, -c.max_x, c.max_y, -(SELECT type FROM sqlite_master WHERE lower(name) = -lower(c.table_name) AND type IN ('table', 'view')) AS object_type - FROM gpkg_geometry_columns g - JOIN gpkg_contents c ON (g.table_name = c.table_name) - WHERE - c.data_type = 'features' LIMIT $layer - """ - ) - tb = map(table) do row - tn = row.table_name - tableinfo = SQLite.tableinfo(db, tn).name - # returns NamedTuple of AbstractVectors, also known as a "column table" - - # if there are no aspatial fields in column table then return nothing to table - if isone(length(tableinfo)) - return nothing - end - # remove `column_name` field from tableinfo to avoid querying the GeoPackage geometry column - deleteat!(tableinfo, findall(x -> isequal(x, row.column_name), tableinfo)) - columns = join(tableinfo, ", ") - - rowvals = map(DBInterface.execute(db, "SELECT $columns from $tn")) do rv - NamedTuple(rv) - end - rowvals - end - isnothing(first(tb)) ? first(tb) : vcat(tb...) -end - +# According to Geometry Columns Table Requirements # https://www.geopackage.org/spec/#:~:text=2.1.5.1.2.%20Table%20Data%20Values #------------------------------------------------------------------------------ -# # Requirement 21: a gpkg_contents table row with a "features" data_type -# SHALL contain a gpkg_geometry_columns table -# # Requirement 22: gpkg_geometry_columns table # SHALL contain one row record for the geometry column # in each vector feature data table @@ -103,91 +62,159 @@ end # Requirement 146: The srs_id value in a gpkg_geometry_columns table row # SHALL match the srs_id column value from the corresponding row in the # gpkg_contents table. -function gpkggeoms(db, ; layer=1) - tb = DBInterface.execute( +function gpkgtable(db, ; layer=1) + resultrows = DBInterface.execute( + # According to https://www.geopackage.org/spec/#r16 + # Values of the gpkg_contents table srs_id column + # SHALL reference values in the gpkg_spatial_ref_sys table srs_id column + # According to https://www.geopackage.org/spec/#r18 + # The gpkg_contents table SHALL contain a row + # with a lowercase data_type column value of "features" + # for each vector features user data table or view. db, """ -SELECT g.table_name AS tn, g.column_name AS cn, c.srs_id as crs, g.z as elev, srs.organization as org, srs.organization_coordsys_id as org_coordsys_id, -( SELECT type FROM sqlite_master WHERE lower(name) = lower(c.table_name) AND type IN ('table', 'view')) AS object_type -FROM gpkg_geometry_columns g, gpkg_spatial_ref_sys srs -JOIN gpkg_contents c ON ( g.table_name = c.table_name ) -WHERE c.data_type = 'features' -AND (SELECT type FROM sqlite_master WHERE lower(name) = lower(c.table_name) AND type IN ('table', 'view')) IS NOT NULL -AND g.srs_id = srs.srs_id -AND g.srs_id = c.srs_id -AND g.z IN (0, 1, 2) -AND g.m = 0 - LIMIT $layer; + SELECT g.table_name AS tablename, g.column_name AS columnname, + c.srs_id AS srsid, g.z, srs.organization AS org, srs.organization_coordsys_id AS orgcoordsysid, + ( SELECT type FROM sqlite_master WHERE lower(name) = lower(c.table_name) AND type IN ('table', 'view')) AS object_type + FROM gpkg_geometry_columns g, gpkg_spatial_ref_sys srs + JOIN gpkg_contents c ON ( g.table_name = c.table_name ) + WHERE c.data_type = 'features' + AND object_type IS NOT NULL + AND g.srs_id = srs.srs_id + AND g.srs_id = c.srs_id + AND g.z IN (0, 1, 2) + AND g.m = 0 + LIMIT $layer; """ ) - firstrow = [row for row in first(tb)] + # Note: first feature table that is read specifies the CRS to be used on all feature tables resulted from SELECT statement + firstrow = first(resultrows) - srsid, org, orgcoordsysid = firstrow[3], firstrow[5], firstrow[6] - crs = Cartesian{NoDatum} # an srs_id of -1 uses undefined Cartesian CRS - if iszero(srsid) # an srs_id of 0 uses undefined Geographic CRS + # According to https://www.geopackage.org/spec/#r33, feature table geometry columns + # SHALL contain geometries with the srs_id specified + # for the column by the gpkg_geometry_columns table srs_id column value. + srsid, org, orgcoordsysid = firstrow.srsid, firstrow.org, firstrow.orgcoordsysid + # defaults to undefined Cartesian CRS + crs = Cartesian{NoDatum} + # an srs_id of 0 uses undefined Geographic CRS + if iszero(srsid) crs = LatLon{WGS84Latest} - elseif !isone(abs(srsid)) + # if srs_id not equal to -1 + elseif !iszero(srsid + 1) + # CRS assigned by the org if org == "EPSG" + # orgcoordsysid is the numeric id of the CRS assigned by the org crs = CoordRefSystems.get(EPSG{orgcoordsysid}) elseif org == "ESRI" crs = CoordRefSystems.get(ERSI{orgcoordsysid}) end end - featuretablegeoms = map((row.tn, row.cn) for row in tb) do (tn, cn) - # get feature geometry from geometry column in feature table - gpkgbinary = DBInterface.execute(db, "SELECT $cn FROM $tn;") + results = map((row.tablename, row.columnname) for row in resultrows) do (tablename, columnname) + # According to https://www.geopackage.org/spec/#r14 + # The table_name column value in a gpkg_contents table row + # SHALL contain the name of a SQLite table or view. + gpkgbinary = DBInterface.execute(db, "SELECT * FROM $tablename;") + # length of the GeoPackageBinaryHeader headerlen = 0 - geomcollection = map(gpkgbinary) do blob - if blob[1][1:2] != UInt8[0x47, 0x50] + geomcollection = map(gpkgbinary) do row + # According to https://www.geopackage.org/spec/#r30 + # A feature table or view SHALL have only one geometry column. + columnindex = findfirst(==(Symbol(columnname)), keys(row)) + rowvals = map(keys(row)[[begin:(columnindex - 1); (columnindex + 1):end]]) do key + key, getproperty(row, key) + end + # get the column of SQL Geometry Binary specified by gpkg_geometry_columns table in column_name field + blob = getproperty(row, Symbol(columnname)) + + # According to https://www.geopackage.org/spec/#r19 + # A GeoPackage SHALL store feature table geometries in SQL BLOBs using the Standard GeoPackageBinary format + # check the GeoPackageBinaryHeader for the first byte[2] to be 'GP' in ASCII + if blob[1:2] != UInt8[0x47, 0x50] @warn "Missing magic 'GP' string in GPkgBinaryGeometry" end - io = IOBuffer(blob[1]) + + # create in-memory I/O stream of GeoPackage SQL Geometry Binary Format starting after byte[2] magic = 0x4750; + io = IOBuffer(blob) + # skip the magic string in header seek(io, 3) + + # bit layout of GeoPackageBinary flags byte + # --------------------------------------- + # bit # 7 # 6 # 5 # 4 # 3 # 2 # 1 # 0 # + # use # R # R # X # Y # E # E # E # B # + # --------------------------------------- + # R: reserved for future use; set to 0 + # X: GeoPackageBinary type + # Y: empty geometry flag + # E: envelope contents indicator code (3-bit unsigned integer) + # B: byte order for SRS_ID and envelope values in header flag = read(io, UInt8) - # envelope contents indicator code (3-bit unsigned integer) + # 0x07 is a 3-bit mask 0x00001110 + # left-shift moves the 3-bit mask by one to align with E bits in flag layout + # bitwise AND operation isolates the E bits + # right-shift moves the E bits by one to align with the least significant bits + # results in a 3-bit unsigned integer envelope = (flag & (0x07 << 1)) >> 1 + envelopecode = 0 if !iszero(envelope) if isone(envelope) - envelopecode = 2 # 2D envelope [minx, maxx, miny, maxy], 32 bytes + # 2D envelope [minx, maxx, miny, maxy], 32 bytes + envelopecode = 2 elseif isequal(2, envelope) - envelopecode = 3 # 2D+Z envelope [minx, maxx, miny, maxy, minz, maxz], 48 bytes + # 2D+Z envelope [minx, maxx, miny, maxy, minz, maxz], 48 bytes + envelopecode = 3 elseif isequal(3, envelope) - envelopecode = 4 # 2D+M envelope [minx, maxx, miny, maxy, minm, maxm] (is not supported) + # 2D+M envelope [minx, maxx, miny, maxy, minm, maxm] (is not supported) + envelopecode = 4 elseif isequal(4, envelope) - envelopecode = 5 # 2D+ZM envelope [minx, maxx, miny, maxy, minz, maxz, minm, maxm] (is not supported) - else # 5-7: invalid + # 2D+ZM envelope [minx, maxx, miny, maxy, minz, maxz, minm, maxm] (is not supported) + envelopecode = 5 + else + # 5-7: invalid throw(ErrorException("exceeded dimensional limit for geometry")) end - end # else no envelope (space saving slower indexing option), 0 bytes + # else no envelope (space saving slower indexing option), 0 bytes + end + + # calculate GeoPackageBinaryHeader size in byte stream given extent of envelope: + # byte[2] magic + byte[1] version + byte[1] flags + byte[4] srs_id + byte[16*E] envelope + # where E is the code that indicates how many pairs of floating-point numbers exist in envelope + headerlen = 8 + 8 * 2 * envelopecode + + # Skip reading the double[] envelope and start reading Well-Known Binary geometry + seek(io, headerlen) - headerlen = 8 + 8 * 2 * envelopecode # calculate header size in byte stream - seek(io, headerlen) # skip reading envelope bytes - # start reading Well-Known Binary geometry - wkbbyteswap = isone(read(io, UInt8)) ? ltoh : ntoh # Note that Julia does not convert the endianness for you. # Use ntoh or ltoh for this purpose. + wkbbyteswap = isone(read(io, UInt8)) ? ltoh : ntoh wkbtypebits = read(io, UInt32) zextent = isequal(envelopecode, 3) - if zextent # if the geometry type is a 3D geometry type - # if WKBGeometry is specified in `extended WKB` remove the the dimensionality bit flag that indicates a Z dimension - wkbtype = !iszero(wkbtypebits & ewkbmaskbits) ? wkbtypebits & 0x000000F : wkbtypebits - 1000 - # if WKBGeometry is specified in `ISO WKB` and we simply subtract the round number added to the type number that indicates a Z dimensions. + # if the geometry type is a 3D geometry type + if zextent + wkbtype = !iszero(wkbtypebits & ewkbmaskbits) ? + # if WKBGeometry is specified in `extended WKB` remove the the dimensionality bit flag that indicates a Z dimension + wkbtypebits & 0x000000F : + # if WKBGeometry is specified in `ISO WKB` and we simply subtract the round number added to the type number that indicates a Z dimensions. + wkbtypebits - 1000 else + # WKBGeometry is specified in 'Standard WKB' wkbtype = wkbtypebits end geom = gpkgwkbgeom(io, crs, wkbtype, zextent, wkbbyteswap) if !isnothing(geom) - geom + return (NamedTuple(rowvals), geom) end end geomcollection end # efficient method for concatenating arrays of arrays - reduce(vcat, featuretablegeoms) # Future versions of Julia might change the reduce algorithm + table = reduce(vcat, results) + # unpack results to return results + getindex.(table, 1), getindex.(table, 2) end diff --git a/src/extra/gpkg/write.jl b/src/extra/gpkg/write.jl index f1f93ea..d864248 100644 --- a/src/extra/gpkg/write.jl +++ b/src/extra/gpkg/write.jl @@ -16,9 +16,12 @@ function gpkgwrite(fname, geotable;) domain = GeoTables.domain(geotable) crs = GeoTables.crs(domain) geom = collect(domain) + DBInterface.execute(db, "PRAGMA application_id = $GPKG_APPLICATION_ID ") DBInterface.execute(db, "PRAGMA user_version = $GPKG_1_4_VERSION ") + creategpkgtables(db, table, domain, crs, geom) + DBInterface.execute(db, "PRAGMA optimize;") # https://sqlite.org/pragma.html#pragma_optimize # Applications with short-lived database connections should run "PRAGMA optimize;" @@ -44,10 +47,13 @@ function creategpkgtables(db, table, domain, crs, geom) vcat(gpkgbinheader, take!(io)) end - table = + features = + # if no values in table then store only geometry in features isnothing(table) ? [(; geom=g,) for (_, g) in zip(1:length(gpkgbinary), gpkgbinary)] : + # else store the geometry as the first column and the remaining table columns in features [(; geom=g, t...) for (t, g) in zip(Tables.rowtable(table), gpkgbinary)] - rows = Tables.rows(table) + + rows = Tables.rows(features) sch = Tables.schema(rows) columns = [ string(SQLite.esc_id(String(sch.names[i])), ' ', SQLite.sqlitetype(sch.types !== nothing ? sch.types[i] : Any)) @@ -62,8 +68,9 @@ function creategpkgtables(db, table, domain, crs, geom) params = chop(repeat("?,", length(sch.names))) columns = join(SQLite.esc_id.(string.(sch.names)), ",") stmt = SQLite.Stmt(db, "INSERT INTO features ($columns) VALUES ($params)";) - handle = SQLite._get_stmt_handle(stmt) # # used for holding references to bound statement values via bind! - SQLite.transaction(db) do # default mode = "DEFERRED" + # used for holding references to bound statement values via bind! + handle = SQLite._get_stmt_handle(stmt) + SQLite.transaction(db) do row = nothing if row === nothing state = iterate(rows) @@ -74,9 +81,10 @@ function creategpkgtables(db, table, domain, crs, geom) Tables.eachcolumn(sch, row) do val, col, _ SQLite.bind!(stmt, col, val) end - r = GC.@preserve row SQLite.C.sqlite3_step(handle) # Evaluate An SQL Statement - # the return value will be either SQLITE_BUSY, SQLITE_DONE, SQLITE_ROW, SQLITE_ERROR, or SQLITE_MISUSE. + # Evaluates a SQL Statement and returns a value will be either SQLITE_BUSY, SQLITE_DONE, SQLITE_ROW, SQLITE_ERROR, or SQLITE_MISUSE. # To ensure that a statement has "finished" is to invoke sqlite3_reset() or sqlite3_finalize(). + r = GC.@preserve row SQLite.C.sqlite3_step(handle) + if r == SQLite.C.SQLITE_DONE SQLite.C.sqlite3_reset(handle) elseif r != SQLite.C.SQLITE_ROW @@ -95,6 +103,8 @@ function creategpkgtables(db, table, domain, crs, geom) minx, miny, maxx, maxy = mincoords[1], mincoords[2], maxcoords[1], maxcoords[2] z = paramdim(first(geom)) > 2 ? 1 : 0 + # According to https://www.geopackage.org/spec/#r10 + # A GeoPackage SHALL include a gpkg_spatial_ref_sys table DBInterface.execute( db, """ @@ -125,6 +135,9 @@ function creategpkgtables(db, table, domain, crs, geom) ) # Insert non-existing CRS record into gpkg_spatial_ref_sys table. if srid != 4326 && srid > 0 + # According to https://www.geopackage.org/spec/#r115 + # This also conforms to the WKT for CRS extension + # the gpkg_spatial_ref_sys table SHALL have an additional column called definition_12_063 DBInterface.execute( db, """ @@ -137,6 +150,8 @@ function creategpkgtables(db, table, domain, crs, geom) ) end + # According to https://www.geopackage.org/spec/#r13 + # A GeoPackage SHALL include a gpkg_contents table DBInterface.execute( db, """ @@ -167,6 +182,9 @@ function creategpkgtables(db, table, domain, crs, geom) ["features", "features", "features", minx, miny, maxx, maxy, srid] ) + # According to https://www.geopackage.org/spec/#r21 + # A GeoPackage with a gpkg_contents table row with a "features" data_type + # SHALL contain a gpkg_geometry_columns table DBInterface.execute( db, """ @@ -197,7 +215,8 @@ function creategpkgtables(db, table, domain, crs, geom) ) # https://www.geopackage.org/spec/#r77 - # Extended GeoPackage requires spatial indexes on feature table geometry columns using the SQLite Virtual Table R-trees + # Extended GeoPackage requires spatial indexes on feature table geometry columns + # using the SQLite Virtual Table R-trees DBInterface.execute( db, """ From ec9de5a468e5476737c48b15b11cf8cce306000a Mon Sep 17 00:00:00 2001 From: jph6366 Date: Thu, 30 Oct 2025 18:52:07 -0400 Subject: [PATCH 42/90] write.jl comments; wkb.jl remains for review --- src/extra/gpkg/read.jl | 19 +++---- src/extra/gpkg/write.jl | 107 ++++++++++++++++++++++++++++------------ 2 files changed, 85 insertions(+), 41 deletions(-) diff --git a/src/extra/gpkg/read.jl b/src/extra/gpkg/read.jl index f612957..23f8391 100644 --- a/src/extra/gpkg/read.jl +++ b/src/extra/gpkg/read.jl @@ -11,15 +11,15 @@ function gpkgread(fname; layer=1) end function assertgpkg(db) - # Requirement 6: PRAGMA integrity_check returns a single row with the value 'ok' - # Requirement 7: PRAGMA foreign_key_check (w/ no parameter value) returns an empty result set + # According to https://www.geopackage.org/spec/#r6 and https://www.geopackage.org/spec/#r7 + # PRAGMA integrity_check returns a single row with the value 'ok' + # PRAGMA foreign_key_check (w/ no parameter value) returns an empty result set if first(DBInterface.execute(db, "PRAGMA integrity_check;")).integrity_check != "ok" || !(isempty(DBInterface.execute(db, "PRAGMA foreign_key_check;"))) throw(ErrorException("database integrity at risk or foreign key violation(s)")) end - - # Requirement 10: must include a gpkg_spatial_ref_sys table - # Requirement 13: must include a gpkg_contents table + # According to https://www.geopackage.org/spec/#r10 and https://www.geopackage.org/spec/#r13 + # A GeoPackage SHALL include a 'gpkg_spatial_ref_sys' table and a 'gpkg_contents table' if first(DBInterface.execute( db, """ @@ -141,6 +141,7 @@ function gpkgtable(db, ; layer=1) seek(io, 3) # bit layout of GeoPackageBinary flags byte + # https://www.geopackage.org/spec/#flags_layout # --------------------------------------- # bit # 7 # 6 # 5 # 4 # 3 # 2 # 1 # 0 # # use # R # R # X # Y # E # E # E # B # @@ -159,6 +160,7 @@ function gpkgtable(db, ; layer=1) # results in a 3-bit unsigned integer envelope = (flag & (0x07 << 1)) >> 1 + # No envelope [] (space saving slower indexing option), 0 bytes envelopecode = 0 if !iszero(envelope) if isone(envelope) @@ -177,12 +179,10 @@ function gpkgtable(db, ; layer=1) # 5-7: invalid throw(ErrorException("exceeded dimensional limit for geometry")) end - # else no envelope (space saving slower indexing option), 0 bytes end # calculate GeoPackageBinaryHeader size in byte stream given extent of envelope: - # byte[2] magic + byte[1] version + byte[1] flags + byte[4] srs_id + byte[16*E] envelope - # where E is the code that indicates how many pairs of floating-point numbers exist in envelope + # byte[2] magic + byte[1] version + byte[1] flags + byte[4] srs_id + byte[16×NumberOfAxes] envelope headerlen = 8 + 8 * 2 * envelopecode # Skip reading the double[] envelope and start reading Well-Known Binary geometry @@ -208,6 +208,7 @@ function gpkgtable(db, ; layer=1) geom = gpkgwkbgeom(io, crs, wkbtype, zextent, wkbbyteswap) if !isnothing(geom) + # returns a tuple of the corresponding aspatial attributes and the geometries for each row in the feature table return (NamedTuple(rowvals), geom) end end @@ -215,6 +216,6 @@ function gpkgtable(db, ; layer=1) end # efficient method for concatenating arrays of arrays table = reduce(vcat, results) - # unpack results to return results + # unpack results to return aspatial and spatial results getindex.(table, 1), getindex.(table, 2) end diff --git a/src/extra/gpkg/write.jl b/src/extra/gpkg/write.jl index d864248..0316775 100644 --- a/src/extra/gpkg/write.jl +++ b/src/extra/gpkg/write.jl @@ -5,45 +5,50 @@ function gpkgwrite(fname, geotable;) db = SQLite.DB(fname) - DBInterface.execute(db, "PRAGMA synchronous=0") # https://sqlite.org/pragma.html#pragma_synchronous # Commits can be orders of magnitude faster with # Setting PRAGMA synchronous=OFF but, # can cause the database to go corrupt # if there is an operating system crash or power failure. + DBInterface.execute(db, "PRAGMA synchronous=0") table = values(geotable) domain = GeoTables.domain(geotable) crs = GeoTables.crs(domain) geom = collect(domain) + # According to https://www.geopackage.org/spec/#r2 + # A GeoPackage SHALL contain a value of 0x47504B47 ("GPKG" in ASCII) + # in the "application_id" field and an appropriate value in "user_version" field + # of the SQLite database header to indicate that it is a GeoPackage DBInterface.execute(db, "PRAGMA application_id = $GPKG_APPLICATION_ID ") DBInterface.execute(db, "PRAGMA user_version = $GPKG_1_4_VERSION ") - + creategpkgtables(db, table, domain, crs, geom) - DBInterface.execute(db, "PRAGMA optimize;") # https://sqlite.org/pragma.html#pragma_optimize # Applications with short-lived database connections should run "PRAGMA optimize;" # just once, prior to closing each database connection. + DBInterface.execute(db, "PRAGMA optimize;") + DBInterface.close!(db) end function creategpkgtables(db, table, domain, crs, geom) if crs <: Cartesian - srs = "" - srid = -1 + org = "" + srsid = -1 elseif crs <: LatLon{WGS84Latest} - srs = "EPSG" - srid = 4326 + org = "EPSG" + srsid = 4326 else - srs = string(CoordRefSystems.code(crs))[1:4] - srid = parse(Int32, string(CoordRefSystems.code(crs))[6:(end - 1)]) + org = string(CoordRefSystems.code(crs))[1:4] + srsid = parse(Int32, string(CoordRefSystems.code(crs))[6:(end - 1)]) end - gpkgbinary = map(geom) do ft - gpkgbinheader = writegpkgheader(srid, ft) + gpkgbinary = map(geom) do feature + gpkgbinheader = writegpkgheader(srsid, feature) io = IOBuffer() - writewkbgeom(io, ft) + writewkbgeom(io, feature) vcat(gpkgbinheader, take!(io)) end @@ -61,46 +66,68 @@ function creategpkgtables(db, table, domain, crs, geom) ] # https://www.geopackage.org/spec/#r29 - # A feature table SHALL have a primary key column of type INTEGER and that column SHALL act as a rowid alias. - DBInterface.execute(db, "CREATE TABLE features ( $(join(columns, ',')));") + # A feature table SHALL have a primary key column of type INTEGER and that column SHALL act as a rowid alias. # The use of the AUTOINCREMENT keyword is optional but recommended. + DBInterface.execute(db, "CREATE TABLE features ( $(join(columns, ',')));") + # generate the SQL parameter string for binding values, chop removes the last comma, resulting in "?,?,?" params = chop(repeat("?,", length(sch.names))) + # generate the comma-separated list of escaped column names for the SQL query columns = join(SQLite.esc_id.(string.(sch.names)), ",") + # Note: the `sql` statement is not actually executed, but only compiled + # mainly for usage where the same statement is executed multiple times with different parameters bound as values stmt = SQLite.Stmt(db, "INSERT INTO features ($columns) VALUES ($params)";) + # used for holding references to bound statement values via bind! handle = SQLite._get_stmt_handle(stmt) + + # an explicit write transaction is started by statements like CREATE, DELETE, DROP, INSERT, or UPDATE + # the default transaction behavior is DEFERRED. + # DEFERRED means that the transaction does not actually start until the database is first accessed SQLite.transaction(db) do row = nothing if row === nothing + # advance the iterator to obtain the next element state = iterate(rows) + # exit transaction if iterator is empty state === nothing && return row, st = state end while true + # bind the values of the current row to the prepared SQL statement Tables.eachcolumn(sch, row) do val, col, _ SQLite.bind!(stmt, col, val) end - # Evaluates a SQL Statement and returns a value will be either SQLITE_BUSY, SQLITE_DONE, SQLITE_ROW, SQLITE_ERROR, or SQLITE_MISUSE. - # To ensure that a statement has "finished" is to invoke sqlite3_reset() or sqlite3_finalize(). - r = GC.@preserve row SQLite.C.sqlite3_step(handle) + # executes the prepared statement and GC.@preserve prevents the 'row' object from being garbage collected + r = GC.@preserve row SQLite.C.sqlite3_step(handle) if r == SQLite.C.SQLITE_DONE + # insertion successful, reset for next execution SQLite.C.sqlite3_reset(handle) elseif r != SQLite.C.SQLITE_ROW + # error occurred (e.g., SQLITE_BUSY, SQLITE_ERROR, or others). + # throw a Julia-specific SQLite exception after resetting the statement handle. e = SQLite.sqliteexception(db, stmt) SQLite.C.sqlite3_reset(handle) throw(e) end + # advance to the next row state = iterate(rows, st) + # break the loop if iterator is exhausted state === nothing && break row, st = state end + # collect bounding box for all content in geotable bbox = boundingbox(domain) + # bounding box minimum easting or longitude, and northing or latitude mincoords = CoordRefSystems.raw(coords(bbox.min)) + # bounding box maximum easting or longitude, and northing or latitude maxcoords = CoordRefSystems.raw(coords(bbox.max)) + # the bounding box (min_x, min_y, max_x, max_y) provides an informative bounding box of the content minx, miny, maxx, maxy = mincoords[1], mincoords[2], maxcoords[1], maxcoords[2] + # 0: z values prohibited; 1: z values mandatory; + # (x,y{,z}) where x is easting or longitude, y is northing or latitude, and z is optional elevation z = paramdim(first(geom)) > 2 ? 1 : 0 # According to https://www.geopackage.org/spec/#r10 @@ -134,9 +161,9 @@ function creategpkgtables(db, table, domain, crs, geom) """ ) # Insert non-existing CRS record into gpkg_spatial_ref_sys table. - if srid != 4326 && srid > 0 + if srsid != 4326 && srsid > 0 # According to https://www.geopackage.org/spec/#r115 - # This also conforms to the WKT for CRS extension + # This conforms to the Well-Known Text for Coordinate Reference Systems extension # the gpkg_spatial_ref_sys table SHALL have an additional column called definition_12_063 DBInterface.execute( db, @@ -146,7 +173,7 @@ function creategpkgtables(db, table, domain, crs, geom) VALUES (?, ?, ?, ?, ?, ?, ?); """, - ["", srid, srs, srid, CoordRefSystems.wkt2(crs), "", CoordRefSystems.wkt2(crs)] + ["", srsid, org, srsid, CoordRefSystems.wkt2(crs), "", CoordRefSystems.wkt2(crs)] ) end @@ -179,7 +206,7 @@ function creategpkgtables(db, table, domain, crs, geom) VALUES (?, ?, ?, ?, ?, ?, ?, ?); """, - ["features", "features", "features", minx, miny, maxx, maxy, srid] + ["features", "features", "features", minx, miny, maxx, maxy, srsid] ) # According to https://www.geopackage.org/spec/#r21 @@ -211,7 +238,7 @@ function creategpkgtables(db, table, domain, crs, geom) VALUES (?, ?, ?, ?, ?, ?); """, - ["features", "geom", "GEOMETRY", srid, z, 0] + ["features", "geom", "GEOMETRY", srsid, z, 0] ) # https://www.geopackage.org/spec/#r77 @@ -219,6 +246,8 @@ function creategpkgtables(db, table, domain, crs, geom) # using the SQLite Virtual Table R-trees DBInterface.execute( db, + # creates a spatial index using rtree__ + # where and are replaced with the names of the feature table and geometry column being indexed. """ CREATE VIRTUAL TABLE rtree_features_geom USING rtree(id, minx, maxx, miny, maxy) @@ -239,18 +268,24 @@ function creategpkgtables(db, table, domain, crs, geom) stmt = SQLite.Stmt(db, "INSERT INTO rtree_features_geom VALUES (?, ?, ?, ?, ?)") handle = SQLite._get_stmt_handle(stmt) for i in 1:length(gpkgbinary) - fx_minx, fx_maxx, fx_miny, fx_maxy = bboxes[i] # the min/max x/y parameters are min- and max-value pairs (stored as 32-bit floating point numbers) - SQLite.bind!(stmt, 1, i) # The R-tree function id parameter becomes the virtual table 64-bit signed integer primary key id column - SQLite.bind!(stmt, 2, fx_minx) - SQLite.bind!(stmt, 3, fx_maxx) - SQLite.bind!(stmt, 4, fx_miny) - SQLite.bind!(stmt, 5, fx_maxy) - r = SQLite.C.sqlite3_step(handle) + # min-value and max-value pairs (stored as 32-bit floating point numbers) + minx, maxx, miny, maxy = bboxes[i] + # virtual table 64-bit signed integer primary key id column + SQLite.bind!(stmt, 1, i) + # min/max x/y parameters + SQLite.bind!(stmt, 2, minx) + SQLite.bind!(stmt, 3, maxx) + SQLite.bind!(stmt, 4, miny) + SQLite.bind!(stmt, 5, maxy) + + # Evaluates a SQL Statement and returns SQLITE_DONE, or SQLITE_BUSY, SQLITE_ROW, SQLITE_ERROR, or SQLITE_MISUSE. + r = GC.@preserve row SQLite.C.sqlite3_step(handle) if r != SQLite.C.SQLITE_DONE e = SQLite.sqliteexception(db, stmt) SQLite.C.sqlite3_reset(handle) throw(e) end + # invoke sqlite3_reset() to ensure a statement is finished SQLite.C.sqlite3_reset(handle) end @@ -282,21 +317,29 @@ end function writegpkgheader(srsid, geom) io = IOBuffer() - write(io, [0x47, 0x50]) # 'GP' in ASCII - write(io, zero(UInt8)) # 0 = version 1 + # 'GP' in ASCII + write(io, [0x47, 0x50]) + # 8-bit unsigned integer, 0 = version 1 + write(io, zero(UInt8)) + # bit layout of GeoPackageBinary flags byte indicates: + # The geometry header includes an envelope [minx, maxx, miny, maxy] + # and Little Endian (least significant byte first) is the byte order used for SRS ID and envelope values in the header. flagsbyte = UInt8(0x07 >> 1) write(io, flagsbyte) + # write the SRS ID, with the endianness specified by the byte order flag write(io, htol(Int32(srsid))) + # write the envelope for all content in GeoPackage SQL Geometry Binary Format bbox = boundingbox(geom) + # [minx, maxx, miny, maxy] write(io, htol(Float64(CoordRefSystems.raw(coords(bbox.min))[2]))) write(io, htol(Float64(CoordRefSystems.raw(coords(bbox.max))[2]))) write(io, htol(Float64(CoordRefSystems.raw(coords(bbox.min))[1]))) write(io, htol(Float64(CoordRefSystems.raw(coords(bbox.max))[1]))) - if paramdim(geom) >= 3 + # [..., minz, maxz] write(io, htol(Float64(CoordRefSystems.raw(coords(bbox.min))[3]))) write(io, htol(Float64(CoordRefSystems.raw(coords(bbox.max))[3]))) end From c23fdb2988de9ee042c28754ec99f90ad04cbece Mon Sep 17 00:00:00 2001 From: jph6366 Date: Sun, 2 Nov 2025 20:12:29 -0500 Subject: [PATCH 43/90] wkb comments --- src/extra/gpkg/wkb.jl | 172 ++++++++++++++++++++++++------------------ 1 file changed, 97 insertions(+), 75 deletions(-) diff --git a/src/extra/gpkg/wkb.jl b/src/extra/gpkg/wkb.jl index 42fa0be..fb2892b 100644 --- a/src/extra/gpkg/wkb.jl +++ b/src/extra/gpkg/wkb.jl @@ -66,106 +66,105 @@ GeoIO GeoPackage writer supports X,Y,Z coordinate offset of 1000 (Z) for wkbGeom ``` """ -const ewkbmaskbits = 0x40000000 | 0x80000000 -# Requirement 20: GeoPackage SHALL store feature table geometries -# with the basic simple feature geometry types -# https://www.geopackage.org/spec140/index.html#geometry_types +# According to https://www.geopackage.org/spec/#r20 +# GeoPackage SHALL store feature table geometries with the basic simple feature geometry types. +# Geometry Types (Normative): https://www.geopackage.org/spec140/index.html#geometry_types +# Note: this implementation supports (Core) Geometry Type Codes function gpkgwkbgeom(io, crs, wkbtype, zextent, bswap) if wkbtype > 3 + # 4 - 7 [MultiPoint, MultiLinestring, MultiPolygon, GeometryCollection] elems = wkbmulti(io, crs, zextent, bswap) Multi(elems) else + # 0 - 3 [Geometry, Point, Linestring, Polygon] elem = wkbsimple(io, crs, wkbtype, zextent, bswap) elem end end +#------- +# WKB GEOMETRY READER UTILS +#------- + +# read simple features from Well-Known Binary IO Buffer and return Concrete Geometry function wkbsimple(io, crs, wkbtype, zextent, bswap) if isequal(wkbtype, 1) - elem = wkbcoordinate(io, zextent, bswap) - Point(crs(elem...)) + geom = wkbcoordinate(io, zextent, bswap) + # return point given coordinates with respect to CRS + Point(crs(geom...)) elseif isequal(wkbtype, 2) - elem = wkblinestring(io, zextent, bswap) - if length(elem) >= 2 && first(elem) != last(elem) - Rope([Point(crs(coords...)) for coords in elem]...) + geom = wkblinestring(io, zextent, bswap) + if length(geom) >= 2 && first(geom) != last(geom) + # return open polygonal chain from sequence of points w.r.t CRS + Rope([Point(crs(points...)) for points in geom]...) else - Ring([Point(crs(coords...)) for coords in elem[1:(end - 1)]]...) + # return closed polygonal chain from sequence of points w.r.t CRS + Ring([Point(crs(points...)) for points in geom[1:(end - 1)]]...) end elseif isequal(wkbtype, 3) - elem = wkbpolygon(io, zextent, bswap) - rings = map(elem) do ring + geom = wkbpolygon(io, zextent, bswap) + rings = map(geom) do ring coords = map(ring) do point Point(crs(point...)) end Ring(coords) end - outerring = first(rings) holes = isone(length(rings)) ? rings[2:end] : Ring[] + # return polygonal area with outer ring, and optional inner rings PolyArea(outerring, holes...) end end -function wkbcoordinate(io, z, bswap) - x = bswap(read(io, Float64)) - y = bswap(read(io, Float64)) - if z +function wkbcoordinate(io, zextent, bswap) + x, y = bswap(read(io, Float64)), bswap(read(io, Float64)) + if zextent z = bswap(read(io, Float64)) return x, y, z end - x, y end -function wkblinestring(io, z, bswap) +function wkblinestring(io, zextent, bswap) npoints = bswap(read(io, UInt32)) - points = map(1:npoints) do _ - wkbcoordinate(io, z, bswap) + wkbcoordinate(io, zextent, bswap) end points end -function wkbpolygon(io, z, bswap) +function wkbpolygon(io, zextent, bswap) nrings = bswap(read(io, UInt32)) - rings = map(1:nrings) do _ - wkblinestring(io, z, bswap) + wkblinestring(io, zextent, bswap) end rings end -function wkbmulti(io, crs, z, bswap) +function wkbmulti(io, crs, zextent, bswap) ngeoms = bswap(read(io, UInt32)) - - geomcollection = map(1:ngeoms) do _ + geoms = map(1:ngeoms) do _ wkbbswap = isone(read(io, UInt8)) ? ltoh : ntoh wkbtypebits = read(io, UInt32) - if z - if iszero(wkbtypebits & ewkbmaskbits) - wkbtypebits = wkbtypebits & 0x000000F # extended WKB - else - wkbtypebits = wkbtypebits - 1000 # ISO WKB + # if 2D+Z the dimensionality flag is present + if zextent + if iszero(wkbtypebits & 0x80000000) + # Extended WKB: wkbtype + 0x80000000 = wkbTypeZ + wkbtypebits = wkbtypebits & 0x000000F + elseif wkbtypebits > 1000 + # ISO WKB: wkbType + 1000 = wkbTypeZ + wkbtypebits = wkbtypebits - 1000 end end - wkbsimple(io, crs, wkbtypebits, z, wkbbswap) + wkbsimple(io, crs, wkbtypebits, zextent, wkbbswap) end - geomcollection + geoms end -function _wkbtype(geometry) - if geometry isa Point - return 1 # wkbPoint - elseif geometry isa Rope || geometry isa Ring - return 2 # wkbLineString - elseif geometry isa PolyArea - return 3 # wkbPolygon - elseif geometry isa Multi - fg = first(parent(geometry)) - return _wkbtype(fg) + 3 # wkbMulti - end -end +#------- +# WKB GEOMETRY WRITER UTILS +#------- function writewkbgeom(io, geom) wkbtype = _wkbtype(geom) @@ -174,22 +173,40 @@ function writewkbgeom(io, geom) _wkbgeom(io, wkbtype, geom) end -#------- -# WKB GEOMETRY WRITER UTILS -#------- + +function _wkbtype(geometry) + if geometry isa Point + # wkbPoint + return 1 + elseif geometry isa Rope || geometry isa Ring + # wkbLineString + return 2 + elseif geometry isa PolyArea + # wkbPolygon + return 3 + elseif geometry isa Multi + # wkbMulti + fg = first(parent(geometry)) + return _wkbtype(fg) + 3 + end +end function _writewkbsimple(io, wkbtype, geom) - if isequal(wkbtype, 3) # wkbPolygon - _wkbpolygon(io, [boundary(geom::PolyArea)]) - elseif isequal(wkbtype, 2) # wkbLineString + # wkbPolygon + if isequal(wkbtype, 3) + _wkbpolyarea(io, [boundary(geom::PolyArea)]) + elseif isequal(wkbtype, 2) coordlist = vertices(geom) if typeof(geom) <: Ring + # wkbLineString[length+1] return _wkbchainring(io, coordlist) end + # wkbLineString _wkbchainrope(io, coordlist) - elseif isequal(wkbtype, 1) # wkbPoint + elseif isequal(wkbtype, 1) coordinates = CoordRefSystems.raw(coords(geom)) - _wkbcoordinates(io, coordinates) + # wkbPoint + _wkbpoint(io, coordinates) else throw(ErrorException("Well-Known Binary Geometry unknown: $wkbtype")) end @@ -203,44 +220,49 @@ function _wkbgeom(io, wkbtype, geom) end end -function _wkbcoordinates(io, coords) - write(io, htol(coords[2])) - write(io, htol(coords[1])) - if length(coords) == 3 - write(io, htol(coords[3])) +function _wkbpoint(io, coordinates) + write(io, htol(coordinates[2])) + write(io, htol(coordinates[1])) + if length(coordinates) == 3 + write(io, htol(coordinates[3])) end end -function _wkbchainrope(io, coord_list) - write(io, htol(UInt32(length(coord_list)))) - for n_coords in coord_list - coordinates = CoordRefSystems.raw(coords(n_coords)) - _wkbcoordinates(io, coordinates) +function _wkbchainrope(io, points) + write(io, htol(UInt32(length(points)))) + for coordinates in points + point = CoordRefSystems.raw(coords(coordinates)) + _wkbpoint(io, point) end end -function _wkbchainring(io, coord_list) - write(io, htol(UInt32(length(coord_list) + 1))) - for n_coords in coord_list - coordinates = CoordRefSystems.raw(coords(n_coords)) - _wkbcoordinates(io, coordinates) +function _wkbchainring(io, points) + # add a point to close linestring + write(io, htol(UInt32(length(points) + 1))) + for point in points + point = CoordRefSystems.raw(coords(point)) + _wkbpoint(io, point) end - _wkbcoordinates(io, CoordRefSystems.raw(first(coord_list) |> coords)) + # write a point to close linestring + _wkbpoint(io, CoordRefSystems.raw(first(points) |> coords)) end -function _wkbpolygon(io, rings) +function _wkbpolyarea(io, rings) write(io, htol(UInt32(length(rings)))) for ring in rings - coord_list = vertices(ring) - _wkbchainring(io, coord_list) + points = vertices(ring) + _wkbchainrope(io, points) end end function _wkbmulti(io, multiwkbtype, geoms) + # `geoms` is treated as a single [`Geometry`] + # `parent(geoms)` returns the collection of geometries with the same types write(io, htol(UInt32(length(parent(geoms))))) - for sf in parent(geoms) + for geom in parent(geoms) write(io, one(UInt8)) + # wkbGeometryType + 3 = Multi-wkbGeometryType write(io, multiwkbtype - 3) - _writewkbsimple(io, multiwkbtype - 3, sf) + _writewkbsimple(io, multiwkbtype - 3, geom) end end \ No newline at end of file From 59428a4ecccd00c1af960e3764704b280301e269 Mon Sep 17 00:00:00 2001 From: jph6366 Date: Sun, 2 Nov 2025 20:25:34 -0500 Subject: [PATCH 44/90] removed isequal --- src/extra/gpkg/wkb.jl | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/extra/gpkg/wkb.jl b/src/extra/gpkg/wkb.jl index fb2892b..dcc3d0a 100644 --- a/src/extra/gpkg/wkb.jl +++ b/src/extra/gpkg/wkb.jl @@ -89,11 +89,11 @@ end # read simple features from Well-Known Binary IO Buffer and return Concrete Geometry function wkbsimple(io, crs, wkbtype, zextent, bswap) - if isequal(wkbtype, 1) + if wkbtype == 1 geom = wkbcoordinate(io, zextent, bswap) # return point given coordinates with respect to CRS Point(crs(geom...)) - elseif isequal(wkbtype, 2) + elseif wkbtype == 2 geom = wkblinestring(io, zextent, bswap) if length(geom) >= 2 && first(geom) != last(geom) # return open polygonal chain from sequence of points w.r.t CRS @@ -102,7 +102,7 @@ function wkbsimple(io, crs, wkbtype, zextent, bswap) # return closed polygonal chain from sequence of points w.r.t CRS Ring([Point(crs(points...)) for points in geom[1:(end - 1)]]...) end - elseif isequal(wkbtype, 3) + elseif wkbtype == 3 geom = wkbpolygon(io, zextent, bswap) rings = map(geom) do ring coords = map(ring) do point @@ -111,7 +111,7 @@ function wkbsimple(io, crs, wkbtype, zextent, bswap) Ring(coords) end outerring = first(rings) - holes = isone(length(rings)) ? rings[2:end] : Ring[] + holes = length(rings) > 1 ? rings[2:end] : Ring[] # return polygonal area with outer ring, and optional inner rings PolyArea(outerring, holes...) end @@ -193,9 +193,9 @@ end function _writewkbsimple(io, wkbtype, geom) # wkbPolygon - if isequal(wkbtype, 3) + if wkbtype == 3 _wkbpolyarea(io, [boundary(geom::PolyArea)]) - elseif isequal(wkbtype, 2) + elseif wkbtype == 2 coordlist = vertices(geom) if typeof(geom) <: Ring # wkbLineString[length+1] @@ -203,7 +203,7 @@ function _writewkbsimple(io, wkbtype, geom) end # wkbLineString _wkbchainrope(io, coordlist) - elseif isequal(wkbtype, 1) + elseif wkbtype == 1 coordinates = CoordRefSystems.raw(coords(geom)) # wkbPoint _wkbpoint(io, coordinates) From 62c9393fbb3da5e424c1435216f9f0b8efe0b8ba Mon Sep 17 00:00:00 2001 From: jph6366 Date: Mon, 3 Nov 2025 08:26:12 -0500 Subject: [PATCH 45/90] GeoIO.load changes only --- src/extra/gpkg/wkb.jl | 105 ------------ src/extra/gpkg/write.jl | 348 ---------------------------------------- 2 files changed, 453 deletions(-) delete mode 100644 src/extra/gpkg/write.jl diff --git a/src/extra/gpkg/wkb.jl b/src/extra/gpkg/wkb.jl index dcc3d0a..8549ffb 100644 --- a/src/extra/gpkg/wkb.jl +++ b/src/extra/gpkg/wkb.jl @@ -160,109 +160,4 @@ function wkbmulti(io, crs, zextent, bswap) wkbsimple(io, crs, wkbtypebits, zextent, wkbbswap) end geoms -end - -#------- -# WKB GEOMETRY WRITER UTILS -#------- - -function writewkbgeom(io, geom) - wkbtype = _wkbtype(geom) - write(io, htol(one(UInt8))) - write(io, htol(UInt32(wkbtype))) - _wkbgeom(io, wkbtype, geom) -end - - -function _wkbtype(geometry) - if geometry isa Point - # wkbPoint - return 1 - elseif geometry isa Rope || geometry isa Ring - # wkbLineString - return 2 - elseif geometry isa PolyArea - # wkbPolygon - return 3 - elseif geometry isa Multi - # wkbMulti - fg = first(parent(geometry)) - return _wkbtype(fg) + 3 - end -end - -function _writewkbsimple(io, wkbtype, geom) - # wkbPolygon - if wkbtype == 3 - _wkbpolyarea(io, [boundary(geom::PolyArea)]) - elseif wkbtype == 2 - coordlist = vertices(geom) - if typeof(geom) <: Ring - # wkbLineString[length+1] - return _wkbchainring(io, coordlist) - end - # wkbLineString - _wkbchainrope(io, coordlist) - elseif wkbtype == 1 - coordinates = CoordRefSystems.raw(coords(geom)) - # wkbPoint - _wkbpoint(io, coordinates) - else - throw(ErrorException("Well-Known Binary Geometry unknown: $wkbtype")) - end -end - -function _wkbgeom(io, wkbtype, geom) - if wkbtype > 3 - _wkbmulti(io, wkbtype, geom) - else - _writewkbsimple(io, wkbtype, geom) - end -end - -function _wkbpoint(io, coordinates) - write(io, htol(coordinates[2])) - write(io, htol(coordinates[1])) - if length(coordinates) == 3 - write(io, htol(coordinates[3])) - end -end - -function _wkbchainrope(io, points) - write(io, htol(UInt32(length(points)))) - for coordinates in points - point = CoordRefSystems.raw(coords(coordinates)) - _wkbpoint(io, point) - end -end - -function _wkbchainring(io, points) - # add a point to close linestring - write(io, htol(UInt32(length(points) + 1))) - for point in points - point = CoordRefSystems.raw(coords(point)) - _wkbpoint(io, point) - end - # write a point to close linestring - _wkbpoint(io, CoordRefSystems.raw(first(points) |> coords)) -end - -function _wkbpolyarea(io, rings) - write(io, htol(UInt32(length(rings)))) - for ring in rings - points = vertices(ring) - _wkbchainrope(io, points) - end -end - -function _wkbmulti(io, multiwkbtype, geoms) - # `geoms` is treated as a single [`Geometry`] - # `parent(geoms)` returns the collection of geometries with the same types - write(io, htol(UInt32(length(parent(geoms))))) - for geom in parent(geoms) - write(io, one(UInt8)) - # wkbGeometryType + 3 = Multi-wkbGeometryType - write(io, multiwkbtype - 3) - _writewkbsimple(io, multiwkbtype - 3, geom) - end end \ No newline at end of file diff --git a/src/extra/gpkg/write.jl b/src/extra/gpkg/write.jl deleted file mode 100644 index 0316775..0000000 --- a/src/extra/gpkg/write.jl +++ /dev/null @@ -1,348 +0,0 @@ -# ------------------------------------------------------------------ -# Licensed under the MIT License. See LICENSE in the project root. -# ------------------------------------------------------------------ - -function gpkgwrite(fname, geotable;) - db = SQLite.DB(fname) - - # https://sqlite.org/pragma.html#pragma_synchronous - # Commits can be orders of magnitude faster with - # Setting PRAGMA synchronous=OFF but, - # can cause the database to go corrupt - # if there is an operating system crash or power failure. - DBInterface.execute(db, "PRAGMA synchronous=0") - - table = values(geotable) - domain = GeoTables.domain(geotable) - crs = GeoTables.crs(domain) - geom = collect(domain) - - # According to https://www.geopackage.org/spec/#r2 - # A GeoPackage SHALL contain a value of 0x47504B47 ("GPKG" in ASCII) - # in the "application_id" field and an appropriate value in "user_version" field - # of the SQLite database header to indicate that it is a GeoPackage - DBInterface.execute(db, "PRAGMA application_id = $GPKG_APPLICATION_ID ") - DBInterface.execute(db, "PRAGMA user_version = $GPKG_1_4_VERSION ") - - creategpkgtables(db, table, domain, crs, geom) - - # https://sqlite.org/pragma.html#pragma_optimize - # Applications with short-lived database connections should run "PRAGMA optimize;" - # just once, prior to closing each database connection. - DBInterface.execute(db, "PRAGMA optimize;") - - DBInterface.close!(db) -end - -function creategpkgtables(db, table, domain, crs, geom) - if crs <: Cartesian - org = "" - srsid = -1 - elseif crs <: LatLon{WGS84Latest} - org = "EPSG" - srsid = 4326 - else - org = string(CoordRefSystems.code(crs))[1:4] - srsid = parse(Int32, string(CoordRefSystems.code(crs))[6:(end - 1)]) - end - gpkgbinary = map(geom) do feature - gpkgbinheader = writegpkgheader(srsid, feature) - io = IOBuffer() - writewkbgeom(io, feature) - vcat(gpkgbinheader, take!(io)) - end - - features = - # if no values in table then store only geometry in features - isnothing(table) ? [(; geom=g,) for (_, g) in zip(1:length(gpkgbinary), gpkgbinary)] : - # else store the geometry as the first column and the remaining table columns in features - [(; geom=g, t...) for (t, g) in zip(Tables.rowtable(table), gpkgbinary)] - - rows = Tables.rows(features) - sch = Tables.schema(rows) - columns = [ - string(SQLite.esc_id(String(sch.names[i])), ' ', SQLite.sqlitetype(sch.types !== nothing ? sch.types[i] : Any)) - for i in eachindex(sch.names) - ] - - # https://www.geopackage.org/spec/#r29 - # A feature table SHALL have a primary key column of type INTEGER and that column SHALL act as a rowid alias. - # The use of the AUTOINCREMENT keyword is optional but recommended. - DBInterface.execute(db, "CREATE TABLE features ( $(join(columns, ',')));") - - # generate the SQL parameter string for binding values, chop removes the last comma, resulting in "?,?,?" - params = chop(repeat("?,", length(sch.names))) - # generate the comma-separated list of escaped column names for the SQL query - columns = join(SQLite.esc_id.(string.(sch.names)), ",") - # Note: the `sql` statement is not actually executed, but only compiled - # mainly for usage where the same statement is executed multiple times with different parameters bound as values - stmt = SQLite.Stmt(db, "INSERT INTO features ($columns) VALUES ($params)";) - - # used for holding references to bound statement values via bind! - handle = SQLite._get_stmt_handle(stmt) - - # an explicit write transaction is started by statements like CREATE, DELETE, DROP, INSERT, or UPDATE - # the default transaction behavior is DEFERRED. - # DEFERRED means that the transaction does not actually start until the database is first accessed - SQLite.transaction(db) do - row = nothing - if row === nothing - # advance the iterator to obtain the next element - state = iterate(rows) - # exit transaction if iterator is empty - state === nothing && return - row, st = state - end - while true - # bind the values of the current row to the prepared SQL statement - Tables.eachcolumn(sch, row) do val, col, _ - SQLite.bind!(stmt, col, val) - end - - # executes the prepared statement and GC.@preserve prevents the 'row' object from being garbage collected - r = GC.@preserve row SQLite.C.sqlite3_step(handle) - if r == SQLite.C.SQLITE_DONE - # insertion successful, reset for next execution - SQLite.C.sqlite3_reset(handle) - elseif r != SQLite.C.SQLITE_ROW - # error occurred (e.g., SQLITE_BUSY, SQLITE_ERROR, or others). - # throw a Julia-specific SQLite exception after resetting the statement handle. - e = SQLite.sqliteexception(db, stmt) - SQLite.C.sqlite3_reset(handle) - throw(e) - end - # advance to the next row - state = iterate(rows, st) - # break the loop if iterator is exhausted - state === nothing && break - row, st = state - end - - # collect bounding box for all content in geotable - bbox = boundingbox(domain) - # bounding box minimum easting or longitude, and northing or latitude - mincoords = CoordRefSystems.raw(coords(bbox.min)) - # bounding box maximum easting or longitude, and northing or latitude - maxcoords = CoordRefSystems.raw(coords(bbox.max)) - # the bounding box (min_x, min_y, max_x, max_y) provides an informative bounding box of the content - minx, miny, maxx, maxy = mincoords[1], mincoords[2], maxcoords[1], maxcoords[2] - # 0: z values prohibited; 1: z values mandatory; - # (x,y{,z}) where x is easting or longitude, y is northing or latitude, and z is optional elevation - z = paramdim(first(geom)) > 2 ? 1 : 0 - - # According to https://www.geopackage.org/spec/#r10 - # A GeoPackage SHALL include a gpkg_spatial_ref_sys table - DBInterface.execute( - db, - """ - CREATE TABLE gpkg_spatial_ref_sys ( - srs_name TEXT NOT NULL, srs_id INTEGER NOT NULL PRIMARY KEY, - organization TEXT NOT NULL, organization_coordsys_id INTEGER NOT NULL, - definition TEXT NOT NULL, description TEXT, - definition_12_063 TEXT NOT NULL - ); - """ - ) - - # According to https://www.geopackage.org/spec/#r11 - # The gpkg_spatial_ref_sys table SHALL contain at a minimum - # 1. the record with an srs_id of 4326 SHALL correspond to WGS-84 as defined by EPSG in 4326 - # 2. the record with an srs_id of -1 SHALL be used for undefined Cartesian coordinate reference systems - # 3. the record with an srs_id of 0 SHALL be used for undefined geographic coordinate reference systems - DBInterface.execute( - db, - """ - INSERT INTO gpkg_spatial_ref_sys - (srs_name, srs_id, organization, organization_coordsys_id, definition, description, definition_12_063) - VALUES - ('Undefined Cartesian SRS', -1, 'NONE', -1, 'undefined', 'undefined geographic coordinate reference system', 'undefined'), - ('Undefined geographic SRS', 0, 'NONE', 0, 'undefined', 'undefined geographic coordinate reference system', 'undefined'), - ('WGS 84 geodectic', 4326, 'EPSG', 4326, 'GEOGCRS["WGS 84",DATUM["World Geodetic System 1984",ELLIPSOID["WGS 84",6378137,298.257223563,LENGTHUNIT["metre",1]]],PRIMEM["Greenwich",0,ANGLEUNIT["degree",0.0174532925199433]],CS[ellipsoidal,2],AXIS["geodetic latitude (Lat)",north,ORDER[1],ANGLEUNIT["degree",0.0174532925199433]],AXIS["geodetic longitude (Lon)",east,ORDER[2],ANGLEUNIT["degree",0.0174532925199433]],ID["EPSG",4326]]', 'longitude/latitude coordinates in decimal degrees on the WGS 84 spheroid', 'GEOGCRS["WGS 84",DATUM["World Geodetic System 1984",ELLIPSOID["WGS 84",6378137,298.257223563,LENGTHUNIT["metre",1]]],PRIMEM["Greenwich",0,ANGLEUNIT["degree",0.0174532925199433]],CS[ellipsoidal,2],AXIS["geodetic latitude (Lat)",north,ORDER[1],ANGLEUNIT["degree",0.0174532925199433]],AXIS["geodetic longitude (Lon)",east,ORDER[2],ANGLEUNIT["degree",0.0174532925199433]],ID["EPSG",4326]]'); - """ - ) - # Insert non-existing CRS record into gpkg_spatial_ref_sys table. - if srsid != 4326 && srsid > 0 - # According to https://www.geopackage.org/spec/#r115 - # This conforms to the Well-Known Text for Coordinate Reference Systems extension - # the gpkg_spatial_ref_sys table SHALL have an additional column called definition_12_063 - DBInterface.execute( - db, - """ - INSERT INTO gpkg_spatial_ref_sys - (srs_name, srs_id, organization, organization_coordsys_id, definition, description, definition_12_063) - VALUES - (?, ?, ?, ?, ?, ?, ?); - """, - ["", srsid, org, srsid, CoordRefSystems.wkt2(crs), "", CoordRefSystems.wkt2(crs)] - ) - end - - # According to https://www.geopackage.org/spec/#r13 - # A GeoPackage SHALL include a gpkg_contents table - DBInterface.execute( - db, - """ - CREATE TABLE gpkg_contents ( - table_name TEXT NOT NULL PRIMARY KEY, - data_type TEXT NOT NULL, - identifier TEXT UNIQUE NOT NULL, - description TEXT DEFAULT '', - last_change DATETIME NOT NULL DEFAULT - (strftime('%Y-%m-%dT%H:%M:%fZ','now')), - min_x DOUBLE, min_y DOUBLE, - max_x DOUBLE, max_y DOUBLE, - srs_id INTEGER, - CONSTRAINT fk_gc_r_srs_id FOREIGN KEY (srs_id) REFERENCES - gpkg_spatial_ref_sys(srs_id) - ); - """ - ) - - DBInterface.execute( - db, - """ - INSERT INTO gpkg_contents - (table_name, data_type, identifier, min_x, min_y, max_x, max_y, srs_id) - VALUES - (?, ?, ?, ?, ?, ?, ?, ?); - """, - ["features", "features", "features", minx, miny, maxx, maxy, srsid] - ) - - # According to https://www.geopackage.org/spec/#r21 - # A GeoPackage with a gpkg_contents table row with a "features" data_type - # SHALL contain a gpkg_geometry_columns table - DBInterface.execute( - db, - """ - CREATE TABLE gpkg_geometry_columns ( - table_name TEXT NOT NULL, - column_name TEXT NOT NULL, - geometry_type_name TEXT NOT NULL, - srs_id INTEGER NOT NULL, - z TINYINT NOT NULL, - m TINYINT NOT NULL, - CONSTRAINT pk_geom_cols PRIMARY KEY (table_name, column_name), - CONSTRAINT uk_gc_table_name UNIQUE (table_name), - CONSTRAINT fk_gc_tn FOREIGN KEY (table_name) REFERENCES gpkg_contents(table_name), - CONSTRAINT fk_gc_srs FOREIGN KEY (srs_id) REFERENCES gpkg_spatial_ref_sys (srs_id) - ); - """ - ) - - DBInterface.execute( - db, - """ - INSERT INTO gpkg_geometry_columns - (table_name, column_name, geometry_type_name, srs_id, z, m) - VALUES - (?, ?, ?, ?, ?, ?); - """, - ["features", "geom", "GEOMETRY", srsid, z, 0] - ) - - # https://www.geopackage.org/spec/#r77 - # Extended GeoPackage requires spatial indexes on feature table geometry columns - # using the SQLite Virtual Table R-trees - DBInterface.execute( - db, - # creates a spatial index using rtree__ - # where and are replaced with the names of the feature table and geometry column being indexed. - """ - CREATE VIRTUAL TABLE rtree_features_geom USING - rtree(id, minx, maxx, miny, maxy) - """ - ) - bboxes = map(geom) do ft - bbox = boundingbox(ft) - mincoords = CoordRefSystems.raw(coords(bbox.min)) - maxcoords = CoordRefSystems.raw(coords(bbox.max)) - minx, miny, maxx, maxy = mincoords[1], mincoords[2], maxcoords[1], maxcoords[2] - return (minx, maxx, miny, maxy) - end - - # The R-tree Spatial Indexes extension provides a means to encode an R-tree index for geometry values - # And provides a significant performance advantage for searches with basic envelope spatial criteria - # that return subsets of the rows in a feature table with a non-trivial number (thousands or more) of rows. - # The index data structure needs to be manually populated, updated and queried. - stmt = SQLite.Stmt(db, "INSERT INTO rtree_features_geom VALUES (?, ?, ?, ?, ?)") - handle = SQLite._get_stmt_handle(stmt) - for i in 1:length(gpkgbinary) - # min-value and max-value pairs (stored as 32-bit floating point numbers) - minx, maxx, miny, maxy = bboxes[i] - # virtual table 64-bit signed integer primary key id column - SQLite.bind!(stmt, 1, i) - # min/max x/y parameters - SQLite.bind!(stmt, 2, minx) - SQLite.bind!(stmt, 3, maxx) - SQLite.bind!(stmt, 4, miny) - SQLite.bind!(stmt, 5, maxy) - - # Evaluates a SQL Statement and returns SQLITE_DONE, or SQLITE_BUSY, SQLITE_ROW, SQLITE_ERROR, or SQLITE_MISUSE. - r = GC.@preserve row SQLite.C.sqlite3_step(handle) - if r != SQLite.C.SQLITE_DONE - e = SQLite.sqliteexception(db, stmt) - SQLite.C.sqlite3_reset(handle) - throw(e) - end - # invoke sqlite3_reset() to ensure a statement is finished - SQLite.C.sqlite3_reset(handle) - end - - DBInterface.execute( - db, - """ - CREATE TABLE gpkg_extensions ( - table_name TEXT, - column_name TEXT, - extension_name TEXT NOT NULL, - definition TEXT NOT NULL, - scope TEXT NOT NULL, - CONSTRAINT ge_tce UNIQUE (table_name, column_name, extension_name) - ) - """ - ) - - DBInterface.execute( - db, - """ - INSERT INTO gpkg_extensions - (table_name, column_name, extension_name, definition, scope) - VALUES - ('features', 'geom', 'gpkg_rtree_index', 'http://www.geopackage.org/spec120/#extension_rtree', 'write-only'); - """ - ) - end -end - -function writegpkgheader(srsid, geom) - io = IOBuffer() - # 'GP' in ASCII - write(io, [0x47, 0x50]) - # 8-bit unsigned integer, 0 = version 1 - write(io, zero(UInt8)) - - # bit layout of GeoPackageBinary flags byte indicates: - # The geometry header includes an envelope [minx, maxx, miny, maxy] - # and Little Endian (least significant byte first) is the byte order used for SRS ID and envelope values in the header. - flagsbyte = UInt8(0x07 >> 1) - write(io, flagsbyte) - - # write the SRS ID, with the endianness specified by the byte order flag - write(io, htol(Int32(srsid))) - - # write the envelope for all content in GeoPackage SQL Geometry Binary Format - bbox = boundingbox(geom) - # [minx, maxx, miny, maxy] - write(io, htol(Float64(CoordRefSystems.raw(coords(bbox.min))[2]))) - write(io, htol(Float64(CoordRefSystems.raw(coords(bbox.max))[2]))) - write(io, htol(Float64(CoordRefSystems.raw(coords(bbox.min))[1]))) - write(io, htol(Float64(CoordRefSystems.raw(coords(bbox.max))[1]))) - if paramdim(geom) >= 3 - # [..., minz, maxz] - write(io, htol(Float64(CoordRefSystems.raw(coords(bbox.min))[3]))) - write(io, htol(Float64(CoordRefSystems.raw(coords(bbox.max))[3]))) - end - - return take!(io) -end From e4a40f036c301a2692c344334fe881dd60e1a7bc Mon Sep 17 00:00:00 2001 From: Jackson Hardee <42678151+jph6366@users.noreply.github.com> Date: Mon, 3 Nov 2025 08:27:03 -0500 Subject: [PATCH 46/90] Remove inclusion of write.jl in gpkg.jl Removed the inclusion of the write.jl file. --- src/extra/gpkg.jl | 1 - 1 file changed, 1 deletion(-) diff --git a/src/extra/gpkg.jl b/src/extra/gpkg.jl index 8b8fbcd..f4dc043 100644 --- a/src/extra/gpkg.jl +++ b/src/extra/gpkg.jl @@ -17,4 +17,3 @@ end include("gpkg/wkb.jl") include("gpkg/read.jl") -include("gpkg/write.jl") From a83e48b58230caa269d7dfa35200ea4d3dbfa362 Mon Sep 17 00:00:00 2001 From: jph6366 Date: Mon, 3 Nov 2025 09:19:40 -0500 Subject: [PATCH 47/90] fixed tests; ArchGDAL AUTOINCREMENT issue --- src/extra/gis.jl | 2 -- src/extra/gpkg/read.jl | 14 ++++++++------ src/extra/gpkg/wkb.jl | 10 +++++----- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/src/extra/gis.jl b/src/extra/gis.jl index 85c5420..f544832 100644 --- a/src/extra/gis.jl +++ b/src/extra/gis.jl @@ -50,8 +50,6 @@ function giswrite(fname, geotable; warn, kwargs...) elseif endswith(fname, ".parquet") CRS = crs(domain(geotable)) GPQ.write(fname, geotable, (:geometry,), projjson(CRS); kwargs...) - elseif endswith(fname, ".gpkg") - gpkgwrite(fname, geotable; kwargs...) else # fallback to GDAL agwrite(fname, geotable; kwargs...) end diff --git a/src/extra/gpkg/read.jl b/src/extra/gpkg/read.jl index 23f8391..f29b836 100644 --- a/src/extra/gpkg/read.jl +++ b/src/extra/gpkg/read.jl @@ -63,7 +63,7 @@ end # SHALL match the srs_id column value from the corresponding row in the # gpkg_contents table. function gpkgtable(db, ; layer=1) - resultrows = DBInterface.execute( + rowtable = DBInterface.execute( # According to https://www.geopackage.org/spec/#r16 # Values of the gpkg_contents table srs_id column # SHALL reference values in the gpkg_spatial_ref_sys table srs_id column @@ -89,19 +89,18 @@ function gpkgtable(db, ; layer=1) ) # Note: first feature table that is read specifies the CRS to be used on all feature tables resulted from SELECT statement - firstrow = first(resultrows) + firstrow = first(rowtable) # According to https://www.geopackage.org/spec/#r33, feature table geometry columns # SHALL contain geometries with the srs_id specified # for the column by the gpkg_geometry_columns table srs_id column value. srsid, org, orgcoordsysid = firstrow.srsid, firstrow.org, firstrow.orgcoordsysid - # defaults to undefined Cartesian CRS - crs = Cartesian{NoDatum} + # an srs_id of 0 uses undefined Geographic CRS if iszero(srsid) crs = LatLon{WGS84Latest} # if srs_id not equal to -1 - elseif !iszero(srsid + 1) + elseif srsid != -1 # CRS assigned by the org if org == "EPSG" # orgcoordsysid is the numeric id of the CRS assigned by the org @@ -109,9 +108,12 @@ function gpkgtable(db, ; layer=1) elseif org == "ESRI" crs = CoordRefSystems.get(ERSI{orgcoordsysid}) end + else + # defaults to undefined Cartesian CRS + crs = Cartesian{NoDatum} end - results = map((row.tablename, row.columnname) for row in resultrows) do (tablename, columnname) + results = map((row.tablename, row.columnname) for row in rowtable) do (tablename, columnname) # According to https://www.geopackage.org/spec/#r14 # The table_name column value in a gpkg_contents table row # SHALL contain the name of a SQLite table or view. diff --git a/src/extra/gpkg/wkb.jl b/src/extra/gpkg/wkb.jl index 8549ffb..34799e2 100644 --- a/src/extra/gpkg/wkb.jl +++ b/src/extra/gpkg/wkb.jl @@ -78,7 +78,7 @@ function gpkgwkbgeom(io, crs, wkbtype, zextent, bswap) Multi(elems) else # 0 - 3 [Geometry, Point, Linestring, Polygon] - elem = wkbsimple(io, crs, wkbtype, zextent, bswap) + elem = wkbsingle(io, crs, wkbtype, zextent, bswap) elem end end @@ -87,8 +87,8 @@ end # WKB GEOMETRY READER UTILS #------- -# read simple features from Well-Known Binary IO Buffer and return Concrete Geometry -function wkbsimple(io, crs, wkbtype, zextent, bswap) +# read single features from Well-Known Binary IO Buffer and return Concrete Geometry +function wkbsingle(io, crs, wkbtype, zextent, bswap) if wkbtype == 1 geom = wkbcoordinate(io, zextent, bswap) # return point given coordinates with respect to CRS @@ -118,7 +118,7 @@ function wkbsimple(io, crs, wkbtype, zextent, bswap) end function wkbcoordinate(io, zextent, bswap) - x, y = bswap(read(io, Float64)), bswap(read(io, Float64)) + y, x = bswap(read(io, Float64)), bswap(read(io, Float64)) if zextent z = bswap(read(io, Float64)) return x, y, z @@ -157,7 +157,7 @@ function wkbmulti(io, crs, zextent, bswap) wkbtypebits = wkbtypebits - 1000 end end - wkbsimple(io, crs, wkbtypebits, zextent, wkbbswap) + wkbsingle(io, crs, wkbtypebits, zextent, wkbbswap) end geoms end \ No newline at end of file From 7e43151de2c252503bf79b38cefe41cd147ee708 Mon Sep 17 00:00:00 2001 From: jph6366 Date: Mon, 3 Nov 2025 21:13:17 -0500 Subject: [PATCH 48/90] primary key one-indexed in PRAGMA tableinfo --- src/extra/gpkg/read.jl | 19 +++++++++++-------- src/extra/gpkg/wkb.jl | 2 +- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/src/extra/gpkg/read.jl b/src/extra/gpkg/read.jl index f29b836..827fae6 100644 --- a/src/extra/gpkg/read.jl +++ b/src/extra/gpkg/read.jl @@ -74,16 +74,16 @@ function gpkgtable(db, ; layer=1) db, """ SELECT g.table_name AS tablename, g.column_name AS columnname, - c.srs_id AS srsid, g.z, srs.organization AS org, srs.organization_coordsys_id AS orgcoordsysid, - ( SELECT type FROM sqlite_master WHERE lower(name) = lower(c.table_name) AND type IN ('table', 'view')) AS object_type + c.srs_id AS srsid, g.z, srs.organization AS org, srs.organization_coordsys_id AS orgcoordsysid, + ( SELECT type FROM sqlite_master WHERE lower(name) = lower(c.table_name) AND type IN ('table', 'view')) AS object_type FROM gpkg_geometry_columns g, gpkg_spatial_ref_sys srs JOIN gpkg_contents c ON ( g.table_name = c.table_name ) WHERE c.data_type = 'features' - AND object_type IS NOT NULL - AND g.srs_id = srs.srs_id - AND g.srs_id = c.srs_id - AND g.z IN (0, 1, 2) - AND g.m = 0 + AND object_type IS NOT NULL + AND g.srs_id = srs.srs_id + AND g.srs_id = c.srs_id + AND g.z IN (0, 1, 2) + AND g.m = 0 LIMIT $layer; """ ) @@ -117,7 +117,10 @@ function gpkgtable(db, ; layer=1) # According to https://www.geopackage.org/spec/#r14 # The table_name column value in a gpkg_contents table row # SHALL contain the name of a SQLite table or view. - gpkgbinary = DBInterface.execute(db, "SELECT * FROM $tablename;") + tableinfo = SQLite.tableinfo(db, tablename) + # "pk" (either zero for columns that are not part of the primary key, or the 1-based index of the column within the primary key) + columns = [name for (name, pk) in zip(tableinfo.name, tableinfo.pk) if pk == 0] + gpkgbinary = DBInterface.execute(db, "SELECT $(join(columns, ',')) FROM $tablename;") # length of the GeoPackageBinaryHeader headerlen = 0 geomcollection = map(gpkgbinary) do row diff --git a/src/extra/gpkg/wkb.jl b/src/extra/gpkg/wkb.jl index 34799e2..aa35f95 100644 --- a/src/extra/gpkg/wkb.jl +++ b/src/extra/gpkg/wkb.jl @@ -3,7 +3,7 @@ # ------------------------------------------------------------------ """ - GeoIO.meshfromwkb(io, crs, wkbtype, haszextent, wkbbyteswap); + GeoIO.gpkgwkbgeom(io, crs, wkbtype, zextent, wkbbyteswap); Flavors of WKB supported: From 57f3a66ff69251bf04471f39bf87a6b2b93aaa72 Mon Sep 17 00:00:00 2001 From: jph6366 Date: Tue, 4 Nov 2025 14:14:56 -0500 Subject: [PATCH 49/90] update dependency versions --- Project.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Project.toml b/Project.toml index 1a1ae78..2973f5f 100644 --- a/Project.toml +++ b/Project.toml @@ -42,7 +42,7 @@ ArchGDAL = "0.10" CSV = "0.10" Colors = "0.12, 0.13" CommonDataModel = "0.2, 0.3, 0.4" -CoordRefSystems = "0.18" +CoordRefSystems = "0.19" DBInterface = "2.6" FileIO = "1.16" Format = "1.3" @@ -56,7 +56,7 @@ GeoTables = "1.25" GslibIO = "1.6" ImageIO = "0.6" JSON3 = "1.14" -Meshes = "0.54" +Meshes = "0.55" NCDatasets = "0.13, 0.14" PlyIO = "1.1" PrecompileTools = "1.2" From 72335cdea5582380017616ee3382007c36c2f01a Mon Sep 17 00:00:00 2001 From: jph6366 Date: Tue, 4 Nov 2025 16:29:39 -0500 Subject: [PATCH 50/90] removed consts used for writer --- src/extra/gpkg.jl | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/src/extra/gpkg.jl b/src/extra/gpkg.jl index f4dc043..a83d124 100644 --- a/src/extra/gpkg.jl +++ b/src/extra/gpkg.jl @@ -2,18 +2,5 @@ # Licensed under the MIT License. See LICENSE in the project root. # ------------------------------------------------------------------ -# According to https://www.geopackage.org/spec/#r2 -# a GeoPackage should contain "GPKG" in ASCII in -# "application_id" field of SQLite db header -const GPKG_APPLICATION_ID = Int(0x47504B47) -const GPKG_1_4_VERSION = 10400 - -# If the geometry type_name value is "GEOMETRY" -# then the feature table geometry column MAY contain: -# geometries of any allowed geometry type. -function SQLite.sqlitetype_(::Type{Vector{UInt8}}) - return "GEOMETRY" -end - include("gpkg/wkb.jl") include("gpkg/read.jl") From 97ca89fb011357408704d1a6dbe8bd9b2f918623 Mon Sep 17 00:00:00 2001 From: jph6366 Date: Thu, 6 Nov 2025 21:44:59 -0500 Subject: [PATCH 51/90] trimming fat; refactor wkb; clean comments --- src/extra/gpkg/read.jl | 48 ++------------- src/extra/gpkg/wkb.jl | 131 ++++++++++++----------------------------- 2 files changed, 43 insertions(+), 136 deletions(-) diff --git a/src/extra/gpkg/read.jl b/src/extra/gpkg/read.jl index 827fae6..29107b3 100644 --- a/src/extra/gpkg/read.jl +++ b/src/extra/gpkg/read.jl @@ -164,54 +164,16 @@ function gpkgtable(db, ; layer=1) # right-shift moves the E bits by one to align with the least significant bits # results in a 3-bit unsigned integer envelope = (flag & (0x07 << 1)) >> 1 - - # No envelope [] (space saving slower indexing option), 0 bytes - envelopecode = 0 - if !iszero(envelope) - if isone(envelope) - # 2D envelope [minx, maxx, miny, maxy], 32 bytes - envelopecode = 2 - elseif isequal(2, envelope) - # 2D+Z envelope [minx, maxx, miny, maxy, minz, maxz], 48 bytes - envelopecode = 3 - elseif isequal(3, envelope) - # 2D+M envelope [minx, maxx, miny, maxy, minm, maxm] (is not supported) - envelopecode = 4 - elseif isequal(4, envelope) - # 2D+ZM envelope [minx, maxx, miny, maxy, minz, maxz, minm, maxm] (is not supported) - envelopecode = 5 - else - # 5-7: invalid - throw(ErrorException("exceeded dimensional limit for geometry")) - end - end - + # calculate GeoPackageBinaryHeader size in byte stream given extent of envelope: - # byte[2] magic + byte[1] version + byte[1] flags + byte[4] srs_id + byte[16×NumberOfAxes] envelope - headerlen = 8 + 8 * 2 * envelopecode + # envelope is [minx, maxx, miny, maxy, minz, maxz], 48 bytes or envelope is [minx, maxx, miny, maxy], 32 bytes or no envelope, 0 bytes + # byte[2] magic + byte[1] version + byte[1] flags + byte[4] srs_id + byte[(8*2)×(x,y{,z})] envelope + headerlen = iszero(envelope) ? 8 : 8 + 8 * 2 * (envelope + 1) # Skip reading the double[] envelope and start reading Well-Known Binary geometry seek(io, headerlen) - # Note that Julia does not convert the endianness for you. - # Use ntoh or ltoh for this purpose. - wkbbyteswap = isone(read(io, UInt8)) ? ltoh : ntoh - - wkbtypebits = read(io, UInt32) - zextent = isequal(envelopecode, 3) - # if the geometry type is a 3D geometry type - if zextent - wkbtype = !iszero(wkbtypebits & ewkbmaskbits) ? - # if WKBGeometry is specified in `extended WKB` remove the the dimensionality bit flag that indicates a Z dimension - wkbtypebits & 0x000000F : - # if WKBGeometry is specified in `ISO WKB` and we simply subtract the round number added to the type number that indicates a Z dimensions. - wkbtypebits - 1000 - else - # WKBGeometry is specified in 'Standard WKB' - wkbtype = wkbtypebits - end - - geom = gpkgwkbgeom(io, crs, wkbtype, zextent, wkbbyteswap) + geom = gpkgwkbgeom(io, crs) if !isnothing(geom) # returns a tuple of the corresponding aspatial attributes and the geometries for each row in the feature table return (NamedTuple(rowvals), geom) diff --git a/src/extra/gpkg/wkb.jl b/src/extra/gpkg/wkb.jl index aa35f95..6fe5240 100644 --- a/src/extra/gpkg/wkb.jl +++ b/src/extra/gpkg/wkb.jl @@ -2,104 +2,45 @@ # Licensed under the MIT License. See LICENSE in the project root. # ------------------------------------------------------------------ -""" - GeoIO.gpkgwkbgeom(io, crs, wkbtype, zextent, wkbbyteswap); - -Flavors of WKB supported: - -- Standard WKB supports two-dimensional geometry, and is a proper subset of both extended WKB and ISO WKB. - -## Reading WKB Geometry BLOB - -``` julia - io = IOBuffer(WKBGeometryBLOB) - # load byte order in order to transport geometry - # easily between systems of different endianness - wkbByteOrder = read(io, UInt8) - # load simple feature wkb geometry types supported - wkbGeometryType = read(io, UInt32) - # wkb points do not have a `numPoints` field - if wkbGeometryType != 1 - # load number of geometries in geometry set - numWkbGeometries = read(io, UInt32) - end - # load in WKBGeometry that contain geometry values - # w/ double precision numbers in the coordinates - # that are also subject to byte order rules. - wkbEndianness = isone(wkbByteOrder) ? ltoh : ntoh - # Note that Julia does not convert the endianness for you. - wkbGeometryBlob = wkbEndianness(read(io, Vector{UInt8})) -``` - -- Extended WKB allows applications to optionally add extra dimensions, and optionally embed an SRID - 99-402 was a short-lived extension to SFSQL 1.1 that used a high-bit flag -to indicate the presence of Z coordinates in a WKB geometry. - When the optional wkbSRID is added to the wkbType, an SRID number is inserted after the wkbType number. -⚠ This optional behaviour is not supported and will likely fail loading this variant - -- ISO WKB allows for higher dimensional geometries. -SQL/MM Part 3 and SFSQL 1.2 use offsets of 1000 (Z), 2000 (M) and 3000 (ZM) -⚠ only offsets of 1000 are recognized and supported and will likely fali loading this variant - -Other systems like GDAL supports three wkbVariants deviated from the wkbStandard -- wkbVariantOldOgc:: Old-style 99-402 -- wkbVariantIso:: SFSQL 1.2 and ISO SQL/MM Part 3 -- wkbVariantPostGIS1::PostGIS 1.X -PostGIS supports a wider range of types (for example, CircularString, CurvePolygon) - -GeoIO GeoPackage reader supports wkbGeometryType using the SFSQL 1.2 use offset of 1000 (Z) and SFSQL 1.1 that used a high-bit flag, restricting some optional features - -GeoIO GeoPackage writer supports X,Y,Z coordinate offset of 1000 (Z) for wkbGeometryType - -## Example - -``` julia - geoms = [] - io = IOBuffer - for row in wkbGeometryColumn - wkbbyteswap = isone(read(io, UInt8)) ? ltoh : ntoh - wkbtype = read(io, UInt32) - crs = LatLon{WGS84Latest} - haszextent = false - push!(geoms, gpkgwkbgeom(io, crs, wkbtype, haszextent, wkbbyteswap)) - end -``` - -""" +function gpkgwkbgeom(io, crs) + # Note: the coordinates are subject to byte order rules specified here + wkbbyteswap = isone(read(io, UInt8)) ? ltoh : ntoh + wkbtypebits = read(io, UInt32) + # reader supports wkbGeometryType using the SFSQL 1.2 use offset of 1000 (Z) and SFSQL 1.1 that used a high-bit flag 0x80000000 (Z) + if _haszextent(wkbtypebits) + wkbtype = iszero(wkbtypebits & 0x80000000) ? wkbtypebits - 1000 : wkbtypebits & 0x7FFFFFFF + zextent = true + else + wkbtype = wkbtypebits + zextent = false + end -# According to https://www.geopackage.org/spec/#r20 -# GeoPackage SHALL store feature table geometries with the basic simple feature geometry types. -# Geometry Types (Normative): https://www.geopackage.org/spec140/index.html#geometry_types -# Note: this implementation supports (Core) Geometry Type Codes -function gpkgwkbgeom(io, crs, wkbtype, zextent, bswap) if wkbtype > 3 # 4 - 7 [MultiPoint, MultiLinestring, MultiPolygon, GeometryCollection] - elems = wkbmulti(io, crs, zextent, bswap) - Multi(elems) + geoms = wkbmulti(io, crs, zextent, wkbbyteswap) + Multi(geoms) else # 0 - 3 [Geometry, Point, Linestring, Polygon] - elem = wkbsingle(io, crs, wkbtype, zextent, bswap) - elem + wkbsingle(io, crs, wkbtype, zextent, wkbbyteswap) end end -#------- -# WKB GEOMETRY READER UTILS -#------- - # read single features from Well-Known Binary IO Buffer and return Concrete Geometry function wkbsingle(io, crs, wkbtype, zextent, bswap) if wkbtype == 1 geom = wkbcoordinate(io, zextent, bswap) - # return point given coordinates with respect to CRS + + # return point given coordinates Point(crs(geom...)) elseif wkbtype == 2 geom = wkblinestring(io, zextent, bswap) if length(geom) >= 2 && first(geom) != last(geom) - # return open polygonal chain from sequence of points w.r.t CRS + + # return open polygonal chain from sequence of points Rope([Point(crs(points...)) for points in geom]...) else - # return closed polygonal chain from sequence of points w.r.t CRS + + # return closed polygonal chain from sequence of points Ring([Point(crs(points...)) for points in geom[1:(end - 1)]]...) end elseif wkbtype == 3 @@ -110,10 +51,9 @@ function wkbsingle(io, crs, wkbtype, zextent, bswap) end Ring(coords) end - outerring = first(rings) - holes = length(rings) > 1 ? rings[2:end] : Ring[] - # return polygonal area with outer ring, and optional inner rings - PolyArea(outerring, holes...) + + # return a polygonal area from rings + PolyArea(rings) end end @@ -148,16 +88,21 @@ function wkbmulti(io, crs, zextent, bswap) wkbbswap = isone(read(io, UInt8)) ? ltoh : ntoh wkbtypebits = read(io, UInt32) # if 2D+Z the dimensionality flag is present - if zextent - if iszero(wkbtypebits & 0x80000000) - # Extended WKB: wkbtype + 0x80000000 = wkbTypeZ - wkbtypebits = wkbtypebits & 0x000000F - elseif wkbtypebits > 1000 - # ISO WKB: wkbType + 1000 = wkbTypeZ - wkbtypebits = wkbtypebits - 1000 - end + if _haszextent(wkbtypebits) + wkbtype = iszero(wkbtypebits & 0x80000000) ? wkbtypebits - 1000 : wkbtypebits & 0x7FFFFFFF + zextent = true + else + zextent = false end - wkbsingle(io, crs, wkbtypebits, zextent, wkbbswap) + # read single geometry from Well-Known Binary IO Buffer + wkbsingle(io, crs, wkbtype, zextent, wkbbswap) end geoms +end + +function _haszextent(wkbtypebits) + if !iszero(wkbtypebits & 0x80000000) || wkbtypebits > 1000 + return true + end + return false end \ No newline at end of file From 694853a4d8330845420d6499ba7a0576a5e51366 Mon Sep 17 00:00:00 2001 From: jph6366 Date: Fri, 7 Nov 2025 09:28:24 -0500 Subject: [PATCH 52/90] mv Multi(geoms); simpler wkb2geom; test GeoArtifacst --- src/extra/gpkg/wkb.jl | 60 +++++++++++++++---------------------------- 1 file changed, 20 insertions(+), 40 deletions(-) diff --git a/src/extra/gpkg/wkb.jl b/src/extra/gpkg/wkb.jl index 6fe5240..9d2be6b 100644 --- a/src/extra/gpkg/wkb.jl +++ b/src/extra/gpkg/wkb.jl @@ -17,8 +17,7 @@ function gpkgwkbgeom(io, crs) if wkbtype > 3 # 4 - 7 [MultiPoint, MultiLinestring, MultiPolygon, GeometryCollection] - geoms = wkbmulti(io, crs, zextent, wkbbyteswap) - Multi(geoms) + wkbmulti(io, crs, zextent, wkbbyteswap) else # 0 - 3 [Geometry, Point, Linestring, Polygon] wkbsingle(io, crs, wkbtype, zextent, wkbbyteswap) @@ -28,58 +27,41 @@ end # read single features from Well-Known Binary IO Buffer and return Concrete Geometry function wkbsingle(io, crs, wkbtype, zextent, bswap) if wkbtype == 1 - geom = wkbcoordinate(io, zextent, bswap) - - # return point given coordinates - Point(crs(geom...)) + wkb2point(io, crs, zextent, bswap) elseif wkbtype == 2 - geom = wkblinestring(io, zextent, bswap) - if length(geom) >= 2 && first(geom) != last(geom) - - # return open polygonal chain from sequence of points - Rope([Point(crs(points...)) for points in geom]...) - else - - # return closed polygonal chain from sequence of points - Ring([Point(crs(points...)) for points in geom[1:(end - 1)]]...) - end + wkb2chain(io, crs, zextent, bswap) elseif wkbtype == 3 - geom = wkbpolygon(io, zextent, bswap) - rings = map(geom) do ring - coords = map(ring) do point - Point(crs(point...)) - end - Ring(coords) - end - - # return a polygonal area from rings - PolyArea(rings) + wkb2poly(io, crs, zextent, bswap) end end -function wkbcoordinate(io, zextent, bswap) +function wkb2point(io, crs, zextent, bswap) y, x = bswap(read(io, Float64)), bswap(read(io, Float64)) if zextent z = bswap(read(io, Float64)) return x, y, z end - x, y + Point(crs(x, y)) end -function wkblinestring(io, zextent, bswap) +function wkb2chain(io, crs, zextent, bswap) npoints = bswap(read(io, UInt32)) - points = map(1:npoints) do _ - wkbcoordinate(io, zextent, bswap) + chain = map(1:npoints) do _ + wkb2point(io, crs, zextent, bswap) + end + if length(chain) >= 2 && first(chain) != last(chain) + Rope(chain) + else + Ring(chain) end - points end -function wkbpolygon(io, zextent, bswap) +function wkb2poly(io, crs, zextent, bswap) nrings = bswap(read(io, UInt32)) rings = map(1:nrings) do _ - wkblinestring(io, zextent, bswap) + wkb2chain(io, crs, zextent, bswap) end - rings + PolyArea(rings) end function wkbmulti(io, crs, zextent, bswap) @@ -90,14 +72,12 @@ function wkbmulti(io, crs, zextent, bswap) # if 2D+Z the dimensionality flag is present if _haszextent(wkbtypebits) wkbtype = iszero(wkbtypebits & 0x80000000) ? wkbtypebits - 1000 : wkbtypebits & 0x7FFFFFFF - zextent = true + wkbsingle(io, crs, wkbtype, true, wkbbswap) else - zextent = false + wkbsingle(io, crs, wkbtypebits, false, wkbbswap) end - # read single geometry from Well-Known Binary IO Buffer - wkbsingle(io, crs, wkbtype, zextent, wkbbswap) end - geoms + Multi(geoms) end function _haszextent(wkbtypebits) From b99a299a1d3a69f80c6443a42503df866bd7c6ba Mon Sep 17 00:00:00 2001 From: jph6366 Date: Sat, 8 Nov 2025 17:53:54 -0500 Subject: [PATCH 53/90] commit suggestions; modular gpkg utils; refactor varnames; handle mutliple layers --- src/GeoIO.jl | 2 +- src/extra/gpkg/read.jl | 123 ++++++++++++++++++++++------------------- src/extra/gpkg/wkb.jl | 7 +-- 3 files changed, 68 insertions(+), 64 deletions(-) diff --git a/src/GeoIO.jl b/src/GeoIO.jl index 0436598..bdaf148 100644 --- a/src/GeoIO.jl +++ b/src/GeoIO.jl @@ -76,7 +76,7 @@ const CDMEXTS = [".grib", ".nc"] const FORMATS = [ (extension=".csv", load="CSV.jl", save="CSV.jl"), (extension=".geojson", load="GeoJSON.jl", save="GeoJSON.jl"), - (extension=".gpkg", load="GeoIO.jl", save="GeoIO.jl"), + (extension=".gpkg", load="GeoIO.jl", save="ArchGDAL.jl"), (extension=".grib", load="GRIBDatasets.jl", save=""), (extension=".gslib", load="GslibIO.jl", save="GslibIO.jl"), (extension=".jpeg", load="ImageIO.jl", save="ImageIO.jl"), diff --git a/src/extra/gpkg/read.jl b/src/extra/gpkg/read.jl index 29107b3..601dd60 100644 --- a/src/extra/gpkg/read.jl +++ b/src/extra/gpkg/read.jl @@ -32,6 +32,52 @@ function assertgpkg(db) end end +function iohandle(row, geomcolumn) + # get the column of SQL Geometry Binary specified by gpkg_geometry_columns table in column_name field + blob = getproperty(row, Symbol(geomcolumn)) + # According to https://www.geopackage.org/spec/#r19 + # A GeoPackage SHALL store feature table geometries in SQL BLOBs using the Standard GeoPackageBinary format + # check the GeoPackageBinaryHeader for the first byte[2] to be 'GP' in ASCII + if blob[1:2] != [0x47, 0x50] + @warn "Missing magic 'GP' string in GPkgBinaryGeometry" + end + # create in-memory I/O stream of GeoPackage SQL Geometry Binary Format + io = IOBuffer(blob) + # skip the magic string in header + seek(io, 3) + io +end + +function skipheader(io) + # bit layout of GeoPackageBinary flags byte + # https://www.geopackage.org/spec/#flags_layout + # --------------------------------------- + # bit # 7 # 6 # 5 # 4 # 3 # 2 # 1 # 0 # + # use # R # R # X # Y # E # E # E # B # + # --------------------------------------- + # R: reserved for future use; set to 0 + # X: GeoPackageBinary type + # Y: empty geometry flag + # E: envelope contents indicator code (3-bit unsigned integer) + # B: byte order for SRS_ID and envelope values in header + flag = read(io, UInt8) + + # 0x07 is a 3-bit mask 0x00001110 + # left-shift moves the 3-bit mask by one to align with E bits in flag layout + # bitwise AND operation isolates the E bits + # right-shift moves the E bits by one to align with the least significant bits + # results in a 3-bit unsigned integer + envelope = (flag & (0x07 << 1)) >> 1 + + # calculate GeoPackageBinaryHeader size in byte stream given extent of envelope: + # envelope is [minx, maxx, miny, maxy, minz, maxz], 48 bytes or envelope is [minx, maxx, miny, maxy], 32 bytes or no envelope, 0 bytes + # byte[2] magic + byte[1] version + byte[1] flags + byte[4] srs_id + byte[(8*2)×(x,y{,z})] envelope + headerlen = iszero(envelope) ? 8 : 8 + 8 * 2 * (envelope + 1) + + # Skip reading the double[] envelope and start reading Well-Known Binary geometry + seek(io, headerlen) +end + # According to Geometry Columns Table Requirements # https://www.geopackage.org/spec/#:~:text=2.1.5.1.2.%20Table%20Data%20Values #------------------------------------------------------------------------------ @@ -73,7 +119,7 @@ function gpkgtable(db, ; layer=1) # for each vector features user data table or view. db, """ - SELECT g.table_name AS tablename, g.column_name AS columnname, + SELECT g.table_name AS tablename, g.column_name AS geomcolumn, c.srs_id AS srsid, g.z, srs.organization AS org, srs.organization_coordsys_id AS orgcoordsysid, ( SELECT type FROM sqlite_master WHERE lower(name) = lower(c.table_name) AND type IN ('table', 'view')) AS object_type FROM gpkg_geometry_columns g, gpkg_spatial_ref_sys srs @@ -90,12 +136,9 @@ function gpkgtable(db, ; layer=1) # Note: first feature table that is read specifies the CRS to be used on all feature tables resulted from SELECT statement firstrow = first(rowtable) - # According to https://www.geopackage.org/spec/#r33, feature table geometry columns - # SHALL contain geometries with the srs_id specified - # for the column by the gpkg_geometry_columns table srs_id column value. + # SHALL contain geometries with the srs_id specified for the column by the gpkg_geometry_columns table srs_id column value. srsid, org, orgcoordsysid = firstrow.srsid, firstrow.org, firstrow.orgcoordsysid - # an srs_id of 0 uses undefined Geographic CRS if iszero(srsid) crs = LatLon{WGS84Latest} @@ -113,7 +156,7 @@ function gpkgtable(db, ; layer=1) crs = Cartesian{NoDatum} end - results = map((row.tablename, row.columnname) for row in rowtable) do (tablename, columnname) + results = map((row.tablename, row.geomcolumn) for row in rowtable) do (tablename, geomcolumn) # According to https://www.geopackage.org/spec/#r14 # The table_name column value in a gpkg_contents table row # SHALL contain the name of a SQLite table or view. @@ -121,68 +164,32 @@ function gpkgtable(db, ; layer=1) # "pk" (either zero for columns that are not part of the primary key, or the 1-based index of the column within the primary key) columns = [name for (name, pk) in zip(tableinfo.name, tableinfo.pk) if pk == 0] gpkgbinary = DBInterface.execute(db, "SELECT $(join(columns, ',')) FROM $tablename;") - # length of the GeoPackageBinaryHeader - headerlen = 0 - geomcollection = map(gpkgbinary) do row + resultrow = map(gpkgbinary) do row # According to https://www.geopackage.org/spec/#r30 # A feature table or view SHALL have only one geometry column. - columnindex = findfirst(==(Symbol(columnname)), keys(row)) - rowvals = map(keys(row)[[begin:(columnindex - 1); (columnindex + 1):end]]) do key + geomindex = findfirst(==(Symbol(geomcolumn)), keys(row)) + rowvals = map(keys(row)[[begin:(geomindex - 1); (geomindex + 1):end]]) do key key, getproperty(row, key) end - # get the column of SQL Geometry Binary specified by gpkg_geometry_columns table in column_name field - blob = getproperty(row, Symbol(columnname)) - - # According to https://www.geopackage.org/spec/#r19 - # A GeoPackage SHALL store feature table geometries in SQL BLOBs using the Standard GeoPackageBinary format - # check the GeoPackageBinaryHeader for the first byte[2] to be 'GP' in ASCII - if blob[1:2] != UInt8[0x47, 0x50] - @warn "Missing magic 'GP' string in GPkgBinaryGeometry" - end - - # create in-memory I/O stream of GeoPackage SQL Geometry Binary Format starting after byte[2] magic = 0x4750; - io = IOBuffer(blob) - # skip the magic string in header - seek(io, 3) - - # bit layout of GeoPackageBinary flags byte - # https://www.geopackage.org/spec/#flags_layout - # --------------------------------------- - # bit # 7 # 6 # 5 # 4 # 3 # 2 # 1 # 0 # - # use # R # R # X # Y # E # E # E # B # - # --------------------------------------- - # R: reserved for future use; set to 0 - # X: GeoPackageBinary type - # Y: empty geometry flag - # E: envelope contents indicator code (3-bit unsigned integer) - # B: byte order for SRS_ID and envelope values in header - flag = read(io, UInt8) - - # 0x07 is a 3-bit mask 0x00001110 - # left-shift moves the 3-bit mask by one to align with E bits in flag layout - # bitwise AND operation isolates the E bits - # right-shift moves the E bits by one to align with the least significant bits - # results in a 3-bit unsigned integer - envelope = (flag & (0x07 << 1)) >> 1 - - # calculate GeoPackageBinaryHeader size in byte stream given extent of envelope: - # envelope is [minx, maxx, miny, maxy, minz, maxz], 48 bytes or envelope is [minx, maxx, miny, maxy], 32 bytes or no envelope, 0 bytes - # byte[2] magic + byte[1] version + byte[1] flags + byte[4] srs_id + byte[(8*2)×(x,y{,z})] envelope - headerlen = iszero(envelope) ? 8 : 8 + 8 * 2 * (envelope + 1) - - # Skip reading the double[] envelope and start reading Well-Known Binary geometry - seek(io, headerlen) - geom = gpkgwkbgeom(io, crs) + # check for magic 'GP' string and create IOBuffer + io = iohandle(row, geomcolumn) + # skip the GeoPackageBinaryHeader to start reading Well-Known Binary geometry + skipheader(io) + geom = wkbgeom(io, crs) if !isnothing(geom) # returns a tuple of the corresponding aspatial attributes and the geometries for each row in the feature table return (NamedTuple(rowvals), geom) end end - geomcollection + resultrow end - # efficient method for concatenating arrays of arrays + # efficient method for concatenating arrays of arrays table = reduce(vcat, results) - # unpack results to return aspatial and spatial results - getindex.(table, 1), getindex.(table, 2) + # get the aspatial attributes + aspatial = getindex.(table, 1) + # find the NamedTuple with the maximum number of fields to ensure all fields are included + maxfields = argmax(length, aspatial) + emptytuple = NamedTuple(k => "" for k in keys(maxfields)) + return [merge(emptytuple, f) for f in aspatial], getindex.(table, 2) end diff --git a/src/extra/gpkg/wkb.jl b/src/extra/gpkg/wkb.jl index 9d2be6b..18971b2 100644 --- a/src/extra/gpkg/wkb.jl +++ b/src/extra/gpkg/wkb.jl @@ -2,7 +2,7 @@ # Licensed under the MIT License. See LICENSE in the project root. # ------------------------------------------------------------------ -function gpkgwkbgeom(io, crs) +function wkbgeom(io, crs) # Note: the coordinates are subject to byte order rules specified here wkbbyteswap = isone(read(io, UInt8)) ? ltoh : ntoh wkbtypebits = read(io, UInt32) @@ -81,8 +81,5 @@ function wkbmulti(io, crs, zextent, bswap) end function _haszextent(wkbtypebits) - if !iszero(wkbtypebits & 0x80000000) || wkbtypebits > 1000 - return true - end - return false + !iszero(wkbtypebits & 0x80000000) || wkbtypebits > 1000 end \ No newline at end of file From c47aeebff63eb7eff3465519223724a607fc7382 Mon Sep 17 00:00:00 2001 From: jph6366 Date: Sat, 8 Nov 2025 17:55:23 -0500 Subject: [PATCH 54/90] remove return --- src/extra/gpkg/read.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/extra/gpkg/read.jl b/src/extra/gpkg/read.jl index 601dd60..b5d68aa 100644 --- a/src/extra/gpkg/read.jl +++ b/src/extra/gpkg/read.jl @@ -191,5 +191,5 @@ function gpkgtable(db, ; layer=1) # find the NamedTuple with the maximum number of fields to ensure all fields are included maxfields = argmax(length, aspatial) emptytuple = NamedTuple(k => "" for k in keys(maxfields)) - return [merge(emptytuple, f) for f in aspatial], getindex.(table, 2) + [merge(emptytuple, f) for f in aspatial], getindex.(table, 2) end From 1ad329c64e4de0a17e5f0ae56055836d3e8eda27 Mon Sep 17 00:00:00 2001 From: jph6366 Date: Sat, 8 Nov 2025 20:56:09 -0500 Subject: [PATCH 55/90] better multi-layer behavior --- src/extra/gpkg/read.jl | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/src/extra/gpkg/read.jl b/src/extra/gpkg/read.jl index b5d68aa..45b9805 100644 --- a/src/extra/gpkg/read.jl +++ b/src/extra/gpkg/read.jl @@ -186,10 +186,16 @@ function gpkgtable(db, ; layer=1) end # efficient method for concatenating arrays of arrays table = reduce(vcat, results) - # get the aspatial attributes - aspatial = getindex.(table, 1) - # find the NamedTuple with the maximum number of fields to ensure all fields are included - maxfields = argmax(length, aspatial) + # separate aspatial attributes and spatial geometries + aspatial, spatial = getindex.(table, 1), getindex.(table, 2) + # if only one layer is requested, return as is (otherwise integrity of fields must be maintained for Tables.schema) + if isone(layer) + return aspatial, spatial + end + # merge aspatial as a single NamedTuple to ensure all distinct fields are present for Tables.schema + maxfields = merge(first(aspatial), aspatial[2:end]...) + # replace the fields with with empty strings emptytuple = NamedTuple(k => "" for k in keys(maxfields)) - [merge(emptytuple, f) for f in aspatial], getindex.(table, 2) + # overwrite the empty tuple with the actual values from each row in aspatial values + [merge(emptytuple, f) for f in aspatial], spatial end From db61a1ccbf0d74408d5eb73275b2306d8a03d7e1 Mon Sep 17 00:00:00 2001 From: jph6366 Date: Sat, 8 Nov 2025 21:10:45 -0500 Subject: [PATCH 56/90] idioms --- src/extra/gpkg/read.jl | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/extra/gpkg/read.jl b/src/extra/gpkg/read.jl index 45b9805..9578b24 100644 --- a/src/extra/gpkg/read.jl +++ b/src/extra/gpkg/read.jl @@ -187,15 +187,15 @@ function gpkgtable(db, ; layer=1) # efficient method for concatenating arrays of arrays table = reduce(vcat, results) # separate aspatial attributes and spatial geometries - aspatial, spatial = getindex.(table, 1), getindex.(table, 2) + values, geoms = getindex.(table, 1), getindex.(table, 2) # if only one layer is requested, return as is (otherwise integrity of fields must be maintained for Tables.schema) if isone(layer) - return aspatial, spatial + return values, geoms end - # merge aspatial as a single NamedTuple to ensure all distinct fields are present for Tables.schema - maxfields = merge(first(aspatial), aspatial[2:end]...) + # merge aspatial as a single NamedTuple to ensure all distinct fields are present for Tables.schema + mergetuple = merge(first(values), values[2:end]...) # replace the fields with with empty strings - emptytuple = NamedTuple(k => "" for k in keys(maxfields)) + emptytuple = NamedTuple(k => "" for k in keys(mergetuple)) # overwrite the empty tuple with the actual values from each row in aspatial values - [merge(emptytuple, f) for f in aspatial], spatial + [merge(emptytuple, f) for f in values], geoms end From 00e988dd15cdf6705d5dbf14e3f7ec40ed10f1c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlio=20Hoffimann?= Date: Sun, 9 Nov 2025 09:03:00 -0300 Subject: [PATCH 57/90] More adjustments --- src/extra/gpkg/read.jl | 38 +++++++++++++++++++++++++++----------- 1 file changed, 27 insertions(+), 11 deletions(-) diff --git a/src/extra/gpkg/read.jl b/src/extra/gpkg/read.jl index 9578b24..40fd329 100644 --- a/src/extra/gpkg/read.jl +++ b/src/extra/gpkg/read.jl @@ -3,14 +3,16 @@ # ------------------------------------------------------------------ function gpkgread(fname; layer=1) - db = SQLite.DB(fname) - assertgpkg(db) - table, geoms = gpkgtable(db; layer) + db = gpkgdatabase(fname) + table, geoms = gpkgextract(db; layer) DBInterface.close!(db) georef(table, geoms) end -function assertgpkg(db) +function gpkgdatabase(fname) + # connect to SQLite database on disk + db = SQLite.DB(fname) + # According to https://www.geopackage.org/spec/#r6 and https://www.geopackage.org/spec/#r7 # PRAGMA integrity_check returns a single row with the value 'ok' # PRAGMA foreign_key_check (w/ no parameter value) returns an empty result set @@ -18,6 +20,7 @@ function assertgpkg(db) !(isempty(DBInterface.execute(db, "PRAGMA foreign_key_check;"))) throw(ErrorException("database integrity at risk or foreign key violation(s)")) end + # According to https://www.geopackage.org/spec/#r10 and https://www.geopackage.org/spec/#r13 # A GeoPackage SHALL include a 'gpkg_spatial_ref_sys' table and a 'gpkg_contents table' if first(DBInterface.execute( @@ -30,21 +33,27 @@ function assertgpkg(db) )).n != 2 throw(ErrorException("missing required metadata tables in the GeoPackage SQL database")) end + + db end function iohandle(row, geomcolumn) # get the column of SQL Geometry Binary specified by gpkg_geometry_columns table in column_name field blob = getproperty(row, Symbol(geomcolumn)) + # According to https://www.geopackage.org/spec/#r19 # A GeoPackage SHALL store feature table geometries in SQL BLOBs using the Standard GeoPackageBinary format # check the GeoPackageBinaryHeader for the first byte[2] to be 'GP' in ASCII if blob[1:2] != [0x47, 0x50] @warn "Missing magic 'GP' string in GPkgBinaryGeometry" end + # create in-memory I/O stream of GeoPackage SQL Geometry Binary Format io = IOBuffer(blob) + # skip the magic string in header seek(io, 3) + io end @@ -108,7 +117,7 @@ end # Requirement 146: The srs_id value in a gpkg_geometry_columns table row # SHALL match the srs_id column value from the corresponding row in the # gpkg_contents table. -function gpkgtable(db, ; layer=1) +function gpkgextract(db, ; layer=1) rowtable = DBInterface.execute( # According to https://www.geopackage.org/spec/#r16 # Values of the gpkg_contents table srs_id column @@ -136,23 +145,21 @@ function gpkgtable(db, ; layer=1) # Note: first feature table that is read specifies the CRS to be used on all feature tables resulted from SELECT statement firstrow = first(rowtable) + # According to https://www.geopackage.org/spec/#r33, feature table geometry columns # SHALL contain geometries with the srs_id specified for the column by the gpkg_geometry_columns table srs_id column value. srsid, org, orgcoordsysid = firstrow.srsid, firstrow.org, firstrow.orgcoordsysid - # an srs_id of 0 uses undefined Geographic CRS + + # identify coordinate reference system from srsid if iszero(srsid) crs = LatLon{WGS84Latest} - # if srs_id not equal to -1 elseif srsid != -1 - # CRS assigned by the org if org == "EPSG" - # orgcoordsysid is the numeric id of the CRS assigned by the org crs = CoordRefSystems.get(EPSG{orgcoordsysid}) elseif org == "ESRI" crs = CoordRefSystems.get(ERSI{orgcoordsysid}) end - else - # defaults to undefined Cartesian CRS + else # defaults to Cartesian CRS with no datum crs = Cartesian{NoDatum} end @@ -161,6 +168,7 @@ function gpkgtable(db, ; layer=1) # The table_name column value in a gpkg_contents table row # SHALL contain the name of a SQLite table or view. tableinfo = SQLite.tableinfo(db, tablename) + # "pk" (either zero for columns that are not part of the primary key, or the 1-based index of the column within the primary key) columns = [name for (name, pk) in zip(tableinfo.name, tableinfo.pk) if pk == 0] gpkgbinary = DBInterface.execute(db, "SELECT $(join(columns, ',')) FROM $tablename;") @@ -174,8 +182,10 @@ function gpkgtable(db, ; layer=1) # check for magic 'GP' string and create IOBuffer io = iohandle(row, geomcolumn) + # skip the GeoPackageBinaryHeader to start reading Well-Known Binary geometry skipheader(io) + geom = wkbgeom(io, crs) if !isnothing(geom) # returns a tuple of the corresponding aspatial attributes and the geometries for each row in the feature table @@ -184,18 +194,24 @@ function gpkgtable(db, ; layer=1) end resultrow end + # efficient method for concatenating arrays of arrays table = reduce(vcat, results) + # separate aspatial attributes and spatial geometries values, geoms = getindex.(table, 1), getindex.(table, 2) + # if only one layer is requested, return as is (otherwise integrity of fields must be maintained for Tables.schema) if isone(layer) return values, geoms end + # merge aspatial as a single NamedTuple to ensure all distinct fields are present for Tables.schema mergetuple = merge(first(values), values[2:end]...) + # replace the fields with with empty strings emptytuple = NamedTuple(k => "" for k in keys(mergetuple)) + # overwrite the empty tuple with the actual values from each row in aspatial values [merge(emptytuple, f) for f in values], geoms end From 00a5290354a5448145747a50c3b93f7f7bcda9c9 Mon Sep 17 00:00:00 2001 From: jph6366 Date: Sun, 9 Nov 2025 09:10:29 -0500 Subject: [PATCH 58/90] k-th layers, hoist IOBlubber --- src/extra/gpkg/read.jl | 86 +++++++++++++++++------------------------- 1 file changed, 34 insertions(+), 52 deletions(-) diff --git a/src/extra/gpkg/read.jl b/src/extra/gpkg/read.jl index 40fd329..9b7318a 100644 --- a/src/extra/gpkg/read.jl +++ b/src/extra/gpkg/read.jl @@ -41,18 +41,19 @@ function iohandle(row, geomcolumn) # get the column of SQL Geometry Binary specified by gpkg_geometry_columns table in column_name field blob = getproperty(row, Symbol(geomcolumn)) + # create in-memory I/O stream of GeoPackage SQL Geometry Binary Format + io = IOBuffer(blob) + # According to https://www.geopackage.org/spec/#r19 # A GeoPackage SHALL store feature table geometries in SQL BLOBs using the Standard GeoPackageBinary format # check the GeoPackageBinaryHeader for the first byte[2] to be 'GP' in ASCII - if blob[1:2] != [0x47, 0x50] + magic = read(io, UInt16) + if magic != 0x5047 # 'GP' @warn "Missing magic 'GP' string in GPkgBinaryGeometry" end - # create in-memory I/O stream of GeoPackage SQL Geometry Binary Format - io = IOBuffer(blob) - - # skip the magic string in header - seek(io, 3) + # byte[1] version: 8-bit unsigned integer, 0 = version 1 + read(io, UInt8) io end @@ -139,7 +140,7 @@ function gpkgextract(db, ; layer=1) AND g.srs_id = c.srs_id AND g.z IN (0, 1, 2) AND g.m = 0 - LIMIT $layer; + LIMIT 1 OFFSET ($layer-1); """ ) @@ -148,6 +149,7 @@ function gpkgextract(db, ; layer=1) # According to https://www.geopackage.org/spec/#r33, feature table geometry columns # SHALL contain geometries with the srs_id specified for the column by the gpkg_geometry_columns table srs_id column value. + tablename, geomcolumn = firstrow.tablename, firstrow.geomcolumn srsid, org, orgcoordsysid = firstrow.srsid, firstrow.org, firstrow.orgcoordsysid # identify coordinate reference system from srsid @@ -163,55 +165,35 @@ function gpkgextract(db, ; layer=1) crs = Cartesian{NoDatum} end - results = map((row.tablename, row.geomcolumn) for row in rowtable) do (tablename, geomcolumn) - # According to https://www.geopackage.org/spec/#r14 - # The table_name column value in a gpkg_contents table row - # SHALL contain the name of a SQLite table or view. - tableinfo = SQLite.tableinfo(db, tablename) - - # "pk" (either zero for columns that are not part of the primary key, or the 1-based index of the column within the primary key) - columns = [name for (name, pk) in zip(tableinfo.name, tableinfo.pk) if pk == 0] - gpkgbinary = DBInterface.execute(db, "SELECT $(join(columns, ',')) FROM $tablename;") - resultrow = map(gpkgbinary) do row - # According to https://www.geopackage.org/spec/#r30 - # A feature table or view SHALL have only one geometry column. - geomindex = findfirst(==(Symbol(geomcolumn)), keys(row)) - rowvals = map(keys(row)[[begin:(geomindex - 1); (geomindex + 1):end]]) do key - key, getproperty(row, key) - end - - # check for magic 'GP' string and create IOBuffer - io = iohandle(row, geomcolumn) - - # skip the GeoPackageBinaryHeader to start reading Well-Known Binary geometry - skipheader(io) - - geom = wkbgeom(io, crs) - if !isnothing(geom) - # returns a tuple of the corresponding aspatial attributes and the geometries for each row in the feature table - return (NamedTuple(rowvals), geom) - end + # According to https://www.geopackage.org/spec/#r14 + # The table_name column value in a gpkg_contents table row + # SHALL contain the name of a SQLite table or view. + tableinfo = SQLite.tableinfo(db, tablename) + + # "pk" (either zero for columns that are not part of the primary key, or the 1-based index of the column within the primary key) + columns = [name for (name, pk) in zip(tableinfo.name, tableinfo.pk) if pk == 0] + gpkgbinary = DBInterface.execute(db, "SELECT $(join(columns, ',')) FROM $tablename;") + table = map(gpkgbinary) do row + # According to https://www.geopackage.org/spec/#r30 + # A feature table or view SHALL have only one geometry column. + geomindex = findfirst(==(Symbol(geomcolumn)), keys(row)) + rowvals = map(keys(row)[[begin:(geomindex - 1); (geomindex + 1):end]]) do key + key, getproperty(row, key) end - resultrow - end - # efficient method for concatenating arrays of arrays - table = reduce(vcat, results) + # check for magic 'GP' string and create IOBuffer + io = iohandle(row, geomcolumn) - # separate aspatial attributes and spatial geometries - values, geoms = getindex.(table, 1), getindex.(table, 2) + # skip the GeoPackageBinaryHeader to start reading Well-Known Binary geometry + skipheader(io) - # if only one layer is requested, return as is (otherwise integrity of fields must be maintained for Tables.schema) - if isone(layer) - return values, geoms + geom = wkbgeom(io, crs) + if !isnothing(geom) + # returns a tuple of the corresponding aspatial attributes and the geometries for each row in the feature table + return (NamedTuple(rowvals), geom) + end end - # merge aspatial as a single NamedTuple to ensure all distinct fields are present for Tables.schema - mergetuple = merge(first(values), values[2:end]...) - - # replace the fields with with empty strings - emptytuple = NamedTuple(k => "" for k in keys(mergetuple)) - - # overwrite the empty tuple with the actual values from each row in aspatial values - [merge(emptytuple, f) for f in values], geoms + # aspatial attributes and geometries + getindex.(table, 1), getindex.(table, 2) end From fb6ef04ee41fa77cf5f209ba6bce4a3c24e91fd2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlio=20Hoffimann?= Date: Sun, 9 Nov 2025 16:15:17 -0300 Subject: [PATCH 59/90] More adjustments --- src/extra/gpkg/read.jl | 101 ++++++++++++++++++----------------------- 1 file changed, 45 insertions(+), 56 deletions(-) diff --git a/src/extra/gpkg/read.jl b/src/extra/gpkg/read.jl index 9b7318a..f52d095 100644 --- a/src/extra/gpkg/read.jl +++ b/src/extra/gpkg/read.jl @@ -37,57 +37,6 @@ function gpkgdatabase(fname) db end -function iohandle(row, geomcolumn) - # get the column of SQL Geometry Binary specified by gpkg_geometry_columns table in column_name field - blob = getproperty(row, Symbol(geomcolumn)) - - # create in-memory I/O stream of GeoPackage SQL Geometry Binary Format - io = IOBuffer(blob) - - # According to https://www.geopackage.org/spec/#r19 - # A GeoPackage SHALL store feature table geometries in SQL BLOBs using the Standard GeoPackageBinary format - # check the GeoPackageBinaryHeader for the first byte[2] to be 'GP' in ASCII - magic = read(io, UInt16) - if magic != 0x5047 # 'GP' - @warn "Missing magic 'GP' string in GPkgBinaryGeometry" - end - - # byte[1] version: 8-bit unsigned integer, 0 = version 1 - read(io, UInt8) - - io -end - -function skipheader(io) - # bit layout of GeoPackageBinary flags byte - # https://www.geopackage.org/spec/#flags_layout - # --------------------------------------- - # bit # 7 # 6 # 5 # 4 # 3 # 2 # 1 # 0 # - # use # R # R # X # Y # E # E # E # B # - # --------------------------------------- - # R: reserved for future use; set to 0 - # X: GeoPackageBinary type - # Y: empty geometry flag - # E: envelope contents indicator code (3-bit unsigned integer) - # B: byte order for SRS_ID and envelope values in header - flag = read(io, UInt8) - - # 0x07 is a 3-bit mask 0x00001110 - # left-shift moves the 3-bit mask by one to align with E bits in flag layout - # bitwise AND operation isolates the E bits - # right-shift moves the E bits by one to align with the least significant bits - # results in a 3-bit unsigned integer - envelope = (flag & (0x07 << 1)) >> 1 - - # calculate GeoPackageBinaryHeader size in byte stream given extent of envelope: - # envelope is [minx, maxx, miny, maxy, minz, maxz], 48 bytes or envelope is [minx, maxx, miny, maxy], 32 bytes or no envelope, 0 bytes - # byte[2] magic + byte[1] version + byte[1] flags + byte[4] srs_id + byte[(8*2)×(x,y{,z})] envelope - headerlen = iszero(envelope) ? 8 : 8 + 8 * 2 * (envelope + 1) - - # Skip reading the double[] envelope and start reading Well-Known Binary geometry - seek(io, headerlen) -end - # According to Geometry Columns Table Requirements # https://www.geopackage.org/spec/#:~:text=2.1.5.1.2.%20Table%20Data%20Values #------------------------------------------------------------------------------ @@ -181,11 +130,8 @@ function gpkgextract(db, ; layer=1) key, getproperty(row, key) end - # check for magic 'GP' string and create IOBuffer - io = iohandle(row, geomcolumn) - - # skip the GeoPackageBinaryHeader to start reading Well-Known Binary geometry - skipheader(io) + # create IOBuffer and seek geometry binary data + io = geomio(row, geomcolumn) geom = wkbgeom(io, crs) if !isnothing(geom) @@ -197,3 +143,46 @@ function gpkgextract(db, ; layer=1) # aspatial attributes and geometries getindex.(table, 1), getindex.(table, 2) end + +function geomio(row, geomcolumn) + # get the column of SQL Geometry Binary specified by gpkg_geometry_columns table in column_name field + io = IOBuffer(getproperty(row, Symbol(geomcolumn))) + + # According to https://www.geopackage.org/spec/#r19 + # A GeoPackage SHALL store feature table geometries in SQL BLOBs using the Standard GeoPackageBinary format + # check the GeoPackageBinaryHeader for the first byte[2] to be 'GP' in ASCII + read(io, UInt16) != 0x5047 || @warn "Missing magic 'GP' string in GPkgBinaryGeometry" + + # byte[1] version: 8-bit unsigned integer, 0 = version 1 + read(io, UInt8) + + # bit layout of GeoPackageBinary flags byte + # https://www.geopackage.org/spec/#flags_layout + # --------------------------------------- + # bit # 7 # 6 # 5 # 4 # 3 # 2 # 1 # 0 # + # use # R # R # X # Y # E # E # E # B # + # --------------------------------------- + # R: reserved for future use; set to 0 + # X: GeoPackageBinary type + # Y: empty geometry flag + # E: envelope contents indicator code (3-bit unsigned integer) + # B: byte order for SRS_ID and envelope values in header + flag = read(io, UInt8) + + # 0x07 is a 3-bit mask 0x00001110 + # left-shift moves the 3-bit mask by one to align with E bits in flag layout + # bitwise AND operation isolates the E bits + # right-shift moves the E bits by one to align with the least significant bits + # results in a 3-bit unsigned integer + envelope = (flag & (0x07 << 1)) >> 1 + + # calculate GeoPackageBinaryHeader size in byte stream given extent of envelope: + # envelope is [minx, maxx, miny, maxy, minz, maxz], 48 bytes or envelope is [minx, maxx, miny, maxy], 32 bytes or no envelope, 0 bytes + # byte[2] magic + byte[1] version + byte[1] flags + byte[4] srs_id + byte[(8*2)×(x,y{,z})] envelope + headerlen = iszero(envelope) ? 8 : 8 + 8 * 2 * (envelope + 1) + + # Skip reading the double[] envelope and start reading Well-Known Binary geometry + seek(io, headerlen) + + io +end From 79560a3201c84b7db5df1090ba5854ccf7962bfa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlio=20Hoffimann?= Date: Sun, 9 Nov 2025 16:36:54 -0300 Subject: [PATCH 60/90] More adjustments --- src/extra/gpkg/read.jl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/extra/gpkg/read.jl b/src/extra/gpkg/read.jl index f52d095..7f62c7e 100644 --- a/src/extra/gpkg/read.jl +++ b/src/extra/gpkg/read.jl @@ -131,9 +131,9 @@ function gpkgextract(db, ; layer=1) end # create IOBuffer and seek geometry binary data - io = geomio(row, geomcolumn) + buff = wkbgeombuffer(row, geomcolumn) - geom = wkbgeom(io, crs) + geom = wkbgeom(buff, crs) if !isnothing(geom) # returns a tuple of the corresponding aspatial attributes and the geometries for each row in the feature table return (NamedTuple(rowvals), geom) @@ -144,7 +144,7 @@ function gpkgextract(db, ; layer=1) getindex.(table, 1), getindex.(table, 2) end -function geomio(row, geomcolumn) +function wkbgeombuffer(row, geomcolumn) # get the column of SQL Geometry Binary specified by gpkg_geometry_columns table in column_name field io = IOBuffer(getproperty(row, Symbol(geomcolumn))) From f3dce85d78c488db3b56e4af4b13cafcfa9510c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlio=20Hoffimann?= Date: Sun, 9 Nov 2025 16:44:02 -0300 Subject: [PATCH 61/90] More adjustments --- src/extra/gpkg/read.jl | 2 +- src/extra/gpkg/wkb.jl | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/extra/gpkg/read.jl b/src/extra/gpkg/read.jl index 7f62c7e..c7a2ff3 100644 --- a/src/extra/gpkg/read.jl +++ b/src/extra/gpkg/read.jl @@ -133,7 +133,7 @@ function gpkgextract(db, ; layer=1) # create IOBuffer and seek geometry binary data buff = wkbgeombuffer(row, geomcolumn) - geom = wkbgeom(buff, crs) + geom = wkb2geom(buff, crs) if !isnothing(geom) # returns a tuple of the corresponding aspatial attributes and the geometries for each row in the feature table return (NamedTuple(rowvals), geom) diff --git a/src/extra/gpkg/wkb.jl b/src/extra/gpkg/wkb.jl index 18971b2..39fa9e9 100644 --- a/src/extra/gpkg/wkb.jl +++ b/src/extra/gpkg/wkb.jl @@ -2,7 +2,7 @@ # Licensed under the MIT License. See LICENSE in the project root. # ------------------------------------------------------------------ -function wkbgeom(io, crs) +function wkb2geom(io, crs) # Note: the coordinates are subject to byte order rules specified here wkbbyteswap = isone(read(io, UInt8)) ? ltoh : ntoh wkbtypebits = read(io, UInt32) @@ -17,15 +17,15 @@ function wkbgeom(io, crs) if wkbtype > 3 # 4 - 7 [MultiPoint, MultiLinestring, MultiPolygon, GeometryCollection] - wkbmulti(io, crs, zextent, wkbbyteswap) + wkb2multi(io, crs, zextent, wkbbyteswap) else # 0 - 3 [Geometry, Point, Linestring, Polygon] - wkbsingle(io, crs, wkbtype, zextent, wkbbyteswap) + wkb2single(io, crs, wkbtype, zextent, wkbbyteswap) end end # read single features from Well-Known Binary IO Buffer and return Concrete Geometry -function wkbsingle(io, crs, wkbtype, zextent, bswap) +function wkb2single(io, crs, wkbtype, zextent, bswap) if wkbtype == 1 wkb2point(io, crs, zextent, bswap) elseif wkbtype == 2 @@ -64,7 +64,7 @@ function wkb2poly(io, crs, zextent, bswap) PolyArea(rings) end -function wkbmulti(io, crs, zextent, bswap) +function wkb2multi(io, crs, zextent, bswap) ngeoms = bswap(read(io, UInt32)) geoms = map(1:ngeoms) do _ wkbbswap = isone(read(io, UInt8)) ? ltoh : ntoh @@ -72,9 +72,9 @@ function wkbmulti(io, crs, zextent, bswap) # if 2D+Z the dimensionality flag is present if _haszextent(wkbtypebits) wkbtype = iszero(wkbtypebits & 0x80000000) ? wkbtypebits - 1000 : wkbtypebits & 0x7FFFFFFF - wkbsingle(io, crs, wkbtype, true, wkbbswap) + wkb2single(io, crs, wkbtype, true, wkbbswap) else - wkbsingle(io, crs, wkbtypebits, false, wkbbswap) + wkb2single(io, crs, wkbtypebits, false, wkbbswap) end end Multi(geoms) From 12a25975b40fe396d110d46ca2da7317e41cb119 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlio=20Hoffimann?= Date: Sun, 9 Nov 2025 16:47:32 -0300 Subject: [PATCH 62/90] More adjustments --- src/extra/gpkg/read.jl | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/extra/gpkg/read.jl b/src/extra/gpkg/read.jl index c7a2ff3..5037fee 100644 --- a/src/extra/gpkg/read.jl +++ b/src/extra/gpkg/read.jl @@ -146,15 +146,15 @@ end function wkbgeombuffer(row, geomcolumn) # get the column of SQL Geometry Binary specified by gpkg_geometry_columns table in column_name field - io = IOBuffer(getproperty(row, Symbol(geomcolumn))) + buff = IOBuffer(getproperty(row, Symbol(geomcolumn))) # According to https://www.geopackage.org/spec/#r19 # A GeoPackage SHALL store feature table geometries in SQL BLOBs using the Standard GeoPackageBinary format # check the GeoPackageBinaryHeader for the first byte[2] to be 'GP' in ASCII - read(io, UInt16) != 0x5047 || @warn "Missing magic 'GP' string in GPkgBinaryGeometry" + read(buff, UInt16) != 0x5047 || @warn "Missing magic 'GP' string in GPkgBinaryGeometry" # byte[1] version: 8-bit unsigned integer, 0 = version 1 - read(io, UInt8) + read(buff, UInt8) # bit layout of GeoPackageBinary flags byte # https://www.geopackage.org/spec/#flags_layout @@ -167,7 +167,7 @@ function wkbgeombuffer(row, geomcolumn) # Y: empty geometry flag # E: envelope contents indicator code (3-bit unsigned integer) # B: byte order for SRS_ID and envelope values in header - flag = read(io, UInt8) + flag = read(buff, UInt8) # 0x07 is a 3-bit mask 0x00001110 # left-shift moves the 3-bit mask by one to align with E bits in flag layout @@ -182,7 +182,7 @@ function wkbgeombuffer(row, geomcolumn) headerlen = iszero(envelope) ? 8 : 8 + 8 * 2 * (envelope + 1) # Skip reading the double[] envelope and start reading Well-Known Binary geometry - seek(io, headerlen) + seek(buff, headerlen) - io + buff end From 76193b6be31bedf3f2bb4d9ce38c10d8953ad914 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlio=20Hoffimann?= Date: Sun, 9 Nov 2025 16:53:28 -0300 Subject: [PATCH 63/90] More adjustments --- src/extra/gpkg/wkb.jl | 48 +++++++++++++++++++++---------------------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/src/extra/gpkg/wkb.jl b/src/extra/gpkg/wkb.jl index 39fa9e9..07e1f97 100644 --- a/src/extra/gpkg/wkb.jl +++ b/src/extra/gpkg/wkb.jl @@ -2,10 +2,10 @@ # Licensed under the MIT License. See LICENSE in the project root. # ------------------------------------------------------------------ -function wkb2geom(io, crs) +function wkb2geom(buff, crs) # Note: the coordinates are subject to byte order rules specified here - wkbbyteswap = isone(read(io, UInt8)) ? ltoh : ntoh - wkbtypebits = read(io, UInt32) + wkbbyteswap = isone(read(buff, UInt8)) ? ltoh : ntoh + wkbtypebits = read(buff, UInt32) # reader supports wkbGeometryType using the SFSQL 1.2 use offset of 1000 (Z) and SFSQL 1.1 that used a high-bit flag 0x80000000 (Z) if _haszextent(wkbtypebits) wkbtype = iszero(wkbtypebits & 0x80000000) ? wkbtypebits - 1000 : wkbtypebits & 0x7FFFFFFF @@ -17,37 +17,37 @@ function wkb2geom(io, crs) if wkbtype > 3 # 4 - 7 [MultiPoint, MultiLinestring, MultiPolygon, GeometryCollection] - wkb2multi(io, crs, zextent, wkbbyteswap) + wkb2multi(buff, crs, zextent, wkbbyteswap) else # 0 - 3 [Geometry, Point, Linestring, Polygon] - wkb2single(io, crs, wkbtype, zextent, wkbbyteswap) + wkb2single(buff, crs, wkbtype, zextent, wkbbyteswap) end end # read single features from Well-Known Binary IO Buffer and return Concrete Geometry -function wkb2single(io, crs, wkbtype, zextent, bswap) +function wkb2single(buff, crs, wkbtype, zextent, bswap) if wkbtype == 1 - wkb2point(io, crs, zextent, bswap) + wkb2point(buff, crs, zextent, bswap) elseif wkbtype == 2 - wkb2chain(io, crs, zextent, bswap) + wkb2chain(buff, crs, zextent, bswap) elseif wkbtype == 3 - wkb2poly(io, crs, zextent, bswap) + wkb2poly(buff, crs, zextent, bswap) end end -function wkb2point(io, crs, zextent, bswap) - y, x = bswap(read(io, Float64)), bswap(read(io, Float64)) +function wkb2point(buff, crs, zextent, bswap) + y, x = bswap(read(buff, Float64)), bswap(read(buff, Float64)) if zextent - z = bswap(read(io, Float64)) + z = bswap(read(buff, Float64)) return x, y, z end Point(crs(x, y)) end -function wkb2chain(io, crs, zextent, bswap) - npoints = bswap(read(io, UInt32)) +function wkb2chain(buff, crs, zextent, bswap) + npoints = bswap(read(buff, UInt32)) chain = map(1:npoints) do _ - wkb2point(io, crs, zextent, bswap) + wkb2point(buff, crs, zextent, bswap) end if length(chain) >= 2 && first(chain) != last(chain) Rope(chain) @@ -56,25 +56,25 @@ function wkb2chain(io, crs, zextent, bswap) end end -function wkb2poly(io, crs, zextent, bswap) - nrings = bswap(read(io, UInt32)) +function wkb2poly(buff, crs, zextent, bswap) + nrings = bswap(read(buff, UInt32)) rings = map(1:nrings) do _ - wkb2chain(io, crs, zextent, bswap) + wkb2chain(buff, crs, zextent, bswap) end PolyArea(rings) end -function wkb2multi(io, crs, zextent, bswap) - ngeoms = bswap(read(io, UInt32)) +function wkb2multi(buff, crs, zextent, bswap) + ngeoms = bswap(read(buff, UInt32)) geoms = map(1:ngeoms) do _ - wkbbswap = isone(read(io, UInt8)) ? ltoh : ntoh - wkbtypebits = read(io, UInt32) + wkbbswap = isone(read(buff, UInt8)) ? ltoh : ntoh + wkbtypebits = read(buff, UInt32) # if 2D+Z the dimensionality flag is present if _haszextent(wkbtypebits) wkbtype = iszero(wkbtypebits & 0x80000000) ? wkbtypebits - 1000 : wkbtypebits & 0x7FFFFFFF - wkb2single(io, crs, wkbtype, true, wkbbswap) + wkb2single(buff, crs, wkbtype, true, wkbbswap) else - wkb2single(io, crs, wkbtypebits, false, wkbbswap) + wkb2single(buff, crs, wkbtypebits, false, wkbbswap) end end Multi(geoms) From 6aba38893d4fb9e161e343a6a7869fde7f71ff73 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlio=20Hoffimann?= Date: Sun, 9 Nov 2025 16:54:47 -0300 Subject: [PATCH 64/90] More adjustments --- src/extra/gpkg/read.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/extra/gpkg/read.jl b/src/extra/gpkg/read.jl index 5037fee..e385eb1 100644 --- a/src/extra/gpkg/read.jl +++ b/src/extra/gpkg/read.jl @@ -67,7 +67,7 @@ end # Requirement 146: The srs_id value in a gpkg_geometry_columns table row # SHALL match the srs_id column value from the corresponding row in the # gpkg_contents table. -function gpkgextract(db, ; layer=1) +function gpkgextract(db; layer=1) rowtable = DBInterface.execute( # According to https://www.geopackage.org/spec/#r16 # Values of the gpkg_contents table srs_id column From 3dd8bda0f097962790ca25d77e53c617c835e6ab Mon Sep 17 00:00:00 2001 From: jph6366 Date: Sun, 9 Nov 2025 17:41:58 -0500 Subject: [PATCH 65/90] hotfixes and revert adjustment --- src/extra/gpkg/read.jl | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/src/extra/gpkg/read.jl b/src/extra/gpkg/read.jl index 5037fee..371b481 100644 --- a/src/extra/gpkg/read.jl +++ b/src/extra/gpkg/read.jl @@ -93,16 +93,15 @@ function gpkgextract(db, ; layer=1) """ ) - # Note: first feature table that is read specifies the CRS to be used on all feature tables resulted from SELECT statement - firstrow = first(rowtable) + features = first(rowtable) # According to https://www.geopackage.org/spec/#r33, feature table geometry columns # SHALL contain geometries with the srs_id specified for the column by the gpkg_geometry_columns table srs_id column value. - tablename, geomcolumn = firstrow.tablename, firstrow.geomcolumn - srsid, org, orgcoordsysid = firstrow.srsid, firstrow.org, firstrow.orgcoordsysid + tablename, geomcolumn = features.tablename, features.geomcolumn + srsid, org, orgcoordsysid = features.srsid, features.org, features.orgcoordsysid - # identify coordinate reference system from srsid if iszero(srsid) + # srs_id of 0 SHALL be used for undefined geographic coordinate reference system based on WGS84 crs = LatLon{WGS84Latest} elseif srsid != -1 if org == "EPSG" @@ -110,7 +109,8 @@ function gpkgextract(db, ; layer=1) elseif org == "ESRI" crs = CoordRefSystems.get(ERSI{orgcoordsysid}) end - else # defaults to Cartesian CRS with no datum + else + # srs_id of -1 SHALL be used for undefined Cartesian coordinate reference systems crs = Cartesian{NoDatum} end @@ -134,10 +134,9 @@ function gpkgextract(db, ; layer=1) buff = wkbgeombuffer(row, geomcolumn) geom = wkb2geom(buff, crs) - if !isnothing(geom) - # returns a tuple of the corresponding aspatial attributes and the geometries for each row in the feature table - return (NamedTuple(rowvals), geom) - end + + # returns a tuple of the corresponding aspatial attributes and the geometries for each row in the feature table + return (NamedTuple(rowvals), geom) end # aspatial attributes and geometries @@ -151,7 +150,7 @@ function wkbgeombuffer(row, geomcolumn) # According to https://www.geopackage.org/spec/#r19 # A GeoPackage SHALL store feature table geometries in SQL BLOBs using the Standard GeoPackageBinary format # check the GeoPackageBinaryHeader for the first byte[2] to be 'GP' in ASCII - read(buff, UInt16) != 0x5047 || @warn "Missing magic 'GP' string in GPkgBinaryGeometry" + read(buff, UInt16) == 0x5047 ? nothing : @warn "Missing magic 'GP' string in GPkgBinaryGeometry" # byte[1] version: 8-bit unsigned integer, 0 = version 1 read(buff, UInt8) @@ -183,6 +182,4 @@ function wkbgeombuffer(row, geomcolumn) # Skip reading the double[] envelope and start reading Well-Known Binary geometry seek(buff, headerlen) - - buff end From c563881c214ff78c9519afcaec41557e60b07f15 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlio=20Hoffimann?= Date: Mon, 10 Nov 2025 06:54:16 -0300 Subject: [PATCH 66/90] Simplify CRS logic --- src/extra/gpkg/read.jl | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/extra/gpkg/read.jl b/src/extra/gpkg/read.jl index 4c617a0..18903be 100644 --- a/src/extra/gpkg/read.jl +++ b/src/extra/gpkg/read.jl @@ -100,18 +100,16 @@ function gpkgextract(db; layer=1) tablename, geomcolumn = features.tablename, features.geomcolumn srsid, org, orgcoordsysid = features.srsid, features.org, features.orgcoordsysid - if iszero(srsid) - # srs_id of 0 SHALL be used for undefined geographic coordinate reference system based on WGS84 + if srsid == 0 crs = LatLon{WGS84Latest} - elseif srsid != -1 + elseif srsid == -1 + crs = Cartesian{NoDatum} + else if org == "EPSG" crs = CoordRefSystems.get(EPSG{orgcoordsysid}) elseif org == "ESRI" crs = CoordRefSystems.get(ERSI{orgcoordsysid}) end - else - # srs_id of -1 SHALL be used for undefined Cartesian coordinate reference systems - crs = Cartesian{NoDatum} end # According to https://www.geopackage.org/spec/#r14 From 0b5223331c7dfddaa18f15eb108778be7ad6c633 Mon Sep 17 00:00:00 2001 From: jph6366 Date: Mon, 10 Nov 2025 12:01:51 -0500 Subject: [PATCH 67/90] featuretable varname; magic gp. --- src/extra/gpkg/read.jl | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/extra/gpkg/read.jl b/src/extra/gpkg/read.jl index 18903be..d556665 100644 --- a/src/extra/gpkg/read.jl +++ b/src/extra/gpkg/read.jl @@ -68,7 +68,7 @@ end # SHALL match the srs_id column value from the corresponding row in the # gpkg_contents table. function gpkgextract(db; layer=1) - rowtable = DBInterface.execute( + featuretable = DBInterface.execute( # According to https://www.geopackage.org/spec/#r16 # Values of the gpkg_contents table srs_id column # SHALL reference values in the gpkg_spatial_ref_sys table srs_id column @@ -93,7 +93,7 @@ function gpkgextract(db; layer=1) """ ) - features = first(rowtable) + features = first(featuretable) # According to https://www.geopackage.org/spec/#r33, feature table geometry columns # SHALL contain geometries with the srs_id specified for the column by the gpkg_geometry_columns table srs_id column value. @@ -148,7 +148,9 @@ function wkbgeombuffer(row, geomcolumn) # According to https://www.geopackage.org/spec/#r19 # A GeoPackage SHALL store feature table geometries in SQL BLOBs using the Standard GeoPackageBinary format # check the GeoPackageBinaryHeader for the first byte[2] to be 'GP' in ASCII - read(buff, UInt16) == 0x5047 ? nothing : @warn "Missing magic 'GP' string in GPkgBinaryGeometry" + if read(buff, UInt16) == 0x5047 + @warn "Missing magic 'GP' string in GPkgBinaryGeometry" + end # byte[1] version: 8-bit unsigned integer, 0 = version 1 read(buff, UInt8) From ed7ac0c65eb6af29db6bf04189a6f4208114dc60 Mon Sep 17 00:00:00 2001 From: jph6366 Date: Mon, 10 Nov 2025 15:25:09 -0500 Subject: [PATCH 68/90] redo adjustment; simplfiy query; remove unneccessary param. --- src/extra/gpkg/read.jl | 60 +++++++++++++++++++++--------------------- src/extra/gpkg/wkb.jl | 4 +-- 2 files changed, 32 insertions(+), 32 deletions(-) diff --git a/src/extra/gpkg/read.jl b/src/extra/gpkg/read.jl index d556665..4e9cc41 100644 --- a/src/extra/gpkg/read.jl +++ b/src/extra/gpkg/read.jl @@ -68,33 +68,35 @@ end # SHALL match the srs_id column value from the corresponding row in the # gpkg_contents table. function gpkgextract(db; layer=1) - featuretable = DBInterface.execute( - # According to https://www.geopackage.org/spec/#r16 - # Values of the gpkg_contents table srs_id column - # SHALL reference values in the gpkg_spatial_ref_sys table srs_id column - # According to https://www.geopackage.org/spec/#r18 - # The gpkg_contents table SHALL contain a row - # with a lowercase data_type column value of "features" - # for each vector features user data table or view. - db, - """ - SELECT g.table_name AS tablename, g.column_name AS geomcolumn, - c.srs_id AS srsid, g.z, srs.organization AS org, srs.organization_coordsys_id AS orgcoordsysid, - ( SELECT type FROM sqlite_master WHERE lower(name) = lower(c.table_name) AND type IN ('table', 'view')) AS object_type - FROM gpkg_geometry_columns g, gpkg_spatial_ref_sys srs - JOIN gpkg_contents c ON ( g.table_name = c.table_name ) - WHERE c.data_type = 'features' - AND object_type IS NOT NULL - AND g.srs_id = srs.srs_id - AND g.srs_id = c.srs_id - AND g.z IN (0, 1, 2) - AND g.m = 0 - LIMIT 1 OFFSET ($layer-1); - """ + # get the first (and only) feature table returned in sqlite query results + # sqlite query results are forward-only iterators where each row is only valid when `iterate(rows)` is called + features = first( + DBInterface.execute( + # According to https://www.geopackage.org/spec/#r16 + # Values of the gpkg_contents table srs_id column + # SHALL reference values in the gpkg_spatial_ref_sys table srs_id column + # According to https://www.geopackage.org/spec/#r18 + # The gpkg_contents table SHALL contain a row + # with a lowercase data_type column value of "features" + # for each vector features user data table or view. + db, + """ + SELECT g.table_name AS tablename, g.column_name AS geomcolumn, + c.srs_id AS srsid, g.z, srs.organization AS org, srs.organization_coordsys_id AS orgcoordsysid, + ( SELECT type FROM sqlite_master WHERE lower(name) = lower(c.table_name) AND type IN ('table', 'view')) AS object_type + FROM gpkg_geometry_columns g, gpkg_spatial_ref_sys srs + JOIN gpkg_contents c ON ( g.table_name = c.table_name ) + WHERE c.data_type = 'features' + AND object_type IS NOT NULL + AND g.srs_id = srs.srs_id + AND g.srs_id = c.srs_id + AND g.z IN (0, 1, 2) + AND g.m = 0 + LIMIT 1 OFFSET ($layer-1); + """ + ) ) - features = first(featuretable) - # According to https://www.geopackage.org/spec/#r33, feature table geometry columns # SHALL contain geometries with the srs_id specified for the column by the gpkg_geometry_columns table srs_id column value. tablename, geomcolumn = features.tablename, features.geomcolumn @@ -124,7 +126,7 @@ function gpkgextract(db; layer=1) # According to https://www.geopackage.org/spec/#r30 # A feature table or view SHALL have only one geometry column. geomindex = findfirst(==(Symbol(geomcolumn)), keys(row)) - rowvals = map(keys(row)[[begin:(geomindex - 1); (geomindex + 1):end]]) do key + values = map(keys(row)[[begin:(geomindex - 1); (geomindex + 1):end]]) do key key, getproperty(row, key) end @@ -134,7 +136,7 @@ function gpkgextract(db; layer=1) geom = wkb2geom(buff, crs) # returns a tuple of the corresponding aspatial attributes and the geometries for each row in the feature table - return (NamedTuple(rowvals), geom) + return (NamedTuple(values), geom) end # aspatial attributes and geometries @@ -148,9 +150,7 @@ function wkbgeombuffer(row, geomcolumn) # According to https://www.geopackage.org/spec/#r19 # A GeoPackage SHALL store feature table geometries in SQL BLOBs using the Standard GeoPackageBinary format # check the GeoPackageBinaryHeader for the first byte[2] to be 'GP' in ASCII - if read(buff, UInt16) == 0x5047 - @warn "Missing magic 'GP' string in GPkgBinaryGeometry" - end + read(buff, UInt16) == 0x5047 || @warn "Missing magic 'GP' string in GPkgBinaryGeometry" # byte[1] version: 8-bit unsigned integer, 0 = version 1 read(buff, UInt8) diff --git a/src/extra/gpkg/wkb.jl b/src/extra/gpkg/wkb.jl index 07e1f97..108feed 100644 --- a/src/extra/gpkg/wkb.jl +++ b/src/extra/gpkg/wkb.jl @@ -17,7 +17,7 @@ function wkb2geom(buff, crs) if wkbtype > 3 # 4 - 7 [MultiPoint, MultiLinestring, MultiPolygon, GeometryCollection] - wkb2multi(buff, crs, zextent, wkbbyteswap) + wkb2multi(buff, crs, wkbbyteswap) else # 0 - 3 [Geometry, Point, Linestring, Polygon] wkb2single(buff, crs, wkbtype, zextent, wkbbyteswap) @@ -64,7 +64,7 @@ function wkb2poly(buff, crs, zextent, bswap) PolyArea(rings) end -function wkb2multi(buff, crs, zextent, bswap) +function wkb2multi(buff, crs, bswap) ngeoms = bswap(read(buff, UInt32)) geoms = map(1:ngeoms) do _ wkbbswap = isone(read(buff, UInt8)) ? ltoh : ntoh From 99a37da2e8e6bcf5dd3e9a68ab9f636fc489adcc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlio=20Hoffimann?= Date: Tue, 11 Nov 2025 13:34:52 -0300 Subject: [PATCH 69/90] More adjustments --- src/extra/gpkg/read.jl | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/extra/gpkg/read.jl b/src/extra/gpkg/read.jl index 4e9cc41..25fbdd9 100644 --- a/src/extra/gpkg/read.jl +++ b/src/extra/gpkg/read.jl @@ -82,7 +82,7 @@ function gpkgextract(db; layer=1) db, """ SELECT g.table_name AS tablename, g.column_name AS geomcolumn, - c.srs_id AS srsid, g.z, srs.organization AS org, srs.organization_coordsys_id AS orgcoordsysid, + c.srs_id AS srsid, g.z, srs.organization AS org, srs.organization_coordsys_id AS code, ( SELECT type FROM sqlite_master WHERE lower(name) = lower(c.table_name) AND type IN ('table', 'view')) AS object_type FROM gpkg_geometry_columns g, gpkg_spatial_ref_sys srs JOIN gpkg_contents c ON ( g.table_name = c.table_name ) @@ -99,24 +99,26 @@ function gpkgextract(db; layer=1) # According to https://www.geopackage.org/spec/#r33, feature table geometry columns # SHALL contain geometries with the srs_id specified for the column by the gpkg_geometry_columns table srs_id column value. - tablename, geomcolumn = features.tablename, features.geomcolumn - srsid, org, orgcoordsysid = features.srsid, features.org, features.orgcoordsysid - + org = features.org + code = features.code + srsid = features.srsid if srsid == 0 crs = LatLon{WGS84Latest} elseif srsid == -1 crs = Cartesian{NoDatum} else if org == "EPSG" - crs = CoordRefSystems.get(EPSG{orgcoordsysid}) + crs = CoordRefSystems.get(EPSG{code}) elseif org == "ESRI" - crs = CoordRefSystems.get(ERSI{orgcoordsysid}) + crs = CoordRefSystems.get(ERSI{code}) end end # According to https://www.geopackage.org/spec/#r14 # The table_name column value in a gpkg_contents table row # SHALL contain the name of a SQLite table or view. + tablename = features.tablename + geomcolumn = features.geomcolumn tableinfo = SQLite.tableinfo(db, tablename) # "pk" (either zero for columns that are not part of the primary key, or the 1-based index of the column within the primary key) From 435f6bb5c4c5b5d12ce975807e2a2fd96c205c02 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlio=20Hoffimann?= Date: Tue, 11 Nov 2025 13:47:25 -0300 Subject: [PATCH 70/90] More adjusments --- src/extra/gpkg/read.jl | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/extra/gpkg/read.jl b/src/extra/gpkg/read.jl index 25fbdd9..ffd210f 100644 --- a/src/extra/gpkg/read.jl +++ b/src/extra/gpkg/read.jl @@ -70,7 +70,7 @@ end function gpkgextract(db; layer=1) # get the first (and only) feature table returned in sqlite query results # sqlite query results are forward-only iterators where each row is only valid when `iterate(rows)` is called - features = first( + metadata = first( DBInterface.execute( # According to https://www.geopackage.org/spec/#r16 # Values of the gpkg_contents table srs_id column @@ -99,9 +99,9 @@ function gpkgextract(db; layer=1) # According to https://www.geopackage.org/spec/#r33, feature table geometry columns # SHALL contain geometries with the srs_id specified for the column by the gpkg_geometry_columns table srs_id column value. - org = features.org - code = features.code - srsid = features.srsid + org = metadata.org + code = metadata.code + srsid = metadata.srsid if srsid == 0 crs = LatLon{WGS84Latest} elseif srsid == -1 @@ -117,8 +117,8 @@ function gpkgextract(db; layer=1) # According to https://www.geopackage.org/spec/#r14 # The table_name column value in a gpkg_contents table row # SHALL contain the name of a SQLite table or view. - tablename = features.tablename - geomcolumn = features.geomcolumn + tablename = metadata.tablename + geomcolumn = metadata.geomcolumn tableinfo = SQLite.tableinfo(db, tablename) # "pk" (either zero for columns that are not part of the primary key, or the 1-based index of the column within the primary key) From 7751cd1a2df146249c86287bfb38a6bcd4bec6e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlio=20Hoffimann?= Date: Tue, 11 Nov 2025 13:54:46 -0300 Subject: [PATCH 71/90] More adjustments --- src/extra/gpkg/read.jl | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/src/extra/gpkg/read.jl b/src/extra/gpkg/read.jl index ffd210f..39c0bd3 100644 --- a/src/extra/gpkg/read.jl +++ b/src/extra/gpkg/read.jl @@ -40,6 +40,15 @@ end # According to Geometry Columns Table Requirements # https://www.geopackage.org/spec/#:~:text=2.1.5.1.2.%20Table%20Data%20Values #------------------------------------------------------------------------------ +# Requirement 16: https://www.geopackage.org/spec/#r16 +# Values of the gpkg_contents table srs_id column +# SHALL reference values in the gpkg_spatial_ref_sys table srs_id column +# +# Requirement 18: https://www.geopackage.org/spec/#r18 +# The gpkg_contents table SHALL contain a row +# with a lowercase data_type column value of "features" +# for each vector features user data table or view. +# # Requirement 22: gpkg_geometry_columns table # SHALL contain one row record for the geometry column # in each vector feature data table @@ -52,9 +61,6 @@ end # SHALL be the name of a column in the table or view specified by the table_name # column value for that row. # -# Requirement 25: The geometry_type_name value in a gpkg_geometry_columns row -# SHALL be one of the uppercase geometry type names specified - # Requirement 26: The srs_id value in a gpkg_geometry_columns table row # SHALL be an srs_id column value from the gpkg_spatial_ref_sys table. # @@ -69,16 +75,8 @@ end # gpkg_contents table. function gpkgextract(db; layer=1) # get the first (and only) feature table returned in sqlite query results - # sqlite query results are forward-only iterators where each row is only valid when `iterate(rows)` is called metadata = first( DBInterface.execute( - # According to https://www.geopackage.org/spec/#r16 - # Values of the gpkg_contents table srs_id column - # SHALL reference values in the gpkg_spatial_ref_sys table srs_id column - # According to https://www.geopackage.org/spec/#r18 - # The gpkg_contents table SHALL contain a row - # with a lowercase data_type column value of "features" - # for each vector features user data table or view. db, """ SELECT g.table_name AS tablename, g.column_name AS geomcolumn, From c1a2ea720b6dae13804d65d2f06a454b6255fcc4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlio=20Hoffimann?= Date: Tue, 11 Nov 2025 13:57:53 -0300 Subject: [PATCH 72/90] More adjustments --- src/extra/gpkg/read.jl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/extra/gpkg/read.jl b/src/extra/gpkg/read.jl index 39c0bd3..28c8424 100644 --- a/src/extra/gpkg/read.jl +++ b/src/extra/gpkg/read.jl @@ -26,9 +26,9 @@ function gpkgdatabase(fname) if first(DBInterface.execute( db, """ - SELECT COUNT(*) AS n FROM sqlite_master WHERE - name IN ('gpkg_spatial_ref_sys', 'gpkg_contents') AND - type IN ('table', 'view'); + SELECT COUNT(*) AS n FROM sqlite_master WHERE + name IN ('gpkg_spatial_ref_sys', 'gpkg_contents') AND + type IN ('table', 'view'); """ )).n != 2 throw(ErrorException("missing required metadata tables in the GeoPackage SQL database")) From 7399fa28e1ad680d1d95b374302c426b7d9de42d Mon Sep 17 00:00:00 2001 From: jph6366 Date: Tue, 11 Nov 2025 17:36:02 -0500 Subject: [PATCH 73/90] simply select stmt; zextent refactor --- src/extra/gpkg/read.jl | 23 ++++++----------------- src/extra/gpkg/wkb.jl | 12 ++++-------- 2 files changed, 10 insertions(+), 25 deletions(-) diff --git a/src/extra/gpkg/read.jl b/src/extra/gpkg/read.jl index 28c8424..f57b6c8 100644 --- a/src/extra/gpkg/read.jl +++ b/src/extra/gpkg/read.jl @@ -61,14 +61,8 @@ end # SHALL be the name of a column in the table or view specified by the table_name # column value for that row. # -# Requirement 26: The srs_id value in a gpkg_geometry_columns table row -# SHALL be an srs_id column value from the gpkg_spatial_ref_sys table. -# -# Requirement 27: The z value in a gpkg_geometry_columns table row SHALL be one -# of 0, 1, or 2. -# -# Requirement 28: The m value in a gpkg_geometry_columns table row SHALL be one -# of 0, 1, or 2. +# Requirement 25: The geometry_type_name value in a gpkg_geometry_columns row +# SHALL be one of the uppercase geometry type names specified # # Requirement 146: The srs_id value in a gpkg_geometry_columns table row # SHALL match the srs_id column value from the corresponding row in the @@ -80,27 +74,22 @@ function gpkgextract(db; layer=1) db, """ SELECT g.table_name AS tablename, g.column_name AS geomcolumn, - c.srs_id AS srsid, g.z, srs.organization AS org, srs.organization_coordsys_id AS code, - ( SELECT type FROM sqlite_master WHERE lower(name) = lower(c.table_name) AND type IN ('table', 'view')) AS object_type + c.srs_id AS srsid, srs.organization AS org, srs.organization_coordsys_id AS code FROM gpkg_geometry_columns g, gpkg_spatial_ref_sys srs JOIN gpkg_contents c ON ( g.table_name = c.table_name ) WHERE c.data_type = 'features' - AND object_type IS NOT NULL - AND g.srs_id = srs.srs_id AND g.srs_id = c.srs_id - AND g.z IN (0, 1, 2) - AND g.m = 0 LIMIT 1 OFFSET ($layer-1); """ ) ) - # According to https://www.geopackage.org/spec/#r33, feature table geometry columns # SHALL contain geometries with the srs_id specified for the column by the gpkg_geometry_columns table srs_id column value. org = metadata.org code = metadata.code srsid = metadata.srsid - if srsid == 0 + + if srsid == 0 || srsid == 4326 crs = LatLon{WGS84Latest} elseif srsid == -1 crs = Cartesian{NoDatum} @@ -108,7 +97,7 @@ function gpkgextract(db; layer=1) if org == "EPSG" crs = CoordRefSystems.get(EPSG{code}) elseif org == "ESRI" - crs = CoordRefSystems.get(ERSI{code}) + crs = CoordRefSystems.get(ESRI{code}) end end diff --git a/src/extra/gpkg/wkb.jl b/src/extra/gpkg/wkb.jl index 108feed..eb32ecc 100644 --- a/src/extra/gpkg/wkb.jl +++ b/src/extra/gpkg/wkb.jl @@ -7,7 +7,7 @@ function wkb2geom(buff, crs) wkbbyteswap = isone(read(buff, UInt8)) ? ltoh : ntoh wkbtypebits = read(buff, UInt32) # reader supports wkbGeometryType using the SFSQL 1.2 use offset of 1000 (Z) and SFSQL 1.1 that used a high-bit flag 0x80000000 (Z) - if _haszextent(wkbtypebits) + if !iszero(wkbtypebits & 0x80000000) || wkbtypebits > 1000 wkbtype = iszero(wkbtypebits & 0x80000000) ? wkbtypebits - 1000 : wkbtypebits & 0x7FFFFFFF zextent = true else @@ -17,7 +17,7 @@ function wkb2geom(buff, crs) if wkbtype > 3 # 4 - 7 [MultiPoint, MultiLinestring, MultiPolygon, GeometryCollection] - wkb2multi(buff, crs, wkbbyteswap) + wkb2multi(buff, crs, zextent, wkbbyteswap) else # 0 - 3 [Geometry, Point, Linestring, Polygon] wkb2single(buff, crs, wkbtype, zextent, wkbbyteswap) @@ -64,13 +64,13 @@ function wkb2poly(buff, crs, zextent, bswap) PolyArea(rings) end -function wkb2multi(buff, crs, bswap) +function wkb2multi(buff, crs, zextent, bswap) ngeoms = bswap(read(buff, UInt32)) geoms = map(1:ngeoms) do _ wkbbswap = isone(read(buff, UInt8)) ? ltoh : ntoh wkbtypebits = read(buff, UInt32) # if 2D+Z the dimensionality flag is present - if _haszextent(wkbtypebits) + if zextent wkbtype = iszero(wkbtypebits & 0x80000000) ? wkbtypebits - 1000 : wkbtypebits & 0x7FFFFFFF wkb2single(buff, crs, wkbtype, true, wkbbswap) else @@ -78,8 +78,4 @@ function wkb2multi(buff, crs, bswap) end end Multi(geoms) -end - -function _haszextent(wkbtypebits) - !iszero(wkbtypebits & 0x80000000) || wkbtypebits > 1000 end \ No newline at end of file From ef477bb75cff89d0bff6ffac9eadc14a6a368f0b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlio=20Hoffimann?= Date: Thu, 13 Nov 2025 07:19:54 -0300 Subject: [PATCH 74/90] Refactor wkb2geom --- src/extra/gpkg/wkb.jl | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/src/extra/gpkg/wkb.jl b/src/extra/gpkg/wkb.jl index eb32ecc..37e3d58 100644 --- a/src/extra/gpkg/wkb.jl +++ b/src/extra/gpkg/wkb.jl @@ -3,24 +3,27 @@ # ------------------------------------------------------------------ function wkb2geom(buff, crs) - # Note: the coordinates are subject to byte order rules specified here - wkbbyteswap = isone(read(buff, UInt8)) ? ltoh : ntoh - wkbtypebits = read(buff, UInt32) - # reader supports wkbGeometryType using the SFSQL 1.2 use offset of 1000 (Z) and SFSQL 1.1 that used a high-bit flag 0x80000000 (Z) - if !iszero(wkbtypebits & 0x80000000) || wkbtypebits > 1000 - wkbtype = iszero(wkbtypebits & 0x80000000) ? wkbtypebits - 1000 : wkbtypebits & 0x7FFFFFFF + byteswap = isone(read(buff, UInt8)) ? ltoh : ntoh + typebits = read(buff, UInt32) + # supports wkbGeometryType according to + # SFSQL 1.1 high-bit flag 0x80000000 (Z) + # SFSQL 1.2 offset of 1000 (Z) + sfsql11 = iszero(typebits & 0x80000000) + sfsql12 = typebits > 1000 + if !sfsql11 || sfsql12 + wkbtype = sfsql11 ? typebits - 1000 : typebits & 0x7FFFFFFF zextent = true else - wkbtype = wkbtypebits + wkbtype = typebits zextent = false end if wkbtype > 3 # 4 - 7 [MultiPoint, MultiLinestring, MultiPolygon, GeometryCollection] - wkb2multi(buff, crs, zextent, wkbbyteswap) + wkb2multi(buff, crs, zextent, byteswap) else # 0 - 3 [Geometry, Point, Linestring, Polygon] - wkb2single(buff, crs, wkbtype, zextent, wkbbyteswap) + wkb2single(buff, crs, wkbtype, zextent, byteswap) end end From b912fa89ffd3e5b58f0dc64868536bb85e19b271 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlio=20Hoffimann?= Date: Thu, 13 Nov 2025 07:22:25 -0300 Subject: [PATCH 75/90] More adjustments --- src/extra/gpkg/wkb.jl | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/src/extra/gpkg/wkb.jl b/src/extra/gpkg/wkb.jl index 37e3d58..4211e81 100644 --- a/src/extra/gpkg/wkb.jl +++ b/src/extra/gpkg/wkb.jl @@ -28,29 +28,29 @@ function wkb2geom(buff, crs) end # read single features from Well-Known Binary IO Buffer and return Concrete Geometry -function wkb2single(buff, crs, wkbtype, zextent, bswap) +function wkb2single(buff, crs, wkbtype, zextent, byteswap) if wkbtype == 1 - wkb2point(buff, crs, zextent, bswap) + wkb2point(buff, crs, zextent, byteswap) elseif wkbtype == 2 - wkb2chain(buff, crs, zextent, bswap) + wkb2chain(buff, crs, zextent, byteswap) elseif wkbtype == 3 - wkb2poly(buff, crs, zextent, bswap) + wkb2poly(buff, crs, zextent, byteswap) end end -function wkb2point(buff, crs, zextent, bswap) - y, x = bswap(read(buff, Float64)), bswap(read(buff, Float64)) +function wkb2point(buff, crs, zextent, byteswap) + y, x = byteswap(read(buff, Float64)), byteswap(read(buff, Float64)) if zextent - z = bswap(read(buff, Float64)) + z = byteswap(read(buff, Float64)) return x, y, z end Point(crs(x, y)) end -function wkb2chain(buff, crs, zextent, bswap) - npoints = bswap(read(buff, UInt32)) +function wkb2chain(buff, crs, zextent, byteswap) + npoints = byteswap(read(buff, UInt32)) chain = map(1:npoints) do _ - wkb2point(buff, crs, zextent, bswap) + wkb2point(buff, crs, zextent, byteswap) end if length(chain) >= 2 && first(chain) != last(chain) Rope(chain) @@ -59,16 +59,16 @@ function wkb2chain(buff, crs, zextent, bswap) end end -function wkb2poly(buff, crs, zextent, bswap) - nrings = bswap(read(buff, UInt32)) +function wkb2poly(buff, crs, zextent, byteswap) + nrings = byteswap(read(buff, UInt32)) rings = map(1:nrings) do _ - wkb2chain(buff, crs, zextent, bswap) + wkb2chain(buff, crs, zextent, byteswap) end PolyArea(rings) end -function wkb2multi(buff, crs, zextent, bswap) - ngeoms = bswap(read(buff, UInt32)) +function wkb2multi(buff, crs, zextent, byteswap) + ngeoms = byteswap(read(buff, UInt32)) geoms = map(1:ngeoms) do _ wkbbswap = isone(read(buff, UInt8)) ? ltoh : ntoh wkbtypebits = read(buff, UInt32) From 2aa6ce02475fa0b149c6c754144a7434f240a263 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlio=20Hoffimann?= Date: Thu, 13 Nov 2025 07:24:33 -0300 Subject: [PATCH 76/90] More adjustments --- src/extra/gpkg/wkb.jl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/extra/gpkg/wkb.jl b/src/extra/gpkg/wkb.jl index 4211e81..a398671 100644 --- a/src/extra/gpkg/wkb.jl +++ b/src/extra/gpkg/wkb.jl @@ -39,7 +39,8 @@ function wkb2single(buff, crs, wkbtype, zextent, byteswap) end function wkb2point(buff, crs, zextent, byteswap) - y, x = byteswap(read(buff, Float64)), byteswap(read(buff, Float64)) + y = byteswap(read(buff, Float64)) + x = byteswap(read(buff, Float64)) if zextent z = byteswap(read(buff, Float64)) return x, y, z From 5efb822b9eb55c853405695dc9254f122f3377ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlio=20Hoffimann?= Date: Thu, 13 Nov 2025 07:30:13 -0300 Subject: [PATCH 77/90] More adjustments --- src/extra/gpkg/read.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/extra/gpkg/read.jl b/src/extra/gpkg/read.jl index f57b6c8..cc48aba 100644 --- a/src/extra/gpkg/read.jl +++ b/src/extra/gpkg/read.jl @@ -83,12 +83,12 @@ function gpkgextract(db; layer=1) """ ) ) + # According to https://www.geopackage.org/spec/#r33, feature table geometry columns # SHALL contain geometries with the srs_id specified for the column by the gpkg_geometry_columns table srs_id column value. org = metadata.org code = metadata.code srsid = metadata.srsid - if srsid == 0 || srsid == 4326 crs = LatLon{WGS84Latest} elseif srsid == -1 From f0ff9b2373133840eabbdccfe19388c1b12a86fb Mon Sep 17 00:00:00 2001 From: jph6366 Date: Thu, 13 Nov 2025 16:41:46 -0500 Subject: [PATCH 78/90] wkb refactors --- src/extra/gpkg/wkb.jl | 95 ++++++++++++++++++++++++++----------------- 1 file changed, 58 insertions(+), 37 deletions(-) diff --git a/src/extra/gpkg/wkb.jl b/src/extra/gpkg/wkb.jl index a398671..6c98779 100644 --- a/src/extra/gpkg/wkb.jl +++ b/src/extra/gpkg/wkb.jl @@ -5,53 +5,80 @@ function wkb2geom(buff, crs) byteswap = isone(read(buff, UInt8)) ? ltoh : ntoh typebits = read(buff, UInt32) - # supports wkbGeometryType according to - # SFSQL 1.1 high-bit flag 0x80000000 (Z) - # SFSQL 1.2 offset of 1000 (Z) - sfsql11 = iszero(typebits & 0x80000000) - sfsql12 = typebits > 1000 - if !sfsql11 || sfsql12 - wkbtype = sfsql11 ? typebits - 1000 : typebits & 0x7FFFFFFF - zextent = true + # Input variants of WKB supported are standard, extended, and ISO WKB geometry with Z dimensions (M/ZM not supported) + if CoordRefSystems.ncoords(crs) == 3 + # SQL/MM Part 3 and SFSQL 1.2 use offsets of 1000 (Z), 2000 (M) and 3000 (ZM) + # to indicate the present of higher dimensional coordinates in a WKB geometry + if typebits > 3007 + # 99-402 was a short-lived extension to SFSQL 1.1 that used a high-bit flag to indicate the presence of Z coordinates in a WKB geometry + # the high-bit flag 0x80000000 for Z (or 0x40000000 for M) is set and masking it off gives the standard WKB type + wkbtype = typebits & 0x7FFFFFFF + # check that the WKB typebits is not outside the SFSQL 1.2 Z encoding range (1001-1007) + elseif typebits <= 1007 + # the SFSQL 1.2 offset of 1000 (Z) is present and subtracting a round number of 1000 gives the standard WKB type + wkbtype = typebits - 1000 + else + @error "Unsupported WKB Geometry Type with M or ZM dimension encoding: $typebits" + end else + # standard WKB typebits without Z dimension encoding wkbtype = typebits - zextent = false end - if wkbtype > 3 - # 4 - 7 [MultiPoint, MultiLinestring, MultiPolygon, GeometryCollection] - wkb2multi(buff, crs, zextent, byteswap) - else + if wkbtype <= 3 # 0 - 3 [Geometry, Point, Linestring, Polygon] - wkb2single(buff, crs, wkbtype, zextent, byteswap) + wkb2single(buff, crs, wkbtype, byteswap) + else + # 4 - 7 [MultiPoint, MultiLinestring, MultiPolygon, GeometryCollection] + wkb2multi(buff, crs, byteswap) end end # read single features from Well-Known Binary IO Buffer and return Concrete Geometry -function wkb2single(buff, crs, wkbtype, zextent, byteswap) +function wkb2single(buff, crs, wkbtype, byteswap) if wkbtype == 1 - wkb2point(buff, crs, zextent, byteswap) + wkb2point(buff, crs, byteswap) elseif wkbtype == 2 - wkb2chain(buff, crs, zextent, byteswap) + wkb2chain(buff, crs, byteswap) elseif wkbtype == 3 - wkb2poly(buff, crs, zextent, byteswap) + wkb2poly(buff, crs, byteswap) + else + error("Unsupported WKB Geometry Type: $wkbtype") end end -function wkb2point(buff, crs, zextent, byteswap) - y = byteswap(read(buff, Float64)) - x = byteswap(read(buff, Float64)) - if zextent +function wkb2point(buff, crs, byteswap) + coordinates = wkb2coords(buff, crs, byteswap) + Point(referencecoords(coordinates, crs)) +end + +function wkb2coords(buff, crs, byteswap) + if CoordRefSystems.ncoords(crs) == 2 + x = byteswap(read(buff, Float64)) + y = byteswap(read(buff, Float64)) + return (x, y) + elseif CoordRefSystems.ncoords(crs) == 3 + x = byteswap(read(buff, Float64)) + y = byteswap(read(buff, Float64)) z = byteswap(read(buff, Float64)) - return x, y, z + return (x, y, z) end - Point(crs(x, y)) end -function wkb2chain(buff, crs, zextent, byteswap) +function referencecoords(coordinates, crs) + if crs <: LatLon + crs(coordinates[2], coordinates[1]) + elseif crs <: LatLonAlt + crs(coordinates[2], coordinates[1], coordinates[3]) + else + crs(coordinates...) + end +end + +function wkb2chain(buff, crs, byteswap) npoints = byteswap(read(buff, UInt32)) chain = map(1:npoints) do _ - wkb2point(buff, crs, zextent, byteswap) + wkb2point(buff, crs, byteswap) end if length(chain) >= 2 && first(chain) != last(chain) Rope(chain) @@ -60,26 +87,20 @@ function wkb2chain(buff, crs, zextent, byteswap) end end -function wkb2poly(buff, crs, zextent, byteswap) +function wkb2poly(buff, crs, byteswap) nrings = byteswap(read(buff, UInt32)) rings = map(1:nrings) do _ - wkb2chain(buff, crs, zextent, byteswap) + wkb2chain(buff, crs, byteswap) end PolyArea(rings) end -function wkb2multi(buff, crs, zextent, byteswap) +function wkb2multi(buff, crs, byteswap) ngeoms = byteswap(read(buff, UInt32)) geoms = map(1:ngeoms) do _ wkbbswap = isone(read(buff, UInt8)) ? ltoh : ntoh - wkbtypebits = read(buff, UInt32) - # if 2D+Z the dimensionality flag is present - if zextent - wkbtype = iszero(wkbtypebits & 0x80000000) ? wkbtypebits - 1000 : wkbtypebits & 0x7FFFFFFF - wkb2single(buff, crs, wkbtype, true, wkbbswap) - else - wkb2single(buff, crs, wkbtypebits, false, wkbbswap) - end + wkbtype = read(buff, UInt32) + wkb2single(buff, crs, wkbtype, wkbbswap) end Multi(geoms) end \ No newline at end of file From 2a012bc5b2676bb33d8418b5cda8b2d3be8f35f3 Mon Sep 17 00:00:00 2001 From: jph6366 Date: Sun, 16 Nov 2025 14:43:31 -0500 Subject: [PATCH 79/90] simplify and rewrite headerskip --- src/extra/gpkg/read.jl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/extra/gpkg/read.jl b/src/extra/gpkg/read.jl index cc48aba..75c5a9c 100644 --- a/src/extra/gpkg/read.jl +++ b/src/extra/gpkg/read.jl @@ -166,9 +166,9 @@ function wkbgeombuffer(row, geomcolumn) # calculate GeoPackageBinaryHeader size in byte stream given extent of envelope: # envelope is [minx, maxx, miny, maxy, minz, maxz], 48 bytes or envelope is [minx, maxx, miny, maxy], 32 bytes or no envelope, 0 bytes - # byte[2] magic + byte[1] version + byte[1] flags + byte[4] srs_id + byte[(8*2)×(x,y{,z})] envelope - headerlen = iszero(envelope) ? 8 : 8 + 8 * 2 * (envelope + 1) + # byte[2] magic + byte[1] version + byte[1] flag + byte[4] srs_id + byte[(8*2)×(x,y{,z})] envelope + skiplen = iszero(envelope) ? 4 : 4 + 8 * 2 * (envelope + 1) # Skip reading the double[] envelope and start reading Well-Known Binary geometry - seek(buff, headerlen) + skip(buff, skiplen) end From bd2a0ed2a13bd5d8ee863717e05c8a23ea821111 Mon Sep 17 00:00:00 2001 From: jph6366 Date: Mon, 17 Nov 2025 21:08:32 -0500 Subject: [PATCH 80/90] batch wkb2points --- src/extra/gpkg/wkb.jl | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/extra/gpkg/wkb.jl b/src/extra/gpkg/wkb.jl index 6c98779..be80c36 100644 --- a/src/extra/gpkg/wkb.jl +++ b/src/extra/gpkg/wkb.jl @@ -75,14 +75,22 @@ function referencecoords(coordinates, crs) end end +function wkb2points(buff, npoints, crs, byteswap) + map(1:npoints) do _ + coordinates = wkb2coords(buff, crs, byteswap) + Point(referencecoords(coordinates, crs)) + end +end + function wkb2chain(buff, crs, byteswap) npoints = byteswap(read(buff, UInt32)) - chain = map(1:npoints) do _ - wkb2point(buff, crs, byteswap) - end - if length(chain) >= 2 && first(chain) != last(chain) + chain = wkb2points(buff, npoints, crs, byteswap) + if length(chain) >= 2 && first(chain) == last(chain) + Ring(chain[1:end-1]) + elseif length(chain) >= 2 Rope(chain) else + # single point or closed single point Ring(chain) end end From b154726cbf789d410179268a3f7516d0fab7cf62 Mon Sep 17 00:00:00 2001 From: jph6366 Date: Mon, 17 Nov 2025 22:01:34 -0500 Subject: [PATCH 81/90] refactor wkbtype z mask --- src/extra/gpkg/read.jl | 26 ++++++++++++-------------- src/extra/gpkg/wkb.jl | 23 ++++++++++------------- 2 files changed, 22 insertions(+), 27 deletions(-) diff --git a/src/extra/gpkg/read.jl b/src/extra/gpkg/read.jl index 75c5a9c..66e215f 100644 --- a/src/extra/gpkg/read.jl +++ b/src/extra/gpkg/read.jl @@ -69,20 +69,18 @@ end # gpkg_contents table. function gpkgextract(db; layer=1) # get the first (and only) feature table returned in sqlite query results - metadata = first( - DBInterface.execute( - db, - """ - SELECT g.table_name AS tablename, g.column_name AS geomcolumn, - c.srs_id AS srsid, srs.organization AS org, srs.organization_coordsys_id AS code - FROM gpkg_geometry_columns g, gpkg_spatial_ref_sys srs - JOIN gpkg_contents c ON ( g.table_name = c.table_name ) - WHERE c.data_type = 'features' - AND g.srs_id = c.srs_id - LIMIT 1 OFFSET ($layer-1); - """ - ) - ) + metadata = first(DBInterface.execute( + db, + """ + SELECT g.table_name AS tablename, g.column_name AS geomcolumn, + c.srs_id AS srsid, srs.organization AS org, srs.organization_coordsys_id AS code + FROM gpkg_geometry_columns g, gpkg_spatial_ref_sys srs + JOIN gpkg_contents c ON ( g.table_name = c.table_name ) + WHERE c.data_type = 'features' + AND g.srs_id = c.srs_id + LIMIT 1 OFFSET ($layer-1); + """ + )) # According to https://www.geopackage.org/spec/#r33, feature table geometry columns # SHALL contain geometries with the srs_id specified for the column by the gpkg_geometry_columns table srs_id column value. diff --git a/src/extra/gpkg/wkb.jl b/src/extra/gpkg/wkb.jl index be80c36..e8a4112 100644 --- a/src/extra/gpkg/wkb.jl +++ b/src/extra/gpkg/wkb.jl @@ -4,25 +4,22 @@ function wkb2geom(buff, crs) byteswap = isone(read(buff, UInt8)) ? ltoh : ntoh - typebits = read(buff, UInt32) + wkbtype = read(buff, UInt32) # Input variants of WKB supported are standard, extended, and ISO WKB geometry with Z dimensions (M/ZM not supported) if CoordRefSystems.ncoords(crs) == 3 # SQL/MM Part 3 and SFSQL 1.2 use offsets of 1000 (Z), 2000 (M) and 3000 (ZM) # to indicate the present of higher dimensional coordinates in a WKB geometry - if typebits > 3007 + if wkbtype >= 1001 && wkbtype <= 1007 + # the SFSQL 1.2 offset of 1000 (Z) is present and subtracting a round number of 1000 gives the standard WKB type + wkbtype -= 1000 # 99-402 was a short-lived extension to SFSQL 1.1 that used a high-bit flag to indicate the presence of Z coordinates in a WKB geometry # the high-bit flag 0x80000000 for Z (or 0x40000000 for M) is set and masking it off gives the standard WKB type - wkbtype = typebits & 0x7FFFFFFF - # check that the WKB typebits is not outside the SFSQL 1.2 Z encoding range (1001-1007) - elseif typebits <= 1007 - # the SFSQL 1.2 offset of 1000 (Z) is present and subtracting a round number of 1000 gives the standard WKB type - wkbtype = typebits - 1000 + elseif wkbtype > 0x80000000 + # the SFSQL 1.1 high-bit flag 0x80000000 (Z) is present and removing the flag reveals the standard WKB type + wkbtype -= 0x80000000 else - @error "Unsupported WKB Geometry Type with M or ZM dimension encoding: $typebits" + @error "Unsupported WKB Geometry Type with M or ZM dimension encoding: $wkbtype" end - else - # standard WKB typebits without Z dimension encoding - wkbtype = typebits end if wkbtype <= 3 @@ -86,7 +83,7 @@ function wkb2chain(buff, crs, byteswap) npoints = byteswap(read(buff, UInt32)) chain = wkb2points(buff, npoints, crs, byteswap) if length(chain) >= 2 && first(chain) == last(chain) - Ring(chain[1:end-1]) + Ring(chain[1:(end - 1)]) elseif length(chain) >= 2 Rope(chain) else @@ -98,7 +95,7 @@ end function wkb2poly(buff, crs, byteswap) nrings = byteswap(read(buff, UInt32)) rings = map(1:nrings) do _ - wkb2chain(buff, crs, byteswap) + wkb2chain(buff, crs, byteswap) end PolyArea(rings) end From f2fcb78f6f568137ddf138758e0abf3f2aeda27f Mon Sep 17 00:00:00 2001 From: jph6366 Date: Mon, 17 Nov 2025 22:34:57 -0500 Subject: [PATCH 82/90] tested on 3dLineString --- src/extra/gpkg/wkb.jl | 25 ++++++++++--------------- 1 file changed, 10 insertions(+), 15 deletions(-) diff --git a/src/extra/gpkg/wkb.jl b/src/extra/gpkg/wkb.jl index e8a4112..fa6b48d 100644 --- a/src/extra/gpkg/wkb.jl +++ b/src/extra/gpkg/wkb.jl @@ -6,22 +6,17 @@ function wkb2geom(buff, crs) byteswap = isone(read(buff, UInt8)) ? ltoh : ntoh wkbtype = read(buff, UInt32) # Input variants of WKB supported are standard, extended, and ISO WKB geometry with Z dimensions (M/ZM not supported) - if CoordRefSystems.ncoords(crs) == 3 - # SQL/MM Part 3 and SFSQL 1.2 use offsets of 1000 (Z), 2000 (M) and 3000 (ZM) - # to indicate the present of higher dimensional coordinates in a WKB geometry - if wkbtype >= 1001 && wkbtype <= 1007 - # the SFSQL 1.2 offset of 1000 (Z) is present and subtracting a round number of 1000 gives the standard WKB type - wkbtype -= 1000 - # 99-402 was a short-lived extension to SFSQL 1.1 that used a high-bit flag to indicate the presence of Z coordinates in a WKB geometry - # the high-bit flag 0x80000000 for Z (or 0x40000000 for M) is set and masking it off gives the standard WKB type - elseif wkbtype > 0x80000000 - # the SFSQL 1.1 high-bit flag 0x80000000 (Z) is present and removing the flag reveals the standard WKB type - wkbtype -= 0x80000000 - else - @error "Unsupported WKB Geometry Type with M or ZM dimension encoding: $wkbtype" - end + # SQL/MM Part 3 and SFSQL 1.2 use offsets of 1000 (Z), 2000 (M) and 3000 (ZM) + # to indicate the present of higher dimensional coordinates in a WKB geometry + if wkbtype >= 1001 && wkbtype <= 1007 + # the SFSQL 1.2 offset of 1000 (Z) is present and subtracting a round number of 1000 gives the standard WKB type + wkbtype -= UInt32(1000) + # 99-402 was a short-lived extension to SFSQL 1.1 that used a high-bit flag to indicate the presence of Z coordinates in a WKB geometry + # the high-bit flag 0x80000000 for Z (or 0x40000000 for M) is set and masking it off gives the standard WKB type + elseif wkbtype > 0x80000000 + # the SFSQL 1.1 high-bit flag 0x80000000 (Z) is present and removing the flag reveals the standard WKB type + wkbtype -= 0x80000000 end - if wkbtype <= 3 # 0 - 3 [Geometry, Point, Linestring, Polygon] wkb2single(buff, crs, wkbtype, byteswap) From 6f62749df2165b4914afd6ab79759d5689373e2f Mon Sep 17 00:00:00 2001 From: jph6366 Date: Sat, 22 Nov 2025 13:07:53 -0500 Subject: [PATCH 83/90] hoisted gpkgread out of gis.jl; clean code; novalues condition --- src/extra/gis.jl | 2 - src/extra/gpkg/read.jl | 120 +++++++++++++++++++---------------------- src/load.jl | 5 ++ 3 files changed, 59 insertions(+), 68 deletions(-) diff --git a/src/extra/gis.jl b/src/extra/gis.jl index f544832..90b7d1f 100644 --- a/src/extra/gis.jl +++ b/src/extra/gis.jl @@ -63,8 +63,6 @@ function gistable(fname; layer, numtype, kwargs...) return GJS.read(fname; numbertype=numtype, kwargs...) elseif endswith(fname, ".parquet") return GPQ.read(fname; kwargs...) - elseif endswith(fname, ".gpkg") - return gpkgread(fname; layer, kwargs...) else # fallback to GDAL data = AG.read(fname; kwargs...) return AG.getlayer(data, layer - 1) diff --git a/src/extra/gpkg/read.jl b/src/extra/gpkg/read.jl index 66e215f..f93ed95 100644 --- a/src/extra/gpkg/read.jl +++ b/src/extra/gpkg/read.jl @@ -13,81 +13,67 @@ function gpkgdatabase(fname) # connect to SQLite database on disk db = SQLite.DB(fname) - # According to https://www.geopackage.org/spec/#r6 and https://www.geopackage.org/spec/#r7 + # According to https://www.geopackage.org/spec/#r6 # PRAGMA integrity_check returns a single row with the value 'ok' + if first(DBInterface.execute(db, "PRAGMA integrity_check;")).integrity_check != "ok" + throw(ErrorException("database integrity at risk")) + end + # According to https://www.geopackage.org/spec/#r7 # PRAGMA foreign_key_check (w/ no parameter value) returns an empty result set - if first(DBInterface.execute(db, "PRAGMA integrity_check;")).integrity_check != "ok" || - !(isempty(DBInterface.execute(db, "PRAGMA foreign_key_check;"))) - throw(ErrorException("database integrity at risk or foreign key violation(s)")) + if !(isempty(DBInterface.execute(db, "PRAGMA foreign_key_check;"))) + throw(ErrorException("foreign key violation(s)")) end - # According to https://www.geopackage.org/spec/#r10 and https://www.geopackage.org/spec/#r13 - # A GeoPackage SHALL include a 'gpkg_spatial_ref_sys' table and a 'gpkg_contents table' - if first(DBInterface.execute( - db, - """ - SELECT COUNT(*) AS n FROM sqlite_master WHERE - name IN ('gpkg_spatial_ref_sys', 'gpkg_contents') AND - type IN ('table', 'view'); - """ - )).n != 2 + # According to https://www.geopackage.org/spec/#r10 + # A GeoPackage SHALL include a 'gpkg_spatial_ref_sys' table + if isnothing(SQLite.tableinfo(db, "gpkg_spatial_ref_sys")) + throw(ErrorException("missing required metadata tables in the GeoPackage SQL database")) + end + # According to https://www.geopackage.org/spec/#r13 + # A GeoPackage SHALL include a 'gpkg_contents` table + if isnothing(SQLite.tableinfo(db, "gpkg_contents")) throw(ErrorException("missing required metadata tables in the GeoPackage SQL database")) end - db end -# According to Geometry Columns Table Requirements -# https://www.geopackage.org/spec/#:~:text=2.1.5.1.2.%20Table%20Data%20Values -#------------------------------------------------------------------------------ -# Requirement 16: https://www.geopackage.org/spec/#r16 -# Values of the gpkg_contents table srs_id column -# SHALL reference values in the gpkg_spatial_ref_sys table srs_id column -# -# Requirement 18: https://www.geopackage.org/spec/#r18 -# The gpkg_contents table SHALL contain a row -# with a lowercase data_type column value of "features" -# for each vector features user data table or view. -# -# Requirement 22: gpkg_geometry_columns table -# SHALL contain one row record for the geometry column -# in each vector feature data table -# -# Requirement 23: gpkg_geometry_columns table_name column -# SHALL reference values in the gpkg_contents table_name column -# for rows with a data_type of 'features' -# -# Requirement 24: The column_name column value in a gpkg_geometry_columns row -# SHALL be the name of a column in the table or view specified by the table_name -# column value for that row. -# -# Requirement 25: The geometry_type_name value in a gpkg_geometry_columns row -# SHALL be one of the uppercase geometry type names specified -# -# Requirement 146: The srs_id value in a gpkg_geometry_columns table row -# SHALL match the srs_id column value from the corresponding row in the -# gpkg_contents table. function gpkgextract(db; layer=1) - # get the first (and only) feature table returned in sqlite query results - metadata = first(DBInterface.execute( - db, - """ - SELECT g.table_name AS tablename, g.column_name AS geomcolumn, - c.srs_id AS srsid, srs.organization AS org, srs.organization_coordsys_id AS code - FROM gpkg_geometry_columns g, gpkg_spatial_ref_sys srs - JOIN gpkg_contents c ON ( g.table_name = c.table_name ) - WHERE c.data_type = 'features' - AND g.srs_id = c.srs_id - LIMIT 1 OFFSET ($layer-1); - """ - )) + # get the feature table or Any[] returned in sqlite query results + metadata = first( + DBInterface.execute( + db, + """ + SELECT g.table_name AS tablename, g.column_name AS geomcolumn, + c.srs_id AS srsid, srs.organization AS org, srs.organization_coordsys_id AS code + FROM gpkg_geometry_columns g, gpkg_spatial_ref_sys srs + """ * + # According to https://www.geopackage.org/spec/#r24 + # The column_name column value in a gpkg_geometry_columns row SHALL be the name of a column in the table or view specified by the table_name column value for that row. + """ + JOIN gpkg_contents c ON ( g.table_name = c.table_name ) + """ * + # According to https://www.geopackage.org/spec/#r23 + # gpkg_geometry_columns table_name column SHALL reference values in the gpkg_contents table_name column for rows with a data_type of 'features' + """ + WHERE c.data_type = 'features' + """ * + # According to https://www.geopackage.org/spec/#r146 + # The srs_id value in a gpkg_geometry_columns table row SHALL match the srs_id column value from the corresponding row in the gpkg_contents table. + """ + AND g.srs_id = c.srs_id + LIMIT 1 OFFSET ($layer-1); + """ + ) + ) # According to https://www.geopackage.org/spec/#r33, feature table geometry columns # SHALL contain geometries with the srs_id specified for the column by the gpkg_geometry_columns table srs_id column value. org = metadata.org code = metadata.code srsid = metadata.srsid - if srsid == 0 || srsid == 4326 + if srsid == 0 + crs = LatLon{NoDatum} + elseif srsid == 4326 crs = LatLon{WGS84Latest} elseif srsid == -1 crs = Cartesian{NoDatum} @@ -112,10 +98,7 @@ function gpkgextract(db; layer=1) table = map(gpkgbinary) do row # According to https://www.geopackage.org/spec/#r30 # A feature table or view SHALL have only one geometry column. - geomindex = findfirst(==(Symbol(geomcolumn)), keys(row)) - values = map(keys(row)[[begin:(geomindex - 1); (geomindex + 1):end]]) do key - key, getproperty(row, key) - end + values = [(key, getproperty(row, key)) for key in keys(row) if key != Symbol(geomcolumn)] # create IOBuffer and seek geometry binary data buff = wkbgeombuffer(row, geomcolumn) @@ -123,11 +106,16 @@ function gpkgextract(db; layer=1) geom = wkb2geom(buff, crs) # returns a tuple of the corresponding aspatial attributes and the geometries for each row in the feature table - return (NamedTuple(values), geom) + NamedTuple(values), geom + end + # separate aspatial attributes into its own vector + aspatial = getindex.(table, 1) + # check if type of aspatial elements generated are calls to the macro for declaring `NamedTuple` types instead of `NamedTuple` types + if eltype(aspatial) == @NamedTuple{} + aspatial = nothing end - # aspatial attributes and geometries - getindex.(table, 1), getindex.(table, 2) + aspatial, getindex.(table, 2) end function wkbgeombuffer(row, geomcolumn) diff --git a/src/load.jl b/src/load.jl index 7d904f0..f43117e 100644 --- a/src/load.jl +++ b/src/load.jl @@ -156,6 +156,11 @@ function load(fname; repair=true, layer=1, lenunit=nothing, numtype=Float64, kwa return cdmread(fname; kwargs...) end + # GeoPackage formats + if endswith(fname, ".gpkg") + return gpkgread(fname; layer, kwargs...) + end + # GIS formats gisread(fname; repair, layer, numtype, kwargs...) end From 5da73de162ba505b16531c925f02b7224b24c435 Mon Sep 17 00:00:00 2001 From: jph6366 Date: Mon, 24 Nov 2025 09:40:00 -0500 Subject: [PATCH 84/90] clean code --- src/extra/gpkg/read.jl | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/extra/gpkg/read.jl b/src/extra/gpkg/read.jl index f93ed95..d870724 100644 --- a/src/extra/gpkg/read.jl +++ b/src/extra/gpkg/read.jl @@ -110,11 +110,10 @@ function gpkgextract(db; layer=1) end # separate aspatial attributes into its own vector aspatial = getindex.(table, 1) - # check if type of aspatial elements generated are calls to the macro for declaring `NamedTuple` types instead of `NamedTuple` types - if eltype(aspatial) == @NamedTuple{} - aspatial = nothing - end - # aspatial attributes and geometries + + # check if type of aspatial elements generated are calls to the macro for declaring `NamedTuple` types + eltype(aspatial) != @NamedTuple{} || return nothing, getindex.(table, 2) + # aspatial attributes and corresponding geometries aspatial, getindex.(table, 2) end From 6f4273bb5a245739263581b56353bce2ebcae7da Mon Sep 17 00:00:00 2001 From: jph6366 Date: Mon, 24 Nov 2025 09:40:24 -0500 Subject: [PATCH 85/90] format --- src/extra/gpkg/read.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/extra/gpkg/read.jl b/src/extra/gpkg/read.jl index d870724..98998a3 100644 --- a/src/extra/gpkg/read.jl +++ b/src/extra/gpkg/read.jl @@ -110,7 +110,7 @@ function gpkgextract(db; layer=1) end # separate aspatial attributes into its own vector aspatial = getindex.(table, 1) - + # check if type of aspatial elements generated are calls to the macro for declaring `NamedTuple` types eltype(aspatial) != @NamedTuple{} || return nothing, getindex.(table, 2) # aspatial attributes and corresponding geometries From f0f71887b2173464a0e858a4b03ff4e5da23aa2d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlio=20Hoffimann?= Date: Fri, 12 Dec 2025 07:58:41 -0300 Subject: [PATCH 86/90] Apply repairs with gpkg --- src/extra/gis.jl | 14 ++------------ src/load.jl | 16 +++++++++++----- 2 files changed, 13 insertions(+), 17 deletions(-) diff --git a/src/extra/gis.jl b/src/extra/gis.jl index 90b7d1f..f3b2787 100644 --- a/src/extra/gis.jl +++ b/src/extra/gis.jl @@ -2,22 +2,12 @@ # Licensed under the MIT License. See LICENSE in the project root. # ------------------------------------------------------------------ -function gisread(fname; repair, layer, numtype, kwargs...) +function gisread(fname; layer, numtype, kwargs...) # extract Tables.jl table from GIS format table = gistable(fname; layer, numtype, kwargs...) # convert Tables.jl table to GeoTable - geotable = asgeotable(table) - - # repair pipeline - pipeline = if repair - Repair(11) → Repair(12) - else - Identity() - end - - # perform repairs - geotable |> pipeline + asgeotable(table) end function giswrite(fname, geotable; warn, kwargs...) diff --git a/src/load.jl b/src/load.jl index f43117e..6e9f136 100644 --- a/src/load.jl +++ b/src/load.jl @@ -156,13 +156,19 @@ function load(fname; repair=true, layer=1, lenunit=nothing, numtype=Float64, kwa return cdmread(fname; kwargs...) end - # GeoPackage formats - if endswith(fname, ".gpkg") - return gpkgread(fname; layer, kwargs...) + # GIS formats + geotable = if endswith(fname, ".gpkg") + gpkgread(fname; layer, kwargs...) + else + gisread(fname; layer, numtype, kwargs...) end - # GIS formats - gisread(fname; repair, layer, numtype, kwargs...) + # repair geometries + if repair + geotable |> Repair(11) |> Repair(12) + else + geotable + end end """ From 4826b46f30bda824be50adc701a70f015c8e9d18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlio=20Hoffimann?= Date: Fri, 12 Dec 2025 08:20:22 -0300 Subject: [PATCH 87/90] Minor adjustments --- src/extra/gpkg/read.jl | 29 ++++++++++++++++++++--------- 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/src/extra/gpkg/read.jl b/src/extra/gpkg/read.jl index 98998a3..9e81aea 100644 --- a/src/extra/gpkg/read.jl +++ b/src/extra/gpkg/read.jl @@ -18,6 +18,7 @@ function gpkgdatabase(fname) if first(DBInterface.execute(db, "PRAGMA integrity_check;")).integrity_check != "ok" throw(ErrorException("database integrity at risk")) end + # According to https://www.geopackage.org/spec/#r7 # PRAGMA foreign_key_check (w/ no parameter value) returns an empty result set if !(isempty(DBInterface.execute(db, "PRAGMA foreign_key_check;"))) @@ -29,11 +30,13 @@ function gpkgdatabase(fname) if isnothing(SQLite.tableinfo(db, "gpkg_spatial_ref_sys")) throw(ErrorException("missing required metadata tables in the GeoPackage SQL database")) end + # According to https://www.geopackage.org/spec/#r13 # A GeoPackage SHALL include a 'gpkg_contents` table if isnothing(SQLite.tableinfo(db, "gpkg_contents")) throw(ErrorException("missing required metadata tables in the GeoPackage SQL database")) end + db end @@ -48,17 +51,23 @@ function gpkgextract(db; layer=1) FROM gpkg_geometry_columns g, gpkg_spatial_ref_sys srs """ * # According to https://www.geopackage.org/spec/#r24 - # The column_name column value in a gpkg_geometry_columns row SHALL be the name of a column in the table or view specified by the table_name column value for that row. + # The column_name column value in a gpkg_geometry_columns row + # SHALL be the name of a column in the table or view specified + # by the table_name column value for that row. """ JOIN gpkg_contents c ON ( g.table_name = c.table_name ) """ * # According to https://www.geopackage.org/spec/#r23 - # gpkg_geometry_columns table_name column SHALL reference values in the gpkg_contents table_name column for rows with a data_type of 'features' + # gpkg_geometry_columns table_name column SHALL reference values + # in the gpkg_contents table_name column for rows with a data_type + # of 'features' """ WHERE c.data_type = 'features' """ * # According to https://www.geopackage.org/spec/#r146 - # The srs_id value in a gpkg_geometry_columns table row SHALL match the srs_id column value from the corresponding row in the gpkg_contents table. + # The srs_id value in a gpkg_geometry_columns table row SHALL + # match the srs_id column value from the corresponding row in + # the gpkg_contents table. """ AND g.srs_id = c.srs_id LIMIT 1 OFFSET ($layer-1); @@ -66,8 +75,10 @@ function gpkgextract(db; layer=1) ) ) - # According to https://www.geopackage.org/spec/#r33, feature table geometry columns - # SHALL contain geometries with the srs_id specified for the column by the gpkg_geometry_columns table srs_id column value. + # According to https://www.geopackage.org/spec/#r33 + # feature table geometry columns SHALL contain geometries + # with the srs_id specified for the column by the + # gpkg_geometry_columns table srs_id column value. org = metadata.org code = metadata.code srsid = metadata.srsid @@ -92,10 +103,11 @@ function gpkgextract(db; layer=1) geomcolumn = metadata.geomcolumn tableinfo = SQLite.tableinfo(db, tablename) - # "pk" (either zero for columns that are not part of the primary key, or the 1-based index of the column within the primary key) + # pk is the index of the column within the primary key, + # or 0 for columns that are not part of the primary key columns = [name for (name, pk) in zip(tableinfo.name, tableinfo.pk) if pk == 0] - gpkgbinary = DBInterface.execute(db, "SELECT $(join(columns, ',')) FROM $tablename;") - table = map(gpkgbinary) do row + gpkgtab = DBInterface.execute(db, "SELECT $(join(columns, ',')) FROM $tablename;") + table = map(gpkgtab) do row # According to https://www.geopackage.org/spec/#r30 # A feature table or view SHALL have only one geometry column. values = [(key, getproperty(row, key)) for key in keys(row) if key != Symbol(geomcolumn)] @@ -105,7 +117,6 @@ function gpkgextract(db; layer=1) geom = wkb2geom(buff, crs) - # returns a tuple of the corresponding aspatial attributes and the geometries for each row in the feature table NamedTuple(values), geom end # separate aspatial attributes into its own vector From a5aeb0009283e0d18574f1b1402b7eeda8311ce2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlio=20Hoffimann?= Date: Fri, 12 Dec 2025 09:14:46 -0300 Subject: [PATCH 88/90] Minor adjustments --- src/extra/gpkg/read.jl | 42 +++++++++++++++++++++++++----------------- 1 file changed, 25 insertions(+), 17 deletions(-) diff --git a/src/extra/gpkg/read.jl b/src/extra/gpkg/read.jl index 9e81aea..ec5cf94 100644 --- a/src/extra/gpkg/read.jl +++ b/src/extra/gpkg/read.jl @@ -76,7 +76,7 @@ function gpkgextract(db; layer=1) ) # According to https://www.geopackage.org/spec/#r33 - # feature table geometry columns SHALL contain geometries + # Feature table geometry columns SHALL contain geometries # with the srs_id specified for the column by the # gpkg_geometry_columns table srs_id column value. org = metadata.org @@ -100,37 +100,45 @@ function gpkgextract(db; layer=1) # The table_name column value in a gpkg_contents table row # SHALL contain the name of a SQLite table or view. tablename = metadata.tablename - geomcolumn = metadata.geomcolumn - tableinfo = SQLite.tableinfo(db, tablename) + # According to https://www.geopackage.org/spec/#r30 + # A feature table or view SHALL have only one geometry column. + geomcolumn = metadata.geomcolumn |> Symbol + + # Retrieve names of columns with attributes (i.e., ≠ geometry) # pk is the index of the column within the primary key, # or 0 for columns that are not part of the primary key - columns = [name for (name, pk) in zip(tableinfo.name, tableinfo.pk) if pk == 0] - gpkgtab = DBInterface.execute(db, "SELECT $(join(columns, ',')) FROM $tablename;") - table = map(gpkgtab) do row - # According to https://www.geopackage.org/spec/#r30 - # A feature table or view SHALL have only one geometry column. - values = [(key, getproperty(row, key)) for key in keys(row) if key != Symbol(geomcolumn)] + tabinfo = SQLite.tableinfo(db, tablename) + columns = [Symbol(name) for (name, pk) in zip(tabinfo.name, tabinfo.pk) if pk == 0] + attribs = setdiff(columns, [geomcolumn]) + + # load feature table from database + gpkgtable = DBInterface.execute(db, "SELECT $(join(columns, ',')) FROM $tablename;") + + # extract attribute table and geometries + pairs = map(Tables.rows(gpkgtable)) do row + # retrieve attribute values as a named tuple + vals = (; (col => Tables.getcolumn(row, col) for col in attribs)...) # create IOBuffer and seek geometry binary data buff = wkbgeombuffer(row, geomcolumn) + # convert buffer into Meshes.jl geometry geom = wkb2geom(buff, crs) - NamedTuple(values), geom + vals, geom end - # separate aspatial attributes into its own vector - aspatial = getindex.(table, 1) - # check if type of aspatial elements generated are calls to the macro for declaring `NamedTuple` types - eltype(aspatial) != @NamedTuple{} || return nothing, getindex.(table, 2) - # aspatial attributes and corresponding geometries - aspatial, getindex.(table, 2) + # handle tables without attributes + table = isempty(attribs) ? nothing : first.(pairs) + geoms = GeometrySet(last.(pairs)) + + table, geoms end function wkbgeombuffer(row, geomcolumn) # get the column of SQL Geometry Binary specified by gpkg_geometry_columns table in column_name field - buff = IOBuffer(getproperty(row, Symbol(geomcolumn))) + buff = IOBuffer(getproperty(row, geomcolumn)) # According to https://www.geopackage.org/spec/#r19 # A GeoPackage SHALL store feature table geometries in SQL BLOBs using the Standard GeoPackageBinary format From 6c51bdf584dc885f78427c6f61e2f2bb2fe495ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlio=20Hoffimann?= Date: Fri, 12 Dec 2025 09:28:59 -0300 Subject: [PATCH 89/90] Minor adjustments --- src/extra/gpkg/read.jl | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/src/extra/gpkg/read.jl b/src/extra/gpkg/read.jl index ec5cf94..af2bea9 100644 --- a/src/extra/gpkg/read.jl +++ b/src/extra/gpkg/read.jl @@ -120,11 +120,14 @@ function gpkgextract(db; layer=1) # retrieve attribute values as a named tuple vals = (; (col => Tables.getcolumn(row, col) for col in attribs)...) - # create IOBuffer and seek geometry binary data - buff = wkbgeombuffer(row, geomcolumn) + # retrieve geometry binary data as IO buffer + buff = IOBuffer(Tables.getcolumn(row, geomcolumn)) + + # seek start of geometry (e.g., discard envelope) + gbuff = seekgeom(buff) # convert buffer into Meshes.jl geometry - geom = wkb2geom(buff, crs) + geom = wkb2geom(gbuff, crs) vals, geom end @@ -136,13 +139,12 @@ function gpkgextract(db; layer=1) table, geoms end -function wkbgeombuffer(row, geomcolumn) - # get the column of SQL Geometry Binary specified by gpkg_geometry_columns table in column_name field - buff = IOBuffer(getproperty(row, geomcolumn)) - +function seekgeom(buff) # According to https://www.geopackage.org/spec/#r19 - # A GeoPackage SHALL store feature table geometries in SQL BLOBs using the Standard GeoPackageBinary format - # check the GeoPackageBinaryHeader for the first byte[2] to be 'GP' in ASCII + # A GeoPackage SHALL store feature table geometries in + # SQL BLOBs using the Standard GeoPackageBinary format + # check the GeoPackageBinaryHeader for the first byte[2] + # to be 'GP' in ASCII read(buff, UInt16) == 0x5047 || @warn "Missing magic 'GP' string in GPkgBinaryGeometry" # byte[1] version: 8-bit unsigned integer, 0 = version 1 From f18b1233d881d57467d31f2635de16316642ca5a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlio=20Hoffimann?= Date: Fri, 12 Dec 2025 09:41:25 -0300 Subject: [PATCH 90/90] Minor adjustments --- src/extra/gpkg/read.jl | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/src/extra/gpkg/read.jl b/src/extra/gpkg/read.jl index af2bea9..6232e51 100644 --- a/src/extra/gpkg/read.jl +++ b/src/extra/gpkg/read.jl @@ -139,15 +139,14 @@ function gpkgextract(db; layer=1) table, geoms end +# According to https://www.geopackage.org/spec/#r19 +# A GeoPackage SHALL store feature table geometries in +# SQL BLOBs using the Standard GeoPackageBinary format function seekgeom(buff) - # According to https://www.geopackage.org/spec/#r19 - # A GeoPackage SHALL store feature table geometries in - # SQL BLOBs using the Standard GeoPackageBinary format - # check the GeoPackageBinaryHeader for the first byte[2] - # to be 'GP' in ASCII + # check the GeoPackageBinaryHeader for the 'GP' magic read(buff, UInt16) == 0x5047 || @warn "Missing magic 'GP' string in GPkgBinaryGeometry" - # byte[1] version: 8-bit unsigned integer, 0 = version 1 + # skip version (0 => version 1) read(buff, UInt8) # bit layout of GeoPackageBinary flags byte @@ -163,18 +162,16 @@ function seekgeom(buff) # B: byte order for SRS_ID and envelope values in header flag = read(buff, UInt8) - # 0x07 is a 3-bit mask 0x00001110 - # left-shift moves the 3-bit mask by one to align with E bits in flag layout - # bitwise AND operation isolates the E bits - # right-shift moves the E bits by one to align with the least significant bits - # results in a 3-bit unsigned integer + # 0x07 is the mask 0x00001110 envelope = (flag & (0x07 << 1)) >> 1 # calculate GeoPackageBinaryHeader size in byte stream given extent of envelope: - # envelope is [minx, maxx, miny, maxy, minz, maxz], 48 bytes or envelope is [minx, maxx, miny, maxy], 32 bytes or no envelope, 0 bytes + # [no envelope] => 0 bytes + # [minx, maxx, miny, maxy] => 32 bytes + # [minx, maxx, miny, maxy, minz, maxz] => 48 bytes # byte[2] magic + byte[1] version + byte[1] flag + byte[4] srs_id + byte[(8*2)×(x,y{,z})] envelope skiplen = iszero(envelope) ? 4 : 4 + 8 * 2 * (envelope + 1) - # Skip reading the double[] envelope and start reading Well-Known Binary geometry + # skip reading the double[] envelope and start reading Well-Known Binary geometry skip(buff, skiplen) end