Merge pull request #757 from Maxxen/v1.5-variegata-dev

Author: Maxxen

duckdb

@@ -6,5 +6,5 @@ add_subdirectory(operators)
 
 set(EXTENSION_SOURCES
     ${EXTENSION_SOURCES} ${CMAKE_CURRENT_SOURCE_DIR}/spatial_extension.cpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/spatial_types.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/spatial_types.cpp ${CMAKE_CURRENT_SOURCE_DIR}/spatial_settings.cpp
     PARENT_SCOPE)

src/spatial/CMakeLists.txt

@@ -162,8 +162,7 @@ void RTreeIndex::CommitDrop(IndexLock &index_lock) {
 	tree->Reset();
 }
 
-
-template<class CALLBACK = std::function<void(const RTreeEntry &)>>
+template <class CALLBACK = std::function<void(const RTreeEntry &)>>
 static void ConvertToEntries(Vector &box_vec, Vector &rowid_vec, idx_t count, CALLBACK &&callback) {
 	const auto &box_validity = FlatVector::Validity(box_vec);
 	const auto &row_validity = FlatVector::Validity(rowid_vec);
@@ -181,7 +180,7 @@ static void ConvertToEntries(Vector &box_vec, Vector &rowid_vec, idx_t count, CA
 			continue;
 		}
 
-		Box2D <float> box;
+		Box2D<float> box;
 		box.min.x = box_xmin_data[i];
 		box.min.y = box_ymin_data[i];
 		box.max.x = box_xmax_data[i];
@@ -204,9 +203,7 @@ ErrorData RTreeIndex::Insert(IndexLock &lock, DataChunk &input, Vector &row_vec)
 
 	auto &box_vec = key_chunk.data[0];
 
-	ConvertToEntries(box_vec, row_vec, input.size(), [&](const RTreeEntry &entry) {
-		tree->Insert(entry);
-	});
+	ConvertToEntries(box_vec, row_vec, input.size(), [&](const RTreeEntry &entry) { tree->Insert(entry); });
 
 	return ErrorData {};
 }
@@ -230,9 +227,7 @@ void RTreeIndex::Delete(IndexLock &lock, DataChunk &input, Vector &row_vec) {
 	key_chunk.Flatten();
 
 	auto &box_vec = key_chunk.data[0];
-	ConvertToEntries(box_vec, row_vec, count, [&](const RTreeEntry &entry) {
-		tree->Delete(entry);
-	});
+	ConvertToEntries(box_vec, row_vec, count, [&](const RTreeEntry &entry) { tree->Delete(entry); });
 }
 
 IndexStorageInfo RTreeIndex::SerializeToDisk(QueryContext context, const case_insensitive_map_t<Value> &options) {

src/spatial/index/rtree/rtree_index.cpp

@@ -35,7 +35,8 @@ class RTreeIndex final : public BoundIndex {
 
 	static unique_ptr<BoundIndex> Create(CreateIndexInput &input) {
 		auto res = make_uniq<RTreeIndex>(input.name, input.constraint_type, input.column_ids, input.table_io_manager,
-		                                 input.unbound_expressions, input.db, input.options, input.context, input.storage_info);
+		                                 input.unbound_expressions, input.db, input.options, input.context,
+		                                 input.storage_info);
 		return std::move(res);
 	}
 

src/spatial/index/rtree/rtree_index.hpp

@@ -28,7 +28,6 @@
 #include "duckdb/planner/filter/expression_filter.hpp"
 #include "duckdb/main/database.hpp"
 
-
 namespace duckdb {
 //-----------------------------------------------------------------------------
 // Plan rewriter
@@ -126,7 +125,8 @@ class RTreeIndexScanOptimizer : public OptimizerExtension {
 		vector<unique_ptr<Expression>> children;
 		children.push_back(expr.Copy());
 
-		const auto bbox_expr = make_uniq<BoundFunctionExpression>(GeoTypes::BOX_2DF(), func, std::move(children), nullptr);
+		const auto bbox_expr =
+		    make_uniq<BoundFunctionExpression>(GeoTypes::BOX_2DF(), func, std::move(children), nullptr);
 
 		Value result;
 		if (!ExpressionExecutor::TryEvaluateScalar(context, *bbox_expr, result)) {
@@ -216,9 +216,9 @@ class RTreeIndexScanOptimizer : public OptimizerExtension {
 		auto &table_info = *table.GetStorage().GetDataTableInfo();
 		unique_ptr<RTreeIndexScanBindData> bind_data = nullptr;
 
-		unordered_set<string> spatial_predicates = {"ST_Equals",    "ST_Intersects",      "ST_Touches",  "ST_Crosses",
-		                                            "ST_Within",    "ST_Contains",        "ST_Overlaps", "ST_Covers",
-		                                            "ST_CoveredBy", "ST_ContainsProperly", "&&", "ST_IntersectsExtent"};
+		unordered_set<string> spatial_predicates = {
+		    "ST_Equals",   "ST_Intersects", "ST_Touches",   "ST_Crosses",          "ST_Within", "ST_Contains",
+		    "ST_Overlaps", "ST_Covers",     "ST_CoveredBy", "ST_ContainsProperly", "&&",        "ST_IntersectsExtent"};
 
 		table_info.BindIndexes(context, RTreeIndex::TYPE_NAME);
 

src/spatial/index/rtree/rtree_index_plan_scan.cpp

@@ -29,6 +29,8 @@
 #include "cpl_vsi_virtual.h"
 #include "duckdb/common/types/geometry_crs.hpp"
 #include "duckdb/main/settings.hpp"
+#include "spatial/spatial_settings.hpp"
+#include "spatial/modules/proj/proj_module.hpp"
 
 namespace duckdb {
 namespace {
@@ -393,6 +395,8 @@ class DuckDBFileSystemHandler final : public VSIFilesystemHandler {
 
 class DuckDBFileSystemPrefix final : public ClientContextState {
 public:
+	static mutex vsi_mutex;
+
 	explicit DuckDBFileSystemPrefix(ClientContext &context) : context(context) {
 		// Create a new random prefix for this client
 		client_prefix = StringUtil::Format("/vsiduckdb-%s/", UUID::ToString(UUID::GenerateRandomUUID()));
@@ -401,11 +405,13 @@ class DuckDBFileSystemPrefix final : public ClientContextState {
 		fs_handler = make_uniq<DuckDBFileSystemHandler>(client_prefix, context);
 
 		// Register the file handler
+		lock_guard<mutex> lock(vsi_mutex);
 		VSIFileManager::InstallHandler(client_prefix, fs_handler.get());
 	}
 
 	~DuckDBFileSystemPrefix() override {
 		// Uninstall the file handler for this prefix
+		lock_guard<mutex> lock(vsi_mutex);
 		VSIFileManager::RemoveHandler(client_prefix);
 	}
 
@@ -430,6 +436,8 @@ class DuckDBFileSystemPrefix final : public ClientContextState {
 	unique_ptr<DuckDBFileSystemHandler> fs_handler;
 };
 
+mutex DuckDBFileSystemPrefix::vsi_mutex;
+
 //======================================================================================================================
 // GDAL READ
 //======================================================================================================================
@@ -628,6 +636,35 @@ auto Bind(ClientContext &ctx, TableFunctionBindInput &input, vector<LogicalType>
 				col_names.push_back(child_schema.name);
 			}
 
+			if (duck_type.id() != LogicalTypeId::GEOMETRY) {
+				col_types.push_back(std::move(duck_type));
+				continue;
+			}
+
+			if (!GeoType::HasCRS(duck_type)) {
+				col_types.push_back(std::move(duck_type));
+				continue;
+			}
+
+			// Try to identify the CRS
+			const auto &gdal_crs = GeoType::GetCRS(duck_type);
+
+			// FIXME: GDAL should be able to do this on its own, but somehow
+			// OGRSpatialReference::FindBestMatch() dont work
+
+			string auth_name;
+			string auth_code;
+			if (IdentifyProjCRS(gdal_crs.GetDefinition().c_str(), auth_name, auth_code)) {
+				// If we can identify the CRS using PROJ, we can be pretty sure that DuckDB will recognize it.
+				auto authcode = auth_name + ":" + auth_code;
+				auto duck_crs = CoordinateReferenceSystem::TryIdentify(ctx, authcode);
+				if (duck_crs) {
+					col_types.push_back(LogicalType::GEOMETRY(*duck_crs));
+					continue;
+				}
+			}
+
+			// Otherwise just pass on what we got
 			col_types.push_back(std::move(duck_type));
 		}
 
@@ -1295,7 +1332,25 @@ auto InitGlobal(ClientContext &context, FunctionData &bdata_p, const string &rea
 	if (!bdata.target_srs.empty()) {
 		// Make a new spatial reference object, and set it from the user input
 		result->srs = OSRNewSpatialReference(nullptr);
-		OSRSetFromUserInput(result->srs, bdata.target_srs.c_str());
+
+		// Try get the complete SRS from duckdb
+		auto converted =
+		    CoordinateReferenceSystem::TryConvert(context, bdata.target_srs, CoordinateReferenceSystemType::PROJJSON);
+		if (!converted) {
+			converted = CoordinateReferenceSystem::TryConvert(context, bdata.target_srs,
+			                                                  CoordinateReferenceSystemType::WKT2_2019);
+			if (!converted) {
+				converted = CoordinateReferenceSystem::TryConvert(context, bdata.target_srs,
+				                                                  CoordinateReferenceSystemType::AUTH_CODE);
+			}
+		}
+
+		if (converted) {
+			OSRSetFromUserInput(result->srs, converted->GetDefinition().c_str());
+		} else {
+			// Try to set it directly from the user input, in case it's in a format that duckdb doesn't recognize but GDAL does
+			OSRSetFromUserInput(result->srs, bdata.target_srs.c_str());
+		}
 	}
 
 	// Create Layer