Merge remote-tracking branch 'origin/master' into add_option_to_avoid_blank_nodes

Author: nicolano

.github/workflows/native-build.yml

@@ -12,7 +12,7 @@ jobs:
       - name: update apt
         run: sudo apt update
       - name: install dependencies
-        run: sudo apt install -y gcc g++ libboost-dev libboost-serialization-dev libexpat1-dev cmake libbz2-dev zlib1g-dev libomp-dev
+        run: sudo apt install -y gcc g++ libexpat1-dev cmake libbz2-dev zlib1g-dev libomp-dev
       - name: cmake
         run: mkdir build && cd build && cmake ..
       - name: make
@@ -29,7 +29,7 @@ jobs:
       - name: update apt
         run: sudo apt update
       - name: install dependencies
-        run: sudo apt install -y clang libboost-dev libboost-serialization-dev libexpat1-dev cmake libbz2-dev zlib1g-dev libomp-dev
+        run: sudo apt install -y clang libexpat1-dev cmake libbz2-dev zlib1g-dev libomp-dev
       - name: cmake
         run: mkdir build && cd build && cmake ..
         shell: bash

CMakeLists.txt

@@ -67,20 +67,6 @@ find_package(EXPAT REQUIRED)
 find_package(BZip2 REQUIRED)
 find_package(OpenMP)
 
-# zlib-ng
-set(ZLIB_COMPAT ON)
-set(WITH_GTEST OFF)
-set(ZLIB_ENABLE_TESTS OFF)
-add_subdirectory(vendor/zlib-ng/zlib-ng)
-
-set(ZLIB_USE_STATIC_LIBS ON)
-set(ZLIB_INCLUDE_DIR "${CMAKE_BINARY_DIR}/vendor/zlib-ng/zlib-ng")
-set(ZLIB_LIBRARY "${CMAKE_BINARY_DIR}/vendor/zlib-ng/zlib-ng/libz.a")
-
-# Disable installation of google stuff
-set(INSTALL_GMOCK OFF)
-set(INSTALL_GTEST OFF)
-
 find_package(Git QUIET)
 if (GIT_FOUND AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/.git")
 	# Update submodules as needed
@@ -100,6 +86,21 @@ if (NOT EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/vendor/google/googletest/CMakeLists.
     message(FATAL_ERROR "The submodules were not downloaded! GIT_SUBMODULE was turned off or failed. Please update submodules and try again.")
 endif ()
 
+# zlib-ng
+set(ZLIB_COMPAT ON)
+set(WITH_GTEST OFF)
+set(ZLIB_ENABLE_TESTS OFF)
+add_subdirectory(vendor/zlib-ng/zlib-ng)
+
+set(ZLIB_USE_STATIC_LIBS ON)
+set(ZLIB_INCLUDE_DIR "${CMAKE_BINARY_DIR}/vendor/zlib-ng/zlib-ng")
+set(ZLIB_LIBRARY "${CMAKE_BINARY_DIR}/vendor/zlib-ng/zlib-ng/libz.a")
+
+# Disable installation of google stuff
+set(INSTALL_GMOCK OFF)
+set(INSTALL_GTEST OFF)
+
+
 find_package(POPL REQUIRED)
 include_directories(SYSTEM ${POPL_INCLUDE_DIR})
 find_package(Protozero REQUIRED)

Dockerfile

@@ -1,6 +1,6 @@
 FROM ubuntu:24.04
 
-RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y git clang clang-tidy g++ libboost-dev libboost-serialization-dev libexpat1-dev cmake libbz2-dev zlib1g-dev libomp-dev
+RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y git g++ libexpat1-dev cmake libbz2-dev libomp-dev zlib1g-dev
 COPY . /app/
-RUN cd /app/ && make
+RUN cd /app/ && mkdir -p build && cd build && cmake .. && make -j
 ENTRYPOINT ["/app/build/apps/osm2rdf"]

Makefile

@@ -1,82 +1,62 @@
 # osm2rdf
 
-`osm2rdf` is a tool for converting the complete [OpenStreetMap](https://www.openstreetmap.org) (OSM) data to [RDF Turtle](https://www.w3.org/TR/turtle) (TTL). 
+`osm2rdf` is a tool for converting [OpenStreetMap](https://www.openstreetmap.org) (OSM) data to [RDF Turtle](https://www.w3.org/TR/turtle) (TTL).
 
-We consider the following prefixes being used in our code and documentation:
+It also pre-computes the spatial relations `sfIntersects`, `sfContains`, `sfCovers`, `sfTouches`, `sfCrosses`, `sfOverlaps`, and `sfEquals` between all OSM objects.
 
-```
-PREFIX ogc: <http://www.opengis.net/rdf#>
-PREFIX geo: <http://www.opengis.net/ont/geosparql#>
-PREFIX osm2rdf: <https://osm2rdf.cs.uni-freiburg.de/rdf#>
-```
-
-The produced triples include:
-
-1. One triple `<o> <key> <value> .` for each key-value pair (called "tag" in OSM) of each object.
-2. Two triples `<o> geo:hasGeometry <geom_o> . <geom_o> geo:asWKT <wkt> .` for the shape of each object (using [WKT](https://en.wikipedia.org/wiki/Well-known_text_representation_of_geometry) to represent the shape).
-3. One triple `<o1> ogc:sfContains <o2> .` for each pair of objects, where `<o1>` geometrically contains `<o2>`.
-4. One triple `<o1> ogc:sfIntersects <o2> .` for each pair of objects, where `<o1>` geometrically intersects `<o2>`.
-5. Triples for various other kinds of information (see `osm2rdf --help` for the many options of the tool).
-
-For the complete OSM data, the tool takes around 15 hours on standard hardware and produces around 40 billion triples with a size of around 200 GB for the compressed TTL file. 
-
-## Accompanying services and materials
+## Services and materials
 
-Weekly updated downloads for the complete data as well as for each individual country are provided on https://osm2rdf.cs.uni-freiburg.de .
+Weekly updated downloads for the complete OSM as well as for individual countries are provided [here](https://osm2rdf.cs.uni-freiburg.de).
 
-A SPARQL endpoint (based on [the QLever SPARQL engine](https://github.com/ad-freiburg/qlever)) for the complete OSM data is available under https://qlever.cs.uni-freiburg.de/osm-planet . The UI provides many example queries (click on the "Examples" button).
+A SPARQL endpoint (based on [the QLever SPARQL engine](https://github.com/ad-freiburg/qlever)) for the complete OSM data is available under https://qlever.dev/osm-planet.
+Example: [all buildings under a powerline](https://qlever.dev/osm-planet/Wd07W1).
+The UI provides more example queries (click "Examples").
 
 The basic method behind `osm2rdf` is described in [this SIGSPATIAL'21 paper](https://ad-publications.cs.uni-freiburg.de/SIGSPATIAL_osm2rdf_BBKL_2021.pdf).
 
-The algorithm behind computing the predicates `geo:sfContains` and `geo:sfIntersects` will be published soon. For the complete OSM data, there are around 2 billion non-trivial geometric objects (this count excludes points that are merely base points of more complex shapes). It is very challenging to compute geometric relations between such a large number of objects in finite time.
+The methods behind computing the spatial relations are described in [this SIGSPATIAL'25 paper](https://ad-publications.cs.uni-freiburg.de/SIGSPATIAL_spatialjoin_BBK_2025.pdf).
 
-## Getting started
+## Requirements
 
-Some features require [boost](https://www.boost.org/) version **1.78 or newer**.
-To enable these features boost has to be updated manually on most systems as distributions do ship older versions.
+  * cmake
+  * Compiler for C++17
+  * libexpat1
+  * zlib
+  * libbz2
+  * libomp (optional, for multithreading)
 
-### Docker
+## Quickstart
+
+Fetch this repo:
 
-The provided `Makefile` contains targets which build a Docker image, prepare input and output directories and downloads data.
-To run `osm2rdf` on germany it is enough to call:
 ```
-make docker-de
+$ git clone https://github.com/ad-freiburg/osm2rdf.git
 ```
-This creates the directories `input`, `output`, and `scratch`, downloads the data and runs `osm2rdf` with default parameters.
-
-Alternatively use the provided `Dockerfile` to compile and run `osm2rdf`:
+Build `osm2rdf`:
+```
+$ cd osm2rdf
+$ mkdir -p build && cd build
+$ cmake ..
+$ make -j
+```
+Fetch an OSM input file and print TTL to stdout:
+```
+$ wget https://download.geofabrik.de/europe/germany/baden-wuerttemberg/freiburg-regbez-latest.osm.pbf
+$ ./apps/osm2rdf freiburg-regbex-latest.osm.pbf
 ```
-$ mkdir input
-$ mkdir output
-$ mkdir scratch
 
-$ docker build -t osm2rdf .
+For more options, see `--help`, `-h`, `-hh`, and `-hhh`.
 
-$ wget -O ./input/germany-latest.osm.pbf https://download.geofabrik.de/europe/germany-latest.osm.pbf
-$ docker run --rm -v `pwd`/input/:/input/ -v `pwd`/output/:/output/ -v `pwd`/scratch/:/scratch/ -it osm2rdf /input/germany-latest.osm.pbf -o /output/germany-latest.osm.ttl -t /scratch/
-```
 
-### Without Docker
+## Docker
 
-On Ubuntu 20.04 the following packages are required to build osm2rdf:
+Alternatively you may use the provided `Dockerfile` to compile and run `osm2rdf`:
 ```
-clang clang-tidy g++ libboost-dev libboost-serialization-dev libexpat1-dev cmake libbz2-dev zlib1g-dev libomp-dev
+$ docker build -t osm2rdf .
 ```
-`clang` is optional, but without it `clang-tidy` has [unrelated error messages](https://stackoverflow.com/a/52728225).
 
-Clone and build `osm2rdf`:
-```
-$ git clone https://github.com/ad-freiburg/osm2rdf.git
-$ cd osm2rdf
-$ mkdir -p build && cd build && cmake -DCMAKE_BUILD_TYPE=Debug .. && make -j4 && cd ..
-```
-Prepare `input`, `output` and `scratch` folders and download some data and convert the data to `ttl`:
 ```
 $ mkdir input
-$ mkdir output
-$ mkdir scratch
-$ wget -O ./input/freiburg-regbez-latest.osm.pbf https://download.geofabrik.de/europe/germany/baden-wuerttemberg/freiburg-regbez-latest.osm.pbf
-$ ./build/osm2rdf ./input/freiburg-regbez-latest.osm.pbf -o ./output/freiburg-regbez-latest.osm.ttl
+$ wget -O input/freiburg-regbez-latest.osm.pbf https://download.geofabrik.de/europe/germany/baden-wuerttemberg/freiburg-regbez-latest.osm.pbf
+$ docker run -v `pwd`/input/:/input/ -it osm2rdf /input/freiburg-regbex-latest.osm.pbf
 ```
-
-More `.pbf` files can be found on the [Geofabrik Download Server](https://download.geofabrik.de/index.html)

README.md

@@ -21,7 +21,8 @@
 
 namespace osm2rdf::osm::constants {
 
-static const int AREA_PRECISION = 12;
+static const int AREA_PRECISION = 4;
+static const int LENGTH_PRECISION = 2;
 static const int BASE10_BASE = 10;
 static const double BASE_SIMPLIFICATION_FACTOR = 0.001;
 

include/osm2rdf/osm/Constants.h

@@ -88,6 +88,8 @@ class OsmiumHandler : public osmium::handler::Handler {
   void handleBuffers(
       osmium::memory::Buffer& buffer,
       osmium::area::MultipolygonManager<osmium::area::Assembler>& mp_manager);
+  void handleRelBuffers(
+      osmium::memory::Buffer& buffer);
   void handleAreaBuffers(osmium::memory::Buffer& areaBuffer);
 };
 }  // namespace osm2rdf::osm

include/osm2rdf/osm/OsmiumHandler.h

@@ -17,10 +17,11 @@
 // You should have received a copy of the GNU General Public License
 // along with osm2rdf.  If not, see <https://www.gnu.org/licenses/>.
 
+#include "osm2rdf/osm/Area.h"
+
 #include <iostream>
 #include <limits>
 
-#include "osm2rdf/osm/Area.h"
 #include "osm2rdf/osm/Constants.h"
 #include "osmium/osm/area.hpp"
 #include "util/geo/Geo.h"
@@ -33,7 +34,35 @@ osm2rdf::osm::Area::Area() {
 
 // ____________________________________________________________________________
 void osm2rdf::osm::Area::finalize() noexcept {
-  _geomArea = ::util::geo::area(_geom);
+  // poly in area preserving projection
+  ::util::geo::DMultiPolygon lambertPoly;
+  lambertPoly.resize(_geom.size());
+
+  double EARTH_RAD = 6371008.7714;  // mean radius
+
+  for (size_t i = 0; i < _geom.size(); i++) {
+    const auto& poly = _geom[i];
+    lambertPoly[i].getOuter().reserve(poly.getOuter().size());
+    for (const auto& p : poly.getOuter()) {
+      lambertPoly[i].getOuter().push_back(
+          ::util::geo::DPoint{EARTH_RAD * (p.getX() * util::geo::RAD),
+                              EARTH_RAD * (sin(p.getY() * util::geo::RAD))});
+    }
+
+    lambertPoly[i].getInners().reserve(poly.getInners().size());
+
+    for (const auto& inner : poly.getInners()) {
+      lambertPoly[i].getInners().push_back({});
+      lambertPoly[i].getInners().back().reserve(inner.size());
+      for (const auto& p : inner) {
+        lambertPoly[i].getInners().back().push_back(
+            ::util::geo::DPoint{EARTH_RAD * (p.getX() * util::geo::RAD),
+                                EARTH_RAD * (sin(p.getY() * util::geo::RAD))});
+      }
+    }
+  }
+
+  _geomArea = ::util::geo::area(lambertPoly);
 }
 
 // ____________________________________________________________________________
@@ -115,8 +144,7 @@ ::util::geo::DPolygon osm2rdf::osm::Area::convexHull() const noexcept {
 }
 
 // ____________________________________________________________________________
-::util::geo::DPolygon osm2rdf::osm::Area::orientedBoundingBox()
-    const noexcept {
+::util::geo::DPolygon osm2rdf::osm::Area::orientedBoundingBox() const noexcept {
   return ::util::geo::convexHull(::util::geo::getOrientedEnvelope(_geom));
 }
 
@@ -129,8 +157,7 @@ const ::util::geo::DPoint osm2rdf::osm::Area::centroid() const noexcept {
 bool osm2rdf::osm::Area::operator==(
     const osm2rdf::osm::Area& other) const noexcept {
   return _id == other._id && _objId == other._objId &&
-         _geomArea == other._geomArea &&
-         _envelope == other._envelope &&
+         _geomArea == other._geomArea && _envelope == other._envelope &&
          _geom == other._geom;
 }