tilemaker 3.0

Author: systemed

.gitignore

@@ -1,6 +1,7 @@
 # object files created during the build and the output binary itself
 *.o
 tilemaker
+tilemaker-server
 build/
 tilemaker.dSYM/
 

CHANGELOG.md

@@ -1,5 +1,46 @@
 # Changelog
 
+## [3.0.0] - 2024-01-15
+
+3.0 is a major release that significantly reduces tilemaker's memory footprint and improves running time. Note that it has __breaking changes__ in the way you write your Lua scripts (`way:Layer` becomes simply `Layer`, and so on).
+
+### Added
+- PMTiles output (@systemed)
+- C++ tilemaker-server for quick prototyping (@bdon)
+- GeoJSON supported as an alternative to shapefiles (@systemed)
+- Support nodes in relations and relation roles (@cldellow)
+- Nested relations support (@systemed/@cldellow)
+- `LayerAsCentroid` can use positions from relation child nodes (@cldellow)
+- Add polylabel algorithm to `LayerAsCentroid` (@cldellow)
+- Filter input .pbf by way keys (@cldellow)
+- GeoJSON writer for debugging (@systemed)
+- Warn about PBFs with large blocks (@cldellow)
+- Unit tests for various features (@cldellow)
+- `RestartRelations()` to reset relation iterator (@systemed)
+- Per-layer, zoom-dependent feature_limit (@systemed after an original by @keichan34)
+- Report OSM ID on Lua processing error (@systemed)
+- Docker OOM killer warning (@Firefishy)
+- Push Docker image to Github package (@JinIgarashi)
+- Support `type=boundary` relations as native multipolygons (@systemed)
+
+### Changed
+- __BREAKING__: Lua calls use the global namespace, so `Layer` instead of `way:Layer` etc. (@cldellow)
+- __BREAKING__: Mapsplit (.msf) support removed (@systemed)
+- Widespread speed improvements (@cldellow, @systemed)
+- Reduced memory consumption (@cldellow)
+- protobuf dependency removed: protozero/vtzero used instead (@cldellow)
+- Better Lua detection in Makefile (@systemed)
+- z-order is now a lossy float: compile-time flag not needed (@systemed)
+- --input and --output parameter names no longer required explicitly (@systemed)
+- Docker image improvements (@Booligoosh)
+
+### Fixed
+- Improved polygon correction (@systemed)
+- Add missing attributes to OMT layers (@Nakaner)
+- Use different OSM tags for OMT subclasses (@Nakaner)
+- Add access and mtb_scale attributes to OMT (@dschep)
+- Fix CMake build on Arch Linux (@holzgeist)
+
 
 ## [2.4.0] - 2023-03-28
 

CMakeLists.txt

@@ -96,8 +96,10 @@ file(GLOB tilemaker_src_files
 	src/shared_data.cpp
 	src/shp_mem_tiles.cpp
 	src/shp_processor.cpp
+	src/significant_tags.cpp
 	src/sorted_node_store.cpp
 	src/sorted_way_store.cpp
+	src/tag_map.cpp
 	src/tile_data.cpp
 	src/tilemaker.cpp
 	src/tile_worker.cpp

Makefile

@@ -1,82 +1,71 @@
 # See what Lua versions are installed
-# order of preference: LuaJIT 2.1, LuaJIT 2.0, any generic Lua, Lua 5.1
+# order of preference: LuaJIT, any generic Lua, then versions from 5.4 down
 
 PLATFORM_PATH := /usr/local
 
-ifneq ("$(wildcard /usr/local/include/luajit-2.1/lua.h)","")
-  LUA_VER := LuaJIT 2.1
-  LUA_CFLAGS := -I/usr/local/include/luajit-2.1 -DLUAJIT
-  LUA_LIBS := -lluajit-5.1
-  LUAJIT := 1
-
-else ifneq ("$(wildcard /usr/include/luajit-2.1/lua.h)","")
-  LUA_VER := LuaJIT 2.1
-  LUA_CFLAGS := -I/usr/include/luajit-2.1 -DLUAJIT
-  LUA_LIBS := -lluajit-5.1
-  LUAJIT := 1
-
-else ifneq ("$(wildcard /usr/local/include/luajit-2.0/lua.h)","")
-  LUA_VER := LuaJIT 2.0
-  LUA_CFLAGS := -I/usr/local/include/luajit-2.0 -DLUAJIT
-  LUA_LIBS := -lluajit-5.1
-  LUAJIT := 1
-
-else ifneq ("$(wildcard /usr/include/luajit-2.0/lua.h)","")
-  LUA_VER := LuaJIT 2.0
-  LUA_CFLAGS := -I/usr/include/luajit-2.0 -DLUAJIT
-  LUA_LIBS := -lluajit-5.1
-  LUAJIT := 1
-
-else ifneq ("$(wildcard /usr/local/include/lua/lua.h)","")
-  LUA_VER := system Lua
-  LUA_CFLAGS := -I/usr/local/include/lua
-  LUA_LIBS := -llua
-
-else ifneq ("$(wildcard /usr/include/lua/lua.h)","")
-  LUA_VER := system Lua
-  LUA_CFLAGS := -I/usr/include/lua
-  LUA_LIBS := -llua
-
-else ifneq ("$(wildcard /usr/include/lua.h)","")
-  LUA_VER := system Lua
+# First, find what the Lua executable is called
+# - when a new Lua is released, then add it before 5.4 here
+LUA_CMD := $(shell luajit -e 'print("luajit")' 2> /dev/null || lua -e 'print("lua")' 2> /dev/null || lua5.4 -e 'print("lua5.4")' 2> /dev/null || lua5.3 -e 'print("lua5.3")' 2> /dev/null || lua5.2 -e 'print("lua5.2")' 2> /dev/null || lua5.1 -e 'print("lua5.1")' 2> /dev/null)
+ifeq ($(LUA_CMD),"")
+  $(error Couldn't find Lua interpreter)
+endif
+$(info Using ${LUA_CMD})
+
+# Find the language version
+LUA_LANGV := $(shell ${LUA_CMD} -e 'print(string.match(_VERSION, "%d+.%d+"))')
+$(info - Lua language version ${LUA_LANGV})
+
+# Find the directory where Lua might be
+ifeq ($(LUA_CMD),luajit)
+  # We need the LuaJIT version (2.0/2.1) to find this
+  LUA_JITV := $(shell luajit -e 'a,b,c=string.find(jit.version,"LuaJIT (%d.%d)");print(c)')
+  $(info - LuaJIT version ${LUA_JITV})
+  LUA_DIR := luajit-${LUA_JITV}
+  LUA_LIBS := -lluajit-${LUA_LANGV}
+else
+  LUA_DIR := $(LUA_CMD)
+  LUA_LIBS := -l${LUA_CMD}
+endif
+
+# Find the include path by looking in the most likely locations
+ifneq ('$(wildcard /usr/local/include/${LUA_DIR}/lua.h)','')
+  LUA_CFLAGS := -I/usr/local/include/${LUA_DIR}
+else ifneq ('$(wildcard /usr/local/include/${LUA_DIR}${LUA_LANGV}/lua.h)','')
+  LUA_CFLAGS := -I/usr/local/include/${LUA_DIR}${LUA_LANGV}
+  LUA_LIBS := -l${LUA_CMD}${LUA_LANGV}
+else ifneq ('$(wildcard /usr/include/${LUA_DIR}/lua.h)','')
+  LUA_CFLAGS := -I/usr/include/${LUA_DIR}
+else ifneq ('$(wildcard /usr/include/${LUA_DIR}${LUA_LANGV}/lua.h)','')
+  LUA_CFLAGS := -I/usr/include/${LUA_DIR}${LUA_LANGV}
+  LUA_LIBS := -l${LUA_CMD}${LUA_LANGV}
+else ifneq ('$(wildcard /usr/include/lua.h)','')
   LUA_CFLAGS := -I/usr/include
-  LUA_LIBS := -llua
-
-else ifneq ("$(wildcard /usr/local/include/lua5.1/lua.h)","")
-  LUA_VER := Lua 5.1
-  LUA_CFLAGS := -I/usr/local/include/lua5.1
-  LUA_LIBS := -llua5.1
-
-else ifneq ("$(wildcard /usr/include/lua5.1/lua.h)","")
-  LUA_VER := Lua 5.1
-  LUA_CFLAGS := -I/usr/include/lua5.1
-  LUA_LIBS := -llua5.1
-
-else ifneq ("$(wildcard /usr/include/lua5.3/lua.h)","")
-  LUA_VER := Lua 5.3
-  LUA_CFLAGS := -I/usr/include/lua5.3
-  LUA_LIBS := -llua5.3
-
-else ifneq ("$(wildcard /opt/homebrew/include/lua5.1/lua.h)","")
-  LUA_VER := Lua 5.1
-  LUA_CFLAGS := -I/opt/homebrew/include/lua5.1
-  LUA_LIBS := -llua5.1
+else ifneq ('$(wildcard /opt/homebrew/include/${LUA_DIR}/lua.h)','')
+  LUA_CFLAGS := -I/opt/homebrew/include/${LUA_DIR}
+  PLATFORM_PATH := /opt/homebrew
+else ifneq ('$(wildcard /opt/homebrew/include/${LUA_DIR}${LUA_LANGV}/lua.h)','')
+  LUA_CFLAGS := -I/opt/homebrew/include/${LUA_DIR}${LUA_LANGV}
+  LUA_LIBS := -l${LUA_CMD}${LUA_LANGV}
   PLATFORM_PATH := /opt/homebrew
-
 else
-  $(error Couldn't find Lua)
+  $(error Couldn't find Lua libraries)
 endif
 
-$(info Using ${LUA_VER} (include path is ${LUA_CFLAGS}, library path is ${LUA_LIBS}))
-ifneq ($(OS),Windows_NT)
-  ifeq ($(shell uname -s), Darwin)
-    ifeq ($(LUAJIT), 1)
+# Append LuaJIT-specific flags if needed
+ifeq ($(LUA_CMD),luajit)
+  LUA_CFLAGS := ${LUA_CFLAGS} -DLUAJIT
+  ifneq ($(OS),Windows_NT)
+    ifeq ($(shell uname -s), Darwin)
       LDFLAGS := -pagezero_size 10000 -image_base 100000000
       $(info - with MacOS LuaJIT linking)
     endif
   endif
 endif
 
+# Report success
+$(info - include path is ${LUA_CFLAGS})
+$(info - library path is ${LUA_LIBS})
+
 # Main includes
 
 prefix = /usr/local
@@ -121,8 +110,10 @@ tilemaker: \
 	src/shared_data.o \
 	src/shp_mem_tiles.o \
 	src/shp_processor.o \
+	src/significant_tags.o \
 	src/sorted_node_store.o \
 	src/sorted_way_store.o \
+	src/tag_map.o \
 	src/tile_data.o \
 	src/tilemaker.o \
 	src/tile_worker.o \
@@ -136,6 +127,7 @@ test: \
 	test_pbf_reader \
 	test_pooled_string \
 	test_relation_roles \
+	test_significant_tags \
 	test_sorted_node_store \
 	test_sorted_way_store
 
@@ -166,11 +158,18 @@ test_pooled_string: \
 	test/pooled_string.test.o
 	$(CXX) $(CXXFLAGS) -o test.pooled_string $^ $(INC) $(LIB) $(LDFLAGS) && ./test.pooled_string
 
+
 test_relation_roles: \
 	src/relation_roles.o \
 	test/relation_roles.test.o
 	$(CXX) $(CXXFLAGS) -o test.relation_roles $^ $(INC) $(LIB) $(LDFLAGS) && ./test.relation_roles
 
+test_significant_tags: \
+	src/significant_tags.o \
+	src/tag_map.o \
+	test/significant_tags.test.o
+	$(CXX) $(CXXFLAGS) -o test.significant_tags $^ $(INC) $(LIB) $(LDFLAGS) && ./test.significant_tags
+
 test_sorted_node_store: \
 	src/external/streamvbyte_decode.o \
 	src/external/streamvbyte_encode.o \
@@ -209,8 +208,8 @@ install:
 	install -m 0755 -d $(DESTDIR)$(prefix)/bin/
 	install -m 0755 tilemaker $(DESTDIR)$(prefix)/bin/
 	install -m 0755 tilemaker-server $(DESTDIR)$(prefix)/bin/
-	install -m 0755 -d ${DESTDIR}${MANPREFIX}/man1/
-	install docs/man/tilemaker.1 ${DESTDIR}${MANPREFIX}/man1/
+	@install -m 0755 -d ${DESTDIR}${MANPREFIX}/man1/ || true
+	@install docs/man/tilemaker.1 ${DESTDIR}${MANPREFIX}/man1/ || true
 
 clean:
 	rm -f tilemaker tilemaker-server src/*.o src/external/*.o include/*.o include/*.pb.h server/*.o test/*.o

README.md

@@ -12,14 +12,13 @@ See an example of a vector tile map produced by tilemaker at [tilemaker.org](htt
 
 tilemaker is written in C++14. The chief dependencies are:
 
-* Google Protocol Buffers
 * Boost (latest version advised, 1.66 minimum)
 * Lua (5.1 or later) or LuaJIT
 * sqlite3
 * shapelib
 * rapidjson
 
-sqlite_modern_cpp, and kaguya are bundled in the include/ directory.
+Other third-party code is bundled in the include/ directory.
 
 You can then simply install with:
 
@@ -32,20 +31,19 @@ For detailed installation instructions for your operating system, see [INSTALL.m
 
 tilemaker comes with configuration files compatible with the popular [OpenMapTiles](https://openmaptiles.org) schema, and a demonstration map server. You'll run tilemaker to make vector tiles from your `.osm.pbf` source data. To create the tiles, run this from the tilemaker directory:
 
-    tilemaker --input /path/to/your/input.osm.pbf \
-        --output /path/to/your/output.mbtiles
+    tilemaker /path/to/your/input.osm.pbf /path/to/your/output.mbtiles
 
 If you want to include sea tiles, then create a directory called `coastline` in the same place you're running tilemaker from, and then save the files from https://osmdata.openstreetmap.de/download/water-polygons-split-4326.zip in it, such that tilemaker can find a file at `coastline/water_polygons.shp`.
 
+_(If you want to include optional small-scale landcover, create a `landcover` directory, and download the appropriate 10m files from 'Features' at https://www.naturalearthdata.com so that you have `landcover/ne_10m_antarctic_ice_shelves_polys/ne_10m_antarctic_ice_shelves_polys.shp`, `landcover/ne_10m_urban_areas/ne_10m_urban_areas.shp`, `landcover/ne_10m_glaciated_areas/ne_10m_glaciated_areas.shp`.)_
+
 Then, to serve your tiles using the demonstration server:
 
     cd server
-    ruby server.rb /path/to/your/output.mbtiles
+	tilemaker-server /path/to/your/output.mbtiles
 
 You can now navigate to http://localhost:8080/ and see your map!
 
-(If you don't already have them, you'll need to install Ruby and the required gems to run the demonstration server. On Ubuntu, for example, `sudo apt install sqlite3 libsqlite3-dev ruby ruby-dev` and then `sudo gem install sqlite3 cgi glug rack rackup`.)
-
 ## Your own configuration
 
 Vector tiles contain (generally thematic) 'layers'. For example, your tiles might contain river, cycleway and railway layers. It's up to you what OSM data goes into each layer. You configure this in tilemaker with two files:
@@ -77,27 +75,31 @@ You might use tilemaker if:
 But don't use tilemaker if:
 
 * You want someone else to create and host the tiles for you
-* You want the entire planet
 * You want continuous updates with the latest OSM data
 
 ## Contributing
 
-Bug reports, suggestions and (especially!) pull requests are very welcome on the Github issue tracker. Please check the tracker to see if your issue is already known, and be nice. For questions, please use IRC (irc.oftc.net or https://irc.osm.org, channel #osm-dev) and https://help.osm.org.
+Bug reports, suggestions and (especially!) pull requests are very welcome on the Github issue tracker. Please check the tracker to see if your issue is already known, and be nice. For questions, please use IRC (irc.oftc.net or https://irc.osm.org, channel #osm-dev) and https://community.osm.org.
 
 Formatting: braces and indents as shown, hard tabs (4sp). (Yes, I know.) Please be conservative about adding dependencies or increasing the memory requirement.
 
 ## Copyright
 
 tilemaker is maintained by Richard Fairhurst and supported by [many contributors](https://github.com/systemed/tilemaker/graphs/contributors).
 
-Copyright tilemaker contributors, 2015-2023.
+Copyright tilemaker contributors, 2015-2024.
 
 The tilemaker code is licensed as FTWPL; you may do anything you like with this code and there is no warranty.
 
 Licenses of third-party libraries:
 
-- sqlite_modern_cpp (Amin Roosta) is licensed under MIT
+- [sqlite_modern_cpp](https://github.com/SqliteModernCpp/sqlite_modern_cpp) is licensed under MIT
 - [kaguya](https://github.com/satoren/kaguya) is licensed under the Boost Software Licence
 - [libpopcnt](https://github.com/kimwalisch/libpopcnt) is licensed under BSD 2-clause
 - [pmtiles](https://github.com/protomaps/PMTiles) is licensed under BSD 3-clause
 - [streamvbyte](https://github.com/lemire/streamvbyte) is licensed under Apache 2
+- [polylabel](https://github.com/mapbox/polylabel) is licensed under ISC
+- [protozero](https://github.com/mapbox/protozero) is licensed under BSD 2-clause
+- [vtzero](https://github.com/mapbox/vtzero) is licensed under BSD 2-clause
+- [minunit](https://github.com/siu/minunit) is licensed under MIT
+- [Simple-Web-Server](https://gitlab.com/eidheim/Simple-Web-Server) is licensed under MIT