Merge pull request #2008 from joto/add-tile-id-columns

Add special id column type "tile" for generalized data

Author: lonvia

flex-config/gen/forests.lua

@@ -51,13 +51,12 @@ tables.forests = osm2pgsql.define_area_table('forests', {
 
 -- These defines the tables for three levels of generalized data. In this
 -- case, the geometries generated by the generalization are always polygons.
--- You must define x and y integer columns.
 for _, level in ipairs(gen_levels) do
     tables['forests_' .. level] = osm2pgsql.define_table({
         name = 'forests_' .. level,
+        -- Define the x and y integer columns and add an index for them.
+        ids = { type = 'tile' },
         columns = {
-            { column = 'x', type = 'int', not_null = true },
-            { column = 'y', type = 'int', not_null = true },
             { column = 'geom', type = 'polygon', not_null = true }
         }
     })

src/flex-lua-table.cpp

@@ -99,6 +99,19 @@ static flex_table_t &create_flex_table(lua_State *lua_state,
     return new_table;
 }
 
+static void parse_create_index(lua_State *lua_state, flex_table_t *table)
+{
+    std::string const create_index = luaX_get_table_string(
+        lua_state, "create_index", -1, "The ids field", "auto");
+    lua_pop(lua_state, 1); // "create_index"
+    if (create_index == "always") {
+        table->set_always_build_id_index();
+    } else if (create_index != "auto") {
+        throw fmt_error("Unknown value '{}' for 'create_index' field of ids",
+                        create_index);
+    }
+}
+
 static void setup_flex_table_id_columns(lua_State *lua_state,
                                         flex_table_t *table)
 {
@@ -119,15 +132,15 @@ static void setup_flex_table_id_columns(lua_State *lua_state,
     lua_pop(lua_state, 1); // "type"
 
     if (type == "node") {
-        table->set_id_type(osmium::item_type::node);
+        table->set_id_type(flex_table_index_type::node);
     } else if (type == "way") {
-        table->set_id_type(osmium::item_type::way);
+        table->set_id_type(flex_table_index_type::way);
     } else if (type == "relation") {
-        table->set_id_type(osmium::item_type::relation);
+        table->set_id_type(flex_table_index_type::relation);
     } else if (type == "area") {
-        table->set_id_type(osmium::item_type::area);
+        table->set_id_type(flex_table_index_type::area);
     } else if (type == "any") {
-        table->set_id_type(osmium::item_type::undefined);
+        table->set_id_type(flex_table_index_type::any_object);
         lua_getfield(lua_state, -1, "type_column");
         if (lua_isstring(lua_state, -1)) {
             std::string const column_name =
@@ -139,6 +152,13 @@ static void setup_flex_table_id_columns(lua_State *lua_state,
             throw std::runtime_error{"type_column must be a string or nil."};
         }
         lua_pop(lua_state, 1); // "type_column"
+    } else if (type == "tile") {
+        table->set_id_type(flex_table_index_type::tile);
+        parse_create_index(lua_state, table);
+        table->add_column("x", "int", "int").set_not_null();
+        table->add_column("y", "int", "int").set_not_null();
+        lua_pop(lua_state, 1); // "ids"
+        return;
     } else {
         throw fmt_error("Unknown ids type: {}.", type);
     }
@@ -148,15 +168,7 @@ static void setup_flex_table_id_columns(lua_State *lua_state,
     lua_pop(lua_state, 1); // "id_column"
     check_identifier(name, "column names");
 
-    std::string const create_index = luaX_get_table_string(
-        lua_state, "create_index", -1, "The ids field", "auto");
-    lua_pop(lua_state, 1); // "create_index"
-    if (create_index == "always") {
-        table->set_always_build_id_index();
-    } else if (create_index != "auto") {
-        throw fmt_error("Unknown value '{}' for 'create_index' field of ids",
-                        create_index);
-    }
+    parse_create_index(lua_state, table);
 
     auto &column = table->add_column(name, "id_num", "");
     column.set_not_null();

src/flex-table.cpp

@@ -77,30 +77,35 @@ bool flex_table_t::has_id_column() const noexcept
 bool flex_table_t::matches_type(osmium::item_type type) const noexcept
 {
     // This table takes any type -> okay
-    if (m_id_type == osmium::item_type::undefined) {
+    if (m_id_type == flex_table_index_type::any_object) {
         return true;
     }
 
     // Type and table type match -> okay
-    if (type == m_id_type) {
+    if ((type == osmium::item_type::node &&
+         m_id_type == flex_table_index_type::node) ||
+        (type == osmium::item_type::way &&
+         m_id_type == flex_table_index_type::way) ||
+        (type == osmium::item_type::relation &&
+         m_id_type == flex_table_index_type::relation)) {
         return true;
     }
 
     // Relations can be written as linestrings into way tables -> okay
     if (type == osmium::item_type::relation &&
-        m_id_type == osmium::item_type::way) {
+        m_id_type == flex_table_index_type::way) {
         return true;
     }
 
     // Area tables can take ways or relations, but not nodes
-    return m_id_type == osmium::item_type::area &&
+    return m_id_type == flex_table_index_type::area &&
            type != osmium::item_type::node;
 }
 
 /// Map way/node/relation ID to id value used in database table column
 osmid_t flex_table_t::map_id(osmium::item_type type, osmid_t id) const noexcept
 {
-    if (m_id_type == osmium::item_type::undefined) {
+    if (m_id_type == flex_table_index_type::any_object) {
         if (has_multicolumn_id_index()) {
             return id;
         }
@@ -117,7 +122,7 @@ osmid_t flex_table_t::map_id(osmium::item_type type, osmid_t id) const noexcept
         }
     }
 
-    if (m_id_type != osmium::item_type::relation &&
+    if (m_id_type != flex_table_index_type::relation &&
         type == osmium::item_type::relation) {
         return -id;
     }

src/flex-table.hpp

@@ -27,6 +27,21 @@
 #include <utility>
 #include <vector>
 
+/**
+ * This defines the type of "primary key" for the tables generated in the flex
+ * output. This is not a real primary key, because the values are not
+ * necessarily unique.
+ */
+enum class flex_table_index_type {
+    no_index,
+    node, // index by node id
+    way, // index by way id
+    relation, // index by relation id
+    area, // index by way (positive) or relation (negative) id
+    any_object, // any OSM object, two columns for type and id
+    tile // index by tile with x and y columns (used for generalized data)
+};
+
 /**
  * An output table (in the SQL sense) for the flex backend.
  */
@@ -85,9 +100,9 @@ class flex_table_t
         m_index_tablespace = std::move(tablespace);
     }
 
-    osmium::item_type id_type() const noexcept { return m_id_type; }
+    flex_table_index_type id_type() const noexcept { return m_id_type; }
 
-    void set_id_type(osmium::item_type type) noexcept { m_id_type = type; }
+    void set_id_type(flex_table_index_type type) noexcept { m_id_type = type; }
 
     bool has_id_column() const noexcept;
 
@@ -213,10 +228,9 @@ class flex_table_t
     std::size_t m_geom_column = std::numeric_limits<std::size_t>::max();
 
     /**
-     * Type of Id stored in this table (node, way, relation, area, or
-     * undefined for any type).
+     * Type of id stored in this table.
      */
-    osmium::item_type m_id_type = osmium::item_type::undefined;
+    flex_table_index_type m_id_type = flex_table_index_type::no_index;
 
     /// Cluster the table by geometry.
     bool m_cluster_by_geom = true;

src/gen/osm2pgsql-gen.cpp

@@ -33,6 +33,7 @@
 #include "params.hpp"
 #include "pgsql-capabilities.hpp"
 #include "pgsql.hpp"
+#include "properties.hpp"
 #include "tile.hpp"
 #include "util.hpp"
 #include "version.hpp"
@@ -265,7 +266,7 @@ class genproc_t
 {
 public:
     genproc_t(std::string const &filename, std::string conninfo, bool append,
-              uint32_t jobs);
+              bool updatable, uint32_t jobs);
 
     int app_define_table()
     {
@@ -491,6 +492,7 @@ class genproc_t
     std::size_t m_gen_run = 0;
     uint32_t m_jobs;
     bool m_append;
+    bool m_updatable;
 }; // class genproc_t
 
 TRAMPOLINE(app_define_table, define_table)
@@ -499,8 +501,9 @@ TRAMPOLINE(app_run_gen, run_gen)
 TRAMPOLINE(app_run_sql, run_sql)
 
 genproc_t::genproc_t(std::string const &filename, std::string conninfo,
-                     bool append, uint32_t jobs)
-: m_conninfo(std::move(conninfo)), m_jobs(jobs), m_append(append)
+                     bool append, bool updatable, uint32_t jobs)
+: m_conninfo(std::move(conninfo)), m_jobs(jobs), m_append(append),
+  m_updatable(updatable)
 {
     setup_lua_environment(lua_state(), filename, append);
 
@@ -558,6 +561,22 @@ void genproc_t::run()
             "Failed to execute Lua function 'osm2pgsql.process_gen': {}.",
             lua_tostring(lua_state(), -1));
     }
+
+    if (!m_append) {
+        pg_conn_t const db_connection{m_conninfo};
+        for (auto const &table : m_tables) {
+            if (table.id_type() == flex_table_index_type::tile &&
+                (table.always_build_id_index() || m_updatable)) {
+                log_info("Creating tile (x/y) index on table '{}'...",
+                         table.name());
+                auto const sql =
+                    fmt::format("CREATE INDEX ON {} USING BTREE (x, y) {}",
+                                table.full_name(),
+                                tablespace_clause(table.index_tablespace()));
+                db_connection.exec(sql);
+            }
+        }
+    }
 }
 
 int main(int argc, char *argv[])
@@ -677,7 +696,11 @@ int main(int argc, char *argv[])
             init_database_capabilities(db_connection);
         }
 
-        genproc_t gen{style, conninfo, append, jobs};
+        properties_t properties{conninfo, ""};
+        properties.load();
+
+        bool const updatable = properties.get_bool("updatable", false);
+        genproc_t gen{style, conninfo, append, updatable, jobs};
         gen.run();
 
         osmium::MemoryUsage const mem;