Skip to content

Commit a6d6ab1

Browse files
committed
Add special id column type "tile" for generalized data
Generalized data is often stored in tables accessed by tile (x/y). This adds special code for that so we don't have to defined these columns explicitly in the Lua config. The columns are logically very similar to the "id column(s)" stored for "normal" output tables in that they are used by osm2pgsql to update the data correctly. So we are using the same mechanism for these columns. In "create" mode an index is automatically generated on these columns after generalization. As with the other id indexes this will happen either when the database is updatable or when the "create_index" option on the ids setting is set to "always". This commit introduces a new enum "flex_table_index_type" which is used instead of osmium::item_type to define the index type, because osmium::item_type has no value for "tile".
1 parent 5dba240 commit a6d6ab1

File tree

5 files changed

+85
-32
lines changed

5 files changed

+85
-32
lines changed

flex-config/gen/forests.lua

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -51,13 +51,12 @@ tables.forests = osm2pgsql.define_area_table('forests', {
5151

5252
-- These defines the tables for three levels of generalized data. In this
5353
-- case, the geometries generated by the generalization are always polygons.
54-
-- You must define x and y integer columns.
5554
for _, level in ipairs(gen_levels) do
5655
tables['forests_' .. level] = osm2pgsql.define_table({
5756
name = 'forests_' .. level,
57+
-- Define the x and y integer columns and add an index for them.
58+
ids = { type = 'tile' },
5859
columns = {
59-
{ column = 'x', type = 'int', not_null = true },
60-
{ column = 'y', type = 'int', not_null = true },
6160
{ column = 'geom', type = 'polygon', not_null = true }
6261
}
6362
})

src/flex-lua-table.cpp

Lines changed: 26 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -99,6 +99,19 @@ static flex_table_t &create_flex_table(lua_State *lua_state,
9999
return new_table;
100100
}
101101

102+
static void parse_create_index(lua_State *lua_state, flex_table_t *table)
103+
{
104+
std::string const create_index = luaX_get_table_string(
105+
lua_state, "create_index", -1, "The ids field", "auto");
106+
lua_pop(lua_state, 1); // "create_index"
107+
if (create_index == "always") {
108+
table->set_always_build_id_index();
109+
} else if (create_index != "auto") {
110+
throw fmt_error("Unknown value '{}' for 'create_index' field of ids",
111+
create_index);
112+
}
113+
}
114+
102115
static void setup_flex_table_id_columns(lua_State *lua_state,
103116
flex_table_t *table)
104117
{
@@ -119,15 +132,15 @@ static void setup_flex_table_id_columns(lua_State *lua_state,
119132
lua_pop(lua_state, 1); // "type"
120133

121134
if (type == "node") {
122-
table->set_id_type(osmium::item_type::node);
135+
table->set_id_type(flex_table_index_type::node);
123136
} else if (type == "way") {
124-
table->set_id_type(osmium::item_type::way);
137+
table->set_id_type(flex_table_index_type::way);
125138
} else if (type == "relation") {
126-
table->set_id_type(osmium::item_type::relation);
139+
table->set_id_type(flex_table_index_type::relation);
127140
} else if (type == "area") {
128-
table->set_id_type(osmium::item_type::area);
141+
table->set_id_type(flex_table_index_type::area);
129142
} else if (type == "any") {
130-
table->set_id_type(osmium::item_type::undefined);
143+
table->set_id_type(flex_table_index_type::any_object);
131144
lua_getfield(lua_state, -1, "type_column");
132145
if (lua_isstring(lua_state, -1)) {
133146
std::string const column_name =
@@ -139,6 +152,13 @@ static void setup_flex_table_id_columns(lua_State *lua_state,
139152
throw std::runtime_error{"type_column must be a string or nil."};
140153
}
141154
lua_pop(lua_state, 1); // "type_column"
155+
} else if (type == "tile") {
156+
table->set_id_type(flex_table_index_type::tile);
157+
parse_create_index(lua_state, table);
158+
table->add_column("x", "int", "int").set_not_null();
159+
table->add_column("y", "int", "int").set_not_null();
160+
lua_pop(lua_state, 1); // "ids"
161+
return;
142162
} else {
143163
throw fmt_error("Unknown ids type: {}.", type);
144164
}
@@ -148,15 +168,7 @@ static void setup_flex_table_id_columns(lua_State *lua_state,
148168
lua_pop(lua_state, 1); // "id_column"
149169
check_identifier(name, "column names");
150170

151-
std::string const create_index = luaX_get_table_string(
152-
lua_state, "create_index", -1, "The ids field", "auto");
153-
lua_pop(lua_state, 1); // "create_index"
154-
if (create_index == "always") {
155-
table->set_always_build_id_index();
156-
} else if (create_index != "auto") {
157-
throw fmt_error("Unknown value '{}' for 'create_index' field of ids",
158-
create_index);
159-
}
171+
parse_create_index(lua_state, table);
160172

161173
auto &column = table->add_column(name, "id_num", "");
162174
column.set_not_null();

src/flex-table.cpp

Lines changed: 11 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -77,30 +77,35 @@ bool flex_table_t::has_id_column() const noexcept
7777
bool flex_table_t::matches_type(osmium::item_type type) const noexcept
7878
{
7979
// This table takes any type -> okay
80-
if (m_id_type == osmium::item_type::undefined) {
80+
if (m_id_type == flex_table_index_type::any_object) {
8181
return true;
8282
}
8383

8484
// Type and table type match -> okay
85-
if (type == m_id_type) {
85+
if ((type == osmium::item_type::node &&
86+
m_id_type == flex_table_index_type::node) ||
87+
(type == osmium::item_type::way &&
88+
m_id_type == flex_table_index_type::way) ||
89+
(type == osmium::item_type::relation &&
90+
m_id_type == flex_table_index_type::relation)) {
8691
return true;
8792
}
8893

8994
// Relations can be written as linestrings into way tables -> okay
9095
if (type == osmium::item_type::relation &&
91-
m_id_type == osmium::item_type::way) {
96+
m_id_type == flex_table_index_type::way) {
9297
return true;
9398
}
9499

95100
// Area tables can take ways or relations, but not nodes
96-
return m_id_type == osmium::item_type::area &&
101+
return m_id_type == flex_table_index_type::area &&
97102
type != osmium::item_type::node;
98103
}
99104

100105
/// Map way/node/relation ID to id value used in database table column
101106
osmid_t flex_table_t::map_id(osmium::item_type type, osmid_t id) const noexcept
102107
{
103-
if (m_id_type == osmium::item_type::undefined) {
108+
if (m_id_type == flex_table_index_type::any_object) {
104109
if (has_multicolumn_id_index()) {
105110
return id;
106111
}
@@ -117,7 +122,7 @@ osmid_t flex_table_t::map_id(osmium::item_type type, osmid_t id) const noexcept
117122
}
118123
}
119124

120-
if (m_id_type != osmium::item_type::relation &&
125+
if (m_id_type != flex_table_index_type::relation &&
121126
type == osmium::item_type::relation) {
122127
return -id;
123128
}

src/flex-table.hpp

Lines changed: 19 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,21 @@
2727
#include <utility>
2828
#include <vector>
2929

30+
/**
31+
* This defines the type of "primary key" for the tables generated in the flex
32+
* output. This is not a real primary key, because the values are not
33+
* necessarily unique.
34+
*/
35+
enum class flex_table_index_type {
36+
no_index,
37+
node, // index by node id
38+
way, // index by way id
39+
relation, // index by relation id
40+
area, // index by way (positive) or relation (negative) id
41+
any_object, // any OSM object, two columns for type and id
42+
tile // index by tile with x and y columns (used for generalized data)
43+
};
44+
3045
/**
3146
* An output table (in the SQL sense) for the flex backend.
3247
*/
@@ -85,9 +100,9 @@ class flex_table_t
85100
m_index_tablespace = std::move(tablespace);
86101
}
87102

88-
osmium::item_type id_type() const noexcept { return m_id_type; }
103+
flex_table_index_type id_type() const noexcept { return m_id_type; }
89104

90-
void set_id_type(osmium::item_type type) noexcept { m_id_type = type; }
105+
void set_id_type(flex_table_index_type type) noexcept { m_id_type = type; }
91106

92107
bool has_id_column() const noexcept;
93108

@@ -213,10 +228,9 @@ class flex_table_t
213228
std::size_t m_geom_column = std::numeric_limits<std::size_t>::max();
214229

215230
/**
216-
* Type of Id stored in this table (node, way, relation, area, or
217-
* undefined for any type).
231+
* Type of id stored in this table.
218232
*/
219-
osmium::item_type m_id_type = osmium::item_type::undefined;
233+
flex_table_index_type m_id_type = flex_table_index_type::no_index;
220234

221235
/// Cluster the table by geometry.
222236
bool m_cluster_by_geom = true;

src/gen/osm2pgsql-gen.cpp

Lines changed: 27 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@
3333
#include "params.hpp"
3434
#include "pgsql-capabilities.hpp"
3535
#include "pgsql.hpp"
36+
#include "properties.hpp"
3637
#include "tile.hpp"
3738
#include "util.hpp"
3839
#include "version.hpp"
@@ -265,7 +266,7 @@ class genproc_t
265266
{
266267
public:
267268
genproc_t(std::string const &filename, std::string conninfo, bool append,
268-
uint32_t jobs);
269+
bool updatable, uint32_t jobs);
269270

270271
int app_define_table()
271272
{
@@ -491,6 +492,7 @@ class genproc_t
491492
std::size_t m_gen_run = 0;
492493
uint32_t m_jobs;
493494
bool m_append;
495+
bool m_updatable;
494496
}; // class genproc_t
495497

496498
TRAMPOLINE(app_define_table, define_table)
@@ -499,8 +501,9 @@ TRAMPOLINE(app_run_gen, run_gen)
499501
TRAMPOLINE(app_run_sql, run_sql)
500502

501503
genproc_t::genproc_t(std::string const &filename, std::string conninfo,
502-
bool append, uint32_t jobs)
503-
: m_conninfo(std::move(conninfo)), m_jobs(jobs), m_append(append)
504+
bool append, bool updatable, uint32_t jobs)
505+
: m_conninfo(std::move(conninfo)), m_jobs(jobs), m_append(append),
506+
m_updatable(updatable)
504507
{
505508
setup_lua_environment(lua_state(), filename, append);
506509

@@ -558,6 +561,22 @@ void genproc_t::run()
558561
"Failed to execute Lua function 'osm2pgsql.process_gen': {}.",
559562
lua_tostring(lua_state(), -1));
560563
}
564+
565+
if (!m_append) {
566+
pg_conn_t const db_connection{m_conninfo};
567+
for (auto const &table : m_tables) {
568+
if (table.id_type() == flex_table_index_type::tile &&
569+
(table.always_build_id_index() || m_updatable)) {
570+
log_info("Creating tile (x/y) index on table '{}'...",
571+
table.name());
572+
auto const sql =
573+
fmt::format("CREATE INDEX ON {} USING BTREE (x, y) {}",
574+
table.full_name(),
575+
tablespace_clause(table.index_tablespace()));
576+
db_connection.exec(sql);
577+
}
578+
}
579+
}
561580
}
562581

563582
int main(int argc, char *argv[])
@@ -677,7 +696,11 @@ int main(int argc, char *argv[])
677696
init_database_capabilities(db_connection);
678697
}
679698

680-
genproc_t gen{style, conninfo, append, jobs};
699+
properties_t properties{conninfo, ""};
700+
properties.load();
701+
702+
bool const updatable = properties.get_bool("updatable", false);
703+
genproc_t gen{style, conninfo, append, updatable, jobs};
681704
gen.run();
682705

683706
osmium::MemoryUsage const mem;

0 commit comments

Comments
 (0)