Skip to content

Commit e6a3096

Browse files
lonvialonvia
authored
Merge pull request #2134 from joto/fewer-db-conns
Significantly reduce number of db connection in flex output
2 parents 68eae3f + ef8f6e5 commit e6a3096

File tree

6 files changed

+83
-86
lines changed

6 files changed

+83
-86
lines changed

‎src/flex-lua-table.cpp‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,8 @@ static flex_table_t &create_flex_table(lua_State *lua_state,
4141
throw fmt_error("Table with name '{}' already exists.", table_name);
4242
}
4343

44-
auto &new_table = tables->emplace_back(default_schema, table_name);
44+
auto &new_table =
45+
tables->emplace_back(default_schema, table_name, tables->size());
4546

4647
lua_pop(lua_state, 1); // "name"
4748

‎src/flex-table.cpp‎

Lines changed: 36 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -167,14 +167,15 @@ std::string flex_table_t::build_sql_prepare_get_wkb() const
167167

168168
if (has_multicolumn_id_index()) {
169169
return fmt::format(
170-
R"(PREPARE get_wkb(char(1), bigint) AS)"
170+
R"(PREPARE get_wkb_{}(char(1), bigint) AS)"
171171
R"( SELECT {} FROM {} WHERE "{}" = $1 AND "{}" = $2)",
172-
columns, full_name(), m_columns[0].name(), m_columns[1].name());
172+
m_table_num, columns, full_name(), m_columns[0].name(),
173+
m_columns[1].name());
173174
}
174175

175-
return fmt::format(R"(PREPARE get_wkb(bigint) AS)"
176+
return fmt::format(R"(PREPARE get_wkb_{}(bigint) AS)"
176177
R"( SELECT {} FROM {} WHERE "{}" = $1)",
177-
columns, full_name(), id_column_names());
178+
m_table_num, columns, full_name(), id_column_names());
178179
}
179180

180181
std::string
@@ -241,14 +242,6 @@ bool flex_table_t::has_columns_with_expire() const noexcept
241242
[](auto const &column) { return column.has_expire(); });
242243
}
243244

244-
void table_connection_t::connect(connection_params_t const &connection_params)
245-
{
246-
assert(!m_db_connection);
247-
248-
m_db_connection =
249-
std::make_unique<pg_conn_t>(connection_params, "out.flex.table");
250-
}
251-
252245
static void enable_check_trigger(pg_conn_t const &db_connection,
253246
flex_table_t const &table)
254247
{
@@ -273,50 +266,46 @@ static void enable_check_trigger(pg_conn_t const &db_connection,
273266
checks);
274267
}
275268

276-
void table_connection_t::start(bool append)
269+
void table_connection_t::start(pg_conn_t const &db_connection, bool append)
277270
{
278-
assert(m_db_connection);
279-
280271
if (!append) {
281-
m_db_connection->exec("DROP TABLE IF EXISTS {} CASCADE",
282-
table().full_name());
272+
db_connection.exec("DROP TABLE IF EXISTS {} CASCADE",
273+
table().full_name());
283274
}
284275

285276
// These _tmp tables can be left behind if we run out of disk space.
286-
m_db_connection->exec("DROP TABLE IF EXISTS {}", table().full_tmp_name());
277+
db_connection.exec("DROP TABLE IF EXISTS {}", table().full_tmp_name());
287278

288279
if (!append) {
289-
m_db_connection->exec(table().build_sql_create_table(
280+
db_connection.exec(table().build_sql_create_table(
290281
table().cluster_by_geom() ? flex_table_t::table_type::interim
291282
: flex_table_t::table_type::permanent,
292283
table().full_name()));
293284

294-
enable_check_trigger(*m_db_connection, table());
285+
enable_check_trigger(db_connection, table());
295286
}
296287

297-
prepare();
288+
prepare(db_connection);
298289
}
299290

300-
void table_connection_t::stop(bool updateable, bool append)
291+
void table_connection_t::stop(pg_conn_t const &db_connection, bool updateable,
292+
bool append)
301293
{
302-
assert(m_db_connection);
303-
304294
m_copy_mgr.sync();
305295

306296
if (append) {
307-
teardown();
308297
return;
309298
}
310299

311300
if (table().cluster_by_geom()) {
312301
if (table().geom_column().needs_isvalid()) {
313-
drop_geom_check_trigger(*m_db_connection, table().schema(),
302+
drop_geom_check_trigger(db_connection, table().schema(),
314303
table().name());
315304
}
316305

317306
log_info("Clustering table '{}' by geometry...", table().name());
318307

319-
m_db_connection->exec(table().build_sql_create_table(
308+
db_connection.exec(table().build_sql_create_table(
320309
flex_table_t::table_type::permanent, table().full_tmp_name()));
321310

322311
std::string const columns = table().build_sql_column_list();
@@ -349,15 +338,15 @@ void table_connection_t::stop(bool updateable, bool append)
349338
sql += geom_column_name;
350339
}
351340

352-
m_db_connection->exec(sql);
341+
db_connection.exec(sql);
353342

354-
m_db_connection->exec("DROP TABLE {}", table().full_name());
355-
m_db_connection->exec(R"(ALTER TABLE {} RENAME TO "{}")",
356-
table().full_tmp_name(), table().name());
343+
db_connection.exec("DROP TABLE {}", table().full_name());
344+
db_connection.exec(R"(ALTER TABLE {} RENAME TO "{}")",
345+
table().full_tmp_name(), table().name());
357346
m_id_index_created = false;
358347

359348
if (updateable) {
360-
enable_check_trigger(*m_db_connection, table());
349+
enable_check_trigger(db_connection, table());
361350
}
362351
}
363352

@@ -369,55 +358,53 @@ void table_connection_t::stop(bool updateable, bool append)
369358
index.columns());
370359
auto const sql = index.create_index(
371360
qualified_name(table().schema(), table().name()));
372-
m_db_connection->exec(sql);
361+
db_connection.exec(sql);
373362
}
374363
}
375364

376365
if ((table().always_build_id_index() || updateable) &&
377366
table().has_id_column()) {
378-
create_id_index();
367+
create_id_index(db_connection);
379368
}
380369

381370
log_info("Analyzing table '{}'...", table().name());
382-
analyze();
383-
384-
teardown();
371+
analyze(db_connection);
385372
}
386373

387-
void table_connection_t::prepare()
374+
void table_connection_t::prepare(pg_conn_t const &db_connection)
388375
{
389-
assert(m_db_connection);
390376
if (table().has_id_column() && table().has_columns_with_expire()) {
391-
m_db_connection->exec(table().build_sql_prepare_get_wkb());
377+
db_connection.exec(table().build_sql_prepare_get_wkb());
392378
}
393379
}
394380

395-
void table_connection_t::analyze()
381+
void table_connection_t::analyze(pg_conn_t const &db_connection)
396382
{
397-
analyze_table(*m_db_connection, table().schema(), table().name());
383+
analyze_table(db_connection, table().schema(), table().name());
398384
}
399385

400-
void table_connection_t::create_id_index()
386+
void table_connection_t::create_id_index(pg_conn_t const &db_connection)
401387
{
402388
if (m_id_index_created) {
403389
log_debug("Id index on table '{}' already created.", table().name());
404390
} else {
405391
log_info("Creating id index on table '{}'...", table().name());
406-
m_db_connection->exec(table().build_sql_create_id_index());
392+
db_connection.exec(table().build_sql_create_id_index());
407393
m_id_index_created = true;
408394
}
409395
}
410396

411-
pg_result_t table_connection_t::get_geoms_by_id(osmium::item_type type,
397+
pg_result_t table_connection_t::get_geoms_by_id(pg_conn_t const &db_connection,
398+
osmium::item_type type,
412399
osmid_t id) const
413400
{
414401
assert(table().has_geom_column());
415-
assert(m_db_connection);
402+
std::string const stmt = fmt::format("get_wkb_{}", table().num());
416403
if (table().has_multicolumn_id_index()) {
417-
return m_db_connection->exec_prepared_as_binary("get_wkb",
418-
type_to_char(type), id);
404+
return db_connection.exec_prepared_as_binary(stmt.c_str(),
405+
type_to_char(type), id);
419406
}
420-
return m_db_connection->exec_prepared_as_binary("get_wkb", id);
407+
return db_connection.exec_prepared_as_binary(stmt.c_str(), id);
421408
}
422409

423410
void table_connection_t::delete_rows_with(osmium::item_type type, osmid_t id)

‎src/flex-table.hpp‎

Lines changed: 15 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -59,8 +59,8 @@ class flex_table_t
5959
permanent
6060
};
6161

62-
flex_table_t(std::string schema, std::string name)
63-
: m_schema(std::move(schema)), m_name(std::move(name))
62+
flex_table_t(std::string schema, std::string name, std::size_t num)
63+
: m_schema(std::move(schema)), m_name(std::move(name)), m_table_num(num)
6464
{
6565
}
6666

@@ -197,6 +197,8 @@ class flex_table_t
197197

198198
bool has_columns_with_expire() const noexcept;
199199

200+
std::size_t num() const noexcept { return m_table_num; }
201+
200202
private:
201203
/// The schema this table is in
202204
std::string m_schema;
@@ -230,6 +232,9 @@ class flex_table_t
230232
*/
231233
std::size_t m_geom_column = std::numeric_limits<std::size_t>::max();
232234

235+
/// Unique number for each table.
236+
std::size_t m_table_num;
237+
233238
/**
234239
* Type of id stored in this table.
235240
*/
@@ -255,31 +260,28 @@ class table_connection_t
255260
m_target(std::make_shared<db_target_descr_t>(
256261
table->schema(), table->name(), table->id_column_names(),
257262
table->build_sql_column_list())),
258-
m_copy_mgr(copy_thread), m_db_connection(nullptr)
263+
m_copy_mgr(copy_thread)
259264
{
260265
}
261266

262-
void connect(connection_params_t const &connection_params);
263-
264-
void start(bool append);
267+
void start(pg_conn_t const &db_connection, bool append);
265268

266-
void stop(bool updateable, bool append);
269+
void stop(pg_conn_t const &db_connection, bool updateable, bool append);
267270

268271
flex_table_t const &table() const noexcept { return *m_table; }
269272

270-
void teardown() { m_db_connection.reset(); }
273+
void prepare(pg_conn_t const &db_connection);
271274

272-
void prepare();
275+
void analyze(pg_conn_t const &db_connection);
273276

274-
void analyze();
275-
276-
void create_id_index();
277+
void create_id_index(pg_conn_t const &db_connection);
277278

278279
/**
279280
* Get all geometries that have at least one expire config defined
280281
* from the database and return the result set.
281282
*/
282-
pg_result_t get_geoms_by_id(osmium::item_type type, osmid_t id) const;
283+
pg_result_t get_geoms_by_id(pg_conn_t const &db_connection,
284+
osmium::item_type type, osmid_t id) const;
283285

284286
void flush() { m_copy_mgr.flush(); }
285287

@@ -327,9 +329,6 @@ class table_connection_t
327329
*/
328330
db_copy_mgr_t<db_deleter_by_type_and_id_t> m_copy_mgr;
329331

330-
/// The connection to the database server.
331-
std::unique_ptr<pg_conn_t> m_db_connection;
332-
333332
task_result_t m_task_result;
334333

335334
std::size_t m_count_insert = 0;

‎src/output-flex.cpp‎

Lines changed: 14 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1076,7 +1076,10 @@ void output_flex_t::stop()
10761076
{
10771077
for (auto &table : m_table_connections) {
10781078
table.task_set(thread_pool().submit([&]() {
1079-
table.stop(get_options()->slim && !get_options()->droptemp,
1079+
pg_conn_t const db_connection{get_options()->connection_params,
1080+
"out.flex.stop"};
1081+
table.stop(db_connection,
1082+
get_options()->slim && !get_options()->droptemp,
10801083
get_options()->append);
10811084
}));
10821085
}
@@ -1151,13 +1154,15 @@ void output_flex_t::relation_add(osmium::Relation const &relation)
11511154
}
11521155

11531156
void output_flex_t::delete_from_table(table_connection_t *table_connection,
1157+
pg_conn_t const &db_connection,
11541158
osmium::item_type type, osmid_t osm_id)
11551159
{
11561160
assert(table_connection);
11571161
auto const id = table_connection->table().map_id(type, osm_id);
11581162

11591163
if (table_connection->table().has_columns_with_expire()) {
1160-
auto const result = table_connection->get_geoms_by_id(type, id);
1164+
auto const result =
1165+
table_connection->get_geoms_by_id(db_connection, type, id);
11611166
auto const num_tuples = result.num_tuples();
11621167
if (num_tuples > 0) {
11631168
int col = 0;
@@ -1180,7 +1185,7 @@ void output_flex_t::delete_from_tables(osmium::item_type type, osmid_t osm_id)
11801185
{
11811186
for (auto &table : m_table_connections) {
11821187
if (table.table().matches_type(type) && table.table().has_id_column()) {
1183-
delete_from_table(&table, type, osm_id);
1188+
delete_from_table(&table, m_db_connection, type, osm_id);
11841189
}
11851190
}
11861191
}
@@ -1225,8 +1230,7 @@ void output_flex_t::relation_modify(osmium::Relation const &rel)
12251230
void output_flex_t::start()
12261231
{
12271232
for (auto &table : m_table_connections) {
1228-
table.connect(get_options()->connection_params);
1229-
table.start(get_options()->append);
1233+
table.start(m_db_connection, get_options()->append);
12301234
}
12311235
}
12321236

@@ -1254,6 +1258,7 @@ output_flex_t::output_flex_t(output_flex_t const *other,
12541258
std::shared_ptr<db_copy_thread_t> copy_thread)
12551259
: output_t(other, std::move(mid)), m_tables(other->m_tables),
12561260
m_expire_outputs(other->m_expire_outputs),
1261+
m_db_connection(get_options()->connection_params, "out.flex.thread"),
12571262
m_stage2_way_ids(other->m_stage2_way_ids),
12581263
m_copy_thread(std::move(copy_thread)), m_lua_state(other->m_lua_state),
12591264
m_process_node(other->m_process_node), m_process_way(other->m_process_way),
@@ -1262,8 +1267,7 @@ output_flex_t::output_flex_t(output_flex_t const *other,
12621267
{
12631268
for (auto &table : *m_tables) {
12641269
auto &tc = m_table_connections.emplace_back(&table, m_copy_thread);
1265-
tc.connect(get_options()->connection_params);
1266-
tc.prepare();
1270+
tc.prepare(m_db_connection);
12671271
}
12681272

12691273
for (auto &expire_output : *m_expire_outputs) {
@@ -1283,6 +1287,7 @@ output_flex_t::output_flex_t(std::shared_ptr<middle_query_t> const &mid,
12831287
std::shared_ptr<thread_pool_t> thread_pool,
12841288
options_t const &options)
12851289
: output_t(mid, std::move(thread_pool), options),
1290+
m_db_connection(get_options()->connection_params, "out.flex.main"),
12861291
m_copy_thread(std::make_shared<db_copy_thread_t>(options.connection_params))
12871292
{
12881293
init_lua(options.style);
@@ -1509,8 +1514,8 @@ void output_flex_t::reprocess_marked()
15091514
for (auto &table : m_table_connections) {
15101515
if (table.table().matches_type(osmium::item_type::way) &&
15111516
table.table().has_id_column()) {
1512-
table.analyze();
1513-
table.create_id_index();
1517+
table.analyze(m_db_connection);
1518+
table.create_id_index(m_db_connection);
15141519
}
15151520
}
15161521

‎src/output-flex.hpp‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -224,7 +224,9 @@ class output_flex_t : public output_t
224224
void add_row(table_connection_t *table_connection, OBJECT const &object);
225225

226226
void delete_from_table(table_connection_t *table_connection,
227+
pg_conn_t const &db_connection,
227228
osmium::item_type type, osmid_t osm_id);
229+
228230
void delete_from_tables(osmium::item_type type, osmid_t osm_id);
229231

230232
lua_State *lua_state() noexcept { return m_lua_state.get(); }
@@ -288,6 +290,9 @@ class output_flex_t : public output_t
288290

289291
std::vector<table_connection_t> m_table_connections;
290292

293+
/// The connection to the database server.
294+
pg_conn_t m_db_connection;
295+
291296
// This is shared between all clones of the output and must only be
292297
// accessed while protected using the lua_mutex.
293298
std::shared_ptr<idset_t> m_stage2_way_ids = std::make_shared<idset_t>();

0 commit comments

Comments
 (0)