Skip to content

Commit d34c23b

Browse files
lonvialonvia
authored
Merge pull request #2002 from joto/single-query-per-exec
Don't do multi-statement queries
2 parents 1cfb05b + 7529244 commit d34c23b

File tree

3 files changed

+101
-83
lines changed

3 files changed

+101
-83
lines changed

‎src/middle-pgsql.cpp‎

Lines changed: 91 additions & 79 deletions
Original file line numberDiff line numberDiff line change
@@ -116,10 +116,22 @@ static std::string build_sql(options_t const &options, std::string const &templ)
116116
: ""));
117117
}
118118

119+
static std::vector<std::string>
120+
build_sql(options_t const &options, std::vector<std::string> const &templs)
121+
{
122+
std::vector<std::string> out;
123+
124+
for (auto const &templ : templs) {
125+
out.push_back(build_sql(options, templ));
126+
}
127+
128+
return out;
129+
}
130+
119131
middle_pgsql_t::table_desc::table_desc(options_t const &options,
120132
table_sql const &ts)
121133
: m_create_table(build_sql(options, ts.create_table)),
122-
m_prepare_query(build_sql(options, ts.prepare_query)),
134+
m_prepare_queries(build_sql(options, ts.prepare_queries)),
123135
m_copy_target(std::make_shared<db_target_descr_t>())
124136
{
125137
m_copy_target->name = build_sql(options, ts.name);
@@ -161,7 +173,9 @@ void middle_pgsql_t::table_desc::build_index(std::string const &conninfo) const
161173
pg_conn_t const db_connection{conninfo};
162174

163175
log_info("Building index on table '{}'", name());
164-
db_connection.exec(m_create_fw_dep_indexes);
176+
for (auto const &query : m_create_fw_dep_indexes) {
177+
db_connection.exec(query);
178+
}
165179
}
166180

167181
/**
@@ -1307,7 +1321,7 @@ void middle_pgsql_t::update_users_table()
13071321

13081322
m_db_connection.exec("PREPARE insert_user(int8, text) AS"
13091323
" INSERT INTO {}\"{}\" (id, name) VALUES ($1, $2)"
1310-
" ON CONFLICT (id) DO UPDATE SET id=EXCLUDED.id;\n",
1324+
" ON CONFLICT (id) DO UPDATE SET id=EXCLUDED.id",
13111325
m_users_table.schema(), m_users_table.name());
13121326

13131327
for (auto const &[id, name] : m_users) {
@@ -1361,7 +1375,7 @@ static table_sql sql_for_users(middle_pgsql_options const &store_options)
13611375
sql.create_table = "CREATE TABLE {schema}\"{prefix}_users\" ("
13621376
" id INT4 PRIMARY KEY {using_tablespace},"
13631377
" name TEXT NOT NULL"
1364-
") {data_tablespace};\n";
1378+
") {data_tablespace}";
13651379
}
13661380

13671381
return sql;
@@ -1379,15 +1393,15 @@ static table_sql sql_for_nodes_format1(bool create_table)
13791393
" id int8 PRIMARY KEY {using_tablespace},"
13801394
" lat int4 NOT NULL,"
13811395
" lon int4 NOT NULL"
1382-
") {data_tablespace};\n";
1396+
") {data_tablespace}";
13831397

1384-
sql.prepare_query =
1398+
sql.prepare_queries = {
13851399
"PREPARE get_node_list(int8[]) AS"
13861400
" SELECT id, lon, lat FROM {schema}\"{prefix}_nodes\""
1387-
" WHERE id = ANY($1::int8[]);\n"
1401+
" WHERE id = ANY($1::int8[])",
13881402
"PREPARE get_node(int8) AS"
13891403
" SELECT id, lon, lat FROM {schema}\"{prefix}_nodes\""
1390-
" WHERE id = $1;\n";
1404+
" WHERE id = $1"};
13911405
}
13921406

13931407
return sql;
@@ -1407,15 +1421,15 @@ static table_sql sql_for_nodes_format2(middle_pgsql_options const &options)
14071421
" lon int4 NOT NULL,"
14081422
"{attribute_columns_definition}"
14091423
" tags jsonb NOT NULL"
1410-
") {data_tablespace};\n";
1424+
") {data_tablespace}";
14111425

1412-
sql.prepare_query =
1426+
sql.prepare_queries = {
14131427
"PREPARE get_node_list(int8[]) AS"
14141428
" SELECT id, lon, lat FROM {schema}\"{prefix}_nodes\""
1415-
" WHERE id = ANY($1::int8[]);\n"
1429+
" WHERE id = ANY($1::int8[])",
14161430
"PREPARE get_node(int8) AS"
14171431
" SELECT id, lon, lat FROM {schema}\"{prefix}_nodes\""
1418-
" WHERE id = $1;\n";
1432+
" WHERE id = $1"};
14191433
}
14201434

14211435
return sql;
@@ -1431,32 +1445,32 @@ static table_sql sql_for_ways_format1(uint8_t way_node_index_id_shift)
14311445
" id int8 PRIMARY KEY {using_tablespace},"
14321446
" nodes int8[] NOT NULL,"
14331447
" tags text[]"
1434-
") {data_tablespace};\n";
1448+
") {data_tablespace}";
14351449

1436-
sql.prepare_query = "PREPARE get_way(int8) AS"
1437-
" SELECT nodes, tags"
1438-
" FROM {schema}\"{prefix}_ways\" WHERE id = $1;\n"
1439-
"PREPARE get_way_list(int8[]) AS"
1440-
" SELECT id, nodes, tags"
1441-
" FROM {schema}\"{prefix}_ways\""
1442-
" WHERE id = ANY($1::int8[]);\n";
1450+
sql.prepare_queries = {"PREPARE get_way(int8) AS"
1451+
" SELECT nodes, tags"
1452+
" FROM {schema}\"{prefix}_ways\" WHERE id = $1",
1453+
"PREPARE get_way_list(int8[]) AS"
1454+
" SELECT id, nodes, tags"
1455+
" FROM {schema}\"{prefix}_ways\""
1456+
" WHERE id = ANY($1::int8[])"};
14431457

14441458
if (way_node_index_id_shift == 0) {
1445-
sql.create_fw_dep_indexes =
1459+
sql.create_fw_dep_indexes = {
14461460
"CREATE INDEX ON {schema}\"{prefix}_ways\" USING GIN (nodes)"
1447-
" WITH (fastupdate = off) {index_tablespace};\n";
1461+
" WITH (fastupdate = off) {index_tablespace}"};
14481462
} else {
1449-
sql.create_fw_dep_indexes =
1463+
sql.create_fw_dep_indexes = {
14501464
"CREATE OR REPLACE FUNCTION"
14511465
" {schema}\"{prefix}_index_bucket\"(int8[])"
1452-
" RETURNS int8[] AS $$\n"
1466+
" RETURNS int8[] AS $$"
14531467
" SELECT ARRAY(SELECT DISTINCT"
1454-
" unnest($1) >> {way_node_index_id_shift})\n"
1455-
"$$ LANGUAGE SQL IMMUTABLE;\n"
1468+
" unnest($1) >> {way_node_index_id_shift})"
1469+
"$$ LANGUAGE SQL IMMUTABLE",
14561470
"CREATE INDEX \"{prefix}_ways_nodes_bucket_idx\""
14571471
" ON {schema}\"{prefix}_ways\""
14581472
" USING GIN ({schema}\"{prefix}_index_bucket\"(nodes))"
1459-
" WITH (fastupdate = off) {index_tablespace};\n";
1473+
" WITH (fastupdate = off) {index_tablespace}"};
14601474
}
14611475

14621476
return sql;
@@ -1473,36 +1487,35 @@ static table_sql sql_for_ways_format2(middle_pgsql_options const &options)
14731487
"{attribute_columns_definition}"
14741488
" nodes int8[] NOT NULL,"
14751489
" tags jsonb NOT NULL"
1476-
") {data_tablespace};\n";
1477-
1478-
sql.prepare_query = "PREPARE get_way(int8) AS"
1479-
" SELECT nodes, tags{attribute_columns_use}"
1480-
" FROM {schema}\"{prefix}_ways\" o"
1481-
" {users_table_access}"
1482-
" WHERE o.id = $1;\n";
1483-
1484-
sql.prepare_query += "PREPARE get_way_list(int8[]) AS"
1485-
" SELECT o.id, nodes, tags{attribute_columns_use}"
1486-
" FROM {schema}\"{prefix}_ways\" o"
1487-
" {users_table_access}"
1488-
" WHERE o.id = ANY($1::int8[]);\n";
1490+
") {data_tablespace}";
1491+
1492+
sql.prepare_queries = {"PREPARE get_way(int8) AS"
1493+
" SELECT nodes, tags{attribute_columns_use}"
1494+
" FROM {schema}\"{prefix}_ways\" o"
1495+
" {users_table_access}"
1496+
" WHERE o.id = $1",
1497+
"PREPARE get_way_list(int8[]) AS"
1498+
" SELECT o.id, nodes, tags{attribute_columns_use}"
1499+
" FROM {schema}\"{prefix}_ways\" o"
1500+
" {users_table_access}"
1501+
" WHERE o.id = ANY($1::int8[])"};
14891502

14901503
if (options.way_node_index_id_shift == 0) {
1491-
sql.create_fw_dep_indexes =
1504+
sql.create_fw_dep_indexes = {
14921505
"CREATE INDEX ON {schema}\"{prefix}_ways\" USING GIN (nodes)"
1493-
" WITH (fastupdate = off) {index_tablespace};\n";
1506+
" WITH (fastupdate = off) {index_tablespace}"};
14941507
} else {
1495-
sql.create_fw_dep_indexes =
1508+
sql.create_fw_dep_indexes = {
14961509
"CREATE OR REPLACE FUNCTION"
14971510
" {schema}\"{prefix}_index_bucket\"(int8[])"
1498-
" RETURNS int8[] AS $$\n"
1511+
" RETURNS int8[] AS $$"
14991512
" SELECT ARRAY(SELECT DISTINCT"
1500-
" unnest($1) >> {way_node_index_id_shift})\n"
1501-
"$$ LANGUAGE SQL IMMUTABLE;\n"
1513+
" unnest($1) >> {way_node_index_id_shift})"
1514+
"$$ LANGUAGE SQL IMMUTABLE",
15021515
"CREATE INDEX \"{prefix}_ways_nodes_bucket_idx\""
15031516
" ON {schema}\"{prefix}_ways\""
15041517
" USING GIN ({schema}\"{prefix}_index_bucket\"(nodes))"
1505-
" WITH (fastupdate = off) {index_tablespace};\n";
1518+
" WITH (fastupdate = off) {index_tablespace}"};
15061519
}
15071520

15081521
return sql;
@@ -1521,15 +1534,15 @@ static table_sql sql_for_relations_format1()
15211534
" parts int8[],"
15221535
" members text[],"
15231536
" tags text[]"
1524-
") {data_tablespace};\n";
1537+
") {data_tablespace}";
15251538

1526-
sql.prepare_query = "PREPARE get_rel(int8) AS"
1527-
" SELECT members, tags"
1528-
" FROM {schema}\"{prefix}_rels\" WHERE id = $1;\n";
1539+
sql.prepare_queries = {"PREPARE get_rel(int8) AS"
1540+
" SELECT members, tags"
1541+
" FROM {schema}\"{prefix}_rels\" WHERE id = $1"};
15291542

1530-
sql.create_fw_dep_indexes =
1543+
sql.create_fw_dep_indexes = {
15311544
"CREATE INDEX ON {schema}\"{prefix}_rels\" USING GIN (parts)"
1532-
" WITH (fastupdate = off) {index_tablespace};\n";
1545+
" WITH (fastupdate = off) {index_tablespace}"};
15331546

15341547
return sql;
15351548
}
@@ -1540,36 +1553,35 @@ static table_sql sql_for_relations_format2()
15401553

15411554
sql.name = "{prefix}_rels";
15421555

1543-
sql.create_table += "CREATE {unlogged} TABLE {schema}\"{prefix}_rels\" ("
1544-
" id int8 PRIMARY KEY {using_tablespace},"
1545-
"{attribute_columns_definition}"
1546-
" members jsonb NOT NULL,"
1547-
" tags jsonb NOT NULL"
1548-
") {data_tablespace};\n"
1549-
1550-
"CREATE OR REPLACE FUNCTION"
1551-
" {schema}\"{prefix}_member_ids\"(jsonb, char)"
1552-
" RETURNS int8[] AS $$\n"
1553-
" SELECT array_agg((el->>'ref')::int8)"
1554-
" FROM jsonb_array_elements($1) AS el"
1555-
" WHERE el->>'type' = $2\n"
1556-
"$$ LANGUAGE SQL IMMUTABLE;\n";
1557-
1558-
sql.prepare_query = "PREPARE get_rel(int8) AS"
1559-
" SELECT members, tags{attribute_columns_use}"
1560-
" FROM {schema}\"{prefix}_rels\" o"
1561-
" {users_table_access}"
1562-
" WHERE o.id = $1;\n";
1563-
1564-
sql.create_fw_dep_indexes =
1556+
sql.create_table = "CREATE {unlogged} TABLE {schema}\"{prefix}_rels\" ("
1557+
" id int8 PRIMARY KEY {using_tablespace},"
1558+
"{attribute_columns_definition}"
1559+
" members jsonb NOT NULL,"
1560+
" tags jsonb NOT NULL"
1561+
") {data_tablespace}";
1562+
1563+
sql.prepare_queries = {"PREPARE get_rel(int8) AS"
1564+
" SELECT members, tags{attribute_columns_use}"
1565+
" FROM {schema}\"{prefix}_rels\" o"
1566+
" {users_table_access}"
1567+
" WHERE o.id = $1"};
1568+
1569+
sql.create_fw_dep_indexes = {
1570+
"CREATE OR REPLACE FUNCTION"
1571+
" {schema}\"{prefix}_member_ids\"(jsonb, char)"
1572+
" RETURNS int8[] AS $$"
1573+
" SELECT array_agg((el->>'ref')::int8)"
1574+
" FROM jsonb_array_elements($1) AS el"
1575+
" WHERE el->>'type' = $2"
1576+
"$$ LANGUAGE SQL IMMUTABLE",
15651577
"CREATE INDEX \"{prefix}_rels_node_members\""
15661578
" ON {schema}\"{prefix}_rels\" USING GIN"
15671579
" (({schema}\"{prefix}_member_ids\"(members, 'N'::char)))"
1568-
" WITH (fastupdate = off) {index_tablespace};\n"
1580+
" WITH (fastupdate = off) {index_tablespace}",
15691581
"CREATE INDEX \"{prefix}_rels_way_members\""
15701582
" ON {schema}\"{prefix}_rels\" USING GIN"
15711583
" (({schema}\"{prefix}_member_ids\"(members, 'W'::char)))"
1572-
" WITH (fastupdate = off) {index_tablespace};\n";
1584+
" WITH (fastupdate = off) {index_tablespace}"};
15731585

15741586
return sql;
15751587
}
@@ -1659,8 +1671,8 @@ middle_pgsql_t::get_query_instance()
16591671

16601672
// We use a connection per table to enable the use of COPY
16611673
for (auto &table : m_tables) {
1662-
if (!table.m_prepare_query.empty()) {
1663-
mid->exec_sql(table.m_prepare_query);
1674+
for (auto const &query : table.m_prepare_queries) {
1675+
mid->exec_sql(query);
16641676
}
16651677
}
16661678

‎src/middle-pgsql.hpp‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -94,8 +94,8 @@ struct table_sql
9494
{
9595
std::string name;
9696
std::string create_table;
97-
std::string prepare_query;
98-
std::string create_fw_dep_indexes;
97+
std::vector<std::string> prepare_queries;
98+
std::vector<std::string> create_fw_dep_indexes;
9999
};
100100

101101
struct middle_pgsql_t : public middle_t
@@ -150,8 +150,8 @@ struct middle_pgsql_t : public middle_t
150150
void build_index(std::string const &conninfo) const;
151151

152152
std::string m_create_table;
153-
std::string m_prepare_query;
154-
std::string m_create_fw_dep_indexes;
153+
std::vector<std::string> m_prepare_queries;
154+
std::vector<std::string> m_create_fw_dep_indexes;
155155

156156
void task_set(std::future<std::chrono::microseconds> &&future)
157157
{

‎tests/test-middle.cpp‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1092,6 +1092,9 @@ TEMPLATE_TEST_CASE("middle: change nodes in way", "", options_slim_default,
10921092
check_way_nodes(mid, way21.id(), {&node11, &node12});
10931093

10941094
REQUIRE_FALSE(dependency_manager.has_pending());
1095+
1096+
mid->stop();
1097+
mid->wait();
10951098
}
10961099

10971100
// From now on use append mode to not destroy the data we just added.
@@ -1235,6 +1238,9 @@ TEMPLATE_TEST_CASE("middle: change nodes in relation", "", options_slim_default,
12351238
mid->relation(rel30);
12361239
mid->relation(rel31);
12371240
mid->after_relations();
1241+
1242+
mid->stop();
1243+
mid->wait();
12381244
}
12391245

12401246
// From now on use append mode to not destroy the data we just added.

0 commit comments

Comments
 (0)