Merge pull request #2059 from joto/find-changed-ways

Use roundabout way to find changed ways

Author: lonvia

src/middle-pgsql.cpp

@@ -73,8 +73,8 @@ static void load_id_list(pg_conn_t const &db_connection,
                          std::string const &table,
                          osmium::index::IdSetSmall<osmid_t> *ids)
 {
-    auto const res =
-        db_connection.exec(fmt::format("SELECT id FROM {} ORDER BY id", table));
+    auto const res = db_connection.exec(
+        fmt::format("SELECT DISTINCT id FROM {} ORDER BY id", table));
     for (int n = 0; n < res.num_tuples(); ++n) {
         ids->set(osmium::string_to_object_id(res.get_value(n, 0)));
     }
@@ -822,48 +822,69 @@ void middle_pgsql_t::get_node_parents(
 
     send_id_list(m_db_connection, "osm2pgsql_changed_nodes", changed_nodes);
 
-    m_db_connection.exec("ANALYZE osm2pgsql_changed_nodes");
+    std::vector<std::string> queries;
+
+    queries.emplace_back("ANALYZE osm2pgsql_changed_nodes");
 
     bool const has_bucket_index =
         check_bucket_index(&m_db_connection, m_options->prefix);
 
     if (has_bucket_index) {
-        m_db_connection.exec(build_sql(*m_options, R"(
-WITH changed_buckets AS (
-  SELECT array_agg(id) AS node_ids, id >> {way_node_index_id_shift} AS bucket
-    FROM osm2pgsql_changed_nodes GROUP BY id >> {way_node_index_id_shift}
-)
-INSERT INTO osm2pgsql_changed_ways
-  SELECT DISTINCT w.id
-    FROM {schema}"{prefix}_ways" w, changed_buckets b
-    WHERE w.nodes && b.node_ids
-      AND {schema}"{prefix}_index_bucket"(w.nodes)
-       && ARRAY[b.bucket];
-        )"));
+        // The query to get the parent ways of changed nodes is "hidden"
+        // inside a PL/pgSQL function so that the query planner only sees
+        // a single node id that is being queried for. If we ask for all
+        // nodes at the same time the query planner sometimes thinks it is
+        // better to do a full table scan which totally destroys performance.
+        // This is due to the PostgreSQL statistics on ARRAYs being way off.
+        queries.emplace_back(R"(
+CREATE OR REPLACE FUNCTION osm2pgsql_find_changed_ways() RETURNS void AS $$
+DECLARE
+  changed_buckets RECORD;
+BEGIN
+  FOR changed_buckets IN
+    SELECT array_agg(id) AS node_ids, id >> {way_node_index_id_shift} AS bucket
+      FROM osm2pgsql_changed_nodes GROUP BY id >> {way_node_index_id_shift}
+  LOOP
+    INSERT INTO osm2pgsql_changed_ways
+    SELECT DISTINCT w.id
+      FROM {schema}"{prefix}_ways" w
+      WHERE w.nodes && changed_buckets.node_ids
+        AND {schema}"{prefix}_index_bucket"(w.nodes)
+         && ARRAY[changed_buckets.bucket];
+  END LOOP;
+END;
+$$ LANGUAGE plpgsql
+)");
+        queries.emplace_back("SELECT osm2pgsql_find_changed_ways()");
+        queries.emplace_back("DROP FUNCTION osm2pgsql_find_changed_ways()");
     } else {
-        m_db_connection.exec(build_sql(*m_options, R"(
+        queries.emplace_back(R"(
 INSERT INTO osm2pgsql_changed_ways
-  SELECT DISTINCT w.id
+  SELECT w.id
     FROM {schema}"{prefix}_ways" w, osm2pgsql_changed_nodes n
     WHERE w.nodes && ARRAY[n.id]
-        )"));
+        )");
     }
 
     if (m_options->middle_database_format == 1) {
-        m_db_connection.exec(build_sql(*m_options, R"(
+        queries.emplace_back(R"(
 INSERT INTO osm2pgsql_changed_relations
-  SELECT DISTINCT r.id
+  SELECT r.id
     FROM {schema}"{prefix}_rels" r, osm2pgsql_changed_nodes n
     WHERE r.parts && ARRAY[n.id]
       AND r.parts[1:way_off] && ARRAY[n.id]
-        )"));
+        )");
     } else {
-        m_db_connection.exec(build_sql(*m_options, R"(
+        queries.emplace_back(R"(
 INSERT INTO osm2pgsql_changed_relations
-  SELECT DISTINCT r.id
+  SELECT r.id
     FROM {schema}"{prefix}_rels" r, osm2pgsql_changed_nodes c
     WHERE {schema}"{prefix}_member_ids"(r.members, 'N'::char) && ARRAY[c.id];
-        )"));
+        )");
+    }
+
+    for (auto const &query : queries) {
+        m_db_connection.exec(build_sql(*m_options, query));
     }
 
     load_id_list(m_db_connection, "osm2pgsql_changed_ways", parent_ways);