Merge pull request #1608 from joto/fix-large-relations

Again: ignore relations with more than 32k members

Author: lonvia

src/osmdata.cpp

@@ -88,6 +88,13 @@ void osmdata_t::after_ways() { m_mid->after_ways(); }
 
 void osmdata_t::relation(osmium::Relation const &rel)
 {
+    if (rel.members().size() > 32767) {
+        log_warn(
+            "Relation id {} ignored, because it has more than 32767 members",
+            rel.id());
+        return;
+    }
+
     if (m_append && !rel.deleted()) {
         m_output->select_relation_members(rel.id());
     }
@@ -97,9 +104,6 @@ void osmdata_t::relation(osmium::Relation const &rel)
     if (rel.deleted()) {
         relation_delete(rel.id());
     } else {
-        if (rel.members().size() > 32767) {
-            return;
-        }
         if (m_append) {
             relation_modify(rel);
         } else {

tests/regression.py

@@ -18,6 +18,7 @@
 
 import logging
 from os import path as op
+from io import StringIO
 import os
 import subprocess
 import sys
@@ -193,6 +194,27 @@ def run_import(cls, params, filename):
             raise RuntimeError("Import failed.")
 
 
+    @classmethod
+    def import_from_string(cls, content, format='opl'):
+        cmdline = [CONFIG['executable']]
+        params = cls.get_def_params() + cls.extra_params
+        if not '-d' in params and not '--database' in params:
+            cmdline.extend(('-d', CONFIG['test_database']))
+        cmdline.extend(params)
+        cmdline.extend(('-r', format, '-'))
+        logging.info("Executing command: {}".format(' '.join(cmdline)))
+
+        proc = subprocess.Popen(cmdline, stdin=subprocess.PIPE,
+                                stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+
+        (outp, err) = proc.communicate(input=content.encode('utf-8'))
+
+        if proc.returncode == 0:
+            logging.debug(err.decode('utf-8'))
+        else:
+            logging.warning(err.decode('utf-8'))
+            raise RuntimeError("Import failed.")
+
     def assert_count(self, count, table, where=None):
         if self.schema:
             table = self.schema + '.' + table
@@ -873,3 +895,21 @@ class TestDBOutputSchema(BaseUpdateRunnerWithOutputSchema, unittest.TestCase,
                          PgsqlBaseTests):
     extra_params = ['--slim', '--output-pgsql-schema=osm']
 
+# Bad data tests
+
+class TestBadOSMData(BaseRunner, unittest.TestCase):
+
+    extra_params = ['--slim']
+
+    def test_large_relation(self):
+        content = StringIO()
+        content.write('n1 x45 y34\n')
+        content.write('r1 Ttype=multipolygon M')
+        for _ in range(33000):
+            content.write('n1@,')
+        content.write('n1@')
+        self.import_from_string(content.getvalue())
+
+        self.assert_count(1, 'planet_osm_nodes')
+        self.assert_count(0, 'planet_osm_rels')
+