Skip to content

Commit 5a90fb5

Browse files
authored
Merge pull request #1250 from joto/warning-order
Check order of input files and warn if they are out of order
2 parents bf337cf + 39a3a34 commit 5a90fb5

File tree

8 files changed

+174
-51
lines changed

8 files changed

+174
-51
lines changed

src/CMakeLists.txt

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11

22
set(osm2pgsql_lib_SOURCES
3+
check-order.cpp
34
db-copy.cpp
45
dependency-manager.cpp
56
expire-tiles.cpp

src/check-order.cpp

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
#include "check-order.hpp"
2+
#include "format.hpp"
3+
4+
#include <osmium/osm/relation.hpp>
5+
#include <osmium/osm/types.hpp>
6+
#include <osmium/osm/way.hpp>
7+
8+
void check_order_t::warning(char const *msg, osmid_t id)
9+
{
10+
fmt::print(stderr,
11+
"\nWARNING: {}: {}\n"
12+
" Unordered input files do not work correctly in all\n"
13+
" cases. Future versions of osm2pgsql will require\n"
14+
" ordered files. Use the 'sort' command of osmium tool\n"
15+
" to sort them first.\n\n",
16+
msg, id);
17+
m_issued_warning = true;
18+
}
19+
20+
void check_order_t::node(const osmium::Node &node)
21+
{
22+
if (m_issued_warning) {
23+
return;
24+
}
25+
26+
if (m_max_way_id > std::numeric_limits<osmid_t>::min()) {
27+
warning("Found a node after a way", node.id());
28+
}
29+
if (m_max_relation_id > std::numeric_limits<osmid_t>::min()) {
30+
warning("Found a node after a relation", node.id());
31+
}
32+
33+
if (m_max_node_id == node.id()) {
34+
warning("Node ID twice in input. Maybe you are using a history or "
35+
"non-simplified change file?",
36+
node.id());
37+
}
38+
if (node.id() < m_max_node_id) {
39+
warning("Node IDs out of order", node.id());
40+
}
41+
m_max_node_id = node.id();
42+
}
43+
44+
void check_order_t::way(const osmium::Way &way)
45+
{
46+
if (m_issued_warning) {
47+
return;
48+
}
49+
50+
if (m_max_relation_id > std::numeric_limits<osmid_t>::min()) {
51+
warning("Found a way after a relation", way.id());
52+
}
53+
54+
if (m_max_way_id == way.id()) {
55+
warning("Way ID twice in input. Maybe you are using a history or "
56+
"non-simplified change file?",
57+
way.id());
58+
}
59+
if (way.id() < m_max_way_id) {
60+
warning("Way IDs out of order", way.id());
61+
}
62+
m_max_way_id = way.id();
63+
}
64+
65+
void check_order_t::relation(const osmium::Relation &relation)
66+
{
67+
if (m_issued_warning) {
68+
return;
69+
}
70+
71+
if (m_max_relation_id == relation.id()) {
72+
warning("Relation ID twice in input. Maybe you are using a history or "
73+
"non-simplified change file?",
74+
relation.id());
75+
}
76+
if (relation.id() < m_max_relation_id) {
77+
warning("Relation IDs out of order", relation.id());
78+
}
79+
m_max_relation_id = relation.id();
80+
}

src/check-order.hpp

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
#ifndef OSM2PGSQL_CHECK_ORDER_HPP
2+
#define OSM2PGSQL_CHECK_ORDER_HPP
3+
4+
#include "osmtypes.hpp"
5+
6+
#include <osmium/fwd.hpp>
7+
#include <osmium/handler.hpp>
8+
9+
#include <limits>
10+
11+
/**
12+
* Handler that can be used to check that an OSM file is ordered
13+
* correctly. Ordered in this case refers to the usual order in OSM
14+
* files: First nodes in the order of their IDs, then ways in the order
15+
* of their IDs, then relations in the order or their IDs.
16+
*
17+
* IDs have to be unique for each type. This check will fail for
18+
* history files.
19+
*/
20+
class check_order_t : public osmium::handler::Handler
21+
{
22+
23+
public:
24+
void node(const osmium::Node &node);
25+
26+
void way(const osmium::Way &way);
27+
28+
void relation(const osmium::Relation &relation);
29+
30+
private:
31+
void warning(char const *msg, osmid_t id);
32+
33+
osmid_t m_max_node_id = std::numeric_limits<osmid_t>::min();
34+
osmid_t m_max_way_id = std::numeric_limits<osmid_t>::min();
35+
osmid_t m_max_relation_id = std::numeric_limits<osmid_t>::min();
36+
37+
bool m_issued_warning = false;
38+
}; // class check_order_t
39+
40+
#endif // OSM2PGSQL_CHECK_ORDER_HPP

src/osmdata.cpp

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
#include <osmium/io/any_input.hpp>
1212
#include <osmium/visitor.hpp>
1313

14+
#include "check-order.hpp"
1415
#include "db-copy.hpp"
1516
#include "format.hpp"
1617
#include "input-handler.hpp"
@@ -375,9 +376,10 @@ progress_display_t osmdata_t::process_file(osmium::io::File const &file,
375376
fmt::print(stderr, "Using {} parser.\n",
376377
osmium::io::as_string(file.format()));
377378

379+
check_order_t check_order{};
378380
input_handler_t handler{bbox, m_append, this};
379381
osmium::io::Reader reader{file};
380-
osmium::apply(reader, handler);
382+
osmium::apply(reader, check_order, handler);
381383
reader.close();
382384

383385
return handler.progress();

tests/data/hstore-match-only.osm

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -140,15 +140,6 @@ gis=> select count(*) from planet_osm_line;
140140
<node id='1143' visible='true' version='1' lat='48.95200639595' lon='8.00379105913' />
141141
<node id='1145' visible='true' version='1' lat='49.03381156891' lon='7.95894426887' />
142142
<node id='1147' visible='true' version='1' lat='49.0861198919' lon='7.97508911336' />
143-
<way id='107' visible='true' version='1'>
144-
<nd ref='46' />
145-
<nd ref='32' />
146-
<nd ref='34' />
147-
<nd ref='44' />
148-
<nd ref='36' />
149-
<tag k='admin_level' v='3' />
150-
<tag k='boundary' v='administrative' />
151-
</way>
152143
<way id='98' visible='true' version='1'>
153144
<nd ref='14' />
154145
<nd ref='12' />
@@ -185,6 +176,15 @@ gis=> select count(*) from planet_osm_line;
185176
<tag k='addr:housenumber' v='23' />
186177
<tag k='building' v='yes' />
187178
</way>
179+
<way id='107' visible='true' version='1'>
180+
<nd ref='46' />
181+
<nd ref='32' />
182+
<nd ref='34' />
183+
<nd ref='44' />
184+
<nd ref='36' />
185+
<tag k='admin_level' v='3' />
186+
<tag k='boundary' v='administrative' />
187+
</way>
188188
<way id='108' visible='true' version='1'>
189189
<nd ref='88' />
190190
<nd ref='86' />

tests/data/test_multipolygon_diff.osc

Lines changed: 14 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -5,14 +5,14 @@
55
<node id="88" version="2" timestamp="2013-08-07T15:58:07Z" uid="5" user="Test123" changeset="1" lat="-0.0034521" lon="-0.0025154"/>
66
</modify>
77
<create>
8-
<node id="110739" version="1" timestamp="2013-08-30T00:11:07Z" uid="5" user="Test123" changeset="5" lat="-0.0117967" lon="0.0073064"/>
9-
<node id="110737" version="1" timestamp="2013-08-30T00:11:07Z" uid="5" user="Test123" changeset="5" lat="-0.011878" lon="0.0082495"/>
10-
<node id="110735" version="1" timestamp="2013-08-30T00:11:07Z" uid="5" user="Test123" changeset="5" lat="-0.0113739" lon="0.0083145"/>
11-
<node id="110734" version="1" timestamp="2013-08-30T00:11:07Z" uid="5" user="Test123" changeset="5" lat="-0.0111951" lon="0.0072739"/>
12-
<node id="110730" version="1" timestamp="2013-08-30T00:11:07Z" uid="5" user="Test123" changeset="5" lat="-0.011943" lon="0.0068349"/>
13-
<node id="110728" version="1" timestamp="2013-08-30T00:11:07Z" uid="5" user="Test123" changeset="5" lat="-0.0124308" lon="0.0088023"/>
14-
<node id="110726" version="1" timestamp="2013-08-30T00:11:07Z" uid="5" user="Test123" changeset="5" lat="-0.0110162" lon="0.0088673"/>
158
<node id="110725" version="1" timestamp="2013-08-30T00:11:07Z" uid="5" user="Test123" changeset="5" lat="-0.0108536" lon="0.0067049"/>
9+
<node id="110726" version="1" timestamp="2013-08-30T00:11:07Z" uid="5" user="Test123" changeset="5" lat="-0.0110162" lon="0.0088673"/>
10+
<node id="110728" version="1" timestamp="2013-08-30T00:11:07Z" uid="5" user="Test123" changeset="5" lat="-0.0124308" lon="0.0088023"/>
11+
<node id="110730" version="1" timestamp="2013-08-30T00:11:07Z" uid="5" user="Test123" changeset="5" lat="-0.011943" lon="0.0068349"/>
12+
<node id="110734" version="1" timestamp="2013-08-30T00:11:07Z" uid="5" user="Test123" changeset="5" lat="-0.0111951" lon="0.0072739"/>
13+
<node id="110735" version="1" timestamp="2013-08-30T00:11:07Z" uid="5" user="Test123" changeset="5" lat="-0.0113739" lon="0.0083145"/>
14+
<node id="110737" version="1" timestamp="2013-08-30T00:11:07Z" uid="5" user="Test123" changeset="5" lat="-0.011878" lon="0.0082495"/>
15+
<node id="110739" version="1" timestamp="2013-08-30T00:11:07Z" uid="5" user="Test123" changeset="5" lat="-0.0117967" lon="0.0073064"/>
1616
</create>
1717
<modify>
1818
<way id="15" version="1" timestamp="2013-08-07T15:58:09Z" uid="5" user="Test123" changeset="1">
@@ -102,20 +102,20 @@
102102
<nd ref="182"/>
103103
<tag k="landuse" v="residential"/>
104104
</way>
105-
<way id="110736" version="1" timestamp="2013-08-07T15:58:08Z" uid="5" user="Test123" changeset="1">
106-
<nd ref="110734"/>
107-
<nd ref="110735"/>
108-
<nd ref="110737"/>
109-
<nd ref="110739"/>
110-
<nd ref="110734"/>
111-
</way>
112105
<way id="110727" version="1" timestamp="2013-08-07T15:58:08Z" uid="5" user="Test123" changeset="1">
113106
<nd ref="110725"/>
114107
<nd ref="110726"/>
115108
<nd ref="110728"/>
116109
<nd ref="110730"/>
117110
<nd ref="110725"/>
118111
</way>
112+
<way id="110736" version="1" timestamp="2013-08-07T15:58:08Z" uid="5" user="Test123" changeset="1">
113+
<nd ref="110734"/>
114+
<nd ref="110735"/>
115+
<nd ref="110737"/>
116+
<nd ref="110739"/>
117+
<nd ref="110734"/>
118+
</way>
119119
</create>
120120
<modify>
121121
<relation id="14" version="2" timestamp="2013-08-07T15:58:12Z" uid="5" user="Test123" changeset="1">

tests/data/test_multipolygon_postdiff.osm

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -768,17 +768,17 @@
768768
<nd ref='230' />
769769
<nd ref='228' />
770770
</way>
771+
<way id='101420' timestamp='2013-08-07T15:58:08Z' uid='5' user='Test123' visible='true' version='1' changeset='1'>
772+
<nd ref='121' />
773+
<nd ref='179' />
774+
<nd ref='181' />
775+
</way>
771776
<way id='101426' timestamp='2013-08-07T15:58:08Z' uid='5' user='Test123' visible='true' version='1' changeset='2'>
772777
<nd ref='162' />
773778
<nd ref='178' />
774779
<nd ref='182' />
775780
<tag k='landuse' v='residential' />
776781
</way>
777-
<way id='101420' timestamp='2013-08-07T15:58:08Z' uid='5' user='Test123' visible='true' version='1' changeset='1'>
778-
<nd ref='121' />
779-
<nd ref='179' />
780-
<nd ref='181' />
781-
</way>
782782
<relation id='1' timestamp='2013-08-07T15:58:11Z' uid='5' user='Test123' visible='true' version='1' changeset='1'>
783783
<member type='way' ref='71' role='outer' />
784784
<member type='way' ref='74' role='inner' />

tests/data/test_output_pgsql_validgeom.osm

Lines changed: 22 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -150,6 +150,28 @@
150150
<nd ref="3357025791"/>
151151
<tag k="leisure" v="garden"/>
152152
</way>
153+
<way id="328880344" visible="true">
154+
<nd ref="3357025912"/>
155+
<nd ref="3357025911"/>
156+
<nd ref="3357025929"/>
157+
<nd ref="3357025928"/>
158+
<nd ref="3357025927"/>
159+
<nd ref="3357025926"/>
160+
<nd ref="3357025925"/>
161+
<nd ref="3357025924"/>
162+
<nd ref="3357025923"/>
163+
<nd ref="3357025915"/>
164+
<nd ref="3357025914"/>
165+
<nd ref="3357025922"/>
166+
<nd ref="3357025921"/>
167+
<nd ref="3357025920"/>
168+
<nd ref="3357025919"/>
169+
<nd ref="3357025918"/>
170+
<nd ref="3357025917"/>
171+
<nd ref="3357025916"/>
172+
<nd ref="3357025913"/>
173+
<nd ref="3357025912"/>
174+
</way>
153175
<way id="328880345" visible="true">
154176
<nd ref="3357025902"/>
155177
<nd ref="3357025792"/>
@@ -190,28 +212,6 @@
190212
<nd ref="3357025909"/>
191213
<tag k="leisure" v="garden"/>
192214
</way>
193-
<way id="328880344" visible="true">
194-
<nd ref="3357025912"/>
195-
<nd ref="3357025911"/>
196-
<nd ref="3357025929"/>
197-
<nd ref="3357025928"/>
198-
<nd ref="3357025927"/>
199-
<nd ref="3357025926"/>
200-
<nd ref="3357025925"/>
201-
<nd ref="3357025924"/>
202-
<nd ref="3357025923"/>
203-
<nd ref="3357025915"/>
204-
<nd ref="3357025914"/>
205-
<nd ref="3357025922"/>
206-
<nd ref="3357025921"/>
207-
<nd ref="3357025920"/>
208-
<nd ref="3357025919"/>
209-
<nd ref="3357025918"/>
210-
<nd ref="3357025917"/>
211-
<nd ref="3357025916"/>
212-
<nd ref="3357025913"/>
213-
<nd ref="3357025912"/>
214-
</way>
215215
<way id="439432117" visible="true" version="1">
216216
<nd ref="4370458249"/>
217217
<nd ref="4370458250"/>

0 commit comments

Comments
 (0)