Skip to content

Commit 9d1e4f3

Browse files
lonvialonvia
authored
Merge pull request #956 from gravitystorm/int4
Avoid crashing on out-of-range inputs for int4 columns
2 parents 545c7d3 + 5b4b391 commit 9d1e4f3

File tree

5 files changed

+209
-4
lines changed

5 files changed

+209
-4
lines changed

‎table.cpp‎

Lines changed: 13 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -457,12 +457,21 @@ void table_t::escape_type(const string &value, ColumnType flags)
457457
switch (flags) {
458458
case COLUMN_TYPE_INT: {
459459
// For integers we take the first number, or the average if it's a-b
460-
long from, to;
461-
int items = sscanf(value.c_str(), "%ld-%ld", &from, &to);
462-
if (items == 1) {
460+
long long from, to;
461+
// limit number of digits parsed to avoid undefined behaviour in sscanf
462+
int items = sscanf(value.c_str(), "%18lld-%18lld", &from, &to);
463+
if (items == 1 && from <= std::numeric_limits<int32_t>::max() &&
464+
from >= std::numeric_limits<int32_t>::min()) {
463465
m_copy.add_column(from);
464466
} else if (items == 2) {
465-
m_copy.add_column((from + to) / 2);
467+
// calculate mean while avoiding overflows
468+
int64_t mean = (from / 2) + (to / 2) + ((from % 2 + to % 2) / 2);
469+
if (mean <= std::numeric_limits<int32_t>::max() &&
470+
mean >= std::numeric_limits<int32_t>::min()) {
471+
m_copy.add_column(mean);
472+
} else {
473+
m_copy.add_null_column();
474+
}
466475
} else {
467476
m_copy.add_null_column();
468477
}

‎tests/CMakeLists.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ set(TESTS
2121
test-output-multi-tags.cpp
2222
test-output-pgsql-area.cpp
2323
test-output-pgsql-schema.cpp
24+
test-output-pgsql-int4.cpp
2425
test-output-pgsql-tablespace.cpp
2526
test-output-pgsql-validgeom.cpp
2627
test-output-pgsql-z_order.cpp

‎tests/test-output-pgsql-int4.cpp‎

Lines changed: 117 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,117 @@
1+
#include <iostream>
2+
#include <stdlib.h>
3+
#include <stdio.h>
4+
#include <string.h>
5+
#include <cassert>
6+
#include <sstream>
7+
#include <stdexcept>
8+
#include <memory>
9+
10+
#include "osmtypes.hpp"
11+
#include "osmdata.hpp"
12+
#include "output-pgsql.hpp"
13+
#include "options.hpp"
14+
#include "middle-pgsql.hpp"
15+
#include "middle-ram.hpp"
16+
#include "taginfo_impl.hpp"
17+
18+
#include <sys/types.h>
19+
#include <unistd.h>
20+
21+
#include <boost/lexical_cast.hpp>
22+
23+
#include "tests/middle-tests.hpp"
24+
#include "tests/common-pg.hpp"
25+
#include "tests/common.hpp"
26+
27+
namespace {
28+
29+
struct skip_test : public std::exception {
30+
const char *what() const noexcept { return "Test skipped."; }
31+
};
32+
33+
void run_test(const char* test_name, void (*testfunc)()) {
34+
try {
35+
fprintf(stderr, "%s\n", test_name);
36+
testfunc();
37+
38+
} catch (const skip_test &) {
39+
exit(77); // <-- code to skip this test.
40+
41+
} catch (const std::exception& e) {
42+
fprintf(stderr, "%s\n", e.what());
43+
fprintf(stderr, "FAIL\n");
44+
exit(EXIT_FAILURE);
45+
}
46+
47+
fprintf(stderr, "PASS\n");
48+
}
49+
#define RUN_TEST(x) run_test(#x, &(x))
50+
51+
// "simple" test modeled on the basic regression test from
52+
// the python script. this is just to check everything is
53+
// working as expected before we start the complex stuff.
54+
void test_int4() {
55+
std::unique_ptr<pg::tempdb> db;
56+
57+
try {
58+
db.reset(new pg::tempdb);
59+
} catch (const std::exception &e) {
60+
std::cerr << "Unable to setup database: " << e.what() << "\n";
61+
throw skip_test();
62+
}
63+
64+
std::string proc_name("test-output-pgsql-int4"), input_file("-");
65+
char *argv[] = { &proc_name[0], &input_file[0], nullptr };
66+
67+
options_t options = options_t(2, argv);
68+
options.database_options = db->database_options;
69+
options.num_procs = 1;
70+
options.slim = 1;
71+
options.prefix = "osm2pgsql_test";
72+
options.style = "tests/test_output_pgsql_int4.style";
73+
74+
testing::run_osm2pgsql(options, "tests/test_output_pgsql_int4.osm",
75+
"xml");
76+
77+
db->assert_has_table("osm2pgsql_test_point");
78+
db->assert_has_table("osm2pgsql_test_line");
79+
db->assert_has_table("osm2pgsql_test_polygon");
80+
db->assert_has_table("osm2pgsql_test_roads");
81+
82+
// First three nodes have population values that are out of range for int4 columns
83+
db->check_string("", "SELECT population FROM osm2pgsql_test_point WHERE osm_id = 1");
84+
db->check_string("", "SELECT population FROM osm2pgsql_test_point WHERE osm_id = 2");
85+
db->check_string("", "SELECT population FROM osm2pgsql_test_point WHERE osm_id = 3");
86+
// Check values that are valid for int4 columns, including limits
87+
db->check_count(2147483647, "SELECT population FROM osm2pgsql_test_point WHERE osm_id = 4");
88+
db->check_count(10000, "SELECT population FROM osm2pgsql_test_point WHERE osm_id = 5");
89+
db->check_count(-10000, "SELECT population FROM osm2pgsql_test_point WHERE osm_id = 6");
90+
db->check_count(-2147483648, "SELECT population FROM osm2pgsql_test_point WHERE osm_id = 7");
91+
// More out of range negative values
92+
db->check_string("", "SELECT population FROM osm2pgsql_test_point WHERE osm_id = 8");
93+
db->check_string("", "SELECT population FROM osm2pgsql_test_point WHERE osm_id = 9");
94+
db->check_string("", "SELECT population FROM osm2pgsql_test_point WHERE osm_id = 10");
95+
96+
// Ranges are also parsed into int4 columns
97+
db->check_string("", "SELECT population FROM osm2pgsql_test_point WHERE osm_id = 11");
98+
db->check_string("", "SELECT population FROM osm2pgsql_test_point WHERE osm_id = 12");
99+
// Check values that are valid for int4 columns, including limits
100+
db->check_count(2147483647, "SELECT population FROM osm2pgsql_test_point WHERE osm_id =13");
101+
db->check_count(15000, "SELECT population FROM osm2pgsql_test_point WHERE osm_id = 14");
102+
db->check_count(-15000, "SELECT population FROM osm2pgsql_test_point WHERE osm_id = 15");
103+
db->check_count(-2147483648, "SELECT population FROM osm2pgsql_test_point WHERE osm_id = 16");
104+
// More out of range negative values
105+
db->check_string("", "SELECT population FROM osm2pgsql_test_point WHERE osm_id = 17");
106+
db->check_string("", "SELECT population FROM osm2pgsql_test_point WHERE osm_id = 18");
107+
}
108+
109+
} // anonymous namespace
110+
111+
int main(int argc, char *argv[]) {
112+
(void)argc;
113+
(void)argv;
114+
RUN_TEST(test_int4);
115+
116+
return 0;
117+
}

‎tests/test_output_pgsql_int4.osm‎

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<osm version="0.6">
3+
<node id="1" visible="true" version="1" changeset="1" timestamp="2018-10-31T10:20:19Z" user="a" uid="1" lat="51.4779481" lon="-0.0014863">
4+
<tag k="population" v="10000000000000000000" />
5+
<tag k="name" v="longer than long" />
6+
</node>
7+
<node id="2" visible="true" version="1" changeset="1" timestamp="2018-10-31T10:20:19Z" user="a" uid="1" lat="51.4779481" lon="-0.0014863">
8+
<tag k="population" v="10000000000" />
9+
<tag k="name" v="long (ten billion)" />
10+
</node>
11+
<node id="3" visible="true" version="1" changeset="1" timestamp="2018-10-31T10:20:19Z" user="a" uid="1" lat="51.4779481" lon="-0.0014863">
12+
<tag k="population" v="2147483648" />
13+
<tag k="name" v="postgresql one more than int4 type" />
14+
</node>
15+
<node id="4" visible="true" version="1" changeset="1" timestamp="2018-10-31T10:20:19Z" user="a" uid="1" lat="51.4779481" lon="-0.0014863">
16+
<tag k="population" v="2147483647" />
17+
<tag k="name" v="postgresql max int4 type" />
18+
</node>
19+
<node id="5" visible="true" version="1" changeset="1" timestamp="2018-10-31T10:20:19Z" user="a" uid="1" lat="51.4779481" lon="-0.0014863">
20+
<tag k="population" v="10000" />
21+
<tag k="name" v="ten thousand" />
22+
</node>
23+
<node id="6" visible="true" version="1" changeset="1" timestamp="2018-10-31T10:20:19Z" user="a" uid="1" lat="51.4779481" lon="-0.0014863">
24+
<tag k="population" v="-10000" />
25+
<tag k="name" v="minus ten thousand" />
26+
</node>
27+
<node id="7" visible="true" version="1" changeset="1" timestamp="2018-10-31T10:20:19Z" user="a" uid="1" lat="51.4779481" lon="-0.0014863">
28+
<tag k="population" v="-2147483648" />
29+
<tag k="name" v="postgresql min int4 type" />
30+
</node>
31+
<node id="8" visible="true" version="1" changeset="1" timestamp="2018-10-31T10:20:19Z" user="a" uid="1" lat="51.4779481" lon="-0.0014863">
32+
<tag k="population" v="-2147483649" />
33+
<tag k="name" v="postgresql one less than min int4 type" />
34+
</node>
35+
<node id="9" visible="true" version="1" changeset="1" timestamp="2018-10-31T10:20:19Z" user="a" uid="1" lat="51.4779481" lon="-0.0014863">
36+
<tag k="population" v="-10000000000" />
37+
<tag k="name" v="minus long (minus ten billion)" />
38+
</node>
39+
<node id="10" visible="true" version="1" changeset="1" timestamp="2018-10-31T10:20:19Z" user="a" uid="1" lat="51.4779481" lon="-0.0014863">
40+
<tag k="population" v="-10000000000000000000" />
41+
<tag k="name" v="minus longer than long" />
42+
</node>
43+
<node id="11" visible="true" version="1" changeset="1" timestamp="2018-10-31T10:20:19Z" user="a" uid="1" lat="51.4779481" lon="-0.0014863">
44+
<tag k="population" v="10000000000000000000-20000000000000000000" />
45+
<tag k="name" v="range, longer than long" />
46+
</node>
47+
<node id="12" visible="true" version="1" changeset="1" timestamp="2018-10-31T10:20:19Z" user="a" uid="1" lat="51.4779481" lon="-0.0014863">
48+
<tag k="population" v="10000000000-20000000000" />
49+
<tag k="name" v="range, 15 billion" />
50+
</node>
51+
<node id="13" visible="true" version="1" changeset="1" timestamp="2018-10-31T10:20:19Z" user="a" uid="1" lat="51.4779481" lon="-0.0014863">
52+
<tag k="population" v="2147483646-2147483648" />
53+
<tag k="name" v="range, mean is max int4" />
54+
</node>
55+
<node id="14" visible="true" version="1" changeset="1" timestamp="2018-10-31T10:20:19Z" user="a" uid="1" lat="51.4779481" lon="-0.0014863">
56+
<tag k="population" v="10000-20000" />
57+
<tag k="name" v="range, 15 thousand" />
58+
</node>
59+
<node id="15" visible="true" version="1" changeset="1" timestamp="2018-10-31T10:20:19Z" user="a" uid="1" lat="51.4779481" lon="-0.0014863">
60+
<tag k="population" v="-10000--20000" />
61+
<tag k="name" v="range, negative 15 thousand" />
62+
</node>
63+
<node id="16" visible="true" version="1" changeset="1" timestamp="2018-10-31T10:20:19Z" user="a" uid="1" lat="51.4779481" lon="-0.0014863">
64+
<tag k="population" v="-2147483647--2147483649" />
65+
<tag k="name" v="range, mean is min int4" />
66+
</node>
67+
<node id="17" visible="true" version="1" changeset="1" timestamp="2018-10-31T10:20:19Z" user="a" uid="1" lat="51.4779481" lon="-0.0014863">
68+
<tag k="population" v="-10000000000--20000000000" />
69+
<tag k="name" v="range, negative 15 billion" />
70+
</node>
71+
<node id="18" visible="true" version="1" changeset="1" timestamp="2018-10-31T10:20:19Z" user="a" uid="1" lat="51.4779481" lon="-0.0014863">
72+
<tag k="population" v="-10000000000000000000--20000000000000000000" />
73+
<tag k="name" v="range, negative longer than long" />
74+
</node>
75+
</osm>

‎tests/test_output_pgsql_int4.style‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
# OsmType Tag DataType Flags
2+
node,way population int4 linear
3+
node,way name text linear

0 commit comments

Comments
 (0)