Skip to content

Commit dbac88f

Browse files
lonvialonvia
committed
add tests for osm2pgsql-replication
Needs mocking of replication functions and url accesses.
1 parent 9985d70 commit dbac88f

File tree

6 files changed

+500
-11
lines changed

6 files changed

+500
-11
lines changed

‎scripts/osm2pgsql-replication‎

Lines changed: 11 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,7 @@ LOG = logging.getLogger()
5454
OSM2PGSQL_PATH = Path(__file__).parent.resolve() / 'osm2pgsql'
5555

5656
def pretty_format_timedelta(seconds):
57+
seconds = int(seconds)
5758
(minutes, seconds) = divmod(seconds, 60)
5859
(hours, minutes) = divmod(minutes, 60)
5960
(days, hours) = divmod(hours, 24)
@@ -62,15 +63,15 @@ def pretty_format_timedelta(seconds):
6263
output = []
6364
# If weeks > 1 but hours == 0, we still want to show "0 hours"
6465
if weeks > 0:
65-
output.append("{:d} week(s)".format(weeks))
66+
output.append("{} week(s)".format(weeks))
6667
if days > 0 or weeks > 0:
67-
output.append("{:d} day(s)".format(days))
68+
output.append("{} day(s)".format(days))
6869
if hours > 0 or days > 0 or weeks > 0:
69-
output.append("{:d} hour(s)".format(hours))
70+
output.append("{} hour(s)".format(hours))
7071
if minutes > 0 or hours > 0 or days > 0 or weeks > 0:
71-
output.append("{:d} minute(s)".format(minutes))
72+
output.append("{} minute(s)".format(minutes))
7273

73-
output.append("{:d} second(s)".format(seconds))
74+
output.append("{} second(s)".format(seconds))
7475

7576
output = " ".join(output)
7677
return output
@@ -201,7 +202,7 @@ class Osm2pgsqlProperties:
201202
date = self._get_prop('replication_timestamp')
202203

203204
if base_url is None or seq is None or date is None:
204-
raise DBError(2, "Updates not set up correctly. Run 'osm2pgsql-updates init' first.")
205+
raise DBError(2, "Updates not set up correctly. Run 'osm2pgsql-replication init' first.")
205206

206207
return base_url, int(seq), from_osm_date(date)
207208

@@ -243,7 +244,7 @@ class LegacyProperties:
243244
LOG.debug("Using way id %d for timestamp lookup", osmid)
244245
# Get the way from the API to find the timestamp when it was created.
245246
url = 'https://www.openstreetmap.org/api/0.6/way/{}/1'.format(osmid)
246-
headers = {"User-Agent" : "osm2pgsql-update",
247+
headers = {"User-Agent" : "osm2pgsql-replication",
247248
"Accept" : "application/json"}
248249
with urlrequest.urlopen(urlrequest.Request(url, headers=headers)) as response:
249250
data = json.loads(response.read().decode('utf-8'))
@@ -272,12 +273,12 @@ class LegacyProperties:
272273
with self.db.conn.cursor() as cur:
273274
cur.execute(sql.SQL('SELECT * FROM {}').format(self.db.table_id(self.prop_table)))
274275
if cur.rowcount != 1:
275-
raise DBError(2, "Updates not set up correctly. Run 'osm2pgsql-updates init' first.")
276+
raise DBError(2, "Updates not set up correctly. Run 'osm2pgsql-replication init' first.")
276277

277278
base_url, seq, date = cur.fetchone()
278279

279280
if base_url is None or seq is None or date is None:
280-
raise DBError(2, "Updates not set up correctly. Run 'osm2pgsql-updates init' first.")
281+
raise DBError(2, "Updates not set up correctly. Run 'osm2pgsql-replication init' first.")
281282

282283
return base_url, seq, date
283284

@@ -491,7 +492,7 @@ def update(props, args):
491492
remote_server_age_sec = int((dt.datetime.now(dt.timezone.utc) - current.timestamp).total_seconds())
492493
LOG.debug("Applying %d sequence(s) (%d → %d), covering %s (%s sec) of changes (%s → %s)",
493494
current.sequence - seq, current.sequence, seq,
494-
pretty_format_timedelta(int((current.timestamp - ts).total_seconds())),
495+
pretty_format_timedelta((current.timestamp - ts).total_seconds()),
495496
int((current.timestamp - ts).total_seconds()),
496497
osm_date(ts.astimezone(dt.timezone.utc)),
497498
osm_date(current.timestamp.astimezone(dt.timezone.utc))

‎tests/bdd/command-line/replication.feature‎

Lines changed: 225 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,11 @@
11
Feature: Tests for the osm2pgsql-replication script with property table
22

3-
Scenario: Replication can be initialised with a osm file after import
3+
Scenario: Replication can be initialised with an osm file after import
44
Given the OSM data
55
"""
66
n34 Tamenity=restaurant x77 y45.3
77
"""
8+
And the replication service at http://example.com/europe/liechtenstein-updates
89
When running osm2pgsql pgsql with parameters
910
| --slim |
1011

@@ -20,6 +21,7 @@ Feature: Tests for the osm2pgsql-replication script with property table
2021

2122
Scenario: Replication will be initialised from the information of the import file
2223
Given the input file 'liechtenstein-2013-08-03.osm.pbf'
24+
And the replication service at http://example.com/europe/liechtenstein-updates
2325
When running osm2pgsql pgsql with parameters
2426
| --slim |
2527

@@ -62,6 +64,7 @@ Feature: Tests for the osm2pgsql-replication script with property table
6264
Scenario: Replication can be initialised for a database in a different schema
6365
Given the database schema foobar
6466
Given the input file 'liechtenstein-2013-08-03.osm.pbf'
67+
And the replication service at http://example.com/europe/liechtenstein-updates
6568
When running osm2pgsql pgsql with parameters
6669
| --slim | --middle-schema=foobar |
6770

@@ -78,6 +81,7 @@ Feature: Tests for the osm2pgsql-replication script with property table
7881
Scenario: Replication initialiasion will fail for a database in a different schema
7982
Given the database schema foobar
8083
Given the input file 'liechtenstein-2013-08-03.osm.pbf'
84+
And the replication service at http://example.com/europe/liechtenstein-updates
8185
When running osm2pgsql pgsql with parameters
8286
| --slim |
8387

@@ -88,3 +92,223 @@ Feature: Tests for the osm2pgsql-replication script with property table
8892
Database needs to be imported in --slim mode.
8993
"""
9094

95+
Scenario: Replication can be initialised with a fixed date (no previous replication info)
96+
Given the OSM data
97+
"""
98+
n34 Tamenity=restaurant x77 y45.3
99+
"""
100+
And the replication service at https://planet.openstreetmap.org/replication/minute
101+
| sequence | timestamp |
102+
| 345 | 2020-10-04T01:00:00Z |
103+
| 346 | 2020-10-04T02:00:00Z |
104+
| 347 | 2020-10-04T03:00:00Z |
105+
When running osm2pgsql pgsql with parameters
106+
| --slim |
107+
108+
And running osm2pgsql-replication
109+
| init | --start-at | 2020-10-04T01:30:00Z |
110+
111+
Then table osm2pgsql_properties contains
112+
| property | value |
113+
| replication_base_url | https://planet.openstreetmap.org/replication/minute |
114+
| replication_sequence_number | 345 |
115+
| replication_timestamp | 2020-10-04T01:30:00Z |
116+
117+
118+
Scenario: Replication can be initialised with a fixed date (with previous replication info)
119+
Given the input file 'liechtenstein-2013-08-03.osm.pbf'
120+
And the replication service at http://example.com/europe/liechtenstein-updates
121+
| sequence | timestamp |
122+
| 345 | 2020-10-04T01:00:00Z |
123+
| 346 | 2020-10-04T02:00:00Z |
124+
| 347 | 2020-10-04T03:00:00Z |
125+
When running osm2pgsql pgsql with parameters
126+
| --slim |
127+
128+
And running osm2pgsql-replication
129+
| init | --start-at | 2020-10-04T03:30:00Z |
130+
131+
Then table osm2pgsql_properties contains
132+
| property | value |
133+
| replication_base_url | http://example.com/europe/liechtenstein-updates |
134+
| replication_sequence_number | 347 |
135+
| replication_timestamp | 2020-10-04T03:30:00Z |
136+
137+
138+
Scenario: Replication can be initialised with a rollback (no previous replication info)
139+
Given the OSM data
140+
"""
141+
n34 Tamenity=restaurant x77 y45.3 t2020-10-04T02:00:01Z
142+
"""
143+
And the replication service at https://planet.openstreetmap.org/replication/minute
144+
| sequence | timestamp |
145+
| 345 | 2020-10-04T01:00:00Z |
146+
| 346 | 2020-10-04T02:00:00Z |
147+
| 347 | 2020-10-04T03:00:00Z |
148+
When running osm2pgsql pgsql with parameters
149+
| --slim |
150+
151+
And running osm2pgsql-replication
152+
| init | --start-at | 60 |
153+
154+
Then table osm2pgsql_properties contains
155+
| property | value |
156+
| replication_base_url | https://planet.openstreetmap.org/replication/minute |
157+
| replication_sequence_number | 345 |
158+
| replication_timestamp | 2020-10-04T01:00:01Z |
159+
160+
161+
Scenario: Replication can be initialised with a rollback (with previous replication info)
162+
Given the input file 'liechtenstein-2013-08-03.osm.pbf'
163+
And the replication service at http://example.com/europe/liechtenstein-updates
164+
| sequence | timestamp |
165+
| 345 | 2020-10-04T01:00:00Z |
166+
| 346 | 2020-10-04T02:00:00Z |
167+
| 347 | 2020-10-04T03:00:00Z |
168+
When running osm2pgsql pgsql with parameters
169+
| --slim |
170+
171+
And running osm2pgsql-replication
172+
| init | --start-at | 60 |
173+
174+
Then table osm2pgsql_properties contains
175+
| property | value |
176+
| replication_base_url | http://example.com/europe/liechtenstein-updates |
177+
| replication_sequence_number | 345 |
178+
| replication_timestamp | 2013-08-03T14:55:30Z |
179+
180+
181+
Scenario: Replication can be initialised from a different server
182+
Given the input file 'liechtenstein-2013-08-03.osm.pbf'
183+
And the replication service at https://custom.replication
184+
| sequence | timestamp |
185+
| 1345 | 2013-07-01T01:00:00Z |
186+
| 1346 | 2013-08-01T01:00:00Z |
187+
| 1347 | 2013-09-01T01:00:00Z |
188+
When running osm2pgsql pgsql with parameters
189+
| --slim |
190+
191+
And running osm2pgsql-replication
192+
| init | --server | https://custom.replication |
193+
194+
Then table osm2pgsql_properties contains
195+
| property | value |
196+
| replication_base_url | https://custom.replication |
197+
| replication_sequence_number | 1346 |
198+
| replication_timestamp | 2013-08-03T15:55:30Z |
199+
200+
201+
Scenario: Updates need an initialised replication
202+
Given the OSM data
203+
"""
204+
n34 Tamenity=restaurant x77 y45.3
205+
"""
206+
And the replication service at https://planet.openstreetmap.org/replication/minute
207+
When running osm2pgsql pgsql with parameters
208+
| --slim |
209+
210+
Then running osm2pgsql-replication fails with returncode 1
211+
| update |
212+
And the error output contains
213+
"""
214+
Updates not set up correctly.
215+
"""
216+
217+
Scenario: Updates run until the end (exactly one application)
218+
Given the input file 'liechtenstein-2013-08-03.osm.pbf'
219+
And the replication service at http://example.com/europe/liechtenstein-updates
220+
| sequence | timestamp |
221+
| 9999999 | 2013-08-01T01:00:02Z |
222+
| 10000000 | 2013-09-01T01:00:00Z |
223+
| 10000001 | 2013-10-01T01:00:00Z |
224+
When running osm2pgsql pgsql with parameters
225+
| --slim |
226+
And running osm2pgsql-replication
227+
| init |
228+
And running osm2pgsql-replication
229+
| update |
230+
231+
Then table osm2pgsql_properties contains
232+
| property | value |
233+
| replication_base_url | http://example.com/europe/liechtenstein-updates |
234+
| replication_sequence_number | 10000001 |
235+
| replication_timestamp | 2013-10-01T01:00:00Z |
236+
237+
238+
Scenario: Updates run until the end (multiple applications)
239+
Given the input file 'liechtenstein-2013-08-03.osm.pbf'
240+
And the replication service at http://example.com/europe/liechtenstein-updates
241+
| sequence | timestamp |
242+
| 9999999 | 2013-08-01T01:00:02Z |
243+
| 10000000 | 2013-09-01T01:00:00Z |
244+
| 10000001 | 2013-10-01T01:00:00Z |
245+
When running osm2pgsql pgsql with parameters
246+
| --slim |
247+
And running osm2pgsql-replication
248+
| init |
249+
And running osm2pgsql-replication
250+
| update | --max-diff-size | 1 |
251+
252+
Then table osm2pgsql_properties contains
253+
| property | value |
254+
| replication_base_url | http://example.com/europe/liechtenstein-updates |
255+
| replication_sequence_number | 10000001 |
256+
| replication_timestamp | 2013-10-01T01:00:00Z |
257+
258+
259+
Scenario: Updates can run only once
260+
Given the input file 'liechtenstein-2013-08-03.osm.pbf'
261+
And the replication service at http://example.com/europe/liechtenstein-updates
262+
| sequence | timestamp |
263+
| 9999999 | 2013-08-01T01:00:02Z |
264+
| 10000000 | 2013-09-01T01:00:00Z |
265+
| 10000001 | 2013-10-01T01:00:00Z |
266+
When running osm2pgsql pgsql with parameters
267+
| --slim |
268+
And running osm2pgsql-replication
269+
| init |
270+
And running osm2pgsql-replication
271+
| update | --once | --max-diff-size | 1 |
272+
273+
Then table osm2pgsql_properties contains
274+
| property | value |
275+
| replication_base_url | http://example.com/europe/liechtenstein-updates |
276+
| replication_sequence_number | 10000000 |
277+
| replication_timestamp | 2013-09-01T01:00:00Z |
278+
279+
280+
Scenario: Status of an uninitialised database fails
281+
Given the OSM data
282+
"""
283+
n34 Tamenity=restaurant x77 y45.3
284+
"""
285+
And the replication service at https://planet.openstreetmap.org/replication/minute
286+
When running osm2pgsql pgsql with parameters
287+
| --slim |
288+
289+
Then running osm2pgsql-replication fails with returncode 2
290+
| status | --json |
291+
And the standard output contains
292+
"""
293+
"status": 2
294+
"error": "Updates not set up correctly.
295+
"""
296+
297+
Scenario: Status of a freshly initialised database
298+
Given the input file 'liechtenstein-2013-08-03.osm.pbf'
299+
And the replication service at http://example.com/europe/liechtenstein-updates
300+
| sequence | timestamp |
301+
| 9999999 | 2013-08-01T01:00:02Z |
302+
| 10000000 | 2013-09-01T01:00:00Z |
303+
| 10000001 | 2013-10-01T01:00:00Z |
304+
When running osm2pgsql pgsql with parameters
305+
| --slim |
306+
307+
And running osm2pgsql-replication
308+
| status | --json |
309+
Then the standard output contains
310+
"""
311+
"status": 0
312+
"server": {"base_url": "http://example.com/europe/liechtenstein-updates", "sequence": 10000001, "timestamp": "2013-10-01T01:00:00Z"
313+
"local": {"sequence": 9999999, "timestamp": "2013-08-03T19:00:02Z"
314+
"""

0 commit comments

Comments
 (0)