Merge pull request #2212 from lonvia/check-date-on-replication-init

Replication: make sure that the computed start state is valid

Author: joto

scripts/osm2pgsql-replication

@@ -449,12 +449,22 @@ def init(props, args):
             raise DBError(1, f"Cannot reach the configured replication service '{base_url}'.\n"
                              "Does the URL point to a directory containing OSM update data?")
 
+    state = repl.get_state_info(seq)
+    if state is None:
+        raise DBError(1, f"Cannot load state information for {seq} from replication service {base_url}.\n"
+                         + (f"The server may not have diffs going as far back as {date}."
+                            if date is not None else
+                            "Does the URL point to a directory containing OSM update data?"))
+
     if date is None:
-        state = repl.get_state_info(seq)
-        if state is None:
-            raise DBError(1, f"Cannot reach the configured replication service '{base_url}'.\n"
-                             "Does the URL point to a directory containing OSM update data?")
         date = from_osm_date(state.timestamp)
+    else:
+        # Sanity check. Is the sequence in line with the date requested?
+        if state.timestamp > date:
+            raise DBError(1, "The replication service does not have diff files for the requested date.\n"
+                             f"Replication service used: {base_url}\n"
+                             f"Most recent diffs available start at: {state.timestamp}\n"
+                             f"Database date: {date}")
 
     props.write_replication_state(base_url, seq, date)
 

tests/bdd/command-line/replication.feature

@@ -6,6 +6,8 @@ Feature: Tests for the osm2pgsql-replication script with property table
             n34 Tamenity=restaurant x77 y45.3
             """
         And the replication service at http://example.com/europe/liechtenstein-updates
+            | sequence | timestamp            |
+            | 9999999  | 2013-08-03T19:00:02Z |
         When running osm2pgsql pgsql with parameters
             | --slim |
 
@@ -22,6 +24,8 @@ Feature: Tests for the osm2pgsql-replication script with property table
     Scenario: Replication will be initialised from the information of the import file
         Given the input file 'liechtenstein-2013-08-03.osm.pbf'
         And the replication service at http://example.com/europe/liechtenstein-updates
+            | sequence | timestamp            |
+            | 9999999  | 2013-08-03T19:00:02Z |
         When running osm2pgsql pgsql with parameters
             | --slim |
 
@@ -65,6 +69,8 @@ Feature: Tests for the osm2pgsql-replication script with property table
         Given the database schema foobar
         Given the input file 'liechtenstein-2013-08-03.osm.pbf'
         And the replication service at http://example.com/europe/liechtenstein-updates
+            | sequence | timestamp            |
+            | 9999999  | 2013-08-03T19:00:02Z |
         When running osm2pgsql pgsql with parameters
             | --slim | --middle-schema=foobar |
 
@@ -82,6 +88,8 @@ Feature: Tests for the osm2pgsql-replication script with property table
         Given the database schema foobar
         Given the input file 'liechtenstein-2013-08-03.osm.pbf'
         And the replication service at http://example.com/europe/liechtenstein-updates
+            | sequence | timestamp            |
+            | 9999999  | 2013-08-03T19:00:02Z |
         When running osm2pgsql pgsql with parameters
             | --slim | --schema=foobar |
 
@@ -100,6 +108,8 @@ Feature: Tests for the osm2pgsql-replication script with property table
         Given the database schema baz
         Given the input file 'liechtenstein-2013-08-03.osm.pbf'
         And the replication service at http://example.com/europe/liechtenstein-updates
+            | sequence | timestamp            |
+            | 9999999  | 2013-08-03T19:00:02Z |
         When running osm2pgsql pgsql with parameters
             | --slim | --middle-schema=foobar | --schema=baz |
 
@@ -220,9 +230,9 @@ Feature: Tests for the osm2pgsql-replication script with property table
         Given the input file 'liechtenstein-2013-08-03.osm.pbf'
         And the replication service at http://example.com/europe/liechtenstein-updates
             | sequence | timestamp            |
-            | 345      | 2020-10-04T01:00:00Z |
-            | 346      | 2020-10-04T02:00:00Z |
-            | 347      | 2020-10-04T03:00:00Z |
+            | 344      | 2013-08-03T13:00:02Z |
+            | 345      | 2013-08-03T14:00:02Z |
+            | 346      | 2013-08-03T15:00:02Z |
         When running osm2pgsql pgsql with parameters
             | --slim |
 
@@ -370,3 +380,37 @@ Feature: Tests for the osm2pgsql-replication script with property table
             "server": {"base_url": "http://example.com/europe/liechtenstein-updates", "sequence": 10000001, "timestamp": "2013-10-01T01:00:00Z"
             "local": {"sequence": 9999999, "timestamp": "2013-08-03T19:00:02Z"
             """
+
+
+    Scenario: Replication initialisation will fail when diffs are not old enough
+        Given the database schema foobar
+        Given the input file 'liechtenstein-2013-08-03.osm.pbf'
+        And the replication service at http://example.com/europe/liechtenstein-updates
+            | sequence | timestamp            |
+            | 10000000 | 2020-01-01T01:00:02Z |
+        When running osm2pgsql pgsql with parameters
+            | --slim |
+
+        Then running osm2pgsql-replication fails with returncode 1
+            | init | --server | http://example.com/europe/liechtenstein-updates |
+        And the error output contains
+            """
+            The replication service does not have diff files for the requested date.
+            """
+
+
+    Scenario: Replication initialisation will fail when diffs are not old enough
+        Given the database schema foobar
+        Given the input file 'liechtenstein-2013-08-03.osm.pbf'
+        And the replication service at http://example.com/europe/liechtenstein-updates
+            | sequence | timestamp            |
+            | 10000000 | 2020-01-01T01:00:02Z |
+        When running osm2pgsql pgsql with parameters
+            | --slim |
+
+        Then running osm2pgsql-replication fails with returncode 1
+            | init |
+        And the error output contains
+            """
+            Cannot load state information for 9999999 from replication service http://example.com/europe/liechtenstein-updates.
+            """

tests/bdd/command-line/replication_legacy.feature

@@ -10,6 +10,8 @@ Feature: Tests for the osm2pgsql-replication script without property table
 
     Scenario: Replication can be initialised with a osm file
         Given the replication service at http://example.com/europe/liechtenstein-updates
+            | sequence | timestamp            |
+            | 9999999  | 2013-08-03T19:00:02Z |
         When running osm2pgsql pgsql with parameters
             | --slim |
         And deleting table osm2pgsql_properties
@@ -38,6 +40,8 @@ Feature: Tests for the osm2pgsql-replication script without property table
 
     Scenario: Replication can be initialised in different schema
         Given the replication service at http://example.com/europe/liechtenstein-updates
+            | sequence | timestamp            |
+            | 9999999  | 2013-08-03T19:00:02Z |
         Given the database schema foobar
         When running osm2pgsql pgsql with parameters
             | --slim | --middle-schema=foobar |

tests/bdd/steps/replication_server_mock.py

@@ -28,7 +28,7 @@ def get_state_info(self, seq=None, retries=2):
             if info.sequence == seq:
                 return info
 
-        assert False, f"No sequence information for sequence ID {seq}."
+        return None
 
     def timestamp_to_sequence(self, timestamp, balanced_search=False):
         assert self.state_infos, 'Replication mock not properly set up'