feat: loop up request IP address in geoip database and store result

Store geoip results alongside map_frame in database

Author: matthiasschaub

sketch_map_tool/config.py

@@ -1,5 +1,6 @@
 import logging
 import os
+from pathlib import Path
 
 from pydantic import computed_field, field_validator
 from pydantic_settings import (
@@ -26,6 +27,7 @@ class Config(BaseSettings):
     cleanup_map_frames_interval: str = "12 months"
     data_dir: str = str(get_project_root() / "data")  # TODO: make this a Path
     esri_api_key: str = ""
+    geo_ip_database: Path | None = None
     log_level: str = "INFO"
     max_nr_simultaneous_uploads: int = 100
     model_type_sam: str = "vit_b"

sketch_map_tool/database/client_celery.py

@@ -40,6 +40,12 @@ def insert_map_frame(
     format_: PaperFormat,
     orientation: str,
     layer: str,
+    ip: str | None = None,
+    user_agent: str | None = None,
+    geo_ip_city: str | None = None,
+    geo_ip_country: str | None = None,
+    geo_ip_country_iso_code: str | None = None,
+    geo_ip_centroid_wgs84: str | None = None,
 ):
     """Insert map frame alongside map generation parameters into the database.
 
@@ -59,7 +65,13 @@ def insert_map_frame(
             layer VARCHAR,
             version VARCHAR,
             created TIMESTAMP WITH TIME ZONE DEFAULT now(),
-            downloaded TIMESTAMP WITH TIME ZONE
+            downloaded TIMESTAMP WITH TIME ZONE,
+            ip VARCHAR DEFAULT NULL,
+            user_agent VARCHAR DEFAULT NULL,
+            geo_ip_city VARCHAR DEFAULT NULL,
+            geo_ip_country VARCHAR DEFAULT NULL,
+            geo_ip_country_iso_code VARCHAR DEFAULT NULL,
+            geo_ip_centroid_wgs84 VARCHAR DEFAULT NULL
             )
     """
     insert_query = """
@@ -73,7 +85,13 @@ def insert_map_frame(
             format,
             orientation,
             layer,
-            version
+            version,
+            ip,
+            user_agent,
+            geo_ip_city,
+            geo_ip_country,
+            geo_ip_country_iso_code,
+            geo_ip_centroid_wgs84
             )
         VALUES (
             %s,
@@ -85,6 +103,12 @@ def insert_map_frame(
             %s,
             %s,
             %s,
+            %s,
+            %s,
+            %s,
+            %s,
+            %s,
+            %s,
             %s)
     """
     with db_conn.cursor() as curs:
@@ -102,6 +126,12 @@ def insert_map_frame(
                 orientation,
                 layer,
                 __version__,
+                ip,
+                user_agent,
+                geo_ip_city,
+                geo_ip_country,
+                geo_ip_country_iso_code,
+                geo_ip_centroid_wgs84,
             ),
         )
 
@@ -112,6 +142,7 @@ def cleanup_map_frames():
     Only set file and bbox to null. Keep metadata.
     This function is called by a periodic celery task.
     """
+    # TODO: Should request IP, user_agent and geo ip data also be removed?
     query = """
     UPDATE
         map_frame

sketch_map_tool/helpers.py

@@ -1,12 +1,15 @@
+import logging
 from io import BytesIO
 from pathlib import Path
 from typing import assert_never
 from zipfile import ZipFile
 
 import cv2
+import geoip2.database
 import numpy as np
 from billiard.exceptions import TimeLimitExceeded
 from celery.result import AsyncResult, GroupResult
+from geoip2.errors import AddressNotFoundError
 from geojson import Feature, FeatureCollection
 from numpy.typing import NDArray
 from reportlab.graphics.shapes import Drawing
@@ -109,3 +112,19 @@ def extract_errors(
                 if len(errors_) > 0:
                     errors = errors + [e.translate() for e in errors_]
     return errors
+
+
+def geo_ip_lookup(ip: str | None, database: Path | None) -> tuple[str | None, ...]:
+    if ip is not None and database is not None:
+        try:
+            with geoip2.database.Reader(str(database)) as reader:
+                geo_ip_city = reader.city(ip)
+                country = geo_ip_city.country.name
+                country_iso_code = geo_ip_city.country.iso_code
+                city = geo_ip_city.city.name
+                location = geo_ip_city.location
+                centroid_wgs84 = f"POINT ({location.longitude} {location.latitude})"
+            return (country, country_iso_code, city, centroid_wgs84)
+        except (FileNotFoundError, AddressNotFoundError) as error:
+            logging.error(error, exc_info=error)
+    return (None, None, None, None)

sketch_map_tool/routes.py

@@ -16,8 +16,9 @@
 )
 from werkzeug import Response
 
-from sketch_map_tool import celery_app, config, definitions, tasks
+from sketch_map_tool import celery_app, definitions, tasks
 from sketch_map_tool import flask_app as app
+from sketch_map_tool.config import CONFIG
 from sketch_map_tool.database import client_flask as db_client_flask
 from sketch_map_tool.definitions import REQUEST_TYPES
 from sketch_map_tool.exceptions import (
@@ -31,6 +32,7 @@
 from sketch_map_tool.helpers import (
     N_,
     extract_errors,
+    geo_ip_lookup,
     merge,
     to_array,
     zip_,
@@ -81,7 +83,7 @@ def create(lang="en") -> str:
     return render_template(
         "create.html.jinja",
         lang=lang,
-        esri_api_key=config.CONFIG.esri_api_key,
+        esri_api_key=CONFIG.esri_api_key,
     )
 
 
@@ -97,9 +99,25 @@ def create_results_post(lang="en") -> Response:
     size = Size(**(json.loads(request.form["size"])))
     scale = float(request.form["scale"])
     layer = validate_layer(request.form["layer"])
+
+    user_agent = request.headers.get("User-Agent", None)
+    ip = request.headers.get("X-Forwarded-For", request.remote_addr)
+    geo_ip_response = geo_ip_lookup(ip, CONFIG.geo_ip_database)
+
     # Tasks
     task_sketch_map = tasks.generate_sketch_map.apply_async(
-        args=(bbox, bbox_wgs84, format_, orientation, size, scale, layer)
+        args=(
+            bbox,
+            bbox_wgs84,
+            format_,
+            orientation,
+            size,
+            scale,
+            layer,
+            ip,
+            user_agent,
+            *geo_ip_response,
+        )
     )
     return redirect(
         url_for(

sketch_map_tool/tasks.py

@@ -98,6 +98,12 @@ def generate_sketch_map(
     size: Size,
     scale: float,
     layer: str,
+    ip: str | None = None,
+    user_agent: str | None = None,
+    geo_ip_city: str | None = None,
+    geo_ip_country: str | None = None,
+    geo_ip_country_iso_code: str | None = None,
+    geo_ip_centroid_wgs84: str | None = None,
 ) -> BytesIO | AsyncResult:
     """Generate and returns a sketch map as PDF and stores the map frame in DB."""
     if layer.startswith("oam"):
@@ -125,6 +131,12 @@ def generate_sketch_map(
         format_,
         orientation,
         layer,
+        ip,
+        user_agent,
+        geo_ip_city,
+        geo_ip_country,
+        geo_ip_country_iso_code,
+        geo_ip_centroid_wgs84,
     )
     return map_pdf
 

tests/approvals/unit/test_helpers.py--test_geo_ip_lookup.approved.txt

@@ -0,0 +1 @@
+('United Kingdom', 'GB', 'Boxford', 'POINT (-1.25 51.75)')

tests/approvals/unit/test_helpers.py--test_geo_ip_lookup_address_not_found.approved.txt

@@ -0,0 +1 @@
+ERROR    root:helpers.py:129 The address 42.2.228.64 is not in the database.

tests/approvals/unit/test_helpers.py--test_geo_ip_lookup_city_not_found.approved.txt

@@ -0,0 +1 @@
+('Germany', 'DE', None, 'POINT (10.5 51.5)')

tests/fixtures/geoip/GeoIP2-City-Test.mmdb

@@ -0,0 +1,9 @@
+GeoIP2-City-Test.mmdb
+
+Source of database
+
+https://github.com/maxmind/MaxMind-DB/blob/94975ab8c103b822a91f7082b5b70794e5b9fb12/test-data/GeoIP2-City-Test.mmdb
+
+Source of data
+
+https://github.com/maxmind/MaxMind-DB/blob/94975ab8c103b822a91f7082b5b70794e5b9fb12/source-data/GeoIP2-City-Test.json