Integrate with MaxMind out of the box (#766)

Integrate with MaxMind out of the box

Author: chadwhitacre

.gitignore

@@ -81,3 +81,6 @@ sentry/requirements.txt
 relay/credentials.json
 relay/config.yml
 symbolicator/config.yml
+geoip/GeoIP.conf
+geoip/*.mmdb
+geoip/.geoipupdate.lock

docker-compose.yml

@@ -31,6 +31,7 @@ x-sentry-defaults: &sentry_defaults
   volumes:
     - 'sentry-data:/data'
     - './sentry:/etc/sentry'
+    - './geoip:/geoip:ro'
 x-snuba-defaults: &snuba_defaults
   << : *restart_policy
   depends_on:
@@ -126,6 +127,14 @@ services:
       # If you have high volume and your search return incomplete results
       # You might want to change this to a higher value (and ensure your host has enough memory)
       MAX_MEMORY_USAGE_RATIO: 0.3
+  geoipupdate:
+    image: 'maxmindinc/geoipupdate:latest'
+    # Override the entrypoint in order to avoid using envvars for config.
+    # Futz with settings so we can keep mmdb and conf in same dir on host
+    # (image looks for them in separate dirs by default).
+    entrypoint: ['/usr/bin/geoipupdate', '-d', '/sentry', '-f', '/sentry/GeoIP.conf']
+    volumes:
+      - './geoip:/sentry'
   snuba-api:
     << : *snuba_defaults
   # Kafka consumer responsible for feeding events into Clickhouse
@@ -233,6 +242,10 @@ services:
         read_only: true
         source: ./relay
         target: /work/.relay
+      - type: bind
+        read_only: true
+        source: ./geoip
+        target: /geoip
     depends_on:
       - kafka
       - redis

geoip/GeoLite2-City.mmdb.empty

@@ -325,6 +325,10 @@ if [[ ! -f "$RELAY_CREDENTIALS_JSON" ]]; then
   echo "Relay credentials written to $RELAY_CREDENTIALS_JSON"
 fi
 
+
+./install/geoip.sh
+
+
 if [[ "$MINIMIZE_DOWNTIME" ]]; then
   # Start the whole setup, except nginx and relay.
   $dc up -d --remove-orphans $($dc config --services | grep -v -E '^(nginx|relay)$')

install.sh

@@ -0,0 +1,39 @@
+#!/usr/bin/env bash
+
+if [ ! -f 'install.sh' ]; then echo 'Where are you?'; exit 1; fi
+
+dc="docker-compose --no-ansi"
+dcr="$dc run --rm"
+
+
+install_geoip() {
+  local mmdb='geoip/GeoLite2-City.mmdb'
+  local conf='geoip/GeoIP.conf'
+  local result='Done'
+
+  echo "Setting up IP address geolocation ..."
+  if [[ ! -f "$mmdb" ]]; then
+    echo -n "Installing (empty) IP address geolocation database ... "
+    cp "$mmdb.empty" "$mmdb"
+    echo "done."
+  else
+    echo "IP address geolocation database already exists."
+  fi
+
+  if [[ ! -f "$conf" ]]; then
+    echo "IP address geolocation is not configured for updates."
+    echo "See https://develop.sentry.dev/self-hosted/geolocation/ for instructions."
+    result='Error'
+  else
+    echo "IP address geolocation is configured for updates."
+    echo "Updating IP address geolocation database ... "
+    $dcr geoipupdate
+    if [ $? -gt 0 ]; then
+      result='Error'
+    fi
+    echo "$result updating IP address geolocation database."
+  fi
+  echo "$result setting up IP address geolocation."
+}
+
+install_geoip

install/geoip.sh

@@ -45,7 +45,7 @@ http {
 	proxy_set_header Connection '';
 	proxy_set_header Host $host;
 	proxy_set_header X-Real-IP $remote_addr;
-	proxy_set_header X-Forwarded-For $remote_addr;
+	proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
 	proxy_set_header X-Forwarded-Proto $scheme;
 	proxy_set_header X-Request-Id $request_id;
 	proxy_read_timeout 30s;

nginx/nginx.conf

@@ -10,3 +10,4 @@ processing:
     - {name: "bootstrap.servers", value: "kafka:9092"}
     - {name: "message.max.bytes", value: 50000000} #50MB or bust
   redis: redis://redis:6379
+  geoip_path: "/geoip/GeoLite2-City.mmdb"

relay/config.example.yml

@@ -260,9 +260,15 @@ def get_internal_network():
     }
 )
 
+#######################
+# MaxMind Integration #
+#######################
+
+GEOIP_PATH_MMDB = '/geoip/GeoLite2-City.mmdb'
+
 #########################
 # Bitbucket Integration #
-########################
+#########################
 
 # BITBUCKET_CONSUMER_KEY = 'YOUR_BITBUCKET_CONSUMER_KEY'
 # BITBUCKET_CONSUMER_SECRET = 'YOUR_BITBUCKET_CONSUMER_SECRET'