Add an enum to the configuration to switch between ip.sb and ipquery.io

README.md

@@ -199,6 +199,31 @@ Screenshot:
 
 ![screenshot-geo-distribution](https://user-images.githubusercontent.com/2396817/40653599-9458b6b8-6333-11e8-8e6e-1d90271fc083.png)
 
+### Configuration
+
+Scylla allows you to configure various settings through command-line arguments. One of the new configuration options is to select the GeoIP API to use. You can choose between `ip.sb` and `ipquery.io` using the `--geoip-api` argument.
+
+#### Command-line Arguments
+
+| Argument            | Description                                                                 |
+|---------------------|-----------------------------------------------------------------------------|
+| `--no-webserver`    | Prevent starting a web server for JSON API                                  |
+| `--web-port`        | The port number for the web server                                          |
+| `--web-host`        | The hostname for the web server                                             |
+| `--skip-scheduler`  | Prevent the scheduler from crawling                                         |
+| `--version`         | Print the version of Scylla                                                 |
+| `--db-path`         | The sqlite database file location                                           |
+| `--validation-pool` | The validation pool size (i.e. the limit of concurrent validation tasks for proxies) |
+| `--no-forward-proxy-server` | Disable the forward proxy server                                    |
+| `--proxy-port`      | The port number for the forward proxy                                       |
+| `--geoip-api`       | Select the GeoIP API to use (`ip.sb` or `ipquery.io`)                        |
+
+Example usage:
+
+```bash
+scylla --geoip-api ipquery.io
+```
+
 API Documentation
 =================
 

frontend/src/utils.ts

@@ -25,6 +25,20 @@ export interface Proxy {
     region: string;
     country: string;
     city: string;
+    asn: string;
+    isp: string;
+    state: string;
+    zipcode: string;
+    latitude: number;
+    longitude: number;
+    timezone: string;
+    localtime: string;
+    is_mobile: boolean;
+    is_vpn: boolean;
+    is_tor: boolean;
+    is_proxy: boolean;
+    is_datacenter: boolean;
+    risk_score: number;
 }
 
 export interface ResponseJSON {
@@ -40,4 +54,4 @@ export interface StatsResponseJSON {
     median: number;
     total_count: number;
     valid_count: number;
-}
+}

scylla/cli.py

@@ -1,7 +1,7 @@
 import argparse
 import sys
 
-from scylla.config import batch_set_config, get_config
+from scylla.config import batch_set_config, get_config, GeoIPAPI
 from ._version import __version__
 
 CMD_DESCRIPTION = """Scylla command line mode
@@ -32,6 +32,8 @@ def main(args) -> int:
                         help='Disable the forward proxy server')
     parser.add_argument('--proxy-port', '-pp', type=int, default=8081,
                         help='The port number for the forward proxy')
+    parser.add_argument('--geoip-api', type=GeoIPAPI, choices=list(GeoIPAPI),
+                        default=GeoIPAPI.IPSB, help='Select the GeoIP API to use')
 
     parsed_args = parser.parse_args(args)
 

scylla/config.py

@@ -1,24 +1,29 @@
 from typing import Union
+from enum import Enum
 
-_config_data = {}
+class GeoIPAPI(Enum):
+    IPSB = "ip.sb"
+    IPQUERY = "ipquery.io"
 
+_config_data = {
+    "geoip_api": GeoIPAPI.IPSB
+}
 
 def _config_data_instance():
     global _config_data
     return _config_data
 
-
-def set_config(key: str, value: str):
+def set_config(key: str, value: Union[str, GeoIPAPI]):
+    if key == "geoip_api" and isinstance(value, str):
+        value = GeoIPAPI(value)
     _config_data_instance()[key] = value
 
-
-def get_config(key: str, default: str = None) -> Union[str, None]:
+def get_config(key: str, default: Union[str, GeoIPAPI] = None) -> Union[str, GeoIPAPI, None]:
     try:
         return _config_data_instance()[key]
     except KeyError:
         return default
 
-
 def batch_set_config(**kwargs):
     for k, v in kwargs.items():
         set_config(k, v)

scylla/database.py

@@ -58,6 +58,20 @@ class Meta:
     region = CharField(null=True)
     country = CharField(null=True)
     city = CharField(null=True)
+    asn = CharField(null=True)
+    isp = CharField(null=True)
+    state = CharField(null=True)
+    zipcode = CharField(null=True)
+    latitude = FloatField(null=True)
+    longitude = FloatField(null=True)
+    timezone = CharField(null=True)
+    localtime = DateTimeField(null=True)
+    is_mobile = BooleanField(default=False)
+    is_vpn = BooleanField(default=False)
+    is_tor = BooleanField(default=False)
+    is_proxy = BooleanField(default=False)
+    is_datacenter = BooleanField(default=False)
+    risk_score = IntegerField(null=True)
 
     def assign_from(self, p):
         self.ip = p.ip
@@ -76,6 +90,20 @@ def assign_from(self, p):
         self.region = p.region
         self.country = p.country
         self.city = p.city
+        self.asn = p.asn
+        self.isp = p.isp
+        self.state = p.state
+        self.zipcode = p.zipcode
+        self.latitude = p.latitude
+        self.longitude = p.longitude
+        self.timezone = p.timezone
+        self.localtime = p.localtime
+        self.is_mobile = p.is_mobile
+        self.is_vpn = p.is_vpn
+        self.is_tor = p.is_tor
+        self.is_proxy = p.is_proxy
+        self.is_datacenter = p.is_datacenter
+        self.risk_score = p.risk_score
         self.updated_at = datetime.datetime.now()
 
     def __str__(self):

scylla/validator.py

@@ -5,6 +5,7 @@
 
 from .loggings import logger
 from .tcpping import ping
+from .config import get_config, GeoIPAPI
 
 IP_CHECKER_API = 'http://api.ipify.org/?format=json'
 IP_CHECKER_API_SSL = 'https://api.ipify.org/?format=json'
@@ -62,19 +63,62 @@ def validate_proxy(self):
                     self._anonymous = True
                 self._valid = True
 
+                geoip_api = get_config('geoip_api', GeoIPAPI.IPSB)
+
+                if geoip_api == GeoIPAPI.IPSB:
+                    geoip_url = 'https://api.ip.sb/geoip/{}'.format(j['ip'])
+                else:
+                    geoip_url = 'https://api.ipquery.io/?format=json&ip={}'.format(j['ip'])
+
                 # A second request for meta info
-                r2 = requests.get('https://api.ip.sb/geoip/{}'.format(j['ip']), timeout=15)
+                r2 = requests.get(geoip_url, timeout=15)
                 jresponse = r2.json()
 
-                # Load meta data
-                # TODO: better location check
-                meta = {
-                    'location': '{},{}'.format(jresponse['latitude'], jresponse['longitude']),
-                    'organization': jresponse['organization'] if 'organization' in jresponse else None,
-                    'region': jresponse['region'],
-                    'country': jresponse['country_code'],
-                    'city': jresponse['city'],
-                }
+                if geoip_api == GeoIPAPI.IPSB:
+                    meta = {
+                        'asn': jresponse['asn'],
+                        'isp': jresponse['isp'],
+                        'state': jresponse['region'],
+                        'zipcode': None,
+                        'latitude': jresponse['latitude'],
+                        'longitude': jresponse['longitude'],
+                        'timezone': jresponse['timezone'],
+                        'localtime': None,
+                        'is_mobile': None,
+                        'is_vpn': None,
+                        'is_tor': None,
+                        'is_proxy': None,
+                        'is_datacenter': None,
+                        'risk_score': None,
+                        'location': '{},{}'.format(jresponse['latitude'], jresponse['longitude']),
+                        'organization': jresponse['organization'],
+                        'region': jresponse['region'],
+                        'country': jresponse['country_code'],
+                        'city': jresponse['city'],
+                    }
+                else:
+                    meta = {
+                        'asn': jresponse['isp']['asn'],
+                        'isp': jresponse['isp']['isp'],
+                        'state': jresponse['location']['state'],
+                        'zipcode': jresponse['location']['zipcode'],
+                        'latitude': jresponse['location']['latitude'],
+                        'longitude': jresponse['location']['longitude'],
+                        'timezone': jresponse['location']['timezone'],
+                        'localtime': jresponse['location']['localtime'],
+                        'is_mobile': jresponse['risk']['is_mobile'],
+                        'is_vpn': jresponse['risk']['is_vpn'],
+                        'is_tor': jresponse['risk']['is_tor'],
+                        'is_proxy': jresponse['risk']['is_proxy'],
+                        'is_datacenter': jresponse['risk']['is_datacenter'],
+                        'risk_score': jresponse['risk']['risk_score'],
+                        'location': '{},{}'.format(jresponse['location']['latitude'], jresponse['location']['longitude']),
+                        'organization': jresponse['isp']['org'],
+                        'region': jresponse['location']['state'],
+                        'country': jresponse['location']['country_code'],
+                        'city': jresponse['location']['city'],
+                    }
+
                 self._meta = meta
 
         except requests.Timeout: