Initial work on async handler.

Mostly copies code, some optimization, brings up
tests cases.

Not actually fully async yet because ultimately
calls sync code via the requests library.

Author: UmanShahzad

.gitignore

@@ -1,3 +1,7 @@
+.vscode/
+.vim/
+.idea/
+
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]

ipinfo/__init__.py

@@ -1,6 +1,12 @@
 from .handler import Handler
+from .handler_async import AsyncHandler
 
 
 def getHandler(access_token=None, **kwargs):
     """Create and return Handler object."""
     return Handler(access_token, **kwargs)
+
+
+def getHandlerAsync(access_token=None, **kwargs):
+    """Create an return an asynchronous Handler object."""
+    return AsyncHandler(access_token, **kwargs)

ipinfo/handler.py

@@ -16,8 +16,8 @@
 
 class Handler:
     """
-    Allows client to request data for specified IP address. Instantiates and
-    and maintains access to cache.
+    Allows client to request data for specified IP address.
+    Instantiates and maintains access to cache.
     """
 
     API_URL = "https://ipinfo.io"
@@ -120,7 +120,7 @@ def _requestDetails(self, ip_address=None):
     def _get_headers(self):
         """Built headers for request to IPinfo API."""
         headers = {
-            "user-agent": "IPinfoClient/Python{version}/2.0.0".format(
+            "user-agent": "IPinfoClient/Python{version}/3.0.0".format(
                 version=sys.version_info[0]
             ),
             "accept": "application/json",

ipinfo/handler_async.py

@@ -0,0 +1,155 @@
+"""
+Main API client asynchronous handler for fetching data from the IPinfo service.
+"""
+
+from ipaddress import IPv4Address, IPv6Address
+import json
+import os
+import sys
+
+import requests
+
+from .cache.default import DefaultCache
+from .details import Details
+from .exceptions import RequestQuotaExceededError
+
+
+class AsyncHandler:
+    """
+    Allows client to request data for specified IP address asynchronously.
+    Instantiates and maintains access to cache.
+    """
+
+    API_URL = "https://ipinfo.io"
+    CACHE_MAXSIZE = 4096
+    CACHE_TTL = 60 * 60 * 24
+    COUNTRY_FILE_DEFAULT = "countries.json"
+    REQUEST_TIMEOUT_DEFAULT = 2
+
+    def __init__(self, access_token=None, **kwargs):
+        """Initialize the Handler object with country name list and the cache initialized."""
+        self.access_token = access_token
+        self.countries = self._read_country_names(kwargs.get("countries_file"))
+        self.request_options = kwargs.get("request_options", {})
+        if "timeout" not in self.request_options:
+            self.request_options["timeout"] = self.REQUEST_TIMEOUT_DEFAULT
+
+        if "cache" in kwargs:
+            self.cache = kwargs["cache"]
+        else:
+            cache_options = kwargs.get("cache_options", {})
+            if "maxsize" not in cache_options:
+                cache_options["maxsize"] = self.CACHE_MAXSIZE
+            if "ttl" not in cache_options:
+                cache_options["ttl"] = self.CACHE_TTL
+            self.cache = DefaultCache(**cache_options)
+
+    async def getDetails(self, ip_address=None):
+        """Get details for specified IP address as a Details object."""
+        # If the supplied IP address uses the objects defined in the built-in
+        # module ipaddress, extract the appropriate string notation before
+        # formatting the URL.
+        if isinstance(ip_address, IPv4Address) or isinstance(ip_address, IPv6Address):
+            ip_address = ip_address.exploded
+
+        if ip_address in self.cache:
+            return Details(self.cache[ip_address])
+
+        # not in cache; get result, format it, and put in cache.
+        url = self.API_URL
+        if ip_address:
+            url += "/" + ip_address
+        response = requests.get(
+            url, headers=self._get_headers(), **self.request_options
+        )
+        if response.status_code == 429:
+            raise RequestQuotaExceededError()
+        response.raise_for_status()
+        raw_details = response.json()
+        self._format_details(raw_details)
+        self.cache[ip_address] = raw_details
+        return Details(raw_details)
+
+    async def getBatchDetails(self, ip_addresses):
+        """Get details for a batch of IP addresses at once."""
+        result = {}
+
+        # Pre-populate with anything we've got in the cache, and keep around
+        # the IPs not in the cache.
+        lookup_addresses = []
+        for ip_address in ip_addresses:
+            # If the supplied IP address uses the objects defined in the
+            # built-in module ipaddress extract the appropriate string notation
+            # before formatting the URL.
+            if isinstance(ip_address, IPv4Address) or isinstance(ip_address, IPv6Address):
+                ip_address = ip_address.exploded
+
+            if ip_address in self.cache:
+                result[ip_address] = self.cache[ip_address]
+            else:
+                lookup_addresses.append(ip_address)
+
+        # all in cache - return early.
+        if len(lookup_addresses) == 0:
+            return result
+
+        # Do the lookup
+        url = self.API_URL + "/batch"
+        headers = self._get_headers()
+        headers["content-type"] = "application/json"
+        response = requests.post(
+            url, json=lookup_addresses, headers=headers, **self.request_options
+        )
+        if response.status_code == 429:
+            raise RequestQuotaExceededError()
+        response.raise_for_status()
+
+        # Format & fill up cache
+        json_response = response.json()
+        for ip_address, details in json_response.items():
+            if isinstance(details, dict):
+                self._format_details(details)
+                self.cache[ip_address] = details
+
+        # Merge cached results with new lookup
+        result.update(json_response)
+
+        return result
+
+    def _get_headers(self):
+        """Built headers for request to IPinfo API."""
+        headers = {
+            "user-agent": "IPinfoClient/Python{version}/3.0.0".format(
+                version=sys.version_info[0]
+            ),
+            "accept": "application/json",
+        }
+
+        if self.access_token:
+            headers["authorization"] = "Bearer {}".format(self.access_token)
+
+        return headers
+
+    def _format_details(self, details):
+        details["country_name"] = self.countries.get(details.get("country"))
+        details["latitude"], details["longitude"] = self._read_coords(
+            details.get("loc")
+        )
+
+    def _read_coords(self, location):
+        lat, lon = None, None
+        coords = tuple(location.split(",")) if location else ""
+        if len(coords) == 2 and coords[0] and coords[1]:
+            lat, lon = coords[0], coords[1]
+        return lat, lon
+
+    def _read_country_names(self, countries_file=None):
+        """Read list of countries from specified country file or default file."""
+        if not countries_file:
+            countries_file = os.path.join(
+                os.path.dirname(__file__), self.COUNTRY_FILE_DEFAULT
+            )
+        with open(countries_file) as f:
+            countries_json = f.read()
+
+        return json.loads(countries_json)

requirements.in

@@ -1,8 +1,10 @@
-# For app
+# base
 requests>=2.18.4
-cachetools==3.1.1
+cachetools==4.1.1
+aiohttp<=4
 
-# For dev
-pytest==4.5.0
-pip-tools==3.7.0
-black==19.3b0
+# dev
+pytest==6.1.2
+pytest-asyncio==0.14.0
+pip-tools==5.3.1
+black==20.8b1

requirements.txt

@@ -2,29 +2,37 @@
 # This file is autogenerated by pip-compile
 # To update, run:
 #
-#    pip-compile --no-emit-trusted-host --no-index
+#    pip-compile --no-emit-index-url --no-emit-trusted-host
 #
+aiohttp==3.7.2            # via -r requirements.in
 appdirs==1.4.3            # via black
-atomicwrites==1.3.0       # via pytest
-attrs==19.3.0             # via black, pytest
-black==19.3b0
-cachetools==3.1.1
+async-timeout==3.0.1      # via aiohttp
+attrs==19.3.0             # via aiohttp, pytest
+black==20.8b1             # via -r requirements.in
+cachetools==4.1.1         # via -r requirements.in
 certifi==2019.9.11        # via requests
-chardet==3.0.4            # via requests
-click==7.0                # via black, pip-tools
-idna==2.8                 # via requests
-importlib-metadata==0.23  # via pluggy
-more-itertools==7.2.0     # via pytest, zipp
-pip-tools==3.7.0
+chardet==3.0.4            # via aiohttp, requests
+click==7.1.2              # via black, pip-tools
+idna==2.8                 # via requests, yarl
+iniconfig==1.1.1          # via pytest
+multidict==5.0.0          # via aiohttp, yarl
+mypy-extensions==0.4.3    # via black
+packaging==20.4           # via pytest
+pathspec==0.8.0           # via black
+pip-tools==5.3.1          # via -r requirements.in
 pluggy==0.13.0            # via pytest
-py==1.8.0                 # via pytest
-pytest==4.5.0
-requests==2.22.0
-six==1.12.0               # via pip-tools, pytest
-toml==0.10.0              # via black
+py==1.9.0                 # via pytest
+pyparsing==2.4.7          # via packaging
+pytest-asyncio==0.14.0    # via -r requirements.in
+pytest==6.1.2             # via -r requirements.in, pytest-asyncio
+regex==2020.10.28         # via black
+requests==2.22.0          # via -r requirements.in
+six==1.12.0               # via packaging, pip-tools
+toml==0.10.2              # via black, pytest
+typed-ast==1.4.1          # via black
+typing-extensions==3.7.4.3  # via aiohttp, black
 urllib3==1.25.6           # via requests
-wcwidth==0.1.7            # via pytest
-zipp==0.6.0               # via importlib-metadata
+yarl==1.6.2               # via aiohttp
 
 # The following packages are considered to be unsafe in a requirements file:
-# setuptools==41.6.0        # via pytest
+# pip

scripts/ctags.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+# Regenerate ctags.
+
+ctags \
+    --recurse=yes \
+    --exclude=node_modules \
+    --exclude=dist \
+    --exclude=build \
+    --exclude=target \
+    -f .vim/tags \
+    --tag-relative=never \
+    --totals=yes \
+        ./ipinfo \
+        ./tests

tests/handler_async_test.py

@@ -0,0 +1,75 @@
+import os
+
+from ipinfo.cache.default import DefaultCache
+from ipinfo.details import Details
+from ipinfo.handler_async import AsyncHandler
+import pytest
+
+@pytest.mark.asyncio
+async def test_init():
+    token = "mytesttoken"
+    handler = AsyncHandler(token)
+    assert handler.access_token == token
+    assert isinstance(handler.cache, DefaultCache)
+    assert "PK" in handler.countries
+
+
+@pytest.mark.asyncio
+async def test_headers():
+    token = "mytesttoken"
+    handler = AsyncHandler(token)
+    headers = handler._get_headers()
+
+    assert "user-agent" in headers
+    assert "accept" in headers
+    assert "authorization" in headers
+
+
+@pytest.mark.asyncio
+async def test_get_details():
+    token = os.environ.get('IPINFO_TOKEN', '')
+    handler = AsyncHandler(token)
+    details = await handler.getDetails("8.8.8.8")
+    assert isinstance(details, Details)
+    assert details.ip == "8.8.8.8"
+    assert details.hostname == "dns.google"
+    assert details.city == "Mountain View"
+    assert details.region == "California"
+    assert details.country == "US"
+    assert details.country_name == "United States"
+    assert details.loc == "37.4056,-122.0775"
+    assert details.latitude == "37.4056"
+    assert details.longitude == "-122.0775"
+    assert details.postal == "94043"
+    assert details.timezone == "America/Los_Angeles"
+    if token:
+        asn = details.asn
+        assert asn["asn"] == "AS15169"
+        assert asn["name"] == "Google LLC"
+        assert asn["domain"] == "google.com"
+        assert asn["route"] == "8.8.8.0/24"
+        assert asn["type"] == "business"
+
+        company = details.company
+        assert company["name"] == "Google LLC"
+        assert company["domain"] == "google.com"
+        assert company["type"] == "business"
+
+        privacy = details.privacy
+        assert privacy["vpn"] == False
+        assert privacy["proxy"] == False
+        assert privacy["tor"] == False
+        assert privacy["hosting"] == False
+
+        abuse = details.abuse
+        assert abuse["address"] == "US, CA, Mountain View, 1600 Amphitheatre Parkway, 94043"
+        assert abuse["country"] == "US"
+        assert abuse["email"] == "[email protected]"
+        assert abuse["name"] == "Abuse"
+        assert abuse["network"] == "8.8.8.0/24"
+        assert abuse["phone"] == "+1-650-253-0000"
+
+        domains = details.domains
+        assert domains["ip"] == "8.8.8.8"
+        assert domains["total"] == 12988
+        assert len(domains["domains"]) == 5