Merge pull request #3: Ready for release v0.1.1

Ready for release v0.1.1

Author: Haannbboo

.gitignore

@@ -6,5 +6,12 @@
 main.py
 __pycache__
 
-setup.py
-doc
+doc
+dist
+*.egg-info
+*.vscode
+*.bat
+venv
+
+*tox
+tox.ini

MANIFEST.in

@@ -0,0 +1,5 @@
+include requirements.txt
+include LICENSE
+global-include *.yaml
+global-include *.bz2
+recursive-include eve_tools/ESI/sso *.py application.json

README.md

@@ -1,36 +1,50 @@
-# EVE tools for trading
+# EVE Tools for trading
 
 > A collection of trading tools that are helpful for EVE game plays.
 
 _Author: Hanbo Guo_
 
-_EVE character name: Hanbie Serine_
+_EVE character: Hanbie Serine_
 
 _Contact: hbopublic@163.com_
 
 
+## What is EVE Tools
+
+EVE Tools is a Python package that simplifies EVE ESI. The goal is to write easier and faster Python scripts that analyze EVE data for ISK making. 
+
 ## Installations
 
-#### 0. Have a working Python environment. 
-#### 1. Download everything to local.
-#### 2. Run the `requirements.txt` file with pip install.
+### 0. Have a working Python environment. 
+### 1. Install using PyPI
+```sh
+pip3 install eve_tools
+```
+-----
+or
+### 1. Install manually
+
+#### 1.1 Download everything under this repo
+
+#### 1.2 Run `requirements.txt` file with pip install.
 ```sh
-# install after downloading
-cd (PATH of the download) 
-(e.g. macOS: /Users/{NAME}/Downloads/EVE_tools;
-Windows: C:\Users\{NAME}\Downloads\EVE_tools) 
+cd path/to/your/download/EVE_tools-master
 
 pip3 install -r requirements.txt
 ```
-#### 3. Run `example.py` by: 
 
-* double clicking it (in Windows)
-* or go to a Python IDE to launch it 
-* or possibly:
+#### 1.3 Setup
 ```sh
-python3 examples.py
+python setup.py install
 ```
-* or maybe try with
+-----
+### 2. Try with `example.py`
+
+Download `examples.py`, run it by:
+* double clicking it (in Windows)
+* or go to a Python IDE to launch it 
+* or
 ```sh
 python examples.py
 ```
+You will see the result `hauling.csv` generated in the same directory as `examples.py`.

eve_tools/ESI/esi.py

@@ -1,7 +1,10 @@
 import asyncio
+from dataclasses import dataclass
+from datetime import datetime
+from email.utils import parsedate
 import logging
 import aiohttp
-from typing import Optional, Union, List
+from typing import Coroutine, Optional, Union, List
 
 from .token import ESITokens
 from .metadata import ESIMetadata, ESIRequest
@@ -14,6 +17,32 @@
 logger = logging.getLogger(__name__)
 
 
+@dataclass
+class ESIResponse:
+    """Response returned by ESI.request() family.
+
+    User should never create ESIResponse but gets it from ESI.request() calls.
+
+    Attributes: (referencing aiohttp doc)
+        status: int
+            HTTP status code of response.
+        method: str
+            Request's method.
+        headers: dict
+            A case insensitive dictionary with HTTP headers of response.
+        data: dict | List | int
+            A json serialized response body, a dictionary or a list or an int.
+        expires: str | None
+            A RFC7231 formatted datetime string, if any.
+    """
+
+    status: int
+    method: str
+    headers: dict
+    data: Optional[Union[dict, List, int]]
+    expires: Optional[str] = None
+
+
 class ESI(object):
     """ESI request client for API requests.
 
@@ -35,6 +64,10 @@ def __init__(self, **kwd):
         ### Exit flag
         self._app_changed = False
 
+        ### API session
+        self._api_session = False
+        self._api_session_record = None
+
     def get(
         self,
         key: str,
@@ -130,7 +163,8 @@ def recursive_looper(async_loop: List, kwd: dict):
 
         ret = []
         for task in tasks:
-            ret.extend(task.result())  # well, let's forget about memory
+            # each task.result() is a ESIResponse instance
+            ret.append(task.result())  # well, let's forget about memory
 
         return ret
 
@@ -193,7 +227,7 @@ async def request(
             kwd: Keywords necessary for sending the request, such as headers, params, and other ESI required inputs.
 
         Returns:
-            A dictionary containing json serialized data from ESI.
+            An instance of ESIResponse containing response of the request.
 
         Raises:
             NotImplementedError: Request type POST/DELETE/PUT is not supported.
@@ -237,6 +271,83 @@ async def request(
 
         return res
 
+    def _api_session_recorder(_coro: Coroutine):
+        """Records useful info from ESIResponse that can be fed to other objects."""
+
+        async def _api_session_recorder_wrapped(_self, *args, **kwd):
+            resp: ESIResponse = await _coro(
+                _self, *args, **kwd
+            )  # this _self should be an instance of ESI
+
+            if not _self._api_session:
+                return resp
+
+            expires = resp.expires
+
+            if not _self._api_session_record:
+                _self._api_session_record = expires
+
+            # Use the earliest expire
+            if expires:
+                expires_dt = datetime(*parsedate(expires)[:6])
+                record_dt = datetime(*parsedate(_self._api_session_record)[:6])
+                if expires_dt < record_dt:
+                    _self._api_session_record = expires
+            return resp
+
+        return _api_session_recorder_wrapped
+
+    @ESIRequestError(attempts=3)
+    @_api_session_recorder
+    async def async_request(self, api_request: ESIRequest, method: str) -> dict:
+        """Asynchronous requests to ESI API.
+
+        Uses aiohttp to asynchronously request GET to ESI API.
+        ClientSession is created once for each instance and shared by multiple async_request call of the instance.
+        Default having maximum 100 open connections (100 async_request pending).
+
+        Args:
+            api_request: ESIRequest
+                A fully initialized ESIRequest with url, params, headers field filled in, given to aiohttp.ClientSession.get.
+            method: str
+                A str for HTTP request method.
+
+        Returns:
+            An instance of ESIResponse containing response of the request. Memory allocation assumed not to be a problem.
+        """
+        if not self._async_session:
+            self._async_session = aiohttp.ClientSession(
+                connector=aiohttp.TCPConnector(ssl=False), raise_for_status=True
+            )  # default maximum 100 connections
+
+        # no encoding: "4-HWF" stays what it is
+        if method == "get":
+            async with self._async_session.get(
+                api_request.url, params=api_request.params, headers=api_request.headers
+            ) as req:
+                data = await req.json()
+                resp = ESIResponse(
+                    req.status,
+                    req.method,
+                    dict(req.headers),
+                    data,
+                    req.headers.get("Expires"),
+                )
+
+        elif method == "head":
+            async with self._async_session.head(
+                api_request.url, params=api_request.params, headers=api_request.headers
+            ) as req:
+                resp = ESIResponse(
+                    req.status,
+                    req.method,
+                    dict(req.headers),
+                    None,
+                    req.headers.get("Expires"),
+                )
+
+        return resp
+
     def add_app_generate_token(
         self, clientId: str, scope: str, callbackURL: Optional[str] = None
     ) -> None:
@@ -280,42 +391,6 @@ def add_app_generate_token(
         with ESITokens(new_app) as token:
             token.generate()
 
-    @ESIRequestError(attempts=3)
-    async def async_request(self, api_request: ESIRequest, method: str) -> dict:
-        """Asynchronous requests to ESI API.
-
-        Uses aiohttp to asynchronously request GET to ESI API.
-        ClientSession is created once for each instance and shared by multiple async_request call of the instance.
-        Default having maximum 100 open connections (100 async_request pending).
-
-        Args:
-            api_request: ESIRequest
-                A fully initialized ESIRequest with url, params, headers field filled in, given to aiohttp.ClientSession.get.
-            method: str
-                A str for HTTP request method.
-
-        Returns:
-            A dictionary containing the response body or response header. Memory allocation assumed not to be a problem.
-        """
-        if not self._async_session:
-            self._async_session = aiohttp.ClientSession(
-                connector=aiohttp.TCPConnector(ssl=False), raise_for_status=True
-            )  # default maximum 100 connections
-
-        # no encoding: "4-HWF" stays what it is
-        if method == "get":
-            async with self._async_session.get(
-                api_request.url, params=api_request.params, headers=api_request.headers
-            ) as req:
-                return (
-                    await req.json()
-                )  # read entire response to memory, which shouldn't be a problem now.
-        elif method == "head":
-            async with self._async_session.head(
-                api_request.url, params=api_request.params, headers=api_request.headers
-            ) as req:
-                return dict(req.headers)
-
     def _get_auth_headers(
         self, tokens: ESITokens, cname: Optional[str] = "any"
     ) -> dict:
@@ -460,3 +535,17 @@ def __del__(self):
             if self._async_session._connector_owner:
                 self._async_session._connector.close()
             self._async_session._connector = None
+
+    def _start_api_session(self):
+        """Starts recording useful ESIResponse from API calls."""
+        self._api_session = True
+        self._api_session_record = None
+
+    def _end_api_session(self):
+        """Ends recording ESIResponse from API calls."""
+        self._api_session = False
+
+    def _clear_api_record(self):
+        """Clears _api_session_record entry of the instance."""
+        self._api_session_record = None
+        self._api_session = False

eve_tools/ESI/utils.py

@@ -5,20 +5,26 @@
 from typing import Coroutine, Optional
 
 
+BAD_REQUEST = 400
+NOT_FOUND = 404
+ERROR_LIMITED = 420
+
+
 class ESIRequestError:
     """A decorator that handles errors in ESI requests.
 
-    Currently, this decorator only retries ESI requests upon error. 
-    In the future, different errors will be detected and handled differently.
-    For example, a 400 error (bad request) should not be repeated, while a 502 (bad gateway) could be repeated.
-    It should be assumed that this decorator does not change behavior of async_request() method.
+    This decorator retries ESI request with error code 502 and 503. 
+    Other codes, such as 400, 404, 420, etc., that are not ESI's error, are not repeated.
+    ESI response headers has an x-error-limit field, which triggers an ERROR_LIMITED error when reaching 0.
 
     Attributes:
         attempts: Number of attempts, default 3.
         raises: Raises ClientResponseError or not. Default True, raising errors when no attempts left.
             If set to False, a None is returned.
     """
 
+    status_raise = [BAD_REQUEST, NOT_FOUND, ERROR_LIMITED]
+
     def __init__(
         self,
         attempts: Optional[int] = 3,
@@ -29,10 +35,13 @@ def __init__(
 
     def wrapper_retry(self, func: Coroutine):
         @wraps(func)
-        async def wrapped_retry(_caller_self, *args, **kwd):
+        async def wrapped_retry(_esi_self, *args, **kwd):
             """Retry ESI request upon error.
-            
-            ESI's async_request has signature (self, ESIRequest, method), so a _caller_self arg is added."""
+
+            ESI's async_request has signature (self, ESIRequest, method), so a _esi_self arg is added."""
+
+            # Can't use something like _caller_self = _args.pop(0),
+            # because deepcopy can't copy ESI.self. 
             _args = copy.deepcopy(args)
             _kwd = copy.deepcopy(kwd)
 
@@ -42,12 +51,16 @@ async def wrapped_retry(_caller_self, *args, **kwd):
             ret = None
             while not success and attempts > 0:
                 try:
-                    ret = await func(_caller_self, *_args, **_kwd)
+                    ret = await func(_esi_self, *_args, **_kwd)
                     success = True
                 except ClientResponseError as exc:
-                    logging.warning("%s | attempts left: %s", exc, self.attempts)
                     attempts -= 1
-                    if attempts == 0 and self.raises:
+                    resp_code = exc.status
+                    if resp_code in self.status_raise:
+                        attempts = 0
+                    logging.warning("%s | attempts left: %s", exc, attempts)
+
+                    if self.raises and attempts == 0:
                         raise
             return ret
 

eve_tools/__init__.py

@@ -1,16 +1,17 @@
 from __future__ import absolute_import
+import os
 import logging
 
 from .api import *
 from .ESI import ESIClient, ESITokens, Application
-from .data import ESIDB, CacheDB, api_cache, SqliteCache
+from .data import *
 from .config import *
 
 logging.basicConfig(
-    filename="esi.log",
+    filename=os.path.join(SOURCE_DIR, "esi.log"),
     format="%(asctime)s %(levelname)s %(module)s.%(funcName)s: %(message)s",
     filemode="w",
     level=logging.WARNING,
 )
 
-__version__ = "0.1.0"
+__version__ = "0.1.1"