Keep test replays in /tmp (#488)

Author: droserasprout

poetry.lock

@@ -8,13 +8,13 @@ apscheduler==3.9.1 ; python_version >= "3.10" and python_version < "3.11"
 argcomplete==2.0.0 ; python_version >= "3.10" and python_version < "3.11"
 astor==0.8.1 ; python_version >= "3.10" and python_version < "3.11"
 async-timeout==4.0.2 ; python_version >= "3.10" and python_version < "3.11"
-asyncclick==8.1.3.2 ; python_version >= "3.10" and python_version < "3.11"
+asyncclick==8.1.3.4 ; python_version >= "3.10" and python_version < "3.11"
 asyncpg==0.26.0 ; python_version >= "3.10" and python_version < "3.11"
 attrs==21.4.0 ; python_version >= "3.10" and python_version < "3.11"
 babel==2.10.3 ; python_version >= "3.10" and python_version < "3.11"
 black==22.8.0 ; python_version >= "3.10" and python_version < "3.11"
 bump2version==1.0.1 ; python_version >= "3.10" and python_version < "3.11"
-certifi==2022.6.15.1 ; python_version >= "3.10" and python_version < "3.11"
+certifi==2022.9.14 ; python_version >= "3.10" and python_version < "3.11"
 chardet==4.0.0 ; python_version >= "3.10" and python_version < "3.11"
 charset-normalizer==2.1.1 ; python_version >= "3.10" and python_version < "3.11"
 click==8.0.4 ; python_version >= "3.10" and python_version < "3.11"
@@ -24,9 +24,9 @@ datamodel-code-generator==0.13.1 ; python_version >= "3.10" and python_version <
 deprecation==2.1.0 ; python_version >= "3.10" and python_version < "3.11"
 diff-cover==6.5.1 ; python_version >= "3.10" and python_version < "3.11"
 dnspython==2.2.1 ; python_version >= "3.10" and python_version < "3.11"
-docker==5.0.3 ; python_version >= "3.10" and python_version < "3.11"
+docker==6.0.0 ; python_version >= "3.10" and python_version < "3.11"
 docutils==0.19 ; python_version >= "3.10" and python_version < "3.11"
-email-validator==1.2.1 ; python_version >= "3.10" and python_version < "3.11"
+email-validator==1.3.0 ; python_version >= "3.10" and python_version < "3.11"
 entrypoints==0.4 ; python_version >= "3.10" and python_version < "3.11"
 execnet==1.9.0 ; python_version >= "3.10" and python_version < "3.11"
 flake8-bugbear==22.9.11 ; python_version >= "3.10" and python_version < "3.11"
@@ -39,7 +39,7 @@ flakeheaven==2.1.3 ; python_version >= "3.10" and python_version < "3.11"
 frozenlist==1.3.1 ; python_version >= "3.10" and python_version < "3.11"
 genson==1.2.2 ; python_version >= "3.10" and python_version < "3.11"
 html2text==2020.1.16 ; python_version >= "3.10" and python_version < "3.11"
-idna==3.3 ; python_version >= "3.10" and python_version < "3.11"
+idna==3.4 ; python_version >= "3.10" and python_version < "3.11"
 imagesize==1.4.1 ; python_version >= "3.10" and python_version < "3.11"
 inflect==5.6.2 ; python_version >= "3.10" and python_version < "3.11"
 iniconfig==1.1.1 ; python_version >= "3.10" and python_version < "3.11"
@@ -83,7 +83,7 @@ pytest==7.1.3 ; python_version >= "3.10" and python_version < "3.11"
 python-dotenv==0.19.2 ; python_version >= "3.10" and python_version < "3.11"
 pytz-deprecation-shim==0.1.0.post0 ; python_version >= "3.10" and python_version < "3.11"
 pytz==2022.2.1 ; python_version >= "3.10" and python_version < "3.11"
-pywin32==227 ; python_version >= "3.10" and python_version < "3.11" and sys_platform == "win32"
+pywin32==304 ; python_version >= "3.10" and python_version < "3.11" and sys_platform == "win32"
 pyyaml==6.0 ; python_version >= "3.10" and python_version < "3.11"
 requests==2.28.1 ; python_version >= "3.10" and python_version < "3.11"
 ruamel-yaml-clib==0.2.6 ; platform_python_implementation == "CPython" and python_version < "3.11" and python_version >= "3.10"

requirements.dev.txt

@@ -10,7 +10,7 @@ argon2-cffi-bindings==21.2.0 ; python_version >= "3.10" and python_version < "3.
 argon2-cffi==21.3.0 ; python_version >= "3.10" and python_version < "3.11"
 asttokens==2.0.8 ; python_version >= "3.10" and python_version < "3.11"
 async-timeout==4.0.2 ; python_version >= "3.10" and python_version < "3.11"
-asyncclick==8.1.3.2 ; python_version >= "3.10" and python_version < "3.11"
+asyncclick==8.1.3.4 ; python_version >= "3.10" and python_version < "3.11"
 asyncpg==0.26.0 ; python_version >= "3.10" and python_version < "3.11"
 attrs==21.4.0 ; python_version >= "3.10" and python_version < "3.11"
 backcall==0.2.0 ; python_version >= "3.10" and python_version < "3.11"
@@ -21,7 +21,7 @@ bleach==5.0.1 ; python_version >= "3.10" and python_version < "3.11"
 bson==0.5.10 ; python_version >= "3.10" and python_version < "3.11"
 cached-property==1.5.2 ; python_version >= "3.10" and python_version < "3.11"
 cattrs==22.1.0 ; python_version >= "3.10" and python_version < "3.11"
-certifi==2022.6.15.1 ; python_version >= "3.10" and python_version < "3.11"
+certifi==2022.9.14 ; python_version >= "3.10" and python_version < "3.11"
 cffi==1.15.1 ; python_version >= "3.10" and python_version < "3.11"
 chardet==4.0.0 ; python_version >= "3.10" and python_version < "3.11"
 charset-normalizer==2.1.1 ; python_version >= "3.10" and python_version < "3.11"
@@ -34,19 +34,19 @@ decorator==5.1.1 ; python_version >= "3.10" and python_version < "3.11"
 defusedxml==0.7.1 ; python_version >= "3.10" and python_version < "3.11"
 deprecation==2.1.0 ; python_version >= "3.10" and python_version < "3.11"
 dnspython==2.2.1 ; python_version >= "3.10" and python_version < "3.11"
-docker==5.0.3 ; python_version >= "3.10" and python_version < "3.11"
-email-validator==1.2.1 ; python_version >= "3.10" and python_version < "3.11"
+docker==6.0.0 ; python_version >= "3.10" and python_version < "3.11"
+email-validator==1.3.0 ; python_version >= "3.10" and python_version < "3.11"
 entrypoints==0.4 ; python_version >= "3.10" and python_version < "3.11"
 eth-hash==0.3.3 ; python_version >= "3.10" and python_version < "3.11"
-eth-typing==3.1.0 ; python_version >= "3.10" and python_version < "3.11"
+eth-typing==3.2.0 ; python_version >= "3.10" and python_version < "3.11"
 eth-utils==2.0.0 ; python_version >= "3.10" and python_version < "3.11"
 exceptiongroup==1.0.0rc9 ; python_version >= "3.10" and python_version < "3.11"
 executing==1.0.0 ; python_version >= "3.10" and python_version < "3.11"
 fastecdsa==2.2.3 ; python_version >= "3.10" and python_version < "3.11"
-fastjsonschema==2.16.1 ; python_version >= "3.10" and python_version < "3.11"
+fastjsonschema==2.16.2 ; python_version >= "3.10" and python_version < "3.11"
 frozenlist==1.3.1 ; python_version >= "3.10" and python_version < "3.11"
 genson==1.2.2 ; python_version >= "3.10" and python_version < "3.11"
-idna==3.3 ; python_version >= "3.10" and python_version < "3.11"
+idna==3.4 ; python_version >= "3.10" and python_version < "3.11"
 inflect==5.6.2 ; python_version >= "3.10" and python_version < "3.11"
 ipykernel==6.15.3 ; python_version >= "3.10" and python_version < "3.11"
 ipython-genutils==0.2.0 ; python_version >= "3.10" and python_version < "3.11"
@@ -106,16 +106,16 @@ pysha3==1.0.2 ; python_version >= "3.10" and python_version < "3.11"
 pysignalr==0.1.2 ; python_version >= "3.10" and python_version < "3.11"
 pysnooper==1.1.1 ; python_version >= "3.10" and python_version < "3.11"
 pysodium==0.7.12 ; python_version >= "3.10" and python_version < "3.11"
-pytezos==3.6.1 ; python_version >= "3.10" and python_version < "3.11"
+pytezos==3.7.0 ; python_version >= "3.10" and python_version < "3.11"
 python-dateutil==2.8.2 ; python_version >= "3.10" and python_version < "3.11"
 python-dotenv==0.19.2 ; python_version >= "3.10" and python_version < "3.11"
 pytz-deprecation-shim==0.1.0.post0 ; python_version >= "3.10" and python_version < "3.11"
 pytz==2022.2.1 ; python_version >= "3.10" and python_version < "3.11"
 pytzdata==2020.1 ; python_version >= "3.10" and python_version < "3.11"
-pywin32==227 ; python_version >= "3.10" and python_version < "3.11" and sys_platform == "win32"
-pywinpty==2.0.7 ; python_version >= "3.10" and python_version < "3.11" and os_name == "nt"
+pywin32==304 ; python_version >= "3.10" and python_version < "3.11" and sys_platform == "win32"
+pywinpty==2.0.8 ; python_version >= "3.10" and python_version < "3.11" and os_name == "nt"
 pyyaml==6.0 ; python_version >= "3.10" and python_version < "3.11"
-pyzmq==23.2.1 ; python_version >= "3.10" and python_version < "3.11"
+pyzmq==24.0.0 ; python_version >= "3.10" and python_version < "3.11"
 requests==2.28.1 ; python_version >= "3.10" and python_version < "3.11"
 ruamel-yaml-clib==0.2.6 ; platform_python_implementation == "CPython" and python_version < "3.11" and python_version >= "3.10"
 ruamel-yaml==0.17.21 ; python_version >= "3.10" and python_version < "3.11"

requirements.pytezos.txt

@@ -5,14 +5,14 @@ aiosqlite==0.17.0 ; python_version >= "3.10" and python_version < "3.11"
 anyio==3.6.1 ; python_version >= "3.10" and python_version < "3.11"
 apscheduler==3.9.1 ; python_version >= "3.10" and python_version < "3.11"
 async-timeout==4.0.2 ; python_version >= "3.10" and python_version < "3.11"
-asyncclick==8.1.3.2 ; python_version >= "3.10" and python_version < "3.11"
+asyncclick==8.1.3.4 ; python_version >= "3.10" and python_version < "3.11"
 asyncpg==0.26.0 ; python_version >= "3.10" and python_version < "3.11"
 attrs==21.4.0 ; python_version >= "3.10" and python_version < "3.11"
-certifi==2022.6.15.1 ; python_version >= "3.10" and python_version < "3.11"
+certifi==2022.9.14 ; python_version >= "3.10" and python_version < "3.11"
 charset-normalizer==2.1.1 ; python_version >= "3.10" and python_version < "3.11"
 colorama==0.4.5 ; python_version >= "3.10" and python_version < "3.11" and platform_system == "Windows"
 frozenlist==1.3.1 ; python_version >= "3.10" and python_version < "3.11"
-idna==3.3 ; python_version >= "3.10" and python_version < "3.11"
+idna==3.4 ; python_version >= "3.10" and python_version < "3.11"
 iso8601==1.0.2 ; python_version >= "3.10" and python_version < "3.11"
 msgpack==1.0.4 ; python_version >= "3.10" and python_version < "3.11"
 multidict==6.0.2 ; python_version >= "3.10" and python_version < "3.11"

requirements.slim.txt

@@ -6,21 +6,21 @@ anyio==3.6.1 ; python_version >= "3.10" and python_version < "3.11"
 apscheduler==3.9.1 ; python_version >= "3.10" and python_version < "3.11"
 argcomplete==2.0.0 ; python_version >= "3.10" and python_version < "3.11"
 async-timeout==4.0.2 ; python_version >= "3.10" and python_version < "3.11"
-asyncclick==8.1.3.2 ; python_version >= "3.10" and python_version < "3.11"
+asyncclick==8.1.3.4 ; python_version >= "3.10" and python_version < "3.11"
 asyncpg==0.26.0 ; python_version >= "3.10" and python_version < "3.11"
 attrs==21.4.0 ; python_version >= "3.10" and python_version < "3.11"
 black==22.8.0 ; python_version >= "3.10" and python_version < "3.11"
-certifi==2022.6.15.1 ; python_version >= "3.10" and python_version < "3.11"
+certifi==2022.9.14 ; python_version >= "3.10" and python_version < "3.11"
 chardet==4.0.0 ; python_version >= "3.10" and python_version < "3.11"
 charset-normalizer==2.1.1 ; python_version >= "3.10" and python_version < "3.11"
 click==8.0.4 ; python_version >= "3.10" and python_version < "3.11"
 colorama==0.4.5 ; python_version >= "3.10" and python_version < "3.11" and platform_system == "Windows"
 datamodel-code-generator==0.13.1 ; python_version >= "3.10" and python_version < "3.11"
 dnspython==2.2.1 ; python_version >= "3.10" and python_version < "3.11"
-email-validator==1.2.1 ; python_version >= "3.10" and python_version < "3.11"
+email-validator==1.3.0 ; python_version >= "3.10" and python_version < "3.11"
 frozenlist==1.3.1 ; python_version >= "3.10" and python_version < "3.11"
 genson==1.2.2 ; python_version >= "3.10" and python_version < "3.11"
-idna==3.3 ; python_version >= "3.10" and python_version < "3.11"
+idna==3.4 ; python_version >= "3.10" and python_version < "3.11"
 inflect==5.6.2 ; python_version >= "3.10" and python_version < "3.11"
 iso8601==1.0.2 ; python_version >= "3.10" and python_version < "3.11"
 isort==5.10.1 ; python_version >= "3.10" and python_version < "3.11"

requirements.txt

@@ -175,6 +175,7 @@ class HTTPConfig:
     :param connection_limit: Number of simultaneous connections
     :param connection_timeout: Connection timeout in seconds
     :param batch_size: Number of items fetched in a single paginated request (for some APIs)
+    :param replay_path: Development-only option to replay HTTP requests from a file
     """
 
     retry_count: Optional[int] = None
@@ -185,6 +186,7 @@ class HTTPConfig:
     connection_limit: Optional[int] = None  # default 100
     connection_timeout: Optional[int] = None  # default 60
     batch_size: Optional[int] = None
+    replay_path: Optional[str] = None
 
     def merge(self, other: Optional['HTTPConfig']) -> 'HTTPConfig':
         """Set missing values from other config"""

src/dipdup/config.py

@@ -1,4 +1,5 @@
 import asyncio
+import hashlib
 import logging
 import platform
 from contextlib import suppress
@@ -13,11 +14,13 @@
 import aiohttp
 import orjson
 from aiolimiter import AsyncLimiter
+from genericpath import isfile
 
 from dipdup import __version__
 from dipdup.config import HTTPConfig
 from dipdup.exceptions import InvalidRequestError
 from dipdup.prometheus import Metrics
+from dipdup.utils import touch
 
 safe_exceptions = (
     aiohttp.ClientConnectionError,
@@ -115,7 +118,13 @@ def _session(self) -> aiohttp.ClientSession:
         return self.__session
 
     # TODO: Move to separate method to cover SignalR negotiations too
-    async def _retry_request(self, method: str, url: str, weight: int = 1, **kwargs):
+    async def _retry_request(
+        self,
+        method: str,
+        url: str,
+        weight: int = 1,
+        **kwargs,
+    ):
         """Retry a request in case of failure sleeping according to config"""
         attempt = 1
         retry_sleep = self._config.retry_sleep or 0
@@ -158,7 +167,13 @@ async def _retry_request(self, method: str, url: str, weight: int = 1, **kwargs)
                 multiplier = 1 if ratelimit_sleep else self._config.retry_multiplier or 1
                 retry_sleep *= multiplier
 
-    async def _request(self, method: str, url: str, weight: int = 1, **kwargs):
+    async def _request(
+        self,
+        method: str,
+        url: str,
+        weight: int = 1,
+        **kwargs,
+    ):
         """Wrapped aiohttp call with preconfigured headers and ratelimiting"""
         if not url.startswith(self._url):
             url = self._url + '/' + url.lstrip('/')
@@ -187,6 +202,31 @@ async def _request(self, method: str, url: str, weight: int = 1, **kwargs):
                 return await response.json()
             return await response.read()
 
+    async def _replay_request(
+        self,
+        method: str,
+        url: str,
+        weight: int = 1,
+        **kwargs,
+    ):
+        request_hash = hashlib.sha256(
+            f'{self._url} {method} {url} {kwargs}'.encode(),
+        ).hexdigest()
+        if not self._config.replay_path:
+            raise Exception
+        replay_path = self._config.replay_path.rstrip('/') + '/' + request_hash
+
+        if isfile(replay_path):
+            with open(replay_path, 'rb') as file:
+                return orjson.loads(file.read())
+        else:
+            response = await self._retry_request(method, url, weight, **kwargs)
+            if response:
+                touch(replay_path)
+                with open(replay_path, 'wb') as file:
+                    file.write(orjson.dumps(response))
+            return response
+
     async def request(
         self,
         method: str,
@@ -195,7 +235,10 @@ async def request(
         **kwargs,
     ) -> Any:
         """Performs an HTTP request."""
-        return await self._retry_request(method, url, weight, **kwargs)
+        if self._config.replay_path:
+            return await self._replay_request(method, url, weight, **kwargs)
+        else:
+            return await self._retry_request(method, url, weight, **kwargs)
 
     def set_user_agent(self, *args: str) -> None:
         """Add list of arguments to User-Agent header"""

src/dipdup/http.py

@@ -17,6 +17,8 @@ datasources:
   tzkt_mainnet:
     kind: tzkt
     url: ${TZKT_URL:-https://api.tzkt.io}
+    http:
+      replay_path: /tmp/dipdup/replays
 
 indexes:
   hen_mainnet:

tests/integration_tests/hic_et_nunc.yml

@@ -23,6 +23,8 @@ datasources:
   tzkt_mainnet:
     kind: tzkt
     url: https://api.tzkt.io
+    http:
+      replay_path: /tmp/dipdup/replays
 
 templates:
   quipuswap_fa12:

tests/integration_tests/quipuswap.yml

@@ -14,6 +14,8 @@ datasources:
   tzkt:
     kind: tzkt
     url: ${TZKT_URL:-https://api.tzkt.io}
+    http:
+      replay_path: /tmp/dipdup/replays
 
 templates: