Fix HTTP config and TzKT fetcher methods (#577)

Author: droserasprout

CHANGELOG.md

@@ -4,6 +4,18 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog], and this project adheres to [Semantic Versioning].
 
+## [Unreleased]
+
+### Added
+
+- context: Allow adding contracts by code hash in runtime.
+- config: Added `http.ratelimit_sleep` option to set fixed sleep time on 429 responses.
+
+### Fixed
+
+- http: Fixed merging user-defined HTTP settings and datasource defaults.
+- tzkt: Fixed iterating over big map keys.
+
 ## [6.4.1] - 2022-12-22
 
 ### Fixed

docs/Makefile

@@ -22,9 +22,9 @@ help:
 docs:
 	poetry run make html
 	# NOTE: A hack to include generated references in mdbook pages
-	tail -n '+35' _build/html/cli-reference.html | head -n '-63' > cli-reference.md
-	tail -n '+35' _build/html/config-reference.html | head -n '-63' > config/reference.md
-	tail -n '+35' _build/html/context-reference.html | head -n '-63' > include/context-reference.md
+	tail -n '+32' _build/html/cli-reference.html | head -n '-63' > cli-reference.md
+	tail -n '+32' _build/html/config-reference.html | head -n '-63' > config/reference.md
+	tail -n '+32' _build/html/context-reference.html | head -n '-63' > include/context-reference.md
 	mdbook build  --dest-dir=dist/docs
 
 homepage:

docs/config/reference.md

@@ -18,15 +18,15 @@ chardet==4.0.0 ; python_version >= "3.10" and python_version < "3.11"
 charset-normalizer==2.1.1 ; python_version >= "3.10" and python_version < "3.11"
 click==8.0.4 ; python_version >= "3.10" and python_version < "3.11"
 colorama==0.4.6 ; python_version >= "3.10" and python_version < "3.11"
-coverage[toml]==7.0.0 ; python_version >= "3.10" and python_version < "3.11"
+coverage[toml]==7.0.1 ; python_version >= "3.10" and python_version < "3.11"
 datamodel-code-generator==0.13.1 ; python_version >= "3.10" and python_version < "3.11"
 diff-cover==7.3.0 ; python_version >= "3.10" and python_version < "3.11"
 dnspython==2.2.1 ; python_version >= "3.10" and python_version < "3.11"
 docker==6.0.1 ; python_version >= "3.10" and python_version < "3.11"
 docutils==0.19 ; python_version >= "3.10" and python_version < "3.11"
 email-validator==1.3.0 ; python_version >= "3.10" and python_version < "3.11"
 entrypoints==0.4 ; python_version >= "3.10" and python_version < "3.11"
-exceptiongroup==1.0.4 ; python_version >= "3.10" and python_version < "3.11"
+exceptiongroup==1.1.0 ; python_version >= "3.10" and python_version < "3.11"
 execnet==1.9.0 ; python_version >= "3.10" and python_version < "3.11"
 flake8-bugbear==22.12.6 ; python_version >= "3.10" and python_version < "3.11"
 flake8-comprehensions==3.10.1 ; python_version >= "3.10" and python_version < "3.11"
@@ -40,21 +40,21 @@ imagesize==1.4.1 ; python_version >= "3.10" and python_version < "3.11"
 inflect==5.6.2 ; python_version >= "3.10" and python_version < "3.11"
 iniconfig==1.1.1 ; python_version >= "3.10" and python_version < "3.11"
 iso8601==1.1.0 ; python_version >= "3.10" and python_version < "3.11"
-isort==5.11.3 ; python_version >= "3.10" and python_version < "3.11"
+isort==5.11.4 ; python_version >= "3.10" and python_version < "3.11"
 jinja2==3.1.2 ; python_version >= "3.10" and python_version < "3.11"
 jsonschema==4.17.3 ; python_version >= "3.10" and python_version < "3.11"
 markupsafe==2.1.1 ; python_version >= "3.10" and python_version < "3.11"
 mccabe==0.6.1 ; python_version >= "3.10" and python_version < "3.11"
 msgpack==1.0.4 ; python_version >= "3.10" and python_version < "3.11"
-multidict==6.0.3 ; python_version >= "3.10" and python_version < "3.11"
+multidict==6.0.4 ; python_version >= "3.10" and python_version < "3.11"
 mypy-extensions==0.4.3 ; python_version >= "3.10" and python_version < "3.11"
 mypy==0.991 ; python_version >= "3.10" and python_version < "3.11"
 openapi-schema-validator==0.3.4 ; python_version >= "3.10" and python_version < "3.11"
 openapi-spec-validator==0.3.1 ; python_version >= "3.10" and python_version < "3.11"
 orjson==3.8.3 ; python_version >= "3.10" and python_version < "3.11"
 packaging==21.3 ; python_version >= "3.10" and python_version < "3.11"
 pathspec==0.10.3 ; python_version >= "3.10" and python_version < "3.11"
-platformdirs==2.6.0 ; python_version >= "3.10" and python_version < "3.11"
+platformdirs==2.6.2 ; python_version >= "3.10" and python_version < "3.11"
 pluggy==1.0.0 ; python_version >= "3.10" and python_version < "3.11"
 prance==0.22.11.4.0 ; python_version >= "3.10" and python_version < "3.11"
 prometheus-client==0.14.1 ; python_version >= "3.10" and python_version < "3.11"
@@ -67,7 +67,7 @@ pygments==2.13.0 ; python_version >= "3.10" and python_version < "3.11"
 pyhumps==3.8.0 ; python_version >= "3.10" and python_version < "3.11"
 pyparsing==3.0.9 ; python_version >= "3.10" and python_version < "3.11"
 pypika-tortoise==0.1.6 ; python_version >= "3.10" and python_version < "3.11"
-pyrsistent==0.19.2 ; python_version >= "3.10" and python_version < "3.11"
+pyrsistent==0.19.3 ; python_version >= "3.10" and python_version < "3.11"
 pysignalr==0.1.2 ; python_version >= "3.10" and python_version < "3.11"
 pysnooper==1.1.1 ; python_version >= "3.10" and python_version < "3.11"
 pytest-aiohttp==1.0.4 ; python_version >= "3.10" and python_version < "3.11"
@@ -91,7 +91,7 @@ sniffio==1.3.0 ; python_version >= "3.10" and python_version < "3.11"
 snowballstemmer==2.2.0 ; python_version >= "3.10" and python_version < "3.11"
 sphinx-click==4.4.0 ; python_version >= "3.10" and python_version < "3.11"
 sphinx-markdown-builder==0.5.5 ; python_version >= "3.10" and python_version < "3.11"
-sphinx==5.3.0 ; python_version >= "3.10" and python_version < "3.11"
+sphinx==6.0.0 ; python_version >= "3.10" and python_version < "3.11"
 sphinxcontrib-applehelp==1.0.2 ; python_version >= "3.10" and python_version < "3.11"
 sphinxcontrib-devhelp==1.0.2 ; python_version >= "3.10" and python_version < "3.11"
 sphinxcontrib-htmlhelp==2.0.0 ; python_version >= "3.10" and python_version < "3.11"

docs/include/context-reference.md

@@ -29,7 +29,7 @@ charset-normalizer==2.1.1 ; python_version >= "3.10" and python_version < "3.11"
 click==8.0.4 ; python_version >= "3.10" and python_version < "3.11"
 colorama==0.4.6 ; python_version >= "3.10" and python_version < "3.11" and platform_system == "Windows" or python_version >= "3.10" and python_version < "3.11" and sys_platform == "win32"
 comm==0.1.2 ; python_version >= "3.10" and python_version < "3.11"
-cytoolz==0.12.0 ; python_version >= "3.10" and python_version < "3.11" and implementation_name == "cpython"
+cytoolz==0.12.1 ; python_version >= "3.10" and python_version < "3.11" and implementation_name == "cpython"
 datamodel-code-generator==0.13.1 ; python_version >= "3.10" and python_version < "3.11"
 debugpy==1.6.4 ; python_version >= "3.10" and python_version < "3.11"
 decorator==5.1.1 ; python_version >= "3.10" and python_version < "3.11"
@@ -42,38 +42,38 @@ entrypoints==0.4 ; python_version >= "3.10" and python_version < "3.11"
 eth-hash==0.5.1 ; python_version >= "3.10" and python_version < "3.11"
 eth-typing==3.2.0 ; python_version >= "3.10" and python_version < "3.11"
 eth-utils==2.1.0 ; python_version >= "3.10" and python_version < "3.11"
-exceptiongroup==1.0.4 ; python_version >= "3.10" and python_version < "3.11"
+exceptiongroup==1.1.0 ; python_version >= "3.10" and python_version < "3.11"
 executing==1.2.0 ; python_version >= "3.10" and python_version < "3.11"
-fastecdsa==2.2.3 ; python_version >= "3.10" and python_version < "3.11"
+fastecdsa==2.3.0 ; python_version >= "3.10" and python_version < "3.11"
 fastjsonschema==2.16.2 ; python_version >= "3.10" and python_version < "3.11"
 fqdn==1.5.1 ; python_version >= "3.10" and python_version < "3.11"
 frozenlist==1.3.3 ; python_version >= "3.10" and python_version < "3.11"
 genson==1.2.2 ; python_version >= "3.10" and python_version < "3.11"
 idna==3.4 ; python_version >= "3.10" and python_version < "3.11"
 inflect==5.6.2 ; python_version >= "3.10" and python_version < "3.11"
-ipykernel==6.19.3 ; python_version >= "3.10" and python_version < "3.11"
+ipykernel==6.19.4 ; python_version >= "3.10" and python_version < "3.11"
 ipython-genutils==0.2.0 ; python_version >= "3.10" and python_version < "3.11"
 ipython==8.7.0 ; python_version >= "3.10" and python_version < "3.11"
 iso8601==1.1.0 ; python_version >= "3.10" and python_version < "3.11"
 isoduration==20.11.0 ; python_version >= "3.10" and python_version < "3.11"
-isort==5.11.3 ; python_version >= "3.10" and python_version < "3.11"
+isort==5.11.4 ; python_version >= "3.10" and python_version < "3.11"
 jedi==0.18.2 ; python_version >= "3.10" and python_version < "3.11"
 jinja2==3.1.2 ; python_version >= "3.10" and python_version < "3.11"
 jsonpointer==2.3 ; python_version >= "3.10" and python_version < "3.11"
 jsonschema==4.17.3 ; python_version >= "3.10" and python_version < "3.11"
 jsonschema[format-nongpl]==4.17.3 ; python_version >= "3.10" and python_version < "3.11"
 jupyter-client==7.4.8 ; python_version >= "3.10" and python_version < "3.11"
-jupyter-core==5.1.0 ; python_version >= "3.10" and python_version < "3.11"
+jupyter-core==5.1.1 ; python_version >= "3.10" and python_version < "3.11"
 jupyter-events==0.5.0 ; python_version >= "3.10" and python_version < "3.11"
 jupyter-server-terminals==0.4.3 ; python_version >= "3.10" and python_version < "3.11"
-jupyter-server==2.0.1 ; python_version >= "3.10" and python_version < "3.11"
+jupyter-server==2.0.6 ; python_version >= "3.10" and python_version < "3.11"
 jupyterlab-pygments==0.2.2 ; python_version >= "3.10" and python_version < "3.11"
 markupsafe==2.1.1 ; python_version >= "3.10" and python_version < "3.11"
 matplotlib-inline==0.1.6 ; python_version >= "3.10" and python_version < "3.11"
 mistune==2.0.4 ; python_version >= "3.10" and python_version < "3.11"
 mnemonic==0.20 ; python_version >= "3.10" and python_version < "3.11"
 msgpack==1.0.4 ; python_version >= "3.10" and python_version < "3.11"
-multidict==6.0.3 ; python_version >= "3.10" and python_version < "3.11"
+multidict==6.0.4 ; python_version >= "3.10" and python_version < "3.11"
 mypy-extensions==0.4.3 ; python_version >= "3.10" and python_version < "3.11"
 nbclassic==0.4.8 ; python_version >= "3.10" and python_version < "3.11"
 nbclient==0.7.2 ; python_version >= "3.10" and python_version < "3.11"
@@ -93,7 +93,7 @@ pathspec==0.10.3 ; python_version >= "3.10" and python_version < "3.11"
 pendulum==2.1.2 ; python_version >= "3.10" and python_version < "3.11"
 pexpect==4.8.0 ; python_version >= "3.10" and python_version < "3.11" and sys_platform != "win32"
 pickleshare==0.7.5 ; python_version >= "3.10" and python_version < "3.11"
-platformdirs==2.6.0 ; python_version >= "3.10" and python_version < "3.11"
+platformdirs==2.6.2 ; python_version >= "3.10" and python_version < "3.11"
 ply==3.11 ; python_version >= "3.10" and python_version < "3.11"
 prance==0.22.11.4.0 ; python_version >= "3.10" and python_version < "3.11"
 prometheus-client==0.14.1 ; python_version >= "3.10" and python_version < "3.11"
@@ -111,7 +111,7 @@ pygments==2.13.0 ; python_version >= "3.10" and python_version < "3.11"
 pyhumps==3.8.0 ; python_version >= "3.10" and python_version < "3.11"
 pyparsing==3.0.9 ; python_version >= "3.10" and python_version < "3.11"
 pypika-tortoise==0.1.6 ; python_version >= "3.10" and python_version < "3.11"
-pyrsistent==0.19.2 ; python_version >= "3.10" and python_version < "3.11"
+pyrsistent==0.19.3 ; python_version >= "3.10" and python_version < "3.11"
 pysha3==1.0.2 ; python_version >= "3.10" and python_version < "3.11"
 pysignalr==0.1.2 ; python_version >= "3.10" and python_version < "3.11"
 pysnooper==1.1.1 ; python_version >= "3.10" and python_version < "3.11"

poetry.lock

@@ -15,7 +15,7 @@ frozenlist==1.3.3 ; python_version >= "3.10" and python_version < "3.11"
 idna==3.4 ; python_version >= "3.10" and python_version < "3.11"
 iso8601==1.1.0 ; python_version >= "3.10" and python_version < "3.11"
 msgpack==1.0.4 ; python_version >= "3.10" and python_version < "3.11"
-multidict==6.0.3 ; python_version >= "3.10" and python_version < "3.11"
+multidict==6.0.4 ; python_version >= "3.10" and python_version < "3.11"
 orjson==3.8.3 ; python_version >= "3.10" and python_version < "3.11"
 prometheus-client==0.14.1 ; python_version >= "3.10" and python_version < "3.11"
 pydantic==1.9.2 ; python_version >= "3.10" and python_version < "3.11"

requirements.dev.txt

@@ -23,27 +23,27 @@ genson==1.2.2 ; python_version >= "3.10" and python_version < "3.11"
 idna==3.4 ; python_version >= "3.10" and python_version < "3.11"
 inflect==5.6.2 ; python_version >= "3.10" and python_version < "3.11"
 iso8601==1.1.0 ; python_version >= "3.10" and python_version < "3.11"
-isort==5.11.3 ; python_version >= "3.10" and python_version < "3.11"
+isort==5.11.4 ; python_version >= "3.10" and python_version < "3.11"
 jinja2==3.1.2 ; python_version >= "3.10" and python_version < "3.11"
 jsonschema==4.17.3 ; python_version >= "3.10" and python_version < "3.11"
 markupsafe==2.1.1 ; python_version >= "3.10" and python_version < "3.11"
 msgpack==1.0.4 ; python_version >= "3.10" and python_version < "3.11"
-multidict==6.0.3 ; python_version >= "3.10" and python_version < "3.11"
+multidict==6.0.4 ; python_version >= "3.10" and python_version < "3.11"
 mypy-extensions==0.4.3 ; python_version >= "3.10" and python_version < "3.11"
 openapi-schema-validator==0.3.4 ; python_version >= "3.10" and python_version < "3.11"
 openapi-spec-validator==0.3.1 ; python_version >= "3.10" and python_version < "3.11"
 orjson==3.8.3 ; python_version >= "3.10" and python_version < "3.11"
 packaging==21.3 ; python_version >= "3.10" and python_version < "3.11"
 pathspec==0.10.3 ; python_version >= "3.10" and python_version < "3.11"
-platformdirs==2.6.0 ; python_version >= "3.10" and python_version < "3.11"
+platformdirs==2.6.2 ; python_version >= "3.10" and python_version < "3.11"
 prance==0.22.11.4.0 ; python_version >= "3.10" and python_version < "3.11"
 prometheus-client==0.14.1 ; python_version >= "3.10" and python_version < "3.11"
 pydantic==1.9.2 ; python_version >= "3.10" and python_version < "3.11"
 pydantic[email]==1.9.2 ; python_version >= "3.10" and python_version < "3.11"
 pyhumps==3.8.0 ; python_version >= "3.10" and python_version < "3.11"
 pyparsing==3.0.9 ; python_version >= "3.10" and python_version < "3.11"
 pypika-tortoise==0.1.6 ; python_version >= "3.10" and python_version < "3.11"
-pyrsistent==0.19.2 ; python_version >= "3.10" and python_version < "3.11"
+pyrsistent==0.19.3 ; python_version >= "3.10" and python_version < "3.11"
 pysignalr==0.1.2 ; python_version >= "3.10" and python_version < "3.11"
 pysnooper==1.1.1 ; python_version >= "3.10" and python_version < "3.11"
 python-dotenv==0.19.2 ; python_version >= "3.10" and python_version < "3.11"

requirements.pytezos.txt

@@ -19,11 +19,11 @@
 import json
 import logging.config
 import re
+import sys
 from abc import ABC
 from abc import abstractmethod
 from collections import Counter
 from contextlib import suppress
-from copy import copy
 from dataclasses import field
 from io import StringIO
 from os import environ as env
@@ -187,27 +187,52 @@ class HTTPConfig:
     :param retry_multiplier: Multiplier for sleep time between retries
     :param ratelimit_rate: Number of requests per period ("drops" in leaky bucket)
     :param ratelimit_period: Time period for rate limiting in seconds
+    :param ratelimit_sleep: Sleep time between requests when rate limit is reached
     :param connection_limit: Number of simultaneous connections
     :param connection_timeout: Connection timeout in seconds
     :param batch_size: Number of items fetched in a single paginated request (for some APIs)
-    :param replay_path: Development-only option to replay HTTP requests from a file
+    :param replay_path: Development-only option to use cached HTTP responses instead of making real requests
     """
 
-    retry_count: int | None = None         # default: inf
-    retry_sleep: float | None = None       # default: 0
-    retry_multiplier: float | None = None  # default: 0
+    retry_count: int | None = None
+    retry_sleep: float | None = None
+    retry_multiplier: float | None = None
     ratelimit_rate: int | None = None
     ratelimit_period: int | None = None
-    connection_limit: int | None = None    # default: 100
-    connection_timeout: int | None = None  # default: 60
+    ratelimit_sleep: float | None = None
+    connection_limit: int | None = None
+    connection_timeout: int | None = None
     batch_size: int | None = None
     replay_path: str | None = None
 
-    def merge(self, other: HTTPConfig | None) -> 'HTTPConfig':
-        """Set missing values from other config"""
-        config = copy(self)
-        if other:
-            for k, v in other.__dict__.items():
+
+@dataclass
+class ResolvedHTTPConfig:
+    """HTTP client configuration with defaults"""
+
+    retry_count: int = sys.maxsize
+    retry_sleep: float = 0.0
+    retry_multiplier: float = 1.0
+    ratelimit_rate: int = 0
+    ratelimit_period: int = 0
+    ratelimit_sleep: float = 5.0
+    connection_limit: int = 100
+    connection_timeout: int = 60
+    batch_size: int = 1000
+    replay_path: str | None = None
+
+    @classmethod
+    def create(
+        cls,
+        default: HTTPConfig,
+        user: HTTPConfig | None,
+    ) -> 'ResolvedHTTPConfig':
+        config = cls()
+        # NOTE: Apply datasource defaults first
+        for merge_config in (default, user):
+            if merge_config is None:
+                continue
+            for k, v in merge_config.__dict__.items():
                 if v is not None:
                     setattr(config, k, v)
         return config
@@ -1402,7 +1427,8 @@ def __post_init_post_parse__(self) -> None:
 
         self.paths: list[Path] = []
         self.environment: dict[str, str] = {}
-        self._contract_addresses = {contract.address for contract in self.contracts.values()}
+        self._contract_addresses = {v.address: k for k, v in self.contracts.items() if v.address is not None}
+        self._contract_code_hashes = {v.code_hash: k for k, v in self.contracts.items() if v.code_hash is not None}
 
     @property
     def schema_name(self) -> str:
@@ -1554,7 +1580,7 @@ def add_index(
         last_level: int = 0,
     ) -> None:
         if name in self.indexes:
-            raise IndexAlreadyExistsError(self, name)
+            raise IndexAlreadyExistsError(name)
         template_config = IndexTemplateConfig(
             template=template,
             values=values,