Generic http datasource (#239)

* Http Datasource

* Datasource Factory

* Changelog

* Cleanup

Co-authored-by: Lev Gorodetskiy <[email protected]>

Author: igorsereda

CHANGELOG.md

@@ -44,6 +44,10 @@ Please use [this](https://docs.gitlab.com/ee/development/changelog.html) documen
 
 ## 4.2.4 - 2022-02-14
 
+### Added
+
+* config: Added `http` datasource to making arbitrary http requests.
+
 ### Fixed
 
 * context: Fixed crash when calling `fire_hook` method.

src/dipdup/config.py

@@ -273,11 +273,22 @@ def __hash__(self):
         return hash(self.kind + self.url)
 
 
+@dataclass
+class HttpDatasourceConfig(NameMixin):
+    kind: Literal['http']
+    url: str
+    http: Optional[HTTPConfig] = None
+
+    def __hash__(self):
+        return hash(self.kind + self.url)
+
+
 DatasourceConfigT = Union[
     TzktDatasourceConfig,
     CoinbaseDatasourceConfig,
     MetadataDatasourceConfig,
     IpfsDatasourceConfig,
+    HttpDatasourceConfig,
 ]
 
 

src/dipdup/context.py

@@ -38,6 +38,7 @@
 from dipdup.config import TzktDatasourceConfig
 from dipdup.datasources.coinbase.datasource import CoinbaseDatasource
 from dipdup.datasources.datasource import Datasource
+from dipdup.datasources.datasource import HttpDatasource
 from dipdup.datasources.ipfs.datasource import IpfsDatasource
 from dipdup.datasources.metadata.datasource import MetadataDatasource
 from dipdup.datasources.tzkt.datasource import TzktDatasource
@@ -298,6 +299,9 @@ def get_metadata_datasource(self, name: str) -> MetadataDatasource:
     def get_ipfs_datasource(self, name: str) -> IpfsDatasource:
         return self._get_datasource(name, IpfsDatasource)
 
+    def get_http_datasource(self, name: str) -> HttpDatasource:
+        return self._get_datasource(name, HttpDatasource)
+
 
 class HookContext(DipDupContext):
     """Hook callback context."""

src/dipdup/datasources/datasource.py

@@ -6,6 +6,8 @@
 from typing import Set
 from typing import Tuple
 
+from aiohttp.hdrs import METH_GET
+
 from dipdup.config import HTTPConfig
 from dipdup.datasources.subscription import HeadSubscription
 from dipdup.datasources.subscription import Subscription
@@ -38,6 +40,26 @@ def set_logger(self, name: str) -> None:
         self._logger = FormattedLogger(self._logger.name, name + ': {}')
 
 
+class HttpDatasource(Datasource):
+    _default_http_config = HTTPConfig(
+        cache=True,
+        retry_count=5,
+        retry_sleep=1,
+        ratelimit_rate=0,
+        ratelimit_period=0,
+    )
+
+    def __init__(self, url: str, http_config: Optional[HTTPConfig] = None) -> None:
+        super().__init__(url, self._default_http_config.merge(http_config))
+        self._logger = _logger
+
+    async def get(self, url: str, cache: bool = False, weight: int = 1, **kwargs):
+        return await self.request(METH_GET, url, cache, weight, **kwargs)
+
+    async def run(self) -> None:
+        pass
+
+
 # TODO: Generic interface
 class GraphQLDatasource(Datasource):
     ...

src/dipdup/datasources/factory.py

@@ -0,0 +1,61 @@
+from dipdup.config import CoinbaseDatasourceConfig
+from dipdup.config import DipDupConfig
+from dipdup.config import HttpDatasourceConfig
+from dipdup.config import IpfsDatasourceConfig
+from dipdup.config import MetadataDatasourceConfig
+from dipdup.config import TzktDatasourceConfig
+from dipdup.datasources.coinbase.datasource import CoinbaseDatasource
+from dipdup.datasources.datasource import Datasource
+from dipdup.datasources.datasource import HttpDatasource
+from dipdup.datasources.ipfs.datasource import IpfsDatasource
+from dipdup.datasources.metadata.datasource import MetadataDatasource
+from dipdup.datasources.tzkt.datasource import TzktDatasource
+
+
+class DatasourceFactory:
+    """Decouple Context logic and knowledge about Datasources implementation and creation"""
+
+    @classmethod
+    def build(cls, name: str, config: DipDupConfig) -> Datasource:
+        datasource = cls._build_datasource(name, config)
+        datasource.set_logger(name)
+        datasource.set_user_agent(config.package)
+
+        return datasource
+
+    @classmethod
+    def _build_datasource(cls, name: str, config: DipDupConfig) -> Datasource:
+        datasource_config = config.get_datasource(name)
+
+        if isinstance(datasource_config, TzktDatasourceConfig):
+            return TzktDatasource(
+                url=datasource_config.url,
+                http_config=datasource_config.http,
+                merge_subscriptions=config.advanced.merge_subscriptions,
+            )
+
+        if isinstance(datasource_config, CoinbaseDatasourceConfig):
+            return CoinbaseDatasource(
+                http_config=datasource_config.http,
+            )
+
+        if isinstance(datasource_config, MetadataDatasourceConfig):
+            return MetadataDatasource(
+                url=datasource_config.url,
+                network=datasource_config.network,
+                http_config=datasource_config.http,
+            )
+
+        if isinstance(datasource_config, IpfsDatasourceConfig):
+            return IpfsDatasource(
+                url=datasource_config.url,
+                http_config=datasource_config.http,
+            )
+
+        if isinstance(datasource_config, HttpDatasourceConfig):
+            return HttpDatasource(
+                url=datasource_config.url,
+                http_config=datasource_config.http,
+            )
+
+        raise NotImplementedError

src/dipdup/dipdup.py

@@ -23,26 +23,20 @@
 from tortoise.transactions import get_connection
 
 from dipdup.codegen import DipDupCodeGenerator
-from dipdup.config import CoinbaseDatasourceConfig
 from dipdup.config import ContractConfig
 from dipdup.config import DatasourceConfigT
 from dipdup.config import DipDupConfig
 from dipdup.config import IndexTemplateConfig
-from dipdup.config import IpfsDatasourceConfig
-from dipdup.config import MetadataDatasourceConfig
 from dipdup.config import OperationIndexConfig
 from dipdup.config import PostgresDatabaseConfig
-from dipdup.config import TzktDatasourceConfig
 from dipdup.config import default_hooks
 from dipdup.context import CallbackManager
 from dipdup.context import DipDupContext
 from dipdup.context import MetadataCursor
 from dipdup.context import pending_indexes
-from dipdup.datasources.coinbase.datasource import CoinbaseDatasource
 from dipdup.datasources.datasource import Datasource
 from dipdup.datasources.datasource import IndexDatasource
-from dipdup.datasources.ipfs.datasource import IpfsDatasource
-from dipdup.datasources.metadata.datasource import MetadataDatasource
+from dipdup.datasources.factory import DatasourceFactory
 from dipdup.datasources.tzkt.datasource import TzktDatasource
 from dipdup.enums import ReindexingReason
 from dipdup.exceptions import ConfigInitializationException
@@ -376,32 +370,8 @@ async def _create_datasources(self) -> None:
             if name in self._datasources:
                 continue
 
-            if isinstance(datasource_config, TzktDatasourceConfig):
-                datasource = TzktDatasource(
-                    url=datasource_config.url,
-                    http_config=datasource_config.http,
-                    merge_subscriptions=self._config.advanced.merge_subscriptions,
-                )
-            elif isinstance(datasource_config, CoinbaseDatasourceConfig):
-                datasource = CoinbaseDatasource(
-                    http_config=datasource_config.http,
-                )
-            elif isinstance(datasource_config, MetadataDatasourceConfig):
-                datasource = MetadataDatasource(
-                    url=datasource_config.url,
-                    network=datasource_config.network,
-                    http_config=datasource_config.http,
-                )
-            elif isinstance(datasource_config, IpfsDatasourceConfig):
-                datasource = IpfsDatasource(
-                    url=datasource_config.url,
-                    http_config=datasource_config.http,
-                )
-            else:
-                raise NotImplementedError
+            datasource = DatasourceFactory.build(name, self._config)
 
-            datasource.set_logger(datasource_config.name)
-            datasource.set_user_agent(self._config.package)
             self._datasources[name] = datasource
             self._datasources_by_config[datasource_config] = datasource