Add option to specify env files, fix index config hash validation (#99)

Author: droserasprout

poetry.lock

@@ -37,6 +37,7 @@ sentry-sdk = "^1.1.0"
 pyhumps = "^3.0.2"
 aiolimiter = "^1.0.0-beta.1"
 tabulate = "^0.8.9"
+python-dotenv = "^0.18.0"
 pytezos = {version = "^3.2.4", optional = true}
 
 [tool.poetry.dev-dependencies]

pyproject.toml

@@ -4,11 +4,12 @@
 import os
 from dataclasses import dataclass
 from functools import wraps
-from os.path import dirname, join
+from os.path import dirname, exists, join
 from typing import List, cast
 
 import click
 import sentry_sdk
+from dotenv import load_dotenv
 from fcache.cache import FileCache  # type: ignore
 from sentry_sdk.integrations.aiohttp import AioHttpIntegration
 
@@ -46,10 +47,11 @@ class CLIContext:
 @click.group()
 @click.version_option(__version__)
 @click.option('--config', '-c', type=str, multiple=True, help='Path to dipdup YAML config', default=['dipdup.yml'])
+@click.option('--env-file', '-e', type=str, multiple=True, help='Path to .env file', default=[])
 @click.option('--logging-config', '-l', type=str, help='Path to logging YAML config', default='logging.yml')
 @click.pass_context
 @click_command_wrapper
-async def cli(ctx, config: List[str], logging_config: str):
+async def cli(ctx, config: List[str], env_file: List[str], logging_config: str):
     try:
         path = join(os.getcwd(), logging_config)
         _logging_config = LoggingConfig.load(path)
@@ -58,6 +60,13 @@ async def cli(ctx, config: List[str], logging_config: str):
         _logging_config = LoggingConfig.load(path)
     _logging_config.apply()
 
+    for env_path in env_file:
+        env_path = join(os.getcwd(), env_path)
+        if not exists(env_path):
+            raise ConfigurationError(f'env file `{env_path}` does not exist')
+        _logger.info('Applying env_file `%s`', env_path)
+        load_dotenv(env_path, override=True)
+
     _config = DipDupConfig.load(config)
     if _config.spec_version not in spec_version_mapping:
         raise ConfigurationError('Unknown `spec_version`, correct ones: {}')

src/dipdup/cli.py

@@ -517,12 +517,9 @@ def __post_init_post_parse__(self) -> None:
         NameMixin.__post_init_post_parse__(self)
 
     def hash(self) -> str:
-        return hashlib.sha256(
-            json.dumps(
-                self,
-                default=pydantic_encoder,
-            ).encode(),
-        ).hexdigest()
+        config_json = json.dumps(self, default=pydantic_encoder)
+        config_hash = hashlib.sha256(config_json.encode()).hexdigest()
+        return config_hash
 
     @property
     def datasource_config(self) -> TzktDatasourceConfig:

src/dipdup/config.py

@@ -76,7 +76,7 @@ async def _process_queue(self) -> None:
 
     async def _initialize_index_state(self) -> None:
         self._logger.info('Getting state for index `%s`', self._config.name)
-        index_hash = self._config.hash()
+        index_config_hash = self._config.hash()
         state = await State.get_or_none(
             index_name=self._config.name,
             index_type=self._config.kind,
@@ -86,11 +86,11 @@ async def _initialize_index_state(self) -> None:
             state = state_cls(
                 index_name=self._config.name,
                 index_type=self._config.kind,
-                index_hash=index_hash,
+                index_hash=index_config_hash,
                 level=self._config.first_block,
             )
 
-        elif state.hash != index_hash:
+        elif state.index_hash != index_config_hash:
             self._logger.warning('Config hash mismatch (config has been changed), reindexing')
             await self._ctx.reindex()