Set config values from environment variables (#4)

Author: droserasprout

README.md

@@ -31,17 +31,14 @@ database:
 
 contracts:
   HEN_objkts:
-    network: mainnet
-    address: KT1Hkg5qeNhfwpKW4fXvq7HGZB9z2EnmCCA9
+    address: ${HEN_OBJKTS:-KT1Hkg5qeNhfwpKW4fXvq7HGZB9z2EnmCCA9}
   HEN_minter:
-    network: mainnet
-    address: KT1RJ6PbjHpwc3M5rw5s2Nbmefwbuwbdxton
+    address: ${HEN_MINTER:-KT1RJ6PbjHpwc3M5rw5s2Nbmefwbuwbdxton}
 
 datasources:
   tzkt_mainnet:
     kind: tzkt
-    network: mainnet
-    url: https://staging.api.tzkt.io
+    url: ${TZKT_URL:-https://staging.api.tzkt.io}
 
 indexes:
   operations_mainnet:
@@ -168,6 +165,12 @@ Now everything is ready to run your indexer:
 $ dipdup -c config.yml run
 ```
 
+Parameters wrapped with `${VARIABLE:-default_value}` in config could be set from corresponding environment variables. For example if you want to use another TzKT instance:
+
+```shell
+$ TZKT_URL=https://api.tzkt.io dipdup -c config.yml run
+```
+
 You can interrupt indexing at any moment, it will start from last processed block next time you run your app again.
 
 Use `docker-compose.yml` included in this repo if you prefer to run dipdup in Docker:
@@ -179,8 +182,6 @@ $ # edit `secrets.env` file, change credentials
 $ docker-compose up dipdup
 ```
 
-Note that can use `DIPDUP_DATABASE_PASSWORD` environment variable to avoid storing database password in `dipdup.yml`.
-
 ### Optional: configure Hasura GraphQL Engine
 
 `init` command generates Hasura metadata JSON in the package root. You can use `configure-graphql` command to apply it to the running GraphQL Engine instance:

secrets.env.example

@@ -1,4 +1,6 @@
-DIPDUP_DATABASE_PASSWORD=changeme
+HEN_OBJKTS=KT1Hkg5qeNhfwpKW4fXvq7HGZB9z2EnmCCA9
+HEN_MINTER=KT1RJ6PbjHpwc3M5rw5s2Nbmefwbuwbdxton
+TZKT_URL=https://staging.api.tzkt.io
 
 POSTGRES_USER=dipdup
 POSTGRES_DB=dipdup

src/dipdup/config.py

@@ -3,6 +3,7 @@
 import json
 import logging.config
 import os
+import re
 import sys
 from os import environ as env
 from os.path import dirname
@@ -17,6 +18,7 @@
 from dipdup.models import IndexType, State
 
 ROLLBACK_HANDLER = 'on_rollback'
+ENV_VARIABLE_REGEX = r'\${([\w]*):-(.*)}'
 
 
 @dataclass
@@ -53,10 +55,6 @@ class DatabaseConfig:
     database: str
     password: str = ''
 
-    def __post_init_post_parse__(self):
-        if not self.password:
-            self.password = env.get('DIPDUP_DATABASE_PASSWORD', '')
-
     @property
     def connection_string(self):
         return f'{self.driver}://{self.user}:{self.password}@{self.host}:{self.port}/{self.database}'
@@ -72,7 +70,6 @@ class TzktDatasourceConfig:
 
     kind: Literal['tzkt']
     url: str
-    network: Optional[str] = None
 
 
 @dataclass
@@ -219,7 +216,6 @@ class ContractConfig:
     """
 
     address: str
-    network: Optional[str] = None
 
 
 @dataclass
@@ -267,8 +263,16 @@ def load(
         filename = os.path.join(current_workdir, filename)
 
         with open(filename) as file:
-            raw_config = YAML(typ='base').load(file.read())
-        config = cls(**raw_config)
+            raw_config = file.read()
+
+        for match in re.finditer(ENV_VARIABLE_REGEX, raw_config):
+            variable, default_value = match.group(1), match.group(2)
+            value = env.get(variable)
+            placeholder = '${' + variable + ':-' + default_value + '}'
+            raw_config = raw_config.replace(placeholder, value or default_value)
+
+        json_config = YAML(typ='base').load(raw_config)
+        config = cls(**json_config)
         return config
 
     async def initialize(self) -> None:

src/dipdup_hic_et_nunc/dipdup-docker.yml

@@ -7,20 +7,18 @@ database:
   port: 5432
   user: dipdup
   database: dipdup
+  password: ${POSTGRES_PASSWORD:-changeme}
 
 contracts:
   HEN_objkts:
-    network: mainnet
-    address: KT1Hkg5qeNhfwpKW4fXvq7HGZB9z2EnmCCA9
+    address: ${HEN_OBJKTS:-KT1Hkg5qeNhfwpKW4fXvq7HGZB9z2EnmCCA9}
   HEN_minter:
-    network: mainnet
-    address: KT1RJ6PbjHpwc3M5rw5s2Nbmefwbuwbdxton
+    address: ${HEN_MINTER:-KT1RJ6PbjHpwc3M5rw5s2Nbmefwbuwbdxton}
 
 datasources:
   tzkt_mainnet:
     kind: tzkt
-    network: mainnet
-    url: https://staging.api.tzkt.io
+    url: ${TZKT_URL:-https://staging.api.tzkt.io}
 
 indexes:
   operations_mainnet:

src/dipdup_hic_et_nunc/dipdup.yml

@@ -6,17 +6,14 @@ database:
 
 contracts:
   HEN_objkts:
-    network: mainnet
-    address: KT1Hkg5qeNhfwpKW4fXvq7HGZB9z2EnmCCA9
+    address: ${HEN_OBJKTS:-KT1Hkg5qeNhfwpKW4fXvq7HGZB9z2EnmCCA9}
   HEN_minter:
-    network: mainnet
-    address: KT1RJ6PbjHpwc3M5rw5s2Nbmefwbuwbdxton
+    address: ${HEN_MINTER:-KT1RJ6PbjHpwc3M5rw5s2Nbmefwbuwbdxton}
 
 datasources:
   tzkt_mainnet:
     kind: tzkt
-    network: mainnet
-    url: https://staging.api.tzkt.io
+    url: ${TZKT_URL:-https://staging.api.tzkt.io}
 
 indexes:
   operations_mainnet:

src/dipdup_hic_et_nunc/hasura_metadata.json

@@ -53,8 +53,8 @@
                     "role": "user",
                     "permission": {
                         "columns": [
-                            "token_id",
                             "token_info",
+                            "token_id",
                             "id"
                         ],
                         "filter": {},

tests/test_dipdup/dipdup.yml

@@ -6,17 +6,14 @@ database:
 
 contracts:
   HEN_objkts:
-    network: mainnet
-    address: KT1Hkg5qeNhfwpKW4fXvq7HGZB9z2EnmCCA9
+    address: ${HEN_OBJKTS:-KT1Hkg5qeNhfwpKW4fXvq7HGZB9z2EnmCCA9}
   HEN_minter:
-    network: mainnet
-    address: KT1RJ6PbjHpwc3M5rw5s2Nbmefwbuwbdxton
+    address: ${HEN_MINTER:-KT1RJ6PbjHpwc3M5rw5s2Nbmefwbuwbdxton}
 
 datasources:
   tzkt_mainnet:
     kind: tzkt
-    network: mainnet
-    url: https://staging.api.tzkt.io
+    url: ${TZKT_URL:-https://staging.api.tzkt.io}
 
 indexes:
   operations_mainnet:

tests/test_dipdup/test_datasources/test_tzkt/test_cache.py

@@ -50,11 +50,14 @@ async def test_process(self):
 
         await self.cache.process(callback_mock)
 
-        callback_mock.assert_awaited_with(
+        self.assertEqual(
+            callback_mock.await_args[0][0],
             OperationHandlerConfig(
                 callback='',
                 pattern=[OperationHandlerPatternConfig(destination='KT1AFA2mwNUMNd4SsujE1YYp29vd8BZejyKW', entrypoint='hDAO_batch')],
             ),
-            [ANY],
-            ANY,
         )
+        self.assertIsInstance(callback_mock.await_args[0][1], list)
+        self.assertIsInstance(callback_mock.await_args[0][1][0], OperationData)
+        self.assertIsInstance(callback_mock.await_args[0][2], list)
+        self.assertIsInstance(callback_mock.await_args[0][2][0], OperationData)