Fix config export command and remove optionals from its output (#234)

* Fix `config export` command and remove optionals from its output

* Cleanup

* Fix changelog

Author: droserasprout

CHANGELOG.md

@@ -7,7 +7,9 @@ Please use [this](https://docs.gitlab.com/ee/development/changelog.html) documen
 ### Fixed
 
 * ci: Removed `black 21.12b0` dependency since bug in `datamodel-codegen-generator` is fixed.
-* config: Verify `advanced.scheduler` config for correctness unsupported features.
+* cli: Fixed `config export` command crash when `advanced.reindex` dictionary is present.
+* cli: Removed optionals from `config export` output so the result can be loaded again.
+* config: Verify `advanced.scheduler` config for the correctness and unsupported features.
 * context: Fixed ignored `wait` argument of `fire_hook` method.
 * hasura: Fixed processing relation fields with missing `related_name`.
 * jobs: Fixed default `apscheduler` config.

poetry.lock

@@ -49,6 +49,7 @@
 from dipdup.enums import SkipHistory
 from dipdup.exceptions import ConfigInitializationException
 from dipdup.exceptions import ConfigurationError
+from dipdup.utils import exclude_none
 from dipdup.utils import import_from
 from dipdup.utils import pascal_to_snake
 from dipdup.utils import snake_to_pascal
@@ -1072,10 +1073,12 @@ def load(
 
     def dump(self) -> str:
         config_json = json.dumps(self, default=pydantic_encoder)
+        config_yaml = yaml.safe_load(config_json)
+
         return cast(
             str,
             yaml.dump(
-                yaml.safe_load(config_json),
+                exclude_none(config_yaml),
                 indent=2,
                 default_flow_style=False,
             ),

src/dipdup/config.py

@@ -35,9 +35,11 @@ class IndexStatus(Enum):
     SYNCING = 'SYNCING'
     REALTIME = 'REALTIME'
     ROLLBACK = 'ROLLBACK'
+    # TODO: Drop in 5.0.0
     ONESHOT = 'ONESHOT'
 
 
+# TODO: Drop in 5.0.0
 class ReindexingReason(ReversedEnum):
     MANUAL = 'triggered manually from callback'
     MIGRATION = 'applied migration requires reindexing'
@@ -48,7 +50,8 @@ class ReindexingReason(ReversedEnum):
     MISSING_INDEX_TEMPLATE = 'index template is missing, can\'t restore index state'
 
 
-class ReindexingReasonC(Enum):
+# NOTE: Used as a key in config, must inherit from str
+class ReindexingReasonC(str, Enum):
     manual = 'manual'
     migration = 'migration'
     rollback = 'rollback'

src/dipdup/enums.py

@@ -177,3 +177,11 @@ def skip_ci(fn):
     if os.environ.get('CI'):
         return skip('CI environment, skipping')(fn)
     return fn
+
+
+def exclude_none(config_json):
+    if isinstance(config_json, (list, tuple)):
+        return [exclude_none(i) for i in config_json if i is not None]
+    if isinstance(config_json, dict):
+        return {k: exclude_none(v) for k, v in config_json.items() if v is not None}
+    return config_json

src/dipdup/utils/__init__.py

@@ -1,3 +1,4 @@
+import tempfile
 from os.path import dirname
 from os.path import join
 from typing import Callable
@@ -34,3 +35,14 @@ async def test_validators(self):
             ContractConfig(address='lalalalalalalalalalalalalalalalalala')
         with self.assertRaises(ConfigurationError):
             TzktDatasourceConfig(kind='tzkt', url='not_an_url')
+
+    async def test_dump(self):
+        config = DipDupConfig.load([self.path])
+        config.initialize()
+
+        tmp = tempfile.mkstemp()[1]
+        with open(tmp, 'w') as f:
+            f.write(config.dump())
+
+        config = DipDupConfig.load([tmp], environment=False)
+        config.initialize()