Skip to content

Commit 027cbed

Browse files
logging config field (#349)
* Just `logging` field * Update references
1 parent 2f5391f commit 027cbed

17 files changed

+83
-20
lines changed

CHANGELOG.md

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,8 +4,13 @@
44

55
### Added
66

7+
* config: Added `logging` config field.
78
* config: Added `hasura.create_source` flag to create PostgreSQL source if missing.
89

10+
### Deprecated
11+
12+
* cli: `--logging-config` option is deprecated.
13+
914
### Fixed
1015

1116
* hasura: Do not apply table customizations to tables from other sources.

docs/config/reference.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -112,7 +112,7 @@
112112

113113
<dl class="py class">
114114
<dt class="sig sig-object py" id="dipdup.config.DipDupConfig">
115-
<em class="property"><span class="pre">class</span><span class="w"> </span></em><span class="sig-prename descclassname"><span class="pre">dipdup.config.</span></span><span class="sig-name descname"><span class="pre">DipDupConfig</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="pre">spec_version:</span> <span class="pre">str,</span> <span class="pre">package:</span> <span class="pre">str,</span> <span class="pre">datasources:</span> <span class="pre">typing.Dict[str,</span> <span class="pre">typing.Union[dipdup.config.TzktDatasourceConfig,</span> <span class="pre">dipdup.config.CoinbaseDatasourceConfig,</span> <span class="pre">dipdup.config.MetadataDatasourceConfig,</span> <span class="pre">dipdup.config.IpfsDatasourceConfig,</span> <span class="pre">dipdup.config.HttpDatasourceConfig]],</span> <span class="pre">database:</span> <span class="pre">typing.Union[dipdup.config.SqliteDatabaseConfig,</span> <span class="pre">dipdup.config.PostgresDatabaseConfig]</span> <span class="pre">=</span> <span class="pre">SqliteDatabaseConfig(kind='sqlite',</span> <span class="pre">path=':memory:'),</span> <span class="pre">contracts:</span> <span class="pre">typing.Dict[str,</span> <span class="pre">dipdup.config.ContractConfig]</span> <span class="pre">=</span> <span class="pre">&lt;factory&gt;,</span> <span class="pre">indexes:</span> <span class="pre">typing.Dict[str,</span> <span class="pre">typing.Union[dipdup.config.OperationIndexConfig,</span> <span class="pre">dipdup.config.BigMapIndexConfig,</span> <span class="pre">dipdup.config.HeadIndexConfig,</span> <span class="pre">dipdup.config.TokenTransferIndexConfig,</span> <span class="pre">dipdup.config.IndexTemplateConfig]]</span> <span class="pre">=</span> <span class="pre">&lt;factory&gt;,</span> <span class="pre">templates:</span> <span class="pre">typing.Dict[str,</span> <span class="pre">typing.Union[dipdup.config.OperationIndexConfig,</span> <span class="pre">dipdup.config.BigMapIndexConfig,</span> <span class="pre">dipdup.config.HeadIndexConfig,</span> <span class="pre">dipdup.config.TokenTransferIndexConfig]]</span> <span class="pre">=</span> <span class="pre">&lt;factory&gt;,</span> <span class="pre">jobs:</span> <span class="pre">typing.Dict[str,</span> <span class="pre">dipdup.config.JobConfig]</span> <span class="pre">=</span> <span class="pre">&lt;factory&gt;,</span> <span class="pre">hooks:</span> <span class="pre">typing.Dict[str,</span> <span class="pre">dipdup.config.HookConfig]</span> <span class="pre">=</span> <span class="pre">&lt;factory&gt;,</span> <span class="pre">hasura:</span> <span class="pre">typing.Optional[dipdup.config.HasuraConfig]</span> <span class="pre">=</span> <span class="pre">None,</span> <span class="pre">sentry:</span> <span class="pre">typing.Optional[dipdup.config.SentryConfig]</span> <span class="pre">=</span> <span class="pre">None,</span> <span class="pre">prometheus:</span> <span class="pre">typing.Optional[dipdup.config.PrometheusConfig]</span> <span class="pre">=</span> <span class="pre">None,</span> <span class="pre">advanced:</span> <span class="pre">dipdup.config.AdvancedConfig</span> <span class="pre">=</span> <span class="pre">AdvancedConfig(reindex={},</span> <span class="pre">scheduler=None,</span> <span class="pre">postpone_jobs=False,</span> <span class="pre">early_realtime=False,</span> <span class="pre">merge_subscriptions=False,</span> <span class="pre">metadata_interface=False,</span> <span class="pre">skip_version_check=False),</span> <span class="pre">custom:</span> <span class="pre">typing.Dict[str,</span> <span class="pre">typing.Any]</span> <span class="pre">=</span> <span class="pre">&lt;factory&gt;</span></em><span class="sig-paren">)</span><a class="headerlink" href="#dipdup.config.DipDupConfig" title="Permalink to this definition">¶</a></dt>
115+
<em class="property"><span class="pre">class</span><span class="w"> </span></em><span class="sig-prename descclassname"><span class="pre">dipdup.config.</span></span><span class="sig-name descname"><span class="pre">DipDupConfig</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="pre">spec_version:</span> <span class="pre">str,</span> <span class="pre">package:</span> <span class="pre">str,</span> <span class="pre">datasources:</span> <span class="pre">typing.Dict[str,</span> <span class="pre">typing.Union[dipdup.config.TzktDatasourceConfig,</span> <span class="pre">dipdup.config.CoinbaseDatasourceConfig,</span> <span class="pre">dipdup.config.MetadataDatasourceConfig,</span> <span class="pre">dipdup.config.IpfsDatasourceConfig,</span> <span class="pre">dipdup.config.HttpDatasourceConfig]],</span> <span class="pre">database:</span> <span class="pre">typing.Union[dipdup.config.SqliteDatabaseConfig,</span> <span class="pre">dipdup.config.PostgresDatabaseConfig]</span> <span class="pre">=</span> <span class="pre">SqliteDatabaseConfig(kind='sqlite',</span> <span class="pre">path=':memory:'),</span> <span class="pre">contracts:</span> <span class="pre">typing.Dict[str,</span> <span class="pre">dipdup.config.ContractConfig]</span> <span class="pre">=</span> <span class="pre">&lt;factory&gt;,</span> <span class="pre">indexes:</span> <span class="pre">typing.Dict[str,</span> <span class="pre">typing.Union[dipdup.config.OperationIndexConfig,</span> <span class="pre">dipdup.config.BigMapIndexConfig,</span> <span class="pre">dipdup.config.HeadIndexConfig,</span> <span class="pre">dipdup.config.TokenTransferIndexConfig,</span> <span class="pre">dipdup.config.IndexTemplateConfig]]</span> <span class="pre">=</span> <span class="pre">&lt;factory&gt;,</span> <span class="pre">templates:</span> <span class="pre">typing.Dict[str,</span> <span class="pre">typing.Union[dipdup.config.OperationIndexConfig,</span> <span class="pre">dipdup.config.BigMapIndexConfig,</span> <span class="pre">dipdup.config.HeadIndexConfig,</span> <span class="pre">dipdup.config.TokenTransferIndexConfig]]</span> <span class="pre">=</span> <span class="pre">&lt;factory&gt;,</span> <span class="pre">jobs:</span> <span class="pre">typing.Dict[str,</span> <span class="pre">dipdup.config.JobConfig]</span> <span class="pre">=</span> <span class="pre">&lt;factory&gt;,</span> <span class="pre">hooks:</span> <span class="pre">typing.Dict[str,</span> <span class="pre">dipdup.config.HookConfig]</span> <span class="pre">=</span> <span class="pre">&lt;factory&gt;,</span> <span class="pre">hasura:</span> <span class="pre">typing.Optional[dipdup.config.HasuraConfig]</span> <span class="pre">=</span> <span class="pre">None,</span> <span class="pre">sentry:</span> <span class="pre">typing.Optional[dipdup.config.SentryConfig]</span> <span class="pre">=</span> <span class="pre">None,</span> <span class="pre">prometheus:</span> <span class="pre">typing.Optional[dipdup.config.PrometheusConfig]</span> <span class="pre">=</span> <span class="pre">None,</span> <span class="pre">advanced:</span> <span class="pre">dipdup.config.AdvancedConfig</span> <span class="pre">=</span> <span class="pre">AdvancedConfig(reindex={},</span> <span class="pre">scheduler=None,</span> <span class="pre">postpone_jobs=False,</span> <span class="pre">early_realtime=False,</span> <span class="pre">merge_subscriptions=False,</span> <span class="pre">metadata_interface=False,</span> <span class="pre">skip_version_check=False),</span> <span class="pre">custom:</span> <span class="pre">typing.Dict[str,</span> <span class="pre">typing.Any]</span> <span class="pre">=</span> <span class="pre">&lt;factory&gt;,</span> <span class="pre">logging:</span> <span class="pre">dipdup.enums.LoggingValues</span> <span class="pre">=</span> <span class="pre">LoggingValues.default</span></em><span class="sig-paren">)</span><a class="headerlink" href="#dipdup.config.DipDupConfig" title="Permalink to this definition">¶</a></dt>
116116
<dd><p>Main indexer config</p>
117117
<dl class="field-list simple">
118118
<dt class="field-odd">Parameters</dt>

src/dipdup/cli.py

Lines changed: 32 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,17 @@
5353
_is_shutting_down = False
5454

5555

56+
def set_up_logging() -> None:
57+
root = logging.getLogger()
58+
handler = logging.StreamHandler()
59+
formatter = logging.Formatter('%(levelname)-8s %(name)-20s %(message)s')
60+
handler.setFormatter(formatter)
61+
root.addHandler(handler)
62+
63+
# NOTE: Nothing useful there
64+
logging.getLogger('tortoise').setLevel(logging.WARNING)
65+
66+
5667
def echo(message: str) -> None:
5768
with suppress(BrokenPipeError):
5869
click.echo(message)
@@ -62,7 +73,6 @@ def echo(message: str) -> None:
6273
class CLIContext:
6374
config_paths: List[str]
6475
config: DipDupConfig
65-
logging_config: LoggingConfig
6676

6777

6878
async def _shutdown() -> None:
@@ -159,7 +169,7 @@ async def _check_version() -> None:
159169
default=[DEFAULT_CONFIG_NAME],
160170
)
161171
@click.option('--env-file', '-e', type=str, multiple=True, help='A path to .env file containing `KEY=value` strings.', default=[])
162-
@click.option('--logging-config', '-l', type=str, help='A path to Python logging config in YAML format.', default='logging.yml')
172+
@click.option('--logging-config', '-l', type=str, help='A path to Python logging config in YAML format.', default=None)
163173
@click.pass_context
164174
@cli_wrapper
165175
async def cli(ctx, config: List[str], env_file: List[str], logging_config: str):
@@ -173,18 +183,19 @@ async def cli(ctx, config: List[str], env_file: List[str], logging_config: str):
173183
if '--help' in sys.argv:
174184
return
175185

176-
# NOTE: Search in current workdir, fallback to builtin configs
177-
try:
178-
path = join(os.getcwd(), logging_config)
179-
_logging_config = LoggingConfig.load(path)
180-
except FileNotFoundError:
181-
path = join(dirname(__file__), 'configs', logging_config)
182-
_logging_config = LoggingConfig.load(path)
183-
_logging_config.apply()
186+
set_up_logging()
184187

185-
# NOTE: Nothing useful there
186-
if 'tortoise' not in _logging_config.config['loggers']:
187-
logging.getLogger('tortoise').setLevel(logging.WARNING)
188+
# TODO: Deprecated, remove in 6.0
189+
if logging_config:
190+
_logger.warning('`--logging-config` option is deprecated. Use `logging` config field.')
191+
# NOTE: Search in the current workdir, fallback to builtin configs
192+
try:
193+
path = os.path.join(os.getcwd(), logging_config)
194+
_logging_config = LoggingConfig.load(path)
195+
except FileNotFoundError:
196+
path = os.path.join(os.path.dirname(__file__), 'configs', logging_config)
197+
_logging_config = LoggingConfig.load(path)
198+
_logging_config.apply()
188199

189200
# NOTE: Apply env files before loading config
190201
for env_path in env_file:
@@ -195,13 +206,21 @@ async def cli(ctx, config: List[str], env_file: List[str], logging_config: str):
195206
load_dotenv(env_path, override=True)
196207

197208
_config = DipDupConfig.load(config)
209+
210+
# TODO: Deprecated, remove in 6.0
211+
# NOTE: Skip if Python config is already applied
212+
if not logging_config:
213+
_config.set_up_logging()
214+
198215
# NOTE: Imports will be loaded later if needed
199216
_config.initialize(skip_imports=True)
200217
_init_sentry(_config)
201218

219+
# NOTE: Fire and forget, do not block instant commands
202220
if not _config.advanced.skip_version_check:
203221
asyncio.ensure_future(_check_version())
204222

223+
# NOTE: Avoid import errors if project package is incomplete
205224
try:
206225
await DipDupCodeGenerator(_config, {}).create_package()
207226
except Exception as e:
@@ -217,7 +236,6 @@ async def cli(ctx, config: List[str], env_file: List[str], logging_config: str):
217236
ctx.obj = CLIContext(
218237
config_paths=config,
219238
config=_config,
220-
logging_config=_logging_config,
221239
)
222240

223241

src/dipdup/config.py

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@
4646
from dipdup.datasources.subscription import Subscription
4747
from dipdup.datasources.subscription import TokenTransferSubscription
4848
from dipdup.datasources.subscription import TransactionSubscription
49+
from dipdup.enums import LoggingValues
4950
from dipdup.enums import OperationType
5051
from dipdup.enums import ReindexingAction
5152
from dipdup.enums import ReindexingReason
@@ -1217,6 +1218,7 @@ class DipDupConfig:
12171218
prometheus: Optional[PrometheusConfig] = None
12181219
advanced: AdvancedConfig = AdvancedConfig()
12191220
custom: Dict[str, Any] = field(default_factory=dict)
1221+
logging: LoggingValues = LoggingValues.default
12201222

12211223
def __post_init_post_parse__(self):
12221224
self.paths: List[str] = []
@@ -1340,6 +1342,16 @@ def get_tzkt_datasource(self, name: str) -> TzktDatasourceConfig:
13401342
raise ConfigurationError('`datasource` field must refer to TzKT datasource')
13411343
return datasource
13421344

1345+
def set_up_logging(self) -> None:
1346+
if self.logging == LoggingValues.default:
1347+
pass
1348+
elif self.logging == LoggingValues.quiet:
1349+
logging.getLogger('dipdup').setLevel(logging.WARNING)
1350+
elif self.logging == LoggingValues.verbose:
1351+
logging.getLogger('dipdup').setLevel(logging.DEBUG)
1352+
else:
1353+
raise RuntimeError(f'Unknown `logging` field value: `{self.logging}`')
1354+
13431355
def _import_index(self, index_config: IndexConfigT) -> None:
13441356
_logger.debug('Loading callbacks and typeclasses of index `%s`', index_config.name)
13451357

src/dipdup/enums.py

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,14 @@
11
from enum import Enum
22

33

4+
class LoggingValues(Enum):
5+
"""Enum for `logging` field values."""
6+
7+
default = 'default'
8+
quiet = 'quiet'
9+
verbose = 'verbose'
10+
11+
412
class MessageType(Enum):
513
operation = 'operation'
614
big_map = 'big_map'

tests/integration_tests/hic_et_nunc.yml

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,4 +49,6 @@ indexes:
4949
destination: HEN_minter
5050
entrypoint: collect
5151
first_level: 1365000
52-
last_level: 1366000
52+
last_level: 1366000
53+
54+
logging: quiet

tests/integration_tests/quipuswap.yml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -171,3 +171,5 @@ indexes:
171171
token_contract: hdao_token_mainnet
172172
symbol: hDAO
173173
decimals: 6
174+
175+
logging: quiet

tests/integration_tests/registrydao.yml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,3 +41,5 @@ indexes:
4141
template: registry_dao
4242
values:
4343
contract: registry
44+
45+
logging: quiet

tests/integration_tests/test_demos.py

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -30,8 +30,6 @@ def run_dipdup(self, config: str) -> None:
3030
subprocess.run(
3131
[
3232
'dipdup',
33-
'-l',
34-
'warning.yml',
3533
'-c',
3634
join(dirname(__file__), config),
3735
'run',

tests/integration_tests/tezos_domains.yml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,3 +40,5 @@ indexes:
4040
values:
4141
datasource: tzkt
4242
name_registry: name_registry
43+
44+
logging: quiet

0 commit comments

Comments
 (0)