diff --git a/elementary/cli/cli.py b/elementary/cli/cli.py index e3cf2f0c4..c3f509f20 100644 --- a/elementary/cli/cli.py +++ b/elementary/cli/cli.py @@ -29,6 +29,13 @@ def get_log_path(ctx): return os.path.join(target_path, "edr.log") +def get_quiet_logs(ctx): + try: + return "--quiet-logs" in ctx.args + except (ValueError, AttributeError): + return False + + class ElementaryCLI(click.MultiCommand): _CMD_MAP = { "monitor": monitor, @@ -57,12 +64,14 @@ def format_help(self, ctx, formatter): def invoke(self, ctx: click.Context) -> Any: files_target_path = get_log_path(ctx) - set_root_logger_handlers("elementary", files_target_path) - click.echo( - "Any feedback and suggestions are welcomed! join our community here - " - "https://bit.ly/slack-elementary\n" - ) - logger.info(f"Running with edr={package.get_package_version()}") + quiet_logs = get_quiet_logs(ctx) + set_root_logger_handlers("elementary", files_target_path, quiet_logs=quiet_logs) + if not quiet_logs: + click.echo( + "Any feedback and suggestions are welcomed! join our community here - " + "https://bit.ly/slack-elementary\n" + ) + logger.info(f"Running with edr={package.get_package_version()}") return super().invoke(ctx) diff --git a/elementary/config/config.py b/elementary/config/config.py index b2faf90e5..924c3c319 100644 --- a/elementary/config/config.py +++ b/elementary/config/config.py @@ -75,6 +75,7 @@ def __init__( env: str = DEFAULT_ENV, run_dbt_deps_if_needed: Optional[bool] = None, project_name: Optional[str] = None, + quiet_logs: Optional[bool] = None, ): self.config_dir = config_dir self.profiles_dir = profiles_dir diff --git a/elementary/monitor/cli.py b/elementary/monitor/cli.py index 2b786786e..9b47133ff 100644 --- a/elementary/monitor/cli.py +++ b/elementary/monitor/cli.py @@ -149,6 +149,12 @@ def decorator(func): else "DEPRECATED! Please use --filters instead! - Filter the alerts by tags: / owners: / models: / " "statuses: / resource_types:.", )(func) + func = click.option( + "--quiet-logs", + is_flag=True, + default=False, + help="Minimize INFO level logs. Only WARNING and above will be shown.", + )(func) return func return decorator @@ -324,12 +330,14 @@ def monitor( excludes, teams_webhook, maximum_columns_in_alert_samples, + quiet_logs, ): """ Get alerts on failures in dbt jobs. """ if ctx.invoked_subcommand is not None: return + if deprecated_slack_webhook is not None: click.secho( '\n"-s" is deprecated and won\'t be supported in the near future.\n' @@ -356,6 +364,7 @@ def monitor( report_url=report_url, teams_webhook=teams_webhook, maximum_columns_in_alert_samples=maximum_columns_in_alert_samples, + quiet_logs=quiet_logs, ) anonymous_tracking = AnonymousCommandLineTracking(config) anonymous_tracking.set_env("use_select", bool(select)) @@ -450,6 +459,7 @@ def report( env, select, target_path, + quiet_logs, ): """ Generate a local observability report of your warehouse. @@ -463,6 +473,7 @@ def report( target_path, dbt_quoting=dbt_quoting, env=env, + quiet_logs=quiet_logs, ) anonymous_tracking = AnonymousCommandLineTracking(config) anonymous_tracking.set_env("use_select", bool(select)) @@ -680,6 +691,7 @@ def send_report( disable, include, target_path, + quiet_logs, ): """ Generate and send the report to an external platform. @@ -722,6 +734,7 @@ def send_report( report_url=report_url, env=env, project_name=project_name, + quiet_logs=quiet_logs, ) anonymous_tracking = AnonymousCommandLineTracking(config) anonymous_tracking.set_env("use_select", bool(select)) diff --git a/elementary/monitor/fetchers/alerts/alerts.py b/elementary/monitor/fetchers/alerts/alerts.py index e14768095..008b36980 100644 --- a/elementary/monitor/fetchers/alerts/alerts.py +++ b/elementary/monitor/fetchers/alerts/alerts.py @@ -1,6 +1,8 @@ import json from typing import Dict, List, Optional +import click + from elementary.clients.dbt.base_dbt_runner import BaseDbtRunner from elementary.clients.fetcher.fetcher import FetcherClient from elementary.config.config import Config @@ -29,7 +31,7 @@ def skip_alerts( ): alert_ids = [alert.id for alert in alerts_to_skip] alert_ids_chunks = self._split_list_to_chunks(alert_ids) - logger.info("Update skipped alerts") + click.echo(f"Update skipped alerts ({len(alerts_to_skip)})") for alert_ids_chunk in alert_ids_chunks: self.dbt_runner.run( select="elementary_cli.update_alerts.update_skipped_alerts", @@ -60,7 +62,7 @@ def query_last_alert_times( def update_sent_alerts(self, alert_ids: List[str]) -> None: alert_ids_chunks = self._split_list_to_chunks(alert_ids) - logger.info("Update sent alerts") + click.echo(f"Update sent alerts ({len(alert_ids)})") for alert_ids_chunk in alert_ids_chunks: self.dbt_runner.run( select="elementary_cli.update_alerts.update_sent_alerts", diff --git a/elementary/utils/log.py b/elementary/utils/log.py index 3d28f9ce0..3d1397616 100644 --- a/elementary/utils/log.py +++ b/elementary/utils/log.py @@ -30,10 +30,15 @@ def format(self, record): ROTATION_BACKUP_COUNT = 4 -def get_console_handler(): +def get_console_handler(quiet_logs: bool = False): console_handler = logging.StreamHandler(sys.stdout) console_handler.setFormatter(FORMATTER) - console_handler.setLevel(logging.DEBUG if is_debug() else logging.INFO) + if is_debug(): + console_handler.setLevel(logging.DEBUG) + elif quiet_logs: + console_handler.setLevel(logging.WARNING) + else: + console_handler.setLevel(logging.INFO) return console_handler @@ -55,7 +60,14 @@ def get_logger(logger_name): return logger -def set_root_logger_handlers(logger_name, files_target_path): +def set_root_logger_handlers(logger_name, files_target_path, quiet_logs: bool = False): logger = logging.getLogger(logger_name) - logger.addHandler(get_console_handler()) + + # Disable propagation to root logger to avoid duplicate logs + logger.propagate = False + + logger.addHandler(get_console_handler(quiet_logs=quiet_logs)) logger.addHandler(get_file_handler(files_target_path)) + + # Set logger level to DEBUG so it doesn't filter messages (handler will filter) + logger.setLevel(logging.DEBUG)