diff --git a/.github/workflows/license-check.yml b/.github/workflows/license-check.yml index 659f41b1b..d80f7fe22 100644 --- a/.github/workflows/license-check.yml +++ b/.github/workflows/license-check.yml @@ -50,8 +50,11 @@ jobs: with: requirements: "backend/requirements-all.txt" fail: "Copyleft,Other,Error" - exclude: '(envier.*0\.5\.0|psycopg2.*2\.9\.3|fqdn.*1\.5\.1|pyzmq.*25\.1\.2|debugpy.*1\.6\.7|certifi.*2024\.8\.30|tqdm.*4\.67\..*|webencodings.*0\.5\.1|torch.*1\.10\.2.*|torch.*1\.11\.0.*|pytorch-ignite.*0\.4\.10.*|torchaudio.*0\.11\.0.*|torchvision.*0\.12\.0.*|terminado.*0\.15\.0|qudida.*0\.0\.4|expiringdict.*1\.2\.2|botocore.*1\.29\.80|orderedmultidict.*1\.0\.1|deepchecks.*)' + exclude: '(category_encoders.*2\.7\..*|attrs.*25\.1\..*|referencing.*0\.36\..*|envier.*0\.5\.0|psycopg2.*2\.9\.3|fqdn.*1\.5\.1|pyzmq.*25\.1\.2|debugpy.*1\.6\.7|certifi.*2025\.1\.31|tqdm.*4\.67\..*|webencodings.*0\.5\.1|torch.*1\.10\.2.*|torch.*1\.11\.0.*|pytorch-ignite.*0\.4\.10.*|torchaudio.*0\.11\.0.*|torchvision.*0\.12\.0.*|terminado.*0\.15\.0|qudida.*0\.0\.4|expiringdict.*1\.2\.2|botocore.*1\.29\.80|orderedmultidict.*1\.0\.1|deepchecks.*)' # psycopg2 is LGPL 2 + # category_encoders is BSD https://github.com/scikit-learn-contrib/category_encoders/tree/master?tab=BSD-3-Clause-1-ov-file + # attrs is MIT https://github.com/python-attrs/attrs/blob/main/LICENSE + # referencing is MIT https://github.com/python-jsonschema/referencing?tab=MIT-1-ov-file # pyzmq is Revised BSD https://github.com/zeromq/pyzmq/blob/main/examples/LICENSE # debugpy is MIT https://github.com/microsoft/debugpy/blob/main/LICENSE # certifi is MPL-2.0 https://github.com/certifi/python-certifi/blob/master/LICENSE diff --git a/backend/deepchecks_monitoring/bgtasks/alert_task.py b/backend/deepchecks_monitoring/bgtasks/alert_task.py index 9f2b2a250..396e75950 100644 --- a/backend/deepchecks_monitoring/bgtasks/alert_task.py +++ b/backend/deepchecks_monitoring/bgtasks/alert_task.py @@ -112,7 +112,7 @@ async def execute_monitor( logger: t.Optional[logging.Logger] = None, ) -> t.List[Alert]: """Execute monitor alert rules.""" - logger = logger or logging.getLogger('monitor-executor') + logger = logger or configure_logger('monitor-executor') logger.info('Execution of Monitor(id:%s) for timestamp %s', monitor_id, timestamp) monitor = t.cast(Monitor, await session.scalar( diff --git a/backend/deepchecks_monitoring/bgtasks/scheduler.py b/backend/deepchecks_monitoring/bgtasks/scheduler.py index 7cedb3b31..cc2738f55 100644 --- a/backend/deepchecks_monitoring/bgtasks/scheduler.py +++ b/backend/deepchecks_monitoring/bgtasks/scheduler.py @@ -67,7 +67,7 @@ def __init__( self.engine = engine self.async_session_factory = sessionmaker(engine, class_=AsyncSession, expire_on_commit=False) self.sleep_seconds = sleep_seconds - self.logger = logger or logging.getLogger('alerts-scheduler') + self.logger = logger or configure_logger('alerts-scheduler') async def run(self): """Start alert scheduler.""" diff --git a/backend/deepchecks_monitoring/ee/integrations/slack.py b/backend/deepchecks_monitoring/ee/integrations/slack.py index ecb61c669..be29c3c96 100644 --- a/backend/deepchecks_monitoring/ee/integrations/slack.py +++ b/backend/deepchecks_monitoring/ee/integrations/slack.py @@ -14,6 +14,7 @@ from deepchecks_monitoring.ee.config import SlackSettings from deepchecks_monitoring.monitoring_utils import CheckParameterTypeEnum as CheckParameterKind from deepchecks_monitoring.monitoring_utils import MonitorCheckConfSchema as MonitorConfig +from deepchecks_monitoring.monitoring_utils import configure_logger from deepchecks_monitoring.schema_models import Alert, AlertRule, AlertSeverity, Check, Model, Monitor from deepchecks_monitoring.utils.alerts import prepare_alert_link from deepchecks_monitoring.utils.text import format_float @@ -84,7 +85,7 @@ def __init__( ): self.settings = settings self.client = SlackClient() - self.logger = logger or logging.getLogger("slack.installation") + self.logger = logger or configure_logger("slack.installation") self.state_utils = OAuthStateUtils() def generate_authorization_url(self, state, redirect_path) -> str: diff --git a/backend/deepchecks_monitoring/ee/middlewares.py b/backend/deepchecks_monitoring/ee/middlewares.py index 2cd132883..d81feb1ee 100644 --- a/backend/deepchecks_monitoring/ee/middlewares.py +++ b/backend/deepchecks_monitoring/ee/middlewares.py @@ -19,6 +19,7 @@ from starlette.types import ASGIApp, Message, Receive, Scope, Send from deepchecks_monitoring.exceptions import LicenseError +from deepchecks_monitoring.monitoring_utils import configure_logger from deepchecks_monitoring.public_models import User from deepchecks_monitoring.utils.auth import CurrentUser @@ -82,7 +83,7 @@ def __init__( log_stream_name=log_stream_name, ) h.setLevel(logging.INFO) - self.logger = logging.getLogger("access-audit") + self.logger = configure_logger("access-audit") self.logger.addHandler(h) self.app = app diff --git a/backend/deepchecks_monitoring/ee/resources.py b/backend/deepchecks_monitoring/ee/resources.py index 45ebf0e27..0181a0218 100644 --- a/backend/deepchecks_monitoring/ee/resources.py +++ b/backend/deepchecks_monitoring/ee/resources.py @@ -18,6 +18,7 @@ from deepchecks_monitoring.ee.notifications import AlertNotificator as EEAlertNotificator from deepchecks_monitoring.features_control import FeaturesControl from deepchecks_monitoring.integrations.email import EmailSender +from deepchecks_monitoring.monitoring_utils import configure_logger from deepchecks_monitoring.public_models import User from deepchecks_monitoring.resources import ResourcesProvider as OpenSourceResourcesProvider @@ -27,6 +28,8 @@ __all__ = ["ResourcesProvider"] +logger: logging.Logger = configure_logger("server") + class ResourcesProvider(OpenSourceResourcesProvider): """Provider of resources.""" @@ -64,7 +67,7 @@ def get_features_control(self, user: User) -> FeaturesControl: def parallel_check_executors_pool(self) -> "ActorPool | None": parallel_check_executor_flag = self.settings.parallel_check_executor_flag - logging.getLogger("server").info({ + logger.info({ "mesage": f"'parallelCheckExecutorEnabled' is set to {parallel_check_executor_flag}" }) if parallel_check_executor_flag: diff --git a/backend/deepchecks_monitoring/integrations/email.py b/backend/deepchecks_monitoring/integrations/email.py index be642e26c..7032f7592 100644 --- a/backend/deepchecks_monitoring/integrations/email.py +++ b/backend/deepchecks_monitoring/integrations/email.py @@ -21,12 +21,13 @@ from typing_extensions import Self from deepchecks_monitoring.config import EmailSettings +from deepchecks_monitoring.monitoring_utils import configure_logger # TODO: change this TEMPLATES_DIR = pathlib.Path(__file__).absolute().parent.parent / "templates" templates = Jinja2Templates(directory=str(TEMPLATES_DIR)) -logger: logging.Logger = logging.getLogger(__name__) +logger: logging.Logger = configure_logger(__name__) class Recepient(BaseModel): diff --git a/backend/deepchecks_monitoring/logic/model_logic.py b/backend/deepchecks_monitoring/logic/model_logic.py index b8b417111..9963532fe 100644 --- a/backend/deepchecks_monitoring/logic/model_logic.py +++ b/backend/deepchecks_monitoring/logic/model_logic.py @@ -22,7 +22,8 @@ from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.orm import selectinload -from deepchecks_monitoring.monitoring_utils import CheckParameterTypeEnum, MonitorCheckConfSchema, fetch_or_404 +from deepchecks_monitoring.monitoring_utils import (CheckParameterTypeEnum, MonitorCheckConfSchema, configure_logger, + fetch_or_404) from deepchecks_monitoring.schema_models import Check, Model, ModelVersion from deepchecks_monitoring.schema_models.column_type import (SAMPLE_LABEL_COL, SAMPLE_PRED_COL, SAMPLE_PRED_PROBA_COL, SAMPLE_TS_COL, ColumnType) @@ -33,6 +34,8 @@ DEFAULT_N_SAMPLES = 5000 +logger: logging.Logger = configure_logger('monitor_run_logger') + async def get_model_versions_for_time_range(session: AsyncSession, model_id: int, @@ -317,7 +320,7 @@ def run_deepchecks( if not (msg := getattr(e, 'message', None)) else msg ) - logging.getLogger('monitor_run_logger').exception( + logger.exception( 'For model(id=%s) version(id=%s) check(%s) ' 'got exception: %s', model.id, @@ -372,7 +375,7 @@ async def get_results_for_model_versions_for_reference( except errors.DeepchecksBaseError as e: message = f'For model(id={model.id}) version(id={model_version.id}) check({dp_check.name()}) ' \ f'got exception: {e.message}' - logging.getLogger('monitor_run_logger').error(message) + logger.error(message) curr_result = None reduced_outs.append({'result': curr_result}) diff --git a/backend/deepchecks_monitoring/notifications.py b/backend/deepchecks_monitoring/notifications.py index f386f180d..03e864dfb 100644 --- a/backend/deepchecks_monitoring/notifications.py +++ b/backend/deepchecks_monitoring/notifications.py @@ -19,6 +19,7 @@ from typing_extensions import Self from deepchecks_monitoring import __version__ +from deepchecks_monitoring.monitoring_utils import configure_logger from deepchecks_monitoring.public_models import Organization, User from deepchecks_monitoring.schema_models import Check, Model from deepchecks_monitoring.schema_models.alert import Alert @@ -85,7 +86,7 @@ def __init__( self.alert_rule = alert_rule self.session = session self.resources_provider = resources_provider - self.logger = logger or logging.getLogger("alert-notificator") + self.logger = logger or configure_logger("alert-notificator") async def send_emails(self) -> bool: """Send notification emails.""" diff --git a/backend/deepchecks_monitoring/resources.py b/backend/deepchecks_monitoring/resources.py index 1faa98e92..88919e50f 100644 --- a/backend/deepchecks_monitoring/resources.py +++ b/backend/deepchecks_monitoring/resources.py @@ -32,7 +32,7 @@ from deepchecks_monitoring.features_control import FeaturesControl from deepchecks_monitoring.integrations.email import EmailSender from deepchecks_monitoring.logic.cache_functions import CacheFunctions -from deepchecks_monitoring.monitoring_utils import ExtendedAsyncSession, json_dumps +from deepchecks_monitoring.monitoring_utils import ExtendedAsyncSession, configure_logger, json_dumps from deepchecks_monitoring.notifications import AlertNotificator from deepchecks_monitoring.public_models import Organization from deepchecks_monitoring.public_models.user import User @@ -42,6 +42,8 @@ __all__ = ["ResourcesProvider"] +logger: logging.Logger = configure_logger("server") + class BaseResourcesProvider: """Base class for all resources provides.""" @@ -348,11 +350,11 @@ def parallel_check_executors_pool(self): import ray # noqa from ray.util.actor_pool import ActorPool # noqa except ImportError: - logging.getLogger("server").info({"message": "Ray is not installed"}) + logger.info({"message": "Ray is not installed"}) return if not ray.is_initialized(): - logging.getLogger("server").info({ + logger.info({ "message": "Ray is not initialized" }) return @@ -389,7 +391,7 @@ def shutdown_parallel_check_executors_pool(self): wait=tenacity.wait_fixed(1), retry=tenacity.retry_if_exception_type(KafkaError), reraise=True, - before_sleep=tenacity.before_sleep_log(logging.getLogger("server"), logging.WARNING), + before_sleep=tenacity.before_sleep_log(logger, logging.WARNING), ) def ensure_kafka_topic(self, topic_name, num_partitions=1) -> bool: """Ensure that kafka topic exist. If not, creating it. @@ -462,14 +464,14 @@ def _get_mixpanel_event_reporter(self) -> MixpanelEventReporter | None: if mixpanel is not None: return mixpanel if self.settings.enable_analytics is False: - logging.getLogger("server").warning({"message": "Analytics gathering is disabled"}) + logger.warning({"message": "Analytics gathering is disabled"}) return if token := self.settings.mixpanel_id: mixpanel = MixpanelEventReporter.from_token(token) self._mixpanel_event_reporter = mixpanel return mixpanel - logging.getLogger("server").warning({"message": "Mixpanel token is not provided"}) + logger.warning({"message": "Mixpanel token is not provided"}) def get_features_control(self, user: User) -> FeaturesControl: # pylint: disable=unused-argument """Return features control.""" diff --git a/backend/deepchecks_monitoring/schema_models/alert_webhook.py b/backend/deepchecks_monitoring/schema_models/alert_webhook.py index ec0cefc63..4145b6981 100644 --- a/backend/deepchecks_monitoring/schema_models/alert_webhook.py +++ b/backend/deepchecks_monitoring/schema_models/alert_webhook.py @@ -11,7 +11,7 @@ from pydantic import AnyUrl, BaseModel, validator from sqlalchemy.dialects.postgresql import JSONB -from deepchecks_monitoring.monitoring_utils import MetadataMixin +from deepchecks_monitoring.monitoring_utils import MetadataMixin, configure_logger from deepchecks_monitoring.schema_models import Base from deepchecks_monitoring.utils.alerts import AlertSeverity @@ -157,7 +157,7 @@ async def execute( True - if webhook executes succesfully False - if webhook execution fails or request returns not successful response status """ - logger = logger or logging.getLogger(f"webhook-{self.name}") + logger = logger or configure_logger(f"webhook-{self.name}") if alert.alert_rule.alert_severity not in self.notification_levels: logger.info( diff --git a/backend/deepchecks_monitoring/utils/database.py b/backend/deepchecks_monitoring/utils/database.py index 029b23b8d..2d398f571 100644 --- a/backend/deepchecks_monitoring/utils/database.py +++ b/backend/deepchecks_monitoring/utils/database.py @@ -13,6 +13,8 @@ from sqlalchemy.orm import Session from sqlalchemy.schema import CreateSchema, DDLElement +from deepchecks_monitoring.monitoring_utils import configure_logger + __all__ = ["SchemaBuilder", "attach_schema_switcher_listener", "attach_schema_switcher", "sqlalchemy_exception_to_asyncpg_exception"] @@ -139,7 +141,7 @@ def __init__( self.name = name self.metadata = metadata self.migrations_location = migrations_location - self.logger = logger or logging.getLogger("schema-builder") + self.logger = logger or configure_logger("schema-builder") async def create(self, engine: AsyncEngine): """Create schema.""" diff --git a/backend/deepchecks_monitoring/utils/mixpanel.py b/backend/deepchecks_monitoring/utils/mixpanel.py index e77a6cda3..f2470b4ca 100644 --- a/backend/deepchecks_monitoring/utils/mixpanel.py +++ b/backend/deepchecks_monitoring/utils/mixpanel.py @@ -13,7 +13,7 @@ import deepchecks_monitoring from deepchecks_monitoring.config import Settings -from deepchecks_monitoring.monitoring_utils import OperatorsEnum +from deepchecks_monitoring.monitoring_utils import OperatorsEnum, configure_logger from deepchecks_monitoring.public_models.organization import OrgTier from deepchecks_monitoring.public_models.user import User from deepchecks_monitoring.schema_models import Alert, AlertRule, Check, Model, ModelVersion, Monitor @@ -655,7 +655,7 @@ def __init__( ): self.mixpanel = mixpanel self.supress_errors = supress_errors - self.logger = logger or logging.getLogger(type(self).__name__) + self.logger = logger or configure_logger(type(self).__name__) def report( self, diff --git a/backend/requirements.txt b/backend/requirements.txt index b2c7e80b4..df06960f7 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -18,7 +18,7 @@ confluent-kafka==2.3.0 kafka-python==2.0.2 uvloop==0.17.0 nbformat>=5.4.0,<6 -deepchecks@git+https://github.com/deepchecks/deepchecks.git@main +deepchecks@git+https://github.com/deepchecks/deepchecks.git@8c15865aaebc7f73faeb0939d07fa982d646b2c4 redis[hiredis]~=4.6.0 pandas~=2.1.0 pyjwt[crypto]==2.4.0 @@ -38,4 +38,5 @@ python-json-logger~=2.0.7 mixpanel==4.10.0 typing_extensions~=4.12.0 tenacity~=8.5.0 -pyarrow==14.0.2 \ No newline at end of file +pyarrow==14.0.2 +plotly>=5.13.1,<6 \ No newline at end of file