Improve Error logging (#502)

Author: Hmnt39

CHANGELOG.md

@@ -13,6 +13,12 @@ Changes are grouped as follows
 - `Security` in case of vulnerabilities.
 
 
+## 7.11.4
+
+### Fixed
+* In the `unstable` package: Improved error logging and error messages when config revision changes are detected
+
+
 ## 7.11.3
 
 ### Fixed

cognite/extractorutils/__init__.py

@@ -16,7 +16,7 @@
 Cognite extractor utils is a Python package that simplifies the development of new extractors.
 """
 
-__version__ = "7.11.3"
+__version__ = "7.11.4"
 from .base import Extractor
 
 __all__ = ["Extractor"]

cognite/extractorutils/unstable/core/base.py

@@ -427,9 +427,9 @@ def run_task(task_context: TaskContext) -> None:
                 target(task_context)
 
             except Exception as e:
-                # Task crashed, record it as a fatal error
+                # Task failed, record it as a fatal error
                 task_context.exception(
-                    f"Task {task.name} crashed unexpectedly",
+                    f"Task {task.name} failed unexpectedly",
                     e,
                     level=ErrorLevel.fatal,
                 )

cognite/extractorutils/unstable/core/checkin_worker.py

@@ -195,14 +195,14 @@ def _handle_checkin_response(self, response: JSONType) -> None:
                     "and configured to use remote config for the new config to take effect.",
                 )
             elif self._active_revision < checkin_response.last_config_revision:
-                self._active_revision = checkin_response.last_config_revision
                 if self._on_revision_change is not None:
                     self._logger.info(
                         "Remote config revision changed %s -> %s. The extractor will now use the new configuration.",
                         self._active_revision,
                         checkin_response.last_config_revision,
                     )
                     self._on_revision_change(checkin_response.last_config_revision)
+                self._active_revision = checkin_response.last_config_revision
 
     def flush(self, cancellation_token: CancellationToken) -> None:
         """

cognite/extractorutils/unstable/core/runtime.py

@@ -337,6 +337,7 @@ def _safe_get_application_config(
                     # Same error as before, no need to log it again
                     self._cancellation_token.wait(randint(1, self.RETRY_CONFIG_INTERVAL))
                     continue
+                self.logger.error(error_message)
                 prev_error = error_message
 
                 ts = now()

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "cognite-extractor-utils"
-version = "7.11.3"
+version = "7.11.4"
 description = "Utilities for easier development of extractors for CDF"
 authors = [
     {name = "Mathias Lohne", email = "mathias.lohne@cognite.com"}

tests/test_unstable/test_errors.py

@@ -177,7 +177,7 @@ def task(_tc: TaskContext) -> None:
     assert len(worker._errors) == 1
 
     error = next(iter(worker._errors.values()))
-    assert error.description == "Task TestTask crashed unexpectedly"
+    assert error.description == "Task TestTask failed unexpectedly"
     assert error.level == ErrorLevel.fatal
 
     # Make sure error was recorded as a task error