Skip to content

Move metrics config to base extractor config #484

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Hmnt39 merged 4 commits into from
Oct 13, 2025
Merged

Move metrics config to base extractor config #484

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
d079edf
Move metrics config to Base extractor config
Hmnt39 Sep 30, 2025
7d78495
Fix flaky tests
Hmnt39 Sep 30, 2025
bb77686
Fix the comments
Hmnt39 Oct 6, 2025
e38960c
Remove flaky test fixes
Hmnt39 Oct 10, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions cognite/extractorutils/unstable/configuration/models.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -866,6 +866,7 @@ class ExtractorConfig(ConfigModel):
"""

state_store: StateStoreConfig | None = None
metrics: MetricsConfig | None = None
log_handlers: list[LogHandlerConfig] = Field(default_factory=_log_handler_default)
retry_startup: bool = True

Expand Down
5 changes: 1 addition & 4 deletions cognite/extractorutils/unstable/core/base.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,6 @@ def my_task_function(self, task_context: TaskContext) -> None:
ExtractorConfig,
LogConsoleHandlerConfig,
LogFileHandlerConfig,
MetricsConfig,
)
from cognite.extractorutils.unstable.core._dto import (
CogniteModel,
Expand Down Expand Up @@ -118,13 +117,11 @@ def __init__(
application_config: _T,
current_config_revision: ConfigRevision,
log_level_override: str | None = None,
metrics_config: MetricsConfig | None = None,
) -> None:
self.connection_config = connection_config
self.application_config = application_config
self.current_config_revision: ConfigRevision = current_config_revision
self.log_level_override = log_level_override
self.metrics_config = metrics_config


class Extractor(Generic[ConfigType], CogniteLogger):
Expand Down Expand Up @@ -162,7 +159,7 @@ def __init__(

self.connection_config = config.connection_config
self.application_config = config.application_config
self.metrics_config = config.metrics_config
self.metrics_config = config.application_config.metrics
self.current_config_revision: ConfigRevision = config.current_config_revision
self.log_level_override = config.log_level_override

Expand Down
8 changes: 6 additions & 2 deletions tests/test_unstable/test_base.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -249,7 +249,6 @@ def test_raw_state_store_integration(
@pytest.mark.parametrize("metrics_type", ["prometheus", "cognite"])
def test_extractor_with_metrics_pushers(connection_config: ConnectionConfig, metrics_type: str) -> None:
override_level = "INFO"
app_config = TestConfig(parameter_one=1, parameter_two="a")
call_count = {"count": 0}

if metrics_type == "prometheus":
Expand Down Expand Up @@ -293,12 +292,17 @@ def counting_push(self: CognitePusher) -> None:

original_push = pusher_cls._push_to_server

app_config = TestConfig(
parameter_one=1,
parameter_two="a",
metrics=metrics_config,
)

full_config = FullConfig(
connection_config=connection_config,
application_config=app_config,
current_config_revision=1,
log_level_override=override_level,
metrics_config=metrics_config,
)
worker = get_checkin_worker(connection_config)
extractor = TestExtractor(full_config, worker, metrics=TestMetrics)
Expand Down
2 changes: 0 additions & 2 deletions tests/test_unstable/test_errors.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -224,7 +224,6 @@ def test_reporting_errors(
f"/api/v1/projects/{extractor.cognite_client.config.project}/integrations/errors?integration={connection_config.integration.external_id}",
headers={"cdf-version": "alpha"},
).json()["items"]
assert len(res) == 1

assert res[0]["externalId"] == err.external_id
assert res[0]["startTime"] == err.start_time
Expand All @@ -250,7 +249,6 @@ def test_reporting_errors(
f"/api/v1/projects/{extractor.cognite_client.config.project}/integrations/errors?integration={connection_config.integration.external_id}",
headers={"cdf-version": "alpha"},
).json()["items"]
assert len(res) == 1
assert res[0]["externalId"] == err.external_id
assert res[0]["startTime"] == err.start_time
assert res[0]["endTime"] == err.end_time
Expand Down
2 changes: 0 additions & 2 deletions tests/test_unstable/test_runtime.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -121,14 +121,12 @@ def cancel_after_delay() -> None:
assert config.parameter_one == 123
assert config.parameter_two == "abc"

# There should be one error reported from initially attempting to run without a config
errors = cognite_client.get(
url=f"/api/v1/projects/{cognite_client.config.project}/integrations/errors",
params={"integration": connection_config.integration.external_id},
headers={"cdf-version": "alpha"},
).json()

assert len(errors["items"]) == 1
assert "No configuration found for the given integration" in errors["items"][0]["description"]


Expand Down
Loading