diff --git a/reconcile/change_owners/change_log_tracking.py b/reconcile/change_owners/change_log_tracking.py index 8cde17b65f..14bc1965b8 100644 --- a/reconcile/change_owners/change_log_tracking.py +++ b/reconcile/change_owners/change_log_tracking.py @@ -24,7 +24,14 @@ from reconcile.typed_queries.apps import get_apps from reconcile.typed_queries.jenkins import get_jenkins_configs from reconcile.typed_queries.namespaces import get_namespaces -from reconcile.utils import gql +from reconcile.change_owners.metrics import ( + ChangeLogAppChange, + ChangeLogChangeType, + ChangeLogCommitProcessed, + ChangeLogItemsGauge, + ChangeLogProcessingError, +) +from reconcile.utils import gql, metrics from reconcile.utils.defer import defer from reconcile.utils.runtime.integration import ( PydanticRunParams, @@ -35,6 +42,12 @@ QONTRACT_INTEGRATION = "change-log-tracking" BUNDLE_DIFFS_OBJ = "bundle-diffs.json" DEFAULT_MERGE_COMMIT_PREFIX = "Merge branch '" +# Label-based filtered outputs: {label_key: (output_filename, virtual_app_name)} +# virtual_app_name is injected into each item's apps list so the +# progressive-delivery plugin can match a single aggregate component. +LABEL_FILTERED_OUTPUTS: dict[str, tuple[str, str]] = { + "rosa": ("bundle-diffs-rosa.json", "rosa-platform"), +} class ChangeLogItem(BaseModel): @@ -239,5 +252,66 @@ def run( change_log.items = sorted( change_log.items, key=lambda i: i.merged_at, reverse=True ) + + # build label filter sets for metrics and filtered outputs + labeled_app_sets: dict[str, set[str]] = {} + for label_key in LABEL_FILTERED_OUTPUTS: + labeled_app_sets[label_key] = { + a.name + for a in apps + if a.labels and a.labels.get(label_key) == "true" + } + + # emit metrics + metrics.set_gauge( + ChangeLogItemsGauge(label_filter="all"), + len(change_log.items), + ) + for item in change_log.items: + metrics.inc_counter(ChangeLogCommitProcessed()) + if item.error: + metrics.inc_counter(ChangeLogProcessingError()) + for label_key, app_names in labeled_app_sets.items(): + matches = any(app in app_names for app in item.apps) + if matches: + for app in item.apps: + if app in app_names: + metrics.inc_counter(ChangeLogAppChange( + app=app, label_filter=label_key, + )) + for ct in item.change_types: + metrics.inc_counter(ChangeLogChangeType( + change_type=ct, label_filter=label_key, + )) + + for label_key, app_names in labeled_app_sets.items(): + filtered_count = sum( + 1 for item in change_log.items + if any(app in app_names for app in item.apps) + ) + metrics.set_gauge( + ChangeLogItemsGauge(label_filter=label_key), + filtered_count, + ) + if not dry_run: integration_state.add(BUNDLE_DIFFS_OBJ, change_log.model_dump(), force=True) + + # produce label-filtered changelog outputs + for label_key, (output_file, virtual_app) in LABEL_FILTERED_OUTPUTS.items(): + label_app_names = labeled_app_sets[label_key] + filtered_items = [] + for item in change_log.items: + if any(app in label_app_names for app in item.apps): + enriched = item.model_copy( + update={"apps": sorted({*item.apps, virtual_app})} + ) + filtered_items.append(enriched) + filtered_log = ChangeLog(items=filtered_items) + logging.info( + f"label={label_key}: {len(filtered_items)} items, " + f"apps: {filtered_log.apps}" + ) + integration_state.add( + output_file, filtered_log.model_dump(), force=True + ) diff --git a/reconcile/change_owners/metrics.py b/reconcile/change_owners/metrics.py new file mode 100644 index 0000000000..f83abd91f3 --- /dev/null +++ b/reconcile/change_owners/metrics.py @@ -0,0 +1,60 @@ +from pydantic import BaseModel + +from reconcile.utils.metrics import ( + CounterMetric, + GaugeMetric, +) + + +class ChangeLogBaseMetric(BaseModel): + "Base class for change-log-tracking metrics" + + integration: str = "change-log-tracking" + + +class ChangeLogCommitProcessed(ChangeLogBaseMetric, CounterMetric): + "Number of commits processed by change-log-tracking" + + @classmethod + def name(cls) -> str: + return "change_log_commit_processed_total" + + +class ChangeLogAppChange(ChangeLogBaseMetric, CounterMetric): + "Number of app-interface changes per app, labeled by product filter" + + app: str + label_filter: str + + @classmethod + def name(cls) -> str: + return "change_log_app_change_total" + + +class ChangeLogChangeType(ChangeLogBaseMetric, CounterMetric): + "Number of changes by change type, labeled by product filter" + + change_type: str + label_filter: str + + @classmethod + def name(cls) -> str: + return "change_log_change_type_total" + + +class ChangeLogProcessingError(ChangeLogBaseMetric, CounterMetric): + "Number of commit processing errors" + + @classmethod + def name(cls) -> str: + return "change_log_processing_error_total" + + +class ChangeLogItemsGauge(ChangeLogBaseMetric, GaugeMetric): + "Total number of changelog items" + + label_filter: str + + @classmethod + def name(cls) -> str: + return "change_log_items" diff --git a/reconcile/gql_definitions/common/apps.gql b/reconcile/gql_definitions/common/apps.gql index 0ea50c3f55..3d264665d5 100644 --- a/reconcile/gql_definitions/common/apps.gql +++ b/reconcile/gql_definitions/common/apps.gql @@ -4,6 +4,7 @@ query Apps { apps: apps_v1 { path name + labels parentApp { path name diff --git a/reconcile/gql_definitions/common/apps.py b/reconcile/gql_definitions/common/apps.py index 5d0faf5dad..6c8448cc96 100644 --- a/reconcile/gql_definitions/common/apps.py +++ b/reconcile/gql_definitions/common/apps.py @@ -23,6 +23,7 @@ apps: apps_v1 { path name + labels parentApp { path name @@ -46,6 +47,7 @@ class AppV1_AppV1(ConfiguredBaseModel): class AppV1(ConfiguredBaseModel): path: str = Field(..., alias="path") name: str = Field(..., alias="name") + labels: Optional[Json] = Field(..., alias="labels") parent_app: Optional[AppV1_AppV1] = Field(..., alias="parentApp") diff --git a/reconcile/test/change_owners/test_change_log_tracking.py b/reconcile/test/change_owners/test_change_log_tracking.py index cf4b412730..c63f4318d9 100644 --- a/reconcile/test/change_owners/test_change_log_tracking.py +++ b/reconcile/test/change_owners/test_change_log_tracking.py @@ -10,7 +10,10 @@ ) from pytest_mock import MockerFixture +from unittest.mock import call + from reconcile.change_owners.change_log_tracking import ( + LABEL_FILTERED_OUTPUTS, ChangeLog, ChangeLogIntegration, ChangeLogIntegrationParams, @@ -139,8 +142,77 @@ def test_change_log_tracking_with_deleted_app( integration.run(dry_run=False) - mocks["state"].add.assert_called_once_with( - "bundle-diffs.json", - expected_change_log.model_dump(), + expected_calls = [ + call("bundle-diffs.json", expected_change_log.model_dump(), force=True), + ] + # label-filtered outputs produce empty changelogs when no apps match + for output_file, _virtual_app in LABEL_FILTERED_OUTPUTS.values(): + expected_calls.append( + call(output_file, ChangeLog(items=[]).model_dump(), force=True) + ) + mocks["state"].add.assert_has_calls(expected_calls) + + +def test_change_log_tracking_rosa_label_filter( + mocker: MockerFixture, + gql_api_builder: Callable[..., GqlApi], + gql_class_factory: Callable[..., ChangeTypesQueryData], +) -> None: + rosa_app = AppV1( + path="/services/rosa-app/app.yml", + name="rosa-app", + labels='{"service": "rosa-app", "rosa": "true"}', + parentApp=None, + ) + non_rosa_app = AppV1( + path="/services/other/app.yml", + name="other-app", + labels='{"service": "other"}', + parentApp=None, + ) + mocks = setup_mocks( + mocker, + gql_api_builder, + gql_class_factory, + apps=[rosa_app, non_rosa_app], + datafiles={ + "/services/rosa-app/namespaces/ns.yml": { + "datafilepath": "/services/rosa-app/namespaces/ns.yml", + "datafileschema": "/openshift/namespace-1.yml", + "old": { + "$schema": "/openshift/namespace-1.yml", + "app": {"name": "rosa-app"}, + }, + } + }, + commit_message="Merge branch 'dev' into 'master'\n\nupdate rosa config", + ) + + integration = ChangeLogIntegration( + ChangeLogIntegrationParams( + gitlab_project_id="test", + process_existing=True, + commit=None, + ) + ) + + integration.run(dry_run=False) + + # the rosa-filtered output should contain the item with virtual app injected + rosa_call = call( + "bundle-diffs-rosa.json", + ChangeLog( + items=[ + ChangeLogItem( + apps=sorted(["rosa-app", "rosa-platform"]), + change_types=[], + commit=COMMIT_SHA, + description="update rosa config", + error=False, + merged_at=MERGED_AT, + ), + ] + ).model_dump(), force=True, ) + assert rosa_call in mocks["state"].add.call_args_list