handle diff summary counts at the database level (#5712)

* WIP move diff summary counts enrichment to a query

* move diff repository tests around

* finish query and tests

* make sure counts are not null when retrieving

* fix flaky test

* replace summary enrichment with query

* update repository tests for counts

Author: ajtmccarty

backend/infrahub/core/diff/combiner.py

@@ -342,6 +342,8 @@ def _combine_relationships(
 
     def _copy_node_without_parents(self, node: EnrichedDiffNode) -> EnrichedDiffNode:
         rels_without_parents = {replace(r, nodes=set()) for r in node.relationships}
+        for rel in rels_without_parents:
+            rel.reset_summaries()
         node_without_parents = replace(node, relationships=rels_without_parents)
         return deepcopy(node_without_parents)
 
@@ -351,15 +353,11 @@ def _combine_nodes(self, node_pairs: list[NodePair]) -> set[EnrichedDiffNode]:
             if node_pair.earlier is None:
                 if node_pair.later is not None:
                     copied = self._copy_node_without_parents(node_pair.later)
-                    for rel in copied.relationships:
-                        rel.reset_summaries()
                     combined_nodes.add(copied)
                 continue
             if node_pair.later is None:
                 if node_pair.earlier is not None:
                     copied = self._copy_node_without_parents(node_pair.earlier)
-                    for rel in copied.relationships:
-                        rel.reset_summaries()
                     combined_nodes.add(copied)
                 continue
             combined_attributes = self._combine_attributes(

backend/infrahub/core/diff/coordinator.py

@@ -30,7 +30,6 @@
     from .data_check_synchronizer import DiffDataCheckSynchronizer
     from .enricher.aggregated import AggregatedDiffEnricher
     from .enricher.labels import DiffLabelsEnricher
-    from .enricher.summary_counts import DiffSummaryCountsEnricher
     from .repository.repository import DiffRepository
 
 
@@ -66,7 +65,6 @@ def __init__(
         diff_combiner: DiffCombiner,
         conflicts_enricher: ConflictsEnricher,
         labels_enricher: DiffLabelsEnricher,
-        summary_counts_enricher: DiffSummaryCountsEnricher,
         data_check_synchronizer: DiffDataCheckSynchronizer,
         conflict_transferer: DiffConflictTransferer,
     ) -> None:
@@ -76,7 +74,6 @@ def __init__(
         self.diff_combiner = diff_combiner
         self.conflicts_enricher = conflicts_enricher
         self.labels_enricher = labels_enricher
-        self.summary_counts_enricher = summary_counts_enricher
         self.data_check_synchronizer = data_check_synchronizer
         self.conflict_transferer = conflict_transferer
         self.lock_registry = lock.registry
@@ -152,10 +149,6 @@ async def update_branch_diff(self, base_branch: Branch, diff_branch: Branch) ->
                 tracking_id=tracking_id,
                 force_branch_refresh=False,
             )
-            if isinstance(enriched_diffs, EnrichedDiffs):
-                await self.summary_counts_enricher.enrich(enriched_diff_root=enriched_diffs.base_branch_diff)
-                await self.summary_counts_enricher.enrich(enriched_diff_root=enriched_diffs.diff_branch_diff)
-
             await self.diff_repo.save(enriched_diffs=enriched_diffs)
             await self._update_core_data_checks(enriched_diff=enriched_diffs.diff_branch_diff)
             log.info(f"Branch diff update complete for {base_branch.name} - {diff_branch.name}")
@@ -183,9 +176,6 @@ async def create_or_update_arbitrary_timeframe_diff(
                 tracking_id=tracking_id,
                 force_branch_refresh=False,
             )
-            if isinstance(enriched_diffs, EnrichedDiffs):
-                await self.summary_counts_enricher.enrich(enriched_diff_root=enriched_diffs.base_branch_diff)
-                await self.summary_counts_enricher.enrich(enriched_diff_root=enriched_diffs.diff_branch_diff)
 
             await self.diff_repo.save(enriched_diffs=enriched_diffs)
             await self._update_core_data_checks(enriched_diff=enriched_diffs.diff_branch_diff)
@@ -228,8 +218,6 @@ async def recalculate(
                     earlier=current_branch_diff, later=enriched_diffs.diff_branch_diff
                 )
 
-            await self.summary_counts_enricher.enrich(enriched_diff_root=enriched_diffs.base_branch_diff)
-            await self.summary_counts_enricher.enrich(enriched_diff_root=enriched_diffs.diff_branch_diff)
             await self.diff_repo.save(enriched_diffs=enriched_diffs)
             await self._update_core_data_checks(enriched_diff=enriched_diffs.diff_branch_diff)
             log.info(f"Diff recalculation complete for {base_branch.name} - {diff_branch.name}")

backend/infrahub/core/diff/enricher/summary_counts.py

@@ -37,11 +37,6 @@ async def query_init(self, db: InfrahubDatabase, **kwargs: Any) -> None:
     SET diff_root.diff_branch = diff_root_map.diff_branch
     SET diff_root.from_time = diff_root_map.from_time
     SET diff_root.to_time = diff_root_map.to_time
-    SET diff_root.num_added = diff_root_map.num_added
-    SET diff_root.num_updated = diff_root_map.num_updated
-    SET diff_root.num_removed = diff_root_map.num_removed
-    SET diff_root.num_conflicts = diff_root_map.num_conflicts
-    SET diff_root.contains_conflict = diff_root_map.contains_conflict
     SET diff_root.tracking_id = diff_root_map.tracking_id
     RETURN diff_root
 }
@@ -68,11 +63,6 @@ def _build_diff_root_params(self, enriched_diffs: EnrichedDiffs | EnrichedDiffsM
                     "to_time": enriched_diff.to_time.to_string(),
                     "uuid": enriched_diff.uuid,
                     "tracking_id": enriched_diff.tracking_id.serialize() if enriched_diff.tracking_id else None,
-                    "num_added": enriched_diff.num_added,
-                    "num_updated": enriched_diff.num_updated,
-                    "num_removed": enriched_diff.num_removed,
-                    "num_conflicts": enriched_diff.num_conflicts,
-                    "contains_conflict": enriched_diff.contains_conflict,
                 }
             )
         return {"diff_root_list": diff_root_list}
@@ -96,7 +86,12 @@ async def query_init(self, db: InfrahubDatabase, **kwargs: Any) -> None:
     WITH root_uuid, node_map
     MATCH (diff_root {uuid: root_uuid})
     MERGE (diff_root)-[:DIFF_HAS_NODE]->(diff_node:DiffNode {uuid: node_map.node_properties.uuid})
-    SET diff_node = node_map.node_properties
+    SET
+        diff_node.kind = node_map.node_properties.kind,
+        diff_node.label = node_map.node_properties.label,
+        diff_node.changed_at = node_map.node_properties.changed_at,
+        diff_node.action = node_map.node_properties.action,
+        diff_node.path_identifier = node_map.node_properties.path_identifier
     // -------------------------
     // add/remove node-level conflict
     // -------------------------
@@ -324,11 +319,6 @@ def _build_diff_attribute_params(self, enriched_attribute: EnrichedDiffAttribute
                 "changed_at": enriched_attribute.changed_at.to_string(),
                 "action": enriched_attribute.action.value,
                 "path_identifier": enriched_attribute.path_identifier,
-                "num_added": enriched_attribute.num_added,
-                "num_updated": enriched_attribute.num_updated,
-                "num_removed": enriched_attribute.num_removed,
-                "num_conflicts": enriched_attribute.num_conflicts,
-                "contains_conflict": enriched_attribute.contains_conflict,
             },
             "properties": property_props,
         }
@@ -349,11 +339,6 @@ def _build_diff_single_relationship_params(
                 "peer_id": enriched_single_relationship.peer_id,
                 "peer_label": enriched_single_relationship.peer_label,
                 "path_identifier": enriched_single_relationship.path_identifier,
-                "num_added": enriched_single_relationship.num_added,
-                "num_updated": enriched_single_relationship.num_updated,
-                "num_removed": enriched_single_relationship.num_removed,
-                "num_conflicts": enriched_single_relationship.num_conflicts,
-                "contains_conflict": enriched_single_relationship.contains_conflict,
             },
             "conflict_params": conflict_params,
             "properties": property_props,
@@ -375,11 +360,6 @@ def _build_diff_relationship_params(self, enriched_relationship: EnrichedDiffRel
                 else None,
                 "action": enriched_relationship.action,
                 "path_identifier": enriched_relationship.path_identifier,
-                "num_added": enriched_relationship.num_added,
-                "num_updated": enriched_relationship.num_updated,
-                "num_removed": enriched_relationship.num_removed,
-                "num_conflicts": enriched_relationship.num_conflicts,
-                "contains_conflict": enriched_relationship.contains_conflict,
             },
             "relationships": single_relationship_props,
         }
@@ -402,11 +382,6 @@ def _build_diff_node_params(self, enriched_node: EnrichedDiffNode) -> dict[str,
                 "changed_at": enriched_node.changed_at.to_string() if enriched_node.changed_at else None,
                 "action": enriched_node.action.value,
                 "path_identifier": enriched_node.path_identifier,
-                "num_added": enriched_node.num_added,
-                "num_updated": enriched_node.num_updated,
-                "num_removed": enriched_node.num_removed,
-                "num_conflicts": enriched_node.num_conflicts,
-                "contains_conflict": enriched_node.contains_conflict,
             },
             "conflict_params": conflict_params,
             "attributes": attribute_props,