Merge pull request #6582 from opsmill/ajtm-06012025-diff-save-query-update

update cypher query to use less memory

Author: ajtmccarty

backend/infrahub/core/diff/query/save.py

@@ -80,6 +80,7 @@ def __init__(self, node_create_batch: list[EnrichedNodeCreateRequest], **kwargs:
 
     async def query_init(self, db: InfrahubDatabase, **kwargs: Any) -> None:  # noqa: ARG002
         self.params = self._build_node_batch_params()
+
         query = """
 UNWIND $node_details_list AS node_details
 WITH
@@ -136,65 +137,82 @@ async def query_init(self, db: InfrahubDatabase, **kwargs: Any) -> None:  # noqa
     OPTIONAL MATCH (diff_node)-[:DIFF_HAS_CONFLICT]->(node_conflict:DiffConflict)
     SET node_conflict = node_conflict_params
 }
+// -------------------------
+// resetting the UNWIND and starting over here reduces memory usage
+// -------------------------
+WITH root_uuid LIMIT 1
+UNWIND $node_details_list AS node_details
+WITH
+    node_details.root_uuid AS root_uuid,
+    node_details.node_map AS node_map,
+    toString(node_details.node_map.node_properties.uuid) AS node_uuid,
+    node_details.node_map.node_properties.db_id AS node_db_id
+MATCH (:DiffRoot {uuid: root_uuid})-[:DIFF_HAS_NODE]->(diff_node:DiffNode {uuid: node_uuid, db_id: node_db_id})
+WITH diff_node, node_map, %(attr_name_list_comp)s AS attr_names
+OPTIONAL MATCH (diff_node)-[:DIFF_HAS_ATTRIBUTE]->(attr_to_delete:DiffAttribute)
+WHERE NOT (attr_to_delete.name IN attr_names)
+OPTIONAL MATCH (attr_to_delete)-[*..6]->(next_to_delete)
+DETACH DELETE next_to_delete
+DETACH DELETE attr_to_delete
+// -------------------------
+// add attributes for this node
+// -------------------------
+WITH DISTINCT diff_node, node_map
 CALL {
-    // -------------------------
-    // remove stale attributes for this node
-    // -------------------------
     WITH diff_node, node_map
-    CALL {
-        WITH diff_node, node_map
-        WITH diff_node, %(attr_name_list_comp)s AS attr_names
-        OPTIONAL MATCH (diff_node)-[:DIFF_HAS_ATTRIBUTE]->(attr_to_delete:DiffAttribute)
-        WHERE NOT (attr_to_delete.name IN attr_names)
-        OPTIONAL MATCH (attr_to_delete)-[*..6]->(next_to_delete)
-        DETACH DELETE next_to_delete
-        DETACH DELETE attr_to_delete
-    }
-    // -------------------------
-    // add attributes for this node
-    // -------------------------
     UNWIND node_map.attributes AS node_attribute
     MERGE (diff_node)-[:DIFF_HAS_ATTRIBUTE]->(diff_attribute:DiffAttribute {name: node_attribute.node_properties.name})
     SET diff_attribute = node_attribute.node_properties
     // -------------------------
-    // add properties for this attribute
+    // remove stale properties for this attribute
     // -------------------------
-    WITH diff_attribute, node_attribute
+    WITH diff_attribute, node_attribute, %(attr_props_list_comp)s AS prop_types
+    OPTIONAL MATCH (diff_attribute)-[:DIFF_HAS_PROPERTY]->(prop_to_delete:DiffProperty)
+    WHERE NOT (prop_to_delete.property_type IN prop_types)
+    OPTIONAL MATCH (prop_to_delete)-[*..4]->(next_to_delete)
+    DETACH DELETE next_to_delete
+    DETACH DELETE prop_to_delete
     // -------------------------
-    // remove stale properties for this attribute
+    // set attribute property values
     // -------------------------
-    CALL {
-        WITH diff_attribute, node_attribute
-        WITH diff_attribute, %(attr_props_list_comp)s AS prop_types
-        OPTIONAL MATCH (diff_attribute)-[:DIFF_HAS_PROPERTY]->(prop_to_delete:DiffProperty)
-        WHERE NOT (prop_to_delete.property_type IN prop_types)
-        OPTIONAL MATCH (prop_to_delete)-[*..4]->(next_to_delete)
-        DETACH DELETE next_to_delete
-        DETACH DELETE prop_to_delete
-    }
+    WITH DISTINCT diff_attribute, node_attribute
     UNWIND node_attribute.properties AS attr_property
     MERGE (diff_attribute)-[:DIFF_HAS_PROPERTY]->(diff_attr_prop:DiffProperty {property_type: attr_property.node_properties.property_type})
     SET diff_attr_prop = attr_property.node_properties
-    // -------------------------
-    // add/remove conflict for this property
-    // -------------------------
     WITH diff_attr_prop, attr_property
     OPTIONAL MATCH (diff_attr_prop)-[:DIFF_HAS_CONFLICT]->(current_attr_prop_conflict:DiffConflict)
     WITH diff_attr_prop, attr_property, current_attr_prop_conflict, (attr_property.conflict_params IS NOT NULL) AS has_prop_conflict
-    FOREACH (i in CASE WHEN has_prop_conflict = FALSE THEN [1] ELSE [] END |
+    CALL {
+        WITH has_prop_conflict, current_attr_prop_conflict
+        WITH has_prop_conflict, current_attr_prop_conflict
+        WHERE has_prop_conflict = FALSE AND current_attr_prop_conflict IS NOT NULL
         DETACH DELETE current_attr_prop_conflict
-    )
-    FOREACH (i in CASE WHEN has_prop_conflict = TRUE THEN [1] ELSE [] END |
+    }
+    CALL {
+        WITH has_prop_conflict, diff_attr_prop, attr_property
+        WITH has_prop_conflict, diff_attr_prop, attr_property
+        WHERE has_prop_conflict = TRUE
         MERGE (diff_attr_prop)-[:DIFF_HAS_CONFLICT]->(diff_attr_prop_conflict:DiffConflict)
         SET diff_attr_prop_conflict = attr_property.conflict_params
-    )
+    }
 }
 // -------------------------
+// resetting the UNWIND and starting over here reduces memory usage
+// -------------------------
+WITH 1 AS resetting LIMIT 1
+UNWIND $node_details_list AS node_details
+WITH
+    node_details.root_uuid AS root_uuid,
+    node_details.node_map AS node_map,
+    toString(node_details.node_map.node_properties.uuid) AS node_uuid,
+    node_details.node_map.node_properties.db_id AS node_db_id
+MATCH (:DiffRoot {uuid: root_uuid})-[:DIFF_HAS_NODE]->(diff_node:DiffNode {uuid: node_uuid, db_id: node_db_id})
+// -------------------------
 // remove stale relationships for this node
 // -------------------------
 CALL {
     WITH diff_node, node_map
-    WITH diff_node, %(rel_name_list_comp)s AS rel_names
+    WITH diff_node, node_map, %(rel_name_list_comp)s AS rel_names
     OPTIONAL MATCH (diff_node)-[:DIFF_HAS_RELATIONSHIP]->(rel_to_delete:DiffRelationship)
     WHERE NOT (rel_to_delete.name IN rel_names)
     OPTIONAL MATCH (rel_to_delete)-[*..8]->(next_to_delete)
@@ -214,7 +232,7 @@ async def query_init(self, db: InfrahubDatabase, **kwargs: Any) -> None:  # noqa
 WITH diff_relationship, node_relationship
 CALL {
     WITH diff_relationship, node_relationship
-    WITH diff_relationship, %(rel_peers_list_comp)s AS rel_peers
+    WITH diff_relationship, node_relationship, %(rel_peers_list_comp)s AS rel_peers
     OPTIONAL MATCH (diff_relationship)-[:DIFF_HAS_ELEMENT]->(element_to_delete:DiffRelationshipElement)
     WHERE NOT (element_to_delete.peer_id IN rel_peers)
     OPTIONAL MATCH (element_to_delete)-[*..6]->(next_to_delete)
@@ -236,20 +254,26 @@ async def query_init(self, db: InfrahubDatabase, **kwargs: Any) -> None:  # noqa
 OPTIONAL MATCH (diff_relationship_element)-[:DIFF_HAS_CONFLICT]->(current_element_conflict:DiffConflict)
 WITH diff_relationship_element, node_single_relationship, current_element_conflict,
     (node_single_relationship.conflict_params IS NOT NULL) AS has_element_conflict
-FOREACH (i in CASE WHEN has_element_conflict = FALSE THEN [1] ELSE [] END |
+CALL {
+    WITH has_element_conflict, current_element_conflict
+    WITH has_element_conflict, current_element_conflict
+    WHERE has_element_conflict = FALSE
     DETACH DELETE current_element_conflict
-)
-FOREACH (i in CASE WHEN has_element_conflict = TRUE THEN [1] ELSE [] END |
+}
+CALL {
+    WITH has_element_conflict, diff_relationship_element, node_single_relationship
+    WITH has_element_conflict, diff_relationship_element, node_single_relationship
+    WHERE has_element_conflict = TRUE
     MERGE (diff_relationship_element)-[:DIFF_HAS_CONFLICT]->(element_conflict:DiffConflict)
     SET element_conflict = node_single_relationship.conflict_params
-)
+}
 // -------------------------
 // remove stale properties for this relationship element
 // -------------------------
 WITH diff_relationship_element, node_single_relationship
 CALL {
     WITH diff_relationship_element, node_single_relationship
-    WITH diff_relationship_element, %(element_props_list_comp)s AS element_props
+    WITH diff_relationship_element, node_single_relationship, %(element_props_list_comp)s AS element_props
     OPTIONAL MATCH (diff_relationship_element)-[:DIFF_HAS_PROPERTY]->(property_to_delete:DiffProperty)
     WHERE NOT (property_to_delete.property_type IN element_props)
     OPTIONAL MATCH (property_to_delete)-[*..4]->(next_to_delete)
@@ -271,13 +295,19 @@ async def query_init(self, db: InfrahubDatabase, **kwargs: Any) -> None:  # noqa
 OPTIONAL MATCH (diff_relationship_property)-[:DIFF_HAS_CONFLICT]->(diff_relationship_property_conflict:DiffConflict)
 WITH diff_relationship_property, node_relationship_property, diff_relationship_property_conflict,
     (node_relationship_property.conflict_params IS NOT NULL) AS has_property_conflict
-FOREACH (i in CASE WHEN has_property_conflict = FALSE THEN [1] ELSE [] END |
+CALL {
+    WITH has_property_conflict, diff_relationship_property_conflict
+    WITH has_property_conflict, diff_relationship_property_conflict
+    WHERE has_property_conflict = FALSE
     DETACH DELETE diff_relationship_property_conflict
-)
-FOREACH (i in CASE WHEN has_property_conflict = TRUE THEN [1] ELSE [] END |
+}
+CALL {
+    WITH has_property_conflict, diff_relationship_property, node_relationship_property
+    WITH has_property_conflict, diff_relationship_property, node_relationship_property
+    WHERE has_property_conflict = TRUE
     MERGE (diff_relationship_property)-[:DIFF_HAS_CONFLICT]->(property_conflict:DiffConflict)
     SET property_conflict = node_relationship_property.conflict_params
-)
+}
         """ % {
             "attr_name_list_comp": db.render_list_comprehension(
                 items="node_map.attributes", item_name="node_properties.name"

changelog/6568.fixed.md

@@ -0,0 +1 @@
+Update the cypher query that saves a diff to use less memory.