Speed up the loading of the entities by using set and also reduce complexity with supplied known hash set (#1316) (#1320)

* Speed up the loading of the entities by using set and also reduce complecity with supplied known hash set

* Remove description

* Ready

* Fix lint

Author: benflexcompute

flow360/component/simulation/entity_info.py

@@ -142,8 +142,9 @@ def group_in_registry(
         Group items with given attribute_name.
         """
         entity_list = self._get_list_of_entities(attribute_name, entity_type_name)
+        known_frozen_hashes = set()
         for item in entity_list:
-            registry.register(item)
+            known_frozen_hashes = registry.fast_register(item, known_frozen_hashes)
         return registry
 
     def _get_list_of_entities(
@@ -516,14 +517,16 @@ def get_registry(self, internal_registry, **_) -> EntityRegistry:
             internal_registry = EntityRegistry()
 
             # Populate boundaries
+            known_frozen_hashes = set()
             # pylint: disable=not-an-iterable
             for boundary in self.boundaries:
-                internal_registry.register(boundary)
+                known_frozen_hashes = internal_registry.fast_register(boundary, known_frozen_hashes)
 
             # Populate zones
             # pylint: disable=not-an-iterable
+            known_frozen_hashes = set()
             for zone in self.zones:
-                internal_registry.register(zone)
+                known_frozen_hashes = internal_registry.fast_register(zone, known_frozen_hashes)
 
         return internal_registry
 
@@ -552,10 +555,11 @@ def get_registry(self, internal_registry, **_) -> EntityRegistry:
         if internal_registry is None:
             # Initialize the local registry
             internal_registry = EntityRegistry()
+            known_frozen_hashes = set()
             # Populate boundaries
             # pylint: disable=not-an-iterable
             for boundary in self.boundaries:
-                internal_registry.register(boundary)
+                known_frozen_hashes = internal_registry.fast_register(boundary, known_frozen_hashes)
             return internal_registry
         return internal_registry
 

flow360/component/simulation/framework/entity_base.py

@@ -3,6 +3,7 @@
 from __future__ import annotations
 
 import copy
+import hashlib
 import uuid
 from abc import ABCMeta
 from collections import defaultdict
@@ -45,6 +46,11 @@ class EntityBase(Flow360BaseModel, metaclass=ABCMeta):
 
     name: str = pd.Field(frozen=True)
 
+    # Whether the entity is dirty and needs to be re-hashed
+    _dirty: bool = pd.PrivateAttr(True)
+    # Cached hash of the entity
+    _hash_cache: str = pd.PrivateAttr(None)
+
     def __init__(self, **data):
         """
         Initializes a new entity and registers it in the global registry.
@@ -110,9 +116,34 @@ def entity_type(self, value: str):
     def __str__(self) -> str:
         return "\n".join([f"        {attr}: {value}" for attr, value in self.__dict__.items()])
 
+    def _recompute_hash(self):
+        new_hash = hashlib.sha256(self.model_dump_json().encode("utf-8")).hexdigest()
+        # Can further speed up 10% by using `object.__setattr__`
+        self._hash_cache = new_hash
+        self._dirty = False
+        return new_hash
+
     def _get_hash(self):
         """hash generator to identify if two entities are the same"""
-        return hash(self.model_dump_json(exclude="private_attribute_id"))
+        # Can further speed up 10% by using `object.__getattribute__`
+        dirty = self._dirty
+        cache = self._hash_cache
+        if dirty or cache is None:
+            return self._recompute_hash()
+        return cache
+
+    def __setattr__(self, name, value):
+        """
+        [Large model performance]
+        Wrapping the __setattr__ to mark the entity as dirty when the attribute is not private
+        This enables caching the hash of the entity to avoid re-calculating the hash when the entity is not changed.
+        """
+
+        super().__setattr__(name, value)
+        if not name.startswith("_") and not self._dirty:
+            # Not using self to avoid invoking
+            # Can further speed up 10% by using `object.__setattr__`
+            self._dirty = True
 
     @property
     def id(self) -> str:
@@ -338,8 +369,8 @@ def _get_expanded_entities(
     # pylint: disable=arguments-differ
     def preprocess(self, **kwargs):
         """
-        Expand and overwrite self.stored_entities in preparation for submissin/serialization.
-        Should only be called as late as possible to incoperate all possible changes.
+        Expand and overwrite self.stored_entities in preparation for submission/serialization.
+        Should only be called as late as possible to incorporate all possible changes.
         """
         # WARNING: this is very expensive all for long lists as it is quadratic
         self.stored_entities = self._get_expanded_entities(create_hard_copy=False)

flow360/component/simulation/framework/entity_registry.py

@@ -43,6 +43,32 @@ class EntityRegistry(Flow360BaseModel):
 
     internal_registry: Dict[str, list[Any]] = pd.Field({})
 
+    def fast_register(self, entity: EntityBase, known_frozen_hashes: set[str]) -> set[str]:
+        """
+        Registers an entity in the registry under its type. Suitable for registering a large number of entities.
+
+        Parameters:
+            entity (EntityBase): The entity instance to register.
+            known_frozen_hashes (Optional[set[str]]): A set of hashes of frozen entities.
+                This is used to speed up checking if the has is already in the registry by avoiding O(N^2) complexity.
+                This can be provided when registering a large number of entities.
+
+        Returns:
+            known_frozen_hashes (set[str])
+        """
+        if entity.entity_bucket not in self.internal_registry:
+            # pylint: disable=unsupported-assignment-operation
+            self.internal_registry[entity.entity_bucket] = []
+
+        # pylint: disable=protected-access
+        if entity._get_hash() in known_frozen_hashes:
+            return known_frozen_hashes
+        known_frozen_hashes.add(entity._get_hash())
+
+        # pylint: disable=unsubscriptable-object
+        self.internal_registry[entity.entity_bucket].append(entity)
+        return known_frozen_hashes
+
     def register(self, entity: EntityBase):
         """
         Registers an entity in the registry under its type.

flow360/component/simulation/framework/param_utils.py

@@ -132,15 +132,16 @@ def register_entity_list(model: Flow360BaseModel, registry: EntityRegistry) -> N
     Returns:
         None
     """
+    known_frozen_hashes = set()
     for field in model.__dict__.values():
         if isinstance(field, EntityBase):
-            registry.register(field)
+            known_frozen_hashes = registry.fast_register(field, known_frozen_hashes)
 
         if isinstance(field, EntityList):
             # pylint: disable=protected-access
             expanded_entities = field._get_expanded_entities(create_hard_copy=False)
             for entity in expanded_entities if expanded_entities else []:
-                registry.register(entity)
+                known_frozen_hashes = registry.fast_register(entity, known_frozen_hashes)
 
         elif isinstance(field, (list, tuple)):
             for item in field: