Merge pull request #611 from aperture-data/release-0.4.50

Release 0.4.50

Author: gsaluja9

aperturedb/ParallelLoader.py

@@ -120,9 +120,8 @@ def get_existing_indices(self):
 
     def query_setup(self, generator: Subscriptable) -> None:
         """
-        Runs the setup for the loader, which includes creating indices
-        Currently, it only creates indices for the properties that are
-        also used for constraint.
+        Runs the setup for the loader, which includes creating indices.
+        Currently, it only creates indices for the properties that are used for constraint.
 
         Will only run when the argument generator has a get_indices method that returns
         a dictionary of the form:
@@ -150,17 +149,20 @@ def query_setup(self, generator: Subscriptable) -> None:
             schema = self.utils.get_schema()
 
             indices_needed = generator.get_indices()
-            for schema_type, schema_type_plural in [("entity", "entities"), ("connection", "connections")]:
-                for entity_class in indices_needed.get(schema_type, {}):
-                    for property_name in indices_needed[schema_type][entity_class]:
-                        schema_type = schema.get(schema_type_plural, {}) or {}
-                        if property_name not in schema_type.get('classes', {}).get(entity_class, {}).get('properties', {}):
-                            if not self.utils.create_entity_index(entity_class, property_name):
-                                logger.warning(
-                                    f"Failed to create index for {entity_class}.{property_name}")
-                        else:
-                            logger.info(
-                                f"Index for {entity_class}.{property_name} already exists")
+
+            # Create indexes for entities
+            for entity_class in indices_needed.get("entity", {}):
+                for property_name in indices_needed["entity"].get(entity_class, []):
+                    if not self.utils.create_entity_index(entity_class, property_name):
+                        logger.warning(
+                            f"Failed to create index for {entity_class}.{property_name}")
+
+            # Create indexes for connections
+            for connection_class in indices_needed.get("connection", {}):
+                for property_name in indices_needed["connection"].get(connection_class, []):
+                    if not self.utils.create_connection_index(connection_class, property_name):
+                        logger.warning(
+                            f"Failed to create index for {connection_class}.{property_name}")
 
     def ingest(self, generator: Subscriptable, batchsize: int = 1, numthreads: int = 4, stats: bool = False) -> None:
         """

aperturedb/__init__.py

@@ -10,7 +10,7 @@
 import signal
 import sys
 
-__version__ = "0.4.49"
+__version__ = "0.4.50"
 
 logger = logging.getLogger(__name__)
 

test/conftest.py

@@ -129,8 +129,8 @@ def insert_data_from_csv(in_csv_file, rec_count=-1, expected_error_count=0, load
             data = data[:rec_count]
 
         loader = ParallelLoader(db)
-        loader.ingest(data, batchsize=99,
-                      numthreads=8,
+        loader.ingest(data, batchsize=503,
+                      numthreads=4,
                       stats=True,
                       )