Latest learnings from running the migration logic

Author: matteius

migrate_collections.py

@@ -7,20 +7,26 @@
 # Access the database
 db = get_open_sensor_db()
 
-collections_to_migrate = ["Temperature", "Humidity", "Pressure", "Lux", "CO2", "PH", "Moisture"]
+collections_to_migrate = ["Temperature", "Humidity", "Pressure", "Lux", "CO2", "pH", "Moisture"]
 
 migration = db.Migration.find_one({"migration_name": "FreeTier"})
 if not migration:
     db["Migration"].insert_one({"migration_name": "FreeTier", "migration_complete": False})
 
 earliest_timestamp = datetime(2023, 1, 1)
 start_date = earliest_timestamp
-one_day = timedelta(days=1)  # Change to one day
+one_day = timedelta(days=1)
+
+
+# Function to create a composite key
+def create_key(timestamp, metadata):
+    return f"{timestamp}_{metadata['device_id']}_{metadata.get('name', 'NA')}_{metadata.get('user_id', 'NA')}"
+
 
 while start_date <= datetime(2023, 11, 10):
-    end_date = start_date + one_day  # Use one day
+    end_date = start_date + one_day
     buffer = {}
-    timestamps_set = set()  # For faster timestamp lookups
+
     print(start_date, end_date)
 
     for collection_name in collections_to_migrate:
@@ -39,28 +45,24 @@
             if unit:
                 new_document[f"{new_collections[collection_name]}_unit"] = unit
 
-            found = False
-            for existing_timestamp in timestamps_set:
-                if abs(existing_timestamp - document["timestamp"]) <= timedelta(seconds=3):
-                    buffer[existing_timestamp][new_collections[collection_name]] = document.get(
-                        old_collections[collection_name]
-                    )
-                    if unit:
-                        buffer[existing_timestamp][
-                            f"{new_collections[collection_name]}_unit"
-                        ] = unit
-                    found = True
-                    break
-
-            if not found:
-                buffer[document["timestamp"]] = new_document
-                timestamps_set.add(document["timestamp"])
+            key = create_key(document["timestamp"], document["metadata"])
+
+            if key in buffer:
+                buffer[key][new_collections[collection_name]] = document.get(
+                    old_collections[collection_name]
+                )
+                if unit:
+                    buffer[key][f"{new_collections[collection_name]}_unit"] = unit
+            else:
+                buffer[key] = new_document
 
     all_documents = sorted(buffer.values(), key=itemgetter("timestamp"))
 
-    if all_documents:  # Only insert if there are documents
+    # Insert the batch of documents for the current day
+    if all_documents:
         db["FreeTier"].insert_many(all_documents)
 
+    # Move to the next day
     start_date = end_date
 
 db["Migration"].update_one({"migration_name": "FreeTier"}, {"$set": {"migration_complete": True}})