Merge pull request #498 from hotosm/refactor-processing

fix: copying file within s3 bucket

Author: nrjadkry

src/backend/app/migrations/versions/e23c05f21542_change_task_events_state.py

@@ -55,17 +55,17 @@ def upgrade():
     # Step 2: Add a new column with the new enum type
     op.add_column("task_events", sa.Column("new_state", new_state_enum, nullable=True))
 
-    # Step 3: Populate the new state column with the transformed data
-    op.execute(
-        """
-        UPDATE task_events
-        SET new_state =
-            CASE
-                WHEN state = 'IMAGE_PROCESSED' THEN 'IMAGE_PROCESSING_FINISHED'
-                ELSE state::text
-            END::state_new
-        """
-    )
+    # # Step 3: Populate the new state column with the transformed data
+    # op.execute(
+    #     """
+    #     UPDATE task_events
+    #     SET new_state =
+    #         CASE
+    #             WHEN state = 'IMAGE_PROCESSED' THEN 'IMAGE_PROCESSING_FINISHED'
+    #             ELSE state::text
+    #         END::state_new
+    #     """
+    # )
 
     # Step 4: Drop the old state column
     op.drop_column("task_events", "state")
@@ -110,17 +110,17 @@ def downgrade():
     # Step 3: Add the old state column with the old enum type
     op.add_column("task_events", sa.Column("state_old", old_state_enum, nullable=True))
 
-    # Step 4: Populate the old state column with the transformed data
-    op.execute(
-        """
-        UPDATE task_events
-        SET state_old =
-            CASE
-                WHEN state = 'IMAGE_PROCESSING_FINISHED' THEN 'IMAGE_PROCESSED'
-                ELSE state::text
-            END::state
-        """
-    )
+    # # Step 4: Populate the old state column with the transformed data
+    # op.execute(
+    #     """
+    #     UPDATE task_events
+    #     SET state_old =
+    #         CASE
+    #             WHEN state = 'IMAGE_PROCESSING_FINISHED' THEN 'IMAGE_PROCESSED'
+    #             ELSE state::text
+    #         END::state
+    #     """
+    # )
 
     # Step 5: Drop the new_state column
     op.drop_column("task_events", "state")

src/backend/app/projects/image_processing.py

@@ -484,37 +484,41 @@ async def process_assets_from_odm(
                             conn, dtm_project_id, dtm_task_id, "assets_url", s3_path_url
                         )
 
-                        # If the project has only one task, copy the s3_ortho path to the project level
-                        # And mark the project processing status as completed, no need to process again
-                        total_tasks = await project_logic.get_all_tasks_for_project(
+                        # If the project has only one task, copy the orthophoto and assets to the project level
+                        # Mark the project processing status as completed to avoid redundant processing
+                        tasks = await project_logic.get_all_tasks_for_project(
                             dtm_project_id, conn
                         )
-                        if len(total_tasks) == 1:
-                            s3_ortho_project_path = f"dtm-data/projects/{dtm_project_id}/orthophoto/odm_orthophoto.tif"
+
+                        if len(tasks) == 1:
+                            project_ortho_path = f"dtm-data/projects/{dtm_project_id}/orthophoto/odm_orthophoto.tif"
                             log.info(
-                                f"Copying orthophoto to project level: {s3_ortho_project_path}"
+                                f"Copying orthophoto to project level: {project_ortho_path}"
                             )
-                            copy_file_within_bucket(
+
+                            ortho_copy_status = copy_file_within_bucket(
                                 settings.S3_BUCKET_NAME,
                                 s3_ortho_path,
-                                s3_ortho_project_path,
+                                project_ortho_path,
                             )
 
-                            # copy the assets.zip to the project level
-                            s3_assets_project_path = (
+                            project_assets_path = (
                                 f"dtm-data/projects/{dtm_project_id}/assets.zip"
                             )
                             log.info(
-                                f"Copying assets to project level: {s3_assets_project_path}"
-                            )
-                            copy_file_within_bucket(
-                                settings.S3_BUCKET_NAME, s3_path, s3_assets_project_path
+                                f"Copying assets to project level: {project_assets_path}"
                             )
-                            # update the image processing status
-                            await project_logic.update_processing_status(
-                                conn, dtm_project_id, ImageProcessingStatus.SUCCESS
+
+                            assets_copy_status = copy_file_within_bucket(
+                                settings.S3_BUCKET_NAME, s3_path, project_assets_path
                             )
 
+                            # Update project processing status if both copies were successful
+                            if ortho_copy_status and assets_copy_status:
+                                await project_logic.update_processing_status(
+                                    conn, dtm_project_id, ImageProcessingStatus.SUCCESS
+                                )
+
             status = ImageProcessingStatus.SUCCESS
         if not dtm_task_id:
             # Update the image processing status

src/backend/app/s3.py

@@ -6,6 +6,7 @@
 from datetime import timedelta
 from urllib.parse import urljoin
 from minio.error import S3Error
+from minio.commonconfig import CopySource
 
 
 def s3_client():
@@ -302,14 +303,16 @@ def copy_file_within_bucket(
             log.warning(f"Source file not found: {source_path} in bucket {bucket_name}")
             return False
 
+        # Create CopySource object
+        copy_source = CopySource(bucket_name, source_path)
+
         # Copy the object within the same bucket
-        client.copy_object(
-            bucket_name, destination_path, f"{bucket_name}/{source_path}"
-        )
+        result = client.copy_object(bucket_name, destination_path, copy_source)
 
         log.debug(
             f"Successfully copied object within {bucket_name} "
-            f"from {source_path} to {destination_path}"
+            f"from {source_path} to {destination_path}. "
+            f"Object name: {result.object_name}, Version ID: {result.version_id}"
         )
         return True