Sync bottleneck: prevent remove chunks

server/mergin/sync/public_api_controller.py

@@ -1028,14 +1028,6 @@ def push_finish(transaction_id):
         # let's move uploaded files where they are expected to be
         os.renames(files_dir, version_dir)
 
-        # remove used chunks
-        for file in upload.changes["added"] + upload.changes["updated"]:
-            file_chunks = file.get("chunks", [])
-            for chunk_id in file_chunks:
-                chunk_file = os.path.join(upload.upload_dir, "chunks", chunk_id)
-                if os.path.exists(chunk_file):
-                    move_to_tmp(chunk_file)
-
         logging.info(
             f"Push finished for project: {project.id}, project version: {v_next_version}, transaction id: {transaction_id}."
         )

server/mergin/sync/public_api_v2_controller.py

@@ -318,14 +318,6 @@ def create_project_version(id):
             temp_files_dir = os.path.join(upload.upload_dir, "files", v_next_version)
             os.renames(temp_files_dir, version_dir)
 
-            # remove used chunks
-            for file in to_be_added_files + to_be_updated_files:
-                file_chunks = file.get("chunks", [])
-                for chunk_id in file_chunks:
-                    chunk_file = get_chunk_location(chunk_id)
-                    if os.path.exists(chunk_file):
-                        move_to_tmp(chunk_file)
-
         logging.info(
             f"Push finished for project: {project.id}, project version: {v_next_version}, upload id: {upload.id}."
         )
@@ -377,7 +369,6 @@ def upload_chunk(id: str):
         # we could have used request.data here, but it could eventually cause OOM issue
         save_to_file(request.stream, dest_file, current_app.config["MAX_CHUNK_SIZE"])
     except IOError:
-        move_to_tmp(dest_file, chunk_id)
         return BigChunkError().response(413)
     except Exception as e:
         return UploadError(error="Error saving chunk").response(400)

server/mergin/tests/test_public_api_v2.py

@@ -2,6 +2,7 @@
 #
 # SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-MerginMaps-Commercial
 
+from mergin.sync.tasks import remove_unused_chunks
 from . import DEFAULT_USER
 from .utils import (
     add_user,
@@ -22,6 +23,7 @@
 
 from mergin.app import db
 from mergin.config import Configuration
+from mergin.sync.config import Configuration as SyncConfiguration
 from mergin.sync.errors import (
     BigChunkError,
     ProjectLocked,
@@ -375,9 +377,14 @@ def test_create_version(client, data, expected, err_code):
 
     response = client.post(f"v2/projects/{project.id}/versions", json=data)
     assert response.status_code == expected
+    # mock chunks expiration to check if removed
     if expected == 201:
         assert response.json["version"] == "v2"
         assert project.latest_version == 2
+        # chunks exists after upload, cleanup job did not remove them
+        assert all(os.path.exists(chunk) for chunk in chunks)
+        with patch.object(SyncConfiguration, "UPLOAD_CHUNKS_EXPIRATION", 0):
+            remove_unused_chunks()
         assert all(not os.path.exists(chunk) for chunk in chunks)
     else:
         assert project.latest_version == 1