Remove diff files (#176)

* add function to clean after push_project

* if file size is zero remove and consider file not_updated

* do not check for size, only remove existing file

* add remove_diff_files()

* call cleaning function

* import

* remove cleaning

* revert changes

* add test checking if diff files are cleaned

---------

Co-authored-by: Jan Caha <[email protected]>

Author: JanCaha

mergin/client_push.py

@@ -14,6 +14,7 @@
 import pprint
 import tempfile
 import concurrent.futures
+import os
 
 from .common import UPLOAD_CHUNK_SIZE, ClientError
 from .merginproject import MerginProject
@@ -286,6 +287,8 @@ def push_project_finalize(job):
 
     job.tmp_dir.cleanup()  # delete our temporary dir and all its content
 
+    remove_diff_files(job)
+
     job.mp.log.info("--- push finished - new project version " + job.server_resp["version"])
 
 
@@ -316,3 +319,13 @@ def _do_upload(item, job):
 
     item.upload_blocking(job.mc, job.mp)
     job.transferred_size += item.size
+
+
+def remove_diff_files(job) -> None:
+    """Looks for diff files in the job and removes them."""
+
+    for change in job.changes["updated"]:
+        if "diff" in change.keys():
+            diff_file = job.mp.fpath_meta(change["diff"]["path"])
+            if os.path.exists(diff_file):
+                os.remove(diff_file)

mergin/merginproject.py

@@ -357,6 +357,8 @@ def get_push_changes(self):
                         "mtime": datetime.fromtimestamp(os.path.getmtime(diff_file), tzlocal()),
                     }
                 else:
+                    if os.path.exists(diff_file):
+                        os.remove(diff_file)
                     not_updated.append(file)
             except (pygeodiff.GeoDiffLibError, pygeodiff.GeoDiffLibConflictError) as e:
                 self.log.warning("failed to create changeset for " + path)

mergin/test/test_client.py

@@ -8,6 +8,7 @@
 import pytest
 import pytz
 import sqlite3
+import glob
 
 from .. import InvalidProject
 from ..client import MerginClient, ClientError, MerginProject, LoginError, decode_token_data, TokenError, ServerType
@@ -1890,3 +1891,27 @@ def test_version_info(mc):
     created = datetime.strptime(info["created"], "%Y-%m-%dT%H:%M:%SZ")
     assert created.date() == date.today()
     assert info["changes"]["updated"][0]["size"] == 98304
+
+
+def test_clean_diff_files(mc):
+    test_project = "test_clean"
+    project = API_USER + "/" + test_project
+    project_dir = os.path.join(TMP_DIR, test_project)  # primary project dir for updates
+    project_dir_2 = os.path.join(TMP_DIR, test_project + "_2")  # concurrent project dir
+
+    cleanup(mc, project, [project_dir, project_dir_2])
+    # create remote project
+    shutil.copytree(TEST_DATA_DIR, project_dir)
+    mc.create_project_and_push(test_project, project_dir)
+
+    # test push changes with diffs:
+    mp = MerginProject(project_dir)
+    f_updated = "base.gpkg"
+    # step 1) base.gpkg updated to inserted_1_A (inserted A feature)
+    shutil.move(mp.fpath(f_updated), mp.fpath_meta(f_updated))  # make local copy for changeset calculation
+    shutil.copy(mp.fpath("inserted_1_A.gpkg"), mp.fpath(f_updated))
+    mc.push_project(project_dir)
+
+    diff_files = glob.glob("*-diff-*", root_dir=os.path.split(mp.fpath_meta("inserted_1_A.gpkg"))[0])
+
+    assert diff_files == []