Merge pull request #546 from MerginMaps/exlude_clone_filenames

Filter out specific filenames when cloning project

Author: MarcelGeo

server/mergin/sync/config.py

@@ -75,3 +75,6 @@ class Configuration(object):
     UPLOAD_CHUNKS_EXPIRATION = config(
         "UPLOAD_CHUNKS_EXPIRATION", default=86400, cast=int
     )
+    EXCLUDED_CLONE_FILENAMES = config(
+        "EXCLUDED_CLONE_FILENAMES", default="qgis_cfg.xml", cast=Csv()
+    )

server/mergin/sync/public_api_controller.py

@@ -1136,9 +1136,12 @@ def clone_project(namespace, project_name):  # noqa: E501
     )
     p.updated = datetime.utcnow()
     db.session.add(p)
+    files_to_exclude = current_app.config.get("EXCLUDED_CLONE_FILENAMES", [])
 
     try:
-        p.storage.initialize(template_project=cloned_project)
+        p.storage.initialize(
+            template_project=cloned_project, excluded_files=files_to_exclude
+        )
     except InitializationError as e:
         abort(400, f"Failed to clone project: {str(e)}")
 
@@ -1149,6 +1152,8 @@ def clone_project(namespace, project_name):  # noqa: E501
     # transform source files to new uploaded files
     file_changes = []
     for file in cloned_project.files:
+        if os.path.basename(file.path) in files_to_exclude:
+            continue
         file_changes.append(
             ProjectFileChange(
                 file.path,

server/mergin/sync/storages/disk.py

@@ -178,7 +178,7 @@ def _project_dir(self):
         )
         return project_dir
 
-    def initialize(self, template_project=None):
+    def initialize(self, template_project=None, excluded_files=None):
         if os.path.exists(self.project_dir):
             raise InitializationError(
                 "Project directory already exists: {}".format(self.project_dir)
@@ -193,8 +193,12 @@ def initialize(self, template_project=None):
             if ws.disk_usage() + template_project.disk_usage > ws.storage:
                 self.delete()
                 raise InitializationError("Disk quota reached")
+            if excluded_files is None:
+                excluded_files = []
 
             for file in template_project.files:
+                if os.path.basename(file.path) in excluded_files:
+                    continue
                 src = os.path.join(template_project.storage.project_dir, file.location)
                 dest = os.path.join(
                     self.project_dir,

server/mergin/tests/test_project_controller.py

@@ -1728,16 +1728,21 @@ def test_clone_project(client, data, username, expected):
         assert resp.json["code"] == "StorageLimitHit"
         assert resp.json["detail"] == "You have reached a data limit (StorageLimitHit)"
     if expected == 200:
+        excluded_filenames = current_app.config.get("EXCLUDED_CLONE_FILENAMES")
+
         proj = data.get("project", test_project).strip()
         template = Project.query.filter_by(
             name=test_project, workspace_id=test_workspace_id
         ).first()
         project = Project.query.filter_by(
             name=proj, workspace_id=test_workspace_id
         ).first()
+        template_files_filtered = [
+            f for f in template.files if f.path not in excluded_filenames
+        ]
         assert not any(
             x.checksum != y.checksum and x.path != y.path
-            for x, y in zip(project.files, template.files)
+            for x, y in zip(project.files, template_files_filtered)
         )
         assert os.path.exists(
             os.path.join(project.storage.project_dir, project.files[0].location)
@@ -1755,6 +1760,12 @@ def test_clone_project(client, data, username, expected):
             item for item in changes if item.change == PushChangeType.UPDATE.value
         ]
         assert pv.device_id == json_headers["X-Device-Id"]
+
+        assert not any(f.path == excluded_filenames[0] for f in project.files)
+        assert not os.path.exists(
+            os.path.join(project.storage.project_dir, excluded_filenames[0])
+        )
+        assert len(project.files) == len(template.files) - 1
         # cleanup
         shutil.rmtree(project.storage.project_dir)
 
@@ -2000,7 +2011,7 @@ def test_get_projects_by_uuids(client):
         {"page": 1, "per_page": 5, "desc": False},
         200,
         "v1",
-        {"added": 12, "removed": 0, "updated": 0, "updated_diff": 0},
+        {"added": 13, "removed": 0, "updated": 0, "updated_diff": 0},
     ),
     (
         {"page": 2, "per_page": 3, "desc": True},

server/mergin/tests/test_projects/test/qgis_cfg.xml

@@ -0,0 +1,4 @@
+<!DOCTYPE qgis_authentication>
+<qgis_authentication salt="852cd01b5ba209804a5d068f5b28c448" hash="cab305f8ee1500ff912f9d68375b49e8" civ="d5472f8b9ad315bdd250e44a3419a2b8b3c9fd01956c002220520097bfbcb099">0737fe398eb9f26bd847fb9da2407646a2e8c89dc2f93eba5a059f19eedd8017e50c557d2c7d435a2701d881cdaf4fbbd3a892e4367053b5bfc348b556ae252314d9b06fc70a4f184362d064023c1ed6c4dd7ee14dce10ea91595e8548f7ba3d3eaca1d41063f50d1ccc12bfb90c059271254dca780e0d60e68bd234844fda81a0781977907485b397aa1263aef81863625eb439dc349fca0dbc641b4a606657f17e55d2c02fbc95388bf9f96977c65fcf7b723689d5fcdaf73190a5597425b3d33c858c2c4ef8c334b5f601c98db05557c8f690cdb9f73c725bf7ee420fffb6037cc9e80c7374a55ab55baf4aaa7f1c957fd40bb69b9fcb41ec42b063330bbddcd73f4de69e47772309167cb20ef4fe3250db96c29b71772edd18c7e73c501d569f4f8deda15fb0bcf6701d81902a6fc6c722db9e0d766d18a45297232224738c07a1a4f8fe490954efcae05fc1e43eac4eb5efc8b9008dcff4cf3688db4b7e268c9adf75d88d4d1d3648232c6fc2ac98b49b3bbb19368ce460b4a7a9828558d473eb0f1ba34c09f1ba9ce0170ac6d6c656176760e4012c56daef3f5f05320d5f84260d2e6b5a0b15620c33802d1c8c2f28084eef63b32f8130edd4789972b25960e12eb79351d11316f78aea3a941b9e7f1f4042f708d873ac7807ce0652819b2e2f77f9aac1c50cf72d2341118c41419f5f4c31474d8dbe56558dab9cb4b8e4fe8df9c8b4a057d9c6fe6b098b78e150aadd2a45cf1ea15e02f8f1f8b1b46d1a5513c26a63ea08788675feaa912e884ceee57adc120393c8a5bc42988f7b210195f6eff5de3e332d0d67321d05b907f836eb0f0f9e97388f89b699638804978639ad8b4c889f1a56952f949242679506cba5cc35538ea01b1621dd6a154f92b721b5247e294a5394df9c87765675b737dcb28346fc4032b68f87f46150aa4aa136378903036aff61fd41cf0cbcdd0865660f26d7f1d49f29ff5962adc209b9db71d12bf49bf67950496f18ca1de0a5cd7186e1bc0fcf826ffd1bb91ab36412c43730db5ff9ec57990fee27c5158446294bf0d8e61e12ee53e80b606b541c754ed45b2289079df8b647a8ca12fb1706e371523a581af50d333adfe5e84bcce2a60e84e24bdc1eb74610bc28b279b15c4f2020b045d2e4a7f846e488d74e761d98c05f105452235f602b3fe8beccf4b11d35ba6042dcc97f68090f40edbd6e8497434c193343cca98ebeabdd8620ec7eec642efda7cd45f0a9547ea821ac193eb1a8fb8c9c71d2e607b4651de5b8b613bc38aa4ba06bdb65a3d6b6e92546f1a4113e0bbce99aadbab3bbb07f31d6f90b3ff58b4494815e97a265c1c5e8a826bf14177427e03247395a18941753c0e580c42661a9c959ad57b93b97fb4adeca49927f3bec95eff361e95c324623f1c7c4d39e71250938d4189461cd6c1e978a5445f88eaf47670f23145cb7c8faf42ac83158743004fefb17a37a25edcf2425d530dd12ca52fdcfc399542cd288773c06931ce9aaac94df69dc6514fa3b1b8629dcfe725c0dcd77b5db967c5620dbc2444f4b78fb247e33a54ed2cbcaba3b92833b6d75b4900697da646f04da9a04d6353556b0ab70f8dd952eed9bd9cee1d53e760b292080862a74f625eb402662aadd94efaa6cce0727d3ccab5b6e112f25562effadfcf70307800e0d28976327576e99380facea2828ddb6a85addb4d4c0cfdb73cd848a9f707f8f978caf5de82756c80f42d53719987d5b4826397de8674d75dc1308dd3e96af37e9b3e42175dca1a5ff58a4aa4881a344113711a93340ee6515e5b9d03d1f4979531c84ec187b9303ea763b2641f530144cf52a81812349511219fc92bb038ec62d438c3beaf723</qgis_authentication>
+
+<!-- HASH: c60949a4387b0efb717777df1cd9e3da459574109674fb0f483e59869e11fe1b -->