Merge pull request #889 from Intevation/dev-issue881

Implement more flexibility for storage/filesystem

Author: Hugo Osvaldo Barrera

docs/config.rst

@@ -408,6 +408,7 @@ Local
       fileext = "..."
       #encoding = "utf-8"
       #post_hook = null
+      #fileignoreext = ".tmp"
 
     Can be used with `khal <http://lostpackets.de/khal/>`_. See :doc:`vdir` for
     a more formal description of the format.
@@ -421,11 +422,15 @@ Local
     :param fileext: The file extension to use (e.g. ``.txt``). Contained in the
         href, so if you change the file extension after a sync, this will
         trigger a re-download of everything (but *should* not cause data-loss
-        of any kind).
+        of any kind). To be compatible with the ``vset`` format you have
+        to either use ``.vcf`` or ``.ics``. Note that metasync won't work
+        if you use an empty string here.
     :param encoding: File encoding for items, both content and filename.
     :param post_hook: A command to call for each item creation and
         modification. The command will be called with the path of the
         new/updated file.
+    :param fileeignoreext: The file extention to ignore. It is only useful
+        if fileext is set to the empty string. The default is ``.tmp``.
 
 .. storage:: singlefile
 

tests/storage/test_filesystem.py

@@ -44,6 +44,36 @@ def test_ident_with_slash(self, tmpdir):
         (item_file,) = tmpdir.listdir()
         assert "/" not in item_file.basename and item_file.isfile()
 
+    def test_ignore_tmp_files(self, tmpdir):
+        """Test that files with .tmp suffix beside .ics files are ignored."""
+        s = self.storage_class(str(tmpdir), '.ics')
+        s.upload(Item('UID:xyzxyz'))
+        item_file, = tmpdir.listdir()
+        item_file.copy(item_file.new(ext='tmp'))
+        assert len(tmpdir.listdir()) == 2
+        assert len(list(s.list())) == 1
+
+    def test_ignore_tmp_files_empty_fileext(self, tmpdir):
+        """Test that files with .tmp suffix are ignored with empty fileext."""
+        s = self.storage_class(str(tmpdir), '')
+        s.upload(Item('UID:xyzxyz'))
+        item_file, = tmpdir.listdir()
+        item_file.copy(item_file.new(ext='tmp'))
+        assert len(tmpdir.listdir()) == 2
+        # assert False, tmpdir.listdir() # enable to see the created filename
+        assert len(list(s.list())) == 1
+
+    def test_ignore_files_typical_backup(self, tmpdir):
+        """Test file-name ignorance with typical backup ending ~."""
+        ignorext = "~" # without dot
+        s = self.storage_class(str(tmpdir), '', fileignoreext="~")
+        s.upload(Item('UID:xyzxyz'))
+        item_file, = tmpdir.listdir()
+        item_file.copy(item_file.new(basename=item_file.basename+'~'))
+        assert len(tmpdir.listdir()) == 2
+        #assert False, tmpdir.listdir() # enable to see the created filename
+        assert len(list(s.list())) == 1
+
     def test_too_long_uid(self, tmpdir):
         s = self.storage_class(str(tmpdir), ".txt")
         item = Item("UID:" + "hue" * 600)

vdirsyncer/storage/filesystem.py

@@ -22,13 +22,22 @@ class FilesystemStorage(Storage):
     storage_name = "filesystem"
     _repr_attributes = ("path",)
 
-    def __init__(self, path, fileext, encoding="utf-8", post_hook=None, **kwargs):
+    def __init__(
+        self,
+        path,
+        fileext,
+        encoding="utf-8",
+        post_hook=None,
+        fileignoreext=".tmp",
+        **kwargs
+    ):
         super().__init__(**kwargs)
         path = expand_path(path)
         checkdir(path, create=False)
         self.path = path
         self.encoding = encoding
         self.fileext = fileext
+        self.fileignoreext = fileignoreext
         self.post_hook = post_hook
 
     @classmethod
@@ -80,7 +89,11 @@ def _get_href(self, ident):
     def list(self):
         for fname in os.listdir(self.path):
             fpath = os.path.join(self.path, fname)
-            if os.path.isfile(fpath) and fname.endswith(self.fileext):
+            if (
+                os.path.isfile(fpath)
+                and fname.endswith(self.fileext)
+                and (not fname.endswith(self.fileignoreext))
+            ):
                 yield fname, get_etag_from_file(fpath)
 
     def get(self, href):