Ensure idempotency close #20

Author: danlooo

pathtraits/cli.py

@@ -34,6 +34,7 @@ def main():
 def batch(path, db_path, exclude_regex, verbose):
     """
     Update database once, searches for all directories recursively.
+    Deletes existing database to ensure its synced with sidecar metadata files.
     """
     scan.batch(path, db_path, exclude_regex, verbose)
 
@@ -86,15 +87,20 @@ def query(query_str, db_path):
 
 
 @main.command()
-@click.argument("path", required=True, type=click.Path(exists=True, file_okay=False, dir_okay=True))
+@click.argument(
+    "path", required=True, type=click.Path(exists=True, file_okay=False, dir_okay=True)
+)
 @click.option(
-    "--needed-until", "-nu",
+    "--needed-until",
+    "-nu",
     default=None,
     help="Optional account termination date (YYYY-MM-DD)",
 )
 @click.option(
-    "--overwrite", "-o",
-    is_flag=True, default=False,
+    "--overwrite",
+    "-o",
+    is_flag=True,
+    default=False,
     help="Overwrite existing metadata.",
 )
 @click.option("-v", "--verbose", is_flag=True, default=False)
@@ -104,5 +110,6 @@ def create(path, needed_until, overwrite, verbose):
     """
     _create.generate_metadata(path, needed_until, overwrite, verbose)
 
+
 if __name__ == "__main__":
     main()

pathtraits/scan.py

@@ -50,6 +50,7 @@ def scan_meta_yml(path, pathpairs=[], exclude_regex=None):
 def batch(path, db_path, exclude_regex, verbose):
     """
     Update database once, searches for all directories recursively.
+    Deletes existing database to ensure its synced with sidecar metadata files.
 
     :param path: path to scan in batch mode recursively
     :param db_path: path to the database
@@ -60,6 +61,11 @@ def batch(path, db_path, exclude_regex, verbose):
 
     if db_path is None:
         db_path = path + "/.pathtraits.db"
+
+    # database is just a cache, deleting specific rows is very slow.
+    if os.path.exists(db_path):
+        os.remove(db_path)
+
     db = TraitsDB(db_path)
     if exclude_regex is not None:
         exclude_regex = re.compile(exclude_regex)

test/test.py

@@ -10,6 +10,7 @@
 import pathtraits.db
 import pathtraits.scan
 import pathtraits.access
+import hashlib
 
 
 class TestMain(unittest.TestCase):
@@ -79,6 +80,17 @@ def test_query(self):
         )
         self.assertEqual(len(q2), 1)
 
+    def test_idempotency_batch(self):
+        db_path = tempfile.mkstemp()[1]
+        pathtraits.scan.batch("test/example", db_path, "", False)
+        hash1 = hashlib.file_digest(open(db_path, "rb"), "sha256").hexdigest()
+
+        pathtraits.scan.batch("test/example", db_path, "", False)
+        hash2 = hashlib.file_digest(open(db_path, "rb"), "sha256").hexdigest()
+
+        os.remove(db_path)
+        self.assertEqual(hash1, hash2)
+
 
 if __name__ == "__main__":
     unittest.main()