The mfn is part of the upload loop,

... so it makes sense to log it as part of the progress.

Previously we'd report 10/10 (or whatever), but still be working on the
manifest upload.

Also a version bump.

Author: sz3

pog/pog.py

@@ -247,6 +247,7 @@ def save_manifest(self, mfn, filename=None):
                 full_manifest_bytes = dumps(mfn).encode('utf-8')
                 self._write(f, _compress(full_manifest_bytes, self.compresslevel))
             self.blob_store.save(filename, temp_path)
+        return filename
 
     def encrypt_single_file(self, filename):
         cctx = zstd.ZstdCompressor(level=self.compresslevel)
@@ -267,7 +268,7 @@ def encrypt(self, *inputs):
         mfn = dict()
         all_inputs = local_file_list(*inputs)
         for count, filename in enumerate(all_inputs):
-            _print_progress(count+1, len(all_inputs), filename)
+            _print_progress(count+1, len(all_inputs)+1, filename)
             outputs = []
             for temp_path in self.encrypt_single_file(filename):
                 blob_name = path.basename(temp_path)
@@ -280,7 +281,8 @@ def encrypt(self, *inputs):
                     'atime': path.getatime(filename),
                     'mtime': path.getmtime(filename),
                 }
-        self.save_manifest(mfn)
+        mfn_filename = self.save_manifest(mfn)
+        _print_progress(len(all_inputs)+1, len(all_inputs)+1, mfn_filename)
 
 
 class Decryptor():
@@ -378,7 +380,7 @@ def decrypt(self, *inputs):
 
 
 def main():
-    args = docopt(__doc__, version='Pog 0.1.2')
+    args = docopt(__doc__, version='Pog 0.1.3')
     chunk_size = parse_size(args.get('--chunk-size', '100MB'))
     compresslevel = int(args.get('--compresslevel', '3'))
     store_absolute_paths = args.get('--store-absolute-paths')

setup.py

@@ -10,7 +10,7 @@
     name='pogcli',
     license='MIT',
     url='https://github.com/sz3/pog',
-    version='0.1.2',
+    version='0.1.3',
 
     entry_points={
         'console_scripts': [

tests/test_pog.py

@@ -52,17 +52,19 @@ class KeyfileTest(TestDirMixin, TestCase):
     def test_round_trip(self):
         # encrypt our sample files
         enc = self.run_command(self.encryption_flag, self.tiny_sample, self.another_sample)
+        manifest_name = glob(path.join(self.working_dir.name, '*.mfn'))[0]
+
         # ordered lexicographically by filename
         self.assertEqual(enc, [
-            f'*** 1/2: {self.another_sample}',
+            f'*** 1/3: {self.another_sample}',
             self.another_sample_blobname,
-            f'*** 2/2: {self.tiny_sample}',
-            self.tiny_sample_blobname
+            f'*** 2/3: {self.tiny_sample}',
+            self.tiny_sample_blobname,
+            '*** 3/3: {}'.format(path.basename(manifest_name)),
         ])
         blobs = [l for l in enc if not l.startswith('***')]
 
         # check that the manifest looks good
-        manifest_name = glob(path.join(self.working_dir.name, '*.mfn'))[0]
         show_mfn = self.run_command(self.decryption_flag, '--dump-manifest', manifest_name)
         self.assertEqual(
             show_mfn, ['* another_sample.txt:', blobs[0], '* tiny_sample.txt:', blobs[1]]
@@ -167,15 +169,17 @@ def test_absolute_paths(self):
         enc = self.run_command(
             self.encryption_flag, self.tiny_sample, self.another_sample, '--store-absolute-paths'
         )
+        manifest_name = glob(path.join(self.working_dir.name, '*.mfn'))[0]
+
         self.assertEqual(enc, [
-            f'*** 1/2: {self.another_sample}',
+            f'*** 1/3: {self.another_sample}',
             self.another_sample_blobname,
-            f'*** 2/2: {self.tiny_sample}',
-            self.tiny_sample_blobname
+            f'*** 2/3: {self.tiny_sample}',
+            self.tiny_sample_blobname,
+            '*** 3/3: {}'.format(path.basename(manifest_name)),
         ])
 
         # check that the manifest looks good
-        manifest_name = glob(path.join(self.working_dir.name, '*.mfn'))[0]
         show_mfn = self.run_command(self.decryption_flag, '--dump-manifest', manifest_name)
         self.assertEqual(
             show_mfn, [
@@ -211,15 +215,17 @@ def test_glob_input_directory(self):
         enc = self.run_command(
             self.encryption_flag, self.input_dir.name, '--store-absolute-paths'
         )
+        manifest_name = glob(path.join(self.working_dir.name, '*.mfn'))[0]
+
         self.assertEqual(enc, [
-            f'*** 1/2: {self.another_sample}',
+            f'*** 1/3: {self.another_sample}',
             self.another_sample_blobname,
-            f'*** 2/2: {self.tiny_sample}',
-            self.tiny_sample_blobname
+            f'*** 2/3: {self.tiny_sample}',
+            self.tiny_sample_blobname,
+            '*** 3/3: {}'.format(path.basename(manifest_name)),
         ])
 
         # check that the manifest looks good
-        manifest_name = glob(path.join(self.working_dir.name, '*.mfn'))[0]
         show_mfn = self.run_command(self.decryption_flag, '--dump-manifest', manifest_name)
         self.assertEqual(
             show_mfn, [
@@ -245,16 +251,18 @@ def test_manifest_index_ordering(self):
         '''
         # encrypt our sample files
         enc = self.run_command(self.encryption_flag, self.tiny_sample, self.another_sample)
+        manifest_name = glob(path.join(self.working_dir.name, '*.mfn'))[0]
+
         self.assertEqual(enc, [
-            f'*** 1/2: {self.another_sample}',
+            f'*** 1/3: {self.another_sample}',
             self.another_sample_blobname,
-            f'*** 2/2: {self.tiny_sample}',
-            self.tiny_sample_blobname
+            f'*** 2/3: {self.tiny_sample}',
+            self.tiny_sample_blobname,
+            '*** 3/3: {}'.format(path.basename(manifest_name)),
         ])
         blobs = [l for l in enc if not l.startswith('***')]
 
         # check that the manifest index looks good
-        manifest_name = glob(path.join(self.working_dir.name, '*.mfn'))[0]
         show_mfn = self.run_command(self.decryption_flag, '--dump-manifest-index', manifest_name)
         # manifest index sorted by blobname
         self.assertEqual(show_mfn, sorted(blobs))
@@ -276,15 +284,17 @@ def tearDownClass(cls):
     def test_with_keyfile(self):
         # encrypt our sample file
         enc = self.run_command(f'--keyfile={POG_ROOT}/tests/samples/only_for_testing.encrypt', BigFileTest.big_sample)
+        manifest_name = glob(path.join(self.working_dir.name, '*.mfn'))[0]
+
         self.assertEqual(enc, [
-            f'*** 1/1: {self.big_sample}',
+            f'*** 1/2: {self.big_sample}',
             'xyQWj-UXXZpwWXPF2c5_MsBm3cTfZFXayUVLLMlkt4Y=',
             'HXBJ_N4EM2rywLdOWT02hccp4c_oLk0QyD2lc3vUttw=',
+            '*** 2/2: {}'.format(path.basename(manifest_name)),
         ])
         blobs = [l for l in enc if not l.startswith('***')]
 
         # check that the manifest looks good
-        manifest_name = glob(path.join(self.working_dir.name, '*.mfn'))[0]
         show_mfn = self.run_command(f'--keyfile={POG_ROOT}/tests/samples/only_for_testing.encrypt', '--dump-manifest',
                                     manifest_name)
         self.assertEqual(show_mfn, ['* big_sample.bin:'] + blobs)
@@ -307,15 +317,17 @@ def test_with_asymmetric(self):
         # encrypt our sample file
         enc = self.run_command(f'--encryption-keyfile={POG_ROOT}/tests/samples/only_for_testing.encrypt',
                                BigFileTest.big_sample)
+        manifest_name = glob(path.join(self.working_dir.name, '*.mfn'))[0]
+
         self.assertEqual(enc, [
-            f'*** 1/1: {self.big_sample}',
+            f'*** 1/2: {self.big_sample}',
             'RiOpsEQbQpxrBvXL1s047hq54EhFXxWqwag-vMuiRfc=',
             'YdK86P4e2191CxVBhZwvvPtwOLU6Ve1NzMhwLjxVXqg=',
+            '*** 2/2: {}'.format(path.basename(manifest_name)),
         ])
         blobs = [l for l in enc if not l.startswith('***')]
 
         # check that the manifest looks good
-        manifest_name = glob(path.join(self.working_dir.name, '*.mfn'))[0]
         show_mfn = self.run_command(f'--decryption-keyfile={POG_ROOT}/tests/samples/only_for_testing.decrypt',
                                     '--dump-manifest', manifest_name)
         self.assertEqual(show_mfn, ['* big_sample.bin:'] + blobs)
@@ -345,16 +357,18 @@ def test_smaller_chunk_size(self):
             f'--encryption-keyfile={POG_ROOT}/tests/samples/only_for_testing.encrypt', BigFileTest.big_sample,
             '--chunk-size=50MB'
         )
+        manifest_name = glob(path.join(self.working_dir.name, '*.mfn'))[0]
+
         self.assertEqual(enc, [
-            f'*** 1/1: {self.big_sample}',
+            f'*** 1/2: {self.big_sample}',
             'PURfe1ei1aqpPRarpAfKkcKPRSHdo5hPH-bvfYND2KM=',
             'nnL4ta-BChpb36CIFeZUG4lJLiz8l0YVv94IaABcgyU=',
             'YdK86P4e2191CxVBhZwvvPtwOLU6Ve1NzMhwLjxVXqg=',
+            '*** 2/2: {}'.format(path.basename(manifest_name)),
         ])
         blobs = [l for l in enc if not l.startswith('***')]
 
         # check that the manifest looks good
-        manifest_name = glob(path.join(self.working_dir.name, '*.mfn'))[0]
         show_mfn = self.run_command(f'--decryption-keyfile={POG_ROOT}/tests/samples/only_for_testing.decrypt',
                                     '--dump-manifest', manifest_name)
         self.assertEqual(show_mfn, ['* big_sample.bin:'] + blobs)