Fix sync pull for files with extension in public id

Author: const-cloudinary

cloudinary_cli/modules/sync.py

@@ -2,15 +2,15 @@
 from functools import reduce
 from itertools import product
 from os import remove
-from os.path import split, join as path_join, abspath
+from os.path import join as path_join, abspath
 
 from click import command, argument, option, style
 from cloudinary import api
 
 from cloudinary_cli.utils.api_utils import query_cld_folder, upload_file, download_file
 from cloudinary_cli.utils.file_utils import walk_dir, delete_empty_dirs, get_destination_folder
 from cloudinary_cli.utils.json_utils import print_json
-from cloudinary_cli.utils.utils import logger, run_tasks_concurrently, confirm_action, get_user_action
+from cloudinary_cli.utils.utils import logger, run_tasks_concurrently, get_user_action
 
 DELETE_ASSETS_BATCH_SIZE = 100
 
@@ -88,8 +88,10 @@ def push(self):
             return False
 
         files_to_push = self.unique_local_file_names | self.out_of_sync_file_names
-        to_upload = list(filter(lambda x: split(x)[1][0] != ".", files_to_push))
-        logger.info(f"Uploading {len(to_upload)} items to Cloudinary folder '{self.remote_dir}'")
+        if not files_to_push:
+            return True
+
+        logger.info(f"Uploading {len(files_to_push)} items to Cloudinary folder '{self.remote_dir}'")
 
         options = {
             'use_filename': True,
@@ -98,7 +100,7 @@ def push(self):
             'resource_type': 'auto'
         }
         uploads = []
-        for file in to_upload:
+        for file in files_to_push:
             folder = get_destination_folder(self.remote_dir, file)
 
             uploads.append((self.local_files[file]['path'], {**options, 'folder': folder}))
@@ -145,12 +147,14 @@ def pull(self):
 
         files_to_pull = self.unique_remote_file_names | self.out_of_sync_file_names
 
+        if not files_to_pull:
+            return True
+
         logger.info(f"Downloading {len(files_to_pull)} files from Cloudinary")
         downloads = []
         for file in files_to_pull:
             remote_file = self.remote_files[file]
-            file_name = file + "." + remote_file['format'] if remote_file['resource_type'] != 'raw' else file
-            local_path = abspath(path_join(self.local_dir, file_name))
+            local_path = abspath(path_join(self.local_dir, file))
 
             downloads.append((remote_file, local_path))
 

cloudinary_cli/utils/api_utils.py

@@ -25,7 +25,7 @@ def query_cld_folder(folder):
         res = expression.execute()
 
         for asset in res['resources']:
-            rel_path = path.relpath(asset['public_id'], folder)
+            rel_path = path.relpath(asset_source(asset), folder)
             files[rel_path] = {
                 "type": asset['type'],
                 "resource_type": asset['resource_type'],
@@ -65,18 +65,35 @@ def upload_file(file_path, options, uploaded=None, skipped=None):
 def download_file(remote_file, local_path):
     makedirs(path.dirname(local_path), exist_ok=True)
     with open(local_path, "wb") as f:
-        f.write(requests.get(cloudinary_url(remote_file['public_id'], resource_type=remote_file['resource_type'],
-                                            type=remote_file['type'])[0]).content)
+        download_url = cloudinary_url(asset_source(remote_file), resource_type=remote_file['resource_type'],
+                                      type=remote_file['type'])[0]
+        f.write(requests.get(download_url).content)
     logger.info(style("Downloaded '{}' to '{}'".format(remote_file['relative_path'], local_path), fg="green"))
 
 
+def asset_source(asset_details):
+    """
+    Public ID of the transformable file (image/video) does not include file extension.
+
+    It needs to be added in order to download file properly (without creating derived asset).
+
+    Raw files are accessed using only public_id.
+
+    :param asset_details: The details of the asset.
+    :rtype asset_details: dict
+
+    :return:
+    """
+    base_name = asset_details['public_id']
+    return base_name + "." + asset_details['format'] if asset_details['resource_type'] != 'raw' else base_name
+
+
 def handle_command(
         params,
         optional_parameter,
         optional_parameter_parsed,
         module,
         module_name):
-
     try:
         func = module.__dict__[params[0]]
     except KeyError:

cloudinary_cli/utils/file_utils.py

@@ -8,10 +8,13 @@
 def walk_dir(root_dir):
     all_files = {}
     for root, _, files in walk(root_dir):
+        relative_path = relpath(root, root_dir) if root_dir != root else ""
         for file in files:
-            all_files[path.splitext(path.join(root, file)[len(root_dir) + 1:])[0]] = {
-                "path": path.join(root, file),
-                "etag": etag(path.join(root, file))
+            full_path = path.join(root, file)
+            relative_file_path = "/".join(p for p in [relative_path, file] if p)
+            all_files[relative_file_path] = {
+                "path": full_path,
+                "etag": etag(full_path)
             }
     return all_files