Add support for dynamic folders mode in sync and upload_dir commands

* Bump `pycloudinary` to `1.35.0`

Author: const-cloudinary

cloudinary_cli/modules/sync.py

@@ -3,14 +3,15 @@
 from itertools import groupby
 from os import path, remove
 
-from click import command, argument, option, style, UsageError
+from click import command, argument, option, style, UsageError, Choice
 from cloudinary import api
 
-from cloudinary_cli.utils.api_utils import query_cld_folder, upload_file, download_file
-from cloudinary_cli.utils.file_utils import walk_dir, delete_empty_dirs, get_destination_folder, \
-    normalize_file_extension, posix_rel_path
+from cloudinary_cli.utils.api_utils import query_cld_folder, upload_file, download_file, get_folder_mode, \
+    get_default_upload_options, get_destination_folder_options
+from cloudinary_cli.utils.file_utils import walk_dir, delete_empty_dirs, normalize_file_extension, posix_rel_path
 from cloudinary_cli.utils.json_utils import print_json, read_json_from_file, write_json_to_file
-from cloudinary_cli.utils.utils import logger, run_tasks_concurrently, get_user_action, invert_dict, chunker
+from cloudinary_cli.utils.utils import logger, run_tasks_concurrently, get_user_action, invert_dict, chunker, \
+    group_params, parse_option_value
 
 _DEFAULT_DELETION_BATCH_SIZE = 30
 _DEFAULT_CONCURRENT_WORKERS = 30
@@ -32,13 +33,18 @@
 @option("-K", "--keep-unique", is_flag=True, help="Keep unique files in the destination folder.")
 @option("-D", "--deletion-batch-size", type=int, default=_DEFAULT_DELETION_BATCH_SIZE,
         help="Specify the batch size for deleting remote assets.")
+@option("-fm", "--folder-mode", type=Choice(['fixed', 'dynamic'], case_sensitive=False),
+        help="Specify folder mode explicitly. By default uses cloud mode configured in your cloud.", hidden=True)
+@option("-o", "--optional_parameter", multiple=True, nargs=2, help="Pass optional parameters as raw strings.")
+@option("-O", "--optional_parameter_parsed", multiple=True, nargs=2,
+        help="Pass optional parameters as interpreted strings.")
 def sync(local_folder, cloudinary_folder, push, pull, include_hidden, concurrent_workers, force, keep_unique,
-         deletion_batch_size):
+         deletion_batch_size, folder_mode, optional_parameter, optional_parameter_parsed):
     if push == pull:
         raise UsageError("Please use either the '--push' OR '--pull' options")
 
     sync_dir = SyncDir(local_folder, cloudinary_folder, include_hidden, concurrent_workers, force, keep_unique,
-                       deletion_batch_size)
+                       deletion_batch_size, folder_mode, optional_parameter, optional_parameter_parsed)
 
     result = True
     if push:
@@ -53,7 +59,7 @@ def sync(local_folder, cloudinary_folder, push, pull, include_hidden, concurrent
 
 class SyncDir:
     def __init__(self, local_dir, remote_dir, include_hidden, concurrent_workers, force, keep_deleted,
-                 deletion_batch_size):
+                 deletion_batch_size, folder_mode, optional_parameter, optional_parameter_parsed):
         self.local_dir = local_dir
         self.remote_dir = remote_dir.strip('/')
         self.user_friendly_remote_dir = self.remote_dir if self.remote_dir else '/'
@@ -63,15 +69,21 @@ def __init__(self, local_dir, remote_dir, include_hidden, concurrent_workers, fo
         self.keep_unique = keep_deleted
         self.deletion_batch_size = deletion_batch_size
 
+        self.folder_mode = folder_mode or get_folder_mode()
+
+        self.optional_parameter = optional_parameter
+        self.optional_parameter_parsed = optional_parameter_parsed
+
         self.sync_meta_file = path.join(self.local_dir, _SYNC_META_FILE)
 
         self.verbose = logger.getEffectiveLevel() < logging.INFO
 
         self.local_files = walk_dir(path.abspath(self.local_dir), include_hidden)
         logger.info(f"Found {len(self.local_files)} items in local folder '{local_dir}'")
 
-        self.remote_files = query_cld_folder(self.remote_dir)
-        logger.info(f"Found {len(self.remote_files)} items in Cloudinary folder '{self.user_friendly_remote_dir}'")
+        self.remote_files = query_cld_folder(self.remote_dir, self.folder_mode)
+        logger.info(f"Found {len(self.remote_files)} items in Cloudinary folder '{self.user_friendly_remote_dir}' "
+                    f"({self.folder_mode} folder mode)")
 
         local_file_names = self.local_files.keys()
         remote_file_names = self.remote_files.keys()
@@ -123,19 +135,20 @@ def push(self):
         logger.info(f"Uploading {len(files_to_push)} items to Cloudinary folder '{self.user_friendly_remote_dir}'")
 
         options = {
-            'use_filename': True,
-            'unique_filename': False,
-            'invalidate': True,
-            'resource_type': 'auto'
+            **get_default_upload_options(self.folder_mode),
+            **group_params(
+                self.optional_parameter,
+                ((k, parse_option_value(v)) for k, v in self.optional_parameter_parsed))
         }
+
         upload_results = {}
         upload_errors = {}
         uploads = []
         for file in files_to_push:
-            folder = get_destination_folder(self.remote_dir, file)
+            folder_options = get_destination_folder_options(file, self.remote_dir, self.folder_mode)
 
             uploads.append(
-                (self.local_files[file]['path'], {**options, 'folder': folder}, upload_results, upload_errors))
+                (self.local_files[file]['path'], {**options, **folder_options}, upload_results, upload_errors))
 
         try:
             run_tasks_concurrently(upload_file, uploads, self.concurrent_workers)
@@ -186,7 +199,8 @@ def _print_sync_status(self, success, errors):
 
     def _save_sync_meta_file(self, upload_results):
         diverse_filenames = {}
-        for local_path, remote_path in upload_results.items():
+        for local_path, remote_res in upload_results.items():
+            remote_path = remote_res["display_path"] if self.folder_mode == "dynamic" else remote_res["path"]
             local = normalize_file_extension(posix_rel_path(local_path, self.local_dir))
             remote = normalize_file_extension(posix_rel_path(remote_path, self.remote_dir))
             if local != remote:
@@ -254,7 +268,10 @@ def _get_out_of_sync_file_names(self, common_file_names):
                 out_of_sync_file_names.add(f)
                 continue
             logger.debug(f"'{f}' is in sync" +
-                         (f" with '{self.diverse_file_names[f]}'" if f in self.diverse_file_names else ""))
+                         (f" with '{self.diverse_file_names[f]}'" if f in self.diverse_file_names else "") +
+                         (f". Public ID: {self.recovered_remote_files[f]['public_id']}"
+                          if self.folder_mode == "dynamic" else "")
+                         )
 
         return out_of_sync_file_names
 

cloudinary_cli/modules/upload_dir.py

@@ -2,9 +2,10 @@
 from os.path import dirname, join as path_join
 from pathlib import Path
 
-from click import command, argument, option, style, launch
+from click import command, argument, option, style, launch, Choice
 
-from cloudinary_cli.utils.api_utils import upload_file
+from cloudinary_cli.utils.api_utils import upload_file, get_default_upload_options, get_folder_mode, \
+    get_destination_folder_options
 from cloudinary_cli.utils.file_utils import get_destination_folder, is_hidden_path
 from cloudinary_cli.utils.utils import parse_option_value, logger, run_tasks_concurrently, group_params
 
@@ -25,6 +26,8 @@
              "The path you specify will be pre-pended to the public IDs of the uploaded assets. "
              "You can specify a whole path, for example path1/path2/path3. "
              "Any folders that do not exist are automatically created.")
+@option("-fm", "--folder-mode", type=Choice(['fixed', 'dynamic'], case_sensitive=False),
+        help="Specify folder mode explicitly. By default uses cloud mode configured in your cloud.", hidden=True)
 @option("-p", "--preset", help="The upload preset to use.")
 @option("-e", "--exclude-dir-name", is_flag=True, default=False,
         help="When this option is used, the contents of the parent directory are uploaded but not the parent "
@@ -33,7 +36,7 @@
 @option("-w", "--concurrent_workers", type=int, default=30, help="Specify the number of concurrent network threads.")
 @option("-d", "--doc", is_flag=True, help="Open upload_dir command documentation page.")
 def upload_dir(directory, glob_pattern, include_hidden, optional_parameter, optional_parameter_parsed, transformation,
-               folder, preset, concurrent_workers, exclude_dir_name, doc):
+               folder, folder_mode, preset, concurrent_workers, exclude_dir_name, doc):
     items, skipped = {}, {}
 
     if doc:
@@ -44,24 +47,27 @@ def upload_dir(directory, glob_pattern, include_hidden, optional_parameter, opti
         logger.error(f"Directory: {dir_to_upload} does not exist")
         return False
 
+    folder_mode = folder_mode or get_folder_mode()
+
     if exclude_dir_name:
-        logger.info(f"Uploading contents of directory '{dir_to_upload}'")
+        contents_str = "contents of"
         parent = dir_to_upload
     else:
-        logger.info(f"Uploading directory '{dir_to_upload}'")
+        contents_str = ""
         parent = dirname(dir_to_upload)
 
-    defaults = {
-        "resource_type": "auto",
-        "invalidate": True,
-        "unique_filename": False,
-        "use_filename": True,
+    logger.info(f"Uploading {contents_str} directory '{dir_to_upload}' ({folder_mode} folder mode)")
+
+    defaults = get_default_upload_options(folder_mode)
+
+    upload_dir_options = {
         "raw_transformation": transformation,
         "upload_preset": preset
     }
 
     options = {
         **defaults,
+        **upload_dir_options,
         **group_params(optional_parameter, ((k, parse_option_value(v)) for k, v in optional_parameter_parsed)),
     }
 
@@ -72,8 +78,8 @@ def upload_dir(directory, glob_pattern, include_hidden, optional_parameter, opti
             if not include_hidden and is_hidden_path(file_path):
                 continue
 
-            options = {**options, "folder": get_destination_folder(folder, str(file_path), parent=parent)}
-            uploads.append((file_path, options, items, skipped))
+            folder_options = get_destination_folder_options(str(file_path), folder, folder_mode, parent)
+            uploads.append((file_path, {**options, **folder_options}, items, skipped))
 
     run_tasks_concurrently(upload_file, uploads, concurrent_workers)
 

cloudinary_cli/utils/api_utils.py

@@ -3,12 +3,12 @@
 
 import requests
 from click import style, launch
-from cloudinary import Search, uploader
+from cloudinary import Search, uploader, api
 from cloudinary.utils import cloudinary_url
 
 from cloudinary_cli.defaults import logger
 from cloudinary_cli.utils.config_utils import is_valid_cloudinary_config
-from cloudinary_cli.utils.file_utils import normalize_file_extension, posix_rel_path
+from cloudinary_cli.utils.file_utils import normalize_file_extension, posix_rel_path, get_destination_folder
 from cloudinary_cli.utils.json_utils import print_json, write_json_to_file
 from cloudinary_cli.utils.utils import log_exception, confirm_action, get_command_params, merge_responses, \
     normalize_list_params, ConfigurationError, print_api_help
@@ -18,7 +18,7 @@
 _cursor_fields = {"resource": "derived_next_cursor"}
 
 
-def query_cld_folder(folder):
+def query_cld_folder(folder, folder_mode):
     files = {}
 
     folder = folder.strip('/')  # omit redundant leading slash and duplicate trailing slashes in query
@@ -31,15 +31,21 @@ def query_cld_folder(folder):
         res = expression.execute()
 
         for asset in res['resources']:
-            rel_path = posix_rel_path(asset_source(asset), folder)
-            files[normalize_file_extension(rel_path)] = {
+            rel_path = _relative_path(asset, folder)
+            rel_display_path = _relative_display_path(asset, folder)
+            path_key = rel_display_path if folder_mode == "dynamic" else rel_path
+            files[normalize_file_extension(path_key)] = {
                 "type": asset['type'],
                 "resource_type": asset['resource_type'],
                 "public_id": asset['public_id'],
                 "format": asset['format'],
                 "etag": asset.get('etag', '0'),
                 "relative_path": rel_path,  # save for inner use
                 "access_mode": asset.get('access_mode', 'public'),
+                # dynamic folder mode fields
+                "asset_folder": asset.get('asset_folder'),
+                "display_name": asset.get('display_name'),
+                "relative_display_path": rel_display_path
             }
         # use := when switch to python 3.8
         next_cursor = res.get('next_cursor')
@@ -48,6 +54,28 @@ def query_cld_folder(folder):
     return files
 
 
+def _display_path(asset):
+    if asset.get("display_name") is None:
+        return ""
+
+    return "/".join([asset["asset_folder"], ".".join([asset["display_name"], asset["format"]])])
+
+
+def _relative_display_path(asset, folder):
+    if asset.get("display_name") is None:
+        return ""
+
+    return posix_rel_path(_display_path(asset), folder)
+
+
+def _relative_path(asset, folder):
+    source = asset_source(asset)
+    if not source.startswith(folder):
+        return source
+
+    return posix_rel_path(asset_source(asset), folder)
+
+
 def regen_derived_version(public_id, delivery_type, res_type,
                           eager_trans, eager_async,
                           eager_notification_url):
@@ -78,15 +106,49 @@ def upload_file(file_path, options, uploaded=None, failed=None):
         if size > 20000000:
             upload_func = uploader.upload_large
         result = upload_func(file_path, **options)
-        logger.info(style(f"Successfully uploaded {file_path} as {result['public_id']}", fg="green"))
+        disp_path = _display_path(result)
+        disp_str = f"as {result['public_id']}" if not disp_path \
+            else f"as {disp_path} with public_id: {result['public_id']}"
+        logger.info(style(f"Successfully uploaded {file_path} {disp_str}", fg="green"))
         if verbose:
             print_json(result)
-        uploaded[file_path] = asset_source(result)
+        uploaded[file_path] = {"path": asset_source(result), "display_path": disp_path}
     except Exception as e:
         log_exception(e, f"Failed uploading {file_path}")
         failed[file_path] = str(e)
 
 
+def get_default_upload_options(folder_mode):
+    options = {
+        'resource_type': 'auto'
+    }
+
+    if folder_mode == 'fixed':
+        options = {
+            **options,
+            'use_filename': True,
+            'unique_filename': False,
+            'invalidate': True,
+        }
+
+    if folder_mode == 'dynamic':
+        options = {
+            **options,
+            'use_filename_as_display_name': True,
+        }
+
+    return options
+
+
+def get_destination_folder_options(file, remote_dir, folder_mode, parent=None):
+    destination_folder = get_destination_folder(remote_dir, file, parent)
+
+    if folder_mode == "dynamic":
+        return {"asset_folder": destination_folder}
+
+    return {"folder": destination_folder}
+
+
 def download_file(remote_file, local_path, downloaded=None, failed=None):
     downloaded = downloaded if downloaded is not None else {}
     failed = failed if failed is not None else {}
@@ -134,12 +196,29 @@ def asset_source(asset_details):
     :return:
     """
     base_name = asset_details['public_id']
+
     if asset_details['resource_type'] == 'raw' or asset_details['type'] == 'fetch':
         return base_name
 
     return base_name + '.' + asset_details['format']
 
 
+def get_folder_mode():
+    """
+    Returns folder mode of the cloud.
+
+    :return: String representing folder mode. Can be "fixed" or "dynamic".
+    """
+    try:
+        config_res = api.config(settings="true")
+        mode = config_res["settings"]["folder_mode"]
+    except Exception as e:
+        log_exception(e, f"Failed getting cloud configuration")
+        raise
+
+    return mode
+
+
 def call_api(func, args, kwargs):
     try:
         return func(*args, **kwargs)

requirements.txt

@@ -1,4 +1,4 @@
-cloudinary>=1.33.0
+cloudinary>=1.35.0
 pygments
 jinja2
 click

test/helper_test.py

@@ -43,7 +43,8 @@ def uploader_response_mock():
       "resource_type": "raw",
       "type": "upload",
       "format":"bin",
-      "foo": "bar"
+      "foo": "bar",
+      "resources": []
     }''')