[Storage] Fix #29929: az storage copy: Fix when wildcard * is in --source-file-path (#30569)

* fix `az storage copy` with wildcard '*' in source share file path

* fix cases when * is in --source-file-path

* fix cases when * is in --source-blob or --source-container

Author: calvinhzy

src/azure-cli/azure/cli/command_modules/storage/_validators.py

@@ -2025,18 +2025,28 @@ def get_url_with_sas(cmd, namespace, url=None, container=None, blob=None, share=
         client = cf_blob_service(cmd.cli_ctx, kwargs)
         if blob is None:
             blob = ''
-        from .operations.blob import create_blob_url
-        url = create_blob_url(client, container, blob, snapshot=None)
+        if '*' in container or '*' in blob:
+            url = client.url
+            if not url.endswith('/'):
+                url = url + '/'
+            url = url + container + '/' + blob
+        else:
+            from .operations.blob import create_blob_url
+            url = create_blob_url(client, container, blob, snapshot=None)
         service = 'blob'
     elif share:
         if hasattr(namespace, 'enable_file_backup_request_intent'):
             kwargs.update({'enable_file_backup_request_intent': namespace.enable_file_backup_request_intent})
         client = cf_share_service(cmd.cli_ctx, kwargs)
         client = client.get_share_client(share)
-        dir_name, file_name = os.path.split(file_path) if file_path else (None, '')
-        dir_name = None if dir_name in ('', '.') else dir_name
-        from .operations.file import create_file_url
-        url = create_file_url(client, directory_name=dir_name, file_name=file_name)
+        # if wildcard '*' in file path, skip manually parsing the url
+        if file_path and '*' in file_path:
+            url = client.url + '/' + file_path
+        else:
+            dir_name, file_name = os.path.split(file_path) if file_path else (None, '')
+            dir_name = None if dir_name in ('', '.') else dir_name
+            from .operations.file import create_file_url
+            url = create_file_url(client, directory_name=dir_name, file_name=file_name)
         service = 'file'
     elif not any([url, container, share]):  # In account level, only blob service is supported
         service = 'blob'

src/azure-cli/azure/cli/command_modules/storage/tests/latest/test_storage_azcopy_scenarios.py

@@ -615,9 +615,8 @@ def test_storage_azcopy_blob_url(self, resource_group, first_account, second_acc
         local_file = self.create_temp_file(20480.5)
         self.cmd('disk create -n {} -g {} --for-upload --upload-size-bytes 20972032'
                  .format(diskname, resource_group))
-        sasURL = self.cmd(
-            'disk grant-access --access-level Write --duration-in-seconds 3600 -n {} -g {} --query accessSas -o tsv'
-            .format(diskname, resource_group)).output.strip('\n')
+        sasURL = self.cmd('disk grant-access --access-level Write --duration-in-seconds 3600 -n {} -g {} -o tsv'
+                          .format(diskname, resource_group)).output.strip('\n')
         self.cmd('storage copy -s "{}" -d "{}" --blob-type PageBlob'
                  .format(local_file, sasURL))
 
@@ -632,6 +631,7 @@ def test_storage_azcopy_blob_account(self, resource_group, first_account, second
 
         first_container = self.create_container(first_account_info)
         second_container = self.create_container(second_account_info)
+        third_container = self.create_container(second_account_info)
 
         first_account_url = 'https://{}.blob.core.windows.net'.format(first_account)
 
@@ -708,13 +708,20 @@ def test_storage_azcopy_blob_account(self, resource_group, first_account, second
                  .format(second_container, second_account), checks=JMESPathCheck('length(@)', 11))
 
         # Copy an entire storage account data to another blob account
-        self.cmd('storage copy --source-account-name {} --destination-account-name {} --recursive --preserve-s2s-access-tier false'
-                 .format(first_account, second_account))
+        self.cmd('storage copy --source-account-name {} --destination-account-name {} --recursive '
+                 '--preserve-s2s-access-tier false'.format(first_account, second_account))
         self.cmd('storage container list --account-name {}'
-                 .format(second_account), checks=JMESPathCheck('length(@)', 2))
+                 .format(second_account), checks=JMESPathCheck('length(@)', 3))
         self.cmd('storage blob list -c {} --account-name {}'
                  .format(first_container, second_account), checks=JMESPathCheck('length(@)', 22))
 
+        # Copy an entire directory from blob virtual directory to another blob virtual directory with wildcard
+        self.cmd('storage copy --source-account-name {} --source-container {} --source-blob {} '
+                 '--destination-account-name {} --destination-container {} --recursive --preserve-s2s-access-tier false'
+                 .format(second_account, second_container, 'apple/*', second_account, third_container))
+        self.cmd('storage blob list -c {} --account-name {}'
+                 .format(third_container, second_account), checks=JMESPathCheck('length(@)', 10))
+
     @ResourceGroupPreparer()
     @StorageAccountPreparer(parameter_name='first_account')
     @StorageAccountPreparer(parameter_name='second_account', sku='Premium_LRS', kind='BlockBlobStorage')
@@ -881,6 +888,26 @@ def test_storage_azcopy_file_account(self, resource_group, storage_account_info,
         self.assertEqual(3, sum(len(d) for r, d, f in os.walk(local_folder)))
         self.assertEqual(21, sum(len(f) for r, d, f in os.walk(local_folder)))
 
+        # Copy with wildcard *
+        self.cmd('storage directory create -s {} -n {} --account-name {}'.format(share, 'parent', storage_account))
+        self.cmd('storage directory create -s {} -n {} --account-name {}'
+                 .format(share, 'parent/source', storage_account))
+        self.cmd('storage copy --source-local-path "{}" --destination-account-name {} --destination-share {} '
+                 '--destination-file-path {} --recursive'
+                 .format(os.path.join(test_dir, 'butter/file_*'), storage_account, share, 'parent/source'))
+        self.cmd('storage copy --source-account-name {} --source-share {} --source-file-path {} --account-name {} '
+                 '--destination-share {} --destination-file-path {} --recursive -- --as-subdir=false'
+                 .format(storage_account, share, 'parent/source/*', storage_account, share, 'parent/target'))
+        self.cmd('storage file list -s {} --path {} --account-name {}'.format(
+            share, 'parent/target', storage_account), checks=JMESPathCheck('length(@)', 10))
+        self.cmd('storage copy --source-account-name {} --source-share {} --source-file-path {} --account-name {} '
+                 '--destination-share {} --destination-file-path {} --recursive -- --as-subdir=false'
+                 .format(storage_account, share, 'parent/*', storage_account, share, 'parentdst'))
+        self.cmd('storage file list -s {} --path {} --account-name {}'
+                 .format(share, 'parentdst/source', storage_account), checks=JMESPathCheck('length(@)', 10))
+        self.cmd('storage file list -s {} --path {} --account-name {}'
+                 .format(share, 'parentdst/target', storage_account), checks=JMESPathCheck('length(@)', 10))
+
     @ResourceGroupPreparer()
     @StorageAccountPreparer(parameter_name='first_account', allow_shared_key_access=False)
     @StorageAccountPreparer(parameter_name='second_account', sku='Premium_LRS', kind='BlockBlobStorage',
@@ -972,9 +999,8 @@ def test_storage_azcopy_blob_url_oauth(self, resource_group, first_account, seco
         local_file = self.create_temp_file(20480.5)
         self.cmd('disk create -n {} -g {} --for-upload --upload-size-bytes 20972032'
                  .format(diskname, resource_group))
-        sasURL = self.cmd(
-            'disk grant-access --access-level Write --duration-in-seconds 3600 -n {} -g {} --query accessSas -o tsv'
-            .format(diskname, resource_group)).output.strip('\n')
+        sasURL = self.cmd('disk grant-access --access-level Write --duration-in-seconds 3600 -n {} -g {} -o tsv'
+                          .format(diskname, resource_group)).output.strip('\n')
         self.oauth_cmd('storage copy -s "{}" -d "{}" --blob-type PageBlob'.format(
             local_file, sasURL))
 
@@ -986,8 +1012,10 @@ def test_storage_azcopy_blob_url_oauth(self, resource_group, first_account, seco
     def test_storage_azcopy_blob_account_oauth(self, resource_group, first_account, second_account, test_dir):
         first_container = self.create_random_name(prefix='container', length=24)
         second_container = self.create_random_name(prefix='container', length=24)
+        third_container = self.create_random_name(prefix='container', length=24)
         self.oauth_cmd('storage container create -n {} --account-name {}', first_container, first_account)
         self.oauth_cmd('storage container create -n {} --account-name {}', second_container, second_account)
+        self.oauth_cmd('storage container create -n {} --account-name {}', third_container, second_account)
 
         first_account_url = 'https://{}.blob.core.windows.net'.format(first_account)
 
@@ -1072,10 +1100,18 @@ def test_storage_azcopy_blob_account_oauth(self, resource_group, first_account,
                        '--preserve-s2s-access-tier false'.format(
             first_account, second_account))
         self.oauth_cmd('storage container list --account-name {}'.format(
-            second_account), checks=JMESPathCheck('length(@)', 2))
+            second_account), checks=JMESPathCheck('length(@)', 3))
         self.oauth_cmd('storage blob list -c {} --account-name {}'.format(
             first_container, second_account), checks=JMESPathCheck('length(@)', 22))
 
+        # Copy an entire directory from blob virtual directory to another blob virtual directory with wildcard
+        self.oauth_cmd('storage copy --source-account-name {} --source-container {} --source-blob {} '
+                       '--destination-account-name {} --destination-container {} --recursive '
+                       '--preserve-s2s-access-tier false'
+                       .format(second_account, second_container, 'apple/*', second_account, third_container))
+        self.oauth_cmd('storage blob list -c {} --account-name {}'
+                       .format(third_container, second_account), checks=JMESPathCheck('length(@)', 10))
+
     @ResourceGroupPreparer()
     @StorageAccountPreparer(parameter_name='first_account', allow_shared_key_access=False)
     @StorageAccountPreparer(parameter_name='second_account', sku='Premium_LRS', kind='BlockBlobStorage',
@@ -1236,3 +1272,24 @@ def test_storage_azcopy_file_account_oauth(self, resource_group, storage_account
             storage_account, share, local_folder))
         self.assertEqual(3, sum(len(d) for r, d, f in os.walk(local_folder)))
         self.assertEqual(21, sum(len(f) for r, d, f in os.walk(local_folder)))
+
+        # Copy with wildcard *
+        self.file_oauth_cmd('storage directory create -s {} -n {} --account-name {}'
+                            .format(share, 'parent', storage_account))
+        self.file_oauth_cmd('storage directory create -s {} -n {} --account-name {}'
+                            .format(share, 'parent/source', storage_account))
+        self.oauth_cmd('storage copy --source-local-path "{}" --destination-account-name {} --destination-share {} '
+                       '--destination-file-path {} --recursive'
+                       .format(os.path.join(test_dir, 'butter/file_*'), storage_account, share, 'parent/source'))
+        self.cmd('storage copy --source-account-name {} --source-share {} --source-file-path {} --account-name {} '
+                 '--destination-share {} --destination-file-path {} --recursive --auth-mode login -- --as-subdir=false'
+                 .format(storage_account, share, 'parent/source/*', storage_account, share, 'parent/target'))
+        self.file_oauth_cmd('storage file list -s {} --path {} --account-name {}'
+                            .format(share, 'parent/target', storage_account), checks=JMESPathCheck('length(@)', 10))
+        self.cmd('storage copy --source-account-name {} --source-share {} --source-file-path {} --account-name {} '
+                 '--destination-share {} --destination-file-path {} --recursive --auth-mode login -- --as-subdir=false'
+                 .format(storage_account, share, 'parent/*', storage_account, share, 'parentdst'))
+        self.file_oauth_cmd('storage file list -s {} --path {} --account-name {}'
+                            .format(share, 'parentdst/source', storage_account), checks=JMESPathCheck('length(@)', 10))
+        self.file_oauth_cmd('storage file list -s {} --path {} --account-name {}'
+                            .format(share, 'parentdst/target', storage_account), checks=JMESPathCheck('length(@)', 10))