Storage soft-delete feature (#5515)

* rebased soft-delete work with latest cli and sdk updates

* tests and history

* re-recorded storage tests, fixes due to service changes, version, history

* review feedback, remove test version constraint

* re-recorded tests with new shell

* new recording for blob incremental copy

Author: williexu

src/azure-cli-core/HISTORY.rst

@@ -6,7 +6,6 @@ Release History
 ++++++
 * auth: key on both subscription id and name on msi login
 * Add events module in core for EVENT_INVOKER_PRE_CMD_TBL_TRUNCATE
-* Minor fixes
 
 2.0.26
 ++++++

src/azure-cli-core/azure/cli/core/profiles/_shared.py

@@ -79,7 +79,7 @@ def __init__(self, default_api_version, profile=None):
         ResourceType.MGMT_RESOURCE_POLICY: '2017-06-01-preview',
         ResourceType.MGMT_RESOURCE_RESOURCES: '2017-05-10',
         ResourceType.MGMT_RESOURCE_SUBSCRIPTIONS: '2016-06-01',
-        ResourceType.DATA_STORAGE: '2017-04-17',
+        ResourceType.DATA_STORAGE: '2017-07-29',
         ResourceType.DATA_COSMOS_TABLE: '2017-04-17'
     },
     '2017-03-09-profile': {

src/command_modules/azure-cli-storage/HISTORY.rst

@@ -5,7 +5,7 @@ Release History
 
 2.0.25
 ++++++
-* Minor fixes.
+* Added `storage blob service-properties delete-policy` and `storage blob undelete` commands to enable soft-delete.
 
 2.0.24
 ++++++

src/command_modules/azure-cli-storage/azure/cli/command_modules/storage/_help.py

@@ -196,8 +196,7 @@
     short-summary: List blobs in a given container.
     parameters:
         - name: --include
-          short-summary: 'Specifies additional datasets to include: (c)opy-info, (m)etadata, (s)napshots. Can be
-                         combined.'
+          short-summary: 'Specifies additional datasets to include: (c)opy-info, (m)etadata, (s)napshots, (d)eleted-soft. Can be combined.'
 """
 
 helps['storage blob copy'] = """
@@ -225,6 +224,19 @@
     short-summary: Manage storage blob service properties.
 """
 
+helps['storage blob service-properties delete-policy'] = """
+    type: group
+    short-summary: Manage storage blob delete-policy service properties.
+"""
+helps['storage blob service-properties delete-policy show'] = """
+    type: command
+    short-summary: Show the storage blob delete-policy.
+"""
+helps['storage blob service-properties delete-policy update'] = """
+    type: command
+    short-summary: Update the storage blob delete-policy.
+"""
+
 helps['storage blob set-tier'] = """
     type: command
     short-summary: Set the block or page tiers on the blob.

src/command_modules/azure-cli-storage/azure/cli/command_modules/storage/_params.py

@@ -220,6 +220,11 @@ def load_arguments(self, _):  # pylint: disable=too-many-locals, too-many-statem
         c.argument('tier', validator=blob_tier_validator)
         c.argument('timeout', type=int)
 
+    with self.argument_context('storage blob service-properties delete-policy update') as c:
+        c.argument('enable', arg_type=get_enum_type(['true', 'false']), help='Enables/disables soft-delete.')
+        c.argument('days_retained', type=int,
+                   help='Number of days that soft-deleted blob will be retained. Must be in range [1,365].')
+
     with self.argument_context('storage blob upload') as c:
         from ._validators import page_blob_tier_validator
         from .sdkutil import get_blob_types, get_blob_tier_names

src/command_modules/azure-cli-storage/azure/cli/command_modules/storage/_validators.py

@@ -468,11 +468,11 @@ def validator(namespace):
 def validate_included_datasets(cmd, namespace):
     if namespace.include:
         include = namespace.include
-        if set(include) - set('cms'):
-            help_string = '(c)opy-info (m)etadata (s)napshots'
+        if set(include) - set('cmsd'):
+            help_string = '(c)opy-info (m)etadata (s)napshots (d)eleted'
             raise ValueError('valid values are {} or a combination thereof.'.format(help_string))
         t_blob_include = cmd.get_models('blob#Include')
-        namespace.include = t_blob_include('s' in include, 'm' in include, False, 'c' in include)
+        namespace.include = t_blob_include('s' in include, 'm' in include, False, 'c' in include, 'd' in include)
 
 
 def validate_key(namespace):

src/command_modules/azure-cli-storage/azure/cli/command_modules/storage/commands.py

@@ -104,6 +104,8 @@ def get_custom_sdk(custom_module, client_factory, resource_type=ResourceType.DAT
         g.storage_command('exists', 'exists', transform=create_boolean_result_output_transformer('exists'))
         g.storage_command('delete', 'delete_blob', transform=create_boolean_result_output_transformer('deleted'),
                           table_transformer=transform_boolean_for_table)
+        g.storage_command('undelete', 'undelete_blob', transform=create_boolean_result_output_transformer('undeleted'),
+                          table_transformer=transform_boolean_for_table, min_api='2017-07-29')
 
         g.storage_custom_command('set-tier', 'set_blob_tier')
         g.storage_custom_command('upload', 'upload_blob',
@@ -139,6 +141,14 @@ def get_custom_sdk(custom_module, client_factory, resource_type=ResourceType.DAT
                             min_api='2016-05-31') as g:
         g.storage_command('cancel', 'abort_copy_blob')
 
+    with self.command_group('storage blob service-properties delete-policy', command_type=base_blob_sdk,
+                            min_api='2017-07-29',
+                            custom_command_type=get_custom_sdk('blob', blob_data_service_factory)) as g:
+        g.storage_command('show', 'get_blob_service_properties',
+                          transform=lambda x: getattr(x, 'delete_retention_policy', x),
+                          exception_handler=g.get_handler_suppress_404())
+        g.storage_custom_command('update', 'set_delete_policy')
+
     with self.command_group('storage blob service-properties', command_type=base_blob_sdk) as g:
         g.storage_command('show', 'get_blob_service_properties', exception_handler=g.get_handler_suppress_404())
 

src/command_modules/azure-cli-storage/azure/cli/command_modules/storage/operations/blob.py

@@ -27,6 +27,22 @@ def set_blob_tier(client, container_name, blob_name, tier, blob_type='block', ti
         raise ValueError('Blob tier is only applicable to block or page blob.')
 
 
+def set_delete_policy(client, enable=None, days_retained=None):
+    policy = client.get_blob_service_properties().delete_retention_policy
+
+    if enable is not None:
+        policy.enabled = enable == 'true'
+    if days_retained is not None:
+        policy.days = days_retained
+
+    if policy.enabled and not policy.days:
+        from knack.util import CLIError
+        raise CLIError("must specify days-retained")
+
+    client.set_blob_service_properties(delete_retention_policy=policy)
+    return client.get_blob_service_properties().delete_retention_policy
+
+
 def storage_blob_copy_batch(cmd, client, source_client,
                             destination_container=None, source_container=None, source_share=None,
                             source_sas=None, pattern=None, dryrun=False):