[Storage] STG 82 (Azure#23937)

Author: jalauzon-msft

sdk/storage/azure-storage-blob/CHANGELOG.md

@@ -1,8 +1,11 @@
 # Release History
 
-## 12.12.0b1 (Unreleased)
+## 12.12.0b1 (2022-04-14)
 
 ### Features Added
+- Added support for service version 2021-06-08.
+- Added a new paginated method for listing page ranges, `list_page_ranges()`. This replaces `get_page_ranges()` which has been deprecated.
+- Added support for copying source blob tags with `start_copy_from_url()` by specifying `"COPY"` for the `tags` keyword.
 
 ## 12.11.0 (2022-03-29)
 

sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py

@@ -10,13 +10,15 @@
     Union, Optional, Any, IO, Iterable, AnyStr, Dict, List, Tuple,
     TYPE_CHECKING,
     TypeVar, Type)
+import warnings
 
 try:
     from urllib.parse import urlparse, quote, unquote
 except ImportError:
     from urlparse import urlparse  # type: ignore
     from urllib2 import quote, unquote  # type: ignore
 import six
+from azure.core.paging import ItemPaged
 from azure.core.pipeline import Pipeline
 from azure.core.tracing.decorator import distributed_trace
 from azure.core.exceptions import ResourceNotFoundError, HttpResponseError, ResourceExistsError
@@ -55,7 +57,7 @@
     upload_append_blob,
     upload_page_blob, _any_conditions)
 from ._models import BlobType, BlobBlock, BlobProperties, BlobQueryError, QuickQueryDialect, \
-    DelimitedJsonDialect, DelimitedTextDialect
+    DelimitedJsonDialect, DelimitedTextDialect, PageRangePaged, PageRange
 from ._download import StorageStreamDownloader
 from ._lease import BlobLeaseClient
 
@@ -1866,37 +1868,52 @@ def _start_copy_from_url_options(self, source_url, metadata=None, incremental_co
         if 'source_lease' in kwargs:
             source_lease = kwargs.pop('source_lease')
             try:
-                headers['x-ms-source-lease-id'] = source_lease.id # type: str
+                headers['x-ms-source-lease-id'] = source_lease.id
             except AttributeError:
                 headers['x-ms-source-lease-id'] = source_lease
 
         tier = kwargs.pop('premium_page_blob_tier', None) or kwargs.pop('standard_blob_tier', None)
+        tags = kwargs.pop('tags', None)
+
+        # Options only available for sync copy
         requires_sync = kwargs.pop('requires_sync', None)
         encryption_scope_str = kwargs.pop('encryption_scope', None)
         source_authorization = kwargs.pop('source_authorization', None)
+        # If tags is a str, interpret that as copy_source_tags
+        copy_source_tags = isinstance(tags, str)
+
+        if incremental_copy:
+            if source_authorization:
+                raise ValueError("Source authorization tokens are not applicable for incremental copying.")
+            if copy_source_tags:
+                raise ValueError("Copying source tags is not applicable for incremental copying.")
 
-        if not requires_sync and encryption_scope_str:
-            raise ValueError("Encryption_scope is only supported for sync copy, please specify requires_sync=True")
-        if source_authorization and incremental_copy:
-            raise ValueError("Source authorization tokens are not applicable for incremental copying.")
-        #
         # TODO: refactor start_copy_from_url api in _blob_client.py. Call _generated/_blob_operations.py copy_from_url
         #  when requires_sync=True is set.
         #  Currently both sync copy and async copy are calling _generated/_blob_operations.py start_copy_from_url.
         #  As sync copy diverges more from async copy, more problem will surface.
-        if encryption_scope_str:
-            headers.update({'x-ms-encryption-scope': encryption_scope_str})
-
         if requires_sync is True:
             headers['x-ms-requires-sync'] = str(requires_sync)
+            if encryption_scope_str:
+                headers['x-ms-encryption-scope'] = encryption_scope_str
             if source_authorization:
                 headers['x-ms-copy-source-authorization'] = source_authorization
+            if copy_source_tags:
+                headers['x-ms-copy-source-tag-option'] = tags
         else:
+            if encryption_scope_str:
+                raise ValueError(
+                    "Encryption_scope is only supported for sync copy, please specify requires_sync=True")
             if source_authorization:
-                raise ValueError("Source authorization tokens are only applicable for synchronous copy operations.")
+                raise ValueError(
+                    "Source authorization tokens are only supported for sync copy, please specify requires_sync=True")
+            if copy_source_tags:
+                raise ValueError(
+                    "Copying source tags is only supported for sync copy, please specify requires_sync=True")
+
         timeout = kwargs.pop('timeout', None)
         dest_mod_conditions = get_modify_conditions(kwargs)
-        blob_tags_string = serialize_blob_tags_header(kwargs.pop('tags', None))
+        blob_tags_string = serialize_blob_tags_header(tags) if not copy_source_tags else None
 
         immutability_policy = kwargs.pop('immutability_policy', None)
         if immutability_policy:
@@ -1985,11 +2002,14 @@ def start_copy_from_url(self, source_url, metadata=None, incremental_copy=False,
             The tag set may contain at most 10 tags.  Tag keys must be between 1 and 128 characters,
             and tag values must be between 0 and 256 characters.
             Valid tag key and value characters include: lowercase and uppercase letters, digits (0-9),
-            space (` `), plus (+), minus (-), period (.), solidus (/), colon (:), equals (=), underscore (_)
+            space (` `), plus (+), minus (-), period (.), solidus (/), colon (:), equals (=), underscore (_).
+
+            The (case-sensitive) literal "COPY" can instead be passed to copy tags from the source blob.
+            This option is only available when `incremental_copy=False` and `requires_sync=True`.
 
             .. versionadded:: 12.4.0
 
-        :paramtype tags: dict(str, str)
+        :paramtype tags: dict(str, str) or Literal["COPY"]
         :keyword ~azure.storage.blob.ImmutabilityPolicy immutability_policy:
             Specifies the immutability policy of a blob, blob snapshot or blob version.
 
@@ -2879,7 +2899,7 @@ def get_page_ranges( # type: ignore
             **kwargs
         ):
         # type: (...) -> Tuple[List[Dict[str, int]], List[Dict[str, int]]]
-        """Returns the list of valid page ranges for a Page Blob or snapshot
+        """DEPRECATED: Returns the list of valid page ranges for a Page Blob or snapshot
         of a page blob.
 
         :param int offset:
@@ -2934,6 +2954,11 @@ def get_page_ranges( # type: ignore
             The first element are filled page ranges, the 2nd element is cleared page ranges.
         :rtype: tuple(list(dict(str, str), list(dict(str, str))
         """
+        warnings.warn(
+            "get_page_ranges is deprecated, use list_page_ranges instead",
+            DeprecationWarning
+        )
+
         options = self._get_page_ranges_options(
             offset=offset,
             length=length,
@@ -2948,6 +2973,92 @@ def get_page_ranges( # type: ignore
             process_storage_error(error)
         return get_page_ranges_result(ranges)
 
+    @distributed_trace
+    def list_page_ranges(
+            self,
+            *,
+            offset: Optional[int] = None,
+            length: Optional[int] = None,
+            previous_snapshot: Optional[Union[str, Dict[str, Any]]] = None,
+            **kwargs: Any
+        ) -> ItemPaged[PageRange]:
+        """Returns the list of valid page ranges for a Page Blob or snapshot
+        of a page blob. If `previous_snapshot` is specified, the result will be
+        a diff of changes between the target blob and the previous snapshot.
+
+        :keyword int offset:
+            Start of byte range to use for getting valid page ranges.
+            If no length is given, all bytes after the offset will be searched.
+            Pages must be aligned with 512-byte boundaries, the start offset
+            must be a modulus of 512 and the length must be a modulus of
+            512.
+        :keyword int length:
+            Number of bytes to use for getting valid page ranges.
+            If length is given, offset must be provided.
+            This range will return valid page ranges from the offset start up to
+            the specified length.
+            Pages must be aligned with 512-byte boundaries, the start offset
+            must be a modulus of 512 and the length must be a modulus of
+            512.
+        :keyword previous_snapshot:
+            A snapshot value that specifies that the response will contain only pages that were changed
+            between target blob and previous snapshot. Changed pages include both updated and cleared
+            pages. The target blob may be a snapshot, as long as the snapshot specified by `previous_snapshot`
+            is the older of the two.
+        :paramtype previous_snapshot: str or Dict[str, Any]
+        :keyword lease:
+            Required if the blob has an active lease. Value can be a BlobLeaseClient object
+            or the lease ID as a string.
+        :paramtype lease: ~azure.storage.blob.BlobLeaseClient or str
+        :keyword ~datetime.datetime if_modified_since:
+            A DateTime value. Azure expects the date value passed in to be UTC.
+            If timezone is included, any non-UTC datetimes will be converted to UTC.
+            If a date is passed in without timezone info, it is assumed to be UTC.
+            Specify this header to perform the operation only
+            if the resource has been modified since the specified time.
+        :keyword ~datetime.datetime if_unmodified_since:
+            A DateTime value. Azure expects the date value passed in to be UTC.
+            If timezone is included, any non-UTC datetimes will be converted to UTC.
+            If a date is passed in without timezone info, it is assumed to be UTC.
+            Specify this header to perform the operation only if
+            the resource has not been modified since the specified date/time.
+        :keyword str etag:
+            An ETag value, or the wildcard character (*). Used to check if the resource has changed,
+            and act according to the condition specified by the `match_condition` parameter.
+        :keyword ~azure.core.MatchConditions match_condition:
+            The match condition to use upon the etag.
+        :keyword str if_tags_match_condition:
+            Specify a SQL where clause on blob tags to operate only on blob with a matching value.
+            eg. ``\"\\\"tagname\\\"='my tag'\"``
+
+            .. versionadded:: 12.4.0
+
+        :keyword int results_per_page:
+            The maximum number of page ranges to retrieve per API call.
+        :keyword int timeout:
+            The timeout parameter is expressed in seconds.
+        :returns: An iterable (auto-paging) of PageRange.
+        :rtype: ~azure.core.paging.ItemPaged[~azure.storage.blob.PageRange]
+        """
+        results_per_page = kwargs.pop('results_per_page', None)
+        options = self._get_page_ranges_options(
+            offset=offset,
+            length=length,
+            previous_snapshot_diff=previous_snapshot,
+            **kwargs)
+
+        if previous_snapshot:
+            command = partial(
+                self._client.page_blob.get_page_ranges_diff,
+                **options)
+        else:
+            command = partial(
+                self._client.page_blob.get_page_ranges,
+                **options)
+        return ItemPaged(
+            command, results_per_page=results_per_page,
+            page_iterator_class=PageRangePaged)
+
     @distributed_trace
     def get_page_range_diff_for_managed_disk(
             self, previous_snapshot_url,  # type: str

sdk/storage/azure-storage-blob/azure/storage/blob/_generated/aio/operations/_blob_operations.py

@@ -2049,6 +2049,7 @@ async def copy_from_url(  # pylint: disable=inconsistent-return-statements
         immutability_policy_mode: Optional[Union[str, "_models.BlobImmutabilityPolicyMode"]] = None,
         legal_hold: Optional[bool] = None,
         copy_source_authorization: Optional[str] = None,
+        copy_source_tags: Optional[Union[str, "_models.BlobCopySourceTags"]] = None,
         source_modified_access_conditions: Optional["_models.SourceModifiedAccessConditions"] = None,
         modified_access_conditions: Optional["_models.ModifiedAccessConditions"] = None,
         lease_access_conditions: Optional["_models.LeaseAccessConditions"] = None,
@@ -2099,6 +2100,9 @@ async def copy_from_url(  # pylint: disable=inconsistent-return-statements
         :param copy_source_authorization: Only Bearer type is supported. Credentials should be a valid
          OAuth access token to copy source. Default value is None.
         :type copy_source_authorization: str
+        :param copy_source_tags: Optional, default 'replace'.  Indicates if source tags should be
+         copied or replaced with the tags specified by x-ms-tags. Default value is None.
+        :type copy_source_tags: str or ~azure.storage.blob.models.BlobCopySourceTags
         :param source_modified_access_conditions: Parameter group. Default value is None.
         :type source_modified_access_conditions:
          ~azure.storage.blob.models.SourceModifiedAccessConditions
@@ -2178,6 +2182,7 @@ async def copy_from_url(  # pylint: disable=inconsistent-return-statements
             legal_hold=legal_hold,
             copy_source_authorization=copy_source_authorization,
             encryption_scope=_encryption_scope,
+            copy_source_tags=copy_source_tags,
             template_url=self.copy_from_url.metadata['url'],
         )
         request = _convert_request(request)

sdk/storage/azure-storage-blob/azure/storage/blob/_generated/aio/operations/_block_blob_operations.py

@@ -256,6 +256,7 @@ async def put_blob_from_url(  # pylint: disable=inconsistent-return-statements
         blob_tags_string: Optional[str] = None,
         copy_source_blob_properties: Optional[bool] = None,
         copy_source_authorization: Optional[str] = None,
+        copy_source_tags: Optional[Union[str, "_models.BlobCopySourceTags"]] = None,
         blob_http_headers: Optional["_models.BlobHTTPHeaders"] = None,
         lease_access_conditions: Optional["_models.LeaseAccessConditions"] = None,
         cpk_info: Optional["_models.CpkInfo"] = None,
@@ -311,6 +312,9 @@ async def put_blob_from_url(  # pylint: disable=inconsistent-return-statements
         :param copy_source_authorization: Only Bearer type is supported. Credentials should be a valid
          OAuth access token to copy source. Default value is None.
         :type copy_source_authorization: str
+        :param copy_source_tags: Optional, default 'replace'.  Indicates if source tags should be
+         copied or replaced with the tags specified by x-ms-tags. Default value is None.
+        :type copy_source_tags: str or ~azure.storage.blob.models.BlobCopySourceTags
         :param blob_http_headers: Parameter group. Default value is None.
         :type blob_http_headers: ~azure.storage.blob.models.BlobHTTPHeaders
         :param lease_access_conditions: Parameter group. Default value is None.
@@ -427,6 +431,7 @@ async def put_blob_from_url(  # pylint: disable=inconsistent-return-statements
             blob_tags_string=blob_tags_string,
             copy_source_blob_properties=copy_source_blob_properties,
             copy_source_authorization=copy_source_authorization,
+            copy_source_tags=copy_source_tags,
             template_url=self.put_blob_from_url.metadata['url'],
         )
         request = _convert_request(request)