[Storage] Refactor Blob download to address Unicode issues (#35740)

Author: jalauzon-msft

sdk/storage/azure-storage-blob/CHANGELOG.md

@@ -3,7 +3,9 @@
 ## 12.21.0 (Unreleased)
 
 ### Features Added
-
+- Added new `chars` keyword to the `StorageStreamDownloader.read` method to support reading an arbitrary number of
+characters from the stream rather than bytes. This can only be used when `encoding` is specified on `download_blob`
+but can help prevent decoding errors in certain scenarios.
 
 ## 12.21.0b1 (2024-06-11)
 

sdk/storage/azure-storage-blob/assets.json

@@ -2,5 +2,5 @@
   "AssetsRepo": "Azure/azure-sdk-assets",
   "AssetsRepoPrefixPath": "python",
   "TagPrefix": "python/storage/azure-storage-blob",
-  "Tag": "python/storage/azure-storage-blob_275000b78a"
+  "Tag": "python/storage/azure-storage-blob_4bb162f320"
 }

sdk/storage/azure-storage-blob/azure/storage/blob/_download.py

@@ -859,6 +859,8 @@ async def test_get_blob_read_with_other_read_operations_ranged(self, **kwargs):
         await self._setup(storage_account_name, storage_account_key)
         self.bsc.require_encryption = True
         self.bsc.key_encryption_key = KeyWrapper('key1')
+        self.bsc._config.max_single_get_size = 1024
+        self.bsc._config.max_chunk_get_size = 1024
 
         data = b'12345' * 205 * 10  # 10250 bytes
         blob = self.bsc.get_blob_client(self.container_name, self._get_blob_reference(BlobType.BLOCKBLOB))

sdk/storage/azure-storage-blob/azure/storage/blob/aio/_download_async.py

@@ -1098,6 +1098,41 @@ def test_get_blob_read_with_other_read_operations_ranged(self, **kwargs):
         assert second == data[offset + read_size:offset + length]
         assert read_length == len(second)
 
+    @pytest.mark.live_test_only
+    @BlobPreparer()
+    def test_get_blob_using_read_chars(self, **kwargs):
+        storage_account_name = kwargs.pop("storage_account_name")
+        storage_account_key = kwargs.pop("storage_account_key")
+
+        self._setup(storage_account_name, storage_account_key)
+        kek = KeyWrapper('key1')
+        bsc = BlobServiceClient(
+            self.account_url(storage_account_name, "blob"),
+            credential=storage_account_key,
+            max_single_get_size=1024,
+            max_chunk_get_size=1024,
+            require_encryption=True,
+            encryption_version='2.0',
+            key_encryption_key=kek)
+
+        blob = bsc.get_blob_client(self.container_name, self._get_blob_reference())
+        data = '你好世界' * 1024  # 12 KiB
+        blob.upload_blob(data, overwrite=True, encoding='utf-8')
+
+        # Act / Assert
+        stream = blob.download_blob(max_concurrency=2, encoding='utf-8')
+        assert stream.read() == data
+
+        result = ''
+        stream = blob.download_blob(encoding='utf-8')
+        for _ in range(4):
+            chunk = stream.read(chars=300)
+            result += chunk
+            assert len(chunk) == 300
+
+        result += stream.readall()
+        assert result == data
+
     @pytest.mark.skip(reason="Intended for manual testing due to blob size.")
     @pytest.mark.live_test_only
     @BlobPreparer()

sdk/storage/azure-storage-blob/tests/test_blob_encryption_async.py

@@ -1110,6 +1110,41 @@ async def test_get_blob_read_with_other_read_operations_ranged(self, **kwargs):
         assert second == data[offset + read_size:offset + length]
         assert read_length == len(second)
 
+    @pytest.mark.live_test_only
+    @BlobPreparer()
+    async def test_get_blob_using_read_chars(self, **kwargs):
+        storage_account_name = kwargs.pop("storage_account_name")
+        storage_account_key = kwargs.pop("storage_account_key")
+
+        await self._setup(storage_account_name, storage_account_key)
+        kek = KeyWrapper('key1')
+        bsc = BlobServiceClient(
+            self.account_url(storage_account_name, "blob"),
+            credential=storage_account_key,
+            max_single_get_size=1024,
+            max_chunk_get_size=1024,
+            require_encryption=True,
+            encryption_version='2.0',
+            key_encryption_key=kek)
+
+        blob = bsc.get_blob_client(self.container_name, self._get_blob_reference())
+        data = '你好世界' * 1024  # 12 KiB
+        await blob.upload_blob(data, overwrite=True, encoding='utf-8')
+
+        # Act / Assert
+        stream = await blob.download_blob(max_concurrency=2, encoding='utf-8')
+        assert await stream.read() == data
+
+        result = ''
+        stream = await blob.download_blob(encoding='utf-8')
+        for _ in range(4):
+            chunk = await stream.read(chars=300)
+            result += chunk
+            assert len(chunk) == 300
+
+        result += await stream.readall()
+        assert result == data
+
     @pytest.mark.skip(reason="Intended for manual testing due to blob size.")
     @pytest.mark.live_test_only
     @BlobPreparer()