Azure Storage Sample Automation (Azure#39944)

* complete storage automation

* fix storage

* fix

* Create cspell.yaml

* skip mypy for specified line

* Add type annotations for key_bytes and key_id

* Fix container conflicts

* Fix type issues in KeyVaultKey initialization

* Fix container conflicts

* Replace SecretClient with KeyClient in sample

* skip container access policy samples

* create containers in the samples

* Handle existing container creation error

* remove keyvault dependency and skip keyvault sample tests

---------

Co-authored-by: ChenxiJiang333 <[email protected]>
Co-authored-by: ChenxiJiang333 <[email protected]>
Co-authored-by: Tong Xu (MSFT) <[email protected]>

Author: 3 people

scripts/devops_tasks/test_run_samples.py

@@ -153,6 +153,12 @@
         "sample_abstract_summary.py",
         "sample_abstract_summary_async.py",
     ],
+    "azure-storage-blob": [
+        "blob_samples_proxy_configuration.py",
+        "blob_samples_container_access_policy.py",
+        "blob_samples_container_access_policy_async.py",
+        "blob_samples_client_side_encryption_keyvault.py"
+    ],
 }
 
 def run_check_call_with_timeout(

sdk/storage/azure-storage-blob/samples/blob_samples_authentication.py

@@ -16,29 +16,29 @@
 USAGE:
     python blob_samples_authentication.py
     Set the environment variables with your own values before running the sample:
-    1) AZURE_STORAGE_CONNECTION_STRING - the connection string to your storage account
+    1) STORAGE_CONNECTION_STRING - the connection string to your storage account
     2) OAUTH_STORAGE_ACCOUNT_NAME - the oauth storage account name
-    3) AZURE_STORAGE_ACCOUNT_NAME - the name of the storage account
-    4) AZURE_STORAGE_ACCESS_KEY - the storage account access key
+    3) STORAGE_ACCOUNT_NAME - the name of the storage account
+    4) STORAGE_ACCOUNT_KEY - the storage account access key
 """
 
 import os
 import sys
 
 class AuthSamples(object):
     url = "https://{}.blob.core.windows.net".format(
-        os.getenv("AZURE_STORAGE_ACCOUNT_NAME")
+        os.getenv("STORAGE_ACCOUNT_NAME")
     )
     oauth_url = "https://{}.blob.core.windows.net".format(
         os.getenv("OAUTH_STORAGE_ACCOUNT_NAME")
     )
 
-    connection_string = os.getenv("AZURE_STORAGE_CONNECTION_STRING")
-    shared_access_key = os.getenv("AZURE_STORAGE_ACCESS_KEY")
+    connection_string = os.getenv("STORAGE_CONNECTION_STRING")
+    shared_access_key = os.getenv("STORAGE_ACCOUNT_KEY")
 
     def auth_connection_string(self):
         if self.connection_string is None:
-            print("Missing required environment variable: AZURE_STORAGE_CONNECTION_STRING." + '\n' +
+            print("Missing required environment variable: STORAGE_CONNECTION_STRING." + '\n' +
                   "Test: auth_connection_string")
             sys.exit(1)
         # [START auth_from_connection_string]
@@ -63,7 +63,7 @@ def auth_connection_string(self):
 
     def auth_shared_key(self):
         if self.shared_access_key is None:
-            print("Missing required environment variable: AZURE_STORAGE_ACCESS_KEY." + '\n' +
+            print("Missing required environment variable: STORAGE_ACCOUNT_KEY." + '\n' +
                   "Test: auth_shared_key")
             sys.exit(1)
         # [START create_blob_service_client]
@@ -89,7 +89,7 @@ def auth_blob_url(self):
 
     def auth_shared_access_signature(self):
         if self.connection_string is None:
-            print("Missing required environment variable: AZURE_STORAGE_CONNECTION_STRING." + '\n' +
+            print("Missing required environment variable: STORAGE_CONNECTION_STRING." + '\n' +
                   "Test: auth_shared_access_signature")
             sys.exit(1)
         # Instantiate a BlobServiceClient using a connection string

sdk/storage/azure-storage-blob/samples/blob_samples_authentication_async.py

@@ -16,10 +16,10 @@
 USAGE:
     python blob_samples_authentication_async.py
     Set the environment variables with your own values before running the sample:
-    1) AZURE_STORAGE_CONNECTION_STRING - the connection string to your storage account
+    1) STORAGE_CONNECTION_STRING - the connection string to your storage account
     2) OAUTH_STORAGE_ACCOUNT_NAME - the oauth storage account name
-    3) AZURE_STORAGE_ACCOUNT_NAME - the name of the storage account
-    4) AZURE_STORAGE_ACCESS_KEY - the storage account access key
+    3) STORAGE_ACCOUNT_NAME - the name of the storage account
+    4) STORAGE_ACCOUNT_KEY - the storage account access key
 """
 
 
@@ -29,18 +29,18 @@
 
 class AuthSamplesAsync(object):
     url = "https://{}.blob.core.windows.net".format(
-        os.getenv("AZURE_STORAGE_ACCOUNT_NAME")
+        os.getenv("STORAGE_ACCOUNT_NAME")
     )
     oauth_url = "https://{}.blob.core.windows.net".format(
         os.getenv("OAUTH_STORAGE_ACCOUNT_NAME")
     )
 
-    connection_string = os.getenv("AZURE_STORAGE_CONNECTION_STRING")
-    shared_access_key = os.getenv("AZURE_STORAGE_ACCESS_KEY")
+    connection_string = os.getenv("STORAGE_CONNECTION_STRING")
+    shared_access_key = os.getenv("STORAGE_ACCOUNT_KEY")
 
     async def auth_connection_string_async(self):
         if self.connection_string is None:
-            print("Missing required environment variable: AZURE_STORAGE_CONNECTION_STRING." + '\n' +
+            print("Missing required environment variable: STORAGE_CONNECTION_STRING." + '\n' +
                   "Test: auth_connection_string_async")
             sys.exit(1)
         # [START auth_from_connection_string]
@@ -62,7 +62,7 @@ async def auth_connection_string_async(self):
 
     async def auth_shared_key_async(self):
         if self.shared_access_key is None:
-            print("Missing required environment variable: AZURE_STORAGE_ACCESS_KEY." + '\n' +
+            print("Missing required environment variable: STORAGE_ACCOUNT_KEY." + '\n' +
                   "Test: auth_shared_key_async")
             sys.exit(1)
         # [START create_blob_service_client]
@@ -85,7 +85,7 @@ async def auth_blob_url_async(self):
 
     async def auth_shared_access_signature_async(self):
         if self.connection_string is None:
-            print("Missing required environment variable: AZURE_STORAGE_CONNECTION_STRING." + '\n' +
+            print("Missing required environment variable: STORAGE_CONNECTION_STRING." + '\n' +
                   "Test: auth_shared_access_signature_async")
             sys.exit(1)
         # Instantiate a BlobServiceClient using a connection string

sdk/storage/azure-storage-blob/samples/blob_samples_batch_delete_blobs.py

@@ -10,18 +10,19 @@
 USAGE:
     python blob_samples_batch_delete_blobs.py
     Set the environment variables with your own values before running the sample:
-    1) AZURE_STORAGE_CONNECTION_STRING - the connection string to your storage account
+    1) STORAGE_CONNECTION_STRING - the connection string to your storage account
 """
 
-SOURCE_FOLDER = "./sample-blobs/"
+current_dir = os.path.dirname(os.path.abspath(__file__))
+SOURCE_FOLDER = os.path.join(current_dir, "./sample-blobs/")
 
 
 def batch_delete_blobs_sample(local_path):
     # Set the connection string and container name values to initialize the Container Client
-    connection_string = os.getenv('AZURE_STORAGE_CONNECTION_STRING')
+    connection_string = os.getenv('STORAGE_CONNECTION_STRING')
 
     if connection_string is None:
-        print("Missing required environment variable: AZURE_STORAGE_CONNECTION_STRING." + '\n' +
+        print("Missing required environment variable: STORAGE_CONNECTION_STRING." + '\n' +
               "Test: batch_delete_blobs_sample")
         sys.exit(1)
 

sdk/storage/azure-storage-blob/samples/blob_samples_client_side_encryption.py

@@ -13,12 +13,12 @@
     This example contains sample code for the KeyWrapper and KeyResolver classes
     needed to use Storage client side encryption, as well as code that illustrates
     key usage patterns for client side encryption features. This sample expects that
-    the `AZURE_STORAGE_CONNECTION_STRING` environment variable is set. It SHOULD NOT
+    the `STORAGE_CONNECTION_STRING` environment variable is set. It SHOULD NOT
     be hardcoded in any code derived from this sample.
 
 USAGE: python blob_samples_client_side_encryption.py
     Set the environment variables with your own values before running the sample:
-    1) AZURE_STORAGE_CONNECTION_STRING - the connection string to your storage account
+    1) STORAGE_CONNECTION_STRING - the connection string to your storage account
 """
 
 import os
@@ -275,9 +275,9 @@ def alternate_key_algorithms(self):
             self.container_client.delete_container()
 
 try:
-    CONNECTION_STRING = os.environ['AZURE_STORAGE_CONNECTION_STRING']
+    CONNECTION_STRING = os.environ['STORAGE_CONNECTION_STRING']
 except KeyError:
-    print("AZURE_STORAGE_CONNECTION_STRING must be set.")
+    print("STORAGE_CONNECTION_STRING must be set.")
     sys.exit(1)
 
 # Configure max_single_put_size to make blobs in this sample smaller

sdk/storage/azure-storage-blob/samples/blob_samples_client_side_encryption_keyvault.py

@@ -14,38 +14,30 @@
     service for client side encryption, storing and retrieving the key encryption key
     (kek) from within Azure KeyVault. This sample requires a service principal be set
     configured with access to KeyVault, and that the vault contains a 256-bit base64-
-    encoded secret named "symmetric-key". Additionally, a number of environment
+    encoded key named "symmetric-key". Additionally, a number of environment
     variables, listed below, must be set. Since these often contain sensitive information,
     they SHOULD NOT be replaced with hardcoded values in any code derived from this sample.
 
 USAGE: python blob_samples_client_side_encryption_keyvault.py
     Set the environment variables with your own values before running the sample:
     1) AZURE_STORAGE_ACCOUNT_URL - the storage account url
-    2) ACTIVE_DIRECTORY_APPLICATION_ID - Azure Active Directory application ID
-    3) ACTIVE_DIRECTORY_APPLICATION_SECRET - Azure Active Directory application secret
-    4) ACTIVE_DIRECTORY_TENANT_ID - Azure Active Directory tenant ID
-    5) AZURE_KEYVAULT_DNS_NAME: The keyvault account dns name
+    2) AZURE_KEYVAULT_DNS_NAME: The keyvault account dns name
 """
 
-import base64
 import os
 import sys
 import uuid
 
 from azure.identity import DefaultAzureCredential
 
 from azure.keyvault.keys.crypto import CryptographyClient, KeyWrapAlgorithm
-from azure.keyvault.keys import KeyVaultKey, KeyType
-from azure.keyvault.secrets import SecretClient
+from azure.keyvault.keys import KeyClient
 
 from azure.storage.blob import BlobServiceClient
 
 # Environment variable keys which must be set to run this sample
-STORAGE_URL = 'AZURE_STORAGE_ACCOUNT_URL'
-KEYVAULT_URL = 'AZURE_KEYVAULT_DNS_NAME'
-CLIENT_ID = 'ACTIVE_DIRECTORY_APPLICATION_ID'
-CLIENT_SECRET = 'ACTIVE_DIRECTORY_APPLICATION_SECRET'
-TENANT_ID = 'ACTIVE_DIRECTORY_TENANT_ID'
+STORAGE_URL = 'STORAGE_ACCOUNT_BLOB_URL'
+KEYVAULT_URL = 'KEYVAULT_URL'
 
 def get_env_var(key):
     try:
@@ -62,19 +54,19 @@ class KeyWrapper:
         automatic client-side encyrption and decryption routines. """
 
     def __init__(self, kek, credential):
-        self.algorithm = KeyWrapAlgorithm.aes_256
+        self.algorithm = KeyWrapAlgorithm.rsa_oaep_256
         self.kek = kek
         self.kid = kek.id
         self.client = CryptographyClient(kek, credential)
 
     def wrap_key(self, key):
-        if self.algorithm != KeyWrapAlgorithm.aes_256:
+        if self.algorithm != KeyWrapAlgorithm.rsa_oaep_256:
             raise ValueError('Unknown key wrap algorithm. {}'.format(self.algorithm))
         wrapped = self.client.wrap_key(key=key, algorithm=self.algorithm)
         return wrapped.encrypted_key
 
     def unwrap_key(self, key, _):
-        if self.algorithm != KeyWrapAlgorithm.aes_256:
+        if self.algorithm != KeyWrapAlgorithm.rsa_oaep_256:
             raise ValueError('Unknown key wrap algorithm. {}'.format(self.algorithm))
         unwrapped = self.client.unwrap_key(encrypted_key=key, algorithm=self.algorithm)
         return unwrapped.key
@@ -91,12 +83,10 @@ def get_kid(self):
 
 # Construct a token credential for use by Storage and KeyVault clients.
 credential = DefaultAzureCredential()
-secret_client = SecretClient(keyvault_url, credential=credential)
+key_client = KeyClient(keyvault_url, credential=credential)
 
-# The secret is url-safe base64 encoded bytes, content type 'application/octet-stream'
-secret = secret_client.get_secret('symmetric-key')
-key_bytes = base64.urlsafe_b64decode(secret.value)
-kvk = KeyVaultKey(key_id=secret.id, key_ops=['unwrapKey', 'wrapKey'], k=key_bytes, kty=KeyType.oct)
+# The key is url-safe base64 encoded bytes
+kvk = key_client.create_rsa_key(name="symmetric-key", size=2048, key_operations=["unwrapKey", "wrapKey"])
 kek = KeyWrapper(kvk, credential)
 
 storage_client = BlobServiceClient(storage_url, credential=credential)

sdk/storage/azure-storage-blob/samples/blob_samples_common.py

@@ -14,7 +14,7 @@
 USAGE:
     python blob_samples_common.py
     Set the environment variables with your own values before running the sample.
-    1) AZURE_STORAGE_CONNECTION_STRING - the connection string to your storage account
+    1) STORAGE_CONNECTION_STRING - the connection string to your storage account
 """
 
 import os
@@ -23,18 +23,19 @@
 from azure.core.exceptions import HttpResponseError, ResourceExistsError
 from azure.storage.blob import BlobServiceClient
 
-SOURCE_FILE = 'SampleSource.txt'
+current_dir = os.path.dirname(os.path.abspath(__file__))
+SOURCE_FILE = os.path.join(current_dir, 'SampleSource.txt')
 
 
 class CommonBlobSamples(object):
 
-    connection_string = os.getenv("AZURE_STORAGE_CONNECTION_STRING")
+    connection_string = os.getenv("STORAGE_CONNECTION_STRING_SOFT")
 
     #--Begin Blob Samples-----------------------------------------------------------------
 
     def blob_snapshots(self):
         if self.connection_string is None:
-            print("Missing required environment variable: AZURE_STORAGE_CONNECTION_STRING." + '\n' +
+            print("Missing required environment variable: STORAGE_CONNECTION_STRING." + '\n' +
                   "Test: blob_snapshots")
             sys.exit(1)
 
@@ -74,7 +75,7 @@ def blob_snapshots(self):
 
     def soft_delete_and_undelete_blob(self):
         if self.connection_string is None:
-            print("Missing required environment variable: AZURE_STORAGE_CONNECTION_STRING." + '\n' +
+            print("Missing required environment variable: STORAGE_CONNECTION_STRING." + '\n' +
                   "Test: soft_delete_and_undelete_blob")
             sys.exit(1)
 
@@ -120,7 +121,7 @@ def soft_delete_and_undelete_blob(self):
 
     def delete_multiple_blobs(self):
         if self.connection_string is None:
-            print("Missing required environment variable: AZURE_STORAGE_CONNECTION_STRING." + '\n' +
+            print("Missing required environment variable: STORAGE_CONNECTION_STRING." + '\n' +
                   "Test: delete_multiple_blobs")
             sys.exit(1)
         # Instantiate a BlobServiceClient using a connection string
@@ -157,7 +158,7 @@ def delete_multiple_blobs(self):
 
     def acquire_lease_on_blob(self):
         if self.connection_string is None:
-            print("Missing required environment variable: AZURE_STORAGE_CONNECTION_STRING." + '\n' +
+            print("Missing required environment variable: STORAGE_CONNECTION_STRING." + '\n' +
                   "Test: acquire_lease_on_blob")
             sys.exit(1)
 
@@ -194,7 +195,7 @@ def acquire_lease_on_blob(self):
 
     def start_copy_blob_from_url_and_abort_copy(self):
         if self.connection_string is None:
-            print("Missing required environment variable: AZURE_STORAGE_CONNECTION_STRING." + '\n' +
+            print("Missing required environment variable: STORAGE_CONNECTION_STRING." + '\n' +
                   "Test: start_copy_blob_from_url_and_abort_copy")
             sys.exit(1)
         # Instantiate a BlobServiceClient using a connection string

sdk/storage/azure-storage-blob/samples/blob_samples_common_async.py

@@ -14,27 +14,27 @@
 USAGE:
     python blob_samples_common_async.py
     Set the environment variables with your own values before running the sample.
-    1) AZURE_STORAGE_CONNECTION_STRING - the connection string to your storage account
+    1) STORAGE_CONNECTION_STRING - the connection string to your storage account
 """
 
 import os
 import sys
 import asyncio
 from azure.core.exceptions import ResourceExistsError
 
-
-SOURCE_FILE = './SampleSource.txt'
+current_dir = os.path.dirname(os.path.abspath(__file__))
+SOURCE_FILE = os.path.join(current_dir, 'SampleSource.txt')
 
 
 class CommonBlobSamplesAsync(object):
 
-    connection_string = os.getenv("AZURE_STORAGE_CONNECTION_STRING")
+    connection_string = os.getenv("STORAGE_CONNECTION_STRING_SOFT")
 
     #--Begin Blob Samples-----------------------------------------------------------------
 
     async def blob_snapshots_async(self):
         if self.connection_string is None:
-            print("Missing required environment variable: AZURE_STORAGE_CONNECTION_STRING." + '\n' +
+            print("Missing required environment variable: STORAGE_CONNECTION_STRING." + '\n' +
                   "Test: blob_snapshots_async")
             sys.exit(1)
         # Instantiate a BlobServiceClient using a connection string
@@ -74,7 +74,7 @@ async def blob_snapshots_async(self):
 
     async def soft_delete_and_undelete_blob_async(self):
         if self.connection_string is None:
-            print("Missing required environment variable: AZURE_STORAGE_CONNECTION_STRING." + '\n' +
+            print("Missing required environment variable: STORAGE_CONNECTION_STRING." + '\n' +
                   "Test: soft_delete_and_undelete_blob_async")
             sys.exit(1)
         # Instantiate a BlobServiceClient using a connection string
@@ -120,7 +120,7 @@ async def soft_delete_and_undelete_blob_async(self):
 
     async def delete_multiple_blobs_async(self):
         if self.connection_string is None:
-            print("Missing required environment variable: AZURE_STORAGE_CONNECTION_STRING." + '\n' +
+            print("Missing required environment variable: STORAGE_CONNECTION_STRING." + '\n' +
                   "Test: delete_multiple_blobs_async")
             sys.exit(1)
         # Instantiate a BlobServiceClient using a connection string
@@ -158,7 +158,7 @@ async def delete_multiple_blobs_async(self):
 
     async def acquire_lease_on_blob_async(self):
         if self.connection_string is None:
-            print("Missing required environment variable: AZURE_STORAGE_CONNECTION_STRING." + '\n' +
+            print("Missing required environment variable: STORAGE_CONNECTION_STRING." + '\n' +
                   "Test: acquire_lease_on_blob_async")
             sys.exit(1)
         # Instantiate a BlobServiceClient using a connection string
@@ -195,7 +195,7 @@ async def acquire_lease_on_blob_async(self):
 
     async def start_copy_blob_from_url_and_abort_copy_async(self):
         if self.connection_string is None:
-            print("Missing required environment variable: AZURE_STORAGE_CONNECTION_STRING." + '\n' +
+            print("Missing required environment variable: STORAGE_CONNECTION_STRING." + '\n' +
                   "Test: start_copy_blob_from_url_and_abort_copy_async")
             sys.exit(1)
         # Instantiate a BlobServiceClient using a connection string