Fix storage perf (Azure#49082)

jaschrep-msft · web-flow · commit fc81d2e452a1 · 2025-04-03T12:25:34.000-04:00
* add datamovement perf to main storage sln

* Fix blob perf.

Use token credentials via DefaultAzureCredential.
Some tests still require shared key for SAS. Generally we are locked out
of shared key by default, but these tests are probably still important.
Made shared key an optional variable to be fetched. Those tests will
probably still break, but they were already broken. This at least
compiles.

* convert shares and datalake to tokencreds

* revert sln

* missing env var
diff --git a/sdk/storage/Azure.Storage.Blobs/perf.yml b/sdk/storage/Azure.Storage.Blobs/perf.yml
@@ -38,3 +38,5 @@ extends:
     Iterations: ${{ parameters.Iterations }}
     Profile: ${{ parameters.Profile }}
     AdditionalArguments: ${{ parameters.AdditionalArguments }}
+    EnvVars:
+      AZURE_AUTHORITY_HOST: $(AZURE_STORAGE_BLOBS_AZURE_AUTHORITY_HOST)
diff --git a/sdk/storage/Azure.Storage.Blobs/perf/Azure.Storage.Blobs.Perf/Infrastructure/PerfTestEnvironment.cs b/sdk/storage/Azure.Storage.Blobs/perf/Azure.Storage.Blobs.Perf/Infrastructure/PerfTestEnvironment.cs
@@ -25,25 +25,25 @@ internal sealed class PerfTestEnvironment : TestEnvironment
         /// The name of the Blob storage account to test against.
         /// </summary>
         /// <value>The Blob storage account name, read from the "AZURE_STORAGE_ACCOUNT_NAME" environment variable.</value>
-        public string BlobStorageAccountName => GetVariable("AZURE_STORAGE_ACCOUNT_NAME");
+        public string StorageAccountName => GetVariable("AZURE_STORAGE_ACCOUNT_NAME");
 
         /// <summary>
         /// The shared access key of the Blob storage account to test against.
         /// </summary>
         /// <value>The Blob storage account key, read from the "AZURE_STORAGE_ACCOUNT_KEY" environment variable.</value>
-        public string BlobStorageAccountKey => GetVariable("AZURE_STORAGE_ACCOUNT_KEY");
+        public string StorageAccountKey => GetOptionalVariable("AZURE_STORAGE_ACCOUNT_KEY");
 
         /// <summary>
-        /// The connection string for accessing the Blob storage account used for testing.
+        /// The Blob storage endpoint.
         /// </summary>
-        public string BlobStorageConnectionString { get; }
+        public Uri StorageEndpoint { get; }
 
         /// <summary>
         /// Initializes a new instance of the <see cref="PerfTestEnvironment"/> class.
         /// </summary>
         public PerfTestEnvironment()
         {
-            BlobStorageConnectionString = $"DefaultEndpointsProtocol={Uri.UriSchemeHttps};AccountName={BlobStorageAccountName};AccountKey={BlobStorageAccountKey};EndpointSuffix={StorageEndpointSuffix}";
+            StorageEndpoint = new Uri($"https://{StorageAccountName}.blob.{StorageEndpointSuffix}");
         }
     }
 }
diff --git a/sdk/storage/Azure.Storage.Blobs/perf/Azure.Storage.Blobs.Perf/Infrastructure/ServiceTest.cs b/sdk/storage/Azure.Storage.Blobs/perf/Azure.Storage.Blobs.Perf/Infrastructure/ServiceTest.cs
@@ -16,10 +16,16 @@ public ServiceTest(TOptions options) : base(options)
                 ? clientOptionsOptions.ClientOptions
                 : new BlobClientOptions();
             BlobServiceClient = new BlobServiceClient(
-                PerfTestEnvironment.Instance.BlobStorageConnectionString, ConfigureClientOptions(clientOptions));
+                PerfTestEnvironment.Instance.StorageEndpoint,
+                PerfTestEnvironment.Instance.Credential,
+                ConfigureClientOptions(clientOptions));
 
-            StorageSharedKeyCredential = new StorageSharedKeyCredential(
-                PerfTestEnvironment.Instance.BlobStorageAccountName, PerfTestEnvironment.Instance.BlobStorageAccountKey);
+            // Can't do shared key tests if shared key wasn't provided
+            if (PerfTestEnvironment.Instance.StorageAccountKey != null)
+            {
+                StorageSharedKeyCredential = new StorageSharedKeyCredential(
+                    PerfTestEnvironment.Instance.StorageAccountName, PerfTestEnvironment.Instance.StorageAccountKey);
+            }
         }
     }
 }
diff --git a/sdk/storage/Azure.Storage.Files.DataLake/perf/Azure.Storage.Files.DataLake.Perf/Infrastructure/PerfTestEnvironment.cs b/sdk/storage/Azure.Storage.Files.DataLake/perf/Azure.Storage.Files.DataLake.Perf/Infrastructure/PerfTestEnvironment.cs
@@ -27,26 +27,21 @@ public sealed class PerfTestEnvironment : TestEnvironment
         public new string StorageEndpointSuffix => base.StorageEndpointSuffix ?? "core.windows.net";
 
         /// <summary>
-        ///   The name of the Data Lake storage account to test against.
+        /// The name of the Blob storage account to test against.
         /// </summary>
-        ///
-        /// <value>The Data Lake storage account name, read from the "DATALAKE_STORAGE_ACCOUNT_NAME" environment variable.</value>
-        ///
-        public string DataLakeAccountName => GetVariable("DATALAKE_STORAGE_ACCOUNT_NAME");
+        /// <value>The Blob storage account name, read from the "AZURE_STORAGE_ACCOUNT_NAME" environment variable.</value>
+        public string StorageAccountName => GetVariable("AZURE_STORAGE_ACCOUNT_NAME");
 
         /// <summary>
-        ///   The shared access key of the Data Lake storage account to test against.
+        /// The shared access key of the Blob storage account to test against.
         /// </summary>
-        ///
-        /// <value>The Data Lake storage account key, read from the "DATALAKE_STORAGE_ACCOUNT_KEY" environment variable.</value>
-        ///
-        public string DataLakeAccountKey => GetVariable("DATALAKE_STORAGE_ACCOUNT_KEY");
+        /// <value>The Blob storage account key, read from the "AZURE_STORAGE_ACCOUNT_KEY" environment variable.</value>
+        public string StorageAccountKey => GetOptionalVariable("AZURE_STORAGE_ACCOUNT_KEY");
 
         /// <summary>
-        ///   The fully-qualified URI for the Data Lake storage account to test against.
+        /// The Blob storage endpoint.
         /// </summary>
-        ///
-        public Uri DataLakeServiceUri { get; }
+        public Uri StorageEndpoint { get; }
 
         /// <summary>
         ///   The credential for accessing the Data Lake storage account used for testing.
@@ -62,8 +57,11 @@ public sealed class PerfTestEnvironment : TestEnvironment
         ///
         public PerfTestEnvironment()
         {
-            DataLakeServiceUri = new Uri($"{ Uri.UriSchemeHttps }{ Uri.SchemeDelimiter }{ DataLakeAccountName }.dfs.{ StorageEndpointSuffix }");
-            DataLakeCredential = new StorageSharedKeyCredential(DataLakeAccountName, DataLakeAccountKey);
+            StorageEndpoint = new Uri($"{ Uri.UriSchemeHttps }{ Uri.SchemeDelimiter }{ StorageAccountName }.dfs.{ StorageEndpointSuffix }");
+            if (StorageAccountKey != null)
+            {
+                DataLakeCredential = new StorageSharedKeyCredential(StorageAccountName, StorageAccountKey);
+            }
         }
     }
 }
diff --git a/sdk/storage/Azure.Storage.Files.DataLake/perf/Azure.Storage.Files.DataLake.Perf/Infrastructure/ServiceTest.cs b/sdk/storage/Azure.Storage.Files.DataLake/perf/Azure.Storage.Files.DataLake.Perf/Infrastructure/ServiceTest.cs
@@ -16,12 +16,16 @@ public ServiceTest(TOptions options) : base(options)
                 ? clientOptionsProvider.ClientOptions
                 : new DataLakeClientOptions();
             DataLakeServiceClient = new DataLakeServiceClient(
-                PerfTestEnvironment.Instance.DataLakeServiceUri,
-                PerfTestEnvironment.Instance.DataLakeCredential,
+                PerfTestEnvironment.Instance.StorageEndpoint,
+                PerfTestEnvironment.Instance.Credential,
                 ConfigureClientOptions(clientOptions));
 
-            StorageSharedKeyCredential = new StorageSharedKeyCredential(
-                PerfTestEnvironment.Instance.DataLakeAccountName, PerfTestEnvironment.Instance.DataLakeAccountKey);
+            // Can't do shared key tests if shared key wasn't provided
+            if (PerfTestEnvironment.Instance.StorageAccountKey != null)
+            {
+                StorageSharedKeyCredential = new StorageSharedKeyCredential(
+                    PerfTestEnvironment.Instance.StorageAccountName, PerfTestEnvironment.Instance.StorageAccountKey);
+            }
         }
     }
 }
diff --git a/sdk/storage/Azure.Storage.Files.DataLake/perf/Azure.Storage.Files.DataLake.Perf/Scenarios/CreateClients.cs b/sdk/storage/Azure.Storage.Files.DataLake/perf/Azure.Storage.Files.DataLake.Perf/Scenarios/CreateClients.cs
@@ -17,9 +17,9 @@ public class CreateClients : PerfTest<PerfOptions>
     {
         private static readonly TokenCredential s_tokenCredential = new ClientSecretCredential("foo", "bar", "baz");
         private static readonly PerfTestEnvironment s_testEnvironment = PerfTestEnvironment.Instance;
-        private static readonly Uri s_fileSystemUri = new DataLakeUriBuilder(s_testEnvironment.DataLakeServiceUri) { FileSystemName = "foo" }.ToUri();
-        private static readonly Uri s_directoryUri = new DataLakeUriBuilder(s_testEnvironment.DataLakeServiceUri) { FileSystemName = "foo", DirectoryOrFilePath = "bar" }.ToUri();
-        private static readonly Uri s_fileUri = new DataLakeUriBuilder(s_testEnvironment.DataLakeServiceUri) { FileSystemName = "foo", DirectoryOrFilePath = "bar/baz" }.ToUri();
+        private static readonly Uri s_fileSystemUri = new DataLakeUriBuilder(s_testEnvironment.StorageEndpoint) { FileSystemName = "foo" }.ToUri();
+        private static readonly Uri s_directoryUri = new DataLakeUriBuilder(s_testEnvironment.StorageEndpoint) { FileSystemName = "foo", DirectoryOrFilePath = "bar" }.ToUri();
+        private static readonly Uri s_fileUri = new DataLakeUriBuilder(s_testEnvironment.StorageEndpoint) { FileSystemName = "foo", DirectoryOrFilePath = "bar/baz" }.ToUri();
 
         public CreateClients(PerfOptions options) : base(options)
         {
@@ -28,9 +28,9 @@ public CreateClients(PerfOptions options) : base(options)
 #pragma warning disable CA1806 // Do not ignore method results
         public override void Run(CancellationToken cancellationToken)
         {
-            var serviceClient = new DataLakeServiceClient(s_testEnvironment.DataLakeServiceUri);
-            new DataLakeServiceClient(s_testEnvironment.DataLakeServiceUri, s_tokenCredential);
-            new DataLakeServiceClient(s_testEnvironment.DataLakeServiceUri, s_testEnvironment.DataLakeCredential);
+            var serviceClient = new DataLakeServiceClient(s_testEnvironment.StorageEndpoint);
+            new DataLakeServiceClient(s_testEnvironment.StorageEndpoint, s_tokenCredential);
+            new DataLakeServiceClient(s_testEnvironment.StorageEndpoint, new StorageSharedKeyCredential("s_testEnvironment.StorageAccountKey", Convert.ToBase64String(Guid.NewGuid().ToByteArray())));
 
             var fileSystemClient = new DataLakeFileSystemClient(s_fileSystemUri);
             new DataLakeFileSystemClient(s_fileSystemUri, s_tokenCredential);
diff --git a/sdk/storage/Azure.Storage.Files.Shares/perf/Azure.Storage.Files.Shares.Perf/Infrastructure/PerfTestEnvironment.cs b/sdk/storage/Azure.Storage.Files.Shares/perf/Azure.Storage.Files.Shares.Perf/Infrastructure/PerfTestEnvironment.cs
@@ -22,28 +22,28 @@ internal sealed class PerfTestEnvironment : TestEnvironment
         public new string StorageEndpointSuffix => base.StorageEndpointSuffix ?? "core.windows.net";
 
         /// <summary>
-        /// The name of the Files Shares storage account to test against.
+        /// The name of the Blob storage account to test against.
         /// </summary>
-        /// <value>The Files Shares storage account name, read from the "AZURE_STORAGE_ACCOUNT_NAME" environment variable.</value>
-        public string FilesSharesAccountName => GetVariable("AZURE_STORAGE_ACCOUNT_NAME");
+        /// <value>The Blob storage account name, read from the "AZURE_STORAGE_ACCOUNT_NAME" environment variable.</value>
+        public string StorageAccountName => GetVariable("AZURE_STORAGE_ACCOUNT_NAME");
 
         /// <summary>
-        /// The shared access key of the Files Shares storage account to test against.
+        /// The shared access key of the Blob storage account to test against.
         /// </summary>
-        /// <value>The Files Shares storage account key, read from the "AZURE_STORAGE_ACCOUNT_KEY" environment variable.</value>
-        public string FilesSharesAccountKey => GetVariable("AZURE_STORAGE_ACCOUNT_KEY");
+        /// <value>The Blob storage account key, read from the "AZURE_STORAGE_ACCOUNT_KEY" environment variable.</value>
+        public string StorageAccountKey => GetOptionalVariable("AZURE_STORAGE_ACCOUNT_KEY");
 
         /// <summary>
-        /// The connection string for accessing the Files Shares storage account used for testing.
+        /// The Blob storage endpoint.
         /// </summary>
-        public string FileSharesConnectionString { get; }
+        public Uri StorageEndpoint { get; }
 
         /// <summary>
         /// Initializes a new instance of the <see cref="PerfTestEnvironment"/> class.
         /// </summary>
         public PerfTestEnvironment()
         {
-            FileSharesConnectionString = $"DefaultEndpointsProtocol={Uri.UriSchemeHttps};AccountName={FilesSharesAccountName};AccountKey={FilesSharesAccountKey};EndpointSuffix={StorageEndpointSuffix}";
+            StorageEndpoint = new Uri($"https://{StorageAccountName}.file.{StorageEndpointSuffix}");
         }
     }
 }
diff --git a/sdk/storage/Azure.Storage.Files.Shares/perf/Azure.Storage.Files.Shares.Perf/Infrastructure/ServiceTest.cs b/sdk/storage/Azure.Storage.Files.Shares/perf/Azure.Storage.Files.Shares.Perf/Infrastructure/ServiceTest.cs
@@ -1,6 +1,7 @@
 ﻿// Copyright (c) Microsoft Corporation. All rights reserved.
 // Licensed under the MIT License.
 
+using Azure.Storage.Files.Shares.Models;
 using Azure.Test.Perf;
 
 namespace Azure.Storage.Files.Shares.Perf
@@ -15,11 +16,18 @@ public ServiceTest(TOptions options) : base(options)
             ShareClientOptions clientOptions = options is Options.IShareClientOptionsProvider clientOptionsProvider
                 ? clientOptionsProvider.ClientOptions
                 : new ShareClientOptions();
+            clientOptions.ShareTokenIntent = ShareTokenIntent.Backup;
             ShareServiceClient = new ShareServiceClient(
-                PerfTestEnvironment.Instance.FileSharesConnectionString, ConfigureClientOptions(clientOptions));
+                PerfTestEnvironment.Instance.StorageEndpoint,
+                PerfTestEnvironment.Instance.Credential,
+                ConfigureClientOptions(clientOptions));
 
-            StorageSharedKeyCredential = new StorageSharedKeyCredential(
-                PerfTestEnvironment.Instance.FilesSharesAccountName, PerfTestEnvironment.Instance.FilesSharesAccountKey);
+            // Can't do shared key tests if shared key wasn't provided
+            if (PerfTestEnvironment.Instance.StorageAccountKey != null)
+            {
+                StorageSharedKeyCredential = new StorageSharedKeyCredential(
+                    PerfTestEnvironment.Instance.StorageAccountName, PerfTestEnvironment.Instance.StorageAccountKey);
+            }
         }
     }
 }

Original file line number	Diff line number	Diff line change
`@@ -16,10 +16,16 @@ public ServiceTest(TOptions options) : base(options)`
`16`	`16`	`? clientOptionsOptions.ClientOptions`
`17`	`17`	`: new BlobClientOptions();`
`18`	`18`	`BlobServiceClient = new BlobServiceClient(`
`19`		`- PerfTestEnvironment.Instance.BlobStorageConnectionString, ConfigureClientOptions(clientOptions));`
	`19`	`+ PerfTestEnvironment.Instance.StorageEndpoint,`
	`20`	`+ PerfTestEnvironment.Instance.Credential,`
	`21`	`+ ConfigureClientOptions(clientOptions));`
`20`	`22`
`21`		`- StorageSharedKeyCredential = new StorageSharedKeyCredential(`
`22`		`- PerfTestEnvironment.Instance.BlobStorageAccountName, PerfTestEnvironment.Instance.BlobStorageAccountKey);`
	`23`	`+ // Can't do shared key tests if shared key wasn't provided`
	`24`	`+ if (PerfTestEnvironment.Instance.StorageAccountKey != null)`
	`25`	`+ {`
	`26`	`+ StorageSharedKeyCredential = new StorageSharedKeyCredential(`
	`27`	`+ PerfTestEnvironment.Instance.StorageAccountName, PerfTestEnvironment.Instance.StorageAccountKey);`
	`28`	`+ }`
`23`	`29`	`}`
`24`	`30`	`}`
`25`	`31`	`}`