[Corehttp] add perf tests (#34412)

* copy over core perf tests

* update readme

* update perf tests to corehttp

* add query attribute to httprequest in perf tests

* update httppolicy checks in pipeline base

* add perf tests yml for corehttp

* move perf tests yml into corehttp folder

* fix merge

* move perf files to corehttp

* black

* add dev reqs for perf

* add identity to dev reqs

* remove 1.0.0b3 from perf tests

* update corehttp aad envvars get

* update perf tests.yml

* revert azure core perf yml

* update perf readme

* remove unneeded policies

* address kashif/paul comments

* black

* black on azure core

Author: swathipil

sdk/core/azure-core/perf-tests.yml

@@ -13,7 +13,7 @@ Tests:
   Class: UploadBinaryDataTest
   Arguments:
   - --size 1024 --parallel 64 --duration 60 --policies all
-  - --size 1024 --parallel 64 --duration 60 --policies all --aad
+  - --size 1024 --parallel 64 --duration 60 --policies all --use-entra-id
   - --size 10240 --parallel 32 --duration 60
   - --size 10240 --parallel 32 --duration 60 --transport requests
 
@@ -22,29 +22,29 @@ Tests:
   Arguments:
   - --size 1024 --parallel 64 --duration 60
   - --size 1024 --parallel 64 --duration 60 --transport requests
-  - --size 1024 --parallel 64 --duration 60 --aad
+  - --size 1024 --parallel 64 --duration 60 --use-entra-id
   - --size 10240 --parallel 32 --duration 60 --policies all
 
 - Test: update-entity
   Class: UpdateEntityJSONTest
   Arguments:
   - --size 1024 --parallel 64 --duration 60
   - --size 1024 --parallel 64 --duration 60 --transport requests
-  - --size 1024 --parallel 64 --duration 60 --aad
+  - --size 1024 --parallel 64 --duration 60 --use-entra-id
   - --size 1024 --parallel 64 --duration 60 --policies all
 
 - Test: query-entities
   Class: QueryEntitiesJSONTest
   Arguments:
   - --size 1024 --parallel 64 --duration 60
   - --size 1024 --parallel 64 --duration 60 --transport requests
-  - --size 1024 --parallel 64 --duration 60 --aad
+  - --size 1024 --parallel 64 --duration 60 --use-entra-id
   - --size 1024 --parallel 64 --duration 60 --policies all
 
 - Test: list-entities
   Class: ListEntitiesPageableTest
   Arguments:
   - --count 500 --parallel 32 --warmup 60 --duration 60
   - --count 500 --parallel 32 --warmup 60 --duration 60 --transport requests
-  - --count 500 --parallel 32 --warmup 60 --duration 60 --aad
+  - --count 500 --parallel 32 --warmup 60 --duration 60 --use-entra-id
   - --count 500 --parallel 32 --warmup 60 --duration 60 --policies all

sdk/core/azure-core/tests/perf_tests/README.md

@@ -65,8 +65,12 @@ The options that are available for all Core perf tests:
     - `"requests"`: AsyncioRequestsTransport
   - For sync:
     - `"requests"`: RequestsTransport (default)
-- `--aad` - Flag to pass in to use Azure Active Directory as the authentication. By default, set to False.
+- `--use-entra-id` - Flag to pass in to use Microsoft Entra ID as the authentication. By default, set to False.
 - `--size=10240` - Size of request content (in bytes). Defaults to 10240. (Not used by `ListEntitiesPageableTest`.)
+- `--policies` - List of policies to pass in to the pipeline. Options:
+  - None: No extra policies passed in, except for authentication policy. This is the default.
+  - 'all': All policies added automatically by autorest.
+  - 'policy1,policy2': Comma-separated list of policies, such as 'RetryPolicy,HttpLoggingPolicy'"
 
 #### Additional ListEntitiesPageableTest command line options
 
@@ -78,5 +82,5 @@ The options that are additionally available for `ListEntitiesPageableTest`:
 ## Example command
 
 ```cmd
-(env) ~/azure-core> perfstress DownloadBinaryDataTest --aad --transport requests --size=20480 --parallel=2
+(env) ~/azure-core> perfstress DownloadBinaryDataTest --use-entra-id --transport requests --size=20480 --parallel=2
 ```

sdk/core/azure-core/tests/perf_tests/_test_base.py

@@ -159,7 +159,7 @@ def _build_async_pipeline_client(self, auth_policy):
         return AsyncPipelineClient(self.account_endpoint, pipeline=async_pipeline)
 
     def _set_auth_policies(self):
-        if not self.args.aad:
+        if not self.args.use_entra_id:
             # if tables, create table credential policy, else blob policy
             if "tables" in self.sdk_moniker:
                 self.sync_auth_policy = TableSharedKeyCredentialPolicy(
@@ -199,7 +199,9 @@ def add_arguments(parser):
             """\n- 'policy1,policy2': Comma-separated list of policies, such as 'RetryPolicy,HttpLoggingPolicy'""",
             default=None,
         )
-        parser.add_argument("--aad", action="store_true", help="Use AAD authentication instead of shared key.")
+        parser.add_argument(
+            "--use-entra-id", action="store_true", help="Use Microsoft Entra ID authentication instead of shared key."
+        )
 
 
 class _BlobTest(_ServiceTest):

sdk/core/corehttp/dev_requirements.txt

@@ -7,3 +7,6 @@ httpx>=0.25.0
 -e ../../../tools/azure-devtools
 -e tests/testserver_tests/coretestserver
 ../../core/azure-core
+azure-storage-blob
+azure-data-tables
+azure-identity

sdk/core/corehttp/perf-resources.bicep

@@ -0,0 +1,74 @@
+param baseName string = resourceGroup().name
+param location string = resourceGroup().location
+param storageEndpointSuffix string = 'core.windows.net'
+param testApplicationOid string
+
+resource storageAccount 'Microsoft.Storage/storageAccounts@2019-06-01' = {
+  name: '${baseName}storage'
+  location: location
+  kind: 'StorageV2'
+  sku: {
+    name: 'Standard_RAGRS'
+  }
+}
+
+resource storageAccountBlobServices 'Microsoft.Storage/storageAccounts/blobServices@2021-09-01' = {
+  name: 'default'
+  parent: storageAccount
+}
+
+resource testContainer 'Microsoft.Storage/storageAccounts/blobServices/containers@2021-09-01' = {
+  name: 'testcontainer'
+  parent: storageAccountBlobServices
+
+}
+
+@description('This is the Blob owner role.')
+resource blobOwnerRoleDefinition 'Microsoft.Authorization/roleDefinitions@2018-01-01-preview' existing = {
+  scope: resourceGroup()
+  name: 'b7e6dc6d-f1e8-4753-8033-0f276bb0955b'
+}
+
+resource blobRoleAssignment 'Microsoft.Authorization/roleAssignments@2022-04-01' = {
+  name: guid(resourceGroup().id, testApplicationOid, blobOwnerRoleDefinition.id)
+  properties: {
+    roleDefinitionId: blobOwnerRoleDefinition.id
+    principalId: testApplicationOid
+    principalType: 'ServicePrincipal'
+  }
+}
+
+resource tableServices 'Microsoft.Storage/storageAccounts/tableServices@2022-09-01' = {
+  name: 'default'
+  parent: storageAccount
+}
+
+resource tables 'Microsoft.Storage/storageAccounts/tableServices/tables@2022-09-01' = {
+  name: 'default'
+  parent: tableServices
+}
+
+@description('This is the Blob owner role.')
+resource tableOwnerRoleDefinition 'Microsoft.Authorization/roleDefinitions@2018-01-01-preview' existing = {
+  scope: resourceGroup()
+  name: '0a9a7e1f-b9d0-4cc4-a60d-0319b160aaa3'
+}
+
+resource tableRoleAssignment 'Microsoft.Authorization/roleAssignments@2022-04-01' = {
+  name: guid(resourceGroup().id, testApplicationOid, tableOwnerRoleDefinition.id)
+  properties: {
+    roleDefinitionId: tableOwnerRoleDefinition.id
+    principalId: testApplicationOid
+    principalType: 'ServicePrincipal'
+  }
+}
+
+var storageAccountKey = storageAccount.listKeys('2021-09-01').keys[0].value
+output AZURE_STORAGE_ACCOUNT_NAME string = storageAccount.name
+output AZURE_STORAGE_BLOBS_ENDPOINT string = storageAccount.properties.primaryEndpoints.blob
+output AZURE_STORAGE_ACCOUNT_KEY string = storageAccountKey
+output AZURE_STORAGE_CONN_STR string = 'DefaultEndpointsProtocol=https;AccountName=${storageAccount.name};AccountKey=${storageAccountKey};EndpointSuffix=${storageEndpointSuffix}'
+output AZURE_STORAGE_CONTAINER_NAME string = testContainer.name
+
+output AZURE_STORAGE_TABLE_NAME string = tables.name
+output AZURE_STORAGE_TABLES_ENDPOINT string = 'https://${storageAccount.name}.table.${storageEndpointSuffix}/'

sdk/core/corehttp/perf-tests.yml

@@ -0,0 +1,50 @@
+Service: core
+
+Project: sdk/core/corehttp
+
+PrimaryPackage: corehttp
+
+PackageVersions:
+- corehttp: 1.0.0b4
+- corehttp: source
+
+Tests:
+- Test: upload-binary
+  Class: UploadBinaryDataTest
+  Arguments:
+  - --size 1024 --parallel 64 --duration 60 --policies all
+  - --size 1024 --parallel 64 --duration 60 --policies all --use-entra-id
+  - --size 10240 --parallel 32 --duration 60
+  - --size 10240 --parallel 32 --duration 60 --transport httpx
+
+- Test: download-binary
+  Class: DownloadBinaryDataTest
+  Arguments:
+  - --size 1024 --parallel 64 --duration 60
+  - --size 1024 --parallel 64 --duration 60 --transport httpx
+  - --size 1024 --parallel 64 --duration 60 --use-entra-id
+  - --size 10240 --parallel 32 --duration 60 --policies all
+
+- Test: update-entity
+  Class: UpdateEntityJSONTest
+  Arguments:
+  - --size 1024 --parallel 64 --duration 60
+  - --size 1024 --parallel 64 --duration 60 --transport httpx
+  - --size 1024 --parallel 64 --duration 60 --use-entra-id
+  - --size 1024 --parallel 64 --duration 60 --policies all
+
+- Test: query-entities
+  Class: QueryEntitiesJSONTest
+  Arguments:
+  - --size 1024 --parallel 64 --duration 60
+  - --size 1024 --parallel 64 --duration 60 --transport httpx
+  - --size 1024 --parallel 64 --duration 60 --use-entra-id
+  - --size 1024 --parallel 64 --duration 60 --policies all
+
+- Test: list-entities
+  Class: ListEntitiesPageableTest
+  Arguments:
+  - --count 500 --parallel 32 --warmup 60 --duration 60
+  - --count 500 --parallel 32 --warmup 60 --duration 60 --transport httpx
+  - --count 500 --parallel 32 --warmup 60 --duration 60 --use-entra-id
+  - --count 500 --parallel 32 --warmup 60 --duration 60 --policies all

sdk/core/corehttp/perf.yml

@@ -0,0 +1,37 @@
+parameters:
+- name: LanguageVersion
+  displayName: LanguageVersion (3.8, 3.9, 3.10, 3.11, 3.12)
+  type: string
+  default: '3.11'
+- name: PackageVersions
+  displayName: PackageVersions (regex of package versions to run)
+  type: string
+  default: '1|source'
+- name: Tests
+  displayName: Tests (regex of tests to run)
+  type: string
+  default: '^(upload-binary|download-binary|update-entity|query-entities|list-entities)$'
+- name: Arguments
+  displayName: Arguments (regex of arguments to run)
+  type: string
+  default: '(1024)|(10240)|(500)'
+- name: Iterations
+  displayName: Iterations (times to run each test)
+  type: number
+  default: '5'
+- name: AdditionalArguments
+  displayName: AdditionalArguments (passed to PerfAutomation)
+  type: string
+  default: ' '
+
+extends:
+  template: /eng/pipelines/templates/jobs/perf.yml
+  parameters:
+    TimeoutInMinutes: 720
+    ServiceDirectory: core/corehttp
+    LanguageVersion: ${{ parameters.LanguageVersion }}
+    PackageVersions: ${{ parameters.PackageVersions }}
+    Tests: ${{ parameters.Tests }}
+    Arguments: ${{ parameters.Arguments }}
+    Iterations: ${{ parameters.Iterations }}
+    AdditionalArguments: ${{ parameters.AdditionalArguments }}

sdk/core/corehttp/tests/perf_tests/README.md

@@ -0,0 +1,87 @@
+# Core Python client performance tests
+
+In order to run the performance tests, the `devtools_testutils` package must be installed. This is done as part of the `dev_requirements.txt` installation. Start be creating a new virtual environment for your perf tests. This will need to be a Python 3 environment, preferably >=3.7.
+
+### Setup for test resources
+The following environment variables will need to be set for the tests to access the live resources:
+
+```
+AZURE_STORAGE_CONN_STR=<the connection string to the Storage account>
+AZURE_STORAGE_ACCOUNT_NAME=<the Storage account name>
+AZURE_STORAGE_ACCOUNT_KEY=<the Storage account key>
+
+AZURE_STORAGE_CONTAINER_NAME=<the container name>
+AZURE_STORAGE_BLOBS_ENDPOINT=<The Storage Blobs endpoint in the format 'https://{storageAccountName}.blob.core.windows.net'>
+
+AZURE_STORAGE_TABLE_NAME=<The name to use for the Storage Table>
+AZURE_STORAGE_TABLES_ENDPOINT=<The Storage Tables endpoint in the format 'https://{storageAccountName}.table.core.windows.net'>
+```
+
+### Setup for perf test runs
+
+```cmd
+(env) ~/core/corehttp> pip install -r dev_requirements.txt
+(env) ~/core/corehttp> pip install .
+```
+
+## Test commands
+
+When `devtools_testutils` is installed, you will have access to the `perfstress` command line tool, which will scan the current module for runable perf tests. Only a specific test can be run at a time (i.e. there is no "run all" feature).
+
+```cmd
+(env) ~/core/corehttp> cd tests
+(env) ~/core/corehttp/tests> perfstress
+```
+
+Using the `perfstress` command alone will list the available perf tests found.
+
+### Tests
+
+The tests currently available:
+
+- `UploadBinaryDataTest` - Puts binary data of `size` in a Storage Blob (corresponds to the `upload_blob` Blob operation).
+- `DownloadBinaryDataTest` - Gets binary data of `size` from a Storage Blob (corresponds to the `download_blob` Blob operation).
+- `UpdateEntityJSONTest` - Puts JSON data of `size` in a Storage Table (corresponds to the `update_entity` Tables operation).
+- `QueryEntitiesJSONTest` - Gets JSON data of `size` from a Storage Table (corresponds to the `query_entities` Tables operation).
+- `ListEntitiesPageableTest` - Gets pageable data from a Storage Table (corresponds to the `list_entities` Tables operation).
+
+### Common perf command line options
+
+The `perfstress` framework has a series of common command line options built in. View them [here](https://github.com/Azure/azure-sdk-for-python/blob/main/doc/dev/perfstress_tests.md#default-command-options).
+
+- `--sync` Whether to run the tests in sync or async. Default is False (async).
+- `-d --duration=10` Number of seconds to run as many operations (the "run" function) as possible. Default is 10.
+- `-i --iterations=1` Number of test iterations to run. Default is 1.
+- `-p --parallel=1` Number of tests to run in parallel. Default is 1.
+- `-w --warm-up=5` Number of seconds to spend warming up the connection before measuring begins. Default is 5.
+
+#### Core perf test common command line options
+
+The options that are available for all Core perf tests:
+
+- `--transport` - By default, uses AiohttpTransport ("aiohttp") for async. By default, uses RequestsTransport ("requests") for sync. All options:
+  - For async:
+    - `"aiohttp"`: AiohttpTransport (default)
+    - `"httpx"`: AsyncHttpXTransport
+  - For sync:
+    - `"requests"`: RequestsTransport (default)
+    - `"httpx"`: HttpXTransport
+- `--use-entra-id` - Flag to pass in to use Microsoft Entra ID as the authentication. By default, set to False.
+- `--size=10240` - Size of request content (in bytes). Defaults to 10240. (Not used by `ListEntitiesPageableTest`.)
+- `--policies` - List of policies to pass in to the pipeline. Options:
+  - None: No extra policies passed in, except for authentication policy. This is the default.
+  - 'all': All policies added automatically by autorest.
+  - 'policy1,policy2': Comma-separated list of policies, such as 'RetryPolicy,UserAgentPolicy'"
+
+#### Additional ListEntitiesPageableTest command line options
+
+The options that are additionally available for `ListEntitiesPageableTest`:
+
+- `--count=100` - Number of table entities to list. Defaults to 100.
+- `--page-size=None` - Maximum number of entities to list per page. Default is None, which will return all possible results per page.
+
+## Example command
+
+```cmd
+(env) ~/core/corehttp> perfstress DownloadBinaryDataTest --use-entra-id --transport httpx --size=20480 --parallel=2
+```