[KeyVault] Enable Test Proxy for KeyVault Secrets (Azure#22807)

* switch to assert from self.assert*

* Adding in conftest

* Make sleep patches into fixtures

* Change class name for pytest

* move from aztestcase to azrectestcase

* fix import

* change class name for pytest

* decorate sync tests with proxy

* decorate async tests

* missing imports

* changes to get the tests collectd by pytest

* add in sanitizers & test proxy changes

* secrets client async preparer

* changes for preparer

* update with further sanitizers

* update to test and recordings

* changes to async test case

* update test_parse to record output

* minor cleanups

* move to async prepaper

* updates

* minor changes to get tests working

* more recordings from test proxy

* switch to assert from self.assert*

* Adding in conftest

* updates

* updates

* further merges

* merges

* merges

* changes

* more recordings for async tests

* Changes for async tests and work around for test proxy

* Changes to to get tests working

* change for EnvironmentVariablesLoader & recordings

* clean up for client creation

* fix patches for sleep

* add in http cache clean

* mods for win-http

* recordings using test proxy

* set default vault url

* changes for auth headers

* changes for auth update

* new recordings

* delete vcrpy recordings

* remove unused imports & code cleanup

* minor changes around patching sleep

* new recordings for 7.3

* new recordings

* clean up recordings of 7.3 preview

* remove 7.3 preview tests

* removing customer default handler for auth

* new recording for test

* code clean up and formatting

* remove temp fix for async recordings

* code clean up for imports, better error messages

* add cache challenge in to a teardown

* new recordings with cache challenge change

* change kv url error

* fix wrong masking name

* Fix SecretsClientPreparer typo

* remove code block for async tests

* remove unused imports

Author: kashifkhan

sdk/keyvault/azure-keyvault-secrets/tests/_async_test_case.py

@@ -0,0 +1,43 @@
+# ------------------------------------
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+# ------------------------------------
+import os
+
+import pytest
+from azure.keyvault.secrets._shared.client_base import DEFAULT_VERSION
+from devtools_testutils import AzureRecordedTestCase, is_live
+
+
+class AsyncSecretsClientPreparer(AzureRecordedTestCase):
+    def __init__(self, **kwargs) -> None:
+        self.azure_keyvault_url = "https://vaultname.vault.azure.net"
+
+        if is_live():
+            self.azure_keyvault_url = os.environ["AZURE_KEYVAULT_URL"]
+
+        self.is_logging_enabled = kwargs.pop("logging_enable", True)
+        if is_live():
+            os.environ["AZURE_TENANT_ID"] = os.environ["KEYVAULT_TENANT_ID"]
+            os.environ["AZURE_CLIENT_ID"] = os.environ["KEYVAULT_CLIENT_ID"]
+            os.environ["AZURE_CLIENT_SECRET"] = os.environ["KEYVAULT_CLIENT_SECRET"]
+
+    def __call__(self, fn):
+        async def _preparer(test_class, api_version, **kwargs):
+            self._skip_if_not_configured(api_version)
+            if not self.is_logging_enabled:
+                kwargs.update({"logging_enable": False})
+            client = self.create_client(self.azure_keyvault_url, api_version=api_version, **kwargs)
+            await fn(test_class, client)
+
+        return _preparer
+
+    def create_client(self, vault_uri, **kwargs):
+        from azure.keyvault.secrets.aio import SecretClient
+
+        credential = self.get_credential(SecretClient, is_async=True)
+        return self.create_client_from_credential(SecretClient, credential=credential, vault_url=vault_uri, **kwargs)
+
+    def _skip_if_not_configured(self, api_version, **kwargs):
+        if is_live() and api_version != DEFAULT_VERSION:
+            pytest.skip("This test only uses the default API version for live tests")

sdk/keyvault/azure-keyvault-secrets/tests/_shared/test_case.py

@@ -4,22 +4,11 @@
 # ------------------------------------
 import time
 
-from azure_devtools.scenario_tests.patches import patch_time_sleep_api
-from devtools_testutils import AzureTestCase
+from azure.keyvault.secrets._shared import HttpChallengeCache
+from devtools_testutils import AzureRecordedTestCase
 
 
-class KeyVaultTestCase(AzureTestCase):
-    def __init__(self, *args, **kwargs):
-        if "match_body" not in kwargs:
-            kwargs["match_body"] = True
-
-        super(KeyVaultTestCase, self).__init__(*args, **kwargs)
-        self.replay_patches.append(patch_time_sleep_api)
-
-    def setUp(self):
-        self.list_test_size = 7
-        super(KeyVaultTestCase, self).setUp()
-
+class KeyVaultTestCase(AzureRecordedTestCase):
     def get_resource_name(self, name):
         """helper to create resources with a consistent, test-indicative prefix"""
         return super(KeyVaultTestCase, self).get_resource_name("livekvtest{}".format(name))
@@ -48,3 +37,7 @@ def _poll_until_exception(self, fn, expected_exception, max_retries=20, retry_de
                 return
 
         self.fail("expected exception {expected_exception} was not raised")
+
+    def teardown_method(self, method):
+        HttpChallengeCache.clear()
+        assert len(HttpChallengeCache._cache) == 0

sdk/keyvault/azure-keyvault-secrets/tests/_shared/test_case_async.py

@@ -4,26 +4,11 @@
 # ------------------------------------
 import asyncio
 
-from azure_devtools.scenario_tests.patches import mock_in_unit_test
-from devtools_testutils import AzureTestCase
+from azure.keyvault.secrets._shared import HttpChallengeCache
+from devtools_testutils import AzureRecordedTestCase
 
 
-def skip_sleep(unit_test):
-    async def immediate_return(_):
-        return
-
-    return mock_in_unit_test(unit_test, "asyncio.sleep", immediate_return)
-
-
-class KeyVaultTestCase(AzureTestCase):
-    def __init__(self, *args, match_body=True, **kwargs):
-        super().__init__(*args, match_body=match_body, **kwargs)
-        self.replay_patches.append(skip_sleep)
-
-    def setUp(self):
-        self.list_test_size = 7
-        super(KeyVaultTestCase, self).setUp()
-
+class KeyVaultTestCase(AzureRecordedTestCase):
     def get_resource_name(self, name):
         """helper to create resources with a consistent, test-indicative prefix"""
         return super(KeyVaultTestCase, self).get_resource_name("livekvtest{}".format(name))
@@ -51,3 +36,7 @@ async def _poll_until_exception(self, fn, expected_exception, max_retries=20, re
             except expected_exception:
                 return
         self.fail("expected exception {expected_exception} was not raised")
+
+    def teardown_method(self, method):
+        HttpChallengeCache.clear()
+        assert len(HttpChallengeCache._cache) == 0

sdk/keyvault/azure-keyvault-secrets/tests/_test_case.py

@@ -2,62 +2,49 @@
 # Copyright (c) Microsoft Corporation.
 # Licensed under the MIT License.
 # ------------------------------------
-import functools
+import os
 
+import pytest
 from azure.keyvault.secrets import ApiVersion
-from azure.keyvault.secrets._shared import HttpChallengeCache
 from azure.keyvault.secrets._shared.client_base import DEFAULT_VERSION
-from devtools_testutils import AzureTestCase, PowerShellPreparer
-from parameterized import parameterized, param
-import pytest
-
-
-def client_setup(testcase_func):
-    """decorator that creates a client to be passed in to a test method"""
-    @PowerShellPreparer("keyvault", azure_keyvault_url="https://vaultname.vault.azure.net")
-    @functools.wraps(testcase_func)
-    def wrapper(test_class_instance, azure_keyvault_url, api_version, **kwargs):
-        test_class_instance._skip_if_not_configured(api_version)
-        client = test_class_instance.create_client(azure_keyvault_url, api_version=api_version, **kwargs)
-
-        if kwargs.get("is_async"):
-            import asyncio
-
-            coroutine = testcase_func(test_class_instance, client)
-            loop = asyncio.get_event_loop()
-            loop.run_until_complete(coroutine)
-        else:
-            testcase_func(test_class_instance, client)
-    return wrapper
+from devtools_testutils import AzureRecordedTestCase, is_live
 
 
 def get_decorator(**kwargs):
     """returns a test decorator for test parameterization"""
-    params = [param(api_version=api_version, **kwargs) for api_version in ApiVersion]
-    return functools.partial(parameterized.expand, params, name_func=suffixed_test_name)
+    return [(api_version) for api_version in ApiVersion]
+
 
+class SecretsClientPreparer(AzureRecordedTestCase):
+    def __init__(self, **kwargs) -> None:
+        self.azure_keyvault_url = "https://vaultname.vault.azure.net"
 
-def suffixed_test_name(testcase_func, param_num, param):
-    return "{}_{}".format(testcase_func.__name__, parameterized.to_safe_name(param.kwargs.get("api_version")))
+        if is_live():
+            self.azure_keyvault_url = os.environ["AZURE_KEYVAULT_URL"]
 
+        self.is_logging_enabled = kwargs.pop("logging_enable", True)
+        if is_live():
+            os.environ["AZURE_TENANT_ID"] = os.environ["KEYVAULT_TENANT_ID"]
+            os.environ["AZURE_CLIENT_ID"] = os.environ["KEYVAULT_CLIENT_ID"]
+            os.environ["AZURE_CLIENT_SECRET"] = os.environ["KEYVAULT_CLIENT_SECRET"]
 
-class SecretsTestCase(AzureTestCase):
-    def tearDown(self):
-        HttpChallengeCache.clear()
-        assert len(HttpChallengeCache._cache) == 0
-        super(SecretsTestCase, self).tearDown()
+    def __call__(self, fn):
+        def _preparer(test_class, api_version, **kwargs):
+            self._skip_if_not_configured(api_version)
+            if not self.is_logging_enabled:
+                kwargs.update({"logging_enable": False})
+            client = self.create_client(self.azure_keyvault_url, **kwargs, api_version=api_version)
+            with client:
+                fn(test_class, client)
+
+        return _preparer
 
     def create_client(self, vault_uri, **kwargs):
-        if kwargs.pop("is_async", False):
-            from azure.keyvault.secrets.aio import SecretClient
-            credential = self.get_credential(SecretClient, is_async=True)
-        else:
-            from azure.keyvault.secrets import SecretClient
-            credential = self.get_credential(SecretClient)
-        return self.create_client_from_credential(
-            SecretClient, credential=credential, vault_url=vault_uri, **kwargs
-        )
+        from azure.keyvault.secrets import SecretClient
+
+        credential = self.get_credential(SecretClient)
+        return self.create_client_from_credential(SecretClient, credential=credential, vault_url=vault_uri, **kwargs)
 
     def _skip_if_not_configured(self, api_version, **kwargs):
-        if self.is_live and api_version != DEFAULT_VERSION:
+        if is_live() and api_version != DEFAULT_VERSION:
             pytest.skip("This test only uses the default API version for live tests")

sdk/keyvault/azure-keyvault-secrets/tests/conftest.py

@@ -0,0 +1,68 @@
+# --------------------------------------------------------------------------
+#
+# Copyright (c) Microsoft Corporation. All rights reserved.
+#
+# The MIT License (MIT)
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the ""Software""), to
+# deal in the Software without restriction, including without limitation the
+# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+# sell copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+# IN THE SOFTWARE.
+#
+# --------------------------------------------------------------------------
+import os
+import pytest
+from unittest import mock
+from devtools_testutils import is_live, test_proxy, add_oauth_response_sanitizer, add_general_regex_sanitizer
+
+
+@pytest.fixture(scope="session", autouse=True)
+def add_sanitizers(test_proxy):
+    azure_keyvault_url = os.getenv("azure_keyvault_url", "https://vaultname.vault.azure.net")
+    azure_keyvault_url = azure_keyvault_url.rstrip("/")
+    keyvault_tenant_id = os.getenv("keyvault_tenant_id", "keyvault_tenant_id")
+    keyvault_subscription_id = os.getenv("keyvault_subscription_id", "keyvault_subscription_id")
+
+    add_general_regex_sanitizer(regex=azure_keyvault_url, value="https://vaultname.vault.azure.net")
+    add_general_regex_sanitizer(regex=keyvault_tenant_id, value="00000000-0000-0000-0000-000000000000")
+    add_general_regex_sanitizer(regex=keyvault_subscription_id, value="00000000-0000-0000-0000-000000000000")
+    add_oauth_response_sanitizer()
+
+
+@pytest.fixture(scope="session", autouse=True)
+def patch_async_sleep():
+    async def immediate_return(_):
+        return
+
+    if not is_live():
+        with mock.patch("asyncio.sleep", immediate_return):
+            yield
+
+    else:
+        yield
+
+
+@pytest.fixture(scope="session", autouse=True)
+def patch_sleep():
+    def immediate_return(_):
+        return
+
+    if not is_live():
+        with mock.patch("time.sleep", immediate_return):
+            yield
+
+    else:
+        yield

sdk/keyvault/azure-keyvault-secrets/tests/perfstress_tests/get_secret.py

@@ -10,7 +10,6 @@
 
 
 class GetSecretTest(PerfStressTest):
-
     def __init__(self, arguments):
         super().__init__(arguments)
 

sdk/keyvault/azure-keyvault-secrets/tests/perfstress_tests/list_secrets.py

@@ -12,7 +12,6 @@
 
 
 class ListSecretsTest(PerfStressTest):
-
     def __init__(self, arguments):
         super().__init__(arguments)
 
@@ -31,11 +30,15 @@ async def global_setup(self):
         # Validate that vault contains 0 secrets (including soft-deleted secrets), since additional secrets
         # (including soft-deleted) impact performance.
         async for secret in self.async_client.list_properties_of_secrets():
-            raise Exception("KeyVault %s must contain 0 secrets (including soft-deleted) before starting perf test" \
-                % self.async_client.vault_url)
+            raise Exception(
+                "KeyVault %s must contain 0 secrets (including soft-deleted) before starting perf test"
+                % self.async_client.vault_url
+            )
         async for secret in self.async_client.list_deleted_secrets():
-            raise Exception("KeyVault %s must contain 0 secrets (including soft-deleted) before starting perf test" \
-                % self.async_client.vault_url)
+            raise Exception(
+                "KeyVault %s must contain 0 secrets (including soft-deleted) before starting perf test"
+                % self.async_client.vault_url
+            )
 
         await super().global_setup()
         create = [self.async_client.set_secret(name, "secret-value") for name in self.secret_names]
@@ -72,5 +75,5 @@ async def run_async(self):
     def add_arguments(parser):
         super(ListSecretsTest, ListSecretsTest).add_arguments(parser)
         parser.add_argument(
-            '--count', nargs='?', type=int, help='Number of secrets to list. Defaults to 10', default=10
+            "--count", nargs="?", type=int, help="Number of secrets to list. Defaults to 10", default=10
         )