[translation] convert to test proxy (Azure#21764)

* remove unittest dependency

* initial work

* some small changse

* remove endpoint trailing slash on sanitizer value

* updates for playback

* test translation - convert to proxy + recordings

* cancel, document status, supported formats tests - conversion + recordings

* list translations/documents - convert and record

* add test proxy to ci/tests.yml

* fixes

* rstrip endpoint

* add oauth sanitizer

* update conftest to automatically start proxy

* update rstrip endpoint with better err msg

* try pinning storage api version for sas urls

* set bodiless matcher for sas urls

* remove proxy from tests.yml

* rerecord

* rerecord with 1251067 image tag

* use latest tag to get logging

* go back to tag 1209124

* try rerecord

* pin new version of proxy tool

* patch from main, bumping proxy version

* additional logging output

* sanitizers to only be test level.

* import appropriate deferred_send function from sanitizers.go

* this will work, but is extremely inefficient. need to do better with the SAS token overrides

* few test fixes

* restore everything to align with main

* rerecord all tests

* try modifying ppe to only run with one python version

* try fixing python version filtering for ppe

* switch to passing kwargs based on latest test guidance

* pop variables

Co-authored-by: scbedd <[email protected]>

Author: kristapratico

sdk/translation/azure-ai-translation-document/azure/ai/translation/document/_client.py

@@ -74,14 +74,17 @@ def __init__(
                 :dedent: 4
                 :caption: Creating the DocumentTranslationClient with a token credential.
         """
-        self._endpoint = endpoint
+        try:
+            self._endpoint = endpoint.rstrip("/")
+        except AttributeError:
+            raise ValueError("Parameter 'endpoint' must be a string.")
         self._credential = credential
         self._api_version = kwargs.pop("api_version", None)
 
         authentication_policy = get_authentication_policy(credential)
         polling_interval = kwargs.pop("polling_interval", POLLING_INTERVAL)
         self._client = _BatchDocumentTranslationClient(
-            endpoint=endpoint,
+            endpoint=self._endpoint,
             credential=credential,  # type: ignore
             api_version=self._api_version,
             sdk_moniker=USER_AGENT,

sdk/translation/azure-ai-translation-document/azure/ai/translation/document/aio/_client_async.py

@@ -80,14 +80,17 @@ def __init__(
                 :dedent: 4
                 :caption: Creating the DocumentTranslationClient with a token credential.
         """
-        self._endpoint = endpoint
+        try:
+            self._endpoint = endpoint.rstrip("/")
+        except AttributeError:
+            raise ValueError("Parameter 'endpoint' must be a string.")
         self._credential = credential
         self._api_version = kwargs.pop("api_version", None)
 
         authentication_policy = get_authentication_policy(credential)
         polling_interval = kwargs.pop("polling_interval", POLLING_INTERVAL)
         self._client = _BatchDocumentTranslationClient(
-            endpoint=endpoint,
+            endpoint=self._endpoint,
             credential=credential,  # type: ignore
             api_version=self._api_version,
             sdk_moniker=USER_AGENT,

sdk/translation/azure-ai-translation-document/tests/asynctestcase.py

@@ -3,29 +3,20 @@
 # Licensed under the MIT License.
 # ------------------------------------
 
-import os
+from devtools_testutils import (
+    AzureRecordedTestCase
+)
 from testcase import DocumentTranslationTest, Document
 from azure.ai.translation.document import DocumentTranslationInput, TranslationTarget
 
-class AsyncDocumentTranslationTest(DocumentTranslationTest):
 
-    def __init__(self, method_name):
-        super().__init__(method_name)
-
-    def generate_oauth_token(self):
-        if self.is_live:
-            from azure.identity.aio import ClientSecretCredential
-            return ClientSecretCredential(
-                os.getenv("TRANSLATION_TENANT_ID"),
-                os.getenv("TRANSLATION_CLIENT_ID"),
-                os.getenv("TRANSLATION_CLIENT_SECRET"),
-            )
+class AsyncDocumentTranslationTest(DocumentTranslationTest, AzureRecordedTestCase):
 
     async def _begin_and_validate_translation_async(self, async_client, translation_inputs, total_docs_count, language=None):
         # submit operation
         poller = await async_client.begin_translation(translation_inputs)
-        self.assertIsNotNone(poller.id)
-        self.assertIsNotNone(poller.details.id)
+        assert poller.id is not None
+        assert poller.details.id is not None
         # wait for result
         doc_statuses = await poller.result()
         # validate
@@ -36,6 +27,8 @@ async def _begin_and_validate_translation_async(self, async_client, translation_
 
     # client helpers
     async def _begin_multiple_translations_async(self, async_client, operations_count, **kwargs):
+        container_suffix = kwargs.pop('container_suffix', "")
+        variables = kwargs.pop('variables', {})
         wait_for_operation = kwargs.pop('wait', True)
         language_code = kwargs.pop('language_code', "es")
         docs_per_operation = kwargs.pop('docs_per_operation', 2)
@@ -49,8 +42,8 @@ async def _begin_multiple_translations_async(self, async_client, operations_coun
                 no need for async container clients!
             '''
             blob_data = Document.create_dummy_docs(docs_per_operation)
-            source_container_sas_url = self.create_source_container(data=blob_data)
-            target_container_sas_url = self.create_target_container()
+            source_container_sas_url = self.create_source_container(data=blob_data, variables=variables, container_suffix=str(i)+container_suffix)
+            target_container_sas_url = self.create_target_container(variables=variables, container_suffix=str(i)+container_suffix)
 
             # prepare translation inputs
             translation_inputs = [
@@ -67,7 +60,7 @@ async def _begin_multiple_translations_async(self, async_client, operations_coun
 
             # submit multiple operations
             poller = await async_client.begin_translation(translation_inputs)
-            self.assertIsNotNone(poller.id)
+            assert poller.id is not None
             if wait_for_operation:
                 await poller.result()
             else:
@@ -78,13 +71,14 @@ async def _begin_multiple_translations_async(self, async_client, operations_coun
 
     async def _begin_and_validate_translation_with_multiple_docs_async(self, async_client, docs_count, **kwargs):
         # get input params
+        variables = kwargs.pop('variables', {})
         wait_for_operation = kwargs.pop('wait', False)
         language_code = kwargs.pop('language_code', "es")
 
         # prepare containers and test data
         blob_data = Document.create_dummy_docs(docs_count=docs_count)
-        source_container_sas_url = self.create_source_container(data=blob_data)
-        target_container_sas_url = self.create_target_container()
+        source_container_sas_url = self.create_source_container(data=blob_data, variables=variables)
+        target_container_sas_url = self.create_target_container(variables=variables)
 
         # prepare translation inputs
         translation_inputs = [
@@ -98,10 +92,9 @@ async def _begin_and_validate_translation_with_multiple_docs_async(self, async_c
                 ]
             )
         ]
-
         # submit operation
         poller = await async_client.begin_translation(translation_inputs)
-        self.assertIsNotNone(poller.id)
+        assert poller.id is not None
         # wait for result
         if wait_for_operation:
             result = await poller.result()

sdk/translation/azure-ai-translation-document/tests/conftest.py

@@ -0,0 +1,52 @@
+
+# coding: utf-8
+# -------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for
+# license information.
+# --------------------------------------------------------------------------
+
+import os
+import sys
+import pytest
+from devtools_testutils import (
+    test_proxy,
+    is_live,
+    add_remove_header_sanitizer,
+    add_general_regex_sanitizer,
+    add_oauth_response_sanitizer
+)
+from azure.storage.blob import BlobServiceClient
+
+# Ignore async tests for Python < 3.5
+collect_ignore_glob = []
+if sys.version_info < (3, 5):
+    collect_ignore_glob.append("*_async.py")
+
+
+@pytest.fixture(scope="session", autouse=True)
+def add_sanitizers(test_proxy):
+    add_remove_header_sanitizer(headers="Ocp-Apim-Subscription-Key")
+    add_remove_header_sanitizer(headers="Retry-After")
+    add_general_regex_sanitizer(
+        value="fakeendpoint",
+        regex="(?<=\\/\\/)[a-z-]+(?=\\.cognitiveservices\\.azure\\.com)"
+    )
+    add_general_regex_sanitizer(
+        regex="(?<=\\/\\/)[a-z]+(?=(?:|-secondary)\\.(?:table|blob|queue)\\.core\\.windows\\.net)",
+        value="fakeendpoint",
+    )
+    add_oauth_response_sanitizer()
+
+    # run tests
+    yield
+
+    # Dogfood env uses a static storage account so we clean up the blob resources
+    # This is unnecessary for AzureCloud where each storage account is deleted at the end of testing
+    if is_live() and os.getenv("TRANSLATION_ENVIRONMENT") == "Dogfood":
+        client = BlobServiceClient(
+            "https://" + os.getenv("TRANSLATION_DOCUMENT_STORAGE_NAME") + ".blob.core.windows.net/",
+            os.getenv("TRANSLATION_DOCUMENT_STORAGE_KEY")
+        )
+        for container in client.list_containers():
+            client.delete_container(container)

sdk/translation/azure-ai-translation-document/tests/preparer.py

@@ -13,7 +13,7 @@
 DocumentTranslationPreparer = functools.partial(
     PowerShellPreparer,
     'translation',
-    translation_document_test_endpoint="https://redacted.cognitiveservices.azure.com/",
+    translation_document_test_endpoint="https://fakeendpoint.cognitiveservices.azure.com",
     translation_document_test_api_key="fakeZmFrZV9hY29jdW50X2tleQ==",
     translation_document_name="redacted",
     translation_document_storage_name="redacted",