Merge branch 'main' into feature/storage-stg101

Author: weirongw23-msft

doc/dev/test_proxy_troubleshooting.md

@@ -275,6 +275,9 @@ To run recorded tests successfully when recorded values are inconsistent or rand
 proxy provides a `variables` API. This makes it possible for a test to record the values of variables that were used
 during recording and use the same values in playback mode without a sanitizer.
 
+Note that the recorded variables **must** have string values. For example, trying to record an integer value for a
+variable will cause a test proxy error.
+
 For example, imagine that a test uses a randomized `table_uuid` variable when creating resources. The same random value
 for `table_uuid` can be used in playback mode by using this `variables` API.
 
@@ -293,14 +296,14 @@ class TestExample(AzureRecordedTestCase):
     @recorded_by_proxy
     def test_example(self, **kwargs):
         # In live mode, variables is an empty dictionary
-        # In playback mode, the value of variables is {"table_uuid": "random-value"}
+        # In playback mode, the value of variables is {"current_time": "<previously recorded time>"}
         variables = kwargs.pop("variables", {})
 
-        # To fetch variable values, use the `setdefault` method to look for a key ("table_uuid")
-        # and set a real value for that key if it's not present ("random-value")
-        table_uuid = variables.setdefault("table_uuid", "random-value")
+        # To fetch variable values, use the `setdefault` method to look for a key ("current_time")
+        # and set a real value for that key if it's not present (str(time.time()))
+        # Note that time.time() is converted from a float to a string to record it properly
+        current_time = variables.setdefault("current_time", str(time.time()))
 
-        # use variables["table_uuid"] when using the table UUID throughout the test
         ...
 
         # return the variables at the end of the test to record them

eng/pipelines/templates/steps/build-package-artifacts.yml

@@ -109,6 +109,7 @@ steps:
     condition: and(succeeded(), or(eq(variables['ENABLE_EXTENSION_BUILD'], 'true'), eq('${{ parameters.ArtifactSuffix }}', 'linux')))
 
   - script: |
+      sudo dpkg --configure -a
       sudo apt-get update
       sudo apt-get install -y qemu-user-static binfmt-support
       sudo update-binfmts --enable qemu-aarch64

eng/pipelines/templates/steps/install-portaudio.yml

@@ -1,6 +1,7 @@
 steps:
   - script: |
       if [[ "$AGENT_OS" == "Linux" ]]; then
+        sudo dpkg --configure -a
         sudo apt-get update
         sudo apt-get install -y portaudio19-dev libasound2-dev
       elif [[ "$AGENT_OS" == "Darwin" ]]; then

eng/tools/azure-sdk-tools/devtools_testutils/proxy_testcase.py

@@ -114,32 +114,31 @@ def start_record_or_playback(test_id: str) -> "Tuple[str, Dict[str, str]]":
 
 
 def stop_record_or_playback(test_id: str, recording_id: str, test_variables: "Dict[str, str]") -> None:
-    try:
-        http_client = get_http_client()
-        if is_live():
-            http_client.request(
-                method="POST",
-                url=RECORDING_STOP_URL,
-                headers={
-                    "x-recording-file": test_id,
-                    "x-recording-id": recording_id,
-                    "x-recording-save": "true",
-                    "Content-Type": "application/json",
-                },
-                # tests don't record successfully unless test_variables is a dictionary
-                body=json.dumps(test_variables).encode("utf-8") if test_variables else "{}",
-            )
-        else:
-            http_client.request(
-                method="POST",
-                url=PLAYBACK_STOP_URL,
-                headers={"x-recording-id": recording_id},
-            )
-    except HTTPError as e:
+    http_client = get_http_client()
+    if is_live():
+        response = http_client.request(
+            method="POST",
+            url=RECORDING_STOP_URL,
+            headers={
+                "x-recording-file": test_id,
+                "x-recording-id": recording_id,
+                "x-recording-save": "true",
+                "Content-Type": "application/json",
+            },
+            # tests don't record successfully unless test_variables is a dictionary
+            body=json.dumps(test_variables or {}).encode("utf-8"),
+        )
+    else:
+        response = http_client.request(
+            method="POST",
+            url=PLAYBACK_STOP_URL,
+            headers={"x-recording-id": recording_id},
+        )
+    if response.status >= 400:
         raise HttpResponseError(
             "The test proxy ran into an error while ending the session. Make sure any test variables you record have "
-            "string values."
-        ) from e
+            f"string values. Full error details: {response.data}"
+        )
 
 
 def get_proxy_netloc() -> "Dict[str, str]":

sdk/communication/azure-communication-callautomation/tests/conftest.py

@@ -3,6 +3,7 @@
 # Licensed under the MIT License. See License.txt in the project root for
 # license information.
 # --------------------------------------------------------------------------
+import sys
 import pytest
 import os
 import asyncio
@@ -14,15 +15,21 @@
     add_body_key_sanitizer,
     add_general_string_sanitizer,
     remove_batch_sanitizers,
+    set_custom_default_matcher
 )
 
 
 # autouse=True will trigger this fixture on each pytest run, even if it's not explicitly used by a test method
 @pytest.fixture(scope="session", autouse=True)
 def start_proxy(test_proxy):
-
     set_default_session_settings()
 
+    # On python 3.14, azure-core sends an additional 'Accept-Encoding' header value that causes playback issues.
+    # By ignoring it, we can avoid really wonky mismatch errors, while still validating the other headers
+    if sys.version_info >= (3, 14):
+        headers_to_ignore = "Accept-Encoding"
+        set_custom_default_matcher(ignored_headers=headers_to_ignore)
+
     fake_connection_str = "endpoint=https://sanitized.communication.azure.com/;accesskey=fake=="
     connection_str = os.getenv("COMMUNICATION_LIVETEST_STATIC_CONNECTION_STRING", fake_connection_str)
     add_general_string_sanitizer(target=connection_str, value=fake_connection_str)

sdk/communication/azure-communication-identity/tests/conftest.py

@@ -27,7 +27,9 @@
 # cSpell:ignore ests
 import pytest
 import os
+import sys
 from devtools_testutils import (
+    set_custom_default_matcher,
     test_proxy,
     add_general_regex_sanitizer,
     add_header_regex_sanitizer,
@@ -42,6 +44,13 @@
 @pytest.fixture(scope="session", autouse=True)
 def add_sanitizers(test_proxy):
     set_default_session_settings()
+
+    # On python 3.14, azure-core sends an additional 'Accept-Encoding' header value that causes playback issues.
+    # By ignoring it, we can avoid really wonky mismatch errors, while still validating the other headers
+    if sys.version_info >= (3, 14):
+        headers_to_ignore = "Accept-Encoding"
+        set_custom_default_matcher(ignored_headers=headers_to_ignore)
+
     add_oauth_response_sanitizer()
 
     connection_str = os.environ.get("COMMUNICATION_LIVETEST_DYNAMIC_CONNECTION_STRING")

sdk/communication/azure-communication-jobrouter/tests/conftest.py

@@ -36,6 +36,7 @@
     add_uri_regex_sanitizer,
     add_body_key_sanitizer,
     remove_batch_sanitizers,
+    set_custom_default_matcher
 )
 from router_test_constants import SANITIZED, FAKE_FUNCTION_URI, FAKE_ENDPOINT, FAKE_CONNECTION_STRING
 from azure.communication.jobrouter._shared.utils import parse_connection_str
@@ -49,6 +50,12 @@
 def start_proxy(test_proxy):
     set_default_session_settings()
 
+    # On python 3.14, azure-core sends an additional 'Accept-Encoding' header value that causes playback issues.
+    # By ignoring it, we can avoid really wonky mismatch errors, while still validating the other headers
+    if sys.version_info >= (3, 14):
+        headers_to_ignore = "Accept-Encoding"
+        set_custom_default_matcher(ignored_headers=headers_to_ignore)
+
     communication_connection_string = os.getenv(
         "COMMUNICATION_LIVETEST_DYNAMIC_CONNECTION_STRING", FAKE_CONNECTION_STRING
     )

sdk/communication/azure-communication-messages/tests/conftest.py

@@ -27,7 +27,9 @@
 # cSpell:ignore ests
 import pytest
 import os
+import sys
 from devtools_testutils import (
+    set_custom_default_matcher,
     test_proxy,
     add_header_regex_sanitizer,
     set_default_session_settings,
@@ -43,6 +45,13 @@
 @pytest.fixture(scope="session", autouse=True)
 def start_proxy(test_proxy):
     set_default_session_settings()
+
+    # On python 3.14, azure-core sends an additional 'Accept-Encoding' header value that causes playback issues.
+    # By ignoring it, we can avoid really wonky mismatch errors, while still validating the other headers
+    if sys.version_info >= (3, 14):
+        headers_to_ignore = "Accept-Encoding"
+        set_custom_default_matcher(ignored_headers=headers_to_ignore)
+
     add_oauth_response_sanitizer()
 
     FAKE_CONNECTION_STRING = "endpoint=https://sanitized.unitedstates.ppe.communication.azure.net/;accesskey=fake==="

sdk/communication/azure-communication-rooms/tests/conftest.py

@@ -25,12 +25,14 @@
 # --------------------------------------------------------------------------
 import pytest
 import os
+import sys
 from devtools_testutils import (
     add_general_string_sanitizer,
     add_header_regex_sanitizer,
     set_default_session_settings,
     remove_batch_sanitizers,
     add_uri_regex_sanitizer,
+    set_custom_default_matcher
 )
 from azure.communication.rooms._shared.utils import parse_connection_str
 
@@ -39,6 +41,12 @@
 def add_sanitizers(test_proxy):
     set_default_session_settings()
 
+    # On python 3.14, azure-core sends an additional 'Accept-Encoding' header value that causes playback issues.
+    # By ignoring it, we can avoid really wonky mismatch errors, while still validating the other headers
+    if sys.version_info >= (3, 14):
+        headers_to_ignore = "Accept-Encoding"
+        set_custom_default_matcher(ignored_headers=headers_to_ignore)
+
     communication_connection_string = os.getenv(
         "COMMUNICATION_CONNECTION_STRING_ROOMS", "endpoint=https://sanitized.communication.azure.com/;accesskey=fake==="
     )

sdk/communication/azure-communication-sms/tests/conftest.py

@@ -27,20 +27,29 @@
 # cSpell:ignore ests
 import pytest
 import os
+import sys
 from devtools_testutils import (
     test_proxy,
     add_header_regex_sanitizer,
     set_default_session_settings,
     add_body_key_sanitizer,
     add_oauth_response_sanitizer,
     add_general_string_sanitizer,
+    set_custom_default_matcher
 )
 from azure.communication.sms._shared.utils import parse_connection_str
 
 
 @pytest.fixture(scope="session", autouse=True)
 def add_sanitizers(test_proxy):
     set_default_session_settings()
+
+    # On python 3.14, azure-core sends an additional 'Accept-Encoding' header value that causes playback issues.
+    # By ignoring it, we can avoid really wonky mismatch errors, while still validating the other headers
+    if sys.version_info >= (3, 14):
+        headers_to_ignore = "Accept-Encoding"
+        set_custom_default_matcher(ignored_headers=headers_to_ignore)
+
     add_oauth_response_sanitizer()
 
     connection_str = os.environ.get("COMMUNICATION_LIVETEST_STATIC_CONNECTION_STRING")