Eliminate Remaining requests usage within azure-sdk-tools (Azure#27101)

* eliminate remaining requests usage from azure-sdk-tools
* directly add Paul's recommendations, definitely on point
* ensure that a None header gets flattened to empty string.
* ensure that the urllib3 clients honor the same certificate bundle that requests does

Author: scbedd

tools/azure-sdk-tools/devtools_testutils/proxy_startup.py

@@ -6,7 +6,6 @@
 
 import os
 import logging
-import requests
 import shlex
 import sys
 import time
@@ -15,6 +14,8 @@
 import pytest
 import subprocess
 
+import urllib3
+
 from .config import PROXY_URL
 from .helpers import is_live_and_not_recording
 from .sanitizers import add_remove_header_sanitizer, set_custom_default_matcher
@@ -33,6 +34,12 @@
 
 discovered_roots = []
 
+from urllib3 import PoolManager, Retry
+from urllib3.exceptions import HTTPError
+
+http_client = PoolManager(retries=Retry(total=1, raise_on_status=False))
+
+
 def get_image_tag(repo_root: str) -> str:
     """Gets the test proxy Docker image tag from the target_version.txt file in /eng/common/testproxy"""
     version_file_location = os.path.relpath("eng/common/testproxy/target_version.txt")
@@ -84,10 +91,10 @@ def delete_container() -> None:
 def check_availability() -> None:
     """Attempts request to /Info/Available. If a test-proxy instance is responding, we should get a response."""
     try:
-        response = requests.get(PROXY_CHECK_URL, timeout=10)
-        return response.status_code
+        response = http_client.request(method="GET", url=PROXY_CHECK_URL, timeout=10)
+        return response.status
     # We get an SSLError if the container is started but the endpoint isn't available yet
-    except requests.exceptions.SSLError as sslError:
+    except urllib3.exceptions.SSLError as sslError:
         _LOGGER.debug(sslError)
         return 404
     except Exception as e:
@@ -171,7 +178,7 @@ def start_test_proxy(request) -> None:
                 proc = subprocess.Popen(
                     shlex.split('test-proxy start --storage-location="{}" -- --urls "{}"'.format(root, PROXY_URL)),
                     stdout=log,
-                    stderr=log
+                    stderr=log,
                 )
                 os.environ[TOOL_ENV_VAR] = str(proc.pid)
         else:

tools/azure-sdk-tools/devtools_testutils/proxy_testcase.py

@@ -4,7 +4,6 @@
 # license information.
 # --------------------------------------------------------------------------
 import logging
-import requests
 import six
 import os
 from typing import TYPE_CHECKING
@@ -18,9 +17,13 @@
 
 # the trimming function to clean up incoming arguments to the test function we are wrapping
 from azure_devtools.scenario_tests.utilities import trim_kwargs_from_test_function
+
 from .config import PROXY_URL
 from .helpers import get_test_id, is_live, is_live_and_not_recording, set_recording_id
 from .proxy_startup import discovered_roots
+from urllib3 import PoolManager, Retry
+from urllib3.exceptions import HTTPError
+import json
 
 if TYPE_CHECKING:
     from typing import Callable, Dict, Tuple
@@ -29,13 +32,22 @@
 # To learn about how to migrate SDK tests to the test proxy, please refer to the migration guide at
 # https://github.com/Azure/azure-sdk-for-python/blob/main/doc/dev/test_proxy_migration_guide.md
 
+if os.getenv("REQUESTS_CA_BUNDLE"):
+    http_client = PoolManager(
+        retries=Retry(total=3, raise_on_status=False),
+        cert_reqs="CERT_REQUIRED",
+        ca_certs=os.getenv("REQUESTS_CA_BUNDLE"),
+    )
+else:
+    http_client = PoolManager(retries=Retry(total=3, raise_on_status=False))
 
 # defaults
 RECORDING_START_URL = "{}/record/start".format(PROXY_URL)
 RECORDING_STOP_URL = "{}/record/stop".format(PROXY_URL)
 PLAYBACK_START_URL = "{}/playback/start".format(PROXY_URL)
 PLAYBACK_STOP_URL = "{}/playback/stop".format(PROXY_URL)
 
+
 def get_recording_assets(test_id: str) -> str:
     """
     Used to retrieve the assets.json given a PYTEST_CURRENT_TEST test id.
@@ -58,45 +70,50 @@ def get_recording_assets(test_id: str) -> str:
 
     return None
 
+
 def start_record_or_playback(test_id: str) -> "Tuple[str, Dict[str, str]]":
     """Sends a request to begin recording or playing back the provided test.
 
     This returns a tuple, (a, b), where a is the recording ID of the test and b is the `variables` dictionary that maps
     test variables to values. If no variable dictionary was stored when the test was recorded, b is an empty dictionary.
     """
     variables = {}  # this stores a dictionary of test variable values that could have been stored with a recording
-    
+
     json_payload = {"x-recording-file": test_id}
     assets_json = get_recording_assets(test_id)
     if assets_json:
-        json_payload["x-recording-assets-file"] = assets_json    
+        json_payload["x-recording-assets-file"] = assets_json
+
+    encoded_payload = json.dumps(json_payload).encode("utf-8")
 
     if is_live():
-        result = requests.post(
-            RECORDING_START_URL,
-            json=json_payload,
+        result = http_client.request(
+            method="POST",
+            url=RECORDING_START_URL,
+            body=encoded_payload,
         )
-        if result.status_code != 200:
-            message = six.ensure_str(result._content)
+        if result.status != 200:
+            message = six.ensure_str(result.data)
             raise HttpResponseError(message=message)
         recording_id = result.headers["x-recording-id"]
 
     else:
-        result = requests.post(
-            PLAYBACK_START_URL,
-            json=json_payload,
+        result = http_client.request(
+            method="POST",
+            url=PLAYBACK_START_URL,
+            body=encoded_payload,
         )
-        if result.status_code != 200:
-            message = six.ensure_str(result._content)
+        if result.status != 200:
+            message = six.ensure_str(result.data)
             raise HttpResponseError(message=message)
 
         try:
             recording_id = result.headers["x-recording-id"]
         except KeyError as ex:
             six.raise_from(ValueError("No recording file found for {}".format(test_id)), ex)
-        if result.text:
+        if result.data:
             try:
-                variables = result.json()
+                variables = json.loads(result.data.decode("utf-8"))
             except ValueError as ex:  # would be a JSONDecodeError on Python 3, which subclasses ValueError
                 six.raise_from(
                     ValueError("The response body returned from starting playback did not contain valid JSON"),
@@ -109,25 +126,27 @@ def start_record_or_playback(test_id: str) -> "Tuple[str, Dict[str, str]]":
 
 
 def stop_record_or_playback(test_id: str, recording_id: str, test_variables: "Dict[str, str]") -> None:
-    if is_live():
-        response = requests.post(
-            RECORDING_STOP_URL,
-            headers={
-                "x-recording-file": test_id,
-                "x-recording-id": recording_id,
-                "x-recording-save": "true",
-                "Content-Type": "application/json",
-            },
-            json=test_variables or {},  # tests don't record successfully unless test_variables is a dictionary
-        )
-    else:
-        response = requests.post(
-            PLAYBACK_STOP_URL,
-            headers={"x-recording-id": recording_id},
-        )
     try:
-        response.raise_for_status()
-    except requests.HTTPError as e:
+        if is_live():
+            http_client.request(
+                method="POST",
+                url=RECORDING_STOP_URL,
+                headers={
+                    "x-recording-file": test_id,
+                    "x-recording-id": recording_id,
+                    "x-recording-save": "true",
+                    "Content-Type": "application/json",
+                },
+                # tests don't record successfully unless test_variables is a dictionary
+                body=json.dumps(test_variables).encode("utf-8") if test_variables else "{}",
+            )
+        else:
+            http_client.request(
+                method="POST",
+                url=PLAYBACK_STOP_URL,
+                headers={"x-recording-id": recording_id},
+            )
+    except HTTPError as e:
         raise HttpResponseError(
             "The test proxy ran into an error while ending the session. Make sure any test variables you record have "
             "string values."

tools/azure-sdk-tools/devtools_testutils/sanitizers.py

@@ -4,15 +4,25 @@
 # license information.
 # --------------------------------------------------------------------------
 from typing import TYPE_CHECKING
-from urllib.error import HTTPError
-import requests
+
+
+from urllib3 import PoolManager, Retry
+import json, os
 
 from .config import PROXY_URL
 from .helpers import get_recording_id, is_live, is_live_and_not_recording
 
 if TYPE_CHECKING:
     from typing import Any, Iterable, Optional
 
+if os.getenv("REQUESTS_CA_BUNDLE"):
+    http_client = PoolManager(
+        retries=Retry(total=3, raise_on_status=True),
+        cert_reqs="CERT_REQUIRED",
+        ca_certs=os.getenv("REQUESTS_CA_BUNDLE"),
+    )
+else:
+    http_client = PoolManager(retries=Retry(total=3, raise_on_status=True))
 
 # This file contains methods for adjusting many aspects of test proxy behavior:
 #
@@ -552,9 +562,17 @@ def _send_matcher_request(matcher: str, headers: dict, parameters: "Optional[dic
         return
 
     headers_to_send = {"x-abstraction-identifier": matcher}
-    headers_to_send.update(headers)
-    response = requests.post(f"{PROXY_URL}/Admin/SetMatcher", headers=headers_to_send, json=parameters)
-    response.raise_for_status()
+
+    for key in headers:
+        if headers[key] is not None:
+            headers_to_send[key] = headers[key]
+
+    http_client.request(
+        method="POST",
+        url=f"{PROXY_URL}/Admin/SetMatcher",
+        headers=headers_to_send,
+        body=json.dumps(parameters).encode("utf-8"),
+    )
 
 
 def _send_recording_options_request(parameters: dict, headers: "Optional[dict]" = None) -> None:
@@ -571,8 +589,17 @@ def _send_recording_options_request(parameters: dict, headers: "Optional[dict]"
     if is_live_and_not_recording():
         return
 
-    response = requests.post(f"{PROXY_URL}/Admin/SetRecordingOptions", headers=headers, json=parameters)
-    response.raise_for_status()
+    headers_to_send = {}
+    for key in headers:
+        if headers[key] is not None:
+            headers_to_send[key] = headers[key]
+
+    http_client.request(
+        method="POST",
+        url=f"{PROXY_URL}/Admin/SetRecordingOptions",
+        headers=headers_to_send,
+        body=json.dumps(parameters).encode("utf-8"),
+    )
 
 
 def _send_reset_request(headers: dict) -> None:
@@ -587,8 +614,12 @@ def _send_reset_request(headers: dict) -> None:
     if is_live_and_not_recording():
         return
 
-    response = requests.post(f"{PROXY_URL}/Admin/Reset", headers=headers)
-    response.raise_for_status()
+    headers_to_send = {}
+    for key in headers:
+        if headers[key] is not None:
+            headers_to_send[key] = headers[key]
+
+    request = http_client.request(method="POST", url=f"{PROXY_URL}/Admin/Reset", headers=headers_to_send)
 
 
 def _send_sanitizer_request(sanitizer: str, parameters: dict) -> None:
@@ -604,15 +635,15 @@ def _send_sanitizer_request(sanitizer: str, parameters: dict) -> None:
     if is_live_and_not_recording():
         return
 
-    response = requests.post(
-        "{}/Admin/AddSanitizer".format(PROXY_URL),
+    http_client.request(
+        method="POST",
+        url="{}/Admin/AddSanitizer".format(PROXY_URL),
         headers={
             "x-abstraction-identifier": sanitizer,
             "Content-Type": "application/json",
         },
-        json=parameters,
+        body=json.dumps(parameters).encode("utf-8"),
     )
-    response.raise_for_status()
 
 
 def _send_transform_request(transform: str, parameters: dict) -> None:
@@ -627,9 +658,9 @@ def _send_transform_request(transform: str, parameters: dict) -> None:
     if is_live():
         return
 
-    response = requests.post(
-        f"{PROXY_URL}/Admin/AddTransform",
+    http_client.request(
+        method="POST",
+        url=f"{PROXY_URL}/Admin/AddTransform",
         headers={"x-abstraction-identifier": transform, "Content-Type": "application/json"},
-        json=parameters,
+        body=json.dumps(parameters).encode("utf-8"),
     )
-    response.raise_for_status()

tools/azure-sdk-tools/packaging_tools/swaggertosdk/SwaggerToSdkCore.py

@@ -4,11 +4,12 @@
 import json
 import logging
 import os
+from pickle import GET
 import re
 import tempfile
 from pathlib import Path
 
-import requests
+from urllib3 import PoolManager, Retry
 
 from github import Github, UnknownObjectException
 
@@ -26,6 +27,8 @@
 
 DEFAULT_COMMIT_MESSAGE = "Generated from {hexsha}"
 
+http_client = PoolManager(retries=Retry(total=3, raise_on_status=False))
+
 
 def build_file_content():
     autorest_version = autorest_latest_version_finder()
@@ -176,14 +179,14 @@ def read_config_from_github(sdk_id, branch="main", gh_token=None):
     _LOGGER.debug("Will try to download: %s", raw_link)
     _LOGGER.debug("Token is defined: %s", gh_token is not None)
     headers = {"Authorization": "token {}".format(gh_token)} if gh_token else {}
-    response = requests.get(raw_link, headers=headers)
-    if response.status_code != 200:
+    response = http_client.request(method="GET", url=raw_link, headers=headers)
+    if response.status != 200:
         raise ValueError(
             "Unable to download conf file for SDK {} branch {}: status code {}".format(
-                sdk_id, branch, response.status_code
+                sdk_id, branch, response.status
             )
         )
-    return json.loads(response.text)
+    return json.loads(response.data.decode('utf-8'))
 
 
 def extract_conf_from_readmes(swagger_files_in_pr, restapi_git_folder, sdk_git_id, config, force_generation=False):