Use prebuilt wheel dir to resolve necessary azure requirements before using repo version (#33911)

* enable our prebuilt wheel dir to override a located relative path

Author: scbedd

tools/azure-sdk-tools/ci_tools/functions.py

@@ -28,8 +28,7 @@
     "azure-ai-anomalydetector",
 ]
 
-NO_TESTS_ALLOWED = [
-]
+NO_TESTS_ALLOWED = []
 
 
 META_PACKAGES = ["azure", "azure-mgmt", "azure-keyvault"]
@@ -282,6 +281,27 @@ def get_package_from_repo(pkg_name: str, repo_root: str = None) -> ParsedSetup:
     return None
 
 
+def get_package_from_repo_or_folder(req: str, prebuilt_wheel_dir: str = None) -> str:
+    """Takes a package name and a possible prebuilt wheel directory. Attempts to resolve a wheel that matches the package name, and if it can't,
+    attempts to find the package within the repo to install directly from path on disk.
+
+    During a CI build, it is preferred that the package is installed from a prebuilt wheel directory, as multiple CI environments attempting to install the relative
+    req can cause inconsistent installation issues during parallel tox environment execution.
+    """
+
+    local_package = get_package_from_repo(req)
+
+    if prebuilt_wheel_dir and os.path.exists(prebuilt_wheel_dir):
+        prebuilt_package = discover_prebuilt_package(prebuilt_wheel_dir, local_package.setup_filename, "wheel")
+        if prebuilt_package:
+            # return the first package found, there should only be a single one matching given that our prebuilt wheel directory
+            # is populated by the replacement of dev_reqs.txt with the prebuilt wheels
+            # ref tox_harness replace_dev_reqs() calls
+            return os.path.join(prebuilt_wheel_dir, prebuilt_package[0])
+
+    return local_package.folder
+
+
 def get_version_from_repo(pkg_name: str, repo_root: str = None) -> str:
     pkg_info = get_package_from_repo(pkg_name, repo_root)
     if pkg_info:
@@ -411,6 +431,7 @@ def pip_install(requirements: List[str], include_dependencies: bool = True, pyth
 
     return True
 
+
 def pip_uninstall(requirements: List[str], python_executable: str) -> bool:
     """
     Attempts to invoke an install operation using the invoking python's pip. Empty requirements are auto-success.
@@ -480,7 +501,7 @@ def pytest(args: [], cwd: str = None, python_executable: str = None) -> bool:
         result = subprocess.run(commands, cwd=cwd)
     else:
         result = subprocess.run(commands)
-    
+
     return result.returncode == 0
 
 

tools/azure-sdk-tools/ci_tools/scenario/generation.py

@@ -16,7 +16,7 @@
 )
 from ci_tools.build import cleanup_build_artifacts, create_package
 from ci_tools.parsing import ParsedSetup, parse_require
-from ci_tools.functions import get_package_from_repo, find_whl, get_pip_list_output, pytest
+from ci_tools.functions import get_package_from_repo_or_folder, find_whl, get_pip_list_output, pytest
 from .managed_virtual_env import ManagedVirtualEnv
 
 
@@ -71,6 +71,8 @@ def create_package_and_install(
         setup_py_path, distribution_directory, target_setup, package_type, force_create
     )
 
+    target_package = ParsedSetup.from_path(setup_py_path)
+
     if skip_install:
         logging.info("Flag to skip install whl is passed. Skipping package installation")
     else:
@@ -116,6 +118,7 @@ def create_package_and_install(
                         addition_necessary = True
                         # get all installed packages
                         installed_pkgs = get_pip_list_output(python_exe)
+                        logging.info("Installed packages: {}".format(installed_pkgs))
 
                         # parse the specifier
                         req_name, req_specifier = parse_require(req)
@@ -150,7 +153,12 @@ def create_package_and_install(
 
                         additional_downloaded_reqs = [
                             os.path.abspath(os.path.join(tmp_dl_folder, pth)) for pth in os.listdir(tmp_dl_folder)
-                        ] + [get_package_from_repo(relative_req).folder for relative_req in non_present_reqs]
+                        ] + [
+                            get_package_from_repo_or_folder(
+                                package_name, os.path.join(target_package.folder, ".tmp_whl_dir")
+                            )
+                            for package_name in non_present_reqs
+                        ]
 
             commands = [python_exe, "-m", "pip", "install", built_pkg_path]
             commands.extend(additional_downloaded_reqs)
@@ -343,7 +351,9 @@ def prepare_and_test_optional(mapped_args: argparse.Namespace) -> int:
 
         if mapped_args.optional:
             if env_name != mapped_args.optional:
-                logging.info(f"{env_name} does not match targeted environment {mapped_args.optional}, skipping this environment.")
+                logging.info(
+                    f"{env_name} does not match targeted environment {mapped_args.optional}, skipping this environment."
+                )
                 continue
 
         environment_exe = prepare_environment(mapped_args.target, mapped_args.temp_dir, env_name)