(engsys) feat: Make mindependency aware of environment_markers (#37357)

* refactor: Return full Requirements object instead of [name, specifier]

* feat: Filter out requirements that apply to the current environment

* feat: Add a log message when a requirement is skipped

* small update covering a couple extra usages of parse_require

---------

Co-authored-by: Scott Beddall <[email protected]>

Author: kdestin

eng/tox/install_depend_packages.py

@@ -13,7 +13,7 @@
 import re
 from subprocess import check_call
 from typing import TYPE_CHECKING
-from pkg_resources import parse_version
+from pkg_resources import parse_version, Requirement
 from pypi_tools.pypi import PyPIClient
 from packaging.specifiers import SpecifierSet
 from packaging.version import Version, parse
@@ -231,7 +231,18 @@ def process_requirement(req, dependency_type, orig_pkg_name):
     # this method finds either latest or minimum version of a package that is available on PyPI
 
     # find package name and requirement specifier from requires
-    pkg_name, spec = parse_require(req)
+    requirement = parse_require(req)
+    pkg_name = requirement.key
+    spec = requirement.specifier if len(requirement.specifier) else None
+
+    # Filter out requirements with environment markers that don't match the current environment
+    # e.g. `; python_version > 3.10` when running on Python3.9
+    if not (requirement.marker is None or requirement.marker.evaluate()):
+        logging.info(
+            f"Skipping requirement {req!r}. Environment marker {str(requirement.marker)!r} "
+            + "does not apply to current environment."
+        )
+        return ""
 
     # get available versions on PyPI
     client = PyPIClient()

scripts/devops_tasks/common_tasks.py

@@ -225,13 +225,13 @@ def find_packages_missing_on_pypi(path: str) -> Iterable[str]:
         requires = ParsedSetup.from_path(path).requires
 
     # parse pkg name and spec
-    pkg_spec_dict = dict(parse_require(req) for req in requires)
+    pkg_spec_dict = [parse_require(req) for req in requires]
     logging.info("Package requirement: {}".format(pkg_spec_dict))
     # find if version is available on pypi
     missing_packages = [
-        "{0}{1}".format(pkg, pkg_spec_dict[pkg])
-        for pkg in pkg_spec_dict.keys()
-        if not is_required_version_on_pypi(pkg, pkg_spec_dict[pkg])
+        f"{pkg.key}{pkg.specifier}"
+        for pkg in pkg_spec_dict
+        if not is_required_version_on_pypi(pkg.key, str(pkg.specifier))
     ]
     if missing_packages:
         logging.error("Packages not found on PyPI: {}".format(missing_packages))

scripts/devops_tasks/test_regression.py

@@ -328,7 +328,7 @@ def find_package_dependency(glob_string, repo_root_dir, dependent_service):
             parsed = ParsedSetup.from_path(pkg_root)
 
             # Get a list of package names from install requires
-            required_pkgs = [parse_require(r)[0] for r in parsed.requires]
+            required_pkgs = [parse_require(r).key for r in parsed.requires]
             required_pkgs = [p for p in required_pkgs if p.startswith("azure")]
 
             for req_pkg in required_pkgs:

tools/azure-sdk-tools/ci_tools/dependency_analysis.py

@@ -68,7 +68,9 @@ def get_lib_deps(base_dir: str) -> Tuple[Dict[str, Dict[str, Any]], Dict[str, Di
             packages[lib_name] = {"version": version, "source": lib_dir, "deps": []}
 
             for req in requires:
-                req_name, spec = parse_require(req)
+                req_obj = parse_require(req)
+                req_name = req_obj.key
+                spec = req_obj.specifier if len(req_obj.specifier) else None
                 if spec is None:
                     spec = ""
 
@@ -97,7 +99,10 @@ def get_wheel_deps(wheel_dir: str) -> Tuple[Dict[str, Dict[str, Any]], Dict[str,
                 for req in requires:
                     req = req.split(";")[0]  # Extras conditions appear after a semicolon
                     req = re.sub(r"[\s\(\)]", "", req)  # Version specifiers appear in parentheses
-                    req_name, spec = parse_require(req)
+                    req_obj = parse_require(req)
+
+                    req_name = req_obj.key
+                    spec = req_obj.specifier if len(req_obj.specifier) else None
                     if spec is None:
                         spec = ""
 

tools/azure-sdk-tools/ci_tools/parsing/parse_functions.py

@@ -272,7 +272,7 @@ def setup(*args, **kwargs):
     classifiers = kwargs.get("classifiers", [])
     keywords = kwargs.get("keywords", [])
 
-    is_new_sdk = name in NEW_REQ_PACKAGES or any(map(lambda x: (parse_require(x)[0] in NEW_REQ_PACKAGES), requires))
+    is_new_sdk = name in NEW_REQ_PACKAGES or any(map(lambda x: (parse_require(x).key in NEW_REQ_PACKAGES), requires))
 
     ext_package = kwargs.get("ext_package", None)
     ext_modules = kwargs.get("ext_modules", [])
@@ -301,23 +301,13 @@ def get_install_requires(setup_path: str) -> List[str]:
     return ParsedSetup.from_path(setup_path).requires
 
 
-def parse_require(req: str) -> Tuple[str, SpecifierSet]:
+def parse_require(req: str) -> Requirement:
     """
     Parses the incoming version specification and returns a tuple of the requirement name and specifier.
 
     "azure-core<2.0.0,>=1.11.0" -> [azure-core, <2.0.0,>=1.11.0]
     """
-    req_object = Requirement.parse(req.split(";")[0].lower())
-    pkg_name = req_object.key
-
-    # we were not passed a full requirement. Instead we were passed a value of "readme-renderer" or another string without a version.
-    if not req_object.specifier:
-        return [pkg_name, None]
-
-    # regex details ripped from https://peps.python.org/pep-0508/
-    isolated_spec = re.sub(r"^([a-zA-Z0-9\-\_\.]+)(\[[a-zA-Z0-9\-\_\.\,]*\])?", "", str(req_object))
-    spec = SpecifierSet(isolated_spec)
-    return (pkg_name, spec)
+    return Requirement.parse(req)
 
 
 def parse_freeze_output(file_location: str) -> Dict[str, str]:

tools/azure-sdk-tools/ci_tools/scenario/generation.py

@@ -133,7 +133,9 @@ def create_package_and_install(
                         logging.info("Installed packages: {}".format(installed_pkgs))
 
                         # parse the specifier
-                        req_name, req_specifier = parse_require(req)
+                        requirement = parse_require(req)
+                        req_name = requirement.key
+                        req_specifier = requirement.specifier if len(requirement.specifier) else None
 
                         # if we have the package already present...
                         if req_name in installed_pkgs:
@@ -160,8 +162,8 @@ def create_package_and_install(
                                     env=dict(os.environ, PIP_EXTRA_INDEX_URL=""),
                                 )
                             except subprocess.CalledProcessError as e:
-                                req_name, req_specifier = parse_require(addition)
-                                non_present_reqs.append(req_name)
+                                requirement = parse_require(addition)
+                                non_present_reqs.append(requirement.key)
 
                         additional_downloaded_reqs = [
                             os.path.abspath(os.path.join(tmp_dl_folder, pth)) for pth in os.listdir(tmp_dl_folder)

tools/azure-sdk-tools/tests/test_parse_functionality.py

@@ -24,12 +24,11 @@ def test_parse_require():
 
     for scenario in test_scenarios:
         result = parse_require(scenario[0])
-        assert result[0] is not None
+        assert result.key is not None
         if scenario[2] is not None:
-            assert result[1] is not None
-            assert isinstance(result[1], SpecifierSet)
-        assert result[0] == scenario[1]
-        assert result[1] == scenario[2]
+            assert len(result.specifier) != 0
+        assert result.key == scenario[1]
+        assert str(result.specifier) == (scenario[2] or "")
 
 
 def test_parse_require_with_no_spec():
@@ -38,8 +37,8 @@ def test_parse_require_with_no_spec():
     for scenario in spec_scenarios:
         result = parse_require(scenario)
 
-        assert result[0] == scenario.replace("_", "-")
-        assert result[1] is None
+        assert result.key == scenario.replace("_", "-")
+        assert len(result.specifier) == 0
 
 
 @patch("ci_tools.parsing.parse_functions.read_setup_py_content")