Support pure pyproject-toml packages (Azure#37882)

* add support for parsing and building pure pyproject.toml packages within azure-sdk-tools
* various typing updates across azure-sdk-tools, cleaning up what code paths I'm touching with the new build and parse support
* add tests for various pyproject.toml parse scenarios
* bump setuptools to 74.1.3 to support pure-pyproject.toml extension packages (eg without a setup.py)

---------

Co-authored-by: McCoy Patiño <[email protected]>

Author: scbedd

eng/ci_tools.txt

@@ -1,5 +1,5 @@
 # requirements leveraged by ci tools
-setuptools==72.2.0
+setuptools==74.1.3
 virtualenv==20.25.1
 wheel==0.43.0
 packaging==23.1

eng/conda_test_requirements.txt

@@ -8,7 +8,7 @@ trio
 typing_extensions>=3.7.2
 cryptography
 adal
-setuptools==72.2.0
+setuptools==74.1.3
 pytest-asyncio==0.12.0
 -e sdk/core/azure-core/tests/testserver_tests/coretestserver
 azure-mgmt-storage

eng/pipelines/templates/jobs/tests-nightly-python.yml

@@ -64,7 +64,7 @@ jobs:
             export PATH=~/.local/bin:$PATH
             curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py
             python3 get-pip.py
-            python3 -m pip install setuptools==72.2.0 wheel
+            python3 -m pip install setuptools==74.1.3 wheel
             python3 -m pip install tox packaging twine beautifulsoup4
             python3 --version
             cd $(Build.SourcesDirectory)

eng/regression_tools.txt

@@ -1,5 +1,5 @@
 # requirements leveraged by ci tools
-setuptools==72.2.0
+setuptools==74.1.3
 virtualenv==20.23.0
 wheel==0.43.0
 Jinja2==3.1.2

scripts/repo_health_status_report/dev_requirements.txt

@@ -1,6 +1,6 @@
 httpx==0.25.2
 markdown==3.6
 PyGitHub>=1.59.0
-setuptools==72.2.0
+setuptools==74.1.3
 -e ./sdk/identity/azure-identity
 -e ./tools/azure-sdk-tools

sdk/ml/azure-ai-ml/dev_requirements.txt

@@ -12,7 +12,7 @@ pytest-mock
 pytest
 pydash
 azure-mgmt-msi
-pywin32==304 ; sys_platform == 'win32'
+pywin32==306 ; sys_platform == 'win32'
 docker;platform.python_implementation!="PyPy"
 numpy;platform.python_implementation!="PyPy"
 scikit-image;platform.python_implementation!="PyPy"

tools/azure-sdk-tools/ci_tools/build.py

@@ -2,9 +2,9 @@
 
 from subprocess import run
 
-from typing import List
-from ci_tools.functions import discover_targeted_packages, process_requires
-from ci_tools.parsing import ParsedSetup
+from typing import List, Optional
+from ci_tools.functions import discover_targeted_packages, process_requires, get_pip_list_output
+from ci_tools.parsing import ParsedSetup, parse_require
 from ci_tools.variables import DEFAULT_BUILD_ID, str_to_bool, discover_repo_root, get_artifact_directory
 from ci_tools.versioning.version_shared import set_version_py, set_dev_classifier
 from ci_tools.versioning.version_set_dev import get_dev_version, format_build_id
@@ -84,7 +84,7 @@ def build() -> None:
         help=(
             "Where is the start directory that we are building against? If not provided, the current working directory will be used. Please ensure you are within the azure-sdk-for-python repository."
         ),
-    ),
+    )
 
     parser.add_argument(
         "--build_id",
@@ -124,7 +124,6 @@ def build() -> None:
         targeted_packages,
         artifact_directory,
         str_to_bool(args.is_dev_build),
-        str_to_bool(args.apiview_closure),
         build_id,
     )
 
@@ -144,17 +143,20 @@ def cleanup_build_artifacts(build_folder):
 
 def build_packages(
     targeted_packages: List[str],
-    distribution_directory: str = None,
+    distribution_directory: Optional[str] = None,
     is_dev_build: bool = False,
-    build_apiview_artifact: bool = False,
     build_id: str = "",
 ):
     logging.log(level=logging.INFO, msg=f"Generating {targeted_packages} using python{sys.version}")
 
     for package_root in targeted_packages:
         setup_parsed = ParsedSetup.from_path(package_root)
         package_name_in_artifacts = os.path.join(os.path.basename(package_root))
-        dist_dir = os.path.join(distribution_directory, package_name_in_artifacts)
+
+        if distribution_directory:
+            dist_dir = os.path.join(distribution_directory, package_name_in_artifacts)
+        else:
+            dist_dir = package_name_in_artifacts
 
         if is_dev_build:
             process_requires(package_root, True)
@@ -170,7 +172,7 @@ def build_packages(
 
 
 def create_package(
-    setup_directory_or_file: str, dest_folder: str = None, enable_wheel: bool = True, enable_sdist: bool = True
+    setup_directory_or_file: str, dest_folder: str, enable_wheel: bool = True, enable_sdist: bool = True
 ):
     """
     Uses the invoking python executable to build a wheel and sdist file given a setup.py or setup.py directory. Outputs
@@ -180,11 +182,24 @@ def create_package(
     dist = get_artifact_directory(dest_folder)
     setup_parsed = ParsedSetup.from_path(setup_directory_or_file)
 
-    if enable_wheel:
-        if setup_parsed.ext_modules:
-            run([sys.executable, "-m", "cibuildwheel", "--output-dir", dist], cwd=setup_parsed.folder, check=True)
-        else:
-            run([sys.executable, "setup.py", "bdist_wheel", "-d", dist], cwd=setup_parsed.folder, check=True)
-
-    if enable_sdist:
-        run([sys.executable, "setup.py", "sdist", "-d", dist], cwd=setup_parsed.folder, check=True)
+    if setup_parsed.is_pyproject:
+        # when building with pyproject, we will use `python -m build` to build the package
+        # -n argument will not use an isolated environment, which means the current environment must have all the dependencies of the package installed, to successfully
+        # pull in the dynamic `__version__` attribute. This is because setuptools is actually walking the __init__.py to get that attribute, which will fail
+        # if the imports within the setup.py don't work. Perhaps an isolated environment is better, pulling all the "dependencies" into the [build-system].requires list
+
+        # given the additional requirements of the package, we should install them in the current environment before attempting to build the package
+        # we assume the presence of `wheel`, `build`, `setuptools>=61.0.0`
+        pip_output = get_pip_list_output(sys.executable)
+        necessary_install_requirements = [req for req in setup_parsed.requires if parse_require(req).key not in pip_output.keys()]
+        run([sys.executable, "-m", "pip", "install", *necessary_install_requirements], cwd=setup_parsed.folder)
+        run([sys.executable, "-m", "build", f"-n{'s' if enable_sdist else ''}{'w' if enable_wheel else ''}", "-o", dist], cwd=setup_parsed.folder)
+    else:
+        if enable_wheel:
+            if setup_parsed.ext_modules:
+                run([sys.executable, "-m", "cibuildwheel", "--output-dir", dist], cwd=setup_parsed.folder, check=True)
+            else:
+                run([sys.executable, "setup.py", "bdist_wheel", "-d", dist], cwd=setup_parsed.folder, check=True)
+
+        if enable_sdist:
+            run([sys.executable, "setup.py", "sdist", "-d", dist], cwd=setup_parsed.folder, check=True)

tools/azure-sdk-tools/ci_tools/functions.py

@@ -10,7 +10,7 @@
 from pkg_resources import Requirement
 
 from ci_tools.variables import discover_repo_root, DEV_BUILD_IDENTIFIER, str_to_bool
-from ci_tools.parsing import ParsedSetup, get_config_setting
+from ci_tools.parsing import ParsedSetup, get_config_setting, get_pyproject
 from pypi_tools.pypi import PyPIClient
 
 import os, sys, platform, glob, re, logging
@@ -166,6 +166,15 @@ def glob_packages(glob_string: str, target_root_dir: str) -> List[str]:
         )
         collected_top_level_directories.extend([os.path.dirname(p) for p in globbed])
 
+    # handle pyproject.toml separately, as we need to filter them by the presence of a `[project]` section
+    for glob_string in individual_globs:
+        globbed = glob.glob(os.path.join(target_root_dir, glob_string, "pyproject.toml")) + glob.glob(
+            os.path.join(target_root_dir, "sdk/*/", glob_string, "pyproject.toml")
+        )
+        for p in globbed:
+            if get_pyproject(os.path.dirname(p)):
+                collected_top_level_directories.append(os.path.dirname(p))
+
     # deduplicate, in case we have double coverage from the glob strings. Example: "azure-mgmt-keyvault,azure-mgmt-*"
     return list(set(collected_top_level_directories))
 
@@ -249,10 +258,10 @@ def apply_inactive_filter(collected_packages: List[str]) -> List[str]:
     return packages
 
 
-def update_requires(setup_py_path, requires_dict):
+def update_requires(setup_path, requires_dict):
     # This method changes package requirement by overriding the specifier
     contents = []
-    with open(setup_py_path, "r") as setup_file:
+    with open(setup_path, "r") as setup_file:
         contents = setup_file.readlines()
 
     # find and replace all existing package requirement with new requirement
@@ -261,7 +270,7 @@ def update_requires(setup_py_path, requires_dict):
         for key in keys:
             contents[i] = contents[i].replace(key, requires_dict[key])
 
-    with open(setup_py_path, "w") as setup_file:
+    with open(setup_path, "w") as setup_file:
         setup_file.writelines(contents)
 
 

tools/azure-sdk-tools/ci_tools/parsing/__init__.py

@@ -9,6 +9,11 @@
     update_build_config,
     compare_string_to_glob_array,
     get_ci_config,
+    get_version_py,
+    get_pyproject,
+    VERSION_REGEX,
+    VERSION_PY,
+    OLD_VERSION_PY
 )
 
 __all__ = [
@@ -22,4 +27,9 @@
     "update_build_config",
     "compare_string_to_glob_array",
     "get_ci_config",
+    "get_version_py",
+    "get_pyproject",
+    "VERSION_REGEX",
+    "VERSION_PY",
+    "OLD_VERSION_PY"
 ]