Skip to content

Commit da88eec

Browse files
JennyPngCopilot
andauthored
Add devtest to azpysdk (#44284)
* base * minor * move dependency installs * working * proxy url * refactor pytest args * uv fix * more uv fix! * more uv fix! except...finding pkg names must not be using the right venv... * saved by glob * try catch * clean * oops clean uninstall * Apply suggestions from code review Co-authored-by: Copilot <[email protected]> * minor code review fixes * cross-platform o_o * try except --------- Co-authored-by: Copilot <[email protected]>
1 parent e0b3863 commit da88eec

File tree

6 files changed

+275
-24
lines changed

6 files changed

+275
-24
lines changed

doc/tool_usage_guide.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ This repo is currently migrating all checks from a slower `tox`-based framework,
3030
|`import_all`| Installs the package w/ default dependencies, then attempts to `import *` from the base namespace. Ensures that all imports will resolve after a base install and import. | `azpysdk import_all .` |
3131
|`generate`| Regenerates the code. | `azpysdk generate .` |
3232
|`breaking`| Checks for breaking changes. | `azpysdk breaking .` |
33+
|`devtest`| Tests a package against dependencies installed from a dev index. | `azpysdk devtest .` |
3334

3435
## Common arguments
3536

eng/tools/azure-sdk-tools/azpysdk/Check.py

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -248,3 +248,33 @@ def pip_freeze(self, executable: str) -> None:
248248
logger.error(f"Failed to run pip freeze: {e}")
249249
logger.error(e.stdout)
250250
logger.error(e.stderr)
251+
252+
def _build_pytest_args(self, package_dir: str, args: argparse.Namespace) -> List[str]:
253+
"""
254+
Builds the pytest arguments used for the given package directory.
255+
256+
:param package_dir: The package directory to build pytest args for.
257+
:param args: The argparse.Namespace object containing command-line arguments.
258+
:return: A list of pytest arguments.
259+
"""
260+
log_level = os.getenv("PYTEST_LOG_LEVEL", "51")
261+
junit_path = os.path.join(package_dir, f"test-junit-{args.command}.xml")
262+
263+
default_args = [
264+
"-rsfE",
265+
f"--junitxml={junit_path}",
266+
"--verbose",
267+
"--cov-branch",
268+
"--durations=10",
269+
"--ignore=azure",
270+
"--ignore=.tox",
271+
"--ignore-glob=.venv*",
272+
"--ignore=build",
273+
"--ignore=.eggs",
274+
"--ignore=samples",
275+
f"--log-cli-level={log_level}",
276+
]
277+
278+
additional = args.pytest_args if args.pytest_args else []
279+
280+
return [*default_args, *additional, package_dir]
Lines changed: 221 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,221 @@
1+
import argparse
2+
from subprocess import CalledProcessError
3+
import sys
4+
import os
5+
import glob
6+
7+
from typing import Optional, List
8+
9+
from .Check import Check
10+
from ci_tools.functions import (
11+
install_into_venv,
12+
uninstall_from_venv,
13+
is_error_code_5_allowed,
14+
discover_targeted_packages,
15+
)
16+
from ci_tools.scenario.generation import create_package_and_install
17+
from ci_tools.variables import discover_repo_root, set_envvar_defaults
18+
from ci_tools.logging import logger
19+
20+
REPO_ROOT = discover_repo_root()
21+
common_task_path = os.path.abspath(os.path.join(REPO_ROOT, "scripts", "devops_tasks"))
22+
sys.path.append(common_task_path)
23+
24+
from common_tasks import get_installed_packages
25+
26+
EXCLUDED_PKGS = [
27+
"azure-common",
28+
]
29+
30+
# index URL to devops feed
31+
DEV_INDEX_URL = "https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-python/pypi/simple"
32+
33+
TEST_TOOLS_REQUIREMENTS = os.path.join(REPO_ROOT, "eng/test_tools.txt")
34+
35+
36+
def get_installed_azure_packages(executable: str, pkg_name_to_exclude: str) -> List[str]:
37+
"""
38+
Returns a list of installed Azure SDK packages in the venv, excluding specified packages.
39+
40+
:param executable: Path to the Python executable in the venv.
41+
:param pkg_name_to_exclude: Package name to exclude from the result.
42+
:return: List of installed Azure SDK package names.
43+
"""
44+
venv_root = os.path.dirname(os.path.dirname(executable))
45+
# Find site-packages directory within the venv
46+
if os.name == "nt":
47+
site_packages_pattern = os.path.join(venv_root, "Lib", "site-packages")
48+
else:
49+
site_packages_pattern = os.path.join(venv_root, "lib", "python*", "site-packages")
50+
site_packages_dirs = glob.glob(site_packages_pattern)
51+
installed_pkgs = [p.split("==")[0] for p in get_installed_packages(site_packages_dirs) if p.startswith("azure-")]
52+
53+
# Get valid list of Azure SDK packages in repo
54+
pkgs = discover_targeted_packages("", REPO_ROOT)
55+
valid_azure_packages = [os.path.basename(p) for p in pkgs if "mgmt" not in p and "-nspkg" not in p]
56+
57+
# Filter current package and any excluded package
58+
pkg_names = [
59+
p for p in installed_pkgs if p in valid_azure_packages and p != pkg_name_to_exclude and p not in EXCLUDED_PKGS
60+
]
61+
62+
logger.info("Installed azure sdk packages: %s", pkg_names)
63+
return pkg_names
64+
65+
66+
def uninstall_packages(executable: str, packages: List[str], working_directory: str):
67+
"""
68+
Uninstalls a list of packages from the virtual environment so dev build versions can be reinstalled.
69+
70+
:param executable: Path to the Python executable in the virtual environment.
71+
:param packages: List of package names to uninstall.
72+
:param working_directory: Directory from which to run the uninstall command.
73+
:raises Exception: If uninstallation fails.
74+
:return: None
75+
"""
76+
if len(packages) == 0:
77+
logger.warning("No packages to uninstall.")
78+
return
79+
80+
logger.info("Uninstalling packages: %s", packages)
81+
82+
try:
83+
uninstall_from_venv(executable, packages, working_directory)
84+
except Exception as e:
85+
logger.error(f"Failed to uninstall packages: {e}")
86+
raise e
87+
logger.info("Uninstalled packages")
88+
89+
90+
def install_packages(executable: str, packages: List[str], working_directory: str):
91+
"""
92+
Installs a list of packages from the devops feed into the virtual environment.
93+
94+
:param executable: Path to the Python executable in the virtual environment.
95+
:param packages: List of package names to install.
96+
:param working_directory: Directory from which to run the install command.
97+
:raises Exception: If installation fails.
98+
:return: None
99+
"""
100+
101+
if len(packages) == 0:
102+
logger.warning("No packages to install.")
103+
return
104+
105+
logger.info("Installing dev build version for packages: %s", packages)
106+
107+
commands = [*packages, "--index-url", DEV_INDEX_URL]
108+
109+
# install dev build of azure packages
110+
try:
111+
install_into_venv(executable, commands, working_directory)
112+
except Exception as e:
113+
logger.error(f"Failed to install packages: {e}")
114+
raise e
115+
logger.info("Installed dev build version for packages")
116+
117+
118+
def install_dev_build_packages(executable: str, pkg_name_to_exclude: str, working_directory: str):
119+
# Uninstall GA version and reinstall dev build version of dependent packages
120+
azure_pkgs = get_installed_azure_packages(executable, pkg_name_to_exclude)
121+
uninstall_packages(executable, azure_pkgs, working_directory)
122+
install_packages(executable, azure_pkgs, working_directory)
123+
124+
125+
class devtest(Check):
126+
def __init__(self) -> None:
127+
super().__init__()
128+
129+
def register(
130+
self, subparsers: "argparse._SubParsersAction", parent_parsers: Optional[List[argparse.ArgumentParser]] = None
131+
) -> None:
132+
"""Register the devtest check. The devtest check tests a package against dependencies installed from a dev index."""
133+
parents = parent_parsers or []
134+
p = subparsers.add_parser(
135+
"devtest",
136+
parents=parents,
137+
help="Run the devtest check to test a package against dependencies installed from a dev index",
138+
)
139+
p.set_defaults(func=self.run)
140+
p.add_argument(
141+
"--pytest-args",
142+
nargs=argparse.REMAINDER,
143+
help="Additional arguments forwarded to pytest.",
144+
)
145+
146+
def run(self, args: argparse.Namespace) -> int:
147+
"""Run the devtest check command."""
148+
logger.info("Running devtest check...")
149+
150+
set_envvar_defaults({"PROXY_URL": "http://localhost:5002"})
151+
targeted = self.get_targeted_directories(args)
152+
153+
results: List[int] = []
154+
155+
for parsed in targeted:
156+
package_dir = parsed.folder
157+
package_name = parsed.name
158+
executable, staging_directory = self.get_executable(args.isolate, args.command, sys.executable, package_dir)
159+
logger.info(f"Processing {package_name} for devtest check")
160+
161+
# install dependencies
162+
try:
163+
self.install_dev_reqs(executable, args, package_dir)
164+
except CalledProcessError as e:
165+
logger.error(f"Failed to install dev requirements: {e}")
166+
results.append(1)
167+
continue
168+
169+
try:
170+
create_package_and_install(
171+
distribution_directory=staging_directory,
172+
target_setup=package_dir,
173+
skip_install=False,
174+
cache_dir=None,
175+
work_dir=staging_directory,
176+
force_create=False,
177+
package_type="sdist",
178+
pre_download_disabled=False,
179+
python_executable=executable,
180+
)
181+
except CalledProcessError as e:
182+
logger.error(f"Failed to create and install package {package_name}: {e}")
183+
results.append(1)
184+
continue
185+
186+
if os.path.exists(TEST_TOOLS_REQUIREMENTS):
187+
try:
188+
install_into_venv(executable, ["-r", TEST_TOOLS_REQUIREMENTS], package_dir)
189+
except Exception as e:
190+
logger.error(f"Failed to install test tools requirements: {e}")
191+
results.append(1)
192+
continue
193+
else:
194+
logger.warning(f"Test tools requirements file not found at {TEST_TOOLS_REQUIREMENTS}.")
195+
196+
try:
197+
install_dev_build_packages(executable, package_name, package_dir)
198+
except Exception as e:
199+
logger.error(f"Failed to install dev build packages: {e}")
200+
results.append(1)
201+
continue
202+
203+
pytest_args = self._build_pytest_args(package_dir, args)
204+
205+
pytest_result = self.run_venv_command(
206+
executable, ["-m", "pytest", *pytest_args], cwd=package_dir, immediately_dump=True
207+
)
208+
209+
if pytest_result.returncode != 0:
210+
if pytest_result.returncode == 5 and is_error_code_5_allowed(package_dir, package_name):
211+
logger.info(
212+
"pytest exited with code 5 for %s, which is allowed for management or opt-out packages.",
213+
package_name,
214+
)
215+
# Align with tox: skip coverage when tests are skipped entirely
216+
continue
217+
218+
logger.error(f"pytest failed for {package_name} with exit code {pytest_result.returncode}.")
219+
results.append(pytest_result.returncode)
220+
221+
return max(results) if results else 0

eng/tools/azure-sdk-tools/azpysdk/main.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@
3232
from .verify_keywords import verify_keywords
3333
from .generate import generate
3434
from .breaking import breaking
35+
from .devtest import devtest
3536

3637
from ci_tools.logging import configure_logging, logger
3738

@@ -95,6 +96,7 @@ def build_parser() -> argparse.ArgumentParser:
9596
verify_keywords().register(subparsers, [common])
9697
generate().register(subparsers, [common])
9798
breaking().register(subparsers, [common])
99+
devtest().register(subparsers, [common])
98100

99101
return parser
100102

eng/tools/azure-sdk-tools/azpysdk/whl.py

Lines changed: 0 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -132,26 +132,3 @@ def _install_common_requirements(self, executable: str, package_dir: str) -> Non
132132
install_into_venv(executable, ["-r", TEST_TOOLS_REQUIREMENTS], package_dir)
133133
else:
134134
logger.warning(f"Test tools requirements file not found at {TEST_TOOLS_REQUIREMENTS}.")
135-
136-
def _build_pytest_args(self, package_dir: str, args: argparse.Namespace) -> List[str]:
137-
log_level = os.getenv("PYTEST_LOG_LEVEL", "51")
138-
junit_path = os.path.join(package_dir, f"test-junit-{args.command}.xml")
139-
140-
default_args = [
141-
"-rsfE",
142-
f"--junitxml={junit_path}",
143-
"--verbose",
144-
"--cov-branch",
145-
"--durations=10",
146-
"--ignore=azure",
147-
"--ignore=.tox",
148-
"--ignore-glob=.venv*",
149-
"--ignore=build",
150-
"--ignore=.eggs",
151-
"--ignore=samples",
152-
f"--log-cli-level={log_level}",
153-
]
154-
155-
additional = args.pytest_args if args.pytest_args else []
156-
157-
return [*default_args, *additional, package_dir]

eng/tools/azure-sdk-tools/ci_tools/functions.py

Lines changed: 21 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -519,7 +519,7 @@ def pip_uninstall(requirements: List[str], python_executable: str) -> bool:
519519
"""
520520
Attempts to invoke an install operation using the invoking python's pip. Empty requirements are auto-success.
521521
"""
522-
# we do not use get_pip_command here because uv pip doesn't have an uninstall command
522+
# use uninstall_from_venv() for uv venvs
523523
exe = python_executable or sys.executable
524524
command = [exe, "-m", "pip", "uninstall", "-y"]
525525

@@ -565,10 +565,30 @@ def install_into_venv(venv_path_or_executable: str, requirements: List[str], wor
565565

566566
if pip_cmd[0] == "uv":
567567
cmd += ["--python", py]
568+
568569
# todo: clean this up so that we're using run_logged from #42862
569570
subprocess.check_call(cmd, cwd=working_directory)
570571

571572

573+
def uninstall_from_venv(venv_path_or_executable: str, requirements: List[str], working_directory: str) -> None:
574+
"""
575+
Uninstalls the requirements from an existing venv (venv_path) without activating it.
576+
"""
577+
py = get_venv_python(venv_path_or_executable)
578+
pip_cmd = get_pip_command(py)
579+
580+
install_targets = [r.strip() for r in requirements]
581+
cmd = pip_cmd + ["uninstall"]
582+
if pip_cmd[0] != "uv":
583+
cmd += ["-y"]
584+
cmd.extend(install_targets)
585+
586+
if pip_cmd[0] == "uv":
587+
cmd += ["--python", py]
588+
589+
subprocess.check_call(cmd, cwd=working_directory)
590+
591+
572592
def pip_install_requirements_file(requirements_file: str, python_executable: Optional[str] = None) -> bool:
573593
return pip_install(["-r", requirements_file], True, python_executable)
574594

0 commit comments

Comments
 (0)