Skip to content

Commit 1526205

Browse files
JennyPnglmazuelscbeddsemick-devCopilot
authored
Rewrite Mypy Check Without Tox (#42706)
Co-authored-by: Scott Beddall <[email protected]> * add mypy check * add mypy version * add get_targeted_directories helper function * clean * append wildcard to target glob * revert adding /* to target * change default to ** * remove comment * update import_all * update import_all and whl * copilot formatting fixes Co-authored-by: Copilot <[email protected]> * update exception logging * update readme --------- Co-authored-by: Laurent Mazuel <[email protected]> Co-authored-by: Scott Beddall <[email protected]> Co-authored-by: Scott Beddall <[email protected]> Co-authored-by: Scott Beddall <[email protected]> Co-authored-by: Copilot <[email protected]>
1 parent 9712ea7 commit 1526205

File tree

6 files changed

+199
-26
lines changed

6 files changed

+199
-26
lines changed

eng/tools/azure-sdk-tools/README.md

Lines changed: 6 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -210,16 +210,14 @@ class my_check(Check):
210210
"""This is the recommended """
211211
set_envvar_defaults()
212212

213-
if args.target == ".":
214-
targeted = [os.getcwd()]
215-
else:
216-
target_dir = os.getcwd()
217-
targeted = discover_targeted_packages(args.target, target_dir)
213+
targeted = self.get_targeted_directories(args)
214+
218215
results: List[int] = []
219216

220-
for pkg in targeted:
221-
parsed = ParsedSetup.from_path(pkg)
222-
print(f"Processing {pkg.name} for my_check")
217+
for parsed in targeted:
218+
pkg_dir = parsed.folder
219+
pkg_name = parsed.name
220+
print(f"Processing {pkg_name} for my_check")
223221
```
224222

225223
- Once the new check has been created, register it in `azpysdk/main.py` on line 58. This will likely be automated by decorator in the near future but this is out how it should be done for now.

eng/tools/azure-sdk-tools/azpysdk/Check.py

Lines changed: 31 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,10 @@
11
import abc
2+
import os
23
import argparse
4+
import traceback
35
from typing import Sequence, Optional, List, Any
6+
from ci_tools.parsing import ParsedSetup
7+
from ci_tools.functions import discover_targeted_packages
48

59
class Check(abc.ABC):
610
"""
@@ -28,4 +32,30 @@ def run(self, args: argparse.Namespace) -> int:
2832
2933
Subclasses can override this to perform the actual work.
3034
"""
31-
return 0
35+
return 0
36+
37+
def get_targeted_directories(self, args: argparse.Namespace) -> List[ParsedSetup]:
38+
"""
39+
Get the directories that are targeted for the check.
40+
"""
41+
targeted: List[ParsedSetup] = []
42+
targeted_dir = os.getcwd()
43+
44+
if args.target == ".":
45+
try:
46+
targeted.append(ParsedSetup.from_path(targeted_dir))
47+
except Exception as e:
48+
print("Error: Current directory does not appear to be a Python package (no setup.py or setup.cfg found). Remove '.' argument to run on child directories.")
49+
print(f"Exception: {e}")
50+
return []
51+
else:
52+
targeted_packages = discover_targeted_packages(args.target, targeted_dir)
53+
for pkg in targeted_packages:
54+
try:
55+
targeted.append(ParsedSetup.from_path(pkg))
56+
except Exception as e:
57+
print(f"Unable to parse {pkg} as a Python package. Dumping exception detail and skipping.")
58+
print(f"Exception: {e}")
59+
print(traceback.format_exc())
60+
61+
return targeted

eng/tools/azure-sdk-tools/azpysdk/import_all.py

Lines changed: 4 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -44,20 +44,16 @@ def run(self, args: argparse.Namespace) -> int:
4444
"""Run the import_all check command."""
4545
print("Running import_all check in isolated venv...")
4646

47-
# this is common. we should have an abstraction layer for this somehow
48-
if args.target == ".":
49-
targeted = [os.getcwd()]
50-
else:
51-
target_dir = os.getcwd()
52-
targeted = discover_targeted_packages(args.target, target_dir)
47+
targeted = self.get_targeted_directories(args)
5348

5449
# {[tox]pip_command} freeze
5550
# python {repository_root}/eng/tox/import_all.py -t {tox_root}
5651

5752
outcomes: List[int] = []
5853

59-
for pkg in targeted:
60-
parsed = ParsedSetup.from_path(pkg)
54+
for parsed in targeted:
55+
pkg = parsed.folder
56+
6157

6258
staging_area = tempfile.mkdtemp()
6359
create_package_and_install(

eng/tools/azure-sdk-tools/azpysdk/main.py

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515

1616
from .whl import whl
1717
from .import_all import import_all
18+
from .mypy import mypy
1819

1920
from ci_tools.scenario import install_into_venv, get_venv_python
2021
from ci_tools.functions import get_venv_call
@@ -42,8 +43,8 @@ def build_parser() -> argparse.ArgumentParser:
4243
common.add_argument(
4344
"target",
4445
nargs="?",
45-
default=".",
46-
help="Glob pattern for packages. Defaults to '.', but will match patterns below CWD if a value is provided."
46+
default="**",
47+
help="Glob pattern for packages. Defaults to '**', but will match patterns below CWD if a value is provided."
4748
)
4849
# allow --isolate to be specified after the subcommand as well
4950
common.add_argument(
@@ -58,6 +59,7 @@ def build_parser() -> argparse.ArgumentParser:
5859
# register our checks with the common params as their parent
5960
whl().register(subparsers, [common])
6061
import_all().register(subparsers, [common])
62+
mypy().register(subparsers, [common])
6163

6264
return parser
6365

Lines changed: 150 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,150 @@
1+
import argparse
2+
import os
3+
import sys
4+
import logging
5+
import tempfile
6+
7+
from typing import Optional, List
8+
from subprocess import CalledProcessError, check_call
9+
10+
from .Check import Check
11+
from ci_tools.parsing import ParsedSetup
12+
from ci_tools.functions import discover_targeted_packages
13+
from ci_tools.scenario.generation import create_package_and_install
14+
from ci_tools.variables import in_ci, set_envvar_defaults
15+
from ci_tools.environment_exclusions import (
16+
is_check_enabled, is_typing_ignored
17+
)
18+
19+
logging.getLogger().setLevel(logging.INFO)
20+
21+
PYTHON_VERSION = "3.9"
22+
MYPY_VERSION = "1.14.1"
23+
24+
class mypy(Check):
25+
def __init__(self) -> None:
26+
super().__init__()
27+
28+
def register(self, subparsers: "argparse._SubParsersAction", parent_parsers: Optional[List[argparse.ArgumentParser]] = None) -> None:
29+
"""Register the `mypy` check. The mypy check installs mypy and runs mypy against the target package.
30+
"""
31+
parents = parent_parsers or []
32+
p = subparsers.add_parser("mypy", parents=parents, help="Run the mypy check")
33+
p.set_defaults(func=self.run)
34+
35+
p.add_argument(
36+
"--next",
37+
default=False,
38+
help="Next version of mypy is being tested",
39+
required=False
40+
)
41+
42+
def run(self, args: argparse.Namespace) -> int:
43+
"""Run the mypy check command."""
44+
print("Running mypy check in isolated venv...")
45+
46+
set_envvar_defaults()
47+
48+
targeted = self.get_targeted_directories(args)
49+
50+
results: List[int] = []
51+
52+
for parsed in targeted:
53+
package_dir = parsed.folder
54+
package_name = parsed.name
55+
print(f"Processing {package_name} for mypy check")
56+
57+
staging_area = tempfile.mkdtemp()
58+
create_package_and_install(
59+
distribution_directory=staging_area,
60+
target_setup=package_dir,
61+
skip_install=False,
62+
cache_dir=None,
63+
work_dir=staging_area,
64+
force_create=False,
65+
package_type="wheel",
66+
pre_download_disabled=False,
67+
)
68+
69+
# install mypy
70+
try:
71+
if (args.next):
72+
# use latest version of mypy
73+
check_call([sys.executable, "-m", "pip", "install", "mypy"])
74+
else:
75+
check_call([sys.executable, "-m", "pip", "install", f"mypy=={MYPY_VERSION}"])
76+
except CalledProcessError as e:
77+
print("Failed to install mypy:", e)
78+
return e.returncode
79+
80+
logging.info(f"Running mypy against {package_name}")
81+
82+
if not args.next and in_ci():
83+
if not is_check_enabled(package_dir, "mypy", True) or is_typing_ignored(package_name):
84+
logging.info(
85+
f"Package {package_name} opts-out of mypy check. See https://aka.ms/python/typing-guide for information."
86+
)
87+
continue
88+
89+
top_level_module = parsed.namespace.split(".")[0]
90+
91+
commands = [
92+
sys.executable,
93+
"-m",
94+
"mypy",
95+
"--python-version",
96+
PYTHON_VERSION,
97+
"--show-error-codes",
98+
"--ignore-missing-imports",
99+
]
100+
src_code = [*commands, os.path.join(package_dir, top_level_module)]
101+
src_code_error = None
102+
sample_code_error = None
103+
try:
104+
logging.info(
105+
f"Running mypy commands on src code: {src_code}"
106+
)
107+
results.append(check_call(src_code))
108+
logging.info("Verified mypy, no issues found")
109+
except CalledProcessError as src_error:
110+
src_code_error = src_error
111+
results.append(src_error.returncode)
112+
113+
if not args.next and in_ci() and not is_check_enabled(package_dir, "type_check_samples", True):
114+
logging.info(
115+
f"Package {package_name} opts-out of mypy check on samples."
116+
)
117+
continue
118+
else:
119+
# check if sample dirs exists, if not, skip sample code check
120+
samples = os.path.exists(os.path.join(package_dir, "samples"))
121+
generated_samples = os.path.exists(os.path.join(package_dir, "generated_samples"))
122+
if not samples and not generated_samples:
123+
logging.info(
124+
f"Package {package_name} does not have a samples directory."
125+
)
126+
else:
127+
sample_code = [
128+
*commands,
129+
"--check-untyped-defs",
130+
"--follow-imports=silent",
131+
os.path.join(package_dir, "samples" if samples else "generated_samples"),
132+
]
133+
try:
134+
logging.info(
135+
f"Running mypy commands on sample code: {sample_code}"
136+
)
137+
results.append(check_call(sample_code))
138+
except CalledProcessError as sample_error:
139+
sample_code_error = sample_error
140+
results.append(sample_error.returncode)
141+
142+
if args.next and in_ci() and not is_typing_ignored(package_name):
143+
from gh_tools.vnext_issue_creator import create_vnext_issue, close_vnext_issue
144+
if src_code_error or sample_code_error:
145+
create_vnext_issue(package_dir, "mypy")
146+
else:
147+
close_vnext_issue(package_name, "mypy")
148+
149+
return max(results) if results else 0
150+

eng/tools/azure-sdk-tools/azpysdk/whl.py

Lines changed: 4 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -33,15 +33,12 @@ def run(self, args: argparse.Namespace) -> int:
3333

3434
set_envvar_defaults()
3535

36-
if args.target == ".":
37-
targeted = [os.getcwd()]
38-
else:
39-
target_dir = os.getcwd()
40-
targeted = discover_targeted_packages(args.target, target_dir)
36+
targeted = self.get_targeted_directories(args)
37+
4138
results: List[int] = []
4239

43-
for pkg in targeted:
44-
parsed = ParsedSetup.from_path(pkg)
40+
for parsed in targeted:
41+
pkg = parsed.folder
4542

4643
dev_requirements = os.path.join(pkg, "dev_requirements.txt")
4744

0 commit comments

Comments
 (0)