Use New VerifyTypes and Pyright Checks in CI (#43774)

* cut

* debug

* huh

* double check pip freeze

* revert cwd...doesn't seem to change anything

* log

* pass executable in freeze

* clean and run_venv_command fix

* clean

* add stdout prints and use venv abstraction in pyright

* add stdout prints and use venv abstraction in pyright

* format

* log stdout

* add the additional version that should match the requirement that uv and pip is unable to honor

* Update eng/tools/azure-sdk-tools/azpysdk/verifytypes.py

Co-authored-by: Copilot <[email protected]>

---------

Co-authored-by: Scott Beddall <[email protected]>
Co-authored-by: Copilot <[email protected]>

Author: 3 people

eng/pipelines/templates/steps/run_pyright.yml

@@ -11,27 +11,31 @@ steps:
   - task: PythonScript@0
     displayName: 'Run Pyright'
     inputs:
-      scriptPath: 'scripts/devops_tasks/dispatch_tox.py'
+      scriptPath: 'eng/scripts/dispatch_checks.py'
       arguments: >-
         "$(TargetingString)"
-        --mark_arg="${{ parameters.TestMarkArgument }}"
         --service="${{ parameters.ServiceDirectory }}"
-        --toxenv="pyright"
+        --check="pyright"
         --disable-compatibility-filter
-        --disablecov
         ${{ parameters.AdditionalTestArgs }}
+    env:
+      TOX_PIP_IMPL: "uv"
+      VIRTUAL_ENV: ""
+      PYTHONHOME: ""
     condition: and(succeededOrFailed(), ne(variables['Skip.Pyright'],'true'))
 
   - task: PythonScript@0
     displayName: 'Run verifytypes'
     inputs:
-      scriptPath: 'scripts/devops_tasks/dispatch_tox.py'
+      scriptPath: 'eng/scripts/dispatch_checks.py'
       arguments: >-
         "$(TargetingString)"
-        --mark_arg="${{ parameters.TestMarkArgument }}"
         --service="${{ parameters.ServiceDirectory }}"
-        --toxenv="verifytypes"
+        --check="verifytypes"
         --disable-compatibility-filter
-        --disablecov
         ${{ parameters.AdditionalTestArgs }}
+    env:
+      TOX_PIP_IMPL: "uv"
+      VIRTUAL_ENV: ""
+      PYTHONHOME: ""
     condition: and(succeededOrFailed(), ne(variables['Skip.Verifytypes'],'true'))

eng/tools/azure-sdk-tools/azpysdk/pyright.py

@@ -134,8 +134,11 @@ def run(self, args: argparse.Namespace) -> int:
             commands.extend(paths)
 
             try:
-                check_call(commands)
+                self.run_venv_command(executable, commands[1:], package_dir, check=True)
+                logger.info(f"Pyright check passed for {package_name}.")
             except CalledProcessError as error:
+                logger.error("Pyright reported issues:")
+                logger.error(error.stdout)
                 if (
                     args.next
                     and in_ci()

eng/tools/azure-sdk-tools/azpysdk/verifytypes.py

@@ -12,7 +12,8 @@
 
 from .Check import Check
 from ci_tools.functions import install_into_venv
-from ci_tools.variables import discover_repo_root, in_ci, set_envvar_defaults, set_envvar_defaults
+from ci_tools.scenario.generation import create_package_and_install
+from ci_tools.variables import discover_repo_root, in_ci, set_envvar_defaults
 from ci_tools.environment_exclusions import is_check_enabled, is_typing_ignored
 from ci_tools.functions import get_pip_command
 from ci_tools.logging import logger
@@ -21,70 +22,6 @@
 REPO_ROOT = discover_repo_root()
 
 
-def install_from_main(setup_path: str) -> int:
-    path = pathlib.Path(setup_path)
-    subdirectory = path.relative_to(REPO_ROOT)
-    cwd = os.getcwd()
-    with tempfile.TemporaryDirectory() as temp_dir_name:
-        os.chdir(temp_dir_name)
-        try:
-            subprocess.check_call(["git", "init"], stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT)
-            subprocess.check_call(
-                ["git", "clone", "--no-checkout", "https://github.com/Azure/azure-sdk-for-python.git", "--depth", "1"],
-                stdout=subprocess.DEVNULL,
-                stderr=subprocess.STDOUT,
-            )
-            os.chdir("azure-sdk-for-python")
-            subprocess.check_call(
-                ["git", "sparse-checkout", "init", "--cone"], stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT
-            )
-            subprocess.check_call(
-                ["git", "sparse-checkout", "set", subdirectory], stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT
-            )
-            subprocess.check_call(["git", "checkout", "main"], stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT)
-
-            if not os.path.exists(os.path.join(os.getcwd(), subdirectory)):
-                # code is not checked into main yet, nothing to compare
-                logger.info(f"{subdirectory} is not checked into main, nothing to compare.")
-                return 1
-
-            os.chdir(subdirectory)
-
-            command = get_pip_command() + ["install", ".", "--force-reinstall"]
-
-            subprocess.check_call(command, stdout=subprocess.DEVNULL)
-        finally:
-            os.chdir(cwd)  # allow temp dir to be deleted
-        return 0
-
-
-def get_type_complete_score(commands: typing.List[str], check_pytyped: bool = False) -> float:
-    try:
-        response = subprocess.run(
-            commands,
-            check=True,
-            capture_output=True,
-        )
-    except subprocess.CalledProcessError as e:
-        if e.returncode != 1:
-            logger.error(
-                f"Running verifytypes failed: {e.stderr}. See https://aka.ms/python/typing-guide for information."
-            )
-            return -1.0
-
-        report = json.loads(e.output)
-        if check_pytyped:
-            pytyped_present = report["typeCompleteness"].get("pyTypedPath", None)
-            if not pytyped_present:
-                logger.error(f"No py.typed file was found. See https://aka.ms/python/typing-guide for information.")
-                return -1.0
-        return report["typeCompleteness"]["completenessScore"]
-
-    # library scores 100%
-    report = json.loads(response.stdout)
-    return report["typeCompleteness"]["completenessScore"]
-
-
 class verifytypes(Check):
     def __init__(self) -> None:
         super().__init__()
@@ -124,6 +61,18 @@ def run(self, args: argparse.Namespace) -> int:
                 logger.error(f"Failed to install pyright: {e}")
                 return e.returncode
 
+            create_package_and_install(
+                distribution_directory=staging_directory,
+                target_setup=package_dir,
+                skip_install=False,
+                cache_dir=None,
+                work_dir=staging_directory,
+                force_create=False,
+                package_type="sdist",
+                pre_download_disabled=False,
+                python_executable=executable,
+            )
+
             if in_ci():
                 if not is_check_enabled(package_dir, "verifytypes") or is_typing_ignored(package_name):
                     logger.info(
@@ -142,25 +91,27 @@ def run(self, args: argparse.Namespace) -> int:
             ]
 
             # get type completeness score from current code
-            score_from_current = get_type_complete_score(commands, check_pytyped=True)
+            score_from_current = self.get_type_complete_score(executable, commands, package_dir, check_pytyped=True)
             if score_from_current == -1.0:
                 results.append(1)
                 continue
 
+            # show output
             try:
-                subprocess.check_call(commands[:-1])
-            except subprocess.CalledProcessError:
+                response = self.run_venv_command(executable, commands[1:-1], package_dir, check=True)
+                logger.info(response.stdout)
+            except subprocess.CalledProcessError as e:
                 logger.warning(
-                    "verifytypes reported issues."
+                    f"verifytypes reported issues: {e.stdout}"
                 )  # we don't fail on verifytypes, only if type completeness score worsens from main
 
             if in_ci():
                 # get type completeness score from main
                 logger.info("Getting the type completeness score from the code in main...")
-                if install_from_main(os.path.abspath(package_dir)) > 0:
+                if self.install_from_main(os.path.abspath(package_dir), executable) > 0:
                     continue
 
-                score_from_main = get_type_complete_score(commands)
+                score_from_main = self.get_type_complete_score(executable, commands, package_dir)
                 if score_from_main == -1.0:
                     results.append(1)
                     continue
@@ -177,3 +128,70 @@ def run(self, args: argparse.Namespace) -> int:
                     )
                     results.append(1)
         return max(results) if results else 0
+
+    def install_from_main(self, setup_path: str, python_executable: Optional[str] = None) -> int:
+        path = pathlib.Path(setup_path)
+
+        subdirectory = path.relative_to(REPO_ROOT)
+        cwd = os.getcwd()
+        with tempfile.TemporaryDirectory() as temp_dir_name:
+            os.chdir(temp_dir_name)
+            try:
+                subprocess.check_call(["git", "init"], stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT)
+                subprocess.check_call(
+                    [
+                        "git",
+                        "clone",
+                        "--no-checkout",
+                        "https://github.com/Azure/azure-sdk-for-python.git",
+                        "--depth",
+                        "1",
+                    ],
+                    stdout=subprocess.DEVNULL,
+                    stderr=subprocess.STDOUT,
+                )
+                os.chdir("azure-sdk-for-python")
+                subprocess.check_call(
+                    ["git", "sparse-checkout", "init", "--cone"], stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT
+                )
+                subprocess.check_call(
+                    ["git", "sparse-checkout", "set", subdirectory], stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT
+                )
+                subprocess.check_call(["git", "checkout", "main"], stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT)
+
+                if not os.path.exists(os.path.join(os.getcwd(), subdirectory)):
+                    # code is not checked into main yet, nothing to compare
+                    logger.info(f"{subdirectory} is not checked into main, nothing to compare.")
+                    return 1
+
+                os.chdir(subdirectory)
+
+                command = get_pip_command(python_executable) + ["install", ".", "--force-reinstall"]
+
+                subprocess.check_call(command, stdout=subprocess.DEVNULL)
+            finally:
+                os.chdir(cwd)  # allow temp dir to be deleted
+            return 0
+
+    def get_type_complete_score(
+        self, executable, commands: typing.List[str], cwd: str, check_pytyped: bool = False
+    ) -> float:
+        try:
+            response = self.run_venv_command(executable, commands[1:], cwd, check=True)
+        except subprocess.CalledProcessError as e:
+            if e.returncode != 1:
+                logger.error(
+                    f"Running verifytypes failed: {e.stderr}. See https://aka.ms/python/typing-guide for information."
+                )
+                return -1.0
+            report = json.loads(e.output)
+            if check_pytyped:
+                pytyped_present = report["typeCompleteness"].get("pyTypedPath", None)
+                if not pytyped_present:
+                    logger.error(f"No py.typed file was found. See https://aka.ms/python/typing-guide for information.")
+                    return -1.0
+            return report["typeCompleteness"]["completenessScore"]
+
+        # library scores 100%
+        report = json.loads(response.stdout)
+        return report["typeCompleteness"]["completenessScore"]