update parsed verifysdist and verifywhl to check metadata (#42619)

* add metadata extraction to ParsedSetup

* add check to verify sdist metadata

* add verify against prior version metadata

* normalize metadata b/w pep 621 and 566

* verify whl metadata

* clean up

* more clean up

* cleanup 3

* copilot comments

* add tests

* address comment

* nit

* uncomment tests

Author: swathipil

eng/tools/azure-sdk-tools/ci_tools/parsing/__init__.py

@@ -11,6 +11,7 @@
     get_ci_config,
     get_version_py,
     get_pyproject,
+    extract_package_metadata,
     VERSION_REGEX,
     VERSION_PY,
     OLD_VERSION_PY
@@ -29,6 +30,7 @@
     "get_ci_config",
     "get_version_py",
     "get_pyproject",
+    "extract_package_metadata",
     "VERSION_REGEX",
     "VERSION_PY",
     "OLD_VERSION_PY"

eng/tools/azure-sdk-tools/ci_tools/parsing/parse_functions.py

@@ -12,7 +12,7 @@
     # otherwise fall back to pypi package tomli
     import tomli as toml
 
-from typing import Dict, List, Tuple, Any, Optional
+from typing import Dict, List, Tuple, Any, Optional, Union
 
 # this assumes the presence of "packaging"
 from packaging.requirements import Requirement
@@ -50,20 +50,179 @@
 }
 
 
+def extract_package_metadata(package_path: str) -> Dict[str, Any]:
+    """Extract package metadata from a built package or source directory with comprehensive PEP 566/621 normalization."""
+    from pkginfo import get_metadata
+    try:
+        # Note: metadata may be different between source directory and built packages since
+        # some metadata may be normalized/transformed during build process
+        pkg_info = get_metadata(package_path)
+
+        if not pkg_info:
+            return {}
+
+        # Convert pkginfo object to dictionary with normalized keys
+        metadata: Dict[str, Any] = {}
+
+        # Core metadata fields (always normalized to same key names)
+        if pkg_info.name:
+            metadata['name'] = pkg_info.name
+        if pkg_info.version:
+            metadata['version'] = pkg_info.version
+        if pkg_info.keywords:
+            metadata['keywords'] = pkg_info.keywords
+
+        # Summary/Description normalization
+        if pkg_info.summary:
+            metadata['summary'] = pkg_info.summary
+        if pkg_info.description:
+            metadata['description'] = pkg_info.description
+
+        # Classifiers (consistent across PEPs)
+        if pkg_info.classifiers:
+            metadata['classifiers'] = pkg_info.classifiers
+
+        # Python version requirements
+        if pkg_info.requires_python:
+            metadata['requires_python'] = pkg_info.requires_python
+
+        # Dependencies normalization
+        if pkg_info.requires_dist:
+            metadata['requires_dist'] = pkg_info.requires_dist
+
+        # Author/Maintainer normalization - handle both simple and complex formats
+        _normalize_person_fields(pkg_info, metadata, 'author')
+        _normalize_person_fields(pkg_info, metadata, 'maintainer')
+
+        # License normalization - handle both PEP 566 and PEP 621 formats
+        _normalize_license_field(pkg_info, metadata)
+
+        # URL normalization - handle both home_page and project_urls
+        _normalize_url_fields(pkg_info, metadata)
+
+        # Additional optional fields
+        _add_optional_fields(pkg_info, metadata)
+        return metadata
+
+    except Exception as e:
+        logging.warning(f"Error extracting metadata from {package_path}: {e}")
+        return {}
+
+
+def _normalize_person_fields(pkg_info, metadata: Dict[str, Any], role: str) -> None:
+    """Normalize author/maintainer fields from both PEP 566 and PEP 621 formats."""
+    name_attr = getattr(pkg_info, role, None)
+    email_attr = getattr(pkg_info, f'{role}_email', None)
+
+    # Handle PEP 566 style (separate fields)
+    if name_attr and email_attr:
+        metadata[role] = name_attr
+        metadata[f'{role}_email'] = email_attr
+    # Handle PEP 621 style where name might be embedded in email field
+    elif email_attr:
+        # Check if email contains name in format "Name <email>"
+        if '<' in email_attr and '>' in email_attr:
+            # Extract name and email from "Name <email>" format
+            match = re.match(r'^(.+?)\s*<(.+?)>$', email_attr.strip())
+            if match:
+                name_part = match.group(1).strip()
+                email_part = match.group(2).strip()
+                metadata[role] = name_part
+                metadata[f'{role}_email'] = email_part
+            else:
+                metadata[f'{role}_email'] = email_attr
+        else:
+            metadata[f'{role}_email'] = email_attr
+    # Handle case where only name is provided
+    elif name_attr:
+        metadata[f'{role}'] = name_attr
+
+
+def _normalize_license_field(pkg_info, metadata: Dict[str, Any]) -> None:
+    """Normalize license field from both PEP 566 and PEP 621 formats."""
+    if pkg_info.license:
+        metadata['license'] = pkg_info.license
+    # Handle license expression (PEP 639) if available
+    if hasattr(pkg_info, 'license_expression') and getattr(pkg_info, 'license_expression', None):
+        metadata['license'] = pkg_info.license_expression
+    # Handle license file references if available
+    if hasattr(pkg_info, 'license_file') and getattr(pkg_info, 'license_file', None):
+        metadata['license'] = pkg_info.license_file
+
+
+def _normalize_url_fields(pkg_info, metadata: Dict[str, Any]) -> None:
+    """Normalize URL fields from both PEP 566 and PEP 621 formats."""
+    # Homepage from PEP 566 style
+    if pkg_info.home_page:
+        metadata['homepage'] = pkg_info.home_page
+
+    # Handle project URLs (can be in various formats)
+    if pkg_info.project_urls:
+        metadata['project_urls'] = pkg_info.project_urls
+
+        # Try to extract homepage from project_urls if not already set
+        if 'homepage' not in metadata:
+            homepage = _extract_homepage_from_project_urls(pkg_info.project_urls)
+            if homepage:
+                metadata['homepage'] = homepage
+
+    # Download URL
+    if hasattr(pkg_info, 'download_url') and getattr(pkg_info, 'download_url', None):
+        metadata['download_url'] = pkg_info.download_url
+
+
+def _extract_homepage_from_project_urls(project_urls) -> Optional[str]:
+    """Extract homepage URL from project_urls in various formats."""
+    if not project_urls:
+        return None
+
+    # Handle different project_urls formats
+    if isinstance(project_urls, (list, tuple)):
+        for url_entry in project_urls:
+            if isinstance(url_entry, str) and ',' in url_entry:
+                # Format: "Homepage, https://example.com"
+                url_type, url_value = url_entry.split(',', 1)
+                url_type = url_type.strip().lower()
+                url_value = url_value.strip()
+                if url_type in ['homepage', 'home-page', 'home', 'website']:
+                    return url_value
+    elif isinstance(project_urls, dict):
+        # Handle dictionary format
+        for key, value in project_urls.items():
+            if key.lower() in ['homepage', 'home-page', 'home', 'website']:
+                return value
+
+    return None
+
+
+def _add_optional_fields(pkg_info, metadata: Dict[str, Any]) -> None:
+    """Add optional metadata fields that may be present."""
+    optional_fields = [
+        'obsoletes_dist', 'provides_dist', 'requires_external',
+        'platform', 'supported_platform'
+    ]
+
+    for field in optional_fields:
+        if hasattr(pkg_info, field):
+            value = getattr(pkg_info, field, None)
+            if value:
+                metadata[field] = value
+
+
 def discover_namespace(package_root_path: str) -> Optional[str]:
     """
     Discover the true namespace of a package by walking through its directory structure
     and finding the first __init__.py that contains actual content (not just namespace extension).
-    
+
     :param str package_root_path: Root path of the package directory
     :rtype: str or None
     :return: The discovered namespace string, or None if no suitable namespace found
     """
     if not os.path.exists(package_root_path):
         return None
-        
+
     namespace = None
-    
+
     for root, subdirs, files in os.walk(package_root_path):
         # Ignore any modules with name starts with "_"
         # For e.g. _generated, _shared etc
@@ -73,26 +232,26 @@ def discover_namespace(package_root_path: str) -> Optional[str]:
         for d in dirs_to_skip:
             logging.debug("Dirs to skip: {}".format(dirs_to_skip))
             subdirs.remove(d)
-            
+
         if INIT_PY_FILE in files:
             module_name = os.path.relpath(root, package_root_path).replace(
                 os.path.sep, "."
             )
-            
+
             # If namespace has not been set yet, try to find the first __init__.py that's not purely for extension.
             if not namespace:
                 namespace = _set_root_namespace(
                     os.path.join(root, INIT_PY_FILE), module_name
                 )
-    
+
     return namespace
 
 
 def _set_root_namespace(init_file_path: str, module_name: str) -> Optional[str]:
     """
     Examine an __init__.py file to determine if it represents a substantial namespace 
     or is just a namespace extension file.
-    
+
     :param str init_file_path: Path to the __init__.py file
     :param str module_name: The module name corresponding to this __init__.py
     :rtype: str or None
@@ -111,16 +270,16 @@ def _set_root_namespace(init_file_path: str, module_name: str) -> Optional[str]:
                 # If comment, skip line. Otherwise, add to content.
                 if not in_docstring and not stripped_line.startswith("#"):
                     content.append(line)
-            
+
             # If there's more than one line of content, or if there's one line that's not just namespace extension
             if len(content) > 1 or (
                 len(content) == 1 and INIT_EXTENSION_SUBSTRING not in content[0]
             ):
                 return module_name
-                
+
     except Exception as e:
         logging.error(f"Error reading {init_file_path}: {e}")
-        
+
     return None
 
 
@@ -168,7 +327,8 @@ def __init__(
     @classmethod
     def from_path(cls, parse_directory_or_file: str):
         """
-        Creates a new ParsedSetup instance from a path to a setup.py, pyproject.toml (with [project] member), or a directory containing either of those files.
+        Creates a new ParsedSetup instance from a path to a setup.py, pyproject.toml (with [project] member), 
+        or a directory containing either of those files.
         """
         (
             name,
@@ -370,9 +530,9 @@ def setup(*args, **kwargs):
 
     fixed = ast.fix_missing_locations(parsed)
     codeobj = compile(fixed, setup_filename, "exec")
-    local_vars = {}
+    local_vars: Dict[str, Any] = {}
     kwargs = {}
-    global_vars = {"__setup_calls__": []}
+    global_vars: Dict[str, Any] = {"__setup_calls__": []}
     current_dir = os.getcwd()
     working_dir = os.path.dirname(setup_filename)
     os.chdir(working_dir)
@@ -390,7 +550,7 @@ def setup(*args, **kwargs):
 
     version = kwargs.get("version")
     name = kwargs.get("name")
-    name_space = name.replace("-", ".")
+    name_space = name.replace("-", ".") if name else ""
     packages = kwargs.get("packages", [])
 
     if packages:
@@ -588,11 +748,12 @@ def parse_setup(
         <keywords>,
         <ext_packages>,
         <ext_modules>,
-        <is_metapackage>
+        <is_metapackage>,
     )
 
     If a pyproject.toml (containing [project]) or a setup.py is NOT found, a ValueError will be raised.
     """
+
     targeted_path = setup_filename_or_folder
     if os.path.isfile(setup_filename_or_folder):
         targeted_path = os.path.dirname(setup_filename_or_folder)
@@ -602,9 +763,11 @@ def parse_setup(
         raise ValueError(f"Unable to find a setup.py or pyproject.toml in {setup_filename_or_folder}")
 
     if resolved_filename.endswith(".toml"):
-        return parse_pyproject(resolved_filename)
+        result = parse_pyproject(resolved_filename)
     else:
-        return parse_setup_py(resolved_filename)
+        result = parse_setup_py(resolved_filename)
+
+    return result
 
 
 def get_pyproject_dict(pyproject_file: str) -> Dict[str, Any]:

eng/tools/azure-sdk-tools/pypi_tools/pypi.py

@@ -81,3 +81,20 @@ def get_relevant_versions(self, package_name):
         versions = self.get_ordered_versions(package_name)
         stable_releases = [version for version in versions if not version.is_prerelease]
         return (versions[-1], stable_releases[-1])
+
+def retrieve_versions_from_pypi(package_name: str) -> List[str]:
+    """
+    Retrieve all published versions on PyPI for the package.
+
+    :param str package_name: The name of the package.
+    :rtype: List[str]
+    :return: List of all version strings (sorted ascending).
+    """
+    try:
+        client = PyPIClient()
+        all_versions = client.get_ordered_versions(package_name)
+        # Return all versions as strings
+        return [str(v) for v in all_versions]
+    except Exception as ex:
+        logging.warning("Failed to retrieve PyPI data for %s: %s", package_name, ex)
+        return []

eng/tools/azure-sdk-tools/tests/integration/scenarios/pyproject_beta_invalid_metadata/README.md

@@ -0,0 +1 @@
+This folder contains a beta release `pyproject.toml` scenario for azure-mgmt-planetarycomputer with invalid metadata (missing author name and homepage). We use this to ensure that our metadata verification within ParsedSetup properly detects missing required fields for beta packages.

eng/tools/azure-sdk-tools/tests/integration/scenarios/pyproject_beta_invalid_metadata/azure/mgmt/planetarycomputer/_version.py

@@ -0,0 +1,6 @@
+# --------------------------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for license information.
+# --------------------------------------------------------------------------------------------
+
+VERSION = "1.0.0b2"

eng/tools/azure-sdk-tools/tests/integration/scenarios/pyproject_beta_invalid_metadata/pyproject.toml

@@ -0,0 +1,45 @@
+[build-system]
+requires = ["setuptools>=77.0.3", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "azure-mgmt-planetarycomputer"
+authors = [
+    {name = "Microsoft Corporation"},
+]
+keywords = ["azure", "azure sdk"]
+requires-python = ">=3.9"
+license = "MIT"
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Programming Language :: Python",
+    "Programming Language :: Python :: 3 :: Only",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+]
+dependencies = [
+    "isodate>=0.6.1",
+    "typing-extensions>=4.6.0",
+    "azure-common>=1.1",
+    "azure-mgmt-core>=1.5.0"
+]
+dynamic = ["version", "readme"]
+
+[project.urls]
+Homepage = "https://github.com/Azure/azure-sdk-for-python"
+
+[tool.setuptools]
+include-package-data = true
+
+[tool.setuptools.packages.find]
+exclude = ["tests", "azure", "azure.mgmt"]
+
+[tool.setuptools.package-data]
+pytyped = ["py.typed"]
+
+[tool.setuptools.dynamic]
+version = {attr = "azure.mgmt.planetarycomputer._version.VERSION"}
+readme = {file = "README.md", content-type = "text/markdown"}

eng/tools/azure-sdk-tools/tests/integration/scenarios/pyproject_beta_metadata/README.md

@@ -0,0 +1 @@
+This folder contains a basic beta release `pyproject.toml` scenario for azure-mgmt-planetarycomputer. We use this to ensure that our metadata verification within ParsedSetup is working properly for beta packages.