update verifywhl to check that one of homepage/repository exists (#43188)

* update verifywhl check to alias homepage/repository and validate project urls

* black

* check that homepage or repository link exists, don't check other project.urls

* black

Author: swathipil

eng/tools/azure-sdk-tools/ci_tools/parsing/parse_functions.py

@@ -156,46 +156,31 @@ def _normalize_url_fields(pkg_info, metadata: Dict[str, Any]) -> None:
     # Homepage from PEP 566 style
     if pkg_info.home_page:
         metadata["homepage"] = pkg_info.home_page
-
-    # Handle project URLs (can be in various formats)
-    if pkg_info.project_urls:
-        metadata["project_urls"] = pkg_info.project_urls
-
-        # Try to extract homepage from project_urls if not already set
-        if "homepage" not in metadata:
-            homepage = _extract_homepage_from_project_urls(pkg_info.project_urls)
-            if homepage:
-                metadata["homepage"] = homepage
+    elif pkg_info.project_urls:  # If homepage not found, try to extract from project_urls
+        if isinstance(pkg_info.project_urls, (list, tuple)):
+            for url_entry in pkg_info.project_urls:
+                if isinstance(url_entry, str) and "," in url_entry:
+                    # Format: "Label, https://example.com"
+                    label, url_value = url_entry.split(",", 1)
+                    label_lower = label.strip().lower()
+                    url_value = url_value.strip()
+                    if label_lower in ["homepage", "home-page", "home"]:
+                        metadata["homepage"] = url_value
+                    elif label_lower == "repository":
+                        metadata["repository"] = url_value
+        elif isinstance(pkg_info.project_urls, dict):
+            for key, value in pkg_info.project_urls.items():
+                key_lower = key.lower()
+                if key_lower in ["homepage", "home-page", "home"]:
+                    metadata["homepage"] = value
+                elif key_lower == "repository":
+                    metadata["repository"] = value
 
     # Download URL
     if hasattr(pkg_info, "download_url") and getattr(pkg_info, "download_url", None):
         metadata["download_url"] = pkg_info.download_url
 
 
-def _extract_homepage_from_project_urls(project_urls) -> Optional[str]:
-    """Extract homepage URL from project_urls in various formats."""
-    if not project_urls:
-        return None
-
-    # Handle different project_urls formats
-    if isinstance(project_urls, (list, tuple)):
-        for url_entry in project_urls:
-            if isinstance(url_entry, str) and "," in url_entry:
-                # Format: "Homepage, https://example.com"
-                url_type, url_value = url_entry.split(",", 1)
-                url_type = url_type.strip().lower()
-                url_value = url_value.strip()
-                if url_type in ["homepage", "home-page", "home", "website"]:
-                    return url_value
-    elif isinstance(project_urls, dict):
-        # Handle dictionary format
-        for key, value in project_urls.items():
-            if key.lower() in ["homepage", "home-page", "home", "website"]:
-                return value
-
-    return None
-
-
 def _add_optional_fields(pkg_info, metadata: Dict[str, Any]) -> None:
     """Add optional metadata fields that may be present."""
     optional_fields = ["obsoletes_dist", "provides_dist", "requires_external", "platform", "supported_platform"]

eng/tools/azure-sdk-tools/devtools_testutils/helpers.py

@@ -19,6 +19,7 @@
 this = sys.modules[__name__]
 this.recording_ids = {}
 
+
 def locate_assets(current_test_file: str) -> str:
     """Locate the test assets directory for the targeted testfile.
 
@@ -75,6 +76,7 @@ def locate_assets(current_test_file: str) -> str:
 
     raise FileNotFoundError(f"No matching breadcrumb file found for asset path {relative_asset_path}")
 
+
 def get_http_client(**kwargs):
     """Returns a `urllib3` client that provides the test proxy's self-signed certificate if it's available.
 

eng/tools/azure-sdk-tools/tests/integration/scenarios/pyproject_invalid_metadata/pyproject.toml

@@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta"
 [project]
 name = "azure-storage-blob"
 authors = [
-    {email = "[email protected]"},
+    {name = "Microsoft Corporation", email = "[email protected]"},
 ]
 description = "Microsoft Azure Blob Storage Client Library for Python"
 keywords = ["azure", "azure sdk"]
@@ -35,7 +35,7 @@ aio = [
 ]
 
 [project.urls]
-repository = "https://github.com/Azure/azure-sdk-for-python"
+source = "https://github.com/Azure/azure-sdk-for-python"
 
 [tool.setuptools.dynamic]
 version = {attr = "azure.storage.blob._version.VERSION"}

eng/tools/azure-sdk-tools/tests/integration/scenarios/pyproject_metadata/pyproject.toml

@@ -35,7 +35,6 @@ aio = [
 ]
 
 [project.urls]
-homepage = "https://github.com/Azure/azure-sdk-for-python/tree/main/sdk/storage/azure-storage-blob"
 repository = "https://github.com/Azure/azure-sdk-for-python"
 
 [tool.setuptools.dynamic]

eng/tools/azure-sdk-tools/tests/test_metadata_verification.py

@@ -113,15 +113,15 @@ def test_verify_valid_metadata_passes(package_type, scenario_name, scenario_path
 
 
 @pytest.mark.parametrize(
-    "package_type,scenario_name,scenario_path",
+    "package_type,scenario_name,scenario_path,missing_keys",
     [
-        ("wheel", "stable", "pyproject_invalid_metadata_scenario"),
-        ("sdist", "stable", "pyproject_invalid_metadata_scenario"),
-        ("wheel", "beta", "pyproject_beta_invalid_metadata_scenario"),
-        ("sdist", "beta", "pyproject_beta_invalid_metadata_scenario"),
+        ("wheel", "stable", "pyproject_invalid_metadata_scenario", ["homepage", "repository"]),
+        ("sdist", "stable", "pyproject_invalid_metadata_scenario", ["homepage", "repository"]),
+        ("wheel", "beta", "pyproject_beta_invalid_metadata_scenario", ["author_email", "summary"]),
+        ("sdist", "beta", "pyproject_beta_invalid_metadata_scenario", ["author_email", "summary"]),
     ],
 )
-def test_verify_invalid_metadata_fails_with_missing_keys(package_type, scenario_name, scenario_path, caplog):
+def test_verify_invalid_metadata_fails(package_type, scenario_name, scenario_path, missing_keys, caplog):
     """Test that verify_whl/verify_sdist fails for scenarios with invalid metadata and reports missing author_name and homepage."""
     # Get the actual scenario path from globals
     actual_scenario_path = globals()[scenario_path]
@@ -154,22 +154,20 @@ def test_verify_invalid_metadata_fails_with_missing_keys(package_type, scenario_
         # Check that the error log contains information about missing keys
         error_logs = [record.message for record in caplog.records if record.levelname == "ERROR"]
 
-        # Different scenarios have different missing keys
-        if scenario_name == "stable":
-            # Stable scenario is missing author name and homepage
-            expected_missing_keys = ["author", "homepage"]
-        else:  # beta scenario
-            # Beta scenario is missing author email and description
-            expected_missing_keys = ["author_email", "summary"]
-
-        # Check for either order of the missing keys
-        missing_keys_pattern1 = f"Missing keys: {{'{expected_missing_keys[0]}', '{expected_missing_keys[1]}'}}"
-        missing_keys_pattern2 = f"Missing keys: {{'{expected_missing_keys[1]}', '{expected_missing_keys[0]}'}}"
-        has_missing_keys_error = any(missing_keys_pattern1 in msg or missing_keys_pattern2 in msg for msg in error_logs)
-
-        assert (
-            has_missing_keys_error
-        ), f"Expected error log about missing keys '{expected_missing_keys[0]}' and '{expected_missing_keys[1]}' for {scenario_name} scenario, but got: {error_logs}"
+        # Raise error if homepage AND repository not found in current version
+        if "homepage" in missing_keys:
+            assert f"Current metadata must contain at least one of: {missing_keys}" in error_logs
+        # Otherwise, check for missing keys from prior version
+        else:
+            missing_keys_pattern1 = f"Missing keys: {{'{missing_keys[0]}', '{missing_keys[1]}'}}"
+            missing_keys_pattern2 = f"Missing keys: {{'{missing_keys[1]}', '{missing_keys[0]}'}}"
+            has_missing_keys_error = any(
+                missing_keys_pattern1 in msg or missing_keys_pattern2 in msg for msg in error_logs
+            )
+
+            assert (
+                has_missing_keys_error
+            ), f"Expected error log about Missing keys: '{missing_keys[0]}' and '{missing_keys[1]}' for {scenario_name} scenario, but got: {error_logs}"
 
     finally:
         # Cleanup dist directory

eng/tox/verify_whl.py

@@ -125,26 +125,47 @@ def verify_prior_version_metadata(package_name: str, prior_version: str, current
                 f"{package_name}=={prior_version}", "--dest", tmp_dir
             ], check=True, capture_output=True)
             zip_files = glob.glob(os.path.join(tmp_dir, package_type))
-            if not zip_files:
+            # If no match and we're not constrained to wheel-only, attempt legacy sdist (zip) once.
+            if not zip_files and package_type != "*.whl":
+                zip_files = glob.glob(os.path.join(tmp_dir, "*.zip"))
+            if not zip_files:  # Still nothing -> treat as no prior artifact to compare.
                 return True
 
             prior_metadata: Dict[str, Any] = extract_package_metadata(zip_files[0])
             is_compatible = verify_metadata_compatibility(current_metadata, prior_metadata)
-            if not is_compatible:
-                missing_keys = set(prior_metadata.keys()) - set(current_metadata.keys())
-                logging.error(f"Metadata compatibility failed for {package_name}. Missing keys: {missing_keys}")
             return is_compatible
         except Exception:
             return True
 
 
 def verify_metadata_compatibility(current_metadata: Dict[str, Any], prior_metadata: Dict[str, Any]) -> bool:
-    """Verify that all keys from prior version metadata are present in current version."""
-    if not prior_metadata:
-        return True
+    """Verify that all keys from prior version metadata are present in current version.
+
+    Special handling: homepage/repository keys are exempt from prior compatibility check,
+    but current version must have at least one of them.
+    """
     if not current_metadata:
         return False
-    return set(prior_metadata.keys()).issubset(set(current_metadata.keys()))
+    # Check that current version has at least one homepage or repository URL
+    repo_urls = ['homepage', 'repository']
+    current_keys_lower = {k.lower() for k in current_metadata.keys()}
+    if not any(key in current_keys_lower for key in repo_urls):
+        logging.error(f"Current metadata must contain at least one of: {repo_urls}")
+        return False
+
+    if not prior_metadata:
+        return True
+
+    # For backward compatibility check, exclude homepage/repository from prior requirements
+    prior_keys_filtered = {k for k in prior_metadata.keys() if k.lower() not in repo_urls}
+    current_keys = set(current_metadata.keys())
+
+    is_compatible = prior_keys_filtered.issubset(current_keys)
+    if not is_compatible:
+        missing_keys = prior_keys_filtered - current_keys
+        logging.error("Metadata compatibility failed. Missing keys: %s", missing_keys)
+    return is_compatible
+
 
 def get_path_to_zip(dist_dir: str, version: str, package_type: str = "*.whl") -> str:
     return glob.glob(os.path.join(dist_dir, "**", "*{}{}".format(version, package_type)), recursive=True)[0]