[AI-5518] Add Python version filtering to integrations to the ddev size command (#20742)

* wip

* fix tests

* comment

* changelog

* remove print

* rename variables and method

* rewrite changelog and change is_valid_integration method name

* put comment in is_valid_integration and change regex

Author: lucia-sb

ddev/changelog.d/20742.added

@@ -0,0 +1 @@
+Adds logic to ensure `ddev size` filters integrations by the specified Python version.

ddev/src/ddev/cli/size/diff.py

@@ -204,13 +204,13 @@ def get_repo_info(
         repo = gitRepo.repo_dir
         task = progress.add_task("[cyan]Calculating sizes for the first commit...", total=None)
         gitRepo.checkout_commit(first_commit)
-        files_b = get_files(repo, compressed)
+        files_b = get_files(repo, compressed, version)
         dependencies_b = get_dependencies(repo, platform, version, compressed)
         progress.remove_task(task)
 
         task = progress.add_task("[cyan]Calculating sizes for the second commit...", total=None)
         gitRepo.checkout_commit(second_commit)
-        files_a = get_files(repo, compressed)
+        files_a = get_files(repo, compressed, version)
         dependencies_a = get_dependencies(repo, platform, version, compressed)
         progress.remove_task(task)
 

ddev/src/ddev/cli/size/status.py

@@ -94,7 +94,7 @@ def status_mode(
     params: CLIParameters,
 ) -> list[FileDataEntryPlatformVersion]:
     with console.status("[cyan]Calculating sizes...", spinner="dots"):
-        modules = get_files(repo_path, params["compressed"]) + get_dependencies(
+        modules = get_files(repo_path, params["compressed"], params["version"]) + get_dependencies(
             repo_path, params["platform"], params["version"], params["compressed"]
         )
 

ddev/src/ddev/cli/size/timeline.py

@@ -25,11 +25,10 @@
     compress,
     convert_to_human_readable_size,
     extract_version_from_about_py,
-    get_gitignore_files,
     get_valid_platforms,
     get_valid_versions,
     is_correct_dependency,
-    is_valid_integration,
+    is_valid_integration_file,
     print_table,
     save_csv,
     save_json,
@@ -454,10 +453,6 @@ def get_files(
         )
         return file_data
 
-    ignored_files = {"datadog_checks_dev", "datadog_checks_tests_helper"}
-    git_ignore = get_gitignore_files(repo_path)
-    included_folder = "datadog_checks" + os.sep
-
     total_size = 0
     version = ""
 
@@ -466,7 +461,7 @@ def get_files(
             file_path = os.path.join(root, file)
             relative_path = os.path.relpath(file_path, repo_path)
 
-            if not is_valid_integration(relative_path, included_folder, ignored_files, git_ignore):
+            if not is_valid_integration_file(relative_path, repo_path):
                 continue
 
             if file == "__about__.py" and "datadog_checks" in relative_path:

ddev/src/ddev/cli/size/utils/common_funcs.py

@@ -21,6 +21,7 @@
 from datadog import api, initialize
 
 from ddev.cli.application import Application
+from ddev.utils.toml import load_toml_file
 
 METRIC_VERSION = 2
 
@@ -108,8 +109,9 @@ def get_valid_versions(repo_path: Path | str) -> set[str]:
     """
     resolved_path = os.path.join(repo_path, os.path.join(repo_path, ".deps", "resolved"))
     versions = []
+    pattern = re.compile(r"\d+\.\d+")
     for file in os.listdir(resolved_path):
-        match = re.search(r"\d+\.\d+", file)
+        match = pattern.search(file)
         if match:
             versions.append(match.group())
     return set(versions)
@@ -119,7 +121,40 @@ def is_correct_dependency(platform: str, version: str, name: str) -> bool:
     return platform in name and version in name
 
 
-def is_valid_integration(path: str, included_folder: str, ignored_files: set[str], git_ignore: list[str]) -> bool:
+def is_valid_integration_file(
+    path: str,
+    repo_path: str,
+    ignored_files: set[str] | None = None,
+    included_folder: str | None = None,
+    git_ignore: list[str] | None = None,
+) -> bool:
+    """
+    Check if a file would be packaged with an integration.
+
+    Used to estimate integration package size by excluding:
+    - Hidden files (starting with ".")
+    - Files outside "datadog_checks"
+    - Helper/test-only packages (e.g. datadog_checks_dev)
+    - Files ignored by .gitignore
+
+    Args:
+        path (str): File path to check.
+        repo_path (str): Repository root, for loading .gitignore rules.
+
+    Returns:
+        bool: True if the file would be packaged, False otherwise.
+    """
+    if ignored_files is None:
+        ignored_files = {
+            "datadog_checks_dev",
+            "datadog_checks_tests_helper",
+        }
+
+    if included_folder is None:
+        included_folder = "datadog_checks" + os.sep
+
+    if git_ignore is None:
+        git_ignore = get_gitignore_files(repo_path)
     # It is not an integration
     if path.startswith("."):
         return False
@@ -166,29 +201,31 @@ def compress(file_path: str) -> int:
     return compressed_size
 
 
-def get_files(repo_path: str | Path, compressed: bool) -> list[FileDataEntry]:
+def get_files(repo_path: str | Path, compressed: bool, py_version: str) -> list[FileDataEntry]:
     """
     Calculates integration file sizes and versions from a repository.
+    Only takes into account integrations with a valid version looking at the pyproject.toml file
+    The pyproject.toml file should have a classifier with this format:
+        classifiers = [
+            ...
+            "Programming Language :: Python :: 3.12",
+            ...
+        ]
     """
-    ignored_files = {"datadog_checks_dev", "datadog_checks_tests_helper"}
-    git_ignore = get_gitignore_files(repo_path)
-    included_folder = "datadog_checks" + os.sep
-
     integration_sizes: dict[str, int] = {}
     integration_versions: dict[str, str] = {}
+    py_major_version = py_version.split(".")[0]
 
     for root, _, files in os.walk(repo_path):
+        integration_name = str(os.path.relpath(root, repo_path).split(os.sep)[0])
+
+        if not check_python_version(str(repo_path), integration_name, py_major_version):
+            continue
         for file in files:
             file_path = os.path.join(root, file)
             relative_path = os.path.relpath(file_path, repo_path)
-
-            if not is_valid_integration(relative_path, included_folder, ignored_files, git_ignore):
+            if not is_valid_integration_file(relative_path, str(repo_path)):
                 continue
-            path = Path(relative_path)
-            parts = path.parts
-
-            integration_name = parts[0]
-
             size = compress(file_path) if compressed else os.path.getsize(file_path)
             integration_sizes[integration_name] = integration_sizes.get(integration_name, 0) + size
 
@@ -208,6 +245,23 @@ def get_files(repo_path: str | Path, compressed: bool) -> list[FileDataEntry]:
     ]
 
 
+def check_python_version(repo_path: str, integration_name: str, py_major_version: str) -> bool:
+    pyproject_path = os.path.join(repo_path, integration_name, "pyproject.toml")
+    if os.path.exists(pyproject_path):
+        pyproject = load_toml_file(pyproject_path)
+        if "project" not in pyproject or "classifiers" not in pyproject["project"]:
+            return False
+        classifiers = pyproject["project"]["classifiers"]
+        integration_py_version = ""
+        pattern = re.compile(r"Programming Language :: Python :: (\d+)")
+        for classifier in classifiers:
+            match = pattern.match(classifier)
+            if match:
+                integration_py_version = match.group(1)
+                return integration_py_version == py_major_version
+    return False
+
+
 def extract_version_from_about_py(path: str) -> str:
     """
     Extracts the __version__ string from a given __about__.py file.
@@ -248,8 +302,9 @@ def get_dependencies_list(file_path: str) -> tuple[list[str], list[str], list[st
     versions = []
     with open(file_path, "r", encoding="utf-8") as file:
         file_content = file.read()
+        pattern = re.compile(r"([\w\-\d\.]+) @ (https?://[^\s#]+)")
         for line in file_content.splitlines():
-            match = re.search(r"([\w\-\d\.]+) @ (https?://[^\s#]+)", line)
+            match = pattern.search(line)
             if not match:
                 raise WrongDependencyFormat("The dependency format 'name @ link' is no longer supported.")
             name = match.group(1)
@@ -327,43 +382,43 @@ def get_dependencies_sizes(
 
 
 def is_excluded_from_wheel(path: str) -> bool:
-    '''
+    """
     These files are excluded from the wheel in the agent build:
     https://github.com/DataDog/datadog-agent/blob/main/omnibus/config/software/datadog-agent-integrations-py3.rb
     In order to have more accurate results, this files are excluded when computing the size of the dependencies while
     the wheels still include them.
-    '''
+    """
     excluded_test_paths = [
         os.path.normpath(path)
         for path in [
-            'idlelib/idle_test',
-            'bs4/tests',
-            'Cryptodome/SelfTest',
-            'gssapi/tests',
-            'keystoneauth1/tests',
-            'openstack/tests',
-            'os_service_types/tests',
-            'pbr/tests',
-            'pkg_resources/tests',
-            'psutil/tests',
-            'securesystemslib/_vendor/ed25519/test_data',
-            'setuptools/_distutils/tests',
-            'setuptools/tests',
-            'simplejson/tests',
-            'stevedore/tests',
-            'supervisor/tests',
-            'test',  # cm-client
-            'vertica_python/tests',
-            'websocket/tests',
+            "idlelib/idle_test",
+            "bs4/tests",
+            "Cryptodome/SelfTest",
+            "gssapi/tests",
+            "keystoneauth1/tests",
+            "openstack/tests",
+            "os_service_types/tests",
+            "pbr/tests",
+            "pkg_resources/tests",
+            "psutil/tests",
+            "securesystemslib/_vendor/ed25519/test_data",
+            "setuptools/_distutils/tests",
+            "setuptools/tests",
+            "simplejson/tests",
+            "stevedore/tests",
+            "supervisor/tests",
+            "test",  # cm-client
+            "vertica_python/tests",
+            "websocket/tests",
         ]
     ]
 
     type_annot_libraries = [
-        'krb5',
-        'Cryptodome',
-        'ddtrace',
-        'pyVmomi',
-        'gssapi',
+        "krb5",
+        "Cryptodome",
+        "ddtrace",
+        "pyVmomi",
+        "gssapi",
     ]
     rel_path = Path(path).as_posix()
 
@@ -377,7 +432,7 @@ def is_excluded_from_wheel(path: str) -> bool:
     if path_parts:
         dependency_name = path_parts[0]
         if dependency_name in type_annot_libraries:
-            if path.endswith('.pyi') or os.path.basename(path) == 'py.typed':
+            if path.endswith(".pyi") or os.path.basename(path) == "py.typed":
                 return True
 
     return False
@@ -830,14 +885,14 @@ def send_metrics_to_dd(
                 ],
             }
         )
-        key_count = (item['Platform'], item['Python_Version'])
+        key_count = (item["Platform"], item["Python_Version"])
         if key_count not in n_integrations:
             n_integrations[key_count] = 0
         if key_count not in n_dependencies:
             n_dependencies[key_count] = 0
-        if item['Type'] == 'Integration':
+        if item["Type"] == "Integration":
             n_integrations[key_count] += 1
-        elif item['Type'] == 'Dependency':
+        elif item["Type"] == "Dependency":
             n_dependencies[key_count] += 1
 
     for (platform, py_version), count in n_integrations.items():
@@ -919,8 +974,8 @@ def get_last_commit_timestamp() -> int:
 
 def get_last_commit_data() -> tuple[str, list[str], list[str]]:
     result = subprocess.run(["git", "log", "-1", "--format=%s"], capture_output=True, text=True, check=True)
-    ticket_pattern = r'\b(?:DBMON|SAASINT|AGENT|AI)-\d+\b'
-    pr_pattern = r'#(\d+)'
+    ticket_pattern = r"\b(?:DBMON|SAASINT|AGENT|AI)-\d+\b"
+    pr_pattern = r"#(\d+)"
 
     message = result.stdout.strip()
     tickets = re.findall(ticket_pattern, message)

ddev/tests/cli/size/test_diff.py

@@ -18,7 +18,7 @@ def mock_size_diff_dependencies():
     mock_git_repo.repo_dir = "fake_repo"
     mock_git_repo.get_commit_metadata.return_value = ("Feb 1 2025", "", "")
 
-    def get_compressed_files_side_effect(_, __):
+    def get_compressed_files_side_effect(*args, **kwargs):
         get_compressed_files_side_effect.counter += 1
         if get_compressed_files_side_effect.counter % 2 == 1:
             return [{"Name": "path1.py", "Version": "1.1.1", "Size_Bytes": 1000, "Type": "Integration"}]  # before
@@ -30,7 +30,7 @@ def get_compressed_files_side_effect(_, __):
 
     get_compressed_files_side_effect.counter = 0
 
-    def get_compressed_dependencies_side_effect(_, __, ___, ____):
+    def get_compressed_dependencies_side_effect(*args, **kwargs):
         get_compressed_dependencies_side_effect.counter += 1
         if get_compressed_dependencies_side_effect.counter % 2 == 1:
             return [{"Name": "dep1", "Version": "1.0.0", "Size_Bytes": 2000, "Type": "Dependency"}]  # before
@@ -45,11 +45,11 @@ def get_compressed_dependencies_side_effect(_, __, ___, ____):
     with (
         patch(
             "ddev.cli.size.diff.get_valid_platforms",
-            return_value=({'linux-x86_64', 'macos-x86_64', 'linux-aarch64', 'macos-aarch64', 'windows-x86_64'}),
+            return_value=({"linux-x86_64", "macos-x86_64", "linux-aarch64", "macos-aarch64", "windows-x86_64"}),
         ),
         patch(
             "ddev.cli.size.diff.get_valid_versions",
-            return_value=({'3.12'}),
+            return_value=({"3.12"}),
         ),
         patch("ddev.cli.size.diff.GitRepo.__enter__", return_value=mock_git_repo),
         patch("ddev.cli.size.diff.GitRepo.__exit__", return_value=None),
@@ -111,11 +111,11 @@ def test_diff_no_differences(ddev):
         patch("ddev.cli.size.diff.GitRepo.__exit__", return_value=None),
         patch(
             "ddev.cli.size.diff.get_valid_platforms",
-            return_value=({'linux-x86_64', 'macos-x86_64', 'linux-aarch64', 'macos-aarch64', 'windows-x86_64'}),
+            return_value=({"linux-x86_64", "macos-x86_64", "linux-aarch64", "macos-aarch64", "windows-x86_64"}),
         ),
         patch(
             "ddev.cli.size.diff.get_valid_versions",
-            return_value=({'3.12'}),
+            return_value=({"3.12"}),
         ),
         patch.object(fake_repo, "checkout_commit"),
         patch("ddev.cli.size.utils.common_funcs.tempfile.mkdtemp", return_value="fake_repo"),
@@ -162,11 +162,11 @@ def test_diff_invalid_platform(ddev):
         patch("ddev.cli.size.diff.GitRepo", return_value=mock_git_repo),
         patch(
             "ddev.cli.size.diff.get_valid_platforms",
-            return_value=({'linux-x86_64', 'macos-x86_64', 'linux-aarch64', 'macos-aarch64', 'windows-x86_64'}),
+            return_value=({"linux-x86_64", "macos-x86_64", "linux-aarch64", "macos-aarch64", "windows-x86_64"}),
         ),
         patch(
             "ddev.cli.size.diff.get_valid_versions",
-            return_value=({'3.12'}),
+            return_value=({"3.12"}),
         ),
     ):
         result = ddev("size", "diff", "commit1", "commit2", "--platform", "linux", "--python", "3.12", "--compressed")
@@ -184,11 +184,11 @@ def test_diff_invalid_version(ddev):
         patch("ddev.cli.size.diff.GitRepo", return_value=mock_git_repo),
         patch(
             "ddev.cli.size.diff.get_valid_platforms",
-            return_value=({'linux-x86_64', 'macos-x86_64', 'linux-aarch64', 'macos-aarch64', 'windows-x86_64'}),
+            return_value=({"linux-x86_64", "macos-x86_64", "linux-aarch64", "macos-aarch64", "windows-x86_64"}),
         ),
         patch(
             "ddev.cli.size.diff.get_valid_versions",
-            return_value=({'3.12'}),
+            return_value=({"3.12"}),
         ),
     ):
         result = ddev(
@@ -215,11 +215,11 @@ def test_diff_invalid_platform_and_version(ddev):
         patch("ddev.cli.size.diff.GitRepo", return_value=mock_git_repo),
         patch(
             "ddev.cli.size.diff.get_valid_platforms",
-            return_value=({'linux-x86_64', 'macos-x86_64', 'linux-aarch64', 'macos-aarch64', 'windows-x86_64'}),
+            return_value=({"linux-x86_64", "macos-x86_64", "linux-aarch64", "macos-aarch64", "windows-x86_64"}),
         ),
         patch(
             "ddev.cli.size.diff.get_valid_versions",
-            return_value=({'3.12'}),
+            return_value=({"3.12"}),
         ),
     ):
         result = ddev("size", "diff", "commit1", "commit2", "--platform", "linux", "--python", "2.10", "--compressed")