SNOW-2337918 Fix for duplicated dependencies in Snowpark (#2609)

* SNOW-2337918 Fix for duplicated dependencies in Snowpark

* update RELEASE-NOTES.md

* fit this in 3.12

* bring empty line back

* one more empty line... we should have linter for Release Notes...

* removed redundant tests and obvious comments; drop local imports

* make real integration test

Author: sfc-gh-daniszewski

RELEASE-NOTES.md

@@ -40,6 +40,7 @@
 * Fixed issues when pasting content with trailing new lines.
 * Improved output handling  with streaming
 * Bumped `snowflake-connector-python` to 3.17.3
+* Fixed `snow snowpark deploy` failing on duplicated packages
 * Extend `Decimal` precision to 38
 
 

src/snowflake/cli/_plugins/snowpark/package/anaconda_packages.py

@@ -153,13 +153,37 @@ def write_requirements_file_in_snowflake_format(
     ):
         """Saves requirements to a file in format accepted by Snowflake SQL commands."""
         log.info("Writing requirements into file %s", file_path.path)
-        formatted_requirements = []
+
+        # Deduplicate requirements by package name, keeping the first occurrence
+        seen_packages = set()
+        deduplicated_requirements = []
+        duplicate_packages = set()
+
         for requirement in requirements:
             if requirement.name and requirement.name in self._packages:
-                snowflake_name = self._packages[requirement.name].snowflake_name
-                formatted_requirements.append(
-                    snowflake_name + requirement.formatted_specs
-                )
+                if requirement.name in seen_packages:
+                    duplicate_packages.add(requirement.name)
+                    log.warning(
+                        "Duplicate package '%s' found in Anaconda requirements. "
+                        "Ignoring: %s",
+                        requirement.name,
+                        requirement.name_and_version,
+                    )
+                else:
+                    seen_packages.add(requirement.name)
+                    deduplicated_requirements.append(requirement)
+
+        if duplicate_packages:
+            log.warning(
+                "Found duplicate Anaconda packages: %s. "
+                "Consider consolidating package versions in requirements.txt.",
+                ", ".join(sorted(duplicate_packages)),
+            )
+
+        formatted_requirements = []
+        for requirement in deduplicated_requirements:
+            snowflake_name = self._packages[requirement.name].snowflake_name
+            formatted_requirements.append(snowflake_name + requirement.formatted_specs)
 
         if formatted_requirements:
             file_path.write_text("\n".join(formatted_requirements))

src/snowflake/cli/_plugins/snowpark/package_utils.py

@@ -255,14 +255,55 @@ def split_downloaded_dependencies(
     anaconda_packages: AnacondaPackages,
     skip_version_check: bool,
 ) -> SplitDownloadedDependenciesResult:
-    packages_metadata: Dict[str, WheelMetadata] = {
-        meta.name: meta
+    # Build metadata for all downloaded wheels
+    all_wheels_metadata = [
+        meta
         for meta in (
             WheelMetadata.from_wheel(wheel_path)
             for wheel_path in downloads_dir.glob("*.whl")
         )
         if meta is not None
-    }
+    ]
+
+    # Detect and handle duplicate packages
+    packages_metadata: Dict[str, WheelMetadata] = {}
+    duplicate_packages = set()
+
+    for meta in all_wheels_metadata:
+        if meta.name in packages_metadata:
+            duplicate_packages.add(meta.name)
+            log.warning(
+                "Multiple versions of package '%s' found in dependencies. "
+                "Using: %s, Ignoring: %s",
+                meta.name,
+                packages_metadata[meta.name].wheel_path.name,
+                meta.wheel_path.name,
+            )
+        else:
+            packages_metadata[meta.name] = meta
+
+    if duplicate_packages:
+        log.warning(
+            "Found duplicate packages: %s. This may cause deployment issues. "
+            "Consider pinning package versions in requirements.txt to avoid conflicts.",
+            ", ".join(sorted(duplicate_packages)),
+        )
+
+        # Remove duplicate wheel files to prevent them from being extracted
+        for meta in all_wheels_metadata:
+            if (
+                meta.name in duplicate_packages
+                and meta not in packages_metadata.values()
+            ):
+                try:
+                    meta.wheel_path.unlink()
+                    log.debug("Removed duplicate wheel file: %s", meta.wheel_path.name)
+                except Exception as e:
+                    log.warning(
+                        "Failed to remove duplicate wheel file %s: %s",
+                        meta.wheel_path.name,
+                        e,
+                    )
     available_in_snowflake_dependencies: Dict = {}
     unavailable_dependencies: Dict = {}
 

tests/snowpark/test_build.py

@@ -12,13 +12,23 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 from typing import Set
-from unittest.mock import patch
+from unittest.mock import MagicMock, patch
 from zipfile import ZipFile
 
 import pytest
+from snowflake.cli._plugins.snowpark.models import (
+    Requirement,
+    WheelMetadata,
+)
+from snowflake.cli._plugins.snowpark.package.anaconda_packages import (
+    AnacondaPackages,
+    AvailablePackage,
+)
 from snowflake.cli._plugins.snowpark.package_utils import (
     DownloadUnavailablePackagesResult,
+    split_downloaded_dependencies,
 )
+from snowflake.cli.api.secure_path import SecurePath
 
 
 @patch("snowflake.cli._plugins.snowpark.package_utils.download_unavailable_packages")
@@ -66,3 +76,222 @@ def test_build_with_glob_patterns_in_artifacts(
 def _assert_zip_contains(app_zip: str, expected_files: Set[str]):
     zip_file = ZipFile(app_zip)
     assert set(zip_file.namelist()) == expected_files
+
+
+@patch("snowflake.cli._plugins.snowpark.package_utils.log")
+def test_split_downloaded_dependencies_handles_duplicates(mock_log, tmp_path):
+    """Test that split_downloaded_dependencies properly handles duplicate package versions.
+
+    This test prevents regression of the bug where multiple versions of the same package
+    (e.g., httpx-0.27.0.whl and httpx-0.28.1.whl) would both be included in dependencies.zip,
+    causing Snowflake deployment to fail with 'Package specified with multiple versions'.
+    """
+    downloads_dir = tmp_path / "downloads"
+    downloads_dir.mkdir()
+
+    httpx_v1_wheel = downloads_dir / "httpx-0.27.0-py3-none-any.whl"
+    httpx_v2_wheel = downloads_dir / "httpx-0.28.1-py3-none-any.whl"
+    httpx_v1_wheel.touch()
+    httpx_v2_wheel.touch()
+
+    requirements_file = tmp_path / "requirements.txt"
+    requirements_file.write_text("httpx\n")
+
+    original_from_wheel = WheelMetadata.from_wheel
+
+    def mock_from_wheel(wheel_path):
+        if "httpx-0.27.0" in str(wheel_path):
+            return WheelMetadata(name="httpx", wheel_path=wheel_path, dependencies=[])
+        elif "httpx-0.28.1" in str(wheel_path):
+            return WheelMetadata(name="httpx", wheel_path=wheel_path, dependencies=[])
+        return original_from_wheel(wheel_path)
+
+    with patch.object(WheelMetadata, "from_wheel", side_effect=mock_from_wheel):
+        mock_anaconda = MagicMock(spec=AnacondaPackages)
+        mock_anaconda.is_package_available.return_value = False
+
+        result = split_downloaded_dependencies(
+            requirements_file=SecurePath(requirements_file),
+            downloads_dir=downloads_dir,
+            anaconda_packages=mock_anaconda,
+            skip_version_check=False,
+        )
+
+        # Verify that 2 warnings were logged about duplicate packages
+        assert mock_log.warning.call_count >= 2
+
+        # Check the first warning call (multiple versions found)
+        first_call = mock_log.warning.call_args_list[0]
+        assert "Multiple versions of package '%s' found" in first_call.args[0]
+        assert first_call.args[1] == "httpx"  # package name
+        assert "httpx-" in first_call.args[2]  # using wheel filename
+        assert "httpx-" in first_call.args[3]  # ignoring wheel filename
+
+        # Check the second warning call (duplicate packages summary)
+        second_call = mock_log.warning.call_args_list[1]
+        assert "Found duplicate packages: %s" in second_call.args[0]
+        assert second_call.args[1] == "httpx"
+
+        # Verify that only one version of httpx is in the result
+        httpx_packages = [
+            pkg
+            for pkg in result.unavailable_dependencies_wheels
+            if pkg.requirement.name == "httpx"
+        ]
+        assert (
+            len(httpx_packages) == 1
+        ), f"Expected 1 httpx package, got {len(httpx_packages)}"
+
+        # Verify that one of the duplicate wheel files was removed
+        remaining_wheels = list(downloads_dir.glob("httpx-*.whl"))
+        assert (
+            len(remaining_wheels) == 1
+        ), f"Expected 1 remaining wheel file, got {len(remaining_wheels)}"
+
+
+@patch("snowflake.cli._plugins.snowpark.package.anaconda_packages.log")
+def test_write_requirements_file_deduplicates_anaconda_packages(mock_log, tmp_path):
+    """Test that write_requirements_file_in_snowflake_format deduplicates packages.
+
+    This test prevents regression of the bug where multiple entries for the same package
+    (e.g., 'httpx==0.28.1' and 'httpx>=0.20.0') would both be written to requirements.snowflake.txt,
+    causing Snowflake deployment issues.
+    """
+    packages = {
+        "httpx": AvailablePackage(snowflake_name="httpx", versions={"0.28.1", "0.27.0"})
+    }
+
+    anaconda_packages = AnacondaPackages(packages)
+
+    requirements = [
+        Requirement.parse_line("httpx==0.28.1"),
+        Requirement.parse_line("httpx>=0.20.0"),
+    ]
+
+    output_file = tmp_path / "requirements.snowflake.txt"
+
+    anaconda_packages.write_requirements_file_in_snowflake_format(
+        file_path=SecurePath(output_file), requirements=requirements
+    )
+
+    # Verify 2 warnings were logged
+    assert mock_log.warning.call_count >= 2
+
+    # Check the first warning call (duplicate package found)
+    first_call = mock_log.warning.call_args_list[0]
+    assert "Duplicate package '%s' found in Anaconda requirements" in first_call.args[0]
+    assert first_call.args[1] == "httpx"  # package name
+    assert first_call.args[2] == "httpx>=0.20.0"  # ignored requirement
+
+    # Check the second warning call (duplicate packages summary)
+    second_call = mock_log.warning.call_args_list[1]
+    assert "Found duplicate Anaconda packages: %s" in second_call.args[0]
+    assert second_call.args[1] == "httpx"
+
+    # Verify only one entry was written to the file
+    content = output_file.read_text().strip()
+    lines = [line.strip() for line in content.split("\n") if line.strip()]
+
+    # Should only have one httpx entry
+    httpx_lines = [line for line in lines if "httpx" in line]
+    assert (
+        len(httpx_lines) == 1
+    ), f"Expected 1 httpx line, got {len(httpx_lines)}: {httpx_lines}"
+    assert httpx_lines[0] == "httpx==0.28.1"  # Should keep the first one
+
+
+def test_similar_package_names_not_treated_as_duplicates():
+    """Test that packages with similar names are treated as separate packages.
+
+    This test ensures that packages like 'httpx' and 'httpx-retries' are correctly
+    treated as different packages and don't trigger duplicate detection.
+    """
+    req1 = Requirement.parse_line("httpx==0.28.1")
+    req2 = Requirement.parse_line("httpx-retries==0.4.2")
+
+    assert req1.name == "httpx"
+    assert req2.name == "httpx_retries"  # Note: hyphen becomes underscore
+    assert req1.name != req2.name
+
+    wheel1 = "httpx-0.28.1-py3-none-any.whl"
+    wheel2 = "httpx_retries-0.4.2-py3-none-any.whl"
+
+    name1 = WheelMetadata._get_name_from_wheel_filename(wheel1)  # noqa: SLF001
+    name2 = WheelMetadata._get_name_from_wheel_filename(wheel2)  # noqa: SLF001
+
+    assert name1 == "httpx"
+    assert name2 == "httpx_retries"
+    assert name1 != name2
+
+
+@patch("snowflake.cli._plugins.snowpark.package_utils.log")
+def test_multiple_different_packages_no_duplicates_detected(mock_log, tmp_path):
+    """Test that multiple different packages don't trigger duplicate detection.
+
+    This is a regression test to ensure that legitimate different packages
+    (like httpx, httpx-retries, requests, etc.) don't get flagged as duplicates.
+    """
+    downloads_dir = tmp_path / "downloads"
+    downloads_dir.mkdir()
+
+    wheels = [
+        "httpx-0.28.1-py3-none-any.whl",
+        "httpx_retries-0.4.2-py3-none-any.whl",
+        "requests-2.31.0-py3-none-any.whl",
+    ]
+
+    for wheel in wheels:
+        (downloads_dir / wheel).touch()
+
+    requirements_file = tmp_path / "requirements.txt"
+    requirements_file.write_text("httpx\nhttpx-retries\nrequests\n")
+
+    def mock_from_wheel(wheel_path):
+        wheel_name = wheel_path.name
+        if "httpx-0.28.1" in wheel_name:
+            return WheelMetadata(name="httpx", wheel_path=wheel_path, dependencies=[])
+        elif "httpx_retries-0.4.2" in wheel_name:
+            return WheelMetadata(
+                name="httpx_retries", wheel_path=wheel_path, dependencies=[]
+            )
+        elif "requests-2.31.0" in wheel_name:
+            return WheelMetadata(
+                name="requests", wheel_path=wheel_path, dependencies=[]
+            )
+        return None
+
+    with patch.object(WheelMetadata, "from_wheel", side_effect=mock_from_wheel):
+        mock_anaconda = MagicMock(spec=AnacondaPackages)
+        mock_anaconda.is_package_available.return_value = False
+
+        result = split_downloaded_dependencies(
+            requirements_file=SecurePath(requirements_file),
+            downloads_dir=downloads_dir,
+            anaconda_packages=mock_anaconda,
+            skip_version_check=False,
+        )
+
+        # Verify NO duplicate warnings were logged
+        warning_calls = [str(call) for call in mock_log.warning.call_args_list]
+        duplicate_warnings = [
+            call
+            for call in warning_calls
+            if "Multiple versions of package" in call
+            or "Found duplicate packages" in call
+        ]
+        assert (
+            len(duplicate_warnings) == 0
+        ), f"Unexpected duplicate warnings: {duplicate_warnings}"
+
+        # Verify three packages are in the result
+        package_names = {
+            pkg.requirement.name for pkg in result.unavailable_dependencies_wheels
+        }
+        assert "httpx" in package_names
+        assert "httpx_retries" in package_names
+        assert "requests" in package_names
+        assert len(package_names) == 3
+
+        # Verify all wheel files are still present
+        remaining_wheels = list(downloads_dir.glob("*.whl"))
+        assert len(remaining_wheels) == 3

tests_integration/test_data/projects/snowpark_duplicate_test/app.py

@@ -0,0 +1,2 @@
+def main():
+    return "hello"

tests_integration/test_data/projects/snowpark_duplicate_test/requirements.txt

@@ -0,0 +1,3 @@
+httpx>=0.20.0
+httpx-retries==0.4.2  
+aiohttp>=3.8.0

tests_integration/test_data/projects/snowpark_duplicate_test/snowflake.yml

@@ -0,0 +1,11 @@
+definition_version: 1
+snowpark:
+  project_name: "test_snowpark_project"
+  stage_name: "test_stage"
+  src: "app.py"
+  functions:
+    - name: func1
+      handler: "app.main"
+      signature: ""
+      returns: string
+      runtime: 3.10