diff --git a/src/sagemaker/serve/detector/dependency_manager.py b/src/sagemaker/serve/detector/dependency_manager.py index e72a84da30..a6e6e0a091 100644 --- a/src/sagemaker/serve/detector/dependency_manager.py +++ b/src/sagemaker/serve/detector/dependency_manager.py @@ -34,27 +34,7 @@ def capture_dependencies(dependencies: dict, work_dir: Path, capture_all: bool = """Placeholder docstring""" path = work_dir.joinpath("requirements.txt") if "auto" in dependencies and dependencies["auto"]: - command = [ - sys.executable, - Path(__file__).parent.joinpath("pickle_dependencies.py"), - "--pkl_path", - work_dir.joinpath(PKL_FILE_NAME), - "--dest", - path, - ] - - if capture_all: - command.append("--capture_all") - - subprocess.run( - command, - env={"SETUPTOOLS_USE_DISTUTILS": "stdlib"}, - check=True, - ) - - with open(path, "r") as f: - autodetect_depedencies = f.read().splitlines() - autodetect_depedencies.append("sagemaker[huggingface]>=2.199") + raise ValueError("Auto requirements.txt generation is temporarily disabled in this version of sagemaker") else: autodetect_depedencies = ["sagemaker[huggingface]>=2.199"] diff --git a/src/sagemaker/serve/detector/pickle_dependencies.py b/src/sagemaker/serve/detector/pickle_dependencies.py index 5a1cd43869..b8b97981e5 100644 --- a/src/sagemaker/serve/detector/pickle_dependencies.py +++ b/src/sagemaker/serve/detector/pickle_dependencies.py @@ -100,25 +100,6 @@ def get_currently_used_packages(): return currently_used_packages -def get_requirements_for_pkl_file(pkl_path: Path, dest: Path): - """Placeholder docstring""" - with open(pkl_path, mode="rb") as file: - cloudpickle.load(file) - - currently_used_packages = get_currently_used_packages() - - with open(dest, mode="w+") as out: - for x in get_all_installed_packages(): - name = x["name"] - version = x["version"] - # skip only for dev - if name == "boto3": - boto3_version = boto3.__version__ - out.write(f"boto3=={boto3_version}\n") - elif name in currently_used_packages: - out.write(f"{name}=={version}\n") - - def get_all_requirements(dest: Path): """Placeholder docstring""" all_installed_packages = get_all_installed_packages() @@ -146,15 +127,3 @@ def parse_args(): args = parser.parse_args() return (Path(args.pkl_path), Path(args.dest), args.capture_all) - -def main(): - """Placeholder docstring""" - pkl_path, dest, capture_all = parse_args() - if capture_all: - get_all_requirements(dest) - else: - get_requirements_for_pkl_file(pkl_path, dest) - - -if __name__ == "__main__": - main() diff --git a/tests/unit/sagemaker/serve/detector/test_dependency_manager.py b/tests/unit/sagemaker/serve/detector/test_dependency_manager.py deleted file mode 100644 index 491968dd25..0000000000 --- a/tests/unit/sagemaker/serve/detector/test_dependency_manager.py +++ /dev/null @@ -1,141 +0,0 @@ -# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"). You -# may not use this file except in compliance with the License. A copy of -# the License is located at -# -# http://aws.amazon.com/apache2.0/ -# -# or in the "license" file accompanying this file. This file is -# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF -# ANY KIND, either express or implied. See the License for the specific -# language governing permissions and limitations under the License. -from __future__ import absolute_import - -import unittest -from unittest.mock import patch, mock_open, call -from pathlib import Path - -from sagemaker.serve.detector.dependency_manager import _parse_dependency_list, capture_dependencies - - -DEPENDENCY_LIST = [ - "requests==2.26.0", - "numpy>=1.20.0", - "pandas<=1.3.3", - "matplotlib<3.5.0", - "scikit-learn>0.24.1", - "Django!=4.0.0", - "attrs>=23.1.0,<24", - "torch@https://download.pytorch.org/whl/cpu/torch-2.0.0%2Bcpu-cp310-cp310-linux_x86_64.whl", - "# some comment", - "boto3==1.26.*", -] - -EXPECTED_DEPENDENCY_MAP = { - "requests": "==2.26.0", - "numpy": ">=1.20.0", - "pandas": "<=1.3.3", - "matplotlib": "<3.5.0", - "scikit-learn": ">0.24.1", - "Django": "!=4.0.0", - "attrs": ">=23.1.0,<24", - "torch": "@https://download.pytorch.org/whl/cpu/torch-2.0.0%2Bcpu-cp310-cp310-linux_x86_64.whl", - "boto3": "==1.26.*", -} - -DEPENDENCY_CONFIG = { - "auto": True, - "requirements": "/path/to/requirements.txt", - "custom": ["custom_module==1.2.3", "other_module@http://some/website.whl"], -} - -NO_AUTO_DEPENDENCY_CONFIG = { - "auto": False, - "requirements": "/path/to/requirements.txt", - "custom": ["custom_module==1.2.3", "other_module@http://some/website.whl"], -} - -WORK_DIR = Path("/path/to/working/dir") - -AUTODETECTED_REQUIREMENTS = """module==1.2 -custom_module==1.2.0 -numpy==4.5 -boto3==1.26.135 -""" - -NO_AUTODETECTED_REQUIREMENTS = """ -""" - -CUSTOM_REQUIREMENT_FILE = """boto3=1.28.* -""" - - -class DepedencyManagerTest(unittest.TestCase): - @patch("sagemaker.serve.detector.dependency_manager.Path") - @patch("builtins.open", new_callable=mock_open, read_data=AUTODETECTED_REQUIREMENTS) - @patch("sagemaker.serve.detector.dependency_manager.subprocess") - def test_capture_dependencies(self, mock_subprocess, mock_file, mock_path): - mock_open_custom_file = mock_open(read_data=CUSTOM_REQUIREMENT_FILE) - mock_open_write = mock_open(read_data="") - handlers = ( - mock_file.return_value, - mock_open_custom_file.return_value, - mock_open_write.return_value, - ) - mock_file.side_effect = handlers - - mock_path.is_file.return_value = True - - capture_dependencies(dependencies=DEPENDENCY_CONFIG, work_dir=WORK_DIR, capture_all=False) - mock_subprocess.run.assert_called_once() - - mocked_writes = mock_open_write.return_value.__enter__().write - - assert 6 == mocked_writes.call_count - - expected_calls = [ - call("module==1.2\n"), - call("custom_module==1.2.3\n"), - call("numpy==4.5\n"), - call("boto3=1.28.*\n"), - call("sagemaker[huggingface]>=2.199\n"), - call("other_module@http://some/website.whl\n"), - ] - mocked_writes.assert_has_calls(expected_calls) - - @patch("sagemaker.serve.detector.dependency_manager.Path") - @patch("builtins.open", new_callable=mock_open, read_data=NO_AUTODETECTED_REQUIREMENTS) - @patch("sagemaker.serve.detector.dependency_manager.subprocess") - def test_capture_dependencies_no_auto_detect(self, mock_path, mock_file, mock_subprocess): - mock_open_custom_file = mock_open(read_data=CUSTOM_REQUIREMENT_FILE) - mock_open_write = mock_open(read_data="") - handlers = ( - mock_open_custom_file.return_value, - mock_open_write.return_value, - ) - mock_file.side_effect = handlers - - mock_path.is_file.return_value = True - - capture_dependencies( - dependencies=NO_AUTO_DEPENDENCY_CONFIG, work_dir=WORK_DIR, capture_all=False - ) - mock_subprocess.run().assert_not_called() - mocked_writes = mock_open_write.return_value.__enter__().write - - assert 4 == mocked_writes.call_count - - expected_calls = [ - call("boto3=1.28.*\n"), - call("custom_module==1.2.3\n"), - call("other_module@http://some/website.whl\n"), - ] - mocked_writes.assert_has_calls(expected_calls) - - def test_parse_dependency_list(self): - dependency_map = _parse_dependency_list(DEPENDENCY_LIST) - - for key, value in dependency_map.items(): - self.assertTrue(key in EXPECTED_DEPENDENCY_MAP.keys()) - self.assertEqual(value, EXPECTED_DEPENDENCY_MAP.get(key)) diff --git a/tests/unit/sagemaker/serve/detector/test_pickle_dependencies.py b/tests/unit/sagemaker/serve/detector/test_pickle_dependencies.py deleted file mode 100644 index 34cab8a526..0000000000 --- a/tests/unit/sagemaker/serve/detector/test_pickle_dependencies.py +++ /dev/null @@ -1,244 +0,0 @@ -# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"). You -# may not use this file except in compliance with the License. A copy of -# the License is located at -# -# http://aws.amazon.com/apache2.0/ -# -# or in the "license" file accompanying this file. This file is -# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF -# ANY KIND, either express or implied. See the License for the specific -# language governing permissions and limitations under the License. -from __future__ import absolute_import -from sagemaker.serve.detector.pickle_dependencies import get_requirements_for_pkl_file - -import boto3 -import json -import io -import sys -import subprocess -from mock import MagicMock, mock_open, patch, call -from types import ModuleType - -import logging - -logger = logging.getLogger(__name__) - - -def create_mock_modules(name, doc, file): - m = ModuleType(name, doc) - m.__file__ = file - return m - - -INSTALLED_PKG_JSON = [ - {"name": "sagemaker", "version": "1.2.3"}, - {"name": "boto3", "version": "1.2.3"}, - {"name": "xgboost", "version": "1.2.3"}, - {"name": "scipy", "version": "1.2.3"}, -] -INSTALLED_PKG_LST = ["sagemaker", "boto3", "xgboost", "scipy"] -INSTALLED_PKG_JSON_UNUSED = INSTALLED_PKG_JSON + [{"name": "unused", "version": "1.2.3"}] -INSTALLED_PKG_LST_WITH_UNUSED = INSTALLED_PKG_LST + ["unused"] -INSTALLED_PKG_READLINE_LST = [ - "Name: sagemaker", - "Location: /tmp/to/site-packages/sagemaker", - "Files: sagemaker/__init__.py", - "---", - "Name: boto3", - "Location: /tmp/to/site-packages/boto3", - "Files: boto3/__init__.py", - "---", - "Name: xgboost", - "Location: /tmp/to/site-packages/xgboost", - "Files: xgboost/__init__.py", - "---", - "Name: scipy", - "Location: /tmp/to/site-packages/scipy", - "Files: scipy/__init__.py", -] -INSTALLED_PKG_READLINE = io.BytesIO("\n".join(INSTALLED_PKG_READLINE_LST).encode("utf-8")) -INSTALLED_PKG_READLINE_UNUSED_LST = INSTALLED_PKG_READLINE_LST + [ - "---", - "Name: unused", - "Location: /tmp/to/site-packages/unused", - "Files: unused/__init__.py", -] -INSTALLED_PKG_READLINE_UNUSED = io.BytesIO( - "\n".join(INSTALLED_PKG_READLINE_UNUSED_LST).encode("utf-8") -) -CURRENTLY_USED_FILES = { - "sagemaker": create_mock_modules( - "sagemaker", - "mock sagemaker module", - "/tmp/to/site-packages/sagemaker/sagemaker/__init__.py", - ), - "boto3": create_mock_modules( - "boto3", "mock boto3 module", "/tmp/to/site-packages/boto3/boto3/__init__.py" - ), - "xgboost": create_mock_modules( - "xgboost", - "mock xgboost module", - "/tmp/to/site-packages/xgboost/xgboost/__init__.py", - ), - "scipy": create_mock_modules( - "scipy", "mock scipy module", "/tmp/to/site-packages/scipy/scipy/__init__.py" - ), -} -EXPECTED_SM_WHL = "/opt/ml/model/whl/sagemaker-2.197.1.dev0-py2.py3-none-any.whl" -EXPECTED_BOTO3_MAPPING = f"boto3=={boto3.__version__}" -EXPECTED_SUBPROCESS_CMD = [sys.executable, "-m", "pip", "--disable-pip-version-check"] - - -# happy case -def test_generate_requirements_exact_match(monkeypatch): - with patch("cloudpickle.load"), patch("tqdm.tqdm"), patch( - "sagemaker.serve.detector.pickle_dependencies.subprocess.run" - ) as subprocess_run, patch( - "sagemaker.serve.detector.pickle_dependencies.subprocess.Popen" - ) as subprocess_popen, patch( - "builtins.open" - ) as mocked_open, monkeypatch.context() as m: - mock_run_stdout = MagicMock() - mock_run_stdout.stdout = json.dumps(INSTALLED_PKG_JSON).encode("utf-8") - subprocess_run.return_value = mock_run_stdout - - mock_popen_stdout = MagicMock() - mock_popen_stdout.stdout = INSTALLED_PKG_READLINE - subprocess_popen.return_value = mock_popen_stdout - - m.setattr(sys, "modules", CURRENTLY_USED_FILES) - - open_dest_file = mock_open() - mocked_open.side_effect = [ - mock_open().return_value, - open_dest_file.return_value, - ] - - get_requirements_for_pkl_file("/path/to/serve.pkl", "path/to/requirements.txt") - - mocked_open.assert_any_call("/path/to/serve.pkl", mode="rb") - mocked_open.assert_any_call("path/to/requirements.txt", mode="w+") - - assert 2 == subprocess_run.call_count - subprocess_run_call = call( - EXPECTED_SUBPROCESS_CMD + ["list", "--format", "json"], - stdout=subprocess.PIPE, - check=True, - ) - subprocess_run.assert_has_calls([subprocess_run_call, subprocess_run_call]) - - subprocess_popen.assert_called_once_with( - EXPECTED_SUBPROCESS_CMD + ["show", "-f"] + INSTALLED_PKG_LST, - stdout=subprocess.PIPE, - ) - - mocked_writes = open_dest_file.return_value.__enter__().write - assert 4 == mocked_writes.call_count - - expected_calls = [ - call("sagemaker==1.2.3\n"), - call(f"{EXPECTED_BOTO3_MAPPING}\n"), - call("xgboost==1.2.3\n"), - call("scipy==1.2.3\n"), - ] - mocked_writes.assert_has_calls(expected_calls) - - -def test_generate_requirements_txt_pruning_unused_packages(monkeypatch): - with patch("cloudpickle.load"), patch("tqdm.tqdm"), patch( - "sagemaker.serve.detector.pickle_dependencies.subprocess.run" - ) as subprocess_run, patch( - "sagemaker.serve.detector.pickle_dependencies.subprocess.Popen" - ) as subprocess_popen, patch( - "builtins.open" - ) as mocked_open, monkeypatch.context() as m: - mock_run_stdout = MagicMock() - mock_run_stdout.stdout = json.dumps(INSTALLED_PKG_JSON_UNUSED).encode("utf-8") - subprocess_run.return_value = mock_run_stdout - - mock_popen_stdout = MagicMock() - mock_popen_stdout.stdout = INSTALLED_PKG_READLINE_UNUSED - subprocess_popen.return_value = mock_popen_stdout - - m.setattr(sys, "modules", CURRENTLY_USED_FILES) - - open_dest_file = mock_open() - mocked_open.side_effect = [ - mock_open().return_value, - open_dest_file.return_value, - ] - - get_requirements_for_pkl_file("/path/to/serve.pkl", "path/to/requirements.txt") - - mocked_open.assert_any_call("/path/to/serve.pkl", mode="rb") - mocked_open.assert_any_call("path/to/requirements.txt", mode="w+") - - assert 2 == subprocess_run.call_count - subprocess_run_call = call( - EXPECTED_SUBPROCESS_CMD + ["list", "--format", "json"], - stdout=subprocess.PIPE, - check=True, - ) - subprocess_run.assert_has_calls([subprocess_run_call, subprocess_run_call]) - - subprocess_popen.assert_called_once_with( - EXPECTED_SUBPROCESS_CMD + ["show", "-f"] + INSTALLED_PKG_LST_WITH_UNUSED, - stdout=subprocess.PIPE, - ) - - mocked_writes = open_dest_file.return_value.__enter__().write - assert 4 == mocked_writes.call_count - - expected_calls = [ - call("sagemaker==1.2.3\n"), - call(f"{EXPECTED_BOTO3_MAPPING}\n"), - call("xgboost==1.2.3\n"), - call("scipy==1.2.3\n"), - ] - mocked_writes.assert_has_calls(expected_calls) - - -def test_generate_requirements_txt_no_currently_used_packages(monkeypatch): - with patch("cloudpickle.load"), patch("tqdm.tqdm"), patch( - "sagemaker.serve.detector.pickle_dependencies.subprocess.run" - ) as subprocess_run, patch( - "sagemaker.serve.detector.pickle_dependencies.subprocess.Popen" - ) as subprocess_popen, patch( - "builtins.open" - ) as mocked_open, monkeypatch.context() as m: - mock_run_stdout = MagicMock() - mock_run_stdout.stdout = json.dumps([]).encode("utf-8") - subprocess_run.return_value = mock_run_stdout - - mock_popen_stdout = MagicMock() - mock_popen_stdout.stdout = [] - subprocess_popen.return_value = mock_popen_stdout - - m.setattr(sys, "modules", CURRENTLY_USED_FILES) - - open_dest_file = mock_open() - mocked_open.side_effect = [ - mock_open().return_value, - open_dest_file.return_value, - ] - - get_requirements_for_pkl_file("/path/to/serve.pkl", "path/to/requirements.txt") - - mocked_open.assert_any_call("/path/to/serve.pkl", mode="rb") - mocked_open.assert_any_call("path/to/requirements.txt", mode="w+") - - assert 2 == subprocess_run.call_count - subprocess_run_call = call( - EXPECTED_SUBPROCESS_CMD + ["list", "--format", "json"], - stdout=subprocess.PIPE, - check=True, - ) - subprocess_run.assert_has_calls([subprocess_run_call, subprocess_run_call]) - - assert 0 == subprocess_popen.call_count - - mocked_writes = open_dest_file.return_value.__enter__().write - - assert 0 == mocked_writes.call_count