support and deprecate legacy installations

Author: ofek

MANIFEST.in

@@ -0,0 +1,15 @@
+recursive-include src/sagemaker *.py
+
+include src/sagemaker/image_uri_config/*.json
+include src/sagemaker/serve/schema/*.json
+include src/sagemaker/serve/requirements.txt
+recursive-include requirements *
+
+include VERSION
+include LICENSE.txt
+include README.rst
+
+prune tests
+
+recursive-exclude * __pycache__
+recursive-exclude * *.py[co]

hatch_build.py

@@ -1,25 +1,41 @@
+import os
 import sys
 
 from hatchling.metadata.plugin.interface import MetadataHookInterface
 
 
 class CustomMetadataHook(MetadataHookInterface):
     def update(self, metadata):
-        optional_dependencies = metadata["optional-dependencies"]
-        optional_dependencies["all"] = [
-            dependency
-            for optional_dependencies in optional_dependencies.values()
-            for dependency in optional_dependencies
-        ]
-        optional_dependencies["test"].extend(optional_dependencies["all"])
-
-        # remove torch and torchvision if python version is not 3.10/3.11
-        if sys.version_info.minor not in (10, 11):
-            optional_dependencies["test"] = list(
-                filter(
-                    lambda d: not d.startswith(
-                        ("sentencepiece", "transformers", "torch", "torchvision")
-                    ),
-                    optional_dependencies["test"],
-                )
+        metadata["optional-dependencies"] = get_optional_dependencies(self.root)
+
+
+def get_optional_dependencies(root):
+
+    def read_feature_deps(feature):
+        req_file = os.path.join(root, "requirements", "extras", f"{feature}_requirements.txt")
+        with open(req_file, encoding="utf-8") as f:
+            return list(filter(lambda d: not d.startswith("#"), f.read().splitlines()))
+
+    optional_dependencies = {"all": []}
+
+    for feature in ("feature-processor", "huggingface", "local", "scipy"):
+        dependencies = read_feature_deps(feature)
+        optional_dependencies[feature] = dependencies
+        optional_dependencies["all"].extend(dependencies)
+
+    # Test dependencies come last because we don't want them in `all`
+    optional_dependencies["test"] = read_feature_deps("test")
+    optional_dependencies["test"].extend(optional_dependencies["all"])
+
+    # remove torch and torchvision if python version is not 3.10/3.11
+    if sys.version_info.minor not in (10, 11):
+        optional_dependencies["test"] = list(
+            filter(
+                lambda d: not d.startswith(
+                    ("sentencepiece", "transformers", "torch", "torchvision")
+                ),
+                optional_dependencies["test"],
             )
+        )
+
+    return optional_dependencies

pyproject.toml

@@ -1,5 +1,5 @@
 [build-system]
-requires = ["hatchling", "hatch-requirements-txt>=0.3.0"]
+requires = ["hatchling"]
 build-backend = "hatchling.build"
 
 [project]
@@ -65,15 +65,8 @@ Homepage = "https://github.com/aws/sagemaker-python-sdk"
 path = "VERSION"
 pattern = "(?P<version>.+)"
 
-# Keep format of *_requirements.txt to be tracked by dependabot
-[tool.hatch.metadata.hooks.requirements_txt.optional-dependencies]
-feature-processor = ["requirements/extras/feature-processor_requirements.txt"]
-huggingface = ["requirements/extras/huggingface_requirements.txt"]
-local = ["requirements/extras/local_requirements.txt"]
-scipy = ["requirements/extras/scipy_requirements.txt"]
-test = ["requirements/extras/test_requirements.txt"]
-
-# Then do some postprocessing of optional dependencies
+# Dynamically define optional dependencies from requirements.txt files so
+# they can be be tracked by Dependabot
 [tool.hatch.metadata.hooks.custom]
 
 [tool.hatch.build.targets.wheel]

setup.py

@@ -0,0 +1,74 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#     http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+"""Placeholder docstring"""
+from __future__ import absolute_import
+
+import os
+import re
+import sys
+from ast import literal_eval
+from glob import glob
+from pathlib import Path
+
+from setuptools import find_packages, setup
+
+sys.stderr.write(
+    """
+===============================
+Unsupported installation method
+===============================
+
+This version of sagemaker no longer supports installation with `python setup.py install`.
+
+Please use `python -m pip install .` instead.
+"""
+)
+
+HERE = Path(__file__).parent.absolute()
+PYPROJECT = HERE.joinpath("pyproject.toml").read_text(encoding="utf-8")
+BUILD_SCRIPT = HERE.joinpath("hatch_build.py").read_text(encoding="utf-8")
+
+
+def get_dependencies():
+    pattern = r"^dependencies = (\[.*?\])$"
+    array = re.search(pattern, PYPROJECT, flags=re.MULTILINE | re.DOTALL).group(1)
+    return literal_eval(array)
+
+
+def get_optional_dependencies():
+    pattern = r"^def get_optional_dependencies.+"
+    function = re.search(pattern, BUILD_SCRIPT, flags=re.MULTILINE | re.DOTALL).group(0)
+    identifiers = {}
+    exec(function, None, identifiers)
+    return identifiers["get_optional_dependencies"](str(HERE))
+
+
+setup(
+    name="sagemaker",
+    version=HERE.joinpath("VERSION").read_text().strip(),
+    packages=find_packages("src"),
+    package_dir={"": "src"},
+    package_data={"": ["*.whl"]},
+    py_modules=[os.path.splitext(os.path.basename(path))[0] for path in glob("src/*.py")],
+    include_package_data=True,
+    install_requires=get_dependencies(),
+    extras_require=get_optional_dependencies(),
+    entry_points={
+        "console_scripts": [
+            "sagemaker-upgrade-v2=sagemaker.cli.compatibility.v2.sagemaker_upgrade_v2:main",
+        ]
+    },
+    scripts=[
+        "src/sagemaker/serve/model_server/triton/pack_conda_env.sh",
+    ],
+)