change: Update package metadata (#3529)

* change: Update package metadata

* address feedback

* support and deprecate legacy installations

* Update MANIFEST.in

* Update test_requirements.txt

* make tests faster

* apply correct formatting

* add required __future__ import

* Update tox.ini

* Update pyproject.toml

* address feedback

---------

Co-authored-by: Erick Benitez-Ramos <[email protected]>
Co-authored-by: SageMaker Bot <[email protected]>

Author: 3 people

MANIFEST.in

@@ -8,6 +8,7 @@ recursive-include requirements *
 include VERSION
 include LICENSE.txt
 include README.rst
+include hatch_build.py
 
 prune tests
 

hatch_build.py

@@ -0,0 +1,43 @@
+from __future__ import absolute_import
+
+import os
+import sys
+
+from hatchling.metadata.plugin.interface import MetadataHookInterface
+
+
+class CustomMetadataHook(MetadataHookInterface):
+    def update(self, metadata):
+        metadata["optional-dependencies"] = get_optional_dependencies(self.root)
+
+
+def get_optional_dependencies(root):
+
+    def read_feature_deps(feature):
+        req_file = os.path.join(root, "requirements", "extras", f"{feature}_requirements.txt")
+        with open(req_file, encoding="utf-8") as f:
+            return list(filter(lambda d: not d.startswith("#"), f.read().splitlines()))
+
+    optional_dependencies = {"all": []}
+
+    for feature in ("feature-processor", "huggingface", "local", "scipy"):
+        dependencies = read_feature_deps(feature)
+        optional_dependencies[feature] = dependencies
+        optional_dependencies["all"].extend(dependencies)
+
+    # Test dependencies come last because we don't want them in `all`
+    optional_dependencies["test"] = read_feature_deps("test")
+    optional_dependencies["test"].extend(optional_dependencies["all"])
+
+    # remove torch and torchvision if python version is not 3.10/3.11
+    if sys.version_info.minor not in (10, 11):
+        optional_dependencies["test"] = list(
+            filter(
+                lambda d: not d.startswith(
+                    ("sentencepiece", "transformers", "torch", "torchvision")
+                ),
+                optional_dependencies["test"],
+            )
+        )
+
+    return optional_dependencies

pyproject.toml

@@ -1,2 +1,90 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "sagemaker"
+dynamic = ["version", "optional-dependencies"]
+description = "Open source library for training and deploying models on Amazon SageMaker."
+readme = "README.rst"
+requires-python = ">=3.8"
+authors = [
+  { name = "Amazon Web Services" },
+]
+keywords = [
+  "AI",
+  "AWS",
+  "Amazon",
+  "ML",
+  "MXNet",
+  "Tensorflow",
+]
+classifiers = [
+  "Development Status :: 5 - Production/Stable",
+  "Intended Audience :: Developers",
+  "License :: OSI Approved :: Apache Software License",
+  "Natural Language :: English",
+  "Programming Language :: Python",
+  "Programming Language :: Python :: 3.8",
+  "Programming Language :: Python :: 3.9",
+  "Programming Language :: Python :: 3.10",
+  "Programming Language :: Python :: 3.11",
+]
+dependencies = [
+  "attrs>=23.1.0,<24",
+  "boto3>=1.34.142,<2.0",
+  "cloudpickle==2.2.1",
+  "docker",
+  "google-pasta",
+  "importlib-metadata>=1.4.0,<7.0",
+  "jsonschema",
+  "numpy>=1.9.0,<2.0",
+  "packaging>=20.0",
+  "pandas",
+  "pathos",
+  "platformdirs",
+  "protobuf>=3.12,<5.0",
+  "psutil",
+  "PyYAML~=6.0",
+  "requests",
+  "schema",
+  "smdebug_rulesconfig==1.0.1",
+  "tblib>=1.7.0,<4",
+  "tqdm",
+  "urllib3>=1.26.8,<3.0.0",
+]
+
+[project.scripts]
+sagemaker-upgrade-v2 = "sagemaker.cli.compatibility.v2.sagemaker_upgrade_v2:main"
+
+[project.urls]
+Homepage = "https://github.com/aws/sagemaker-python-sdk"
+
+[tool.hatch.version]
+path = "VERSION"
+pattern = "(?P<version>.+)"
+
+# Dynamically define optional dependencies from requirements.txt files so
+# they can be be tracked by Dependabot
+[tool.hatch.metadata.hooks.custom]
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/sagemaker"]
+exclude = ["src/sagemaker/serve/model_server/triton/pack_conda_env.sh"]
+
+[tool.hatch.build.targets.wheel.shared-scripts]
+"src/sagemaker/serve/model_server/triton/pack_conda_env.sh" = "pack_conda_env.sh"
+
+[tool.hatch.build.targets.sdist]
+only-include = [
+  "/requirements/extras",
+  "/src",
+  "/VERSION",
+]
+
+[tool.pytest.ini_options]
+addopts = ["-vv"]
+testpaths = ["tests"]
+
 [tool.black]
 line-length = 100

requirements/extras/test_requirements.txt

@@ -1,4 +1,5 @@
 tox==3.24.5
+build[virtualenv]==1.2.1
 flake8==4.0.1
 pytest==6.2.5
 pytest-cov==3.0.0

requirements/tox/twine_requirements.txt

@@ -1 +1,2 @@
+build==1.2.1
 twine==5.0.0

setup.cfg

@@ -14,122 +14,55 @@
 from __future__ import absolute_import
 
 import os
-from glob import glob
+import re
 import sys
+from ast import literal_eval
+from glob import glob
+from pathlib import Path
 
 from setuptools import find_packages, setup
 
-
-def read(fname):
+sys.stderr.write(
     """
-    Args:
-        fname:
-    """
-    return open(os.path.join(os.path.dirname(__file__), fname)).read()
-
+===============================
+Unsupported installation method
+===============================
 
-def read_version():
-    return read("VERSION").strip()
+This version of sagemaker no longer supports installation with `python setup.py install`.
 
+Please use `python -m pip install .` instead.
+"""
+)
 
-def read_requirements(filename):
-    """Reads requirements file which lists package dependencies.
-
-    Args:
-        filename: type(str) Relative file path of requirements.txt file
+HERE = Path(__file__).parent.absolute()
+PYPROJECT = HERE.joinpath("pyproject.toml").read_text(encoding="utf-8")
+BUILD_SCRIPT = HERE.joinpath("hatch_build.py").read_text(encoding="utf-8")
 
-    Returns:
-        list of dependencies extracted from file
-    """
-    with open(os.path.abspath(filename)) as fp:
-        deps = [line.strip() for line in fp.readlines()]
-    return deps
 
+def get_dependencies():
+    pattern = r"^dependencies = (\[.*?\])$"
+    array = re.search(pattern, PYPROJECT, flags=re.MULTILINE | re.DOTALL).group(1)
+    return literal_eval(array)
 
-# Declare minimal set for installation
-required_packages = [
-    "attrs>=23.1.0,<24",
-    "boto3>=1.34.142,<2.0",
-    "cloudpickle==2.2.1",
-    "google-pasta",
-    "numpy>=1.9.0,<2.0",
-    "protobuf>=3.12,<5.0",
-    "smdebug_rulesconfig==1.0.1",
-    "importlib-metadata>=1.4.0,<7.0",
-    "packaging>=20.0",
-    "pandas",
-    "pathos",
-    "schema",
-    "PyYAML~=6.0",
-    "jsonschema",
-    "platformdirs",
-    "tblib>=1.7.0,<4",
-    "urllib3>=1.26.8,<3.0.0",
-    "requests",
-    "docker",
-    "tqdm",
-    "psutil",
-]
 
-# Specific use case dependencies
-# Keep format of *_requirements.txt to be tracked by dependabot
-extras = {
-    "local": read_requirements("requirements/extras/local_requirements.txt"),
-    "scipy": read_requirements("requirements/extras/scipy_requirements.txt"),
-    "feature-processor": read_requirements(
-        "requirements/extras/feature-processor_requirements.txt"
-    ),
-    "huggingface": read_requirements("requirements/extras/huggingface_requirements.txt"),
-}
-# Meta dependency groups
-extras["all"] = [item for group in extras.values() for item in group]
-# Tests specific dependencies (do not need to be included in 'all')
-test_dependencies = read_requirements("requirements/extras/test_requirements.txt")
-# test dependencies are a superset of testing and extra dependencies
-test_dependencies.extend(extras["all"])
-# remove torch and torchvision if python version is not 3.10/3.11
-if sys.version_info.minor != 10 or sys.version_info.minor != 11:
-    test_dependencies = [
-        module
-        for module in test_dependencies
-        if not (
-            module.startswith("transformers")
-            or module.startswith("sentencepiece")
-            or module.startswith("torch")
-            or module.startswith("torchvision")
-        )
-    ]
+def get_optional_dependencies():
+    pattern = r"^def get_optional_dependencies.+"
+    function = re.search(pattern, BUILD_SCRIPT, flags=re.MULTILINE | re.DOTALL).group(0)
+    identifiers = {}
+    exec(function, None, identifiers)
+    return identifiers["get_optional_dependencies"](str(HERE))
 
-extras["test"] = (test_dependencies,)
 
 setup(
     name="sagemaker",
-    version=read_version(),
-    description="Open source library for training and deploying models on Amazon SageMaker.",
+    version=HERE.joinpath("VERSION").read_text().strip(),
     packages=find_packages("src"),
     package_dir={"": "src"},
     package_data={"": ["*.whl"]},
     py_modules=[os.path.splitext(os.path.basename(path))[0] for path in glob("src/*.py")],
     include_package_data=True,
-    long_description=read("README.rst"),
-    author="Amazon Web Services",
-    url="https://github.com/aws/sagemaker-python-sdk/",
-    license="Apache License 2.0",
-    keywords="ML Amazon AWS AI Tensorflow MXNet",
-    python_requires=">= 3.8",
-    classifiers=[
-        "Development Status :: 5 - Production/Stable",
-        "Intended Audience :: Developers",
-        "Natural Language :: English",
-        "License :: OSI Approved :: Apache Software License",
-        "Programming Language :: Python",
-        "Programming Language :: Python :: 3.8",
-        "Programming Language :: Python :: 3.9",
-        "Programming Language :: Python :: 3.10",
-        "Programming Language :: Python :: 3.11",
-    ],
-    install_requires=required_packages,
-    extras_require=extras,
+    install_requires=get_dependencies(),
+    extras_require=get_optional_dependencies(),
     entry_points={
         "console_scripts": [
             "sagemaker-upgrade-v2=sagemaker.cli.compatibility.v2.sagemaker_upgrade_v2:main",

setup.py

@@ -68,7 +68,7 @@ def __init__(
         encrypt_inter_container_traffic: Union[bool, PipelineVariable] = False,
         use_spot_instances: Union[bool, PipelineVariable] = False,
         max_wait: Optional[Union[int, PipelineVariable]] = None,
-        **kwargs  # pylint: disable=W0613
+        **kwargs,  # pylint: disable=W0613
     ):
         """Initialize an ``AlgorithmEstimator`` instance.
 
@@ -271,7 +271,7 @@ def create_model(
         serializer=IdentitySerializer(),
         deserializer=BytesDeserializer(),
         vpc_config_override=vpc_utils.VPC_CONFIG_DEFAULT,
-        **kwargs
+        **kwargs,
     ):
         """Create a model to deploy.
 
@@ -325,7 +325,7 @@ def predict_wrapper(endpoint, session):
             vpc_config=self.get_vpc_config(vpc_config_override),
             sagemaker_session=self.sagemaker_session,
             predictor_cls=predictor_cls,
-            **kwargs
+            **kwargs,
         )
 
     def transformer(

src/sagemaker/algorithm.py

@@ -55,7 +55,7 @@ def __init__(
         instance_type: Optional[Union[str, PipelineVariable]] = None,
         data_location: Optional[str] = None,
         enable_network_isolation: Union[bool, PipelineVariable] = False,
-        **kwargs
+        **kwargs,
     ):
         """Initialize an AmazonAlgorithmEstimatorBase.
 
@@ -91,7 +91,7 @@ def __init__(
             instance_count,
             instance_type,
             enable_network_isolation=enable_network_isolation,
-            **kwargs
+            **kwargs,
         )
 
         data_location = data_location or (

src/sagemaker/amazon/amazon_estimator.py

@@ -114,7 +114,7 @@ def __init__(
         factors_init_scale: Optional[float] = None,
         factors_init_sigma: Optional[float] = None,
         factors_init_value: Optional[float] = None,
-        **kwargs
+        **kwargs,
     ):
         """Factorization Machines is :class:`Estimator` for general-purpose supervised learning.
 
@@ -266,7 +266,7 @@ def create_model(self, vpc_config_override=VPC_CONFIG_DEFAULT, **kwargs):
             self.role,
             sagemaker_session=self.sagemaker_session,
             vpc_config=self.get_vpc_config(vpc_config_override),
-            **kwargs
+            **kwargs,
         )
 
 
@@ -332,7 +332,7 @@ def __init__(
         model_data: Union[str, PipelineVariable],
         role: Optional[str] = None,
         sagemaker_session: Optional[Session] = None,
-        **kwargs
+        **kwargs,
     ):
         """Initialization for FactorizationMachinesModel class.
 
@@ -365,5 +365,5 @@ def __init__(
             role,
             predictor_cls=FactorizationMachinesPredictor,
             sagemaker_session=sagemaker_session,
-            **kwargs
+            **kwargs,
         )