Merge branch 'master' into fastapi-inprocess

Author: bryannahm1

.github/workflows/codeql.yml

@@ -0,0 +1,35 @@
+name: "CodeQL"
+on:
+  push:
+    branches: [ "master" ]
+  pull_request:
+    branches: [ "master" ]
+  schedule:
+    - cron: '30 8 * * *'
+jobs:
+  analyze:
+    name: Analyze (${{ matrix.language }})
+    runs-on: ${{ 'ubuntu-latest' }}
+    permissions:
+      security-events: write
+      packages: read
+
+    strategy:
+      matrix:
+        include:
+          - language: python
+            build-mode: none
+          - language: java-kotlin
+            build-mode: none
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@6ccd57f4c5d15bdc2fef309bd9fb6cc9db2ef1c6
+      - name: Initialize CodeQL
+        uses: github/codeql-action/init@4b1d7da102ff94aca014c0245062b1a463356d72
+        with:
+          languages: ${{ matrix.language }}
+          build-mode: ${{ matrix.build-mode }}
+      - name: Perform CodeQL Analysis
+        uses: github/codeql-action/analyze@4b1d7da102ff94aca014c0245062b1a463356d72
+        with:
+          category: "/language:${{matrix.language}}"

CHANGELOG.md

@@ -1,5 +1,40 @@
 # Changelog
 
+## v2.227.0 (2024-07-30)
+
+### Features
+
+ * added code scanning through CodeQL
+
+### Bug Fixes and Other Changes
+
+ * Fixed cpu isntance type for the estimator register test
+ * update image_uri_configs  07-29-2024 11:28:28 PST
+ * avoid AccessDenied error for a while on SageMaker Studio wtih do…
+ * SMP PT 2.3 Fix
+ * chore: pin framework version in serverless inference tests
+ * image uri in TGI 2.2.0 image
+ * explicitly access enum member values to avoid Python version related regression
+ * chore: add huggingface TGI 2.2.0 config
+ * update image_uri_configs  07-22-2024 11:53:54 PST
+ * update image_uri_configs  07-17-2024 07:17:38 PST
+ * update image_uri_configs  07-16-2024 07:17:45 PST
+ * add support for new regions
+
+## v2.226.1 (2024-07-17)
+
+## v2.226.0 (2024-07-12)
+
+### Features
+
+ * Curated hub improvements
+ * InferenceSpec support for MMS and testing
+
+### Bug Fixes and Other Changes
+
+ * ModelBuilder not passing HF_TOKEN to model.
+ * update image_uri_configs  07-10-2024 07:18:04 PST
+
 ## v2.225.0 (2024-07-10)
 
 ### Features

MANIFEST.in

@@ -8,6 +8,7 @@ recursive-include requirements *
 include VERSION
 include LICENSE.txt
 include README.rst
+include hatch_build.py
 
 prune tests
 

VERSION

@@ -1 +1 @@
-2.225.1.dev0
+2.227.1.dev0

hatch_build.py

@@ -0,0 +1,43 @@
+from __future__ import absolute_import
+
+import os
+import sys
+
+from hatchling.metadata.plugin.interface import MetadataHookInterface
+
+
+class CustomMetadataHook(MetadataHookInterface):
+    def update(self, metadata):
+        metadata["optional-dependencies"] = get_optional_dependencies(self.root)
+
+
+def get_optional_dependencies(root):
+
+    def read_feature_deps(feature):
+        req_file = os.path.join(root, "requirements", "extras", f"{feature}_requirements.txt")
+        with open(req_file, encoding="utf-8") as f:
+            return list(filter(lambda d: not d.startswith("#"), f.read().splitlines()))
+
+    optional_dependencies = {"all": []}
+
+    for feature in ("feature-processor", "huggingface", "local", "scipy"):
+        dependencies = read_feature_deps(feature)
+        optional_dependencies[feature] = dependencies
+        optional_dependencies["all"].extend(dependencies)
+
+    # Test dependencies come last because we don't want them in `all`
+    optional_dependencies["test"] = read_feature_deps("test")
+    optional_dependencies["test"].extend(optional_dependencies["all"])
+
+    # remove torch and torchvision if python version is not 3.10/3.11
+    if sys.version_info.minor not in (10, 11):
+        optional_dependencies["test"] = list(
+            filter(
+                lambda d: not d.startswith(
+                    ("sentencepiece", "transformers", "torch", "torchvision")
+                ),
+                optional_dependencies["test"],
+            )
+        )
+
+    return optional_dependencies

pyproject.toml

@@ -1,2 +1,90 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "sagemaker"
+dynamic = ["version", "optional-dependencies"]
+description = "Open source library for training and deploying models on Amazon SageMaker."
+readme = "README.rst"
+requires-python = ">=3.8"
+authors = [
+  { name = "Amazon Web Services" },
+]
+keywords = [
+  "AI",
+  "AWS",
+  "Amazon",
+  "ML",
+  "MXNet",
+  "Tensorflow",
+]
+classifiers = [
+  "Development Status :: 5 - Production/Stable",
+  "Intended Audience :: Developers",
+  "License :: OSI Approved :: Apache Software License",
+  "Natural Language :: English",
+  "Programming Language :: Python",
+  "Programming Language :: Python :: 3.8",
+  "Programming Language :: Python :: 3.9",
+  "Programming Language :: Python :: 3.10",
+  "Programming Language :: Python :: 3.11",
+]
+dependencies = [
+  "attrs>=23.1.0,<24",
+  "boto3>=1.34.142,<2.0",
+  "cloudpickle==2.2.1",
+  "docker",
+  "google-pasta",
+  "importlib-metadata>=1.4.0,<7.0",
+  "jsonschema",
+  "numpy>=1.9.0,<2.0",
+  "packaging>=20.0",
+  "pandas",
+  "pathos",
+  "platformdirs",
+  "protobuf>=3.12,<5.0",
+  "psutil",
+  "PyYAML~=6.0",
+  "requests",
+  "schema",
+  "smdebug_rulesconfig==1.0.1",
+  "tblib>=1.7.0,<4",
+  "tqdm",
+  "urllib3>=1.26.8,<3.0.0",
+]
+
+[project.scripts]
+sagemaker-upgrade-v2 = "sagemaker.cli.compatibility.v2.sagemaker_upgrade_v2:main"
+
+[project.urls]
+Homepage = "https://github.com/aws/sagemaker-python-sdk"
+
+[tool.hatch.version]
+path = "VERSION"
+pattern = "(?P<version>.+)"
+
+# Dynamically define optional dependencies from requirements.txt files so
+# they can be be tracked by Dependabot
+[tool.hatch.metadata.hooks.custom]
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/sagemaker"]
+exclude = ["src/sagemaker/serve/model_server/triton/pack_conda_env.sh"]
+
+[tool.hatch.build.targets.wheel.shared-scripts]
+"src/sagemaker/serve/model_server/triton/pack_conda_env.sh" = "pack_conda_env.sh"
+
+[tool.hatch.build.targets.sdist]
+only-include = [
+  "/requirements/extras",
+  "/src",
+  "/VERSION",
+]
+
+[tool.pytest.ini_options]
+addopts = ["-vv"]
+testpaths = ["tests"]
+
 [tool.black]
 line-length = 100

requirements/extras/test_requirements.txt

@@ -1,4 +1,5 @@
 tox==3.24.5
+build[virtualenv]==1.2.1
 flake8==4.0.1
 pytest==6.2.5
 pytest-cov==3.0.0

requirements/tox/twine_requirements.txt

@@ -1 +1,2 @@
+build==1.2.1
 twine==5.0.0

setup.cfg

@@ -14,122 +14,55 @@
 from __future__ import absolute_import
 
 import os
-from glob import glob
+import re
 import sys
+from ast import literal_eval
+from glob import glob
+from pathlib import Path
 
 from setuptools import find_packages, setup
 
-
-def read(fname):
+sys.stderr.write(
     """
-    Args:
-        fname:
-    """
-    return open(os.path.join(os.path.dirname(__file__), fname)).read()
-
+===============================
+Unsupported installation method
+===============================
 
-def read_version():
-    return read("VERSION").strip()
+This version of sagemaker no longer supports installation with `python setup.py install`.
 
+Please use `python -m pip install .` instead.
+"""
+)
 
-def read_requirements(filename):
-    """Reads requirements file which lists package dependencies.
-
-    Args:
-        filename: type(str) Relative file path of requirements.txt file
+HERE = Path(__file__).parent.absolute()
+PYPROJECT = HERE.joinpath("pyproject.toml").read_text(encoding="utf-8")
+BUILD_SCRIPT = HERE.joinpath("hatch_build.py").read_text(encoding="utf-8")
 
-    Returns:
-        list of dependencies extracted from file
-    """
-    with open(os.path.abspath(filename)) as fp:
-        deps = [line.strip() for line in fp.readlines()]
-    return deps
 
+def get_dependencies():
+    pattern = r"^dependencies = (\[.*?\])$"
+    array = re.search(pattern, PYPROJECT, flags=re.MULTILINE | re.DOTALL).group(1)
+    return literal_eval(array)
 
-# Declare minimal set for installation
-required_packages = [
-    "attrs>=23.1.0,<24",
-    "boto3>=1.34.142,<2.0",
-    "cloudpickle==2.2.1",
-    "google-pasta",
-    "numpy>=1.9.0,<2.0",
-    "protobuf>=3.12,<5.0",
-    "smdebug_rulesconfig==1.0.1",
-    "importlib-metadata>=1.4.0,<7.0",
-    "packaging>=20.0",
-    "pandas",
-    "pathos",
-    "schema",
-    "PyYAML~=6.0",
-    "jsonschema",
-    "platformdirs",
-    "tblib>=1.7.0,<4",
-    "urllib3>=1.26.8,<3.0.0",
-    "requests",
-    "docker",
-    "tqdm",
-    "psutil",
-]
 
-# Specific use case dependencies
-# Keep format of *_requirements.txt to be tracked by dependabot
-extras = {
-    "local": read_requirements("requirements/extras/local_requirements.txt"),
-    "scipy": read_requirements("requirements/extras/scipy_requirements.txt"),
-    "feature-processor": read_requirements(
-        "requirements/extras/feature-processor_requirements.txt"
-    ),
-    "huggingface": read_requirements("requirements/extras/huggingface_requirements.txt"),
-}
-# Meta dependency groups
-extras["all"] = [item for group in extras.values() for item in group]
-# Tests specific dependencies (do not need to be included in 'all')
-test_dependencies = read_requirements("requirements/extras/test_requirements.txt")
-# test dependencies are a superset of testing and extra dependencies
-test_dependencies.extend(extras["all"])
-# remove torch and torchvision if python version is not 3.10/3.11
-if sys.version_info.minor != 10 or sys.version_info.minor != 11:
-    test_dependencies = [
-        module
-        for module in test_dependencies
-        if not (
-            module.startswith("transformers")
-            or module.startswith("sentencepiece")
-            or module.startswith("torch")
-            or module.startswith("torchvision")
-        )
-    ]
+def get_optional_dependencies():
+    pattern = r"^def get_optional_dependencies.+"
+    function = re.search(pattern, BUILD_SCRIPT, flags=re.MULTILINE | re.DOTALL).group(0)
+    identifiers = {}
+    exec(function, None, identifiers)
+    return identifiers["get_optional_dependencies"](str(HERE))
 
-extras["test"] = (test_dependencies,)
 
 setup(
     name="sagemaker",
-    version=read_version(),
-    description="Open source library for training and deploying models on Amazon SageMaker.",
+    version=HERE.joinpath("VERSION").read_text().strip(),
     packages=find_packages("src"),
     package_dir={"": "src"},
     package_data={"": ["*.whl"]},
     py_modules=[os.path.splitext(os.path.basename(path))[0] for path in glob("src/*.py")],
     include_package_data=True,
-    long_description=read("README.rst"),
-    author="Amazon Web Services",
-    url="https://github.com/aws/sagemaker-python-sdk/",
-    license="Apache License 2.0",
-    keywords="ML Amazon AWS AI Tensorflow MXNet",
-    python_requires=">= 3.8",
-    classifiers=[
-        "Development Status :: 5 - Production/Stable",
-        "Intended Audience :: Developers",
-        "Natural Language :: English",
-        "License :: OSI Approved :: Apache Software License",
-        "Programming Language :: Python",
-        "Programming Language :: Python :: 3.8",
-        "Programming Language :: Python :: 3.9",
-        "Programming Language :: Python :: 3.10",
-        "Programming Language :: Python :: 3.11",
-    ],
-    install_requires=required_packages,
-    extras_require=extras,
+    install_requires=get_dependencies(),
+    extras_require=get_optional_dependencies(),
     entry_points={
         "console_scripts": [
             "sagemaker-upgrade-v2=sagemaker.cli.compatibility.v2.sagemaker_upgrade_v2:main",