Merge branch 'master' into hot-fix

Author: makungaj1

CHANGELOG.md

@@ -1,5 +1,17 @@
 # Changelog
 
+## v2.225.0 (2024-07-10)
+
+### Features
+
+ * model optimization
+
+### Bug Fixes and Other Changes
+
+ * fix integ test
+ * update uris for v1.1.1
+ * update image_uri_configs  07-04-2024 07:17:24 PST
+
 ## v2.224.4 (2024-07-04)
 
 ### Bug Fixes and Other Changes

VERSION

@@ -1 +1 @@
-2.224.5.dev0
+2.225.1.dev0

src/sagemaker/image_uri_config/tensorflow.json

@@ -2195,7 +2195,8 @@
             "2.9": "2.9.1",
             "2.12": "2.12.1",
             "2.13": "2.13.0",
-            "2.14": "2.14.1"
+            "2.14": "2.14.1",
+            "2.16": "2.16.1"
         },
         "versions": {
             "2.9.1": {
@@ -2381,6 +2382,50 @@
                     "us-west-2": "763104351884"
                 },
                 "repository": "tensorflow-inference-graviton"
+            },
+            "2.16.1": {
+                "container_version": {
+                    "cpu": "ubuntu20.04"
+                },
+                "py_versions": [
+                    "py310"
+                ],
+                "registries": {
+                    "af-south-1": "626614931356",
+                    "ap-east-1": "871362719292",
+                    "ap-northeast-1": "763104351884",
+                    "ap-northeast-2": "763104351884",
+                    "ap-northeast-3": "364406365360",
+                    "ap-south-1": "763104351884",
+                    "ap-south-2": "772153158452",
+                    "ap-southeast-1": "763104351884",
+                    "ap-southeast-2": "763104351884",
+                    "ap-southeast-3": "907027046896",
+                    "ap-southeast-4": "457447274322",
+                    "ca-central-1": "763104351884",
+                    "ca-west-1": "204538143572",
+                    "cn-north-1": "727897471807",
+                    "cn-northwest-1": "727897471807",
+                    "eu-central-1": "763104351884",
+                    "eu-central-2": "380420809688",
+                    "eu-north-1": "763104351884",
+                    "eu-south-1": "692866216735",
+                    "eu-south-2": "503227376785",
+                    "eu-west-1": "763104351884",
+                    "eu-west-2": "763104351884",
+                    "eu-west-3": "763104351884",
+                    "il-central-1": "780543022126",
+                    "me-central-1": "914824155844",
+                    "me-south-1": "217643126080",
+                    "sa-east-1": "763104351884",
+                    "us-east-1": "763104351884",
+                    "us-east-2": "763104351884",
+                    "us-gov-east-1": "446045086412",
+                    "us-gov-west-1": "442386744353",
+                    "us-west-1": "763104351884",
+                    "us-west-2": "763104351884"
+                },
+                "repository": "tensorflow-inference-graviton"
             }
         }
     },

src/sagemaker/serve/builder/transformers_builder.py

@@ -13,8 +13,10 @@
 """Transformers build logic with model builder"""
 from __future__ import absolute_import
 import logging
+import os
 from abc import ABC, abstractmethod
 from typing import Type
+from pathlib import Path
 from packaging.version import Version
 
 from sagemaker.model import Model
@@ -26,7 +28,12 @@
 from sagemaker.huggingface import HuggingFaceModel
 from sagemaker.serve.model_server.multi_model_server.prepare import (
     _create_dir_structure,
+    prepare_for_mms,
 )
+from sagemaker.serve.detector.image_detector import (
+    auto_detect_container,
+)
+from sagemaker.serve.detector.pickler import save_pkl
 from sagemaker.serve.utils.optimize_utils import _is_optimized
 from sagemaker.serve.utils.predictors import TransformersLocalModePredictor
 from sagemaker.serve.utils.types import ModelServer
@@ -73,6 +80,8 @@ def __init__(self):
         self.pytorch_version = None
         self.instance_type = None
         self.schema_builder = None
+        self.inference_spec = None
+        self.shared_libs = None
 
     @abstractmethod
     def _prepare_for_mode(self):
@@ -110,7 +119,7 @@ def _get_hf_metadata_create_model(self) -> Type[Model]:
         """
 
         hf_model_md = get_huggingface_model_metadata(
-            self.model, self.env_vars.get("HUGGING_FACE_HUB_TOKEN")
+            self.env_vars.get("HF_MODEL_ID"), self.env_vars.get("HUGGING_FACE_HUB_TOKEN")
         )
         hf_config = image_uris.config_for_framework("huggingface").get("inference")
         config = hf_config["versions"]
@@ -245,18 +254,22 @@ def _build_transformers_env(self):
 
         _create_dir_structure(self.model_path)
         if not hasattr(self, "pysdk_model"):
-            self.env_vars.update({"HF_MODEL_ID": self.model})
+
+            if self.inference_spec is not None:
+                self.env_vars.update({"HF_MODEL_ID": self.inference_spec.get_model()})
+            else:
+                self.env_vars.update({"HF_MODEL_ID": self.model})
 
             logger.info(self.env_vars)
 
             # TODO: Move to a helper function
             if hasattr(self.env_vars, "HF_API_TOKEN"):
                 self.hf_model_config = _get_model_config_properties_from_hf(
-                    self.model, self.env_vars.get("HF_API_TOKEN")
+                    self.env_vars.get("HF_MODEL_ID"), self.env_vars.get("HF_API_TOKEN")
                 )
             else:
                 self.hf_model_config = _get_model_config_properties_from_hf(
-                    self.model, self.env_vars.get("HUGGING_FACE_HUB_TOKEN")
+                    self.env_vars.get("HF_MODEL_ID"), self.env_vars.get("HUGGING_FACE_HUB_TOKEN")
                 )
 
         self.pysdk_model = self._create_transformers_model()
@@ -292,6 +305,42 @@ def _get_supported_version(self, hf_config, hugging_face_version, base_fw):
                 versions_to_return.append(base_fw_version)
         return sorted(versions_to_return, reverse=True)[0]
 
+    def _auto_detect_container(self):
+        """Set image_uri by detecting container via model name or inference spec"""
+        # Auto detect the container image uri
+        if self.image_uri:
+            logger.info(
+                "Skipping auto detection as the image uri is provided %s",
+                self.image_uri,
+            )
+            return
+
+        if self.model:
+            logger.info(
+                "Auto detect container url for the provided model and on instance %s",
+                self.instance_type,
+            )
+            self.image_uri = auto_detect_container(
+                self.model, self.sagemaker_session.boto_region_name, self.instance_type
+            )
+
+        elif self.inference_spec:
+            # TODO: this won't work for larger image.
+            # Fail and let the customer include the image uri
+            logger.warning(
+                "model_path provided with no image_uri. Attempting to autodetect the image\
+                    by loading the model using inference_spec.load()..."
+            )
+            self.image_uri = auto_detect_container(
+                self.inference_spec.load(self.model_path),
+                self.sagemaker_session.boto_region_name,
+                self.instance_type,
+            )
+        else:
+            raise ValueError(
+                "Cannot detect and set image_uri. Please pass model or inference spec."
+            )
+
     def _build_for_transformers(self):
         """Method that triggers model build
 
@@ -300,6 +349,26 @@ def _build_for_transformers(self):
         self.secret_key = None
         self.model_server = ModelServer.MMS
 
+        if self.inference_spec:
+
+            os.makedirs(self.model_path, exist_ok=True)
+
+            code_path = Path(self.model_path).joinpath("code")
+
+            save_pkl(code_path, (self.inference_spec, self.schema_builder))
+            logger.info("PKL file saved to file: %s", code_path)
+
+            self._auto_detect_container()
+
+            self.secret_key = prepare_for_mms(
+                model_path=self.model_path,
+                shared_libs=self.shared_libs,
+                dependencies=self.dependencies,
+                session=self.sagemaker_session,
+                image_uri=self.image_uri,
+                inference_spec=self.inference_spec,
+            )
+
         self._build_transformers_env()
 
         if self.role_arn:

src/sagemaker/serve/mode/sagemaker_endpoint_mode.py

@@ -130,6 +130,7 @@ def prepare(
                 model_path=model_path,
                 sagemaker_session=sagemaker_session,
                 s3_model_data_url=s3_model_data_url,
+                secret_key=secret_key,
                 image=image,
                 should_upload_artifacts=should_upload_artifacts,
             )

src/sagemaker/serve/model_server/multi_model_server/inference.py

@@ -0,0 +1,100 @@
+"""This module is for SageMaker inference.py."""
+
+from __future__ import absolute_import
+import os
+import io
+import cloudpickle
+import shutil
+import platform
+from pathlib import Path
+from functools import partial
+from sagemaker.serve.spec.inference_spec import InferenceSpec
+from sagemaker.serve.validations.check_integrity import perform_integrity_check
+import logging
+
+logger = logging.getLogger(__name__)
+
+inference_spec = None
+schema_builder = None
+SHARED_LIBS_DIR = Path(__file__).parent.parent.joinpath("shared_libs")
+SERVE_PATH = Path(__file__).parent.joinpath("serve.pkl")
+METADATA_PATH = Path(__file__).parent.joinpath("metadata.json")
+
+
+def model_fn(model_dir):
+    """Overrides default method for loading a model"""
+    shared_libs_path = Path(model_dir + "/shared_libs")
+
+    if shared_libs_path.exists():
+        # before importing, place dynamic linked libraries in shared lib path
+        shutil.copytree(shared_libs_path, "/lib", dirs_exist_ok=True)
+
+    serve_path = Path(__file__).parent.joinpath("serve.pkl")
+    with open(str(serve_path), mode="rb") as file:
+        global inference_spec, schema_builder
+        obj = cloudpickle.load(file)
+        if isinstance(obj[0], InferenceSpec):
+            inference_spec, schema_builder = obj
+
+    if inference_spec:
+        return partial(inference_spec.invoke, model=inference_spec.load(model_dir))
+
+
+def input_fn(input_data, content_type):
+    """Deserializes the bytes that were received from the model server"""
+    try:
+        if hasattr(schema_builder, "custom_input_translator"):
+            return schema_builder.custom_input_translator.deserialize(
+                io.BytesIO(input_data), content_type
+            )
+        else:
+            return schema_builder.input_deserializer.deserialize(
+                io.BytesIO(input_data), content_type[0]
+            )
+    except Exception as e:
+        logger.error("Encountered error: %s in deserialize_response." % e)
+        raise Exception("Encountered error in deserialize_request.") from e
+
+
+def predict_fn(input_data, predict_callable):
+    """Invokes the model that is taken in by model server"""
+    return predict_callable(input_data)
+
+
+def output_fn(predictions, accept_type):
+    """Prediction is serialized to bytes and sent back to the customer"""
+    try:
+        if hasattr(schema_builder, "custom_output_translator"):
+            return schema_builder.custom_output_translator.serialize(predictions, accept_type)
+        else:
+            return schema_builder.output_serializer.serialize(predictions)
+    except Exception as e:
+        logger.error("Encountered error: %s in serialize_response." % e)
+        raise Exception("Encountered error in serialize_response.") from e
+
+
+def _run_preflight_diagnostics():
+    _py_vs_parity_check()
+    _pickle_file_integrity_check()
+
+
+def _py_vs_parity_check():
+    container_py_vs = platform.python_version()
+    local_py_vs = os.getenv("LOCAL_PYTHON")
+
+    if not local_py_vs or container_py_vs.split(".")[1] != local_py_vs.split(".")[1]:
+        logger.warning(
+            f"The local python version {local_py_vs} differs from the python version "
+            f"{container_py_vs} on the container. Please align the two to avoid unexpected behavior"
+        )
+
+
+def _pickle_file_integrity_check():
+    with open(SERVE_PATH, "rb") as f:
+        buffer = f.read()
+
+    perform_integrity_check(buffer=buffer, metadata_path=METADATA_PATH)
+
+
+# on import, execute
+_run_preflight_diagnostics()