mb_inprocess updates

Author: Bryannah Hernandez

src/sagemaker/serve/builder/transformers_builder.py

@@ -13,10 +13,13 @@
 """Transformers build logic with model builder"""
 from __future__ import absolute_import
 import logging
+import os
 from abc import ABC, abstractmethod
 from typing import Type
 from packaging.version import Version
 
+from pathlib import Path
+
 from sagemaker.model import Model
 from sagemaker import image_uris
 from sagemaker.serve.utils.local_hardware import (
@@ -26,7 +29,12 @@
 from sagemaker.huggingface import HuggingFaceModel
 from sagemaker.serve.model_server.multi_model_server.prepare import (
     _create_dir_structure,
+    prepare_for_mms
+)
+from sagemaker.serve.detector.image_detector import (
+    auto_detect_container,
 )
+from sagemaker.serve.detector.pickler import save_pkl
 from sagemaker.serve.utils.predictors import TransformersLocalModePredictor
 from sagemaker.serve.utils.types import ModelServer
 from sagemaker.serve.mode.function_pointers import Mode
@@ -73,6 +81,7 @@ def __init__(self):
         self.instance_type = None
         self.schema_builder = None
         self.inference_spec = None
+        self.shared_libs = None
 
     @abstractmethod
     def _prepare_for_mode(self):
@@ -298,6 +307,40 @@ def _get_supported_version(self, hf_config, hugging_face_version, base_fw):
                 versions_to_return.append(base_fw_version)
         return sorted(versions_to_return, reverse=True)[0]
 
+    def _auto_detect_container(self):
+        """Placeholder docstring"""
+        # Auto detect the container image uri
+        if self.image_uri:
+            logger.info(
+                "Skipping auto detection as the image uri is provided %s",
+                self.image_uri,
+            )
+            return
+
+        if self.model:
+            logger.info(
+                "Auto detect container url for the provided model and on instance %s",
+                self.instance_type,
+            )
+            self.image_uri = auto_detect_container(
+                self.model, self.sagemaker_session.boto_region_name, self.instance_type
+            )
+
+        elif self.inference_spec:
+            # TODO: this won't work for larger image.
+            # Fail and let the customer include the image uri
+            logger.warning(
+                "model_path provided with no image_uri. Attempting to autodetect the image\
+                    by loading the model using inference_spec.load()..."
+            )
+            self.image_uri = auto_detect_container(
+                self.inference_spec.load(self.model_path),
+                self.sagemaker_session.boto_region_name,
+                self.instance_type,
+            )
+        else:
+            raise ValueError("Cannot detect required model or inference spec")
+        
     def _build_for_transformers(self):
         """Method that triggers model build
 
@@ -306,6 +349,26 @@ def _build_for_transformers(self):
         self.secret_key = None
         self.model_server = ModelServer.MMS
 
+        if not os.path.exists(self.model_path):
+            os.makedirs(self.model_path)
+
+        code_path = Path(self.model_path).joinpath("code")
+        # save the model or inference spec in cloud pickle format
+        if self.inference_spec:
+            save_pkl(code_path, (self.inference_spec, self.schema_builder))
+            logger.info("PKL file saved to file: {}".format(code_path))
+
+        self._auto_detect_container()
+
+        self.secret_key = prepare_for_mms(
+            model_path=self.model_path,
+            shared_libs=self.shared_libs,
+            dependencies=self.dependencies,
+            session=self.sagemaker_session,
+            image_uri=self.image_uri,
+            inference_spec=self.inference_spec,
+        )
+
         self._build_transformers_env()
 
         return self.pysdk_model

src/sagemaker/serve/model_server/multi_model_server/inference.py

@@ -1,18 +1,24 @@
 """This module is for SageMaker inference.py."""
 
 from __future__ import absolute_import
+import os
 import io
 import cloudpickle
 import shutil
+import platform
 from pathlib import Path
 from functools import partial
 from sagemaker.serve.spec.inference_spec import InferenceSpec
+from sagemaker.serve.validations.check_integrity import perform_integrity_check
 import logging
 
 logger = logging.getLogger(__name__)
 
 inference_spec = None
 schema_builder = None
+SHARED_LIBS_DIR = Path(__file__).parent.parent.joinpath("shared_libs")
+SERVE_PATH = Path(__file__).parent.joinpath("serve.pkl")
+METADATA_PATH = Path(__file__).parent.joinpath("metadata.json")
 
 
 def model_fn(model_dir):
@@ -27,9 +33,11 @@ def model_fn(model_dir):
     with open(str(serve_path), mode="rb") as file:
         global inference_spec, schema_builder
         obj = cloudpickle.load(file)
-        if isinstance(obj[0], InferenceSpec): 
+        if isinstance(obj[0], InferenceSpec):
             inference_spec, schema_builder = obj
-            
+
+    logger.info("in model_fn")
+
     if inference_spec:
         return partial(inference_spec.invoke, model=inference_spec.load(model_dir))
 
@@ -46,11 +54,13 @@ def input_fn(input_data, content_type):
                 io.BytesIO(input_data), content_type[0]
             )
     except Exception as e:
+        logger.error("Encountered error: %s in deserialize_response." % e)
         raise Exception("Encountered error in deserialize_request.") from e
 
 
 def predict_fn(input_data, predict_callable):
     """Placeholder docstring"""
+    logger.info("in predict_fn")
     return predict_callable(input_data)
 
 
@@ -66,3 +76,28 @@ def output_fn(predictions, accept_type):
         raise Exception("Encountered error in serialize_response.") from e
 
 
+def _run_preflight_diagnostics():
+    _py_vs_parity_check()
+    _pickle_file_integrity_check()
+
+
+def _py_vs_parity_check():
+    container_py_vs = platform.python_version()
+    local_py_vs = os.getenv("LOCAL_PYTHON")
+
+    if not local_py_vs or container_py_vs.split(".")[1] != local_py_vs.split(".")[1]:
+        logger.warning(
+            f"The local python version {local_py_vs} differs from the python version "
+            f"{container_py_vs} on the container. Please align the two to avoid unexpected behavior"
+        )
+
+
+def _pickle_file_integrity_check():
+    with open(SERVE_PATH, "rb") as f:
+        buffer = f.read()
+
+    perform_integrity_check(buffer=buffer, metadata_path=METADATA_PATH)
+
+
+# on import, execute
+_run_preflight_diagnostics()

src/sagemaker/serve/model_server/multi_model_server/prepare.py

@@ -14,12 +14,23 @@
 
 from __future__ import absolute_import
 import logging
-from pathlib import Path
-from typing import List
 
 from sagemaker.serve.model_server.tgi.prepare import _copy_jumpstart_artifacts
 from sagemaker.serve.utils.local_hardware import _check_disk_space, _check_docker_disk_usage
 
+from pathlib import Path
+import shutil
+from typing import List
+
+from sagemaker.session import Session
+from sagemaker.serve.spec.inference_spec import InferenceSpec
+from sagemaker.serve.detector.dependency_manager import capture_dependencies
+from sagemaker.serve.validations.check_integrity import (
+    generate_secret_key,
+    compute_hash,
+)
+from sagemaker.remote_function.core.serialization import _MetaData
+
 logger = logging.getLogger(__name__)
 
 
@@ -63,3 +74,54 @@ def prepare_mms_js_resources(
     model_path, code_dir = _create_dir_structure(model_path)
 
     return _copy_jumpstart_artifacts(model_data, js_id, code_dir)
+
+def prepare_for_mms(
+        model_path: str,
+        shared_libs: List[str],
+        dependencies: dict,
+        session: Session,
+        image_uri: str,
+        inference_spec: InferenceSpec = None,
+) -> str:
+    """This is a one-line summary of the function.
+    Args:to
+        model_path (str) : Argument
+        shared_libs (List[]) : Argument
+        dependencies (dict) : Argument
+        session (Session) : Argument
+        inference_spec (InferenceSpec, optional) : Argument
+            (default is None)
+    Returns:
+        ( str ) :
+    """
+    model_path = Path(model_path)
+    if not model_path.exists():
+        model_path.mkdir()
+    elif not model_path.is_dir():
+        raise Exception("model_dir is not a valid directory")
+
+    if inference_spec:
+        inference_spec.prepare(str(model_path))
+
+    code_dir = model_path.joinpath("code")
+    code_dir.mkdir(exist_ok=True)
+
+    shutil.copy2(Path(__file__).parent.joinpath("inference.py"), code_dir)
+
+    logger.info("Finished writing inference.py to code directory")
+
+    shared_libs_dir = model_path.joinpath("shared_libs")
+    shared_libs_dir.mkdir(exist_ok=True)
+    for shared_lib in shared_libs:
+        shutil.copy2(Path(shared_lib), shared_libs_dir)
+
+    capture_dependencies(dependencies=dependencies, work_dir=code_dir)
+
+    secret_key = generate_secret_key()
+    with open(str(code_dir.joinpath("serve.pkl")), "rb") as f:
+        buffer = f.read()
+    hash_value = compute_hash(buffer=buffer, secret_key=secret_key)
+    with open(str(code_dir.joinpath("metadata.json")), "wb") as metadata:
+        metadata.write(_MetaData(hash_value).to_json())
+
+    return secret_key