Add files for building container image for serving LLM apps.

Author: qiuosier

LLM/deployment/container/Dockerfile

@@ -0,0 +1,29 @@
+FROM ghcr.io/oracle/oraclelinux:9-slim
+
+RUN curl -L -o ./miniconda.sh -O https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh \
+    && chmod +x ./miniconda.sh \
+    && ./miniconda.sh -b -p /opt/conda \
+    && rm ./miniconda.sh \
+    && echo ". /opt/conda/etc/profile.d/conda.sh" >> ~/.bashrc \
+    && /opt/conda/bin/conda update python cryptography \
+    && /opt/conda/bin/conda clean -ya
+
+RUN microdnf install -y expat-devel which wget && microdnf clean all
+
+ARG USERNAME=uwsgi
+RUN useradd -ms /bin/bash $USERNAME
+USER $USERNAME
+WORKDIR /home/$USERNAME
+
+# Create conda env
+RUN /opt/conda/bin/conda create -n conda_env python=3.10 pip -y
+SHELL ["/opt/conda/bin/conda", "run", "-n", "conda_env", "/bin/bash", "-c"]
+
+ADD requirements.txt /opt/requirements.txt
+RUN pip install -r /opt/requirements.txt && pip cache purge
+ADD app.py /opt/app.py
+
+ENV MODEL_DIR="/opt/ds/model/deployed_model"
+ENV PATH=/home/$USERNAME/.conda/envs/conda_env/bin:/opt/conda/bin/:$PATH
+ENTRYPOINT [ "uwsgi" ]
+CMD [ "--http", "0.0.0.0:8080", "--master", "--enable-threads", "--single-interpreter", "-p", "5", "--chdir", "/opt", "--module", "app:app" ]

LLM/deployment/container/Readme.md

@@ -0,0 +1,8 @@
+# Container for serving LLM apps with Flask and uWSGI
+
+This directory contains files for building a container image for serving LLM applications on OCI Data Science model deployment service.
+* `Dockerfile`: The Dockerfile for building the image based on Oracle Linux 9.
+* `requirements.txt`: The Python dependencies for serving the application.
+* `app.py`: The Flask application for serving the LLM applications.
+
+This container image will basic LangChain/LangGraph applications, including the LLM/Chat Models for OCI Model Deployment and OCI Generative AI. You may add additional dependencies into `requirements.txt` as needed.

LLM/deployment/container/app.py

@@ -0,0 +1,67 @@
+"""Flask app with /health and /predict endpoint."""
+
+import importlib.util
+import logging
+import os
+import sys
+import traceback
+
+from flask import Flask, request
+
+
+# Get logging and debugging settings from environment variables
+LOG_LEVEL = os.environ.get("LOG_LEVEL", logging.INFO)
+MODEL_DIR = os.environ.get("MODEL_DIR", "/opt/ds/model/deployed_model")
+FLASK_DEBUG = os.environ.get("FLASK_DEBUG", False)
+
+
+def set_log_level(the_logger: logging.Logger, log_level=None):
+    """Sets the log level of a logger based on the environment variable.
+    This will also set the log level of logging.lastResort.
+    """
+    if not log_level:
+        return the_logger
+    try:
+        the_logger.setLevel(log_level)
+        logging.lastResort.setLevel(log_level)
+        the_logger.info(f"Log level set to {log_level}")
+    except Exception:
+        # Catching all exceptions here
+        # Setting log level should not interrupt the job run even if there is an exception.
+        the_logger.warning("Failed to set log level.")
+        the_logger.debug(traceback.format_exc())
+    return the_logger
+
+
+def import_from_path(file_path, module_name="score"):
+    """Imports a module from file path."""
+    spec = importlib.util.spec_from_file_location(module_name, file_path)
+    module = importlib.util.module_from_spec(spec)
+    sys.modules[module_name] = module
+    spec.loader.exec_module(module)
+    return module
+
+
+logger = logging.getLogger(__name__)
+set_log_level(logger, LOG_LEVEL)
+
+score = import_from_path(os.path.join(MODEL_DIR, "score.py"))
+app = Flask(__name__)
+
+
+@app.route("/health")
+def health():
+    """Health check."""
+    return {"status": "success"}
+
+
+@app.route("/predict", methods=["POST"])
+def predict():
+    """Make prediction."""
+    payload = request.get_data()
+    results = score.predict(payload)
+    return results
+
+
+if __name__ == "__main__":
+    app.run(debug=FLASK_DEBUG, host="0.0.0.0", port=8080)

LLM/deployment/container/requirements.txt

@@ -0,0 +1,8 @@
+flask
+langchain>=0.3
+langchain-community>=0.3
+langchain-openai>=0.2
+langchain-experimental>=0.3
+langgraph>=0.2
+oracle-ads>=2.12
+pyuwsgi