Merge pull request #1 from trustyai-explainability/text_generation_detection

Text generation detection

Author: m-misiura

detectors/Dockerfile.hf

@@ -0,0 +1,31 @@
+FROM registry.access.redhat.com/ubi9/ubi-minimal as base
+RUN microdnf update -y && \
+    microdnf install -y --nodocs \
+        python-pip python-devel && \
+    pip install --upgrade --no-cache-dir pip wheel && \
+    microdnf clean all
+RUN pip install --no-cache-dir torch
+
+# FROM icr.io/fm-stack/ubi9-minimal-py39-torch as builder
+FROM base as builder
+
+COPY ./common/requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
+
+COPY ./huggingface/requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
+
+FROM builder
+
+WORKDIR /app
+ARG CACHEBUST=1
+RUN echo "$CACHEBUST"
+COPY ./common /common
+COPY ./huggingface/app.py /app
+COPY ./huggingface/detector.py /app
+COPY ./huggingface/scheme.py /app
+
+EXPOSE 8000
+CMD ["uvicorn", "app:app" "--workers", "4", "--host", "0.0.0.0", "--port", "8000", "--log-config", "/common/log_conf.yaml"]
+
+# gunicorn main:app --workers 4 --worker-class uvicorn.workers.UvicornWorker --bind 0.0.0.0:8000

detectors/common/app.py

@@ -14,9 +14,11 @@
 
 from fastapi import FastAPI, status
 from starlette.exceptions import HTTPException as StarletteHTTPException
+from prometheus_fastapi_instrumentator import Instrumentator
 
 logger = logging.getLogger(__name__)
-
+uvicorn_error_logger = logging.getLogger("uvicorn.error")
+uvicorn_error_logger.name = "uvicorn"
 
 app = FastAPI(
     title="WxPE Detectors API",
@@ -38,6 +40,7 @@ def __init__(self, *args, **kwargs):
         self.add_exception_handler(StarletteHTTPException, self.http_exception_handler)
         self.add_api_route("/health", health, description="Check if server is alive")
 
+
     async def validation_exception_handler(self, request, exc):
         errors = exc.errors()
         if len(errors) > 0 and errors[0]["type"] == "missing":
@@ -95,7 +98,6 @@ async def http_exception_handler(self, request, exc):
 async def health():
     return "ok"
 
-
 def main(app):
     # "loop": "uvloop", (thats default in our setting)
     # "backlog": 10000
@@ -112,6 +114,8 @@ def main(app):
         }
     }
 
+    logger.info("config:", os.getenv("CONFIG_FILE_PATH"))
+
     try:
         with open(os.getenv("CONFIG_FILE_PATH", "config.yaml")) as stream:
             config = yaml.safe_load(stream)

detectors/common/requirements.txt

@@ -1,3 +1,5 @@
 fastapi==0.112.0
 uvicorn==0.30.5
-httpx==0.27.0
+httpx==0.27.0
+prometheus_client >= 0.18.0
+prometheus-fastapi-instrumentator >= 7.0.0

detectors/huggingface/README.md

@@ -0,0 +1,5 @@
+```
+oc apply -f deployment/model_container.yaml
+oc apply -f deployment/servingruntime.yaml
+oc apply -f deployment/isvc.yaml
+```

detectors/huggingface/__init__.py

@@ -0,0 +1,47 @@
+import os
+import sys
+from contextlib import asynccontextmanager
+from typing import Annotated
+
+from fastapi import Header
+from prometheus_fastapi_instrumentator import Instrumentator
+sys.path.insert(0, os.path.abspath(".."))
+
+from common.app import DetectorBaseAPI as FastAPI
+from detector import Detector
+from scheme import (
+    ContentAnalysisHttpRequest,
+    ContentsAnalysisResponse,
+    Error,
+)
+
+detector_objects = {}
+
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    detector_objects["detector"] = Detector()
+    yield
+    # Clean up the ML models and release the resources
+    detector_objects.clear()
+
+
+app = FastAPI(lifespan=lifespan, dependencies=[])
+Instrumentator().instrument(app).expose(app)
+
+
+@app.post(
+    "/api/v1/text/contents",
+    response_model=ContentsAnalysisResponse,
+    description="""Detectors that work on content text, be it user prompt or generated text. \
+                    Generally classification type detectors qualify for this. <br>""",
+    responses={
+        404: {"model": Error, "description": "Resource Not Found"},
+        422: {"model": Error, "description": "Validation Error"},
+    },
+)
+async def detector_unary_handler(
+    request: ContentAnalysisHttpRequest,
+    detector_id: Annotated[str, Header(example="en_syntax_slate.38m.hap")],
+):
+    return ContentsAnalysisResponse(root=detector_objects["detector"].run(request))

detectors/huggingface/app.py

@@ -0,0 +1,25 @@
+apiVersion: serving.kserve.io/v1beta1
+kind: InferenceService
+metadata:
+  name: guardrails-detector-ibm-guardian
+  labels:
+    opendatahub.io/dashboard: 'true'
+  annotations:
+    openshift.io/display-name: guardrails-detector-ibm-guardian
+    security.opendatahub.io/enable-auth: 'true'
+    serving.knative.openshift.io/enablePassthrough: 'true'
+    sidecar.istio.io/inject: 'true'
+    sidecar.istio.io/rewriteAppHTTPProbers: 'true'
+    serving.kserve.io/deploymentMode: RawDeployment
+spec:
+  predictor:
+    maxReplicas: 1
+    minReplicas: 1
+    model:
+      modelFormat:
+        name: guardrails-detector-huggingface
+      name: ''
+      runtime: guardrails-detector-runtime-guardian
+      storage:
+        key: aws-connection-minio-data-connection-guardrails-guardian
+        path: granite-guardian-3.0-2b

detectors/huggingface/deploy/isvc.yaml

@@ -0,0 +1,127 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: minio-guardrails-guardian
+spec:
+  ports:
+    - name: minio-client-port
+      port: 9000
+      protocol: TCP
+      targetPort: 9000
+  selector:
+    app: minio-guardrails-guardian
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: guardrails-models-claim-guardian
+spec:
+  accessModes:
+    - ReadWriteOnce
+  volumeMode: Filesystem
+  # storageClassName: gp3-csi
+  resources:
+    requests:
+      storage: 100Gi
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: guardrails-container-deployment-guardian # <--- change this
+labels:
+    app: minio-guardrails-guardian # <--- change this to match label on the pod
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: minio-guardrails-guardian  # <--- change this to match label on the pod
+  template: # => from here down copy and paste the pods metadata: and spec: sections
+    metadata:
+      labels:
+        app: minio-guardrails-guardian
+        maistra.io/expose-route: 'true'
+      name: minio-guardrails-guardian
+    spec:
+      volumes:
+      - name: model-volume
+        persistentVolumeClaim:
+          claimName: guardrails-models-claim-guardian
+      initContainers:
+        - name: download-model
+          image: quay.io/rgeada/llm_downloader:latest
+          securityContext:
+            fsGroup: 1001
+          command:
+            - bash
+            - -c
+            - |
+              # model="ibm-granite/granite-guardian-hap-38m"
+              # model="h2oai/deberta_finetuned_pii"
+              model="ibm-granite/granite-guardian-3.0-2b"
+              # model="microsoft/Phi-3-mini-4k-instruct"
+              echo "starting download"
+              /tmp/venv/bin/huggingface-cli download $model --local-dir /mnt/models/huggingface/$(basename $model)
+              echo "Done!"
+          resources:
+            limits:
+              memory: "2Gi"
+              cpu: "2"
+          volumeMounts:
+            - mountPath: "/mnt/models/"
+              name: model-volume
+      containers:
+        - args:
+            - server
+            - /models
+          env:
+            - name: MINIO_ACCESS_KEY
+              value:  THEACCESSKEY
+            - name: MINIO_SECRET_KEY
+              value: THESECRETKEY
+          image: quay.io/trustyai/modelmesh-minio-examples:latest
+          name: minio
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop:
+                - ALL
+            seccompProfile:
+              type: RuntimeDefault
+          volumeMounts:
+            - mountPath: "/models/"
+              name: model-volume
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: aws-connection-minio-data-connection-guardrails-guardian
+  labels:
+    opendatahub.io/dashboard: 'true'
+    opendatahub.io/managed: 'true'
+  annotations:
+    opendatahub.io/connection-type: s3
+    openshift.io/display-name: Minio Data Connection
+data:
+  AWS_ACCESS_KEY_ID: VEhFQUNDRVNTS0VZ
+  AWS_DEFAULT_REGION: dXMtc291dGg=
+  AWS_S3_BUCKET: aHVnZ2luZ2ZhY2U=
+  AWS_S3_ENDPOINT: aHR0cDovL21pbmlvLWd1YXJkcmFpbHMtZ3VhcmRpYW46OTAwMA==
+  AWS_SECRET_ACCESS_KEY: VEhFU0VDUkVUS0VZ
+type: Opaque
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: user-one
+---
+kind: RoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: user-one-view
+subjects:
+  - kind: ServiceAccount
+    name: user-one
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: view

detectors/huggingface/deploy/model_container.yaml

@@ -0,0 +1,47 @@
+apiVersion: serving.kserve.io/v1alpha1
+kind: ServingRuntime
+metadata:
+  name: guardrails-detector-runtime-guardian
+  annotations:
+    openshift.io/display-name: Guardrails Detector ServingRuntime for KServe
+    opendatahub.io/recommended-accelerators: '["nvidia.com/gpu"]'
+  labels:
+    opendatahub.io/dashboard: 'true'
+spec:
+  annotations:
+    prometheus.io/port: '8080'
+    prometheus.io/path: '/metrics'
+  multiModel: false
+  supportedModelFormats:
+    - autoSelect: true
+      name: guardrails-detector-huggingface
+  containers:
+    - name: kserve-container
+      image: quay.io/rh-ee-mmisiura/guardrails-detector-huggingface:3d51741
+      command:
+        - uvicorn
+        - app:app
+      args:
+        - "--workers"
+        - "1"
+        - "--host"
+        - "0.0.0.0"
+        - "--port"
+        - "8000"
+        - "--log-config"
+        - "/common/log_conf.yaml"
+      env:
+        - name: MODEL_DIR
+          value: /mnt/models
+        - name: HF_HOME
+          value: /tmp/hf_home
+      ports:
+        - containerPort: 8000
+          protocol: TCP
+      resources:
+        requests:
+          memory: "18Gi"  # pre-allocate 18Gi of memory -- might be needed for larger models
+          cpu: "1"       # reserve 1 CPU core
+        limits:
+          memory: "20Gi"  # limit to 20Gi of memory
+          cpu: "2"       # limit to 2 CPU cores