Skip to content

Commit 3a4965a

Browse files
saichandrapandrajusaichandrapandraju
committed
add tests, dockerfile, deploy manifests
1 parent 072ebc3 commit 3a4965a

File tree

12 files changed

+491
-6
lines changed

12 files changed

+491
-6
lines changed

‎detectors/Dockerfile.judge

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
FROM registry.access.redhat.com/ubi9/ubi-minimal as base
2+
RUN microdnf update -y && \
3+
microdnf install -y --nodocs \
4+
python-pip python-devel && \
5+
pip install --upgrade --no-cache-dir pip wheel && \
6+
microdnf clean all
7+
RUN pip install --no-cache-dir torch
8+
9+
# FROM icr.io/fm-stack/ubi9-minimal-py39-torch as builder
10+
FROM base as builder
11+
12+
COPY ./common/requirements.txt .
13+
RUN pip install --no-cache-dir -r requirements.txt
14+
15+
COPY ./llm_judge/requirements.txt .
16+
RUN pip install --no-cache-dir -r requirements.txt
17+
18+
FROM builder
19+
20+
WORKDIR /app
21+
ARG CACHEBUST=1
22+
RUN echo "$CACHEBUST"
23+
COPY ./common /common
24+
COPY ./llm_judge/app.py /app
25+
COPY ./llm_judge/detector.py /app
26+
COPY ./llm_judge/scheme.py /app
27+
28+
EXPOSE 8000
29+
CMD ["uvicorn", "app:app", "--workers", "4", "--host", "0.0.0.0", "--port", "8000", "--log-config", "/common/log_conf.yaml"]
30+
31+
# gunicorn main:app --workers 4 --worker-class uvicorn.workers.UvicornWorker --bind 0.0.0.0:8000

‎detectors/llm_judge/README.md

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
# LLM Judge Detector
2+
3+
The LLM Judge detector integrates the [vLLM Judge](https://github.com/saichandrapandraju/vllm_judge) into the Guardrails Detector ecosystem.
4+
5+
```
6+
oc apply -f deploy/servingruntime.yaml
7+
oc apply -f deploy/isvc.yaml
8+
```

‎detectors/llm_judge/__init__.py

Whitespace-only changes.

‎detectors/llm_judge/app.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,8 @@
88
sys.path.insert(0, os.path.abspath(".."))
99

1010
from common.app import DetectorBaseAPI as FastAPI
11-
from detector import LLMJudgeDetector
12-
from scheme import (
11+
from .detector import LLMJudgeDetector
12+
from .scheme import (
1313
ContentAnalysisHttpRequest,
1414
ContentsAnalysisResponse,
1515
MetricsListResponse,

‎detectors/llm_judge/deploy/isvc.yaml

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
apiVersion: serving.kserve.io/v1beta1
2+
kind: InferenceService
3+
metadata:
4+
name: guardrails-detector-llm-judge
5+
namespace: model-namespace
6+
labels:
7+
opendatahub.io/dashboard: 'true'
8+
annotations:
9+
openshift.io/display-name: guardrails-detector-llm-judge
10+
security.opendatahub.io/enable-auth: 'true'
11+
serving.knative.openshift.io/enablePassthrough: 'true'
12+
sidecar.istio.io/inject: 'true'
13+
sidecar.istio.io/rewriteAppHTTPProbers: 'true'
14+
serving.kserve.io/deploymentMode: RawDeployment
15+
spec:
16+
predictor:
17+
maxReplicas: 1
18+
minReplicas: 1
19+
model:
20+
modelFormat:
21+
name: guardrails-detector-llm-judge
22+
name: ''
23+
runtime: guardrails-detector-runtime-judge

‎detectors/llm_judge/deploy/servingruntime.yaml

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
apiVersion: serving.kserve.io/v1alpha1
2+
kind: ServingRuntime
3+
metadata:
4+
name: guardrails-detector-runtime-judge
5+
namespace: model-namespace
6+
annotations:
7+
openshift.io/display-name: Guardrails LLM Judge Detector ServingRuntime for KServe
8+
labels:
9+
opendatahub.io/dashboard: 'true'
10+
spec:
11+
annotations:
12+
prometheus.io/port: '8080'
13+
prometheus.io/path: '/metrics'
14+
multiModel: false
15+
supportedModelFormats:
16+
- autoSelect: true
17+
name: guardrails-detector-llm-judge
18+
containers:
19+
- name: kserve-container
20+
image: quay.io/rh-ee-spandraj/guardrails-detector-judge:latest
21+
command:
22+
- uvicorn
23+
- app:app
24+
args:
25+
- "--workers"
26+
- "1"
27+
- "--host"
28+
- "0.0.0.0"
29+
- "--port"
30+
- "8000"
31+
- "--log-config"
32+
- "/common/log_conf.yaml"
33+
env:
34+
- name: VLLM_BASE_URL
35+
value: "http://qwen2-predictor:8080" # <-- Change this to your vLLM URL
36+
ports:
37+
- containerPort: 8000
38+
protocol: TCP
39+
resources:
40+
requests:
41+
memory: "5Gi"
42+
cpu: "1"
43+
limits:
44+
memory: "10Gi"
45+
cpu: "2"

‎detectors/llm_judge/detector.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77
from vllm_judge import Judge, EvaluationResult, BUILTIN_METRICS
88
from vllm_judge.exceptions import MetricNotFoundError
99
from common.app import logger
10-
from scheme import (
10+
from .scheme import (
1111
ContentAnalysisHttpRequest,
1212
ContentAnalysisResponse,
1313
ContentsAnalysisResponse,
@@ -41,7 +41,7 @@ def _initialize_judge(self) -> None:
4141
logger.info(f"Available metrics: {', '.join(sorted(self.available_metrics))}")
4242

4343
except Exception as e:
44-
logger.error(f"Failed to initialize LLM Judge: {e}")
44+
logger.error(f"Failed to detect model: {e}")
4545
raise
4646

4747
async def evaluate_single_content(self, content: str, params: Dict[str, Any]) -> ContentAnalysisResponse:

‎detectors/llm_judge/requirements.txt

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1,2 @@
1-
vllm-judge>=0.1.5
1+
vllm-judge>=0.1.5
2+
pyyaml>=6.0.2

‎tests/conftest.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,10 +9,12 @@ def setup_imports():
99
project_root = os.path.dirname(os.path.dirname(__file__))
1010
detectors_path = os.path.join(project_root, "detectors")
1111
huggingface_path = os.path.join(detectors_path, "huggingface")
12+
llm_judge_path = os.path.join(detectors_path, "llm_judge")
1213
paths = [
1314
huggingface_path,
1415
detectors_path,
1516
project_root,
17+
llm_judge_path,
1618
]
1719

1820
for path in paths:

0 commit comments

Comments
 (0)