add more

Author: ddynwzh1992

oss/litellm/Dockerfile

@@ -0,0 +1,23 @@
+FROM cgr.dev/chainguard/python:latest-dev
+
+USER root
+WORKDIR /app
+
+ENV HOME=/home/litellm
+ENV PATH="${HOME}/venv/bin:$PATH"
+
+# Install runtime dependencies
+RUN apk update && \
+    apk add --no-cache gcc python3-dev openssl openssl-dev
+
+RUN python -m venv ${HOME}/venv
+RUN ${HOME}/venv/bin/pip install --no-cache-dir --upgrade pip
+
+COPY requirements.txt .
+RUN --mount=type=cache,target=${HOME}/.cache/pip \
+    ${HOME}/venv/bin/pip install -r requirements.txt
+
+EXPOSE 4000/tcp
+
+ENTRYPOINT ["litellm"]
+CMD ["--port", "4000"]

oss/litellm/combined.yaml

@@ -0,0 +1,108 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: litellm-config-file
+data:
+  config.yaml: |
+      model_list: 
+        - model_name: DeepSeek-R1-Distill-Qwen-7B-GGUF
+          litellm_params:
+            model: unsloth/DeepSeek-R1-Distill-Qwen-7B-GGUF
+            api_base: http://llama-cpp-cpu-lb-2137543273.us-west-2.elb.amazonaws.com/v1/chat/completions
+            api_key: os.environ/OPENAI_API_KEY
+        - model_name: QwQ-32B
+          litellm_params:
+            model: Qwen/QwQ-32B
+            api_base: http://54.159.11.218:8080/v1
+            api_key: os.environ/OPENAI_API_KEY
+      general_settings: 
+            master_key: os.environ/LITELLM_MASTER_KEY
+---
+apiVersion: v1
+kind: Secret
+type: Opaque
+metadata:
+  name: litellm-secrets
+data:
+  OPENAI_API_KEY: c2stMTIzNA== # your api key in base64
+  LITELLM_MASTER_KEY: c2stMTIzNA== # master key in base64
+stringData:
+  DATABASE_URL: postgresql://<user>:<password>@<host>:<port>/<dbname>
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: litellm-deployment
+  labels:
+    app: litellm
+spec:
+  selector:
+    matchLabels:
+      app: litellm
+  template:
+    metadata:
+      labels:
+        app: litellm
+    spec:
+      nodeSelector:
+            kubernetes.io/arch: arm64
+      affinity:
+            nodeAffinity:
+              requiredDuringSchedulingIgnoredDuringExecution:
+                nodeSelectorTerms:
+                - matchExpressions:
+                  - key: kubernetes.io/arch
+                    operator: In
+                    values:
+                    - arm64
+      containers:
+      - name: litellm
+        image: 412381761882.dkr.ecr.us-west-2.amazonaws.com/litellm-proxy:latest # it is recommended to fix a version generally
+        ports:
+        - containerPort: 4000
+        volumeMounts:
+        - name: config-volume
+          mountPath: /app/proxy_server_config.yaml
+          subPath: config.yaml
+        envFrom:
+        - secretRef:
+            name: litellm-secrets
+      volumes:
+        - name: config-volume
+          configMap:
+            name: litellm-config-file
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: litellm-service
+  labels:
+    app: litellm
+spec:
+  selector:
+    app: litellm
+  ports:
+  - port: 80
+    targetPort: 4000
+    protocol: TCP
+  type: ClusterIP
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: litellm-ingress
+  annotations:
+    kubernetes.io/ingress.class: "alb"
+    alb.ingress.kubernetes.io/scheme: internet-facing
+    alb.ingress.kubernetes.io/target-type: ip
+spec:
+  rules:
+  - http:
+      paths:
+      - path: /
+        pathType: Prefix
+        backend:
+          service:
+            name: litellm-service
+            port:
+              number: 80

oss/litellm/requirements.txt

@@ -0,0 +1,4 @@
+litellm[proxy]==1.57.3 # Specify the litellm version you want to use
+prometheus_client
+langfuse
+prisma