feat: add LiteLLM

README.md

@@ -2,18 +2,22 @@
 
 ## Charts
 
-- vLLM: inference and serving engine for LLMs (NVIDIA device plugin required).
+- vLLM (NVIDIA device plugin required): inference and serving engine for LLMs.
 
 `helm upgrade --install vllm charts/vllm --namespace vllm --create-namespace -f values/production/vllm.yaml`
 
 - CloudNativePG (should be installed in a separate namespace, i.e. `cnpg-system`): PostgreSQL operator.
 
 `helm upgrade --install cloudnative-pg charts/cloudnative-pg --namespace cnpg-system --create-namespace -f values/production/cloudnative-pg.yaml`
 
-- PostgreSQL: CloudNativePG required
+- PostgreSQL (CloudNativePG required)
 
 `helm upgrade --install pg-cluster charts/pg-cluster --namespace pg-cluster --create-namespace -f values/production/pg-cluster.yaml`
 
+- LiteLLM (PostgreSQL required): LLM Gateway to call 100+ LLMs, load balance, cost tracking across projects.
+
+`helm upgrade --install litellm charts/litellm --namespace litellm --create-namespace -f values/production/litellm.yaml`
+
 ## Dependencies
 
 - NVIDIA device plugin: used to manage NVIDIA GPUs in a K8s cluster.

charts/litellm/.helmignore

@@ -0,0 +1,23 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/

charts/litellm/Chart.lock

@@ -0,0 +1,9 @@
+dependencies:
+- name: postgresql
+  repository: oci://registry-1.docker.io/bitnamicharts
+  version: 14.3.1
+- name: redis
+  repository: oci://registry-1.docker.io/bitnamicharts
+  version: 18.19.1
+digest: sha256:8660fe6287f9941d08c0902f3f13731079b8cecd2a5da2fbc54e5b7aae4a6f62
+generated: "2024-03-10T02:28:52.275022+05:30"

charts/litellm/Chart.yaml

@@ -0,0 +1,37 @@
+apiVersion: v2
+
+# We can't call ourselves just "litellm" because then we couldn't publish to the
+#  same OCI repository as the "litellm" OCI image
+name: litellm-helm
+description: Call all LLM APIs using the OpenAI format
+
+# A chart can be either an 'application' or a 'library' chart.
+#
+# Application charts are a collection of templates that can be packaged into versioned archives
+# to be deployed.
+#
+# Library charts provide useful utilities or functions for the chart developer. They're included as
+# a dependency of application charts to inject those utilities and functions into the rendering
+# pipeline. Library charts do not define any templates and therefore cannot be deployed.
+type: application
+
+# This is the chart version. This version number should be incremented each time you make changes
+# to the chart and its templates, including the app version.
+# Versions are expected to follow Semantic Versioning (https://semver.org/)
+version: 0.4.7
+
+# This is the version number of the application being deployed. This version number should be
+# incremented each time you make changes to the application. Versions are not expected to
+# follow Semantic Versioning. They should reflect the version the application is using.
+# It is recommended to use it with quotes.
+appVersion: v1.50.2
+
+dependencies:
+  - name: "postgresql"
+    version: ">=13.3.0"
+    repository: oci://registry-1.docker.io/bitnamicharts
+    condition: db.deployStandalone
+  - name: redis
+    version: ">=18.0.0"
+    repository: oci://registry-1.docker.io/bitnamicharts 
+    condition: redis.enabled

charts/litellm/ci/test-values.yaml

@@ -0,0 +1,15 @@
+fullnameOverride: ""
+# Disable database deployment and configuration
+db:
+  deployStandalone: false
+  useExisting: false
+
+# Test environment variables
+envVars:
+  DD_ENV: "dev_helm"
+  DD_SERVICE: "litellm"
+  USE_DDTRACE: "true"
+
+# Disable migration job since we're not using a database
+migrationJob:
+  enabled: false

charts/litellm/templates/NOTES.txt

@@ -0,0 +1,23 @@
+1. Get the application URL by running these commands:
+{{- if .Values.ingress.enabled }}
+{{- range $host := .Values.ingress.hosts }}
+  {{- range .paths }}
+  http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host.host }}{{ .path }}
+  {{- end }}
+{{- end }}
+{{- else if contains "NodePort" .Values.service.type }}
+  export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "litellm.fullname" . }})
+  export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}")
+  echo http://$NODE_IP:$NODE_PORT
+{{- else if contains "LoadBalancer" .Values.service.type }}
+     NOTE: It may take a few minutes for the LoadBalancer IP to be available.
+           You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "litellm.fullname" . }}'
+  export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "litellm.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}")
+  echo http://$SERVICE_IP:{{ .Values.service.port }}
+{{- else if contains "ClusterIP" .Values.service.type }}
+  export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "litellm.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}")
+  export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}")
+  echo "Visit http://127.0.0.1:8080 to use your application"
+  kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT
+{{- end }}
+PDB: {{ if .Values.pdb.enabled }}enabled{{ else }}disabled{{ end }}. Configure via .Values.pdb.*

charts/litellm/templates/_helpers.tpl

@@ -0,0 +1,84 @@
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "litellm.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "litellm.fullname" -}}
+{{- if .Values.fullnameOverride }}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- $name := default .Chart.Name .Values.nameOverride }}
+{{- if contains $name .Release.Name }}
+{{- .Release.Name | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
+{{- end }}
+{{- end }}
+{{- end }}
+
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "litellm.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Common labels
+*/}}
+{{- define "litellm.labels" -}}
+helm.sh/chart: {{ include "litellm.chart" . }}
+{{ include "litellm.selectorLabels" . }}
+{{- if .Chart.AppVersion }}
+app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+{{- end }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- end }}
+
+{{/*
+Selector labels
+*/}}
+{{- define "litellm.selectorLabels" -}}
+app.kubernetes.io/name: {{ include "litellm.name" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+{{- end }}
+
+{{/*
+Create the name of the service account to use
+*/}}
+{{- define "litellm.serviceAccountName" -}}
+{{- if .Values.serviceAccount.create }}
+{{- default (include "litellm.fullname" .) .Values.serviceAccount.name }}
+{{- else }}
+{{- default "default" .Values.serviceAccount.name }}
+{{- end }}
+{{- end }}
+
+{{/*
+Get redis service name
+*/}}
+{{- define "litellm.redis.serviceName" -}}
+{{- if and (eq .Values.redis.architecture "standalone") .Values.redis.sentinel.enabled -}}
+{{- printf "%s-%s" .Release.Name (default "redis" .Values.redis.nameOverride | trunc 63 | trimSuffix "-") -}}
+{{- else -}}
+{{- printf "%s-%s-master" .Release.Name (default "redis" .Values.redis.nameOverride | trunc 63 | trimSuffix "-") -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Get redis service port
+*/}}
+{{- define "litellm.redis.port" -}}
+{{- if .Values.redis.sentinel.enabled -}}
+{{ .Values.redis.sentinel.service.ports.sentinel }}
+{{- else -}}
+{{ .Values.redis.master.service.ports.redis }}
+{{- end -}}
+{{- end -}}

charts/litellm/templates/configmap-litellm.yaml

@@ -0,0 +1,9 @@
+{{- if .Values.proxyConfigMap.create }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ include "litellm.fullname" . }}-config
+data:
+  config.yaml: |
+{{ .Values.proxy_config | toYaml | indent 6 }}
+{{- end }}