Skip to content

Commit 71bb1f5

Browse files
rennokkirennokki
committed
Added litellm
1 parent ce926af commit 71bb1f5

File tree

2 files changed

+178
-0
lines changed

2 files changed

+178
-0
lines changed

‎public/svgs/litellm.svg

Lines changed: 1 addition & 0 deletions
Loading

‎templates/compose/litellm.yaml

Lines changed: 177 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,177 @@
1+
# documentation: https://docs.litellm.ai
2+
# slogan: Call all LLM APIs using the OpenAI format. Use Bedrock, Azure, OpenAI, Cohere, Anthropic, Ollama, Sagemaker, HuggingFace, Replicate, Groq (100+ LLMs)
3+
# tags: ai, qdrant, weaviate, langchain, openai, gpt, llm, lmops, anthropic, cohere, ollama, sagemaker, huggingface, replicate, groq
4+
# logo: svgs/litellm.svg
5+
# port: 4000
6+
7+
volumes:
8+
pg-data: null
9+
redis-data: null
10+
11+
services:
12+
litellm:
13+
image: "ghcr.io/berriai/litellm-database:main-latest"
14+
restart: always
15+
depends_on:
16+
postgres:
17+
condition: service_healthy
18+
redis:
19+
condition: service_healthy
20+
environment:
21+
- LITELLM_LOG=ERROR
22+
- LITELLM_MODE=PRODUCTION
23+
- "LITELLM_MASTER_KEY=${SERVICE_PASSWORD_32_LITELLM_MASTER_KEY}"
24+
- "UI_USERNAME=${SERVICE_USER_UI}"
25+
- "UI_PASSWORD=${SERVICE_PASSWORD_UI}"
26+
- "DATABASE_URL=postgresql://${SERVICE_USER_POSTGRES}:${SERVICE_PASSWORD_POSTGRES}@postgres:5432/${POSTGRES_DB:-litellm}"
27+
- REDIS_HOST=redis
28+
- REDIS_PORT=6379
29+
- "POSTGRES_USER=${SERVICE_USER_POSTGRES}"
30+
- "POSTGRES_PASSWORD=${SERVICE_PASSWORD_POSTGRES}"
31+
- "POSTGRES_DB=${POSTGRES_DB:-litellm}"
32+
- "OPENAI_API_KEY=${OPENAI_API_KEY}"
33+
- "OPENAI_API_BASE=${OPENAI_API_BASE}"
34+
- "ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY}"
35+
- "ANTHROPIC_API_BASE=${ANTHROPIC_API_BASE}"
36+
- "VOYAGE_API_KEY=${VOYAGE_API_KEY}"
37+
- "VOYAGE_API_BASE=${VOYAGE_API_BASE}"
38+
volumes:
39+
- "./litellm-config.yaml:/app/config.yaml"
40+
- type: bind
41+
source: ./docker/app/config.yaml
42+
target: /app/config.yaml
43+
content: |
44+
general_settings:
45+
proxy_batch_write_at: 60
46+
47+
router_settings:
48+
routing_strategy: usage-based-routing-v2
49+
redis_host: os.environ/REDIS_HOST
50+
redis_port: os.environ/REDIS_PORT
51+
redis_password: os.environ/REDIS_PASSWORD
52+
enable_pre_call_check: true
53+
54+
litellm_settings:
55+
set_verbose: false
56+
json_logs: true
57+
log_raw_request_response: true
58+
# turn_off_message_logging: false
59+
# redact_user_api_key_info: false
60+
drop_params: true
61+
# max_budget: 100
62+
# budget_duration: 30d
63+
num_retries: 3
64+
request_timeout: 600
65+
telemetry: false
66+
cache: true
67+
cache_params:
68+
type: redis
69+
host: os.environ/REDIS_HOST
70+
port: os.environ/REDIS_PORT
71+
password: os.environ/REDIS_PASSWORD
72+
namespace: "litellm_cache"
73+
ttl: 600
74+
# success_callback:
75+
# - "langfuse"
76+
# - "prometheus"
77+
# failure_callback:
78+
# - "langfuse"
79+
# - "prometheus"
80+
model_list:
81+
# OpenAI
82+
- model_name: gpt-3.5-turbo
83+
litellm_params:
84+
model: openai/gpt-3.5-turbo
85+
api_key: os.environ/OPENAI_API_KEY
86+
api_base: os.environ/OPENAI_API_BASE
87+
- model_name: gpt-4
88+
litellm_params:
89+
model: openai/gpt-4
90+
api_key: os.environ/OPENAI_API_KEY
91+
api_base: os.environ/OPENAI_API_BASE
92+
- model_name: gpt-4o
93+
litellm_params:
94+
model: openai/gpt-4o
95+
api_key: os.environ/OPENAI_API_KEY
96+
api_base: os.environ/OPENAI_API_BASE
97+
- model_name: gpt-4o-mini
98+
litellm_params:
99+
model: openai/gpt-4o-mini
100+
api_key: os.environ/OPENAI_API_KEY
101+
api_base: os.environ/OPENAI_API_BASE
102+
# Anthropic
103+
- model_name: claude-3-haiku
104+
litellm_params:
105+
model: claude-3-haiku-20240307
106+
api_key: "os.environ/ANTHROPIC_API_KEY"
107+
api_base: "os.environ/ANTHROPIC_API_BASE"
108+
- model_name: claude-3.5-sonnet
109+
litellm_params:
110+
model: claude-3-5-sonnet-20240620
111+
api_key: "os.environ/ANTHROPIC_API_KEY"
112+
api_base: "os.environ/ANTHROPIC_API_BASE"
113+
# VoyageAI
114+
- model_name: voyage-law-2
115+
model_info:
116+
output_vector_size: 1024
117+
litellm_params:
118+
model: voyage/voyage-law-2
119+
api_key: "os.environ/VOYAGE_API_KEY"
120+
api_base: "os.environ/VOYAGE_API_BASE"
121+
rpm: 300
122+
tpm: 1000000
123+
- model_name: voyage-multilingual-2
124+
model_info:
125+
mode: embedding
126+
max_tokens: 32000
127+
max_input_tokens: 32000
128+
output_vector_size: 1024
129+
litellm_params:
130+
model: voyage/voyage-multilingual-2
131+
api_key: "os.environ/VOYAGE_API_KEY"
132+
api_base: "os.environ/VOYAGE_API_BASE"
133+
input_cost_per_token: 0.00000012
134+
output_cost_per_token: 0
135+
rpm: 300
136+
tpm: 1000000
137+
healthcheck:
138+
test:
139+
- CMD
140+
- python
141+
- "-c"
142+
- "import requests as r;r.get('http://127.0.0.1:4000/health/liveliness').raise_for_status()"
143+
interval: 5s
144+
timeout: 5s
145+
retries: 3
146+
command:
147+
- "--config"
148+
- /app/config.yaml
149+
- "--port"
150+
- "4000"
151+
- "--num_workers"
152+
- "8"
153+
postgres:
154+
image: "postgres:16-alpine"
155+
environment:
156+
- POSTGRES_DB=${POSTGRES_DB:-litellm}
157+
- POSTGRES_PASSWORD=$SERVICE_PASSWORD_POSTGRES
158+
- POSTGRES_USER=$SERVICE_USER_POSTGRES
159+
volumes:
160+
- "pg-data:/var/lib/postgresql/data"
161+
healthcheck:
162+
test:
163+
- CMD-SHELL
164+
- "pg_isready -h localhost -U ${POSTGRES_USER} -d ${POSTGRES_DB}"
165+
interval: 5s
166+
timeout: 5s
167+
retries: 3
168+
redis:
169+
image: redis:7-alpine
170+
command: redis-server --appendonly yes
171+
volumes:
172+
- redis-data:/data
173+
healthcheck:
174+
test: ["CMD", "redis-cli", "ping"]
175+
interval: 5s
176+
timeout: 5s
177+
retries: 3

0 commit comments

Comments
 (0)