Skip to content

Commit 195cc50

Browse files
author
gwizz
committed
Initial public deployment pack
0 parents  commit 195cc50

12 files changed

+944
-0
lines changed

‎.compose.dynamic.rendered.yml‎

Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,101 @@
1+
name: litellm-coolify-deployment-pack
2+
services:
3+
litellm_glm5_pool:
4+
entrypoint:
5+
- /bin/sh
6+
- -lc
7+
- |
8+
set -eu
9+
cat > /app/config.yaml <<'CONFIG'
10+
model_list:
11+
- model_name: glm-5-nvidia
12+
litellm_params:
13+
model: z-ai/glm5
14+
custom_llm_provider: openai
15+
api_base: os.environ/NVIDIA_API_BASE
16+
api_key: os.environ/NVIDIA_API_KEY_POOL_1
17+
- model_name: glm-5-nvidia
18+
litellm_params:
19+
model: z-ai/glm5
20+
custom_llm_provider: openai
21+
api_base: os.environ/NVIDIA_API_BASE
22+
api_key: os.environ/NVIDIA_API_KEY_POOL_2
23+
- model_name: glm-5-nvidia
24+
litellm_params:
25+
model: z-ai/glm5
26+
custom_llm_provider: openai
27+
api_base: os.environ/NVIDIA_API_BASE
28+
api_key: os.environ/NVIDIA_API_KEY_POOL_3
29+
- model_name: glm-5-nvidia
30+
litellm_params:
31+
model: z-ai/glm5
32+
custom_llm_provider: openai
33+
api_base: os.environ/NVIDIA_API_BASE
34+
api_key: os.environ/NVIDIA_API_KEY_POOL_4
35+
- model_name: glm-5-nvidia
36+
litellm_params:
37+
model: z-ai/glm5
38+
custom_llm_provider: openai
39+
api_base: os.environ/NVIDIA_API_BASE
40+
api_key: os.environ/NVIDIA_API_KEY_POOL_5
41+
- model_name: glm-5-nvidia
42+
litellm_params:
43+
model: z-ai/glm5
44+
custom_llm_provider: openai
45+
api_base: os.environ/NVIDIA_API_BASE
46+
api_key: os.environ/NVIDIA_API_KEY_POOL_6
47+
- model_name: glm-5-nvidia
48+
litellm_params:
49+
model: z-ai/glm5
50+
custom_llm_provider: openai
51+
api_base: os.environ/NVIDIA_API_BASE
52+
api_key: os.environ/NVIDIA_API_KEY_POOL_7
53+
- model_name: glm-5-nvidia
54+
litellm_params:
55+
model: z-ai/glm5
56+
custom_llm_provider: openai
57+
api_base: os.environ/NVIDIA_API_BASE
58+
api_key: os.environ/NVIDIA_API_KEY_POOL_8
59+
router_settings:
60+
routing_strategy: simple-shuffle
61+
allowed_fails: 1
62+
cooldown_time: 60
63+
litellm_settings:
64+
master_key: os.environ/LITELLM_MASTER_KEY
65+
drop_params: true
66+
general_settings:
67+
disable_master_key_return: true
68+
CONFIG
69+
exec docker/prod_entrypoint.sh --config /app/config.yaml --host 0.0.0.0 --port 4000
70+
environment:
71+
LITELLM_MASTER_KEY: replace-with-random-master-key
72+
NVIDIA_API_BASE: https://integrate.api.nvidia.com/v1
73+
NVIDIA_API_KEY_POOL_1: replace-me
74+
NVIDIA_API_KEY_POOL_2: replace-me
75+
NVIDIA_API_KEY_POOL_3: replace-me
76+
NVIDIA_API_KEY_POOL_4: replace-me
77+
NVIDIA_API_KEY_POOL_5: replace-me
78+
NVIDIA_API_KEY_POOL_6: replace-me
79+
NVIDIA_API_KEY_POOL_7: replace-me
80+
NVIDIA_API_KEY_POOL_8: replace-me
81+
SSL_CERT_FILE: /etc/ssl/certs/ca-certificates.crt
82+
healthcheck:
83+
test:
84+
- CMD-SHELL
85+
- python3 -c "import sys,urllib.request;sys.exit(0 if urllib.request.urlopen('http://127.0.0.1:4000/health/liveliness', timeout=5).status == 200 else 1)"
86+
timeout: 6s
87+
interval: 10s
88+
retries: 12
89+
start_period: 20s
90+
image: ghcr.io/berriai/litellm@sha256:d6580beba82a69e4cfb6598c300b7c524d9ea6f67592226fdec7f6a9aba34eb2
91+
networks:
92+
default: null
93+
ports:
94+
- mode: ingress
95+
target: 4000
96+
published: "4000"
97+
protocol: tcp
98+
restart: unless-stopped
99+
networks:
100+
default:
101+
name: litellm-coolify-deployment-pack_default

‎.compose.dynamic.yml‎

Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,92 @@
1+
services:
2+
litellm_glm5_pool:
3+
image: "ghcr.io/berriai/litellm@sha256:d6580beba82a69e4cfb6598c300b7c524d9ea6f67592226fdec7f6a9aba34eb2"
4+
restart: unless-stopped
5+
ports:
6+
- "${HOST_PORT:-4000}:4000"
7+
environment:
8+
LITELLM_MASTER_KEY: "${LITELLM_MASTER_KEY}"
9+
NVIDIA_API_BASE: "${NVIDIA_API_BASE}"
10+
NVIDIA_API_KEY_POOL_1: "${NVIDIA_API_KEY_POOL_1}"
11+
NVIDIA_API_KEY_POOL_2: "${NVIDIA_API_KEY_POOL_2}"
12+
NVIDIA_API_KEY_POOL_3: "${NVIDIA_API_KEY_POOL_3}"
13+
NVIDIA_API_KEY_POOL_4: "${NVIDIA_API_KEY_POOL_4}"
14+
NVIDIA_API_KEY_POOL_5: "${NVIDIA_API_KEY_POOL_5}"
15+
NVIDIA_API_KEY_POOL_6: "${NVIDIA_API_KEY_POOL_6}"
16+
NVIDIA_API_KEY_POOL_7: "${NVIDIA_API_KEY_POOL_7}"
17+
NVIDIA_API_KEY_POOL_8: "${NVIDIA_API_KEY_POOL_8}"
18+
SSL_CERT_FILE: "${SSL_CERT_FILE:-/etc/ssl/certs/ca-certificates.crt}"
19+
entrypoint:
20+
- /bin/sh
21+
- -lc
22+
- |
23+
set -eu
24+
cat > /app/config.yaml <<'CONFIG'
25+
model_list:
26+
- model_name: glm-5-nvidia
27+
litellm_params:
28+
model: z-ai/glm5
29+
custom_llm_provider: openai
30+
api_base: os.environ/NVIDIA_API_BASE
31+
api_key: os.environ/NVIDIA_API_KEY_POOL_1
32+
- model_name: glm-5-nvidia
33+
litellm_params:
34+
model: z-ai/glm5
35+
custom_llm_provider: openai
36+
api_base: os.environ/NVIDIA_API_BASE
37+
api_key: os.environ/NVIDIA_API_KEY_POOL_2
38+
- model_name: glm-5-nvidia
39+
litellm_params:
40+
model: z-ai/glm5
41+
custom_llm_provider: openai
42+
api_base: os.environ/NVIDIA_API_BASE
43+
api_key: os.environ/NVIDIA_API_KEY_POOL_3
44+
- model_name: glm-5-nvidia
45+
litellm_params:
46+
model: z-ai/glm5
47+
custom_llm_provider: openai
48+
api_base: os.environ/NVIDIA_API_BASE
49+
api_key: os.environ/NVIDIA_API_KEY_POOL_4
50+
- model_name: glm-5-nvidia
51+
litellm_params:
52+
model: z-ai/glm5
53+
custom_llm_provider: openai
54+
api_base: os.environ/NVIDIA_API_BASE
55+
api_key: os.environ/NVIDIA_API_KEY_POOL_5
56+
- model_name: glm-5-nvidia
57+
litellm_params:
58+
model: z-ai/glm5
59+
custom_llm_provider: openai
60+
api_base: os.environ/NVIDIA_API_BASE
61+
api_key: os.environ/NVIDIA_API_KEY_POOL_6
62+
- model_name: glm-5-nvidia
63+
litellm_params:
64+
model: z-ai/glm5
65+
custom_llm_provider: openai
66+
api_base: os.environ/NVIDIA_API_BASE
67+
api_key: os.environ/NVIDIA_API_KEY_POOL_7
68+
- model_name: glm-5-nvidia
69+
litellm_params:
70+
model: z-ai/glm5
71+
custom_llm_provider: openai
72+
api_base: os.environ/NVIDIA_API_BASE
73+
api_key: os.environ/NVIDIA_API_KEY_POOL_8
74+
router_settings:
75+
routing_strategy: simple-shuffle
76+
allowed_fails: 1
77+
cooldown_time: 60
78+
litellm_settings:
79+
master_key: os.environ/LITELLM_MASTER_KEY
80+
drop_params: true
81+
general_settings:
82+
disable_master_key_return: true
83+
CONFIG
84+
exec docker/prod_entrypoint.sh --config /app/config.yaml --host 0.0.0.0 --port 4000
85+
healthcheck:
86+
test:
87+
- CMD-SHELL
88+
- "python3 -c \"import sys,urllib.request;sys.exit(0 if urllib.request.urlopen('http://127.0.0.1:4000/health/liveliness', timeout=5).status == 200 else 1)\""
89+
interval: 10s
90+
timeout: 6s
91+
retries: 12
92+
start_period: 20s

‎.compose.rendered.yml‎

Lines changed: 99 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,99 @@
1+
name: litellm-coolify-deployment-pack
2+
services:
3+
litellm_glm5_pool:
4+
entrypoint:
5+
- /bin/sh
6+
- -lc
7+
- |
8+
set -eu
9+
cat > /app/config.yaml <<'CONFIG'
10+
model_list:
11+
- model_name: glm-5-nvidia
12+
litellm_params:
13+
model: z-ai/glm5
14+
custom_llm_provider: openai
15+
api_base: os.environ/NVIDIA_API_BASE
16+
api_key: os.environ/NVIDIA_API_KEY_POOL_1
17+
- model_name: glm-5-nvidia
18+
litellm_params:
19+
model: z-ai/glm5
20+
custom_llm_provider: openai
21+
api_base: os.environ/NVIDIA_API_BASE
22+
api_key: os.environ/NVIDIA_API_KEY_POOL_2
23+
- model_name: glm-5-nvidia
24+
litellm_params:
25+
model: z-ai/glm5
26+
custom_llm_provider: openai
27+
api_base: os.environ/NVIDIA_API_BASE
28+
api_key: os.environ/NVIDIA_API_KEY_POOL_3
29+
- model_name: glm-5-nvidia
30+
litellm_params:
31+
model: z-ai/glm5
32+
custom_llm_provider: openai
33+
api_base: os.environ/NVIDIA_API_BASE
34+
api_key: os.environ/NVIDIA_API_KEY_POOL_4
35+
- model_name: glm-5-nvidia
36+
litellm_params:
37+
model: z-ai/glm5
38+
custom_llm_provider: openai
39+
api_base: os.environ/NVIDIA_API_BASE
40+
api_key: os.environ/NVIDIA_API_KEY_POOL_5
41+
- model_name: glm-5-nvidia
42+
litellm_params:
43+
model: z-ai/glm5
44+
custom_llm_provider: openai
45+
api_base: os.environ/NVIDIA_API_BASE
46+
api_key: os.environ/NVIDIA_API_KEY_POOL_6
47+
- model_name: glm-5-nvidia
48+
litellm_params:
49+
model: z-ai/glm5
50+
custom_llm_provider: openai
51+
api_base: os.environ/NVIDIA_API_BASE
52+
api_key: os.environ/NVIDIA_API_KEY_POOL_7
53+
- model_name: glm-5-nvidia
54+
litellm_params:
55+
model: z-ai/glm5
56+
custom_llm_provider: openai
57+
api_base: os.environ/NVIDIA_API_BASE
58+
api_key: os.environ/NVIDIA_API_KEY_POOL_8
59+
router_settings:
60+
routing_strategy: simple-shuffle
61+
allowed_fails: 1
62+
cooldown_time: 60
63+
litellm_settings:
64+
master_key: os.environ/LITELLM_MASTER_KEY
65+
drop_params: true
66+
CONFIG
67+
exec docker/prod_entrypoint.sh --config /app/config.yaml --host 0.0.0.0 --port 4000
68+
environment:
69+
LITELLM_MASTER_KEY: replace-with-random-master-key
70+
NVIDIA_API_BASE: https://integrate.api.nvidia.com/v1
71+
NVIDIA_API_KEY_POOL_1: replace-me
72+
NVIDIA_API_KEY_POOL_2: replace-me
73+
NVIDIA_API_KEY_POOL_3: replace-me
74+
NVIDIA_API_KEY_POOL_4: replace-me
75+
NVIDIA_API_KEY_POOL_5: replace-me
76+
NVIDIA_API_KEY_POOL_6: replace-me
77+
NVIDIA_API_KEY_POOL_7: replace-me
78+
NVIDIA_API_KEY_POOL_8: replace-me
79+
SSL_CERT_FILE: /etc/ssl/certs/ca-certificates.crt
80+
healthcheck:
81+
test:
82+
- CMD-SHELL
83+
- python3 -c "import sys,urllib.request;sys.exit(0 if urllib.request.urlopen('http://127.0.0.1:4000/health/liveliness', timeout=5).status == 200 else 1)"
84+
timeout: 6s
85+
interval: 10s
86+
retries: 12
87+
start_period: 20s
88+
image: ghcr.io/berriai/litellm@sha256:d6580beba82a69e4cfb6598c300b7c524d9ea6f67592226fdec7f6a9aba34eb2
89+
networks:
90+
default: null
91+
ports:
92+
- mode: ingress
93+
target: 4000
94+
published: "4000"
95+
protocol: tcp
96+
restart: unless-stopped
97+
networks:
98+
default:
99+
name: litellm-coolify-deployment-pack_default

‎.env.example‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
# Change HOST_PORT when deploying a second LiteLLM instance on the same server.
2+
HOST_PORT=4000
3+
LITELLM_MASTER_KEY=replace-with-random-master-key
4+
NVIDIA_API_BASE=https://integrate.api.nvidia.com/v1
5+
# Add or remove NVIDIA_API_KEY_POOL_N lines as needed.
6+
NVIDIA_API_KEY_POOL_1=replace-me
7+
NVIDIA_API_KEY_POOL_2=replace-me
8+
NVIDIA_API_KEY_POOL_3=replace-me
9+
NVIDIA_API_KEY_POOL_4=replace-me
10+
NVIDIA_API_KEY_POOL_5=replace-me
11+
NVIDIA_API_KEY_POOL_6=replace-me
12+
NVIDIA_API_KEY_POOL_7=replace-me
13+
NVIDIA_API_KEY_POOL_8=replace-me
14+
SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt

‎.gitignore‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
.env
2+
.env.*
3+
!.env.example
4+
.DS_Store

‎LICENSE‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
MIT License
2+
3+
Copyright (c) 2026 Jacob Rothfield
4+
5+
Permission is hereby granted, free of charge, to any person obtaining a copy
6+
of this software and associated documentation files (the "Software"), to deal
7+
in the Software without restriction, including without limitation the rights
8+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9+
copies of the Software, and to permit persons to whom the Software is
10+
furnished to do so, subject to the following conditions:
11+
12+
The above copyright notice and this permission notice shall be included in all
13+
copies or substantial portions of the Software.
14+
15+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21+
SOFTWARE.

0 commit comments

Comments
 (0)