litellm-coolify-nvidia-multi-key/.compose.rendered.yml at main · jroth1111/litellm-coolify-nvidia-multi-key · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
name: litellm-coolify-deployment-pack
services:
  litellm_glm5_pool:
    entrypoint:
      - /bin/sh
      - -lc
      - |
        set -eu
        cat > /app/config.yaml <<'CONFIG'
        model_list:
          - model_name: glm-5-nvidia
            litellm_params:
              model: z-ai/glm5
              custom_llm_provider: openai
              api_base: os.environ/NVIDIA_API_BASE
              api_key: os.environ/NVIDIA_API_KEY_POOL_1
          - model_name: glm-5-nvidia
            litellm_params:
              model: z-ai/glm5
              custom_llm_provider: openai
              api_base: os.environ/NVIDIA_API_BASE
              api_key: os.environ/NVIDIA_API_KEY_POOL_2
          - model_name: glm-5-nvidia
            litellm_params:
              model: z-ai/glm5
              custom_llm_provider: openai
              api_base: os.environ/NVIDIA_API_BASE
              api_key: os.environ/NVIDIA_API_KEY_POOL_3
          - model_name: glm-5-nvidia
            litellm_params:
              model: z-ai/glm5
              custom_llm_provider: openai
              api_base: os.environ/NVIDIA_API_BASE
              api_key: os.environ/NVIDIA_API_KEY_POOL_4
          - model_name: glm-5-nvidia
            litellm_params:
              model: z-ai/glm5
              custom_llm_provider: openai
              api_base: os.environ/NVIDIA_API_BASE
              api_key: os.environ/NVIDIA_API_KEY_POOL_5
          - model_name: glm-5-nvidia
            litellm_params:
              model: z-ai/glm5
              custom_llm_provider: openai
              api_base: os.environ/NVIDIA_API_BASE
              api_key: os.environ/NVIDIA_API_KEY_POOL_6
          - model_name: glm-5-nvidia
            litellm_params:
              model: z-ai/glm5
              custom_llm_provider: openai
              api_base: os.environ/NVIDIA_API_BASE
              api_key: os.environ/NVIDIA_API_KEY_POOL_7
          - model_name: glm-5-nvidia
            litellm_params:
              model: z-ai/glm5
              custom_llm_provider: openai
              api_base: os.environ/NVIDIA_API_BASE
              api_key: os.environ/NVIDIA_API_KEY_POOL_8
        router_settings:
          routing_strategy: simple-shuffle
          allowed_fails: 1
          cooldown_time: 60
        litellm_settings:
          master_key: os.environ/LITELLM_MASTER_KEY
          drop_params: true
        CONFIG
        exec docker/prod_entrypoint.sh --config /app/config.yaml --host 0.0.0.0 --port 4000
    environment:
      LITELLM_MASTER_KEY: replace-with-random-master-key
      NVIDIA_API_BASE: https://integrate.api.nvidia.com/v1
      NVIDIA_API_KEY_POOL_1: replace-me
      NVIDIA_API_KEY_POOL_2: replace-me
      NVIDIA_API_KEY_POOL_3: replace-me
      NVIDIA_API_KEY_POOL_4: replace-me
      NVIDIA_API_KEY_POOL_5: replace-me
      NVIDIA_API_KEY_POOL_6: replace-me
      NVIDIA_API_KEY_POOL_7: replace-me
      NVIDIA_API_KEY_POOL_8: replace-me
      SSL_CERT_FILE: /etc/ssl/certs/ca-certificates.crt
    healthcheck:
      test:
        - CMD-SHELL
        - python3 -c "import sys,urllib.request;sys.exit(0 if urllib.request.urlopen('http://127.0.0.1:4000/health/liveliness', timeout=5).status == 200 else 1)"
      timeout: 6s
      interval: 10s
      retries: 12
      start_period: 20s
    image: ghcr.io/berriai/litellm@sha256:d6580beba82a69e4cfb6598c300b7c524d9ea6f67592226fdec7f6a9aba34eb2
    networks:
      default: null
    ports:
      - mode: ingress
        target: 4000
        published: "4000"
        protocol: tcp
    restart: unless-stopped
networks:
  default:
    name: litellm-coolify-deployment-pack_default