litellm-coolify-nvidia-multi-key/.compose.dynamic.yml at main · jroth1111/litellm-coolify-nvidia-multi-key · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
services:
  litellm_glm5_pool:
    image: "ghcr.io/berriai/litellm@sha256:d6580beba82a69e4cfb6598c300b7c524d9ea6f67592226fdec7f6a9aba34eb2"
    restart: unless-stopped
    ports:
      - "${HOST_PORT:-4000}:4000"
    environment:
      LITELLM_MASTER_KEY: "${LITELLM_MASTER_KEY}"
      NVIDIA_API_BASE: "${NVIDIA_API_BASE}"
      NVIDIA_API_KEY_POOL_1: "${NVIDIA_API_KEY_POOL_1}"
      NVIDIA_API_KEY_POOL_2: "${NVIDIA_API_KEY_POOL_2}"
      NVIDIA_API_KEY_POOL_3: "${NVIDIA_API_KEY_POOL_3}"
      NVIDIA_API_KEY_POOL_4: "${NVIDIA_API_KEY_POOL_4}"
      NVIDIA_API_KEY_POOL_5: "${NVIDIA_API_KEY_POOL_5}"
      NVIDIA_API_KEY_POOL_6: "${NVIDIA_API_KEY_POOL_6}"
      NVIDIA_API_KEY_POOL_7: "${NVIDIA_API_KEY_POOL_7}"
      NVIDIA_API_KEY_POOL_8: "${NVIDIA_API_KEY_POOL_8}"
      SSL_CERT_FILE: "${SSL_CERT_FILE:-/etc/ssl/certs/ca-certificates.crt}"
    entrypoint:
      - /bin/sh
      - -lc
      - |
        set -eu
        cat > /app/config.yaml <<'CONFIG'
        model_list:
          - model_name: glm-5-nvidia
            litellm_params:
              model: z-ai/glm5
              custom_llm_provider: openai
              api_base: os.environ/NVIDIA_API_BASE
              api_key: os.environ/NVIDIA_API_KEY_POOL_1
          - model_name: glm-5-nvidia
            litellm_params:
              model: z-ai/glm5
              custom_llm_provider: openai
              api_base: os.environ/NVIDIA_API_BASE
              api_key: os.environ/NVIDIA_API_KEY_POOL_2
          - model_name: glm-5-nvidia
            litellm_params:
              model: z-ai/glm5
              custom_llm_provider: openai
              api_base: os.environ/NVIDIA_API_BASE
              api_key: os.environ/NVIDIA_API_KEY_POOL_3
          - model_name: glm-5-nvidia
            litellm_params:
              model: z-ai/glm5
              custom_llm_provider: openai
              api_base: os.environ/NVIDIA_API_BASE
              api_key: os.environ/NVIDIA_API_KEY_POOL_4
          - model_name: glm-5-nvidia
            litellm_params:
              model: z-ai/glm5
              custom_llm_provider: openai
              api_base: os.environ/NVIDIA_API_BASE
              api_key: os.environ/NVIDIA_API_KEY_POOL_5
          - model_name: glm-5-nvidia
            litellm_params:
              model: z-ai/glm5
              custom_llm_provider: openai
              api_base: os.environ/NVIDIA_API_BASE
              api_key: os.environ/NVIDIA_API_KEY_POOL_6
          - model_name: glm-5-nvidia
            litellm_params:
              model: z-ai/glm5
              custom_llm_provider: openai
              api_base: os.environ/NVIDIA_API_BASE
              api_key: os.environ/NVIDIA_API_KEY_POOL_7
          - model_name: glm-5-nvidia
            litellm_params:
              model: z-ai/glm5
              custom_llm_provider: openai
              api_base: os.environ/NVIDIA_API_BASE
              api_key: os.environ/NVIDIA_API_KEY_POOL_8
        router_settings:
          routing_strategy: simple-shuffle
          allowed_fails: 1
          cooldown_time: 60
        litellm_settings:
          master_key: os.environ/LITELLM_MASTER_KEY
          drop_params: true
        general_settings:
          disable_master_key_return: true
        CONFIG
        exec docker/prod_entrypoint.sh --config /app/config.yaml --host 0.0.0.0 --port 4000
    healthcheck:
      test:
        - CMD-SHELL
        - "python3 -c \"import sys,urllib.request;sys.exit(0 if urllib.request.urlopen('http://127.0.0.1:4000/health/liveliness', timeout=5).status == 200 else 1)\""
      interval: 10s
      timeout: 6s
      retries: 12
      start_period: 20s