Skip to content

Commit 0a2c1ba

Browse files
drupoldrupol
authored
nixos/litellm: init (#390340)
2 parents 0d0b65d + c8a823d commit 0a2c1ba

File tree

7 files changed

+234
-3
lines changed

7 files changed

+234
-3
lines changed

‎nixos/doc/manual/release-notes/rl-2505.section.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -178,6 +178,8 @@
178178

179179
- [echoip](https://github.com/mpolden/echoip), a simple service for looking up your IP address. Available as [services.echoip](#opt-services.echoip.enable).
180180

181+
- [LiteLLM](https://github.com/BerriAI/litellm), a LLM Gateway to provide model access, fallbacks and spend tracking across 100+ LLMs. All in the OpenAI format. Available as [services.litellm](#opt-services.litellm.enable).
182+
181183
- [Buffyboard](https://gitlab.postmarketos.org/postmarketOS/buffybox/-/tree/master/buffyboard), a framebuffer on-screen keyboard. Available as [services.buffyboard](option.html#opt-services.buffyboard).
182184

183185
- [KanBoard](https://github.com/kanboard/kanboard), a project management tool that focuses on the Kanban methodology. Available as [services.kanboard](#opt-services.kanboard.enable).

‎nixos/modules/module-list.nix‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -825,6 +825,7 @@
825825
./services/misc/languagetool.nix
826826
./services/misc/leaps.nix
827827
./services/misc/lifecycled.nix
828+
./services/misc/litellm.nix
828829
./services/misc/llama-cpp.nix
829830
./services/misc/logkeys.nix
830831
./services/misc/mame.nix

‎nixos/modules/services/misc/litellm.nix‎

Lines changed: 182 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,182 @@
1+
{
2+
config,
3+
lib,
4+
pkgs,
5+
...
6+
}:
7+
let
8+
inherit (lib) types;
9+
10+
cfg = config.services.litellm;
11+
settingsFormat = pkgs.formats.yaml { };
12+
in
13+
{
14+
options = {
15+
services.litellm = {
16+
enable = lib.mkEnableOption "LiteLLM server";
17+
package = lib.mkPackageOption pkgs "litellm" { };
18+
19+
stateDir = lib.mkOption {
20+
type = types.path;
21+
default = "/var/lib/litellm";
22+
example = "/home/foo";
23+
description = "State directory of LiteLLM.";
24+
};
25+
26+
host = lib.mkOption {
27+
type = types.str;
28+
default = "127.0.0.1";
29+
example = "0.0.0.0";
30+
description = ''
31+
The host address which the LiteLLM server HTTP interface listens to.
32+
'';
33+
};
34+
35+
port = lib.mkOption {
36+
type = types.port;
37+
default = 8080;
38+
example = 11111;
39+
description = ''
40+
Which port the LiteLLM server listens to.
41+
'';
42+
};
43+
44+
settings = lib.mkOption {
45+
type = types.submodule {
46+
freeformType = settingsFormat.type;
47+
options = {
48+
model_list = lib.mkOption {
49+
type = settingsFormat.type;
50+
description = ''
51+
List of supported models on the server, with model-specific configs.
52+
'';
53+
default = [ ];
54+
};
55+
router_settings = lib.mkOption {
56+
type = settingsFormat.type;
57+
description = ''
58+
LiteLLM Router settings
59+
'';
60+
default = { };
61+
};
62+
63+
litellm_settings = lib.mkOption {
64+
type = settingsFormat.type;
65+
description = ''
66+
LiteLLM Module settings
67+
'';
68+
default = { };
69+
};
70+
71+
general_settings = lib.mkOption {
72+
type = settingsFormat.type;
73+
description = ''
74+
LiteLLM Server settings
75+
'';
76+
default = { };
77+
};
78+
79+
environment_variables = lib.mkOption {
80+
type = settingsFormat.type;
81+
description = ''
82+
Environment variables to pass to the Lite
83+
'';
84+
default = { };
85+
};
86+
};
87+
};
88+
default = { };
89+
description = ''
90+
Configuration for LiteLLM.
91+
See <https://docs.litellm.ai/docs/proxy/configs> for more.
92+
'';
93+
};
94+
95+
environment = lib.mkOption {
96+
type = types.attrsOf types.str;
97+
default = {
98+
SCARF_NO_ANALYTICS = "True";
99+
DO_NOT_TRACK = "True";
100+
ANONYMIZED_TELEMETRY = "False";
101+
};
102+
example = ''
103+
{
104+
NO_DOCS="True";
105+
}
106+
'';
107+
description = ''
108+
Extra environment variables for LiteLLM.
109+
'';
110+
};
111+
112+
environmentFile = lib.mkOption {
113+
description = ''
114+
Environment file to be passed to the systemd service.
115+
Useful for passing secrets to the service to prevent them from being
116+
world-readable in the Nix store.
117+
'';
118+
type = lib.types.nullOr lib.types.path;
119+
default = null;
120+
example = "/var/lib/secrets/liteLLMSecrets";
121+
};
122+
123+
openFirewall = lib.mkOption {
124+
type = types.bool;
125+
default = false;
126+
description = ''
127+
Whether to open the firewall for LiteLLM.
128+
This adds `services.litellm.port` to `networking.firewall.allowedTCPPorts`.
129+
'';
130+
};
131+
};
132+
};
133+
134+
config = lib.mkIf cfg.enable {
135+
systemd.services.litellm = {
136+
description = "LLM Gateway to provide model access, fallbacks and spend tracking across 100+ LLMs.";
137+
wantedBy = [ "multi-user.target" ];
138+
after = [ "network.target" ];
139+
140+
environment = cfg.environment;
141+
142+
serviceConfig =
143+
let
144+
configFile = settingsFormat.generate "config.yaml" cfg.settings;
145+
in
146+
{
147+
ExecStart = "${lib.getExe cfg.package} --host \"${cfg.host}\" --port ${toString cfg.port} --config ${configFile}";
148+
EnvironmentFile = lib.optional (cfg.environmentFile != null) cfg.environmentFile;
149+
WorkingDirectory = cfg.stateDir;
150+
StateDirectory = "litellm";
151+
RuntimeDirectory = "litellm";
152+
RuntimeDirectoryMode = "0755";
153+
PrivateTmp = true;
154+
DynamicUser = true;
155+
DevicePolicy = "closed";
156+
LockPersonality = true;
157+
PrivateUsers = true;
158+
ProtectHome = true;
159+
ProtectHostname = true;
160+
ProtectKernelLogs = true;
161+
ProtectKernelModules = true;
162+
ProtectKernelTunables = true;
163+
ProtectControlGroups = true;
164+
RestrictNamespaces = true;
165+
RestrictRealtime = true;
166+
SystemCallArchitectures = "native";
167+
UMask = "0077";
168+
RestrictAddressFamilies = [
169+
"AF_INET"
170+
"AF_INET6"
171+
"AF_UNIX"
172+
];
173+
ProtectClock = true;
174+
ProtectProc = "invisible";
175+
};
176+
};
177+
178+
networking.firewall = lib.mkIf cfg.openFirewall { allowedTCPPorts = [ cfg.port ]; };
179+
};
180+
181+
meta.maintainers = with lib.maintainers; [ drupol ];
182+
}

‎nixos/tests/all-tests.nix‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -634,6 +634,7 @@ in {
634634
limesurvey = handleTest ./limesurvey.nix {};
635635
limine = import ./limine { inherit runTest; };
636636
listmonk = handleTestOn [ "x86_64-linux" "aarch64-linux" ] ./listmonk.nix {};
637+
litellm = runTest ./litellm.nix;
637638
litestream = handleTest ./litestream.nix {};
638639
lldap = handleTest ./lldap.nix {};
639640
localsend = handleTest ./localsend.nix {};

‎nixos/tests/litellm.nix‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
{ lib, ... }:
2+
let
3+
mainPort = "8080";
4+
in
5+
{
6+
name = "litellm";
7+
8+
nodes = {
9+
machine =
10+
{ ... }:
11+
{
12+
services.litellm = {
13+
enable = true;
14+
};
15+
};
16+
};
17+
18+
testScript = ''
19+
machine.start()
20+
machine.wait_for_unit("litellm.service")
21+
machine.wait_for_open_port(${mainPort})
22+
'';
23+
24+
meta = with lib.maintainers; {
25+
maintainers = [ drupol ];
26+
};
27+
}

‎pkgs/by-name/li/litellm/package.nix‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
{ python3Packages }:
2+
3+
let
4+
litellm = python3Packages.litellm;
5+
in
6+
python3Packages.toPythonApplication (
7+
litellm.overridePythonAttrs (oldAttrs: {
8+
dependencies =
9+
(oldAttrs.dependencies or [ ])
10+
++ litellm.optional-dependencies.proxy
11+
++ litellm.optional-dependencies.extra_proxy;
12+
})
13+
)

‎pkgs/development/python-modules/litellm/default.nix‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@
3535
tokenizers,
3636
uvloop,
3737
uvicorn,
38+
nixosTests,
3839
}:
3940

4041
buildPythonPackage rec {
@@ -99,12 +100,16 @@ buildPythonPackage rec {
99100
# access network
100101
doCheck = false;
101102

102-
meta = with lib; {
103+
passthru.tests = {
104+
inherit (nixosTests) litellm;
105+
};
106+
107+
meta = {
103108
description = "Use any LLM as a drop in replacement for gpt-3.5-turbo. Use Azure, OpenAI, Cohere, Anthropic, Ollama, VLLM, Sagemaker, HuggingFace, Replicate (100+ LLMs)";
104109
mainProgram = "litellm";
105110
homepage = "https://github.com/BerriAI/litellm";
106111
changelog = "https://github.com/BerriAI/litellm/releases/tag/${src.tag}";
107-
license = licenses.mit;
108-
maintainers = with maintainers; [ happysalada ];
112+
license = lib.licenses.mit;
113+
maintainers = with lib.maintainers; [ happysalada ];
109114
};
110115
}

0 commit comments

Comments
 (0)