feat: llamacpp chat completion

Author: cgarvis

README.md

@@ -28,7 +28,8 @@ config :hyper_llm,
   ],
   anthropic: [
     api_key: "sk-..."
-  ]
+  ],
+  # ...
 ```
 
 ## Usage
@@ -142,6 +143,7 @@ end
 | Anthropic          | ✅         | ❌        |
 | Cloudflare         | ✅         | ❌        |
 | Groq               | ✅         | ❌        |
+| LlamaCPP           | ✅         | ❌        |
 | Mistral            | ✅         | ❌        |
 | Ollama             | ✅         | ❌        |
 | OpenAI             | ✅         | ❌        |

lib/model.ex

@@ -7,6 +7,7 @@ defmodule HyperLLM.Model do
     "anthropic" => HyperLLM.Provider.Anthropic,
     "cloudflare" => HyperLLM.Provider.Cloudflare,
     "groq" => HyperLLM.Provider.Groq,
+    "llama_cpp" => HyperLLM.Provider.LlamaCPP,
     "mistral" => HyperLLM.Provider.Mistral,
     "openai" => HyperLLM.Provider.OpenAI,
     "ollama" => HyperLLM.Provider.Ollama,
@@ -81,7 +82,7 @@ defmodule HyperLLM.Model do
   Example:
 
       iex> HyperLLM.Model.list_providers()
-      ["anthropic", "cloudflare", "groq", "mistral", "ollama", "openai", "x_ai"]
+      ["anthropic", "cloudflare", "groq", "llama_cpp", "mistral", "ollama", "openai", "x_ai"]
   """
   def list_providers, do: Map.keys(@providers)
 end

lib/providers/llama_cpp.ex

@@ -0,0 +1,78 @@
+defmodule HyperLLM.Provider.LlamaCPP do
+  @behaviour HyperLLM.Provider
+
+  @moduledoc """
+  Provider implementation for LlamaCPP.
+
+  LlamaCPP server that implements the OpenAI API format.
+  https://github.com/ggerganov/llama.cpp/tree/master/examples/server
+
+  ## Configuration
+
+  `api_key` - The API key for the LlamaCPP API (optional).
+  `base_url` - The base URL for the LlamaCPP API.
+
+      config :hyper_llm, 
+        llama_cpp: [
+          api_key: "llamacpp",
+          base_url: "http://localhost:8080"
+        ]
+  """
+
+  @impl true
+  def completion(messages, config) do
+    model = Keyword.get(config, :model)
+
+    {_request, response} =
+      request("/chat/completions",
+        method: :post,
+        receive_timeout: 30_000,
+        json: %{
+          model: model,
+          messages: messages
+        }
+      )
+
+    case response do
+      %{status: 200, body: body} ->
+        {:ok, body}
+
+      %{status: 400, body: body} ->
+        {:error, body.error.message}
+
+      %{status: 401} ->
+        {:error, "LlamaCPP API key is invalid"}
+
+      %{status: 404} ->
+        {:error, "LlamaCPP API not found"}
+
+      %{status: 500} ->
+        {:error, "LlamaCPP Server error"}
+
+      _ ->
+        {:error, "Unknown error"}
+    end
+  end
+
+  @impl true
+  @doc """
+  Check if a model is supported by the provider.
+
+  All models are supported as they are loaded into the server directly.
+  """
+  def model_supported?(_), do: true
+
+  defp request(url, opts) do
+    api_key = HyperLLM.config!(:llama_cpp, :api_key)
+    base_url = HyperLLM.config(:llama_cpp, :base_url, "http://localhost:8080")
+
+    req =
+      Req.new(
+        auth: {:bearer, api_key},
+        base_url: base_url,
+        url: url
+      )
+
+    Req.run(req, opts)
+  end
+end

test/model_test.exs

@@ -54,7 +54,16 @@ defmodule HyperLLM.ModelTest do
 
   describe "list_providers/0" do
     test "returns a list of providers" do
-      assert ["anthropic", "cloudflare", "groq", "mistral", "ollama", "openai", "x_ai"] =
+      assert [
+               "anthropic",
+               "cloudflare",
+               "groq",
+               "llama_cpp",
+               "mistral",
+               "ollama",
+               "openai",
+               "x_ai"
+             ] =
                HyperLLM.Model.list_providers()
     end
   end