Merge pull request #101 from IBM/99-openai-lite-llm-connection

LiteLLM AI Gateway support

Author: D3f0

.env_sample

@@ -1,38 +1,135 @@
-## Set to any of the following values: gemini, openai, watsonx
-## You'll need to set corresponding APIKeys below to use the selected LLM
+################################################################################
+# AGENTICS LLM CONFIGURATION
+################################################################################
+# This file contains all the environment variables needed to configure LLM
+# providers for use with Agentics. All LLM providers are optional - configure
+# only the ones you need. The system will auto-discover which LLMs are available
+# based on the environment variables present.
+################################################################################
 
-SELECTED_LLM = "gemini" 
 
-## GEMINI
+################################################################################
+# GEMINI (Optional)
+################################################################################
 GEMINI_API_KEY=
-GEMINI_MODEL_ID= "gemini/gemini-2.0-flash"
+GEMINI_MODEL_ID="gemini/gemini-2.0-flash"
 
 
-## Set one of the following LLM providers before using agentics
-## WatsonX AI credentials (Optional)
+################################################################################
+# OPENAI (Optional)
+################################################################################
+OPENAI_API_KEY=
+OPENAI_MODEL_ID="openai/gpt-4"
+
+
+################################################################################
+# OPENAI-COMPATIBLE LLMs (Optional)
+#
+# Configure OpenAI-compatible API providers with custom prefixes.
+#
+# Supported patterns:
+# - Default variant: OPENAI_COMPATIBLE_*
+# - Custom variants: OPENAI_COMPATIBLE_<VARIANT>_*
+#
+# Each variant requires three variables:
+# - <PREFIX>_API_KEY: Your API key
+# - <PREFIX>_MODEL_ID: The model identifier
+# - <PREFIX>_BASE_URL: The base URL of the API
+#
+# Examples:
+################################################################################
+
+# Default OpenAI-compatible variant
+OPENAI_COMPATIBLE_API_KEY=
+OPENAI_COMPATIBLE_MODEL_ID=
+OPENAI_COMPATIBLE_BASE_URL=
+
+
+################################################################################
+# WATSONX (Optional)
+################################################################################
 MODEL_ID=watsonx/meta-llama/llama-3-3-70b-instruct
-WATSONX_URL=https://us-south.ml.cloud.ibm.com 
+WATSONX_URL=https://us-south.ml.cloud.ibm.com
 WATSONX_APIKEY=
 WATSONX_PROJECTID=
 
-## OpenAI credentials (Optional)
-OPENAI_API_KEY=
-OPENAI_MODEL_ID="openai/gpt-4"
 
-## VLLM (Optional)
-VLLM_URL=<http://base_url:PORT/v1>
-VLLM_MODEL_ID="hosted_vllm/meta-llama/Llama-3.3-70B-Instruct"
+################################################################################
+# VLLM - vLLM Server (Optional)
+#
+# vLLM is an open-source LLM serving framework.
+# Configure if you have a vLLM server running locally or remotely.
+################################################################################
+
+# Required: vLLM server URL
+VLLM_URL=http://localhost:8000/v1
+
+# Optional: Specific model ID for vLLM
+VLLM_MODEL_ID="meta-llama/Llama-3.3-70B-Instruct"
+
+
+################################################################################
+# OLLAMA (Optional)
+#
+# Ollama is an open-source LLM runner that can run models locally.
+# Configure if you have Ollama installed and running.
+################################################################################
+
+# The model to use with Ollama
+# OLLAMA_MODEL_ID="deepseek-r1:latest"
+# OLLAMA_MODEL_ID="llama2:latest"
+# OLLAMA_MODEL_ID="mistral:latest"
+
+
+################################################################################
+# LITELLM (Optional)
+#
+# LiteLLM provides access to 100+ LLM providers through a unified interface.
+# Use model format: "litellm/provider/model-name"
+# See: https://docs.litellm.ai/docs/providers
+################################################################################
+
+# Model in format: litellm/provider/model-name
+# LITELLM_MODEL="litellm/openai/gpt-4"
+# LITELLM_MODEL="litellm/anthropic/claude-3-sonnet"
+# LITELLM_MODEL="litellm/cohere/command-r"
+
+# Optional temperature and top_p settings
+# LITELLM_TEMPERATURE=0.8
+# LITELLM_TOP_P=0.9
+
+
+################################################################################
+# LITELLM PROXY (Optional)
+#
+# LiteLLM Proxy allows you to run a local proxy server that manages API keys
+# and provides a unified interface for multiple LLM providers. In some
+# organizations this also provides a way to centralize frontier LLM
+# access to employees.
+#
+################################################################################
+
+# Required: Proxy server URL (typically http://localhost:4000 if running locally, or
+https://litellm.your.organization/, no v1)
+# LITELLM_PROXY_URL=http://localhost:4000
 
-## OLLAMA (Optional)
-OLLAMA_MODEL_ID="ollama/deepseek-r1:latest"
+# Required: API key for proxy authentication
+# LITELLM_PROXY_API_KEY=
 
+################################################################################
+# Models
+# Once you've defined the above environment variables, you can run this command
+# to check the available models.
+#   uvx --from litellm[proxy] litellm-proxy models list
+#
+# This command will display all available models configured on your proxy.
+################################################################################
 
-#### CrewAI
+# Required: Model name in format: litellm_proxy/<model_name>
+# Must start with "litellm_proxy/"
+# LITELLM_PROXY_MODEL="litellm_proxy/gpt-4"
+# LITELLM_PROXY_MODEL="litellm_proxy/claude-3-sonnet"
 
-CREWAI_DISABLE_TELEMETRY=true
-CREWAI_DISABLE_TRACING=true
-CREWAI_TELEMETRY=false
-OTEL_SDK_DISABLED=true
-CREWAI_TRACING_DISABLED=true
-CREWAI_SILENT=true
-AGENTICS_TRACE_MODE="off"
+# Optional temperature and top_p settings
+# LITELLM_PROXY_TEMPERATURE=0.8
+# LITELLM_PROXY_TOP_P=0.9

.gitignore

@@ -40,3 +40,5 @@ __*__/
 _[a-z]*.py
 invoke.json
 invoke.y*ml
+# Igore profiling flamegraphs
+profile*.svg

.pre-commit-config.yaml

@@ -19,7 +19,6 @@ repos:
   #       # when "--baseline" with "--use-all-plugins", pre-commit scan with all available plugins
   #       # add "--fail-on-unaudited" to fail pre-commit for unaudited potential secrets
   #       args: [--baseline, .secrets.baseline, --use-all-plugins]
-        
 
   # - repo: https://github.com/astral-sh/ruff-pre-commit
   #   # Ruff version.
@@ -45,13 +44,14 @@ repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v5.0.0
     hooks:
-    -   id: trailing-whitespace
-        exclude: 'docs/.*|\.*.lock|'
-    - id: check-ast
-    -   id: end-of-file-fixer
+      - id: trailing-whitespace
+        exclude: 'docs/.*|\.*.lock|data/.*|tutorials/data/.*'
+      - id: check-ast
+      - id: end-of-file-fixer
+        exclude: "data/.*|tutorials/data/.*"
         exclude_types:
           - svg
-    -   id: debug-statements
+      - id: debug-statements
     # TODO: Fix this
     # -   id: name-tests-test
 

docs/getting_started.md

@@ -72,15 +72,80 @@ ollama pull ollama/deepseek-r1:latest
 - `VLLM_URL` - <http://base_url:PORT/v1>
 - `VLLM_MODEL_ID` - Your model id (e.g. "hosted_vllm/meta-llama/Llama-3.3-70B-Instruct" )
 
+#### LiteLLM (100+ providers via single interface)
+
+LiteLLM provides a unified interface to access 100+ LLM providers. You can use models from OpenAI, Anthropic, Google, Cohere, Azure, Hugging Face, and more.
+
+**Basic Setup (Local LiteLLM)**:
+
+- `LITELLM_MODEL` - Model in format `provider/model-name` (e.g., `openai/gpt-4`, `claude/claude-opus-4-5-20251101`, `gemini/gemini-2.0-flash`)
+- The required API key for your provider should be in environment variables (e.g., `OPENAI_API_KEY`, `ANTHROPIC_API_KEY`, etc.)
+- Optional: `LITELLM_TEMPERATURE` - Set temperature (default: varies by provider)
+- Optional: `LITELLM_TOP_P` - Set top-p sampling (default: varies by provider)
+
+**Examples**:
+
+OpenAI via LiteLLM:
+```bash
+export LITELLM_MODEL="openai/gpt-4"
+export OPENAI_API_KEY="sk-..."
+```
+
+Anthropic Claude via LiteLLM:
+```bash
+export LITELLM_MODEL="claude/claude-opus-4-5-20251101"
+export ANTHROPIC_API_KEY="sk-ant-..."
+```
+
+Google Gemini via LiteLLM:
+```bash
+export LITELLM_MODEL="gemini/gemini-2.0-flash"
+export GOOGLE_API_KEY="..."
+```
+
+**LiteLLM Proxy Server**
+
+If you have a self-hosted LiteLLM proxy server:
+
+- `LITELLM_PROXY_URL` - Base URL of your LiteLLM proxy (e.g., `http://localhost:8000`)
+- `LITELLM_PROXY_API_KEY` - API key for the proxy
+- `LITELLM_PROXY_MODEL` - Model name in format `litellm_proxy/<model-name>` (e.g., `litellm_proxy/gpt-4`)
+- Optional: `LITELLM_PROXY_TEMPERATURE` - Set temperature
+- Optional: `LITELLM_PROXY_TOP_P` - Set top-p sampling
+
+**Example**:
+```bash
+export LITELLM_PROXY_URL="http://localhost:8000"
+export LITELLM_PROXY_API_KEY="sk-proxy-key-123"
+export LITELLM_PROXY_MODEL="litellm_proxy/my-model"
+```
+
+Also you can use the provided script for configuration in the git repo (⚠️not available
+through `pip install`)
+
+```bash
+uv run tasks.py setup
+```
+
+**Checking LiteLLM Status**
+
+After configuration, you can check if your LiteLLM setup is working:
+
+```bash
+show-llms
+```
+
+This will display a table showing the authentication status of all configured LLMs, including LiteLLM.
+
 
 ## Test Installation
 
 test hello world example (need to set up llm credentials first)
 
 ```bash
-python python examples/hello_world.py
-python examples/self_transduction.py
-python examples/agentics_web_search_report.py
+uv run examples/hello_world.py
+uv run examples/self_transduction.py
+uv run examples/agentics_web_search_report.py
 
 ```
 

examples/emotion_extractor.py

@@ -34,7 +34,7 @@ def split_into_chunks(text, chunk_size=200):
 
 
 async def main():
-    emotion_detector = AG(atype=EmotionDetector, llm="watsonx/openai/gpt-oss-120b")
+    emotion_detector = AG(atype=EmotionDetector)
 
     current_file = Path(__file__).resolve()
     text = None

examples/generate_tweets.py

@@ -19,7 +19,7 @@ async def main():
         "data/categorization_example/movies.csv",
         max_rows=2,
     )
-    movies.llm = "watsonx/openai/gpt-oss-120b"
+    # movies.llm = "watsonx/openai/gpt-oss-120b"
     ## Step 2. add attribute tweet to be used as a target for transduction
     extended_movies = movies.add_attribute(
         "tweet", slot_type="str", description="Tweet used to advertise the movie"

examples/relation_extraction.py

@@ -11,10 +11,12 @@
 
 N_TERMS = 500
 N_CLUSTERS = 5
-llm = "watsonx/openai/gpt-oss-120b"
+# llm = "watsonx/openai/gpt-oss-120b"
+
+
+llm = AG.get_llm_provider()
 
 
-# llm=AG.get_llm_provider()
 class InputTerms(BaseModel):
     terms: list[str] = None
 
@@ -73,7 +75,6 @@ async def relation_extracton_map_reduce(
 
 
 async def evaluate_relation_extraction(outout_file: str = None, n_terms=10000):
-
     terms = random_frequent_terms(n=n_terms)
     # for i in range(1000, 1001, 500):
 

pyproject.toml

@@ -39,6 +39,7 @@ dependencies = [
     "litellm",
     "mellea",
     "plotly>=6.5.0",
+    "rich>=13.0.0,<14.0.0",
 ]
 
 
@@ -63,6 +64,8 @@ packages = ["src/agentics"]
 [tool.hatch.build.targets.sdist]
 packages = ["src/agentics"]
 
+[project.scripts]
+show-llms = "agentics.scripts.show_llms:main"
 
 [dependency-groups]
 dev = [
@@ -77,6 +80,7 @@ dev = [
   "papermill>=2.6.0",
   "pytest-xdist>=3.8.0",
   "pytest-subtests>=0.15.0",
+  "hunter>=3.9.0",
 ]
 docling = [
   "langchain-docling >=0.2.0,<0.3.0"