diff --git a/.cursor/worktrees.json b/.cursor/worktrees.json new file mode 100644 index 000000000..235fa1c06 --- /dev/null +++ b/.cursor/worktrees.json @@ -0,0 +1,7 @@ +{ + "setup-worktree": [ + "cp $ROOT_WORKTREE_PATH/.env .env", + "make install-minimal", + "make check" + ] +} diff --git a/docs/howtos/customizations/customize_models.md b/docs/howtos/customizations/customize_models.md index 848f48786..7319a907b 100644 --- a/docs/howtos/customizations/customize_models.md +++ b/docs/howtos/customizations/customize_models.md @@ -1,170 +1,142 @@ ## Customize Models -Ragas may use a LLM and or Embedding for evaluation and synthetic data generation. Both of these models can be customised according to you availability. +Ragas may use a LLM and or Embedding for evaluation and synthetic data generation. Both of these models can be customised according to your availability. -!!! note - Ragas supports all the [LLMs](https://python.langchain.com/docs/integrations/chat/) and [Embeddings](https://python.langchain.com/docs/integrations/text_embedding/) available in LangChain +Ragas provides factory functions (`llm_factory` and `embedding_factory`) that support multiple providers: -- `BaseRagasLLM` and `BaseRagasEmbeddings` are the base classes Ragas uses internally for LLMs and Embeddings. Any custom LLM or Embeddings should be a subclass of these base classes. +- **Direct provider support**: OpenAI, Anthropic, Google +- **Other providers via LiteLLM**: Azure OpenAI, AWS Bedrock, Google Vertex AI, and 100+ other providers -- If you are using LangChain, you can pass the LangChain LLM and Embeddings directly and Ragas will wrap it with `LangchainLLMWrapper` or `LangchainEmbeddingsWrapper` as needed. +The factory functions use the [Instructor](https://python.useinstructor.com/) library for structured outputs and [LiteLLM](https://docs.litellm.ai/) for unified access to multiple LLM providers. ## Examples -- [Azure OpenAI](#azure-openai) -- [Google Vertex](#google-vertex) -- [AWS Bedrock](#aws-bedrock) +- [Customize Models](#customize-models) +- [Examples](#examples) + - [Azure OpenAI](#azure-openai) + - [Google Vertex](#google-vertex) + - [AWS Bedrock](#aws-bedrock) ### Azure OpenAI ```bash -pip install langchain_openai +pip install litellm ``` ```python - -from langchain_openai.chat_models import AzureChatOpenAI -from langchain_openai.embeddings import AzureOpenAIEmbeddings -from ragas.llms import LangchainLLMWrapper -from ragas.embeddings import LangchainEmbeddingsWrapper +import litellm +from ragas.llms import llm_factory +from ragas.embeddings.base import embedding_factory azure_configs = { - "base_url": "https://.openai.azure.com/", + "api_base": "https://.openai.azure.com/", + "api_key": "your-api-key", + "api_version": "2024-02-15-preview", "model_deployment": "your-deployment-name", - "model_name": "your-model-name", - "embedding_deployment": "your-deployment-name", - "embedding_name": "text-embedding-ada-002", # most likely + "embedding_deployment": "your-embedding-deployment-name", } - -azure_llm = AzureChatOpenAI( - openai_api_version="2023-05-15", - azure_endpoint=azure_configs["base_url"], - azure_deployment=azure_configs["model_deployment"], - model=azure_configs["model_name"], - validate_base_url=False, +# Configure LiteLLM for Azure OpenAI +litellm.api_base = azure_configs["api_base"] +litellm.api_key = azure_configs["api_key"] +litellm.api_version = azure_configs["api_version"] + +# Create LLM using llm_factory with litellm provider +# Note: Use deployment name, not model name for Azure +azure_llm = llm_factory( + f"azure/{azure_configs['model_deployment']}", + provider="litellm", + client=litellm, ) -# init the embeddings for answer_relevancy, answer_correctness and answer_similarity -azure_embeddings = AzureOpenAIEmbeddings( - openai_api_version="2023-05-15", - azure_endpoint=azure_configs["base_url"], - azure_deployment=azure_configs["embedding_deployment"], - model=azure_configs["embedding_name"], +# Create embeddings using embedding_factory +# Note: Embeddings use the global litellm configuration set above +azure_embeddings = embedding_factory( + "litellm", + model=f"azure/{azure_configs['embedding_deployment']}", ) - -azure_llm = LangchainLLMWrapper(azure_llm) -azure_embeddings = LangchainEmbeddingsWrapper(azure_embeddings) ``` Yay! Now you are ready to use ragas with Azure OpenAI endpoints ### Google Vertex ```bash -!pip install langchain_google_vertexai +pip install litellm google-cloud-aiplatform ``` ```python -import google.auth -from langchain_google_vertexai import ChatVertexAI, VertexAIEmbeddings -from ragas.llms import LangchainLLMWrapper -from ragas.embeddings import LangchainEmbeddingsWrapper -from langchain_core.outputs import LLMResult, ChatGeneration +import litellm +import os +from ragas.llms import llm_factory +from ragas.embeddings.base import embedding_factory config = { "project_id": "", + "location": "us-central1", # e.g., "us-central1", "us-east1" "chat_model_id": "gemini-1.5-pro-002", "embedding_model_id": "text-embedding-005", } -# authenticate to GCP -creds, _ = google.auth.default(quota_project_id=config["project_id"]) +# Set environment variables for Vertex AI +os.environ["VERTEXAI_PROJECT"] = config["project_id"] +os.environ["VERTEXAI_LOCATION"] = config["location"] -# create LangChain LLM and Embeddings -vertextai_llm = ChatVertexAI( - credentials=creds, - model_name=config["chat_model_id"], -) -vertextai_embeddings = VertexAIEmbeddings( - credentials=creds, model_name=config["embedding_model_id"] +# Create LLM using llm_factory with litellm provider +vertex_llm = llm_factory( + f"vertex_ai/{config['chat_model_id']}", + provider="litellm", + client=litellm, ) -# Create a custom is_finished_parser to capture Gemini generation completion signals -def gemini_is_finished_parser(response: LLMResult) -> bool: - is_finished_list = [] - for g in response.flatten(): - resp = g.generations[0][0] - - # Check generation_info first - if resp.generation_info is not None: - finish_reason = resp.generation_info.get("finish_reason") - if finish_reason is not None: - is_finished_list.append( - finish_reason in ["STOP", "MAX_TOKENS"] - ) - continue - - # Check response_metadata as fallback - if isinstance(resp, ChatGeneration) and resp.message is not None: - metadata = resp.message.response_metadata - if metadata.get("finish_reason"): - is_finished_list.append( - metadata["finish_reason"] in ["STOP", "MAX_TOKENS"] - ) - elif metadata.get("stop_reason"): - is_finished_list.append( - metadata["stop_reason"] in ["STOP", "MAX_TOKENS"] - ) - - # If no finish reason found, default to True - if not is_finished_list: - is_finished_list.append(True) - - return all(is_finished_list) - - -vertextai_llm = LangchainLLMWrapper(vertextai_llm, is_finished_parser=gemini_is_finished_parser) -vertextai_embeddings = LangchainEmbeddingsWrapper(vertextai_embeddings) +# Create embeddings using embedding_factory +# Note: Embeddings use the environment variables set above +vertex_embeddings = embedding_factory( + "litellm", + model=f"vertex_ai/{config['embedding_model_id']}", +) ``` Yay! Now you are ready to use ragas with Google VertexAI endpoints ### AWS Bedrock ```bash -pip install langchain_aws +pip install litellm ``` ```python -from langchain_aws import ChatBedrockConverse -from langchain_aws import BedrockEmbeddings -from ragas.llms import LangchainLLMWrapper -from ragas.embeddings import LangchainEmbeddingsWrapper +import litellm +import os +from ragas.llms import llm_factory +from ragas.embeddings.base import embedding_factory config = { - "credentials_profile_name": "your-profile-name", # E.g "default" - "region_name": "your-region-name", # E.g. "us-east-1" - "llm": "your-llm-model-id", # E.g "anthropic.claude-3-5-sonnet-20241022-v2:0" - "embeddings": "your-embedding-model-id", # E.g "amazon.titan-embed-text-v2:0" + "region_name": "us-east-1", # E.g. "us-east-1" + "llm": "anthropic.claude-3-5-sonnet-20241022-v2:0", # Your LLM model ID + "embeddings": "amazon.titan-embed-text-v2:0", # Your embedding model ID "temperature": 0.4, } -bedrock_llm = ChatBedrockConverse( - credentials_profile_name=config["credentials_profile_name"], - region_name=config["region_name"], - base_url=f"https://bedrock-runtime.{config['region_name']}.amazonaws.com", - model=config["llm"], +# Set AWS credentials as environment variables +# Option 1: Use AWS credentials file (~/.aws/credentials) +# Option 2: Set environment variables directly +os.environ["AWS_REGION_NAME"] = config["region_name"] +# os.environ["AWS_ACCESS_KEY_ID"] = "your-access-key" +# os.environ["AWS_SECRET_ACCESS_KEY"] = "your-secret-key" + +# Create LLM using llm_factory with litellm provider +bedrock_llm = llm_factory( + f"bedrock/{config['llm']}", + provider="litellm", + client=litellm, temperature=config["temperature"], ) -# init the embeddings -bedrock_embeddings = BedrockEmbeddings( - credentials_profile_name=config["credentials_profile_name"], - region_name=config["region_name"], - model_id=config["embeddings"], +# Create embeddings using embedding_factory +# Note: Embeddings use the environment variables set above +bedrock_embeddings = embedding_factory( + "litellm", + model=f"bedrock/{config['embeddings']}", ) - -bedrock_llm = LangchainLLMWrapper(bedrock_llm) -bedrock_embeddings = LangchainEmbeddingsWrapper(bedrock_embeddings) ``` Yay! Now you are ready to use ragas with AWS Bedrock endpoints