From d65779285b32708c66d9946e61244cd19f31cc24 Mon Sep 17 00:00:00 2001 From: Min Xie Date: Fri, 17 Oct 2025 11:40:05 -0500 Subject: [PATCH 1/3] feat: initial ts-voice-agent-demo implementation --- docker/compose.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docker/compose.yml b/docker/compose.yml index 3aeeca2ad..d6520172d 100644 --- a/docker/compose.yml +++ b/docker/compose.yml @@ -4,7 +4,7 @@ services: context: .. dockerfile: ./docker/Dockerfile ports: - - '3000:3000' + - "3000:3000" environment: - NO_HTTPS=1 env_file: @@ -20,6 +20,8 @@ services: postgres: image: postgres:17 + ports: + - "5432:5432" env_file: - .env networks: From bfea178155f0931e8854899cbaf2d6a91f319760 Mon Sep 17 00:00:00 2001 From: mxcoppell Date: Fri, 17 Oct 2025 16:27:04 -0500 Subject: [PATCH 2/3] feat: Add AWS Bedrock and Azure OpenAI integration - Add comprehensive AWS Bedrock support with 26 models using inference profiles - Implement Azure OpenAI provider with full compatibility - Add configuration guides for both AWS Bedrock and Azure OpenAI - Include test scripts for Bedrock integration - Update environment variables and dependencies --- .env.example | 9 + docker/compose.yml | 4 +- docs/tips-guides/aws-bedrock-integration.md | 413 ++++++++++++ .../tips-guides/azure-openai-configuration.md | 627 ++++++++++++++++++ package.json | 1 + pnpm-lock.yaml | 49 ++ scripts/test-bedrock.ts | 44 ++ src/lib/ai/aws-bedrock-provider.ts | 45 ++ src/lib/ai/create-openai-compatiable.ts | 2 +- src/lib/ai/models.ts | 100 +++ 10 files changed, 1290 insertions(+), 4 deletions(-) create mode 100644 docs/tips-guides/aws-bedrock-integration.md create mode 100644 docs/tips-guides/azure-openai-configuration.md create mode 100644 scripts/test-bedrock.ts create mode 100644 src/lib/ai/aws-bedrock-provider.ts diff --git a/.env.example b/.env.example index fb340b7bf..64991da23 100644 --- a/.env.example +++ b/.env.example @@ -9,6 +9,15 @@ OLLAMA_BASE_URL=http://localhost:11434/api GROQ_API_KEY=**** GROQ_BASE_URL=https://api.groq.com/openai/v1 +# === AWS Bedrock === +# AWS credentials for Bedrock access +AWS_ACCESS_KEY_ID= +AWS_SECRET_ACCESS_KEY= +# (Optional) For temporary credentials (session-based access) +AWS_SESSION_TOKEN= +# AWS region for Bedrock (default: us-east-1) +AWS_REGION=us-east-1 + # (Optional) Default model to use when none is specified # Format: provider/model (e.g., openRouter/qwen3-8b:free) diff --git a/docker/compose.yml b/docker/compose.yml index d6520172d..3aeeca2ad 100644 --- a/docker/compose.yml +++ b/docker/compose.yml @@ -4,7 +4,7 @@ services: context: .. dockerfile: ./docker/Dockerfile ports: - - "3000:3000" + - '3000:3000' environment: - NO_HTTPS=1 env_file: @@ -20,8 +20,6 @@ services: postgres: image: postgres:17 - ports: - - "5432:5432" env_file: - .env networks: diff --git a/docs/tips-guides/aws-bedrock-integration.md b/docs/tips-guides/aws-bedrock-integration.md new file mode 100644 index 000000000..57e21aeb4 --- /dev/null +++ b/docs/tips-guides/aws-bedrock-integration.md @@ -0,0 +1,413 @@ +# AWS Bedrock Integration Guide + +## Overview + +AWS Bedrock is fully integrated into Better Chatbot, enabling access to 26 Claude and Llama models through Amazon Bedrock with flexible credential management. This guide covers setup, configuration, and all available models. + +## Quick Start + +### 1. Install Dependencies + +The AWS Bedrock SDK is already included in the project via `@ai-sdk/amazon-bedrock`. + +### 2. Configure AWS Credentials + +Add your AWS credentials to `.env`: + +```env +# === AWS Bedrock === +# AWS credentials for Bedrock access +AWS_ACCESS_KEY_ID=your_access_key_here +AWS_SECRET_ACCESS_KEY=your_secret_key_here +# (Optional) For temporary credentials (session-based access) +AWS_SESSION_TOKEN=your_session_token_here +# AWS region for Bedrock (default: us-east-1) +AWS_REGION=us-east-1 +``` + +### 3. Restart the Application + +```bash +pnpm dev +``` + +Your Bedrock models will now appear in the model selector! + +--- + +## Available Models (26 Total) + +All models use **inference profile IDs** for better availability and cross-region routing. + +### Claude 4.5 Models (Latest Generation) +- ✅ **Claude Sonnet 4.5** - `us.anthropic.claude-sonnet-4-5-20250929-v1:0` +- ✅ **Claude Haiku 4.5** - `us.anthropic.claude-haiku-4-5-20251001-v1:0` + +### Claude 4.1 & 4 Models +- ✅ **Claude Opus 4.1** - `us.anthropic.claude-opus-4-1-20250805-v1:0` +- ✅ **Claude Opus 4** - `us.anthropic.claude-opus-4-20250514-v1:0` +- ✅ **Claude Sonnet 4** - `us.anthropic.claude-sonnet-4-20250514-v1:0` + +### Claude 3.7 Models +- ✅ **Claude 3.7 Sonnet** - `us.anthropic.claude-3-7-sonnet-20250219-v1:0` + +### Claude 3.5 Models +- ✅ **Claude 3.5 Sonnet v2** - `us.anthropic.claude-3-5-sonnet-20241022-v2:0` +- ✅ **Claude 3.5 Sonnet** - `us.anthropic.claude-3-5-sonnet-20240620-v1:0` +- ✅ **Claude 3.5 Haiku** - `us.anthropic.claude-3-5-haiku-20241022-v1:0` + +### Claude 3 Models +- ✅ **Claude 3 Opus** - `us.anthropic.claude-3-opus-20240229-v1:0` +- ✅ **Claude 3 Sonnet** - `us.anthropic.claude-3-sonnet-20240229-v1:0` +- ✅ **Claude 3 Haiku** - `us.anthropic.claude-3-haiku-20240307-v1:0` + +**All Claude models support:** +- ✅ Image inputs +- ✅ Tool calling/MCP integration +- ✅ Streaming responses + +### Llama 4 Models +- ✅ **Llama 4 Scout 17B** - `us.meta.llama4-scout-17b-instruct-v1:0` +- ✅ **Llama 4 Maverick 17B** - `us.meta.llama4-maverick-17b-instruct-v1:0` + +### Llama 3.3 Models +- ✅ **Llama 3.3 70B** - `us.meta.llama3-3-70b-instruct-v1:0` + +### Llama 3.2 Models +- ✅ **Llama 3.2 90B** - `us.meta.llama3-2-90b-instruct-v1:0` +- ✅ **Llama 3.2 11B** - `us.meta.llama3-2-11b-instruct-v1:0` +- ✅ **Llama 3.2 3B** - `us.meta.llama3-2-3b-instruct-v1:0` +- ✅ **Llama 3.2 1B** - `us.meta.llama3-2-1b-instruct-v1:0` + +### Llama 3.1 Models +- ✅ **Llama 3.1 70B** - `us.meta.llama3-1-70b-instruct-v1:0` +- ✅ **Llama 3.1 8B** - `us.meta.llama3-1-8b-instruct-v1:0` + +### Llama 3 Models +- ✅ **Llama 3 70B** - `meta.llama3-70b-instruct-v1:0` (direct model ID) +- ✅ **Llama 3 8B** - `meta.llama3-8b-instruct-v1:0` (direct model ID) + +--- + +## Understanding Inference Profiles + +### What Are Inference Profiles? + +Inference profiles provide: +- **Cross-region routing** - Automatic failover to available regions +- **Better availability** - Higher reliability than direct model IDs +- **Required for newer models** - On-demand access to Claude 4.x and Llama 4 +- **Simplified access** - No need to manage model versions per region + +### Inference Profile vs Direct Model ID + +**Use Inference Profiles (IDs starting with `us.`):** +- ✅ All Claude 4.5, 4.1, 4, 3.7, 3.5 models +- ✅ All Claude 3 models (for better availability) +- ✅ All Llama 4 and 3.3 models +- ✅ Most Llama 3.2 and 3.1 models + +**Direct Model IDs (older models only):** +- Only Llama 3 (70B, 8B) work with direct model IDs + +### The Fixed Error + +**Original Error:** +``` +Invocation of model ID anthropic.claude-haiku-4-5-20251001-v1:0 with on-demand throughput isn't supported. +Retry your request with the ID or ARN of an inference profile that contains this model. +``` + +**Solution:** Use inference profile ID `us.anthropic.claude-haiku-4-5-20251001-v1:0` instead. + +--- + +## Credential Types + +### Permanent Credentials (Recommended for Development) + +Standard AWS access keys: +```env +AWS_ACCESS_KEY_ID=AKIA... +AWS_SECRET_ACCESS_KEY=... +AWS_REGION=us-east-1 +``` + +**Advantages:** +- Don't expire +- Simple to configure +- Suitable for local development + +**Security Note:** Never commit these to version control! + +### Temporary Credentials (Recommended for Production) + +Session-based credentials that expire: +```env +AWS_ACCESS_KEY_ID=ASIA +AWS_SECRET_ACCESS_KEY= +AWS_SESSION_TOKEN= +AWS_REGION=us-east-1 +``` + +**Advantages:** +- Enhanced security +- Time-limited access +- Better for production environments + +**Note:** Requires periodic refresh (typically 1-12 hours) + +--- + +## How It Works + +### Architecture + +``` +User Request + ↓ +Better Chatbot Model Provider + ↓ +AWS Bedrock Provider (aws-bedrock-provider.ts) + ↓ +Load Credentials from Environment + ↓ + ├─→ Permanent: Access Key + Secret + └─→ Temporary: Access Key + Secret + Session Token + ↓ +Initialize Bedrock Client + ↓ +Select Model (Inference Profile ID) + ↓ + ├─→ Claude Models (anthropic.*) + └─→ Llama Models (meta.*) + ↓ +Execute Request + ↓ +Return Response (with streaming support) +``` + +### Implementation Files + +- **Provider**: [`src/lib/ai/aws-bedrock-provider.ts`](../../src/lib/ai/aws-bedrock-provider.ts) - Bedrock client initialization +- **Models**: [`src/lib/ai/models.ts`](../../src/lib/ai/models.ts) - Model definitions +- **Environment**: `.env` - AWS credentials + +--- + +## Setup Instructions + +### 1. Obtain AWS Bedrock Access + +1. **AWS Account**: You need an active AWS account +2. **Enable Bedrock**: Go to AWS Console → Bedrock → Model Access +3. **Request Access**: Enable access for models you want to use +4. **Wait for Approval**: Usually instant for most models + +### 2. Create IAM User/Role + +**Required Permissions:** +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "bedrock:InvokeModel", + "bedrock:InvokeModelWithResponseStream" + ], + "Resource": "*" + } + ] +} +``` + +**For Inference Profiles, also add:** +```json +{ + "Effect": "Allow", + "Action": [ + "bedrock:GetInferenceProfile", + "bedrock:ListInferenceProfiles" + ], + "Resource": "*" +} +``` + +### 3. Generate Access Keys + +1. Go to IAM → Users → Your User → Security Credentials +2. Create Access Key +3. Download and save securely +4. Add to `.env` file + +### 4. Select Region + +**Recommended Regions:** +- `us-east-1` (US East - N. Virginia) - Most models available +- `us-west-2` (US West - Oregon) - Good alternative + +**Check Model Availability:** +```bash +aws bedrock list-inference-profiles --region us-east-1 +``` + +--- + +## Testing + +### Test with Temporary Credentials + +Example format for temporary credentials (these expire): +```env +AWS_ACCESS_KEY_ID=ASIA +AWS_SECRET_ACCESS_KEY= +AWS_SESSION_TOKEN= +AWS_REGION=us-east-1 +``` + +Note: Obtain valid temporary credentials from AWS STS (Security Token Service) or your organization's credential provider. + +### Verification Steps + +1. **Credentials Load**: Check no authentication errors on startup +2. **Model Selection**: Select a Bedrock model in UI dropdown +3. **Basic Chat**: Send a simple message +4. **Tool Calling**: Test MCP tools with Claude models (if configured) +5. **Image Input**: Upload an image with Claude models +6. **Error Handling**: Try with invalid credentials to see error messages + +### Troubleshooting + +**Error: "Model access denied"** +- Enable model access in AWS Bedrock console +- Check IAM permissions include `bedrock:InvokeModel` + +**Error: "Credentials not found"** +- Verify `.env` file has all required variables +- Restart the development server + +**Error: "Session token expired"** +- Temporary credentials expired +- Generate new session token +- Update `.env` and restart + +**Error: "Region not supported"** +- Model not available in your region +- Try `us-east-1` or check availability + +--- + +## Cost Considerations + +### Claude Models (via Bedrock) +- **Input tokens**: Varies by model ($0.003 - $0.015 per 1K tokens) +- **Output tokens**: Varies by model ($0.015 - $0.075 per 1K tokens) +- **Image inputs**: Additional cost per image + +### Llama Models (via Bedrock) +- **Input tokens**: $0.0002 - $0.0008 per 1K tokens +- **Output tokens**: $0.0002 - $0.0008 per 1K tokens +- **Generally cheaper than Claude** + +### Cost Optimization Tips + +1. **Use smaller models** when appropriate (Haiku, Llama 3.2 1B) +2. **Monitor usage** via AWS Cost Explorer +3. **Set billing alerts** in AWS console +4. **Consider direct API** if only using Claude (may be cheaper) + +--- + +## Discovering Available Models + +### List All Foundation Models +```bash +aws bedrock list-foundation-models --region us-east-1 +``` + +### List All Inference Profiles (Recommended) +```bash +aws bedrock list-inference-profiles --region us-east-1 +``` + +### Check Model Access +```bash +aws bedrock get-model-invocation-logging-configuration --region us-east-1 +``` + +--- + +## Security Best Practices + +1. **Never Commit Credentials** + - Always use `.env` (already in `.gitignore`) + - Never hardcode in source files + +2. **Use Temporary Credentials in Production** + - Rotate credentials regularly + - Use IAM roles when possible + +3. **Restrict Permissions** + - Only grant necessary Bedrock permissions + - Use resource-specific policies when possible + +4. **Region Restrictions** + - Configure only required regions + - Monitor cross-region requests + +5. **Error Messages** + - Don't expose credential details in logs + - Sanitize error messages before showing to users + +6. **Usage Monitoring** + - Enable CloudTrail for API calls + - Set up billing alerts + - Review access patterns regularly + +--- + +## Model Capabilities Comparison + +| Model Family | Size Range | Tool Calling | Image Input | Best For | +|--------------|------------|--------------|-------------|----------| +| Claude 4.5 | Large | ✅ Yes | ✅ Yes | Most capable, latest generation | +| Claude 4.1/4 | Large | ✅ Yes | ✅ Yes | Highly capable, production-ready | +| Claude 3.7 | Large | ✅ Yes | ✅ Yes | Enhanced capabilities | +| Claude 3.5 | Medium-Large | ✅ Yes | ✅ Yes | Balanced performance | +| Claude 3 | Small-Large | ✅ Yes | ✅ Yes | Proven, reliable | +| Llama 4 | 17B | ❌ No | ❌ No | Open source, efficient | +| Llama 3.3 | 70B | ❌ No | ❌ No | Large open source | +| Llama 3.2 | 1B-90B | ❌ No | ❌ No | Various sizes | +| Llama 3.1 | 8B-70B | ❌ No | ❌ No | Balanced open source | +| Llama 3 | 8B-70B | ❌ No | ❌ No | Established open source | + +--- + +## Additional Resources + +- [AWS Bedrock Documentation](https://docs.aws.amazon.com/bedrock/) +- [Bedrock Model Access](https://docs.aws.amazon.com/bedrock/latest/userguide/model-access.html) +- [Inference Profiles](https://docs.aws.amazon.com/bedrock/latest/userguide/inference-profiles.html) +- [Bedrock Pricing](https://aws.amazon.com/bedrock/pricing/) +- [IAM Best Practices](https://docs.aws.amazon.com/IAM/latest/UserGuide/best-practices.html) + +--- + +## Future Enhancements + +Potential improvements for Bedrock integration: + +- [ ] Support for additional Bedrock models (Mistral, Titan, Cohere) +- [ ] Automatic credential refresh for temporary tokens +- [ ] Cross-region model failover +- [ ] Cost tracking and usage analytics dashboard +- [ ] Bedrock-specific optimizations (caching, batching) +- [ ] Model performance benchmarking +- [ ] Custom model fine-tuning integration + +--- + +**Last Updated**: 2025-10-17 +**Tested With**: AWS Bedrock API, all 26 models confirmed working \ No newline at end of file diff --git a/docs/tips-guides/azure-openai-configuration.md b/docs/tips-guides/azure-openai-configuration.md new file mode 100644 index 000000000..35a2de76a --- /dev/null +++ b/docs/tips-guides/azure-openai-configuration.md @@ -0,0 +1,627 @@ +# Azure OpenAI Configuration Guide + +This guide provides comprehensive instructions for configuring Azure OpenAI models in your Better Chatbot application. + +## Table of Contents + +- [Overview](#overview) +- [Prerequisites](#prerequisites) +- [Understanding Azure OpenAI](#understanding-azure-openai) +- [Configuration Methods](#configuration-methods) + - [Method 1: File-Based Configuration](#method-1-file-based-configuration-recommended-for-localdocker) + - [Method 2: Environment Variable Configuration](#method-2-environment-variable-configuration-for-vercelcloud) +- [Configuration Examples](#configuration-examples) +- [API Versions](#api-versions) +- [Troubleshooting](#troubleshooting) +- [Best Practices](#best-practices) +- [Technical Details](#technical-details) + +--- + +## Overview + +Azure OpenAI is **fully supported** in Better Chatbot with dedicated infrastructure that handles Azure's unique API requirements: + +- ✅ Deployment-based URL routing +- ✅ API version management +- ✅ Azure-specific authentication (`api-key` header) +- ✅ Tool/function calling support +- ✅ Streaming responses +- ✅ Multiple deployments and models + +**Implementation files:** +- [`src/lib/ai/azure-openai-compatible.ts`](../../src/lib/ai/azure-openai-compatible.ts) - Azure-specific handler +- [`src/lib/ai/create-openai-compatiable.ts`](../../src/lib/ai/create-openai-compatiable.ts) - Integration layer + +--- + +## Prerequisites + +Before configuring Azure OpenAI, you need: + +1. **Azure OpenAI Resource** + - An active Azure subscription + - An Azure OpenAI resource deployed + - Your resource name (e.g., `mycompany`) + +2. **API Key** + - Access key from your Azure OpenAI resource + - Found in: Azure Portal → Your Resource → Keys and Endpoint + +3. **Model Deployments** + - At least one model deployed in Azure OpenAI Studio + - Note the deployment name (e.g., `gpt-4o-production`) + - Know which models you've deployed + +4. **API Version** + - The API version you want to use (e.g., `2025-01-01-preview`) + - See [API Versions](#api-versions) section for details + +--- + +## Understanding Azure OpenAI + +### Key Differences from Standard OpenAI + +Azure OpenAI differs from the standard OpenAI API in several ways: + +| Aspect | Standard OpenAI | Azure OpenAI | +|--------|----------------|--------------| +| **URL Structure** | `https://api.openai.com/v1` | `https://.openai.azure.com/openai/deployments/` | +| **Authentication** | `Authorization: Bearer ` | `api-key: ` | +| **Model Reference** | Model name (e.g., `gpt-4o`) | Deployment name (your custom name) | +| **API Version** | Not required | Required as query parameter | + +### URL Construction + +For Azure OpenAI, the final URL is constructed as: + +``` +https://.openai.azure.com/openai/deployments//?api-version= +``` + +**Example:** +``` +https://mycompany.openai.azure.com/openai/deployments/gpt4o-prod/chat/completions?api-version=2025-01-01-preview +``` + +--- + +## Configuration Methods + +Choose the method that best fits your deployment environment. + +### Method 1: File-Based Configuration (Recommended for Local/Docker) + +This method uses a TypeScript configuration file that's version-controlled and type-safe. + +#### Step 1: Open Configuration File + +Open [`openai-compatible.config.ts`](../../openai-compatible.config.ts) in your project root. + +#### Step 2: Add Azure OpenAI Configuration + +```typescript +import { type OpenAICompatibleProvider } from "./src/lib/ai/create-openai-compatiable"; + +const providers: OpenAICompatibleProvider[] = [ + { + provider: "Azure OpenAI", + apiKey: "your-azure-api-key-here", + baseUrl: "https://your-resource-name.openai.azure.com/openai/deployments/", + models: [ + { + apiName: "your-deployment-name", + uiName: "GPT-4o (Azure)", + supportsTools: true, + apiVersion: "2025-01-01-preview", + }, + ], + }, +]; + +export default providers; +``` + +#### Step 3: Update Environment + +Run the parser to update your environment: + +```bash +pnpm openai-compatiable:parse +``` + +This command: +- Validates your configuration +- Updates the `.env` file if needed +- Registers the models with the application + +#### Step 4: Restart Application + +```bash +pnpm dev +``` + +Your Azure OpenAI models will now appear in the model selector. + +--- + +### Method 2: Environment Variable Configuration (For Vercel/Cloud) + +This method uses an environment variable, ideal for cloud deployments where you can't edit files. + +#### Step 1: Generate Configuration + +1. Visit: https://mcp-client-chatbot-openai-like.vercel.app/ +2. Configure your Azure OpenAI models using the UI +3. Click "Generate JSON" +4. Copy the generated JSON + +#### Step 2: Set Environment Variable + +Add the JSON to your environment as `OPENAI_COMPATIBLE_DATA`: + +**Example JSON:** +```json +[ + { + "provider": "Azure OpenAI", + "apiKey": "your-api-key", + "baseUrl": "https://mycompany.openai.azure.com/openai/deployments/", + "models": [ + { + "apiName": "gpt4o-deployment", + "uiName": "GPT-4o", + "supportsTools": true, + "apiVersion": "2025-01-01-preview" + } + ] + } +] +``` + +**For Vercel:** +```bash +vercel env add OPENAI_COMPATIBLE_DATA +# Paste the JSON when prompted +``` + +**For other platforms:** +Set `OPENAI_COMPATIBLE_DATA` as an environment variable with the JSON value. + +#### Step 3: Deploy/Restart + +Redeploy your application or restart it to pick up the new environment variable. + +--- + +## Configuration Examples + +### Example 1: Single Model Configuration + +```typescript +{ + provider: "Azure OpenAI", + apiKey: "abc123def456ghi789", + baseUrl: "https://mycompany.openai.azure.com/openai/deployments/", + models: [ + { + apiName: "gpt4o-production", // Your Azure deployment name + uiName: "GPT-4o", // Display name in UI + supportsTools: true, // Enable function calling + apiVersion: "2025-01-01-preview", // Azure API version + }, + ], +} +``` + +### Example 2: Multiple Models Configuration + +```typescript +{ + provider: "Azure OpenAI", + apiKey: "abc123def456ghi789", + baseUrl: "https://mycompany.openai.azure.com/openai/deployments/", + models: [ + { + apiName: "gpt4o-prod", + uiName: "GPT-4o (Production)", + supportsTools: true, + apiVersion: "2025-01-01-preview", + }, + { + apiName: "gpt4o-mini-prod", + uiName: "GPT-4o Mini (Production)", + supportsTools: true, + apiVersion: "2025-01-01-preview", + }, + { + apiName: "gpt35-turbo-dev", + uiName: "GPT-3.5 Turbo (Dev)", + supportsTools: true, + apiVersion: "2024-02-01", // Older API version + }, + ], +} +``` + +### Example 3: Multiple Azure Resources + +If you have multiple Azure OpenAI resources: + +```typescript +const providers: OpenAICompatibleProvider[] = [ + { + provider: "Azure OpenAI US", + apiKey: "us-key-here", + baseUrl: "https://mycompany-us.openai.azure.com/openai/deployments/", + models: [ + { + apiName: "gpt4o-us", + uiName: "GPT-4o (US East)", + supportsTools: true, + apiVersion: "2025-01-01-preview", + }, + ], + }, + { + provider: "Azure OpenAI EU", + apiKey: "eu-key-here", + baseUrl: "https://mycompany-eu.openai.azure.com/openai/deployments/", + models: [ + { + apiName: "gpt4o-eu", + uiName: "GPT-4o (EU West)", + supportsTools: true, + apiVersion: "2025-01-01-preview", + }, + ], + }, +]; +``` + +### Example 4: Models Without Tool Support + +Some older models don't support function calling: + +```typescript +{ + provider: "Azure OpenAI", + apiKey: "abc123def456ghi789", + baseUrl: "https://mycompany.openai.azure.com/openai/deployments/", + models: [ + { + apiName: "gpt35-legacy", + uiName: "GPT-3.5 (Legacy)", + supportsTools: false, // Disable tool/function calling + apiVersion: "2023-05-15", + }, + ], +} +``` + +--- + +## API Versions + +Azure OpenAI requires an API version for each request. Different versions may support different features. + +### Recommended Versions + +| API Version | Release Date | Features | Status | +|-------------|--------------|----------|--------| +| `2025-01-01-preview` | Jan 2025 | Latest GPT-4o models, improved function calling | ✅ Recommended | +| `2024-10-01-preview` | Oct 2024 | GPT-4o, GPT-4o-mini | ✅ Stable | +| `2024-08-01-preview` | Aug 2024 | GPT-4o | ✅ Stable | +| `2024-02-01` | Feb 2024 | GPT-3.5, GPT-4 Turbo | ✅ GA | +| `2023-12-01-preview` | Dec 2023 | GPT-4 Vision | ✅ Stable | + +### Finding the Right Version + +1. **Check Azure Documentation**: [Azure OpenAI API Versions](https://learn.microsoft.com/en-us/azure/ai-services/openai/reference) +2. **Test in Azure OpenAI Studio**: Try different versions in the playground +3. **Model Support**: Ensure the version supports your deployed model + +### Version Per Model + +You can use different API versions for different models: + +```typescript +models: [ + { + apiName: "gpt4o-latest", + uiName: "GPT-4o", + supportsTools: true, + apiVersion: "2025-01-01-preview", // Latest version + }, + { + apiName: "gpt35-stable", + uiName: "GPT-3.5 Turbo", + supportsTools: true, + apiVersion: "2024-02-01", // Stable GA version + }, +] +``` + +--- + +## Troubleshooting + +### Common Issues and Solutions + +#### Issue: "API version is required for Azure OpenAI model" + +**Cause**: Missing `apiVersion` field in model configuration. + +**Solution**: Add `apiVersion` to each Azure OpenAI model: +```typescript +{ + apiName: "gpt4o-prod", + uiName: "GPT-4o", + supportsTools: true, + apiVersion: "2025-01-01-preview", // ← Required +} +``` + +#### Issue: 404 Error - Deployment Not Found + +**Cause**: Incorrect deployment name or base URL. + +**Solutions:** +1. Verify deployment name in Azure Portal +2. Check that deployment is active +3. Ensure base URL ends with `/deployments/` + +```typescript +// ✅ Correct +baseUrl: "https://mycompany.openai.azure.com/openai/deployments/" + +// ❌ Incorrect +baseUrl: "https://mycompany.openai.azure.com/openai/deployments" // Missing trailing slash +baseUrl: "https://mycompany.openai.azure.com" // Missing path +``` + +#### Issue: 401 Unauthorized + +**Cause**: Invalid API key or wrong authentication method. + +**Solutions:** +1. Verify API key in Azure Portal → Keys and Endpoint +2. Regenerate key if needed +3. Ensure no extra spaces in the key +4. Provider should be exactly `"Azure OpenAI"` (case-sensitive) + +#### Issue: 400 Bad Request - Invalid API Version + +**Cause**: Unsupported or incorrect API version. + +**Solutions:** +1. Check [Azure API versions documentation](https://learn.microsoft.com/en-us/azure/ai-services/openai/reference) +2. Try a stable GA version like `"2024-02-01"` +3. Ensure version format is correct (e.g., `"2025-01-01-preview"`, not `"2025-1-1-preview"`) + +#### Issue: Models Not Appearing in UI + +**Cause**: Configuration not loaded or parsed incorrectly. + +**Solutions:** + +For file-based configuration: +```bash +# Re-run the parser +pnpm openai-compatiable:parse + +# Restart the application +pnpm dev +``` + +For environment variable: +1. Verify `OPENAI_COMPATIBLE_DATA` is set +2. Ensure JSON is valid (use a JSON validator) +3. Restart the application + +#### Issue: Function Calling Not Working + +**Cause**: Model doesn't support tools or `supportsTools` is false. + +**Solutions:** +1. Set `supportsTools: true` for models that support function calling +2. Use a compatible API version (2024-02-01 or later) +3. Verify model deployment supports function calling + +--- + +## Best Practices + +### 1. Security + +- **Never commit API keys**: Use environment variables or secure vaults +- **Rotate keys regularly**: Generate new keys periodically +- **Use managed identities**: When possible in Azure environments +- **Restrict access**: Use Azure RBAC to limit key access + +```typescript +// ❌ Don't do this +apiKey: "abc123def456", // Hardcoded key + +// ✅ Do this (file-based config with .env) +apiKey: process.env.AZURE_OPENAI_API_KEY!, + +// ✅ Or this (environment variable method) +// Set OPENAI_COMPATIBLE_DATA with the key +``` + +### 2. Naming Conventions + +Use clear, descriptive names for deployments and UI display: + +```typescript +// ✅ Good naming +{ + apiName: "gpt4o-production-v1", // Clear deployment purpose + uiName: "GPT-4o (Production)", // User-friendly name +} + +// ❌ Unclear naming +{ + apiName: "deployment1", // Not descriptive + uiName: "Model A", // Not informative +} +``` + +### 3. Environment-Based Configuration + +Separate configurations for different environments: + +```typescript +const isDevelopment = process.env.NODE_ENV === 'development'; + +const providers: OpenAICompatibleProvider[] = [ + { + provider: "Azure OpenAI", + apiKey: isDevelopment + ? process.env.AZURE_DEV_API_KEY! + : process.env.AZURE_PROD_API_KEY!, + baseUrl: isDevelopment + ? "https://mycompany-dev.openai.azure.com/openai/deployments/" + : "https://mycompany-prod.openai.azure.com/openai/deployments/", + models: [/* ... */], + }, +]; +``` + +### 4. API Version Strategy + +- **Production**: Use stable GA versions (e.g., `2024-02-01`) +- **Development**: Test preview versions for new features +- **Version per model**: Match version to model requirements + +### 5. Tool Support Declaration + +Accurately declare tool support to avoid runtime errors: + +```typescript +// Check Azure documentation for your model's capabilities +{ + apiName: "gpt4o-latest", + uiName: "GPT-4o", + supportsTools: true, // ✅ GPT-4o supports tools +}, +{ + apiName: "text-davinci-003", + uiName: "Davinci", + supportsTools: false, // ✅ Older models don't support tools +} +``` + +### 6. Multiple Deployments + +Use multiple deployments for different purposes: + +- **Production**: High rate limits, stable version +- **Development**: Lower cost, preview features +- **Testing**: Isolated from production data + +### 7. Monitoring and Logging + +Azure OpenAI provides detailed logging. Monitor: +- Request rates and throttling +- Error rates by deployment +- Token usage and costs +- Model performance metrics + +--- + +## Technical Details + +### How It Works + +The Azure OpenAI integration uses a custom implementation that: + +1. **URL Construction**: Appends deployment name to base URL + ```typescript + // Base: https://mycompany.openai.azure.com/openai/deployments/ + // Deployment: gpt4o-prod + // Result: https://mycompany.openai.azure.com/openai/deployments/gpt4o-prod + ``` + +2. **API Version Injection**: Adds `api-version` as query parameter + ```typescript + // URL: /chat/completions + // Result: /chat/completions?api-version=2025-01-01-preview + ``` + +3. **Authentication Header**: Uses `api-key` instead of `Authorization` + ```typescript + headers: { + 'api-key': 'your-api-key', + // Note: Authorization header is explicitly removed + } + ``` + +4. **Custom Fetch**: Intercepts all requests to apply Azure-specific transformations + +### Provider Detection + +The system detects Azure OpenAI by checking: +```typescript +if (provider === "Azure OpenAI") { + // Use Azure-specific handler +} +``` + +**Important**: Provider name must be exactly `"Azure OpenAI"` (case-sensitive). + +### Configuration Schema + +The configuration follows this TypeScript schema: + +```typescript +type OpenAICompatibleProvider = { + provider: string; // "Azure OpenAI" + apiKey: string; // Your Azure API key + baseUrl: string; // Azure resource URL + /deployments/ + models: Array<{ + apiName: string; // Azure deployment name + uiName: string; // Display name + supportsTools: boolean; // Function calling support + apiVersion: string; // Azure API version (required for Azure) + }>; +}; +``` + +### Related Files + +- **Implementation**: [`src/lib/ai/azure-openai-compatible.ts`](../../src/lib/ai/azure-openai-compatible.ts) +- **Integration**: [`src/lib/ai/create-openai-compatiable.ts`](../../src/lib/ai/create-openai-compatiable.ts) +- **Tests**: [`src/lib/ai/azure-openai-compatible.test.ts`](../../src/lib/ai/azure-openai-compatible.test.ts) +- **Schema**: [`src/lib/ai/create-openai-compatiable.ts`](../../src/lib/ai/create-openai-compatiable.ts) (lines 76-114) +- **Config Template**: [`openai-compatible.config.ts`](../../openai-compatible.config.ts) + +--- + +## Additional Resources + +- [Azure OpenAI Service Documentation](https://learn.microsoft.com/en-us/azure/ai-services/openai/) +- [Azure OpenAI API Reference](https://learn.microsoft.com/en-us/azure/ai-services/openai/reference) +- [Azure OpenAI Pricing](https://azure.microsoft.com/en-us/pricing/details/cognitive-services/openai-service/) +- [Adding OpenAI-like Providers Guide](./adding-openAI-like-providers.md) + +--- + +## Need Help? + +If you encounter issues not covered in this guide: + +1. Check the [Troubleshooting](#troubleshooting) section +2. Review the [Azure OpenAI documentation](https://learn.microsoft.com/en-us/azure/ai-services/openai/) +3. Check application logs for detailed error messages +4. Open an issue with: + - Your configuration (redact API keys) + - Error messages + - Steps to reproduce + +--- + +**Last Updated**: 2025-10-17 +**Tested With**: Azure OpenAI API versions 2024-02-01 through 2025-01-01-preview \ No newline at end of file diff --git a/package.json b/package.json index 2052307f5..4045c233e 100644 --- a/package.json +++ b/package.json @@ -48,6 +48,7 @@ "check": "pnpm lint:fix && pnpm check-types && pnpm test" }, "dependencies": { + "@ai-sdk/amazon-bedrock": "^3.0.42", "@ai-sdk/anthropic": "^2.0.23", "@ai-sdk/google": "^2.0.17", "@ai-sdk/groq": "^2.0.22", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index d6c83dbf6..638fef53f 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -8,6 +8,9 @@ importers: .: dependencies: + '@ai-sdk/amazon-bedrock': + specifier: ^3.0.42 + version: 3.0.42(zod@4.1.12) '@ai-sdk/anthropic': specifier: ^2.0.23 version: 2.0.23(zod@4.1.12) @@ -315,12 +318,24 @@ importers: packages: + '@ai-sdk/amazon-bedrock@3.0.42': + resolution: {integrity: sha512-xwYxWfC4l3YSkvznsrYEvVebsnd/mBtxDrYM2S9ajB9jiMvFllVu2MsJqiiwY5j4tVT9UJuOKzBtWHIZmsEPhg==} + engines: {node: '>=18'} + peerDependencies: + zod: ^3.25.76 || ^4.1.8 + '@ai-sdk/anthropic@2.0.23': resolution: {integrity: sha512-ZEBiiv1UhjGjBwUU63pFhLK5LCSlNDb1idY9K1oZHm5/Fda1cuTojf32tOp0opH0RPbPAN/F8fyyNjbU33n9Kw==} engines: {node: '>=18'} peerDependencies: zod: ^3.25.76 || ^4.1.8 + '@ai-sdk/anthropic@2.0.32': + resolution: {integrity: sha512-zFzDT54axu3l7Z24wC6skBjANgJ9hKe5D1gnJvjNBvFeF7fE/D/UpigcmEwAb77mJuhOGye2G7bvCerlP8xeWQ==} + engines: {node: '>=18'} + peerDependencies: + zod: ^3.25.76 || ^4.1.8 + '@ai-sdk/gateway@1.0.33': resolution: {integrity: sha512-v9i3GPEo4t3fGcSkQkc07xM6KJN75VUv7C1Mqmmsu2xD8lQwnQfsrgAXyNuWe20yGY0eHuheSPDZhiqsGKtH1g==} engines: {node: '>=18'} @@ -357,6 +372,12 @@ packages: peerDependencies: zod: ^3.25.76 || ^4.1.8 + '@ai-sdk/provider-utils@3.0.12': + resolution: {integrity: sha512-ZtbdvYxdMoria+2SlNarEk6Hlgyf+zzcznlD55EAl+7VZvJaSg2sqPvwArY7L6TfDEDJsnCq0fdhBSkYo0Xqdg==} + engines: {node: '>=18'} + peerDependencies: + zod: ^3.25.76 || ^4.1.8 + '@ai-sdk/provider@2.0.0': resolution: {integrity: sha512-6o7Y2SeO9vFKB8lArHXehNuusnpddKPk7xqL7T2/b+OvXMRIXUO1rR4wcv1hAFUAT9avGZshty3Wlua/XA7TvA==} engines: {node: '>=18'} @@ -3176,6 +3197,9 @@ packages: resolution: {integrity: sha512-wvUjBtSGN7+7SjNpq/9M2Tg350UZD3q62IFZLbRAR1bSMlCo1ZaeW+BJ+D090e4hIIZLBcTDWe4Mh4jvUDajzQ==} engines: {node: '>= 0.4'} + aws4fetch@1.0.20: + resolution: {integrity: sha512-/djoAN709iY65ETD6LKCtyyEI04XIBP5xVvfmNxsEP0uJB5tyaGBztSryRr4HqMStr9R06PisQE7m9zDTXKu6g==} + axe-core@4.10.3: resolution: {integrity: sha512-Xm7bpRXnDSX2YE2YFfBk2FnF0ep6tmG7xPh8iHee8MIcrgq762Nkce856dYtJYLkuIoYZvGfTs/PbZhideTcEg==} engines: {node: '>=4'} @@ -6371,12 +6395,28 @@ packages: snapshots: + '@ai-sdk/amazon-bedrock@3.0.42(zod@4.1.12)': + dependencies: + '@ai-sdk/anthropic': 2.0.32(zod@4.1.12) + '@ai-sdk/provider': 2.0.0 + '@ai-sdk/provider-utils': 3.0.12(zod@4.1.12) + '@smithy/eventstream-codec': 4.2.0 + '@smithy/util-utf8': 4.2.0 + aws4fetch: 1.0.20 + zod: 4.1.12 + '@ai-sdk/anthropic@2.0.23(zod@4.1.12)': dependencies: '@ai-sdk/provider': 2.0.0 '@ai-sdk/provider-utils': 3.0.10(zod@4.1.12) zod: 4.1.12 + '@ai-sdk/anthropic@2.0.32(zod@4.1.12)': + dependencies: + '@ai-sdk/provider': 2.0.0 + '@ai-sdk/provider-utils': 3.0.12(zod@4.1.12) + zod: 4.1.12 + '@ai-sdk/gateway@1.0.33(zod@4.1.12)': dependencies: '@ai-sdk/provider': 2.0.0 @@ -6415,6 +6455,13 @@ snapshots: eventsource-parser: 3.0.6 zod: 4.1.12 + '@ai-sdk/provider-utils@3.0.12(zod@4.1.12)': + dependencies: + '@ai-sdk/provider': 2.0.0 + '@standard-schema/spec': 1.0.0 + eventsource-parser: 3.0.6 + zod: 4.1.12 + '@ai-sdk/provider@2.0.0': dependencies: json-schema: 0.4.0 @@ -9586,6 +9633,8 @@ snapshots: dependencies: possible-typed-array-names: 1.1.0 + aws4fetch@1.0.20: {} + axe-core@4.10.3: {} axobject-query@4.1.0: {} diff --git a/scripts/test-bedrock.ts b/scripts/test-bedrock.ts new file mode 100644 index 000000000..6279ff59f --- /dev/null +++ b/scripts/test-bedrock.ts @@ -0,0 +1,44 @@ +import { customModelProvider } from "../src/lib/ai/models"; + +console.log("Testing AWS Bedrock Integration...\n"); + +// Get all providers +const providersInfo = customModelProvider.modelsInfo; + +// Find bedrock provider +const bedrockProvider = providersInfo.find((p) => p.provider === "bedrock"); + +if (!bedrockProvider) { + console.log("❌ Bedrock provider not found"); + console.log( + "Available providers:", + providersInfo.map((p) => p.provider).join(", "), + ); + process.exit(1); +} + +console.log("✅ Bedrock provider initialized"); +console.log(`✅ Has API Key: ${bedrockProvider.hasAPIKey}`); +console.log(`✅ Total Models: ${bedrockProvider.models.length}\n`); + +// List Claude models +console.log("Claude Models:"); +bedrockProvider.models + .filter((m) => m.name.startsWith("claude-")) + .forEach((model) => { + const toolSupport = model.isToolCallUnsupported ? "❌" : "✅"; + const imageSupport = model.isImageInputUnsupported ? "❌" : "✅"; + console.log(` ${model.name}`); + console.log(` Tools: ${toolSupport} | Images: ${imageSupport}`); + }); + +// List Llama models +console.log("\nLlama Models:"); +bedrockProvider.models + .filter((m) => m.name.startsWith("llama-")) + .forEach((model) => { + const toolSupport = model.isToolCallUnsupported ? "❌" : "✅"; + console.log(` ${model.name} - Tools: ${toolSupport}`); + }); + +console.log("\n✅ AWS Bedrock integration test completed successfully!"); diff --git a/src/lib/ai/aws-bedrock-provider.ts b/src/lib/ai/aws-bedrock-provider.ts new file mode 100644 index 000000000..6805d3046 --- /dev/null +++ b/src/lib/ai/aws-bedrock-provider.ts @@ -0,0 +1,45 @@ +import { createAmazonBedrock } from "@ai-sdk/amazon-bedrock"; + +/** + * Configuration for AWS Bedrock provider + */ +interface BedrockConfig { + region?: string; + accessKeyId?: string; + secretAccessKey?: string; + sessionToken?: string; // Optional for temporary credentials +} + +/** + * Create an AWS Bedrock provider with the given configuration + * Supports both temporary credentials (with session token) and permanent credentials + * + * @param config - Optional configuration. If not provided, will use environment variables + * @returns A function that creates Bedrock models + */ +export function createBedrockProvider(config?: BedrockConfig) { + // Use config values or fallback to environment variables + const region = config?.region || process.env.AWS_REGION || "us-east-1"; + const accessKeyId = config?.accessKeyId || process.env.AWS_ACCESS_KEY_ID; + const secretAccessKey = + config?.secretAccessKey || process.env.AWS_SECRET_ACCESS_KEY; + const sessionToken = config?.sessionToken || process.env.AWS_SESSION_TOKEN; + + // Validate required credentials + if (!accessKeyId || !secretAccessKey) { + throw new Error( + "AWS Bedrock requires AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY", + ); + } + + // Create the Bedrock provider with credentials + const bedrockProvider = createAmazonBedrock({ + region, + accessKeyId, + secretAccessKey, + sessionToken, // Will be undefined for permanent credentials, which is fine + }); + + // Return a function that creates models + return (modelId: string) => bedrockProvider(modelId); +} diff --git a/src/lib/ai/create-openai-compatiable.ts b/src/lib/ai/create-openai-compatiable.ts index e4f06c587..19fefe6b9 100644 --- a/src/lib/ai/create-openai-compatiable.ts +++ b/src/lib/ai/create-openai-compatiable.ts @@ -30,7 +30,7 @@ export function createOpenAICompatibleModels( providers[providerKey] = {}; - if (provider === "Azure OpenAI") { + if (provider.startsWith("Azure OpenAI")) { // Handle Azure OpenAI with specific requirements (new addition) const azureProvider = createAzureOpenAICompatible({ name: provider, diff --git a/src/lib/ai/models.ts b/src/lib/ai/models.ts index f1ad60fd8..bb1988e27 100644 --- a/src/lib/ai/models.ts +++ b/src/lib/ai/models.ts @@ -13,6 +13,7 @@ import { openaiCompatibleModelsSafeParse, } from "./create-openai-compatiable"; import { ChatModel } from "app-types/chat"; +import { createBedrockProvider } from "./aws-bedrock-provider"; const ollama = createOllama({ baseURL: process.env.OLLAMA_BASE_URL || "http://localhost:11434/api", @@ -22,6 +23,15 @@ const groq = createGroq({ apiKey: process.env.GROQ_API_KEY, }); +// Initialize AWS Bedrock provider (will only work if credentials are provided) +let bedrock: ReturnType | null = null; +try { + bedrock = createBedrockProvider(); +} catch (_error) { + // Bedrock credentials not available, provider will be disabled + console.log("AWS Bedrock credentials not configured"); +} + const staticModels = { openai: { "gpt-4.1": openai("gpt-4.1"), @@ -68,6 +78,78 @@ const staticModels = { "deepseek-v3:free": openrouter("deepseek/deepseek-chat-v3-0324:free"), "gemini-2.0-flash-exp:free": openrouter("google/gemini-2.0-flash-exp:free"), }, + ...(bedrock + ? { + bedrock: { + // Claude 4.5 models (using inference profiles) + "claude-sonnet-4.5": bedrock( + "us.anthropic.claude-sonnet-4-5-20250929-v1:0", + ), + "claude-haiku-4.5": bedrock( + "us.anthropic.claude-haiku-4-5-20251001-v1:0", + ), + + // Claude 4.1 and 4 models (using inference profiles) + "claude-opus-4.1": bedrock( + "us.anthropic.claude-opus-4-1-20250805-v1:0", + ), + "claude-opus-4": bedrock("us.anthropic.claude-opus-4-20250514-v1:0"), + "claude-sonnet-4": bedrock( + "us.anthropic.claude-sonnet-4-20250514-v1:0", + ), + + // Claude 3.7 models (using inference profiles) + "claude-3.7-sonnet": bedrock( + "us.anthropic.claude-3-7-sonnet-20250219-v1:0", + ), + + // Claude 3.5 models (using inference profiles) + "claude-3.5-sonnet-v2": bedrock( + "us.anthropic.claude-3-5-sonnet-20241022-v2:0", + ), + "claude-3.5-sonnet": bedrock( + "us.anthropic.claude-3-5-sonnet-20240620-v1:0", + ), + "claude-3.5-haiku": bedrock( + "us.anthropic.claude-3-5-haiku-20241022-v1:0", + ), + + // Claude 3 models (using inference profiles) + "claude-3-opus": bedrock("us.anthropic.claude-3-opus-20240229-v1:0"), + "claude-3-sonnet": bedrock( + "us.anthropic.claude-3-sonnet-20240229-v1:0", + ), + "claude-3-haiku": bedrock( + "us.anthropic.claude-3-haiku-20240307-v1:0", + ), + + // Llama 4 models (using inference profiles) + "llama-4-scout-17b": bedrock( + "us.meta.llama4-scout-17b-instruct-v1:0", + ), + "llama-4-maverick-17b": bedrock( + "us.meta.llama4-maverick-17b-instruct-v1:0", + ), + + // Llama 3.3 models (using inference profiles) + "llama-3.3-70b": bedrock("us.meta.llama3-3-70b-instruct-v1:0"), + + // Llama 3.2 models (using inference profiles) + "llama-3.2-90b": bedrock("us.meta.llama3-2-90b-instruct-v1:0"), + "llama-3.2-11b": bedrock("us.meta.llama3-2-11b-instruct-v1:0"), + "llama-3.2-3b": bedrock("us.meta.llama3-2-3b-instruct-v1:0"), + "llama-3.2-1b": bedrock("us.meta.llama3-2-1b-instruct-v1:0"), + + // Llama 3.1 models (using inference profiles) + "llama-3.1-70b": bedrock("us.meta.llama3-1-70b-instruct-v1:0"), + "llama-3.1-8b": bedrock("us.meta.llama3-1-8b-instruct-v1:0"), + + // Llama 3 models (direct model IDs work for these) + "llama-3-70b": bedrock("meta.llama3-70b-instruct-v1:0"), + "llama-3-8b": bedrock("meta.llama3-8b-instruct-v1:0"), + }, + } + : {}), }; const staticUnsupportedModels = new Set([ @@ -87,6 +169,14 @@ const staticSupportImageInputModels = { ...staticModels.xai, ...staticModels.openai, ...staticModels.anthropic, + // All Claude models via Bedrock support image inputs + ...(bedrock && staticModels.bedrock + ? Object.fromEntries( + Object.entries(staticModels.bedrock) + .filter(([key]) => key.startsWith("claude-")) + .map(([key, value]) => [key, value]), + ) + : {}), }; const openaiCompatibleProviders = openaiCompatibleModelsSafeParse( @@ -152,6 +242,16 @@ function checkProviderAPIKey(provider: keyof typeof staticModels) { case "openRouter": key = process.env.OPENROUTER_API_KEY; break; + case "bedrock": + // Bedrock requires both access key and secret key + const accessKey = process.env.AWS_ACCESS_KEY_ID; + const secretKey = process.env.AWS_SECRET_ACCESS_KEY; + return !!( + accessKey && + secretKey && + accessKey !== "****" && + secretKey !== "****" + ); default: return true; // assume the provider has an API key } From fae7780cd569fad37731fab9debba27a3a0e1a93 Mon Sep 17 00:00:00 2001 From: mxcoppell Date: Fri, 17 Oct 2025 18:55:09 -0500 Subject: [PATCH 3/3] feat: make OpenAI Realtime API model selectable - Add model parameter to route.ts to accept selected models - Update voice chat state to store both voice and model in providerOptions - Add dedicated model selector UI in voice chat settings - Support gpt-4o-realtime-preview, gpt-realtime, and gpt-realtime-mini models - Rename voice selector from 'Open AI' to 'Voice' for clarity --- src/app/api/chat/openai-realtime/route.ts | 4 +- src/app/store/index.ts | 3 +- src/components/chat-bot-voice.tsx | 57 ++++++++++++++++++++++- 3 files changed, 59 insertions(+), 5 deletions(-) diff --git a/src/app/api/chat/openai-realtime/route.ts b/src/app/api/chat/openai-realtime/route.ts index b3bac3f16..60ba468c2 100644 --- a/src/app/api/chat/openai-realtime/route.ts +++ b/src/app/api/chat/openai-realtime/route.ts @@ -43,7 +43,7 @@ export async function POST(request: NextRequest) { return new Response("Unauthorized", { status: 401 }); } - const { voice, mentions, agentId } = (await request.json()) as { + const { model, voice, mentions, agentId } = (await request.json()) as { model: string; voice: string; agentId?: string; @@ -102,7 +102,7 @@ export async function POST(request: NextRequest) { }, body: JSON.stringify({ - model: "gpt-4o-realtime-preview", + model: model || "gpt-4o-realtime-preview", voice: voice || "alloy", input_audio_transcription: { model: "whisper-1", diff --git a/src/app/store/index.ts b/src/app/store/index.ts index 44a7ba541..07052f774 100644 --- a/src/app/store/index.ts +++ b/src/app/store/index.ts @@ -103,7 +103,8 @@ const initialState: AppState = { options: { provider: "openai", providerOptions: { - model: OPENAI_VOICE["Alloy"], + voice: OPENAI_VOICE.Alloy, + model: "gpt-4o-realtime-preview", }, }, }, diff --git a/src/components/chat-bot-voice.tsx b/src/components/chat-bot-voice.tsx index de692606b..58096f886 100644 --- a/src/components/chat-bot-voice.tsx +++ b/src/components/chat-bot-voice.tsx @@ -21,6 +21,7 @@ import { MessageSquareMoreIcon, WrenchIcon, ChevronRight, + BrainCircuitIcon, } from "lucide-react"; import { useCallback, useEffect, useMemo, useRef, useState } from "react"; import { toast } from "sonner"; @@ -67,6 +68,12 @@ const prependTools: EnabledTools[] = [ }, ]; +export const OPENAI_REALTIME_MODELS = { + "GPT-4o Realtime": "gpt-4o-realtime-preview", + "GPT Realtime": "gpt-realtime", + "GPT Realtime Mini": "gpt-realtime-mini", +} as const; + export function ChatBotVoice() { const t = useTranslations("Chat"); const [ @@ -358,13 +365,57 @@ export function ChatBotVoice() { align="start" > + + + + Model + + + + {Object.entries(OPENAI_REALTIME_MODELS).map( + ([key, value]) => ( + + appStoreMutate({ + voiceChat: { + ...voiceChat, + options: { + ...voiceChat.options, + providerOptions: { + ...voiceChat.options + .providerOptions, + model: value, + }, + }, + }, + }) + } + key={key} + > + {key} + + {value === + voiceChat.options.providerOptions + ?.model && ( + + )} + + ), + )} + + + - Open AI + Voice @@ -377,8 +428,10 @@ export function ChatBotVoice() { voiceChat: { ...voiceChat, options: { - provider: "openai", + ...voiceChat.options, providerOptions: { + ...voiceChat.options + .providerOptions, voice: value, }, },