diff --git a/integrations/llms/ai21.mdx b/integrations/llms/ai21.mdx index 1dd46e15..8aeaa4be 100644 --- a/integrations/llms/ai21.mdx +++ b/integrations/llms/ai21.mdx @@ -30,13 +30,13 @@ print(response.choices[0].message.content) ``` ```js Javascript icon="square-js" - import Portkey from 'portkey-ai' +import Portkey from 'portkey-ai' // 1. Install: npm install portkey-ai // 2. Add @ai21 provider in model catalog // 3. Use it: - const portkey = new Portkey({ +const portkey = new Portkey({ apiKey: "PORTKEY_API_KEY" }) @@ -48,7 +48,7 @@ const response = await portkey.chat.completions.create({ console.log(response.choices[0].message.content) ``` -```python OpenAI Py icon="openai" +```python OpenAI Py icon="python" from openai import OpenAI from portkey_ai import PORTKEY_GATEWAY_URL @@ -69,7 +69,7 @@ response = client.chat.completions.create( print(response.choices[0].message.content) ``` -```js OpenAI JS icon="openai" +```js OpenAI JS icon="square-js" import OpenAI from "openai" import { PORTKEY_GATEWAY_URL } from "portkey-ai" diff --git a/integrations/llms/anthropic.mdx b/integrations/llms/anthropic.mdx index cecbb212..f81c74d2 100644 --- a/integrations/llms/anthropic.mdx +++ b/integrations/llms/anthropic.mdx @@ -63,7 +63,7 @@ const response = await portkey.chat.completions.create({ console.log(response.choices[0].message.content) ``` -```python OpenAI Py icon="openai" +```python OpenAI Py icon="python" from openai import OpenAI from portkey_ai import PORTKEY_GATEWAY_URL @@ -85,7 +85,7 @@ response = client.chat.completions.create( print(response.choices[0].message.content) ``` -```js OpenAI JS icon="openai" +```js OpenAI JS icon="square-js" import OpenAI from "openai" import { PORTKEY_GATEWAY_URL } from "portkey-ai" diff --git a/integrations/llms/anyscale-llama2-mistral-zephyr.mdx b/integrations/llms/anyscale-llama2-mistral-zephyr.mdx index 041cf113..cdbe17d0 100644 --- a/integrations/llms/anyscale-llama2-mistral-zephyr.mdx +++ b/integrations/llms/anyscale-llama2-mistral-zephyr.mdx @@ -45,7 +45,7 @@ const response = await portkey.chat.completions.create({ console.log(response.choices[0].message.content) ``` -```python OpenAI Py icon="openai" +```python OpenAI Py icon="python" from openai import OpenAI from portkey_ai import PORTKEY_GATEWAY_URL @@ -66,7 +66,7 @@ response = client.chat.completions.create( print(response.choices[0].message.content) ``` -```js OpenAI JS icon="openai" +```js OpenAI JS icon="square-js" import OpenAI from "openai" import { PORTKEY_GATEWAY_URL } from "portkey-ai" diff --git a/integrations/llms/cerebras.mdx b/integrations/llms/cerebras.mdx index 11152aaa..a37212a0 100644 --- a/integrations/llms/cerebras.mdx +++ b/integrations/llms/cerebras.mdx @@ -1,107 +1,153 @@ --- title: "Cerebras" +description: "Integrate Cerebras models with Portkey's AI Gateway" --- -Portkey provides a robust and secure gateway to facilitate the integration of various Large Language Models (LLMs) into your applications, including the models hosted on [Cerebras Inference API](https://cerebras.ai/inference). +Portkey provides a robust and secure gateway to integrate various Large Language Models (LLMs) into applications, including [Cerebras Inference API](https://cerebras.ai/inference). - -Provider Slug: `cerebras` - +With Portkey, take advantage of features like fast AI gateway access, observability, prompt management, and more, while securely managing API keys through [Model Catalog](/product/model-catalog). -## Portkey SDK Integration with Cerebras +## Quick Start -Portkey provides a consistent API to interact with models from various providers. To integrate Cerebras with Portkey: +Get Cerebras working in 3 steps: -### 1\. Install the Portkey SDK - - - ```sh - npm install --save portkey-ai - ``` - - - ```sh - pip install portkey-ai - ``` - + +```python Python icon="python" +from portkey_ai import Portkey - +# 1. Install: pip install portkey-ai +# 2. Add @cerebras provider in model catalog +# 3. Use it: +portkey = Portkey(api_key="PORTKEY_API_KEY") +response = portkey.chat.completions.create( + model="@cerebras/llama3.1-8b", + messages=[{"role": "user", "content": "Say this is a test"}] +) -### 2\. Initialize Portkey with Cerebras +print(response.choices[0].message.content) +``` -To use Cerebras with Portkey, get your API key from [here](https://cerebras.ai/inference), then add it to Portkey to create the virtual key. - - - ```js - import Portkey from 'portkey-ai' +```js Javascript icon="square-js" +import Portkey from 'portkey-ai' - const portkey = new Portkey({ - apiKey: "PORTKEY_API_KEY", // defaults to process.env["PORTKEY_API_KEY"] - provider:"@CEREBRAS_PROVIDER" // Your Cerebras Inference virtual key - }) - ``` - - - ```py - from portkey_ai import Portkey - - portkey = Portkey( - api_key ="PORTKEY_API_KEY", # Replace with your Portkey API key - provider="@CEREBRAS_PROVIDER" # Your Cerebras Inference virtual key - ) - ``` - +// 1. Install: npm install portkey-ai +// 2. Add @cerebras provider in model catalog +// 3. Use it: + +const portkey = new Portkey({ + apiKey: "PORTKEY_API_KEY" +}) - +const response = await portkey.chat.completions.create({ + model: "@cerebras/llama3.1-8b", + messages: [{ role: "user", content: "Say this is a test" }] +}) +console.log(response.choices[0].message.content) + ``` +```python OpenAI Py icon="python" +from openai import OpenAI +from portkey_ai import PORTKEY_GATEWAY_URL + +# 1. Install: pip install openai portkey-ai +# 2. Add @cerebras provider in model catalog +# 3. Use it: + +client = OpenAI( + api_key="PORTKEY_API_KEY", # Portkey API key + base_url=PORTKEY_GATEWAY_URL +) + +response = client.chat.completions.create( + model="@cerebras/llama3.1-8b", + messages=[{"role": "user", "content": "Say this is a test"}] +) + +print(response.choices[0].message.content) +``` + +```js OpenAI JS icon="square-js" +import OpenAI from "openai" +import { PORTKEY_GATEWAY_URL } from "portkey-ai" + +// 1. Install: npm install openai portkey-ai +// 2. Add @cerebras provider in model catalog +// 3. Use it: + +const client = new OpenAI({ + apiKey: "PORTKEY_API_KEY", // Portkey API key + baseURL: PORTKEY_GATEWAY_URL +}) + +const response = await client.chat.completions.create({ + model: "@cerebras/llama3.1-8b", + messages: [{ role: "user", content: "Say this is a test" }] +}) + +console.log(response.choices[0].message.content) +``` + +```sh cURL icon="square-terminal" +# 1. Add @cerebras provider in model catalog +# 2. Use it: + +curl https://api.portkey.ai/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "x-portkey-api-key: $PORTKEY_API_KEY" \ + -d '{ + "model": "@cerebras/llama3.1-8b", + "messages": [ + { "role": "user", "content": "Say this is a test" } + ] + }' +``` + + +**Tip:** You can also set `provider="@cerebras"` in `Portkey()` and use just `model="llama3.1-8b"` in the request. + -### 3\. Invoke Chat Completions - - - ```js - const chatCompletion = await portkey.chat.completions.create({ - messages: [{ role: 'user', content: 'Say this is a test' }], - model: 'llama3.1-8b', - }); +## Add Provider in Model Catalog - console.log(chatCompletion.choices); - ``` - - - ```python - completion = portkey.chat.completions.create( - messages= [{ "role": 'user', "content": 'Say this is a test' }], - model= 'llama3.1-8b' - ) - - print(completion) - ``` - +1. Go to [**Model Catalog → Add Provider**](https://app.portkey.ai/model-catalog/providers) +2. Select **Cerebras** +3. Choose existing credentials or create new by entering your [Cerebras API key](https://cerebras.ai/inference) +4. Name your provider (e.g., `cerebras-prod`) - + + See all setup options, code examples, and detailed instructions + --- ## Supported Models -Cerebras currently supports `Llama-3.1-8B` and `Llama-3.1-70B`. You can find more info here: - -[![Logo](/images/llms/apple-touch-icon.png)Overview - Starter KitStarter Kit](https://inference-docs.cerebras.ai/introduction) - -## Next Steps - -The complete list of features supported in the SDK are available on the link below. - - + + View all available models and documentation -You'll find more information in the relevant sections: +## Next Steps -1. [Add metadata to your requests](/product/observability/metadata) -2. [Add gateway configs to your Cerebras](/product/ai-gateway/configs)[ requests](/product/ai-gateway/configs) -3. [Tracing Cerebras requests](/product/observability/traces) -4. [Setup a fallback from OpenAI to Cerebras](/product/ai-gateway/fallbacks) + + + Add metadata to your Cerebras requests + + + Add gateway configs to your Cerebras requests + + + Trace your Cerebras requests + + + Setup fallback from OpenAI to Cerebras + + + +For complete SDK documentation: + + + Complete Portkey SDK documentation + diff --git a/integrations/llms/cohere.mdx b/integrations/llms/cohere.mdx index 703ca095..ce721f98 100644 --- a/integrations/llms/cohere.mdx +++ b/integrations/llms/cohere.mdx @@ -1,137 +1,179 @@ --- title: "Cohere" +description: "Integrate Cohere models with Portkey's AI Gateway" --- -Portkey provides a robust and secure gateway to facilitate the integration of various Large Language Models (LLMs) into your applications, including Cohere's generation, embedding, and other endpoints. +Portkey provides a robust and secure gateway to integrate various Large Language Models (LLMs) into applications, including Cohere's generation, embedding, and reranking endpoints. -With Portkey, you can take advantage of features like fast AI gateway access, observability, prompt management, and more, all while ensuring the secure management of your LLM API keys through a [virtual key](/product/ai-gateway/virtual-keys) system. +With Portkey, take advantage of features like fast AI gateway access, observability, prompt management, and more, while securely managing API keys through [Model Catalog](/product/model-catalog). - -Provider Slug. `cohere` - -## Portkey SDK Integration with Cohere +## Quick Start -Portkey provides a consistent API to interact with models from Cohere. To integrate Cohere with Portkey: +Get Cohere working in 3 steps: -### 1\. Install the Portkey SDK + +```python Python icon="python" +from portkey_ai import Portkey -Add the Portkey SDK to your application to interact with Cohere's models through Portkey's gateway. - - - ```sh - npm install --save portkey-ai - ``` - - - ```sh - pip install portkey-ai - ``` - - +# 1. Install: pip install portkey-ai +# 2. Add @cohere provider in model catalog +# 3. Use it: +portkey = Portkey(api_key="PORTKEY_API_KEY") +response = portkey.chat.completions.create( + model="@cohere/command-r-plus", + messages=[{"role": "user", "content": "Say this is a test"}] +) +print(response.choices[0].message.content) +``` -### 2\. Initialize Portkey with the Virtual Key +```js Javascript icon="square-js" +import Portkey from 'portkey-ai' -To use Cohere with Portkey, [get your API key from here](https://dashboard.cohere.com/api-keys), then add it to Portkey to create the virtual key. +// 1. Install: npm install portkey-ai +// 2. Add @cohere provider in model catalog +// 3. Use it: - - - ```js - import Portkey from 'portkey-ai' +const portkey = new Portkey({ + apiKey: "PORTKEY_API_KEY" +}) - const portkey = new Portkey({ - apiKey: "PORTKEY_API_KEY", // defaults to process.env["PORTKEY_API_KEY"] - provider:"@PROVIDER" // Your Cohere Virtual Key - }) - ``` - - - ```python - from portkey_ai import Portkey - - portkey = Portkey( - api_key="PORTKEY_API_KEY", # Replace with your Portkey API key - provider="@PROVIDER" # Replace with your virtual key for Cohere - ) - ``` - - +const response = await portkey.chat.completions.create({ + model: "@cohere/command-r-plus", + messages: [{ role: "user", content: "Say this is a test" }] +}) +console.log(response.choices[0].message.content) +``` -### **3\. Invoke Chat Completions with Cohere** +```python OpenAI Py icon="python" +from openai import OpenAI +from portkey_ai import PORTKEY_GATEWAY_URL -Use the Portkey instance to send requests to Cohere's models. You can also override the virtual key directly in the API call if needed. +# 1. Install: pip install openai portkey-ai +# 2. Add @cohere provider in model catalog +# 3. Use it: - - - ```js - const chatCompletion = await portkey.chat.completions.create({ - messages: [{ role: 'user', content: 'Say this is a test' }], - model: 'command', - }); +client = OpenAI( + api_key="PORTKEY_API_KEY", # Portkey API key + base_url=PORTKEY_GATEWAY_URL +) - console.log(chatCompletion.choices); - ``` - - - ``` - chat_completion = portkey.chat.completions.create( - messages= [{ "role": 'user', "content": 'Say this is a test' }], - model= 'command' - ) +response = client.chat.completions.create( + model="@cohere/command-r-plus", + messages=[{"role": "user", "content": "Say this is a test"}] +) + +print(response.choices[0].message.content) +``` + +```js OpenAI JS icon="square-js" +import OpenAI from "openai" +import { PORTKEY_GATEWAY_URL } from "portkey-ai" + +// 1. Install: npm install openai portkey-ai +// 2. Add @cohere provider in model catalog +// 3. Use it: + +const client = new OpenAI({ + apiKey: "PORTKEY_API_KEY", // Portkey API key + baseURL: PORTKEY_GATEWAY_URL +}) + +const response = await client.chat.completions.create({ + model: "@cohere/command-r-plus", + messages: [{ role: "user", content: "Say this is a test" }] +}) + +console.log(response.choices[0].message.content) +``` + +```sh cURL icon="square-terminal" +# 1. Add @cohere provider in model catalog +# 2. Use it: + +curl https://api.portkey.ai/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "x-portkey-api-key: $PORTKEY_API_KEY" \ + -d '{ + "model": "@cohere/command-r-plus", + "messages": [ + { "role": "user", "content": "Say this is a test" } + ] + }' ``` - - + + +**Tip:** You can also set `provider="@cohere"` in `Portkey()` and use just `model="command-r-plus"` in the request. + -## Managing Cohere Prompts +## Add Provider in Model Catalog -You can manage all prompts to Cohere in the [Prompt Library](/product/prompt-library). All the current models of Cohere are supported and you can easily start testing different prompts. +1. Go to [**Model Catalog → Add Provider**](https://app.portkey.ai/model-catalog/providers) +2. Select **Cohere** +3. Choose existing credentials or create new by entering your [Cohere API key](https://dashboard.cohere.com/api-keys) +4. Name your provider (e.g., `cohere-prod`) -Once you're ready with your prompt, you can use the `portkey.prompts.completions.create` interface to use the prompt in your application. + + See all setup options, code examples, and detailed instructions + ## Other Cohere Endpoints ### Embeddings -Embedding endpoints are natively supported within Portkey like this: +Embedding endpoints are natively supported within Portkey: + + +```python Python +from portkey_ai import Portkey + +portkey = Portkey( + api_key="PORTKEY_API_KEY", + provider="@cohere" +) + +embedding = portkey.embeddings.create( + input="Name the tallest buildings in Hawaii", + model="embed-english-v3.0" +) + +print(embedding) +``` + +```js Javascript +import Portkey from 'portkey-ai' + +const portkey = new Portkey({ + apiKey: "PORTKEY_API_KEY", + provider: "@cohere" +}) -```js const embedding = await portkey.embeddings.create({ - input: 'Name the tallest buildings in Hawaii' -}); + input: "Name the tallest buildings in Hawaii", + model: "embed-english-v3.0" +}) -console.log(embedding); +console.log(embedding) ``` + ### Re-ranking -You can use cohere reranking the `portkey.post` method with the body expected by [Cohere's reranking API](https://docs.cohere.com/reference/rerank-1). - - - - ```js - const response = await portkey.post( - "/rerank", - { - "return_documents": false, - "max_chunks_per_doc": 10, - "model": "rerank-english-v2.0", - "query": "What is the capital of the United States?", - "documents": [ - "Carson City is the capital city of the American state of Nevada.", - "The Commonwealth of the Northern Mariana Islands is a group of islands in the Pacific Ocean. Its capital is Saipan.", - "Washington, D.C. (also known as simply Washington or D.C., and officially as the District of Columbia) is the capital of the United States. It is a federal district.", - "Capital punishment (the death penalty) has existed in the United States since beforethe United States was a country. As of 2017, capital punishment is legal in 30 of the 50 states." - ] - } - ) - ``` - - - ```python +Use Cohere reranking with the `portkey.post` method and the body expected by [Cohere's reranking API](https://docs.cohere.com/reference/rerank-1): + + +```python Python +from portkey_ai import Portkey + +portkey = Portkey( + api_key="PORTKEY_API_KEY", + provider="@cohere" +) + response = portkey.post( "/rerank", return_documents=False, @@ -142,24 +184,66 @@ You can use cohere reranking the `portkey.post` method with the body expected by "Carson City is the capital city of the American state of Nevada.", "The Commonwealth of the Northern Mariana Islands is a group of islands in the Pacific Ocean. Its capital is Saipan.", "Washington, D.C. (also known as simply Washington or D.C., and officially as the District of Columbia) is the capital of the United States. It is a federal district.", - "Capital punishment (the death penalty) has existed in the United States since beforethe United States was a country. As of 2017, capital punishment is legal in 30 of the 50 states." + "Capital punishment (the death penalty) has existed in the United States since before the United States was a country. As of 2017, capital punishment is legal in 30 of the 50 states." ] ) - ``` - - +print(response) +``` +```js Javascript +import Portkey from 'portkey-ai' + +const portkey = new Portkey({ + apiKey: "PORTKEY_API_KEY", + provider: "@cohere" +}) + +const response = await portkey.post( + "/rerank", + { + return_documents: false, + max_chunks_per_doc: 10, + model: "rerank-english-v2.0", + query: "What is the capital of the United States?", + documents: [ + "Carson City is the capital city of the American state of Nevada.", + "The Commonwealth of the Northern Mariana Islands is a group of islands in the Pacific Ocean. Its capital is Saipan.", + "Washington, D.C. (also known as simply Washington or D.C., and officially as the District of Columbia) is the capital of the United States. It is a federal district.", + "Capital punishment (the death penalty) has existed in the United States since before the United States was a country. As of 2017, capital punishment is legal in 30 of the 50 states." + ] + } +) -## Next Steps +console.log(response) +``` + -The complete list of features supported in the SDK are available on the link below. - - +## Managing Cohere Prompts + +Manage all prompt templates to Cohere in the [Prompt Library](/product/prompt-library). All current Cohere models are supported, and you can easily test different prompts. -You'll find more information in the relevant sections: +Use the `portkey.prompts.completions.create` interface to use the prompt in an application. -1. [Add metadata to your requests](/product/observability/metadata) -2. [Add gateway configs to your Cohere requests](/product/ai-gateway/configs) -3. [Tracing Cohere requests](/product/observability/traces) -4. [Setup a fallback from OpenAI to Cohere APIs](/product/ai-gateway/fallbacks) +## Next Steps + + + + Add metadata to your Cohere requests + + + Add gateway configs to your Cohere requests + + + Trace your Cohere requests + + + Setup fallback from OpenAI to Cohere + + + +For complete SDK documentation: + + + Complete Portkey SDK documentation + diff --git a/integrations/llms/dashscope.mdx b/integrations/llms/dashscope.mdx index f4acb9a5..a29fa97d 100644 --- a/integrations/llms/dashscope.mdx +++ b/integrations/llms/dashscope.mdx @@ -1,351 +1,307 @@ --- title: 'Dashscope' -description: 'Integrate Dashscope with Portkey for seamless completions, prompt management, and advanced features like streaming, function calling, and fine-tuning.' +description: 'Integrate Dashscope with Portkey for seamless completions, embeddings, and advanced features.' --- - -**Portkey Provider Slug:** `dashscope` - +## Quick Start -## Overview +Get started with Dashscope in under 2 minutes: -Portkey offers native integrations with [dashscope](https://dashscope.aliyun.com/) for Node.js, Python, and REST APIs. By combining Portkey with Dashscope, you can create production-grade AI applications with enhanced reliability, observability, and advanced features. - - - - Explore the official Dashscope documentation for comprehensive details on their APIs and models. - - - -## Getting Started - - - - Visit the [Dashscope dashboard](https://help.aliyun.com/zh/model-studio/developer-reference/get-api-key) to generate your API key. - - - - Portkey's virtual key vault simplifies your interaction with Dashscope. Virtual keys act as secure aliases for your actual API keys, offering enhanced security and easier management through [budget limits](/product/ai-gateway/virtual-keys/budget-limits) to control your API usage. - - Use the Portkey app to create a [virtual key](/product/ai-gateway/virtual-keys) associated with your Dashscope API key. - - - - Now that you have your virtual key, set up the Portkey client: - - ### Portkey Hosted App - Use the Portkey API key and the Dashscope virtual key to initialize the client in your preferred programming language. + - - ```python Python - from portkey_ai import Portkey +```python Python icon="python" +from portkey_ai import Portkey - portkey = Portkey( - api_key="PORTKEY_API_KEY", # Replace with your Portkey API key - provider="@PROVIDER" # Replace with your virtual key for Dashscope - ) - ``` +# 1. Install: pip install portkey-ai +# 2. Add @dashscope provider in model catalog +# 3. Use it: - ```javascript Node.js - import Portkey from 'portkey-ai' +portkey = Portkey(api_key="PORTKEY_API_KEY") - const portkey = new Portkey({ - apiKey: "PORTKEY_API_KEY", // defaults to process.env["PORTKEY_API_KEY"] - provider:"@PROVIDER" // Your Dashscope Virtual Key - }) - ``` - +response = portkey.chat.completions.create( + model="@dashscope/qwen-turbo", + messages=[{"role": "user", "content": "Hello!"}] +) - ### Open Source Use - Alternatively, use Portkey's Open Source AI Gateway to enhance your app's reliability with minimal code: +print(response.choices[0].message.content) +``` - - ```python Python - from portkey_ai import Portkey, PORTKEY_GATEWAY_URL +```js Javascript icon="square-js" +import Portkey from 'portkey-ai' - portkey = Portkey( - api_key="dummy", # Replace with your Portkey API key - base_url=PORTKEY_GATEWAY_URL, - Authorization="DASHSCOPE_API_KEY", # Replace with your Dashscope API Key - provider="dashscope" - ) - ``` +// 1. Install: npm install portkey-ai +// 2. Add @dashscope provider in model catalog +// 3. Use it: - ```javascript Node.js - import Portkey, { PORTKEY_GATEWAY_URL } from 'portkey-ai' +const portkey = new Portkey({ + apiKey: "PORTKEY_API_KEY" +}) - const portkey = new Portkey({ - apiKey: "dummy", // Replace with your Portkey API key - baseUrl: PORTKEY_GATEWAY_URL, - Authorization: "DASHSCOPE_API_KEY", // Replace with your Dashscope API Key - provider: "dashscope" - }) - ``` - - - +const response = await portkey.chat.completions.create({ + model: "@dashscope/qwen-turbo", + messages: [{ role: "user", content: "Hello!" }] +}) -🔥 That's it! You've integrated Portkey into your application with just a few lines of code. Now let's explore making requests using the Portkey client. +console.log(response.choices[0].message.content) +``` -## Supported Models +```python OpenAI Py icon="python" +from openai import OpenAI +from portkey_ai import PORTKEY_GATEWAY_URL - +# 1. Install: pip install openai portkey-ai +# 2. Add @dashscope provider in model catalog +# 3. Use it: -`Chat` - qwen-long, qwen-max, qwen-max-0428, qwen-max-0403, qwen-max-0107, qwen-plus, qwen-plus-0806, qwen-plus-0723, qwen-plus-0624, qwen-plus-0206, qwen-turbo, qwen-turbo-0624, qwen-turbo-0206, qwen2-57b-a14b-instruct, qwen2-72b-instruct, qwen2-7b-instruct, qwen2-1.5b-instruct, qwen2-0.5b-instruct, qwen1.5-110b-chat, qwen1.5-72b-chat, qwen1.5-32b-chat, qwen1.5-14b-chat, qwen1.5-7b-chat, qwen1.5-1.8b-chat, qwen1.5-0.5b-chat, codeqwen1.5-7b-chat, qwen-72b-chat, qwen-14b-chat, qwen-7b-chat, qwen-1.8b-longcontext-chat, qwen-1.8b-chat, qwen2-math-72b-instruct, qwen2-math-7b-instruct, qwen2-math-1.5b-instruct +client = OpenAI( + api_key="PORTKEY_API_KEY", # Portkey API key + base_url=PORTKEY_GATEWAY_URL +) +response = client.chat.completions.create( + model="@dashscope/qwen-turbo", + messages=[{"role": "user", "content": "Hello!"}] +) -`Embedding`- text-embedding-v1, text-embedding-v2, text-embedding-v3 +print(response.choices[0].message.content) +``` - +```js OpenAI JS icon="square-js" +import OpenAI from "openai" +import { PORTKEY_GATEWAY_URL } from "portkey-ai" +// 1. Install: npm install openai portkey-ai +// 2. Add @dashscope provider in model catalog +// 3. Use it: -## Supported Endpoints and Parameters +const client = new OpenAI({ + apiKey: "PORTKEY_API_KEY", // Portkey API key + baseURL: PORTKEY_GATEWAY_URL +}) -| Endpoint | Supported Parameters | -| ----------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `chatComplete` | messages, max_tokens, temperature, top_p, stream, presence_penalty, frequency_penalty | -| `embed` | model, input, encoding_format, dimensions, user | +const response = await client.chat.completions.create({ + model: "@dashscope/qwen-turbo", + messages: [{ role: "user", content: "Hello!" }] +}) +console.log(response.choices[0].message.content) +``` +```sh cURL icon="square-terminal" +# 1. Add @dashscope provider in model catalog +# 2. Use it: + +curl https://api.portkey.ai/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "x-portkey-api-key: $PORTKEY_API_KEY" \ + -d '{ + "model": "@dashscope/qwen-turbo", + "messages": [{"role": "user", "content": "Hello!"}] + }' +``` -## Dashscope Supported Features + -### Chat Completions +## Add Provider in Model Catalog -Generate chat completions using Dashscope models through Portkey: +Before making requests, add Dashscope to your Model Catalog: - -```python Python -completion = portkey.chat.completions.create( - messages=[{"role": "user", "content": "Say this is a test"}], - model="qwen-turbo" -) +1. Go to [**Model Catalog → Add Provider**](https://app.portkey.ai/model-catalog/providers) +2. Select **Dashscope** +3. Enter your [Dashscope API key](https://help.aliyun.com/zh/model-studio/developer-reference/get-api-key) +4. Name your provider (e.g., `dashscope`) -print(completion.choices[0].message.content) -``` + + See all setup options and detailed configuration instructions + -```javascript Node.js -const chatCompletion = await portkey.chat.completions.create({ - messages: [{ role: 'user', content: 'Say this is a test' }], - model: 'qwen-turbo', -}); + + Explore the official Dashscope documentation + -console.log(chatCompletion.choices[0].message.content); -``` +--- -```curl REST -curl -X POST "https://api.portkey.ai/v1/chat/completions" \ - -H "Content-Type: application/json" \ - -H "Authorization: Bearer YOUR_PORTKEY_API_KEY" \ - -d '{ - "messages": [{"role": "user", "content": "Say this is a test"}], - "model": "qwen-turbo" - }' -``` - +## Dashscope Capabilities ### Embeddings Generate embeddings for text using Dashscope embedding models: + ```python Python +from portkey_ai import Portkey + +portkey = Portkey(api_key="PORTKEY_API_KEY", provider="@dashscope") + response = portkey.embeddings.create( input="Your text string goes here", - model="text-embedding-v1" + model="text-embedding-v3" ) print(response.data[0].embedding) ``` ```javascript Node.js +import Portkey from 'portkey-ai'; + +const portkey = new Portkey({ + apiKey: 'PORTKEY_API_KEY', + provider: '@dashscope' +}); + const response = await portkey.embeddings.create({ - input: "Your text string goes here", - model: "text-embedding-v1" + input: "Your text string goes here", + model: "text-embedding-v3" }); console.log(response.data[0].embedding); ``` -```curl REST -curl -X POST "https://api.portkey.ai/v1/embeddings" \ - -H "Content-Type: application/json" \ - -H "Authorization: Bearer YOUR_PORTKEY_API_KEY" \ - -d '{ - "input": "Your text string goes here", - "model": "text-embedding-v1" - }' -``` +--- +## Advanced Features +### Track End-User IDs +Monitor user-level costs and requests by passing user IDs: -# Portkey's Advanced Features + -## Track End-User IDs +```python Python +from portkey_ai import Portkey -Portkey allows you to track user IDs passed with the user parameter in Dashscope requests, enabling you to monitor user-level costs, requests, and more: +portkey = Portkey(api_key="PORTKEY_API_KEY", provider="@dashscope") - -```python Python response = portkey.chat.completions.create( - model="qwen-turbo", - messages=[{"role": "user", "content": "Say this is a test"}], - user="user_123456" + model="qwen-turbo", + messages=[{"role": "user", "content": "Hello!"}], + user="user_123456" # Track this user's usage ) ``` ```javascript Node.js -const chatCompletion = await portkey.chat.completions.create({ - messages: [{ role: "user", content: "Say this is a test" }], - model: "qwen-turbo", - user: "user_12345", +import Portkey from 'portkey-ai'; + +const portkey = new Portkey({ + apiKey: 'PORTKEY_API_KEY', + provider: '@dashscope' }); -``` -```curl REST -curl -X POST "https://api.portkey.ai/v1/chat/completions" \ - -H "Content-Type: application/json" \ - -H "Authorization: Bearer YOUR_PORTKEY_API_KEY" \ - -d '{ - "model": "qwen-turbo", - "messages": [{"role": "user", "content": "Say this is a test"}], - "user": "user_123456" - }' +const response = await portkey.chat.completions.create({ + model: "qwen-turbo", + messages: [{ role: "user", content: "Hello!" }], + user: "user_123456" // Track this user's usage +}); ``` - -When you include the user parameter in your requests, Portkey logs will display the associated user ID, as shown in the image below: + Portkey Logs with User ID -In addition to the `user` parameter, Portkey allows you to send arbitrary custom metadata with your requests. This powerful feature enables you to associate additional context or information with each request, which can be useful for analysis, debugging, or other custom use cases. + + Explore how to use custom metadata to enhance your request tracking and analysis + - - - Explore how to use custom metadata to enhance your request tracking and analysis. - - +### Gateway Configurations + +Use Portkey's Gateway features for advanced routing and reliability: -## Using The Gateway Config - -Here's a simplified version of how to use Portkey's Gateway Configuration: - - - - You can create a Gateway configuration using the Portkey Config Dashboard or by writing a JSON configuration in your code. In this example, requests are routed based on the user's subscription plan (paid or free). - - ```json - config = { - "strategy": { - "mode": "conditional", - "conditions": [ - { - "query": { "metadata.user_plan": { "$eq": "paid" } }, - "then": "qwen-turbo" - }, - { - "query": { "metadata.user_plan": { "$eq": "free" } }, - "then": "gpt-3.5" - } - ], - "default": "base-gpt4" +**Example: Conditional Routing** + +```json +{ + "strategy": { + "mode": "conditional", + "conditions": [ + { + "query": { "metadata.user_plan": { "$eq": "paid" } }, + "then": "qwen-turbo" }, - "targets": [ - { - "name": "qwen-turbo", - "provider":"@xx" - }, - { - "name": "gpt-3.5", - "provider":"@yy" - } - ] + { + "query": { "metadata.user_plan": { "$eq": "free" } }, + "then": "gpt-3.5" + } + ], + "default": "gpt-3.5" + }, + "targets": [ + { + "name": "qwen-turbo", + "provider": "@dashscope" + }, + { + "name": "gpt-3.5", + "provider": "@openai" } - ``` - - - - When a user makes a request, it will pass through Portkey's AI Gateway. Based on the configuration, the Gateway routes the request according to the user's metadata. - Conditional Routing Diagram - - - - Pass the Gateway configuration to your Portkey client. You can either use the config object or the Config ID from Portkey's hosted version. - - - ```python Python - from portkey_ai import Portkey - - portkey = Portkey( - api_key="PORTKEY_API_KEY", - provider="@PROVIDER", - config=portkey_config - ) - ``` - - ```javascript Node.js - import Portkey from 'portkey-ai' - - const portkey = new Portkey({ - apiKey: "PORTKEY_API_KEY", - provider:"@PROVIDER", - config: portkeyConfig - }) - ``` - - - - -That's it! Portkey seamlessly allows you to make your AI app more robust using built-in gateway features. Learn more about advanced gateway features: + ] +} +``` + +Conditional Routing Diagram - Distribute requests across multiple targets based on defined weights. + Distribute requests across multiple targets - Automatically switch to backup targets if the primary target fails. + Automatically switch to backup targets - Route requests to different targets based on specified conditions. + Route requests based on conditions - Enable caching of responses to improve performance and reduce costs. + Cache responses to reduce costs -## Guardrails +### Guardrails -Portkey's AI gateway enables you to enforce input/output checks on requests by applying custom hooks before and after processing. Protect your user's/company's data by using PII guardrails and many more available on Portkey Guardrails: +Enforce input/output checks with custom hooks: ```json { - "provider:"@dashscope-xxx", - "before_request_hooks": [{ - "id": "input-guardrail-id-xx" - }], - "after_request_hooks": [{ - "id": "output-guardrail-id-xx" - }] + "provider": "@dashscope", + "before_request_hooks": [{ + "id": "input-guardrail-id-xx" + }], + "after_request_hooks": [{ + "id": "output-guardrail-id-xx" + }] } ``` - Explore Portkey's guardrail features to enhance the security and reliability of your AI applications. + Enhance security and reliability with Portkey Guardrails +--- + ## Next Steps -The complete list of features supported in the SDK are available in our comprehensive documentation: + + + Add fallbacks, load balancing, and more + + + Monitor and trace your Dashscope requests + + + Manage and version your prompts + + + Add custom metadata to requests + + + +For complete SDK documentation: - - Explore the full capabilities of the Portkey SDK and how to leverage them in your projects. + + Complete Portkey SDK documentation --- - - For the most up-to-date information on supported features and endpoints, please refer to our [API Reference](/api-reference/inference-api/introduction). diff --git a/integrations/llms/deepbricks.mdx b/integrations/llms/deepbricks.mdx index 76c2abd7..d8b0e1c2 100644 --- a/integrations/llms/deepbricks.mdx +++ b/integrations/llms/deepbricks.mdx @@ -1,103 +1,149 @@ --- title: "Deepbricks" -description: "Portkey provides a robust and secure gateway to facilitate the integration of various Large Language Models (LLMs) into your applications, including [Deepbricks](https://deepbricks.ai/). -" +description: Use Deepbricks' AI inference platform through Portkey for fast model deployment. --- +## Quick Start -With Portkey, you can take advantage of features like fast AI gateway access, observability, prompt management, and more, all while ensuring the secure management of your LLM API keys through a [virtual key](/product/ai-gateway/virtual-keys) system. - - -Provider Slug: `deepbricks` - - -## Portkey SDK Integration with Deepbricks Models - -Portkey provides a consistent API to interact with models from various providers. To integrate Deepbricks with Portkey: - -### 1. Install the Portkey SDK - -Add the Portkey SDK to your application to interact with Deepbricks API through Portkey's gateway. - - - - ```sh - npm install --save portkey-ai - ``` - - - ```sh - pip install portkey-ai - ``` - - - -### 2. Initialize Portkey with the Virtual Key - -To use Deepbricks with Portkey, [get your API key from here](https://deepbricks.ai/pricing), then add it to Portkey to create the virtual key. - - - - ```javascript - import Portkey from 'portkey-ai' - - const portkey = new Portkey({ - apiKey: "PORTKEY_API_KEY", // defaults to process.env["PORTKEY_API_KEY"] - provider:"@PROVIDER" // Your Deepbricks - }) - ``` - - - ```python - from portkey_ai import Portkey - portkey = Portkey( - api_key="PORTKEY_API_KEY", # Replace with your Portkey API key - provider="@PROVIDER" # Replace with your virtual key for Deepbricks - ) - ``` - - - -### 3. Invoke Chat Completions with Deepbricks - -Use the Portkey instance to send requests to Deepbricks. You can also override the virtual key directly in the API call if needed. - - - - ```javascript - const chatCompletion = await portkey.chat.completions.create({ - messages: [{ role: 'user', content: 'Say this is a test' }], - model: 'deepseek-ai/DeepSeek-V2-Chat', - }); - console.log(chatCompletion.choices); - ``` - - - ```python - completion = portkey.chat.completions.create( - messages= [{ "role": 'user', "content": 'Say this is a test' }], - model= 'deepseek-ai/DeepSeek-V2-Chat' - ) - print(completion) - ``` - - - -## Managing Deepbricks Prompts - -You can manage all prompts to Deepbricks in the [Prompt Library](/product/prompt-library). All the current models of Deepbricks are supported and you can easily start testing different prompts. - -Once you're ready with your prompt, you can use the `portkey.prompts.completions.create` interface to use the prompt in your application. - -The complete list of features supported in the SDK are available on the link below. - - - Explore the Portkey SDK Client documentation +Get started with Deepbricks in under 2 minutes: + + + +```python Python icon="python" +from portkey_ai import Portkey + +# 1. Install: pip install portkey-ai +# 2. Add @deepbricks provider in model catalog +# 3. Use it: + +portkey = Portkey(api_key="PORTKEY_API_KEY") + +response = portkey.chat.completions.create( + model="@deepbricks/deepseek-ai/DeepSeek-V2-Chat", + messages=[{"role": "user", "content": "Hello!"}] +) + +print(response.choices[0].message.content) +``` + +```js Javascript icon="square-js" +import Portkey from 'portkey-ai' + +// 1. Install: npm install portkey-ai +// 2. Add @deepbricks provider in model catalog +// 3. Use it: + +const portkey = new Portkey({ + apiKey: "PORTKEY_API_KEY" +}) + +const response = await portkey.chat.completions.create({ + model: "@deepbricks/deepseek-ai/DeepSeek-V2-Chat", + messages: [{ role: "user", content: "Hello!" }] +}) + +console.log(response.choices[0].message.content) +``` + +```python OpenAI Py icon="python" +from openai import OpenAI +from portkey_ai import PORTKEY_GATEWAY_URL + +# 1. Install: pip install openai portkey-ai +# 2. Add @deepbricks provider in model catalog +# 3. Use it: + +client = OpenAI( + api_key="PORTKEY_API_KEY", # Portkey API key + base_url=PORTKEY_GATEWAY_URL +) + +response = client.chat.completions.create( + model="@deepbricks/deepseek-ai/DeepSeek-V2-Chat", + messages=[{"role": "user", "content": "Hello!"}] +) + +print(response.choices[0].message.content) +``` + +```js OpenAI JS icon="square-js" +import OpenAI from "openai" +import { PORTKEY_GATEWAY_URL } from "portkey-ai" + +// 1. Install: npm install openai portkey-ai +// 2. Add @deepbricks provider in model catalog +// 3. Use it: + +const client = new OpenAI({ + apiKey: "PORTKEY_API_KEY", // Portkey API key + baseURL: PORTKEY_GATEWAY_URL +}) + +const response = await client.chat.completions.create({ + model: "@deepbricks/deepseek-ai/DeepSeek-V2-Chat", + messages: [{ role: "user", content: "Hello!" }] +}) + +console.log(response.choices[0].message.content) +``` + +```sh cURL icon="square-terminal" +# 1. Add @deepbricks provider in model catalog +# 2. Use it: + +curl https://api.portkey.ai/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "x-portkey-api-key: $PORTKEY_API_KEY" \ + -d '{ + "model": "@deepbricks/deepseek-ai/DeepSeek-V2-Chat", + "messages": [{"role": "user", "content": "Hello!"}] + }' +``` + + + +## Add Provider in Model Catalog + +Before making requests, add Deepbricks to your Model Catalog: + +1. Go to [**Model Catalog → Add Provider**](https://app.portkey.ai/model-catalog/providers) +2. Select **Deepbricks** +3. Enter your [Deepbricks API key](https://deepbricks.ai/pricing) +4. Name your provider (e.g., `deepbricks`) + + + See all setup options and detailed configuration instructions -You'll find more information in the relevant sections: +--- + +## Supported Models + +Deepbricks provides fast inference for various models: -1. [Add metadata to your requests](/product/observability/metadata) -2. [Add gateway configs to your Deepbricks requests](/product/ai-gateway/configs) -3. [Tracing Deepbricks requests](/product/observability/traces) -4. [Setup a fallback from OpenAI to Deepbricks APIs](/product/ai-gateway/fallbacks) +Check [Deepbricks' documentation](https://deepbricks.ai/) for the complete model list. + +--- + +## Next Steps + + + + Add fallbacks, load balancing, and more + + + Monitor and trace your Deepbricks requests + + + Manage and version your prompts + + + Add custom metadata to requests + + + +For complete SDK documentation: + + + Complete Portkey SDK documentation + diff --git a/integrations/llms/deepgram.mdx b/integrations/llms/deepgram.mdx index ffa2d9b7..cf957396 100644 --- a/integrations/llms/deepgram.mdx +++ b/integrations/llms/deepgram.mdx @@ -1,40 +1,136 @@ --- title: "Deepgram" -description: Portkey provides a robust and secure gateway to use and observe Deepgrm's Speech-to-Text API. +description: Use Deepgram's Speech-to-Text API through Portkey for audio transcription. --- -Deepgram API is currently supported on Portkey's REST API, with support for Python & Node SDKs coming soon. +Deepgram provides industry-leading speech-to-text capabilities. Portkey allows you to use Deepgram's API with full observability and reliability features. -## Speech to Text API + +Deepgram currently uses a custom host setup. SDK support is available through the custom host pattern. + -* We set the target Deepgram API URL with the `x-portkey-custom-host` header -* We set the target provider as `openai` to let Portkey know that this request should be handled similarly to OpenAI +## Quick Start - - -```sh -curl 'https://api.portkey.ai/v1/listen' \ - -H 'Authorization: Token $DEEPGRAM_API_KEY' \ - -H 'Content-Type: audio/mp3' \ - -H 'x-portkey-custom-host: https://api.deepgram.com/v1' \ - -H 'x-portkey-provider: openai' \ - -H 'x-portkey-api-key: $PORTKEY_API_KEY' \ +Get started with Deepgram in under 2 minutes: + +### Speech to Text - Local File + +Transcribe audio from a local file: + +```bash cURL +curl https://api.portkey.ai/v1/listen \ + -H "Authorization: Token $DEEPGRAM_API_KEY" \ + -H "Content-Type: audio/mp3" \ + -H "x-portkey-custom-host: https://api.deepgram.com/v1" \ + -H "x-portkey-provider: openai" \ + -H "x-portkey-api-key: $PORTKEY_API_KEY" \ --data-binary '@audio.mp3' ``` - - -```sh -curl 'https://api.portkey.ai/v1/listen' \ - -H 'Authorization: Token $DEEPGRAM_API_KEY' \ - -H 'Accept: application/json' \ - -H 'Content-Type: application/json' \ - -H 'x-portkey-custom-host: https://api.deepgram.com/v1' \ - -H 'x-portkey-provider: openai' \ - -H 'x-portkey-api-key: $PORTKEY_API_KEY' \ + +### Speech to Text - Remote File + +Transcribe audio from a URL: + +```bash cURL +curl https://api.portkey.ai/v1/listen \ + -H "Authorization: Token $DEEPGRAM_API_KEY" \ + -H "Accept: application/json" \ + -H "Content-Type: application/json" \ + -H "x-portkey-custom-host: https://api.deepgram.com/v1" \ + -H "x-portkey-provider: openai" \ + -H "x-portkey-api-key: $PORTKEY_API_KEY" \ -d '{"url": "https://dpgr.am/spacewalk.wav"}' ``` - - - +## Add Provider in Model Catalog + +Before making requests, add Deepgram to your Model Catalog: + +1. Go to [**Model Catalog → Add Provider**](https://app.portkey.ai/model-catalog/providers) +2. Select **Deepgram** +3. Enter your [Deepgram API key](https://console.deepgram.com/) +4. Set the custom host to `https://api.deepgram.com/v1` +5. Name your provider (e.g., `deepgram`) + + + See all setup options and detailed configuration instructions + + +--- + +## Using with Portkey SDK + +You can also use Deepgram with the Portkey SDK using custom host configuration: + + + +```python Python +from portkey_ai import Portkey + +portkey = Portkey( + api_key="PORTKEY_API_KEY", + provider="openai", + custom_host="https://api.deepgram.com/v1", + Authorization="Token DEEPGRAM_API_KEY" +) + +# Use with audio transcription +# Note: Specific implementation depends on your use case +``` + +```javascript Node.js +import Portkey from 'portkey-ai'; + +const portkey = new Portkey({ + apiKey: 'PORTKEY_API_KEY', + provider: 'openai', + customHost: 'https://api.deepgram.com/v1', + Authorization: 'Token DEEPGRAM_API_KEY' +}); + +// Use with audio transcription +// Note: Specific implementation depends on your use case +``` + + + +--- + +## Deepgram Features + +Deepgram offers advanced speech recognition capabilities: + +- **Real-time Streaming**: Transcribe audio in real-time +- **Multiple Languages**: Support for 30+ languages +- **Custom Models**: Train custom models for domain-specific terminology +- **Diarization**: Identify different speakers in audio +- **Punctuation & Formatting**: Automatic punctuation and formatting + + + Explore the complete Deepgram API documentation + + +--- + +## Next Steps + + + + Add retries and timeouts for audio processing + + + Monitor and trace your Deepgram requests + + + Learn more about custom host configuration + + + Add custom metadata to track audio sources + + + +For complete SDK documentation: + + + Complete Portkey SDK documentation diff --git a/integrations/llms/deepinfra.mdx b/integrations/llms/deepinfra.mdx index 7cf506f2..f086bc11 100644 --- a/integrations/llms/deepinfra.mdx +++ b/integrations/llms/deepinfra.mdx @@ -1,112 +1,158 @@ --- title: "Deepinfra" +description: "Integrate Deepinfra models with Portkey's AI Gateway" --- -Portkey provides a robust and secure gateway to facilitate the integration of various Large Language Models (LLMs) into your applications, including the models hosted on [Deepinfra API](https://deepinfra.com/models/text-generation). +Portkey provides a robust and secure gateway to integrate various Large Language Models (LLMs) into applications, including [Deepinfra's hosted models](https://deepinfra.com/models/text-generation). - -Provider Slug. `deepinfra` - +With Portkey, take advantage of features like fast AI gateway access, observability, prompt management, and more, while securely managing API keys through [Model Catalog](/product/model-catalog). -## Portkey SDK Integration with Deepinfra Models +## Quick Start -Portkey provides a consistent API to interact with models from various providers. To integrate Deepinfra with Portkey: +Get Deepinfra working in 3 steps: -### 1\. Install the Portkey SDK + +```python Python icon="python" +from portkey_ai import Portkey -Add the Portkey SDK to your application to interact with Mistral AI's API through Portkey's gateway. - - - ```sh - npm install --save portkey-ai - ``` - - - ```sh - pip install portkey-ai - ``` - +# 1. Install: pip install portkey-ai +# 2. Add @deepinfra provider in model catalog +# 3. Use it: - +portkey = Portkey(api_key="PORTKEY_API_KEY") +response = portkey.chat.completions.create( + model="@deepinfra/nvidia/Nemotron-4-340B-Instruct", + messages=[{"role": "user", "content": "Say this is a test"}] +) +print(response.choices[0].message.content) +``` -### 2\. Initialize Portkey with the Virtual Key +```js Javascript icon="square-js" +import Portkey from 'portkey-ai' -To use Deepinfra with Virtual Key, [get your API key from here](https://deepinfra.com/dash/api%5Fkeys). Then add it to Portkey to create the virtual key - - - ```js - import Portkey from 'portkey-ai' +// 1. Install: npm install portkey-ai +// 2. Add @deepinfra provider in model catalog +// 3. Use it: - const portkey = new Portkey({ - apiKey: "PORTKEY_API_KEY", // defaults to process.env["PORTKEY_API_KEY"] - provider:"@PROVIDER" // Your Deepinfra Virtual Key - }) - ``` - - - ```python - from portkey_ai import Portkey +const portkey = new Portkey({ + apiKey: "PORTKEY_API_KEY" +}) - portkey = Portkey( - api_key="PORTKEY_API_KEY", # Replace with your Portkey API key - provider="@DEEPINFRA_PROVIDER" - ) - ``` - +const response = await portkey.chat.completions.create({ + model: "@deepinfra/nvidia/Nemotron-4-340B-Instruct", + messages: [{ role: "user", content: "Say this is a test" }] +}) - +console.log(response.choices[0].message.content) +``` +```python OpenAI Py icon="python" +from openai import OpenAI +from portkey_ai import PORTKEY_GATEWAY_URL +# 1. Install: pip install openai portkey-ai +# 2. Add @deepinfra provider in model catalog +# 3. Use it: +client = OpenAI( + api_key="PORTKEY_API_KEY", # Portkey API key + base_url=PORTKEY_GATEWAY_URL +) -### 3\. Invoke Chat Completions - - - ```js - const chatCompletion = await portkey.chat.completions.create({ - messages: [{ role: 'user', content: 'Say this is a test' }], - model: 'nvidia/Nemotron-4-340B-Instruct', - }); +response = client.chat.completions.create( + model="@deepinfra/nvidia/Nemotron-4-340B-Instruct", + messages=[{"role": "user", "content": "Say this is a test"}] +) - console.log(chatCompletion.choices); - ``` - - - ```python - completion = portkey.chat.completions.create( - messages= [{ "role": 'user', "content": 'Say this is a test' }], - model= 'nvidia/Nemotron-4-340B-Instruct' - ) +print(response.choices[0].message.content) +``` - print(completion) - ``` - +```js OpenAI JS icon="square-js" +import OpenAI from "openai" +import { PORTKEY_GATEWAY_URL } from "portkey-ai" - +// 1. Install: npm install openai portkey-ai +// 2. Add @deepinfra provider in model catalog +// 3. Use it: +const client = new OpenAI({ + apiKey: "PORTKEY_API_KEY", // Portkey API key + baseURL: PORTKEY_GATEWAY_URL +}) +const response = await client.chat.completions.create({ + model: "@deepinfra/nvidia/Nemotron-4-340B-Instruct", + messages: [{ role: "user", content: "Say this is a test" }] +}) +console.log(response.choices[0].message.content) +``` ---- +```sh cURL icon="square-terminal" +# 1. Add @deepinfra provider in model catalog +# 2. Use it: -## Supported Models +curl https://api.portkey.ai/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "x-portkey-api-key: $PORTKEY_API_KEY" \ + -d '{ + "model": "@deepinfra/nvidia/Nemotron-4-340B-Instruct", + "messages": [ + { "role": "user", "content": "Say this is a test" } + ] + }' +``` + -Here's the list of all the Deepinfra models you can route to using Portkey - + +**Tip:** You can also set `provider="@deepinfra"` in `Portkey()` and use just `model="nvidia/Nemotron-4-340B-Instruct"` in the request. + - -[![Logo](/images/llms/deepinfra-logo-64.webp)Models | Machine Learning Inference | Deep InfraDeepInfra](https://deepinfra.com/models/text-generation) - -## Next Steps +## Add Provider in Model Catalog + +1. Go to [**Model Catalog → Add Provider**](https://app.portkey.ai/model-catalog/providers) +2. Select **Deepinfra** +3. Choose existing credentials or create new by entering your [Deepinfra API key](https://deepinfra.com/dash/api_keys) +4. Name your provider (e.g., `deepinfra-prod`) -The complete list of features supported in the SDK are available on the link below. - + + See all setup options, code examples, and detailed instructions -You'll find more information in the relevant sections: +## Supported Models + +Deepinfra hosts a wide range of open-source models for text generation. View the complete list: + + + Browse all available models on Deepinfra + -1. [Add metadata to your requests](/product/observability/metadata) -2. [Add gateway configs to your Deepinfra](/product/ai-gateway/configs)[ requests](/product/ai-gateway/configs) -3. [Tracing Deepinfra requests](/product/observability/traces) -4. [Setup a fallback from OpenAI to Deepinfra](/product/ai-gateway/fallbacks) +Popular models include: +- `nvidia/Nemotron-4-340B-Instruct` +- `meta-llama/Meta-Llama-3.1-405B-Instruct` +- `Qwen/Qwen2.5-72B-Instruct` + +## Next Steps + + + + Add metadata to your Deepinfra requests + + + Add gateway configs to your Deepinfra requests + + + Trace your Deepinfra requests + + + Setup fallback from OpenAI to Deepinfra + + + +For complete SDK documentation: + + + Complete Portkey SDK documentation + diff --git a/integrations/llms/deepseek.mdx b/integrations/llms/deepseek.mdx index 5ca33d62..cd430c9d 100644 --- a/integrations/llms/deepseek.mdx +++ b/integrations/llms/deepseek.mdx @@ -1,140 +1,139 @@ --- title: "DeepSeek" -description: "Portkey provides a robust and secure gateway to facilitate the integration of various Large Language Models (LLMs) into your applications, including DeepSeek models. -" +description: "Integrate DeepSeek models with Portkey's AI Gateway" --- +Portkey provides a robust and secure gateway to integrate various Large Language Models (LLMs) into applications, including DeepSeek's models. -With Portkey, you can take advantage of features like fast AI gateway access, observability, prompt management, and more, all while ensuring the secure management of your LLM API keys through a [virtual key](/product/ai-gateway/virtual-keys) system. +With Portkey, take advantage of features like fast AI gateway access, observability, prompt management, and more, while securely managing API keys through [Model Catalog](/product/model-catalog). - -Provider Slug: **deepseek** - - -## Portkey SDK Integration with DeepSeek Models +## Quick Start -Portkey provides a consistent API to interact with models from various providers. To integrate DeepSeek with Portkey: +Get DeepSeek working in 3 steps: -### 1. Install the Portkey SDK + +```python Python icon="python" +from portkey_ai import Portkey -Add the Portkey SDK to your application to interact with DeepSeek AI's API through Portkey's gateway. +# 1. Install: pip install portkey-ai +# 2. Add @deepseek provider in model catalog +# 3. Use it: - - - ```sh - npm install --save portkey-ai - ``` - - - ```sh - pip install portkey-ai - ``` - - +portkey = Portkey(api_key="PORTKEY_API_KEY") -### 2. Initialize Portkey with the Virtual Key +response = portkey.chat.completions.create( + model="@deepseek/deepseek-chat", + messages=[{"role": "user", "content": "Say this is a test"}] +) -To use DeepSeek with Portkey, [get your API key from here](https://platform.deepseek.com/api_keys), then add it to Portkey to create the virtual key. +print(response.choices[0].message.content) +``` - - - ```javascript +```js Javascript icon="square-js" import Portkey from 'portkey-ai' +// 1. Install: npm install portkey-ai +// 2. Add @deepseek provider in model catalog +// 3. Use it: + const portkey = new Portkey({ - apiKey: "PORTKEY_API_KEY", // defaults to process.env["PORTKEY_API_KEY"] - provider:"@PROVIDER" // Your DeepSeek Virtual Key - }) - ``` - - - ```python - from portkey_ai import Portkey - - portkey = Portkey( - api_key="PORTKEY_API_KEY", # Replace with your Portkey API key - provider="@PROVIDER" # Replace with your virtual key for DeepSeek - ) - ``` - - + apiKey: "PORTKEY_API_KEY" +}) + +const response = await portkey.chat.completions.create({ + model: "@deepseek/deepseek-chat", + messages: [{ role: "user", content: "Say this is a test" }] +}) + +console.log(response.choices[0].message.content) +``` + +```python OpenAI Py icon="python" +from openai import OpenAI +from portkey_ai import PORTKEY_GATEWAY_URL + +# 1. Install: pip install openai portkey-ai +# 2. Add @deepseek provider in model catalog +# 3. Use it: + +client = OpenAI( + api_key="PORTKEY_API_KEY", # Portkey API key + base_url=PORTKEY_GATEWAY_URL +) + +response = client.chat.completions.create( + model="@deepseek/deepseek-chat", + messages=[{"role": "user", "content": "Say this is a test"}] +) + +print(response.choices[0].message.content) +``` + +```js OpenAI JS icon="square-js" +import OpenAI from "openai" +import { PORTKEY_GATEWAY_URL } from "portkey-ai" + +// 1. Install: npm install openai portkey-ai +// 2. Add @deepseek provider in model catalog +// 3. Use it: + +const client = new OpenAI({ + apiKey: "PORTKEY_API_KEY", // Portkey API key + baseURL: PORTKEY_GATEWAY_URL +}) + +const response = await client.chat.completions.create({ + model: "@deepseek/deepseek-chat", + messages: [{ role: "user", content: "Say this is a test" }] +}) + +console.log(response.choices[0].message.content) +``` + +```sh cURL icon="square-terminal" +# 1. Add @deepseek provider in model catalog +# 2. Use it: + +curl https://api.portkey.ai/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "x-portkey-api-key: $PORTKEY_API_KEY" \ + -d '{ + "model": "@deepseek/deepseek-chat", + "messages": [ + { "role": "user", "content": "Say this is a test" } + ] + }' +``` + + + +**Tip:** You can also set `provider="@deepseek"` in `Portkey()` and use just `model="deepseek-chat"` in the request. + -### 3. Invoke Chat Completions with DeepSeek +## Add Provider in Model Catalog -Use the Portkey instance to send requests to DeepSeek. You can also override the virtual key directly in the API call if needed. +1. Go to [**Model Catalog → Add Provider**](https://app.portkey.ai/model-catalog/providers) +2. Select **DeepSeek** +3. Choose existing credentials or create new by entering your [DeepSeek API key](https://platform.deepseek.com/api_keys) +4. Name your provider (e.g., `deepseek-prod`) - - - ```javascript - const chatCompletion = await portkey.chat.completions.create({ - messages: [{ role: 'user', content: 'Say this is a test' }], - model: 'deepseek-chat', - }); + + See all setup options, code examples, and detailed instructions + - console.log(chatCompletion.choices); - ``` - - - ```python - completion = portkey.chat.completions.create( - messages= [{ "role": 'user', "content": 'Say this is a test' }], - model= 'deepseek-chat' - ) +## Advanced Features - print(completion) - ``` - - +### Multi-round Conversations -### 4. Invoke Multi-round Conversation with DeepSeek +DeepSeek supports multi-turn conversations where context is maintained across messages: - - - ```javascript - const client = new Portkey({ - apiKey: "PORTKEY_API_KEY", // defaults to process.env["PORTKEY_API_KEY"] - provider:"@PROVIDER" // Your DeepSeek Virtual Key - }) - - // Function to send chat messages and get a response - async function sendChatMessages(messages) { - try { - const response = await axios.post(baseURL, { - model: 'deepseek-chat', - messages: messages - }, { headers: headers }); - return response.data; - } catch (error) { - console.error('Error during the API request:', error.response ? error.response.data : error.message); - return null; - } - } + +```python Python +from portkey_ai import Portkey - // Round 1 - (async () => { - let messages = [{ role: 'user', content: "What's the highest mountain in the world?" }]; - - let response = await sendChatMessages(messages); - if (response) { - messages.push(response.choices[0].message); - console.log(`Messages Round 1: ${JSON.stringify(messages, null, 2)}`); - } - - // Round 2 - messages.push({ role: 'user', content: 'What is the second?' }); - response = await sendChatMessages(messages); - if (response) { - messages.push(response.choices[0].message); - console.log(`Messages Round 2: ${JSON.stringify(messages, null, 2)}`); - } - })(); - ``` - - - ```python client = Portkey( - api_key="PORTKEY_API_KEY", # Replace with your Portkey API key - provider="@PROVIDER" # Replace with your virtual key for DeepSeek + api_key="PORTKEY_API_KEY", + provider="@deepseek" ) # Round 1 @@ -157,59 +156,50 @@ Use the Portkey instance to send requests to DeepSeek. You can also override the messages.append(response.choices[0].message) print(f"Messages Round 2: {messages}") ``` - - -### 5. JSON Output with DeepSeek +```js Javascript +import Portkey from 'portkey-ai' - - - ```javascript const client = new Portkey({ - apiKey: "PORTKEY_API_KEY", // defaults to process.env["PORTKEY_API_KEY"] - provider:"@PROVIDER" // Your DeepSeek Virtual Key - }) - - const systemPrompt = ` - The user will provide some exam text. Please parse the "question" and "answer" and output them in JSON format. - - EXAMPLE INPUT: - Which is the highest mountain in the world? Mount Everest. + apiKey: "PORTKEY_API_KEY", + provider: "@deepseek" +}) - EXAMPLE JSON OUTPUT: - { - "question": "Which is the highest mountain in the world?", - "answer": "Mount Everest" - } - `; +// Round 1 +let messages = [{ role: "user", content: "What's the highest mountain in the world?" }] - const userPrompt = "Which is the longest river in the world? The Nile River."; +let response = await client.chat.completions.create({ + model: "deepseek-chat", + messages: messages +}) - const messages = [ - { role: "system", content: systemPrompt }, - { role: "user", content: userPrompt } - ]; +messages.push(response.choices[0].message) +console.log(`Messages Round 1: ${JSON.stringify(messages, null, 2)}`) - client.chat.completions.create({ +// Round 2 +messages.push({ role: "user", content: "What is the second?" }) +response = await client.chat.completions.create({ model: "deepseek-chat", - messages: messages, - responseFormat: { - type: 'json_object' - } - }).then(response => { - console.log(JSON.parse(response.choices[0].message.content)); - }).catch(error => { - console.error('Error:', error); - }); + messages: messages +}) + +messages.push(response.choices[0].message) +console.log(`Messages Round 2: ${JSON.stringify(messages, null, 2)}`) ``` - - - ```python + + +### JSON Output + +Force structured JSON responses from DeepSeek models: + + +```python Python import json +from portkey_ai import Portkey client = Portkey( - api_key="PORTKEY_API_KEY", # Replace with your Portkey API key - provider="@PROVIDER" # Replace with your virtual key for DeepSeek + api_key="PORTKEY_API_KEY", + provider="@deepseek" ) system_prompt = """ @@ -227,42 +217,88 @@ Use the Portkey instance to send requests to DeepSeek. You can also override the user_prompt = "Which is the longest river in the world? The Nile River." - messages = [{"role": "system", "content": system_prompt}, - {"role": "user", "content": user_prompt}] +messages = [ + {"role": "system", "content": system_prompt}, + {"role": "user", "content": user_prompt} +] response = client.chat.completions.create( model="deepseek-chat", messages=messages, - response_format={ - 'type': 'json_object' - } + response_format={"type": "json_object"} ) print(json.loads(response.choices[0].message.content)) ``` - - + +```js Javascript +import Portkey from 'portkey-ai' + +const client = new Portkey({ + apiKey: "PORTKEY_API_KEY", + provider: "@deepseek" +}) + +const systemPrompt = ` +The user will provide some exam text. Please parse the "question" and "answer" and output them in JSON format. + +EXAMPLE INPUT: +Which is the highest mountain in the world? Mount Everest. + +EXAMPLE JSON OUTPUT: +{ + "question": "Which is the highest mountain in the world?", + "answer": "Mount Everest" +} +` + +const userPrompt = "Which is the longest river in the world? The Nile River." + +const messages = [ + { role: "system", content: systemPrompt }, + { role: "user", content: userPrompt } +] + +const response = await client.chat.completions.create({ + model: "deepseek-chat", + messages: messages, + responseFormat: { type: "json_object" } +}) + +console.log(JSON.parse(response.choices[0].message.content)) +``` + ## Managing DeepSeek Prompts -You can manage all prompts to DeepSeek in the [Prompt Library](/product/prompt-library). All the current models of DeepSeek are supported and you can easily start testing different prompts. +Manage all prompt templates to DeepSeek in the [Prompt Library](/product/prompt-library). All current DeepSeek models are supported, and you can easily test different prompts. -Once you're ready with your prompt, you can use the `portkey.prompts.completions.create` interface to use the prompt in your application. +Use the `portkey.prompts.completions.create` interface to use the prompt in an application. -### Supported Endpoints +## Supported Endpoints -1. `CHAT_COMPLETIONS` -2. `STREAM_CHAT_COMPLETIONS` +- Chat Completions +- Streaming Chat Completions -The complete list of features supported in the SDK is available on the link below. +## Next Steps - - Learn more about the Portkey SDK Client - + + + Add metadata to your DeepSeek requests + + + Add gateway configs to your DeepSeek requests + + + Trace your DeepSeek requests + + + Setup fallback from OpenAI to DeepSeek + + -You'll find more information in the relevant sections: +For complete SDK documentation: -1. [Add metadata to your requests](/product/observability/metadata) -2. [Add gateway configs to your DeepSeek requests](/product/ai-gateway/configs) -3. [Tracing DeepSeek requests](/product/observability/traces) -4. [Setup a fallback from OpenAI to DeepSeek APIs](/product/ai-gateway/fallbacks) + + Complete Portkey SDK documentation + diff --git a/integrations/llms/featherless.mdx b/integrations/llms/featherless.mdx index a1a05a7a..264d82cd 100644 --- a/integrations/llms/featherless.mdx +++ b/integrations/llms/featherless.mdx @@ -1,114 +1,149 @@ --- title: "Featherless AI" +description: Access 11,900+ open-source models through Featherless AI and Portkey. --- -Portkey provides a robust and secure gateway to facilitate the integration of various Large Language Models (LLMs) into your applications, including [Featherless](https://featherless.ai/). +## Quick Start -Featherless.ai is one of the largest AI inference access to 11,900+ open source models. You can instantly deploy at scale for fine-tuning, testing, and production with unlimited tokens +Get started with Featherless AI in under 2 minutes: -With Portkey, you can take advantage of features like fast AI gateway access, observability, prompt management, and more, all while ensuring the secure management of your LLM API keys Portkey's [model catalog](/product/model-catalog). + - -Provider Slug. `featherless-ai` - -## Portkey SDK Integration with Featherless AI Models +```python Python icon="python" +from portkey_ai import Portkey -Portkey provides a consistent API to interact with models from various providers. To integrate Featherless AI with Portkey: +# 1. Install: pip install portkey-ai +# 2. Add @featherless-ai provider in model catalog +# 3. Use it: -### 1\. Install the Portkey SDK +portkey = Portkey(api_key="PORTKEY_API_KEY") -Add the Portkey SDK to your application to interact with Featherless AI's API through Portkey's gateway. - - - ```sh - npm install --save portkey-ai - ``` - - - ```sh - pip install portkey-ai - ``` - +response = portkey.chat.completions.create( + model="@featherless-ai/google/gemma-3-4b-it", + messages=[{"role": "user", "content": "Hello!"}] +) - +print(response.choices[0].message.content) +``` +```js Javascript icon="square-js" +import Portkey from 'portkey-ai' +// 1. Install: npm install portkey-ai +// 2. Add @featherless-ai provider in model catalog +// 3. Use it: +const portkey = new Portkey({ + apiKey: "PORTKEY_API_KEY" +}) -### 2\. Initialize Portkey with the Virtual Key +const response = await portkey.chat.completions.create({ + model: "@featherless-ai/google/gemma-3-4b-it", + messages: [{ role: "user", content: "Hello!" }] +}) -To use Featherless AI with Portkey, [get your API key from here](https://featherless.ai/), then add it to Portkey to create the virtual key. - - - ```js - import Portkey from 'portkey-ai' +console.log(response.choices[0].message.content) +``` - const portkey = new Portkey({ - apiKey: "PORTKEY_API_KEY", // defaults to process.env["PORTKEY_API_KEY"] - provider:"@PROVIDER" // Your featherless provider slug from model catalog - }) - ``` - - - ```python - from portkey_ai import Portkey +```python OpenAI Py icon="python" +from openai import OpenAI +from portkey_ai import PORTKEY_GATEWAY_URL - portkey = Portkey( - api_key="PORTKEY_API_KEY", # Replace with your Portkey API key - provider="@PROVIDER" # Replace with your model provider slug for featherless AI - ) - ``` - +# 1. Install: pip install openai portkey-ai +# 2. Add @featherless-ai provider in model catalog +# 3. Use it: - +client = OpenAI( + api_key="PORTKEY_API_KEY", # Portkey API key + base_url=PORTKEY_GATEWAY_URL +) +response = client.chat.completions.create( + model="@featherless-ai/google/gemma-3-4b-it", + messages=[{"role": "user", "content": "Hello!"}] +) +print(response.choices[0].message.content) +``` -### 3\. Invoke Chat Completions with Featherless AI +```js OpenAI JS icon="square-js" +import OpenAI from "openai" +import { PORTKEY_GATEWAY_URL } from "portkey-ai" -Use the Portkey instance to send requests to Featherless AI. +// 1. Install: npm install openai portkey-ai +// 2. Add @featherless-ai provider in model catalog +// 3. Use it: - - - ```js - const chatCompletion = await portkey.chat.completions.create({ - messages: [{ role: 'user', content: 'Say this is a test' }], - model: 'google/gemma-3-4b-it', - }); +const client = new OpenAI({ + apiKey: "PORTKEY_API_KEY", // Portkey API key + baseURL: PORTKEY_GATEWAY_URL +}) - console.log(chatCompletion.choices);d - ``` - - - ```python - completion = portkey.chat.completions.create( - messages= [{ "role": 'user', "content": 'Say this is a test' }], - model= 'google/gemma-3-4b-it' - ) +const response = await client.chat.completions.create({ + model: "@featherless-ai/google/gemma-3-4b-it", + messages: [{ role: "user", content: "Hello!" }] +}) - print(completion) - ``` - - +console.log(response.choices[0].message.content) +``` +```sh cURL icon="square-terminal" +# 1. Add @featherless-ai provider in model catalog +# 2. Use it: +curl https://api.portkey.ai/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "x-portkey-api-key: $PORTKEY_API_KEY" \ + -d '{ + "model": "@featherless-ai/google/gemma-3-4b-it", + "messages": [{"role": "user", "content": "Hello!"}] + }' +``` + -## Managing Featherless AI Prompts +## Add Provider in Model Catalog -You can manage all prompts to Featherless AI in the [Prompt Library](/product/prompt-library). All the current models of Featherless AI are supported and you can easily start testing different prompts. +Before making requests, add Featherless AI to your Model Catalog: -Once you're ready with your prompt, you can use the `portkey.prompts.completions.create` interface to use the prompt in your application. +1. Go to [**Model Catalog → Add Provider**](https://app.portkey.ai/model-catalog/providers) +2. Select **Featherless AI** +3. Enter your [Featherless AI API key](https://featherless.ai/) +4. Name your provider (e.g., `featherless-ai`) + + + See all setup options and detailed configuration instructions + + +--- ## Supported Models -The complete list of features supported in the SDK are available on the link below. - - +Featherless AI provides access to 11,900+ open-source models with unlimited tokens: + +Check [Featherless AI's documentation](https://featherless.ai/) for the complete model catalog. -You'll find more information in the relevant sections: +--- -1. [Add metadata to your requests](/product/observability/metadata) -2. A[dd gateway configs to your Featherless](/product/ai-gateway/configs) -3. [Tracing Featherless requests](/product/observability/traces) -4. [Setup a fallback from OpenAI to Featherless APIs](/product/ai-gateway/fallbacks) +## Next Steps + + + + Add fallbacks, load balancing, and more + + + Monitor and trace your Featherless requests + + + Manage and version your prompts + + + Add custom metadata to requests + + + +For complete SDK documentation: + + + Complete Portkey SDK documentation + diff --git a/integrations/llms/fireworks.mdx b/integrations/llms/fireworks.mdx index b7739066..a09efa9d 100644 --- a/integrations/llms/fireworks.mdx +++ b/integrations/llms/fireworks.mdx @@ -1,329 +1,364 @@ --- title: "Fireworks" +description: Use Fireworks for chat, vision, embeddings, and image generation with advanced grammar and JSON modes through Portkey. --- -Portkey provides a robust and secure gateway to facilitate the integration of various models into your apps, including [chat](/integrations/llms/fireworks#id-3.-invoke-chat-completions-with-fireworks), [vision](/integrations/llms/fireworks#using-vision-models), [image generation](/integrations/llms/fireworks#using-image-generation-models), and [embedding](/integrations/llms/fireworks#using-embeddings-models) models hosted on the [Fireworks platform](https://fireworks.ai/). +## Quick Start -With Portkey, you can take advantage of features like fast AI gateway access, observability, prompt management, and more, all while ensuring the secure management of your LLM API keys through a [virtual key](/product/ai-gateway/virtual-keys) system. - -Provider Slug. `fireworks-ai` - -## Portkey SDK Integration with Fireworks Models +Get started with Fireworks in under 2 minutes: -Portkey provides a consistent API to interact with models from various providers. To integrate Fireworks with Portkey: + -### 1\. Install the Portkey SDK - - - ```sh - npm install --save portkey-ai - ``` +```python Python icon="python" +from portkey_ai import Portkey - - - ```sh - pip install portkey-ai - ``` +# 1. Install: pip install portkey-ai +# 2. Add @fireworks-ai provider in model catalog +# 3. Use it: - +portkey = Portkey(api_key="PORTKEY_API_KEY") - +response = portkey.chat.completions.create( + model="@fireworks-ai/accounts/fireworks/models/llama-v3-70b-instruct", + messages=[{"role": "user", "content": "Hello!"}] +) +print(response.choices[0].message.content) +``` -### 2\. Initialize Portkey with the Virtual Key +```js Javascript icon="square-js" +import Portkey from 'portkey-ai' -To use Fireworks with Portkey, [get your API key from here](https://fireworks.ai/api-keys), then add it to Portkey to create the virtual key. - - - ```js - import Portkey from 'portkey-ai' +// 1. Install: npm install portkey-ai +// 2. Add @fireworks-ai provider in model catalog +// 3. Use it: - const portkey = new Portkey({ - apiKey: "PORTKEY_API_KEY", // Defaults to process.env["PORTKEY_API_KEY"] - provider:"@FIREWORKS_PROVIDER" // Your Virtual Key - }) - ``` - - - ```python - from portkey_ai import Portkey +const portkey = new Portkey({ + apiKey: "PORTKEY_API_KEY" +}) - portkey = Portkey( - api_key="PORTKEY_API_KEY", # Defaults to os.env("PORTKEY_API_KEY") - provider="@FIREWORKS_PROVIDER" # Your Virtual Key - ) - ``` - +const response = await portkey.chat.completions.create({ + model: "@fireworks-ai/accounts/fireworks/models/llama-v3-70b-instruct", + messages: [{ role: "user", content: "Hello!" }] +}) - +console.log(response.choices[0].message.content) +``` +```python OpenAI Py icon="python" +from openai import OpenAI +from portkey_ai import PORTKEY_GATEWAY_URL +# 1. Install: pip install openai portkey-ai +# 2. Add @fireworks-ai provider in model catalog +# 3. Use it: +client = OpenAI( + api_key="PORTKEY_API_KEY", # Portkey API key + base_url=PORTKEY_GATEWAY_URL +) -### **3\. Invoke Chat Completions with** Fireworks +response = client.chat.completions.create( + model="@fireworks-ai/accounts/fireworks/models/llama-v3-70b-instruct", + messages=[{"role": "user", "content": "Hello!"}] +) -You can use the Portkey instance now to send requests to Fireworks API. - - - ```js - const chatCompletion = await portkey.chat.completions.create({ - messages: [{ role: 'user', content: 'Say this is a test' }], - model: 'accounts/fireworks/models/llama-v3-70b-instruct', - }); +print(response.choices[0].message.content) +``` - console.log(chatCompletion.choices); - ``` - - - ```python - completion = portkey.chat.completions.create( - messages= [{ "role": 'user', "content": 'Say this is a test' }], - model= 'accounts/fireworks/models/llama-v3-70b-instruct' - ) +```js OpenAI JS icon="square-js" +import OpenAI from "openai" +import { PORTKEY_GATEWAY_URL } from "portkey-ai" - print(completion) - ``` - +// 1. Install: npm install openai portkey-ai +// 2. Add @fireworks-ai provider in model catalog +// 3. Use it: - +const client = new OpenAI({ + apiKey: "PORTKEY_API_KEY", // Portkey API key + baseURL: PORTKEY_GATEWAY_URL +}) +const response = await client.chat.completions.create({ + model: "@fireworks-ai/accounts/fireworks/models/llama-v3-70b-instruct", + messages: [{ role: "user", content: "Hello!" }] +}) -Now, let's explore how you can use Portkey to call other models (vision, embedding, image) on the Fireworks API: +console.log(response.choices[0].message.content) +``` -### Using Embeddings Models +```sh cURL icon="square-terminal" +# 1. Add @fireworks-ai provider in model catalog +# 2. Use it: -Call any [embedding model hosted on Fireworks](https://readme.fireworks.ai/docs/querying-embeddings-models#list-of-available-models) with the familiar OpenAI embeddings signature: - - - ```js - const embeddings = await portkey.embeddings.create({ - input: "create vector representation on this sentence", - model: "thenlper/gte-large", - }); +curl https://api.portkey.ai/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "x-portkey-api-key: $PORTKEY_API_KEY" \ + -d '{ + "model": "@fireworks-ai/accounts/fireworks/models/llama-v3-70b-instruct", + "messages": [{"role": "user", "content": "Hello!"}] + }' +``` - console.log(embeddings); - ``` - - - ```python - embeddings = portkey.embeddings.create( - input='create vector representation on this sentence', - model='thenlper/gte-large' - ) + - print(embeddings) - ``` - +## Add Provider in Model Catalog - +Before making requests, add Fireworks to your Model Catalog: +1. Go to [**Model Catalog → Add Provider**](https://app.portkey.ai/model-catalog/providers) +2. Select **Fireworks** +3. Enter your [Fireworks API key](https://fireworks.ai/api-keys) +4. Name your provider (e.g., `fireworks-ai`) -### Using Vision Models + + See all setup options and detailed configuration instructions + -Portkey natively supports [vision models hosted on Fireworks](https://readme.fireworks.ai/docs/querying-vision-language-models): - - - ```js - const completion = await portkey.chat.completions.create( - messages: [ - { "role": "user", "content": [ - { "type": "text","text": "Can you describe this image?" }, - { "type": "image_url", "image_url": - { "url": "https://images.unsplash.com/photo-1582538885592-e70a5d7ab3d3?ixlib=rb-4.0.3&ixid=M3wxMjA3fDB8MHxwaG90by1wYWdlfHx8fGVufDB8fHx8fA%3D%3D&auto=format&fit=crop&w=1770&q=80" } - } - ] - } - ], - model: 'accounts/fireworks/models/firellava-13b' - ) +--- - console.log(completion); - ``` - - - ```python - completion = portkey.chat.completions.create( - messages= [ - { "role": "user", "content": [ - { "type": "text","text": "Can you describe this image?" }, - { "type": "image_url", "image_url": - { "url": "https://images.unsplash.com/photo-1582538885592-e70a5d7ab3d3?ixlib=rb-4.0.3&ixid=M3wxMjA3fDB8MHxwaG90by1wYWdlfHx8fGVufDB8fHx8fA%3D%3D&auto=format&fit=crop&w=1770&q=80" } - } - ] - } - ], - model= 'accounts/fireworks/models/firellava-13b' - ) +## Fireworks Capabilities - print(completion) - ``` - +### Embeddings - +Generate embeddings using Fireworks models: + +```python Python +from portkey_ai import Portkey -### Using Image Generation Models +portkey = Portkey(api_key="PORTKEY_API_KEY", provider="@fireworks-ai") -Portkey also supports calling [image generation models hosted on Fireworks](https://readme.fireworks.ai/reference/image%5Fgenerationaccountsfireworksmodelsstable-diffusion-xl-1024-v1-0) in the familiar OpenAI signature: - - - ```js - import Portkey from 'portkey-ai'; - import fs from 'fs'; +embeddings = portkey.embeddings.create( + model="thenlper/gte-large", + input="create vector representation on this sentence" +) - const portkey = new Portkey({ - apiKey: "PORTKEY_API_KEY", - provider:"@FIREWORKS_PROVIDER" - }); +print(embeddings.data[0].embedding) +``` - async function main(){ - const image = await portkey.images.generate({ - model: "accounts/fireworks/models/stable-diffusion-xl-1024-v1-0", - prompt: "An orange elephant in a purple pond" - }); +```javascript Node.js +import Portkey from 'portkey-ai'; - const imageData = image.data[0].b64_json as string; +const portkey = new Portkey({ + apiKey: 'PORTKEY_API_KEY', + provider: '@fireworks-ai' +}); - fs.writeFileSync("fireworks-image-gen.png", Buffer.from(imageData, 'base64')); - } +const embeddings = await portkey.embeddings.create({ + model: "thenlper/gte-large", + input: "create vector representation on this sentence" +}); - main() - ``` - - - ```python +console.log(embeddings.data[0].embedding); +``` + + + +### Vision Models + +Process images with vision models: + + + +```python Python +from portkey_ai import Portkey + +portkey = Portkey(api_key="PORTKEY_API_KEY", provider="@fireworks-ai") + +response = portkey.chat.completions.create( + model="accounts/fireworks/models/firellava-13b", + messages=[{ + "role": "user", + "content": [ + {"type": "text", "text": "Can you describe this image?"}, + { + "type": "image_url", + "image_url": { + "url": "https://images.unsplash.com/photo-1582538885592-e70a5d7ab3d3" + } + } + ] + }] + ) + +print(response.choices[0].message.content) +``` + +```javascript Node.js +import Portkey from 'portkey-ai'; + +const portkey = new Portkey({ + apiKey: 'PORTKEY_API_KEY', + provider: '@fireworks-ai' +}); + +const response = await portkey.chat.completions.create({ + model: "accounts/fireworks/models/firellava-13b", + messages: [{ + role: "user", + content: [ + { type: "text", text: "Can you describe this image?" }, + { + type: "image_url", + image_url: { + url: "https://images.unsplash.com/photo-1582538885592-e70a5d7ab3d3" + } + } + ] + }] +}); + +console.log(response.choices[0].message.content); +``` + + + +### Image Generation + +Generate images with Stable Diffusion: + + + +```python Python from portkey_ai import Portkey - import base64 +import base64 from io import BytesIO from PIL import Image - portkey = Portkey( - api_key="PORTKEY_API_KEY", - provider="@FIREWORKS_PROVIDER" - ) +portkey = Portkey(api_key="PORTKEY_API_KEY", provider="@fireworks-ai") image = portkey.images.generate( model="accounts/fireworks/models/stable-diffusion-xl-1024-v1-0", prompt="An orange elephant in a purple pond" ) - Image.open(BytesIO(base64.b64decode(image.data[0].b64_json))).save("fireworks-image-gen.png") +Image.open(BytesIO(base64.b64decode(image.data[0].b64_json))).save("generated.png") ``` - - +```javascript Node.js +import Portkey from 'portkey-ai'; +import fs from 'fs'; ---- +const portkey = new Portkey({ + apiKey: 'PORTKEY_API_KEY', + provider: '@fireworks-ai' +}); -## Fireworks Grammar Mode +const image = await portkey.images.generate({ + model: "accounts/fireworks/models/stable-diffusion-xl-1024-v1-0", + prompt: "An orange elephant in a purple pond" +}); -Fireworks lets you define [formal grammars](https://en.wikipedia.org/wiki/Formal%5Fgrammar) to constrain model outputs. You can use it to force the model to generate valid JSON, speak only in emojis, or anything else. ([Originally created by GGML](https://github.com/ggerganov/llama.cpp/tree/master/grammars)) +const imageData = image.data[0].b64_json; +fs.writeFileSync("generated.png", Buffer.from(imageData, 'base64')); +``` -Grammar mode is set with the `response_format` param. Just pass your grammar definition with `{"type": "grammar", "grammar": grammar_definition}` + + +### Function Calling + +Use function calling with Fireworks models: + + + Explore function calling examples and best practices + -Let's say you want to classify patient requests into 3 pre-defined classes: +### Grammar Mode - - - ```python +Define formal grammars to constrain model outputs: + + + +```python Python from portkey_ai import Portkey - portkey = Portkey( - api_key="PORTKEY_API_KEY", # Defaults to os.env("PORTKEY_API_KEY") - provider="@FIREWORKS_PROVIDER" # Your Virtual Key - ) +portkey = Portkey(api_key="PORTKEY_API_KEY", provider="@fireworks-ai") patient_classification = """ root ::= diagnosis diagnosis ::= "flu" | "dengue" | "malaria" """ - completion = portkey.chat.completions.create( - messages= [{ "role": 'user', "content": 'Say this is a test' }], - response_format={"type": "grammar", "grammar": patient_classification}, - model= 'accounts/fireworks/models/llama-v3-70b-instruct' - ) +response = portkey.chat.completions.create( + model="accounts/fireworks/models/llama-v3-70b-instruct", + messages=[{"role": "user", "content": "Patient has fever and chills"}], + response_format={"type": "grammar", "grammar": patient_classification} +) - print(completion) +print(response.choices[0].message.content) ``` - - - ```js - import Portkey from 'portkey-ai' - - const portkey = new Portkey({ - apiKey: "PORTKEY_API_KEY", // Defaults to process.env["PORTKEY_API_KEY"] - provider:"@FIREWORKS_PROVIDER" // Your Virtual Key - }) - - const patient_classification = ` - root ::= diagnosis + +```javascript Node.js +import Portkey from 'portkey-ai'; + +const portkey = new Portkey({ + apiKey: 'PORTKEY_API_KEY', + provider: '@fireworks-ai' +}); + +const patient_classification = ` +root ::= diagnosis diagnosis ::= "flu" | "dengue" | "malaria" `; - const chatCompletion = await portkey.chat.completions.create({ - messages: [{ role: 'user', content: 'Say this is a test' }], - response_format: {"type": "grammar", "grammar": patient_classification}, - model: 'accounts/fireworks/models/llama-v3-70b-instruct', +const response = await portkey.chat.completions.create({ + model: "accounts/fireworks/models/llama-v3-70b-instruct", + messages: [{ role: "user", content: "Patient has fever and chills" }], + response_format: { type: "grammar", grammar: patient_classification } }); - console.log(chatCompletion.choices); +console.log(response.choices[0].message.content); ``` - - + + + + Learn more about grammar mode and examples + - -NOTE: Fireworks Grammer Mode is not supported on Portkey prompts playground - -[Explore the Fireworks guide for more examples and a deeper dive on Grammer node](https://readme.fireworks.ai/docs/structured-output-grammar-based). +### JSON Mode -## Fireworks JSON Mode +Force JSON output with optional schema validation: -You can force the model to return (1) **An arbitrary JSON**, or (2) **JSON with given schema** with Fireworks' JSON mode. - + - - ```python +```python Python from portkey_ai import Portkey +from pydantic import BaseModel +from typing import List - portkey = Portkey( - api_key="PORTKEY_API_KEY", # Defaults to os.env("PORTKEY_API_KEY") - provider="@FIREWORKS_PROVIDER" # Your Virtual Key - ) +portkey = Portkey(api_key="PORTKEY_API_KEY", provider="@fireworks-ai") class Recipe(BaseModel): title: str description: str steps: List[str] - json_response = portkey.chat.completions.create( - messages = [{ "role": 'user', "content": 'Give me a recipe for making Ramen, in JSON format' }], - model = 'accounts/fireworks/models/llama-v3-70b-instruct', - response_format = { - "type":"json_object", +response = portkey.chat.completions.create( + model="accounts/fireworks/models/llama-v3-70b-instruct", + messages=[{"role": "user", "content": "Give me a recipe for making Ramen"}], + response_format={ + "type": "json_object", "schema": Recipe.schema_json() } ) - print(json_response.choices[0].message.content) +print(response.choices[0].message.content) ``` - - - ```js - import Portkey from 'portkey-ai' +```javascript Node.js +import Portkey from 'portkey-ai'; - const portkey = new Portkey({ - apiKey: "PORTKEY_API_KEY", // Defaults to process.env["PORTKEY_API_KEY"] - provider:"@FIREWORKS_PROVIDER" // Your Virtual Key - }) +const portkey = new Portkey({ + apiKey: 'PORTKEY_API_KEY', + provider: '@fireworks-ai' +}); - asyn function main(){ - const json_response = await portkey.chat.completions.create({ - messages: [{role: "user",content: `Give me a recipe for making Ramen, in JSON format`}], +const response = await portkey.chat.completions.create({ model: "accounts/fireworks/models/llama-v3-70b-instruct", + messages: [{ role: "user", content: "Give me a recipe for making Ramen" }], response_format: { type: "json_object", schema: { @@ -336,40 +371,50 @@ You can force the model to return (1) **An arbitrary JSON**, or (2) **JSON with } } }); - } - - console.log(json_response.choices[0].message.content); - main() +console.log(response.choices[0].message.content); ``` - - - + + + Learn more about JSON mode + -[Explore Fireworks docs for JSON mode for more examples](https://readme.fireworks.ai/docs/structured-response-formatting). +--- -## Fireworks Function Calling +## Supported Models -Portkey also supports function calling mode on Fireworks. [Explore this cookbook for a deep dive and examples](/guides/getting-started/function-calling). +Fireworks provides fast inference for 49+ language models including: -## Managing Fireworks Prompts +- Llama models (v3-70b-instruct, v3-8b-instruct) +- FireLLaVA vision models +- Stable Diffusion XL image generation +- Embedding models (gte-large, nomic-embed) -You can manage all Fireworks prompts in the [Prompt Library](/product/prompt-library). All the current 49+ language models available on Fireworks are supported and you can easily start testing different prompts. +Check [Fireworks documentation](https://readme.fireworks.ai/docs/querying-text-models) for the complete model list. -Once you're ready with your prompt, you can use the `portkey.prompts.completions.create` interface to use the prompt in your application. +--- ## Next Steps -The complete list of features supported in the SDK are available on the link below. - - + + + Add fallbacks, load balancing, and more + + + Monitor and trace your Fireworks requests + + + Manage and version your prompts + + + Add custom metadata to requests + + + +For complete SDK documentation: + + + Complete Portkey SDK documentation - -You'll find more information in the relevant sections: - -1. [Add metadata to your requests](/product/observability/metadata) -2. [Add gateway configs to your ](/product/ai-gateway/configs)[requests](/product/ai-gateway/configs) -3. [Tracing requests](/product/observability/traces) -4. [Setup a fallback from OpenAI to Firework APIs](/product/ai-gateway/fallbacks) diff --git a/integrations/llms/fireworks/files.mdx b/integrations/llms/fireworks/files.mdx index 0f3d3d39..cad533fa 100644 --- a/integrations/llms/fireworks/files.mdx +++ b/integrations/llms/fireworks/files.mdx @@ -53,7 +53,7 @@ await uploadFile(); ```sh -# you can also use a virtual key here +# you can also use a provider from Model Catalog here curl --location 'https://api.portkey.ai/v1/files' \ --header 'x-portkey-api-key: ' \ --header 'x-portkey-provider: fireworks-ai' \ diff --git a/integrations/llms/gemini.mdx b/integrations/llms/gemini.mdx index 3b5d6061..7b32efd4 100644 --- a/integrations/llms/gemini.mdx +++ b/integrations/llms/gemini.mdx @@ -4,95 +4,136 @@ title: "Google Gemini" Portkey provides a robust and secure gateway to facilitate the integration of various Large Language Models (LLMs) into your applications, including [Google Gemini APIs](https://cloud.google.com/vertex-ai/docs/generative-ai/model-reference/gemini). -With Portkey, you can take advantage of features like fast AI gateway access, observability, prompt management, and more, all while ensuring the secure management of your LLM API keys through a [virtual key](/product/ai-gateway/virtual-keys) system. -Provider Slug. `google` +With Portkey, you can take advantage of features like fast AI gateway access, observability, prompt management, and more, all while ensuring the secure management of your LLM API keys through [Model Catalog](/product/model-catalog). +## Quick Start -## Portkey SDK Integration with Google Gemini Models - -Portkey provides a consistent API to interact with models from various providers. To integrate Google Gemini with Portkey: + -### 1\. Install the Portkey SDK +```python Python icon="python" +from portkey_ai import Portkey -Add the Portkey SDK to your application to interact with Google Gemini's API through Portkey's gateway. - - +# 1. Install: pip install portkey-ai +# 2. Add @google provider in Model Catalog +# 3. Use it: -```sh -npm install --save portkey-ai -``` - - -```sh -pip install portkey-ai -``` - - +portkey = Portkey(api_key="PORTKEY_API_KEY") +response = portkey.chat.completions.create( + model="@google/gemini-1.5-pro", + messages=[{"role": "user", "content": "Say this is a test"}] +) -### 2\. Initialize Portkey with the Virtual Key +print(response.choices[0].message.content) +``` -To use Gemini with Portkey, [get your API key from here](https://aistudio.google.com/app/apikey), then add it to Portkey to create the virtual key. - - -```js +```js Javascript icon="square-js" import Portkey from 'portkey-ai' +// 1. Install: npm install portkey-ai +// 2. Add @google provider in Model Catalog +// 3. Use it: + const portkey = new Portkey({ - apiKey: "PORTKEY_API_KEY", // defaults to process.env["PORTKEY_API_KEY"] - provider:"@PROVIDER" // Your Google Virtual Key + apiKey: "PORTKEY_API_KEY" }) + +const response = await portkey.chat.completions.create({ + model: "@google/gemini-1.5-pro", + messages: [{ role: "user", content: "Say this is a test" }] +}) + +console.log(response.choices[0].message.content) ``` - - - ```python - from portkey_ai import Portkey - portkey = Portkey( - api_key="PORTKEY_API_KEY", # Replace with your Portkey API key - provider="@PROVIDER" # Replace with your virtual key for Google - ) - ``` - +```python OpenAI Py icon="python" +from openai import OpenAI +from portkey_ai import PORTKEY_GATEWAY_URL - +# 1. Install: pip install openai portkey-ai +# 2. Add @google provider in Model Catalog +# 3. Use it: +client = OpenAI( + api_key="PORTKEY_API_KEY", # Portkey API key + base_url=PORTKEY_GATEWAY_URL +) +response = client.chat.completions.create( + model="@google/gemini-1.5-pro", + messages=[{"role": "user", "content": "Say this is a test"}] +) -### **3\. Invoke Chat Completions with** Google Gemini +print(response.choices[0].message.content) +``` -Use the Portkey instance to send requests to Google Gemini. You can also override the virtual key directly in the API call if needed. - - - ```js - const chatCompletion = await portkey.chat.completions.create({ - messages: [ - { role: 'system', content: 'You are not a helpful assistant' }, - { role: 'user', content: 'Say this is a test' } - ], - model: 'gemini-1.5-pro', - }); +```js OpenAI JS icon="square-js" +import OpenAI from "openai" +import { PORTKEY_GATEWAY_URL } from "portkey-ai" - console.log(chatCompletion.choices); - ``` - - +// 1. Install: npm install openai portkey-ai +// 2. Add @google provider in Model Catalog +// 3. Use it: -```python -completion = portkey.chat.completions.create( - messages= [ - { "role": 'system', "content": 'You are not a helpful assistant' }, - { "role": 'user', "content": 'Say this is a test' } - ], - model= 'gemini-1.5-pro' -) +const client = new OpenAI({ + apiKey: "PORTKEY_API_KEY", // Portkey API key + baseURL: PORTKEY_GATEWAY_URL +}) + +const response = await client.chat.completions.create({ + model: "@google/gemini-1.5-pro", + messages: [{ role: "user", content: "Say this is a test" }] +}) + +console.log(response.choices[0].message.content) +``` + +```sh cURL icon="square-terminal" +# 1. Add @google provider in Model Catalog +# 2. Use it: -print(completion) +curl https://api.portkey.ai/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "x-portkey-api-key: $PORTKEY_API_KEY" \ + -d '{ + "model": "@google/gemini-1.5-pro", + "messages": [{"role": "user", "content": "Say this is a test"}] + }' ``` - - + + +--- + +## Add Provider in Model Catalog + + + + +Go to [**Model Catalog → Add Provider**](https://app.portkey.ai/model-catalog/providers) in your Portkey dashboard. + + + + + +Find and select **Google** from the provider list. + + + + + +Get your API key from [Google AI Studio](https://aistudio.google.com/app/apikey) and enter it in Model Catalog. + + + + + +Save your configuration. Your provider slug will be `@google` (or a custom name you specify). + + + + Portkey supports the `system_instructions` parameter for Google Gemini 1.5 - allowing you to control the behavior and output of your Gemini-powered applications with ease. @@ -100,14 +141,17 @@ Portkey supports the `system_instructions` parameter for Google Gemini 1.5 - all Simply include your Gemini system prompt as part of the `{"role":"system"}` message within the `messages` array of your request body. Portkey Gateway will automatically transform your message to ensure seamless compatibility with the Google Gemini API. -## Function Calling +--- -Portkey supports function calling mode on Google's Gemini Models. Explore this Cookbook for a deep dive and examples: +## Gemini Capabilities -[Function Calling](/guides/getting-started/function-calling) +### Function Calling +Portkey supports function calling mode on Google's Gemini Models. Explore this cookbook for a deep dive and examples: +[Function Calling](/guides/getting-started/function-calling) +--- ## Advanced Multimodal Capabilities with Gemini @@ -1961,13 +2005,13 @@ Note that you will have to set [`strict_open_ai_compliance=False`](/product/ai-g // Initialize the Portkey client const portkey = new Portkey({ apiKey: "PORTKEY_API_KEY", // Replace with your Portkey API key - provider:"@PROVIDER", // your vertex-ai virtual key + provider:"@PROVIDER", // your Vertex AI provider slug strictOpenAiCompliance: false }); // Generate a chat completion async function getChatCompletionFunctions() { - const response = await portkey.chat.completions.create({ +const response = await portkey.chat.completions.create({ model: "gemini-2.5-flash-preview-04-17", max_tokens: 3000, thinking: { @@ -2133,29 +2177,14 @@ Note that you will have to set [`strict_open_ai_compliance=False`](/product/ai-g ``` -### Using reasoning_effort Parameter - -You can also control thinking using the OpenAI-compatible `reasoning_effort` parameter instead of `thinking.budget_tokens`: - -```python -response = portkey.chat.completions.create( - model="gemini-2.5-flash-preview-04-17", - max_tokens=3000, - reasoning_effort="medium", # Options: "none", "minimal", "low", "medium", "high" - messages=[{"role": "user", "content": "Explain quantum computing"}] -) -``` - -| reasoning_effort | Gemini thinkingLevel | -|------------------|---------------------| -| `none` | Disabled | -| `minimal`, `low` | `low` | -| `medium`, `high` | `high` | - Gemini grounding mode may not work via Portkey SDK. Contact support@portkey.ai for assistance. +--- + + + ## Image Generation (nano banana 🍌) Gemini models like `gemini-3-pro-image-preview` support native image generation capabilities. You can generate images by setting `modalities` to include `"image"` in your request. @@ -2541,13 +2570,26 @@ console.log(editResponse); ## Next Steps -The complete list of features supported in the SDK are available on the link below. - + + + +Complete SDK documentation and API reference -You'll find more information in the relevant sections: + +Add metadata to your Gemini requests + + + +Configure advanced gateway features + + + +Trace and monitor your Gemini requests + + + +Create fallback configurations between providers + -1. [Add metadata to your requests](/product/observability/metadata) -2. [Add gateway configs to your Gemini requests](/product/ai-gateway/configs) -3. [Tracing Google Gemini requests](/product/observability/traces) -4. [Setup a fallback from OpenAI to Gemini APIs](/product/ai-gateway/fallbacks) + diff --git a/integrations/llms/github.mdx b/integrations/llms/github.mdx index 652a2294..405b4255 100644 --- a/integrations/llms/github.mdx +++ b/integrations/llms/github.mdx @@ -1,105 +1,142 @@ --- -title: "Github" + +title: "GitHub Models" +description: Use GitHub Models Marketplace through Portkey for AI model integration. --- -Portkey provides a robust and secure gateway to facilitate the integration of various Large Language Models (LLMs) into your applications, including the models hosted on [Github Models Marketplace](https://github.com/marketplace/models). +## Quick Start - -Provider Slug: `github` - +Get started with GitHub Models in under 2 minutes: -## Portkey SDK Integration with Github Models + -Portkey provides a consistent API to interact with models from various providers. To integrate Github Models with Portkey: +```python Python icon="python" +from portkey_ai import Portkey -### 1\. Install the Portkey SDK - - - ```sh - npm install --save portkey-ai - ``` - - - ```sh - pip install -U portkey-ai - ``` - +# 1. Install: pip install portkey-ai +# 2. Add @github provider in model catalog +# 3. Use it: - +portkey = Portkey(api_key="PORTKEY_API_KEY") +response = portkey.chat.completions.create( + model="@github/Phi-3-small-128k-instruct", + messages=[{"role": "user", "content": "Hello!"}] +) +print(response.choices[0].message.content) +``` -### 2\. Initialize Portkey with Github Models +```js Javascript icon="square-js" +import Portkey from 'portkey-ai' -To use Github with Portkey, get your API key [from here](https://github.com/settings/tokens), then add it to Portkey to create the virtual key. - - - ```js - import Portkey from 'portkey-ai' +// 1. Install: npm install portkey-ai +// 2. Add @github provider in model catalog +// 3. Use it: - const portkey = new Portkey({ - apiKey: "PORTKEY_API_KEY", // defaults to process.env["PORTKEY_API_KEY"] - provider:"@GITHUB_PROVIDER" // Your Github Models virtual key - }) - ``` - - - ```py - from portkey_ai import Portkey +const portkey = new Portkey({ + apiKey: "PORTKEY_API_KEY" +}) - portkey = Portkey( - api_key ="PORTKEY_API_KEY", # Replace with your Portkey API key - provider="@GITHUB_PROVIDER" # Your Github Models virtual key - ) - ``` - +const response = await portkey.chat.completions.create({ + model: "@github/Phi-3-small-128k-instruct", + messages: [{ role: "user", content: "Hello!" }] +}) - +console.log(response.choices[0].message.content) +``` +```python OpenAI Py icon="python" +from openai import OpenAI +from portkey_ai import PORTKEY_GATEWAY_URL +# 1. Install: pip install openai portkey-ai +# 2. Add @github provider in model catalog +# 3. Use it: +client = OpenAI( + api_key="PORTKEY_API_KEY", # Portkey API key + base_url=PORTKEY_GATEWAY_URL +) -### 3\. Invoke Chat Completions - - - ```js - const chatCompletion = await portkey.chat.completions.create({ - messages: [{ role: 'user', content: 'Say this is a test' }], - model: 'Phi-3-small-128k-instruct', - }); +response = client.chat.completions.create( + model="@github/Phi-3-small-128k-instruct", + messages=[{"role": "user", "content": "Hello!"}] +) - console.log(chatCompletion.choices); - ``` - - - ```python - completion = portkey.chat.completions.create( - messages= [{ "role": 'user', "content": 'Say this is a test' }], - model= 'Phi-3-small-128k-instruct' - ) +print(response.choices[0].message.content) +``` - print(completion) - ``` - +```js OpenAI JS icon="square-js" +import OpenAI from "openai" +import { PORTKEY_GATEWAY_URL } from "portkey-ai" - +// 1. Install: npm install openai portkey-ai +// 2. Add @github provider in model catalog +// 3. Use it: ---- +const client = new OpenAI({ + apiKey: "PORTKEY_API_KEY", // Portkey API key + baseURL: PORTKEY_GATEWAY_URL +}) -## Supported Models +const response = await client.chat.completions.create({ + model: "@github/Phi-3-small-128k-instruct", + messages: [{ role: "user", content: "Hello!" }] +}) -Portkey supports *all* the models (both `Chat/completion` and `Embeddings` capabilities) on the Github Models marketplace. +console.log(response.choices[0].message.content) +``` -## Next Steps +```sh cURL icon="square-terminal" +# 1. Add @github provider in model catalog +# 2. Use it: + +curl https://api.portkey.ai/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "x-portkey-api-key: $PORTKEY_API_KEY" \ + -d '{ + "model": "@github/Phi-3-small-128k-instruct", + "messages": [{"role": "user", "content": "Hello!"}] + }' +``` + + + +## Add Provider in Model Catalog -The complete list of features supported in the SDK are available on the link below. +Before making requests, add GitHub Models to your Model Catalog: - +1. Go to [**Model Catalog → Add Provider**](https://app.portkey.ai/model-catalog/providers) +2. Select **GitHub** +3. Enter your [GitHub personal access token](https://github.com/settings/tokens) +4. Name your provider (e.g., `github`) + + + See all setup options and detailed configuration instructions -You'll find more information in the relevant sections: +--- -1. [Add metadata to your requests](/product/observability/metadata) -2. [Add gateway configs to your requests](/product/ai-gateway/configs)[ requests](/product/ai-gateway/configs) -3. [Tracing Github requests](/product/observability/traces) -4. [Setup a fallback from OpenAI to Github](/product/ai-gateway/fallbacks) +## Next Steps + + + + Add fallbacks, load balancing, and more + + + Monitor and trace your GitHub Models requests + + + Manage and version your prompts + + + Add custom metadata to requests + + + +For complete SDK documentation: + + + Complete Portkey SDK documentation + diff --git a/integrations/llms/google-palm.mdx b/integrations/llms/google-palm.mdx index c225d153..295fafde 100644 --- a/integrations/llms/google-palm.mdx +++ b/integrations/llms/google-palm.mdx @@ -38,10 +38,10 @@ Set up Portkey with your virtual key as part of the initialization configuration ```js - import Portkey from 'portkey-ai' +import Portkey from 'portkey-ai' - const portkey = new Portkey({ +const portkey = new Portkey({ apiKey: "PORTKEY_API_KEY", // defaults to process.env["PORTKEY_API_KEY"] @@ -75,7 +75,7 @@ Use the Portkey instance to send requests to Google Palm. You can also override ```js - const chatCompletion = await portkey.chat.completions.create({ +const chatCompletion = await portkey.chat.completions.create({ messages: [{ role: 'user', content: 'Say this is a test' }], diff --git a/integrations/llms/groq.mdx b/integrations/llms/groq.mdx index a6728562..1fda554d 100644 --- a/integrations/llms/groq.mdx +++ b/integrations/llms/groq.mdx @@ -1,153 +1,141 @@ --- title: "Groq" +description: Use Groq's ultra-fast inference for chat completions, tool calling, and audio processing through Portkey. --- -Portkey provides a robust and secure gateway to facilitate the integration of various Large Language Models (LLMs) into your applications, including [Groq APIs](https://console.groq.com/docs/quickstart). +## Quick Start -With Portkey, you can take advantage of features like fast AI gateway access, observability, prompt management, and more, all while ensuring the secure management of your LLM API keys through a [virtual key](/product/ai-gateway/virtual-keys) system. - -Provider Slug. `groq` - -## Portkey SDK Integration with Groq Models - -Portkey provides a consistent API to interact with models from various providers. To integrate Groq with Portkey: - -### 1\. Install the Portkey SDK - -Add the Portkey SDK to your application to interact with Groq AI's API through Portkey's gateway. - - - - ```sh - npm install --save portkey-ai - ``` - - - ```sh - pip install portkey-ai - ``` - - - +Get started with Groq in under 2 minutes: + +```python Python icon="python" +from portkey_ai import Portkey +# 1. Install: pip install portkey-ai +# 2. Add @groq provider in model catalog +# 3. Use it: -### 2\. Initialize Portkey with the Virtual Key +portkey = Portkey(api_key="PORTKEY_API_KEY") -To use Groq with Portkey, [get your API key from here](https://console.groq.com/keys), then add it to Portkey to create the virtual key. +response = portkey.chat.completions.create( + model="@groq/llama-3.3-70b-versatile", + messages=[{"role": "user", "content": "Hello!"}] +) - - +print(response.choices[0].message.content) +``` -```js +```js Javascript icon="square-js" import Portkey from 'portkey-ai' +// 1. Install: npm install portkey-ai +// 2. Add @groq provider in model catalog +// 3. Use it: + const portkey = new Portkey({ - apiKey: "PORTKEY_API_KEY", // defaults to process.env["PORTKEY_API_KEY"] - provider:"@PROVIDER" // Your Groq Virtual Key + apiKey: "PORTKEY_API_KEY" }) -``` - - - ```python - from portkey_ai import Portkey - portkey = Portkey( - api_key="PORTKEY_API_KEY", # Replace with your Portkey API key - provider="@PROVIDER" # Replace with your virtual key for Groq - ) - ``` +const response = await portkey.chat.completions.create({ + model: "@groq/llama-3.3-70b-versatile", + messages: [{ role: "user", content: "Hello!" }] +}) - +console.log(response.choices[0].message.content) +``` - +```python OpenAI Py icon="python" +from openai import OpenAI +from portkey_ai import PORTKEY_GATEWAY_URL +# 1. Install: pip install openai portkey-ai +# 2. Add @groq provider in model catalog +# 3. Use it: +client = OpenAI( + api_key="PORTKEY_API_KEY", # Portkey API key + base_url=PORTKEY_GATEWAY_URL +) -### **3\. Invoke Chat Completions with** Groq +response = client.chat.completions.create( + model="@groq/llama-3.3-70b-versatile", + messages=[{"role": "user", "content": "Hello!"}] +) -Use the Portkey instance to send requests to Groq. You can also override the virtual key directly in the API call if needed. +print(response.choices[0].message.content) +``` - - +```js OpenAI JS icon="square-js" +import OpenAI from "openai" +import { PORTKEY_GATEWAY_URL } from "portkey-ai" - ```js - const chatCompletion = await portkey.chat.completions.create({ - messages: [{ role: 'user', content: 'Say this is a test' }], - model: 'mixtral-8x7b-32768', - }); +// 1. Install: npm install openai portkey-ai +// 2. Add @groq provider in model catalog +// 3. Use it: - console.log(chatCompletion.choices); - ``` - - +const client = new OpenAI({ + apiKey: "PORTKEY_API_KEY", // Portkey API key + baseURL: PORTKEY_GATEWAY_URL +}) -```python -completion = portkey.chat.completions.create( - messages= [{ "role": 'user', "content": 'Say this is a test' }], - model= 'mistral-medium' -) +const response = await client.chat.completions.create({ + model: "@groq/llama-3.3-70b-versatile", + messages: [{ role: "user", content: "Hello!" }] +}) -print(completion) +console.log(response.choices[0].message.content) ``` - +```sh cURL icon="square-terminal" +# 1. Add @groq provider in model catalog +# 2. Use it: + +curl https://api.portkey.ai/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "x-portkey-api-key: $PORTKEY_API_KEY" \ + -d '{ + "model": "@groq/llama-3.3-70b-versatile", + "messages": [{"role": "user", "content": "Hello!"}] + }' +``` - + +## Add Provider in Model Catalog -## Managing Groq Prompts +Before making requests, add Groq to your Model Catalog: -You can manage all prompts to Groq in the [Prompt Library](/product/prompt-library). All the current models of Groq are supported and you can easily start testing different prompts. +1. Go to [**Model Catalog → Add Provider**](https://app.portkey.ai/model-catalog/providers) +2. Select **Groq** +3. Enter your [Groq API key](https://console.groq.com/keys) +4. Name your provider (e.g., `groq`) -Once you're ready with your prompt, you can use the `portkey.prompts.completions.create` interface to use the prompt in your application. + + See all setup options and detailed configuration instructions + +--- +## Groq Capabilities -### Groq Tool Calling -Tool calling feature lets models trigger external tools based on conversation context. You define available functions, the model chooses when to use them, and your application executes them and returns results. +### Tool Calling -Portkey supports Groq Tool Calling and makes it interoperable across multiple providers. With Portkey Prompts, you can templatize various your prompts & tool schemas as well. +Use Groq's tool calling feature to trigger external functions: - + + + View Groq models that support tool calling + + - + - - -```javascript Get Weather Tool -let tools = [{ - type: "function", - function: { - name: "getWeather", - description: "Get the current weather", - parameters: { - type: "object", - properties: { - location: { type: "string", description: "City and state" }, - unit: { type: "string", enum: ["celsius", "fahrenheit"] } - }, - required: ["location"] - } - } -}]; +```python Python +from portkey_ai import Portkey -let response = await portkey.chat.completions.create({ - model: "llama-3.3-70b-versatile", - messages: [ - { role: "system", content: "You are a helpful assistant." }, - { role: "user", content: "What's the weather like in Delhi - respond in JSON" } - ], - tools, - tool_choice: "auto", -}); +portkey = Portkey(api_key="PORTKEY_API_KEY") -console.log(response.choices[0].finish_reason); -``` - - -```python Get Weather Tool tools = [{ "type": "function", "function": { @@ -165,10 +153,10 @@ tools = [{ }] response = portkey.chat.completions.create( - model="llama-3.3-70b-versatile", + model="@groq/llama-3.3-70b-versatile", messages=[ {"role": "system", "content": "You are a helpful assistant."}, - {"role": "user", "content": "What's the weather like in Delhi - respond in JSON"} + {"role": "user", "content": "What's the weather like in Delhi?"} ], tools=tools, tool_choice="auto" @@ -176,17 +164,53 @@ response = portkey.chat.completions.create( print(response.choices[0].finish_reason) ``` - - -```curl Get Weather Tool -curl -X POST "https://api.portkey.ai/v1/chat/completions" \ + +```javascript Node.js +import Portkey from 'portkey-ai'; + +const portkey = new Portkey({ + apiKey: 'PORTKEY_API_KEY' +}); + +const tools = [{ + type: "function", + function: { + name: "getWeather", + description: "Get the current weather", + parameters: { + type: "object", + properties: { + location: { type: "string", description: "City and state" }, + unit: { type: "string", enum: ["celsius", "fahrenheit"] } + }, + required: ["location"] + } + } +}]; + +const response = await portkey.chat.completions.create({ + model: "@groq/llama-3.3-70b-versatile", + messages: [ + { role: "system", content: "You are a helpful assistant." }, + { role: "user", content: "What's the weather like in Delhi?" } + ], + tools, + tool_choice: "auto" +}); + +console.log(response.choices[0].finish_reason); +``` + +```bash cURL +curl https://api.portkey.ai/v1/chat/completions \ -H "Content-Type: application/json" \ - -H "Authorization: Bearer YOUR_PORTKEY_API_KEY" \ + -H "x-portkey-api-key: $PORTKEY_API_KEY" \ + -H "x-portkey-provider: @groq" \ -d '{ "model": "llama-3.3-70b-versatile", "messages": [ {"role": "system", "content": "You are a helpful assistant."}, - {"role": "user", "content": "What'\''s the weather like in Delhi - respond in JSON"} + {"role": "user", "content": "What'\''s the weather like in Delhi?"} ], "tools": [{ "type": "function", @@ -206,25 +230,21 @@ curl -X POST "https://api.portkey.ai/v1/chat/completions" \ "tool_choice": "auto" }' ``` - - - - - - - + +### Speech to Text (Whisper) +Transcribe or translate audio using Groq's Whisper model: + -### Groq Speech to Text (Whisper) +```python Python +from portkey_ai import Portkey -OpenAI's Audio API converts speech to text using the Whisper model. It offers transcription in the original language and translation to English, supporting multiple file formats and languages with high accuracy. +portkey = Portkey(api_key="PORTKEY_API_KEY", provider="@groq") - -```python Python -audio_file= open("/path/to/file.mp3", "rb") +audio_file = open("/path/to/file.mp3", "rb") # Transcription transcription = portkey.audio.transcriptions.create( @@ -242,13 +262,19 @@ print(translation.text) ``` ```javascript Node.js -import fs from "fs"; +import Portkey from 'portkey-ai'; +import fs from 'fs'; + +const portkey = new Portkey({ + apiKey: 'PORTKEY_API_KEY', + provider: '@groq' +}); // Transcription async function transcribe() { const transcription = await portkey.audio.transcriptions.create({ file: fs.createReadStream("/path/to/file.mp3"), - model: "whisper-large-v3", + model: "whisper-large-v3" }); console.log(transcription.text); } @@ -258,41 +284,45 @@ transcribe(); async function translate() { const translation = await portkey.audio.translations.create({ file: fs.createReadStream("/path/to/file.mp3"), - model: "whisper-large-v3", + model: "whisper-large-v3" }); console.log(translation.text); } translate(); ``` -```curl REST +```bash cURL # Transcription -curl -X POST "https://api.portkey.ai/v1/audio/transcriptions" \ - -H "Authorization: Bearer YOUR_PORTKEY_API_KEY" \ +curl https://api.portkey.ai/v1/audio/transcriptions \ + -H "x-portkey-api-key: $PORTKEY_API_KEY" \ + -H "x-portkey-provider: @groq" \ -H "Content-Type: multipart/form-data" \ -F "file=@/path/to/file.mp3" \ -F "model=whisper-large-v3" # Translation -curl -X POST "https://api.portkey.ai/v1/audio/translations" \ - -H "Authorization: Bearer YOUR_PORTKEY_API_KEY" \ +curl https://api.portkey.ai/v1/audio/translations \ + -H "x-portkey-api-key: $PORTKEY_API_KEY" \ + -H "x-portkey-provider: @groq" \ -H "Content-Type: multipart/form-data" \ -F "file=@/path/to/file.mp3" \ -F "model=whisper-large-v3" ``` - - ---- + -### Groq Text to Speech +### Text to Speech -Groq's Text to Speech (TTS) API converts written text into natural-sounding audio using six distinct voices. It supports multiple languages, streaming capabilities, and various audio formats for different use cases. +Convert text to natural-sounding audio: + ```python Python +from portkey_ai import Portkey from pathlib import Path +portkey = Portkey(api_key="PORTKEY_API_KEY", provider="@groq") + speech_file_path = Path(__file__).parent / "speech.mp3" response = portkey.audio.speech.create( model="playai-tts", @@ -305,19 +335,22 @@ with open(speech_file_path, "wb") as f: ``` ```javascript Node.js +import Portkey from 'portkey-ai'; import path from 'path'; import fs from 'fs'; +const portkey = new Portkey({ + apiKey: 'PORTKEY_API_KEY', + provider: '@groq' +}); + const speechFile = path.resolve("./speech.mp3"); async function main() { - const mp3 = await portkey.audio.speech.createCertainly! I'll continue with the Text to Speech section and then move on to the additional features and sections: - -```javascript Node.js -({ + const mp3 = await portkey.audio.speech.create({ model: "playai-tts", voice: "Fritz-PlayAI", - input: "Today is a wonderful day to build something people love!", + input: "Today is a wonderful day to build something people love!" }); const buffer = Buffer.from(await mp3.arrayBuffer()); await fs.promises.writeFile(speechFile, buffer); @@ -326,9 +359,10 @@ async function main() { main(); ``` -```curl REST -curl -X POST "https://api.portkey.ai/v1/audio/speech" \ - -H "Authorization: Bearer YOUR_PORTKEY_API_KEY" \ +```bash cURL +curl https://api.portkey.ai/v1/audio/speech \ + -H "x-portkey-api-key: $PORTKEY_API_KEY" \ + -H "x-portkey-provider: @groq" \ -H "Content-Type: application/json" \ -d '{ "model": "playai-tts", @@ -337,13 +371,46 @@ curl -X POST "https://api.portkey.ai/v1/audio/speech" \ }' \ --output speech.mp3 ``` + --- -You'll find more information in the relevant sections: +## Supported Models -1. [Add metadata to your requests](/product/observability/metadata) -2. [Add gateway configs to your Groq](/product/ai-gateway/configs)[ requests](/product/ai-gateway/configs) -3. [Tracing Groq requests](/product/observability/traces) -4. [Setup a fallback from OpenAI to Groq APIs](/product/ai-gateway/fallbacks) +Groq provides ultra-fast inference for various open-source models: + +| Model Family | Models Available | +|--------------|-----------------| +| Llama | llama-3.3-70b-versatile, llama-3.1-70b-versatile, llama-3.1-8b-instant | +| Mixtral | mixtral-8x7b-32768 | +| Gemma | gemma-7b-it, gemma2-9b-it | +| Whisper | whisper-large-v3, whisper-large-v3-turbo | +| TTS | playai-tts | + +Check [Groq's documentation](https://console.groq.com/docs/models) for the complete list of available models. + +--- + +## Next Steps + + + + Add fallbacks, load balancing, and more + + + Monitor and trace your Groq requests + + + Manage and version your prompts + + + Add custom metadata to requests + + + +For complete SDK documentation: + + + Complete Portkey SDK documentation + diff --git a/integrations/llms/huggingface.mdx b/integrations/llms/huggingface.mdx index d9a5be92..7f9a69a8 100644 --- a/integrations/llms/huggingface.mdx +++ b/integrations/llms/huggingface.mdx @@ -1,241 +1,161 @@ --- title: "Hugging Face" +description: "Use Hugging Face Inference endpoints through Portkey for thousands of open-source models." --- -Portkey provides a robust and secure gateway to facilitate the integration of various Large Language Models (LLMs) into your applications, including all the text generation models supported by [Huggingface's Inference endpoints](https://huggingface.co/docs/api-inference/index). +## Quick Start -With Portkey, you can take advantage of features like fast AI gateway access, observability, prompt management, and more, all while ensuring the secure management of your LLM API keys through a [virtual key](/product/ai-gateway/virtual-keys) system. - - -Provider Slug. `huggingface` - -## Portkey SDK Integration with Huggingface - -Portkey provides a consistent API to interact with models from various providers. To integrate Huggingface with Portkey: - -### 1\. Install the Portkey SDK - -Add the Portkey SDK to your application to interact with Huggingface's API through Portkey's gateway. - - - - ```sh - npm install --save portkey-ai - ``` - - - ```sh - pip install portkey-ai - ``` - - - +Get started with Hugging Face in under 2 minutes: + +```python Python icon="python" +from portkey_ai import Portkey +# 1. Install: pip install portkey-ai +# 2. Add @huggingface provider in model catalog +# 3. Use it: -### 2\. Initialize Portkey with the Virtual Key +portkey = Portkey(api_key="PORTKEY_API_KEY") -To use Huggingface with Portkey, [get your Huggingface Access token from here](https://huggingface.co/settings/tokens), then add it to Portkey to create the virtual key. +response = portkey.chat.completions.create( + model="@huggingface/meta-llama/Llama-3.2-3B-Instruct", + messages=[{"role": "user", "content": "Hello!"}] +) - - +print(response.choices[0].message.content) +``` -```js +```js Javascript icon="square-js" import Portkey from 'portkey-ai' +// 1. Install: npm install portkey-ai +// 2. Add @huggingface provider in model catalog +// 3. Use it: + const portkey = new Portkey({ - apiKey: "PORTKEY_API_KEY", // defaults to process.env["PORTKEY_API_KEY"] - provider:"@PROVIDER", // Your Huggingface Virtual Key - huggingfaceBaseUrl: "HUGGINGFACE_DEDICATED_URL" // Optional: Use this if you have a dedicated server hosted on Huggingface + apiKey: "PORTKEY_API_KEY" }) -``` - - -```py -from portkey_ai import Portkey +const response = await portkey.chat.completions.create({ + model: "@huggingface/meta-llama/Llama-3.2-3B-Instruct", + messages: [{ role: "user", content: "Hello!" }] +}) -portkey = Portkey( - api_key="PORTKEY_API_KEY", # Replace with your Portkey API key - provider="@PROVIDER", # Replace with your virtual key for Huggingface - huggingface_base_url="HUGGINGFACE_DEDICATED_URL" # Optional: Use this if you have a dedicated server hosted on Huggingface -) +console.log(response.choices[0].message.content) ``` - - -```py +```python OpenAI Py icon="python" from openai import OpenAI -from portkey_ai import PORTKEY_GATEWAY_URL, createHeaders +from portkey_ai import PORTKEY_GATEWAY_URL + +# 1. Install: pip install openai portkey-ai +# 2. Add @huggingface provider in model catalog +# 3. Use it: client = OpenAI( - api_key="HUGGINGFACE_ACCESS_TOKEN", - base_url=PORTKEY_GATEWAY_URL, - default_headers=createHeaders( - api_key="PORTKEY_API_KEY", - provider="huggingface", - huggingface_base_url="HUGGINGFACE_DEDICATED_URL" - ) + api_key="PORTKEY_API_KEY", # Portkey API key + base_url=PORTKEY_GATEWAY_URL ) -``` - - - - - ```js - import OpenAI from "openai"; - import { PORTKEY_GATEWAY_URL, createHeaders } from "portkey-ai"; - - const client = new OpenAI({ - apiKey: "HUGGINGFACE_ACCESS_TOKEN", - baseURL: PORTKEY_GATEWAY_URL, - defaultHeaders: createHeaders({ - provider: "huggingface", - apiKey: "PORTKEY_API_KEY", - huggingfaceBaseUrl: "HUGGINGFACE_DEDICATED_URL" - }), - }); - ``` - - - - -### **3\. Invoke Chat Completions with** Huggingface - -Use the Portkey instance to send requests to Huggingface. You can also override the virtual key directly in the API call if needed. - - - - -```js -const chatCompletion = await portkey.chat.completions.create({ - messages: [{ role: 'user', content: 'Say this is a test' }], - model: 'meta-llama/Meta-Llama-3.1-8B-Instruct', // make sure your model is hot -}); - -console.log(chatCompletion.choices[0].message.content); -``` - - -```py -chat_completion = portkey.chat.completions.create( - messages= [{ "role": 'user', "content": 'Say this is a test' }], - model= 'meta-llama/meta-llama-3.1-8b-instruct', # make sure your model is hot +response = client.chat.completions.create( + model="@huggingface/meta-llama/Llama-3.2-3B-Instruct", + messages=[{"role": "user", "content": "Hello!"}] ) -print(chat_completion.choices[0].message.content) +print(response.choices[0].message.content) ``` - - -```py -chat_completion = client.chat.completions.create( - messages = [{ "role": 'user', "content": 'Say this is a test' }], - model = 'meta-llama/meta-llama-3.1-8b-instruct', # make sure your model is hot -) +```js OpenAI JS icon="square-js" +import OpenAI from "openai" +import { PORTKEY_GATEWAY_URL } from "portkey-ai" -print(chat_completion.choices[0].message.content) -``` - +// 1. Install: npm install openai portkey-ai +// 2. Add @huggingface provider in model catalog +// 3. Use it: - +const client = new OpenAI({ + apiKey: "PORTKEY_API_KEY", // Portkey API key + baseURL: PORTKEY_GATEWAY_URL +}) -```js -async function main() { - const chatCompletion = await client.chat.completions.create({ - model: "meta-llama/meta-llama-3.1-8b-instruct", // make sure your model is hot - messages: [{ role: "user", content: "How many points to Gryffindor?" }], - }); - console.log(chatCompletion.choices[0].message.content); -} +const response = await client.chat.completions.create({ + model: "@huggingface/meta-llama/Llama-3.2-3B-Instruct", + messages: [{ role: "user", content: "Hello!" }] +}) -main(); +console.log(response.choices[0].message.content) ``` - - -```py -chat_completion = client.chat.completions.create( - messages = [{ "role": 'user', "content": 'Say this is a test' }], - model = 'meta-llama/meta-llama-3.1-8b-instruct', # make sure your model is hot -) - -print(chat_completion.choices[0].message.content) +```sh cURL icon="square-terminal" +# 1. Add @huggingface provider in model catalog +# 2. Use it: + +curl https://api.portkey.ai/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "x-portkey-api-key: $PORTKEY_API_KEY" \ + -d '{ + "model": "@huggingface/meta-llama/Llama-3.2-3B-Instruct", + "messages": [{"role": "user", "content": "Hello!"}] + }' ``` - - - - -## [Using Virtual Keys](https://app.portkey.ai/virtual-keys) - -Virtual Keys serve as Portkey's unified authentication system for all LLM interactions, simplifying the use of multiple providers and Portkey features within your application. For self-hosted LLMs, you can configure custom authentication requirements including authorization keys, bearer tokens, or any other headers needed to access your model: - - - + -1. Navigate to [Virtual Keys](https://app.portkey.ai/virtual-keys) in your Portkey dashboard -2. Click **"Add Key"** and enable the **"Local/Privately hosted provider"** toggle -3. Configure your deployment: - - Select the matching provider API specification (typically `OpenAI`) - - Enter your model's base URL in the `Custom Host` field - - Add required authentication headers and their values -4. Click **"Create"** to generate your virtual key +## Add Provider in Model Catalog -You can now use this virtual key in your requests: +Before making requests, add Hugging Face to your Model Catalog: - - - ```js - const portkey = new Portkey({ - apiKey: "PORTKEY_API_KEY", - provider:"@YOUR_SELF_HOSTED_LLM_PROVIDER" +1. Go to [**Model Catalog → Add Provider**](https://app.portkey.ai/model-catalog/providers) +2. Select **Hugging Face** +3. Enter your [Hugging Face access token](https://huggingface.co/settings/tokens) +4. (Optional) Add a **Custom Host** if using a dedicated Hugging Face Inference Endpoint +5. Name your provider (e.g., `huggingface`) - async function main() { - const response = await client.chat.completions.create({ - messages: [{ role: "user", content: "Bob the builder.." }], - model: "your-self-hosted-model-name", - }); + +If you have a dedicated server hosted on Hugging Face, enter your dedicated endpoint URL in the **Custom Host** field during provider setup. This allows you to route requests to your private Hugging Face deployment. + - console.log(response.choices[0].message.content); - }) - ``` - - - ```python - portkey = Portkey( - api_key="PORTKEY_API_KEY", - provider="@YOUR_SELF_HOSTED_LLM_PROVIDER" - ) + + See all setup options and detailed configuration instructions + - response = portkey.chat.completions.create( - model="your-self-hosted-model-name", - messages=[ - {"role": "system", "content": "You are a helpful assistant."}, - {"role": "user", "content": "Hello!"} - ] +--- - print(response) - ) - ``` - - +## Supported Models -For more information about managing self-hosted LLMs with Portkey, see [Bring Your Own LLM](/integrations/llms/byollm). +Hugging Face provides access to thousands of text generation models through their Inference endpoints, including: +- Meta Llama 3.2, Llama 3.1, Llama 3 +- Mistral, Mixtral +- Qwen 2.5 +- Phi-3 +- Gemma, Gemma 2 +- And thousands more! +Browse the complete catalog at [Hugging Face Models](https://huggingface.co/models?pipeline_tag=text-generation). -# Next Steps +--- -The complete list of features supported in the SDK are available on the link below. - +## Next Steps + + + + Add fallbacks, load balancing, and more + + + Monitor and trace your Hugging Face requests + + + Manage and version your prompts + + + Add custom metadata to requests + + + +For complete SDK documentation: + + + Complete Portkey SDK documentation - -You'll find more information in the relevant sections: - -1. [Add metadata to your requests](/product/observability/metadata) -2. [Add gateway configs to your Huggingface requests](/product/ai-gateway/configs)[ requests](/product/ai-gateway/configs) -3. [Tracing Huggingface requests](/product/observability/traces) -4. [Setup a fallback from OpenAI to Huggingface APIs](/product/ai-gateway/fallbacks) diff --git a/integrations/llms/inference.net.mdx b/integrations/llms/inference.net.mdx index cfc54f83..d7c320a6 100644 --- a/integrations/llms/inference.net.mdx +++ b/integrations/llms/inference.net.mdx @@ -1,101 +1,153 @@ --- title: "Inference.net" -description: "Portkey provides a robust and secure gateway to facilitate the integration of various Large Language Models (LLMs) into your applications, including the models hosted on [Inference.net](https://www.inference.net/). -" +description: Use Inference.net's distributed GPU compute platform through Portkey. --- +## Quick Start - -Provider slug: `inference-net` - - -## Portkey SDK Integration with Inference.net - -Portkey provides a consistent API to interact with models from various providers. To integrate Inference.net with Portkey: - -### 1. Install the Portkey SDK - - - - ```sh - npm install --save portkey-ai - ``` - - - ```sh - pip install portkey-ai - ``` - - - -### 2. Initialize Portkey with Inference.net Authorization - -* Set `provider` name as `inference-net` -* Pass your API key with `Authorization` header - - - - ```javascript - import Portkey from 'portkey-ai' - - const portkey = new Portkey({ - apiKey: "PORTKEY_API_KEY", // defaults to process.env["PORTKEY_API_KEY"] - provider: "inference-net", - Authorization: "Bearer INFERENCE-NET API KEY" - }) - ``` - - - ```python - from portkey_ai import Portkey - portkey = Portkey( - api_key="PORTKEY_API_KEY", # Replace with your Portkey API key - provider="inference-net", - Authorization="Bearer INFERENCE-NET API KEY" - ) - ``` - - - -### 3. Invoke Chat Completions - - - - ```javascript - const chatCompletion = await portkey.chat.completions.create({ - messages: [{ role: 'user', content: 'Say this is a test' }], - model: 'llama3', - }); - console.log(chatCompletion.choices); - ``` - - - ```python - completion = portkey.chat.completions.create( - messages= [{ "role": 'user', "content": 'Say this is a test' }], - model= 'llama3' - ) - print(completion) - ``` - - +Get started with Inference.net in under 2 minutes: -## Supported Models + -Find more info about models supported by Inference.net here: +```python Python icon="python" +from portkey_ai import Portkey -[Inference.net](https://www.inference.net/) +# 1. Install: pip install portkey-ai +# 2. Add @inference-net provider in model catalog +# 3. Use it: -## Next Steps +portkey = Portkey(api_key="PORTKEY_API_KEY") + +response = portkey.chat.completions.create( + model="@inference-net/llama3", + messages=[{"role": "user", "content": "Hello!"}] +) + +print(response.choices[0].message.content) +``` + +```js Javascript icon="square-js" +import Portkey from 'portkey-ai' + +// 1. Install: npm install portkey-ai +// 2. Add @inference-net provider in model catalog +// 3. Use it: + +const portkey = new Portkey({ + apiKey: "PORTKEY_API_KEY" +}) + +const response = await portkey.chat.completions.create({ + model: "@inference-net/llama3", + messages: [{ role: "user", content: "Hello!" }] +}) + +console.log(response.choices[0].message.content) +``` + +```python OpenAI Py icon="python" +from openai import OpenAI +from portkey_ai import PORTKEY_GATEWAY_URL + +# 1. Install: pip install openai portkey-ai +# 2. Add @inference-net provider in model catalog +# 3. Use it: + +client = OpenAI( + api_key="PORTKEY_API_KEY", # Portkey API key + base_url=PORTKEY_GATEWAY_URL +) + +response = client.chat.completions.create( + model="@inference-net/llama3", + messages=[{"role": "user", "content": "Hello!"}] +) + +print(response.choices[0].message.content) +``` + +```js OpenAI JS icon="square-js" +import OpenAI from "openai" +import { PORTKEY_GATEWAY_URL } from "portkey-ai" + +// 1. Install: npm install openai portkey-ai +// 2. Add @inference-net provider in model catalog +// 3. Use it: + +const client = new OpenAI({ + apiKey: "PORTKEY_API_KEY", // Portkey API key + baseURL: PORTKEY_GATEWAY_URL +}) + +const response = await client.chat.completions.create({ + model: "@inference-net/llama3", + messages: [{ role: "user", content: "Hello!" }] +}) -The complete list of features supported in the SDK are available on the link below. +console.log(response.choices[0].message.content) +``` - +```sh cURL icon="square-terminal" +# 1. Add @inference-net provider in model catalog +# 2. Use it: + +curl https://api.portkey.ai/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "x-portkey-api-key: $PORTKEY_API_KEY" \ + -d '{ + "model": "@inference-net/llama3", + "messages": [{"role": "user", "content": "Hello!"}] + }' +``` + + + +## Add Provider in Model Catalog + +Before making requests, add Inference.net to your Model Catalog: + +1. Go to [**Model Catalog → Add Provider**](https://app.portkey.ai/model-catalog/providers) +2. Select **Inference.net** +3. Enter your Inference.net API key +4. Name your provider (e.g., `inference-net`) + + + See all setup options and detailed configuration instructions -You'll find more information in the relevant sections: +--- + +## Supported Models -1. [Add metadata to your requests](/product/observability/metadata) -2. [Add gateway configs to your Inference.net requests](/product/ai-gateway/configs) -3. [Tracing Inference.net requests](/product/observability/traces) -4. [Setup a fallback from OpenAI to Inference.net](/product/ai-gateway/fallbacks) +Inference.net provides distributed GPU compute for various open-source models including: + +- Llama 3 +- Mistral +- And other popular open-source models + +Check [Inference.net's documentation](https://www.inference.net/) for the complete model list. + +--- + +## Next Steps + + + + Add fallbacks, load balancing, and more + + + Monitor and trace your Inference.net requests + + + Manage and version your prompts + + + Add custom metadata to requests + + + +For complete SDK documentation: + + + Complete Portkey SDK documentation + diff --git a/integrations/llms/jina-ai.mdx b/integrations/llms/jina-ai.mdx index df755dc4..0a8d0f45 100644 --- a/integrations/llms/jina-ai.mdx +++ b/integrations/llms/jina-ai.mdx @@ -1,133 +1,263 @@ --- title: "Jina AI" +description: Use Jina AI's embedding and reranker models through Portkey. --- -Portkey provides a robust and secure gateway to facilitate the integration of various models into your applications, including [Jina AI embedding & reranker models](https://jina.ai/). +## Quick Start -With Portkey, you can take advantage of features like fast AI gateway access, observability, and more, all while ensuring the secure management of your API keys through a [virtual key](/product/ai-gateway/virtual-keys) system. - -Provider Slug. `jina` - -## Portkey SDK Integration with Jina AI Models +Get started with Jina AI in under 2 minutes: -Portkey provides a consistent API to interact with models from various providers. To integrate Jina AI with Portkey: + -### 1\. Install the Portkey SDK +```python Python icon="python" +from portkey_ai import Portkey -Add the Portkey SDK to your application to interact with Jina AI's API through Portkey's gateway. +# 1. Install: pip install portkey-ai +# 2. Add @jina provider in model catalog +# 3. Use it: - - +portkey = Portkey(api_key="PORTKEY_API_KEY") - ```sh - npm install --save portkey-ai - ``` - - +response = portkey.chat.completions.create( + model="@jina/jina-embeddings-v2-base-en", + messages=[{"role": "user", "content": "Hello!"}] +) -```sh -pip install portkey-ai +print(response.choices[0].message.content) ``` - - +```js Javascript icon="square-js" +import Portkey from 'portkey-ai' +// 1. Install: npm install portkey-ai +// 2. Add @jina provider in model catalog +// 3. Use it: +const portkey = new Portkey({ + apiKey: "PORTKEY_API_KEY" +}) -### 2\. Initialize Portkey with the Virtual Key +const response = await portkey.chat.completions.create({ + model: "@jina/jina-embeddings-v2-base-en", + messages: [{ role: "user", content: "Hello!" }] +}) -To use JinaAI with Portkey, [get your API key from here](https://jina.ai/), then add it to Portkey to create the virtual key. - - - ```js - import Portkey from 'portkey-ai' +console.log(response.choices[0].message.content) +``` - const portkey = new Portkey({ - apiKey: "PORTKEY_API_KEY", // defaults to process.env["PORTKEY_API_KEY"] - provider:"@JINA_AI_PROVIDER" // Your Jina AI Virtual Key - }) - ``` - - -```python -from portkey_ai import Portkey +```python OpenAI Py icon="python" +from openai import OpenAI +from portkey_ai import PORTKEY_GATEWAY_URL + +# 1. Install: pip install openai portkey-ai +# 2. Add @jina provider in model catalog +# 3. Use it: + +client = OpenAI( + api_key="PORTKEY_API_KEY", # Portkey API key + base_url=PORTKEY_GATEWAY_URL +) -portkey = Portkey( - api_key="PORTKEY_API_KEY", # Replace with your Portkey API key - provider="@JINA_AI_PROVIDER" # Replace with your virtual key for Jina AI +response = client.chat.completions.create( + model="@jina/jina-embeddings-v2-base-en", + messages=[{"role": "user", "content": "Hello!"}] ) + +print(response.choices[0].message.content) +``` + +```js OpenAI JS icon="square-js" +import OpenAI from "openai" +import { PORTKEY_GATEWAY_URL } from "portkey-ai" + +// 1. Install: npm install openai portkey-ai +// 2. Add @jina provider in model catalog +// 3. Use it: + +const client = new OpenAI({ + apiKey: "PORTKEY_API_KEY", // Portkey API key + baseURL: PORTKEY_GATEWAY_URL +}) + +const response = await client.chat.completions.create({ + model: "@jina/jina-embeddings-v2-base-en", + messages: [{ role: "user", content: "Hello!" }] +}) + +console.log(response.choices[0].message.content) +``` + +```bash cURL +curl https://api.portkey.ai/v1/embeddings \ + -H "Content-Type: application/json" \ + -H "x-portkey-api-key: $PORTKEY_API_KEY" \ + -H "x-portkey-provider: @jina" \ + -d '{ + "model": "jina-embeddings-v2-base-en", + "input": "embed this text" + }' ``` - - + +## Add Provider in Model Catalog +Before making requests, add Jina AI to your Model Catalog: +1. Go to [**Model Catalog → Add Provider**](https://app.portkey.ai/model-catalog/providers) +2. Select **Jina AI** +3. Enter your [Jina AI API key](https://jina.ai/) +4. Name your provider (e.g., `jina`) -### **3\. Invoke Embeddings with** Jina AI + + See all setup options and detailed configuration instructions + + +--- + +## Jina AI Capabilities + +### Embeddings + +Generate embeddings with language-specific models: + + + +```python Python +from portkey_ai import Portkey + +portkey = Portkey(api_key="PORTKEY_API_KEY", provider="@jina") -Use the Portkey instance to send your embeddings requests to Jina AI. You can also override the virtual key directly in the API call if needed. - - - ```js - const embeddings = await portkey.embeddings.create({ - input: "embed this", - model: "jina-embeddings-v2-base-es", - }); - ``` - - -```py embeddings = portkey.embeddings.create( - input = "embed this", - model = "jina-embeddings-v2-base-de" + model="jina-embeddings-v2-base-en", + input="embed this text" ) + +print(embeddings.data[0].embedding) +``` + +```javascript Node.js +import Portkey from 'portkey-ai'; + +const portkey = new Portkey({ + apiKey: 'PORTKEY_API_KEY', + provider: '@jina' +}); + +const embeddings = await portkey.embeddings.create({ + model: "jina-embeddings-v2-base-en", + input: "embed this text" +}); + +console.log(embeddings.data[0].embedding); ``` - - + -### Using Jina AI Reranker Models +### Reranking -Portkey also supports the Reranker models by Jina AI through the REST API. - - - ```sh - curl https://api.portkey.ai/v1/rerank \ - -H "Content-Type: application/json" \ - -H "Authorization: Bearer $JINA_AI_API_KEY" \ - -H "x-portkey-provider: jina" \ - -d '{ +Rerank documents for better search results: + + + +```bash cURL +curl https://api.portkey.ai/v1/rerank \ + -H "Content-Type: application/json" \ + -H "x-portkey-api-key: $PORTKEY_API_KEY" \ + -H "x-portkey-provider: @jina" \ + -d '{ "model": "jina-reranker-v1-base-en", "query": "Organic skincare products for sensitive skin", "documents": [ - "Eco-friendly kitchenware for modern homes", - "Biodegradable cleaning supplies for eco-conscious consumers", - "Organic cotton baby clothes for sensitive skin" + "Eco-friendly kitchenware for modern homes", + "Biodegradable cleaning supplies for eco-conscious consumers", + "Organic cotton baby clothes for sensitive skin" ], "top_n": 2 }' - ``` - +``` - +```python Python +from portkey_ai import Portkey +portkey = Portkey(api_key="PORTKEY_API_KEY", provider="@jina") + +response = portkey.post( + "/rerank", + model="jina-reranker-v1-base-en", + query="Organic skincare products for sensitive skin", + documents=[ + "Eco-friendly kitchenware for modern homes", + "Biodegradable cleaning supplies for eco-conscious consumers", + "Organic cotton baby clothes for sensitive skin" + ], + top_n=2 +) +print(response) +``` +```javascript Node.js +import Portkey from 'portkey-ai'; + +const portkey = new Portkey({ + apiKey: 'PORTKEY_API_KEY', + provider: '@jina' +}); + +const response = await portkey.post( + "/rerank", + { + model: "jina-reranker-v1-base-en", + query: "Organic skincare products for sensitive skin", + documents: [ + "Eco-friendly kitchenware for modern homes", + "Biodegradable cleaning supplies for eco-conscious consumers", + "Organic cotton baby clothes for sensitive skin" + ], + top_n: 2 + } +); + +console.log(response); +``` + + + +--- ## Supported Models -Portkey works with all the embedding & reranker models offered by Jina AI. You can browse the full list of Jina AI models [here](https://jina.ai/embeddings#apiform). +Jina AI provides embedding models in multiple languages: -## Next Steps +| Model Family | Languages | Description | +|--------------|-----------|-------------| +| jina-embeddings-v2-base | en, de, es, zh, and more | Multilingual embeddings | +| jina-reranker-v1-base | en, de, es, zh | Reranking models | -The complete list of features supported in the SDK are available on the link below. - - +Check [Jina AI's model page](https://jina.ai/embeddings) for the complete list. + +--- -You'll find more information in the relevant sections: +## Next Steps -1. [Add metadata to your requests](/product/observability/metadata) -2. [Add gateway configs to your J](/product/ai-gateway/configs)ina AI[ requests](/product/ai-gateway/configs) -3. [Tracing Jina AI requests](/product/observability/traces) -4. [Setup a fallback from OpenAI Embeddings to Jina AI](/product/ai-gateway/fallbacks) + + + Add fallbacks, load balancing, and more + + + Monitor and trace your Jina AI requests + + + Cache embeddings for faster responses + + + Add custom metadata to requests + + + +For complete SDK documentation: + + + Complete Portkey SDK documentation + diff --git a/integrations/llms/lambda.mdx b/integrations/llms/lambda.mdx index a64983cd..24eabb37 100644 --- a/integrations/llms/lambda.mdx +++ b/integrations/llms/lambda.mdx @@ -1,235 +1,177 @@ --- -title: 'Lambda Labs' -description: 'Integrate Lambda with Portkey AI for seamless completions, prompt management, and advanced features like streaming and function calling.' +title: "Lambda Labs" +description: Use Lambda's GPU-powered inference for Llama and open-source models through Portkey. --- - -**Portkey Provider Slug:** `lambda` - +## Quick Start -## Overview +Get started with Lambda Labs in under 2 minutes: -Portkey offers native integrations with [Lambda](https://lambdalabs.com/) for Node.js, Python, and REST APIs. By combining Portkey with Lambda, you can create production-grade AI applications with enhanced reliability, observability, and advanced features. + - - - Explore the official Lambda documentation for comprehensive details on their APIs and models. - - +```python Python icon="python" +from portkey_ai import Portkey -## Getting Started +# 1. Install: pip install portkey-ai +# 2. Add @lambda provider in model catalog +# 3. Use it: - - - Visit the [Lambda dashboard](https://cloud.lambdalabs.com/api-keys) to generate your API key. - +portkey = Portkey(api_key="PORTKEY_API_KEY") - - Portkey's virtual key vault simplifies your interaction with Lambda. Virtual keys act as secure aliases for your actual API keys, offering enhanced security and easier management through [budget limits](/product/ai-gateway/virtual-keys/budget-limits) to control your API usage. +response = portkey.chat.completions.create( + model="@lambda/llama3.1-8b-instruct", + messages=[{"role": "user", "content": "Hello!"}] +) - Use the Portkey app to create a [virtual key](/product/ai-gateway/virtual-keys) associated with your Lambda API key. - +print(response.choices[0].message.content) +``` - - Now that you have your virtual key, set up the Portkey client: +```js Javascript icon="square-js" +import Portkey from 'portkey-ai' - ### Portkey Hosted App - Use the Portkey API key and the Lambda virtual key to initialize the client in your preferred programming language. +// 1. Install: npm install portkey-ai +// 2. Add @lambda provider in model catalog +// 3. Use it: - - ```python Python - from portkey_ai import Portkey +const portkey = new Portkey({ + apiKey: "PORTKEY_API_KEY" +}) - portkey = Portkey( - api_key="PORTKEY_API_KEY", # Replace with your Portkey API key - provider="@PROVIDER" # Replace with your virtual key for Lambda - ) - ``` +const response = await portkey.chat.completions.create({ + model: "@lambda/llama3.1-8b-instruct", + messages: [{ role: "user", content: "Hello!" }] +}) - ```javascript Node.js - import Portkey from 'portkey-ai' +console.log(response.choices[0].message.content) +``` - const portkey = new Portkey({ - apiKey: "PORTKEY_API_KEY", // defaults to process.env["PORTKEY_API_KEY"] - provider:"@PROVIDER" // Your Lambda Virtual Key - }) - ``` - +```python OpenAI Py icon="python" +from openai import OpenAI +from portkey_ai import PORTKEY_GATEWAY_URL - ### Open Source Use - Alternatively, use Portkey's Open Source AI Gateway to enhance your app's reliability with minimal code: +# 1. Install: pip install openai portkey-ai +# 2. Add @lambda provider in model catalog +# 3. Use it: - - ```python Python - from portkey_ai import Portkey, PORTKEY_GATEWAY_URL +client = OpenAI( + api_key="PORTKEY_API_KEY", # Portkey API key + base_url=PORTKEY_GATEWAY_URL +) - portkey = Portkey( - api_key="dummy", # Replace with your Portkey API key - base_url=PORTKEY_GATEWAY_URL, - Authorization="LAMBDA_API_KEY", # Replace with your Lambda API Key - provider="lambda" - ) - ``` +response = client.chat.completions.create( + model="@lambda/llama3.1-8b-instruct", + messages=[{"role": "user", "content": "Hello!"}] +) - ```javascript Node.js - import Portkey, { PORTKEY_GATEWAY_URL } from 'portkey-ai' +print(response.choices[0].message.content) +``` - const portkey = new Portkey({ - apiKey: "dummy", // Replace with your Portkey API key - baseUrl: PORTKEY_GATEWAY_URL, - Authorization: "LAMBDA_API_KEY", // Replace with your Lambda API Key - provider: "lambda" - }) - ``` - - - +```js OpenAI JS icon="square-js" +import OpenAI from "openai" +import { PORTKEY_GATEWAY_URL } from "portkey-ai" -🔥 That's it! You've integrated Portkey into your application with just a few lines of code. Now let's explore making requests using the Portkey client. +// 1. Install: npm install openai portkey-ai +// 2. Add @lambda provider in model catalog +// 3. Use it: -## Supported Models +const client = new OpenAI({ + apiKey: "PORTKEY_API_KEY", // Portkey API key + baseURL: PORTKEY_GATEWAY_URL +}) - +const response = await client.chat.completions.create({ + model: "@lambda/llama3.1-8b-instruct", + messages: [{ role: "user", content: "Hello!" }] +}) -- deepseek-coder-v2-lite-instruct -- dracarys2-72b-instruct -- hermes3-405b -- hermes3-405b-fp8-128k -- hermes3-70b -- hermes3-8b -- lfm-40b -- llama3.1-405b-instruct-fp8 -- llama3.1-70b-instruct-fp8 -- llama3.1-8b-instruct -- llama3.2-3b-instruct -- llama3.1-nemotron-70b-instruct +console.log(response.choices[0].message.content) +``` +```sh cURL icon="square-terminal" +# 1. Add @lambda provider in model catalog +# 2. Use it: + +curl https://api.portkey.ai/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "x-portkey-api-key: $PORTKEY_API_KEY" \ + -d '{ + "model": "@lambda/llama3.1-8b-instruct", + "messages": [{"role": "user", "content": "Hello!"}] + }' +``` + - +## Add Provider in Model Catalog +Before making requests, add Lambda Labs to your Model Catalog: -## Supported Endpoints and Parameters +1. Go to [**Model Catalog → Add Provider**](https://app.portkey.ai/model-catalog/providers) +2. Select **Lambda Labs** +3. Enter your [Lambda API key](https://cloud.lambdalabs.com/api-keys) +4. Name your provider (e.g., `lambda`) -| Endpoint | Supported Parameters | -| ----------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `chatComplete` | messages, max_tokens, temperature, top_p, stream, presence_penalty, frequency_penalty | -| `complete` | model, prompt, max_tokens, temperature, top_p, n, stream, logprobs, echo, stop, presence_penalty, frequency_penalty, best_of, logit_bias, user, seed, suffix | + + See all setup options and detailed configuration instructions + +--- -## Lambda Supported Features +## Lambda Capabilities -### Chat Completions +### Streaming -Generate chat completions using Lambda models through Portkey: +Stream responses for real-time output: -```python Python -completion = portkey.chat.completions.create( - messages=[{"role": "user", "content": "Say this is a test"}], - model="llama3.1-8b-instruct" -) - -print(completion.choices[0].message.content) -``` -```javascript Node.js -const chatCompletion = await portkey.chat.completions.create({ - messages: [{ role: 'user', content: 'Say this is a test' }], - model: 'llama3.1-8b-instruct', -}); - -console.log(chatCompletion.choices[0].message.content); -``` - -```curl REST -curl -X POST "https://api.portkey.ai/v1/chat/completions" \ - -H "Content-Type: application/json" \ - -H "Authorization: Bearer YOUR_PORTKEY_API_KEY" \ - -d '{ - "messages": [{"role": "user", "content": "Say this is a test"}], - "model": "llama3.1-8b-instruct" - }' -``` - - -### Streaming +```python Python +from portkey_ai import Portkey -Stream responses for real-time output in your applications: +portkey = Portkey(api_key="PORTKEY_API_KEY", provider="@lambda") - -```python Python -chat_complete = portkey.chat.completions.create( +stream = portkey.chat.completions.create( model="llama3.1-8b-instruct", - messages=[{"role": "user", "content": "Say this is a test"}], + messages=[{"role": "user", "content": "Tell me a story"}], stream=True ) -for chunk in chat_complete: +for chunk in stream: print(chunk.choices[0].delta.content or "", end="", flush=True) ``` ```javascript Node.js +import Portkey from 'portkey-ai'; + +const portkey = new Portkey({ + apiKey: 'PORTKEY_API_KEY', + provider: '@lambda' +}); + const stream = await portkey.chat.completions.create({ - model: 'gpt-4', - messages: [{ role: 'user', content: 'Say this is a test' }], - stream: true, + model: 'llama3.1-8b-instruct', + messages: [{ role: 'user', content: 'Tell me a story' }], + stream: true }); for await (const chunk of stream) { - process.stdout.write(chunk.choices[0]?.delta?.content || ''); + process.stdout.write(chunk.choices[0]?.delta?.content || ''); } ``` -```curl REST -curl -X POST "https://api.portkey.ai/v1/chat/completions" \ - -H "Content-Type: application/json" \ - -H "Authorization: Bearer YOUR_PORTKEY_API_KEY" \ - -d '{ - "model": "llama3.1-8b-instruct", - "messages": [{"role": "user", "content": "Say this is a test"}], - "stream": true - }' -``` - ### Function Calling -Leverage Lambda's function calling capabilities through Portkey: +Use Lambda's function calling capabilities: -```javascript Node.js -let tools = [{ - type: "function", - function: { - name: "getWeather", - description: "Get the current weather", - parameters: { - type: "object", - properties: { - location: { type: "string", description: "City and state" }, - unit: { type: "string", enum: ["celsius", "fahrenheit"] } - }, - required: ["location"] - } - } -}]; -let response = await portkey.chat.completions.create({ - model: "llama3.1-8b-instruct", - messages: [ - { role: "system", content: "You are a helpful assistant." }, - { role: "user", content: "What's the weather like in Delhi - respond in JSON" } - ], - tools, - tool_choice: "auto", -}); +```python Python +from portkey_ai import Portkey -console.log(response.choices[0].finish_reason); -``` +portkey = Portkey(api_key="PORTKEY_API_KEY", provider="@lambda") -```python Python tools = [{ "type": "function", "function": { @@ -250,212 +192,86 @@ response = portkey.chat.completions.create( model="llama3.1-8b-instruct", messages=[ {"role": "system", "content": "You are a helpful assistant."}, - {"role": "user", "content": "What's the weather like in Delhi - respond in JSON"} + {"role": "user", "content": "What's the weather in Delhi?"} ], tools=tools, tool_choice="auto" ) -print(response.choices[0].finish_reason) -``` - -```curl REST -curl -X POST "https://api.portkey.ai/v1/chat/completions" \ - -H "Content-Type: application/json" \ - -H "Authorization: Bearer YOUR_PORTKEY_API_KEY" \ - -d '{ - "model": "llama3.1-8b-instruct", - "messages": [ - {"role": "system", "content": "You are a helpful assistant."}, - {"role": "user", "content": "What'\''s the weather like in Delhi - respond in JSON"} - ], - "tools": [{ - "type": "function", - "function": { - "name": "getWeather", - "description": "Get the current weather", - "parameters": { - "type": "object", - "properties": { - "location": {"type": "string", "description": "City and state"}, - "unit": {"type": "string", "enum": ["celsius", "fahrenheit"]} - }, - "required": ["location"] - } - } - }], - "tool_choice": "auto" - }' +print(response.choices[0].message) ``` - - - - -# Portkey's Advanced Features -## Track End-User IDs +```javascript Node.js +import Portkey from 'portkey-ai'; -Portkey allows you to track user IDs passed with the user parameter in Lambda requests, enabling you to monitor user-level costs, requests, and more: +const portkey = new Portkey({ + apiKey: 'PORTKEY_API_KEY', + provider: '@lambda' +}); - -```python Python -response = portkey.chat.completions.create( - model="llama3.1-8b-instruct", - messages=[{"role": "user", "content": "Say this is a test"}], - user="user_123456" -) -``` +const tools = [{ + type: "function", + function: { + name: "getWeather", + description: "Get the current weather", + parameters: { + type: "object", + properties: { + location: { type: "string", description: "City and state" }, + unit: { type: "string", enum: ["celsius", "fahrenheit"] } + }, + required: ["location"] + } + } +}]; -```javascript Node.js -const chatCompletion = await portkey.chat.completions.create({ - messages: [{ role: "user", content: "Say this is a test" }], - model: "llama3.1-8b-instruct", - user: "user_12345", +const response = await portkey.chat.completions.create({ + model: "llama3.1-8b-instruct", + messages: [ + { role: "system", content: "You are a helpful assistant." }, + { role: "user", content: "What's the weather in Delhi?" } + ], + tools, + tool_choice: "auto" }); -``` -```curl REST -curl -X POST "https://api.portkey.ai/v1/chat/completions" \ - -H "Content-Type: application/json" \ - -H "Authorization: Bearer YOUR_PORTKEY_API_KEY" \ - -d '{ - "model": "llama3.1-8b-instruct", - "messages": [{"role": "user", "content": "Say this is a test"}], - "user": "user_123456" - }' +console.log(response.choices[0].message); ``` + -When you include the user parameter in your requests, Portkey logs will display the associated user ID, as shown in the image below: +--- -Portkey Logs with User ID +## Supported Endpoints and Parameters -In addition to the `user` parameter, Portkey allows you to send arbitrary custom metadata with your requests. This powerful feature enables you to associate additional context or information with each request, which can be useful for analysis, debugging, or other custom use cases. +| Endpoint | Supported Parameters | +|----------|---------------------| +| `/chat/completions` | messages, max_tokens, temperature, top_p, stream, presence_penalty, frequency_penalty, tools, tool_choice | +| `/completions` | model, prompt, max_tokens, temperature, top_p, n, stream, logprobs, echo, stop, presence_penalty, frequency_penalty, best_of, logit_bias, user, seed, suffix | - - - Explore how to use custom metadata to enhance your request tracking and analysis. - - +Check [Lambda's documentation](https://docs.lambdalabs.com/) for more details. -## Using The Gateway Config - -Here's a simplified version of how to use Portkey's Gateway Configuration: - - - - You can create a Gateway configuration using the Portkey Config Dashboard or by writing a JSON configuration in your code. In this example, requests are routed based on the user's subscription plan (paid or free). - - ```json - config = { - "strategy": { - "mode": "conditional", - "conditions": [ - { - "query": { "metadata.user_plan": { "$eq": "paid" } }, - "then": "llama3.1" - }, - { - "query": { "metadata.user_plan": { "$eq": "free" } }, - "then": "gpt-3.5" - } - ], - "default": "gpt-3.5" - }, - "targets": [ - { - "name": "llama3.1", - "provider":"@xx" - }, - { - "name": "gpt-3.5", - "provider":"@yy" - } - ] - } - ``` - - - - When a user makes a request, it will pass through Portkey's AI Gateway. Based on the configuration, the Gateway routes the request according to the user's metadata. - Conditional Routing Diagram - - - - Pass the Gateway configuration to your Portkey client. You can either use the config object or the Config ID from Portkey's hosted version. - - - ```python Python - from portkey_ai import Portkey - - portkey = Portkey( - api_key="PORTKEY_API_KEY", - provider="@PROVIDER", - config=portkey_config - ) - ``` - - ```javascript Node.js - import Portkey from 'portkey-ai' - - const portkey = new Portkey({ - apiKey: "PORTKEY_API_KEY", - provider:"@PROVIDER", - config: portkeyConfig - }) - ``` - - - - -That's it! Portkey seamlessly allows you to make your AI app more robust using built-in gateway features. Learn more about advanced gateway features: +--- + +## Next Steps - - Distribute requests across multiple targets based on defined weights. + + Add fallbacks, load balancing, and more - - Automatically switch to backup targets if the primary target fails. + + Monitor and trace your Lambda requests - - Route requests to different targets based on specified conditions. + + Manage and version your prompts - - Enable caching of responses to improve performance and reduce costs. + + Add custom metadata to requests -## Guardrails +For complete SDK documentation: -Portkey's AI gateway enables you to enforce input/output checks on requests by applying custom hooks before and after processing. Protect your user's/company's data by using PII guardrails and many more available on Portkey Guardrails: - -```json -{ - "provider:"@lambda-xxx", - "before_request_hooks": [{ - "id": "input-guardrail-id-xx" - }], - "after_request_hooks": [{ - "id": "output-guardrail-id-xx" - }] -} -``` - - - Explore Portkey's guardrail features to enhance the security and reliability of your AI applications. + + Complete Portkey SDK documentation - -## Next Steps - -The complete list of features supported in the SDK are available in our comprehensive documentation: - - - Explore the full capabilities of the Portkey SDK and how to leverage them in your projects. - - ---- - - - - -For the most up-to-date information on supported features and endpoints, please refer to our [API Reference](/api-reference/inference-api/introduction). diff --git a/integrations/llms/lemon-fox.mdx b/integrations/llms/lemon-fox.mdx index 4c0f42ad..0684e0c6 100644 --- a/integrations/llms/lemon-fox.mdx +++ b/integrations/llms/lemon-fox.mdx @@ -1,458 +1,278 @@ --- title: 'Lemonfox-AI' -description: 'Integrate LemonFox with Portkey for seamless completions, prompt management, and advanced features like streaming, function calling, and fine-tuning.' +description: 'Integrate LemonFox with Portkey for chat, image generation, and speech-to-text.' --- - -**Portkey Provider Slug:** `lemonfox-ai` - +## Quick Start -## Overview +Get started with LemonFox AI in under 2 minutes: -Portkey offers native integrations with [LemonFox-AI](https://www.lemonfox.ai/) for Node.js, Python, and REST APIs. By combining Portkey with Lemonfox AI, you can create production-grade AI applications with enhanced reliability, observability, and advanced features. - - - - Explore the official Lemonfox AI documentation for comprehensive details on their APIs and models. - - + -## Getting Started +```python Python icon="python" +from portkey_ai import Portkey - - - Visit the [LemonFox dashboard](https://www.lemonfox.ai/apis/keys) to generate your API key. - +# 1. Install: pip install portkey-ai +# 2. Add @lemonfox-ai provider in model catalog +# 3. Use it: - - Portkey's virtual key vault simplifies your interaction with LemonFox AI. Virtual keys act as secure aliases for your actual API keys, offering enhanced security and easier management through [budget limits](/product/ai-gateway/virtual-keys/budget-limits) to control your API usage. +portkey = Portkey(api_key="PORTKEY_API_KEY") - Use the Portkey app to create a [virtual key](/product/ai-gateway/virtual-keys) associated with your Lemonfox AI API key. - +response = portkey.chat.completions.create( + model="@lemonfox-ai/llama-8b-chat", + messages=[{"role": "user", "content": "Hello!"}] +) - - Now that you have your virtual key, set up the Portkey client: +print(response.choices[0].message.content) +``` - ### Portkey Hosted App - Use the Portkey API key and the LemonFox AI virtual key to initialize the client in your preferred programming language. +```js Javascript icon="square-js" +import Portkey from 'portkey-ai' - - ```python Python - from portkey_ai import Portkey +// 1. Install: npm install portkey-ai +// 2. Add @lemonfox-ai provider in model catalog +// 3. Use it: - portkey = Portkey( - api_key="PORTKEY_API_KEY", # Replace with your Portkey API key - provider="@PROVIDER" # Replace with your virtual key for LemonFox AI - ) - ``` +const portkey = new Portkey({ + apiKey: "PORTKEY_API_KEY" +}) - ```javascript Node.js - import Portkey from 'portkey-ai' +const response = await portkey.chat.completions.create({ + model: "@lemonfox-ai/llama-8b-chat", + messages: [{ role: "user", content: "Hello!" }] +}) - const portkey = new Portkey({ - apiKey: "PORTKEY_API_KEY", // defaults to process.env["PORTKEY_API_KEY"] - provider:"@PROVIDER" // Your LemonFox AI Virtual Key - }) - ``` - +console.log(response.choices[0].message.content) +``` - ### Open Source Use - Alternatively, use Portkey's Open Source AI Gateway to enhance your app's reliability with minimal code: +```python OpenAI Py icon="python" +from openai import OpenAI +from portkey_ai import PORTKEY_GATEWAY_URL - - ```python Python - from portkey_ai import Portkey, PORTKEY_GATEWAY_URL +# 1. Install: pip install openai portkey-ai +# 2. Add @lemonfox-ai provider in model catalog +# 3. Use it: - portkey = Portkey( - api_key="dummy", # Replace with your Portkey API key - base_url=PORTKEY_GATEWAY_URL, - Authorization="LEMONFOX_AI_API_KEY", # Replace with your Lemonfox AI API Key - provider="lemonfox-ai" - ) - ``` +client = OpenAI( + api_key="PORTKEY_API_KEY", # Portkey API key + base_url=PORTKEY_GATEWAY_URL +) - ```javascript Node.js - import Portkey, { PORTKEY_GATEWAY_URL } from 'portkey-ai' +response = client.chat.completions.create( + model="@lemonfox-ai/llama-8b-chat", + messages=[{"role": "user", "content": "Hello!"}] +) - const portkey = new Portkey({ - apiKey: "dummy", // Replace with your Portkey API key - baseUrl: PORTKEY_GATEWAY_URL, - Authorization: "LEMONFOX_AI_API_KEY", // Replace with your Lemonfox AI API Key - provider: "lemonfox-ai" - }) - ``` - - - +print(response.choices[0].message.content) +``` -🔥 That's it! You've integrated Portkey into your application with just a few lines of code. Now let's explore making requests using the Portkey client. +```js OpenAI JS icon="square-js" +import OpenAI from "openai" +import { PORTKEY_GATEWAY_URL } from "portkey-ai" -## Supported Models +// 1. Install: npm install openai portkey-ai +// 2. Add @lemonfox-ai provider in model catalog +// 3. Use it: - +const client = new OpenAI({ + apiKey: "PORTKEY_API_KEY", // Portkey API key + baseURL: PORTKEY_GATEWAY_URL +}) -`Chat` - Mixtral AI, Llama 3.1 8B and Llama 3.1 70B +const response = await client.chat.completions.create({ + model: "@lemonfox-ai/llama-8b-chat", + messages: [{ role: "user", content: "Hello!" }] +}) +console.log(response.choices[0].message.content) +``` -`Speech-To-Text`- Whisper large-v3 +```sh cURL icon="square-terminal" +# 1. Add @lemonfox-ai provider in model catalog +# 2. Use it: +curl https://api.portkey.ai/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "x-portkey-api-key: $PORTKEY_API_KEY" \ + -d '{ + "model": "@lemonfox-ai/llama-8b-chat", + "messages": [{"role": "user", "content": "Hello!"}] + }' +``` -`Vision`- Stable Diffusion XL (SDXL) + +## Add Provider in Model Catalog - +Before making requests, add LemonFox AI to your Model Catalog: +1. Go to [**Model Catalog → Add Provider**](https://app.portkey.ai/model-catalog/providers) +2. Select **LemonFox AI** +3. Enter your [LemonFox API key](https://www.lemonfox.ai/apis/keys) +4. Name your provider (e.g., `lemonfox-ai`) -## Supported Endpoints and Parameters + + See all setup options and detailed configuration instructions + -| Endpoint | Supported Parameters | -| ----------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `chatComplete` | messages, max_tokens, temperature, top_p, stream, presence_penalty, frequency_penalty | -| `imageGenerate` | prompt, response_format, negative_prompt, size, n | -| `createTranscription` | translate, language, prompt, response_format, file | | + + Explore the official Lemonfox AI documentation + +--- -## Lemonfox AI Supported Features +## LemonFox Capabilities -### Chat Completions +### Streaming -Generate chat completions using Lemonfox AI models through Portkey: +Stream responses for real-time output: -```python Python -completion = portkey.chat.completions.create( - messages=[{"role": "user", "content": "Say this is a test"}], - model="llama-8b-chat" -) - -print(completion.choices[0].message.content) -``` -```javascript Node.js -const chatCompletion = await portkey.chat.completions.create({ - messages: [{ role: 'user', content: 'Say this is a test' }], - model: 'llama-8b-chat', -}); - -console.log(chatCompletion.choices[0].message.content); -``` - -```curl REST -curl -X POST "https://api.portkey.ai/v1/chat/completions" \ - -H "Content-Type: application/json" \ - -H "Authorization: Bearer YOUR_PORTKEY_API_KEY" \ - -d '{ - "messages": [{"role": "user", "content": "Say this is a test"}], - "model": "llama-8b-chat" - }' -``` - - -### Streaming +```python Python +from portkey_ai import Portkey -Stream responses for real-time output in your applications: +portkey = Portkey(api_key="PORTKEY_API_KEY", provider="@lemonfox-ai") - -```python Python -chat_complete = portkey.chat.completions.create( +stream = portkey.chat.completions.create( model="llama-8b-chat", - messages=[{"role": "user", "content": "Say this is a test"}], + messages=[{"role": "user", "content": "Tell me a story"}], stream=True ) -for chunk in chat_complete: +for chunk in stream: print(chunk.choices[0].delta.content or "", end="", flush=True) ``` ```javascript Node.js +import Portkey from 'portkey-ai'; + +const portkey = new Portkey({ + apiKey: 'PORTKEY_API_KEY', + provider: '@lemonfox-ai' +}); + const stream = await portkey.chat.completions.create({ - model: 'gpt-4', - messages: [{ role: 'user', content: 'Say this is a test' }], - stream: true, + model: 'llama-8b-chat', + messages: [{ role: 'user', content: 'Tell me a story' }], + stream: true }); for await (const chunk of stream) { - process.stdout.write(chunk.choices[0]?.delta?.content || ''); + process.stdout.write(chunk.choices[0]?.delta?.content || ''); } ``` -```curl REST -curl -X POST "https://api.portkey.ai/v1/chat/completions" \ - -H "Content-Type: application/json" \ - -H "Authorization: Bearer YOUR_PORTKEY_API_KEY" \ - -d '{ - "model": "llama-8b-chat", - "messages": [{"role": "user", "content": "Say this is a test"}], - "stream": true - }' -``` -### Image Generate +### Image Generation -Here's how you can generate images using Lemonfox AI +Generate images with Stable Diffusion XL: - - ```python Python from portkey_ai import Portkey -client = Portkey( - api_key = "PORTKEY_API_KEY", - virtual_key = "PROVIDER" -) +portkey = Portkey(api_key="PORTKEY_API_KEY", provider="@lemonfox-ai") -client.images.generate( - prompt="A cute baby sea otter", - n=1, - size="1024x1024" +image = portkey.images.generate( + prompt="A cute baby sea otter", + size="1024x1024" ) +print(image.data[0].url) ``` ```javascript Node.js import Portkey from 'portkey-ai'; -const client = new Portkey({ - apiKey: 'PORTKEY_API_KEY', - provider:'@PROVIDER' +const portkey = new Portkey({ + apiKey: 'PORTKEY_API_KEY', + provider: '@lemonfox-ai' }); -async function main() { - const image = await client.images.generate({prompt: "A cute baby sea otter" }); - - console.log(image.data); -} -main(); +const image = await portkey.images.generate({ + prompt: "A cute baby sea otter", + size: "1024x1024" +}); +console.log(image.data[0].url); ``` -```curl REST -curl https://api.portkey.ai/v1/images/generations \ - -H "Content-Type: application/json" \ - -H "x-portkey-api-key: $PORTKEY_API_KEY" \ - -H "x-portkey-provider: $PORTKEY_PROVIDER" \ - -d '{ - "prompt": "A cute baby sea otter", - "n": 1, - "size": "1024x1024" - }' - -``` -### Transcription +### Speech-to-Text -Portkey supports both `Transcription` methods for STT models: +Transcribe audio with Whisper: + ```python Python -audio_file= open("/path/to/file.mp3", "rb") +from portkey_ai import Portkey + +portkey = Portkey(api_key="PORTKEY_API_KEY", provider="@lemonfox-ai") + +audio_file = open("/path/to/file.mp3", "rb") -# Transcription transcription = portkey.audio.transcriptions.create( - model="whisper-1", - file=audio_file + model="whisper-1", + file=audio_file ) + print(transcription.text) ``` ```javascript Node.js -import fs from "fs"; - -// Transcription -async function transcribe() { - const transcription = await portkey.audio.transcriptions.create({ - file: fs.createReadStream("/path/to/file.mp3"), - model: "whisper-1", - }); - console.log(transcription.text); -} -transcribe(); - - -``` - -```curl REST -# Transcription -curl -X POST "https://api.portkey.ai/v1/audio/transcriptions" \ - -H "Authorization: Bearer YOUR_PORTKEY_API_KEY" \ - -H "Content-Type: multipart/form-data" \ - -F "file=@/path/to/file.mp3" \ - -F "model=whisper-1" - -``` - - - - - -# Portkey's Advanced Features - -## Track End-User IDs - -Portkey allows you to track user IDs passed with the user parameter in Lemonfox AI requests, enabling you to monitor user-level costs, requests, and more: +import Portkey from 'portkey-ai'; +import fs from 'fs'; - -```python Python -response = portkey.chat.completions.create( - model="llama-8b-chat", - messages=[{"role": "user", "content": "Say this is a test"}], - user="user_123456" -) -``` +const portkey = new Portkey({ + apiKey: 'PORTKEY_API_KEY', + provider: '@lemonfox-ai' +}); -```javascript Node.js -const chatCompletion = await portkey.chat.completions.create({ - messages: [{ role: "user", content: "Say this is a test" }], - model: "llama-8b-chat", - user: "user_12345", +const transcription = await portkey.audio.transcriptions.create({ + file: fs.createReadStream("/path/to/file.mp3"), + model: "whisper-1" }); -``` -```curl REST -curl -X POST "https://api.portkey.ai/v1/chat/completions" \ - -H "Content-Type: application/json" \ - -H "Authorization: Bearer YOUR_PORTKEY_API_KEY" \ - -d '{ - "model": "llama-8b-chat", - "messages": [{"role": "user", "content": "Say this is a test"}], - "user": "user_123456" - }' +console.log(transcription.text); ``` + -When you include the user parameter in your requests, Portkey logs will display the associated user ID, as shown in the image below: +--- -Portkey Logs with User ID +## Supported Endpoints and Parameters -In addition to the `user` parameter, Portkey allows you to send arbitrary custom metadata with your requests. This powerful feature enables you to associate additional context or information with each request, which can be useful for analysis, debugging, or other custom use cases. +| Endpoint | Supported Parameters | +|----------|---------------------| +| `/chat/completions` | messages, max_tokens, temperature, top_p, stream, presence_penalty, frequency_penalty | +| `/images/generations` | prompt, response_format, negative_prompt, size, n | +| `/audio/transcriptions` | translate, language, prompt, response_format, file | - - - Explore how to use custom metadata to enhance your request tracking and analysis. - - +--- -## Using The Gateway Config - -Here's a simplified version of how to use Portkey's Gateway Configuration: - - - - You can create a Gateway configuration using the Portkey Config Dashboard or by writing a JSON configuration in your code. In this example, requests are routed based on the user's subscription plan (paid or free). - - ```json - config = { - "strategy": { - "mode": "conditional", - "conditions": [ - { - "query": { "metadata.user_plan": { "$eq": "paid" } }, - "then": "llama-8b-chat" - }, - { - "query": { "metadata.user_plan": { "$eq": "free" } }, - "then": "gpt-3.5" - } - ], - "default": "base-gpt4" - }, - "targets": [ - { - "name": "llama-8b-chat", - "provider":"@xx" - }, - { - "name": "gpt-3.5", - "provider":"@yy" - } - ] - } - ``` - - - - When a user makes a request, it will pass through Portkey's AI Gateway. Based on the configuration, the Gateway routes the request according to the user's metadata. - Conditional Routing Diagram - - - - Pass the Gateway configuration to your Portkey client. You can either use the config object or the Config ID from Portkey's hosted version. - - - ```python Python - from portkey_ai import Portkey - - portkey = Portkey( - api_key="PORTKEY_API_KEY", - provider="@PROVIDER", - config=portkey_config - ) - ``` - - ```javascript Node.js - import Portkey from 'portkey-ai' - - const portkey = new Portkey({ - apiKey: "PORTKEY_API_KEY", - provider:"@PROVIDER", - config: portkeyConfig - }) - ``` - - - - -That's it! Portkey seamlessly allows you to make your AI app more robust using built-in gateway features. Learn more about advanced gateway features: +## Next Steps - - Distribute requests across multiple targets based on defined weights. + + Add fallbacks, load balancing, and more - - Automatically switch to backup targets if the primary target fails. + + Monitor and trace your LemonFox requests - - Route requests to different targets based on specified conditions. + + Manage and version your prompts - - Enable caching of responses to improve performance and reduce costs. + + Add custom metadata to requests -## Guardrails +For complete SDK documentation: -Portkey's AI gateway enables you to enforce input/output checks on requests by applying custom hooks before and after processing. Protect your user's/company's data by using PII guardrails and many more available on Portkey Guardrails: - -```json -{ - "provider:"@lemonfox-ai-xxx", - "before_request_hooks": [{ - "id": "input-guardrail-id-xx" - }], - "after_request_hooks": [{ - "id": "output-guardrail-id-xx" - }] -} -``` - - - Explore Portkey's guardrail features to enhance the security and reliability of your AI applications. - - -## Next Steps - -The complete list of features supported in the SDK are available in our comprehensive documentation: - - - Explore the full capabilities of the Portkey SDK and how to leverage them in your projects. + + Complete Portkey SDK documentation - ---- - - - -For the most up-to-date information on supported features and endpoints, please refer to our [API Reference](/api-reference/inference-api/introduction). diff --git a/integrations/llms/lepton.mdx b/integrations/llms/lepton.mdx index e00f3dbe..b8d320c5 100644 --- a/integrations/llms/lepton.mdx +++ b/integrations/llms/lepton.mdx @@ -1,188 +1,277 @@ --- title: "Lepton AI" +description: Use Lepton AI's serverless AI endpoints for chat completions and speech-to-text through Portkey. --- -Portkey provides a robust and secure gateway to facilitate the integration of various Large Language Models (LLMs) into your applications, including [Lepton AI APIs](https://www.lepton.ai/docs/guides). - -With Portkey, you can take advantage of features like fast AI gateway access, observability, prompt management, and more, all while ensuring the secure management of your LLM API keys through a [virtual key](/product/ai-gateway/virtual-keys) system. - -Provider Slug. `lepton` - - -## Portkey SDK Integration with Lepton AI Models - -Portkey provides a consistent API to interact with models from various providers. To integrate Lepton AI with Portkey: - -### 1\. Install the Portkey SDK - -Add the Portkey SDK to your application to interact with Lepton AI's API through Portkey's gateway. - - - - ```sh - npm install --save portkey-ai - ``` - - - ```sh - pip install portkey-ai - ``` - - - -### 2\. Initialize Portkey with the Virtual Key - -To use Lepton AI with Portkey, [get your API key from Lepton AI](https://console.lepton.ai/), then add it to Portkey to create the virtual key. - - - - ```js - import Portkey from 'portkey-ai' - - const portkey = new Portkey({ - apiKey: "PORTKEY_API_KEY", // defaults to process.env["PORTKEY_API_KEY"] - provider:"@PROVIDER" // Your Lepton AI Virtual Key - }) - ``` - - - ```python - from portkey_ai import Portkey - - portkey = Portkey( - api_key="PORTKEY_API_KEY", # Replace with your Portkey API key - provider="@PROVIDER" # Replace with your virtual key for Lepton - ) - ``` - - - -### 3\. Invoke Chat Completions with Lepton AI - -Use the Portkey instance to send requests to Lepton AI. You can also override the virtual key directly in the API call if needed. - - - - ```js - const chatCompletion = await portkey.chat.completions.create({ - messages: [{ role: 'user', content: 'Say this is a test' }], - model: 'llama-3-8b-sft-v1', - }); - - console.log(chatCompletion.choices); - ``` - - - ```python - completion = portkey.chat.completions.create( - messages= [{ "role": 'user', "content": 'Say this is a test' }], - model= 'llama-3-8b-sft-v1' - ) - - print(completion) - ``` - - - ```sh - curl --location 'https://api.portkey.ai/v1/chat/completions' \ - -H 'Content-Type: application/json' \ - -H 'x-portkey-api-key: PORTKEY_API_KEY' \ - -H 'x-portkey-provider: PROVIDER' \ - --data '{ - "model": "llama-3-8b-sft-v1", - "messages": [ - { - "role": "user", - "content": "Say this is a test" - } - ] - }' - ``` - - - -## Speech-to-Text (Transcription) - -Lepton AI provides speech-to-text capabilities through Portkey's unified API: - - - - ```js - import fs from 'fs'; - - const transcription = await portkey.audio.transcriptions.create({ - file: fs.createReadStream('audio.mp3'), - model: 'whisper-large-v3', - }); - - console.log(transcription.text); - ``` - - - ```python - with open("audio.mp3", "rb") as audio_file: - transcription = portkey.audio.transcriptions.create( - file=audio_file, - model="whisper-large-v3" - ) - - print(transcription.text) - ``` - - - -## Advanced Features - -### Streaming Responses - -Lepton AI supports streaming responses to provide real-time generation: - - - - ```js - const stream = await portkey.chat.completions.create({ - messages: [{ role: 'user', content: 'Write a story about a robot' }], - model: 'llama-3-8b-sft-v1', - stream: true, - }); - - for await (const chunk of stream) { - process.stdout.write(chunk.choices[0]?.delta?.content || ''); - } - ``` - - - ```python - stream = portkey.chat.completions.create( - messages=[{"role": "user", "content": "Write a story about a robot"}], - model="llama-3-8b-sft-v1", - stream=True - ) - - for chunk in stream: - if chunk.choices[0].delta.content: - print(chunk.choices[0].delta.content, end="") - ``` - - - - -## Managing Lepton AI Prompts - -You can manage all prompts to Lepton AI in the [Prompt Library](/product/prompt-library). All the current models of Lepton AI are supported and you can easily start testing different prompts. - -Once you're ready with your prompt, you can use the `portkey.prompts.completions.create` interface to use the prompt in your application. +## Quick Start -## Next Steps +Get started with Lepton AI in under 2 minutes: + + + +```python Python icon="python" +from portkey_ai import Portkey + +# 1. Install: pip install portkey-ai +# 2. Add @lepton provider in model catalog +# 3. Use it: + +portkey = Portkey(api_key="PORTKEY_API_KEY") + +response = portkey.chat.completions.create( + model="@lepton/llama-3.1-8b", + messages=[{"role": "user", "content": "Hello!"}] +) + +print(response.choices[0].message.content) +``` + +```js Javascript icon="square-js" +import Portkey from 'portkey-ai' + +// 1. Install: npm install portkey-ai +// 2. Add @lepton provider in model catalog +// 3. Use it: + +const portkey = new Portkey({ + apiKey: "PORTKEY_API_KEY" +}) + +const response = await portkey.chat.completions.create({ + model: "@lepton/llama-3.1-8b", + messages: [{ role: "user", content: "Hello!" }] +}) + +console.log(response.choices[0].message.content) +``` + +```python OpenAI Py icon="python" +from openai import OpenAI +from portkey_ai import PORTKEY_GATEWAY_URL + +# 1. Install: pip install openai portkey-ai +# 2. Add @lepton provider in model catalog +# 3. Use it: + +client = OpenAI( + api_key="PORTKEY_API_KEY", # Portkey API key + base_url=PORTKEY_GATEWAY_URL +) + +response = client.chat.completions.create( + model="@lepton/llama-3.1-8b", + messages=[{"role": "user", "content": "Hello!"}] +) + +print(response.choices[0].message.content) +``` + +```js OpenAI JS icon="square-js" +import OpenAI from "openai" +import { PORTKEY_GATEWAY_URL } from "portkey-ai" + +// 1. Install: npm install openai portkey-ai +// 2. Add @lepton provider in model catalog +// 3. Use it: -The complete list of features supported in the SDK are available on the link below. +const client = new OpenAI({ + apiKey: "PORTKEY_API_KEY", // Portkey API key + baseURL: PORTKEY_GATEWAY_URL +}) - +const response = await client.chat.completions.create({ + model: "@lepton/llama-3.1-8b", + messages: [{ role: "user", content: "Hello!" }] +}) + +console.log(response.choices[0].message.content) +``` + +```sh cURL icon="square-terminal" +# 1. Add @lepton provider in model catalog +# 2. Use it: + +curl https://api.portkey.ai/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "x-portkey-api-key: $PORTKEY_API_KEY" \ + -d '{ + "model": "@lepton/llama-3.1-8b", + "messages": [{"role": "user", "content": "Hello!"}] + }' +``` + + + +## Add Provider in Model Catalog + +Before making requests, add Lepton AI to your Model Catalog: + +1. Go to [**Model Catalog → Add Provider**](https://app.portkey.ai/model-catalog/providers) +2. Select **Lepton AI** +3. Enter your [Lepton API key](https://console.lepton.ai/) +4. Name your provider (e.g., `lepton`) + + + See all setup options and detailed configuration instructions -You'll find more information in the relevant sections: +--- + +## Lepton AI Capabilities + +### Chat Completions + +Generate chat completions with Lepton's serverless models: + + + +```python Python +from portkey_ai import Portkey + +portkey = Portkey(api_key="PORTKEY_API_KEY", provider="@lepton") + +response = portkey.chat.completions.create( + model="llama-3.1-8b", + messages=[{"role": "user", "content": "Write a haiku about AI"}] +) + +print(response.choices[0].message.content) +``` + +```javascript Node.js +import Portkey from 'portkey-ai'; + +const portkey = new Portkey({ + apiKey: 'PORTKEY_API_KEY', + provider: '@lepton' +}); -1. [Add metadata to your requests](/product/observability/metadata) -2. [Add gateway configs to your Lepton AI requests](/product/ai-gateway/configs) -3. [Tracing Lepton AI requests](/product/observability/traces) -4. [Setup a fallback from OpenAI to Lepton AI APIs](/product/ai-gateway/fallbacks) +const response = await portkey.chat.completions.create({ + model: "llama-3.1-8b", + messages: [{ role: "user", content: "Write a haiku about AI" }] +}); + +console.log(response.choices[0].message.content); +``` + + + +### Speech-to-Text + +Transcribe audio using Lepton's Whisper models: + + + +```python Python +from portkey_ai import Portkey + +portkey = Portkey(api_key="PORTKEY_API_KEY", provider="@lepton") + +with open("audio.mp3", "rb") as audio_file: + transcription = portkey.audio.transcriptions.create( + file=audio_file, + model="whisper-large-v3" + ) + +print(transcription.text) +``` + +```javascript Node.js +import Portkey from 'portkey-ai'; +import fs from 'fs'; + +const portkey = new Portkey({ + apiKey: 'PORTKEY_API_KEY', + provider: '@lepton' +}); + +const transcription = await portkey.audio.transcriptions.create({ + file: fs.createReadStream('audio.mp3'), + model: 'whisper-large-v3' +}); + +console.log(transcription.text); +``` + + + +### Streaming + +Enable streaming for real-time responses: + + + +```python Python +from portkey_ai import Portkey + +portkey = Portkey(api_key="PORTKEY_API_KEY", provider="@lepton") + +stream = portkey.chat.completions.create( + model="llama-3.1-8b", + messages=[{"role": "user", "content": "Write a story about a robot"}], + stream=True +) + +for chunk in stream: + print(chunk.choices[0].delta.content, end="") +``` + +```javascript Node.js +import Portkey from 'portkey-ai'; + +const portkey = new Portkey({ + apiKey: 'PORTKEY_API_KEY', + provider: '@lepton' +}); + +const stream = await portkey.chat.completions.create({ + model: "llama-3.1-8b", + messages: [{ role: "user", content: "Write a story about a robot" }], + stream: true +}); + +for await (const chunk of stream) { + process.stdout.write(chunk.choices[0]?.delta?.content || ''); +} +``` + + + +--- + +## Supported Models + +Lepton AI provides serverless access to various models: + +| Model | Description | +|-------|-------------| +| llama-3.1-8b | Llama 3.1 8B model | +| llama-3-8b-sft-v1 | Fine-tuned Llama 3 | +| whisper-large-v3 | Speech-to-text | + +Check [Lepton's documentation](https://www.lepton.ai/docs) for the complete model list. + +--- + +## Next Steps + + + + Add fallbacks, load balancing, and more + + + Monitor and trace your Lepton requests + + + Manage and version your prompts + + + Add custom metadata to requests + + + +For complete SDK documentation: + + + Complete Portkey SDK documentation + diff --git a/integrations/llms/lingyi-01.ai.mdx b/integrations/llms/lingyi-01.ai.mdx index 597b3562..6ad48c3e 100644 --- a/integrations/llms/lingyi-01.ai.mdx +++ b/integrations/llms/lingyi-01.ai.mdx @@ -1,109 +1,159 @@ --- title: "Lingyi (01.ai)" +description: Use Lingyi's Yi models through Portkey for advanced Chinese and multilingual AI. --- -Portkey provides a robust and secure gateway to facilitate the integration of various Large Language Models (LLMs) into your applications, including [Lingyi (01.ai).](https://01.ai) +## Quick Start -With Portkey, you can take advantage of features like fast AI gateway access, observability, prompt management, and more, all while ensuring the secure management of your LLM API keys through a [virtual key](/product/ai-gateway/virtual-keys) system. +Get started with Lingyi in under 2 minutes: -Provider Slug: `lingyi` + -## Portkey SDK Integration with Lingyi Models +```python Python icon="python" +from portkey_ai import Portkey -Portkey provides a consistent API to interact with models from various providers. To integrate Lingyi with Portkey: +# 1. Install: pip install portkey-ai +# 2. Add @lingyi provider in model catalog +# 3. Use it: -### 1\. Install the Portkey SDK +portkey = Portkey(api_key="PORTKEY_API_KEY") -Add the Portkey SDK to your application to interact with Lingyi AI's API through Portkey's gateway. - - - ```sh - npm install --save portkey-ai - ``` - - - ```sh - pip install portkey-ai - ``` - +response = portkey.chat.completions.create( + model="@lingyi/Yi-Large-Preview", + messages=[{"role": "user", "content": "Hello!"}] +) - +print(response.choices[0].message.content) +``` +```js Javascript icon="square-js" +import Portkey from 'portkey-ai' +// 1. Install: npm install portkey-ai +// 2. Add @lingyi provider in model catalog +// 3. Use it: -### 2\. Initialize Portkey with the Virtual Key +const portkey = new Portkey({ + apiKey: "PORTKEY_API_KEY" +}) -To use Lingyi with Portkey, [get your API key from here](https://platform.lingyiwanwu.com/apikeys), then add it to Portkey to create the virtual key. - - - ```js - import Portkey from 'portkey-ai' +const response = await portkey.chat.completions.create({ + model: "@lingyi/Yi-Large-Preview", + messages: [{ role: "user", content: "Hello!" }] +}) - const portkey = new Portkey({ - apiKey: "PORTKEY_API_KEY", // defaults to process.env["PORTKEY_API_KEY"] - provider:"@PROVIDER" // Your Lingyi Virtual Key - }) - ``` - - - ```python - from portkey_ai import Portkey +console.log(response.choices[0].message.content) +``` - portkey = Portkey( - api_key="PORTKEY_API_KEY", # Replace with your Portkey API key - provider="@PROVIDER" # Replace with your virtual key for Groq - ) - ``` - +```python OpenAI Py icon="python" +from openai import OpenAI +from portkey_ai import PORTKEY_GATEWAY_URL - +# 1. Install: pip install openai portkey-ai +# 2. Add @lingyi provider in model catalog +# 3. Use it: +client = OpenAI( + api_key="PORTKEY_API_KEY", # Portkey API key + base_url=PORTKEY_GATEWAY_URL +) +response = client.chat.completions.create( + model="@lingyi/Yi-Large-Preview", + messages=[{"role": "user", "content": "Hello!"}] +) +print(response.choices[0].message.content) +``` -### 3\. Invoke Chat Completions with Lingyi +```js OpenAI JS icon="square-js" +import OpenAI from "openai" +import { PORTKEY_GATEWAY_URL } from "portkey-ai" -Use the Portkey instance to send requests to Lingyi. You can also override the virtual key directly in the API call if needed. - - - ```js - const chatCompletion = await portkey.chat.completions.create({ - messages: [{ role: 'user', content: 'Say this is a test' }], - model: 'Yi-Large-Preview', - }); +// 1. Install: npm install openai portkey-ai +// 2. Add @lingyi provider in model catalog +// 3. Use it: - console.log(chatCompletion.choices); - ``` - - - ```python - completion = portkey.chat.completions.create( - messages= [{ "role": 'user', "content": 'Say this is a test' }], - model= 'mistral-medium' - ) +const client = new OpenAI({ + apiKey: "PORTKEY_API_KEY", // Portkey API key + baseURL: PORTKEY_GATEWAY_URL +}) - print(completion) - ``` - +const response = await client.chat.completions.create({ + model: "@lingyi/Yi-Large-Preview", + messages: [{ role: "user", content: "Hello!" }] +}) - +console.log(response.choices[0].message.content) +``` +```sh cURL icon="square-terminal" +# 1. Add @lingyi provider in model catalog +# 2. Use it: +curl https://api.portkey.ai/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "x-portkey-api-key: $PORTKEY_API_KEY" \ + -d '{ + "model": "@lingyi/Yi-Large-Preview", + "messages": [{"role": "user", "content": "Hello!"}] + }' +``` + -## Managing Lingyi Prompts + +**Tip:** You can also set `provider="@lingyi"` in `Portkey()` and use just `model="Yi-Large-Preview"` in the request. + -You can manage all prompts to Lingyi in the [Prompt Library](/product/prompt-library). All the current models of Lingyi are supported and you can easily start testing different prompts. +## Add Provider in Model Catalog -Once you're ready with your prompt, you can use the `portkey.prompts.completions.create` interface to use the prompt in your application. +Before making requests, add Lingyi to your Model Catalog: -The complete list of features supported in the SDK are available on the link below. - +1. Go to [**Model Catalog → Add Provider**](https://app.portkey.ai/model-catalog/providers) +2. Select **Lingyi (01.ai)** +3. Enter your [Lingyi API key](https://platform.lingyiwanwu.com/apikeys) +4. Name your provider (e.g., `lingyi`) + + + See all setup options and detailed configuration instructions -You'll find more information in the relevant sections: +--- + +## Supported Models + +Lingyi (01.ai) provides the Yi model family: + +| Model | Description | +|-------|-------------| +| Yi-Large-Preview | Latest Yi large model preview | +| Yi-Medium | Medium-sized Yi model | +| Yi-Vision | Multimodal Yi model with vision | -1. [Add metadata to your requests](/product/observability/metadata) -2. [Add gateway configs to your Lingyi requests](/product/ai-gateway/configs) -3. [Tracing Lingyi requests](/product/observability/traces) -4. [Setup a fallback from OpenAI to Lingyi APIs](/product/ai-gateway/fallbacks) +Check [Lingyi's documentation](https://01.ai) for the complete model list. + +--- + +## Next Steps + + + + Add fallbacks, load balancing, and more + + + Monitor and trace your Lingyi requests + + + Manage and version your prompts + + + Add custom metadata to requests + + + +For complete SDK documentation: + + + Complete Portkey SDK documentation + diff --git a/integrations/llms/local-ai.mdx b/integrations/llms/local-ai.mdx index d37edf61..8f5a9518 100644 --- a/integrations/llms/local-ai.mdx +++ b/integrations/llms/local-ai.mdx @@ -1,172 +1,139 @@ --- title: "LocalAI" +description: "Integrate LocalAI-hosted models with Portkey for local LLM deployment with observability." --- -Portkey provides a robust and secure gateway to facilitate the integration of various Large Language Models (LLMs) into your applications, including your **locally hosted models through** [**LocalAI**](https://localai.io/). +Portkey provides a robust gateway to facilitate the integration of your **locally hosted models through** [**LocalAI**](https://localai.io/). -## Portkey SDK Integration with LocalAI +## Integration Steps -### 1\. Install the Portkey SDK - - - ```sh - npm install --save portkey-ai - ``` - - + + +Ensure your LocalAI API is externally accessible. If running on `http://localhost`, use a tool like `ngrok` to create a public URL. ```sh -pip install portkey-ai +ngrok http 8080 ``` - + - + +1. Go to [**Model Catalog → Add Provider**](https://app.portkey.ai/model-catalog/providers) +2. Enable **"Local/Privately hosted provider"** toggle +3. Select **OpenAI** as the provider type (LocalAI follows OpenAI API schema) +4. Enter your LocalAI URL with `/v1` in **Custom Host**: `https://your-localai.ngrok-free.app/v1` +5. Name your provider (e.g., `my-localai`) + + See all setup options + + -### 2\. Initialize Portkey with LocalAI URL + -First, ensure that your API is externally accessible. If you're running the API on `http://localhost`, consider using a tool like `ngrok` to create a public URL. Then, instantiate the Portkey client by adding your LocalAI URL (along with the version identifier) to the `customHost` property, and add the provider name as `openai`. - -**Note:** Don't forget to include the version identifier (e.g., `/v1`) in the `customHost` URL - - - - ```js - import Portkey from 'portkey-ai' - - const portkey = new Portkey({ - apiKey: "PORTKEY_API_KEY", // defaults to process.env["PORTKEY_API_KEY"] - provider: "openai", - customHost: "https://7cc4-3-235-157-146.ngrok-free.app/v1" // Your LocalAI ngrok URL - }) - ``` - - - ```python - from portkey_ai import Portkey - - portkey = Portkey( - api_key="PORTKEY_API_KEY", # Replace with your Portkey API key - provider="openai", - custom_host="https://7cc4-3-235-157-146.ngrok-free.app/v1" # Your LocalAI ngrok URL - ) - ``` - - - + + +```python Python +from portkey_ai import Portkey + +portkey = Portkey( + api_key="PORTKEY_API_KEY", + provider="@my-localai" +) + +response = portkey.chat.completions.create( + model="ggml-koala-7b-model-q4_0-r2.bin", + messages=[{"role": "user", "content": "Hello!"}] +) + +print(response.choices[0].message.content) +``` + +```javascript Node.js +import Portkey from 'portkey-ai'; + +const portkey = new Portkey({ + apiKey: 'PORTKEY_API_KEY', + provider: '@my-localai' +}); + +const response = await portkey.chat.completions.create({ + model: 'ggml-koala-7b-model-q4_0-r2.bin', + messages: [{ role: 'user', content: 'Hello!' }] +}); + +console.log(response.choices[0].message.content); +``` + + + +**Or use custom host directly:** + + +```python Python +from portkey_ai import Portkey + +portkey = Portkey( + api_key="PORTKEY_API_KEY", + provider="openai", + custom_host="https://your-localai.ngrok-free.app/v1" +) +``` + +```javascript Node.js +import Portkey from 'portkey-ai'; + +const portkey = new Portkey({ + apiKey: 'PORTKEY_API_KEY', + provider: 'openai', + customHost: 'https://your-localai.ngrok-free.app/v1' +}); +``` + + + + + -Portkey currently supports all endpoints that adhere to the OpenAI specification. This means, you can access and observe any of your LocalAI models that are exposed through OpenAI-compliant routes. +**Important:** +- Don't forget to include the version identifier (`/v1`) in the Custom Host URL +- Portkey supports all endpoints that adhere to the OpenAI specification -[List of supported endpoints here](/integrations/llms/local-ai#localai-endpoints-supported). - -### 3\. Invoke Chat Completions - -Use the Portkey SDK to invoke chat completions from your LocalAI model, just as you would with any other provider. - - - ```js - const chatCompletion = await portkey.chat.completions.create({ - messages: [{ role: 'user', content: 'Say this is a test' }], - model: 'ggml-koala-7b-model-q4_0-r2.bin', - }); - - console.log(chatCompletion.choices); - ``` - - - ```python - completion = portkey.chat.completions.create( - messages= [{ "role": 'user', "content": 'Say this is a test' }], - model= 'ggml-koala-7b-model-q4_0-r2.bin' - ) - - print(completion) - ``` - - - - - -## [Using Virtual Keys](https://app.portkey.ai/virtual-keys) - -Virtual Keys serve as Portkey's unified authentication system for all LLM interactions, simplifying the use of multiple providers and Portkey features within your application. For self-hosted LLMs, you can configure custom authentication requirements including authorization keys, bearer tokens, or any other headers needed to access your model: - - - - - -1. Navigate to [Virtual Keys](https://app.portkey.ai/virtual-keys) in your Portkey dashboard -2. Click **"Add Key"** and enable the **"Local/Privately hosted provider"** toggle -3. Configure your deployment: - - Select the matching provider API specification (typically `OpenAI`) - - Enter your model's base URL in the `Custom Host` field - - Add required authentication headers and their values -4. Click **"Create"** to generate your virtual key - -You can now use this virtual key in your requests: - - - - ```js - const portkey = new Portkey({ - apiKey: "PORTKEY_API_KEY", - provider:"@YOUR_SELF_HOSTED_LLM_PROVIDER" - - async function main() { - const response = await client.chat.completions.create({ - messages: [{ role: "user", content: "Bob the builder.." }], - model: "your-self-hosted-model-name", - }); - - console.log(response.choices[0].message.content); - }) - ``` - - - ```python - portkey = Portkey( - api_key="PORTKEY_API_KEY", - provider="@YOUR_SELF_HOSTED_LLM_PROVIDER" - ) - - response = portkey.chat.completions.create( - model="your-self-hosted-model-name", - messages=[ - {"role": "system", "content": "You are a helpful assistant."}, - {"role": "user", "content": "Hello!"} - ] - - print(response) - ) - ``` - - - -For more information about managing self-hosted LLMs with Portkey, see [Bring Your Own LLM](/integrations/llms/byollm). +--- ## LocalAI Endpoints Supported + | Endpoint | Resource | | :----------------------------------------------- | :---------------------------------------------------------------- | -| /chat/completions (Chat, Vision, Tools support) | [Doc](/provider-endpoints/chat) | -| /images/generations | [Doc](/provider-endpoints/images/create-image) | -| /embeddings | [Doc](/provider-endpoints/embeddings) | -| /audio/transcriptions | [Doc](/product/ai-gateway/multimodal-capabilities/speech-to-text) | - -## Next Steps - -Explore the complete list of features supported in the SDK: - - +| /chat/completions (Chat, Vision, Tools support) | [Doc](/api-reference/inference-api/chat-completion) | +| /images/generations | [Doc](/api-reference/inference-api/images/create-image) | +| /embeddings | [Doc](/api-reference/inference-api/embeddings) | +| /audio/transcriptions | [Doc](/product/ai-gateway/multimodal-capabilities/speech-to-text) | --- -You'll find more information in the relevant sections: +## Next Steps -1. [Add metadata to your requests](/product/observability/metadata) -2. [Add gateway configs to your Ollama requests](/product/ai-gateway/universal-api#ollama-in-configs) -3. [Tracing Ollama requests](/product/observability/traces) -4. [Setup a fallback from OpenAI to Ollama APIs](/product/ai-gateway/fallbacks) + + + Add retries, timeouts, and fallbacks + + + Monitor your LocalAI requests + + + Learn more about custom host setup + + + Complete guide for private LLMs + + + +For complete SDK documentation: + + + Complete Portkey SDK documentation + diff --git a/integrations/llms/mistral-ai.mdx b/integrations/llms/mistral-ai.mdx index 06cddf4f..eca1c963 100644 --- a/integrations/llms/mistral-ai.mdx +++ b/integrations/llms/mistral-ai.mdx @@ -1,209 +1,197 @@ --- title: "Mistral AI" +description: "Integrate Mistral AI models with Portkey's AI Gateway" --- -Portkey provides a robust and secure gateway to facilitate the integration of various Large Language Models (LLMs) into your applications, including [Mistral AI APIs](https://docs.mistral.ai/api/). +Portkey provides a robust and secure gateway to integrate various Large Language Models (LLMs) into applications, including [Mistral AI's models](https://docs.mistral.ai/api/). -With Portkey, you can take advantage of features like fast AI gateway access, observability, prompt management, and more, all while ensuring the secure management of your LLM API keys through a [virtual key](/product/ai-gateway/virtual-keys) system. +With Portkey, take advantage of features like fast AI gateway access, observability, prompt management, and more, while securely managing API keys through [Model Catalog](/product/model-catalog). - -Provider Slug. `mistral-ai` - -## Portkey SDK Integration with Mistral AI Models +## Quick Start + +Get Mistral AI working in 3 steps: -Portkey provides a consistent API to interact with models from various providers. To integrate Mistral AI with Portkey: + +```python Python icon="python" +from portkey_ai import Portkey -### 1\. Install the Portkey SDK +# 1. Install: pip install portkey-ai +# 2. Add @mistral-ai provider in model catalog +# 3. Use it: -Add the Portkey SDK to your application to interact with Mistral AI's API through Portkey's gateway. - - - ```sh - npm install --save portkey-ai - ``` - - - ```sh - pip install portkey-ai - ``` - +portkey = Portkey(api_key="PORTKEY_API_KEY") - +response = portkey.chat.completions.create( + model="@mistral-ai/mistral-large-latest", + messages=[{"role": "user", "content": "Say this is a test"}] +) +print(response.choices[0].message.content) +``` +```js Javascript icon="square-js" +import Portkey from 'portkey-ai' +// 1. Install: npm install portkey-ai +// 2. Add @mistral-ai provider in model catalog +// 3. Use it: -### 2\. Initialize Portkey with the Virtual Key +const portkey = new Portkey({ + apiKey: "PORTKEY_API_KEY" +}) -To use Mistral AI with Portkey, [get your API key from here](https://console.mistral.ai/api-keys/ ), then add it to Portkey to create the virtual key. - - - ```js - import Portkey from 'portkey-ai' +const response = await portkey.chat.completions.create({ + model: "@mistral-ai/mistral-large-latest", + messages: [{ role: "user", content: "Say this is a test" }] +}) - const portkey = new Portkey({ - apiKey: "PORTKEY_API_KEY", // defaults to process.env["PORTKEY_API_KEY"] - provider:"@PROVIDER" // Your Mistral AI Virtual Key - }) - ``` - - - ```python - from portkey_ai import Portkey +console.log(response.choices[0].message.content) +``` - portkey = Portkey( - api_key="PORTKEY_API_KEY", # Replace with your Portkey API key - provider="@PROVIDER" # Replace with your virtual key for Mistral AI - ) - ``` - +```python OpenAI Py icon="python" +from openai import OpenAI +from portkey_ai import PORTKEY_GATEWAY_URL - +# 1. Install: pip install openai portkey-ai +# 2. Add @mistral-ai provider in model catalog +# 3. Use it: +client = OpenAI( + api_key="PORTKEY_API_KEY", # Portkey API key + base_url=PORTKEY_GATEWAY_URL +) -### **3.1\. Invoke Chat Completions with** Mistral AI +response = client.chat.completions.create( + model="@mistral-ai/mistral-large-latest", + messages=[{"role": "user", "content": "Say this is a test"}] +) -Use the Portkey instance to send requests to Mistral AI. You can also override the virtual key directly in the API call if needed. +print(response.choices[0].message.content) +``` -You can also call the new Codestral model here! - - - ```js - const chatCompletion = await portkey.chat.completions.create({ - messages: [{ role: 'user', content: 'Say this is a test' }], - model: 'codestral-latest', - }); +```js OpenAI JS icon="square-js" +import OpenAI from "openai" +import { PORTKEY_GATEWAY_URL } from "portkey-ai" - console.log(chatCompletion.choices); - ``` - - - ```python - completion = portkey.chat.completions.create( - messages= [{ "role": 'user', "content": 'Say this is a test' }], - model= 'codestral-latest' - ) +// 1. Install: npm install openai portkey-ai +// 2. Add @mistral-ai provider in model catalog +// 3. Use it: - print(completion) - ``` - +const client = new OpenAI({ + apiKey: "PORTKEY_API_KEY", // Portkey API key + baseURL: PORTKEY_GATEWAY_URL +}) - +const response = await client.chat.completions.create({ + model: "@mistral-ai/mistral-large-latest", + messages: [{ role: "user", content: "Say this is a test" }] +}) +console.log(response.choices[0].message.content) +``` +```sh cURL icon="square-terminal" +# 1. Add @mistral-ai provider in model catalog +# 2. Use it: + +curl https://api.portkey.ai/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "x-portkey-api-key: $PORTKEY_API_KEY" \ + -d '{ + "model": "@mistral-ai/mistral-large-latest", + "messages": [ + { "role": "user", "content": "Say this is a test" } + ] + }' +``` + ---- + +**Tip:** You can also set `provider="@mistral-ai"` in `Portkey()` and use just `model="mistral-large-latest"` in the request. + -## Invoke Codestral Endpoint +## Add Provider in Model Catalog -Using Portkey, you can also call Mistral API's new Codestral endpoint. Just pass the Codestral URL `https://codestral.mistral.ai/v1` with the `customHost` property. - - - ```js - import Portkey from 'portkey-ai' +1. Go to [**Model Catalog → Add Provider**](https://app.portkey.ai/model-catalog/providers) +2. Select **Mistral AI** +3. Choose existing credentials or create new by entering your [Mistral AI API key](https://console.mistral.ai/api-keys/) +4. Name your provider (e.g., `mistral-ai-prod`) - const portkey = new Portkey({ - apiKey: "PORTKEY_API_KEY", - provider:"@MISTRAL_PROVIDER", - customHost: "https://codestral.mistral.ai/v1" - }) + + See all setup options, code examples, and detailed instructions + - const codeCompletion = await portkey.chat.completions.create({ - model: "codestral-latest", - messages: [{"role": "user", "content": "Write a minimalist Python code to validate the proof for the special number 1729"}] - }); +## Codestral Endpoint - console.log(codeCompletion.choices[0].message.content); - ``` - - - ```python - from portkey_ai import Portkey +Mistral AI provides a dedicated Codestral endpoint for code generation. Use the `customHost` property to access it: - portkey = Portkey( - api_key="PORTKEY_API_KEY", - provider="@MISTRAL_PROVIDER", - custom_host="https://codestral.mistral.ai/v1" - ) + +```python Python +from portkey_ai import Portkey - code_completion = portkey.chat.completions.create( - model="codestral-latest", - messages=[{"role": "user", "content": "Write a minimalist Python code to validate the proof for the special number 1729"}] - ) +portkey = Portkey( + api_key="PORTKEY_API_KEY", + provider="@mistral-ai", + custom_host="https://codestral.mistral.ai/v1" +) - print(code_completion.choices[0].message.content) - ``` - +code_completion = portkey.chat.completions.create( + model="codestral-latest", + messages=[{"role": "user", "content": "Write a minimalist Python code to validate the proof for the special number 1729"}] +) - +print(code_completion.choices[0].message.content) +``` +```js Javascript +import Portkey from 'portkey-ai' +const portkey = new Portkey({ + apiKey: "PORTKEY_API_KEY", + provider: "@mistral-ai", + customHost: "https://codestral.mistral.ai/v1" +}) +const codeCompletion = await portkey.chat.completions.create({ + model: "codestral-latest", + messages: [{ role: "user", content: "Write a minimalist Python code to validate the proof for the special number 1729" }] +}) -#### Your Codestral requests will show up on Portkey logs with the code snippets rendered beautifully! +console.log(codeCompletion.choices[0].message.content) +``` + + +Your Codestral requests will show up on Portkey logs with code snippets rendered beautifully: -## Codestral v/s Mistral API Endpoint +### Codestral vs Mistral API Endpoint + +Here's a guide for when to use the Codestral endpoint vs the standard Mistral API endpoint: -Here's a handy guide for when you might want to make your requests to the Codestral endpoint v/s the original Mistral API endpoint: -[For more, check out Mistral's Code Generation guide here](https://docs.mistral.ai/capabilities/code%5Fgeneration/#operation/listModels). - ---- - -## Managing Mistral AI Prompts - -You can manage all prompts to Mistral AI in the [Prompt Library](/product/prompt-library). All the current models of Mistral AI are supported and you can easily start testing different prompts. - -Once you're ready with your prompt, you can use the `portkey.prompts.completions.create` interface to use the prompt in your application. +[For more, check out Mistral's Code Generation guide](https://docs.mistral.ai/capabilities/code_generation/#operation/listModels) +## Mistral Tool Calling -### Mistral Tool Calling -Tool calling feature lets models trigger external tools based on conversation context. You define available functions, the model chooses when to use them, and your application executes them and returns results. +Tool calling lets models trigger external tools based on conversation context. You define available functions, the model chooses when to use them, and your application executes them and returns results. -Portkey supports Mistral Tool Calling and makes it interoperable across multiple providers. With Portkey Prompts, you can templatize various your prompts & tool schemas as well. +Portkey supports Mistral tool calling and makes it interoperable across multiple providers. With Portkey Prompts, you can templatize your prompts and tool schemas. + +```python Python +from portkey_ai import Portkey - - -```javascript Get Weather Tool -let tools = [{ - type: "function", - function: { - name: "getWeather", - description: "Get the current weather", - parameters: { - type: "object", - properties: { - location: { type: "string", description: "City and state" }, - unit: { type: "string", enum: ["celsius", "fahrenheit"] } - }, - required: ["location"] - } - } -}]; - -let response = await portkey.chat.completions.create({ - model: "your_mistral_model_name", - messages: [ - { role: "system", content: "You are a helpful assistant." }, - { role: "user", content: "What's the weather like in Delhi - respond in JSON" } - ], - tools, - tool_choice: "auto", -}); +portkey = Portkey( + api_key="PORTKEY_API_KEY", + provider="@mistral-ai" +) -console.log(response.choices[0].finish_reason); -``` - - -```python Get Weather Tool tools = [{ "type": "function", "function": { @@ -221,10 +209,10 @@ tools = [{ }] response = portkey.chat.completions.create( - model="your_mistral_model_name", + model="mistral-large-latest", messages=[ {"role": "system", "content": "You are a helpful assistant."}, - {"role": "user", "content": "What's the weather like in Delhi - respond in JSON"} + {"role": "user", "content": "What's the weather like in Delhi?"} ], tools=tools, tool_choice="auto" @@ -232,17 +220,54 @@ response = portkey.chat.completions.create( print(response.choices[0].finish_reason) ``` - - -```curl Get Weather Tool + +```js Javascript +import Portkey from 'portkey-ai' + +const portkey = new Portkey({ + apiKey: "PORTKEY_API_KEY", + provider: "@mistral-ai" +}) + +const tools = [{ + type: "function", + function: { + name: "getWeather", + description: "Get the current weather", + parameters: { + type: "object", + properties: { + location: { type: "string", description: "City and state" }, + unit: { type: "string", enum: ["celsius", "fahrenheit"] } + }, + required: ["location"] + } + } +}] + +const response = await portkey.chat.completions.create({ + model: "mistral-large-latest", + messages: [ + { role: "system", content: "You are a helpful assistant." }, + { role: "user", content: "What's the weather like in Delhi?" } + ], + tools, + tool_choice: "auto" +}) + +console.log(response.choices[0].finish_reason) +``` + +```sh cURL curl -X POST "https://api.portkey.ai/v1/chat/completions" \ -H "Content-Type: application/json" \ - -H "Authorization: Bearer YOUR_PORTKEY_API_KEY" \ + -H "x-portkey-api-key: $PORTKEY_API_KEY" \ + -H "x-portkey-provider: mistral-ai" \ -d '{ - "model": "your_mistral_model_name", + "model": "mistral-large-latest", "messages": [ {"role": "system", "content": "You are a helpful assistant."}, - {"role": "user", "content": "What'\''s the weather like in Delhi - respond in JSON"} + {"role": "user", "content": "What'\''s the weather like in Delhi?"} ], "tools": [{ "type": "function", @@ -262,19 +287,33 @@ curl -X POST "https://api.portkey.ai/v1/chat/completions" \ "tool_choice": "auto" }' ``` - - + +## Managing Mistral AI Prompts -## Next Steps +Manage all prompt templates to Mistral AI in the [Prompt Library](/product/prompt-library). All current Mistral AI models are supported, and you can easily test different prompts. -The complete list of features supported in the SDK are available on the link below. - - +Use the `portkey.prompts.completions.create` interface to use the prompt in an application. -You'll find more information in the relevant sections: +## Next Steps -1. [Add metadata to your requests](/product/observability/metadata) -2. [Add gateway configs to your Mistral AI](/product/ai-gateway/configs)[ requests](/product/ai-gateway/configs) -3. [Tracing Mistral AI requests](/product/observability/traces) -4. [Setup a fallback from OpenAI to Mistral AI APIs](/product/ai-gateway/fallbacks) + + + Add metadata to your Mistral AI requests + + + Add gateway configs to your Mistral AI requests + + + Trace your Mistral AI requests + + + Setup fallback from OpenAI to Mistral AI + + + +For complete SDK documentation: + + + Complete Portkey SDK documentation + diff --git a/integrations/llms/modal.mdx b/integrations/llms/modal.mdx index 8ff26968..48336c80 100644 --- a/integrations/llms/modal.mdx +++ b/integrations/llms/modal.mdx @@ -1,365 +1,208 @@ --- title: 'Modal Labs' -description: 'Integrate Modal with Portkey AI for seamless completions, prompt management, and advanced features like streaming and function calling.' +description: 'Integrate Modal with Portkey for seamless completions and streaming.' --- - -**Portkey Provider Slug:** `modal` - - -## Overview +## Quick Start -Portkey offers native integrations with [Modal](https://modal.com/) for Node.js, Python, and REST APIs. By combining Portkey with Modal, you can create production-grade AI applications with enhanced reliability, observability, and advanced features. +Get started with Modal in under 2 minutes: - - - Explore the official Modal documentation for comprehensive details on their APIs and models. - - + -## Getting Started +```python Python icon="python" +from portkey_ai import Portkey - - - Visit the [Modal dashboard](https://modal.com/) to generate your API credentials. Modal supports two authentication methods: - - **API Key**: Standard Bearer token authentication - - **Custom Headers**: `modal-key` and `modal-secret` headers for advanced use cases - +# 1. Install: pip install portkey-ai +# 2. Add @modal provider in model catalog +# 3. Use it: - - Portkey's virtual key vault simplifies your interaction with Modal. Virtual keys act as secure aliases for your actual API keys, offering enhanced security and easier management through [budget limits](/product/ai-gateway/usage-limits) to control your API usage. +portkey = Portkey(api_key="PORTKEY_API_KEY") - Use the Portkey app to create a [virtual key](/product/ai-gateway/virtual-keys) associated with your Modal API key. - +response = portkey.chat.completions.create( + model="@modal/your-model-name", + messages=[{"role": "user", "content": "Hello!"}] +) - - Now that you have your virtual key, set up the Portkey client: +print(response.choices[0].message.content) +``` - ### Portkey Hosted App - Use the Portkey API key and the Modal virtual key to initialize the client in your preferred programming language. +```js Javascript icon="square-js" +import Portkey from 'portkey-ai' - - ```python Python - from portkey_ai import Portkey +// 1. Install: npm install portkey-ai +// 2. Add @modal provider in model catalog +// 3. Use it: - portkey = Portkey( - api_key="PORTKEY_API_KEY", # Replace with your Portkey API key - provider="@PROVIDER" # Replace with your virtual key for Modal - ) - ``` +const portkey = new Portkey({ + apiKey: "PORTKEY_API_KEY" +}) - ```javascript Node.js - import Portkey from 'portkey-ai' +const response = await portkey.chat.completions.create({ + model: "@modal/your-model-name", + messages: [{ role: "user", content: "Hello!" }] +}) - const portkey = new Portkey({ - apiKey: "PORTKEY_API_KEY", // defaults to process.env["PORTKEY_API_KEY"] - provider:"@PROVIDER" // Your Modal Virtual Key - }) - ``` - +console.log(response.choices[0].message.content) +``` - ### Open Source Use - Alternatively, use Portkey's Open Source AI Gateway to enhance your app's reliability with minimal code: +```python OpenAI Py icon="python" +from openai import OpenAI +from portkey_ai import PORTKEY_GATEWAY_URL - - ```python Python - from portkey_ai import Portkey, PORTKEY_GATEWAY_URL +# 1. Install: pip install openai portkey-ai +# 2. Add @modal provider in model catalog +# 3. Use it: - portkey = Portkey( - api_key="dummy", # Replace with your Portkey API key - base_url=PORTKEY_GATEWAY_URL, - Authorization="MODAL_API_KEY", # Replace with your Modal API Key - provider="modal" - ) - ``` +client = OpenAI( + api_key="PORTKEY_API_KEY", # Portkey API key + base_url=PORTKEY_GATEWAY_URL +) - ```javascript Node.js - import Portkey, { PORTKEY_GATEWAY_URL } from 'portkey-ai' +response = client.chat.completions.create( + model="@modal/your-model-name", + messages=[{"role": "user", "content": "Hello!"}] +) - const portkey = new Portkey({ - apiKey: "dummy", // Replace with your Portkey API key - baseUrl: PORTKEY_GATEWAY_URL, - Authorization: "MODAL_API_KEY", // Replace with your Modal API Key - provider: "modal" - }) - ``` - - - +print(response.choices[0].message.content) +``` -🔥 That's it! You've integrated Portkey into your application with just a few lines of code. Now let's explore making requests using the Portkey client. +```js OpenAI JS icon="square-js" +import OpenAI from "openai" +import { PORTKEY_GATEWAY_URL } from "portkey-ai" -## Custom Host Configuration +// 1. Install: npm install openai portkey-ai +// 2. Add @modal provider in model catalog +// 3. Use it: -Modal deployments typically use custom endpoints. You can configure a custom host for your Modal deployment: +const client = new OpenAI({ + apiKey: "PORTKEY_API_KEY", // Portkey API key + baseURL: PORTKEY_GATEWAY_URL +}) - -```python Python -from portkey_ai import Portkey +const response = await client.chat.completions.create({ + model: "@modal/your-model-name", + messages: [{ role: "user", content: "Hello!" }] +}) -portkey = Portkey( - api_key="PORTKEY_API_KEY", - provider="@PROVIDER", - custom_host="https://your-modal-deployment.modal.run" -) +console.log(response.choices[0].message.content) ``` -```javascript Node.js -import Portkey from 'portkey-ai' - -const portkey = new Portkey({ - apiKey: "PORTKEY_API_KEY", - provider:"@PROVIDER", - customHost: "https://your-modal-deployment.modal.run" -}) +```sh cURL icon="square-terminal" +# 1. Add @modal provider in model catalog +# 2. Use it: + +curl https://api.portkey.ai/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "x-portkey-api-key: $PORTKEY_API_KEY" \ + -d '{ + "model": "@modal/your-model-name", + "messages": [{"role": "user", "content": "Hello!"}] + }' ``` - -## Supported Endpoints and Parameters + -| Endpoint | Supported Parameters | -| ----------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `chatComplete` | messages, max_tokens, temperature, top_p, stream, presence_penalty, frequency_penalty | -| `complete` | model, prompt, max_tokens, temperature, top_p, n, stream, logprobs, echo, stop, presence_penalty, frequency_penalty, best_of, logit_bias, user, seed, suffix | + +**Tip:** You can also set `provider="@modal"` in `Portkey()` and use just `model="your-model-name"` in the request. + +## Add Provider in Model Catalog -## Modal Supported Features +Before making requests, add Modal to your Model Catalog: -### Chat Completions +1. Go to [**Model Catalog → Add Provider**](https://app.portkey.ai/model-catalog/providers) +2. Select **Modal** +3. Enter your [Modal API credentials](https://modal.com/) + - Modal supports **API Key** (Bearer token) or **Custom Headers** (`modal-key` and `modal-secret`) +4. **Custom Host:** Enter your Modal deployment endpoint (e.g., `https://your-modal-deployment.modal.run`) +5. Name your provider (e.g., `modal`) -Generate chat completions using Modal-hosted models through Portkey: + +**Note:** Custom host configuration is set once in Model Catalog and applies to all requests using this provider. + - -```python Python -completion = portkey.chat.completions.create( - messages=[{"role": "user", "content": "Say this is a test"}], - model="your-modal-model" -) + + See all setup options and detailed configuration instructions + -print(completion.choices[0].message.content) -``` + + Explore the official Modal documentation + -```javascript Node.js -const chatCompletion = await portkey.chat.completions.create({ - messages: [{ role: 'user', content: 'Say this is a test' }], - model: 'your-modal-model', -}); +--- -console.log(chatCompletion.choices[0].message.content); -``` - -```curl REST -curl -X POST "https://api.portkey.ai/v1/chat/completions" \ - -H "Content-Type: application/json" \ - -H "Authorization: Bearer YOUR_PORTKEY_API_KEY" \ - -d '{ - "messages": [{"role": "user", "content": "Say this is a test"}], - "model": "your-modal-model" - }' -``` - +## Modal Capabilities ### Streaming -Stream responses for real-time output in your applications: +Stream responses for real-time output: + ```python Python -chat_complete = portkey.chat.completions.create( - model="your-modal-model", - messages=[{"role": "user", "content": "Say this is a test"}], +from portkey_ai import Portkey + +portkey = Portkey(api_key="PORTKEY_API_KEY") + +stream = portkey.chat.completions.create( + model="@modal/your-model", + messages=[{"role": "user", "content": "Tell me a story"}], stream=True ) -for chunk in chat_complete: +for chunk in stream: print(chunk.choices[0].delta.content or "", end="", flush=True) ``` ```javascript Node.js +import Portkey from 'portkey-ai'; + +const portkey = new Portkey({ + apiKey: 'PORTKEY_API_KEY' +}); + const stream = await portkey.chat.completions.create({ - model: 'your-modal-model', - messages: [{ role: 'user', content: 'Say this is a test' }], - stream: true, + model: '@modal/your-model', + messages: [{ role: 'user', content: 'Tell me a story' }], + stream: true }); for await (const chunk of stream) { - process.stdout.write(chunk.choices[0]?.delta?.content || ''); + process.stdout.write(chunk.choices[0]?.delta?.content || ''); } ``` -```curl REST -curl -X POST "https://api.portkey.ai/v1/chat/completions" \ - -H "Content-Type: application/json" \ - -H "Authorization: Bearer YOUR_PORTKEY_API_KEY" \ - -d '{ - "model": "your-modal-model", - "messages": [{"role": "user", "content": "Say this is a test"}], - "stream": true - }' -``` +--- -# Portkey's Advanced Features - -## Track End-User IDs - -Portkey allows you to track user IDs passed with the user parameter in Modal requests, enabling you to monitor user-level costs, requests, and more: - - -```python Python -response = portkey.chat.completions.create( - model="your-modal-model", - messages=[{"role": "user", "content": "Say this is a test"}], - user="user_123456" -) -``` - -```javascript Node.js -const chatCompletion = await portkey.chat.completions.create({ - messages: [{ role: "user", content: "Say this is a test" }], - model: "your-modal-model", - user: "user_12345", -}); -``` - -```curl REST -curl -X POST "https://api.portkey.ai/v1/chat/completions" \ - -H "Content-Type: application/json" \ - -H "Authorization: Bearer YOUR_PORTKEY_API_KEY" \ - -d '{ - "model": "your-modal-model", - "messages": [{"role": "user", "content": "Say this is a test"}], - "user": "user_123456" - }' -``` - - -When you include the user parameter in your requests, Portkey logs will display the associated user ID. +## Supported Endpoints and Parameters -In addition to the `user` parameter, Portkey allows you to send arbitrary custom metadata with your requests. This powerful feature enables you to associate additional context or information with each request, which can be useful for analysis, debugging, or other custom use cases. +| Endpoint | Supported Parameters | +|----------|---------------------| +| `/chat/completions` | messages, max_tokens, temperature, top_p, stream, presence_penalty, frequency_penalty | - - - Explore how to use custom metadata to enhance your request tracking and analysis. - - +--- -## Using The Gateway Config - -Here's a simplified version of how to use Portkey's Gateway Configuration: - - - - You can create a Gateway configuration using the Portkey Config Dashboard or by writing a JSON configuration in your code. In this example, requests are routed based on the user's subscription plan (paid or free). - - ```json - config = { - "strategy": { - "mode": "conditional", - "conditions": [ - { - "query": { "metadata.user_plan": { "$eq": "paid" } }, - "then": "modal-premium" - }, - { - "query": { "metadata.user_plan": { "$eq": "free" } }, - "then": "gpt-3.5" - } - ], - "default": "gpt-3.5" - }, - "targets": [ - { - "name": "modal-premium", - "provider":"@modal-xxx" - }, - { - "name": "gpt-3.5", - "provider":"@openai-yyy" - } - ] - } - ``` - - - - When a user makes a request, it will pass through Portkey's AI Gateway. Based on the configuration, the Gateway routes the request according to the user's metadata. - - - - Pass the Gateway configuration to your Portkey client. You can either use the config object or the Config ID from Portkey's hosted version. - - - ```python Python - from portkey_ai import Portkey - - portkey = Portkey( - api_key="PORTKEY_API_KEY", - provider="@PROVIDER", - config=portkey_config - ) - ``` - - ```javascript Node.js - import Portkey from 'portkey-ai' - - const portkey = new Portkey({ - apiKey: "PORTKEY_API_KEY", - provider:"@PROVIDER", - config: portkeyConfig - }) - ``` - - - - -That's it! Portkey seamlessly allows you to make your AI app more robust using built-in gateway features. Learn more about advanced gateway features: +## Next Steps - - Distribute requests across multiple targets based on defined weights. + + Add fallbacks, load balancing, and more - - Automatically switch to backup targets if the primary target fails. + + Monitor and trace your Modal requests - - Route requests to different targets based on specified conditions. + + Manage and version your prompts - - Enable caching of responses to improve performance and reduce costs. + + Add custom metadata to requests -## Guardrails - -Portkey's AI gateway enables you to enforce input/output checks on requests by applying custom hooks before and after processing. Protect your user's/company's data by using PII guardrails and many more available on Portkey Guardrails: - -```json -{ - "provider:"@modal-xxx", - "before_request_hooks": [{ - "id": "input-guardrail-id-xx" - }], - "after_request_hooks": [{ - "id": "output-guardrail-id-xx" - }] -} -``` +For complete SDK documentation: - - Explore Portkey's guardrail features to enhance the security and reliability of your AI applications. + + Complete Portkey SDK documentation - -## Next Steps - -The complete list of features supported in the SDK are available in our comprehensive documentation: - - - Explore the full capabilities of the Portkey SDK and how to leverage them in your projects. - - ---- - -For the most up-to-date information on supported features and endpoints, please refer to our [API Reference](/docs/api-reference/introduction). - diff --git a/integrations/llms/monster-api.mdx b/integrations/llms/monster-api.mdx index 02b89002..e64d7cc6 100644 --- a/integrations/llms/monster-api.mdx +++ b/integrations/llms/monster-api.mdx @@ -1,105 +1,156 @@ --- title: "Monster API" -description: "MonsterAPIs provides access to generative AI model APIs at 80% lower costs. Connect to MonsterAPI LLM APIs seamlessly through Portkey's AI gateway." +description: Access generative AI models at 80% lower costs through Monster API and Portkey. --- -Portkey provides a robust and secure gateway to facilitate the integration of various Large Language Models (LLMs) into your applications, including [MonsterAPI APIs](https://developer.monsterapi.ai/docs/getting-started). +## Quick Start -With Portkey, you can take advantage of features like fast AI gateway access, observability, prompt management, and more, all while ensuring the secure management of your LLM API keys through a [virtual key](/product/ai-gateway/virtual-keys) system. - -Provider Slug. `monsterapi` - -## Portkey SDK Integration with MonsterAPI Models +Get started with Monster API in under 2 minutes: -Portkey provides a consistent API to interact with models from various providers. To integrate MonsterAPI with Portkey: + -### 1\. Install the Portkey SDK +```python Python icon="python" +from portkey_ai import Portkey -Add the Portkey SDK to your application to interact with MonsterAPI's API through Portkey's gateway. - - - ```sh - npm install --save portkey-ai - ``` - - - ```sh - pip install portkey-ai - ``` - +# 1. Install: pip install portkey-ai +# 2. Add @monsterapi provider in model catalog +# 3. Use it: - +portkey = Portkey(api_key="PORTKEY_API_KEY") +response = portkey.chat.completions.create( + model="@monsterapi/llama-3.1-8b-instruct", + messages=[{"role": "user", "content": "Hello!"}] +) +print(response.choices[0].message.content) +``` +```js Javascript icon="square-js" +import Portkey from 'portkey-ai' -### 2\. Initialize Portkey with the Virtual Key +// 1. Install: npm install portkey-ai +// 2. Add @monsterapi provider in model catalog +// 3. Use it: -To use Monster API with Portkey, [get your API key from here,](https://monsterapi.ai/user/dashboard) then add it to Portkey to create the virtual key. - - - ```js - import Portkey from 'portkey-ai' +const portkey = new Portkey({ + apiKey: "PORTKEY_API_KEY" +}) - const portkey = new Portkey({ - apiKey: "PORTKEY_API_KEY", // defaults to process.env["PORTKEY_API_KEY"] - provider:"@PROVIDER" // Your MonsterAPI Virtual Key - }) - ``` - +const response = await portkey.chat.completions.create({ + model: "@monsterapi/llama-3.1-8b-instruct", + messages: [{ role: "user", content: "Hello!" }] +}) - +console.log(response.choices[0].message.content) +``` +```python OpenAI Py icon="python" +from openai import OpenAI +from portkey_ai import PORTKEY_GATEWAY_URL +# 1. Install: pip install openai portkey-ai +# 2. Add @monsterapi provider in model catalog +# 3. Use it: +client = OpenAI( + api_key="PORTKEY_API_KEY", # Portkey API key + base_url=PORTKEY_GATEWAY_URL +) -### **3\. Invoke Chat Completions with** MonsterAPI +response = client.chat.completions.create( + model="@monsterapi/llama-3.1-8b-instruct", + messages=[{"role": "user", "content": "Hello!"}] +) -Use the Portkey instance to send requests to MonsterAPI. You can also override the virtual key directly in the API call if needed. - - - ```js - const chatCompletion = await portkey.chat.completions.create({ - messages: [{ role: 'user', content: 'Say this is a test' }], - model: 'TinyLlama/TinyLlama-1.1B-Chat-v1.0', - }); +print(response.choices[0].message.content) +``` - console.log(chatCompletion.choices); - ``` - - - ```python - completion = portkey.chat.completions.create( - messages= [{ "role": 'user', "content": 'Say this is a test' }], - model= 'mistral-medium' - ) +```js OpenAI JS icon="square-js" +import OpenAI from "openai" +import { PORTKEY_GATEWAY_URL } from "portkey-ai" - print(completion) - ``` - +// 1. Install: npm install openai portkey-ai +// 2. Add @monsterapi provider in model catalog +// 3. Use it: - +const client = new OpenAI({ + apiKey: "PORTKEY_API_KEY", // Portkey API key + baseURL: PORTKEY_GATEWAY_URL +}) +const response = await client.chat.completions.create({ + model: "@monsterapi/llama-3.1-8b-instruct", + messages: [{ role: "user", content: "Hello!" }] +}) -## Managing MonsterAPI Prompts +console.log(response.choices[0].message.content) +``` -You can manage all prompts to MonsterAPI in the [Prompt Library](/product/prompt-library). All the current models of MonsterAPI are supported and you can easily start testing different prompts. +```sh cURL icon="square-terminal" +# 1. Add @monsterapi provider in model catalog +# 2. Use it: -Once you're ready with your prompt, you can use the `portkey.prompts.completions.create` interface to use the prompt in your application. +curl https://api.portkey.ai/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "x-portkey-api-key: $PORTKEY_API_KEY" \ + -d '{ + "model": "@monsterapi/llama-3.1-8b-instruct", + "messages": [{"role": "user", "content": "Hello!"}] + }' +``` + + + +## Add Provider in Model Catalog + +Before making requests, add Monster API to your Model Catalog: + +1. Go to [**Model Catalog → Add Provider**](https://app.portkey.ai/model-catalog/providers) +2. Select **Monster API** +3. Enter your [Monster API key](https://monsterapi.ai/user/dashboard) +4. Name your provider (e.g., `monsterapi`) + + + See all setup options and detailed configuration instructions + + +--- ## Supported Models -[Find the latest list of supported models here.](https://llm.monsterapi.ai/docs) +Monster API provides cost-effective access to open-source models: -## Next Steps + + View the complete list of Monster API models + + +Popular models include: +- llama-3.1-8b-instruct +- TinyLlama/TinyLlama-1.1B-Chat-v1.0 +- mistral-7b-instruct -The complete list of features supported in the SDK are available on the link below. - -[SDK](/api-reference/sdk) - -You'll find more information in the relevant sections: +--- + +## Next Steps -1. [Add metadata to your requests](/product/observability/metadata) -2. [Add gateway configs to your MonsterAPI requests](/product/ai-gateway/configs) -3. [Tracing MonsterAPI requests](/product/observability/traces) -4. [Setup a fallback from OpenAI to MonsterAPI APIs](/product/ai-gateway/fallbacks) + + + Add fallbacks, load balancing, and more + + + Monitor and trace your Monster API requests + + + Manage and version your prompts + + + Add custom metadata to requests + + + +For complete SDK documentation: + + + Complete Portkey SDK documentation + diff --git a/integrations/llms/moonshot.mdx b/integrations/llms/moonshot.mdx index f24888ce..35afa719 100644 --- a/integrations/llms/moonshot.mdx +++ b/integrations/llms/moonshot.mdx @@ -1,109 +1,155 @@ --- title: "Moonshot" +description: Use Moonshot's AI models through Portkey for Chinese language processing. --- -Portkey provides a robust and secure gateway to facilitate the integration of various Large Language Models (LLMs) into your applications, including [Moonshot. ](https://moonshot.cn) +## Quick Start -With Portkey, you can take advantage of features like fast AI gateway access, observability, prompt management, and more, all while ensuring the secure management of your LLM API keys through a [virtual key](/product/ai-gateway/virtual-keys) system. +Get started with Moonshot in under 2 minutes: - -Provider Slug. `moonshot` - + -## Portkey SDK Integration with Moonshot Models +```python Python icon="python" +from portkey_ai import Portkey -Portkey provides a consistent API to interact with models from various providers. To integrate Moonshot with Portkey: +# 1. Install: pip install portkey-ai +# 2. Add @moonshot provider in model catalog +# 3. Use it: -### 1\. Install the Portkey SDK +portkey = Portkey(api_key="PORTKEY_API_KEY") -Add the Portkey SDK to your application to interact with Moonshot's API through Portkey's gateway. - - - ```sh - npm install --save portkey-ai - ``` - - +response = portkey.chat.completions.create( + model="@moonshot/moonshot-v1-8k", + messages=[{"role": "user", "content": "Hello!"}] +) -```sh -pip install portkey-ai +print(response.choices[0].message.content) ``` - - +```js Javascript icon="square-js" +import Portkey from 'portkey-ai' +// 1. Install: npm install portkey-ai +// 2. Add @moonshot provider in model catalog +// 3. Use it: +const portkey = new Portkey({ + apiKey: "PORTKEY_API_KEY" +}) -### 2\. Initialize Portkey with the Virtual Key +const response = await portkey.chat.completions.create({ + model: "@moonshot/moonshot-v1-8k", + messages: [{ role: "user", content: "Hello!" }] +}) -To use Moonshot with Portkey, [get your API key from here,](https://moonshot.cn) then add it to Portkey to create the virtual key. - - - ```js - import Portkey from 'portkey-ai' +console.log(response.choices[0].message.content) +``` - const portkey = new Portkey({ - apiKey: "PORTKEY_API_KEY", // defaults to process.env["PORTKEY_API_KEY"] - provider:"@PROVIDER" // Your Moonshot Virtual Key - }) - ``` - - - ```python - from portkey_ai import Portkey +```python OpenAI Py icon="python" +from openai import OpenAI +from portkey_ai import PORTKEY_GATEWAY_URL - portkey = Portkey( - api_key="PORTKEY_API_KEY", # Replace with your Portkey API key - provider="@PROVIDER" # Replace with your virtual key for Groq - ) - ``` - +# 1. Install: pip install openai portkey-ai +# 2. Add @moonshot provider in model catalog +# 3. Use it: - +client = OpenAI( + api_key="PORTKEY_API_KEY", # Portkey API key + base_url=PORTKEY_GATEWAY_URL +) +response = client.chat.completions.create( + model="@moonshot/moonshot-v1-8k", + messages=[{"role": "user", "content": "Hello!"}] +) +print(response.choices[0].message.content) +``` -### 3\. Invoke Chat Completions with Moonshot +```js OpenAI JS icon="square-js" +import OpenAI from "openai" +import { PORTKEY_GATEWAY_URL } from "portkey-ai" -Use the Portkey instance to send requests to Moonshot. You can also override the virtual key directly in the API call if needed. - - - ```js - const chatCompletion = await portkey.chat.completions.create({ - messages: [{ role: 'user', content: 'Say this is a test' }], - model: 'moonshot-v1-8k', - }); +// 1. Install: npm install openai portkey-ai +// 2. Add @moonshot provider in model catalog +// 3. Use it: - console.log(chatCompletion.choices);d - ``` - - - ```python - completion = portkey.chat.completions.create( - messages= [{ "role": 'user', "content": 'Say this is a test' }], - model= 'moonshot-v1-8k' - ) +const client = new OpenAI({ + apiKey: "PORTKEY_API_KEY", // Portkey API key + baseURL: PORTKEY_GATEWAY_URL +}) - print(completion) - ``` - +const response = await client.chat.completions.create({ + model: "@moonshot/moonshot-v1-8k", + messages: [{ role: "user", content: "Hello!" }] +}) - +console.log(response.choices[0].message.content) +``` +```sh cURL icon="square-terminal" +# 1. Add @moonshot provider in model catalog +# 2. Use it: + +curl https://api.portkey.ai/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "x-portkey-api-key: $PORTKEY_API_KEY" \ + -d '{ + "model": "@moonshot/moonshot-v1-8k", + "messages": [{"role": "user", "content": "Hello!"}] + }' +``` -## Managing Moonshot Prompts + -You can manage all prompts to Moonshot in the [Prompt Library](/product/prompt-library). All the current models of Moonshot are supported and you can easily start testing different prompts. +## Add Provider in Model Catalog -Once you're ready with your prompt, you can use the `portkey.prompts.completions.create` interface to use the prompt in your application. +Before making requests, add Moonshot to your Model Catalog: -The complete list of features supported in the SDK are available on the link below. - +1. Go to [**Model Catalog → Add Provider**](https://app.portkey.ai/model-catalog/providers) +2. Select **Moonshot** +3. Enter your [Moonshot API key](https://moonshot.cn) +4. Name your provider (e.g., `moonshot`) + + + See all setup options and detailed configuration instructions -You'll find more information in the relevant sections: +--- + +## Supported Models -1. [Add metadata to your requests](/product/observability/metadata) -2. [Add gateway configs to your Moonshot requests](/product/ai-gateway/configs) -3. [Tracing Moonshot requests](/product/observability/traces) -4. [Setup a fallback from OpenAI to Moonshot APIs](/product/ai-gateway/fallbacks) +Moonshot provides Chinese-focused AI models: + +| Model | Context Length | Description | +|-------|---------------|-------------| +| moonshot-v1-8k | 8,192 tokens | 8K context window | +| moonshot-v1-32k | 32,768 tokens | 32K context window | +| moonshot-v1-128k | 131,072 tokens | 128K context window | + +Check [Moonshot's documentation](https://moonshot.cn) for more details. + +--- + +## Next Steps + + + + Add fallbacks, load balancing, and more + + + Monitor and trace your Moonshot requests + + + Manage and version your prompts + + + Add custom metadata to requests + + + +For complete SDK documentation: + + + Complete Portkey SDK documentation + diff --git a/integrations/llms/ncompass.mdx b/integrations/llms/ncompass.mdx index 5dbc5800..75aaca83 100644 --- a/integrations/llms/ncompass.mdx +++ b/integrations/llms/ncompass.mdx @@ -1,118 +1,149 @@ --- title: "Ncompass" +description: Use Ncompass's AI models through Portkey for Snowflake-integrated deployments. --- -Portkey provides a robust and secure gateway to facilitate the integration of various Large Language Models (LLMs) into your applications, including [nCompass APIs](https://www.ncompass.tech/). +## Quick Start -With Portkey, you can take advantage of features like fast AI gateway access, observability, prompt management, and more, all while ensuring the secure management of your LLM API keys through a [virtual key](/product/ai-gateway/virtual-keys) system. - -Provider Slug. `ncompass` - -## Portkey SDK Integration with nCompass Models +Get started with Ncompass in under 2 minutes: -Portkey provides a consistent API to interact with models from various providers. To integrate nCompass with Portkey: + -### 1\. Install the Portkey SDK +```python Python icon="python" +from portkey_ai import Portkey -Add the Portkey SDK to your application to interact with nCompass AI's API through Portkey's gateway. +# 1. Install: pip install portkey-ai +# 2. Add @ncompass provider in model catalog +# 3. Use it: - - - ```sh - npm install --save portkey-ai - ``` - - - ```sh - pip install portkey-ai - ``` - +portkey = Portkey(api_key="PORTKEY_API_KEY") - - - - - -### 2\. Initialize Portkey with the Virtual Key - -To use nCompass with Portkey, get your API key/JWT Token from the Snowflake Platform, then add it to Portkey to create the virtual key. +response = portkey.chat.completions.create( + model="@ncompass/meta-llama/Llama-3.1-8B-Instruct", + messages=[{"role": "user", "content": "Hello!"}] +) - - +print(response.choices[0].message.content) +``` -```js +```js Javascript icon="square-js" import Portkey from 'portkey-ai' +// 1. Install: npm install portkey-ai +// 2. Add @ncompass provider in model catalog +// 3. Use it: + const portkey = new Portkey({ - apiKey: "PORTKEY_API_KEY", // defaults to process.env["PORTKEY_API_KEY"] - provider:"@PROVIDER" // Your nCompass Virtual Key + apiKey: "PORTKEY_API_KEY" }) -``` - - - ```python - from portkey_ai import Portkey - portkey = Portkey( - api_key="PORTKEY_API_KEY", # Replace with your Portkey API key - provider="@PROVIDER" # Replace with your virtual key for nCompass - ) - ``` +const response = await portkey.chat.completions.create({ + model: "@ncompass/meta-llama/Llama-3.1-8B-Instruct", + messages: [{ role: "user", content: "Hello!" }] +}) - +console.log(response.choices[0].message.content) +``` - +```python OpenAI Py icon="python" +from openai import OpenAI +from portkey_ai import PORTKEY_GATEWAY_URL +# 1. Install: pip install openai portkey-ai +# 2. Add @ncompass provider in model catalog +# 3. Use it: +client = OpenAI( + api_key="PORTKEY_API_KEY", # Portkey API key + base_url=PORTKEY_GATEWAY_URL +) -### **3\. Invoke Chat Completions with** nCompass +response = client.chat.completions.create( + model="@ncompass/meta-llama/Llama-3.1-8B-Instruct", + messages=[{"role": "user", "content": "Hello!"}] +) -Use the Portkey instance to send requests to nCompass. You can also override the virtual key directly in the API call if needed. +print(response.choices[0].message.content) +``` - - +```js OpenAI JS icon="square-js" +import OpenAI from "openai" +import { PORTKEY_GATEWAY_URL } from "portkey-ai" - ```js - const chatCompletion = await portkey.chat.completions.create({ - messages: [{ role: 'user', content: 'Say this is a test' }], - model: 'meta-llama/Llama-3.1-8B-Instruct', - }); +// 1. Install: npm install openai portkey-ai +// 2. Add @ncompass provider in model catalog +// 3. Use it: - console.log(chatCompletion.choices); - ``` - - +const client = new OpenAI({ + apiKey: "PORTKEY_API_KEY", // Portkey API key + baseURL: PORTKEY_GATEWAY_URL +}) -```python -completion = portkey.chat.completions.create( - messages= [{ "role": 'user', "content": 'Say this is a test' }], - model= 'meta-llama/Llama-3.1-8B-Instruct' -) +const response = await client.chat.completions.create({ + model: "@ncompass/meta-llama/Llama-3.1-8B-Instruct", + messages: [{ role: "user", content: "Hello!" }] +}) -print(completion) +console.log(response.choices[0].message.content) ``` - - - +```sh cURL icon="square-terminal" +# 1. Add @ncompass provider in model catalog +# 2. Use it: + +curl https://api.portkey.ai/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "x-portkey-api-key: $PORTKEY_API_KEY" \ + -d '{ + "model": "@ncompass/meta-llama/Llama-3.1-8B-Instruct", + "messages": [{"role": "user", "content": "Hello!"}] + }' +``` + -## Managing nCompass Prompts +## Add Provider in Model Catalog -You can manage all prompts to nCompass in the [Prompt Library](/product/prompt-library). All the current models of nCompass are supported and you can easily start testing different prompts. +Before making requests, add Ncompass to your Model Catalog: -Once you're ready with your prompt, you can use the `portkey.prompts.completions.create` interface to use the prompt in your application. +1. Go to [**Model Catalog → Add Provider**](https://app.portkey.ai/model-catalog/providers) +2. Select **Ncompass** +3. Enter your Ncompass API key/JWT Token from Snowflake +4. Name your provider (e.g., `ncompass`) + + See all setup options and detailed configuration instructions + +--- +## Supported Models +Ncompass provides Snowflake-integrated AI models: +Check [Ncompass' documentation](https://www.ncompass.tech/) for the complete model list. --- -You'll find more information in the relevant sections: - -1. [Add metadata to your requests](/product/observability/metadata) -2. [Add gateway configs to your nCompass](/product/ai-gateway/configs)[ requests](/product/ai-gateway/configs) -3. [Tracing nCompass requests](/product/observability/traces) -4. [Setup a fallback from OpenAI to nCompass APIs](/product/ai-gateway/fallbacks) +## Next Steps + + + + Add fallbacks, load balancing, and more + + + Monitor and trace your Ncompass requests + + + Manage and version your prompts + + + Add custom metadata to requests + + + +For complete SDK documentation: + + + Complete Portkey SDK documentation + diff --git a/integrations/llms/nebius.mdx b/integrations/llms/nebius.mdx index 686687f5..24330eb1 100644 --- a/integrations/llms/nebius.mdx +++ b/integrations/llms/nebius.mdx @@ -1,111 +1,149 @@ --- -title: 'Nebius' +title: "Nebius" +description: Use Nebius AI's inference platform through Portkey for scalable model deployment. --- -Portkey provides a robust and secure gateway to facilitate the integration of various Large Language Models (LLMs) into your applications, including [Nebius AI](https://nebius.ai/). -With Portkey, you can take advantage of features like fast AI gateway access, observability, prompt management, and more, all while ensuring the secure management of your LLM API keys through a [virtual key](/product/ai-gateway/virtual-keys) system. +## Quick Start - -Provider Slug. `nebius` - -## Portkey SDK Integration with Nebius AI Models +Get started with Nebius in under 2 minutes: -Portkey provides a consistent API to interact with models from various providers. To integrate Nebius AI with Portkey: + -### 1\. Install the Portkey SDK +```python Python icon="python" +from portkey_ai import Portkey -Add the Portkey SDK to your application to interact with Nebius AI's API through Portkey's gateway. - - - ```sh - npm install --save portkey-ai - ``` - - - ```sh - pip install portkey-ai - ``` - +# 1. Install: pip install portkey-ai +# 2. Add @nebius provider in model catalog +# 3. Use it: - +portkey = Portkey(api_key="PORTKEY_API_KEY") +response = portkey.chat.completions.create( + model="@nebius/deepseek-ai/DeepSeek-V3", + messages=[{"role": "user", "content": "Hello!"}] +) +print(response.choices[0].message.content) +``` +```js Javascript icon="square-js" +import Portkey from 'portkey-ai' -### 2\. Initialize Portkey with the Virtual Key +// 1. Install: npm install portkey-ai +// 2. Add @nebius provider in model catalog +// 3. Use it: -To use Nebius AI with Portkey, [get your API key from here](https://studio.nebius.com/settings/api-keys), then add it to Portkey to create the virtual key. - - - ```js - import Portkey from 'portkey-ai' +const portkey = new Portkey({ + apiKey: "PORTKEY_API_KEY" +}) - const portkey = new Portkey({ - apiKey: "PORTKEY_API_KEY", // defaults to process.env["PORTKEY_API_KEY"] - provider:"@PROVIDER" // Your Nebius Virtual Key - }) - ``` - - - ```python - from portkey_ai import Portkey +const response = await portkey.chat.completions.create({ + model: "@nebius/deepseek-ai/DeepSeek-V3", + messages: [{ role: "user", content: "Hello!" }] +}) - portkey = Portkey( - api_key="PORTKEY_API_KEY", # Replace with your Portkey API key - provider="@PROVIDER" # Replace with your virtual key for Nebius - ) - ``` - +console.log(response.choices[0].message.content) +``` - +```python OpenAI Py icon="python" +from openai import OpenAI +from portkey_ai import PORTKEY_GATEWAY_URL +# 1. Install: pip install openai portkey-ai +# 2. Add @nebius provider in model catalog +# 3. Use it: +client = OpenAI( + api_key="PORTKEY_API_KEY", # Portkey API key + base_url=PORTKEY_GATEWAY_URL +) -### 3\. Invoke Chat Completions with Nebius AI +response = client.chat.completions.create( + model="@nebius/deepseek-ai/DeepSeek-V3", + messages=[{"role": "user", "content": "Hello!"}] +) -Use the Portkey instance to send requests to Nebius AI. You can also override the virtual key directly in the API call if needed. - - - ```js - const chatCompletion = await portkey.chat.completions.create({ - messages: [{ role: 'user', content: 'Say this is a test' }], - model: 'deepseek-ai/DeepSeek-V3', - }); +print(response.choices[0].message.content) +``` - console.log(chatCompletion.choices);d - ``` - - - ```python - completion = portkey.chat.completions.create( - messages= [{ "role": 'user', "content": 'Say this is a test' }], - model= 'deepseek-ai/DeepSeek-V3' - ) +```js OpenAI JS icon="square-js" +import OpenAI from "openai" +import { PORTKEY_GATEWAY_URL } from "portkey-ai" - print(completion) - ``` - +// 1. Install: npm install openai portkey-ai +// 2. Add @nebius provider in model catalog +// 3. Use it: - +const client = new OpenAI({ + apiKey: "PORTKEY_API_KEY", // Portkey API key + baseURL: PORTKEY_GATEWAY_URL +}) +const response = await client.chat.completions.create({ + model: "@nebius/deepseek-ai/DeepSeek-V3", + messages: [{ role: "user", content: "Hello!" }] +}) +console.log(response.choices[0].message.content) +``` +```sh cURL icon="square-terminal" +# 1. Add @nebius provider in model catalog +# 2. Use it: -## Managing Nebius AI Prompts +curl https://api.portkey.ai/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "x-portkey-api-key: $PORTKEY_API_KEY" \ + -d '{ + "model": "@nebius/deepseek-ai/DeepSeek-V3", + "messages": [{"role": "user", "content": "Hello!"}] + }' +``` -You can manage all prompts to Nebius AI in the [Prompt Studio](/product/prompt-library). All the current models of Nebius AI are supported and you can easily start testing different prompts. + -Once you're ready with your prompt, you can use the `portkey.prompts.completions.create` interface to use the prompt in your application. +## Add Provider in Model Catalog -## Supported Models +Before making requests, add Nebius to your Model Catalog: + +1. Go to [**Model Catalog → Add Provider**](https://app.portkey.ai/model-catalog/providers) +2. Select **Nebius** +3. Enter your [Nebius API key](https://studio.nebius.com/settings/api-keys) +4. Name your provider (e.g., `nebius`) -The complete list of features supported in the SDK are available on the link below. - + + See all setup options and detailed configuration instructions -You'll find more information in the relevant sections: +--- + +## Supported Models + +Nebius provides scalable AI inference: + +Check [Nebius' documentation](https://nebius.ai/) for the complete model list. -1. [Add metadata to your requests](/product/observability/metadata) -2. [Add gateway configs to your Nebius](/product/ai-gateway/configs) -3. [Tracing Nebius requests](/product/observability/traces) -4. [Setup a fallback from OpenAI to Nebius APIs](/product/ai-gateway/fallbacks) +--- + +## Next Steps + + + + Add fallbacks, load balancing, and more + + + Monitor and trace your Nebius requests + + + Manage and version your prompts + + + Add custom metadata to requests + + + +For complete SDK documentation: + + + Complete Portkey SDK documentation + diff --git a/integrations/llms/nomic.mdx b/integrations/llms/nomic.mdx index ff14d71c..da200439 100644 --- a/integrations/llms/nomic.mdx +++ b/integrations/llms/nomic.mdx @@ -1,95 +1,152 @@ --- title: "Nomic" +description: Use Nomic's superior embedding models through Portkey. --- -Portkey provides a robust and secure gateway to facilitate the integration of various Large Language Models (LLMs) into your applications, including [Nomic](https://docs.nomic.ai/reference/getting-started/). +## Quick Start -Nomic has especially become popular due to it's superior embeddings and is now available through Portkey's AI gateway as well. +Get started with Nomic in under 2 minutes: -With Portkey, you can take advantage of features like fast AI gateway access, observability, prompt management, and more, all while ensuring the secure management of your LLM API keys through a [virtual key](/product/ai-gateway/virtual-keys) system. - -Provider Slug. `nomic` - -## Portkey SDK Integration with Nomic + -Portkey provides a consistent API to interact with embedding models from various providers. To integrate Nomic with Portkey: +```python Python icon="python" +from portkey_ai import Portkey -### 1\. Integrate Nomic in your Portkey account +# 1. Install: pip install portkey-ai +# 2. Add @nomic provider in model catalog +# 3. Use it: -You can head over to the Integrations tab and connect Nomic with API key. This will be then used to make API requests to Nomic without needing the protected API key. [Grab your Nomic API key from here](https://atlas.nomic.ai/). - -![Logo](/images/llms/nomic.png) - +portkey = Portkey(api_key="PORTKEY_API_KEY") +response = portkey.chat.completions.create( + model="@nomic/nomic-embed-text-v1.5", + messages=[{"role": "user", "content": "Hello!"}] +) -### 2\. Install the Portkey SDK and Initialize with Nomic +print(response.choices[0].message.content) +``` -Add the Portkey SDK to your application to interact with Nomic's API through Portkey's gateway. - - - ```js - import Portkey from 'portkey-ai' +```js Javascript icon="square-js" +import Portkey from 'portkey-ai' - const portkey = new Portkey({ - apiKey: "PORTKEY_API_KEY", // defaults to process.env["PORTKEY_API_KEY"] - provider:"@nomic" // Your Nomic provider slug from Portkey - }) - ``` - - - ```python - from portkey_ai import Portkey +// 1. Install: npm install portkey-ai +// 2. Add @nomic provider in model catalog +// 3. Use it: - portkey = Portkey( - api_key="PORTKEY_API_KEY", # Replace with your Portkey API key - provider="@nomic" # Your Nomic provider slug from Portkey - ) - ``` - +const portkey = new Portkey({ + apiKey: "PORTKEY_API_KEY" +}) - +const response = await portkey.chat.completions.create({ + model: "@nomic/nomic-embed-text-v1.5", + messages: [{ role: "user", content: "Hello!" }] +}) +console.log(response.choices[0].message.content) +``` +```python OpenAI Py icon="python" +from openai import OpenAI +from portkey_ai import PORTKEY_GATEWAY_URL -### 3\. Invoke the Embeddings API with Nomic +# 1. Install: pip install openai portkey-ai +# 2. Add @nomic provider in model catalog +# 3. Use it: -Use the Portkey instance to send requests to your Nomic API. You can also override the virtual key directly in the API call if needed. - - - ```js - const embeddings = await portkey.embeddings.create({ - input: "create vector representation on this sentence", - model: "nomic-embed-text-v1.5", - }); +client = OpenAI( + api_key="PORTKEY_API_KEY", # Portkey API key + base_url=PORTKEY_GATEWAY_URL +) - console.log(embeddings); - ``` - - - ```python - embeddings = portkey.embeddings.create( - input='create vector representation on this sentence', - model='nomic-embed-text-v1.5' - ) +response = client.chat.completions.create( + model="@nomic/nomic-embed-text-v1.5", + messages=[{"role": "user", "content": "Hello!"}] +) - print(embeddings) - ``` +print(response.choices[0].message.content) +``` - +```js OpenAI JS icon="square-js" +import OpenAI from "openai" +import { PORTKEY_GATEWAY_URL } from "portkey-ai" - +// 1. Install: npm install openai portkey-ai +// 2. Add @nomic provider in model catalog +// 3. Use it: +const client = new OpenAI({ + apiKey: "PORTKEY_API_KEY", // Portkey API key + baseURL: PORTKEY_GATEWAY_URL +}) -## Next Steps +const response = await client.chat.completions.create({ + model: "@nomic/nomic-embed-text-v1.5", + messages: [{ role: "user", content: "Hello!" }] +}) + +console.log(response.choices[0].message.content) +``` + +```bash cURL +curl https://api.portkey.ai/v1/embeddings \ + -H "Content-Type: application/json" \ + -H "x-portkey-api-key: $PORTKEY_API_KEY" \ + -H "x-portkey-provider: @nomic" \ + -d '{ + "model": "nomic-embed-text-v1.5", + "input": "create vector representation on this sentence" + }' +``` + + -The complete list of features supported in the SDK are available on the link below. +## Add Provider in Model Catalog - +Before making requests, add Nomic to your Model Catalog: + +1. Go to [**Model Catalog → Add Provider**](https://app.portkey.ai/model-catalog/providers) +2. Select **Nomic** +3. Enter your [Nomic API key](https://atlas.nomic.ai/) +4. Name your provider (e.g., `nomic`) + + + See all setup options and detailed configuration instructions -You'll find more information in the relevant sections: +--- + +## Supported Models -1. [API Reference for Embeddings](/provider-endpoints/embeddings) -2. [Add metadata to your requests](/product/observability/metadata) -3. [Add gateway configs to your Nomic requests](/product/ai-gateway/configs) -4. [Tracing Nomic requests](/product/observability/traces) +Nomic is known for their high-quality embedding models: + +| Model | Context Length | Description | +|-------|---------------|-------------| +| nomic-embed-text-v1.5 | 8,192 tokens | Latest high-performance embedding model | +| nomic-embed-text-v1 | 8,192 tokens | Previous generation embedding model | + +Check [Nomic's documentation](https://docs.nomic.ai/) for more details. + +--- + +## Next Steps + + + + Add fallbacks, load balancing, and more + + + Monitor and trace your Nomic requests + + + Cache embeddings for faster responses + + + Add custom metadata to requests + + + +For complete SDK documentation: + + + Complete Portkey SDK documentation + diff --git a/integrations/llms/novita-ai.mdx b/integrations/llms/novita-ai.mdx index 86a5e29a..2c568c0b 100644 --- a/integrations/llms/novita-ai.mdx +++ b/integrations/llms/novita-ai.mdx @@ -1,112 +1,149 @@ --- title: "Novita AI" +description: Use Novita AI's inference platform through Portkey for diverse model access. --- -Portkey provides a robust and secure gateway to facilitate the integration of various Large Language Models (LLMs) into your applications, including [Novita AI](https://novita.ai/). +## Quick Start -With Portkey, you can take advantage of features like fast AI gateway access, observability, prompt management, and more, all while ensuring the secure management of your LLM API keys through a [virtual key](/product/ai-gateway/virtual-keys) system. +Get started with Novita AI in under 2 minutes: - -Provider Slug. `novita-ai` - -## Portkey SDK Integration with Novita AI Models + -Portkey provides a consistent API to interact with models from various providers. To integrate Novita AI with Portkey: +```python Python icon="python" +from portkey_ai import Portkey -### 1\. Install the Portkey SDK +# 1. Install: pip install portkey-ai +# 2. Add @novita-ai provider in model catalog +# 3. Use it: -Add the Portkey SDK to your application to interact with Novita AI's API through Portkey's gateway. - - - ```sh - npm install --save portkey-ai - ``` - - - ```sh - pip install portkey-ai - ``` - +portkey = Portkey(api_key="PORTKEY_API_KEY") - +response = portkey.chat.completions.create( + model="@novita-ai/Nous-Hermes-2-Mixtral-8x7B-DPO", + messages=[{"role": "user", "content": "Hello!"}] +) +print(response.choices[0].message.content) +``` +```js Javascript icon="square-js" +import Portkey from 'portkey-ai' +// 1. Install: npm install portkey-ai +// 2. Add @novita-ai provider in model catalog +// 3. Use it: -### 2\. Initialize Portkey with the Virtual Key +const portkey = new Portkey({ + apiKey: "PORTKEY_API_KEY" +}) -To use Novita AI with Portkey, [get your API key from here](https://novita.ai/settings), then add it to Portkey to create the virtual key. - - - ```js - import Portkey from 'portkey-ai' +const response = await portkey.chat.completions.create({ + model: "@novita-ai/Nous-Hermes-2-Mixtral-8x7B-DPO", + messages: [{ role: "user", content: "Hello!" }] +}) - const portkey = new Portkey({ - apiKey: "PORTKEY_API_KEY", // defaults to process.env["PORTKEY_API_KEY"] - provider:"@PROVIDER" // Your Novita Virtual Key - }) - ``` - - - ```python - from portkey_ai import Portkey +console.log(response.choices[0].message.content) +``` - portkey = Portkey( - api_key="PORTKEY_API_KEY", # Replace with your Portkey API key - provider="@PROVIDER" # Replace with your virtual key for Groq - ) - ``` - +```python OpenAI Py icon="python" +from openai import OpenAI +from portkey_ai import PORTKEY_GATEWAY_URL - +# 1. Install: pip install openai portkey-ai +# 2. Add @novita-ai provider in model catalog +# 3. Use it: +client = OpenAI( + api_key="PORTKEY_API_KEY", # Portkey API key + base_url=PORTKEY_GATEWAY_URL +) +response = client.chat.completions.create( + model="@novita-ai/Nous-Hermes-2-Mixtral-8x7B-DPO", + messages=[{"role": "user", "content": "Hello!"}] +) -### 3\. Invoke Chat Completions with Novita AI +print(response.choices[0].message.content) +``` -Use the Portkey instance to send requests to Novita AI. You can also override the virtual key directly in the API call if needed. - - - ```js - const chatCompletion = await portkey.chat.completions.create({ - messages: [{ role: 'user', content: 'Say this is a test' }], - model: 'Nous-Hermes-2-Mixtral-8x7B-DPO', - }); +```js OpenAI JS icon="square-js" +import OpenAI from "openai" +import { PORTKEY_GATEWAY_URL } from "portkey-ai" - console.log(chatCompletion.choices);d - ``` - - - ```python - completion = portkey.chat.completions.create( - messages= [{ "role": 'user', "content": 'Say this is a test' }], - model= 'reka-core' - ) +// 1. Install: npm install openai portkey-ai +// 2. Add @novita-ai provider in model catalog +// 3. Use it: - print(completion) - ``` - +const client = new OpenAI({ + apiKey: "PORTKEY_API_KEY", // Portkey API key + baseURL: PORTKEY_GATEWAY_URL +}) - +const response = await client.chat.completions.create({ + model: "@novita-ai/Nous-Hermes-2-Mixtral-8x7B-DPO", + messages: [{ role: "user", content: "Hello!" }] +}) +console.log(response.choices[0].message.content) +``` +```sh cURL icon="square-terminal" +# 1. Add @novita-ai provider in model catalog +# 2. Use it: +curl https://api.portkey.ai/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "x-portkey-api-key: $PORTKEY_API_KEY" \ + -d '{ + "model": "@novita-ai/Nous-Hermes-2-Mixtral-8x7B-DPO", + "messages": [{"role": "user", "content": "Hello!"}] + }' +``` -## Managing Novita AI Prompts + -You can manage all prompts to Novita AI in the [Prompt Library](/product/prompt-library). All the current models of Novita AI are supported and you can easily start testing different prompts. +## Add Provider in Model Catalog -Once you're ready with your prompt, you can use the `portkey.prompts.completions.create` interface to use the prompt in your application. +Before making requests, add Novita AI to your Model Catalog: -## Supported Models +1. Go to [**Model Catalog → Add Provider**](https://app.portkey.ai/model-catalog/providers) +2. Select **Novita AI** +3. Enter your [Novita AI API key](https://novita.ai/settings) +4. Name your provider (e.g., `novita-ai`) -The complete list of features supported in the SDK are available on the link below. - + + See all setup options and detailed configuration instructions -You'll find more information in the relevant sections: +--- + +## Supported Models + +Novita AI provides access to diverse models: -1. [Add metadata to your requests](/product/observability/metadata) -2. A[dd gateway configs to your Novita](/product/ai-gateway/configs) -3. [Tracing Novita requests](/product/observability/traces) -4. [Setup a fallback from OpenAI to Novita APIs](/product/ai-gateway/fallbacks) +Check [Novita AI's documentation](https://novita.ai/) for the complete model list. + +--- + +## Next Steps + + + + Add fallbacks, load balancing, and more + + + Monitor and trace your Novita AI requests + + + Manage and version your prompts + + + Add custom metadata to requests + + + +For complete SDK documentation: + + + Complete Portkey SDK documentation + diff --git a/integrations/llms/nscale.mdx b/integrations/llms/nscale.mdx index ff6152d0..31652f0e 100644 --- a/integrations/llms/nscale.mdx +++ b/integrations/llms/nscale.mdx @@ -1,129 +1,229 @@ --- title: "Nscale (EU Sovereign)" +description: Use Nscale's EU-based sovereign AI infrastructure through Portkey for compliant model deployment. --- -Portkey provides a robust and secure gateway to facilitate the integration of various Large Language Models (LLMs) into your applications, including the models hosted on [Nscale](https://docs.nscale.com/docs/inference/serverless-models/current). - - -Provider Slug. `nscale` - - -## Portkey SDK Integration with Nscale - -Portkey provides a consistent API to interact with models from various providers. To integrate Nscale with Portkey: - -### 1. Install the Portkey SDK - - - ```sh - npm install --save portkey-ai - ``` - - - ```sh - pip install portkey-ai - ``` - - - -### 2. Initialize Portkey with the Virtual Key -To use Nscale with Virtual Key, [get your API key from here](https://console.nscale.com). Then add it to Portkey to create the virtual key. - - - ```js - import Portkey from 'portkey-ai' - - const portkey = new Portkey({ - apiKey: "PORTKEY_API_KEY", // defaults to process.env["PORTKEY_API_KEY"] - provider:"@PROVIDER" // Your Nscale Virtual Key - }) - ``` - - - ```python - from portkey_ai import Portkey - - portkey = Portkey( - api_key="PORTKEY_API_KEY", # Replace with your Portkey API key - provider="@NSCALE_PROVIDER" - ) - ``` - - - -### 3. Invoke Chat Completions - - - ```js - const chatCompletion = await portkey.chat.completions.create({ - messages: [{ role: 'user', content: 'Say this is a test' }], - model: 'meta-llama/Llama-4-Scout-17B-16E-Instruct', - }); - - console.log(chatCompletion.choices); - ``` - - - ```python - completion = portkey.chat.completions.create( - messages= [{ "role": 'user', "content": 'Say this is a test' }], - model= 'meta-llama/Llama-4-Scout-17B-16E-Instruct' - ) - - print(completion) - ``` - - - -### 4. Invoke Image Generation - - - ```js - const response = await portkey.images.generations.create({ - prompt: "A beautiful sunset over mountains", - model: "stabilityai/stable-diffusion-xl-base-1.0", - n: 1, - size: "1024x1024" - }); - - console.log(response.data[0].url); - ``` - - - ```python - response = portkey.images.generate( - prompt="A beautiful sunset over mountains", - model="stabilityai/stable-diffusion-xl-base-1.0", - n=1, - size="1024x1024" - ) - - print(response.data[0].url) - ``` - - +## Quick Start + +Get started with Nscale in under 2 minutes: + + + +```python Python icon="python" +from portkey_ai import Portkey + +# 1. Install: pip install portkey-ai +# 2. Add @nscale provider in model catalog +# 3. Use it: + +portkey = Portkey(api_key="PORTKEY_API_KEY") + +response = portkey.chat.completions.create( + model="@nscale/meta-llama/Llama-4-Scout-17B-16E-Instruct", + messages=[{"role": "user", "content": "Hello!"}] +) + +print(response.choices[0].message.content) +``` + +```js Javascript icon="square-js" +import Portkey from 'portkey-ai' + +// 1. Install: npm install portkey-ai +// 2. Add @nscale provider in model catalog +// 3. Use it: + +const portkey = new Portkey({ + apiKey: "PORTKEY_API_KEY" +}) + +const response = await portkey.chat.completions.create({ + model: "@nscale/meta-llama/Llama-4-Scout-17B-16E-Instruct", + messages: [{ role: "user", content: "Hello!" }] +}) + +console.log(response.choices[0].message.content) +``` + +```python OpenAI Py icon="python" +from openai import OpenAI +from portkey_ai import PORTKEY_GATEWAY_URL + +# 1. Install: pip install openai portkey-ai +# 2. Add @nscale provider in model catalog +# 3. Use it: + +client = OpenAI( + api_key="PORTKEY_API_KEY", # Portkey API key + base_url=PORTKEY_GATEWAY_URL +) + +response = client.chat.completions.create( + model="@nscale/meta-llama/Llama-4-Scout-17B-16E-Instruct", + messages=[{"role": "user", "content": "Hello!"}] +) + +print(response.choices[0].message.content) +``` + +```js OpenAI JS icon="square-js" +import OpenAI from "openai" +import { PORTKEY_GATEWAY_URL } from "portkey-ai" + +// 1. Install: npm install openai portkey-ai +// 2. Add @nscale provider in model catalog +// 3. Use it: + +const client = new OpenAI({ + apiKey: "PORTKEY_API_KEY", // Portkey API key + baseURL: PORTKEY_GATEWAY_URL +}) + +const response = await client.chat.completions.create({ + model: "@nscale/meta-llama/Llama-4-Scout-17B-16E-Instruct", + messages: [{ role: "user", content: "Hello!" }] +}) + +console.log(response.choices[0].message.content) +``` + +```sh cURL icon="square-terminal" +# 1. Add @nscale provider in model catalog +# 2. Use it: + +curl https://api.portkey.ai/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "x-portkey-api-key: $PORTKEY_API_KEY" \ + -d '{ + "model": "@nscale/meta-llama/Llama-4-Scout-17B-16E-Instruct", + "messages": [{"role": "user", "content": "Hello!"}] + }' +``` + + + +## Add Provider in Model Catalog + +Before making requests, add Nscale to your Model Catalog: + +1. Go to [**Model Catalog → Add Provider**](https://app.portkey.ai/model-catalog/providers) +2. Select **Nscale** +3. Enter your [Nscale API key](https://console.nscale.com) +4. Name your provider (e.g., `nscale`) + + + See all setup options and detailed configuration instructions + + +--- + +## Nscale Capabilities + +### Chat Completions + +Generate chat completions with EU-sovereign infrastructure: + + + +```python Python +from portkey_ai import Portkey + +portkey = Portkey(api_key="PORTKEY_API_KEY", provider="@nscale") + +response = portkey.chat.completions.create( + model="meta-llama/Llama-4-Scout-17B-16E-Instruct", + messages=[{"role": "user", "content": "Hello!"}] +) + +print(response.choices[0].message.content) +``` + +```javascript Node.js +import Portkey from 'portkey-ai'; + +const portkey = new Portkey({ + apiKey: 'PORTKEY_API_KEY', + provider: '@nscale' +}); + +const response = await portkey.chat.completions.create({ + model: "meta-llama/Llama-4-Scout-17B-16E-Instruct", + messages: [{ role: "user", content: "Hello!" }] +}); + +console.log(response.choices[0].message.content); +``` + + + +### Image Generation + +Generate images with Stable Diffusion on EU infrastructure: + + + +```python Python +from portkey_ai import Portkey + +portkey = Portkey(api_key="PORTKEY_API_KEY", provider="@nscale") + +image = portkey.images.generate( + model="stabilityai/stable-diffusion-xl-base-1.0", + prompt="A beautiful sunset over mountains", + size="1024x1024" +) + +print(image.data[0].url) +``` + +```javascript Node.js +import Portkey from 'portkey-ai'; + +const portkey = new Portkey({ + apiKey: 'PORTKEY_API_KEY', + provider: '@nscale' +}); + +const image = await portkey.images.generate({ + model: "stabilityai/stable-diffusion-xl-base-1.0", + prompt: "A beautiful sunset over mountains", + size: "1024x1024" +}); + +console.log(image.data[0].url); +``` + + --- ## Supported Models - - - Explore the complete list of available models on Nscale's documentation, including chat models, image generation models, and their pricing details. - - +Nscale provides EU-sovereign AI infrastructure for various models: + +Check [Nscale's documentation](https://docs.nscale.com/) for the complete model list. --- ## Next Steps -The complete list of features supported in the SDK are available on the link below. - - + + + Add fallbacks, load balancing, and more + + + Monitor and trace your Nscale requests + + + Manage and version your prompts + + + Add custom metadata to requests + + -You'll find more information in the relevant sections: +For complete SDK documentation: -1. [Add metadata to your requests](/product/observability/metadata) -2. [Add gateway configs to your Nscale requests](/product/ai-gateway/configs) -3. [Tracing Nscale requests](/product/observability/traces) -4. [Setup a fallback from OpenAI to Nscale](/product/ai-gateway/fallbacks) \ No newline at end of file + + Complete Portkey SDK documentation + diff --git a/integrations/llms/ollama.mdx b/integrations/llms/ollama.mdx index daab3064..f9117eaf 100644 --- a/integrations/llms/ollama.mdx +++ b/integrations/llms/ollama.mdx @@ -1,470 +1,178 @@ --- title: "Ollama" +description: "Integrate Ollama-hosted models with Portkey for local LLM deployment with full observability." --- -Portkey provides a robust and secure gateway to facilitate the integration of various Large Language Models (LLMs) into your applications, including your **locally hosted models through Ollama**. - -Provider Slug. `ollama` - -## Portkey SDK Integration with Ollama Models - -Portkey provides a consistent API to interact with models from various providers. +Portkey provides a robust gateway to facilitate the integration of your **locally hosted models through Ollama**. - -If you are running the open source Portkey Gateway, refer to this guide on how to connect Portkey with Ollama. - + + -### 1\. Expose your Ollama API - -Expose your Ollama API by using a tunneling service like [ngrok](https://ngrok.com/) or any other way you prefer. - -You can skip this step if you're self-hosting the Gateway. +First, install the Gateway locally: -For using Ollama with ngrok, here's a [useful guide](https://github.com/ollama/ollama/blob/main/docs/faq.md#how-can-i-use-ollama-with-ngrok) -```sh -ngrok http 11434 --host-header="localhost:11434" + +```bash npx +npx @portkey-ai/gateway ``` -### 2\. Install the Portkey SDK - -Install the Portkey SDK in your application to interact with your Ollama API through Portkey. - - - -```sh -npm install --save portkey-ai +```bash Docker +docker run -d -p 8787:8787 portkeyai/gateway:latest ``` - - - ```sh - pip install portkey-ai - ``` - - - - - + -### 3\. Initialize Portkey with Ollama URL +Then, connect to your local Ollama instance: -Instantiate the Portkey client by adding your Ollama publicly-exposed URL to the `customHost` property. - - - ```js - import Portkey from 'portkey-ai' +```python Python +from portkey_ai import Portkey - const portkey = new Portkey({ - apiKey: "PORTKEY_API_KEY", // defaults to process.env["PORTKEY_API_KEY"] - provider: "ollama", - customHost: "https://7cc4-3-235-157-146.ngrok-free.app" // Your Ollama ngrok URL - }) - ``` - - - ```python - from portkey_ai import Portkey +portkey = Portkey( + base_url="http://localhost:8787", # Your local Gateway + provider="ollama", + custom_host="http://localhost:11434" # Your Ollama instance +) - portkey = Portkey( - api_key="PORTKEY_API_KEY", # Replace with your Portkey API key - provider="ollama", - custom_host="https://7cc4-3-235-157-146.ngrok-free.app" # Your Ollama ngrok URL - ) - ``` - +response = portkey.chat.completions.create( + model="llama3", + messages=[{"role": "user", "content": "Hello!"}] +) +``` - + + +## Integration Steps - -For the Ollama integration, you only need to pass the base URL to `customHost` without the version identifier (such as `/v1`) - Portkey takes care of the rest! - -### **4\. Invoke Chat Completions with** Ollama - -Use the Portkey SDK to invoke chat completions from your Ollama model, just as you would with any other provider. - - - ```js - const chatCompletion = await portkey.chat.completions.create({ - messages: [{ role: 'user', content: 'Say this is a test' }], - model: 'llama3', - }); - - console.log(chatCompletion.choices); - ``` - - - - ```python - completion = portkey.chat.completions.create( - messages= [{ "role": 'user', "content": 'Say this is a test' }], - model= 'llama3' - ) - - print(completion) - ``` - - - ```sh - curl --location 'https://api.portkey.ai/v1/chat/completions' \ - --header 'Content-Type: application/json' \ - --header 'x-portkey-custom-host: https://1eb6-103-180-45-236.ngrok-free.app' \ - --header 'x-portkey-provider: ollama' \ - --header 'x-portkey-api-key: PORTKEY_API_KEY' \ - --data '{ - "model": "tinyllama", - "max_tokens": 200, - "stream": false, - "messages": [ - { - "role": "system", - "content": [ - { - "type": "text", - "text": "You are Batman" - } - ] - }, - { - "role": "user", - "content": [ - { - "type": "text", - "text": "Who is the greatest detective" - } - ] - }, - { - "role": "assistant", - "content": [ - { - "type": "text", - "text": "is it me?" - } - ] - } - ] - }' - ``` - - - - - -## [Using Virtual Keys](https://app.portkey.ai/virtual-keys) - -Virtual Keys serve as Portkey's unified authentication system for all LLM interactions, simplifying the use of multiple providers and Portkey features within your application. For self-hosted LLMs, you can configure custom authentication requirements including authorization keys, bearer tokens, or any other headers needed to access your model: - - - - - -1. Navigate to [Virtual Keys](https://app.portkey.ai/virtual-keys) in your Portkey dashboard -2. Click **"Add Key"** and enable the **"Local/Privately hosted provider"** toggle -3. Configure your deployment: - - Select the matching provider API specification (typically `OpenAI`) - - Enter your model's base URL in the `Custom Host` field - - Add required authentication headers and their values -4. Click **"Create"** to generate your virtual key - -You can now use this virtual key in your requests: - - - - ```js - const portkey = new Portkey({ - apiKey: "PORTKEY_API_KEY", - provider:"@YOUR_SELF_HOSTED_LLM_PROVIDER" - - async function main() { - const response = await client.chat.completions.create({ - messages: [{ role: "user", content: "Bob the builder.." }], - model: "your-self-hosted-model-name", - }); - - console.log(response.choices[0].message.content); - }) - ``` - - - ```python - portkey = Portkey( - api_key="PORTKEY_API_KEY", - provider="@YOUR_SELF_HOSTED_LLM_PROVIDER" - ) - - response = portkey.chat.completions.create( - model="your-self-hosted-model-name", - messages=[ - {"role": "system", "content": "You are a helpful assistant."}, - {"role": "user", "content": "Hello!"} - ] - - print(response) - ) - ``` - - - -For more information about managing self-hosted LLMs with Portkey, see [Bring Your Own LLM](/integrations/llms/byollm). - - -## Local Setup (npm or docker) + + +Expose your Ollama API using a tunneling service like [ngrok](https://ngrok.com/) or make it publicly accessible. Skip this if you're self-hosting the Gateway. -First, install the Gateway locally: +For using Ollama with ngrok, here's a [useful guide](https://github.com/ollama/ollama/blob/main/docs/faq.md#how-can-i-use-ollama-with-ngrok): - - ```sh -npx @portkey-ai/gateway -``` -| Your Gateway is running on http://localhost:8080/v1 🚀 | | -| - | - | - - -```sh -docker pull portkeyai/gateway +ngrok http 11434 --host-header="localhost:11434" ``` -| Your Gateway is running on http://host.docker.internal:8080/v1 🚀 | | -| - | - | - - + + -Then, just change the `baseURL` to the Gateway URL, `customHost` to the Ollam URL, and make requests. +1. Go to [**Model Catalog → Add Provider**](https://app.portkey.ai/model-catalog/providers) +2. Enable **"Local/Privately hosted provider"** toggle +3. Select **Ollama** as the provider type +4. Enter your Ollama URL in **Custom Host**: `https://your-ollama.ngrok-free.app` +5. Name your provider (e.g., `my-ollama`) - -If you are running Portkey inside a `Docker container`, but Ollama is running natively on your machine (i.e. not in Docker), you will have to refer to Ollama using `http://host.docker.internal:11434` for the Gateway to be able to call it. - - - -```ts NodeJS -import Portkey from 'portkey-ai'; - -const client = new Portkey({ - baseUrl: 'http://localhost:8080/v1', - apiKey: 'PORTKEY_API_KEY', - provider:'@PROVIDER', - customHost: "http://host.docker.internal:11434" // Your Ollama Docker URL -}); + + See all setup options + + -async function main() { - const response = await client.chat.completions.create({ - messages: [{ role: "user", content: "Bob the builder.." }], - model: "gpt-4o", - }); + - console.log(response.choices[0].message.content); -} + -main(); -``` -```py Python +```python Python from portkey_ai import Portkey -client = Portkey( - base_url = 'http://localhost:8080/v1', - api_key = "PORTKEY_API_KEY", - virtual_key = "PROVIDER", - custom_host="http://localhost:11434" # Your Ollama URL +portkey = Portkey( + api_key="PORTKEY_API_KEY", + provider="@my-ollama" ) -response = client.chat.completions.create( - model="gpt-4o", - messages=[ - {"role": "system", "content": "You are a helpful assistant."}, - {"role": "user", "content": "Hello!"} - ] +response = portkey.chat.completions.create( + model="llama3", + messages=[{"role": "user", "content": "Hello!"}] ) -print(response.choices[0].message) -``` -```sh cURL -curl http://localhost:8080/v1/chat/completions \ - -H "Content-Type: application/json" \ - -H "x-portkey-api-key: $PORTKEY_API_KEY" \ - -H "x-portkey-custom-host: http://localhost:11434" \ - -H "x-portkey-provider: $PORTKEY_PROVIDER" \ - -d '{ - "model": "gpt-4o", - "messages": [ - { "role": "user", "content": "Hello!" } - ] - }' +print(response.choices[0].message.content) ``` -```py OpenAI Python SDK -from openai import OpenAI -from portkey_ai import createHeaders, PORTKEY_GATEWAY_URL - -client = OpenAI( - api_key="xx", - base_url="https://localhost:8080/v1", - default_headers=createHeaders( - api_key="PORTKEY_API_KEY", - provider="@OPENAI_PROVIDER", - custom_host="http://localhost:11434" - ) -) -completion = client.chat.completions.create( - model="gpt-4o", - messages=[ - {"role": "system", "content": "You are a helpful assistant."}, - {"role": "user", "content": "Hello!"} - ] -) +```javascript Node.js +import Portkey from 'portkey-ai'; -print(completion.choices[0].message) -``` -```ts OpenAI NodeJS SDK -import OpenAI from 'openai'; -import { PORTKEY_GATEWAY_URL, createHeaders } from 'portkey-ai' - -const openai = new OpenAI({ - apiKey: 'xx', - baseURL: 'https://localhost:8080/v1', - defaultHeaders: createHeaders({ - apiKey: "PORTKEY_API_KEY", - provider:"@OPENAI_PROVIDER", - customHost: "http://localhost:11434" - }) +const portkey = new Portkey({ + apiKey: 'PORTKEY_API_KEY', + provider: '@my-ollama' }); -async function main() { - const completion = await openai.chat.completions.create({ - messages: [{ role: 'user', content: 'Say this is a test' }], - model: 'gpt-4o', - }); - - console.log(chatCompletion.choices); -} +const response = await portkey.chat.completions.create({ + model: 'llama3', + messages: [{ role: 'user', content: 'Hello!' }] +}); -main(); +console.log(response.choices[0].message.content); ``` - + +**Or use custom host directly:** -### Ollama Tool Calling -Tool calling feature lets models trigger external tools based on conversation context. You define available functions, the model chooses when to use them, and your application executes them and returns results. + -Portkey supports Ollama Tool Calling and makes it interoperable across multiple providers. With Portkey Prompts, you can templatize various your prompts & tool schemas as well. +```python Python +from portkey_ai import Portkey - +portkey = Portkey( + api_key="PORTKEY_API_KEY", + provider="ollama", + custom_host="https://your-ollama.ngrok-free.app" +) +``` - +```javascript Node.js +import Portkey from 'portkey-ai'; - - -```javascript Get Weather Tool -let tools = [{ - type: "function", - function: { - name: "getWeather", - description: "Get the current weather", - parameters: { - type: "object", - properties: { - location: { type: "string", description: "City and state" }, - unit: { type: "string", enum: ["celsius", "fahrenheit"] } - }, - required: ["location"] - } - } -}]; - -let response = await portkey.chat.completions.create({ - model: "llama-3.3-70b-versatile", - messages: [ - { role: "system", content: "You are a helpful assistant." }, - { role: "user", content: "What's the weather like in Delhi - respond in JSON" } - ], - tools, - tool_choice: "auto", +const portkey = new Portkey({ + apiKey: 'PORTKEY_API_KEY', + provider: 'ollama', + customHost: 'https://your-ollama.ngrok-free.app' }); - -console.log(response.choices[0].finish_reason); ``` - - -```python Get Weather Tool -tools = [{ - "type": "function", - "function": { - "name": "getWeather", - "description": "Get the current weather", - "parameters": { - "type": "object", - "properties": { - "location": {"type": "string", "description": "City and state"}, - "unit": {"type": "string", "enum": ["celsius", "fahrenheit"]} - }, - "required": ["location"] - } - } -}] -response = portkey.chat.completions.create( - model="llama-3.3-70b-versatile", - messages=[ - {"role": "system", "content": "You are a helpful assistant."}, - {"role": "user", "content": "What's the weather like in Delhi - respond in JSON"} - ], - tools=tools, - tool_choice="auto" -) + -print(response.choices[0].finish_reason) -``` - - -```sh Get Weather Tool -curl -X POST "https://api.portkey.ai/v1/chat/completions" \ - -H "Content-Type: application/json" \ - -H "Authorization: Bearer YOUR_PORTKEY_API_KEY" \ - -d '{ - "model": "llama-3.3-70b-versatile", - "messages": [ - {"role": "system", "content": "You are a helpful assistant."}, - {"role": "user", "content": "What'\''s the weather like in Delhi - respond in JSON"} - ], - "tools": [{ - "type": "function", - "function": { - "name": "getWeather", - "description": "Get the current weather", - "parameters": { - "type": "object", - "properties": { - "location": {"type": "string", "description": "City and state"}, - "unit": {"type": "string", "enum": ["celsius", "fahrenheit"]} - }, - "required": ["location"] - } - } - }], - "tool_choice": "auto" - }' -``` - - - Checkout [Prompt Engineering Studio](/product/prompt-engineering-studio/prompt-playground) - - + + + +**Important:** For Ollama integration, you only need to pass the base URL to `customHost` **without** the version identifier (such as `/v1`) - Portkey handles the rest! + +--- +## Supported Models -## Next Steps +Ollama supports a wide range of models including: -Explore the complete list of features supported in the SDK: - - +- Llama 3, Llama 3.1, Llama 3.2 +- Mistral, Mixtral +- Gemma, Gemma 2 +- Phi-3 +- Qwen 2 +- And many more! + +Check [Ollama's model library](https://ollama.com/library) for the complete list. --- -You'll find more information in the relevant sections: +## Next Steps -1. [Add metadata to your requests](/product/observability/metadata) -2. [Add gateway configs to your Ollama requests](/product/ai-gateway/universal-api#ollama-in-configs) -3. [Tracing Ollama requests](/product/observability/traces) -4. [Setup a fallback from OpenAI to Ollama APIs](/product/ai-gateway/fallbacks) + + + Add retries, timeouts, and fallbacks + + + Monitor your Ollama requests + + + Learn more about custom host setup + + + Complete guide for private LLMs + + + +For complete SDK documentation: + + + Complete Portkey SDK documentation + diff --git a/integrations/llms/openai.mdx b/integrations/llms/openai.mdx index 78b93f31..36036f3a 100644 --- a/integrations/llms/openai.mdx +++ b/integrations/llms/openai.mdx @@ -1,460 +1,307 @@ --- title: "OpenAI" -description: "Learn to integrate OpenAI with Portkey, enabling seamless completions, prompt management, and advanced functionalities like streaming, function calling and fine-tuning." +description: "Integrate OpenAI's GPT models with Portkey's AI Gateway" --- -Portkey has native integrations with OpenAI SDKs for Node.js, Python, and its REST APIs. For OpenAI integration using other frameworks, explore our partnerships, including [Langchain](/integrations/libraries/langchain-python), [LlamaIndex](/integrations/libraries/llama-index-python), among [others](/integrations/llms). - -Provider Slug. `openai` - -## Using the Portkey Gateway +Portkey provides a robust and secure gateway to integrate [OpenAI's APIs](https://platform.openai.com/docs/api-reference/introduction) into your applications, including GPT-4o, o1, DALL·E, Whisper, and more. -To integrate the Portkey gateway with OpenAI, +With Portkey, take advantage of features like fast AI gateway access, observability, prompt management, and more, while securely managing API keys through [Model Catalog](/product/model-catalog). -* Set the `baseURL` to the Portkey Gateway URL -* Include Portkey-specific headers such as `provider`, `apiKey`, 'virtualKey' and others. + + + Full support for GPT-4o, o1, GPT-4, GPT-3.5, and all OpenAI models + + + Chat, completions, embeddings, audio, images, and more fully supported + + + Use with OpenAI SDK, Portkey SDK, or popular frameworks like LangChain + + -Here's how to apply it to a **chat completion** request: +## Quick Start +Get OpenAI working in 3 steps: - - -Install the Portkey SDK with npm -```sh -npm install portkey-ai -``` -```ts Chat Completions -import Portkey from 'portkey-ai'; +```python Python icon="python" +from portkey_ai import Portkey -const client = new Portkey({ - apiKey: 'PORTKEY_API_KEY', - provider:'@PROVIDER' -}); +# 1. Install: pip install portkey-ai +# 2. Add @openai provider in model catalog +# 3. Use it: -async function main() { - const response = await client.chat.completions.create({ - messages: [{ role: "user", content: "Bob the builder.." }], - model: "gpt-4o", - }); +portkey = Portkey(api_key="PORTKEY_API_KEY") - console.log(response.choices[0].message.content); -} +response = portkey.chat.completions.create( + model="@openai/gpt-4o", + messages=[{"role": "user", "content": "Say this is a test"}] +) -main(); +print(response.choices[0].message.content) ``` - - - -Install the Portkey SDK with pip -```sh -pip install portkey-ai -``` - -```py Chat Completions -from portkey_ai import Portkey +```js Javascript icon="square-js" +import Portkey from 'portkey-ai' -client = Portkey( - api_key = "PORTKEY_API_KEY", - virtual_key = "PROVIDER" -) +// 1. Install: npm install portkey-ai +// 2. Add @openai provider in model catalog +// 3. Use it: -response = client.chat.completions.create( - model="gpt-4o", - messages=[ - {"role": "system", "content": "You are a helpful assistant."}, - {"role": "user", "content": "Hello!"} - ] -) +const portkey = new Portkey({ + apiKey: "PORTKEY_API_KEY" +}) -print(response.choices[0].message) -``` +const response = await portkey.chat.completions.create({ + model: "@openai/gpt-4o", + messages: [{ role: "user", content: "Say this is a test" }] +}) - - - - -```sh Chat Completions -curl https://api.portkey.ai/v1/chat/completions \ - -H "Content-Type: application/json" \ - -H "x-portkey-api-key: $PORTKEY_API_KEY" \ - -H "x-portkey-provider: $PORTKEY_PROVIDER" \ - -d '{ - "model": "gpt-4o", - "messages": [ - { "role": "user", "content": "Hello!" } - ] - }' +console.log(response.choices[0].message.content) ``` - - - -Install the OpenAI & Portkey SDKs with pip -```sh -pip install openai portkey-ai -``` - -```py Chat Completions +```python OpenAI Py icon="python" from openai import OpenAI -from portkey_ai import createHeaders, PORTKEY_GATEWAY_URL +from portkey_ai import PORTKEY_GATEWAY_URL + +# 1. Install: pip install openai portkey-ai +# 2. Add @openai provider in model catalog +# 3. Use it: client = OpenAI( - api_key="xx", - base_url=PORTKEY_GATEWAY_URL, - default_headers=createHeaders( - api_key="PORTKEY_API_KEY", - provider="@OPENAI_PROVIDER" - ) + api_key="PORTKEY_API_KEY", # Portkey API key + base_url=PORTKEY_GATEWAY_URL ) -completion = client.chat.completions.create( - model="gpt-4o", - messages=[ - {"role": "system", "content": "You are a helpful assistant."}, - {"role": "user", "content": "Hello!"} - ] +response = client.chat.completions.create( + model="@openai/gpt-4o", + messages=[{"role": "user", "content": "Say this is a test"}] ) -print(completion.choices[0].message) -``` - - - -Install the OpenAI & Portkey SDKs with npm -```sh -npm install openai portkey-ai +print(response.choices[0].message.content) ``` - -```ts Chat Completions -import OpenAI from 'openai'; -import { PORTKEY_GATEWAY_URL, createHeaders } from 'portkey-ai' -const openai = new OpenAI({ - apiKey: 'xx', - baseURL: PORTKEY_GATEWAY_URL, - defaultHeaders: createHeaders({ - apiKey: "PORTKEY_API_KEY", - provider:"@OPENAI_PROVIDER" - }) -}); +```js OpenAI JS icon="square-js" +import OpenAI from "openai" +import { PORTKEY_GATEWAY_URL } from "portkey-ai" -async function main() { - const completion = await openai.chat.completions.create({ - messages: [{ role: 'user', content: 'Say this is a test' }], - model: 'gpt-4o', - }); +// 1. Install: npm install openai portkey-ai +// 2. Add @openai provider in model catalog +// 3. Use it: - console.log(chatCompletion.choices); -} +const client = new OpenAI({ + apiKey: "PORTKEY_API_KEY", // Portkey API key + baseURL: PORTKEY_GATEWAY_URL +}) -main(); -``` - - +const response = await client.chat.completions.create({ + model: "@openai/gpt-4o", + messages: [{ role: "user", content: "Say this is a test" }] +}) - - -```java Chat Completions -package com.example; - -import com.openai.client.OpenAIClient; -import com.openai.client.okhttp.OpenAIOkHttpClient; -import com.openai.models.chat.completions.ChatCompletion; -import com.openai.models.chat.completions.ChatCompletionCreateParams; - -public class App { - - private static final String PORTKEY_API_KEY = "PK_API_KEY"; - private static final String PROVIDER = "< @OPENAI_PROVIDER/gpt-4o >"; - private static final String PORTKEY_BASE_URL = "https://api.portkey.ai/v1"; - - public static void main(String[] args) { - - // Create client with proxy, Portkey API, and custom headers - OpenAIClient client = OpenAIOkHttpClient.builder() - .baseUrl(PORTKEY_BASE_URL) - .apiKey("API_KEY") - .putHeader("x-portkey-api-key", PORTKEY_API_KEY) - .build(); - - ChatCompletionCreateParams params = ChatCompletionCreateParams.builder() - .addUserMessage("Say this is a test") - .model(PROVIDER) - .maxCompletionTokens(10) - .build(); - - ChatCompletion chatCompletion = client.chat().completions().create(params); - System.out.println(chatCompletion); - } -} +console.log(response.choices[0].message.content) ``` - - -See the [OpenAI Examples](https://github.com/openai/openai-java/tree/main/openai-java-example/src/main/java/com/openai/example) directory for complete and runnable examples. - - - +```sh cURL icon="square-terminal" +# 1. Add @openai provider in model catalog +# 2. Use it: -This request will be automatically logged by Portkey. You can view this in your logs dashboard. Portkey logs the tokens utilized, execution time, and cost for each request. Additionally, you can delve into the details to review the precise request and response data. +curl https://api.portkey.ai/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "x-portkey-api-key: $PORTKEY_API_KEY" \ + -H "x-portkey-provider: @openai" \ + -d '{ + "model": "gpt-4o", + "messages": [ + { "role": "user", "content": "Say this is a test" } + ] + }' +``` + -Portkey supports [OpenAI's new "developer" role](https://platform.openai.com/docs/api-reference/chat/create#chat-create-messages) in chat completions. With o1 models and newer, the `developer` role replaces the previous `system` role. +**Tip:** You can also set `provider="@openai"` in `Portkey()` and use just `model="gpt-4o"` in the request. + +**Legacy support:** The `virtual_key` parameter still works for backwards compatibility. +## Add Provider in Model Catalog -### Using the Responses API +1. Go to [**Model Catalog → Add Provider**](https://app.portkey.ai/model-catalog/providers) +2. Select **OpenAI** +3. Choose existing credentials or create new by entering your [OpenAI API key](https://platform.openai.com/api-keys) +4. (Optional) Add your OpenAI **Organization ID** and **Project ID** for better cost tracking +5. Name your provider (e.g., `openai-prod`) -OpenAI has released a new Responses API that combines the best of both Chat Completions and Assistants APIs. Portkey fully supports this new API, enabling you to use it with both the Portkey SDK and OpenAI SDK. + + See all setup options, code examples, and detailed instructions + - - - ```python - from portkey_ai import Portkey +## Basic Usage - portkey = Portkey( - api_key="PORTKEY_API_KEY", - provider="@OPENAI_PROVIDER" - ) +### Streaming - response = portkey.responses.create( - model="gpt-4.1", - input="Tell me a three sentence bedtime story about a unicorn." - ) +Stream responses for real-time output in your applications: - print(response) - ``` - - - ```js - import Portkey from 'portkey-ai'; + +```python Python icon="python" +response = portkey.chat.completions.create( + model="@openai/gpt-4o", + messages=[{"role": "user", "content": "Tell me a story"}], + stream=True +) - const portkey = new Portkey({ - apiKey: "PORTKEY_API_KEY", - provider:"@OPENAI_PROVIDER" - }); +for chunk in response: + if chunk.choices[0].delta.content: + print(chunk.choices[0].delta.content, end="", flush=True) +``` - async function main() { - const response = await portkey.responses.create({ - model: "gpt-4.1", - input: "Tell me a three sentence bedtime story about a unicorn." - }); +```js Javascript icon="square-js" +const response = await portkey.chat.completions.create({ + model: "@openai/gpt-4o", + messages: [{ role: "user", content: "Tell me a story" }], + stream: true +}) - console.log(response); +for await (const chunk of response) { + if (chunk.choices[0]?.delta?.content) { + process.stdout.write(chunk.choices[0].delta.content) } +} +``` + - main(); - ``` - - - ```python - from openai import OpenAI - from portkey_ai import PORTKEY_GATEWAY_URL, createHeaders - - client = OpenAI( - api_key="OPENAI_API_KEY", - base_url=PORTKEY_GATEWAY_URL, - default_headers=createHeaders( - provider="openai", - api_key="PORTKEY_API_KEY", - provider="@OPENAI_PROVIDER" - ) - ) - - response = client.responses.create( - model="gpt-4.1", - input="Tell me a three sentence bedtime story about a unicorn." - ) - - print(response) - ``` - - - ```js - import OpenAI from 'openai'; - import { PORTKEY_GATEWAY_URL, createHeaders } from 'portkey-ai' - - const openai = new OpenAI({ - baseURL: PORTKEY_GATEWAY_URL, - defaultHeaders: createHeaders({ - provider: "openai", - apiKey: "PORTKEY_API_KEY", - provider:"@OPENAI_PROVIDER" - }) - }); - - async function main() { - const response = await openai.responses.create({ - model: "gpt-4.1", - input: "Tell me a three sentence bedtime story about a unicorn." - }); - - console.log(response); - } +## Advanced Features - main(); - ``` - - +### Responses API - -The Responses API provides a more flexible foundation for building agentic applications with built-in tools that execute automatically. - +OpenAI's Responses API combines the best of both Chat Completions and Assistants APIs. Portkey fully supports this API with both the Portkey SDK and OpenAI SDK. + +```python Python icon="python" +from portkey_ai import Portkey - - Portkey supports Remote MCP support by OpenAI on it's Responses API. Learn More - +portkey = Portkey(api_key="PORTKEY_API_KEY") +response = portkey.responses.create( + model="@openai/gpt-4.1", + input="Tell me a three sentence bedtime story about a unicorn." +) +print(response) +``` -## Track End-User IDs +```js Javascript icon="square-js" +import Portkey from 'portkey-ai' -Portkey allows you to track user IDs passed with the `user` parameter in OpenAI requests, enabling you to monitor user-level costs, requests, and more. - - +const portkey = new Portkey({ + apiKey: "PORTKEY_API_KEY" +}) +const response = await portkey.responses.create({ + model: "@openai/gpt-4.1", + input: "Tell me a three sentence bedtime story about a unicorn." +}) -```js -const chatCompletion = await portkey.chat.completions.create({ - messages: [{ role: "user", content: "Say this is a test" }], - model: "gpt-4o", - user: "user_12345", -}); +console.log(response) ``` - +```python OpenAI Py icon="python" +from openai import OpenAI +from portkey_ai import PORTKEY_GATEWAY_URL - -```py -response = portkey.chat.completions.create( - model="gpt-4o", - messages=[{ role: "user", content: "Say this is a test" }] - user="user_123456" +client = OpenAI( + api_key="PORTKEY_API_KEY", + base_url=PORTKEY_GATEWAY_URL ) -``` - - -When you include the `user` parameter in your requests, Portkey logs will display the associated user ID, as shown in the image below: - - -logs - - -In addition to the `user` parameter, Portkey allows you to send arbitrary custom metadata with your requests. This powerful feature enables you to associate additional context or information with each request, which can be useful for analysis, debugging, or other custom use cases. - - - -* The same integration approach applies to APIs for [completions](https://platform.openai.com/docs/guides/text-generation/completions-api), [embeddings](https://platform.openai.com/docs/api-reference/embeddings/create), [vision](https://platform.openai.com/docs/guides/vision/quick-start), [moderation](https://platform.openai.com/docs/api-reference/moderations/create), [transcription](https://platform.openai.com/docs/api-reference/audio/createTranscription), [translation](https://platform.openai.com/docs/api-reference/audio/createTranslation), [speech](https://platform.openai.com/docs/api-reference/audio/createSpeech) and [files](https://platform.openai.com/docs/api-reference/files/create). -* If you are looking for a way to add your **Org ID** & **Project ID** to the requests, head over to [this section](/integrations/llms/openai#managing-openai-projects-and-organizations-in-portkey). - - - -## Using the Prompts API - -Portkey also supports creating and managing prompt templates in the [prompt library](/product/prompt-library). This enables the collaborative development of prompts directly through the user interface. - -1. Create a prompt template with variables and set the hyperparameters. - - -prompt - +response = client.responses.create( + model="@openai/gpt-4.1", + input="Tell me a three sentence bedtime story about a unicorn." +) -2. Use this prompt in your codebase using the Portkey SDK. +print(response) +``` - - +```js OpenAI JS icon="square-js" +import OpenAI from 'openai' +import { PORTKEY_GATEWAY_URL } from 'portkey-ai' -```js -import Portkey from 'portkey-ai' - -const portkey = new Portkey({ +const openai = new OpenAI({ apiKey: "PORTKEY_API_KEY", + baseURL: PORTKEY_GATEWAY_URL }) -// Make the prompt creation call with the variables - -const promptCompletion = await portkey.prompts.completions.create({ - promptID: "Your Prompt ID", - variables: { - // The variables specified in the prompt - } +const response = await openai.responses.create({ + model: "@openai/gpt-4.1", + input: "Tell me a three sentence bedtime story about a unicorn." }) -``` - -```js -// We can also override the hyperparameters - -const promptCompletion = await portkey.prompts.completions.create({ - promptID: "Your Prompt ID", - variables: { - // The variables specified in the prompt - }, - max_tokens: 250, - presence_penalty: 0.2 -}) +console.log(response) ``` + - + +The Responses API provides a more flexible foundation for building agentic applications with built-in tools that execute automatically. + - -```python -from portkey_ai import Portkey + + Portkey supports Remote MCP support by OpenAI on its Responses API. Learn More + -client = Portkey( - api_key="PORTKEY_API_KEY", # defaults to os.environ.get("PORTKEY_API_KEY") -) +#### Streaming with Responses API -prompt_completion = client.prompts.completions.create( - prompt_id="Your Prompt ID", - variables={ - # The variables specified in the prompt - } + +```python Python icon="python" +response = portkey.responses.create( + model="@openai/gpt-4.1", + instructions="You are a helpful assistant.", + input="Hello!", + stream=True ) -print(prompt_completion) +for event in response: + print(event) +``` -# We can also override the hyperparameters +```js Javascript icon="square-js" +const response = await portkey.responses.create({ + model: "@openai/gpt-4.1", + instructions: "You are a helpful assistant.", + input: "Hello!", + stream: true +}) -prompt_completion = client.prompts.completions.create( - prompt_id="Your Prompt ID", - variables={ - # The variables specified in the prompt - }, - max_tokens=250, - presence_penalty=0.2 +for await (const event of response) { + console.log(event) +} +``` + +```python OpenAI Py icon="python" +response = client.responses.create( + model="gpt-4.1", + instructions="You are a helpful assistant.", + input="Hello!", + stream=True ) -print(prompt_completion) -``` - - - -```sh -curl -X POST "https://api.portkey.ai/v1/prompts/:PROMPT_ID/completions" \ --H "Content-Type: application/json" \ --H "x-portkey-api-key: $PORTKEY_API_KEY" \ --d '{ - "variables": { - # The variables to use - }, - "max_tokens": 250, # Optional - "presence_penalty": 0.2 # Optional -}' +for event in response: + print(event) ``` - - -Observe how this streamlines your code readability and simplifies prompt updates via the UI without altering the codebase. +```js OpenAI JS icon="square-js" +const response = await openai.responses.create({ + model: "gpt-4.1", + instructions: "You are a helpful assistant.", + input: "Hello!", + stream: true +}) -## Advanced Use Cases +for await (const event of response) { + console.log(event) +} +``` + ### Realtime API @@ -462,129 +309,6 @@ Portkey supports OpenAI's Realtime API with a seamless integration. This allows -### Streaming Responses - -Portkey supports streaming responses using Server Sent Events (SSE). - - - - ```js - import OpenAI from 'openai'; - - import { PORTKEY_GATEWAY_URL, createHeaders } from 'portkey-ai' - const openai = new OpenAI({ - baseURL: PORTKEY_GATEWAY_URL, - defaultHeaders: createHeaders({ - provider: "openai", - apiKey: "PORTKEY_API_KEY" // defaults to process.env["PORTKEY_API_KEY"] - }) - }); - - async function main() { - const stream = await openai.chat.completions.create({ - model: 'gpt-4', - messages: [{ role: 'user', content: 'Say this is a test' }], - stream: true, - }); - - for await (const chunk of stream) { - process.stdout.write(chunk.choices[0]?.delta?.content || ''); - } - } - - main(); - ``` - - - ```python - from openai import OpenAI - from portkey_ai import PORTKEY_GATEWAY_URL, createHeaders - - client = OpenAI( - api_key="OPENAI_API_KEY", # defaults to os.environ.get("OPENAI_API_KEY") - base_url=PORTKEY_GATEWAY_URL, - default_headers=createHeaders( - provider="openai", - api_key="PORTKEY_API_KEY" # defaults to os.environ.get("PORTKEY_API_KEY") - ) - ) - - chat_complete = client.chat.completions.create( - model="gpt-4", - messages=[{"role": "user", "content": "Say this is a test"}], - stream=True - ) - - for chunk in chat_complete: - print(chunk.choices[0].delta.content, end="", flush=True) - ``` - - - - - - -#### Streaming with the Responses API - -You can also stream responses from the Responses API: - - - - ```python - response = portkey.responses.create( - model="gpt-4.1", - instructions="You are a helpful assistant.", - input="Hello!", - stream=True - ) - - for event in response: - print(event) - ``` - - - ```js - const response = await portkey.responses.create({ - model: "gpt-4.1", - instructions: "You are a helpful assistant.", - input: "Hello!", - stream: true - }); - - for await (const event of response) { - console.log(event); - } - ``` - - - ```python - response = client.responses.create( - model="gpt-4.1", - instructions="You are a helpful assistant.", - input="Hello!", - stream=True - ) - - for event in response: - print(event) - ``` - - - ```js - const response = await openai.responses.create({ - model: "gpt-4.1", - instructions: "You are a helpful assistant.", - input: "Hello!", - stream: true - }); - - for await (const event of response) { - console.log(event); - } - ``` - - - ### Using Vision Models @@ -597,94 +321,89 @@ Portkey's multimodal Gateway fully supports OpenAI vision models as well. See th #### Vision with the Responses API -You can also use the Responses API to process images alongside text: - - - - ```python - response = portkey.responses.create( - model="gpt-4.1", - input=[ - { - "role": "user", - "content": [ - { "type": "input_text", "text": "What is in this image?" }, - { - "type": "input_image", - "image_url": "https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg" - } - ] - } - ] - ) - - print(response) - ``` - - - ```js - const response = await portkey.responses.create({ - model: "gpt-4.1", - input: [ - { - role: "user", - content: [ - { type: "input_text", text: "What is in this image?" }, - { - type: "input_image", - image_url: "https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg" - } - ] - } - ] - }); - - console.log(response); - ``` - - - ```python - response = client.responses.create( - model="gpt-4.1", - input=[ - { - "role": "user", - "content": [ - { "type": "input_text", "text": "What is in this image?" }, - { - "type": "input_image", - "image_url": "https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg" - } - ] - } - ] - ) - - print(response) - ``` - - - ```js - const response = await openai.responses.create({ - model: "gpt-4.1", - input: [ - { - role: "user", - content: [ - { type: "input_text", text: "What is in this image?" }, - { - type: "input_image", - image_url: "https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg" - } - ] - } - ] - }); - - console.log(response); - ``` - - +The Responses API also processes images alongside text: + + +```python Python icon="python" +response = portkey.responses.create( + model="@openai/gpt-4.1", + input=[ + { + "role": "user", + "content": [ + { "type": "input_text", "text": "What is in this image?" }, + { + "type": "input_image", + "image_url": "https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg" + } + ] + } + ] +) + +print(response) +``` + +```js Javascript icon="square-js" +const response = await portkey.responses.create({ + model: "@openai/gpt-4.1", + input: [ + { + role: "user", + content: [ + { type: "input_text", text: "What is in this image?" }, + { + type: "input_image", + image_url: "https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg" + } + ] + } + ] +}) + +console.log(response) +``` + +```python OpenAI Py icon="python" +response = client.responses.create( + model="gpt-4.1", + input=[ + { + "role": "user", + "content": [ + { "type": "input_text", "text": "What is in this image?" }, + { + "type": "input_image", + "image_url": "https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg" + } + ] + } + ] +) + +print(response) +``` + +```js OpenAI JS icon="square-js" +const response = await openai.responses.create({ + model: "gpt-4.1", + input: [ + { + role: "user", + content: [ + { type: "input_text", text: "What is in this image?" }, + { + type: "input_image", + image_url: "https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg" + } + ] + } + ] +}) + +console.log(response) +``` + @@ -699,132 +418,127 @@ Additionally, you can define functions within your prompts and invoke the `portk The Responses API also supports function calling with the same powerful capabilities: - - - ```python - tools = [ - { - "type": "function", - "name": "get_current_weather", - "description": "Get the current weather in a given location", - "parameters": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "The city and state, e.g. San Francisco, CA" - }, - "unit": {"type": "string", "enum": ["celsius", "fahrenheit"]} + +```python Python icon="python" +tools = [ + { + "type": "function", + "name": "get_current_weather", + "description": "Get the current weather in a given location", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA" }, - "required": ["location", "unit"] - } + "unit": {"type": "string", "enum": ["celsius", "fahrenheit"]} + }, + "required": ["location", "unit"] } - ] + } +] - response = portkey.responses.create( - model="gpt-4.1", - tools=tools, - input="What is the weather like in Boston today?", - tool_choice="auto" - ) - - print(response) - ``` - - - ```js - const tools = [ - { - type: "function", - name: "get_current_weather", - description: "Get the current weather in a given location", - parameters: { - type: "object", - properties: { - location: { - type: "string", - description: "The city and state, e.g. San Francisco, CA" - }, - unit: { type: "string", enum: ["celsius", "fahrenheit"] } +response = portkey.responses.create( + model="@openai/gpt-4.1", + tools=tools, + input="What is the weather like in Boston today?", + tool_choice="auto" +) + +print(response) +``` + +```js Javascript icon="square-js" +const tools = [ + { + type: "function", + name: "get_current_weather", + description: "Get the current weather in a given location", + parameters: { + type: "object", + properties: { + location: { + type: "string", + description: "The city and state, e.g. San Francisco, CA" }, - required: ["location", "unit"] - } + unit: { type: "string", enum: ["celsius", "fahrenheit"] } + }, + required: ["location", "unit"] } - ]; - - const response = await portkey.responses.create({ - model: "gpt-4.1", - tools: tools, - input: "What is the weather like in Boston today?", - tool_choice: "auto" - }); - - console.log(response); - ``` - - - ```python - tools = [ - { - "type": "function", - "name": "get_current_weather", - "description": "Get the current weather in a given location", - "parameters": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "The city and state, e.g. San Francisco, CA" - }, - "unit": {"type": "string", "enum": ["celsius", "fahrenheit"]} + } +] + +const response = await portkey.responses.create({ + model: "@openai/gpt-4.1", + tools: tools, + input: "What is the weather like in Boston today?", + tool_choice: "auto" +}) + +console.log(response) +``` + +```python OpenAI Py icon="python" +tools = [ + { + "type": "function", + "name": "get_current_weather", + "description": "Get the current weather in a given location", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA" }, - "required": ["location", "unit"] - } + "unit": {"type": "string", "enum": ["celsius", "fahrenheit"]} + }, + "required": ["location", "unit"] } - ] + } +] - response = client.responses.create( - model="gpt-4.1", - tools=tools, - input="What is the weather like in Boston today?", - tool_choice="auto" - ) - - print(response) - ``` - - - ```js - const tools = [ - { - type: "function", - name: "get_current_weather", - description: "Get the current weather in a given location", - parameters: { - type: "object", - properties: { - location: { - type: "string", - description: "The city and state, e.g. San Francisco, CA" - }, - unit: { type: "string", enum: ["celsius", "fahrenheit"] } +response = client.responses.create( + model="gpt-4.1", + tools=tools, + input="What is the weather like in Boston today?", + tool_choice="auto" +) + +print(response) +``` + +```js OpenAI JS icon="square-js" +const tools = [ + { + type: "function", + name: "get_current_weather", + description: "Get the current weather in a given location", + parameters: { + type: "object", + properties: { + location: { + type: "string", + description: "The city and state, e.g. San Francisco, CA" }, - required: ["location", "unit"] - } + unit: { type: "string", enum: ["celsius", "fahrenheit"] } + }, + required: ["location", "unit"] } - ]; + } +] - const response = await openai.responses.create({ - model: "gpt-4.1", - tools: tools, - input: "What is the weather like in Boston today?", - tool_choice: "auto" - }); +const response = await openai.responses.create({ + model: "gpt-4.1", + tools: tools, + input: "What is the weather like in Boston today?", + tool_choice: "auto" +}) - console.log(response); - ``` - - +console.log(response) +``` + @@ -834,33 +548,29 @@ Please refer to our fine-tuning guides to take advantage of Portkey's advanced [ ### Image Generation -Portkey supports multiple modalities for OpenAI and you can make image generation requests through Portkey's AI Gateway the same way as making completion calls. - - - - ```js - // Define the OpenAI client as shown above - - const image = await openai.images.generate({ - model:"dall-e-3", - prompt:"Lucy in the sky with diamonds", - size:"1024x1024" - }) - ``` - - - - ```python - # Define the OpenAI client as shown above - - image = openai.images.generate( - model="dall-e-3", - prompt="Lucy in the sky with diamonds", - size="1024x1024" - ) - ``` - - +Portkey supports multiple modalities for OpenAI. Make image generation requests through Portkey's AI Gateway the same way as making completion calls. + + +```js Javascript icon="square-js" +// Define the OpenAI client as shown above + +const image = await openai.images.generate({ + model:"dall-e-3", + prompt:"Lucy in the sky with diamonds", + size:"1024x1024" +}) +``` + +```python Python icon="python" +# Define the OpenAI client as shown above + +image = openai.images.generate( + model="dall-e-3", + prompt="Lucy in the sky with diamonds", + size="1024x1024" +) +``` + Portkey's fast AI gateway captures the information about the request on your Portkey Dashboard. On your logs screen, you'd be able to see this request with the request and response. @@ -871,27 +581,22 @@ More information on image generation is available in the [API Reference](/provid ### Video Generation with Sora -Portkey supports OpenAI's Sora video generation models through the AI Gateway. You can generate videos using the Portkey Python SDK. +Portkey supports OpenAI's Sora video generation models through the AI Gateway. Generate videos using the Portkey Python SDK: - - - ```python - from portkey_ai import Portkey +```python +from portkey_ai import Portkey - client = Portkey( - api_key="your-portkey-api-key", - provider="@your-portkey-provider-slug", - ) +client = Portkey( + api_key="PORTKEY_API_KEY" +) - video = client.videos.create( - model="sora-2", - prompt="A video of a cool cat on a motorcycle in the night", - ) +video = client.videos.create( + model="@openai/sora-2", + prompt="A video of a cool cat on a motorcycle in the night", +) - print("Video generation started:", video) - ``` - - +print("Video generation started:", video) +``` Pricing for video generation requests will be visible on your Portkey dashboard, allowing you to track costs alongside your other API usage. @@ -910,54 +615,51 @@ Check out the below guides for more info: --- -## Intergrated Tools with Repsponses API +## Integrated Tools with Responses API ### Web Search Tool Web search delivers accurate and clearly-cited answers from the web, using the same tool as search in ChatGPT: - - - ```python - response = portkey.responses.create( - model="gpt-4.1", - tools=[{ - "type": "web_search_preview", - "search_context_size": "medium", # Options: "high", "medium" (default), or "low" - "user_location": { # Optional - for localized results - "type": "approximate", - "country": "US", - "city": "San Francisco", - "region": "California" - } - }], - input="What was a positive news story from today?" - ) - - print(response) - ``` - - - ```js - const response = await portkey.responses.create({ - model: "gpt-4.1", - tools: [{ - type: "web_search_preview", - search_context_size: "medium", // Options: "high", "medium" (default), or "low" - user_location: { // Optional - for localized results - type: "approximate", - country: "US", - city: "San Francisco", - region: "California" - } - }], - input: "What was a positive news story from today?" - }); - - console.log(response); - ``` - - + +```python Python icon="python" +response = portkey.responses.create( + model="@openai/gpt-4.1", + tools=[{ + "type": "web_search_preview", + "search_context_size": "medium", # Options: "high", "medium" (default), or "low" + "user_location": { # Optional - for localized results + "type": "approximate", + "country": "US", + "city": "San Francisco", + "region": "California" + } + }], + input="What was a positive news story from today?" +) + +print(response) +``` + +```js Javascript icon="square-js" +const response = await portkey.responses.create({ + model: "@openai/gpt-4.1", + tools: [{ + type: "web_search_preview", + search_context_size: "medium", // Options: "high", "medium" (default), or "low" + user_location: { // Optional - for localized results + type: "approximate", + country: "US", + city: "San Francisco", + region: "California" + } + }], + input: "What was a positive news story from today?" +}) + +console.log(response) +``` + **Options for `search_context_size`:** @@ -972,48 +674,45 @@ Web search delivers accurate and clearly-cited answers from the web, using the s File search enables quick retrieval from your knowledge base across multiple file types: - - - ```python - response = portkey.responses.create( - model="gpt-4.1", - tools=[{ - "type": "file_search", - "vector_store_ids": ["vs_1234567890"], - "max_num_results": 20, - "filters": { # Optional - filter by metadata - "type": "eq", - "key": "document_type", - "value": "report" - } - }], - input="What are the attributes of an ancient brown dragon?" - ) - - print(response) - ``` - - - ```js - const response = await portkey.responses.create({ - model: "gpt-4.1", - tools: [{ - type: "file_search", - vector_store_ids: ["vs_1234567890"], - max_num_results: 20, - filters: { // Optional - filter by metadata - type: "eq", - key: "document_type", - value: "report" - } - }], - input: "What are the attributes of an ancient brown dragon?" - }); - - console.log(response); - ``` - - + +```python Python icon="python" +response = portkey.responses.create( + model="@openai/gpt-4.1", + tools=[{ + "type": "file_search", + "vector_store_ids": ["vs_1234567890"], + "max_num_results": 20, + "filters": { # Optional - filter by metadata + "type": "eq", + "key": "document_type", + "value": "report" + } + }], + input="What are the attributes of an ancient brown dragon?" +) + +print(response) +``` + +```js Javascript icon="square-js" +const response = await portkey.responses.create({ + model: "@openai/gpt-4.1", + tools: [{ + type: "file_search", + vector_store_ids: ["vs_1234567890"], + max_num_results: 20, + filters: { // Optional - filter by metadata + type: "eq", + key: "document_type", + value: "report" + } + }], + input: "What are the attributes of an ancient brown dragon?" +}) + +console.log(response) +``` + This tool requires you to first create a vector store and upload files to it. Supports various file formats including PDFs, DOCXs, TXT, and more. Results include file citations in the response. @@ -1023,34 +722,31 @@ File search enables quick retrieval from your knowledge base across multiple fil Control the depth of model reasoning for more comprehensive analysis: - - - ```python - response = portkey.responses.create( - model="o3-mini", - input="How much wood would a woodchuck chuck?", - reasoning={ - "effort": "high" # Options: "high", "medium", or "low" - } - ) - - print(response) - ``` - - - ```js - const response = await portkey.responses.create({ - model: "o3-mini", - input: "How much wood would a woodchuck chuck?", - reasoning: { - effort: "high" // Options: "high", "medium", or "low" - } - }); + +```python Python icon="python" +response = portkey.responses.create( + model="@openai/o3-mini", + input="How much wood would a woodchuck chuck?", + reasoning={ + "effort": "high" # Options: "high", "medium", or "low" + } +) + +print(response) +``` - console.log(response); - ``` - - +```js Javascript icon="square-js" +const response = await portkey.responses.create({ + model: "@openai/o3-mini", + input: "How much wood would a woodchuck chuck?", + reasoning: { + effort: "high" // Options: "high", "medium", or "low" + } +}) + +console.log(response) +``` + ### Computer Use Assistant @@ -1065,42 +761,42 @@ Portkey also supports the Computer Use Assistant (CUA) tool, which helps agents ## Managing OpenAI Projects & Organizations in Portkey -When integrating OpenAI with Portkey, you can specify your OpenAI organization and project IDs along with your API key. This is particularly useful if you belong to multiple organizations or are accessing projects through a legacy user API key. +When integrating OpenAI with Portkey, specify your OpenAI organization and project IDs along with your API key. This is particularly useful if you belong to multiple organizations or are accessing projects through a legacy user API key. Specifying the organization and project IDs helps you maintain better control over your access rules, usage, and costs. -In Portkey, you can add your Org & Project details by, +Add your Org & Project details using: -1. Creating your Virtual Key +1. Adding in Model Catalog (Recommended) 2. Defining a Gateway Config 3. Passing Details in a Request Let's explore each method in more detail. -### Using Virtual Keys +### Using Model Catalog -When selecting OpenAI from the dropdown menu while creating a virtual key, Portkey automatically displays optional fields for the organization ID and project ID alongside the API key field. +When adding OpenAI from the Model Catalog, Portkey automatically displays optional fields for the organization ID and project ID alongside the API key field. -[Get your OpenAI API key from here](https://platform.openai.com/api-keys), then add it to Portkey to create the virtual key that can be used throughout Portkey. +[Get your OpenAI API key from here](https://platform.openai.com/api-keys), then add it to Portkey along with your org/project details. ![LOGO](/images/llms/virtual.png) -[Virtual Keys](/product/ai-gateway/virtual-keys) +[Model Catalog](/product/model-catalog) -Portkey takes budget management a step further than OpenAI. While OpenAI allows setting budget limits per project, Portkey enables you to set budget limits for each virtual key you create. For more information on budget limits, refer to this documentation: +Portkey takes budget management a step further than OpenAI. While OpenAI allows setting budget limits per project, Portkey enables you to set budget limits for each provider you create. For more information on budget limits, refer to this documentation: -[Budget Limits](/product/ai-gateway/virtual-keys/budget-limits) +[Budget Limits](/product/ai-gateway/budget-limits) -### Using The Gateway Config + +### Using the Gateway Config You can also specify the organization and project details in the gateway config, either at the root level or within a specific target. ```json { - "provider": "openai", - "api_key": "OPENAI_API_KEY", + "provider": "@openai", "openai_organization": "org-xxxxxx", "openai_project": "proj_xxxxxxxx" } @@ -1109,135 +805,118 @@ You can also specify the organization and project details in the gateway config, ### While Making a Request You can also pass your organization and project details directly when making a request using curl, the OpenAI SDK, or the Portkey SDK. - - - ```python - from openai import OpenAI - from portkey_ai import PORTKEY_GATEWAY_URL, createHeaders - - client = OpenAI( - api_key="OPENAI_API_KEY", - organization="org-xxxxxxxxxx", - project="proj_xxxxxxxxx", - base_url=PORTKEY_GATEWAY_URL, - default_headers=createHeaders( - provider="openai", - api_key="PORTKEY_API_KEY" - ) - ) - - chat_complete = client.chat.completions.create( - model="gpt-4o", - messages=[{"role": "user", "content": "Say this is a test"}], - ) - - print(chat_complete.choices[0].message.content) - ``` - - - - -```js -import OpenAI from "openai"; -import { PORTKEY_GATEWAY_URL, createHeaders } from "portkey-ai"; + + +```python OpenAI Py icon="python" +from openai import OpenAI +from portkey_ai import PORTKEY_GATEWAY_URL + +client = OpenAI( + api_key="PORTKEY_API_KEY", + organization="org-xxxxxxxxxx", + project="proj_xxxxxxxxx", + base_url=PORTKEY_GATEWAY_URL +) + +chat_complete = client.chat.completions.create( + model="@openai/gpt-4o", + messages=[{"role": "user", "content": "Say this is a test"}], +) + +print(chat_complete.choices[0].message.content) +``` + +```js OpenAI JS icon="square-js" +import OpenAI from "openai" +import { PORTKEY_GATEWAY_URL } from "portkey-ai" const openai = new OpenAI({ - apiKey: "OPENAI_API_KEY", + apiKey: "PORTKEY_API_KEY", organization: "org-xxxxxx", project: "proj_xxxxxxx", - baseURL: PORTKEY_GATEWAY_URL, - defaultHeaders: createHeaders({ - provider: "openai", - apiKey: "PORTKEY_API_KEY", - }), -}); + baseURL: PORTKEY_GATEWAY_URL +}) async function main() { const chatCompletion = await openai.chat.completions.create({ messages: [{ role: "user", content: "Say this is a test" }], - model: "gpt-4o", - }); + model: "@openai/gpt-4o", + }) - console.log(chatCompletion.choices); + console.log(chatCompletion.choices) } -main(); +main() ``` - - - -```sh +```sh cURL icon="square-terminal" curl https://api.portkey.ai/v1/chat/completions \ -H "Content-Type: application/json" \ -H "Authorization: Bearer $OPENAI_API_KEY" \ -H "x-portkey-openai-organization: org-xxxxxxx" \ -H "x-portkey-openai-project: proj_xxxxxxx" \ -H "x-portkey-api-key: $PORTKEY_API_KEY" \ - -H "x-portkey-provider: openai" \ + -H "x-portkey-provider: @openai" \ -d '{ "model": "gpt-4o", "messages": [{"role": "user","content": "Hello!"}] }' ``` - - -```python +```python Python icon="python" from portkey_ai import Portkey portkey = Portkey( api_key="PORTKEY_API_KEY", - provider="openai", Authorization="Bearer OPENAI_API_KEY", openai_organization="org-xxxxxxxxx", openai_project="proj_xxxxxxxxx", ) chat_complete = portkey.chat.completions.create( - model="gpt-4o", + model="@openai/gpt-4o", messages=[{"role": "user", "content": "Say this is a test"}], ) print(chat_complete.choices[0].message.content) ``` - - -```js -import Portkey from "portkey-ai"; +```js Javascript icon="square-js" +import Portkey from "portkey-ai" const portkey = new Portkey({ apiKey: "PORTKEY_API_KEY", - provider: "openai", Authorization: "Bearer OPENAI_API_KEY", openaiOrganization: "org-xxxxxxxxxxx", openaiProject: "proj_xxxxxxxxxxxxx", -}); +}) async function main() { const chatCompletion = await portkey.chat.completions.create({ messages: [{ role: "user", content: "Say this is a test" }], - model: "gpt-4o", - }); + model: "@openai/gpt-4o", + }) - console.log(chatCompletion.choices); + console.log(chatCompletion.choices) } -main(); +main() ``` - - + --- -### Portkey Features - -Portkey supports the complete host of it's functionality via the OpenAI SDK so you don't need to migrate away from it. - -Please find more information in the relevant sections: - -1. [Add metadata to your requests](/product/observability/metadata) -2. [Add gateway configs to the OpenAI client or a single request](/product/ai-gateway/configs) -3. [Tracing OpenAI requests](/product/observability/traces) -4. [Setup a fallback to Azure OpenAI](/product/ai-gateway/fallbacks) +## Frequently Asked Questions + +### General FAQs + + +You can sign up to OpenAI [here](https://platform.openai.com/docs/overview) and grab your scoped API key [here](https://platform.openai.com/api-keys). + + +The OpenAI API can be used by signing up to the OpenAI platform. You can find the pricing info [here](https://openai.com/api/pricing/) + + +You can find your current rate limits imposed by OpenAI [here](https://platform.openai.com/settings/organization/limits). For more tips, check out [this guide](/guides/getting-started/tackling-rate-limiting). + + diff --git a/integrations/llms/openai/prompt-caching-openai.mdx b/integrations/llms/openai/prompt-caching-openai.mdx index 73277479..a1e6fdd0 100644 --- a/integrations/llms/openai/prompt-caching-openai.mdx +++ b/integrations/llms/openai/prompt-caching-openai.mdx @@ -65,7 +65,7 @@ Portkey supports OpenAI's prompt caching feature out of the box. Here is an exam const portkey = new Portkey({ apiKey: "PORTKEY_API_KEY", // defaults to process.env["PORTKEY_API_KEY"] - provider:"@PROVIDER" // Your OpenAI Virtual Key + provider:"@PROVIDER" // Your OpenAI Provider Slug }) // Define tools (for function calling example) diff --git a/integrations/llms/openai3.mdx b/integrations/llms/openai3.mdx index a6aa5d81..ee8f0d71 100644 --- a/integrations/llms/openai3.mdx +++ b/integrations/llms/openai3.mdx @@ -1103,7 +1103,7 @@ When selecting OpenAI from the Virtual Key dropdown menu while creating a virtua -Portkey takes budget management a step further than OpenAI. While OpenAI allows setting budget limits per project, Portkey enables you to set budget limits for each virtual key you create. For more information on budget limits, [refer to this documentation](/product/ai-gateway/virtual-keys/budget-limits) +Portkey takes budget management a step further than OpenAI. While OpenAI allows setting budget limits per project, Portkey enables you to set budget limits for each virtual key you create. For more information on budget limits, [refer to this documentation](/product/model-catalog/integrations#3-budget-%26-rate-limits) diff --git a/integrations/llms/openrouter.mdx b/integrations/llms/openrouter.mdx index d76c07ce..065a2ef0 100644 --- a/integrations/llms/openrouter.mdx +++ b/integrations/llms/openrouter.mdx @@ -1,139 +1,145 @@ --- title: "OpenRouter" +description: "Integrate OpenRouter models with Portkey's AI Gateway" --- -Portkey provides a robust and secure gateway to facilitate the integration of various Large Language Models (LLMs) into your applications, including [OpenRouter](https://openrouter.ai). +Portkey provides a robust and secure gateway to integrate various Large Language Models (LLMs) into applications, including [OpenRouter's unified API](https://openrouter.ai). -With Portkey, you can take advantage of features like fast AI gateway access, observability, prompt management, and more, all while ensuring the secure management of your LLM API keys through a [virtual key](/product/ai-gateway/virtual-keys) system. - -Provider Slug. `openrouter` - -## Portkey SDK Integration with OpenRouter Models +With Portkey, take advantage of features like fast AI gateway access, observability, prompt management, and more, while securely managing API keys through [Model Catalog](/product/model-catalog). + +## Quick Start + +Get OpenRouter working in 3 steps: + + +```python Python icon="python" +from portkey_ai import Portkey -Portkey provides a consistent API to interact with models from various providers. To integrate OpenRouter with Portkey: +# 1. Install: pip install portkey-ai +# 2. Add @openrouter provider in model catalog +# 3. Use it: + +portkey = Portkey(api_key="PORTKEY_API_KEY") + +response = portkey.chat.completions.create( + model="@openrouter/openai/gpt-4o", + messages=[{"role": "user", "content": "Say this is a test"}] +) -### 1\. Install the Portkey SDK +print(response.choices[0].message.content) +``` -Add the Portkey SDK to your application to interact with OpenRouter AI's API through Portkey's gateway. - - - ```sh - npm install --save portkey-ai - ``` - - - ```sh - pip install portkey-ai - ``` +```js Javascript icon="square-js" +import Portkey from 'portkey-ai' - +// 1. Install: npm install portkey-ai +// 2. Add @openrouter provider in model catalog +// 3. Use it: - +const portkey = new Portkey({ + apiKey: "PORTKEY_API_KEY" +}) +const response = await portkey.chat.completions.create({ + model: "@openrouter/openai/gpt-4o", + messages: [{ role: "user", content: "Say this is a test" }] +}) +console.log(response.choices[0].message.content) +``` -### 2\. Initialize Portkey with the Virtual Key +```python OpenAI Py icon="python" +from openai import OpenAI +from portkey_ai import PORTKEY_GATEWAY_URL -To use OpenRouter with Portkey, [get your API key from here](https://openrouter.ai/settings/keys), then add it to Portkey to create the virtual key. - - - ```js - import Portkey from 'portkey-ai' +# 1. Install: pip install openai portkey-ai +# 2. Add @openrouter provider in model catalog +# 3. Use it: - const portkey = new Portkey({ - apiKey: "PORTKEY_API_KEY", // defaults to process.env["PORTKEY_API_KEY"] - provider:"@PROVIDER" // Your OpenRouter Virtual Key - }) - ``` - - - ```python - from portkey_ai import Portkey +client = OpenAI( + api_key="PORTKEY_API_KEY", # Portkey API key + base_url=PORTKEY_GATEWAY_URL +) - portkey = Portkey( - api_key="PORTKEY_API_KEY", # Replace with your Portkey API key - provider="@PROVIDER" # Replace with your virtual key for Open Router - ) - ``` - +response = client.chat.completions.create( + model="@openrouter/openai/gpt-4o", + messages=[{"role": "user", "content": "Say this is a test"}] +) - +print(response.choices[0].message.content) +``` +```js OpenAI JS icon="square-js" +import OpenAI from "openai" +import { PORTKEY_GATEWAY_URL } from "portkey-ai" +// 1. Install: npm install openai portkey-ai +// 2. Add @openrouter provider in model catalog +// 3. Use it: -### **3\. Invoke Chat Completions with** OpenRouter +const client = new OpenAI({ + apiKey: "PORTKEY_API_KEY", // Portkey API key + baseURL: PORTKEY_GATEWAY_URL +}) -Use the Portkey instance to send requests to OpenRouter. You can also override the virtual key directly in the API call if needed. - - +const response = await client.chat.completions.create({ + model: "@openrouter/openai/gpt-4o", + messages: [{ role: "user", content: "Say this is a test" }] +}) -```js -const chatCompletion = await portkey.chat.completions.create({ - messages: [{ role: 'user', content: 'Say this is a test' }], - model: 'openai/gpt-4o-2024-08-06', -}); +console.log(response.choices[0].message.content) +``` -console.log(chatCompletion.choices); +```sh cURL icon="square-terminal" +# 1. Add @openrouter provider in model catalog +# 2. Use it: + +curl https://api.portkey.ai/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "x-portkey-api-key: $PORTKEY_API_KEY" \ + -d '{ + "model": "@openrouter/openai/gpt-4o", + "messages": [ + { "role": "user", "content": "Say this is a test" } + ] + }' ``` - - - ```python - completion = portkey.chat.completions.create( - messages= [{ "role": 'user', "content": 'Say this is a test' }], - model= 'mistral-medium' - ) + - print(completion) - ``` - + +**Tip:** You can also set `provider="@openrouter"` in `Portkey()` and use just `model="openai/gpt-4o"` in the request. + - +## Add Provider in Model Catalog +1. Go to [**Model Catalog → Add Provider**](https://app.portkey.ai/model-catalog/providers) +2. Select **OpenRouter** +3. Choose existing credentials or create new by entering your [OpenRouter API key](https://openrouter.ai/settings/keys) +4. Name your provider (e.g., `openrouter-prod`) + + See all setup options, code examples, and detailed instructions + -### Open Router Tool Calling -Tool calling feature lets models trigger external tools based on conversation context. You define available functions, the model chooses when to use them, and your application executes them and returns results. +## OpenRouter Tool Calling -Portkey supports Open Router Tool Calling and makes it interoperable across multiple providers. With Portkey Prompts, you can templatize various your prompts & tool schemas as well. +Tool calling lets models trigger external tools based on conversation context. You define available functions, the model chooses when to use them, and your application executes them and returns results. - +Portkey supports OpenRouter tool calling and makes it interoperable across multiple providers. With Portkey Prompts, you can templatize your prompts and tool schemas. + + View OpenRouter's tool calling documentation - - -```javascript Get Weather Tool -let tools = [{ - type: "function", - function: { - name: "getWeather", - description: "Get the current weather", - parameters: { - type: "object", - properties: { - location: { type: "string", description: "City and state" }, - unit: { type: "string", enum: ["celsius", "fahrenheit"] } - }, - required: ["location"] - } - } -}]; + +```python Python +from portkey_ai import Portkey -let response = await portkey.chat.completions.create({ - model: "openai/gpt-4o", - messages: [ - { role: "system", content: "You are a helpful assistant." }, - { role: "user", content: "What's the weather like in Delhi - respond in JSON" } - ], - tools, - tool_choice: "auto", -}); +portkey = Portkey( + api_key="PORTKEY_API_KEY", + provider="@openrouter" +) -console.log(response.choices[0].finish_reason); -``` - - -```python Get Weather Tool tools = [{ "type": "function", "function": { @@ -154,7 +160,7 @@ response = portkey.chat.completions.create( model="openai/gpt-4o", messages=[ {"role": "system", "content": "You are a helpful assistant."}, - {"role": "user", "content": "What's the weather like in Delhi - respond in JSON"} + {"role": "user", "content": "What's the weather like in Delhi?"} ], tools=tools, tool_choice="auto" @@ -162,17 +168,54 @@ response = portkey.chat.completions.create( print(response.choices[0].finish_reason) ``` - - -```curl Get Weather Tool + +```js Javascript +import Portkey from 'portkey-ai' + +const portkey = new Portkey({ + apiKey: "PORTKEY_API_KEY", + provider: "@openrouter" +}) + +const tools = [{ + type: "function", + function: { + name: "getWeather", + description: "Get the current weather", + parameters: { + type: "object", + properties: { + location: { type: "string", description: "City and state" }, + unit: { type: "string", enum: ["celsius", "fahrenheit"] } + }, + required: ["location"] + } + } +}] + +const response = await portkey.chat.completions.create({ + model: "openai/gpt-4o", + messages: [ + { role: "system", content: "You are a helpful assistant." }, + { role: "user", content: "What's the weather like in Delhi?" } + ], + tools, + tool_choice: "auto" +}) + +console.log(response.choices[0].finish_reason) +``` + +```sh cURL curl -X POST "https://api.portkey.ai/v1/chat/completions" \ -H "Content-Type: application/json" \ - -H "Authorization: Bearer YOUR_PORTKEY_API_KEY" \ + -H "x-portkey-api-key: $PORTKEY_API_KEY" \ + -H "x-portkey-provider: @openrouter" \ -d '{ "model": "openai/gpt-4o", "messages": [ {"role": "system", "content": "You are a helpful assistant."}, - {"role": "user", "content": "What'\''s the weather like in Delhi - respond in JSON"} + {"role": "user", "content": "What'\''s the weather like in Delhi?"} ], "tools": [{ "type": "function", @@ -192,13 +235,27 @@ curl -X POST "https://api.portkey.ai/v1/chat/completions" \ "tool_choice": "auto" }' ``` - - - - - - -The complete list of features supported in the SDK are available on the link below. - - + + +## Next Steps + + + + Add metadata to your OpenRouter requests + + + Add gateway configs to your OpenRouter requests + + + Trace your OpenRouter requests + + + Setup fallback strategies with OpenRouter + + + +For complete SDK documentation: + + + Complete Portkey SDK documentation diff --git a/integrations/llms/oracle.mdx b/integrations/llms/oracle.mdx index 09b3b202..7b8cecd4 100644 --- a/integrations/llms/oracle.mdx +++ b/integrations/llms/oracle.mdx @@ -45,9 +45,9 @@ Add these credentials to Portkey to create your Oracle virtual key. ```js - import Portkey from 'portkey-ai' +import Portkey from 'portkey-ai' - const portkey = new Portkey({ +const portkey = new Portkey({ apiKey: "PORTKEY_API_KEY", // defaults to process.env["PORTKEY_API_KEY"] provider: "@ORACLE_PROVIDER" // Your Oracle OCI virtual key }) @@ -70,7 +70,7 @@ Add these credentials to Portkey to create your Oracle virtual key. ```js - const chatCompletion = await portkey.chat.completions.create({ +const chatCompletion = await portkey.chat.completions.create({ messages: [{ role: 'user', content: 'Say this is a test' }], model: 'cohere.command-r-plus', }); diff --git a/integrations/llms/perplexity-ai.mdx b/integrations/llms/perplexity-ai.mdx index ba432449..c8eb026a 100644 --- a/integrations/llms/perplexity-ai.mdx +++ b/integrations/llms/perplexity-ai.mdx @@ -1,272 +1,356 @@ --- title: "Perplexity AI" +description: Use Perplexity's online reasoning models with advanced search capabilities through Portkey. --- -Portkey provides a robust and secure gateway to facilitate the integration of various Large Language Models (LLMs) into your applications, including [Perplexity AI APIs](https://docs.perplexity.ai/reference/post%5Fchat%5Fcompletions). +## Quick Start -With Portkey, you can take advantage of features like fast AI gateway access, observability, prompt management, and more, all while ensuring the secure management of your LLM API keys through a [virtual key](/product/ai-gateway/virtual-keys) system. - -Provider Slug. `perplexity-ai` - -## Portkey SDK Integration with Perplexity AI Models +Get started with Perplexity AI in under 2 minutes: -Portkey provides a consistent API to interact with models from various providers. To integrate Perplexity AI with Portkey: + -### 1\. Install the Portkey SDK +```python Python icon="python" +from portkey_ai import Portkey -Add the Portkey SDK to your application to interact with Perplexity AI's API through Portkey's gateway. - - - ```sh - npm install --save portkey-ai - ``` - - - ```sh - pip install portkey-ai - ``` - +# 1. Install: pip install portkey-ai +# 2. Add @perplexity-ai provider in model catalog +# 3. Use it: - +portkey = Portkey(api_key="PORTKEY_API_KEY") +response = portkey.chat.completions.create( + model="@perplexity-ai/sonar", + messages=[{"role": "user", "content": "Hello!"}] +) +print(response.choices[0].message.content) +``` -### 2\. Initialize Portkey with the Virtual Key +```js Javascript icon="square-js" +import Portkey from 'portkey-ai' -To use Perplexity AI with Portkey, [get your API key from here,](https://www.perplexity.ai/settings/api) then add it to Portkey to create the virtual key. - - - ```js - import Portkey from 'portkey-ai' +// 1. Install: npm install portkey-ai +// 2. Add @perplexity-ai provider in model catalog +// 3. Use it: - const portkey = new Portkey({ - apiKey: "PORTKEY_API_KEY", // defaults to process.env["PORTKEY_API_KEY"] - provider:"@PROVIDER" // Your Perplexity AI Virtual Key - }) - ``` - - - ```python - from portkey_ai import Portkey +const portkey = new Portkey({ + apiKey: "PORTKEY_API_KEY" +}) - portkey = Portkey( - api_key="PORTKEY_API_KEY", # Replace with your Portkey API key - provider="@PROVIDER" # Replace with your virtual key for Perplexity AI - ) - ``` - +const response = await portkey.chat.completions.create({ + model: "@perplexity-ai/sonar", + messages: [{ role: "user", content: "Hello!" }] +}) - +console.log(response.choices[0].message.content) +``` +```python OpenAI Py icon="python" +from openai import OpenAI +from portkey_ai import PORTKEY_GATEWAY_URL -### **3\. Invoke Chat Completions with** Perplexity AI +# 1. Install: pip install openai portkey-ai +# 2. Add @perplexity-ai provider in model catalog +# 3. Use it: -Use the Portkey instance to send requests to Perplexity AI. You can also override the virtual key directly in the API call if needed. - - - ```js - const chatCompletion = await portkey.chat.completions.create({ - messages: [{ role: 'user', content: 'Say this is a test' }], - model: 'pplx-70b-chat', - }); +client = OpenAI( + api_key="PORTKEY_API_KEY", # Portkey API key + base_url=PORTKEY_GATEWAY_URL +) - console.log(chatCompletion.choices); - ``` - - - ```python - completion = portkey.chat.completions.create( - messages= [{ "role": 'user', "content": 'Say this is a test' }], - model= 'pplx-70b-chat' - ) +response = client.chat.completions.create( + model="@perplexity-ai/sonar", + messages=[{"role": "user", "content": "Hello!"}] +) - print(completion) - ``` - +print(response.choices[0].message.content) +``` - +```js OpenAI JS icon="square-js" +import OpenAI from "openai" +import { PORTKEY_GATEWAY_URL } from "portkey-ai" -## Fetching citations +// 1. Install: npm install openai portkey-ai +// 2. Add @perplexity-ai provider in model catalog +// 3. Use it: -If you need to obtain citations in the response, you can disable [strict open ai compliance](/product/ai-gateway/strict-open-ai-compliance) +const client = new OpenAI({ + apiKey: "PORTKEY_API_KEY", // Portkey API key + baseURL: PORTKEY_GATEWAY_URL +}) -## Perplexity-Specific Features +const response = await client.chat.completions.create({ + model: "@perplexity-ai/sonar", + messages: [{ role: "user", content: "Hello!" }] +}) -Perplexity AI offers several unique features that can be accessed through additional parameters in your requests: +console.log(response.choices[0].message.content) +``` + +```sh cURL icon="square-terminal" +# 1. Add @perplexity-ai provider in model catalog +# 2. Use it: + +curl https://api.portkey.ai/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "x-portkey-api-key: $PORTKEY_API_KEY" \ + -d '{ + "model": "@perplexity-ai/sonar", + "messages": [{"role": "user", "content": "Hello!"}] + }' +``` + + + +## Add Provider in Model Catalog + +Before making requests, add Perplexity AI to your Model Catalog: + +1. Go to [**Model Catalog → Add Provider**](https://app.portkey.ai/model-catalog/providers) +2. Select **Perplexity AI** +3. Enter your [Perplexity API key](https://www.perplexity.ai/settings/api) +4. Name your provider (e.g., `perplexity-ai`) + + + See all setup options and detailed configuration instructions + + +--- + +## Perplexity AI Capabilities + +### Citations + +To get citations in responses, disable [strict OpenAI compliance](/product/ai-gateway/strict-open-ai-compliance). ### Search Domain Filter (Beta) -You can limit citations to specific domains using the `search_domain_filter` parameter. This feature is currently in closed beta and limited to 3 domains for whitelisting or blacklisting. - - - - ```python - completion = portkey.chat.completions.create( - messages=[{"role": "user", "content": "Tell me about electric cars"}], - model="pplx-70b-chat", - search_domain_filter=["tesla.com", "ford.com", "-competitors.com"] # Use '-' prefix for blacklisting - ) - ``` - - - ```js - const completion = await portkey.chat.completions.create({ - messages: [{ role: "user", content: "Tell me about electric cars" }], - model: "pplx-70b-chat", - search_domain_filter: ["tesla.com", "ford.com", "-competitors.com"] // Use '-' prefix for blacklisting - }); - ``` - - - -### Image Results (Beta) -Enable image results in responses from online models using the `return_images` parameter: - - - - ```python - completion = portkey.chat.completions.create( - messages=[{"role": "user", "content": "Show me pictures of electric cars"}], - model="pplx-70b-chat", - return_images=True # Feature in closed beta - ) - ``` - - - ```js - const completion = await portkey.chat.completions.create({ - messages: [{ role: "user", content: "Show me pictures of electric cars" }], - model: "pplx-70b-chat", - return_images: true // Feature in closed beta - }); - ``` - - - -### Related Questions (Beta) -Get related questions in the response using the `return_related_questions` parameter: - - - - ```python - completion = portkey.chat.completions.create( - messages=[{"role": "user", "content": "Tell me about electric cars"}], - model="pplx-70b-chat", - return_related_questions=True # Feature in closed beta - ) - ``` - - - ```js - const completion = await portkey.chat.completions.create({ - messages: [{ role: "user", content: "Tell me about electric cars" }], - model: "pplx-70b-chat", - return_related_questions: true // Feature in closed beta - }); - ``` - - + +Limit citations to specific domains: + + + +```python Python +from portkey_ai import Portkey + +portkey = Portkey(api_key="PORTKEY_API_KEY") + +response = portkey.chat.completions.create( + model="@perplexity-ai/sonar", + messages=[{"role": "user", "content": "Tell me about electric cars"}], + search_domain_filter=["tesla.com", "ford.com", "-competitors.com"] # '-' prefix for blacklist +) + +print(response.choices[0].message.content) +``` + +```javascript Node.js +import Portkey from 'portkey-ai'; + +const portkey = new Portkey({ + apiKey: 'PORTKEY_API_KEY' +}); + +const response = await portkey.chat.completions.create({ + model: "@perplexity-ai/sonar", + messages: [{ role: "user", content: "Tell me about electric cars" }], + search_domain_filter: ["tesla.com", "ford.com", "-competitors.com"] // '-' prefix for blacklist +}); + +console.log(response.choices[0].message.content); +``` + + ### Search Recency Filter -Filter search results based on time intervals using the `search_recency_filter` parameter: - - - - ```python - completion = portkey.chat.completions.create( - messages=[{"role": "user", "content": "What are the latest developments in electric cars?"}], - model="pplx-70b-chat", - search_recency_filter="week" # Options: month, week, day, hour - ) - ``` - - - ```js - const completion = await portkey.chat.completions.create({ - messages: [{ role: "user", content: "What are the latest developments in electric cars?" }], - model: "pplx-70b-chat", - search_recency_filter: "week" // Options: month, week, day, hour - }); - ``` - - +Filter results by time interval: + + + +```python Python +from portkey_ai import Portkey + +portkey = Portkey(api_key="PORTKEY_API_KEY") + +response = portkey.chat.completions.create( + model="@perplexity-ai/sonar", + messages=[{"role": "user", "content": "Latest developments in AI?"}], + search_recency_filter="week" # Options: month, week, day, hour +) + +print(response.choices[0].message.content) +``` + +```javascript Node.js +import Portkey from 'portkey-ai'; + +const portkey = new Portkey({ + apiKey: 'PORTKEY_API_KEY' +}); + +const response = await portkey.chat.completions.create({ + model: "@perplexity-ai/sonar", + messages: [{ role: "user", content: "Latest developments in AI?" }], + search_recency_filter: "week" // Options: month, week, day, hour +}); + +console.log(response.choices[0].message.content); +``` + + ### Web Search Options -Determines how much search context is retrieved for the model. -Options are: -- `low`: minimizes context for cost savings but less comprehensive answers. -- `medium`: balanced approach suitable for most queries. -- `high`: maximizes context for comprehensive answers but at higher cost. - - - - ```python - completion = portkey.chat.completions.create( - messages=[{"role": "user", "content": "What are the latest developments in electric cars?"}], - model="sonar", - web_search_options={ - "search_context_size": "high" - } - - ) - ``` - - - ```js - const completion = await portkey.chat.completions.create({ - messages: [{ role: "user", content: "What are the latest developments in electric cars?" }], - model: "sonar", - web_search_options: { - "search_context_size": "high" - } - }); - ``` - - +Control search context size: + -### Search Recency Filter -Filters search results based on time (e.g., 'week', 'day'). +```python Python +from portkey_ai import Portkey - - - ```python - completion = portkey.chat.completions.create( - messages=[{"role": "user", "content": "What are the latest developments in electric cars?"}], - model="sonar", - "search_recency_filter": "", - ) - ``` - - - ```js - const completion = await portkey.chat.completions.create({ - messages: [{ role: "user", content: "What are the latest developments in electric cars?" }], - model: "sonar", - search_recency_filter: "", - }); - ``` - - +portkey = Portkey(api_key="PORTKEY_API_KEY") +response = portkey.chat.completions.create( + model="@perplexity-ai/sonar", + messages=[{"role": "user", "content": "Latest developments in AI?"}], + web_search_options={ + "search_context_size": "high" # Options: low, medium, high + } +) +print(response.choices[0].message.content) +``` -## Managing Perplexity AI Prompts +```javascript Node.js +import Portkey from 'portkey-ai'; -You can manage all prompts to Perplexity AI in the [Prompt Library](/product/prompt-library). All the current models of Perplexity AI are supported and you can easily start testing different prompts. +const portkey = new Portkey({ + apiKey: 'PORTKEY_API_KEY' +}); -Once you're ready with your prompt, you can use the `portkey.prompts.completions.create` interface to use the prompt in your application. +const response = await portkey.chat.completions.create({ + model: "@perplexity-ai/sonar", + messages: [{ role: "user", content: "Latest developments in AI?" }], + web_search_options: { + search_context_size: "high" // Options: low, medium, high + } +}); -## Next Steps +console.log(response.choices[0].message.content); +``` -The complete list of features supported in the SDK are available on the link below. - - + + +### Return Images (Beta) + +Enable image results in responses: + + + +```python Python +from portkey_ai import Portkey + +portkey = Portkey(api_key="PORTKEY_API_KEY") + +response = portkey.chat.completions.create( + model="@perplexity-ai/sonar", + messages=[{"role": "user", "content": "Show me pictures of electric cars"}], + return_images=True +) + +print(response.choices[0].message.content) +``` + +```javascript Node.js +import Portkey from 'portkey-ai'; + +const portkey = new Portkey({ + apiKey: 'PORTKEY_API_KEY' +}); + +const response = await portkey.chat.completions.create({ + model: "@perplexity-ai/sonar", + messages: [{ role: "user", content: "Show me pictures of electric cars" }], + return_images: true +}); + +console.log(response.choices[0].message.content); +``` + + + +### Return Related Questions (Beta) + +Get related questions in the response: + + + +```python Python +from portkey_ai import Portkey + +portkey = Portkey(api_key="PORTKEY_API_KEY") -You'll find more information in the relevant sections: +response = portkey.chat.completions.create( + model="@perplexity-ai/sonar", + messages=[{"role": "user", "content": "Tell me about electric cars"}], + return_related_questions=True +) -1. [Add metadata to your requests](/product/observability/metadata) -2. [Add gateway configs to your Perplexity AI](/product/ai-gateway/configs)[ requests](/product/ai-gateway/configs) -3. [Tracing Perplexity AI requests](/product/observability/traces) -4. [Setup a fallback from OpenAI to Perplexity AI APIs](/product/ai-gateway/fallbacks) +print(response.choices[0].message.content) +``` + +```javascript Node.js +import Portkey from 'portkey-ai'; + +const portkey = new Portkey({ + apiKey: 'PORTKEY_API_KEY' +}); + +const response = await portkey.chat.completions.create({ + model: "@perplexity-ai/sonar", + messages: [{ role: "user", content: "Tell me about electric cars" }], + return_related_questions: true +}); + +console.log(response.choices[0].message.content); +``` + + + +--- + +## Supported Models + +Perplexity offers online reasoning models with real-time search capabilities: + +| Model | Context Length | Description | +|-------|---------------|-------------| +| sonar | 127,072 tokens | Latest Sonar model with real-time search | +| sonar-pro | 127,072 tokens | Advanced reasoning with deeper search | + +Check [Perplexity's documentation](https://docs.perplexity.ai/) for the latest models. + +--- + +## Next Steps + + + + Add fallbacks, load balancing, and more + + + Monitor and trace your Perplexity requests + + + Manage and version your prompts + + + Add custom metadata to requests + + + +For complete SDK documentation: + + + Complete Portkey SDK documentation + diff --git a/integrations/llms/predibase.mdx b/integrations/llms/predibase.mdx index 32c40e78..6ffaef98 100644 --- a/integrations/llms/predibase.mdx +++ b/integrations/llms/predibase.mdx @@ -1,294 +1,351 @@ --- title: "Predibase" +description: Use Predibase's open-source and fine-tuned LLMs through Portkey. --- -Portkey provides a robust and secure gateway to seamlessly integrate **open-source** and **fine-tuned** LLMs from Predibase into your applications. With Portkey, you can leverage powerful features like fast AI gateway, caching, observability, prompt management, and more, while securely managing your LLM API keys through a virtual key system. - -Provider Slug. `predibase` - -## Portkey SDK Integration with Predibase - -Using Portkey, you can call your Predibase models in the familar **OpenAI-spec** and try out your existing pipelines on Predibase fine-tuned models with 2 LOC change. - -### 1\. Install the Portkey SDK - -Install the Portkey SDK in your project using npm or pip: - - - ```sh - npm install --save portkey-ai - ``` - - -```sh -pip install portkey-ai +## Quick Start + +Get started with Predibase in under 2 minutes: + + + +```python Python icon="python" +from portkey_ai import Portkey + +# 1. Install: pip install portkey-ai +# 2. Add @predibase provider in model catalog +# 3. Use it: + +portkey = Portkey(api_key="PORTKEY_API_KEY") + +response = portkey.chat.completions.create( + model="@predibase/llama-3-8b-instruct", + messages=[{"role": "user", "content": "Hello!"}] +) + +print(response.choices[0].message.content) ``` - - +```js Javascript icon="square-js" +import Portkey from 'portkey-ai' +// 1. Install: npm install portkey-ai +// 2. Add @predibase provider in model catalog +// 3. Use it: +const portkey = new Portkey({ + apiKey: "PORTKEY_API_KEY" +}) +const response = await portkey.chat.completions.create({ + model: "@predibase/llama-3-8b-instruct", + messages: [{ role: "user", content: "Hello!" }] +}) -### 2\. Initialize Portkey with the Virtual Key +console.log(response.choices[0].message.content) +``` -To use Predibase with Portkey, [get your API key from here](https://app.predibase.com/settings), then add it to Portkey to create the virtual key. - - - ```js - import Portkey from 'portkey-ai' +```python OpenAI Py icon="python" +from openai import OpenAI +from portkey_ai import PORTKEY_GATEWAY_URL - const portkey = new Portkey({ - apiKey: "PORTKEY_API_KEY", // defaults to process.env["PORTKEY_API_KEY"] - provider:"@PROVIDER" // Your Predibase Virtual Key - }) - ``` - - -```python -from portkey_ai import Portkey +# 1. Install: pip install openai portkey-ai +# 2. Add @predibase provider in model catalog +# 3. Use it: + +client = OpenAI( + api_key="PORTKEY_API_KEY", # Portkey API key + base_url=PORTKEY_GATEWAY_URL +) -portkey = Portkey( - api_key="PORTKEY_API_KEY", # Replace with your Portkey API key - provider="@PROVIDER" # Replace with your virtual key for Predibase +response = client.chat.completions.create( + model="@predibase/llama-3-8b-instruct", + messages=[{"role": "user", "content": "Hello!"}] ) + +print(response.choices[0].message.content) ``` - - - -```js -import OpenAI from "openai"; -import { PORTKEY_GATEWAY_URL, createHeaders } from "portkey-ai"; - -const portkey = new OpenAI({ - baseURL: PORTKEY_GATEWAY_URL, - defaultHeaders: createHeaders({ - apiKey: "PORTKEY_API_KEY", - provider:"@PREDIBASE_PROVIDER", - }), -}); + +```js OpenAI JS icon="square-js" +import OpenAI from "openai" +import { PORTKEY_GATEWAY_URL } from "portkey-ai" + +// 1. Install: npm install openai portkey-ai +// 2. Add @predibase provider in model catalog +// 3. Use it: + +const client = new OpenAI({ + apiKey: "PORTKEY_API_KEY", // Portkey API key + baseURL: PORTKEY_GATEWAY_URL +}) + +const response = await client.chat.completions.create({ + model: "@predibase/llama-3-8b-instruct", + messages: [{ role: "user", content: "Hello!" }] +}) + +console.log(response.choices[0].message.content) ``` - - -```python -from openai import OpenAI -from portkey_ai import PORTKEY_GATEWAY_URL, createHeaders - -portkey = OpenAI( - base_url=PORTKEY_GATEWAY_URL, - default_headers=createHeaders( - api_key="PORTKEY_API_KEY", - provider="@PREDIBASE_PROVIDER" - ) -) + +```sh cURL icon="square-terminal" +# 1. Add @predibase provider in model catalog +# 2. Use it: + +curl https://api.portkey.ai/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "x-portkey-api-key: $PORTKEY_API_KEY" \ + -d '{ + "model": "@predibase/llama-3-8b-instruct", + "messages": [{"role": "user", "content": "Hello!"}] + }' ``` - - + + +## Add Provider in Model Catalog + +Before making requests, add Predibase to your Model Catalog: + +1. Go to [**Model Catalog → Add Provider**](https://app.portkey.ai/model-catalog/providers) +2. Select **Predibase** +3. Enter your [Predibase API key](https://app.predibase.com/settings) +4. Name your provider (e.g., `predibase`) + + See all setup options and detailed configuration instructions + + +--- +## Predibase Capabilities -### 3\. Invoke Chat Completions on Predibase Serverless Endpoints +### Serverless Endpoints + +Predibase offers LLMs like **Llama 3**, **Mistral**, **Gemma**, etc. on its [serverless infrastructure](https://docs.predibase.com/user-guide/inference/models#serverless-endpoints) that you can query instantly. -Predibase offers LLMs like **Llama 3**, **Mistral**, **Gemma**, etc. on its [serverless infra](https://docs.predibase.com/user-guide/inference/models#serverless-endpoints) that you can query instantly. -#### Sending Predibase Tenand ID +**Sending Predibase Tenant ID** -Predibase expects your **account tenant ID** along with the API key in each request. With Portkey, you can send [**your Tenand ID**](https://app.predibase.com/settings) with the `user` param while making your request. +Predibase expects your **account tenant ID** along with the API key in each request. With Portkey, you can send [**your Tenant ID**](https://app.predibase.com/settings) with the `user` param while making your request. - - - - ```js - const chatCompletion = await portkey.chat.completions.create({ - messages: [{ role: 'user', content: 'Say this is a test' }], - model: 'llama-3-8b ', - user: 'PREDIBASE_TENANT_ID' - }); - - console.log(chatCompletion.choices); - ``` - - -```python -completion = portkey.chat.completions.create( - messages= [{ "role": 'user', "content": 'Say this is a test' }], - model= 'llama-3-8b', - user= "PREDIBASE_TENANT_ID" + + + +```python Python +from portkey_ai import Portkey + +portkey = Portkey(api_key="PORTKEY_API_KEY", provider="@predibase") + +response = portkey.chat.completions.create( + model="llama-3-8b-instruct", + messages=[{"role": "user", "content": "Hello!"}], + user="PREDIBASE_TENANT_ID" # Required: Your Predibase tenant ID ) -print(completion) +print(response.choices[0].message.content) ``` - - - ```sh - curl https://api.portkey.ai/v1/chat/completions \ - -H "Content-Type: application/json" \ - -H "x-portkey-api-key: $PORTKEY_API_KEY" \ - -H "x-portkey-provider: $PREDIBASE_PROVIDER" \ - -d '{ - "messages": [{"role": "user","content": "Hello!"}], - "model": "llama-3-8b", - "user": "PREDIBASE_TENANT_ID" - }' - ``` - - +```javascript Node.js +import Portkey from 'portkey-ai'; +const portkey = new Portkey({ + apiKey: 'PORTKEY_API_KEY', + provider: '@predibase' +}); +const response = await portkey.chat.completions.create({ + model: "llama-3-8b-instruct", + messages: [{ role: "user", content: "Hello!" }], + user: "PREDIBASE_TENANT_ID" // Required: Your Predibase tenant ID +}); -### 4\. Invoke Predibase Fine-Tuned Models +console.log(response.choices[0].message.content); +``` + + + +### Using Fine-Tuned Models + +Predibase allows you to deploy and use fine-tuned models with adapters. Use the special format with your model identifier from your Predibase dashboard: -With Portkey, you can send your fine-tune model & adapter details directly with the `model` param while making a request. -The format is: +**Fine-Tuned Model Format:** +`model = base_model:adapter-repo-name/adapter-version-number` -`model = :` +For example: `llama-3-8b:sentiment-analysis/1` -For example, if your base model is `llama-3-8b` and the adapter repo name is `sentiment-analysis`, you can make a request like this: - - - - ```js - const chatCompletion = await portkey.chat.completions.create({ - messages: [{ role: 'user', content: 'Say this is a test' }], - model: 'llama-3-8b:sentiment-analysis/1', - user: 'PREDIBASE_TENANT_ID' - }); - - console.log(chatCompletion.choices); - ``` - - -```python -completion = portkey.chat.completions.create( - messages= [{ "role": 'user', "content": 'Say this is a test' }], - model= 'llama-3-8b:sentiment-analysis/1', - user= "PREDIBASE_TENANT_ID" + + +```python Python +from portkey_ai import Portkey + +portkey = Portkey(api_key="PORTKEY_API_KEY", provider="@predibase") + +response = portkey.chat.completions.create( + model="llama-3-8b:sentiment-analysis/1", # Base model + adapter + messages=[{"role": "user", "content": "This product is amazing!"}], + user="PREDIBASE_TENANT_ID" ) -print(completion) +print(response.choices[0].message.content) ``` - - - ```sh - curl https://api.portkey.ai/v1/chat/completions \ - -H "Content-Type: application/json" \ - -H "x-portkey-api-key: $PORTKEY_API_KEY" \ - -H "x-portkey-provider: $PREDIBASE_PROVIDER" \ - -d '{ - "messages": [{"role": "user","content": "Hello!"}], - "model": "llama-3-8b:sentiment-analysis/1", - "user": "PREDIBASE_TENANT_ID" - }' - ``` - - +```javascript Node.js +import Portkey from 'portkey-ai'; +const portkey = new Portkey({ + apiKey: 'PORTKEY_API_KEY', + provider: '@predibase' +}); +const response = await portkey.chat.completions.create({ + model: "llama-3-8b:sentiment-analysis/1", // Base model + adapter + messages: [{ role: "user", content: "This product is amazing!" }], + user: "PREDIBASE_TENANT_ID" +}); ---- +console.log(response.choices[0].message.content); +``` -### Routing to Dedicated Deployments + -Using Portkey, you can easily route to your dedicatedly deployed models as well. Just pass the dedicated deployment name in the `model` param: +### Dedicated Deployments + +Route requests to your dedicated deployed models by passing the deployment name in the `model` parameter: + + + +```python Python +from portkey_ai import Portkey + +portkey = Portkey(api_key="PORTKEY_API_KEY", provider="@predibase") + +response = portkey.chat.completions.create( + model="my-dedicated-mistral-deployment", # Your deployment name + messages=[{"role": "user", "content": "Hello!"}], + user="PREDIBASE_TENANT_ID" +) + +print(response.choices[0].message.content) +``` + +```javascript Node.js +import Portkey from 'portkey-ai'; + +const portkey = new Portkey({ + apiKey: 'PORTKEY_API_KEY', + provider: '@predibase' +}); + +const response = await portkey.chat.completions.create({ + model: "my-dedicated-mistral-deployment", // Your deployment name + messages: [{ role: "user", content: "Hello!" }], + user: "PREDIBASE_TENANT_ID" +}); + +console.log(response.choices[0].message.content); +``` + + - -`model = "my-dedicated-mistral-deployment-name"` - ### JSON Schema Mode -You can enforce JSON schema for all Predibase models - just set the `response_format` to `json_object` and pass the relevant schema while making your request. Portkey logs will show your JSON output separately - - - - ```js - const chatCompletion = await portkey.chat.completions.create({ - messages: [{ role: 'user', content: 'Say this is a test' }], - model: 'llama-3-8b ', - user: 'PREDIBASE_TENANT_ID', - response_format: { - "type": "json_object", - "schema": {"properties": { - "name": {"maxLength": 10, "title": "Name", "type": "string"}, - "age": {"title": "Age", "type": "integer"}, - "required": ["name", "age", "strength"], - "title": "Character", - "type": "object" - } - } - }); - - console.log(chatCompletion.choices); - ``` - - -```python -# Using Pydantic to define the schema +Enforce JSON schema for all Predibase models by setting `response_format` to `json_object` with your schema: + + + +```python Python +from portkey_ai import Portkey from pydantic import BaseModel, constr -# Define JSON Schema +# Define JSON Schema with Pydantic class Character(BaseModel): name: constr(max_length=10) age: int strength: int -completion = portkey.chat.completions.create( - messages= [{ "role": 'user', "content": 'Say this is a test' }], - model= 'llama-3-8b', - user= "PREDIBASE_TENANT_ID", +portkey = Portkey(api_key="PORTKEY_API_KEY", provider="@predibase") + +response = portkey.chat.completions.create( + model="llama-3-8b", + messages=[{"role": "user", "content": "Create a character profile"}], + user="PREDIBASE_TENANT_ID", response_format={ "type": "json_object", - "schema": Character.schema(), - }, + "schema": Character.schema() + } ) -print(completion) +print(response.choices[0].message.content) ``` - - - ```sh - curl https://api.portkey.ai/v1/chat/completions \ - -H "Content-Type: application/json" \ - -H "x-portkey-api-key: $PORTKEY_API_KEY" \ - -H "x-portkey-provider: $PREDIBASE_PROVIDER" \ - -d '{ - "messages": [{"role": "user","content": "Hello!"}], - "model": "llama-3-8b", - "user": "PREDIBASE_TENANT_ID", - "response_format": { - "type": "json_object", - "schema": {"properties": { - "name": {"maxLength": 10, "title": "Name", "type": "string"}, - "age": {"title": "Age", "type": "integer"}, - "required": ["name", "age", "strength"], - "title": "Character", - "type": "object" - } - } - }' - ``` - - +```javascript Node.js +import Portkey from 'portkey-ai'; + +const portkey = new Portkey({ + apiKey: 'PORTKEY_API_KEY', + provider: '@predibase' +}); + +const response = await portkey.chat.completions.create({ + model: "llama-3-8b", + messages: [{ role: "user", content: "Create a character profile" }], + user: "PREDIBASE_TENANT_ID", + response_format: { + type: "json_object", + schema: { + properties: { + name: { maxLength: 10, title: "Name", type: "string" }, + age: { title: "Age", type: "integer" }, + strength: { title: "Strength", type: "integer" } + }, + required: ["name", "age", "strength"], + title: "Character", + type: "object" + } + } +}); +console.log(response.choices[0].message.content); +``` + --- -## Next Steps +## Supported Models -The complete list of features supported in the SDK are available on the link below. +Predibase provides access to various open-source and fine-tuned models: - - +- Llama 3 (various sizes) +- Mistral +- Zephyr +- Your custom fine-tuned models -You'll find more information in the relevant sections: +Check [Predibase's documentation](https://docs.predibase.com/) for the complete model list and fine-tuning options. -1. [Add metadata to your requests](/product/observability/metadata) -2. [Add gateway configs to your Predibase requests](/product/ai-gateway/configs) -3. [Tracing Predibase requests](/product/observability/traces) -4. [Setup a fallback from OpenAI to Predibase](/product/ai-gateway/fallbacks) +--- + +## Next Steps + + + + Add fallbacks, load balancing, and more + + + Monitor and trace your Predibase requests + + + Manage and version your prompts + + + Track performance of fine-tuned models + + + +For complete SDK documentation: + + + Complete Portkey SDK documentation + diff --git a/integrations/llms/recraft-ai.mdx b/integrations/llms/recraft-ai.mdx index cbc0ca84..1a1e8070 100644 --- a/integrations/llms/recraft-ai.mdx +++ b/integrations/llms/recraft-ai.mdx @@ -1,147 +1,161 @@ --- title: "Recraft AI" +description: Use Recraft AI's advanced image generation models through Portkey. --- -Portkey provides a robust and secure gateway to facilitate the integration of various Large Language Models (LLMs) into your applications, including [Recraft AI APIs](https://www.recraft.ai/docs). +## Quick Start -With Portkey, you can take advantage of features like fast AI gateway access, observability, prompt management, and more, all while ensuring the secure management of your LLM API keys through a [virtual key](/product/ai-gateway/virtual-keys) system. - -Provider Slug. `recraft-ai` - -## Portkey SDK Integration with Recraft AI Models - -Portkey provides a consistent API to interact with models from various providers. Recraft AI currently has - the following models available for integration: -- `recraftv3` - - `recraftv2` - -## Image Generation on Recraft AI using Portkey - -Portkey supports the OpenAI signature to make text-to-image requests. - - -```js -import Portkey from 'portkey-ai'; +Get started with Recraft AI image generation in under 2 minutes: -// Initialize the Portkey client -const portkey = new Portkey({ - apiKey: "PORTKEY_API_KEY", // Replace with your Portkey API key - provider: "recraft-ai", - Authorization: "RECRAFT_API_KEY" -}); - -async function main() { - const image = await portkey.images.generate({ - model: "recraftv3", - prompt: "Lucy in the sky with diamonds", - style: 'digital_illustration', - }); - - console.log(image.data); -} - -main(); -``` - - + -```py +```python Python icon="python" from portkey_ai import Portkey -from IPython.display import display, Image -# Initialize the Portkey client -portkey = Portkey( - api_key="PORTKEY_API_KEY", # Replace with your Portkey API key - provider= "recraft-ai", - Authorization= "RECRAFT_API_KEY" -) +# 1. Install: pip install portkey-ai +# 2. Add @recraft-ai provider in model catalog +# 3. Use it: -response = client.images.generate( - model="recraftv3", - prompt='race car on a track', - style='digital_illustration', +portkey = Portkey(api_key="PORTKEY_API_KEY") + +image = portkey.images.generate( + model="@recraft-ai/recraftv3", + prompt="A serene mountain landscape at sunset", + style="digital_illustration" ) -print(response.data[0].url) +print(image.data[0].url) ``` - - -```js -import OpenAI from 'openai'; // We're using the v4 SDK -import { PORTKEY_GATEWAY_URL, createHeaders } from 'portkey-ai' +```js Javascript icon="square-js" +import Portkey from 'portkey-ai' -const client = new OpenAI({ - apiKey: 'RECRAFT_API_KEY', // defaults to process.env["OPENAI_API_KEY"], - baseURL: PORTKEY_GATEWAY_URL, - defaultHeaders: createHeaders({ - provider: "recraft-ai", - apiKey: "PORTKEY_API_KEY" // defaults to process.env["PORTKEY_API_KEY"] - }) -}); - -async function main() { - const image = await openai.images.generate({ - model: "recraftv3", - prompt: 'race car on a track', - style: 'digital_illustration', - }); - - console.log(image.data); -} - -main(); +// 1. Install: npm install portkey-ai +// 2. Add @recraft-ai provider in model catalog +// 3. Use it: + +const portkey = new Portkey({ + apiKey: "PORTKEY_API_KEY" +}) + +const image = await portkey.images.generate({ + model: "@recraft-ai/recraftv3", + prompt: "A serene mountain landscape at sunset", + style: "digital_illustration" +}) + +console.log(image.data[0].url) ``` - - -```py +```python OpenAI Py icon="python" from openai import OpenAI -from portkey_ai import PORTKEY_GATEWAY_URL, createHeaders -from IPython.display import display, Image +from portkey_ai import PORTKEY_GATEWAY_URL + +# 1. Install: pip install openai portkey-ai +# 2. Add @recraft-ai provider in model catalog +# 3. Use it: client = OpenAI( - api_key='RECRAFT_API_KEY', - base_url=PORTKEY_GATEWAY_URL, - default_headers=createHeaders( - provider="recraft-ai", - api_key="PORTKEY_API_KEY" - ) + api_key="PORTKEY_API_KEY", # Portkey API key + base_url=PORTKEY_GATEWAY_URL ) -response = client.images.generate( - model="recraftv3", - prompt='race car on a track', - style='digital_illustration', +image = client.images.generate( + model="@recraft-ai/recraftv3", + prompt="A serene mountain landscape at sunset", + style="digital_illustration" ) -print(response.data[0].url) -) +print(image.data[0].url) +``` + +```js OpenAI JS icon="square-js" +import OpenAI from "openai" +import { PORTKEY_GATEWAY_URL } from "portkey-ai" + +// 1. Install: npm install openai portkey-ai +// 2. Add @recraft-ai provider in model catalog +// 3. Use it: -# Display the image -display(Image(url=image.data[0].url)) +const client = new OpenAI({ + apiKey: "PORTKEY_API_KEY", // Portkey API key + baseURL: PORTKEY_GATEWAY_URL +}) + +const image = await client.images.generate({ + model: "@recraft-ai/recraftv3", + prompt: "A serene mountain landscape at sunset", + style: "digital_illustration" +}) + +console.log(image.data[0].url) ``` - - - -```sh -curl "https://api.portkey.ai/v1/images/generations" \ - -H "Content-Type: application/json" \ - -H "x-portkey-api-key: $PORTKEY_API_KEY" \ - -H "x-portkey-provider: recraft-ai" \ - -H "Authorization: Bearer $RECRAFT_API_KEY" \ - -d '{ - "prompt": "Lucy in the sky with diamonds", - "style": "digital_illustration" - }' + +```bash cURL +curl https://api.portkey.ai/v1/images/generations \ + -H "Content-Type: application/json" \ + -H "x-portkey-api-key: $PORTKEY_API_KEY" \ + -H "x-portkey-provider: @recraft-ai" \ + -d '{ + "model": "recraftv3", + "prompt": "A serene mountain landscape at sunset", + "style": "digital_illustration" + }' ``` - - + + +## Add Provider in Model Catalog + +Before making requests, add Recraft AI to your Model Catalog: + +1. Go to [**Model Catalog → Add Provider**](https://app.portkey.ai/model-catalog/providers) +2. Select **Recraft AI** +3. Enter your [Recraft AI API key](https://www.recraft.ai/) +4. Name your provider (e.g., `recraft-ai`) + + + See all setup options and detailed configuration instructions + + +--- + +## Supported Models + +Recraft AI provides advanced image generation models: + +| Model | Description | +|-------|-------------| +| recraftv3 | Latest Recraft model with enhanced capabilities | +| recraftv2 | Previous generation Recraft model | + +Check [Recraft AI's documentation](https://www.recraft.ai/docs) for more details. -You'll find more information in the relevant sections: + +Portkey uses the OpenAI image generation signature for Recraft AI, allowing you to easily switch between providers. + + +--- -1. [Add metadata to your requests](/product/observability/metadata) -2. [Add gateway configs to your Recraft AI](/product/ai-gateway/configs)[ requests](/product/ai-gateway/configs) -3. [Tracing Recraft AI requests](/product/observability/traces) -4. [Setup a fallback from OpenAI to Recraft AI APIs](/product/ai-gateway/fallbacks) +## Next Steps + + + + Add fallbacks, load balancing, and more + + + Monitor and trace your Recraft AI requests + + + Cache generated images + + + Complete image generation API docs + + + +For complete SDK documentation: + + + Complete Portkey SDK documentation + diff --git a/integrations/llms/reka-ai.mdx b/integrations/llms/reka-ai.mdx index f2a10de0..d9de1b7a 100644 --- a/integrations/llms/reka-ai.mdx +++ b/integrations/llms/reka-ai.mdx @@ -1,119 +1,151 @@ --- title: "Reka AI" +description: "Integrate Reka AI models with Portkey's AI Gateway" --- -Portkey provides a robust and secure gateway to facilitate the integration of various Large Language Models (LLMs) into your applications, including [Reka AI](https://www.reka.ai/). +Portkey provides a robust and secure gateway to integrate various Large Language Models (LLMs) into applications, including [Reka AI's multimodal models](https://www.reka.ai/). -With Portkey, you can take advantage of features like fast AI gateway access, observability, prompt management, and more, all while ensuring the secure management of your LLM API keys through a [virtual key](/product/ai-gateway/virtual-keys) system. - -Provider Slug. `reka` - -## Portkey SDK Integration with Reka Models +With Portkey, take advantage of features like fast AI gateway access, observability, prompt management, and more, while securely managing API keys through [Model Catalog](/product/model-catalog). -Portkey provides a consistent API to interact with models from various providers. To integrate Reka with Portkey: +## Quick Start -### 1\. Install the Portkey SDK +Get Reka AI working in 3 steps: -Add the Portkey SDK to your application to interact with Reka AI's API through Portkey's gateway. - - - ```sh - npm install --save portkey-ai - ``` - - - ```sh - pip install portkey-ai - ``` - + +```python Python icon="python" +from portkey_ai import Portkey - +# 1. Install: pip install portkey-ai +# 2. Add @reka provider in model catalog +# 3. Use it: +portkey = Portkey(api_key="PORTKEY_API_KEY") +response = portkey.chat.completions.create( + model="@reka/reka-core", + messages=[{"role": "user", "content": "Say this is a test"}] +) +print(response.choices[0].message.content) +``` -### 2\. Initialize Portkey with the Virtual Key +```js Javascript icon="square-js" +import Portkey from 'portkey-ai' -To use Reka AI with Portkey, [get your API key from here,](https://platform.reka.ai/apikeys) then add it to Portkey to create the virtual key. - - - ```js - import Portkey from 'portkey-ai' +// 1. Install: npm install portkey-ai +// 2. Add @reka provider in model catalog +// 3. Use it: - const portkey = new Portkey({ - apiKey: "PORTKEY_API_KEY", // defaults to process.env["PORTKEY_API_KEY"] - provider:"@PROVIDER" // Your Reka AI Virtual Key - }) - ``` - - - ```python - from portkey_ai import Portkey +const portkey = new Portkey({ + apiKey: "PORTKEY_API_KEY" +}) - portkey = Portkey( - api_key="PORTKEY_API_KEY", # Replace with your Portkey API key - provider="@PROVIDER" # Replace with your virtual key for Groq - ) - ``` - +const response = await portkey.chat.completions.create({ + model: "@reka/reka-core", + messages: [{ role: "user", content: "Say this is a test" }] +}) - +console.log(response.choices[0].message.content) +``` +```python OpenAI Py icon="python" +from openai import OpenAI +from portkey_ai import PORTKEY_GATEWAY_URL +# 1. Install: pip install openai portkey-ai +# 2. Add @reka provider in model catalog +# 3. Use it: -### 3\. Invoke Chat Completions with Reka AI +client = OpenAI( + api_key="PORTKEY_API_KEY", # Portkey API key + base_url=PORTKEY_GATEWAY_URL +) -Use the Portkey instance to send requests to Reka AI. You can also override the virtual key directly in the API call if needed. - - - ```js - const chatCompletion = await portkey.chat.completions.create({ - messages: [{ role: 'user', content: 'Say this is a test' }], - model: 'reka-core', - }); +response = client.chat.completions.create( + model="@reka/reka-core", + messages=[{"role": "user", "content": "Say this is a test"}] +) - console.log(chatCompletion.choices);d - ``` +print(response.choices[0].message.content) +``` - - +```js OpenAI JS icon="square-js" +import OpenAI from "openai" +import { PORTKEY_GATEWAY_URL } from "portkey-ai" -```python -completion = portkey.chat.completions.create( - messages= [{ "role": 'user', "content": 'Say this is a test' }], - model= 'reka-core' -) +// 1. Install: npm install openai portkey-ai +// 2. Add @reka provider in model catalog +// 3. Use it: -print(completion) +const client = new OpenAI({ + apiKey: "PORTKEY_API_KEY", // Portkey API key + baseURL: PORTKEY_GATEWAY_URL +}) + +const response = await client.chat.completions.create({ + model: "@reka/reka-core", + messages: [{ role: "user", content: "Say this is a test" }] +}) + +console.log(response.choices[0].message.content) ``` - - +```sh cURL icon="square-terminal" +# 1. Add @reka provider in model catalog +# 2. Use it: + +curl https://api.portkey.ai/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "x-portkey-api-key: $PORTKEY_API_KEY" \ + -d '{ + "model": "@reka/reka-core", + "messages": [ + { "role": "user", "content": "Say this is a test" } + ] + }' +``` + + +**Tip:** You can also set `provider="@reka"` in `Portkey()` and use just `model="reka-core"` in the request. + +## Add Provider in Model Catalog +1. Go to [**Model Catalog → Add Provider**](https://app.portkey.ai/model-catalog/providers) +2. Select **Reka AI** +3. Choose existing credentials or create new by entering your [Reka AI API key](https://platform.reka.ai/apikeys) +4. Name your provider (e.g., `reka-prod`) -## Managing Reka Prompts + + See all setup options, code examples, and detailed instructions + -You can manage all prompts to Reka in the [Prompt Library](/product/prompt-library). All the current models of Reka are supported and you can easily start testing different prompts. +## Managing Reka AI Prompts -Once you're ready with your prompt, you can use the `portkey.prompts.completions.create` interface to use the prompt in your application. +Manage all prompt templates to Reka AI in the [Prompt Library](/product/prompt-library). All current Reka AI models are supported, and you can easily test different prompts. -## Supported Models +Use the `portkey.prompts.completions.create` interface to use the prompt in an application. -| Model Name | Model String to Use in API calls | -| ---------- | -------------------------------- | -| Core | reka-core, reka-core-20240415 | -| Edge | reka-edge, reka-edge-20240208 | -| Flash | reka-flash, reka-flash-20240226 | +## Next Steps -The complete list of features supported in the SDK are available on the link below. - - + + + Add metadata to your Reka AI requests + + + Add gateway configs to your Reka AI requests + + + Trace your Reka AI requests + + + Setup fallback from OpenAI to Reka AI + + -You'll find more information in the relevant sections: +For complete SDK documentation: -1. [Add metadata to your requests](/product/observability/metadata) -2. [Add gateway configs to your Reka](/product/ai-gateway/configs)[ requests](/product/ai-gateway/configs) -3. [Tracing Reka requests](/product/observability/traces) -4. [Setup a fallback from OpenAI to Reka APIs](/product/ai-gateway/fallbacks) + + Complete Portkey SDK documentation + diff --git a/integrations/llms/replicate.mdx b/integrations/llms/replicate.mdx index 8144b2f9..5f3e477b 100644 --- a/integrations/llms/replicate.mdx +++ b/integrations/llms/replicate.mdx @@ -1,181 +1,181 @@ --- title: "Replicate" +description: Use Portkey as a proxy to Replicate for auth management and logging. --- -[Replicate](https://replicate.com/) is a platform for building and running machine learning models. +[Replicate](https://replicate.com/) is a platform for running machine learning models in the cloud. -Replicate does not have a standarized JSON body format for their inference API, hence it is not possible to use unified API to interact with Replicate. -Portkey instead provides a proxy to Replicate, allowing you to manage auth on Portkey and log all Replicate requests. + +Replicate doesn't use a standardized JSON format for their API, so Portkey acts as a proxy, managing authentication and logging all requests. + -## Portkey SDK Integration with Replicate +## Quick Start -To integrate Replicate with Portkey: + -### 1\. Install the Portkey SDK - -Add the Portkey SDK to your application to interact with Replicate through Portkey's gateway. +```python Python icon="python" +from portkey_ai import Portkey - - - ```sh - npm install --save portkey-ai - ``` - - - ```sh - pip install portkey-ai - ``` - - +# 1. Install: pip install portkey-ai +# 2. Add @replicate provider in model catalog +# 3. Use it: -### 2\. Create a New Integration +portkey = Portkey( + api_key="PORTKEY_API_KEY", + provider="@replicate" +) -To use Replicate with Portkey, get your Replicate API key from [here](https://replicate.com/account/api-tokens), then add it to Portkey to create your [Replicate virtual key](/product/ai-gateway/virtual-keys#using-virtual-keys). +response = portkey.post( + url="predictions", + data={ + "version": "MODEL_VERSION_ID", + "input": {"prompt": "Hello, world!"} + } +) - - +print(response) +``` -```js +```js Javascript icon="square-js" import Portkey from 'portkey-ai' +// 1. Install: npm install portkey-ai +// 2. Add @replicate provider in model catalog +// 3. Use it: + const portkey = new Portkey({ - apiKey: "PORTKEY_API_KEY", // defaults to process.env["PORTKEY_API_KEY"] - provider:"@PROVIDER" // Your Replicate Virtual Key + apiKey: "PORTKEY_API_KEY", + provider: "@replicate" }) -``` - - +const response = await portkey.post({ + url: "predictions", + data: { + version: "MODEL_VERSION_ID", + input: { prompt: "Hello, world!" } + } +}) -```python -from portkey_ai import Portkey +console.log(response) +``` -portkey = Portkey( - api_key="PORTKEY_API_KEY", # Replace with your Portkey API key - provider="@PROVIDER" # Replace with your virtual key for Replicate -) +```sh cURL icon="square-terminal" +# 1. Add @replicate provider in model catalog +# 2. Use it: + +curl https://api.portkey.ai/v1/predictions \ + -H "Content-Type: application/json" \ + -H "x-portkey-api-key: $PORTKEY_API_KEY" \ + -d '{ + "version": "MODEL_VERSION_ID", + "input": {"prompt": "Hello, world!"} + }' ``` - - -```python -from openai import OpenAI + -from portkey_ai import PORTKEY_GATEWAY_URL, createHeaders +## Add Provider in Model Catalog -client = OpenAI( - api_key="REPLICATE_API_KEY", - base_url=PORTKEY_GATEWAY_URL, - default_headers=createHeaders( - api_key="PORTKEY_API_KEY", - provider="replicate" - ) -) -``` +Before making requests, add Replicate to your Model Catalog: - - +1. Go to [**Model Catalog → Add Provider**](https://app.portkey.ai/model-catalog/providers) +2. Select **Replicate** +3. Enter your [Replicate API token](https://replicate.com/account/api-tokens) +4. Name your provider (e.g., `replicate`) -```js -import OpenAI from "openai"; + + See all setup options and detailed configuration instructions + -import { PORTKEY_GATEWAY_URL, createHeaders } from "portkey-ai"; +--- -const client = new OpenAI({ - apiKey: "REPLICATE_API_KEY", - baseURL: PORTKEY_GATEWAY_URL, - defaultHeaders: createHeaders({ - provider: "replicate", - apiKey: "PORTKEY_API_KEY", - }), -}); -``` - - +## Using Replicate with Portkey + +Since Replicate doesn't follow the OpenAI format, use Portkey's `post()` method to interact with any Replicate endpoint: -### 3\. Use the Portkey SDK to interact with Replicate + - - - ```python +```python Python from portkey_ai import Portkey portkey = Portkey( - api_key="PORTKEY_API_KEY", # Replace with your Portkey API key - provider="@REPLICATE_PROVIDER", + api_key="PORTKEY_API_KEY", + provider="@replicate" ) +# Run a prediction response = portkey.post( - url="predictions", # Replace with the endpoint you want to call + url="predictions", + data={ + "version": "stability-ai/sdxl:...", + "input": { + "prompt": "A serene landscape" + } + } ) print(response) ``` - - - ```javascript - import Portkey from 'portkey-ai'; - - // Initialize the Portkey client - const portkey = new Portkey({ - apiKey: "PORTKEY_API_KEY", // Replace with your Portkey API key - provider:"@REPLICATE_PROVIDER", // Add your Replicate's virtual key - }); - - response = portkey.post( - url="predictions", # Replace with the endpoint you want to call - ) - - print(response) - ``` - - - ```javascript - import OpenAI from 'openai'; // We're using the v4 SDK - import { PORTKEY_GATEWAY_URL, createHeaders } from 'portkey-ai' - - const openai = new OpenAI({ - apiKey: 'REPLICATE_API_KEY', // defaults to process.env["OPENAI_API_KEY"], - baseURL: PORTKEY_GATEWAY_URL, - defaultHeaders: createHeaders({ - provider:"@REPLICATE_PROVIDER", - apiKey: "PORTKEY_API_KEY", // defaults to process.env["PORTKEY_API_KEY"] - }) - }); - - response = openai.post( - url="predictions", # Replace with the endpoint you want to call - ) - - print(response) - ``` - - - ```python - from openai import OpenAI - from portkey_ai import PORTKEY_GATEWAY_URL, createHeaders - - openai = OpenAI( - api_key='REPLICATE_API_KEY', - base_url=PORTKEY_GATEWAY_URL, - default_headers=createHeaders( - provider="replicate", - api_key="PORTKEY_API_KEY" - ) - ) - - response = openai.post( - url="predictions", # Replace with the endpoint you want to call - ) - - print(response) - ``` - - - ```sh -curl --location --request POST 'https://api.portkey.ai/v1/predictions' \ ---header 'x-portkey-provider: REPLICATE_PROVIDER' \ ---header 'x-portkey-api-key: PORTKEY_API_KEY' - ``` - - + +```javascript Node.js +import Portkey from 'portkey-ai'; + +const portkey = new Portkey({ + apiKey: 'PORTKEY_API_KEY', + provider: '@replicate' +}); + +// Run a prediction +const response = await portkey.post({ + url: "predictions", + data: { + version: "stability-ai/sdxl:...", + input: { + prompt: "A serene landscape" + } + } +}); + +console.log(response); +``` + + + +--- + +## Supported Endpoints + +Portkey proxies all Replicate API endpoints: + +- `/predictions` - Create predictions +- `/predictions/{prediction_id}` - Get prediction status +- `/predictions/{prediction_id}/cancel` - Cancel predictions +- `/models` - List models +- `/collections` - List collections + +See [Replicate's API documentation](https://replicate.com/docs/reference/http) for complete endpoint details. + +--- + +## Next Steps + + + + Monitor and trace your Replicate requests + + + Add custom metadata to requests + + + Cache Replicate responses + + + Add input/output checks + + + +For complete SDK documentation: + + + Complete Portkey SDK documentation + diff --git a/integrations/llms/sambanova.mdx b/integrations/llms/sambanova.mdx index 7d0dd908..a40f0081 100644 --- a/integrations/llms/sambanova.mdx +++ b/integrations/llms/sambanova.mdx @@ -1,102 +1,156 @@ --- title: "SambaNova" -description: "Portkey provides a robust and secure gateway to facilitate the integration of various Large Language Models (LLMs) into your applications, including [SambaNova AI](https://sambanova.ai/)." +description: Use SambaNova's fast inference for Llama and other open-source models through Portkey. --- +## Quick Start -With Portkey, you can take advantage of features like fast AI gateway access, observability, prompt management, and more, all while ensuring the secure management of your LLM API keys through a [virtual key](/product/ai-gateway/virtual-keys) system. - - -Provider Slug: `sambanova` - - -## Portkey SDK Integration with SambaNova Models - -### **1. Install the Portkey SDK** - -Add the Portkey SDK to your application to interact with SambaNova's API through Portkey's gateway. - - - - ```sh - npm install --save portkey-ai - ``` - - - ```sh - pip install portkey-ai - ``` - - - -### **2. Initialize Portkey with the Virtual Key** - - - - ```javascript - import Portkey from 'portkey-ai' - - const portkey = new Portkey({ - apiKey: "PORTKEY_API_KEY", // defaults to process.env["PORTKEY_API_KEY"] - provider:"@PROVIDER" // Your SambaNova Virtual Key - }) - ``` - - - ```python - from portkey_ai import Portkey - portkey = Portkey( - api_key="PORTKEY_API_KEY", # Replace with your Portkey API key - provider="@PROVIDER" # Replace with your virtual key for SambaNova AI - ) - ``` - - - -### **3. Invoke Chat Completions** - -Use the Portkey instance to send requests to the SambaNova API. You can also override the virtual key directly in the API call if needed. - - - - ```javascript - const chatCompletion = await portkey.chat.completions.create({ - messages: [{ role: 'user', content: 'Say this is a test' }], - model: 'Meta-Llama-3.1-405B-Instruct', - }); - console.log(chatCompletion.choices); - ``` - - - ```python - completion = portkey.chat.completions.create( - messages= [{ "role": 'user', "content": 'Say this is a test' }], - model= 'Meta-Llama-3.1-405B-Instruct' - ) - print(completion) - ``` - - - -## Managing SambaNova Prompts - -You can manage all prompts to SambaNova models in the [Prompt Library](/product/prompt-library). All the current models of SambaNova are supported and you can easily start testing different prompts. - -Once you're ready with your prompt, you can use the `portkey.prompts.completions.create` interface to use the prompt in your application. - -The complete list of supported models are available here: - - - View the list of supported SambaNova models +Get started with SambaNova in under 2 minutes: + + + +```python Python icon="python" +from portkey_ai import Portkey + +# 1. Install: pip install portkey-ai +# 2. Add @sambanova provider in model catalog +# 3. Use it: + +portkey = Portkey(api_key="PORTKEY_API_KEY") + +response = portkey.chat.completions.create( + model="@sambanova/Meta-Llama-3.1-405B-Instruct", + messages=[{"role": "user", "content": "Hello!"}] +) + +print(response.choices[0].message.content) +``` + +```js Javascript icon="square-js" +import Portkey from 'portkey-ai' + +// 1. Install: npm install portkey-ai +// 2. Add @sambanova provider in model catalog +// 3. Use it: + +const portkey = new Portkey({ + apiKey: "PORTKEY_API_KEY" +}) + +const response = await portkey.chat.completions.create({ + model: "@sambanova/Meta-Llama-3.1-405B-Instruct", + messages: [{ role: "user", content: "Hello!" }] +}) + +console.log(response.choices[0].message.content) +``` + +```python OpenAI Py icon="python" +from openai import OpenAI +from portkey_ai import PORTKEY_GATEWAY_URL + +# 1. Install: pip install openai portkey-ai +# 2. Add @sambanova provider in model catalog +# 3. Use it: + +client = OpenAI( + api_key="PORTKEY_API_KEY", # Portkey API key + base_url=PORTKEY_GATEWAY_URL +) + +response = client.chat.completions.create( + model="@sambanova/Meta-Llama-3.1-405B-Instruct", + messages=[{"role": "user", "content": "Hello!"}] +) + +print(response.choices[0].message.content) +``` + +```js OpenAI JS icon="square-js" +import OpenAI from "openai" +import { PORTKEY_GATEWAY_URL } from "portkey-ai" + +// 1. Install: npm install openai portkey-ai +// 2. Add @sambanova provider in model catalog +// 3. Use it: + +const client = new OpenAI({ + apiKey: "PORTKEY_API_KEY", // Portkey API key + baseURL: PORTKEY_GATEWAY_URL +}) + +const response = await client.chat.completions.create({ + model: "@sambanova/Meta-Llama-3.1-405B-Instruct", + messages: [{ role: "user", content: "Hello!" }] +}) + +console.log(response.choices[0].message.content) +``` + +```sh cURL icon="square-terminal" +# 1. Add @sambanova provider in model catalog +# 2. Use it: + +curl https://api.portkey.ai/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "x-portkey-api-key: $PORTKEY_API_KEY" \ + -d '{ + "model": "@sambanova/Meta-Llama-3.1-405B-Instruct", + "messages": [{"role": "user", "content": "Hello!"}] + }' +``` + + + +## Add Provider in Model Catalog + +Before making requests, add SambaNova to your Model Catalog: + +1. Go to [**Model Catalog → Add Provider**](https://app.portkey.ai/model-catalog/providers) +2. Select **SambaNova** +3. Enter your SambaNova API key +4. Name your provider (e.g., `sambanova`) + + + See all setup options and detailed configuration instructions -You'll find more information in the relevant sections: +--- + +## Supported Models + +SambaNova provides fast inference for open-source models: + + + View the complete list of SambaNova models + -1. [Add metadata to your requests](/product/observability/metadata) -2. [Add gateway configs to your SambaNova requests](/product/ai-gateway/configs) -3. [Tracing SambaNova requests](/product/observability/traces) -4. [Setup a fallback from OpenAI to SambaNova APIs](/product/ai-gateway/fallbacks) +Popular models include: +- Meta-Llama-3.1-405B-Instruct +- Meta-Llama-3.1-70B-Instruct +- Meta-Llama-3.1-8B-Instruct + +--- + +## Next Steps + + + + Add fallbacks, load balancing, and more + + + Monitor and trace your SambaNova requests + + + Manage and version your prompts + + + Add custom metadata to requests + + + +For complete SDK documentation: + + + Complete Portkey SDK documentation + diff --git a/integrations/llms/segmind.mdx b/integrations/llms/segmind.mdx index e05a3d8b..02a2d00e 100644 --- a/integrations/llms/segmind.mdx +++ b/integrations/llms/segmind.mdx @@ -1,144 +1,162 @@ --- title: "Segmind" +description: Use Segmind's Stable Diffusion models for fast, serverless image generation through Portkey. --- -Portkey provides a robust and secure gateway to facilitate the integration of various Large Language Models (LLMs) into your applications, including [Segmind APIs](https://docs.segmind.com/). +## Quick Start -With Portkey, you can take advantage of features like fast AI gateway access, observability, prompt management, and more, all while ensuring the secure management of your LLM API keys through a [virtual key](/product/ai-gateway/virtual-keys) system. - -Provider Slug. `segmind` - -## Portkey SDK Integration with Segmind - -Portkey provides a consistent API to interact with image generation models from various providers. To integrate Segmind with Portkey: - -### 1\. Install the Portkey SDK - -Add the Portkey SDK to your application to interact with the Segmind API through Portkey's gateway. - - - ```sh - npm install --save portkey-ai - ``` - - -```sh -pip install portkey-ai +Get started with Segmind in under 2 minutes: + + + +```python Python icon="python" +from portkey_ai import Portkey + +# 1. Install: pip install portkey-ai +# 2. Add @segmind provider in model catalog +# 3. Use it: + +portkey = Portkey(api_key="PORTKEY_API_KEY") + +response = portkey.chat.completions.create( + model="@segmind/sdxl1.0-txt2img", + messages=[{"role": "user", "content": "Hello!"}] +) + +print(response.choices[0].message.content) ``` - - +```js Javascript icon="square-js" +import Portkey from 'portkey-ai' +// 1. Install: npm install portkey-ai +// 2. Add @segmind provider in model catalog +// 3. Use it: +const portkey = new Portkey({ + apiKey: "PORTKEY_API_KEY" +}) +const response = await portkey.chat.completions.create({ + model: "@segmind/sdxl1.0-txt2img", + messages: [{ role: "user", content: "Hello!" }] +}) -### 2\. Initialize Portkey with the Virtual Key +console.log(response.choices[0].message.content) +``` -To use Segmind with Portkey, [get your API key from here](https://cloud.segmind.com/console/api-keys), then add it to Portkey to create the virtual key. - - - ```js - import Portkey from 'portkey-ai' +```python OpenAI Py icon="python" +from openai import OpenAI +from portkey_ai import PORTKEY_GATEWAY_URL - const portkey = new Portkey({ - apiKey: "PORTKEY_API_KEY", // defaults to process.env["PORTKEY_API_KEY"] - provider:"@PROVIDER" // Your Segmind Virtual Key - }) - ``` - - -```python -from portkey_ai import Portkey +# 1. Install: pip install openai portkey-ai +# 2. Add @segmind provider in model catalog +# 3. Use it: -portkey = Portkey( - api_key="PORTKEY_API_KEY", # Replace with your Portkey API key - provider="@PROVIDER" # Replace with your virtual key for Segmind +client = OpenAI( + api_key="PORTKEY_API_KEY", # Portkey API key + base_url=PORTKEY_GATEWAY_URL ) -``` - - - - - - -### **3\. Invoke Image Generation with** Segmind - -Use the Portkey instance to send requests to Stability AI. You can also override the virtual key directly in the API call if needed. - - - ```js - const image = await portkey.images.generate({ - model:"sdxl1.0-txt2img", - prompt:"Lucy in the sky with diamonds", - size:"1024x1024" - }) - ``` - - -```py -image = portkey.images.generate( - model="sdxl1.0-txt2img", - prompt="Lucy in the sky with diamonds", - size="1024x1024" + +response = client.chat.completions.create( + model="@segmind/sdxl1.0-txt2img", + messages=[{"role": "user", "content": "Hello!"}] ) + +print(response.choices[0].message.content) ``` - - - - - - -Notice how we're using the OpenAI's image generation signature to prompt Segmind's hosted serverless endpoints allowing greater flexibility to change models and providers later if necessary. - -### Supported Models - -The following models are supported, newer models added to Segmind should also be automatically supported. - -| Model String | Model Name | Extra Keys (if any) | -| ------------------------ | --------------------- | ---------------------------------------------------------------------- | -| sdxl1.0-txt2img | SDXL | | -| sd1.5-526mix | 526 Mix | | -| sd1.5-allinonepixel | All In One Pixel | | -| sd1.5-disneyB | Cartoon | | -| sd1.5-colorful | Colorful | | -| sd1.5-cuterichstyle | Cute Rich Style | | -| sd1.5-cyberrealistic | Cyber Realistic | | -| sd1.5-deepspacediffusion | Deep Spaced Diffusion | | -| sd1.5-dreamshaper | Dream Shaper | | -| sd1.5-dvarch | Dv Arch | | -| sd1.5-edgeofrealism | Edge of Realism | | -| sd1.5-epicrealism | Epic Realism | | -| sd1.5-fantassifiedicons | Fantassified Icons | | -| sd1.5-flat2d | Flat 2D | | -| sd1.5-fruitfusion | Fruit Fusion | | -| sd1.5-icbinp | Icbinp | | -| sd1.5-juggernaut | Juggernaut | | -| kandinsky2.2-txt2img | Kandinsky | | -| sd1.5-majicmix | Majicmix | | -| sd1.5-manmarumix | Manmarumix | | -| sd1.5-paragon | Paragon | | -| potraitsd1.5-txt2img | Potrait SD | | -| qrsd1.5-txt2img | QR Generator | control\_scale, control\_scale, control\_scale, qr\_text, invert, size | -| sd1.5-rcnz | RCNZ | | -| sd1.5-rpg | RPG | | -| sd1.5-realisticvision | Realistic Vision | | -| sd1.5-reliberate | Reliberate | | -| sd1.5-revanimated | Revanimated | | -| sd1.5-samaritan-3d | Samaritan | | -| sd1.5-scifi | SciFi | | -| smallsd1.5-txt2img | Small SD | | -| tinysd1.5-txt2img | Tiny SD | | -## Next Steps +```js OpenAI JS icon="square-js" +import OpenAI from "openai" +import { PORTKEY_GATEWAY_URL } from "portkey-ai" + +// 1. Install: npm install openai portkey-ai +// 2. Add @segmind provider in model catalog +// 3. Use it: + +const client = new OpenAI({ + apiKey: "PORTKEY_API_KEY", // Portkey API key + baseURL: PORTKEY_GATEWAY_URL +}) + +const response = await client.chat.completions.create({ + model: "@segmind/sdxl1.0-txt2img", + messages: [{ role: "user", content: "Hello!" }] +}) + +console.log(response.choices[0].message.content) +``` + +```bash cURL +curl https://api.portkey.ai/v1/images/generations \ + -H "Content-Type: application/json" \ + -H "x-portkey-api-key: $PORTKEY_API_KEY" \ + -H "x-portkey-provider: @segmind" \ + -d '{ + "model": "sdxl1.0-txt2img", + "prompt": "Lucy in the sky with diamonds", + "size": "1024x1024" + }' +``` + + + +## Add Provider in Model Catalog -The complete list of features supported in the SDK are available on the link below. - +Before making requests, add Segmind to your Model Catalog: + +1. Go to [**Model Catalog → Add Provider**](https://app.portkey.ai/model-catalog/providers) +2. Select **Segmind** +3. Enter your [Segmind API key](https://cloud.segmind.com/console/api-keys) +4. Name your provider (e.g., `segmind`) + + + See all setup options and detailed configuration instructions -You'll find more information in the relevant sections: +--- + +## Supported Models + +Segmind provides fast, serverless access to various Stable Diffusion models: -1. [Add metadata to your requests](/product/observability/metadata) -2. [Add gateway configs to your Segmind requests](/product/ai-gateway/configs) -3. [Tracing Segmind's requests](/product/observability/traces) -4. [Setup a fallback from OpenAI to Segmind](/product/ai-gateway/fallbacks) -5. [Image generation API Reference](/provider-endpoints/images/create-image) +| Model | Description | +|-------|-------------| +| sdxl1.0-txt2img | Stable Diffusion XL | +| sd1.5-dreamshaper | Dream Shaper | +| sd1.5-realisticvision | Realistic Vision | +| sd1.5-juggernaut | Juggernaut | +| sd1.5-epicrealism | Epic Realism | +| kandinsky2.2-txt2img | Kandinsky | +| qrsd1.5-txt2img | QR Code Generator | + +And 20+ more models. Check [Segmind's documentation](https://docs.segmind.com/) for the complete list. + + +Portkey uses the OpenAI image generation signature for Segmind, allowing you to easily switch between providers. + + +--- + +## Next Steps + + + + Add fallbacks, load balancing, and more + + + Monitor and trace your Segmind requests + + + Cache generated images + + + Complete image generation API docs + + + +For complete SDK documentation: + + + Complete Portkey SDK documentation + diff --git a/integrations/llms/siliconflow.mdx b/integrations/llms/siliconflow.mdx index ab9186a8..70fa9a97 100644 --- a/integrations/llms/siliconflow.mdx +++ b/integrations/llms/siliconflow.mdx @@ -1,103 +1,149 @@ --- title: "SiliconFlow" -Discroption: "Portkey provides a robust and secure gateway to facilitate the integration of various Large Language Models (LLMs) into your applications, including [SiliconFlow](https://siliconflow.cn/). -" +description: Use SiliconFlow's AI inference platform through Portkey for fast, cost-effective model deployment. --- -With Portkey, you can take advantage of features like fast AI gateway access, observability, prompt management, and more, all while ensuring the secure management of your LLM API keys through a [virtual key](/product/ai-gateway/virtual-keys) system. - - -Provider Slug: `siliconflow` - - -## Portkey SDK Integration with SiliconFlow Models - -Portkey provides a consistent API to interact with models from various providers. To integrate SiliconFlow with Portkey: - -### 1. Install the Portkey SDK - -Add the Portkey SDK to your application to interact with SiliconFlow's API through Portkey's gateway. - - - - ```sh - npm install --save portkey-ai - ``` - - - ```sh - pip install portkey-ai - ``` - - - -### 2. Initialize Portkey with the Virtual Key - -To use SiliconFlow with Portkey, [get your API key from here](https://siliconflow.cn/), then add it to Portkey to create the virtual key. - - - - ```javascript - import Portkey from 'portkey-ai' - - const portkey = new Portkey({ - apiKey: "PORTKEY_API_KEY", // defaults to process.env["PORTKEY_API_KEY"] - provider:"@PROVIDER" // Your Silicon Flow - }) - ``` - - - ```python - from portkey_ai import Portkey - - portkey = Portkey( - api_key="PORTKEY_API_KEY", # Replace with your Portkey API key - provider="@PROVIDER" # Replace with your virtual key for SiliconFlow - ) - ``` - - - -### 3. Invoke Chat Completions with SiliconFlow - -Use the Portkey instance to send requests to SiliconFlow. You can also override the virtual key directly in the API call if needed. - - - - ```javascript - const chatCompletion = await portkey.chat.completions.create({ - messages: [{ role: 'user', content: 'Say this is a test' }], - model: 'deepseek-ai/DeepSeek-V2-Chat', - }); - console.log(chatCompletion.choices); - ``` - - - ```python - completion = portkey.chat.completions.create( - messages= [{ "role": 'user', "content": 'Say this is a test' }], - model= 'deepseek-ai/DeepSeek-V2-Chat' - ) - print(completion) - ``` - - - -## Managing SiliconFlow Prompts - -You can manage all prompts to SiliconFlow in the [Prompt Library](/product/prompt-library). All the current models of SiliconFlow are supported and you can easily start testing different prompts. - -Once you're ready with your prompt, you can use the `portkey.prompts.completions.create` interface to use the prompt in your application. - -The complete list of features supported in the SDK are available on the link below. - - - Explore the Portkey SDK Client documentation +## Quick Start + +Get started with SiliconFlow in under 2 minutes: + + + +```python Python icon="python" +from portkey_ai import Portkey + +# 1. Install: pip install portkey-ai +# 2. Add @siliconflow provider in model catalog +# 3. Use it: + +portkey = Portkey(api_key="PORTKEY_API_KEY") + +response = portkey.chat.completions.create( + model="@siliconflow/deepseek-ai/DeepSeek-V2-Chat", + messages=[{"role": "user", "content": "Hello!"}] +) + +print(response.choices[0].message.content) +``` + +```js Javascript icon="square-js" +import Portkey from 'portkey-ai' + +// 1. Install: npm install portkey-ai +// 2. Add @siliconflow provider in model catalog +// 3. Use it: + +const portkey = new Portkey({ + apiKey: "PORTKEY_API_KEY" +}) + +const response = await portkey.chat.completions.create({ + model: "@siliconflow/deepseek-ai/DeepSeek-V2-Chat", + messages: [{ role: "user", content: "Hello!" }] +}) + +console.log(response.choices[0].message.content) +``` + +```python OpenAI Py icon="python" +from openai import OpenAI +from portkey_ai import PORTKEY_GATEWAY_URL + +# 1. Install: pip install openai portkey-ai +# 2. Add @siliconflow provider in model catalog +# 3. Use it: + +client = OpenAI( + api_key="PORTKEY_API_KEY", # Portkey API key + base_url=PORTKEY_GATEWAY_URL +) + +response = client.chat.completions.create( + model="@siliconflow/deepseek-ai/DeepSeek-V2-Chat", + messages=[{"role": "user", "content": "Hello!"}] +) + +print(response.choices[0].message.content) +``` + +```js OpenAI JS icon="square-js" +import OpenAI from "openai" +import { PORTKEY_GATEWAY_URL } from "portkey-ai" + +// 1. Install: npm install openai portkey-ai +// 2. Add @siliconflow provider in model catalog +// 3. Use it: + +const client = new OpenAI({ + apiKey: "PORTKEY_API_KEY", // Portkey API key + baseURL: PORTKEY_GATEWAY_URL +}) + +const response = await client.chat.completions.create({ + model: "@siliconflow/deepseek-ai/DeepSeek-V2-Chat", + messages: [{ role: "user", content: "Hello!" }] +}) + +console.log(response.choices[0].message.content) +``` + +```sh cURL icon="square-terminal" +# 1. Add @siliconflow provider in model catalog +# 2. Use it: + +curl https://api.portkey.ai/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "x-portkey-api-key: $PORTKEY_API_KEY" \ + -d '{ + "model": "@siliconflow/deepseek-ai/DeepSeek-V2-Chat", + "messages": [{"role": "user", "content": "Hello!"}] + }' +``` + + + +## Add Provider in Model Catalog + +Before making requests, add SiliconFlow to your Model Catalog: + +1. Go to [**Model Catalog → Add Provider**](https://app.portkey.ai/model-catalog/providers) +2. Select **SiliconFlow** +3. Enter your [SiliconFlow API key](https://siliconflow.cn/) +4. Name your provider (e.g., `siliconflow`) + + + See all setup options and detailed configuration instructions -You'll find more information in the relevant sections: +--- + +## Supported Models + +SiliconFlow provides cost-effective inference for various models: -1. [Add metadata to your requests](/product/observability/metadata) -2. [Add gateway configs to your SiliconFlow requests](/product/ai-gateway/configs) -3. [Tracing SiliconFlow requests](/product/observability/traces) -4. [Setup a fallback from OpenAI to SiliconFlow APIs](/product/ai-gateway/fallbacks) +Check [SiliconFlow's documentation](https://siliconflow.cn/) for the complete model list. + +--- + +## Next Steps + + + + Add fallbacks, load balancing, and more + + + Monitor and trace your SiliconFlow requests + + + Manage and version your prompts + + + Add custom metadata to requests + + + +For complete SDK documentation: + + + Complete Portkey SDK documentation + diff --git a/integrations/llms/snowflake-cortex.mdx b/integrations/llms/snowflake-cortex.mdx index 4a0e6955..b7883c95 100644 --- a/integrations/llms/snowflake-cortex.mdx +++ b/integrations/llms/snowflake-cortex.mdx @@ -1,118 +1,154 @@ --- title: "Snowflake Cortex" +description: Use Snowflake Cortex AI models through Portkey for enterprise data cloud AI. --- -Portkey provides a robust and secure gateway to facilitate the integration of various Large Language Models (LLMs) into your applications, including [Snowlfake Cortex APIs](https://docs.snowflake.com/). +## Quick Start -With Portkey, you can take advantage of features like fast AI gateway access, observability, prompt management, and more, all while ensuring the secure management of your LLM API keys through a [virtual key](/product/ai-gateway/virtual-keys) system. - -Provider Slug. `cortex` - -## Portkey SDK Integration with Snowflake Cortex Models +Get started with Snowflake Cortex in under 2 minutes: -Portkey provides a consistent API to interact with models from various providers. To integrate Snowflake Cortex with Portkey: + -### 1\. Install the Portkey SDK +```python Python icon="python" +from portkey_ai import Portkey -Add the Portkey SDK to your application to interact with Snowflake Cortex AI's API through Portkey's gateway. +# 1. Install: pip install portkey-ai +# 2. Add @cortex provider in model catalog +# 3. Use it: - - - ```sh - npm install --save portkey-ai - ``` - - - ```sh - pip install portkey-ai - ``` - +portkey = Portkey(api_key="PORTKEY_API_KEY") - - - - - -### 2\. Initialize Portkey with the Virtual Key - -To use Snowflake Cortex with Portkey, get your API key/JWT Token from the Snowflake Platform, then add it to Portkey to create the virtual key. +response = portkey.chat.completions.create( + model="@cortex/claude-3-5-sonnet", + messages=[{"role": "user", "content": "Hello!"}] +) - - +print(response.choices[0].message.content) +``` -```js +```js Javascript icon="square-js" import Portkey from 'portkey-ai' +// 1. Install: npm install portkey-ai +// 2. Add @cortex provider in model catalog +// 3. Use it: + const portkey = new Portkey({ - apiKey: "PORTKEY_API_KEY", // defaults to process.env["PORTKEY_API_KEY"] - provider:"@PROVIDER" // Your Snowflake Cortex Virtual Key + apiKey: "PORTKEY_API_KEY" }) -``` - - - ```python - from portkey_ai import Portkey - portkey = Portkey( - api_key="PORTKEY_API_KEY", # Replace with your Portkey API key - provider="@PROVIDER" # Replace with your virtual key for Snowflake Cortex - ) - ``` +const response = await portkey.chat.completions.create({ + model: "@cortex/claude-3-5-sonnet", + messages: [{ role: "user", content: "Hello!" }] +}) - +console.log(response.choices[0].message.content) +``` - +```python OpenAI Py icon="python" +from openai import OpenAI +from portkey_ai import PORTKEY_GATEWAY_URL +# 1. Install: pip install openai portkey-ai +# 2. Add @cortex provider in model catalog +# 3. Use it: +client = OpenAI( + api_key="PORTKEY_API_KEY", # Portkey API key + base_url=PORTKEY_GATEWAY_URL +) -### **3\. Invoke Chat Completions with** Snowflake Cortex +response = client.chat.completions.create( + model="@cortex/claude-3-5-sonnet", + messages=[{"role": "user", "content": "Hello!"}] +) -Use the Portkey instance to send requests to Snowflake Cortex. You can also override the virtual key directly in the API call if needed. +print(response.choices[0].message.content) +``` - - +```js OpenAI JS icon="square-js" +import OpenAI from "openai" +import { PORTKEY_GATEWAY_URL } from "portkey-ai" - ```js - const chatCompletion = await portkey.chat.completions.create({ - messages: [{ role: 'user', content: 'Say this is a test' }], - model: 'claude-3-5-sonnet', - }); +// 1. Install: npm install openai portkey-ai +// 2. Add @cortex provider in model catalog +// 3. Use it: - console.log(chatCompletion.choices); - ``` - - +const client = new OpenAI({ + apiKey: "PORTKEY_API_KEY", // Portkey API key + baseURL: PORTKEY_GATEWAY_URL +}) -```python -completion = portkey.chat.completions.create( - messages= [{ "role": 'user', "content": 'Say this is a test' }], - model= 'claude-3-5-sonnet' -) +const response = await client.chat.completions.create({ + model: "@cortex/claude-3-5-sonnet", + messages: [{ role: "user", content: "Hello!" }] +}) -print(completion) +console.log(response.choices[0].message.content) ``` - +```sh cURL icon="square-terminal" +# 1. Add @cortex provider in model catalog +# 2. Use it: + +curl https://api.portkey.ai/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "x-portkey-api-key: $PORTKEY_API_KEY" \ + -d '{ + "model": "@cortex/claude-3-5-sonnet", + "messages": [{"role": "user", "content": "Hello!"}] + }' +``` - + +## Add Provider in Model Catalog -## Managing Snowflake Cortex Prompts +Before making requests, add Snowflake Cortex to your Model Catalog: -You can manage all prompts to Snowflake Cortex in the [Prompt Library](/product/prompt-library). All the current models of Snowflake Cortex are supported and you can easily start testing different prompts. +1. Go to [**Model Catalog → Add Provider**](https://app.portkey.ai/model-catalog/providers) +2. Select **Snowflake Cortex** +3. Enter your API key/JWT Token from Snowflake +4. Name your provider (e.g., `cortex`) -Once you're ready with your prompt, you can use the `portkey.prompts.completions.create` interface to use the prompt in your application. + + See all setup options and detailed configuration instructions + +--- +## Supported Models +Snowflake Cortex provides access to various AI models through the Snowflake platform: +- Claude (Anthropic) +- Llama (Meta) +- Mistral +- And other popular models +Check [Snowflake Cortex documentation](https://docs.snowflake.com/) for the complete model list. --- -You'll find more information in the relevant sections: - -1. [Add metadata to your requests](/product/observability/metadata) -2. [Add gateway configs to your Snowflake Cortex](/product/ai-gateway/configs)[ requests](/product/ai-gateway/configs) -3. [Tracing Snowflake Cortex requests](/product/observability/traces) -4. [Setup a fallback from OpenAI to Snowflake Cortex APIs](/product/ai-gateway/fallbacks) +## Next Steps + + + + Add fallbacks, load balancing, and more + + + Monitor and trace your Cortex requests + + + Manage and version your prompts + + + Add custom metadata to requests + + + +For complete SDK documentation: + + + Complete Portkey SDK documentation + diff --git a/integrations/llms/stability-ai.mdx b/integrations/llms/stability-ai.mdx index 5db6f70b..e7086e6e 100644 --- a/integrations/llms/stability-ai.mdx +++ b/integrations/llms/stability-ai.mdx @@ -1,102 +1,208 @@ --- title: "Stability AI" +description: Use Stability AI's Stable Diffusion models for image generation through Portkey. --- -Portkey provides a robust and secure gateway to facilitate the integration of various Large Language Models (LLMs) into your applications, including [Stability AI APIs](https://platform.stability.ai/docs/api-reference). +## Quick Start -With Portkey, you can take advantage of features like fast AI gateway access, observability, prompt management, and more, all while ensuring the secure management of your LLM API keys through a [virtual key](/product/ai-gateway/virtual-keys) system. - -Provider Slug. `stability-ai` - -## Portkey SDK Integration with Stability AI - -Portkey provides a consistent API to interact with image generation models from various providers. To integrate Stability AI with Portkey: - -### 1\. Install the Portkey SDK - -Add the Portkey SDK to your application to interact with the Stability API through Portkey's gateway. - - - ```sh - npm install --save portkey-ai - ``` - - -```sh -pip install portkey-ai -``` - - - - -### 2\. Initialize Portkey with the Virtual Key - -To use Stability AI with Portkey, [get your API key from here](https://platform.stability.ai/account/keys). Then add it to Portkey to create the virtual key - - - ```js - import Portkey from 'portkey-ai' - - const portkey = new Portkey({ - apiKey: "PORTKEY_API_KEY", // defaults to process.env["PORTKEY_API_KEY"] - provider:"@PROVIDER" // Your Stability AI Virtual Key - }) - ``` - - -```python +Get started with Stability AI in under 2 minutes: + + + +```python Python icon="python" from portkey_ai import Portkey -portkey = Portkey( - api_key="PORTKEY_API_KEY", # Replace with your Portkey API key - provider="@PROVIDER" # Replace with your virtual key for Stability AI +# 1. Install: pip install portkey-ai +# 2. Add @stability-ai provider in model catalog +# 3. Use it: + +portkey = Portkey(api_key="PORTKEY_API_KEY") + +image = portkey.images.generate( + model="@stability-ai/stable-diffusion-v1-6", + prompt="A serene landscape with mountains", + size="1024x1024" +) + +print(image.data[0].url) +``` + +```js Javascript icon="square-js" +import Portkey from 'portkey-ai' + +// 1. Install: npm install portkey-ai +// 2. Add @stability-ai provider in model catalog +// 3. Use it: + +const portkey = new Portkey({ + apiKey: "PORTKEY_API_KEY" +}) + +const image = await portkey.images.generate({ + model: "@stability-ai/stable-diffusion-v1-6", + prompt: "A serene landscape with mountains", + size: "1024x1024" +}) + +console.log(image.data[0].url) +``` + +```python OpenAI Py icon="python" +from openai import OpenAI +from portkey_ai import PORTKEY_GATEWAY_URL + +# 1. Install: pip install openai portkey-ai +# 2. Add @stability-ai provider in model catalog +# 3. Use it: + +client = OpenAI( + api_key="PORTKEY_API_KEY", # Portkey API key + base_url=PORTKEY_GATEWAY_URL +) + +image = client.images.generate( + model="@stability-ai/stable-diffusion-v1-6", + prompt="A serene landscape with mountains", + size="1024x1024" ) + +print(image.data[0].url) +``` + +```js OpenAI JS icon="square-js" +import OpenAI from "openai" +import { PORTKEY_GATEWAY_URL } from "portkey-ai" + +// 1. Install: npm install openai portkey-ai +// 2. Add @stability-ai provider in model catalog +// 3. Use it: + +const client = new OpenAI({ + apiKey: "PORTKEY_API_KEY", // Portkey API key + baseURL: PORTKEY_GATEWAY_URL +}) + +const image = await client.images.generate({ + model: "@stability-ai/stable-diffusion-v1-6", + prompt: "A serene landscape with mountains", + size: "1024x1024" +}) + +console.log(image.data[0].url) +``` + +```bash cURL icon="square-terminal" +# 1. Add @stability-ai provider in model catalog +# 2. Use it: + +curl https://api.portkey.ai/v1/images/generations \ + -H "Content-Type: application/json" \ + -H "x-portkey-api-key: $PORTKEY_API_KEY" \ + -d '{ + "model": "@stability-ai/stable-diffusion-v1-6", + "prompt": "A serene landscape with mountains", + "size": "1024x1024" + }' ``` - - + + + +**Tip:** You can also set `provider="@stability-ai"` in `Portkey()` and use just `model="stable-diffusion-v1-6"` in the request. + + +## Add Provider in Model Catalog + +Before making requests, add Stability AI to your Model Catalog: +1. Go to [**Model Catalog → Add Provider**](https://app.portkey.ai/model-catalog/providers) +2. Select **Stability AI** +3. Enter your [Stability AI API key](https://platform.stability.ai/account/keys) +4. Name your provider (e.g., `stability-ai`) + + See all setup options and detailed configuration instructions + -### **3\. Invoke Image Generation with** Stability AI +--- + +## Image Generation + +Generate high-quality images with Stable Diffusion: + + + +```python Python +from portkey_ai import Portkey + +portkey = Portkey(api_key="PORTKEY_API_KEY", provider="@stability-ai") -Use the Portkey instance to send requests to Stability AI. You can also override the virtual key directly in the API call if needed. - - - ```js - const image = await portkey.images.generate({ - model:"stable-diffusion-v1-6", - prompt:"Lucy in the sky with diamonds", - size:"1024x1024" - }) - ``` - - -```py image = portkey.images.generate( model="stable-diffusion-v1-6", - prompt="Lucy in the sky with diamonds", + prompt="A serene landscape with mountains and a lake at sunset", size="1024x1024" ) + +print(image.data[0].url) ``` - - +```javascript Node.js +import Portkey from 'portkey-ai'; +const portkey = new Portkey({ + apiKey: 'PORTKEY_API_KEY', + provider: '@stability-ai' +}); +const image = await portkey.images.generate({ + model: "stable-diffusion-v1-6", + prompt: "A serene landscape with mountains and a lake at sunset", + size: "1024x1024" +}); -Notice how we're using the OpenAI's image generation signature to prompt Stability allowing greater flexibility to change models and providers later if necessary. +console.log(image.data[0].url); +``` -## Next Steps + -The complete list of features supported in the SDK are available on the link below. - - + +Portkey uses the OpenAI image generation signature for Stability AI, allowing you to easily switch between providers. + + +--- + +## Supported Models -You'll find more information in the relevant sections: +Stability AI offers powerful image generation models: -1. [Add metadata to your requests](/product/observability/metadata) -2. [Add gateway configs to your Stability AI requests](/product/ai-gateway/configs) -3. [Tracing Stability AI's requests](/product/observability/traces) -4. [Setup a fallback from OpenAI to Stability](/product/ai-gateway/fallbacks) -5. [Image generation API Reference](/provider-endpoints/images/create-image) +| Model | Description | +|-------|-------------| +| stable-diffusion-v1-6 | High-quality general-purpose image generation | +| stable-diffusion-xl-1024-v1-0 | XL model for larger, more detailed images | + +Check [Stability AI's documentation](https://platform.stability.ai/docs) for the complete model list. + +--- + +## Next Steps + + + + Add fallbacks, load balancing, and more + + + Monitor and trace your Stability AI requests + + + Cache generated images + + + Add custom metadata to requests + + + +For complete SDK documentation: + + + Complete Portkey SDK documentation + diff --git a/integrations/llms/together-ai.mdx b/integrations/llms/together-ai.mdx index 0010b8f5..d992884b 100644 --- a/integrations/llms/together-ai.mdx +++ b/integrations/llms/together-ai.mdx @@ -1,110 +1,151 @@ --- title: "Together AI" +description: "Integrate Together AI models with Portkey's AI Gateway" --- -Portkey provides a robust and secure gateway to facilitate the integration of various Large Language Models (LLMs) into your applications, including [Together AI APIs](https://docs.together.ai/reference/inference). +Portkey provides a robust and secure gateway to integrate various Large Language Models (LLMs) into applications, including [Together AI's hosted models](https://docs.together.ai/reference/inference). -With Portkey, you can take advantage of features like fast AI gateway access, observability, prompt management, and more, all while ensuring the secure management of your LLM API keys through a [virtual key](/product/ai-gateway/virtual-keys) system. - -Provider Slug. `together-ai` - -## Portkey SDK Integration with Together AI Models +With Portkey, take advantage of features like fast AI gateway access, observability, prompt management, and more, while securely managing API keys through [Model Catalog](/product/model-catalog). + +## Quick Start + +Get Together AI working in 3 steps: + + +```python Python icon="python" +from portkey_ai import Portkey -Portkey provides a consistent API to interact with models from various providers. To integrate Together AI with Portkey: +# 1. Install: pip install portkey-ai +# 2. Add @together-ai provider in model catalog +# 3. Use it: -### 1\. Install the Portkey SDK +portkey = Portkey(api_key="PORTKEY_API_KEY") -Add the Portkey SDK to your application to interact with Together AI's API through Portkey's gateway. - - +response = portkey.chat.completions.create( + model="@together-ai/meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo", + messages=[{"role": "user", "content": "Say this is a test"}] +) -```sh -npm install --save portkey-ai +print(response.choices[0].message.content) ``` - - - ```sh - pip install portkey-ai - ``` - - +```js Javascript icon="square-js" +import Portkey from 'portkey-ai' +// 1. Install: npm install portkey-ai +// 2. Add @together-ai provider in model catalog +// 3. Use it: +const portkey = new Portkey({ + apiKey: "PORTKEY_API_KEY" +}) +const response = await portkey.chat.completions.create({ + model: "@together-ai/meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo", + messages: [{ role: "user", content: "Say this is a test" }] +}) -### 2\. Initialize Portkey with the Virtual Key +console.log(response.choices[0].message.content) +``` -To use Together AI with Portkey, [get your API key from here](https://api.together.ai/settings/api-keys). Then add it to Portkey to create the virtual key - - - ```js - import Portkey from 'portkey-ai' +```python OpenAI Py icon="python" +from openai import OpenAI +from portkey_ai import PORTKEY_GATEWAY_URL - const portkey = new Portkey({ - apiKey: "PORTKEY_API_KEY", // defaults to process.env["PORTKEY_API_KEY"] - provider:"@PROVIDER" // Your Together AI Virtual Key - }) - ``` - - - ```python - from portkey_ai import Portkey +# 1. Install: pip install openai portkey-ai +# 2. Add @together-ai provider in model catalog +# 3. Use it: - portkey = Portkey( - api_key="PORTKEY_API_KEY", # Replace with your Portkey API key - provider="@PROVIDER" # Replace with your virtual key for Together AI - ) - ``` - +client = OpenAI( + api_key="PORTKEY_API_KEY", # Portkey API key + base_url=PORTKEY_GATEWAY_URL +) - +response = client.chat.completions.create( + model="@together-ai/meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo", + messages=[{"role": "user", "content": "Say this is a test"}] +) +print(response.choices[0].message.content) +``` -### **3\. Invoke Chat Completions with** Together AI +```js OpenAI JS icon="square-js" +import OpenAI from "openai" +import { PORTKEY_GATEWAY_URL } from "portkey-ai" -Use the Portkey instance to send requests to Together AI. You can also override the virtual key directly in the API call if needed. - - - ```js - const chatCompletion = await portkey.chat.completions.create({ - messages: [{ role: 'user', content: 'Say this is a test' }], - model: 'togethercomputer/llama-2-70b-chat', - }); +// 1. Install: npm install openai portkey-ai +// 2. Add @together-ai provider in model catalog +// 3. Use it: - console.log(chatCompletion.choices); - ``` - - - ```python - completion = portkey.chat.completions.create( - messages= [{ "role": 'user', "content": 'Say this is a test' }], - model='togethercomputer/llama-2-70b-chat' - ) +const client = new OpenAI({ + apiKey: "PORTKEY_API_KEY", // Portkey API key + baseURL: PORTKEY_GATEWAY_URL +}) - print(completion) - ``` - +const response = await client.chat.completions.create({ + model: "@together-ai/meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo", + messages: [{ role: "user", content: "Say this is a test" }] +}) - +console.log(response.choices[0].message.content) +``` +```sh cURL icon="square-terminal" +# 1. Add @together-ai provider in model catalog +# 2. Use it: + +curl https://api.portkey.ai/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "x-portkey-api-key: $PORTKEY_API_KEY" \ + -d '{ + "model": "@together-ai/meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo", + "messages": [ + { "role": "user", "content": "Say this is a test" } + ] + }' +``` + + +**Tip:** You can also set `provider="@together-ai"` in `Portkey()` and use just `model="meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo"` in the request. + + +## Add Provider in Model Catalog + +1. Go to [**Model Catalog → Add Provider**](https://app.portkey.ai/model-catalog/providers) +2. Select **Together AI** +3. Choose existing credentials or create new by entering your [Together AI API key](https://api.together.ai/settings/api-keys) +4. Name your provider (e.g., `together-ai-prod`) + + + See all setup options, code examples, and detailed instructions + ## Managing Together AI Prompts -You can manage all prompts to Together AI in the [Prompt Library](/product/prompt-library). All the current models of Together AI are supported and you can easily start testing different prompts. +Manage all prompt templates to Together AI in the [Prompt Library](/product/prompt-library). All current Together AI models are supported, and you can easily test different prompts. -Once you're ready with your prompt, you can use the `portkey.prompts.completions.create` interface to use the prompt in your application. +Use the `portkey.prompts.completions.create` interface to use the prompt in an application. ## Next Steps -The complete list of features supported in the SDK are available on the link below. - + + + Add metadata to your Together AI requests + + + Add gateway configs to your Together AI requests + + + Trace your Together AI requests + + + Setup fallback from OpenAI to Together AI + + + +For complete SDK documentation: + + + Complete Portkey SDK documentation - -You'll find more information in the relevant sections: - -1. [Add metadata to your requests](/product/observability/metadata) -2. [Add gateway configs to your Together AI](/product/ai-gateway/configs)[ requests](/product/ai-gateway/configs) -3. [Tracing Together AI requests](/product/observability/traces) -4. [Setup a fallback from OpenAI to Together AI APIs](/product/ai-gateway/fallbacks) diff --git a/integrations/llms/triton.mdx b/integrations/llms/triton.mdx index 99667335..b774329f 100644 --- a/integrations/llms/triton.mdx +++ b/integrations/llms/triton.mdx @@ -1,111 +1,129 @@ --- -title: "Triton" -description: "Integrate Trtiton-hosted custom models with Portkey and take them to production" +title: "Triton Inference Server" +description: "Integrate Triton-hosted custom models with Portkey for production observability and reliability." --- -Portkey provides a robust and secure platform to observe, govern, and manage your **locally** or **privately** hosted custom models using Triton. +Portkey provides a robust platform to observe, govern, and manage your **locally** or **privately** hosted custom models using Triton Inference Server. Here's the official [Triton Inference Server documentation](https://docs.nvidia.com/deeplearning/triton-inference-server/user-guide/docs/getting_started/quickstart.html) for more details. -## Integrating Custom Models with Portkey SDK +## Integration Steps + -Expose your Triton server by using a tunneling service like [ngrok](https://ngrok.com/) or any other way you prefer. You can skip this step if you’re self-hosting the Gateway. +Expose your Triton server using a tunneling service like [ngrok](https://ngrok.com/) or make it publicly accessible. Skip this if you're self-hosting the Gateway. ```sh -ngrok http 11434 --host-header="localhost:8080" +ngrok http 8000 --host-header="localhost:8080" ``` - - - - - ```sh - npm install --save portkey-ai - ``` - - - ```sh - pip install portkey-ai - ``` - - + +1. Go to [**Model Catalog → Add Provider**](https://app.portkey.ai/model-catalog/providers) +2. Enable **"Local/Privately hosted provider"** toggle +3. Select **Triton** as the provider type +4. Enter your Triton server URL in **Custom Host**: `http://localhost:8000/v2/models/mymodel` +5. Add authentication headers if needed +6. Name your provider (e.g., `my-triton`) + + + See all setup options + - - -1. Pass your publicly-exposed Triton server URL to Portkey with `customHost` -2. Set target `provider` as `triton`. - - - - ```js - import Portkey from 'portkey-ai' - - const portkey = new Portkey({ - apiKey: "PORTKEY_API_KEY", - provider: "triton", - customHost: "http://localhost:8000/v2/models/mymodel" // Your Triton Hosted URL - Authorization: "AUTH_KEY", // If you need to pass auth - }) - ``` - - - ```python - from portkey_ai import Portkey - - portkey = Portkey( - api_key="PORTKEY_API_KEY", - provider="triton", - custom_host="http://localhost:8000/v2/models/mymodel" # Your Triton Hosted URL - Authorization="AUTH_KEY", # If you need to pass auth - ) - ``` - - - - -More on `custom_host` [here](/product/ai-gateway/universal-api#integrating-local-or-private-models). - - - -Use the Portkey SDK to invoke chat completions (generate) from your model, just as you would with any other provider: - - - - ```js - const chatCompletion = await portkey.chat.completions.create({ - messages: [{ role: 'user', content: 'Say this is a test' }] - }); - - console.log(chatCompletion.choices); - ``` - - -```python -completion = portkey.chat.completions.create( - messages= [{ "role": 'user', "content": 'Say this is a test' }] + + + + + +```python Python +from portkey_ai import Portkey + +portkey = Portkey( + api_key="PORTKEY_API_KEY", + provider="@my-triton" ) -print(completion) +response = portkey.chat.completions.create( + model="your-model-name", + messages=[{"role": "user", "content": "Hello!"}] +) + +print(response.choices[0].message.content) ``` - - + +```javascript Node.js +import Portkey from 'portkey-ai'; + +const portkey = new Portkey({ + apiKey: 'PORTKEY_API_KEY', + provider: '@my-triton' +}); + +const response = await portkey.chat.completions.create({ + model: 'your-model-name', + messages: [{ role: 'user', content: 'Hello!' }] +}); + +console.log(response.choices[0].message.content); +``` + + + +**Or use custom host directly:** + + + +```python Python +from portkey_ai import Portkey + +portkey = Portkey( + api_key="PORTKEY_API_KEY", + provider="triton", + custom_host="http://localhost:8000/v2/models/mymodel", + Authorization="AUTH_KEY" # If needed +) +``` + +```javascript Node.js +import Portkey from 'portkey-ai'; + +const portkey = new Portkey({ + apiKey: 'PORTKEY_API_KEY', + provider: 'triton', + customHost: 'http://localhost:8000/v2/models/mymodel', + Authorization: 'AUTH_KEY' // If needed +}); +``` + + + -## Next Steps - -Explore the complete list of features supported in the SDK: - --- -You'll find more information in the relevant sections: +## Next Steps -1. [Add metadata to your requests](/product/observability/metadata) -2. [Add gateway configs to your requests](/product/ai-gateway/universal-api#ollama-in-configs) -3. [Tracing requests](/product/observability/traces) -4. [Setup a fallback from triton to your local LLM](/product/ai-gateway/fallbacks) + + + Add retries, timeouts, and fallbacks + + + Monitor your Triton deployments + + + Learn more about custom host setup + + + Complete guide for private LLMs + + + +For complete SDK documentation: + + + Complete Portkey SDK documentation + diff --git a/integrations/llms/upstage.mdx b/integrations/llms/upstage.mdx index 5da8242b..343d33b9 100644 --- a/integrations/llms/upstage.mdx +++ b/integrations/llms/upstage.mdx @@ -1,227 +1,181 @@ --- title: 'Upstage AI' -description: 'Integrate Upstage with Portkey AI for seamless completions, prompt management, and advanced features like streaming and embedding.' +description: 'Integrate Upstage with Portkey for chat, embeddings, streaming, and function calling.' --- - -**Portkey Provider Slug:** `upstage` - +## Quick Start -## Overview +Get started with Upstage AI in under 2 minutes: -Portkey offers native integrations with [Upstage](https://www.upstage.ai/) for Node.js, Python, and REST APIs. By combining Portkey with Upstage, you can create production-grade AI applications with enhanced reliability, observability, and advanced features. - - - - Explore the official Upstage documentation for comprehensive details on their APIs and models. - - + -## Getting Started +```python Python icon="python" +from portkey_ai import Portkey - - - Visit the [Upstage dashboard](https://console.upstage.ai/api-keys) to generate your API key. - +# 1. Install: pip install portkey-ai +# 2. Add @upstage provider in model catalog +# 3. Use it: - - Portkey's virtual key vault simplifies your interaction with Upstage. Virtual keys act as secure aliases for your actual API keys, offering enhanced security and easier management through [budget limits](/product/ai-gateway/virtual-keys/budget-limits) to control your API usage. +portkey = Portkey(api_key="PORTKEY_API_KEY") - Use the Portkey app to create a [virtual key](/product/ai-gateway/virtual-keys) associated with your Upstage API key. - +response = portkey.chat.completions.create( + model="@upstage/solar-pro", + messages=[{"role": "user", "content": "Hello!"}] +) - - Now that you have your virtual key, set up the Portkey client: +print(response.choices[0].message.content) +``` - ### Portkey Hosted App - Use the Portkey API key and the Upstage virtual key to initialize the client in your preferred programming language. +```js Javascript icon="square-js" +import Portkey from 'portkey-ai' - - ```python Python - from portkey_ai import Portkey +// 1. Install: npm install portkey-ai +// 2. Add @upstage provider in model catalog +// 3. Use it: - portkey = Portkey( - api_key="PORTKEY_API_KEY", # Replace with your Portkey API key - provider="@PROVIDER" # Replace with your virtual key for Upstage - ) - ``` +const portkey = new Portkey({ + apiKey: "PORTKEY_API_KEY" +}) - ```javascript Node.js - import Portkey from 'portkey-ai' +const response = await portkey.chat.completions.create({ + model: "@upstage/solar-pro", + messages: [{ role: "user", content: "Hello!" }] +}) - const portkey = new Portkey({ - apiKey: "PORTKEY_API_KEY", // defaults to process.env["PORTKEY_API_KEY"] - provider:"@PROVIDER" // Your Upstage Virtual Key - }) - ``` - +console.log(response.choices[0].message.content) +``` - ### Open Source Use - Alternatively, use Portkey's Open Source AI Gateway to enhance your app's reliability with minimal code: +```python OpenAI Py icon="python" +from openai import OpenAI +from portkey_ai import PORTKEY_GATEWAY_URL - - ```python Python - from portkey_ai import Portkey, PORTKEY_GATEWAY_URL +# 1. Install: pip install openai portkey-ai +# 2. Add @upstage provider in model catalog +# 3. Use it: - portkey = Portkey( - api_key="dummy", # Replace with your Portkey API key - base_url=PORTKEY_GATEWAY_URL, - Authorization="UPSTAGE_API_KEY", # Replace with your Upstage API Key - provider="upstage" - ) - ``` +client = OpenAI( + api_key="PORTKEY_API_KEY", # Portkey API key + base_url=PORTKEY_GATEWAY_URL +) - ```javascript Node.js - import Portkey, { PORTKEY_GATEWAY_URL } from 'portkey-ai' +response = client.chat.completions.create( + model="@upstage/solar-pro", + messages=[{"role": "user", "content": "Hello!"}] +) - const portkey = new Portkey({ - apiKey: "dummy", // Replace with your Portkey API key - baseUrl: PORTKEY_GATEWAY_URL, - Authorization: "UPSTAGE_API_KEY", // Replace with your Upstage API Key - provider: "upstage" - }) - ``` - - - +print(response.choices[0].message.content) +``` -🔥 That's it! You've integrated Portkey into your application with just a few lines of code. Now let's explore making requests using the Portkey client. +```js OpenAI JS icon="square-js" +import OpenAI from "openai" +import { PORTKEY_GATEWAY_URL } from "portkey-ai" -## Supported Models +// 1. Install: npm install openai portkey-ai +// 2. Add @upstage provider in model catalog +// 3. Use it: - +const client = new OpenAI({ + apiKey: "PORTKEY_API_KEY", // Portkey API key + baseURL: PORTKEY_GATEWAY_URL +}) -`Chat` - solar-pro, solar-mini and solar-mini-ja +const response = await client.chat.completions.create({ + model: "@upstage/solar-pro", + messages: [{ role: "user", content: "Hello!" }] +}) +console.log(response.choices[0].message.content) +``` -`Embedding`- embedding-passage, embedding-query +```sh cURL icon="square-terminal" +# 1. Add @upstage provider in model catalog +# 2. Use it: + +curl https://api.portkey.ai/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "x-portkey-api-key: $PORTKEY_API_KEY" \ + -d '{ + "model": "@upstage/solar-pro", + "messages": [{"role": "user", "content": "Hello!"}] + }' +``` + +## Add Provider in Model Catalog - +Before making requests, add Upstage to your Model Catalog: +1. Go to [**Model Catalog → Add Provider**](https://app.portkey.ai/model-catalog/providers) +2. Select **Upstage** +3. Enter your [Upstage API key](https://console.upstage.ai/api-keys) +4. Name your provider (e.g., `upstage`) -## Supported Endpoints and Parameters + + See all setup options and detailed configuration instructions + -| Endpoint | Supported Parameters | -| ----------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `chatComplete` | messages, max_tokens, temperature, top_p, stream, presence_penalty, frequency_penalty | -| `embed` | model, input, encoding_format, dimensions, user | + + Explore the official Upstage documentation + +--- -## Upstage Supported Features +## Upstage Capabilities -### Chat Completions +### Streaming -Generate chat completions using Upstage models through Portkey: +Stream responses for real-time output: -```python Python -completion = portkey.chat.completions.create( - messages=[{"role": "user", "content": "Say this is a test"}], - model="solar-pro" -) - -print(completion.choices[0].message.content) -``` - -```javascript Node.js -const chatCompletion = await portkey.chat.completions.create({ - messages: [{ role: 'user', content: 'Say this is a test' }], - model: 'solar-pro', -}); - -console.log(chatCompletion.choices[0].message.content); -``` -```curl REST -curl -X POST "https://api.portkey.ai/v1/chat/completions" \ - -H "Content-Type: application/json" \ - -H "Authorization: Bearer YOUR_PORTKEY_API_KEY" \ - -d '{ - "messages": [{"role": "user", "content": "Say this is a test"}], - "model": "solar-pro" - }' -``` - - -### Streaming +```python Python +from portkey_ai import Portkey -Stream responses for real-time output in your applications: +portkey = Portkey(api_key="PORTKEY_API_KEY", provider="@upstage") - -```python Python -chat_complete = portkey.chat.completions.create( +stream = portkey.chat.completions.create( model="solar-pro", - messages=[{"role": "user", "content": "Say this is a test"}], + messages=[{"role": "user", "content": "Tell me a story"}], stream=True ) -for chunk in chat_complete: +for chunk in stream: print(chunk.choices[0].delta.content or "", end="", flush=True) ``` ```javascript Node.js +import Portkey from 'portkey-ai'; + +const portkey = new Portkey({ + apiKey: 'PORTKEY_API_KEY', + provider: '@upstage' +}); + const stream = await portkey.chat.completions.create({ - model: 'gpt-4', - messages: [{ role: 'user', content: 'Say this is a test' }], - stream: true, + model: 'solar-pro', + messages: [{ role: 'user', content: 'Tell me a story' }], + stream: true }); for await (const chunk of stream) { - process.stdout.write(chunk.choices[0]?.delta?.content || ''); + process.stdout.write(chunk.choices[0]?.delta?.content || ''); } ``` -```curl REST -curl -X POST "https://api.portkey.ai/v1/chat/completions" \ - -H "Content-Type: application/json" \ - -H "Authorization: Bearer YOUR_PORTKEY_API_KEY" \ - -d '{ - "model": "solar-pro", - "messages": [{"role": "user", "content": "Say this is a test"}], - "stream": true - }' -``` - ### Function Calling -Leverage Upstage's function calling capabilities through Portkey: +Use Upstage's function calling capabilities: -```javascript Node.js -let tools = [{ - type: "function", - function: { - name: "getWeather", - description: "Get the current weather", - parameters: { - type: "object", - properties: { - location: { type: "string", description: "City and state" }, - unit: { type: "string", enum: ["celsius", "fahrenheit"] } - }, - required: ["location"] - } - } -}]; -let response = await portkey.chat.completions.create({ - model: "solar-pro", - messages: [ - { role: "system", content: "You are a helpful assistant." }, - { role: "user", content: "What's the weather like in Delhi - respond in JSON" } - ], - tools, - tool_choice: "auto", -}); +```python Python +from portkey_ai import Portkey -console.log(response.choices[0].finish_reason); -``` +portkey = Portkey(api_key="PORTKEY_API_KEY", provider="@upstage") -```python Python tools = [{ "type": "function", "function": { @@ -248,46 +202,59 @@ response = portkey.chat.completions.create( tool_choice="auto" ) -print(response.choices[0].finish_reason) +print(response.choices[0].message) ``` -```curl REST -curl -X POST "https://api.portkey.ai/v1/chat/completions" \ - -H "Content-Type: application/json" \ - -H "Authorization: Bearer YOUR_PORTKEY_API_KEY" \ - -d '{ - "model": "solar-pro", - "messages": [ - {"role": "system", "content": "You are a helpful assistant."}, - {"role": "user", "content": "What'\''s the weather like in Delhi - respond in JSON"} - ], - "tools": [{ - "type": "function", - "function": { - "name": "getWeather", - "description": "Get the current weather", - "parameters": { - "type": "object", - "properties": { - "location": {"type": "string", "description": "City and state"}, - "unit": {"type": "string", "enum": ["celsius", "fahrenheit"]} - }, - "required": ["location"] - } - } - }], - "tool_choice": "auto" - }' +```javascript Node.js +import Portkey from 'portkey-ai'; + +const portkey = new Portkey({ + apiKey: 'PORTKEY_API_KEY', + provider: '@upstage' +}); + +const tools = [{ + type: "function", + function: { + name: "getWeather", + description: "Get the current weather", + parameters: { + type: "object", + properties: { + location: { type: "string", description: "City and state" }, + unit: { type: "string", enum: ["celsius", "fahrenheit"] } + }, + required: ["location"] + } + } +}]; + +const response = await portkey.chat.completions.create({ + model: "solar-pro", + messages: [ + { role: "system", content: "You are a helpful assistant." }, + { role: "user", content: "What's the weather in Delhi?" } + ], + tools, + tool_choice: "auto" +}); + +console.log(response.choices[0].message); ``` - + ### Embeddings -Generate embeddings for text using Upstage embedding models: +Generate embeddings for text: + ```python Python +from portkey_ai import Portkey + +portkey = Portkey(api_key="PORTKEY_API_KEY", provider="@upstage") + response = portkey.embeddings.create( input="Your text string goes here", model="embedding-query" @@ -297,204 +264,68 @@ print(response.data[0].embedding) ``` ```javascript Node.js +import Portkey from 'portkey-ai'; + +const portkey = new Portkey({ + apiKey: 'PORTKEY_API_KEY', + provider: '@upstage' +}); + const response = await portkey.embeddings.create({ - input: "Your text string goes here", - model: "embedding-query" + input: "Your text string goes here", + model: "embedding-query" }); console.log(response.data[0].embedding); ``` -```curl REST -curl -X POST "https://api.portkey.ai/v1/embeddings" \ - -H "Content-Type: application/json" \ - -H "Authorization: Bearer YOUR_PORTKEY_API_KEY" \ - -d '{ - "input": "Your text string goes here", - "model": "embedding-query" - }' -``` +--- -# Portkey's Advanced Features - -## Track End-User IDs - -Portkey allows you to track user IDs passed with the user parameter in Upstage requests, enabling you to monitor user-level costs, requests, and more: +## Supported Models - -```python Python -response = portkey.chat.completions.create( - model="solar-pro", - messages=[{"role": "user", "content": "Say this is a test"}], - user="user_123456" -) -``` + -```javascript Node.js -const chatCompletion = await portkey.chat.completions.create({ - messages: [{ role: "user", content: "Say this is a test" }], - model: "solar-pro", - user: "user_12345", -}); -``` +**Chat Models:** +- solar-pro +- solar-mini +- solar-mini-ja -```curl REST -curl -X POST "https://api.portkey.ai/v1/chat/completions" \ - -H "Content-Type: application/json" \ - -H "Authorization: Bearer YOUR_PORTKEY_API_KEY" \ - -d '{ - "model": "solar-pro", - "messages": [{"role": "user", "content": "Say this is a test"}], - "user": "user_123456" - }' -``` - +**Embedding Models:** +- embedding-passage +- embedding-query -When you include the user parameter in your requests, Portkey logs will display the associated user ID, as shown in the image below: + -Portkey Logs with User ID +## Supported Endpoints and Parameters -In addition to the `user` parameter, Portkey allows you to send arbitrary custom metadata with your requests. This powerful feature enables you to associate additional context or information with each request, which can be useful for analysis, debugging, or other custom use cases. +| Endpoint | Supported Parameters | +|----------|---------------------| +| `/chat/completions` | messages, max_tokens, temperature, top_p, stream, presence_penalty, frequency_penalty, tools, tool_choice | +| `/embeddings` | model, input, encoding_format, dimensions, user | - - - Explore how to use custom metadata to enhance your request tracking and analysis. - - +--- -## Using The Gateway Config - -Here's a simplified version of how to use Portkey's Gateway Configuration: - - - - You can create a Gateway configuration using the Portkey Config Dashboard or by writing a JSON configuration in your code. In this example, requests are routed based on the user's subscription plan (paid or free). - - ```json - config = { - "strategy": { - "mode": "conditional", - "conditions": [ - { - "query": { "metadata.user_plan": { "$eq": "paid" } }, - "then": "solar-pro" - }, - { - "query": { "metadata.user_plan": { "$eq": "free" } }, - "then": "gpt-3.5" - } - ], - "default": "base-gpt4" - }, - "targets": [ - { - "name": "solar-pro", - "provider":"@xx" - }, - { - "name": "gpt-3.5", - "provider":"@yy" - } - ] - } - ``` - - - - When a user makes a request, it will pass through Portkey's AI Gateway. Based on the configuration, the Gateway routes the request according to the user's metadata. - Conditional Routing Diagram - - - - Pass the Gateway configuration to your Portkey client. You can either use the config object or the Config ID from Portkey's hosted version. - - - ```python Python - from portkey_ai import Portkey - - portkey = Portkey( - api_key="PORTKEY_API_KEY", - provider="@PROVIDER", - config=portkey_config - ) - ``` - - ```javascript Node.js - import Portkey from 'portkey-ai' - - const portkey = new Portkey({ - apiKey: "PORTKEY_API_KEY", - provider:"@PROVIDER", - config: portkeyConfig - }) - ``` - - - - -That's it! Portkey seamlessly allows you to make your AI app more robust using built-in gateway features. Learn more about advanced gateway features: +## Next Steps - - Distribute requests across multiple targets based on defined weights. + + Add fallbacks, load balancing, and more - - Automatically switch to backup targets if the primary target fails. + + Monitor and trace your Upstage requests - - Route requests to different targets based on specified conditions. + + Manage and version your prompts - - Enable caching of responses to improve performance and reduce costs. + + Add custom metadata to requests -## Guardrails - -Portkey's AI gateway enables you to enforce input/output checks on requests by applying custom hooks before and after processing. Protect your user's/company's data by using PII guardrails and many more available on Portkey Guardrails: - -```json -{ - "provider:"@upstage-xxx", - "before_request_hooks": [{ - "id": "input-guardrail-id-xx" - }], - "after_request_hooks": [{ - "id": "output-guardrail-id-xx" - }] -} -``` - - - Explore Portkey's guardrail features to enhance the security and reliability of your AI applications. - - -## Next Steps - -The complete list of features supported in the SDK are available in our comprehensive documentation: +For complete SDK documentation: - - Explore the full capabilities of the Portkey SDK and how to leverage them in your projects. + + Complete Portkey SDK documentation - ---- - - -## Limitations - - -Portkey does not support the following Upstage features: -- Document Parse -- Document QA -- Document OCR -- Embeddings -- Translation -- Groundedness Check -- Key Information Extraction - - - - -For the most up-to-date information on supported features and endpoints, please refer to our [API Reference](/api-reference/inference-api/introduction). diff --git a/integrations/llms/vertex-ai.mdx b/integrations/llms/vertex-ai.mdx index 5f3f2a23..9006c5aa 100644 --- a/integrations/llms/vertex-ai.mdx +++ b/integrations/llms/vertex-ai.mdx @@ -4,113 +4,129 @@ title: "Google Vertex AI" Portkey provides a robust and secure gateway to facilitate the integration of various Large Language Models (LLMs), and embedding models into your apps, including [Google Vertex AI](https://cloud.google.com/vertex-ai?hl=en). -With Portkey, you can take advantage of features like fast AI gateway access, observability, prompt management, and more, all while ensuring the secure management of your Vertex auth through a Portkey's [Model Catalog](/product/model-catalog) +With Portkey, you can take advantage of features like fast AI gateway access, observability, prompt management, and more, all while ensuring the secure management of your Vertex auth through [Model Catalog](/product/model-catalog). - -Provider Slug. `vertex-ai` - +## Quick Start -## Portkey SDK Integration with Google Vertex AI + -Portkey provides a consistent API to interact with models from various providers. To integrate Google Vertex AI with Portkey: +```python Python icon="python" +from portkey_ai import Portkey -### 1\. Install the Portkey SDK +# 1. Install: pip install portkey-ai +# 2. Add @vertex-ai provider in Model Catalog with Service Account JSON +# 3. Use it: -Add the Portkey SDK to your application to interact with Google Vertex AI API through Portkey's gateway. - - +portkey = Portkey(api_key="PORTKEY_API_KEY") - ```sh - npm install --save portkey-ai - ``` - - - ```sh -pip install portkey-ai +response = portkey.chat.completions.create( + model="@vertex-ai/gemini-3-pro-preview", + messages=[{"role": "user", "content": "Say this is a test"}] +) + +print(response.choices[0].message.content) ``` - - +```js Javascript icon="square-js" +import Portkey from 'portkey-ai' +// 1. Install: npm install portkey-ai +// 2. Add @vertex-ai provider in Model Catalog with Service Account JSON +// 3. Use it: +const portkey = new Portkey({ + apiKey: "PORTKEY_API_KEY" +}) +const response = await portkey.chat.completions.create({ + model: "@vertex-ai/gemini-3-pro-preview", + messages: [{ role: "user", content: "Say this is a test" }] +}) -### 2\. Initialize Portkey Client +console.log(response.choices[0].message.content) +``` -To integrate Vertex AI with Portkey, you'll need your `Vertex Project Id` Or `Service Account JSON` & `Vertex Region`, with which you can set up the Portkey's AI Provider. +```sh cURL icon="square-terminal" +# 1. Add @vertex-ai provider in Model Catalog with Service Account JSON +# 2. Use it: -[Here's a guide on how to find your Vertex Project details](/integrations/llms/vertex-ai#how-to-find-your-google-vertex-project-details) +curl https://api.portkey.ai/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "x-portkey-api-key: $PORTKEY_API_KEY" \ + -d '{ + "model": "@vertex-ai/gemini-3-pro-preview", + "messages": [{"role": "user", "content": "Say this is a test"}] + }' +``` -If you are integrating through Service Account File, [refer to this guide](/integrations/llms/vertex-ai#get-your-service-account-json). + - - - ```js - import Portkey from 'portkey-ai' + +**Authentication Note:** When you configure Vertex AI in Model Catalog with **Service Account JSON** (recommended), authentication is handled automatically. If you only configure with **Project ID and Region**, you'll need to pass an OAuth2 access token with each request using the `Authorization` header. See the [Making Requests Without Model Catalog](#making-requests-without-portkeys-model-catalog) section for details. + - const portkey = new Portkey({ - apiKey: "PORTKEY_API_KEY", // defaults to process.env["PORTKEY_API_KEY"] +--- - }) - ``` - - - ```python - from portkey_ai import Portkey +## Add Provider in Model Catalog - portkey = Portkey( - api_key="PORTKEY_API_KEY", # Replace with your Portkey API key - ) - ``` - - + + -If you do not want to add your Vertex AI details to Portkey vault, you can directly pass them while instantiating the Portkey client. [More on that here](/integrations/llms/vertex-ai#making-requests-without-portkeys-model-catalog). +Go to [**Model Catalog → Add Provider**](https://app.portkey.ai/model-catalog/providers) in your Portkey dashboard. -### **3\. Invoke Chat Completions with** Vertex AI + -Use the Portkey instance to send requests to any models hosted on Vertex AI. You can also override the Portkey's AI Provider directly in the API call if needed. - -Vertex AI uses OAuth2 to authenticate its requests, so you need to send the **access token** additionally along with the request. - + - - -```js -const chatCompletion = await portkey.chat.completions.create({ - messages: [{ role: 'user', content: 'Say this is a test' }], - model: '@VERTEX_PROVIDER/gemini-1.5-pro-latest', // your model slug from Portkey's Model Catalog -}, {Authorization: "Bearer $YOUR_VERTEX_ACCESS_TOKEN"}); +Find and select **Google Vertex AI** from the provider list. -console.log(chatCompletion.choices); -``` + - - -```python -completion = portkey.with_options(Authorization="Bearer $YOUR_VERTEX_ACCESS_TOKEN").chat.completions.create( - messages= [{ "role": 'user', "content": 'Say this is a test' }], - model= '@VERTEX_PROVIDER/gemini-1.5-pro-latest' # your model slug from Portkey's Model Catalog -) + -print(completion) -``` - - +You'll need your `Vertex Project ID` and `Vertex Region`. You can authenticate using either: + +**Option 1: Service Account JSON** (Recommended for self-deployed models) +- Upload your Google Cloud service account JSON file +- Specify the Vertex Region +- Required for custom endpoints (must have `aiplatform.endpoints.predict` permission) + +**Option 2: Project ID and Region** +- Enter your Vertex Project ID +- Enter your Vertex Region +- Simpler but may not support all features + +[Here's a guide on how to find your Vertex Project details](#how-to-find-your-google-vertex-project-details) + +If you're using Service Account File, [refer to this guide](#get-your-service-account-json). + + + + + +Save your configuration. Your provider slug will be `@vertex-ai` (or a custom name you specify). + + + + -To use Anthopic models on Vertex AI, prepend `anthropic.` to the model name.
-Example: `@VERTEX_PROVIDER/anthropic.claude-3-5-sonnet@20240620` +**To use Anthropic models on Vertex AI**, prepend `anthropic.` to the model name. +Example: `@vertex-ai/anthropic.claude-3-5-sonnet@20240620` -Similarly, for Meta models, prepend `meta.` to the model name.
-Example: `@VERTEX_PROVIDER/meta.llama-3-8b-8192` +Similarly, **for Meta models**, prepend `meta.` to the model name. +Example: `@vertex-ai/meta.llama-3-8b-8192` **Anthropic Beta Header Support**: When using Anthropic models on Vertex AI, you can pass the `anthropic-beta` header (or `x-portkey-anthropic-beta`) to enable beta features. This header is forwarded to the underlying Anthropic API. +--- + +## Vertex AI Capabilities + @@ -154,7 +170,7 @@ This route only works with Claude models. For other models, use the standard Ope ```python Anthropic Python SDK - import anthropic +import anthropic client = anthropic.Anthropic( api_key="dummy", # we will use portkey's provider slug @@ -173,15 +189,15 @@ This route only works with Claude models. For other models, use the standard Ope ```typescript Anthropic TS SDK - import Anthropic from '@anthropic-ai/sdk'; + import Anthropic from '@anthropic-ai/sdk'; - const anthropic = new Anthropic({ + const anthropic = new Anthropic({ apiKey: 'dummy', // we will use portkey's provider slug baseURL: "https://api.portkey.ai/v1", defaultHeaders: { "x-portkey-api-key": "YOUR_PORTKEY_API_KEY" } }); - const msg = await anthropic.messages.create({ + const msg = await anthropic.messages.create({ model: "@your-provider-slug/your-model-name", max_tokens: 1024, messages: [{ role: "user", content: "Hello, Claude" }], @@ -217,17 +233,17 @@ To use self-deployed models on Vertex AI through Portkey: ```js const chatCompletion = await portkey.chat.completions.create({ messages: [{ role: 'user', content: 'Say this is a test' }], - model: 'endpoints.my_custom_llm', // Notice the 'endpoints.' prefix -}, {Authorization: "Bearer $YOUR_VERTEX_ACCESS_TOKEN"}); + model: '@vertex-ai/endpoints.my_custom_llm', // Use Model Catalog slug with 'endpoints.' prefix +}); console.log(chatCompletion.choices); ``` ```python -completion = portkey.with_options(Authorization="Bearer $YOUR_VERTEX_ACCESS_TOKEN").chat.completions.create( +completion = portkey.chat.completions.create( messages= [{ "role": 'user', "content": 'Say this is a test' }], - model= 'endpoints.my_huggingface_model' # Notice the 'endpoints.' prefix + model= '@vertex-ai/endpoints.my_huggingface_model' # Use Model Catalog slug with 'endpoints.' prefix ) print(completion) @@ -249,7 +265,7 @@ This approach works for all models you can self-deploy on Vertex AI Model Garden Vertex AI supports attaching various file types to your Gemini messages including documents (`pdf`), images (`jpg`, `png`), videos (`webm`, `mp4`), and audio files. -**Supported Audio Formats:** `mp3`, `wav`, `opus`, `ogg`, `flac`, `pcm`, `aac`, `m4a`, `mpeg`, `mpga`, `mp4`, `webm` +**Supported Audio Formats:** `mp3`, `wav`, `opus`, `flac`, `pcm`, `aac`, `m4a`, `mpeg`, `mpga`, `mp4`, `webm` Gemini Docs: * [Document Processing](https://ai.google.dev/gemini-api/docs/document-processing?lang=python) @@ -277,7 +293,7 @@ const chatCompletion = await portkey.chat.completions.create({ } ]} ], - model: 'gemini-1.5-pro-001', + model: '@vertex-ai/gemini-3-pro-preview', max_tokens: 200 }); ``` @@ -305,7 +321,7 @@ completion = portkey.chat.completions.create( ] } ], - model='gemini-1.5-pro-001', + model='@vertex-ai/gemini-3-pro-preview', max_tokens=200 ) @@ -314,13 +330,10 @@ print(completion) ```sh cURL curl --location 'https://api.portkey.ai/v1/chat/completions' \ ---header 'x-portkey-provider: vertex-ai' \ ---header 'x-portkey-vertex-region: us-central1' \ --header 'Content-Type: application/json' \ --header 'x-portkey-api-key: PORTKEY_API_KEY' \ ---header 'Authorization: VERTEX_AI_ACCESS_TOKEN' \ --data '{ - "model": "gemini-1.5-pro-001", + "model": "@vertex-ai/gemini-3-pro-preview", "max_tokens": 200, "stream": false, "messages": [ @@ -365,7 +378,7 @@ Upload your PDF using the Files API to get a Google Files URL. ```javascript NodeJS const chatCompletion = await portkey.chat.completions.create({ - model: 'gemini-1.5-pro', + model: '@vertex-ai/gemini-3-pro-preview', messages: [{ role: 'user', content: [ @@ -383,7 +396,7 @@ console.log(chatCompletion.choices[0].message.content); ``` ```python Python completion = portkey.chat.completions.create( - model='gemini-1.5-pro', + model='@vertex-ai/gemini-3-pro-preview', messages=[{ "role": "user", "content": [ @@ -401,10 +414,8 @@ print(completion.choices[0].message.content) ``` ```sh cURL curl --location 'https://api.portkey.ai/v1/chat/completions' \ ---header 'x-portkey-provider: google' \ --header 'x-portkey-api-key: YOUR_PORTKEY_API_KEY' \ --header 'Content-Type: application/json' \ ---header 'Authorization: VERTEX_AI_ACCESS_TOKEN' \ --data '{ "model": "@VERTEX_PROVIDER/MODEL_NAME", "messages": [{ @@ -469,10 +480,8 @@ print(completion.choices[0].message.content) # Then use the encoded content in the request curl --location 'https://api.portkey.ai/v1/chat/completions' \ ---header 'x-portkey-provider: google' \ --header 'x-portkey-api-key: YOUR_PORTKEY_API_KEY' \ --header 'Content-Type: application/json' \ ---header 'Authorization: VERTEX_AI_ACCESS_TOKEN' \ --data '{ "model": "@VERTEX_PROVIDER/MODEL_NAME", "messages": [{ @@ -561,7 +570,7 @@ Note that you will have to set [`strict_open_ai_compliance=False`](/product/ai-g // Generate a chat completion async function getChatCompletionFunctions() { - const response = await portkey.chat.completions.create({ +const response = await portkey.chat.completions.create({ model: "@VERTEX_PROVIDER/anthropic.claude-3-7-sonnet@20250219", // your model slug from Portkey's Model Catalog max_tokens: 3000, thinking: { @@ -721,25 +730,6 @@ Note that you will have to set [`strict_open_ai_compliance=False`](/product/ai-g ``` -### Using reasoning_effort Parameter - -You can also control thinking using the OpenAI-compatible `reasoning_effort` parameter instead of `thinking.budget_tokens`: - -```python -response = portkey.chat.completions.create( - model="@VERTEX_PROVIDER/google.gemini-2.5-flash-preview-04-17", - max_tokens=3000, - reasoning_effort="medium", # Options: "none", "minimal", "low", "medium", "high" - messages=[{"role": "user", "content": "Explain quantum computing"}] -) -``` - -| reasoning_effort | Vertex thinkingLevel | -|------------------|---------------------| -| `none` | Disabled | -| `minimal`, `low` | `low` | -| `medium`, `high` | `high` | - ### Multi turn conversation @@ -800,7 +790,7 @@ response = portkey.chat.completions.create( // Generate a chat completion async function getChatCompletionFunctions() { - const response = await portkey.chat.completions.create({ +const response = await portkey.chat.completions.create({ model: "@VERTEX_PROVIDER/anthropic.claude-3-7-sonnet@20250219", // your model slug from Portkey's Model Catalog max_tokens: 3000, thinking: { @@ -1021,7 +1011,7 @@ async function getEmbeddings() { model: "@VERTEX_PROVIDER/text-multilingual-embedding-002", // your model slug from Portkey's Model Catalog // @ts-ignore (if using typescript) task_type: "CLASSIFICATION", // Optional - }, {Authorization: "Bearer $YOUR_VERTEX_ACCESS_TOKEN"}); + }); console.log(embeddings); } @@ -1039,7 +1029,7 @@ portkey = Portkey( # Generate embeddings def get_embeddings(): - embeddings = portkey.with_options(Authorization="Bearer $YOUR_VERTEX_ACCESS_TOKEN").embeddings.create( + embeddings = portkey.embeddings.create( input='The vector representation for this text', model='@VERTEX_PROVIDER/text-embedding-004', # your model slug from Portkey's Model Catalog task_type="CLASSIFICATION" # Optional @@ -1086,10 +1076,9 @@ Portkey supports the `Imagen API` on Vertex AI for image generations, letting yo curl https://api.portkey.ai/v1/images/generations \ -H "Content-Type: application/json" \ -H "x-portkey-api-key: $PORTKEY_API_KEY" \ - -H "x-portkey-provider: $PORTKEY_PROVIDER" \ -d '{ "prompt": "Cat flying to mars from moon", - "model":"@your-model-slug" + "model":"@vertex-ai/imagen-3.0-generate-001" }' ``` ```py Python @@ -1101,7 +1090,7 @@ client = Portkey( client.images.generate( prompt = "Cat flying to mars from moon", - model = "@VERTEX_PROVIDER/imagen-3.0-generate-001" # your model slug from Portkey's Model Catalog + model = "@vertex-ai/imagen-3.0-generate-001" # your model slug from Portkey's Model Catalog ) ``` ```ts JavaScript @@ -1114,7 +1103,7 @@ const client = new Portkey({ async function main() { const image = await client.images.generate({ prompt: "Cat flying to mars from moon", - model: "@VERTEX_PROVIDER/imagen-3.0-generate-001" # your model slug from Portkey's Model Catalog + model: "@vertex-ai/imagen-3.0-generate-001" # your model slug from Portkey's Model Catalog }); console.log(image.data); @@ -1144,7 +1133,7 @@ Pass labels in your request body or configure them in your gateway config using ```python completion = portkey.chat.completions.create( messages=[{"role": "user", "content": "Say this is a test"}], - model="@VERTEX_PROVIDER/gemini-1.5-pro-latest", + model="@VERTEX_PROVIDER/gemini-3-pro-preview", labels={"service_id": "backend-api", "environment": "production"} ) ``` @@ -1153,7 +1142,7 @@ completion = portkey.chat.completions.create( ```javascript const completion = await portkey.chat.completions.create({ messages: [{ role: 'user', content: 'Say this is a test' }], - model: '@VERTEX_PROVIDER/gemini-1.5-pro-latest', + model: '@VERTEX_PROVIDER/gemini-3-pro-preview', labels: { service_id: "backend-api", environment: "production" } }); ``` @@ -1270,7 +1259,7 @@ The image data is available in the `content_parts` field in the response and it // Generate a chat completion async function getChatCompletionFunctions() { - const response = await portkey.chat.completions.create({ +const response = await portkey.chat.completions.create({ model: "gemini-2.5-flash-image-preview", // your model slug from Portkey's Model Catalog max_tokens: 32768, stream: false, @@ -2316,7 +2305,7 @@ from portkey_ai import PORTKEY_GATEWAY_URL, createHeaders openai = OpenAI( api_key='PORTKEY_API_KEY', base_url=PORTKEY_GATEWAY_URL, - default_headers=createHeaders(provider='vertex-ai', strict_open_ai_compliance=False) + default_headers=createHeaders(provider='@vertex-ai', strict_open_ai_compliance=False) ) response = openai.chat.completions.create( @@ -2426,7 +2415,7 @@ from portkey_ai import PORTKEY_GATEWAY_URL, createHeaders openai = OpenAI( api_key='PORTKEY_API_KEY', base_url=PORTKEY_GATEWAY_URL, - default_headers=createHeaders(provider='vertex-ai', strict_open_ai_compliance=False) + default_headers=createHeaders(provider='@vertex-ai', strict_open_ai_compliance=False) ) response = openai.chat.completions.create( @@ -2545,7 +2534,7 @@ curl --location 'https://api.portkey.ai/v1/chat/completions' \ // Generate a chat completion async function getChatCompletionFunctions() { - const response = await portkey.chat.completions.create({ +const response = await portkey.chat.completions.create({ model: "gemini-2.5-flash-image-preview", // your model slug from Portkey's Model Catalog max_tokens: 32768, stream: false, @@ -2789,222 +2778,6 @@ curl --location 'https://api.portkey.ai/v1/chat/completions' \ ``` -## Safety settings -Gemini models support [configuring safety settings](https://docs.cloud.google.com/vertex-ai/generative-ai/docs/multimodal/configure-safety-filters#unsafe_prompts) to block potentially harmful responses - - - ```ts TypeScript -import Portkey from 'portkey-ai'; - -const portkey = new Portkey({ - apiKey: "PORTKEY_API_KEY", -}); - -async function main() { - const response = await portkey.chat.completions.create({ - model: "@my-vertex-provider/gemini-2.5-flash", - temperature: 0, - stream: false, - messages: [ - { - role: "user", - content: "Speak explicitly like a gangster harrasing someone for money" - } - ], - safety_settings: [ - { - category: "HARM_CATEGORY_SEXUALLY_EXPLICIT", - threshold: "BLOCK_LOW_AND_ABOVE" - }, - { - category: "HARM_CATEGORY_HATE_SPEECH", - threshold: "BLOCK_LOW_AND_ABOVE" - }, - { - category: "HARM_CATEGORY_HARASSMENT", - threshold: "BLOCK_LOW_AND_ABOVE" - }, - { - category: "HARM_CATEGORY_DANGEROUS_CONTENT", - threshold: "BLOCK_LOW_AND_ABOVE" - } - ] - }); - - console.log(response.choices); -} - -main(); - ``` - ```py Python -from portkey_ai import Portkey - -portkey = Portkey( - api_key="PORTKEY_API_KEY", -) - -response = portkey.chat.completions.create( - model="@my-vertex-provider/gemini-2.5-flash", - temperature=0, - stream=False, - messages=[ - { - "role": "user", - "content": "Speak explicitly like a gangster harrasing someone for money" - } - ], - safety_settings=[ - { - "category": "HARM_CATEGORY_SEXUALLY_EXPLICIT", - "threshold": "BLOCK_LOW_AND_ABOVE" - }, - { - "category": "HARM_CATEGORY_HATE_SPEECH", - "threshold": "BLOCK_LOW_AND_ABOVE" - }, - { - "category": "HARM_CATEGORY_HARASSMENT", - "threshold": "BLOCK_LOW_AND_ABOVE" - }, - { - "category": "HARM_CATEGORY_DANGEROUS_CONTENT", - "threshold": "BLOCK_LOW_AND_ABOVE" - } - ] -) - -print(response.choices) - ``` - ```js OpenAI NodeJS -import OpenAI from "openai"; - -const openai = new OpenAI({ - apiKey: process.env.PORTKEY_API_KEY, // Portkey API Key - baseURL: "https://api.portkey.ai/v1", - defaultHeaders: { - "x-portkey-strict-open-ai-compliance": "false", - }, -}); - -async function main() { - const response = await openai.chat.completions.create({ - model: "@my-vertex-provider/gemini-2.5-flash", - temperature: 0, - stream: false, - messages: [ - { - role: "user", - content: "Speak explicitly like a gangster harrasing someone for money" - } - ], - safety_settings: [ - { - category: "HARM_CATEGORY_SEXUALLY_EXPLICIT", - threshold: "BLOCK_LOW_AND_ABOVE" - }, - { - category: "HARM_CATEGORY_HATE_SPEECH", - threshold: "BLOCK_LOW_AND_ABOVE" - }, - { - category: "HARM_CATEGORY_HARASSMENT", - threshold: "BLOCK_LOW_AND_ABOVE" - }, - { - category: "HARM_CATEGORY_DANGEROUS_CONTENT", - threshold: "BLOCK_LOW_AND_ABOVE" - } - ] - }); - - console.log(response.choices); -} - -main(); - ``` - ```py OpenAI Python -import openai - -client = openai.OpenAI( - api_key="PORTKEY_API_KEY", - base_url="https://api.portkey.ai/v1", - default_headers={ - "x-portkey-strict-open-ai-compliance": "false" - } -) - -response = client.chat.completions.create( - model="@my-vertex-provider/gemini-2.5-flash", - temperature=0, - stream=False, - messages=[ - { - "role": "user", - "content": "Speak explicitly like a gangster harrasing someone for money" - } - ], - safety_settings=[ - { - "category": "HARM_CATEGORY_SEXUALLY_EXPLICIT", - "threshold": "BLOCK_LOW_AND_ABOVE" - }, - { - "category": "HARM_CATEGORY_HATE_SPEECH", - "threshold": "BLOCK_LOW_AND_ABOVE" - }, - { - "category": "HARM_CATEGORY_HARASSMENT", - "threshold": "BLOCK_LOW_AND_ABOVE" - }, - { - "category": "HARM_CATEGORY_DANGEROUS_CONTENT", - "threshold": "BLOCK_LOW_AND_ABOVE" - } - ] -) - -print(response.choices) - ``` - ```sh cURL -curl --location 'https://api.portkey.ai/v1/chat/completions' \ ---header 'Content-Type: application/json' \ ---header 'x-portkey-api-key: $PORTKEY_API_KEY' \ ---header 'x-portkey-strict-open-ai-compliance: false' \ ---data '{ - "model": "@my-vertex-provider/gemini-2.5-flash", - "temperature": 0, - "stream": false, - "messages": [ - { - "role": "user", - "content": "Speak explicitly like a gangster harrasing someone for money" - } - ], - "safety_settings": [ - { - "category": "HARM_CATEGORY_SEXUALLY_EXPLICIT", - "threshold": "BLOCK_LOW_AND_ABOVE" - }, - { - "category": "HARM_CATEGORY_HATE_SPEECH", - "threshold": "BLOCK_LOW_AND_ABOVE" - }, - { - "category": "HARM_CATEGORY_HARASSMENT", - "threshold": "BLOCK_LOW_AND_ABOVE" - }, - { - "category": "HARM_CATEGORY_DANGEROUS_CONTENT", - "threshold": "BLOCK_LOW_AND_ABOVE" - } - ] -}' - ``` - - -The response contains the `saftetyRatings` keys in the choice object of the response. -You only need to pass the `strict-open-ai-compliance` flag if you need the `safetyRatings` in response. - ## Making Requests Without Portkey's Model Catalog You can also pass your Vertex AI details & secrets directly without using the Portkey's Model Catalog. @@ -3032,7 +2805,7 @@ Vertex AI expects a `region`, a `project ID` and the `access token` in the reque const chatCompletion = await portkey.chat.completions.create({ messages: [{ role: 'user', content: 'Say this is a test' }], - model: 'gemini-pro', + model: 'gemini-3-pro-preview', }); console.log(chatCompletion.choices); @@ -3052,7 +2825,7 @@ portkey = Portkey( completion = portkey.chat.completions.create( messages= [{ "role": 'user', "content": 'Say this is a test' }], - model= 'gemini-1.5-pro-latest' + model= 'gemini-3-pro-preview' ) print(completion) @@ -3078,7 +2851,7 @@ const portkey = new OpenAI({ async function main() { const response = await portkey.chat.completions.create({ messages: [{ role: "user", content: "1729" }], - model: "gemini-1.5-flash-001", + model: "gemini-3-pro-preview", max_tokens: 32, }); @@ -3098,7 +2871,7 @@ curl 'https://api.portkey.ai/v1/chat/completions' \ -H 'x-portkey-vertex-project-id: sample-94994' \ -H 'x-portkey-vertex-region: us-central1' \ --data '{ - "model": "gemini-1.5-pro", + "model": "gemini-3-pro-preview", "messages": [ { "role": "system", @@ -3163,14 +2936,26 @@ This method is simpler but may not have all the permissions needed for custom en ## Next Steps -The complete list of features supported in the SDK are available on the link below. + - + +Complete SDK documentation and API reference -You'll find more information in the relevant sections: + +Add metadata to your Vertex AI requests + + + +Configure advanced gateway features + + + +Trace and monitor your Vertex AI requests + + + +Create fallback configurations between providers + -1. [Add metadata to your requests](/product/observability/metadata) -2. [Add gateway configs to your Vertex AI requests](/product/ai-gateway/configs) -3. [Tracing Vertex AI requests](/product/observability/traces) -4. [Setup a fallback from OpenAI to Vertex AI APIs](/product/ai-gateway/fallbacks) + diff --git a/integrations/llms/vertex-ai/batches.mdx b/integrations/llms/vertex-ai/batches.mdx index 16192676..570e9a8d 100644 --- a/integrations/llms/vertex-ai/batches.mdx +++ b/integrations/llms/vertex-ai/batches.mdx @@ -9,7 +9,7 @@ With Portkey, you can perform batch inference operations with Vertex AI models. - Perform A/B testing with different foundation models ## Before You Start -1. **Portkey API key** and a **Vertex virtual key** in your workspace. +1. **Portkey API key** and a **Vertex AI provider** configured in Model Catalog. 2. A **GCS bucket** in the same region as your model + `aiplatform-service-agent` permission on the file. 3. *(Only for Portkey-native batching)* A **Portkey File** (`input_file_id`). 4. Familiarity with the [Create Batch OpenAPI spec](/api-reference/inference-api/batch/create-batch). @@ -126,7 +126,7 @@ const openai = new OpenAI({ ```bash curl curl -X POST --header 'x-portkey-api-key: ' \ - --header 'x-portkey-provider: ' \ + --header 'x-portkey-provider: @your-vertex-provider' \ --header 'x-portkey-vertex-storage-bucket-name: ' \ --header 'x-portkey-provider-file-name: .jsonl' \ --header 'x-portkey-provider-model: ' \ @@ -234,7 +234,7 @@ const openai = new OpenAI({ ```bash curl curl -X POST --header 'Content-Type: application/json' \ --header 'x-portkey-api-key: ' \ - --header 'x-portkey-provider: ' \ + --header 'x-portkey-provider: @your-vertex-provider' \ --data \ $'{"input_file_id": "", "endpoint": "/v1/chat/completions", "completion_window": "24h", "model":"gemini-1.5-flash-001"}' \ 'https://api.portkey.ai/v1/batches' @@ -328,7 +328,7 @@ const openai = new OpenAI({ ```bash curl curl -X GET --header 'x-portkey-api-key: ' \ ---header 'x-portkey-provider: ' \ +--header 'x-portkey-provider: @your-vertex-provider' \ 'https://api.portkey.ai/v1/batches' ``` @@ -418,7 +418,7 @@ const openai = new OpenAI({ ```bash curl curl -X GET --header 'x-portkey-api-key: ' \ ---header 'x-portkey-provider: ' \ +--header 'x-portkey-provider: @your-vertex-provider' \ 'https://api.portkey.ai/v1/batches/' ``` @@ -428,7 +428,7 @@ curl -X GET --header 'x-portkey-api-key: ' \ ```bash curl curl -X GET --header 'x-portkey-api-key: ' \ ---header 'x-portkey-provider: ' \ +--header 'x-portkey-provider: @your-vertex-provider' \ 'https://api.portkey.ai/v1/batches//output' ``` \ No newline at end of file diff --git a/integrations/llms/vertex-ai/files.mdx b/integrations/llms/vertex-ai/files.mdx index cf7ebf7a..68782414 100644 --- a/integrations/llms/vertex-ai/files.mdx +++ b/integrations/llms/vertex-ai/files.mdx @@ -58,7 +58,7 @@ uploadFile(); ```sh curl -X POST --header 'x-portkey-api-key: ' \ - --header 'x-portkey-provider: ' \ + --header 'x-portkey-provider: @your-vertex-provider' \ --header 'x-portkey-vertex-storage-bucket-name: ' \ --header 'x-portkey-provider-file-name: .jsonl' \ --header 'x-portkey-provider-model: ' \ @@ -165,7 +165,7 @@ getFile(); ```sh curl -X GET --header 'x-portkey-api-key: ' \ ---header 'x-portkey-provider: ' \ +--header 'x-portkey-provider: @your-vertex-provider' \ 'https://api.portkey.ai/v1/files/' ``` @@ -254,7 +254,7 @@ getFileContent(); ```sh curl -X GET --header 'x-portkey-api-key: ' \ ---header 'x-portkey-provider: ' \ +--header 'x-portkey-provider: @your-vertex-provider' \ 'https://api.portkey.ai/v1/files//content' ``` diff --git a/integrations/llms/vertex-ai/fine-tuning.mdx b/integrations/llms/vertex-ai/fine-tuning.mdx index 494e10d5..66d79039 100644 --- a/integrations/llms/vertex-ai/fine-tuning.mdx +++ b/integrations/llms/vertex-ai/fine-tuning.mdx @@ -111,7 +111,7 @@ const openai = new OpenAI({ ```sh curl -X POST --header 'x-portkey-api-key: ' \ - --header 'x-portkey-provider: ' \ + --header 'x-portkey-provider: @your-vertex-provider' \ --header 'x-portkey-vertex-storage-bucket-name: ' \ --header 'x-portkey-provider-file-name: .jsonl' \ --header 'x-portkey-provider-model: ' \ @@ -233,7 +233,7 @@ const openai = new OpenAI({ ```sh curl -X POST --header 'Content-Type: application/json' \ --header 'x-portkey-api-key: ' \ - --header 'x-portkey-provider: ' \ + --header 'x-portkey-provider: @your-vertex-provider' \ --data \ $'{"model": "", "suffix": "", "training_file": "gs:///.jsonl", "hyperparameters": {"n_epochs": 2}}\n' \ 'https://api.portkey.ai/v1/fine_tuning/jobs' @@ -331,7 +331,7 @@ const openai = new OpenAI({ ```sh curl -X GET --header 'x-portkey-api-key: ' \ ---header 'x-portkey-provider: ' \ +--header 'x-portkey-provider: @your-vertex-provider' \ 'https://api.portkey.ai/v1/fine_tuning/jobs' ``` @@ -427,7 +427,7 @@ const openai = new OpenAI({ ```sh curl -X GET --header 'x-portkey-api-key: ' \ ---header 'x-portkey-provider: ' \ +--header 'x-portkey-provider: @your-vertex-provider' \ 'https://api.portkey.ai/v1/fine_tuning/jobs/' ``` @@ -523,7 +523,7 @@ const openai = new OpenAI({ ```sh curl -X POST --header 'x-portkey-api-key: ' \ ---header 'x-portkey-provider: ' \ +--header 'x-portkey-provider: @your-vertex-provider' \ 'https://api.portkey.ai/v1/fine_tuning/jobs//cancel' ``` diff --git a/integrations/llms/vllm.mdx b/integrations/llms/vllm.mdx index 069d16f4..6509ea12 100644 --- a/integrations/llms/vllm.mdx +++ b/integrations/llms/vllm.mdx @@ -1,171 +1,133 @@ --- title: "vLLM" -description: "Integrate vLLM-hosted custom models with Portkey and take them to production" +description: "Integrate vLLM-hosted custom models with Portkey for production observability and reliability." --- -Portkey provides a robust and secure platform to observe, govern, and manage your **locally** or **privately** hosted custom models using vLLM. +Portkey provides a robust platform to observe, govern, and manage your **locally** or **privately** hosted custom models using vLLM. Here's a [list](https://docs.vllm.ai/en/latest/models/supported_models.html) of all model architectures supported on vLLM. -## Integrating Custom Models with Portkey SDK +## Integration Steps + -Expose your vLLM server by using a tunneling service like [ngrok](https://ngrok.com/) or any other way you prefer. You can skip this step if you’re self-hosting the Gateway. +Expose your vLLM server using a tunneling service like [ngrok](https://ngrok.com/) or make it publicly accessible. Skip this if you're self-hosting the Gateway. ```sh -ngrok http 11434 --host-header="localhost:8080" +ngrok http 8000 --host-header="localhost:8080" ``` - - - - - ```sh - npm install --save portkey-ai - ``` - - - ```sh - pip install portkey-ai - ``` - - + +1. Go to [**Model Catalog → Add Provider**](https://app.portkey.ai/model-catalog/providers) +2. Enable **"Local/Privately hosted provider"** toggle +3. Select **OpenAI** as the provider type (vLLM follows OpenAI API schema) +4. Enter your vLLM server URL in **Custom Host**: `https://your-vllm-server.ngrok-free.app` +5. Add authentication headers if needed +6. Name your provider (e.g., `my-vllm`) + + + See all setup options + - - -1. Pass your publicly-exposed vLLM server URL to Portkey with `customHost` (by default, vLLM is on `http://localhost:8000/v1`) -2. Set target `provider` as `openai` since the server follows OpenAI API schema. - - - - ```js - import Portkey from 'portkey-ai' - - const portkey = new Portkey({ - apiKey: "PORTKEY_API_KEY", - provider: "openai", - customHost: "https://7cc4-3-235-157-146.ngrok-free.app" // Your vLLM ngrok URL - Authorization: "AUTH_KEY", // If you need to pass auth - }) - ``` - - - ```python - from portkey_ai import Portkey - - portkey = Portkey( - api_key="PORTKEY_API_KEY", - provider="openai", - custom_host="https://7cc4-3-235-157-146.ngrok-free.app" # Your vLLM ngrok URL - Authorization="AUTH_KEY", # If you need to pass auth - ) - ``` - - - - -More on `custom_host` [here](/product/ai-gateway/universal-api#integrating-local-or-private-models). - - - -Use the Portkey SDK to invoke chat completions from your model, just as you would with any other provider: - - - - ```js - const chatCompletion = await portkey.chat.completions.create({ - messages: [{ role: 'user', content: 'Say this is a test' }] - }); - - console.log(chatCompletion.choices); - ``` - - -```python -completion = portkey.chat.completions.create( - messages= [{ "role": 'user', "content": 'Say this is a test' }] + + + + + +```python Python +from portkey_ai import Portkey + +portkey = Portkey( + api_key="PORTKEY_API_KEY", + provider="@my-vllm" ) -print(completion) +response = portkey.chat.completions.create( + model="your-model-name", + messages=[{"role": "user", "content": "Hello!"}] +) + +print(response.choices[0].message.content) ``` - - - - -## [Using Virtual Keys](https://app.portkey.ai/virtual-keys) - -Virtual Keys serve as Portkey's unified authentication system for all LLM interactions, simplifying the use of multiple providers and Portkey features within your application. For self-hosted LLMs, you can configure custom authentication requirements including authorization keys, bearer tokens, or any other headers needed to access your model: - - - - - -1. Navigate to [Virtual Keys](https://app.portkey.ai/virtual-keys) in your Portkey dashboard -2. Click **"Add Key"** and enable the **"Local/Privately hosted provider"** toggle -3. Configure your deployment: - - Select the matching provider API specification (typically `OpenAI`) - - Enter your model's base URL in the `Custom Host` field - - Add required authentication headers and their values -4. Click **"Create"** to generate your virtual key - -You can now use this virtual key in your requests: - - - - ```js - const portkey = new Portkey({ - apiKey: "PORTKEY_API_KEY", - provider:"@YOUR_SELF_HOSTED_LLM_PROVIDER" - - async function main() { - const response = await client.chat.completions.create({ - messages: [{ role: "user", content: "Bob the builder.." }], - model: "your-self-hosted-model-name", - }); - - console.log(response.choices[0].message.content); - }) - ``` - - - ```python - portkey = Portkey( - api_key="PORTKEY_API_KEY", - provider="@YOUR_SELF_HOSTED_LLM_PROVIDER" - ) - - response = portkey.chat.completions.create( - model="your-self-hosted-model-name", - messages=[ - {"role": "system", "content": "You are a helpful assistant."}, - {"role": "user", "content": "Hello!"} - ] - - print(response) - ) - ``` - - - -For more information about managing self-hosted LLMs with Portkey, see [Bring Your Own LLM](/integrations/llms/byollm). +```javascript Node.js +import Portkey from 'portkey-ai'; +const portkey = new Portkey({ + apiKey: 'PORTKEY_API_KEY', + provider: '@my-vllm' +}); +const response = await portkey.chat.completions.create({ + model: 'your-model-name', + messages: [{ role: 'user', content: 'Hello!' }] +}); -## Next Steps +console.log(response.choices[0].message.content); +``` + + -Explore the complete list of features supported in the SDK: +**Or use custom host directly:** + + + +```python Python +from portkey_ai import Portkey + +portkey = Portkey( + api_key="PORTKEY_API_KEY", + provider="openai", + custom_host="https://your-vllm-server.ngrok-free.app", + Authorization="AUTH_KEY" # If needed +) +``` + +```javascript Node.js +import Portkey from 'portkey-ai'; + +const portkey = new Portkey({ + apiKey: 'PORTKEY_API_KEY', + provider: 'openai', + customHost: 'https://your-vllm-server.ngrok-free.app', + Authorization: 'AUTH_KEY' // If needed +}); +``` + + + + + + + +**Important:** vLLM follows the OpenAI API specification, so set the provider as `openai` when using custom host directly. By default, vLLM runs on `http://localhost:8000/v1`. + - --- -You'll find more information in the relevant sections: +## Next Steps -1. [Add metadata to your requests](/product/observability/metadata) -2. [Add gateway configs to your requests](/product/ai-gateway/universal-api#ollama-in-configs) -3. [Tracing requests](/product/observability/traces) -4. [Setup a fallback from OpenAI to your local LLM](/product/ai-gateway/fallbacks) + + + Add retries, timeouts, and fallbacks + + + Monitor your vLLM deployments + + + Learn more about custom host setup + + + Complete guide for private LLMs + + + +For complete SDK documentation: + + + Complete Portkey SDK documentation + diff --git a/integrations/llms/voyage-ai.mdx b/integrations/llms/voyage-ai.mdx index fcea3c42..a5035d65 100644 --- a/integrations/llms/voyage-ai.mdx +++ b/integrations/llms/voyage-ai.mdx @@ -1,143 +1,255 @@ --- title: "Voyage AI" -description: "Portkey provides a robust and secure gateway to facilitate the integration of various Large Language Models (LLMs) into your applications, including Voyage AI's embedding and Re-rank endpoints. -" +description: Use Voyage AI's embeddings and reranking models through Portkey. --- -With Portkey, you can take advantage of features like fast AI gateway access, observability, prompt management, and more, all while ensuring the secure management of your LLM API keys through a [virtual key](/product/ai-gateway/virtual-keys) system. +## Quick Start + +Get started with Voyage AI in under 2 minutes: + + + +```python Python icon="python" +from portkey_ai import Portkey + +# 1. Install: pip install portkey-ai +# 2. Add @voyage provider in model catalog +# 3. Use it: + +portkey = Portkey(api_key="PORTKEY_API_KEY") + +embedding = portkey.embeddings.create( + model="@voyage/voyage-3", + input="Name the tallest buildings in Hawaii" +) + +print(embedding.data[0].embedding) +``` + +```js Javascript icon="square-js" +import Portkey from 'portkey-ai' + +// 1. Install: npm install portkey-ai +// 2. Add @voyage provider in model catalog +// 3. Use it: + +const portkey = new Portkey({ + apiKey: "PORTKEY_API_KEY" +}) + +const embedding = await portkey.embeddings.create({ + model: "@voyage/voyage-3", + input: "Name the tallest buildings in Hawaii" +}) + +console.log(embedding.data[0].embedding) +``` + +```python OpenAI Py icon="python" +from openai import OpenAI +from portkey_ai import PORTKEY_GATEWAY_URL + +# 1. Install: pip install openai portkey-ai +# 2. Add @voyage provider in model catalog +# 3. Use it: + +client = OpenAI( + api_key="PORTKEY_API_KEY", # Portkey API key + base_url=PORTKEY_GATEWAY_URL +) + +embedding = client.embeddings.create( + model="@voyage/voyage-3", + input="Name the tallest buildings in Hawaii" +) + +print(embedding.data[0].embedding) +``` + +```js OpenAI JS icon="square-js" +import OpenAI from "openai" +import { PORTKEY_GATEWAY_URL } from "portkey-ai" + +// 1. Install: npm install openai portkey-ai +// 2. Add @voyage provider in model catalog +// 3. Use it: + +const client = new OpenAI({ + apiKey: "PORTKEY_API_KEY", // Portkey API key + baseURL: PORTKEY_GATEWAY_URL +}) + +const embedding = await client.embeddings.create({ + model: "@voyage/voyage-3", + input: "Name the tallest buildings in Hawaii" +}) + +console.log(embedding.data[0].embedding) +``` + +```bash cURL icon="square-terminal" +# 1. Add @voyage provider in model catalog +# 2. Use it: + +curl https://api.portkey.ai/v1/embeddings \ + -H "Content-Type: application/json" \ + -H "x-portkey-api-key: $PORTKEY_API_KEY" \ + -d '{ + "model": "@voyage/voyage-3", + "input": "Name the tallest buildings in Hawaii" + }' +``` + + -Provider Slug: **voyage** +**Tip:** You can also set `provider="@voyage"` in `Portkey()` and use just `model="voyage-3"` in the request. -## Portkey SDK Integration with Voyage +## Add Provider in Model Catalog -Portkey provides a consistent API to interact with models from Voyage. To integrate Voyage with Portkey: +Before making requests, add Voyage AI to your Model Catalog: -### 1. Install the Portkey SDK +1. Go to [**Model Catalog → Add Provider**](https://app.portkey.ai/model-catalog/providers) +2. Select **Voyage AI** +3. Enter your [Voyage API key](https://dash.voyageai.com/) +4. Name your provider (e.g., `voyage`) -Add the Portkey SDK to your application to interact with Voyage AI's models through Portkey's gateway. + + See all setup options and detailed configuration instructions + - - - ```sh - npm install --save portkey-ai - ``` - - - ```sh - pip install portkey-ai - ``` - - +--- -### 2. Initialize Portkey with the API Key +## Voyage AI Capabilities -To use Voyage with Portkey, [get your API key from here](https://dash.voyageai.com/), then add it to Portkey to create the virtual key. +### Embeddings - - - ```javascript - import Portkey from 'portkey-ai' +Generate high-quality embeddings: - const portkey = new Portkey({ - apiKey: "PORTKEY_API_KEY", // defaults to process.env["PORTKEY_API_KEY"] - Authorization: "VOYAGE_API_KEY", // Replace with your Voyage API key - provider: "voyage" - }) - ``` - - - ```python - from portkey_ai import Portkey - - portkey = Portkey( - api_key="PORTKEY_API_KEY", # Replace with your Portkey API key - Authorization="VOYAGE_API_KEY", # Replace with your Voyage API key - provider="voyage" - ) - ``` - - + -### Embeddings +```python Python +from portkey_ai import Portkey + +portkey = Portkey(api_key="PORTKEY_API_KEY", provider="@voyage") -Embedding endpoints are natively supported within Portkey like this: +embedding = portkey.embeddings.create( + model="voyage-3", + input="Name the tallest buildings in Hawaii" +) + +print(embedding.data[0].embedding) +``` + +```javascript Node.js +import Portkey from 'portkey-ai'; + +const portkey = new Portkey({ + apiKey: 'PORTKEY_API_KEY', + provider: '@voyage' +}); - - - ```javascript const embedding = await portkey.embeddings.create({ - input: 'Name the tallest buildings in Hawaii', - model: 'voyage-3' + model: "voyage-3", + input: "Name the tallest buildings in Hawaii" }); - console.log(embedding); - ``` - - - ```python - embedding = portkey.embeddings.create( - input= 'Name the tallest buildings in Hawaii', - model= 'voyage-3' - ) - - print(embedding) +console.log(embedding.data[0].embedding); ``` - - -### Re-ranking + -You can use Voyage reranking the `portkey.post` method with the body expected by Voyage +### Reranking + +Rerank documents for better search results: + + + +```python Python +from portkey_ai import Portkey + +portkey = Portkey(api_key="PORTKEY_API_KEY", provider="@voyage") - - - ```javascript - const response = await portkey.post( - "/rerank", - { - "model": "rerank-2-lite", - "query": "What is the capital of the United States?", - "documents": [ - "Carson City is the capital city of the American state of Nevada.", - "The Commonwealth of the Northern Mariana Islands is a group of islands in the Pacific Ocean. Its capital is Saipan.", - "Washington, D.C. (also known as simply Washington or D.C., and officially as the District of Columbia) is the capital of the United States. It is a federal district.", - "Capital punishment (the death penalty) has existed in the United States since beforethe United States was a country. As of 2017, capital punishment is legal in 30 of the 50 states." - ] - }) - ``` - - - ```python response = portkey.post( "/rerank", model="rerank-2-lite", query="What is the capital of the United States?", documents=[ "Carson City is the capital city of the American state of Nevada.", - "The Commonwealth of the Northern Mariana Islands is a group of islands in the Pacific Ocean. Its capital is Saipan.", - "Washington, D.C. (also known as simply Washington or D.C., and officially as the District of Columbia) is the capital of the United States. It is a federal district.", - "Capital punishment (the death penalty) has existed in the United States since beforethe United States was a country. As of 2017, capital punishment is legal in 30 of the 50 states." + "The Commonwealth of the Northern Mariana Islands is a group of islands in the Pacific Ocean.", + "Washington, D.C. is the capital of the United States.", + "Capital punishment has existed in the United States since before it was a country." ] ) print(response) ``` - - -## Next Steps +```javascript Node.js +import Portkey from 'portkey-ai'; -The complete list of features supported in the SDK is available on the link below. +const portkey = new Portkey({ + apiKey: 'PORTKEY_API_KEY', + provider: '@voyage' +}); - - Explore the SDK documentation - +const response = await portkey.post( + "/rerank", + { + model: "rerank-2-lite", + query: "What is the capital of the United States?", + documents: [ + "Carson City is the capital city of the American state of Nevada.", + "The Commonwealth of the Northern Mariana Islands is a group of islands in the Pacific Ocean.", + "Washington, D.C. is the capital of the United States.", + "Capital punishment has existed in the United States since before it was a country." + ] + } +); + +console.log(response); +``` + + + +--- + +## Supported Models -You'll find more information in the relevant sections: +Voyage AI offers specialized embedding and reranking models: -1. [Add metadata to your requests](/product/observability/metadata) -2. [Add gateway configs to your Voyage requests](/product/ai-gateway/configs) -3. [Tracing Voyage requests](/product/observability/traces) -4. [Setup a fallback from OpenAI to Voyage APIs](/product/ai-gateway/fallbacks) +| Model | Type | Description | +|-------|------|-------------| +| voyage-3 | Embedding | Latest general-purpose embedding model | +| voyage-3-lite | Embedding | Faster, lighter version | +| voyage-code-3 | Embedding | Optimized for code | +| rerank-2 | Reranking | High-quality reranking | +| rerank-2-lite | Reranking | Faster reranking | + +Check [Voyage AI's documentation](https://docs.voyageai.com/) for the complete model list. + +--- + +## Next Steps + + + + Add fallbacks, load balancing, and more + + + Monitor and trace your Voyage requests + + + Cache embeddings for faster responses + + + Add custom metadata to requests + + + +For complete SDK documentation: + + + Complete Portkey SDK documentation + diff --git a/integrations/llms/workers-ai.mdx b/integrations/llms/workers-ai.mdx index b9a1af53..4ebad4a7 100644 --- a/integrations/llms/workers-ai.mdx +++ b/integrations/llms/workers-ai.mdx @@ -1,230 +1,198 @@ --- title: "Workers AI" +description: Use Cloudflare Workers AI models through Portkey for serverless AI inference. --- -Portkey provides a robust and secure gateway to facilitate the integration of various Large Language Models (LLMs) into your applications, including [Workers AI.](https://developers.cloudflare.com/workers-ai/) +## Quick Start -With Portkey, you can take advantage of features like fast AI gateway access, observability, prompt management, and more, all while ensuring the secure management of your LLM API keys through a [virtual key](/product/ai-gateway/virtual-keys) system. - -Provider Slug. **workers-ai** - -## Portkey SDK Integration with Workers AI Models +Get started with Workers AI in under 2 minutes: -Portkey provides a consistent API to interact with models from various providers. To integrate Workers AI with Portkey: + -### 1\. Install the Portkey SDK +```python Python icon="python" +from portkey_ai import Portkey -Add the Portkey SDK to your application to interact with Workers AI's API through Portkey's gateway. - - - ```sh - npm install --save portkey-ai - ``` - - - ```sh - pip install portkey-ai - ``` - +# 1. Install: pip install portkey-ai +# 2. Add @workers-ai provider in model catalog +# 3. Use it: - +portkey = Portkey(api_key="PORTKEY_API_KEY") +response = portkey.chat.completions.create( + model="@workers-ai/@cf/meta/llama-3.2-3b-instruct", + messages=[{"role": "user", "content": "Hello!"}] +) +print(response.choices[0].message.content) +``` -### 2\. Initialize Portkey with the Virtual Key +```js Javascript icon="square-js" +import Portkey from 'portkey-ai' -To use Workers AI with Portkey, [get your API key from here](https://console.groq.com/keys), then add it to Portkey to create the virtual key. - - - ```js - import Portkey from 'portkey-ai' +// 1. Install: npm install portkey-ai +// 2. Add @workers-ai provider in model catalog +// 3. Use it: - const portkey = new Portkey({ - apiKey: "PORTKEY_API_KEY", // defaults to process.env["PORTKEY_API_KEY"] - provider:"@PROVIDER" // Your Workers AI Virtual Key - }) - ``` - - - ```python - from portkey_ai import Portkey +const portkey = new Portkey({ + apiKey: "PORTKEY_API_KEY" +}) - portkey = Portkey( - api_key="PORTKEY_API_KEY", # Replace with your Portkey API key - provider="@PROVIDER" # Replace with your virtual key for Groq - ) - ``` - +const response = await portkey.chat.completions.create({ + model: "@workers-ai/@cf/meta/llama-3.2-3b-instruct", + messages: [{ role: "user", content: "Hello!" }] +}) - +console.log(response.choices[0].message.content) +``` +```python OpenAI Py icon="python" +from openai import OpenAI +from portkey_ai import PORTKEY_GATEWAY_URL +# 1. Install: pip install openai portkey-ai +# 2. Add @workers-ai provider in model catalog +# 3. Use it: +client = OpenAI( + api_key="PORTKEY_API_KEY", # Portkey API key + base_url=PORTKEY_GATEWAY_URL +) -### **3\. Invoke Chat Completions with** Workers AI +response = client.chat.completions.create( + model="@workers-ai/@cf/meta/llama-3.2-3b-instruct", + messages=[{"role": "user", "content": "Hello!"}] +) -Use the Portkey instance to send requests to Workers AI. You can also override the virtual key directly in the API call if needed. - - - ```js - const chatCompletion = await portkey.chat.completions.create({ - messages: [{ role: 'user', content: 'Say "this is a test"' }], - model: '@cf/meta/llama-3.2-3b-instruct', - }); +print(response.choices[0].message.content) +``` - console.log(chatCompletion.choices); - ``` - - - ```python - completion = portkey.chat.completions.create( - messages= [{ "role": 'user', "content": 'Say "this is a test"' }], - model= '@cf/meta/llama-3.2-3b-instruct' - ) +```js OpenAI JS icon="square-js" +import OpenAI from "openai" +import { PORTKEY_GATEWAY_URL } from "portkey-ai" - print(completion) - ``` - +// 1. Install: npm install openai portkey-ai +// 2. Add @workers-ai provider in model catalog +// 3. Use it: - +const client = new OpenAI({ + apiKey: "PORTKEY_API_KEY", // Portkey API key + baseURL: PORTKEY_GATEWAY_URL +}) +const response = await client.chat.completions.create({ + model: "@workers-ai/@cf/meta/llama-3.2-3b-instruct", + messages: [{ role: "user", content: "Hello!" }] +}) +console.log(response.choices[0].message.content) +``` +```sh cURL icon="square-terminal" +# 1. Add @workers-ai provider in model catalog +# 2. Use it: -### **4\. Invoke Image Generation with** Workers AI +curl https://api.portkey.ai/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "x-portkey-api-key: $PORTKEY_API_KEY" \ + -d '{ + "model": "@workers-ai/@cf/meta/llama-3.2-3b-instruct", + "messages": [{"role": "user", "content": "Hello!"}] + }' +``` -Use the Portkey instance to send requests to Workers AI. You can also override the virtual key directly in the API call if needed. + -Portkey supports the OpenAI signature to make text-to-image requests. +## Add Provider in Model Catalog - - -```js -import Portkey from 'portkey-ai'; +Before making requests, add Workers AI to your Model Catalog: -// Initialize the Portkey client -const portkey = new Portkey({ - apiKey: "PORTKEY_API_KEY", // Replace with your Portkey API key - provider:"@PROVIDER" -}); +1. Go to [**Model Catalog → Add Provider**](https://app.portkey.ai/model-catalog/providers) +2. Select **Workers AI** +3. Enter your [Cloudflare API key](https://developers.cloudflare.com/workers-ai/) +4. Name your provider (e.g., `workers-ai`) -async function main() { - const image = await portkey.images.generate({ - model: "image_model_name", - prompt: "Lucy in the sky with diamonds" - }); + + See all setup options and detailed configuration instructions + - console.log(image.data); -} +--- -main(); -``` - - +## Workers AI Capabilities -```py -from portkey_ai import Portkey -from IPython.display import display, Image +### Image Generation -# Initialize the Portkey client -portkey = Portkey( - api_key="PORTKEY_API_KEY", # Replace with your Portkey API key - provider="@PROVIDER" -) +Workers AI supports image generation with various models: -image = portkey.images.generate( - model="image_model_name", - prompt="Lucy in the sky with diamonds" -) + -# Display the image -display(Image(url=image.data[0].url)) -``` - - - -```js -import OpenAI from 'openai'; // We're using the v4 SDK -import { PORTKEY_GATEWAY_URL, createHeaders } from 'portkey-ai' - -const openai = new OpenAI({ - apiKey: 'WORKERS_AI_API_KEY', // defaults to process.env["WORKERS_AI_API_KEY"], - baseURL: PORTKEY_GATEWAY_URL, - defaultHeaders: createHeaders({ - provider: "openai", - apiKey: "PORTKEY_API_KEY" // defaults to process.env["PORTKEY_API_KEY"] - }) -}); +```python Python +from portkey_ai import Portkey -async function main() { - const image = await openai.images.generate({ - model: "image_model_name", - prompt: "Lucy in the sky with diamonds" - }); +portkey = Portkey(api_key="PORTKEY_API_KEY", provider="@workers-ai") - console.log(image.data); -} +image = portkey.images.generate( + model="@cf/stabilityai/stable-diffusion-xl-base-1.0", + prompt="A beautiful sunset over mountains" +) -main(); +print(image.data[0].url) ``` - - - -```py -from openai import OpenAI -from portkey_ai import PORTKEY_GATEWAY_URL, createHeaders -from IPython.display import display, Image -client = OpenAI( - api_key='WORKERS_AI_API_KEY', - base_url=PORTKEY_GATEWAY_URL, - default_headers=createHeaders( - provider="openai", - api_key="PORTKEY_API_KEY" - ) -) +```javascript Node.js +import Portkey from 'portkey-ai'; -image = client.images.generate( - model="image_model_name", - prompt="Lucy in the sky with diamonds", - n=1, - size="1024x1024" -) +const portkey = new Portkey({ + apiKey: 'PORTKEY_API_KEY', + provider: '@workers-ai' +}); -# Display the image -display(Image(url=image.data[0].url)) -``` - - +const image = await portkey.images.generate({ + model: "@cf/stabilityai/stable-diffusion-xl-base-1.0", + prompt: "A beautiful sunset over mountains" +}); -```sh -curl "https://api.portkey.ai/v1/images/generations" \ - -H "Content-Type: application/json" \ - -H "x-portkey-api-key: $PORTKEY_API_KEY" \ - -H "x-portkey-provider: $WORKERS_AI_PROVIDER" \ - -d '{ - "model": "image_model_name", - "prompt": "Lucy in the sky with diamonds" - }' +console.log(image.data[0].url); ``` - - + +--- +## Supported Models -## Managing Workers AI Prompts +Cloudflare Workers AI provides serverless AI inference with various models: -You can manage all prompts to Workers AI in the [Prompt Library](/product/prompt-library). All the current models of Workers AI are supported and you can easily start testing different prompts. +**Chat Models:** +- @cf/meta/llama-3.2-3b-instruct +- @cf/meta/llama-3.1-8b-instruct +- @cf/mistral/mistral-7b-instruct-v0.1 -Once you're ready with your prompt, you can use the `portkey.prompts.completions.create` interface to use the prompt in your application. +**Image Generation Models:** +- @cf/stabilityai/stable-diffusion-xl-base-1.0 -The complete list of features supported in the SDK are available on the link below. - +Check [Cloudflare Workers AI documentation](https://developers.cloudflare.com/workers-ai/) for the complete model list. -You'll find more information in the relevant sections: +--- -1. [Add metadata to your requests](/product/observability/metadata) -2. [Add gateway configs to your Workers requests](/product/ai-gateway/configs) -3. [Tracing Workers AI requests](/product/observability/traces) -4. [Setup a fallback from OpenAI to Workers AI APIs](/product/ai-gateway/fallbacks) +## Next Steps + + + + Add fallbacks, load balancing, and more + + + Monitor and trace your Workers AI requests + + + Manage and version your prompts + + + Cache responses at the edge + + + +For complete SDK documentation: + + + Complete Portkey SDK documentation + diff --git a/integrations/llms/x-ai.mdx b/integrations/llms/x-ai.mdx index 15f00cd0..4256016e 100644 --- a/integrations/llms/x-ai.mdx +++ b/integrations/llms/x-ai.mdx @@ -1,283 +1,132 @@ --- title: 'xAI (Grok)' -description: Portkey supports xAI's chat completions, completions, and embeddings APIs. +description: Use xAI's Grok models through Portkey for chat completions, function calling, and vision capabilities. --- - -[Supported Endpoints](/api-reference/inference-api/supported-providers) - +## Quick Start -## Integrate -Just paste your xAI API Key to [Portkey](https://app.portkey.ai/virtual-keys) to create your Virtual Key. - - -## Sample Request -Portkey is a drop-in replacement for xAI. You can make request using the official Portkey SDK. - -Popular libraries & agent frameworks like LangChain, CrewAI, AutoGen, etc. are [also supported](#popular-libraries). - +Get started with xAI in under 2 minutes: -```ts NodeJS -import Portkey from 'portkey-ai'; +```python Python icon="python" +from portkey_ai import Portkey -const client = new Portkey({ - apiKey: 'PORTKEY_API_KEY', - provider:'@PROVIDER' -}); +# 1. Install: pip install portkey-ai +# 2. Add @x-ai provider in model catalog +# 3. Use it: -async function main() { - const response = await client.chat.completions.create({ - messages: [{ role: "user", content: "Bob the builder.." }], - model: "grok-beta", - }); +portkey = Portkey(api_key="PORTKEY_API_KEY") - console.log(response.choices[0].message.content); -} +response = portkey.chat.completions.create( + model="@x-ai/grok-beta", + messages=[{"role": "user", "content": "Hello!"}] +) -main(); +print(response.choices[0].message.content) ``` -```py Python -from portkey_ai import Portkey -client = Portkey( - api_key = "PORTKEY_API_KEY", - virtual_key = "PROVIDER" -) +```js Javascript icon="square-js" +import Portkey from 'portkey-ai' -response = client.chat.completions.create( - model="grok-beta", - messages=[{"role": "user", "content": "Hello!"}] -) +// 1. Install: npm install portkey-ai +// 2. Add @x-ai provider in model catalog +// 3. Use it: -print(response.choices[0].message) -``` -```sh cURL -curl https://api.portkey.ai/v1/chat/completions \ - -H "Content-Type: application/json" \ - -H "x-portkey-api-key: $PORTKEY_API_KEY" \ - -H "x-portkey-provider: $PORTKEY_PROVIDER" \ - -d '{ - "model": "grok-beta", - "messages": [ - { "role": "user", "content": "Hello!" } - ] - }' +const portkey = new Portkey({ + apiKey: "PORTKEY_API_KEY" +}) + +const response = await portkey.chat.completions.create({ + model: "@x-ai/grok-beta", + messages: [{ role: "user", content: "Hello!" }] +}) + +console.log(response.choices[0].message.content) ``` -```py OpenAI Python SDK + +```python OpenAI Py icon="python" from openai import OpenAI -from portkey_ai import createHeaders, PORTKEY_GATEWAY_URL +from portkey_ai import PORTKEY_GATEWAY_URL + +# 1. Install: pip install openai portkey-ai +# 2. Add @x-ai provider in model catalog +# 3. Use it: client = OpenAI( - api_key="xx", - base_url=PORTKEY_GATEWAY_URL, - default_headers=createHeaders( - api_key="PORTKEY_API_KEY", - provider="@OPENAI_PROVIDER" - ) + api_key="PORTKEY_API_KEY", # Portkey API key + base_url=PORTKEY_GATEWAY_URL ) -completion = client.chat.completions.create( - model="grok-beta", - messages=[ - {"role": "user", "content": "Hello!"} - ] +response = client.chat.completions.create( + model="@x-ai/grok-beta", + messages=[{"role": "user", "content": "Hello!"}] ) -print(completion.choices[0].message) +print(response.choices[0].message.content) ``` -```ts OpenAI NodeJS SDK -import OpenAI from 'openai'; -import { PORTKEY_GATEWAY_URL, createHeaders } from 'portkey-ai' - -const openai = new OpenAI({ - apiKey: 'xx', - baseURL: PORTKEY_GATEWAY_URL, - defaultHeaders: createHeaders({ - apiKey: "PORTKEY_API_KEY", - provider:"@OPENAI_PROVIDER" - }) -}); -async function main() { - const completion = await openai.chat.completions.create({ - messages: [{ role: 'user', content: 'Say this is a test' }], - model: 'grok-beta', - }); +```js OpenAI JS icon="square-js" +import OpenAI from "openai" +import { PORTKEY_GATEWAY_URL } from "portkey-ai" - console.log(chatCompletion.choices); -} +// 1. Install: npm install openai portkey-ai +// 2. Add @x-ai provider in model catalog +// 3. Use it: -main(); -``` - +const client = new OpenAI({ + apiKey: "PORTKEY_API_KEY", // Portkey API key + baseURL: PORTKEY_GATEWAY_URL +}) -## Local Setup -If you do not want to use Portkey's hosted API, you can also run Portkey locally: - - -Portkey runs on our popular [open source Gateway](https://git.new/portkey). You can spin it up locally to make requests without sending them to the Portkey API. - -```sh Install the Gateway -npx @portkey-ai/gateway -``` +const response = await client.chat.completions.create({ + model: "@x-ai/grok-beta", + messages: [{ role: "user", content: "Hello!" }] +}) -```sh Docker Image -npx @portkey-ai/gateway +console.log(response.choices[0].message.content) ``` - -| Your Gateway is running on http://localhost:8080/v1 🚀 | | -| - | - | -Then, just change the `baseURL` to the local Gateway URL, and make requests: - -```ts NodeJS -import Portkey from 'portkey-ai'; - -const client = new Portkey({ - baseUrl: 'http://localhost:8080/v1', - apiKey: 'PORTKEY_API_KEY', - provider:'@PROVIDER' -}); - -async function main() { - const response = await client.chat.completions.create({ - messages: [{ role: "user", content: "Bob the builder.." }], - model: "grok-beta", - }); +```sh cURL icon="square-terminal" +# 1. Add @x-ai provider in model catalog +# 2. Use it: - console.log(response.choices[0].message.content); -} - -main(); -``` -```py Python -from portkey_ai import Portkey - -client = Portkey( - base_url = 'http://localhost:8080/v1', - api_key = "PORTKEY_API_KEY", - virtual_key = "PROVIDER" -) - -response = client.chat.completions.create( - model="grok-beta", - messages=[ - {"role": "user", "content": "Hello!"} - ] -) - -print(response.choices[0].message) -``` -```sh cURL -curl http://localhost:8080/v1/chat/completions \ +curl https://api.portkey.ai/v1/chat/completions \ -H "Content-Type: application/json" \ -H "x-portkey-api-key: $PORTKEY_API_KEY" \ - -H "x-portkey-provider: $PORTKEY_PROVIDER" \ -d '{ - "model": "grok-beta", - "messages": [ - { "role": "user", "content": "Hello!" } - ] + "model": "@x-ai/grok-beta", + "messages": [{"role": "user", "content": "Hello!"}] }' ``` -```py OpenAI Python SDK -from openai import OpenAI -from portkey_ai import createHeaders, PORTKEY_GATEWAY_URL - -client = OpenAI( - api_key="xx", - base_url="https://localhost:8080/v1", - default_headers=createHeaders( - api_key="PORTKEY_API_KEY", - provider="@OPENAI_PROVIDER" - ) -) - -completion = client.chat.completions.create( - model="grok-beta", - messages=[ - {"role": "user", "content": "Hello!"} - ] -) - -print(completion.choices[0].message) -``` -```ts OpenAI NodeJS SDK -import OpenAI from 'openai'; -import { PORTKEY_GATEWAY_URL, createHeaders } from 'portkey-ai' - -const openai = new OpenAI({ - apiKey: 'xx', - baseURL: 'https://localhost:8080/v1', - defaultHeaders: createHeaders({ - apiKey: "PORTKEY_API_KEY", - provider:"@OPENAI_PROVIDER" - }) -}); -async function main() { - const completion = await openai.chat.completions.create({ - messages: [{ role: 'user', content: 'Say this is a test' }], - model: 'grok-beta', - }); + - console.log(chatCompletion.choices); -} +## Add Provider in Model Catalog -main(); -``` - +Before making requests, add xAI to your Model Catalog: - - - Portkey's data & control planes can be fully deployed on-prem with the Enterprise license. +1. Go to [**Model Catalog → Add Provider**](https://app.portkey.ai/model-catalog/providers) +2. Select **xAI** +3. Enter your [xAI API key](https://accounts.x.ai/) +4. Name your provider (e.g., `x-ai`) - [More details here](/self-hosting/hybrid-deployments/architecture) - - + + See all setup options and detailed configuration instructions + --- -## Integration Overview +## xAI Capabilities -### xAI Endpoints & Capabilities +### Tool Calling (Function Calling) -Portkey works with *all* of xAI's endpoints and supports all xAI capabilities like function calling and image understanding. Find examples for each below: - - -```javascript Node.js -let tools = [{ - type: "function", - function: { - name: "getWeather", - description: "Get the current weather", - parameters: { - type: "object", - properties: { - location: { type: "string", description: "City and state" }, - unit: { type: "string", enum: ["celsius", "fahrenheit"] } - }, - required: ["location"] - } - } -}]; -let response = await portkey.chat.completions.create({ - model: "grok-beta", - messages: [ - { role: "system", content: "You are a helpful assistant." }, - { role: "user", content: "What's the weather like in Delhi - respond in JSON" } - ], - tools, - tool_choice: "auto", -}); +```python Python +from portkey_ai import Portkey -console.log(response.choices[0].finish_reason); -``` +portkey = Portkey(api_key="PORTKEY_API_KEY") -```python Python tools = [{ "type": "function", "function": { @@ -295,10 +144,10 @@ tools = [{ }] response = portkey.chat.completions.create( - model="grok-beta", + model="@x-ai/grok-beta", messages=[ {"role": "system", "content": "You are a helpful assistant."}, - {"role": "user", "content": "What's the weather like in Delhi - respond in JSON"} + {"role": "user", "content": "What's the weather like in Delhi?"} ], tools=tools, tool_choice="auto" @@ -307,15 +156,52 @@ response = portkey.chat.completions.create( print(response.choices[0].finish_reason) ``` -```curl REST -curl -X POST "https://api.portkey.ai/v1/chat/completions" \ +```javascript Node.js +import Portkey from 'portkey-ai'; + +const portkey = new Portkey({ + apiKey: 'PORTKEY_API_KEY' +}); + +const tools = [{ + type: "function", + function: { + name: "getWeather", + description: "Get the current weather", + parameters: { + type: "object", + properties: { + location: { type: "string", description: "City and state" }, + unit: { type: "string", enum: ["celsius", "fahrenheit"] } + }, + required: ["location"] + } + } +}]; + +const response = await portkey.chat.completions.create({ + model: "@x-ai/grok-beta", + messages: [ + { role: "system", content: "You are a helpful assistant." }, + { role: "user", content: "What's the weather like in Delhi?" } + ], + tools, + tool_choice: "auto" +}); + +console.log(response.choices[0].finish_reason); +``` + +```bash cURL +curl https://api.portkey.ai/v1/chat/completions \ -H "Content-Type: application/json" \ - -H "Authorization: Bearer YOUR_PORTKEY_API_KEY" \ + -H "x-portkey-api-key: $PORTKEY_API_KEY" \ + -H "x-portkey-provider: @x-ai" \ -d '{ "model": "grok-beta", "messages": [ {"role": "system", "content": "You are a helpful assistant."}, - {"role": "user", "content": "What'\''s the weather like in Delhi - respond in JSON"} + {"role": "user", "content": "What'\''s the weather like in Delhi?"} ], "tools": [{ "type": "function", @@ -335,64 +221,71 @@ curl -X POST "https://api.portkey.ai/v1/chat/completions" \ "tool_choice": "auto" }' ``` + - - +### Vision -Process images alongside text using xAI's vision capabilities: +Process images with Grok's vision capabilities: + ```python Python +from portkey_ai import Portkey + +portkey = Portkey(api_key="PORTKEY_API_KEY") + response = portkey.chat.completions.create( - model="grok-beta", - messages=[ - { + model="@x-ai/grok-beta", + messages=[{ "role": "user", "content": [ {"type": "text", "text": "What's in this image?"}, { "type": "image_url", - "image_url": "https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg", - }, - ], + "image_url": "https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg" } - ], - max_tokens=300, + ] + }], + max_tokens=300 ) -print(response) +print(response.choices[0].message.content) ``` ```javascript Node.js +import Portkey from 'portkey-ai'; + +const portkey = new Portkey({ + apiKey: 'PORTKEY_API_KEY' +}); + const response = await portkey.chat.completions.create({ - model: "grok-beta", - messages: [ - { + model: "@x-ai/grok-beta", + messages: [{ role: "user", content: [ { type: "text", text: "What's in this image?" }, { type: "image_url", - image_url: "https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg", - }, - ], - }, - ], - max_tokens: 300, + image_url: "https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg" + } + ] + }], + max_tokens: 300 }); -console.log(response); +console.log(response.choices[0].message.content); ``` -```curl REST -curl -X POST "https://api.portkey.ai/v1/chat/completions" \ +```bash cURL +curl https://api.portkey.ai/v1/chat/completions \ -H "Content-Type: application/json" \ - -H "Authorization: Bearer YOUR_PORTKEY_API_KEY" \ + -H "x-portkey-api-key: $PORTKEY_API_KEY" \ + -H "x-portkey-provider: @x-ai" \ -d '{ "model": "grok-beta", - "messages": [ - { + "messages": [{ "role": "user", "content": [ {"type": "text", "text": "What'\''s in this image?"}, @@ -401,155 +294,63 @@ curl -X POST "https://api.portkey.ai/v1/chat/completions" \ "image_url": "https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg" } ] - } - ], + }], "max_tokens": 300 }' ``` + - +--- - +## Supported Models -Generate embeddings for text using xAI's embedding models: +xAI offers powerful models through Grok: -`Coming Soon!` - - +| Model | Context Length | Description | +|-------|---------------|-------------| +| grok-beta | 131,072 tokens | Latest Grok model with enhanced capabilities | +| grok-2-1212 | 32,768 tokens | Previous generation Grok model | +Check [xAI's documentation](https://docs.x.ai/docs) for the latest model information. -### Portkey Features +--- - - - -Here's a simplified version of how to use Portkey's Gateway Configuration: - - - - You can create a Gateway configuration using the Portkey Config Dashboard or by writing a JSON configuration in your code. In this example, requests are routed based on the user's subscription plan (paid or free). - - ```json - config = { - "strategy": { - "mode": "conditional", - "conditions": [ - { - "query": { "metadata.user_plan": { "$eq": "paid" } }, - "then": "grok-beta" - }, - { - "query": { "metadata.user_plan": { "$eq": "free" } }, - "then": "grok-2-1212" - } - ], - "default": "grok-beta" - }, - "targets": [ - { - "name": "grok-beta", - "provider":"@xx" - }, - { - "name": "grok-2-1212", - "provider":"@yy" - } - ] - } - ``` - - - - When a user makes a request, it will pass through Portkey's AI Gateway. Based on the configuration, the Gateway routes the request according to the user's metadata. - Conditional Routing Diagram - - - - Pass the Gateway configuration to your Portkey client. You can either use the config object or the Config ID from Portkey's hosted version. - - - ```python Python - from portkey_ai import Portkey - - portkey = Portkey( - api_key="PORTKEY_API_KEY", - provider="@PROVIDER", - config=portkey_config - ) - ``` - - ```javascript Node.js - import Portkey from 'portkey-ai' - - const portkey = new Portkey({ - apiKey: "PORTKEY_API_KEY", - provider:"@PROVIDER", - config: portkeyConfig - }) - ``` - - - - -That's it! Portkey seamlessly allows you to make your AI app more robust using built-in gateway features. Learn more about advanced gateway features: +## Next Steps - - Distribute requests across multiple targets based on defined weights. + + Add fallbacks, load balancing, and more - - Automatically switch to backup targets if the primary target fails. + + Monitor and trace your xAI requests - - Route requests to different targets based on specified conditions. + + Enforce input/output checks on requests - - Enable caching of responses to improve performance and reduce costs. + + Manage and version your prompts - - - +For complete SDK documentation: -Portkey's AI gateway enables you to enforce input/output checks on requests by applying custom hooks before and after processing. Protect your user's/company's data by using PII guardrails and many more available on Portkey Guardrails: - -```json -{ - "provider:"@xai-xxx", - "before_request_hooks": [{ - "id": "input-guardrail-id-xx" - }], - "after_request_hooks": [{ - "id": "output-guardrail-id-xx" - }] -} -``` - - - Explore Portkey's guardrail features to enhance the security and reliability of your AI applications. + + Complete Portkey SDK documentation - - - - - --- -## Appendix - -### FAQs +## FAQs - -You can sign up to xAI [here](https://accounts.x.ai/) and grab your API key. + +Sign up at [xAI](https://accounts.x.ai/) and generate your API key from the console. - -xAI typically gives some amount of free credits without you having to add your credit card. Reach out to their support team if you'd like additional free credits. + +xAI typically provides free credits to start. Contact their support team for additional credits. - -You can find your current rate limits imposed by xAI on the console. Use Portkey's loadbalancer to tackle rate limiting by xAI. + +Check your current rate limits in the xAI console. Use Portkey's [load balancing](/product/ai-gateway/load-balancing) to distribute requests across multiple providers. diff --git a/integrations/llms/zhipu.mdx b/integrations/llms/zhipu.mdx index fc1f8db6..f09590a6 100644 --- a/integrations/llms/zhipu.mdx +++ b/integrations/llms/zhipu.mdx @@ -1,150 +1,141 @@ --- title: "ZhipuAI / ChatGLM / BigModel" +description: Use ZhipuAI's GLM models through Portkey for advanced Chinese and multilingual AI. --- -[ZhipuAI](https://open.bigmodel.cn/) has developed the GLM series of open source LLMs that are some of the world's best performing and capable models today. Portkey provides a robust and secure gateway to seamlessly integrate these LLMs into your applications in the familiar OpenAI spec with just 2 LOC change! +## Quick Start -With Portkey, you can leverage powerful features like fast AI gateway, caching, observability, prompt management, and more, while securely managing your LLM API keys through a virtual key system. - -Provider Slug. `zhipu` - -## Portkey SDK Integration with ZhipuAI +Get started with ZhipuAI in under 2 minutes: -### 1\. Install the Portkey SDK + -Install the Portkey SDK in your project using npm or pip: +```python Python icon="python" +from portkey_ai import Portkey - - - ```sh - npm install --save portkey-ai - ``` - - -```sh -pip install portkey-ai -``` +# 1. Install: pip install portkey-ai +# 2. Add @zhipu provider in model catalog +# 3. Use it: - +portkey = Portkey(api_key="PORTKEY_API_KEY") - +response = portkey.chat.completions.create( + model="@zhipu/glm-4", + messages=[{"role": "user", "content": "Hello!"}] +) +print(response.choices[0].message.content) +``` -### 2\. Initialize Portkey with the Virtual Key +```js Javascript icon="square-js" +import Portkey from 'portkey-ai' -To use ZhipuAI / ChatGLM / BigModel with Portkey, [get your API key from here](https://bigmodel.cn/usercenter/apikeys), then add it to Portkey to create the virtual key. - - - ```js - import Portkey from 'portkey-ai' +// 1. Install: npm install portkey-ai +// 2. Add @zhipu provider in model catalog +// 3. Use it: - const portkey = new Portkey({ - apiKey: "PORTKEY_API_KEY", // defaults to process.env["PORTKEY_API_KEY"] - provider:"@PROVIDER" // Your ZhipuAI Virtual Key - }) - ``` - - -```python -from portkey_ai import Portkey +const portkey = new Portkey({ + apiKey: "PORTKEY_API_KEY" +}) -portkey = Portkey( - api_key="PORTKEY_API_KEY", # Replace with your Portkey API key - provider="@PROVIDER" # Replace with your virtual key for ZhipuAI -) -``` - - - -```js -import OpenAI from "openai"; -import { PORTKEY_GATEWAY_URL, createHeaders } from "portkey-ai"; - -const portkey = new OpenAI({ - baseURL: PORTKEY_GATEWAY_URL, - defaultHeaders: createHeaders({ - apiKey: "PORTKEY_API_KEY", - provider:"@ZHIPUAI_PROVIDER", - }), -}); +const response = await portkey.chat.completions.create({ + model: "@zhipu/glm-4", + messages: [{ role: "user", content: "Hello!" }] +}) + +console.log(response.choices[0].message.content) ``` - - -```python +```python OpenAI Py icon="python" from openai import OpenAI -from portkey_ai import PORTKEY_GATEWAY_URL, createHeaders - -portkey = OpenAI( - base_url=PORTKEY_GATEWAY_URL, - default_headers=createHeaders( - api_key="PORTKEY_API_KEY", - provider="@ZHIPUAI_PROVIDER" - ) +from portkey_ai import PORTKEY_GATEWAY_URL + +# 1. Install: pip install openai portkey-ai +# 2. Add @zhipu provider in model catalog +# 3. Use it: + +client = OpenAI( + api_key="PORTKEY_API_KEY", # Portkey API key + base_url=PORTKEY_GATEWAY_URL ) -``` - - +response = client.chat.completions.create( + model="@zhipu/glm-4", + messages=[{"role": "user", "content": "Hello!"}] +) +print(response.choices[0].message.content) +``` -### 3\. Invoke Chat Completions - - - ```js - const chatCompletion = await portkey.chat.completions.create({ - messages: [{ role: 'user', content: 'Who are you?' }], - model: 'glm-4' - }); +```js OpenAI JS icon="square-js" +import OpenAI from "openai" +import { PORTKEY_GATEWAY_URL } from "portkey-ai" - console.log(chatCompletion.choices); - ``` +// 1. Install: npm install openai portkey-ai +// 2. Add @zhipu provider in model catalog +// 3. Use it: - > I am an AI assistant named ZhiPuQingYan（智谱清言）, you can call me Xiaozhi🤖 +const client = new OpenAI({ + apiKey: "PORTKEY_API_KEY", // Portkey API key + baseURL: PORTKEY_GATEWAY_URL +}) - - -```python -completion = portkey.chat.completions.create( - messages= [{ "role": 'user', "content": 'Say this is a test' }], - model= 'glm-4' -) +const response = await client.chat.completions.create({ + model: "@zhipu/glm-4", + messages: [{ role: "user", content: "Hello!" }] +}) -print(completion) +console.log(response.choices[0].message.content) ``` -> I am an AI assistant named ZhiPuQingYan（智谱清言）, you can call me Xiaozhi🤖 +```sh cURL icon="square-terminal" +# 1. Add @zhipu provider in model catalog +# 2. Use it: + +curl https://api.portkey.ai/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "x-portkey-api-key: $PORTKEY_API_KEY" \ + -d '{ + "model": "@zhipu/glm-4", + "messages": [{"role": "user", "content": "Hello!"}] + }' +``` - - - ```sh - curl https://api.portkey.ai/v1/chat/completions \ - -H "Content-Type: application/json" \ - -H "x-portkey-api-key: $PORTKEY_API_KEY" \ - -H "x-portkey-provider: $ZHIPUAI_PROVIDER" \ - -d '{ - "messages": [{"role": "user","content": "Hello!"}], - "model": "glm-4", - }' - ``` + - > I am an AI assistant named ZhiPuQingYan（智谱清言）, you can call me Xiaozhi🤖 +## Add Provider in Model Catalog - +Before making requests, add ZhipuAI to your Model Catalog: - +1. Go to [**Model Catalog → Add Provider**](https://app.portkey.ai/model-catalog/providers) +2. Select **ZhipuAI** +3. Enter your [ZhipuAI API key](https://bigmodel.cn/usercenter/apikeys) +4. Name your provider (e.g., `zhipu`) + + + See all setup options and detailed configuration instructions + --- ## Next Steps -The complete list of features supported in the SDK are available on the link below. - + + + Add fallbacks, load balancing, and more + + + Monitor and trace your ZhipuAI requests + + + Manage and version your prompts + + + Add custom metadata to requests + + + +For complete SDK documentation: + + + Complete Portkey SDK documentation - -You'll find more information in the relevant sections: - -1. [Add metadata to your requests](/product/observability/metadata) -2. [Add gateway configs to your ZhipuAI requests](/product/ai-gateway/configs) -3. [Tracing ZhipuAI requests](/product/observability/traces) -4. [Setup a fallback from OpenAI to ZhipuAI](/product/ai-gateway/fallbacks)