Skip to content

Commit b7c7746

Browse files
narengoginarengogi
committed
docs for multimodal embeddings
1 parent 9a3940e commit b7c7746

File tree

3 files changed

+374
-2
lines changed

3 files changed

+374
-2
lines changed

‎docs.json

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -239,7 +239,8 @@
239239
"integrations/llms/vertex-ai/files",
240240
"integrations/llms/vertex-ai/batches",
241241
"integrations/llms/vertex-ai/fine-tuning",
242-
"integrations/llms/vertex-ai/controlled-generations"
242+
"integrations/llms/vertex-ai/controlled-generations",
243+
"integrations/llms/vertex-ai/embeddings"
243244
]
244245
},
245246
{

‎integrations/llms/vertex-ai.mdx

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ title: "Google Vertex AI"
44

55
Portkey provides a robust and secure gateway to facilitate the integration of various Large Language Models (LLMs), and embedding models into your apps, including [Google Vertex AI](https://cloud.google.com/vertex-ai?hl=en).
66

7-
With Portkey, you can take advantage of features like fast AI gateway access, observability, prompt management, and more, all while ensuring the secure management of your Vertex auth through a [virtual key](/product/ai-gateway/virtual-keys/) system.s
7+
With Portkey, you can take advantage of features like fast AI gateway access, observability, prompt management, and more, all while ensuring the secure management of your Vertex auth through a [virtual key](/product/ai-gateway/virtual-keys/) system
88
<Note>
99
Provider Slug. `vertex-ai`
1010
</Note>

‎integrations/llms/vertex-ai/embeddings.mdx

Lines changed: 371 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,371 @@
1+
---
2+
title: "Embeddings"
3+
description: "Get embeddings from Vertex AI"
4+
---
5+
6+
Vertex AI offers wide ranging support for embedding text, images and videos.
7+
Portkey provides a standardized interface for embedding multiple modalities.
8+
9+
## Embedding Text
10+
<CodeGroup>
11+
```python Python
12+
from portkey_ai import Portkey
13+
14+
client = Portkey(
15+
api_key="YOUR_PORTKEY_API_KEY", # defaults to os.environ.get("PORTKEY_API_KEY")
16+
virtual_key="VIRTUAL_KEY",
17+
)
18+
19+
embeddings = client.embeddings.create(
20+
model="textembedding-gecko@003",
21+
input_type="classification",
22+
input="The food was delicious and the waiter...",
23+
# input=["text to embed", "more text to embed"], # if you would like to embed multiple texts
24+
)
25+
```
26+
27+
```javascript NodeJS
28+
import { Portkey } from 'portkey-ai';
29+
30+
const portkey = new Portkey({
31+
apiKey: "YOUR_API_KEY",
32+
virtualKey: "YOUR_VIRTUAL_KEY"
33+
});
34+
35+
const embedding = await portkey.embeddings.create({
36+
input: 'Name the tallest buildings in Hawaii',
37+
// input: ['text to embed', 'more text to embed'], // if you would like to embed multiple texts
38+
model: 'textembedding-gecko@003'
39+
});
40+
41+
console.log(embedding);
42+
```
43+
44+
```sh cURL
45+
curl --location 'https://api.portkey.ai/v1/embeddings' \
46+
--header 'Content-Type: application/json' \
47+
--header 'x-portkey-api-key: PORTKEY_API_KEY' \
48+
--header 'x-portkey-virtual-key: PORTKEY_VIRTUAL_KEY' \
49+
--data-raw '{
50+
"model": "textembedding-gecko@003",
51+
"input": [
52+
"A HTTP 246 code is used to signify an AI response containing hallucinations or other inaccuracies",
53+
"246: Partially incorrect response"
54+
],
55+
# "input": "Name the tallest buildings in Hawaii",
56+
"input_type": "classification"
57+
}'
58+
```
59+
60+
```python OpenAI Python
61+
from openai import OpenAI
62+
from portkey_ai import PORTKEY_GATEWAY_URL, createHeaders
63+
64+
portkey_client = OpenAI(
65+
api_key='NOT_REQUIRED',
66+
base_url=PORTKEY_GATEWAY_URL,
67+
default_headers=createHeaders(
68+
provider="openai",
69+
api_key="PORTKEY_API_KEY"
70+
)
71+
)
72+
73+
embeddings = portkey_client.embeddings.create(
74+
model="textembedding-gecko@003",
75+
input_type="classification",
76+
input="The food was delicious and the waiter...",
77+
# input=["text to embed", "more text to embed"], # if you would like to embed multiple texts
78+
)
79+
```
80+
81+
```js OpenAI NodeJS
82+
import OpenAI from 'openai'; // We're using the v4 SDK
83+
import { PORTKEY_GATEWAY_URL, createHeaders } from 'portkey-ai'
84+
85+
const portkeyClient = new OpenAI({
86+
apiKey: 'NOT_REQUIRED', // defaults to process.env["OPENAI_API_KEY"],
87+
baseURL: PORTKEY_GATEWAY_URL,
88+
defaultHeaders: createHeaders({
89+
provider: "vertex-ai",
90+
apiKey: "PORTKEY_API_KEY", // defaults to process.env["PORTKEY_API_KEY"]
91+
virtualKey: "PORTKEY_VIRTUAL_KEY"
92+
})
93+
});
94+
95+
const embedding = await portkeyClient.embeddings.create({
96+
input: 'Name the tallest buildings in Hawaii',
97+
// input: ['text to embed', 'more text to embed'], // if you would like to embed multiple texts
98+
model: 'textembedding-gecko@003'
99+
});
100+
101+
console.log(embedding);
102+
```
103+
104+
</CodeGroup>
105+
106+
## Embeddings Images
107+
108+
<CodeGroup>
109+
```python Python
110+
from portkey_ai import Portkey
111+
112+
client = Portkey(
113+
api_key="YOUR_PORTKEY_API_KEY", # defaults to os.environ.get("PORTKEY_API_KEY")
114+
virtual_key="VIRTUAL_KEY",
115+
)
116+
117+
embeddings = client.embeddings.create(
118+
model="multimodalembedding@001",
119+
input=[
120+
{
121+
"text": "this is the caption of the image",
122+
"image": {
123+
"base64": "UklGRkacAABXRUJQVlA4IDqcAACQggKdASqpAn8B.....",
124+
# "url": "gcs://..." # if you want to use a url
125+
}
126+
}
127+
]
128+
)
129+
```
130+
131+
```javascript NodeJS
132+
import { Portkey } from 'portkey-ai';
133+
134+
const portkey = new Portkey({
135+
apiKey: "YOUR_API_KEY",
136+
virtualKey: "YOUR_VIRTUAL_KEY"
137+
});
138+
139+
const embedding = await portkey.embeddings.create({
140+
input: [
141+
{
142+
"text": "this is the caption of the image",
143+
"image": {
144+
"base64": "UklGRkacAABXRUJQVlA4IDqcAACQggKdASqpAn8B.....",
145+
// "url": "gcs://..." // if you want to use a url
146+
}
147+
}
148+
],
149+
model: 'multimodalembedding@001'
150+
});
151+
152+
console.log(embedding);
153+
```
154+
155+
```sh cURL
156+
curl --location 'https://api.portkey.ai/v1/embeddings' \
157+
--header 'Content-Type: application/json' \
158+
--header 'x-portkey-api-key: PORTKEY_API_KEY' \
159+
--header 'x-portkey-virtual-key: PORTKEY_VIRTUAL_KEY' \
160+
--data-raw '{
161+
"model": "multimodalembedding@001",
162+
"input": [
163+
{
164+
"text": "this is the caption of the image",
165+
"image": {
166+
"base64": "UklGRkacAABXRUJQVlA4IDqcAACQggKdASqpAn8B....."
167+
# "url": "gcs://..." # if you want to use a url
168+
}
169+
}
170+
]
171+
}'
172+
```
173+
174+
```python OpenAI Python
175+
from openai import OpenAI
176+
from portkey_ai import PORTKEY_GATEWAY_URL, createHeaders
177+
178+
portkey_client = OpenAI(
179+
api_key='NOT_REQUIRED',
180+
base_url=PORTKEY_GATEWAY_URL,
181+
default_headers=createHeaders(
182+
provider="openai",
183+
api_key="PORTKEY_API_KEY"
184+
)
185+
)
186+
187+
embeddings = portkey_client.embeddings.create(
188+
model="multimodalembedding@001",
189+
input=[
190+
{
191+
"text": "this is the caption of the image",
192+
"image": {
193+
"base64": "UklGRkacAABXRUJQVlA4IDqcAACQggKdASqpAn8B.....",
194+
# "url": "gcs://..." # if you want to use a url
195+
}
196+
}
197+
]
198+
)
199+
```
200+
201+
```js OpenAI NodeJS
202+
import OpenAI from 'openai'; // We're using the v4 SDK
203+
import { PORTKEY_GATEWAY_URL, createHeaders } from 'portkey-ai'
204+
205+
const portkeyClient = new OpenAI({
206+
apiKey: 'NOT_REQUIRED', // defaults to process.env["OPENAI_API_KEY"],
207+
baseURL: PORTKEY_GATEWAY_URL,
208+
defaultHeaders: createHeaders({
209+
apiKey: "PORTKEY_API_KEY", // defaults to process.env["PORTKEY_API_KEY"]
210+
virtualKey: "PORTKEY_VIRTUAL_KEY"
211+
})
212+
});
213+
214+
const embedding = await portkeyClient.embeddings.create({
215+
input: [
216+
{
217+
"text": "this is the caption of the image",
218+
"image": {
219+
"base64": "UklGRkacAABXRUJQVlA4IDqcAACQggKdASqpAn8B.....",
220+
// "url": "gcs://..." // if you want to use a url
221+
}
222+
}
223+
],
224+
model: 'multimodalembedding@001'
225+
});
226+
227+
console.log(embedding);
228+
```
229+
230+
</CodeGroup>
231+
232+
233+
## Embeddings Videos
234+
235+
<CodeGroup>
236+
```python Python
237+
from portkey_ai import Portkey
238+
239+
client = Portkey(
240+
api_key="YOUR_PORTKEY_API_KEY", # defaults to os.environ.get("PORTKEY_API_KEY")
241+
virtual_key="VIRTUAL_KEY",
242+
)
243+
244+
embeddings = client.embeddings.create(
245+
model="multimodalembedding@001",
246+
input=[
247+
{
248+
"text": "this is the caption of the video",
249+
"video": {
250+
"base64": "UklGRkacAABXRUJQVlA4IDqcAACQggKdASqpAn8B.....",
251+
"start_offset": 0,
252+
"end_offset": 10,
253+
"interval": 5,
254+
# "url": "gcs://..." # if you want to use a url
255+
}
256+
}
257+
]
258+
)
259+
```
260+
```javascript NodeJS
261+
import { Portkey } from 'portkey-ai';
262+
263+
const portkey = new Portkey({
264+
apiKey: "YOUR_API_KEY",
265+
virtualKey: "YOUR_VIRTUAL_KEY"
266+
});
267+
268+
const embedding = await portkey.embeddings.create({
269+
input: [
270+
{
271+
"text": "this is the caption of the video",
272+
"video": {
273+
"base64": "UklGRkacAABXRUJQVlA4IDqcAACQggKdASqpAn8B.....",
274+
"start_offset": 0,
275+
"end_offset": 10,
276+
"interval": 5,
277+
// "url": "gcs://..." // if you want to use a url
278+
}
279+
}
280+
],
281+
model: 'multimodalembedding@001'
282+
});
283+
284+
console.log(embedding);
285+
```
286+
287+
```sh cURL
288+
curl --location 'https://api.portkey.ai/v1/embeddings' \
289+
--header 'Content-Type: application/json' \
290+
--header 'x-portkey-api-key: PORTKEY_API_KEY' \
291+
--header 'x-portkey-virtual-key: PORTKEY_VIRTUAL_KEY' \
292+
--data-raw '{
293+
"model": "multimodalembedding@001",
294+
"input": [
295+
{
296+
"text": "this is the caption of the video",
297+
"video": {
298+
"base64": "UklGRkacAABXRUJQVlA4IDqcAACQggKdASqpAn8B.....",
299+
"start_offset": 0,
300+
"end_offset": 10,
301+
"interval": 5
302+
# "url": "gcs://..." # if you want to use a url
303+
}
304+
}
305+
]
306+
}'
307+
```
308+
309+
```python OpenAI Python
310+
from openai import OpenAI
311+
from portkey_ai import PORTKEY_GATEWAY_URL, createHeaders
312+
313+
portkey_client = OpenAI(
314+
api_key='NOT_REQUIRED',
315+
base_url=PORTKEY_GATEWAY_URL,
316+
default_headers=createHeaders(
317+
provider="openai",
318+
api_key="PORTKEY_API_KEY"
319+
)
320+
)
321+
322+
embeddings = portkey_client.embeddings.create(
323+
model="multimodalembedding@001",
324+
input=[
325+
{
326+
"text": "this is the caption of the video",
327+
"video": {
328+
"base64": "UklGRkacAABXRUJQVlA4IDqcAACQggKdASqpAn8B.....",
329+
"start_offset": 0,
330+
"end_offset": 10,
331+
"interval": 5,
332+
# "url": "gcs://..." # if you want to use a url
333+
}
334+
}
335+
]
336+
)
337+
```
338+
339+
```js OpenAI NodeJS
340+
import OpenAI from 'openai'; // We're using the v4 SDK
341+
import { PORTKEY_GATEWAY_URL, createHeaders } from 'portkey-ai'
342+
343+
const portkeyClient = new OpenAI({
344+
apiKey: 'NOT_REQUIRED', // defaults to process.env["OPENAI_API_KEY"],
345+
baseURL: PORTKEY_GATEWAY_URL,
346+
defaultHeaders: createHeaders({
347+
apiKey: "PORTKEY_API_KEY", // defaults to process.env["PORTKEY_API_KEY"]
348+
virtualKey: "PORTKEY_VIRTUAL_KEY"
349+
})
350+
});
351+
352+
const embedding = await portkeyClient.embeddings.create({
353+
input: [
354+
{
355+
"text": "this is the caption of the video",
356+
"video": {
357+
"base64": "UklGRkacAABXRUJQVlA4IDqcAACQggKdASqpAn8B.....",
358+
"start_offset": 0,
359+
"end_offset": 10,
360+
"interval": 5,
361+
// "url": "gcs://..." // if you want to use a url
362+
}
363+
}
364+
],
365+
model: 'multimodalembedding@001'
366+
});
367+
368+
console.log(embedding);
369+
```
370+
371+
</CodeGroup>

0 commit comments

Comments
 (0)