Skip to content

Commit 6ca72a9

Browse files
indranil786indranil786
authored
Merge branch 'Portkey-AI:main' into main
2 parents 82bd01e + 95c62a1 commit 6ca72a9

File tree

17 files changed

+189
-54
lines changed

17 files changed

+189
-54
lines changed

‎package-lock.json‎

Lines changed: 2 additions & 2 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎package.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"name": "@portkey-ai/gateway",
3-
"version": "1.10.1",
3+
"version": "1.10.2",
44
"description": "A fast AI gateway by Portkey",
55
"repository": {
66
"type": "git",

‎plugins/portkey/gibberish.ts‎

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ import {
44
PluginHandler,
55
PluginParameters,
66
} from '../types';
7-
import { getText } from '../utils';
7+
import { getCurrentContentPart } from '../utils';
88
import { PORTKEY_ENDPOINTS, fetchPortkey } from './globals';
99

1010
export const handler: PluginHandler = async (
@@ -16,9 +16,17 @@ export const handler: PluginHandler = async (
1616
let error = null;
1717
let verdict = false;
1818
let data: any = null;
19-
19+
let text = '';
2020
try {
21-
const text = getText(context, eventType);
21+
const { content, textArray } = getCurrentContentPart(context, eventType);
22+
if (!content) {
23+
return {
24+
error: { message: 'request or response json is empty' },
25+
verdict: true,
26+
data: null,
27+
};
28+
}
29+
text = textArray.filter((text) => text).join('\n');
2230
const not = parameters.not || false;
2331

2432
const response: any = await fetchPortkey(
@@ -47,7 +55,6 @@ export const handler: PluginHandler = async (
4755
};
4856
} catch (e) {
4957
error = e as Error;
50-
const text = getText(context, eventType);
5158
data = {
5259
explanation: `An error occurred while checking for gibberish: ${error.message}`,
5360
not: parameters.not || false,

‎plugins/portkey/language.ts‎

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ import {
44
PluginHandler,
55
PluginParameters,
66
} from '../types';
7-
import { getText } from '../utils';
7+
import { getCurrentContentPart } from '../utils';
88
import { PORTKEY_ENDPOINTS, fetchPortkey } from './globals';
99

1010
export const handler: PluginHandler = async (
@@ -16,9 +16,17 @@ export const handler: PluginHandler = async (
1616
let error = null;
1717
let verdict = false;
1818
let data: any = null;
19-
19+
let text = '';
2020
try {
21-
const text = getText(context, eventType);
21+
const { content, textArray } = getCurrentContentPart(context, eventType);
22+
if (!content) {
23+
return {
24+
error: { message: 'request or response json is empty' },
25+
verdict: true,
26+
data: null,
27+
};
28+
}
29+
text = textArray.filter((text) => text).join('\n');
2230
const languages = parameters.language;
2331
const not = parameters.not || false;
2432

@@ -51,7 +59,6 @@ export const handler: PluginHandler = async (
5159
};
5260
} catch (e) {
5361
error = e as Error;
54-
const text = getText(context, eventType);
5562
data = {
5663
explanation: `An error occurred while checking language: ${error.message}`,
5764
not: parameters.not || false,

‎plugins/portkey/moderateContent.ts‎

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ import {
44
PluginHandler,
55
PluginParameters,
66
} from '../types';
7-
import { getText } from '../utils';
7+
import { getCurrentContentPart } from '../utils';
88
import { PORTKEY_ENDPOINTS, fetchPortkey } from './globals';
99

1010
export const handler: PluginHandler = async (
@@ -16,9 +16,17 @@ export const handler: PluginHandler = async (
1616
let error = null;
1717
let verdict = false;
1818
let data: any = null;
19-
19+
let text = '';
2020
try {
21-
const text = getText(context, eventType);
21+
const { content, textArray } = getCurrentContentPart(context, eventType);
22+
if (!content) {
23+
return {
24+
error: { message: 'request or response json is empty' },
25+
verdict: true,
26+
data: null,
27+
};
28+
}
29+
text = textArray.filter((text) => text).join('\n');
2230
const categories = parameters.categories;
2331
const not = parameters.not || false;
2432

@@ -59,7 +67,6 @@ export const handler: PluginHandler = async (
5967
};
6068
} catch (e) {
6169
error = e as Error;
62-
const text = getText(context, eventType);
6370
data = {
6471
explanation: `An error occurred during content moderation: ${error.message}`,
6572
not: parameters.not || false,

‎src/globals.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -96,6 +96,7 @@ export const KLUSTER_AI: string = 'kluster-ai';
9696
export const NSCALE: string = 'nscale';
9797
export const HYPERBOLIC: string = 'hyperbolic';
9898
export const FEATHERLESS_AI: string = 'featherless-ai';
99+
export const KRUTRIM: string = 'krutrim';
99100

100101
export const VALID_PROVIDERS = [
101102
ANTHROPIC,
@@ -157,6 +158,7 @@ export const VALID_PROVIDERS = [
157158
NSCALE,
158159
HYPERBOLIC,
159160
FEATHERLESS_AI,
161+
KRUTRIM,
160162
];
161163

162164
export const CONTENT_TYPES = {

‎src/handlers/streamHandler.ts‎

Lines changed: 30 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -310,31 +310,41 @@ export function handleStreamingMode(
310310

311311
if (proxyProvider === BEDROCK) {
312312
(async () => {
313-
for await (const chunk of readAWSStream(
314-
reader,
315-
responseTransformer,
316-
fallbackChunkId,
317-
strictOpenAiCompliance,
318-
gatewayRequest
319-
)) {
320-
await writer.write(encoder.encode(chunk));
313+
try {
314+
for await (const chunk of readAWSStream(
315+
reader,
316+
responseTransformer,
317+
fallbackChunkId,
318+
strictOpenAiCompliance,
319+
gatewayRequest
320+
)) {
321+
await writer.write(encoder.encode(chunk));
322+
}
323+
} catch (error) {
324+
console.error(error);
325+
} finally {
326+
writer.close();
321327
}
322-
writer.close();
323328
})();
324329
} else {
325330
(async () => {
326-
for await (const chunk of readStream(
327-
reader,
328-
splitPattern,
329-
responseTransformer,
330-
isSleepTimeRequired,
331-
fallbackChunkId,
332-
strictOpenAiCompliance,
333-
gatewayRequest
334-
)) {
335-
await writer.write(encoder.encode(chunk));
331+
try {
332+
for await (const chunk of readStream(
333+
reader,
334+
splitPattern,
335+
responseTransformer,
336+
isSleepTimeRequired,
337+
fallbackChunkId,
338+
strictOpenAiCompliance,
339+
gatewayRequest
340+
)) {
341+
await writer.write(encoder.encode(chunk));
342+
}
343+
} catch (error) {
344+
console.error(error);
345+
} finally {
346+
writer.close();
336347
}
337-
writer.close();
338348
})();
339349
}
340350

‎src/providers/bedrock/chatComplete.ts‎

Lines changed: 25 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -529,9 +529,8 @@ export const BedrockChatCompleteResponseTransform: (
529529
}
530530

531531
if ('output' in response) {
532-
const shouldSendCacheUsage =
533-
response.usage.cacheWriteInputTokens ||
534-
response.usage.cacheReadInputTokens;
532+
const cacheReadInputTokens = response.usage?.cacheReadInputTokens || 0;
533+
const cacheWriteInputTokens = response.usage?.cacheWriteInputTokens || 0;
535534

536535
let content: string = '';
537536
content = response.output.message.content
@@ -565,12 +564,19 @@ export const BedrockChatCompleteResponseTransform: (
565564
},
566565
],
567566
usage: {
568-
prompt_tokens: response.usage.inputTokens,
567+
prompt_tokens:
568+
response.usage.inputTokens +
569+
cacheReadInputTokens +
570+
cacheWriteInputTokens,
569571
completion_tokens: response.usage.outputTokens,
570572
total_tokens: response.usage.totalTokens, // contains the cache usage as well
571-
...(shouldSendCacheUsage && {
572-
cache_read_input_tokens: response.usage.cacheReadInputTokens,
573-
cache_creation_input_tokens: response.usage.cacheWriteInputTokens,
573+
prompt_tokens_details: {
574+
cached_tokens: cacheReadInputTokens,
575+
},
576+
// we only want to be sending this for anthropic models and this is not openai compliant
577+
...((cacheReadInputTokens > 0 || cacheWriteInputTokens > 0) && {
578+
cache_read_input_tokens: cacheReadInputTokens,
579+
cache_creation_input_tokens: cacheWriteInputTokens,
574580
}),
575581
},
576582
};
@@ -663,9 +669,9 @@ export const BedrockChatCompleteStreamChunkTransform: (
663669

664670
// final chunk
665671
if (parsedChunk.usage) {
666-
const shouldSendCacheUsage =
667-
parsedChunk.usage.cacheWriteInputTokens ||
668-
parsedChunk.usage.cacheReadInputTokens;
672+
const cacheReadInputTokens = parsedChunk.usage?.cacheReadInputTokens || 0;
673+
const cacheWriteInputTokens = parsedChunk.usage?.cacheWriteInputTokens || 0;
674+
669675
return [
670676
`data: ${JSON.stringify({
671677
id: fallbackId,
@@ -684,10 +690,17 @@ export const BedrockChatCompleteStreamChunkTransform: (
684690
},
685691
],
686692
usage: {
687-
prompt_tokens: parsedChunk.usage.inputTokens,
693+
prompt_tokens:
694+
parsedChunk.usage.inputTokens +
695+
cacheReadInputTokens +
696+
cacheWriteInputTokens,
688697
completion_tokens: parsedChunk.usage.outputTokens,
689698
total_tokens: parsedChunk.usage.totalTokens,
690-
...(shouldSendCacheUsage && {
699+
prompt_tokens_details: {
700+
cached_tokens: cacheReadInputTokens,
701+
},
702+
// we only want to be sending this for anthropic models and this is not openai compliant
703+
...((cacheReadInputTokens > 0 || cacheWriteInputTokens > 0) && {
691704
cache_read_input_tokens: parsedChunk.usage.cacheReadInputTokens,
692705
cache_creation_input_tokens:
693706
parsedChunk.usage.cacheWriteInputTokens,

‎src/providers/bedrock/embed.ts‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -52,9 +52,10 @@ export const BedrockCohereEmbedConfig: ProviderConfig = {
5252
encoding_format: {
5353
param: 'embedding_types',
5454
required: false,
55-
transform: (params: any): string[] => {
55+
transform: (params: any): string[] | undefined => {
5656
if (Array.isArray(params.encoding_format)) return params.encoding_format;
57-
return [params.encoding_format];
57+
else if (typeof params.encoding_format === 'string')
58+
return [params.encoding_format];
5859
},
5960
},
6061
};
@@ -115,9 +116,10 @@ export const BedrockTitanEmbedConfig: ProviderConfig = {
115116
encoding_format: {
116117
param: 'embeddingTypes',
117118
required: false,
118-
transform: (params: any): string[] => {
119+
transform: (params: any): string[] | undefined => {
119120
if (Array.isArray(params.encoding_format)) return params.encoding_format;
120-
return [params.encoding_format];
121+
else if (typeof params.encoding_format === 'string')
122+
return [params.encoding_format];
121123
},
122124
},
123125
// Titan specific parameters

‎src/providers/cohere/embed.ts‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -50,9 +50,10 @@ export const CohereEmbedConfig: ProviderConfig = {
5050
encoding_format: {
5151
param: 'embedding_types',
5252
required: false,
53-
transform: (params: any): string[] => {
53+
transform: (params: any): string[] | undefined => {
5454
if (Array.isArray(params.encoding_format)) return params.encoding_format;
55-
return [params.encoding_format];
55+
else if (typeof params.encoding_format === 'string')
56+
return [params.encoding_format];
5657
},
5758
},
5859
//backwards compatibility

0 commit comments

Comments
 (0)