Skip to content

Commit 3a290aa

Browse files
narengoginarengogi
authored
Merge branch 'main' into fix/azure-foundry-fix-mapping
2 parents 059d437 + ac7e691 commit 3a290aa

File tree

18 files changed

+189
-58
lines changed

18 files changed

+189
-58
lines changed

‎package-lock.json‎

Lines changed: 2 additions & 2 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎package.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"name": "@portkey-ai/gateway",
3-
"version": "1.10.1",
3+
"version": "1.10.2",
44
"description": "A fast AI gateway by Portkey",
55
"repository": {
66
"type": "git",

‎plugins/portkey/gibberish.ts‎

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ import {
44
PluginHandler,
55
PluginParameters,
66
} from '../types';
7-
import { getText } from '../utils';
7+
import { getCurrentContentPart } from '../utils';
88
import { PORTKEY_ENDPOINTS, fetchPortkey } from './globals';
99

1010
export const handler: PluginHandler = async (
@@ -16,9 +16,17 @@ export const handler: PluginHandler = async (
1616
let error = null;
1717
let verdict = false;
1818
let data: any = null;
19-
19+
let text = '';
2020
try {
21-
const text = getText(context, eventType);
21+
const { content, textArray } = getCurrentContentPart(context, eventType);
22+
if (!content) {
23+
return {
24+
error: { message: 'request or response json is empty' },
25+
verdict: true,
26+
data: null,
27+
};
28+
}
29+
text = textArray.filter((text) => text).join('\n');
2230
const not = parameters.not || false;
2331

2432
const response: any = await fetchPortkey(
@@ -47,7 +55,6 @@ export const handler: PluginHandler = async (
4755
};
4856
} catch (e) {
4957
error = e as Error;
50-
const text = getText(context, eventType);
5158
data = {
5259
explanation: `An error occurred while checking for gibberish: ${error.message}`,
5360
not: parameters.not || false,

‎plugins/portkey/language.ts‎

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ import {
44
PluginHandler,
55
PluginParameters,
66
} from '../types';
7-
import { getText } from '../utils';
7+
import { getCurrentContentPart } from '../utils';
88
import { PORTKEY_ENDPOINTS, fetchPortkey } from './globals';
99

1010
export const handler: PluginHandler = async (
@@ -16,9 +16,17 @@ export const handler: PluginHandler = async (
1616
let error = null;
1717
let verdict = false;
1818
let data: any = null;
19-
19+
let text = '';
2020
try {
21-
const text = getText(context, eventType);
21+
const { content, textArray } = getCurrentContentPart(context, eventType);
22+
if (!content) {
23+
return {
24+
error: { message: 'request or response json is empty' },
25+
verdict: true,
26+
data: null,
27+
};
28+
}
29+
text = textArray.filter((text) => text).join('\n');
2230
const languages = parameters.language;
2331
const not = parameters.not || false;
2432

@@ -51,7 +59,6 @@ export const handler: PluginHandler = async (
5159
};
5260
} catch (e) {
5361
error = e as Error;
54-
const text = getText(context, eventType);
5562
data = {
5663
explanation: `An error occurred while checking language: ${error.message}`,
5764
not: parameters.not || false,

‎plugins/portkey/moderateContent.ts‎

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ import {
44
PluginHandler,
55
PluginParameters,
66
} from '../types';
7-
import { getText } from '../utils';
7+
import { getCurrentContentPart } from '../utils';
88
import { PORTKEY_ENDPOINTS, fetchPortkey } from './globals';
99

1010
export const handler: PluginHandler = async (
@@ -16,9 +16,17 @@ export const handler: PluginHandler = async (
1616
let error = null;
1717
let verdict = false;
1818
let data: any = null;
19-
19+
let text = '';
2020
try {
21-
const text = getText(context, eventType);
21+
const { content, textArray } = getCurrentContentPart(context, eventType);
22+
if (!content) {
23+
return {
24+
error: { message: 'request or response json is empty' },
25+
verdict: true,
26+
data: null,
27+
};
28+
}
29+
text = textArray.filter((text) => text).join('\n');
2230
const categories = parameters.categories;
2331
const not = parameters.not || false;
2432

@@ -59,7 +67,6 @@ export const handler: PluginHandler = async (
5967
};
6068
} catch (e) {
6169
error = e as Error;
62-
const text = getText(context, eventType);
6370
data = {
6471
explanation: `An error occurred during content moderation: ${error.message}`,
6572
not: parameters.not || false,

‎src/globals.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -96,6 +96,7 @@ export const KLUSTER_AI: string = 'kluster-ai';
9696
export const NSCALE: string = 'nscale';
9797
export const HYPERBOLIC: string = 'hyperbolic';
9898
export const FEATHERLESS_AI: string = 'featherless-ai';
99+
export const KRUTRIM: string = 'krutrim';
99100

100101
export const VALID_PROVIDERS = [
101102
ANTHROPIC,
@@ -157,6 +158,7 @@ export const VALID_PROVIDERS = [
157158
NSCALE,
158159
HYPERBOLIC,
159160
FEATHERLESS_AI,
161+
KRUTRIM,
160162
];
161163

162164
export const CONTENT_TYPES = {

‎src/handlers/retryHandler.ts‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -175,7 +175,6 @@ export const retryRequest = async (
175175
retries: retryCount,
176176
onRetry: (error: Error, attempt: number) => {
177177
lastAttempt = attempt;
178-
console.warn(`Failed in Retry attempt ${attempt}. Error: ${error}`);
179178
},
180179
randomize: false,
181180
}
@@ -206,9 +205,6 @@ export const retryRequest = async (
206205
headers: error.headers,
207206
});
208207
}
209-
console.warn(
210-
`Tried ${lastAttempt ?? 1} time(s) but failed. Error: ${JSON.stringify(error)}`
211-
);
212208
}
213209
return {
214210
response: lastResponse as Response,

‎src/handlers/streamHandler.ts‎

Lines changed: 30 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -310,31 +310,41 @@ export function handleStreamingMode(
310310

311311
if (proxyProvider === BEDROCK) {
312312
(async () => {
313-
for await (const chunk of readAWSStream(
314-
reader,
315-
responseTransformer,
316-
fallbackChunkId,
317-
strictOpenAiCompliance,
318-
gatewayRequest
319-
)) {
320-
await writer.write(encoder.encode(chunk));
313+
try {
314+
for await (const chunk of readAWSStream(
315+
reader,
316+
responseTransformer,
317+
fallbackChunkId,
318+
strictOpenAiCompliance,
319+
gatewayRequest
320+
)) {
321+
await writer.write(encoder.encode(chunk));
322+
}
323+
} catch (error) {
324+
console.error(error);
325+
} finally {
326+
writer.close();
321327
}
322-
writer.close();
323328
})();
324329
} else {
325330
(async () => {
326-
for await (const chunk of readStream(
327-
reader,
328-
splitPattern,
329-
responseTransformer,
330-
isSleepTimeRequired,
331-
fallbackChunkId,
332-
strictOpenAiCompliance,
333-
gatewayRequest
334-
)) {
335-
await writer.write(encoder.encode(chunk));
331+
try {
332+
for await (const chunk of readStream(
333+
reader,
334+
splitPattern,
335+
responseTransformer,
336+
isSleepTimeRequired,
337+
fallbackChunkId,
338+
strictOpenAiCompliance,
339+
gatewayRequest
340+
)) {
341+
await writer.write(encoder.encode(chunk));
342+
}
343+
} catch (error) {
344+
console.error(error);
345+
} finally {
346+
writer.close();
336347
}
337-
writer.close();
338348
})();
339349
}
340350

‎src/providers/bedrock/chatComplete.ts‎

Lines changed: 25 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -529,9 +529,8 @@ export const BedrockChatCompleteResponseTransform: (
529529
}
530530

531531
if ('output' in response) {
532-
const shouldSendCacheUsage =
533-
response.usage.cacheWriteInputTokens ||
534-
response.usage.cacheReadInputTokens;
532+
const cacheReadInputTokens = response.usage?.cacheReadInputTokens || 0;
533+
const cacheWriteInputTokens = response.usage?.cacheWriteInputTokens || 0;
535534

536535
let content: string = '';
537536
content = response.output.message.content
@@ -565,12 +564,19 @@ export const BedrockChatCompleteResponseTransform: (
565564
},
566565
],
567566
usage: {
568-
prompt_tokens: response.usage.inputTokens,
567+
prompt_tokens:
568+
response.usage.inputTokens +
569+
cacheReadInputTokens +
570+
cacheWriteInputTokens,
569571
completion_tokens: response.usage.outputTokens,
570572
total_tokens: response.usage.totalTokens, // contains the cache usage as well
571-
...(shouldSendCacheUsage && {
572-
cache_read_input_tokens: response.usage.cacheReadInputTokens,
573-
cache_creation_input_tokens: response.usage.cacheWriteInputTokens,
573+
prompt_tokens_details: {
574+
cached_tokens: cacheReadInputTokens,
575+
},
576+
// we only want to be sending this for anthropic models and this is not openai compliant
577+
...((cacheReadInputTokens > 0 || cacheWriteInputTokens > 0) && {
578+
cache_read_input_tokens: cacheReadInputTokens,
579+
cache_creation_input_tokens: cacheWriteInputTokens,
574580
}),
575581
},
576582
};
@@ -663,9 +669,9 @@ export const BedrockChatCompleteStreamChunkTransform: (
663669

664670
// final chunk
665671
if (parsedChunk.usage) {
666-
const shouldSendCacheUsage =
667-
parsedChunk.usage.cacheWriteInputTokens ||
668-
parsedChunk.usage.cacheReadInputTokens;
672+
const cacheReadInputTokens = parsedChunk.usage?.cacheReadInputTokens || 0;
673+
const cacheWriteInputTokens = parsedChunk.usage?.cacheWriteInputTokens || 0;
674+
669675
return [
670676
`data: ${JSON.stringify({
671677
id: fallbackId,
@@ -684,10 +690,17 @@ export const BedrockChatCompleteStreamChunkTransform: (
684690
},
685691
],
686692
usage: {
687-
prompt_tokens: parsedChunk.usage.inputTokens,
693+
prompt_tokens:
694+
parsedChunk.usage.inputTokens +
695+
cacheReadInputTokens +
696+
cacheWriteInputTokens,
688697
completion_tokens: parsedChunk.usage.outputTokens,
689698
total_tokens: parsedChunk.usage.totalTokens,
690-
...(shouldSendCacheUsage && {
699+
prompt_tokens_details: {
700+
cached_tokens: cacheReadInputTokens,
701+
},
702+
// we only want to be sending this for anthropic models and this is not openai compliant
703+
...((cacheReadInputTokens > 0 || cacheWriteInputTokens > 0) && {
691704
cache_read_input_tokens: parsedChunk.usage.cacheReadInputTokens,
692705
cache_creation_input_tokens:
693706
parsedChunk.usage.cacheWriteInputTokens,

‎src/providers/bedrock/embed.ts‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -52,9 +52,10 @@ export const BedrockCohereEmbedConfig: ProviderConfig = {
5252
encoding_format: {
5353
param: 'embedding_types',
5454
required: false,
55-
transform: (params: any): string[] => {
55+
transform: (params: any): string[] | undefined => {
5656
if (Array.isArray(params.encoding_format)) return params.encoding_format;
57-
return [params.encoding_format];
57+
else if (typeof params.encoding_format === 'string')
58+
return [params.encoding_format];
5859
},
5960
},
6061
};
@@ -115,9 +116,10 @@ export const BedrockTitanEmbedConfig: ProviderConfig = {
115116
encoding_format: {
116117
param: 'embeddingTypes',
117118
required: false,
118-
transform: (params: any): string[] => {
119+
transform: (params: any): string[] | undefined => {
119120
if (Array.isArray(params.encoding_format)) return params.encoding_format;
120-
return [params.encoding_format];
121+
else if (typeof params.encoding_format === 'string')
122+
return [params.encoding_format];
121123
},
122124
},
123125
// Titan specific parameters

0 commit comments

Comments
 (0)