Skip to content

feat(core): Add byte size limit and oldest first truncation for gen_ai messages #17863

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 9 commits into
base: develop
Choose a base branch
from
Open

feat(core): Add byte size limit and oldest first truncation for gen_ai messages #17863

Show file tree
Hide file tree
Changes from 5 commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
29e4eda
add message truncation utility for gen_ai byte limits
naaa760 Oct 5, 2025
bf1003e
packages/core/src/utils/ai/messageTruncation.ts
naaa760 Oct 5, 2025
c50ad1b
apply message truncation to Google GenAI gen_ai.request.messages
naaa760 Oct 5, 2025
18265ac
apply message truncation to Anthropic gen_ai.request.messages
naaa760 Oct 5, 2025
f1b468f
apply message truncation to Vercel AI prompts and messages
naaa760 Oct 5, 2025
50d1d6d
add message truncation utility with 20KB limit and performance optimi…
naaa760 Oct 7, 2025
7df9993
apply message truncation to Google GenAI with proper type handling
naaa760 Oct 7, 2025
c91de63
fix(ai): add type safety for message truncation
naaa760 Oct 8, 2025
bbea7cb
Merge branch 'develop' into genai-msg-limit
naaa760 Oct 8, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 37 additions & 0 deletions packages/core/src/utils/ai/messageTruncation.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
export function getByteSize(str: string): number {
return new TextEncoder().encode(str).length;
}

export function truncateMessagesByBytes(messages: unknown[], maxBytes: number): unknown[] {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do you mind adding tests for this? You can use the files under dev-packages/node-integration-tests/suites/tracing

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can you add JSDocs for this files?

if (!Array.isArray(messages) || messages.length === 0) {
return messages;
}

const messagesJson = JSON.stringify(messages);
const totalBytes = getByteSize(messagesJson);

if (totalBytes <= maxBytes) {
return messages;
}

let truncatedMessages = [...messages];

while (truncatedMessages.length > 0) {
const truncatedJson = JSON.stringify(truncatedMessages);
const truncatedBytes = getByteSize(truncatedJson);

if (truncatedBytes <= maxBytes) {
break;
}

truncatedMessages.shift();
}

return truncatedMessages;
}

export const DEFAULT_GEN_AI_MESSAGES_BYTE_LIMIT = 100000;

export function truncateGenAiMessages(messages: unknown[]): unknown[] {
return truncateMessagesByBytes(messages, DEFAULT_GEN_AI_MESSAGES_BYTE_LIMIT);
}
13 changes: 7 additions & 6 deletions packages/core/src/utils/anthropic-ai/index.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ import {
GEN_AI_RESPONSE_TOOL_CALLS_ATTRIBUTE,
GEN_AI_SYSTEM_ATTRIBUTE,
} from '../ai/gen-ai-attributes';
import { truncateGenAiMessages } from '../ai/messageTruncation';
import { buildMethodPath, getFinalOperationName, getSpanOperation, setTokenUsageAttributes } from '../ai/utils';
import { handleCallbackErrors } from '../handleCallbackErrors';
import { instrumentAsyncIterableStream, instrumentMessageStream } from './streaming';
Expand Down Expand Up @@ -71,16 +72,16 @@ function extractRequestAttributes(args: unknown[], methodPath: string): Record<s
return attributes;
}

/**
* Add private request attributes to spans.
* This is only recorded if recordInputs is true.
*/
function addPrivateRequestAttributes(span: Span, params: Record<string, unknown>): void {
if ('messages' in params) {
span.setAttributes({ [GEN_AI_REQUEST_MESSAGES_ATTRIBUTE]: JSON.stringify(params.messages) });
const messages = params.messages;
const truncatedMessages = truncateGenAiMessages(messages as unknown[]);
span.setAttributes({ [GEN_AI_REQUEST_MESSAGES_ATTRIBUTE]: JSON.stringify(truncatedMessages) });
}
if ('input' in params) {
span.setAttributes({ [GEN_AI_REQUEST_MESSAGES_ATTRIBUTE]: JSON.stringify(params.input) });
const input = params.input;
const truncatedInput = truncateGenAiMessages(input as unknown[]);
span.setAttributes({ [GEN_AI_REQUEST_MESSAGES_ATTRIBUTE]: JSON.stringify(truncatedInput) });
}
if ('prompt' in params) {
span.setAttributes({ [GEN_AI_PROMPT_ATTRIBUTE]: JSON.stringify(params.prompt) });
Expand Down
21 changes: 10 additions & 11 deletions packages/core/src/utils/google-genai/index.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ import {
GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE,
GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE,
} from '../ai/gen-ai-attributes';
import { truncateGenAiMessages } from '../ai/messageTruncation';
import { buildMethodPath, getFinalOperationName, getSpanOperation } from '../ai/utils';
import { handleCallbackErrors } from '../handleCallbackErrors';
import { CHAT_PATH, CHATS_CREATE_METHOD, GOOGLE_GENAI_SYSTEM_NAME } from './constants';
Expand Down Expand Up @@ -128,25 +129,23 @@ function extractRequestAttributes(
return attributes;
}

/**
* Add private request attributes to spans.
* This is only recorded if recordInputs is true.
* Handles different parameter formats for different Google GenAI methods.
*/
function addPrivateRequestAttributes(span: Span, params: Record<string, unknown>): void {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

can you also revert back this JSDoc comment?

// For models.generateContent: ContentListUnion: Content | Content[] | PartUnion | PartUnion[]
if ('contents' in params) {
span.setAttributes({ [GEN_AI_REQUEST_MESSAGES_ATTRIBUTE]: JSON.stringify(params.contents) });
const contents = params.contents;
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

can you revert the comment removal to help others understand the request structure? this could also be a string

const truncatedContents = truncateGenAiMessages(contents as unknown[]);
span.setAttributes({ [GEN_AI_REQUEST_MESSAGES_ATTRIBUTE]: JSON.stringify(truncatedContents) });
}

// For chat.sendMessage: message can be string or Part[]
if ('message' in params) {
span.setAttributes({ [GEN_AI_REQUEST_MESSAGES_ATTRIBUTE]: JSON.stringify(params.message) });
const message = params.message;
const truncatedMessage = truncateGenAiMessages(message as unknown[]);
span.setAttributes({ [GEN_AI_REQUEST_MESSAGES_ATTRIBUTE]: JSON.stringify(truncatedMessage) });
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Bug: String Messages Bypass Truncation

The addPrivateRequestAttributes function casts params.message to unknown[], but message can also be a string. When message is a string, truncateGenAiMessages receives it as a non-array and returns it unchanged, bypassing the intended truncation for string messages and allowing them to exceed the byte limit.

Fix in Cursor Fix in Web

}

// For chats.create: history contains the conversation history
if ('history' in params) {
span.setAttributes({ [GEN_AI_REQUEST_MESSAGES_ATTRIBUTE]: JSON.stringify(params.history) });
const history = params.history;
const truncatedHistory = truncateGenAiMessages(history as unknown[]);
span.setAttributes({ [GEN_AI_REQUEST_MESSAGES_ATTRIBUTE]: JSON.stringify(truncatedHistory) });
}
}

Expand Down
10 changes: 7 additions & 3 deletions packages/core/src/utils/openai/index.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ import {
GEN_AI_RESPONSE_TOOL_CALLS_ATTRIBUTE,
GEN_AI_SYSTEM_ATTRIBUTE,
} from '../ai/gen-ai-attributes';
import { truncateGenAiMessages } from '../ai/messageTruncation';
import { OPENAI_INTEGRATION_NAME } from './constants';
import { instrumentStream } from './streaming';
import type {
Expand Down Expand Up @@ -188,13 +189,16 @@ function addResponseAttributes(span: Span, result: unknown, recordOutputs?: bool
}
}

// Extract and record AI request inputs, if present. This is intentionally separate from response attributes.
function addRequestAttributes(span: Span, params: Record<string, unknown>): void {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

can you also revert back this JSDoc comment?

if ('messages' in params) {
span.setAttributes({ [GEN_AI_REQUEST_MESSAGES_ATTRIBUTE]: JSON.stringify(params.messages) });
const messages = params.messages;
const truncatedMessages = truncateGenAiMessages(messages as unknown[]);
span.setAttributes({ [GEN_AI_REQUEST_MESSAGES_ATTRIBUTE]: JSON.stringify(truncatedMessages) });
}
if ('input' in params) {
span.setAttributes({ [GEN_AI_REQUEST_MESSAGES_ATTRIBUTE]: JSON.stringify(params.input) });
const input = params.input;
const truncatedInput = truncateGenAiMessages(input as unknown[]);
span.setAttributes({ [GEN_AI_REQUEST_MESSAGES_ATTRIBUTE]: JSON.stringify(truncatedInput) });
}
}

Expand Down
9 changes: 8 additions & 1 deletion packages/core/src/utils/vercel-ai/index.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ import { SEMANTIC_ATTRIBUTE_SENTRY_OP, SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN } from '
import type { Event } from '../../types-hoist/event';
import type { Span, SpanAttributes, SpanAttributeValue, SpanJSON, SpanOrigin } from '../../types-hoist/span';
import { spanToJSON } from '../spanUtils';
import { truncateGenAiMessages } from '../ai/messageTruncation';
import { toolCallSpanMap } from './constants';
import type { TokenSummary } from './types';
import { accumulateTokensForParent, applyAccumulatedTokens } from './utils';
Expand Down Expand Up @@ -187,7 +188,13 @@ function processGenerateSpan(span: Span, name: string, attributes: SpanAttribute
}

if (attributes[AI_PROMPT_ATTRIBUTE]) {
span.setAttribute('gen_ai.prompt', attributes[AI_PROMPT_ATTRIBUTE]);
const prompt = attributes[AI_PROMPT_ATTRIBUTE];
if (Array.isArray(prompt)) {
const truncatedPrompt = truncateGenAiMessages(prompt);
span.setAttribute('gen_ai.prompt', JSON.stringify(truncatedPrompt));
} else {
span.setAttribute('gen_ai.prompt', prompt);
}
}
if (attributes[AI_MODEL_ID_ATTRIBUTE] && !attributes[GEN_AI_RESPONSE_MODEL_ATTRIBUTE]) {
span.setAttribute(GEN_AI_RESPONSE_MODEL_ATTRIBUTE, attributes[AI_MODEL_ID_ATTRIBUTE]);
Expand Down
Loading