Skip to content

Commit 48d6843

Browse files
authored
Merge branch 'main' into bedrock-instr-fix
2 parents 9159941 + cec6770 commit 48d6843

33 files changed

+2483
-830
lines changed

aws-distro-opentelemetry-node-autoinstrumentation/package.json

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,7 @@
7878
"@aws-sdk/client-s3": "3.632.0",
7979
"@aws-sdk/client-secrets-manager": "3.632.0",
8080
"@aws-sdk/client-sfn": "3.632.0",
81+
"@aws-sdk/client-sts": "3.632.0",
8182
"@aws-sdk/client-sns": "3.632.0",
8283
"@opentelemetry/contrib-test-utils": "^0.45.0",
8384
"@smithy/protocol-http": "^5.0.1",

aws-distro-opentelemetry-node-autoinstrumentation/src/aws-attribute-keys.ts

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,12 +5,17 @@ import { SEMATTRS_AWS_DYNAMODB_TABLE_NAMES } from '@opentelemetry/semantic-conve
55

66
// Utility class holding attribute keys with special meaning to AWS components
77
export const AWS_ATTRIBUTE_KEYS = {
8+
AWS_AUTH_ACCOUNT_ACCESS_KEY: 'aws.auth.account.access_key',
9+
AWS_AUTH_REGION: 'aws.auth.region',
810
AWS_SPAN_KIND: 'aws.span.kind',
911
AWS_LOCAL_SERVICE: 'aws.local.service',
1012
AWS_LOCAL_OPERATION: 'aws.local.operation',
1113
AWS_REMOTE_SERVICE: 'aws.remote.service',
1214
AWS_REMOTE_ENVIRONMENT: 'aws.remote.environment',
1315
AWS_REMOTE_OPERATION: 'aws.remote.operation',
16+
AWS_REMOTE_RESOURCE_ACCOUNT_ACCESS_KEY: 'aws.remote.resource.account.access_key',
17+
AWS_REMOTE_RESOURCE_ACCOUNT_ID: 'aws.remote.resource.account.id',
18+
AWS_REMOTE_RESOURCE_REGION: 'aws.remote.resource.region',
1419
AWS_REMOTE_RESOURCE_TYPE: 'aws.remote.resource.type',
1520
AWS_REMOTE_RESOURCE_IDENTIFIER: 'aws.remote.resource.identifier',
1621
AWS_SDK_DESCENDANT: 'aws.sdk.descendant',
@@ -31,7 +36,9 @@ export const AWS_ATTRIBUTE_KEYS = {
3136
AWS_S3_BUCKET: 'aws.s3.bucket',
3237
AWS_SQS_QUEUE_URL: 'aws.sqs.queue.url',
3338
AWS_SQS_QUEUE_NAME: 'aws.sqs.queue.name',
39+
AWS_KINESIS_STREAM_ARN: 'aws.kinesis.stream.arn',
3440
AWS_KINESIS_STREAM_NAME: 'aws.kinesis.stream.name',
41+
AWS_DYNAMODB_TABLE_ARN: 'aws.dynamodb.table.arn',
3542
AWS_DYNAMODB_TABLE_NAMES: SEMATTRS_AWS_DYNAMODB_TABLE_NAMES,
3643
AWS_BEDROCK_DATA_SOURCE_ID: 'aws.bedrock.data_source.id',
3744
AWS_BEDROCK_KNOWLEDGE_BASE_ID: 'aws.bedrock.knowledge_base.id',

aws-distro-opentelemetry-node-autoinstrumentation/src/aws-metric-attribute-generator.ts

Lines changed: 86 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ import {
3232
MetricAttributeGenerator,
3333
SERVICE_METRIC,
3434
} from './metric-attribute-generator';
35+
import { RegionalResourceArnParser } from './regional-resource-arn-parser';
3536
import { SqsUrlParser } from './sqs-url-parser';
3637
import { LAMBDA_APPLICATION_SIGNALS_REMOTE_ENVIRONMENT } from './aws-opentelemetry-configurator';
3738

@@ -112,8 +113,20 @@ export class AwsMetricAttributeGenerator implements MetricAttributeGenerator {
112113
AwsMetricAttributeGenerator.setService(resource, span, attributes);
113114
AwsMetricAttributeGenerator.setEgressOperation(span, attributes);
114115
AwsMetricAttributeGenerator.setRemoteServiceAndOperation(span, attributes);
115-
AwsMetricAttributeGenerator.setRemoteResourceTypeAndIdentifier(span, attributes);
116+
const isRemoteResourceIdentifierPresent = AwsMetricAttributeGenerator.setRemoteResourceTypeAndIdentifier(
117+
span,
118+
attributes
119+
);
116120
AwsMetricAttributeGenerator.setRemoteEnvironment(span, attributes);
121+
if (isRemoteResourceIdentifierPresent) {
122+
const isAccountIdAndRegionPresent = AwsMetricAttributeGenerator.setRemoteResourceAccountIdAndRegion(
123+
span,
124+
attributes
125+
);
126+
if (!isAccountIdAndRegionPresent) {
127+
AwsMetricAttributeGenerator.setRemoteResourceAccessKeyAndRegion(span, attributes);
128+
}
129+
}
117130
AwsMetricAttributeGenerator.setSpanKindForDependency(span, attributes);
118131
AwsMetricAttributeGenerator.setRemoteDbUser(span, attributes);
119132

@@ -369,7 +382,7 @@ export class AwsMetricAttributeGenerator implements MetricAttributeGenerator {
369382
* href="https://docs.aws.amazon.com/cloudcontrolapi/latest/userguide/supported-resources.html">AWS
370383
* Cloud Control resource format</a>.
371384
*/
372-
private static setRemoteResourceTypeAndIdentifier(span: ReadableSpan, attributes: Attributes): void {
385+
private static setRemoteResourceTypeAndIdentifier(span: ReadableSpan, attributes: Attributes): boolean {
373386
let remoteResourceType: AttributeValue | undefined;
374387
let remoteResourceIdentifier: AttributeValue | undefined;
375388
let cloudFormationIdentifier: AttributeValue | undefined;
@@ -383,11 +396,25 @@ export class AwsMetricAttributeGenerator implements MetricAttributeGenerator {
383396
) {
384397
remoteResourceType = NORMALIZED_DYNAMO_DB_SERVICE_NAME + '::Table';
385398
remoteResourceIdentifier = AwsMetricAttributeGenerator.escapeDelimiters(awsTableNames[0]);
399+
} else if (AwsSpanProcessingUtil.isKeyPresent(span, AWS_ATTRIBUTE_KEYS.AWS_DYNAMODB_TABLE_ARN)) {
400+
remoteResourceType = NORMALIZED_DYNAMO_DB_SERVICE_NAME + '::Table';
401+
remoteResourceIdentifier = AwsMetricAttributeGenerator.escapeDelimiters(
402+
RegionalResourceArnParser.extractDynamoDbTableNameFromArn(
403+
span.attributes[AWS_ATTRIBUTE_KEYS.AWS_DYNAMODB_TABLE_ARN]
404+
)
405+
);
386406
} else if (AwsSpanProcessingUtil.isKeyPresent(span, AWS_ATTRIBUTE_KEYS.AWS_KINESIS_STREAM_NAME)) {
387407
remoteResourceType = NORMALIZED_KINESIS_SERVICE_NAME + '::Stream';
388408
remoteResourceIdentifier = AwsMetricAttributeGenerator.escapeDelimiters(
389409
span.attributes[AWS_ATTRIBUTE_KEYS.AWS_KINESIS_STREAM_NAME]
390410
);
411+
} else if (AwsSpanProcessingUtil.isKeyPresent(span, AWS_ATTRIBUTE_KEYS.AWS_KINESIS_STREAM_ARN)) {
412+
remoteResourceType = NORMALIZED_KINESIS_SERVICE_NAME + '::Stream';
413+
remoteResourceIdentifier = AwsMetricAttributeGenerator.escapeDelimiters(
414+
RegionalResourceArnParser.extractKinesisStreamNameFromArn(
415+
span.attributes[AWS_ATTRIBUTE_KEYS.AWS_KINESIS_STREAM_ARN]
416+
)
417+
);
391418
} else if (AwsSpanProcessingUtil.isKeyPresent(span, AWS_ATTRIBUTE_KEYS.AWS_S3_BUCKET)) {
392419
remoteResourceType = NORMALIZED_S3_SERVICE_NAME + '::Bucket';
393420
remoteResourceIdentifier = AwsMetricAttributeGenerator.escapeDelimiters(
@@ -398,31 +425,31 @@ export class AwsMetricAttributeGenerator implements MetricAttributeGenerator {
398425

399426
remoteResourceType = NORMALIZED_SNS_SERVICE_NAME + '::Topic';
400427
remoteResourceIdentifier = AwsMetricAttributeGenerator.escapeDelimiters(
401-
this.extractResourceNameFromArn(snsArn)
428+
RegionalResourceArnParser.extractResourceNameFromArn(snsArn)
402429
);
403430
cloudFormationIdentifier = AwsMetricAttributeGenerator.escapeDelimiters(snsArn);
404431
} else if (AwsSpanProcessingUtil.isKeyPresent(span, AWS_ATTRIBUTE_KEYS.AWS_SECRETSMANAGER_SECRET_ARN)) {
405432
const secretsArn = span.attributes[AWS_ATTRIBUTE_KEYS.AWS_SECRETSMANAGER_SECRET_ARN];
406433

407434
remoteResourceType = NORMALIZED_SECRETSMANAGER_SERVICE_NAME + '::Secret';
408435
remoteResourceIdentifier = AwsMetricAttributeGenerator.escapeDelimiters(
409-
this.extractResourceNameFromArn(secretsArn)
436+
RegionalResourceArnParser.extractResourceNameFromArn(secretsArn)
410437
);
411438
cloudFormationIdentifier = AwsMetricAttributeGenerator.escapeDelimiters(secretsArn);
412439
} else if (AwsSpanProcessingUtil.isKeyPresent(span, AWS_ATTRIBUTE_KEYS.AWS_STEPFUNCTIONS_STATEMACHINE_ARN)) {
413440
const stateMachineArn = span.attributes[AWS_ATTRIBUTE_KEYS.AWS_STEPFUNCTIONS_STATEMACHINE_ARN];
414441

415442
remoteResourceType = NORMALIZED_STEPFUNCTIONS_SERVICE_NAME + '::StateMachine';
416443
remoteResourceIdentifier = AwsMetricAttributeGenerator.escapeDelimiters(
417-
this.extractResourceNameFromArn(stateMachineArn)
444+
RegionalResourceArnParser.extractResourceNameFromArn(stateMachineArn)
418445
);
419446
cloudFormationIdentifier = AwsMetricAttributeGenerator.escapeDelimiters(stateMachineArn);
420447
} else if (AwsSpanProcessingUtil.isKeyPresent(span, AWS_ATTRIBUTE_KEYS.AWS_STEPFUNCTIONS_ACTIVITY_ARN)) {
421448
const activityArn = span.attributes[AWS_ATTRIBUTE_KEYS.AWS_STEPFUNCTIONS_ACTIVITY_ARN];
422449

423450
remoteResourceType = NORMALIZED_STEPFUNCTIONS_SERVICE_NAME + '::Activity';
424451
remoteResourceIdentifier = AwsMetricAttributeGenerator.escapeDelimiters(
425-
this.extractResourceNameFromArn(activityArn)
452+
RegionalResourceArnParser.extractResourceNameFromArn(activityArn)
426453
);
427454
cloudFormationIdentifier = AwsMetricAttributeGenerator.escapeDelimiters(activityArn);
428455
} else if (AwsSpanProcessingUtil.isKeyPresent(span, AWS_ATTRIBUTE_KEYS.AWS_LAMBDA_FUNCTION_NAME)) {
@@ -500,7 +527,10 @@ export class AwsMetricAttributeGenerator implements MetricAttributeGenerator {
500527
attributes[AWS_ATTRIBUTE_KEYS.AWS_REMOTE_RESOURCE_TYPE] = remoteResourceType;
501528
attributes[AWS_ATTRIBUTE_KEYS.AWS_REMOTE_RESOURCE_IDENTIFIER] = remoteResourceIdentifier;
502529
attributes[AWS_ATTRIBUTE_KEYS.AWS_CLOUDFORMATION_PRIMARY_IDENTIFIER] = cloudFormationIdentifier;
530+
return true;
503531
}
532+
533+
return false;
504534
}
505535

506536
/**
@@ -522,6 +552,56 @@ export class AwsMetricAttributeGenerator implements MetricAttributeGenerator {
522552
}
523553
}
524554

555+
private static setRemoteResourceAccountIdAndRegion(span: ReadableSpan, attributes: Attributes): boolean {
556+
const ARN_ATTRIBUTES: string[] = [
557+
AWS_ATTRIBUTE_KEYS.AWS_DYNAMODB_TABLE_ARN,
558+
AWS_ATTRIBUTE_KEYS.AWS_KINESIS_STREAM_ARN,
559+
AWS_ATTRIBUTE_KEYS.AWS_SNS_TOPIC_ARN,
560+
AWS_ATTRIBUTE_KEYS.AWS_SECRETSMANAGER_SECRET_ARN,
561+
AWS_ATTRIBUTE_KEYS.AWS_STEPFUNCTIONS_STATEMACHINE_ARN,
562+
AWS_ATTRIBUTE_KEYS.AWS_STEPFUNCTIONS_ACTIVITY_ARN,
563+
AWS_ATTRIBUTE_KEYS.AWS_LAMBDA_FUNCTION_ARN,
564+
AWS_ATTRIBUTE_KEYS.AWS_BEDROCK_GUARDRAIL_ARN,
565+
];
566+
let remoteResourceAccountId: string | undefined = undefined;
567+
let remoteResourceRegion: string | undefined = undefined;
568+
569+
if (AwsSpanProcessingUtil.isKeyPresent(span, AWS_ATTRIBUTE_KEYS.AWS_SQS_QUEUE_URL)) {
570+
const sqsQueueUrl = AwsMetricAttributeGenerator.escapeDelimiters(
571+
span.attributes[AWS_ATTRIBUTE_KEYS.AWS_SQS_QUEUE_URL]
572+
);
573+
remoteResourceAccountId = SqsUrlParser.getAccountId(sqsQueueUrl);
574+
remoteResourceRegion = SqsUrlParser.getRegion(sqsQueueUrl);
575+
} else {
576+
for (const attributeKey of ARN_ATTRIBUTES) {
577+
if (AwsSpanProcessingUtil.isKeyPresent(span, attributeKey)) {
578+
const arn = span.attributes[attributeKey];
579+
remoteResourceAccountId = RegionalResourceArnParser.getAccountId(arn);
580+
remoteResourceRegion = RegionalResourceArnParser.getRegion(arn);
581+
break;
582+
}
583+
}
584+
}
585+
586+
if (remoteResourceAccountId !== undefined && remoteResourceRegion !== undefined) {
587+
attributes[AWS_ATTRIBUTE_KEYS.AWS_REMOTE_RESOURCE_ACCOUNT_ID] = remoteResourceAccountId;
588+
attributes[AWS_ATTRIBUTE_KEYS.AWS_REMOTE_RESOURCE_REGION] = remoteResourceRegion;
589+
return true;
590+
}
591+
592+
return false;
593+
}
594+
595+
private static setRemoteResourceAccessKeyAndRegion(span: ReadableSpan, attributes: Attributes): void {
596+
if (AwsSpanProcessingUtil.isKeyPresent(span, AWS_ATTRIBUTE_KEYS.AWS_AUTH_ACCOUNT_ACCESS_KEY)) {
597+
attributes[AWS_ATTRIBUTE_KEYS.AWS_REMOTE_RESOURCE_ACCOUNT_ACCESS_KEY] =
598+
span.attributes[AWS_ATTRIBUTE_KEYS.AWS_AUTH_ACCOUNT_ACCESS_KEY];
599+
}
600+
if (AwsSpanProcessingUtil.isKeyPresent(span, AWS_ATTRIBUTE_KEYS.AWS_AUTH_REGION)) {
601+
attributes[AWS_ATTRIBUTE_KEYS.AWS_REMOTE_RESOURCE_REGION] = span.attributes[AWS_ATTRIBUTE_KEYS.AWS_AUTH_REGION];
602+
}
603+
}
604+
525605
/**
526606
* RemoteResourceIdentifier is populated with rule <code>
527607
* ^[{db.name}|]?{address}[|{port}]?
@@ -649,16 +729,6 @@ export class AwsMetricAttributeGenerator implements MetricAttributeGenerator {
649729
return rpcService === 'Lambda' && span.attributes[SEMATTRS_RPC_METHOD] === LAMBDA_INVOKE_OPERATION;
650730
}
651731

652-
// Extracts the name of the resource from an arn
653-
private static extractResourceNameFromArn(attribute: AttributeValue | undefined): string | undefined {
654-
if (typeof attribute === 'string' && attribute.startsWith('arn:aws:')) {
655-
const split = attribute.split(':');
656-
return split[split.length - 1];
657-
}
658-
659-
return undefined;
660-
}
661-
662732
/** Span kind is needed for differentiating metrics in the EMF exporter */
663733
private static setSpanKindForService(span: ReadableSpan, attributes: Attributes): void {
664734
let spanKind: string = SpanKind[span.kind];

aws-distro-opentelemetry-node-autoinstrumentation/src/aws-opentelemetry-configurator.ts

Lines changed: 55 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,6 @@ import {
5454
SpanProcessor,
5555
TraceIdRatioBasedSampler,
5656
} from '@opentelemetry/sdk-trace-base';
57-
5857
import {
5958
BatchLogRecordProcessor,
6059
ConsoleLogRecordExporter,
@@ -80,6 +79,9 @@ import { BaggageSpanProcessor } from '@opentelemetry/baggage-span-processor';
8079
import { logs } from '@opentelemetry/api-logs';
8180
import { AWS_ATTRIBUTE_KEYS } from './aws-attribute-keys';
8281
import { AwsCloudWatchOtlpBatchLogRecordProcessor } from './exporter/otlp/aws/logs/aws-cw-otlp-batch-log-record-processor';
82+
import { ConsoleEMFExporter } from './exporter/aws/metrics/console-emf-exporter';
83+
import { EMFExporterBase } from './exporter/aws/metrics/emf-exporter-base';
84+
import { CompactConsoleLogRecordExporter } from './exporter/console/logs/compact-console-log-exporter';
8385

8486
const AWS_TRACES_OTLP_ENDPOINT_PATTERN = '^https://xray\\.([a-z0-9-]+)\\.amazonaws\\.com/v1/traces$';
8587
const AWS_LOGS_OTLP_ENDPOINT_PATTERN = '^https://logs\\.([a-z0-9-]+)\\.amazonaws\\.com/v1/logs$';
@@ -382,14 +384,17 @@ export class AwsOpentelemetryConfigurator {
382384
}
383385

384386
private customizeMetricReader(isEmfEnabled: boolean) {
387+
let exporter: PushMetricExporter | undefined = undefined;
388+
385389
if (isEmfEnabled) {
386-
const emfExporter = createEmfExporter();
387-
if (emfExporter) {
388-
const periodicExportingMetricReader = new PeriodicExportingMetricReader({
389-
exporter: emfExporter,
390-
});
391-
this.metricReader = periodicExportingMetricReader;
392-
}
390+
exporter = createEmfExporter();
391+
}
392+
393+
if (exporter) {
394+
const periodicExportingMetricReader = new PeriodicExportingMetricReader({
395+
exporter: exporter,
396+
});
397+
this.metricReader = periodicExportingMetricReader;
393398
}
394399
}
395400

@@ -523,8 +528,7 @@ export class AwsLoggerProcessorProvider {
523528
return exporters.map(exporter => {
524529
if (exporter instanceof ConsoleLogRecordExporter) {
525530
return new SimpleLogRecordProcessor(exporter);
526-
}
527-
if (exporter instanceof OTLPAwsLogExporter && isAgentObservabilityEnabled()) {
531+
} else if (exporter instanceof OTLPAwsLogExporter && isAgentObservabilityEnabled()) {
528532
return new AwsCloudWatchOtlpBatchLogRecordProcessor(exporter);
529533
}
530534
return new BatchLogRecordProcessor(exporter);
@@ -609,7 +613,16 @@ export class AwsLoggerProcessorProvider {
609613
}
610614
}
611615
} else if (exporter === 'console') {
612-
exporters.push(new ConsoleLogRecordExporter());
616+
let logExporter: LogRecordExporter | undefined = undefined;
617+
if (isLambdaEnvironment()) {
618+
diag.debug(
619+
'Lambda environment detected, using CompactConsoleLogRecordExporter instead of ConsoleLogRecordExporter'
620+
);
621+
logExporter = new CompactConsoleLogRecordExporter();
622+
} else {
623+
logExporter = new ConsoleLogRecordExporter();
624+
}
625+
exporters.push(logExporter);
613626
} else {
614627
diag.warn(`Unsupported OTEL_LOGS_EXPORTER value: "${exporter}". Supported values are: otlp, console, none.`);
615628
}
@@ -961,15 +974,17 @@ export function validateAndFetchLogsHeader(): OtlpLogHeaderSetting {
961974
const logHeaders = process.env.OTEL_EXPORTER_OTLP_LOGS_HEADERS;
962975

963976
if (!logHeaders) {
964-
diag.warn(
965-
'Missing required configuration: The environment variable OTEL_EXPORTER_OTLP_LOGS_HEADERS must be set with ' +
966-
`required headers ${AWS_OTLP_LOGS_GROUP_HEADER} and ${AWS_OTLP_LOGS_STREAM_HEADER}. ` +
967-
`Example: OTEL_EXPORTER_OTLP_LOGS_HEADERS="${AWS_OTLP_LOGS_GROUP_HEADER}=my-log-group,${AWS_OTLP_LOGS_STREAM_HEADER}=my-log-stream"`
968-
);
977+
if (!isLambdaEnvironment()) {
978+
diag.warn(
979+
'Missing required configuration: The environment variable OTEL_EXPORTER_OTLP_LOGS_HEADERS must be set with ' +
980+
`required headers ${AWS_OTLP_LOGS_GROUP_HEADER} and ${AWS_OTLP_LOGS_STREAM_HEADER}. ` +
981+
`Example: OTEL_EXPORTER_OTLP_LOGS_HEADERS="${AWS_OTLP_LOGS_GROUP_HEADER}=my-log-group,${AWS_OTLP_LOGS_STREAM_HEADER}=my-log-stream"`
982+
);
983+
}
969984
return {
970-
logGroup: '',
971-
logStream: '',
972-
namespace: '',
985+
logGroup: undefined,
986+
logStream: undefined,
987+
namespace: undefined,
973988
isValid: false,
974989
};
975990
}
@@ -1030,16 +1045,27 @@ export function checkEmfExporterEnabled(): boolean {
10301045
return true;
10311046
}
10321047

1033-
export function createEmfExporter(): AWSCloudWatchEMFExporter | undefined {
1034-
const headersResult = validateAndFetchLogsHeader();
1035-
if (!headersResult.isValid) {
1036-
return undefined;
1048+
/**
1049+
* Create the appropriate EMF exporter based on the environment and configuration.
1050+
*
1051+
* @returns {EMFExporterBase | undefined}
1052+
*/
1053+
export function createEmfExporter(): EMFExporterBase | undefined {
1054+
let exporter: EMFExporterBase | undefined = undefined;
1055+
const otlpLogHeaderSetting = validateAndFetchLogsHeader();
1056+
1057+
if (isLambdaEnvironment() && !otlpLogHeaderSetting.isValid) {
1058+
// Lambda without valid logs http headers - use Console EMF exporter
1059+
exporter = new ConsoleEMFExporter(otlpLogHeaderSetting.namespace);
1060+
} else if (otlpLogHeaderSetting.isValid) {
1061+
// Non-Lambda environment - use CloudWatch EMF exporter
1062+
// If headersResult.isValid is true, then headersResult.logGroup and headersResult.logStream are guaranteed to be strings
1063+
exporter = new AWSCloudWatchEMFExporter(
1064+
otlpLogHeaderSetting.namespace,
1065+
otlpLogHeaderSetting.logGroup as string,
1066+
otlpLogHeaderSetting.logStream as string
1067+
);
10371068
}
10381069

1039-
// If headersResult.isValid is true, then headersResult.logGroup and headersResult.logStream are guaranteed to be strings
1040-
return new AWSCloudWatchEMFExporter(
1041-
headersResult.namespace,
1042-
headersResult.logGroup as string,
1043-
headersResult.logStream as string
1044-
);
1070+
return exporter;
10451071
}

0 commit comments

Comments
 (0)