Skip to content

Commit 8a644e9

Browse files
aws-sdk-dotnet-automationaws-sdk-dotnet-automation
committed
With this release, Bedrock Evaluation will now support custom metrics for evaluation.
1 parent ada7f52 commit 8a644e9

File tree

37 files changed

+2362
-55
lines changed

37 files changed

+2362
-55
lines changed

‎docgenerator/AWSSDKDocSamples/Bedrock.GeneratedSamples.extra.xml

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
<?xml version="1.0" encoding="utf-8"?>
2+
<docs>
3+
<doc>
4+
<members>
5+
<member name="M:Amazon.Bedrock.IAmazonBedrock.BatchDeleteEvaluationJob(Amazon.Bedrock.Model.BatchDeleteEvaluationJobRequest)" />
6+
<member name="M:Amazon.Bedrock.AmazonBedrockClient.BatchDeleteEvaluationJob(Amazon.Bedrock.Model.BatchDeleteEvaluationJobRequest)" />
7+
<member name="T:Amazon.Bedrock.Model.BatchDeleteEvaluationJobRequest" />
8+
<member name="T:Amazon.Bedrock.Model.BatchDeleteEvaluationJobResponse" />
9+
</members>
10+
<value>
11+
<example>
12+
<para>
13+
The following example shows a request to delete two model evaluation jobs, where one of the jobs is not found.
14+
</para>
15+
<code
16+
title="Delete evaluation jobs"
17+
source=".\AWSSDKDocSamples\Bedrock\Bedrock.GeneratedSamples.cs"
18+
region="example-1" />
19+
</example>
20+
</value>
21+
</doc>
22+
</docs>

‎docgenerator/AWSSDKDocSamples/Bedrock/Bedrock.GeneratedSamples.cs

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
using System;
2+
using System.Collections.Generic;
3+
using System.Linq;
4+
using System.Text;
5+
using System.Threading.Tasks;
6+
7+
using Amazon.Bedrock;
8+
using Amazon.Bedrock.Model;
9+
10+
namespace AWSSDKDocSamples.Amazon.Bedrock.Generated
11+
{
12+
class BedrockSamples : ISample
13+
{
14+
public void BedrockBatchDeleteEvaluationJob()
15+
{
16+
#region example-1
17+
18+
var client = new AmazonBedrockClient();
19+
var response = client.BatchDeleteEvaluationJob(new BatchDeleteEvaluationJobRequest
20+
{
21+
JobIdentifiers = new List<string> {
22+
"arn:aws:bedrock:us-east-2:123456789012:evaluation-job/12rnxmplqv0v",
23+
"arn:aws:bedrock:us-east-2:123456789012:evaluation-job/rispxmpl12rn"
24+
}
25+
});
26+
27+
List<BatchDeleteEvaluationJobError> errors = response.Errors;
28+
List<BatchDeleteEvaluationJobItem> evaluationJobs = response.EvaluationJobs;
29+
30+
#endregion
31+
}
32+
33+
34+
# region ISample Members
35+
public virtual void Run()
36+
{
37+
38+
}
39+
# endregion
40+
41+
}
42+
}

‎generator/ServiceModels/bedrock/bedrock-2023-04-20.api.json

Lines changed: 120 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1112,9 +1112,34 @@
11121112
"required":["datasetMetricConfigs"],
11131113
"members":{
11141114
"datasetMetricConfigs":{"shape":"EvaluationDatasetMetricConfigs"},
1115-
"evaluatorModelConfig":{"shape":"EvaluatorModelConfig"}
1115+
"evaluatorModelConfig":{"shape":"EvaluatorModelConfig"},
1116+
"customMetricConfig":{"shape":"AutomatedEvaluationCustomMetricConfig"}
11161117
}
11171118
},
1119+
"AutomatedEvaluationCustomMetricConfig":{
1120+
"type":"structure",
1121+
"required":[
1122+
"customMetrics",
1123+
"evaluatorModelConfig"
1124+
],
1125+
"members":{
1126+
"customMetrics":{"shape":"AutomatedEvaluationCustomMetrics"},
1127+
"evaluatorModelConfig":{"shape":"CustomMetricEvaluatorModelConfig"}
1128+
}
1129+
},
1130+
"AutomatedEvaluationCustomMetricSource":{
1131+
"type":"structure",
1132+
"members":{
1133+
"customMetricDefinition":{"shape":"CustomMetricDefinition"}
1134+
},
1135+
"union":true
1136+
},
1137+
"AutomatedEvaluationCustomMetrics":{
1138+
"type":"list",
1139+
"member":{"shape":"AutomatedEvaluationCustomMetricSource"},
1140+
"max":10,
1141+
"min":1
1142+
},
11181143
"BaseModelIdentifier":{
11191144
"type":"string",
11201145
"max":2048,
@@ -1189,7 +1214,7 @@
11891214
"type":"string",
11901215
"max":2048,
11911216
"min":1,
1192-
"pattern":"(arn:aws(-[^:]+)?:bedrock:[a-z0-9-]{1,20}:(([0-9]{12}:custom-model/[a-z0-9-]{1,63}[.]{1}[a-z0-9-]{1,63}/[a-z0-9]{12})|(:foundation-model/[a-z0-9-]{1,63}[.]{1}[a-z0-9-]{1,63}([.:]?[a-z0-9-]{1,63}))))|(arn:aws(|-us-gov|-cn|-iso|-iso-b):bedrock:(|[0-9a-z-]{1,20}):(|[0-9]{12}):inference-profile/[a-zA-Z0-9-:.]+)|([a-z0-9-]{1,63}[.]{1}[a-z0-9-]{1,63}([.:]?[a-z0-9-]{1,63}))|(([0-9a-zA-Z][_-]?)+)"
1217+
"pattern":"(arn:aws(-[^:]+)?:(bedrock|sagemaker):[a-z0-9-]{1,20}:([0-9]{12})?:([a-z-]+/)?)?([a-zA-Z0-9.-]{1,63}){0,2}(([:][a-z0-9-]{1,63}){0,2})?(/[a-z0-9]{1,12})?"
11931218
},
11941219
"BedrockModelId":{
11951220
"type":"string",
@@ -1588,6 +1613,44 @@
15881613
"provisionedModelArn":{"shape":"ProvisionedModelArn"}
15891614
}
15901615
},
1616+
"CustomMetricBedrockEvaluatorModel":{
1617+
"type":"structure",
1618+
"required":["modelIdentifier"],
1619+
"members":{
1620+
"modelIdentifier":{"shape":"EvaluatorModelIdentifier"}
1621+
}
1622+
},
1623+
"CustomMetricBedrockEvaluatorModels":{
1624+
"type":"list",
1625+
"member":{"shape":"CustomMetricBedrockEvaluatorModel"},
1626+
"max":1,
1627+
"min":1
1628+
},
1629+
"CustomMetricDefinition":{
1630+
"type":"structure",
1631+
"required":[
1632+
"name",
1633+
"instructions"
1634+
],
1635+
"members":{
1636+
"name":{"shape":"MetricName"},
1637+
"instructions":{"shape":"CustomMetricInstructions"},
1638+
"ratingScale":{"shape":"RatingScale"}
1639+
},
1640+
"sensitive":true
1641+
},
1642+
"CustomMetricEvaluatorModelConfig":{
1643+
"type":"structure",
1644+
"required":["bedrockEvaluatorModels"],
1645+
"members":{
1646+
"bedrockEvaluatorModels":{"shape":"CustomMetricBedrockEvaluatorModels"}
1647+
}
1648+
},
1649+
"CustomMetricInstructions":{
1650+
"type":"string",
1651+
"max":5000,
1652+
"min":1
1653+
},
15911654
"CustomModelArn":{
15921655
"type":"string",
15931656
"max":1011,
@@ -1841,7 +1904,7 @@
18411904
"type":"string",
18421905
"max":2048,
18431906
"min":1,
1844-
"pattern":"(arn:aws(-[^:]+)?:bedrock:[a-z0-9-]{1,20}:((:foundation-model/[a-z0-9-]{1,63}[.]{1}[a-z0-9-]{1,63}([.:]?[a-z0-9-]{1,63}))|([0-9]{12}:provisioned-model/[a-z0-9]{12})|([0-9]{12}:imported-model/[a-z0-9]{12})|([0-9]{12}:application-inference-profile/[a-z0-9]{12})|([0-9]{12}:inference-profile/(([a-z-]{2,8}.)[a-z0-9-]{1,63}[.]{1}[a-z0-9-]{1,63}([.:]?[a-z0-9-]{1,63})))|([0-9]{12}:(default-prompt-router|prompt-router)/[a-zA-Z0-9-:.]+)))|(([a-z]{2}[.]{1})([a-z0-9-]{1,63}[.]{1}[a-z0-9-]{1,63}([.:]?[a-z0-9-]{1,63})))|([a-z0-9-]{1,63}[.]{1}[a-z0-9-]{1,63}([.:]?[a-z0-9-]{1,63}))|arn:aws(-[^:]+)?:sagemaker:[a-z0-9-]{1,20}:[0-9]{12}:endpoint/[a-z0-9-]{1,63}"
1907+
"pattern":"(arn:aws(-[^:]+)?:bedrock:[a-z0-9-]{1,20}:((:foundation-model/[a-z0-9-]{1,63}[.]{1}[a-z0-9-]{1,63}([.:]?[a-z0-9-]{1,63}))|([0-9]{12}:provisioned-model/[a-z0-9]{12})|([0-9]{12}:imported-model/[a-z0-9]{12})|([0-9]{12}:application-inference-profile/[a-z0-9]{12})|([0-9]{12}:inference-profile/(([a-z-]{2,8}.)[a-z0-9-]{1,63}[.]{1}[a-z0-9-]{1,63}([.:]?[a-z0-9-]{1,63})))|([0-9]{12}:(default-prompt-router|prompt-router)/[a-zA-Z0-9-:.]+)))|(([a-z]{2,4}[.]{1})([a-z0-9-]{1,63}[.]{1}[a-z0-9-]{1,63}([.:]?[a-z0-9-]{1,63})))|([a-z0-9-]{1,63}[.]{1}[a-z0-9-]{1,63}([.:]?[a-z0-9-]{1,63}))|arn:aws(-[^:]+)?:sagemaker:[a-z0-9-]{1,20}:[0-9]{12}:endpoint/[a-z0-9-]{1,63}"
18451908
},
18461909
"EvaluationBedrockModelIdentifiers":{
18471910
"type":"list",
@@ -1980,7 +2043,7 @@
19802043
"EvaluationMetricNames":{
19812044
"type":"list",
19822045
"member":{"shape":"EvaluationMetricName"},
1983-
"max":15,
2046+
"max":25,
19842047
"min":1
19852048
},
19862049
"EvaluationModelConfig":{
@@ -2109,14 +2172,17 @@
21092172
"modelIdentifiers":{
21102173
"shape":"EvaluationBedrockModelIdentifiers",
21112174
"deprecated":true,
2112-
"deprecatedMessage":"Inference identifiers should be retrieved from the inferenceConfigSummary"
2175+
"deprecatedMessage":"Inference identifiers should be retrieved from the inferenceConfigSummary",
2176+
"deprecatedSince":"2025-03-07"
21132177
},
21142178
"ragIdentifiers":{
21152179
"shape":"EvaluationBedrockKnowledgeBaseIdentifiers",
21162180
"deprecated":true,
2117-
"deprecatedMessage":"Inference identifiers should be retrieved from the inferenceConfigSummary"
2181+
"deprecatedMessage":"Inference identifiers should be retrieved from the inferenceConfigSummary",
2182+
"deprecatedSince":"2025-03-07"
21182183
},
21192184
"evaluatorModelIdentifiers":{"shape":"EvaluatorModelIdentifiers"},
2185+
"customMetricsEvaluatorModelIdentifiers":{"shape":"EvaluatorModelIdentifiers"},
21202186
"inferenceConfigSummary":{"shape":"EvaluationInferenceConfigSummary"},
21212187
"applicationType":{"shape":"ApplicationType"}
21222188
}
@@ -2234,6 +2300,10 @@
22342300
"Stopped"
22352301
]
22362302
},
2303+
"Float":{
2304+
"type":"float",
2305+
"box":true
2306+
},
22372307
"FoundationModelArn":{
22382308
"type":"string",
22392309
"pattern":"arn:aws(-[^:]+)?:bedrock:[a-z0-9-]{1,20}::foundation-model/[a-z0-9-]{1,63}[.]{1}([a-z0-9-]{1,63}[.]){0,2}[a-z0-9-]{1,63}([:][a-z0-9-]{1,63}){0,2}"
@@ -4448,12 +4518,19 @@
44484518
"type":"float",
44494519
"box":true
44504520
},
4521+
"MetricName":{
4522+
"type":"string",
4523+
"max":63,
4524+
"min":1,
4525+
"pattern":"[0-9a-zA-Z-_.]+",
4526+
"sensitive":true
4527+
},
44514528
"ModelArchitecture":{"type":"string"},
44524529
"ModelArn":{
44534530
"type":"string",
44544531
"max":1011,
44554532
"min":20,
4456-
"pattern":"arn:aws(-[^:]+)?:bedrock:[a-z0-9-]{1,20}:(([0-9]{12}:custom-model/[a-z0-9-]{1,63}[.]{1}[a-z0-9-]{1,63}/[a-z0-9]{12})|(:foundation-model/[a-z0-9-]{1,63}[.]{1}([a-z0-9-]{1,63}[.]){0,2}[a-z0-9-]{1,63}([:][a-z0-9-]{1,63}){0,2}))"
4533+
"pattern":"arn:aws(-[^:]+)?:bedrock:[a-z0-9-]{1,20}:(([0-9]{12}:custom-model/[a-z0-9-]{1,63}[.]{1}[a-z0-9-]{1,63}(([:][a-z0-9-]{1,63}){0,2})?/[a-z0-9]{12})|(:foundation-model/[a-z0-9-]{1,63}[.]{1}([a-z0-9-]{1,63}[.]){0,2}[a-z0-9-]{1,63}([:][a-z0-9-]{1,63}){0,2}))"
44574534
},
44584535
"ModelCopyJobArn":{
44594536
"type":"string",
@@ -4989,6 +5066,41 @@
49895066
"max":1,
49905067
"min":1
49915068
},
5069+
"RatingScale":{
5070+
"type":"list",
5071+
"member":{"shape":"RatingScaleItem"},
5072+
"max":10,
5073+
"min":1
5074+
},
5075+
"RatingScaleItem":{
5076+
"type":"structure",
5077+
"required":[
5078+
"definition",
5079+
"value"
5080+
],
5081+
"members":{
5082+
"definition":{"shape":"RatingScaleItemDefinition"},
5083+
"value":{"shape":"RatingScaleItemValue"}
5084+
}
5085+
},
5086+
"RatingScaleItemDefinition":{
5087+
"type":"string",
5088+
"max":100,
5089+
"min":1
5090+
},
5091+
"RatingScaleItemValue":{
5092+
"type":"structure",
5093+
"members":{
5094+
"stringValue":{"shape":"RatingScaleItemValueStringValueString"},
5095+
"floatValue":{"shape":"Float"}
5096+
},
5097+
"union":true
5098+
},
5099+
"RatingScaleItemValueStringValueString":{
5100+
"type":"string",
5101+
"max":100,
5102+
"min":1
5103+
},
49925104
"RegisterMarketplaceModelEndpointRequest":{
49935105
"type":"structure",
49945106
"required":[
@@ -5166,7 +5278,7 @@
51665278
"type":"string",
51675279
"max":1024,
51685280
"min":1,
5169-
"pattern":"s3://[a-z0-9][-.a-z0-9]{1,61}(?:/[-!_*'().a-z0-9A-Z]+(?:/[-!_*'().a-z0-9A-Z]+)*)?/?"
5281+
"pattern":"s3://[a-z0-9][-.a-z0-9]{1,61}[a-z0-9](?:/[-!_*'().a-z0-9A-Z]+(?:/[-!_*'().a-z0-9A-Z]+)*)?/?"
51705282
},
51715283
"SageMakerEndpoint":{
51725284
"type":"structure",

0 commit comments

Comments
 (0)