Skip to content

Commit 11f590e

Browse files
brandonskiserbrandonskiser
authored
chore: add http status code to failure telemetry events (#247)
1 parent 7a39c7e commit 11f590e

File tree

8 files changed

+140
-26
lines changed

8 files changed

+140
-26
lines changed

‎crates/chat-cli/src/api_client/clients/streaming_client.rs

Lines changed: 14 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -135,14 +135,16 @@ impl StreamingClient {
135135
match response {
136136
Ok(resp) => Ok(SendMessageOutput::Codewhisperer(resp)),
137137
Err(e) => {
138-
let is_quota_breach = e.raw_response().is_some_and(|resp| resp.status().as_u16() == 429);
138+
let status_code = e.raw_response().map(|res| res.status().as_u16());
139+
140+
let is_quota_breach = status_code.is_some_and(|status| status == 429);
139141
let is_context_window_overflow = e.as_service_error().is_some_and(|err| {
140142
matches!(err, err if err.meta().code() == Some("ValidationException")
141143
&& err.meta().message() == Some("Input is too long."))
142144
});
143145

144146
let is_model_unavailable = model_id_opt.is_some()
145-
&& e.raw_response().is_some_and(|resp| resp.status().as_u16() == 500)
147+
&& status_code.is_some_and(|status| status == 500)
146148
&& e.as_service_error().is_some_and(|err| {
147149
err.meta().message()
148150
== Some("Encountered unexpectedly high load when processing the request, please try again.")
@@ -157,17 +159,23 @@ impl StreamingClient {
157159
.unwrap_or(false);
158160

159161
if is_quota_breach {
160-
Err(ApiClientError::QuotaBreach("quota has reached its limit"))
162+
Err(ApiClientError::QuotaBreach {
163+
message: "quota has reached its limit",
164+
status_code,
165+
})
161166
} else if is_context_window_overflow {
162-
Err(ApiClientError::ContextWindowOverflow)
167+
Err(ApiClientError::ContextWindowOverflow { status_code })
163168
} else if is_model_unavailable {
164169
let request_id = e
165170
.as_service_error()
166171
.and_then(|err| err.meta().request_id())
167172
.map(|s| s.to_string());
168-
Err(ApiClientError::ModelOverloadedError { request_id })
173+
Err(ApiClientError::ModelOverloadedError {
174+
request_id,
175+
status_code,
176+
})
169177
} else if is_monthly_limit_err {
170-
Err(ApiClientError::MonthlyLimitReached)
178+
Err(ApiClientError::MonthlyLimitReached { status_code })
171179
} else {
172180
Err(e.into())
173181
}

‎crates/chat-cli/src/api_client/error.rs

Lines changed: 42 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ use amzn_qdeveloper_streaming_client::types::error::ChatResponseStreamError as Q
1212
use aws_sdk_ssooidc::error::ProvideErrorMetadata;
1313
use aws_smithy_runtime_api::client::orchestrator::HttpResponse;
1414
pub use aws_smithy_runtime_api::client::result::SdkError;
15+
use aws_smithy_runtime_api::http::Response;
1516
use aws_smithy_types::event_stream::RawMessage;
1617
use thiserror::Error;
1718

@@ -51,11 +52,14 @@ pub enum ApiClientError {
5152

5253
// quota breach
5354
#[error("quota has reached its limit")]
54-
QuotaBreach(&'static str),
55+
QuotaBreach {
56+
message: &'static str,
57+
status_code: Option<u16>,
58+
},
5559

5660
// Separate from quota breach (somehow)
5761
#[error("monthly query limit reached")]
58-
MonthlyLimitReached,
62+
MonthlyLimitReached { status_code: Option<u16> },
5963

6064
#[error("{}", SdkErrorDisplay(.0))]
6165
CreateSubscriptionToken(#[from] SdkError<CreateSubscriptionTokenError, HttpResponse>),
@@ -66,7 +70,7 @@ pub enum ApiClientError {
6670
/// Note that we currently do not receive token usage information regarding how large the
6771
/// context window is.
6872
#[error("the context window has overflowed")]
69-
ContextWindowOverflow,
73+
ContextWindowOverflow { status_code: Option<u16> },
7074

7175
#[error(transparent)]
7276
SmithyBuild(#[from] aws_smithy_types::error::operation::BuildError),
@@ -80,7 +84,34 @@ pub enum ApiClientError {
8084
#[error(
8185
"The model you've selected is temporarily unavailable. Please use '/model' to select a different model and try again."
8286
)]
83-
ModelOverloadedError { request_id: Option<String> },
87+
ModelOverloadedError {
88+
request_id: Option<String>,
89+
status_code: Option<u16>,
90+
},
91+
}
92+
93+
impl ApiClientError {
94+
pub fn status_code(&self) -> Option<u16> {
95+
match self {
96+
ApiClientError::GenerateCompletions(e) => sdk_status_code(e),
97+
ApiClientError::GenerateRecommendations(e) => sdk_status_code(e),
98+
ApiClientError::ListAvailableCustomizations(e) => sdk_status_code(e),
99+
ApiClientError::ListAvailableServices(e) => sdk_status_code(e),
100+
ApiClientError::CodewhispererGenerateAssistantResponse(e) => sdk_status_code(e),
101+
ApiClientError::QDeveloperSendMessage(e) => sdk_status_code(e),
102+
ApiClientError::CodewhispererChatResponseStream(_) => None,
103+
ApiClientError::QDeveloperChatResponseStream(_) => None,
104+
ApiClientError::ListAvailableProfilesError(e) => sdk_status_code(e),
105+
ApiClientError::SendTelemetryEvent(e) => sdk_status_code(e),
106+
ApiClientError::CreateSubscriptionToken(e) => sdk_status_code(e),
107+
ApiClientError::QuotaBreach { status_code, .. } => *status_code,
108+
ApiClientError::ContextWindowOverflow { status_code } => *status_code,
109+
ApiClientError::SmithyBuild(_) => None,
110+
ApiClientError::AuthError(_) => None,
111+
ApiClientError::ModelOverloadedError { status_code, .. } => *status_code,
112+
ApiClientError::MonthlyLimitReached { status_code } => *status_code,
113+
}
114+
}
84115
}
85116

86117
impl ReasonCode for ApiClientError {
@@ -97,12 +128,12 @@ impl ReasonCode for ApiClientError {
97128
ApiClientError::ListAvailableProfilesError(e) => sdk_error_code(e),
98129
ApiClientError::SendTelemetryEvent(e) => sdk_error_code(e),
99130
ApiClientError::CreateSubscriptionToken(e) => sdk_error_code(e),
100-
ApiClientError::QuotaBreach(_) => "QuotaBreachError".to_string(),
101-
ApiClientError::ContextWindowOverflow => "ContextWindowOverflow".to_string(),
131+
ApiClientError::QuotaBreach { .. } => "QuotaBreachError".to_string(),
132+
ApiClientError::ContextWindowOverflow { .. } => "ContextWindowOverflow".to_string(),
102133
ApiClientError::SmithyBuild(_) => "SmithyBuildError".to_string(),
103134
ApiClientError::AuthError(_) => "AuthError".to_string(),
104135
ApiClientError::ModelOverloadedError { .. } => "ModelOverloadedError".to_string(),
105-
ApiClientError::MonthlyLimitReached => "MonthlyLimitReached".to_string(),
136+
ApiClientError::MonthlyLimitReached { .. } => "MonthlyLimitReached".to_string(),
106137
}
107138
}
108139
}
@@ -116,6 +147,10 @@ where
116147
.unwrap_or_else(|| e.to_string())
117148
}
118149

150+
fn sdk_status_code<E>(e: &SdkError<E, Response>) -> Option<u16> {
151+
e.raw_response().map(|res| res.status().as_u16())
152+
}
153+
119154
#[cfg(test)]
120155
mod tests {
121156
use std::error::Error as _;

‎crates/chat-cli/src/cli/chat/mod.rs

Lines changed: 52 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -580,6 +580,22 @@ pub enum ChatError {
580580
GetPromptError(#[from] GetPromptError),
581581
}
582582

583+
impl ChatError {
584+
fn status_code(&self) -> Option<u16> {
585+
match self {
586+
ChatError::Client(e) => e.status_code(),
587+
ChatError::Auth(_) => None,
588+
ChatError::ResponseStream(_) => None,
589+
ChatError::Std(_) => None,
590+
ChatError::Readline(_) => None,
591+
ChatError::Custom(_) => None,
592+
ChatError::Interrupted { .. } => None,
593+
ChatError::NonInteractiveToolApproval => None,
594+
ChatError::GetPromptError(_) => None,
595+
}
596+
}
597+
}
598+
583599
impl ReasonCode for ChatError {
584600
fn reason_code(&self) -> String {
585601
match self {
@@ -1001,7 +1017,7 @@ impl ChatContext {
10011017
ChatState::HandleResponseStream(response) => tokio::select! {
10021018
res = self.handle_response(database, telemetry, response) => res,
10031019
Ok(_) = ctrl_c_stream => {
1004-
self.send_chat_telemetry(database, telemetry, None, TelemetryResult::Cancelled, None, None).await;
1020+
self.send_chat_telemetry(database, telemetry, None, TelemetryResult::Cancelled, None, None, None).await;
10051021

10061022
Err(ChatError::Interrupted { tool_uses: None })
10071023
}
@@ -1027,7 +1043,7 @@ impl ChatContext {
10271043
Ok(state) => Ok(state),
10281044
Err(e) => {
10291045
let (reason, reason_desc) = get_error_reason(&e);
1030-
self.send_error_telemetry(database, telemetry, reason, Some(reason_desc))
1046+
self.send_error_telemetry(database, telemetry, reason, Some(reason_desc), e.status_code())
10311047
.await;
10321048

10331049
macro_rules! print_err {
@@ -1096,7 +1112,7 @@ impl ChatContext {
10961112
ChatError::Client(err) => match err {
10971113
// Errors from attempting to send too large of a conversation history. In
10981114
// this case, attempt to automatically compact the history for the user.
1099-
crate::api_client::ApiClientError::ContextWindowOverflow => {
1115+
crate::api_client::ApiClientError::ContextWindowOverflow { .. } => {
11001116
if !self.conversation_state.can_create_summary_request().await {
11011117
execute!(
11021118
self.output,
@@ -1135,10 +1151,10 @@ impl ChatContext {
11351151
help: false,
11361152
});
11371153
},
1138-
crate::api_client::ApiClientError::QuotaBreach(msg) => {
1139-
print_err!(msg, err);
1154+
crate::api_client::ApiClientError::QuotaBreach { message, .. } => {
1155+
print_err!(message, err);
11401156
},
1141-
crate::api_client::ApiClientError::ModelOverloadedError { request_id } => {
1157+
crate::api_client::ApiClientError::ModelOverloadedError { request_id, .. } => {
11421158
queue!(
11431159
self.output,
11441160
style::SetAttribute(Attribute::Bold),
@@ -1162,7 +1178,7 @@ impl ChatContext {
11621178
style::SetForegroundColor(Color::Reset),
11631179
)?;
11641180
},
1165-
crate::api_client::ApiClientError::MonthlyLimitReached => {
1181+
crate::api_client::ApiClientError::MonthlyLimitReached { .. } => {
11661182
let subscription_status = get_subscription_status(database).await;
11671183
if subscription_status.is_err() {
11681184
execute!(
@@ -1303,10 +1319,11 @@ impl ChatContext {
13031319
TelemetryResult::Failed,
13041320
Some(reason),
13051321
Some(reason_desc),
1322+
e.status_code(),
13061323
)
13071324
.await;
13081325
match e {
1309-
crate::api_client::ApiClientError::ContextWindowOverflow => {
1326+
crate::api_client::ApiClientError::ContextWindowOverflow { .. } => {
13101327
self.conversation_state.clear(true);
13111328
if self.interactive {
13121329
self.spinner.take();
@@ -1353,6 +1370,7 @@ impl ChatContext {
13531370
TelemetryResult::Failed,
13541371
Some(reason),
13551372
Some(reason_desc),
1373+
err.status_code(),
13561374
)
13571375
.await;
13581376
return Err(err.into());
@@ -1371,8 +1389,16 @@ impl ChatContext {
13711389
)?;
13721390
}
13731391

1374-
self.send_chat_telemetry(database, telemetry, request_id, TelemetryResult::Succeeded, None, None)
1375-
.await;
1392+
self.send_chat_telemetry(
1393+
database,
1394+
telemetry,
1395+
request_id,
1396+
TelemetryResult::Succeeded,
1397+
None,
1398+
None,
1399+
None,
1400+
)
1401+
.await;
13761402

13771403
self.conversation_state.replace_history_with_summary(summary.clone());
13781404

@@ -3699,6 +3725,7 @@ impl ChatContext {
36993725
TelemetryResult::Failed,
37003726
Some(reason),
37013727
Some(reason_desc),
3728+
recv_error.status_code(),
37023729
)
37033730
.await;
37043731

@@ -3810,8 +3837,16 @@ impl ChatContext {
38103837
}
38113838

38123839
if ended {
3813-
self.send_chat_telemetry(database, telemetry, request_id, TelemetryResult::Succeeded, None, None)
3814-
.await;
3840+
self.send_chat_telemetry(
3841+
database,
3842+
telemetry,
3843+
request_id,
3844+
TelemetryResult::Succeeded,
3845+
None,
3846+
None,
3847+
None,
3848+
)
3849+
.await;
38153850

38163851
if self.interactive
38173852
&& database
@@ -4196,6 +4231,7 @@ impl ChatContext {
41964231
Ok(())
41974232
}
41984233

4234+
#[allow(clippy::too_many_arguments)]
41994235
async fn send_chat_telemetry(
42004236
&self,
42014237
database: &Database,
@@ -4204,6 +4240,7 @@ impl ChatContext {
42044240
result: TelemetryResult,
42054241
reason: Option<String>,
42064242
reason_desc: Option<String>,
4243+
status_code: Option<u16>,
42074244
) {
42084245
telemetry
42094246
.send_chat_added_message(
@@ -4215,6 +4252,7 @@ impl ChatContext {
42154252
result,
42164253
reason,
42174254
reason_desc,
4255+
status_code,
42184256
self.conversation_state.model.clone(),
42194257
)
42204258
.await
@@ -4227,6 +4265,7 @@ impl ChatContext {
42274265
telemetry: &TelemetryThread,
42284266
reason: String,
42294267
reason_desc: Option<String>,
4268+
status_code: Option<u16>,
42304269
) {
42314270
telemetry
42324271
.send_response_error(
@@ -4236,6 +4275,7 @@ impl ChatContext {
42364275
TelemetryResult::Failed,
42374276
Some(reason),
42384277
reason_desc,
4278+
status_code,
42394279
)
42404280
.await
42414281
.ok();

‎crates/chat-cli/src/cli/chat/parser.rs

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,17 @@ pub struct RecvError {
2727
pub source: RecvErrorKind,
2828
}
2929

30+
impl RecvError {
31+
pub fn status_code(&self) -> Option<u16> {
32+
match &self.source {
33+
RecvErrorKind::Client(e) => e.status_code(),
34+
RecvErrorKind::Json(_) => None,
35+
RecvErrorKind::StreamTimeout { .. } => None,
36+
RecvErrorKind::UnexpectedToolUseEos { .. } => None,
37+
}
38+
}
39+
}
40+
3041
impl ReasonCode for RecvError {
3142
fn reason_code(&self) -> String {
3243
"RecvError".to_string()

‎crates/chat-cli/src/telemetry/core.rs

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -133,6 +133,7 @@ impl Event {
133133
result,
134134
reason,
135135
reason_desc,
136+
status_code,
136137
model,
137138
..
138139
} => Some(
@@ -149,6 +150,7 @@ impl Event {
149150
result: result.to_string().into(),
150151
reason: reason.map(Into::into),
151152
reason_desc: reason_desc.map(Into::into),
153+
status_code: status_code.map(|v| v as i64).map(Into::into),
152154
codewhispererterminal_model: model.map(Into::into),
153155
}
154156
.into_metric_datum(),
@@ -251,6 +253,7 @@ impl Event {
251253
result,
252254
reason,
253255
reason_desc,
256+
status_code,
254257
} => Some(
255258
AmazonqMessageResponseError {
256259
create_time: self.created_time,
@@ -262,6 +265,7 @@ impl Event {
262265
result: Some(result.to_string().into()),
263266
reason: reason.map(Into::into),
264267
reason_desc: reason_desc.map(Into::into),
268+
status_code: status_code.map(|v| v as i64).map(Into::into),
265269
}
266270
.into_metric_datum(),
267271
),
@@ -299,6 +303,7 @@ pub enum EventType {
299303
result: TelemetryResult,
300304
reason: Option<String>,
301305
reason_desc: Option<String>,
306+
status_code: Option<u16>,
302307
model: Option<String>,
303308
},
304309
ToolUseSuggested {
@@ -338,6 +343,7 @@ pub enum EventType {
338343
result: TelemetryResult,
339344
reason: Option<String>,
340345
reason_desc: Option<String>,
346+
status_code: Option<u16>,
341347
conversation_id: String,
342348
context_file_length: Option<usize>,
343349
},

‎crates/chat-cli/src/telemetry/definitions.rs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ mod tests {
3030
result: crate::telemetry::definitions::types::Result::new("Succeeded".to_string()),
3131
reason: None,
3232
reason_desc: None,
33+
status_code: None,
3334
codewhispererterminal_model: None,
3435
});
3536

0 commit comments

Comments
 (0)