Skip to content

fix(amazonq): Error handling and telemetry for Unit test generation. #5192

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 25 commits into from
Jan 14, 2025
Merged

fix(amazonq): Error handling and telemetry for Unit test generation. #5192

Show file tree
Hide file tree
Changes from 6 commits
Commits
Show all changes
25 commits
Select commit Hold shift + click to select a range
b733179
Error handling for UTG
laileni-aws Dec 11, 2024
bc08f99
Error handling for UTG
laileni-aws Dec 11, 2024
9ee9b13
Merge branch 'aws:main' into main
laileni-aws Dec 11, 2024
c04de5e
Minor edits addresing comments
laileni-aws Dec 12, 2024
b22eca6
Lint errors
laileni-aws Dec 12, 2024
10b39bb
Merge branch 'main' into main
laileni-aws Dec 12, 2024
7fc0959
Merge branch 'main' into main
laileni-aws Dec 13, 2024
ee4de19
Merge branch 'main' into main
laileni-aws Dec 13, 2024
95b7f1b
Merge branch 'main' into main
laileni-aws Dec 13, 2024
75a6254
Fixing build failures
laileni-aws Dec 13, 2024
3a87313
Fixing build failures
laileni-aws Dec 13, 2024
4461636
Merge branch 'main' into main
laileni-aws Dec 13, 2024
143c8c7
Removing CodeTestServerException from CodeTestException file
laileni-aws Dec 13, 2024
d893719
Adding Generic error message for CX facing UX
laileni-aws Dec 13, 2024
4698590
Changing status code from 4XX to 5XX for CreateTestJobError
laileni-aws Dec 13, 2024
a0c6fc1
Moving statusCode datatype from string to int
laileni-aws Dec 13, 2024
0bab6b6
Adding perfClientLatency to telemetry event (#4)
laileni-aws Dec 14, 2024
1c89a6e
Addressing comments
laileni-aws Dec 14, 2024
f200c7c
Addressing Comments
laileni-aws Dec 14, 2024
f86d1cb
Merge branch 'main' into main
laileni-aws Dec 14, 2024
d62ae9c
Removing the statusCode from UTG telemetry
laileni-aws Dec 14, 2024
584547b
Minor edits
laileni-aws Dec 14, 2024
6bf2386
Merge branch 'main' into main
ashishrp-aws Jan 13, 2025
cf60095
Merge branch 'main' into main
ashishrp-aws Jan 14, 2025
6a566d3
Merge branch 'main' into main
ashishrp-aws Jan 14, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 2 additions & 5 deletions .../software/aws/toolkits/jetbrains/services/amazonqCodeTest/CodeWhispererCodeTestSession.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,9 +34,6 @@ class CodeWhispererCodeTestSession(val sessionContext: CodeTestSessionContext) {
* Run UTG sessions are follow steps:
* 1. Zipping project
* 2. Creating Upload url & Upload to S3 bucket
* 3. StartTestGeneration API -> Get JobId
* 4. GetTestGeneration API
* 5. ExportResultsArchieve API
*/
suspend fun run(codeTestChatHelper: CodeTestChatHelper, previousIterationContext: PreviousUTGIterationContext?): CodeTestResponseContext {
try {
Expand Down Expand Up @@ -94,7 +91,8 @@ class CodeWhispererCodeTestSession(val sessionContext: CodeTestSessionContext) {
sourceZip,
"SourceCode",
CodeWhispererConstants.UploadTaskType.UTG,
taskName
taskName,
CodeWhispererConstants.FeatureName.TEST_GENERATION
)

sourceZipUploadResponse.uploadId()
Expand All @@ -113,7 +111,6 @@ class CodeWhispererCodeTestSession(val sessionContext: CodeTestSessionContext) {
sourceZipUploadResponse,
testSummaryMessageId
)
// TODO send telemetry for upload duration

return codeTestResponseContext
} catch (e: Exception) {
Expand Down
105 changes: 73 additions & 32 deletions ...c/software/aws/toolkits/jetbrains/services/amazonqCodeTest/CodeWhispererUTGChatManager.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,6 @@
import kotlinx.coroutines.Job
import kotlinx.coroutines.delay
import kotlinx.coroutines.launch
import software.amazon.awssdk.services.codewhispererruntime.model.CodeWhispererRuntimeException
import software.amazon.awssdk.services.codewhispererruntime.model.GetTestGenerationResponse
import software.amazon.awssdk.services.codewhispererruntime.model.Range
import software.amazon.awssdk.services.codewhispererruntime.model.StartTestGenerationResponse
Expand All @@ -38,9 +37,14 @@
import software.aws.toolkits.jetbrains.services.amazonqCodeTest.session.Session
import software.aws.toolkits.jetbrains.services.amazonqCodeTest.utils.combineBuildAndExecuteLogFiles
import software.aws.toolkits.jetbrains.services.codemodernizer.utils.calculateTotalLatency
import software.aws.toolkits.jetbrains.services.codewhisperer.codetest.CodeTestException
import software.aws.toolkits.jetbrains.services.codewhisperer.codetest.codeTestServerException
import software.aws.toolkits.jetbrains.services.codewhisperer.codetest.sessionconfig.CodeTestSessionConfig
import software.aws.toolkits.jetbrains.services.codewhisperer.codetest.testGenStoppedError
import software.aws.toolkits.jetbrains.services.codewhisperer.credentials.CodeWhispererClientAdaptor
import software.aws.toolkits.jetbrains.services.codewhisperer.util.CodeWhispererConstants
import software.aws.toolkits.jetbrains.services.codewhisperer.util.CodeWhispererUtil.promptReAuth
import software.aws.toolkits.jetbrains.services.codewhisperer.util.getTelemetryErrorMessage
import software.aws.toolkits.jetbrains.services.cwc.controller.chat.telemetry.getStartUrl
import software.aws.toolkits.jetbrains.services.cwc.messages.ChatMessageType
import software.aws.toolkits.jetbrains.services.cwc.messages.CodeReference
Expand All @@ -67,7 +71,7 @@

private fun throwIfCancelled(session: Session) {
if (!session.isGeneratingTests) {
error(message("testgen.message.cancelled"))
throw testGenStoppedError()
}
}

Expand Down Expand Up @@ -104,24 +108,37 @@
}

// 2nd API call: StartTestGeneration
val startTestGenerationResponse = startTestGeneration(
uploadId = createUploadUrlResponse.uploadId(),
targetCode = listOf(
TargetCode.builder()
.relativeTargetPath(codeTestResponseContext.currentFileRelativePath.toString())
.targetLineRangeList(
if (selectionRange != null) {
listOf(
selectionRange
)
} else {
emptyList()
}
)
.build()
),
userInput = prompt
)
val startTestGenerationResponse = try {
startTestGeneration(
uploadId = createUploadUrlResponse.uploadId(),
targetCode = listOf(
TargetCode.builder()
.relativeTargetPath(codeTestResponseContext.currentFileRelativePath.toString())
.targetLineRangeList(
if (selectionRange != null) {
listOf(
selectionRange
)
} else {
emptyList()
}
)
.build()
),
userInput = prompt
)
} catch (e: Exception) {
LOG.error(e) { "Failed to create test generation job" }
// TODO: Not able to emit e.statusCode directly as statusCode is private property
// Cannot access 'statusCode': it is invisible (private in a supertype) in 'ThrottlingException'
Copy link
Contributor

@rli rli Dec 13, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

try statusCode()

Copy link
Contributor Author

@laileni-aws laileni-aws Dec 13, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yeah, I tried statusCode(), no luck.

Copy link
Contributor

@rli rli Dec 13, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

image

val errorMessage = getTelemetryErrorMessage(e, CodeWhispererConstants.FeatureName.TEST_GENERATION)
throw codeTestServerException(
"CreateTestJobError: $errorMessage",
"400",
Copy link
Contributor

@ctlai95 ctlai95 Dec 13, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should this be renamed? It mentions server exception but uses 4xx

Copy link
Contributor Author

@laileni-aws laileni-aws Dec 13, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yeah this needs to be e.statusCode() but for now this can be 5XX.

"CreateTestJobError",
message("testgen.error.generic_technical_error_message")

Check warning

Code scanning / QDJVMC

Usage of redundant or deprecated syntax or deprecated symbols Warning

'message(String, vararg Any): String' is deprecated. Use extension-specific localization bundle instead
)
}

val job = startTestGenerationResponse.testGenerationJob()
session.startTestGenerationRequestId = startTestGenerationResponse.responseMetadata().requestId()
Expand Down Expand Up @@ -173,7 +190,13 @@
}
// update test summary card
} else {
throw Exception(message("testgen.message.failed"))
// If job status is Completed and has no ShortAnswer then there might be some issue in the backend.
throw codeTestServerException(
Copy link
Contributor

@ctlai95 ctlai95 Dec 13, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Have you verified that this works? It looks like you are throwing a function which throws an exception

Copy link
Contributor Author

@laileni-aws laileni-aws Dec 13, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Verified in Kibana and its working fine. Attached related links in Slack.

"TestGenFailedError: " + message("testgen.message.failed"),
"500",
"TestGenFailedError",
message("testgen.error.generic_technical_error_message")

Check warning

Code scanning / QDJVMC

Usage of redundant or deprecated syntax or deprecated symbols Warning

'message(String, vararg Any): String' is deprecated. Use extension-specific localization bundle instead
)
}
} else if (status == TestGenerationJobStatus.FAILED) {
LOG.debug {
Expand All @@ -184,11 +207,17 @@
shortAnswer = parseShortAnswerString(testGenerationResponse.testGenerationJob().shortAnswer())
if (shortAnswer.stopIteration == "true") {
throw Exception("${shortAnswer.planSummary}")
throw codeTestServerException("TestGenFailedError: ${shortAnswer.planSummary}", "400", "TestGenFailedError", shortAnswer.planSummary)
}
}

// TODO: Modify text according to FnF
throw Exception(message("testgen.message.failed"))
// If job status is Failed and has no ShortAnswer then there might be some issue in the backend.
throw codeTestServerException(
"TestGenFailedError: " + message("testgen.message.failed"),
"500",
"TestGenFailedError",
message("testgen.error.generic_technical_error_message")
)
} else {
// In progress
LOG.debug {
Expand All @@ -200,7 +229,7 @@
if (previousIterationContext == null && testGenerationResponse.testGenerationJob().shortAnswer() != null) {
shortAnswer = parseShortAnswerString(testGenerationResponse.testGenerationJob().shortAnswer())
if (shortAnswer.stopIteration == "true") {
throw Exception("${shortAnswer.planSummary}")
throw codeTestServerException("TestGenFailedError: ${shortAnswer.planSummary}", "400", "TestGenFailedError", shortAnswer.planSummary)
}
codeTestChatHelper.updateAnswer(
CodeTestChatMessageContent(
Expand Down Expand Up @@ -232,6 +261,12 @@
},
{ e ->
LOG.error(e) { "ExportResultArchive failed: ${e.message}" }
throw codeTestServerException(
"ExportResultsArchiveError: ${e.message}",
"500",
"ExportResultsArchiveError",
message("testgen.error.generic_technical_error_message")
)
},
{ startTime ->
LOG.info { "ExportResultArchive latency: ${calculateTotalLatency(startTime, Instant.now())}" }
Expand Down Expand Up @@ -493,14 +528,19 @@
launchTestGenFlow(prompt, codeTestChatHelper, previousIterationContext, selectionRange)
} catch (e: Exception) {
// Add an answer for displaying error message
var errorMessage = e.message
if (e is JsonParseException) {
errorMessage = message("testgen.error.generic_error_message")
val errorMessage = when {
e is CodeTestException && e.statusCode == "400" &&

Check warning on line 532 in plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonqCodeTest/CodeWhispererUTGChatManager.kt

View workflow job for this annotation

GitHub Actions / qodana

Constant conditions 

Condition 'e is CodeTestException' is always true
e.message?.startsWith(
"CreateTestJobError: " +
"Maximum com.amazon.aws.codewhisperer.runtime.StartTestGeneration reached for this month."
) == true ->
message("testgen.error.maximum_generations_reach")

Check warning on line 537 in plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonqCodeTest/CodeWhispererUTGChatManager.kt

View workflow job for this annotation

GitHub Actions / qodana

Usage of redundant or deprecated syntax or deprecated symbols 

'message(String, vararg Any): String' is deprecated. Use extension-specific localization bundle instead

e is CodeTestException -> e.uiMessage

Check warning on line 539 in plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonqCodeTest/CodeWhispererUTGChatManager.kt

View workflow job for this annotation

GitHub Actions / qodana

Constant conditions 

Condition 'e is CodeTestException' is always true
e is JsonParseException -> message("testgen.error.generic_technical_error_message")
else -> e.message
}

if (e is CodeWhispererRuntimeException) {
errorMessage = message("testgen.error.maximum_generations_reach")
}
codeTestChatHelper.addAnswer(
CodeTestChatMessageContent(
message = errorMessage,
Expand All @@ -516,11 +556,12 @@
credentialStartUrl = getStartUrl(project),
jobGroup = session.testGenerationJobGroupName,
jobId = session.testGenerationJob,
result = if (e.message == message("testgen.message.cancelled")) MetricResult.Cancelled else MetricResult.Failed,

Check warning on line 559 in plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonqCodeTest/CodeWhispererUTGChatManager.kt

View workflow job for this annotation

GitHub Actions / qodana

Usage of redundant or deprecated syntax or deprecated symbols 

'message(String, vararg Any): String' is deprecated. Use extension-specific localization bundle instead
reason = e.javaClass.name,
reasonDesc = e.message,
reason = (e as CodeTestException).code ?: "DefaultError",
Copy link
Contributor

@rli rli Dec 13, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

why don't we just let it be null?

Copy link
Contributor Author

@laileni-aws laileni-aws Dec 13, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It can be but placing this under the bucket of DefaultError to be consistent in Kibana.

reasonDesc = if (e.message == message("testgen.message.cancelled")) "${e.code}: ${e.message}" else e.message,

Check warning on line 561 in plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonqCodeTest/CodeWhispererUTGChatManager.kt

View workflow job for this annotation

GitHub Actions / qodana

Usage of redundant or deprecated syntax or deprecated symbols 

'message(String, vararg Any): String' is deprecated. Use extension-specific localization bundle instead
httpStatusCode = e.statusCode ?: "400",
perfClientLatency = (Instant.now().toEpochMilli() - session.startTimeOfTestGeneration),
isCodeBlockSelected = session.isCodeBlockSelected,
artifactsUploadDuration = session.artifactUploadDuration,
buildPayloadBytes = session.srcPayloadSize,
buildZipFileBytes = session.srcZipFileSize,
Expand Down
3 changes: 2 additions & 1 deletion ...ware/aws/toolkits/jetbrains/services/amazonqCodeTest/controller/CodeTestChatController.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -291,7 +291,8 @@ class CodeTestChatController(
credentialStartUrl = getStartUrl(project),
result = MetricResult.Succeeded,
perfClientLatency = (Instant.now().toEpochMilli() - session.startTimeOfTestGeneration),
requestId = id
requestId = id,
httpStatusCode = "200"
)
}
session.isGeneratingTests = false
Expand Down
5 changes: 3 additions & 2 deletions .../software/aws/toolkits/jetbrains/services/codewhisperer/codescan/AmazonQCodeFixSession.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,8 @@ class AmazonQCodeFixSession(val project: Project) {
sourceZip,
"SourceCode",
CodeWhispererConstants.UploadTaskType.CODE_FIX,
codeFixName
codeFixName,
CodeWhispererConstants.FeatureName.CODE_REVIEW
)

/**
Expand Down Expand Up @@ -149,7 +150,7 @@ class AmazonQCodeFixSession(val project: Project) {
)
} catch (e: Exception) {
LOG.debug { "Create Upload URL failed: ${e.message}" }
val errorMessage = getTelemetryErrorMessage(e)
val errorMessage = getTelemetryErrorMessage(e, featureUseCase = CodeWhispererConstants.FeatureName.CODE_REVIEW)
throw codeScanServerException("CreateUploadUrlException: $errorMessage")
}

Expand Down
9 changes: 5 additions & 4 deletions ...re/aws/toolkits/jetbrains/services/codewhisperer/codescan/CodeWhispererCodeScanSession.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -117,7 +117,8 @@ class CodeWhispererCodeScanSession(val sessionContext: CodeScanSessionContext) {
sourceZip,
"SourceCode",
taskType,
codeScanName
codeScanName,
CodeWhispererConstants.FeatureName.CODE_REVIEW
)
if (isProjectScope()) {
LOG.debug {
Expand Down Expand Up @@ -272,7 +273,7 @@ class CodeWhispererCodeScanSession(val sessionContext: CodeScanSessionContext) {
)
} catch (e: Exception) {
LOG.debug { "Creating code review failed: ${e.message}" }
val errorMessage = getTelemetryErrorMessage(e)
val errorMessage = getTelemetryErrorMessage(e, featureUseCase = CodeWhispererConstants.FeatureName.CODE_REVIEW)
throw codeScanServerException(errorMessage)
}
}
Expand All @@ -285,7 +286,7 @@ class CodeWhispererCodeScanSession(val sessionContext: CodeScanSessionContext) {
)
} catch (e: Exception) {
LOG.debug { "Getting code review failed: ${e.message}" }
val errorMessage = getTelemetryErrorMessage(e)
val errorMessage = getTelemetryErrorMessage(e, featureUseCase = CodeWhispererConstants.FeatureName.CODE_REVIEW)
throw codeScanServerException("GetCodeReviewException: $errorMessage")
}

Expand All @@ -299,7 +300,7 @@ class CodeWhispererCodeScanSession(val sessionContext: CodeScanSessionContext) {
)
} catch (e: Exception) {
LOG.debug { "Listing code review failed: ${e.message}" }
val errorMessage = getTelemetryErrorMessage(e)
val errorMessage = getTelemetryErrorMessage(e, featureUseCase = CodeWhispererConstants.FeatureName.CODE_REVIEW)
throw codeScanServerException("ListCodeReviewFindingsException: $errorMessage")
}

Expand Down
10 changes: 5 additions & 5 deletions ...toolkits/jetbrains/services/codewhisperer/codetest/sessionconfig/CodeTestSessionConfig.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,13 +19,13 @@ import software.aws.toolkits.core.utils.debug
import software.aws.toolkits.core.utils.getLogger
import software.aws.toolkits.core.utils.putNextEntry
import software.aws.toolkits.jetbrains.services.amazonq.FeatureDevSessionContext
import software.aws.toolkits.jetbrains.services.codewhisperer.codescan.cannotFindBuildArtifacts
import software.aws.toolkits.jetbrains.services.codewhisperer.codescan.cannotFindFile
import software.aws.toolkits.jetbrains.services.codewhisperer.codescan.fileTooLarge
import software.aws.toolkits.jetbrains.services.codewhisperer.codescan.noFileOpenError
import software.aws.toolkits.jetbrains.services.codewhisperer.codescan.sessionconfig.Payload
import software.aws.toolkits.jetbrains.services.codewhisperer.codescan.sessionconfig.PayloadContext
import software.aws.toolkits.jetbrains.services.codewhisperer.codescan.sessionconfig.PayloadMetadata
import software.aws.toolkits.jetbrains.services.codewhisperer.codetest.cannotFindBuildArtifacts
import software.aws.toolkits.jetbrains.services.codewhisperer.codetest.cannotFindFile
import software.aws.toolkits.jetbrains.services.codewhisperer.codetest.fileTooLarge
import software.aws.toolkits.jetbrains.services.codewhisperer.codetest.noFileOpenError
import software.aws.toolkits.jetbrains.services.codewhisperer.language.CodeWhispererProgrammingLanguage
import software.aws.toolkits.jetbrains.services.codewhisperer.language.languages.CodeWhispererUnknownLanguage
import software.aws.toolkits.jetbrains.services.codewhisperer.language.programmingLanguage
Expand Down Expand Up @@ -94,7 +94,7 @@ class CodeTestSessionConfig(
else -> e.message
}
LOG.debug { "Error creating payload metadata: $errorMessage" }
throw cannotFindBuildArtifacts(errorMessage ?: message("codewhisperer.codescan.run_scan_error_telemetry"))
throw cannotFindBuildArtifacts(errorMessage ?: message("testgen.message.failed"))
}

// Copy all the included source files to the source zip
Expand Down
5 changes: 5 additions & 0 deletions ...src/software/aws/toolkits/jetbrains/services/codewhisperer/util/CodeWhispererConstants.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -127,6 +127,11 @@ object CodeWhispererConstants {
PROJECT("PROJECT"),
}

enum class FeatureName(val value: String) {
TEST_GENERATION("TEST_GENERATION"),
CODE_REVIEW("CODE_REVIEW"),
}

enum class UploadTaskType(val value: String) {
SCAN_FILE("SCAN_FILE"),
SCAN_PROJECT("SCAN_PROJECT"),
Expand Down
Loading
Loading