diff --git a/.changes/next-release/bugfix-552a4676-3214-4321-a332-344bdf9f1830.json b/.changes/next-release/bugfix-552a4676-3214-4321-a332-344bdf9f1830.json new file mode 100644 index 00000000000..e22e1e5b2ea --- /dev/null +++ b/.changes/next-release/bugfix-552a4676-3214-4321-a332-344bdf9f1830.json @@ -0,0 +1,4 @@ +{ + "type" : "bugfix", + "description" : "Amazon Q Code Transformation: retry initial project upload on failure" +} \ No newline at end of file diff --git a/plugins/amazonq/codetransform/jetbrains-community/src/software/aws/toolkits/jetbrains/services/codemodernizer/CodeModernizerSession.kt b/plugins/amazonq/codetransform/jetbrains-community/src/software/aws/toolkits/jetbrains/services/codemodernizer/CodeModernizerSession.kt index e791f298a98..973ac136ed5 100644 --- a/plugins/amazonq/codetransform/jetbrains-community/src/software/aws/toolkits/jetbrains/services/codemodernizer/CodeModernizerSession.kt +++ b/plugins/amazonq/codetransform/jetbrains-community/src/software/aws/toolkits/jetbrains/services/codemodernizer/CodeModernizerSession.kt @@ -8,6 +8,7 @@ import com.intellij.openapi.application.runInEdt import com.intellij.serviceContainer.AlreadyDisposedException import com.intellij.util.io.HttpRequests import kotlinx.coroutines.delay +import kotlinx.coroutines.withContext import org.apache.commons.codec.digest.DigestUtils import software.amazon.awssdk.core.exception.SdkClientException import software.amazon.awssdk.services.codewhispererruntime.model.ResumeTransformationResponse @@ -20,11 +21,13 @@ import software.amazon.awssdk.services.codewhispererruntime.model.Transformation import software.amazon.awssdk.services.codewhispererruntime.model.TransformationUserActionStatus import software.amazon.awssdk.services.codewhispererstreaming.model.TransformationDownloadArtifactType import software.amazon.awssdk.services.ssooidc.model.SsoOidcException +import software.aws.toolkits.core.utils.Waiters.waitUntil import software.aws.toolkits.core.utils.error import software.aws.toolkits.core.utils.exists import software.aws.toolkits.core.utils.getLogger import software.aws.toolkits.core.utils.info import software.aws.toolkits.core.utils.warn +import software.aws.toolkits.jetbrains.core.coroutines.getCoroutineBgContext import software.aws.toolkits.jetbrains.services.codemodernizer.client.GumbyClient import software.aws.toolkits.jetbrains.services.codemodernizer.commands.CodeTransformMessageListener import software.aws.toolkits.jetbrains.services.codemodernizer.model.CodeModernizerException @@ -55,6 +58,8 @@ import java.io.File import java.io.FileInputStream import java.io.IOException import java.net.ConnectException +import java.net.SocketTimeoutException +import java.net.UnknownHostException import java.nio.file.Path import java.time.Instant import java.util.Base64 @@ -142,7 +147,7 @@ class CodeModernizerSession( * * Based on [CodeWhispererCodeScanSession] */ - fun createModernizationJob(copyResult: MavenCopyCommandsResult?): CodeModernizerStartJobResult { + suspend fun createModernizationJob(copyResult: MavenCopyCommandsResult?): CodeModernizerStartJobResult { LOG.info { "Compressing local project" } val payload: File? var payloadSize = 0 @@ -377,8 +382,12 @@ class CodeModernizerSession( /** * Adapted from [CodeWhispererCodeScanSession] */ - fun uploadPayload(payload: File): String { - val sha256checksum: String = Base64.getEncoder().encodeToString(DigestUtils.sha256(FileInputStream(payload))) + suspend fun uploadPayload(payload: File): String { + val sha256checksum: String = Base64.getEncoder().encodeToString( + withContext(getCoroutineBgContext()) { + DigestUtils.sha256(FileInputStream(payload)) + } + ) if (isDisposed.get()) { throw AlreadyDisposedException("Disposed when about to create upload URL") } @@ -394,17 +403,22 @@ class CodeModernizerSession( throw AlreadyDisposedException("Disposed when about to upload project artifact to s3") } val uploadStartTime = Instant.now() - try { + waitUntil( + exceptionsToIgnore = setOf( + UnknownHostException::class, + SocketTimeoutException::class, + HttpRequests.HttpStatusException::class, + ConnectException::class + ) + ) { clientAdaptor.uploadArtifactToS3( createUploadUrlResponse.uploadUrl(), payload, sha256checksum, createUploadUrlResponse.kmsKeyArn().orEmpty(), ) { shouldStop.get() } - } catch (e: Exception) { - LOG.error { "Unexpected error when uploading project artifact to S3: $e" } - throw e // pass along error to callee } + LOG.info { "Upload to S3 succeeded" } if (!shouldStop.get()) { LOG.info { "Uploaded artifact. Latency: ${calculateTotalLatency(uploadStartTime, Instant.now())}ms" } } diff --git a/plugins/amazonq/codetransform/jetbrains-community/src/software/aws/toolkits/jetbrains/services/codemodernizer/client/GumbyClient.kt b/plugins/amazonq/codetransform/jetbrains-community/src/software/aws/toolkits/jetbrains/services/codemodernizer/client/GumbyClient.kt index 2a92dc33d04..097b9231d4a 100644 --- a/plugins/amazonq/codetransform/jetbrains-community/src/software/aws/toolkits/jetbrains/services/codemodernizer/client/GumbyClient.kt +++ b/plugins/amazonq/codetransform/jetbrains-community/src/software/aws/toolkits/jetbrains/services/codemodernizer/client/GumbyClient.kt @@ -148,6 +148,7 @@ class GumbyClient(private val project: Project) { var result: CodeWhispererRuntimeResponse? = null try { result = apiCall() + LOG.info { "$apiName request ID: ${result.responseMetadata()?.requestId()}" } return result } catch (e: Exception) { LOG.error(e) { "$apiName failed: ${e.message}" } diff --git a/plugins/amazonq/codetransform/jetbrains-community/tst/software/aws/toolkits/jetbrains/services/codemodernizer/CodeWhispererCodeModernizerSessionTest.kt b/plugins/amazonq/codetransform/jetbrains-community/tst/software/aws/toolkits/jetbrains/services/codemodernizer/CodeWhispererCodeModernizerSessionTest.kt index a05fb0f6124..bf0edb78620 100644 --- a/plugins/amazonq/codetransform/jetbrains-community/tst/software/aws/toolkits/jetbrains/services/codemodernizer/CodeWhispererCodeModernizerSessionTest.kt +++ b/plugins/amazonq/codetransform/jetbrains-community/tst/software/aws/toolkits/jetbrains/services/codemodernizer/CodeWhispererCodeModernizerSessionTest.kt @@ -410,7 +410,7 @@ class CodeWhispererCodeModernizerSessionTest : CodeWhispererCodeModernizerTestBa } @Test - fun `CodeModernizer can create modernization job`() { + fun `CodeModernizer can create modernization job`() = runTest { doReturn(ZipCreationResult.Succeeded(File("./tst-resources/codemodernizer/test.txt"))) .whenever(testSessionContextSpy).createZipWithModuleFiles(any()) doReturn(exampleCreateUploadUrlResponse).whenever(clientAdaptorSpy).createGumbyUploadUrl(any()) @@ -425,7 +425,7 @@ class CodeWhispererCodeModernizerSessionTest : CodeWhispererCodeModernizerTestBa } @Test - fun `CodeModernizer cannot upload payload due to already disposed`() { + fun `CodeModernizer cannot upload payload due to already disposed`() = runTest { doReturn(ZipCreationResult.Succeeded(File("./tst-resources/codemodernizer/test.txt"))) .whenever(testSessionContextSpy).createZipWithModuleFiles(any()) doReturn(exampleCreateUploadUrlResponse).whenever(clientAdaptorSpy).createGumbyUploadUrl(any()) @@ -435,7 +435,7 @@ class CodeWhispererCodeModernizerSessionTest : CodeWhispererCodeModernizerTestBa } @Test - fun `CodeModernizer returns credentials expired when SsoOidcException during upload`() { + fun `CodeModernizer returns credentials expired when SsoOidcException during upload`() = runTest { setupConnection(BearerTokenAuthState.AUTHORIZED) doReturn(ZipCreationResult.Succeeded(File("./tst-resources/codemodernizer/test.txt"))) .whenever(testSessionContextSpy).createZipWithModuleFiles(any()) @@ -445,7 +445,7 @@ class CodeWhispererCodeModernizerSessionTest : CodeWhispererCodeModernizerTestBa } @Test - fun `CodeModernizer returns credentials expired when expired before upload`() { + fun `CodeModernizer returns credentials expired when expired before upload`() = runTest { listOf(BearerTokenAuthState.NEEDS_REFRESH, BearerTokenAuthState.NOT_AUTHENTICATED).forEach { setupConnection(it) val result = testSessionSpy.createModernizationJob(MavenCopyCommandsResult.Success(File("./mock/path/"))) @@ -454,12 +454,11 @@ class CodeWhispererCodeModernizerSessionTest : CodeWhispererCodeModernizerTestBa } @Test - fun `CodeModernizer cannot upload payload due to presigned url issue`() { + fun `CodeModernizer cannot upload payload due to presigned url issue`() = runTest { doReturn(ZipCreationResult.Succeeded(File("./tst-resources/codemodernizer/test.txt"))) .whenever(testSessionContextSpy).createZipWithModuleFiles(any()) doReturn(exampleCreateUploadUrlResponse).whenever(clientAdaptorSpy).createGumbyUploadUrl(any()) - doAnswer { throw HttpRequests.HttpStatusException("mock error", 403, "mock url") } - .whenever(clientAdaptorSpy).uploadArtifactToS3(any(), any(), any(), any(), any()) + doAnswer { throw HttpRequests.HttpStatusException("mock error", 403, "mock url") }.whenever(testSessionSpy).uploadPayload(any()) val result = testSessionSpy.createModernizationJob(MavenCopyCommandsResult.Success(File("./mock/path/"))) assertEquals(CodeModernizerStartJobResult.ZipUploadFailed(UploadFailureReason.PRESIGNED_URL_EXPIRED), result) verify(testSessionStateSpy, times(1)).putJobHistory(any(), eq(TransformationStatus.FAILED), any(), any()) @@ -467,12 +466,11 @@ class CodeWhispererCodeModernizerSessionTest : CodeWhispererCodeModernizerTestBa } @Test - fun `CodeModernizer cannot upload payload due to other status code`() { + fun `CodeModernizer cannot upload payload due to other status code`() = runTest { doReturn(ZipCreationResult.Succeeded(File("./tst-resources/codemodernizer/test.txt"))) .whenever(testSessionContextSpy).createZipWithModuleFiles(any()) doReturn(exampleCreateUploadUrlResponse).whenever(clientAdaptorSpy).createGumbyUploadUrl(any()) - doAnswer { throw HttpRequests.HttpStatusException("mock error", 407, "mock url") } - .whenever(clientAdaptorSpy).uploadArtifactToS3(any(), any(), any(), any(), any()) + doAnswer { throw HttpRequests.HttpStatusException("mock error", 407, "mock url") }.whenever(testSessionSpy).uploadPayload(any()) val result = testSessionSpy.createModernizationJob(MavenCopyCommandsResult.Success(File("./mock/path/"))) assertEquals(CodeModernizerStartJobResult.ZipUploadFailed(UploadFailureReason.HTTP_ERROR(407)), result) verify(testSessionStateSpy, times(1)).putJobHistory(any(), eq(TransformationStatus.FAILED), any(), any()) @@ -480,7 +478,7 @@ class CodeWhispererCodeModernizerSessionTest : CodeWhispererCodeModernizerTestBa } @Test - fun `CodeModernizer cannot upload payload due to unknown issue`() { + fun `CodeModernizer cannot upload payload due to unknown issue`() = runTest { doReturn(ZipCreationResult.Succeeded(File("./tst-resources/codemodernizer/test.txt"))) .whenever(testSessionContextSpy).createZipWithModuleFiles(any()) doReturn(exampleCreateUploadUrlResponse).whenever(clientAdaptorSpy).createGumbyUploadUrl(any()) @@ -492,11 +490,11 @@ class CodeWhispererCodeModernizerSessionTest : CodeWhispererCodeModernizerTestBa } @Test - fun `CodeModernizer cannot upload payload due to connection refused`() { + fun `CodeModernizer cannot upload payload due to connection refused`() = runTest { doReturn(ZipCreationResult.Succeeded(File("./tst-resources/codemodernizer/test.txt"))) .whenever(testSessionContextSpy).createZipWithModuleFiles(any()) doReturn(exampleCreateUploadUrlResponse).whenever(clientAdaptorSpy).createGumbyUploadUrl(any()) - doAnswer { throw ConnectException("mock exception") }.whenever(clientAdaptorSpy).uploadArtifactToS3(any(), any(), any(), any(), any()) + doAnswer { throw ConnectException("mock exception") }.whenever(testSessionSpy).uploadPayload(any()) val result = testSessionSpy.createModernizationJob(MavenCopyCommandsResult.Success(File("./mock/path/"))) assertEquals(CodeModernizerStartJobResult.ZipUploadFailed(UploadFailureReason.CONNECTION_REFUSED), result) verify(testSessionStateSpy, times(1)).putJobHistory(any(), eq(TransformationStatus.FAILED), any(), any()) @@ -549,7 +547,7 @@ class CodeWhispererCodeModernizerSessionTest : CodeWhispererCodeModernizerTestBa } @Test - fun `test uploadPayload()`() { + fun `test uploadPayload()`() = runTest { val s3endpoint = "http://127.0.0.1:${wireMock.port()}" val gumbyUploadUrlResponse = CreateUploadUrlResponse.builder() .uploadUrl(s3endpoint)