put ZIP contents under dependencies/

Author: David Hasani

plugins/amazonq/codetransform/jetbrains-community/src/software/aws/toolkits/jetbrains/services/codemodernizer/CodeModernizerManager.kt

@@ -687,6 +687,7 @@ class CodeModernizerManager(private val project: Project) : PersistentStateCompo
                 sourceVendor = customerSelection.sourceVendor,
                 targetVendor = customerSelection.targetVendor,
                 sourceServerName = customerSelection.sourceServerName,
+                sqlMetadataZip = customerSelection.sqlMetadataZip,
             ),
         )
     }

plugins/amazonq/codetransform/jetbrains-community/src/software/aws/toolkits/jetbrains/services/codemodernizer/CodeModernizerSession.kt

@@ -5,7 +5,6 @@ package software.aws.toolkits.jetbrains.services.codemodernizer
 
 import com.intellij.openapi.Disposable
 import com.intellij.openapi.application.runInEdt
-import com.intellij.openapi.projectRoots.JavaSdkVersion
 import com.intellij.serviceContainer.AlreadyDisposedException
 import com.intellij.util.io.HttpRequests
 import kotlinx.coroutines.delay
@@ -63,12 +62,7 @@ import java.util.concurrent.CancellationException
 import java.util.concurrent.atomic.AtomicBoolean
 import javax.net.ssl.SSLHandshakeException
 
-const val ZIP_SOURCES_PATH = "sources"
-const val BUILD_LOG_PATH = "build-logs.txt"
-const val UPLOAD_ZIP_MANIFEST_VERSION = 1.0F
 const val MAX_ZIP_SIZE = 2000000000 // 2GB
-const val HIL_1P_UPGRADE_CAPABILITY = "HIL_1pDependency_VersionUpgrade"
-const val EXPLAINABILITY_V1 = "EXPLAINABILITY_V1"
 
 // constants for handling SDKClientException
 const val CONNECTION_REFUSED_ERROR: String = "Connection refused"
@@ -274,7 +268,7 @@ class CodeModernizerSession(
         } finally {
             telemetry.uploadProject(payloadSize, startTime, true, telemetryErrorMessage)
             if (payload != null) {
-                // TODO: revert
+                // TODO: revert this
                 notifyStickyInfo("zip path", payload.path)
                 // deleteUploadArtifact(payload)
             }

plugins/amazonq/codetransform/jetbrains-community/src/software/aws/toolkits/jetbrains/services/codemodernizer/CodeTransformTelemetryManager.kt

@@ -145,7 +145,7 @@ class CodeTransformTelemetryManager(private val project: Project) {
     }
 
     fun getProjectHash(customerSelection: CustomerSelection) = Base64.getEncoder().encodeToString(
-        DigestUtils.sha256(customerSelection.configurationFile.toNioPath().toAbsolutePath().toString())
+        DigestUtils.sha256(customerSelection.configurationFile?.toNioPath()?.toAbsolutePath().toString())
     )
 
     /**

plugins/amazonq/codetransform/jetbrains-community/src/software/aws/toolkits/jetbrains/services/codemodernizer/controller/CodeTransformChatController.kt

@@ -11,6 +11,8 @@ import com.intellij.openapi.fileChooser.FileChooser
 import com.intellij.openapi.fileChooser.FileChooserDescriptorFactory
 import com.intellij.openapi.module.ModuleUtil
 import com.intellij.openapi.projectRoots.JavaSdkVersion
+import com.intellij.openapi.util.io.FileUtil.createTempDirectory
+import com.intellij.openapi.vfs.LocalFileSystem
 import com.intellij.openapi.vfs.VirtualFile
 import kotlinx.coroutines.CoroutineScope
 import kotlinx.coroutines.Dispatchers
@@ -102,10 +104,20 @@ import software.aws.toolkits.jetbrains.services.codemodernizer.utils.getModuleOr
 import software.aws.toolkits.jetbrains.services.codemodernizer.utils.isCodeTransformAvailable
 import software.aws.toolkits.jetbrains.services.codemodernizer.utils.toVirtualFile
 import software.aws.toolkits.jetbrains.services.codemodernizer.utils.tryGetJdk
+import software.aws.toolkits.jetbrains.services.codemodernizer.utils.unzipFile
+import software.aws.toolkits.jetbrains.services.codemodernizer.utils.validateSctMetadata
 import software.aws.toolkits.jetbrains.services.cwc.messages.ChatMessageType
 import software.aws.toolkits.jetbrains.utils.notifyStickyInfo
 import software.aws.toolkits.resources.message
 import software.aws.toolkits.telemetry.CodeTransformVCSViewerSrcComponents
+import java.io.BufferedReader
+import java.io.File
+import java.io.FileInputStream
+import java.io.InputStreamReader
+import java.nio.charset.StandardCharsets
+import java.util.zip.ZipFile
+import java.util.zip.ZipInputStream
+import kotlin.io.path.Path
 
 class CodeTransformChatController(
     private val context: AmazonQAppInitContext,
@@ -289,69 +301,11 @@ class CodeTransformChatController(
         }
     }
 
-    private fun validateSctMetadata(selectedFile: VirtualFile): SqlMetadataValidationResult {
-        val fileContent = selectedFile.contentsToByteArray().toString(Charsets.UTF_8)
-        val xmlDeserializer = XmlMapper(JacksonXmlModule())
-        var sctMetadata: Map<*, *>? = null
-        try {
-            sctMetadata = xmlDeserializer.readValue(fileContent, Any::class.java) as Map<*, *>
-        } catch (e: Exception) {
-            getLogger<CodeTransformChatController>().error { "Error parsing .sct metadata file; invalid XML encountered." }
-            return SqlMetadataValidationResult(false, "Invalid XML encountered.")
-        }
-
-        try {
-            val instances = sctMetadata["instances"] as Map<*, *>
-            val projectModel = instances["ProjectModel"] as Map<*, *>
-            val entities = projectModel["entities"] as Map<*, *>
-
-            val sources = entities["sources"] as Map<*, *>
-            val sourceDbServer = sources["DbServer"] as Map<*, *>
-            val sourceVendor = (sourceDbServer["vendor"] as String).trim().uppercase()
-            if (sourceVendor != "ORACLE") {
-                return SqlMetadataValidationResult(false, "Sorry, your .sct metadata file appears to be invalid; the source DB must be Oracle.")
-            }
-
-            val sourceServerName = (sourceDbServer["name"] as String).trim()
-
-            val targets = entities["targets"] as Map<*, *>
-            val targetDbServer = targets["DbServer"] as Map<*, *>
-            val targetVendor = (targetDbServer["vendor"] as String).trim().uppercase()
-            if (targetVendor != "AURORA_POSTGRESQL" && targetVendor != "RDS_POSTGRESQL") {
-                return SqlMetadataValidationResult(false, "Sorry, your .sct metadata file appears to be invalid; the target DB must be Aurora PostgreSQL or Amazon RDS for PostgreSQL.")
-            }
-
-            val relations = projectModel["relations"] as Map<*, *>
-            val serverNodeLocations = relations["server-node-location"] as List<Map<*, *>>
-            val schemaNames = mutableSetOf<String>()
-            for (serverNodeLocation in serverNodeLocations) {
-                val fullNameNodeInfoList = serverNodeLocation["FullNameNodeInfoList"] as Map<*, *>
-                val nameParts = fullNameNodeInfoList["nameParts"] as Map<*, *>
-                var fullNameNodeInfo = nameParts["FullNameNodeInfo"] // as List<Map<*, *>>
-                if (fullNameNodeInfo is Map<*, *>) {
-                    continue
-                } else {
-                    fullNameNodeInfo = fullNameNodeInfo as List<Map<*, *>>
-                }
-                fullNameNodeInfo.forEach{ node ->
-                    if ((node["typeNode"] as String).lowercase() == "schema") {
-                        schemaNames.add((node["nameNode"] as String).uppercase()) // user will choose one later
-                    }
-                }
-            }
-            // .sct metadata file is valid, return SqlMetadataValidationResult with all data we parsed
-            return SqlMetadataValidationResult(true, "", sourceVendor, targetVendor, sourceServerName, schemaNames)
-        } catch (e: Exception) {
-            getLogger<CodeTransformChatController>().error { "Error parsing .sct metadata file: $e" }
-            return SqlMetadataValidationResult(false, "Sorry, the .sct metadata file you provided appears to be invalid.")
-        }
-    }
-
     override suspend fun processCodeTransformSelectSQLModuleSchemaAction(message: IncomingCodeTransformMessage.CodeTransformSelectSQLModuleSchema) {
-        val moduleName = context.project.getModuleOrProjectNameForFile(message.modulePath.toVirtualFile() as VirtualFile)
+        val moduleName = context.project.getModuleOrProjectNameForFile(message.modulePath.toVirtualFile())
         codeTransformChatHelper.addNewMessage(buildUserSQLConversionSelectionSummaryChatContent(moduleName, message.schema))
         codeModernizerManager.codeTransformationSession?.let {
-            it.sessionContext.configurationFile = message.modulePath.toVirtualFile() as VirtualFile // any file in the selected module works fine; will use to createZip
+            it.sessionContext.configurationFile = message.modulePath.toVirtualFile()
             it.sessionContext.schema = message.schema
         }
         // start the SQL conversion
@@ -363,12 +317,18 @@ class CodeTransformChatController(
     override suspend fun processCodeTransformSelectSQLMetadataAction(message: IncomingCodeTransformMessage.CodeTransformSelectSQLMetadata) {
         runInEdt {
             val descriptor = FileChooserDescriptorFactory.createSingleFileDescriptor()
-                .withDescription("Select .sct metadata file")
-                .withFileFilter { it.extension == "sct" }
+                .withDescription("Select metadata ZIP file")
+                .withFileFilter { it.extension == "zip" }
+
+            val selectedZipFile = FileChooser.chooseFile(descriptor, null, null) ?: return@runInEdt
+            val extractedZip = createTempDirectory("codeTransformSQLMetadata", null) // .path.toVirtualFile()
+            unzipFile(selectedZipFile.toNioPath(), extractedZip.toPath())
+
+            notifyStickyInfo("tempDir", extractedZip.path)
 
-            val selectedFile = FileChooser.chooseFile(descriptor, null, null) ?: return@runInEdt
+            val sctFile = extractedZip.listFiles { file -> file.name.endsWith(".sct") }.firstOrNull()
 
-            val metadataValidationResult = this.validateSctMetadata(selectedFile)
+            val metadataValidationResult = validateSctMetadata(sctFile)
 
             if (!metadataValidationResult.valid) {
                 CoroutineScope(Dispatchers.Main).launch {
@@ -395,7 +355,7 @@ class CodeTransformChatController(
                     sourceVendor = metadataValidationResult.sourceVendor,
                     targetVendor = metadataValidationResult.targetVendor,
                     sourceServerName = metadataValidationResult.sourceServerName,
-                    configurationFile = selectedFile
+                    sqlMetadataZip = extractedZip,
                 )
                 codeModernizerManager.createCodeModernizerSession(selection, context.project)
             }

plugins/amazonq/codetransform/jetbrains-community/src/software/aws/toolkits/jetbrains/services/codemodernizer/model/CodeModernizerSessionContext.kt

@@ -44,6 +44,9 @@ const val MANIFEST_PATH = "manifest.json"
 const val ZIP_SOURCES_PATH = "sources"
 const val ZIP_DEPENDENCIES_PATH = "dependencies"
 const val BUILD_LOG_PATH = "build-logs.txt"
+const val UPLOAD_ZIP_MANIFEST_VERSION = "1.0"
+const val HIL_1P_UPGRADE_CAPABILITY = "HIL_1pDependency_VersionUpgrade"
+const val EXPLAINABILITY_V1 = "EXPLAINABILITY_V1"
 const val MAVEN_CONFIGURATION_FILE_NAME = "pom.xml"
 const val MAVEN_BUILD_RUN_UNIT_TESTS = "clean test"
 const val MAVEN_BUILD_SKIP_UNIT_TESTS = "clean test-compile"
@@ -53,14 +56,15 @@ const val INVALID_SUFFIX_SHA = "sha1"
 const val INVALID_SUFFIX_REPOSITORIES = "repositories"
 data class CodeModernizerSessionContext(
     val project: Project,
-    var configurationFile: VirtualFile, // always needed to ZIP module
+    var configurationFile: VirtualFile? = null, // used to ZIP module
     val sourceJavaVersion: JavaSdkVersion, // always needed for startJob API
     val targetJavaVersion: JavaSdkVersion = JavaSdkVersion.JDK_17, // only one supported
     var customBuildCommand: String = MAVEN_BUILD_RUN_UNIT_TESTS, // run unit tests by default
-    var sourceVendor: String = "ORACLE", // only one supported
-    var targetVendor: String? = null,
-    var sourceServerName: String? = null,
+    val sourceVendor: String = "ORACLE", // only one supported
+    val targetVendor: String? = null,
+    val sourceServerName: String? = null,
     var schema: String? = null,
+    val sqlMetadataZip: File? = null,
 ) {
     private val mapper = jacksonObjectMapper()
     private val ignoredDependencyFileExtensions = setOf(INVALID_SUFFIX_SHA, INVALID_SUFFIX_REPOSITORIES)
@@ -180,23 +184,22 @@ data class CodeModernizerSessionContext(
         }
 
     fun createZipWithModuleFiles(copyResult: MavenCopyCommandsResult?): ZipCreationResult {
-        val root = configurationFile.parent
-        val sourceFolder = File(root.path)
+        val root = configurationFile?.parent
+        val sourceFolder = File(root?.path)
         val buildLogBuilder = StringBuilder("Starting Build Log...\n")
-        // depDirectory will be null for SQL conversions since copyResult will be null
         val depDirectory = if (copyResult is MavenCopyCommandsResult.Success) {
             showTransformationHub()
             copyResult.dependencyDirectory
-        } else {
-            null
-        }
+        } else sqlMetadataZip // null copyResult means doing a SQL conversion
 
         return runReadAction {
             try {
                 val directoriesToExclude = findDirectoriesToExclude(sourceFolder)
-                val files = VfsUtil.collectChildrenRecursively(root).filter { child ->
-                    val childPath = Path(child.path)
-                    !child.isDirectory && directoriesToExclude.none { childPath.startsWith(it.toPath()) }
+                val files = root?.let {
+                    VfsUtil.collectChildrenRecursively(it).filter { child ->
+                        val childPath = Path(child.path)
+                        !child.isDirectory && directoriesToExclude.none { childPath.startsWith(it.toPath()) }
+                    }
                 }
                 val dependencyFiles = if (depDirectory != null) {
                     iterateThroughDependencies(depDirectory)
@@ -208,22 +211,24 @@ data class CodeModernizerSessionContext(
                 val depSources = File(ZIP_DEPENDENCIES_PATH)
                 val outputFile = createTemporaryZipFile { zip ->
                     // 1) Manifest file
-                    val dependenciesRoot = if (depDirectory != null) "$ZIP_DEPENDENCIES_PATH/${depDirectory.name}" else null
-                    var manifest = ZipManifest(dependenciesRoot = dependenciesRoot, customBuildCommand = customBuildCommand)
-                    if (schema != null) {
+                    // val dependenciesRoot = if (depDirectory != null) "$ZIP_DEPENDENCIES_PATH/${depDirectory.name}" else null
+                    var manifest = ZipManifest(customBuildCommand = customBuildCommand)
+                    if (sqlMetadataZip != null) {
                         // doing a SQL conversion, not language upgrade
-                        manifest = ZipManifest(sourceVendor = sourceVendor, targetVendor = targetVendor, sourceServerName = sourceServerName, schema = schema)
+                        // sctFileName below will not work since configurationFile points to any file in the Java module, not the metadata ZIP
+                        manifest = ZipManifest(requestedConversions = RequestedConversions(sqlConversion = SQLConversion(source = sourceVendor, target = targetVendor, schema = schema, host = sourceServerName, sctFileName = sqlMetadataZip.listFiles { file -> file.name.endsWith(".sct") }.first().name)))
                     }
                     mapper.writeValueAsString(manifest)
                         .byteInputStream()
                         .use {
                             zip.putNextEntry(Path(MANIFEST_PATH).toString(), it)
                         }
 
-                    // 2) Dependencies
+                    // 2) Dependencies / SQL conversion metadata
                     if (depDirectory != null) {
+                        notifyStickyInfo("depDirectory", "depDirectory is NOT null")
                         dependencyFiles.forEach { depFile ->
-                            val relativePath = File(depFile.path).relativeTo(depDirectory.parentFile)
+                            val relativePath = File(depFile.path).relativeTo(depDirectory)
                             val paddedPath = depSources.resolve(relativePath)
                             var paddedPathString = paddedPath.toPath().toString()
                             // Convert Windows file path to work on Linux
@@ -239,7 +244,7 @@ data class CodeModernizerSessionContext(
                     LOG.info { "Dependency files size = ${dependencyFiles.sumOf { it.length().toInt() }}" }
 
                     // 3) Sources
-                    files.forEach { file ->
+                    files?.forEach { file ->
                         val relativePath = File(file.path).relativeTo(sourceFolder)
                         val paddedPath = zipSources.resolve(relativePath)
                         var paddedPathString = paddedPath.toPath().toString()

plugins/amazonq/codetransform/jetbrains-community/src/software/aws/toolkits/jetbrains/services/codemodernizer/model/CustomerSelection.kt

@@ -5,13 +5,15 @@ package software.aws.toolkits.jetbrains.services.codemodernizer.model
 
 import com.intellij.openapi.projectRoots.JavaSdkVersion
 import com.intellij.openapi.vfs.VirtualFile
+import java.io.File
 
 data class CustomerSelection(
-    val configurationFile: VirtualFile, // always needed to ZIP module
+    val configurationFile: VirtualFile? = null, // used to ZIP module
     val sourceJavaVersion: JavaSdkVersion, // always needed, use default of JDK_8 for SQL conversions for startJob API call
     val targetJavaVersion: JavaSdkVersion = JavaSdkVersion.JDK_17,
     val sourceVendor: String = "ORACLE", // only one supported
     val targetVendor: String? = null,
     val sourceServerName: String? = null,
+    val sqlMetadataZip: File? = null,
     // note: schema and customBuildCommand are passed in to CodeModernizerSessionContext separately, *after* CodeModernizerSession is created
 )

plugins/amazonq/codetransform/jetbrains-community/src/software/aws/toolkits/jetbrains/services/codemodernizer/model/JobHistoryItem.kt

@@ -7,9 +7,9 @@ import java.time.Instant
 import kotlin.time.Duration.Companion.seconds
 import kotlin.time.toKotlinDuration
 
-data class JobHistoryItem(val moduleName: String, val status: String, val startTime: Instant, val runTime: java.time.Duration, val jobId: String) {
+data class JobHistoryItem(val moduleName: String?, val status: String, val startTime: Instant, val runTime: java.time.Duration, val jobId: String) {
     operator fun get(col: Int): Any = when (col) {
-        0 -> moduleName
+        0 -> moduleName ?: ""
         1 -> status
         2 -> startTime
         3 -> runTime.toKotlinDuration().inWholeSeconds.seconds.toString()

plugins/amazonq/codetransform/jetbrains-community/src/software/aws/toolkits/jetbrains/services/codemodernizer/model/ZipManifest.kt

@@ -3,22 +3,25 @@
 
 package software.aws.toolkits.jetbrains.services.codemodernizer.model
 
-import software.aws.toolkits.jetbrains.services.codemodernizer.BUILD_LOG_PATH
-import software.aws.toolkits.jetbrains.services.codemodernizer.EXPLAINABILITY_V1
-import software.aws.toolkits.jetbrains.services.codemodernizer.HIL_1P_UPGRADE_CAPABILITY
-import software.aws.toolkits.jetbrains.services.codemodernizer.UPLOAD_ZIP_MANIFEST_VERSION
-import software.aws.toolkits.jetbrains.services.codemodernizer.ZIP_SOURCES_PATH
-
 data class ZipManifest(
     val sourcesRoot: String = ZIP_SOURCES_PATH,
-    val dependenciesRoot: String? = null,
+    val dependenciesRoot: String = ZIP_DEPENDENCIES_PATH,
     val buildLogs: String = BUILD_LOG_PATH,
-    val version: String = UPLOAD_ZIP_MANIFEST_VERSION.toString(),
+    val version: String = UPLOAD_ZIP_MANIFEST_VERSION,
     val hilCapabilities: List<String> = listOf(HIL_1P_UPGRADE_CAPABILITY),
     val transformCapabilities: List<String> = listOf(EXPLAINABILITY_V1),
     val customBuildCommand: String = MAVEN_BUILD_RUN_UNIT_TESTS,
-    val sourceVendor: String? = null,
-    val targetVendor: String? = null,
-    val sourceServerName: String? = null,
+    val requestedConversions: RequestedConversions = RequestedConversions(sqlConversion = null),
+)
+
+data class RequestedConversions(
+    val sqlConversion: SQLConversion? = null,
+)
+
+data class SQLConversion(
+    val source: String? = null,
+    val target: String? = null,
     val schema: String? = null,
+    val host: String? = null,
+    val sctFileName: String? = null,
 )