diff --git a/plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonq/startup/AmazonQStartupActivity.kt b/plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonq/startup/AmazonQStartupActivity.kt index b85d94db10f..0f4aca90238 100644 --- a/plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonq/startup/AmazonQStartupActivity.kt +++ b/plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonq/startup/AmazonQStartupActivity.kt @@ -6,14 +6,8 @@ package software.aws.toolkits.jetbrains.services.amazonq.startup import com.intellij.openapi.application.ApplicationManager import com.intellij.openapi.application.runInEdt import com.intellij.openapi.project.Project -import com.intellij.openapi.project.waitForSmartMode import com.intellij.openapi.startup.ProjectActivity import com.intellij.openapi.wm.ToolWindowManager -import kotlinx.coroutines.TimeoutCancellationException -import kotlinx.coroutines.delay -import kotlinx.coroutines.time.withTimeout -import software.aws.toolkits.core.utils.getLogger -import software.aws.toolkits.core.utils.warn import software.aws.toolkits.jetbrains.core.credentials.AwsBearerTokenConnection import software.aws.toolkits.jetbrains.core.credentials.ToolkitConnectionManager import software.aws.toolkits.jetbrains.core.credentials.pinning.QConnection @@ -21,14 +15,11 @@ import software.aws.toolkits.jetbrains.core.gettingstarted.emitUserState import software.aws.toolkits.jetbrains.services.amazonq.CodeWhispererFeatureConfigService import software.aws.toolkits.jetbrains.services.amazonq.lsp.AmazonQLspService import software.aws.toolkits.jetbrains.services.amazonq.profile.QRegionProfileManager -import software.aws.toolkits.jetbrains.services.amazonq.project.ProjectContextController import software.aws.toolkits.jetbrains.services.amazonq.toolwindow.AmazonQToolWindow import software.aws.toolkits.jetbrains.services.amazonq.toolwindow.AmazonQToolWindowFactory import software.aws.toolkits.jetbrains.services.codewhisperer.explorer.CodeWhispererExplorerActionManager import software.aws.toolkits.jetbrains.services.cwc.inline.InlineChatController import software.aws.toolkits.jetbrains.settings.CodeWhispererSettings -import java.lang.management.ManagementFactory -import java.time.Duration import java.util.concurrent.atomic.AtomicBoolean class AmazonQStartupActivity : ProjectActivity { @@ -58,39 +49,8 @@ class AmazonQStartupActivity : ProjectActivity { QRegionProfileManager.getInstance().validateProfile(project) AmazonQLspService.getInstance(project) - startLsp(project) if (runOnce.get()) return emitUserState(project) runOnce.set(true) } - - private suspend fun startLsp(project: Project) { - // Automatically start the project context LSP after some delay when average CPU load is below 30%. - // The CPU load requirement is to avoid competing with native JetBrains indexing and other CPU expensive OS processes - // In the future we will decouple LSP start and indexing start to let LSP perform other tasks. - val startLspIndexingDuration = Duration.ofMinutes(30) - project.waitForSmartMode() - delay(30_000) // Wait for 30 seconds for systemLoadAverage to be more accurate - try { - withTimeout(startLspIndexingDuration) { - while (true) { - val cpuUsage = ManagementFactory.getOperatingSystemMXBean().systemLoadAverage - if (cpuUsage > 0 && cpuUsage < 30) { - ProjectContextController.getInstance(project = project) - break - } else { - delay(60_000) // Wait for 60 seconds - } - } - } - } catch (e: TimeoutCancellationException) { - LOG.warn { "Failed to start LSP server due to time out" } - } catch (e: Exception) { - LOG.warn { "Failed to start LSP server" } - } - } - - companion object { - private val LOG = getLogger() - } } diff --git a/plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/cwc/controller/ChatController.kt b/plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/cwc/controller/ChatController.kt index c8d69e124ab..c061f8be1c3 100644 --- a/plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/cwc/controller/ChatController.kt +++ b/plugins/amazonq/chat/jetbrains-community/src/software/aws/toolkits/jetbrains/services/cwc/controller/ChatController.kt @@ -42,7 +42,6 @@ import software.aws.toolkits.jetbrains.services.amazonq.messages.AmazonQMessage import software.aws.toolkits.jetbrains.services.amazonq.messages.MessagePublisher import software.aws.toolkits.jetbrains.services.amazonq.onboarding.OnboardingPageInteraction import software.aws.toolkits.jetbrains.services.amazonq.onboarding.OnboardingPageInteractionType -import software.aws.toolkits.jetbrains.services.amazonq.project.ProjectContextController import software.aws.toolkits.jetbrains.services.amazonq.project.RelevantDocument import software.aws.toolkits.jetbrains.services.codewhisperer.settings.CodeWhispererConfigurable import software.aws.toolkits.jetbrains.services.cwc.InboundAppMessagesHandler @@ -72,10 +71,8 @@ import software.aws.toolkits.jetbrains.services.cwc.messages.FocusType import software.aws.toolkits.jetbrains.services.cwc.messages.FollowUp import software.aws.toolkits.jetbrains.services.cwc.messages.IncomingCwcMessage import software.aws.toolkits.jetbrains.services.cwc.messages.OnboardingPageInteractionMessage -import software.aws.toolkits.jetbrains.services.cwc.messages.OpenSettingsMessage import software.aws.toolkits.jetbrains.services.cwc.messages.QuickActionMessage import software.aws.toolkits.jetbrains.services.cwc.storage.ChatSessionStorage -import software.aws.toolkits.jetbrains.settings.CodeWhispererSettings import software.aws.toolkits.telemetry.CwsprChatCommandType import java.util.UUID @@ -127,24 +124,11 @@ class ChatController private constructor( } override suspend fun processPromptChatMessage(message: IncomingCwcMessage.ChatPrompt) { - var prompt = message.chatMessage - var queryResult: List = emptyList() + val prompt = message.chatMessage + val queryResult: List = emptyList() val triggerId = UUID.randomUUID().toString() - var shouldAddIndexInProgressMessage: Boolean = false - var shouldUseWorkspaceContext: Boolean = false - - if (prompt.contains("@workspace")) { - if (CodeWhispererSettings.getInstance().isProjectContextEnabled()) { - shouldUseWorkspaceContext = true - prompt = prompt.replace("@workspace", "") - val projectContextController = ProjectContextController.getInstance(context.project) - queryResult = projectContextController.queryChat(prompt, timeout = null) - if (!projectContextController.getProjectContextIndexComplete()) shouldAddIndexInProgressMessage = true - logger.info { "project context relevant document count: ${queryResult.size}" } - } else { - sendOpenSettingsMessage(message.tabId) - } - } + val shouldAddIndexInProgressMessage = false + val shouldUseWorkspaceContext = false handleChat( tabId = message.tabId, @@ -467,13 +451,6 @@ class ChatController private constructor( messagePublisher.publish(message) } - private suspend fun sendOpenSettingsMessage(tabId: String) { - val message = OpenSettingsMessage( - tabId = tabId - ) - messagePublisher.publish(message) - } - private suspend fun sendStaticTextResponse(tabId: String, triggerId: String, response: StaticTextResponse) { val chatMessage = ChatMessage( tabId = tabId, diff --git a/plugins/amazonq/chat/jetbrains-community/tst/software/aws/toolkits/jetbrains/services/amazonq/workspace/context/EncoderServerTest.kt b/plugins/amazonq/chat/jetbrains-community/tst/software/aws/toolkits/jetbrains/services/amazonq/workspace/context/EncoderServerTest.kt deleted file mode 100644 index 7fd76f2c044..00000000000 --- a/plugins/amazonq/chat/jetbrains-community/tst/software/aws/toolkits/jetbrains/services/amazonq/workspace/context/EncoderServerTest.kt +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright 2024 Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: Apache-2.0 - -package software.aws.toolkits.jetbrains.services.amazonq.workspace.context - -import com.intellij.util.io.DigestUtil -import org.apache.commons.codec.digest.DigestUtils -import org.assertj.core.api.Assertions.assertThat -import org.junit.Before -import org.junit.Rule -import org.junit.Test -import software.aws.toolkits.jetbrains.services.amazonq.project.EncoderServer -import software.aws.toolkits.jetbrains.utils.rules.CodeInsightTestFixtureRule -import software.aws.toolkits.jetbrains.utils.rules.JavaCodeInsightTestFixtureRule -import java.math.BigInteger - -class EncoderServerTest { - @Rule @JvmField - val projectRule: CodeInsightTestFixtureRule = JavaCodeInsightTestFixtureRule() - private lateinit var encoderServer: EncoderServer - private val inputBytes = BigInteger(32, DigestUtil.random).toByteArray() - - @Before - fun setup() { - encoderServer = EncoderServer(projectRule.project) - } - - @Test - fun `test download artifacts validate hash if it does not match`() { - val wrongHash = "sha384:ad527e9583d3dc4be3d302bac17f8d5a64eb8f5ab536717982620232e4e4bad82d1041fb73ae27899e9e802f07f61567" - - val actual = encoderServer.validateHash(wrongHash, inputBytes) - assertThat(actual).isEqualTo(false) - } - - @Test - fun `test download artifacts validate hash if it matches`() { - val rightHash = "sha384:${DigestUtils.sha384Hex(inputBytes)}" - - val actual = encoderServer.validateHash(rightHash, inputBytes) - assertThat(actual).isEqualTo(true) - } -} diff --git a/plugins/amazonq/chat/jetbrains-community/tst/software/aws/toolkits/jetbrains/services/amazonq/workspace/context/ProjectContextControllerTest.kt b/plugins/amazonq/chat/jetbrains-community/tst/software/aws/toolkits/jetbrains/services/amazonq/workspace/context/ProjectContextControllerTest.kt deleted file mode 100644 index 385d269bee9..00000000000 --- a/plugins/amazonq/chat/jetbrains-community/tst/software/aws/toolkits/jetbrains/services/amazonq/workspace/context/ProjectContextControllerTest.kt +++ /dev/null @@ -1,79 +0,0 @@ -// Copyright 2024 Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: Apache-2.0 - -package software.aws.toolkits.jetbrains.services.amazonq.workspace.context - -import com.intellij.openapi.Disposable -import com.intellij.openapi.application.ApplicationManager -import com.intellij.openapi.project.Project -import com.intellij.testFramework.ProjectExtension -import com.intellij.testFramework.junit5.TestDisposable -import com.intellij.testFramework.replaceService -import kotlinx.coroutines.CoroutineScope -import kotlinx.coroutines.test.runTest -import org.assertj.core.api.Assertions.assertThat -import org.junit.jupiter.api.Test -import org.junit.jupiter.api.extension.RegisterExtension -import org.mockito.Mockito.mockConstruction -import org.mockito.kotlin.doReturn -import org.mockito.kotlin.mock -import org.mockito.kotlin.times -import org.mockito.kotlin.verify -import software.aws.toolkits.jetbrains.core.coroutines.getCoroutineBgContext -import software.aws.toolkits.jetbrains.services.amazonq.project.EncoderServer -import software.aws.toolkits.jetbrains.services.amazonq.project.ProjectContextController -import software.aws.toolkits.jetbrains.settings.CodeWhispererSettings - -class ProjectContextControllerTest { - lateinit var sut: ProjectContextController - - val project: Project - get() = projectExtension.project - - private companion object { - @JvmField - @RegisterExtension - val projectExtension = ProjectExtension() - } - - @Test - fun `should start encoderServer if chat project context is disabled`(@TestDisposable disposable: Disposable) = runTest { - ApplicationManager.getApplication() - .replaceService( - CodeWhispererSettings::class.java, - mock { on { isProjectContextEnabled() } doReturn false }, - disposable - ) - - assertEncoderServerStarted() - } - - @Test - fun `should start encoderServer if chat project context is enabled`(@TestDisposable disposable: Disposable) = runTest { - ApplicationManager.getApplication() - .replaceService( - CodeWhispererSettings::class.java, - mock { on { isProjectContextEnabled() } doReturn true }, - disposable - ) - - assertEncoderServerStarted() - } - - private fun assertEncoderServerStarted() = runTest { - mockConstruction(EncoderServer::class.java).use { - // TODO: figure out how to make this testScope work -// val cs = TestScope(context = StandardTestDispatcher()) // not works and the test never finish - val cs = CoroutineScope(getCoroutineBgContext()) // works - - assertThat(it.constructed()).isEmpty() - sut = ProjectContextController(project, cs) - assertThat(it.constructed()).hasSize(1) - -// cs.advanceUntilIdle() - sut.initJob.join() - val encoderServer = it.constructed().first() - verify(encoderServer, times(1)).downloadArtifactsAndStartServer() - } - } -} diff --git a/plugins/amazonq/chat/jetbrains-community/tst/software/aws/toolkits/jetbrains/services/amazonq/workspace/context/ProjectContextProviderTest.kt b/plugins/amazonq/chat/jetbrains-community/tst/software/aws/toolkits/jetbrains/services/amazonq/workspace/context/ProjectContextProviderTest.kt deleted file mode 100644 index 399bd114b57..00000000000 --- a/plugins/amazonq/chat/jetbrains-community/tst/software/aws/toolkits/jetbrains/services/amazonq/workspace/context/ProjectContextProviderTest.kt +++ /dev/null @@ -1,524 +0,0 @@ -// Copyright 2024 Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: Apache-2.0 -@file:Suppress("BannedImports") -package software.aws.toolkits.jetbrains.services.amazonq.workspace.context - -import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper -import com.github.tomakehurst.wiremock.client.WireMock.aResponse -import com.github.tomakehurst.wiremock.client.WireMock.any -import com.github.tomakehurst.wiremock.client.WireMock.equalTo -import com.github.tomakehurst.wiremock.client.WireMock.postRequestedFor -import com.github.tomakehurst.wiremock.client.WireMock.stubFor -import com.github.tomakehurst.wiremock.client.WireMock.urlPathEqualTo -import com.github.tomakehurst.wiremock.core.WireMockConfiguration.wireMockConfig -import com.github.tomakehurst.wiremock.http.Body -import com.github.tomakehurst.wiremock.junit.WireMockRule -import com.intellij.openapi.application.ApplicationManager -import com.intellij.openapi.project.Project -import com.intellij.testFramework.DisposableRule -import com.intellij.testFramework.replaceService -import io.mockk.every -import io.mockk.spyk -import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.ExperimentalCoroutinesApi -import kotlinx.coroutines.TimeoutCancellationException -import kotlinx.coroutines.test.StandardTestDispatcher -import kotlinx.coroutines.test.TestScope -import kotlinx.coroutines.test.advanceUntilIdle -import kotlinx.coroutines.test.runTest -import kotlinx.coroutines.withContext -import org.assertj.core.api.Assertions.assertThat -import org.junit.Before -import org.junit.Rule -import org.junit.jupiter.api.assertThrows -import org.mockito.kotlin.any -import org.mockito.kotlin.doReturn -import org.mockito.kotlin.mock -import org.mockito.kotlin.spy -import org.mockito.kotlin.stub -import org.mockito.kotlin.times -import org.mockito.kotlin.verify -import org.mockito.kotlin.whenever -import software.aws.toolkits.jetbrains.core.coroutines.getCoroutineBgContext -import software.aws.toolkits.jetbrains.services.amazonq.project.EncoderServer -import software.aws.toolkits.jetbrains.services.amazonq.project.IndexRequest -import software.aws.toolkits.jetbrains.services.amazonq.project.IndexUpdateMode -import software.aws.toolkits.jetbrains.services.amazonq.project.InlineBm25Chunk -import software.aws.toolkits.jetbrains.services.amazonq.project.InlineContextTarget -import software.aws.toolkits.jetbrains.services.amazonq.project.LspMessage -import software.aws.toolkits.jetbrains.services.amazonq.project.ProjectContextProvider -import software.aws.toolkits.jetbrains.services.amazonq.project.ProjectContextProvider.FileCollectionResult -import software.aws.toolkits.jetbrains.services.amazonq.project.QueryChatRequest -import software.aws.toolkits.jetbrains.services.amazonq.project.QueryInlineCompletionRequest -import software.aws.toolkits.jetbrains.services.amazonq.project.RelevantDocument -import software.aws.toolkits.jetbrains.services.amazonq.project.UpdateIndexRequest -import software.aws.toolkits.jetbrains.settings.CodeWhispererSettings -import software.aws.toolkits.jetbrains.utils.rules.CodeInsightTestFixtureRule -import software.aws.toolkits.jetbrains.utils.rules.JavaCodeInsightTestFixtureRule -import java.net.ConnectException -import kotlin.test.Test - -@OptIn(ExperimentalCoroutinesApi::class) -class ProjectContextProviderTest { - @Rule - @JvmField - val projectRule: CodeInsightTestFixtureRule = JavaCodeInsightTestFixtureRule() - - @Rule - @JvmField - val disposableRule: DisposableRule = DisposableRule() - - @Rule - @JvmField - val wireMock: WireMockRule = createMockServer() - - private val project: Project - get() = projectRule.project - - private lateinit var encoderServer: EncoderServer - private lateinit var sut: ProjectContextProvider - - private val mapper = jacksonObjectMapper() - - private val dispatcher = StandardTestDispatcher() - - @Before - fun setup() { - encoderServer = spy(EncoderServer(project)) - encoderServer.stub { on { port } doReturn wireMock.port() } - encoderServer.stub { on { isNodeProcessRunning() } doReturn true } - sut = spyk(ProjectContextProvider(project, encoderServer, TestScope(context = dispatcher))) - - // initialization - stubFor(any(urlPathEqualTo("/initialize")).willReturn(aResponse().withStatus(200).withResponseBody(Body("initialize response")))) - - // build index - stubFor(any(urlPathEqualTo("/buildIndex")).willReturn(aResponse().withStatus(200).withResponseBody(Body("initialize response")))) - - // update index - stubFor(any(urlPathEqualTo("/updateIndexV2")).willReturn(aResponse().withStatus(200).withResponseBody(Body("initialize response")))) - - // query - stubFor( - any(urlPathEqualTo("/query")).willReturn( - aResponse() - .withStatus(200) - .withResponseBody(Body(validQueryChatResponse)) - ) - ) - stubFor( - any(urlPathEqualTo("/queryInlineProjectContext")).willReturn( - aResponse() - .withStatus(200) - .withResponseBody( - Body(validQueryInlineResponse) - ) - ) - ) - - stubFor( - any(urlPathEqualTo("/getUsage")) - .willReturn( - aResponse() - .withStatus(200) - .withResponseBody(Body(validGetUsageResponse)) - ) - ) - } - - @Test - fun `Lsp endpoint correctness`() { - assertThat(LspMessage.Initialize.endpoint).isEqualTo("initialize") - assertThat(LspMessage.Index.endpoint).isEqualTo("buildIndex") - assertThat(LspMessage.UpdateIndex.endpoint).isEqualTo("updateIndexV2") - assertThat(LspMessage.QueryChat.endpoint).isEqualTo("query") - assertThat(LspMessage.QueryInlineCompletion.endpoint).isEqualTo("queryInlineProjectContext") - assertThat(LspMessage.GetUsageMetrics.endpoint).isEqualTo("getUsage") - } - - @Test - fun `index should send files within the project to lsp - vector index enabled`() = runTest { - ApplicationManager.getApplication().replaceService( - CodeWhispererSettings::class.java, - mock { on { isProjectContextEnabled() } doReturn true }, - disposableRule.disposable - ) - - projectRule.fixture.addFileToProject("Foo.java", "foo") - projectRule.fixture.addFileToProject("Bar.java", "bar") - projectRule.fixture.addFileToProject("Baz.java", "baz") - every { sut.collectFiles() } returns FileCollectionResult( - files = listOf("Foo.java", "Bar.java", "Baz.java"), - fileSize = 10 - ) - sut.index() - - val request = IndexRequest(listOf("/src/Foo.java", "/src/Bar.java", "/src/Baz.java"), "/src", "all", "") - assertThat(request.filePaths).hasSize(3) - assertThat(request.filePaths).satisfies({ - it.contains("/src/Foo.java") && - it.contains("/src/Baz.java") && - it.contains("/src/Bar.java") - }) - assertThat(request.config).isEqualTo("all") - - wireMock.verify( - 1, - postRequestedFor(urlPathEqualTo("/buildIndex")) - .withHeader("Content-Type", equalTo("text/plain")) - // comment it out because order matters and will cause json string different -// .withRequestBody(equalTo(encryptedRequest)) - ) - } - - @Test - fun `index should send files within the project to lsp - vector index disabled`() = runTest { - ApplicationManager.getApplication().replaceService( - CodeWhispererSettings::class.java, - mock { on { isProjectContextEnabled() } doReturn false }, - disposableRule.disposable - ) - - projectRule.fixture.addFileToProject("Foo.java", "foo") - projectRule.fixture.addFileToProject("Bar.java", "bar") - projectRule.fixture.addFileToProject("Baz.java", "baz") - every { sut.collectFiles() } returns FileCollectionResult( - files = listOf("Foo.java", "Bar.java", "Baz.java"), - fileSize = 10 - ) - sut.index() - - val request = IndexRequest(listOf("/src/Foo.java", "/src/Bar.java", "/src/Baz.java"), "/src", "default", "") - assertThat(request.filePaths).hasSize(3) - assertThat(request.filePaths).satisfies({ - it.contains("/src/Foo.java") && - it.contains("/src/Baz.java") && - it.contains("/src/Bar.java") - }) - assertThat(request.config).isEqualTo("default") - - wireMock.verify( - 1, - postRequestedFor(urlPathEqualTo("/buildIndex")) - .withHeader("Content-Type", equalTo("text/plain")) - // comment it out because order matters and will cause json string different -// .withRequestBody(equalTo(encryptedRequest)) - ) - } - - @Test - fun `updateIndex should send correct encrypted request to lsp`() { - sut.updateIndex(listOf("foo.java"), IndexUpdateMode.UPDATE) - val request = UpdateIndexRequest(listOf("foo.java"), IndexUpdateMode.UPDATE.command) - val requestJson = mapper.writeValueAsString(request) - - assertThat(mapper.readTree(requestJson)).isEqualTo(mapper.readTree("""{ "filePaths": ["foo.java"], "mode": "update" }""")) - - val encryptedRequest = encoderServer.encrypt(requestJson) - - wireMock.verify( - 1, - postRequestedFor(urlPathEqualTo("/updateIndexV2")) - .withHeader("Content-Type", equalTo("text/plain")) - .withRequestBody(equalTo(encryptedRequest)) - ) - } - - @Test - fun `query should send correct encrypted request to lsp`() = runTest { - // use real time - withContext(Dispatchers.Default.limitedParallelism(1)) { - sut = ProjectContextProvider(project, encoderServer, this) - val r = sut.query("foo", null) - advanceUntilIdle() - - val request = QueryChatRequest("foo") - val requestJson = mapper.writeValueAsString(request) - - assertThat(mapper.readTree(requestJson)).isEqualTo(mapper.readTree("""{ "query": "foo" }""")) - - val encryptedRequest = encoderServer.encrypt(requestJson) - - wireMock.verify( - 1, - postRequestedFor(urlPathEqualTo("/query")) - .withHeader("Content-Type", equalTo("text/plain")) - .withRequestBody(equalTo(encryptedRequest)) - ) - } - } - - @Test - fun `queryInline should send correct encrypted request to lsp`() = runTest { - // use real time - withContext(Dispatchers.Default.limitedParallelism(1)) { - sut = ProjectContextProvider(project, encoderServer, this) - sut.queryInline("foo", "Foo.java", InlineContextTarget.CODEMAP) - advanceUntilIdle() - - val request = QueryInlineCompletionRequest("foo", "Foo.java", "codemap") - val requestJson = mapper.writeValueAsString(request) - - assertThat(mapper.readTree(requestJson)).isEqualTo(mapper.readTree("""{ "query": "foo", "filePath": "Foo.java", "target": "codemap" }""")) - - val encryptedRequest = encoderServer.encrypt(requestJson) - wireMock.verify( - 1, - postRequestedFor(urlPathEqualTo("/queryInlineProjectContext")) - .withHeader("Content-Type", equalTo("text/plain")) - .withRequestBody(equalTo(encryptedRequest)) - ) - } - } - - @Test - fun `query chat should return empty if result set non deserializable`() = runTest { - stubFor( - any(urlPathEqualTo("/query")).willReturn( - aResponse().withStatus(200).withResponseBody( - Body( - """ - [ - "foo", "bar" - ] - """.trimIndent() - ) - ) - ) - ) - - assertThrows { - sut.query("foo", null) - } - } - - @Test - fun `query chat should return deserialized relevantDocument`() = runTest { - // use real time - withContext(Dispatchers.Default.limitedParallelism(1)) { - sut = ProjectContextProvider(project, encoderServer, this) - val r = sut.query("foo", null) - advanceUntilIdle() - assertThat(r).hasSize(2) - assertThat(r[0]).isEqualTo( - RelevantDocument( - "relativeFilePath1", - "context1" - ) - ) - assertThat(r[1]).isEqualTo( - RelevantDocument( - "relativeFilePath2", - "context2" - ) - ) - } - } - - @Test - fun `query inline should throw if resultset not deserializable`() = - runTest { - sut = ProjectContextProvider(project, encoderServer, this) - stubFor( - any(urlPathEqualTo("/queryInlineProjectContext")).willReturn( - aResponse().withStatus(200).withResponseBody( - Body( - """ - [ - "foo", "bar" - ] - """.trimIndent() - ) - ) - ) - ) - - assertThrows { - withContext(getCoroutineBgContext()) { - sut.queryInline("foo", "filepath", InlineContextTarget.CODEMAP) - } - - advanceUntilIdle() - } - } - - @Test - fun `query inline should return deserialized bm25 chunks`() = runTest { - // use real time - withContext(Dispatchers.Default.limitedParallelism(1)) { - sut = ProjectContextProvider(project, encoderServer, this) - advanceUntilIdle() - val r = sut.queryInline("foo", "filepath", InlineContextTarget.CODEMAP) - assertThat(r).hasSize(3) - assertThat(r[0]).isEqualTo( - InlineBm25Chunk( - "content1", - "file1", - 0.1 - ) - ) - assertThat(r[1]).isEqualTo( - InlineBm25Chunk( - "content2", - "file2", - 0.2 - ) - ) - assertThat(r[2]).isEqualTo( - InlineBm25Chunk( - "content3", - "file3", - 0.3 - ) - ) - } - } - - @Test - fun `get usage should return memory, cpu usage`() = runTest { - val r = sut.getUsage() - assertThat(r).isEqualTo(ProjectContextProvider.Usage(123, 456)) - } - - @Test - fun `queryInline should throw if time elapsed is greater than 50ms`() = runTest { - assertThrows { - sut = ProjectContextProvider(project, encoderServer, this) - stubFor( - any(urlPathEqualTo("/queryInlineProjectContext")).willReturn( - aResponse() - .withStatus(200) - .withResponseBody( - Body(validQueryInlineResponse) - ) - .withFixedDelay(101) // 100 ms - ) - ) - - // it won't throw if it's executed within TestDispatcher context - withContext(getCoroutineBgContext()) { - sut.queryInline("foo", "bar", InlineContextTarget.CODEMAP) - } - - advanceUntilIdle() - } - } - - @Test - fun `queryChat should throw if time elapsed is greather than 500ms`() = runTest { - assertThrows { - sut = ProjectContextProvider(project, encoderServer, this) - stubFor( - any(urlPathEqualTo("/query")).willReturn( - aResponse() - .withStatus(200) - .withResponseBody( - Body(validQueryChatResponse) - ) - .withFixedDelay(501) - ) - ) - - withContext(getCoroutineBgContext()) { - sut.query("foo", timeout = 500L) - } - - advanceUntilIdle() - } - } - - @Test - fun `test index payload is encrypted`() = runTest { - whenever(encoderServer.port).thenReturn(3000) - every { sut.collectFiles() } returns FileCollectionResult( - files = listOf("Foo.java", "Bar.java", "Baz.java"), - fileSize = 10 - ) - try { - sut.index() - } catch (e: ConnectException) { - // no-op - } - verify(encoderServer, times(1)).encrypt(any()) - } - - @Test - fun `test query payload is encrypted`() = runTest { - // use real time - withContext(Dispatchers.Default.limitedParallelism(1)) { - sut = ProjectContextProvider(project, encoderServer, this) - sut.query("what does this project do", null) - advanceUntilIdle() - verify(encoderServer, times(1)).encrypt(any()) - } - } - - private fun createMockServer() = WireMockRule(wireMockConfig().dynamicPort()) -} - -// language=JSON -val validQueryInlineResponse = """ - [ - { - "content": "content1", - "filePath": "file1", - "score": 0.1 - }, - { - "content": "content2", - "filePath": "file2", - "score": 0.2 - }, - { - "content": "content3", - "filePath": "file3", - "score": 0.3 - } - ] -""".trimIndent() - -// language=JSON -val validQueryChatResponse = """ - [ - { - "filePath": "file1", - "content": "content1", - "id": "id1", - "index": "index1", - "vec": [ - "vec_1-1", - "vec_1-2", - "vec_1-3" - ], - "context": "context1", - "prev": "prev1", - "next": "next1", - "relativePath": "relativeFilePath1", - "programmingLanguage": "language1" - }, - { - "filePath": "file2", - "content": "content2", - "id": "id2", - "index": "index2", - "vec": [ - "vec_2-1", - "vec_2-2", - "vec_2-3" - ], - "context": "context2", - "prev": "prev2", - "next": "next2", - "relativePath": "relativeFilePath2", - "programmingLanguage": "language2" - } - ] -""".trimIndent() - -// language=JSON -val validGetUsageResponse = """ - { - "memoryUsage":123, - "cpuUsage":456 - } -""".trimIndent() diff --git a/plugins/amazonq/codewhisperer/jetbrains-community/tst/software/aws/toolkits/jetbrains/services/codewhisperer/CodeWhispererFileContextProviderTest.kt b/plugins/amazonq/codewhisperer/jetbrains-community/tst/software/aws/toolkits/jetbrains/services/codewhisperer/CodeWhispererFileContextProviderTest.kt index c4300ae6b1e..61ed40d41d8 100644 --- a/plugins/amazonq/codewhisperer/jetbrains-community/tst/software/aws/toolkits/jetbrains/services/codewhisperer/CodeWhispererFileContextProviderTest.kt +++ b/plugins/amazonq/codewhisperer/jetbrains-community/tst/software/aws/toolkits/jetbrains/services/codewhisperer/CodeWhispererFileContextProviderTest.kt @@ -15,7 +15,6 @@ import org.junit.Rule import org.junit.Test import org.mockito.kotlin.mock import software.aws.toolkits.jetbrains.services.amazonq.CodeWhispererFeatureConfigService -import software.aws.toolkits.jetbrains.services.amazonq.project.ProjectContextController import software.aws.toolkits.jetbrains.services.codewhisperer.language.languages.CodeWhispererJava import software.aws.toolkits.jetbrains.services.codewhisperer.util.DefaultCodeWhispererFileContextProvider import software.aws.toolkits.jetbrains.services.codewhisperer.util.FileContextProvider @@ -34,7 +33,6 @@ class CodeWhispererFileContextProviderTest { // dependencies lateinit var featureConfigService: CodeWhispererFeatureConfigService - lateinit var mockProjectContext: ProjectContextController lateinit var fixture: JavaCodeInsightTestFixture lateinit var project: Project @@ -53,9 +51,6 @@ class CodeWhispererFileContextProviderTest { featureConfigService, disposableRule.disposable ) - - mockProjectContext = mock() - project.replaceService(ProjectContextController::class.java, mockProjectContext, disposableRule.disposable) } @Test diff --git a/plugins/amazonq/shared/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonq/lsp/artifacts/ArtifactHelper.kt b/plugins/amazonq/shared/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonq/lsp/artifacts/ArtifactHelper.kt index fea7d72ffbb..c1b6121befa 100644 --- a/plugins/amazonq/shared/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonq/lsp/artifacts/ArtifactHelper.kt +++ b/plugins/amazonq/shared/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonq/lsp/artifacts/ArtifactHelper.kt @@ -17,7 +17,6 @@ import software.aws.toolkits.core.utils.getLogger import software.aws.toolkits.core.utils.info import software.aws.toolkits.core.utils.warn import software.aws.toolkits.jetbrains.core.saveFileFromUrl -import software.aws.toolkits.jetbrains.services.amazonq.project.manifest.ManifestManager import software.aws.toolkits.resources.AwsCoreBundle import java.nio.file.Files import java.nio.file.Path @@ -29,7 +28,7 @@ class ArtifactHelper internal constructor( ) { private val currentAttempt = AtomicInteger(0) - fun removeDelistedVersions(delistedVersions: List) { + fun removeDelistedVersions(delistedVersions: List) { val localFolders = getSubFolders(lspArtifactsPath) delistedVersions.forEach { delistedVersion -> @@ -78,7 +77,7 @@ class ArtifactHelper internal constructor( .sortedByDescending { (_, semVer) -> semVer } } - fun getExistingLspArtifacts(versions: List, target: ManifestManager.VersionTarget?): Boolean { + fun getExistingLspArtifacts(versions: List, target: VersionTarget?): Boolean { if (versions.isEmpty() || target?.contents == null) return false val localLSPPath = lspArtifactsPath.resolve(versions.first().serverVersion.toString()) @@ -102,7 +101,7 @@ class ArtifactHelper internal constructor( return !hasInvalidFiles } - suspend fun tryDownloadLspArtifacts(project: Project, versions: List, target: ManifestManager.VersionTarget?): Path? { + suspend fun tryDownloadLspArtifacts(project: Project, versions: List, target: VersionTarget?): Path? { val temporaryDownloadPath = Files.createTempDirectory("lsp-dl") val downloadPath = lspArtifactsPath.resolve(versions.first().serverVersion.toString()) @@ -145,7 +144,7 @@ class ArtifactHelper internal constructor( } @VisibleForTesting - internal fun downloadLspArtifacts(downloadPath: Path, target: ManifestManager.VersionTarget?): Boolean { + internal fun downloadLspArtifacts(downloadPath: Path, target: VersionTarget?): Boolean { if (target == null || target.contents.isNullOrEmpty()) { logger.warn { "No target contents available for download" } return false @@ -199,7 +198,7 @@ class ArtifactHelper internal constructor( return "sha384:$contentHash" == expectedHash } - private fun validateDownloadedFiles(downloadPath: Path, contents: List) { + private fun validateDownloadedFiles(downloadPath: Path, contents: List) { val missingFiles = contents .mapNotNull { it.filename } .filter { filename -> diff --git a/plugins/amazonq/shared/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonq/lsp/artifacts/ArtifactManager.kt b/plugins/amazonq/shared/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonq/lsp/artifacts/ArtifactManager.kt index 08d9cd9729d..7b4e74199c3 100644 --- a/plugins/amazonq/shared/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonq/lsp/artifacts/ArtifactManager.kt +++ b/plugins/amazonq/shared/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonq/lsp/artifacts/ArtifactManager.kt @@ -16,7 +16,6 @@ import org.jetbrains.annotations.VisibleForTesting import software.aws.toolkits.core.utils.error import software.aws.toolkits.core.utils.getLogger import software.aws.toolkits.core.utils.info -import software.aws.toolkits.jetbrains.services.amazonq.project.manifest.ManifestManager import java.nio.file.Path @Service @@ -35,8 +34,8 @@ class ArtifactManager @NonInjectable internal constructor(private val manifestFe val endVersion: SemVer, ) data class LSPVersions( - val deListedVersions: List, - val inRangeVersions: List, + val deListedVersions: List, + val inRangeVersions: List, ) companion object { @@ -91,7 +90,7 @@ class ArtifactManager @NonInjectable internal constructor(private val manifestFe } @VisibleForTesting - internal fun getLSPVersionsFromManifestWithSpecifiedRange(manifest: ManifestManager.Manifest): LSPVersions { + internal fun getLSPVersionsFromManifestWithSpecifiedRange(manifest: Manifest): LSPVersions { if (manifest.versions.isNullOrEmpty()) return LSPVersions(emptyList(), emptyList()) val (deListed, inRange) = manifest.versions.mapNotNull { version -> @@ -112,7 +111,7 @@ class ArtifactManager @NonInjectable internal constructor(private val manifestFe ) } - private fun getTargetFromLspManifest(versions: List): ManifestManager.VersionTarget { + private fun getTargetFromLspManifest(versions: List): VersionTarget { val currentOS = getCurrentOS() val currentArchitecture = getCurrentArchitecture() diff --git a/plugins/amazonq/shared/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonq/lsp/artifacts/ManifestFetcher.kt b/plugins/amazonq/shared/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonq/lsp/artifacts/ManifestFetcher.kt index 9e769fa2f6a..a0932ef1c7d 100644 --- a/plugins/amazonq/shared/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonq/lsp/artifacts/ManifestFetcher.kt +++ b/plugins/amazonq/shared/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonq/lsp/artifacts/ManifestFetcher.kt @@ -3,6 +3,10 @@ package software.aws.toolkits.jetbrains.services.amazonq.lsp.artifacts +import com.fasterxml.jackson.annotation.JsonProperty +import com.fasterxml.jackson.databind.DeserializationFeature +import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper +import com.fasterxml.jackson.module.kotlin.readValue import org.jetbrains.annotations.VisibleForTesting import software.aws.toolkits.core.utils.deleteIfExists import software.aws.toolkits.core.utils.error @@ -10,18 +14,18 @@ import software.aws.toolkits.core.utils.exists import software.aws.toolkits.core.utils.getLogger import software.aws.toolkits.core.utils.info import software.aws.toolkits.core.utils.readText +import software.aws.toolkits.core.utils.warn import software.aws.toolkits.jetbrains.core.getETagFromUrl import software.aws.toolkits.jetbrains.core.getTextFromUrl import software.aws.toolkits.jetbrains.core.saveFileFromUrl -import software.aws.toolkits.jetbrains.services.amazonq.project.manifest.ManifestManager import java.nio.file.Path class ManifestFetcher( private val lspManifestUrl: String = DEFAULT_MANIFEST_URL, - private val manifestManager: ManifestManager = ManifestManager(), private val manifestPath: Path = DEFAULT_MANIFEST_PATH, ) { companion object { + private val mapper = jacksonObjectMapper().apply { configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false) } private val logger = getLogger() private const val DEFAULT_MANIFEST_URL = @@ -41,7 +45,7 @@ class ManifestFetcher( /** * Method which will be used to fetch latest manifest. * */ - fun fetch(): ManifestManager.Manifest? { + fun fetch(): Manifest? { val localManifest = fetchManifestFromLocal() if (localManifest != null) { return localManifest @@ -50,15 +54,16 @@ class ManifestFetcher( } @VisibleForTesting - internal fun fetchManifestFromRemote(): ManifestManager.Manifest? { - val manifest: ManifestManager.Manifest? + internal fun fetchManifestFromRemote(): Manifest? { + val manifest: Manifest? try { val manifestString = getTextFromUrl(lspManifestUrl) - manifest = manifestManager.readManifestFile(manifestString) ?: return null + manifest = readManifestFile(manifestString) ?: return null } catch (e: Exception) { logger.error(e) { "error fetching lsp manifest from remote URL ${e.message}" } return null } + if (manifest.isManifestDeprecated == true) { logger.info { "Manifest is deprecated" } return null @@ -77,7 +82,7 @@ class ManifestFetcher( } @VisibleForTesting - internal fun fetchManifestFromLocal(): ManifestManager.Manifest? { + internal fun fetchManifestFromLocal(): Manifest? { val localETag = getManifestETagFromLocal() val remoteETag = getManifestETagFromUrl() // If local and remote have same ETag, we can re-use the manifest file from local to fetch artifacts. @@ -85,7 +90,7 @@ class ManifestFetcher( if ((localETag != null && remoteETag != null && localETag == remoteETag) or (localETag != null && remoteETag == null)) { try { val manifestContent = lspManifestFilePath.readText() - val manifest = manifestManager.readManifestFile(manifestContent) + val manifest = readManifestFile(manifestContent) if (manifest != null) return manifest lspManifestFilePath.deleteIfExists() // delete manifest if it fails to de-serialize } catch (e: Exception) { @@ -112,4 +117,55 @@ class ManifestFetcher( } return null } + + fun readManifestFile(content: String): Manifest? { + try { + return mapper.readValue(content) + } catch (e: Exception) { + logger.warn { "error parsing manifest file for project context ${e.message}" } + return null + } + } } + +data class TargetContent( + @JsonProperty("filename") + val filename: String? = null, + @JsonProperty("url") + val url: String? = null, + @JsonProperty("hashes") + val hashes: List? = emptyList(), + @JsonProperty("bytes") + val bytes: Number? = null, +) + +data class VersionTarget( + @JsonProperty("platform") + val platform: String? = null, + @JsonProperty("arch") + val arch: String? = null, + @JsonProperty("contents") + val contents: List? = emptyList(), +) + +data class Version( + @JsonProperty("serverVersion") + val serverVersion: String? = null, + @JsonProperty("isDelisted") + val isDelisted: Boolean? = null, + @JsonProperty("targets") + val targets: List? = emptyList(), +) + +data class Manifest( + @JsonProperty("manifestSchemaVersion") + val manifestSchemaVersion: String? = null, + @JsonProperty("artifactId") + val artifactId: String? = null, + @JsonProperty("artifactDescription") + val artifactDescription: String? = null, + @JsonProperty("isManifestDeprecated") + val isManifestDeprecated: Boolean? = null, + @JsonProperty("versions") + val versions: List? = emptyList(), +) diff --git a/plugins/amazonq/shared/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonq/project/EncoderServer.kt b/plugins/amazonq/shared/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonq/project/EncoderServer.kt deleted file mode 100644 index 2da14c2dd50..00000000000 --- a/plugins/amazonq/shared/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonq/project/EncoderServer.kt +++ /dev/null @@ -1,246 +0,0 @@ -// Copyright 2024 Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: Apache-2.0 - -package software.aws.toolkits.jetbrains.services.amazonq.project - -import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper -import com.intellij.execution.configurations.GeneralCommandLine -import com.intellij.execution.process.KillableProcessHandler -import com.intellij.openapi.Disposable -import com.intellij.openapi.application.ApplicationManager -import com.intellij.openapi.project.Project -import com.intellij.openapi.util.process.ProcessCloseUtil -import com.intellij.util.io.HttpRequests -import com.intellij.util.io.createDirectories -import com.intellij.util.net.NetUtils -import com.nimbusds.jose.JOSEObjectType -import com.nimbusds.jose.JWSAlgorithm -import com.nimbusds.jose.JWSHeader -import com.nimbusds.jose.crypto.MACSigner -import com.nimbusds.jwt.JWTClaimsSet -import com.nimbusds.jwt.SignedJWT -import org.apache.commons.codec.digest.DigestUtils -import software.amazon.awssdk.utils.UserHomeDirectoryUtils -import software.aws.toolkits.core.utils.getLogger -import software.aws.toolkits.core.utils.info -import software.aws.toolkits.core.utils.tryDirOp -import software.aws.toolkits.core.utils.warn -import software.aws.toolkits.jetbrains.services.amazonq.lsp.artifacts.extractZipFile -import software.aws.toolkits.jetbrains.services.amazonq.project.manifest.ManifestManager -import software.aws.toolkits.jetbrains.settings.CodeWhispererSettings -import java.io.IOException -import java.nio.file.Files -import java.nio.file.Path -import java.nio.file.Paths -import java.nio.file.attribute.PosixFilePermission -import java.security.Key -import java.security.SecureRandom -import java.util.Base64 -import java.util.concurrent.atomic.AtomicInteger -import javax.crypto.spec.SecretKeySpec - -class EncoderServer(val project: Project) : Disposable { - private val cachePath = Paths.get( - UserHomeDirectoryUtils.userHomeDirectory() - ).resolve(".aws").resolve("amazonq").resolve("cache") - val manifestManager = ManifestManager() - private val serverDirectoryName = "qserver-${manifestManager.currentVersion}.zip" - private val numberOfRetry = AtomicInteger(0) - val port by lazy { NetUtils.findAvailableSocketPort() } - private val nodeRunnableName = if (manifestManager.getOs() == "windows") "node.exe" else "node" - private val maxRetry: Int = 3 - private val key = generateHmacKey() - private var processHandler: KillableProcessHandler? = null - private val mapper = jacksonObjectMapper() - - fun downloadArtifactsAndStartServer() { - if (ApplicationManager.getApplication().isUnitTestMode) { - return - } - downloadArtifactsIfNeeded() - start() - } - - fun isNodeProcessRunning() = processHandler != null && processHandler?.process?.isAlive == true - - private fun generateHmacKey(): Key { - val keyBytes = ByteArray(32) - SecureRandom().nextBytes(keyBytes) - return SecretKeySpec(keyBytes, "HmacSHA256") - } - - fun encrypt(data: String): String { - val header = JWSHeader.Builder(JWSAlgorithm.HS256) - .type(JOSEObjectType.JWT) - .build() - - val claimsSet = JWTClaimsSet.Builder() - .subject(Base64.getUrlEncoder().withoutPadding().encodeToString(data.toByteArray())) - .build() - - val signedJWT = SignedJWT(header, claimsSet) - signedJWT.sign(MACSigner(key.encoded)) - - return signedJWT.serialize() - } - - data class EncryptionRequest( - val version: String = "1.0", - val mode: String = "JWT", - val key: String, - ) - - fun getEncryptionRequest(): String { - val request = EncryptionRequest(key = Base64.getUrlEncoder().withoutPadding().encodeToString(key.encoded)) - return mapper.writeValueAsString(request) - } - - private fun runCommand(command: GeneralCommandLine): Boolean { - try { - logger.info { "starting encoder server for project context on $port for ${project.name}" } - processHandler = KillableProcessHandler(command) - val exitCode = processHandler?.waitFor() - if (exitCode == true) { - throw Exception("Encoder server exited") - } else { - return true - } - } catch (e: Exception) { - logger.warn(e) { "error running encoder server:" } - processHandler?.destroyProcess() - numberOfRetry.incrementAndGet() - return false - } - } - - private fun getCommand(): GeneralCommandLine { - val threadCount = CodeWhispererSettings.getInstance().getProjectContextIndexThreadCount() - val isGpuEnabled = CodeWhispererSettings.getInstance().isProjectContextGpu() - val map = mutableMapOf( - "PORT" to port.toString(), - "START_AMAZONQ_LSP" to "true", - "CACHE_DIR" to cachePath.toString(), - "MODEL_DIR" to cachePath.resolve("qserver").toString() - ) - if (threadCount > 0) { - map["Q_WORKER_THREADS"] = threadCount.toString() - } - if (isGpuEnabled) { - map["Q_ENABLE_GPU"] = "true" - } - val jsPath = cachePath.resolve("qserver").resolve("dist").resolve("extension.js").toString() - val nodePath = cachePath.resolve(nodeRunnableName).toString() - val command = GeneralCommandLine(nodePath, jsPath) - .withParentEnvironmentType(GeneralCommandLine.ParentEnvironmentType.CONSOLE) - .withEnvironment(map) - return command - } - - fun start() { - while (numberOfRetry.get() < maxRetry) { - val isSuccess = runCommand(getCommand()) - if (isSuccess) { - return - } - } - } - - private fun close() { - processHandler?.process?.let { ProcessCloseUtil.close(it) } - } - - private fun downloadArtifactsIfNeeded() { - cachePath.tryDirOp(logger) { createDirectories() } - val nodePath = cachePath.resolve(nodeRunnableName) - val zipFilePath = cachePath.resolve(serverDirectoryName) - val manifest = manifestManager.getManifest() ?: return - try { - if (!Files.exists(nodePath)) { - val nodeContent = manifestManager.getNodeContentFromManifest(manifest) - if (nodeContent?.url != null) { - val bytes = HttpRequests.request(nodeContent.url).readBytes(null) - if (validateHash(nodeContent.hashes?.first(), bytes)) { - downloadFromRemote(nodeContent.url, nodePath) - } - } - } - if (manifestManager.currentOs != "windows") { - makeFileExecutable(nodePath) - } - val files = cachePath.toFile().listFiles() - if (files.isNotEmpty()) { - val filenames = files.map { it.name } - if (filenames.contains(serverDirectoryName)) { - return - } - tryDeleteOldArtifacts(filenames) - } - - val serverContent = manifestManager.getZipContentFromManifest(manifest) - if (serverContent?.url != null) { - if (validateHash(serverContent.hashes?.first(), HttpRequests.request(serverContent.url).readBytes(null))) { - downloadFromRemote(serverContent.url, zipFilePath) - extractZipFile(zipFilePath, cachePath) - } - } - } catch (e: Exception) { - logger.warn(e) { "error downloading artifacts:" } - } - } - - private fun tryDeleteOldArtifacts(filenames: List) { - try { - filenames.forEach { filename -> - if (filename.contains("qserver")) { - val parts = filename.split("-") - val version = if (parts.size > 1) parts[1] else null - if (version != null && version != "${manifestManager.currentVersion}.zip") { - Files.deleteIfExists(cachePath.resolve(filename)) - Files.deleteIfExists(cachePath.resolve("qserver")) - } - } - } - } catch (e: Exception) { - logger.warn(e) { "error deleting old artifacts:" } - } - } - - fun validateHash(expectedHash: String?, input: ByteArray): Boolean { - if (expectedHash == null) { return false } - val sha384 = DigestUtils.sha384Hex(input) - val isValid = ("sha384:$sha384") == expectedHash - if (!isValid) { - logger.warn { "failed validating hash for artifacts $expectedHash" } - } - return isValid - } - - private fun makeFileExecutable(filePath: Path) { - val permissions = setOf( - PosixFilePermission.OWNER_READ, - PosixFilePermission.OWNER_WRITE, - PosixFilePermission.OWNER_EXECUTE, - PosixFilePermission.GROUP_READ, - PosixFilePermission.GROUP_EXECUTE, - PosixFilePermission.OTHERS_READ, - PosixFilePermission.OTHERS_EXECUTE, - ) - Files.setPosixFilePermissions(filePath, permissions) - } - - private fun downloadFromRemote(url: String, path: Path) { - try { - HttpRequests.request(url).saveToFile(path, null) - } catch (e: IOException) { - logger.warn { "error downloading from remote ${e.message}" } - } - } - - override fun dispose() { - close() - } - - companion object { - private val logger = getLogger() - } -} diff --git a/plugins/amazonq/shared/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonq/project/ProjectContextController.kt b/plugins/amazonq/shared/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonq/project/ProjectContextController.kt deleted file mode 100644 index 4c08dcd82e5..00000000000 --- a/plugins/amazonq/shared/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonq/project/ProjectContextController.kt +++ /dev/null @@ -1,94 +0,0 @@ -// Copyright 2024 Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: Apache-2.0 - -package software.aws.toolkits.jetbrains.services.amazonq.project - -import com.intellij.openapi.Disposable -import com.intellij.openapi.components.Service -import com.intellij.openapi.components.service -import com.intellij.openapi.project.Project -import com.intellij.openapi.util.Disposer -import com.intellij.openapi.vfs.VirtualFileManager -import com.intellij.openapi.vfs.newvfs.BulkFileListener -import com.intellij.openapi.vfs.newvfs.events.VFileCreateEvent -import com.intellij.openapi.vfs.newvfs.events.VFileDeleteEvent -import com.intellij.openapi.vfs.newvfs.events.VFileEvent -import com.intellij.util.concurrency.annotations.RequiresBackgroundThread -import kotlinx.coroutines.CoroutineScope -import kotlinx.coroutines.Job -import kotlinx.coroutines.TimeoutCancellationException -import kotlinx.coroutines.launch -import software.aws.toolkits.core.utils.getLogger -import software.aws.toolkits.core.utils.warn -import software.aws.toolkits.jetbrains.core.coroutines.ioDispatcher -import software.aws.toolkits.jetbrains.utils.pluginAwareExecuteOnPooledThread -import java.util.concurrent.TimeoutException - -@Service(Service.Level.PROJECT) -class ProjectContextController(private val project: Project, private val cs: CoroutineScope) : Disposable { - // TODO: Ideally we should inject dependencies via constructor for easier testing, refer to how [TelemetryService] inject publisher and batcher - private val encoderServer: EncoderServer = EncoderServer(project) - private val projectContextProvider: ProjectContextProvider = ProjectContextProvider(project, encoderServer, cs) - val initJob: Job = cs.launch(ioDispatcher(1)) { - encoderServer.downloadArtifactsAndStartServer() - } - - init { - project.messageBus.connect(this).subscribe( - VirtualFileManager.VFS_CHANGES, - object : BulkFileListener { - override fun after(events: MutableList) { - val createdFiles = events.filterIsInstance().mapNotNull { it.file?.path } - val deletedFiles = events.filterIsInstance().map { it.file.path } - - pluginAwareExecuteOnPooledThread { - updateIndex(createdFiles, IndexUpdateMode.ADD) - updateIndex(deletedFiles, IndexUpdateMode.REMOVE) - } - } - } - ) - } - - fun getProjectContextIndexComplete() = projectContextProvider.isIndexComplete.get() - - suspend fun queryChat(prompt: String, timeout: Long?): List { - try { - return projectContextProvider.query(prompt, timeout) - } catch (e: Exception) { - logger.warn { "error while querying for project context $e.message" } - return emptyList() - } - } - - suspend fun queryInline(query: String, filePath: String): List = - try { - projectContextProvider.queryInline(query, filePath, InlineContextTarget.CODEMAP) - } catch (e: Exception) { - var logStr = "error while querying inline for project context $e.message" - if (e is TimeoutCancellationException || e is TimeoutException) { - logStr = "project context times out with 50ms ${e.message}" - } - logger.warn { logStr } - emptyList() - } - - @RequiresBackgroundThread - fun updateIndex(filePaths: List, mode: IndexUpdateMode) { - try { - return projectContextProvider.updateIndex(filePaths, mode) - } catch (e: Exception) { - logger.warn { "error while updating index for project context $e.message" } - } - } - - override fun dispose() { - Disposer.dispose(encoderServer) - Disposer.dispose(projectContextProvider) - } - - companion object { - private val logger = getLogger() - fun getInstance(project: Project) = project.service() - } -} diff --git a/plugins/amazonq/shared/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonq/project/ProjectContextEditorListener.kt b/plugins/amazonq/shared/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonq/project/ProjectContextEditorListener.kt deleted file mode 100644 index ce194da9614..00000000000 --- a/plugins/amazonq/shared/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonq/project/ProjectContextEditorListener.kt +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright 2024 Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: Apache-2.0 - -package software.aws.toolkits.jetbrains.services.amazonq.project - -import com.intellij.openapi.fileEditor.FileDocumentManager -import com.intellij.openapi.fileEditor.FileEditorManagerEvent -import com.intellij.openapi.fileEditor.FileEditorManagerListener -import software.aws.toolkits.jetbrains.utils.pluginAwareExecuteOnPooledThread - -class ProjectContextEditorListener : FileEditorManagerListener { - override fun selectionChanged(event: FileEditorManagerEvent) { - val oldFile = event.oldFile ?: return - - // TODO: should respect isIdeAutosave config - with(FileDocumentManager.getInstance()) { - this.getDocument(oldFile)?.let { - this.saveDocument(it) - } - } - - val project = event.manager.project - pluginAwareExecuteOnPooledThread { - ProjectContextController.getInstance(project).updateIndex(listOf(oldFile.path), IndexUpdateMode.UPDATE) - } - } -} diff --git a/plugins/amazonq/shared/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonq/project/ProjectContextProvider.kt b/plugins/amazonq/shared/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonq/project/ProjectContextProvider.kt deleted file mode 100644 index 854124a180f..00000000000 --- a/plugins/amazonq/shared/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonq/project/ProjectContextProvider.kt +++ /dev/null @@ -1,362 +0,0 @@ -// Copyright 2024 Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: Apache-2.0 - -package software.aws.toolkits.jetbrains.services.amazonq.project - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties -import com.fasterxml.jackson.annotation.JsonProperty -import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper -import com.fasterxml.jackson.module.kotlin.readValue -import com.intellij.openapi.Disposable -import com.intellij.openapi.application.ApplicationManager -import com.intellij.openapi.project.BaseProjectDirectories.Companion.getBaseDirectories -import com.intellij.openapi.project.Project -import com.intellij.openapi.vcs.changes.ChangeListManager -import com.intellij.openapi.vfs.VfsUtilCore -import com.intellij.openapi.vfs.VirtualFile -import com.intellij.openapi.vfs.VirtualFileVisitor -import com.intellij.openapi.vfs.isFile -import com.intellij.util.concurrency.annotations.RequiresBackgroundThread -import kotlinx.coroutines.CoroutineScope -import kotlinx.coroutines.delay -import kotlinx.coroutines.launch -import kotlinx.coroutines.runBlocking -import kotlinx.coroutines.withContext -import kotlinx.coroutines.withTimeout -import software.aws.toolkits.core.utils.debug -import software.aws.toolkits.core.utils.getLogger -import software.aws.toolkits.core.utils.info -import software.aws.toolkits.core.utils.warn -import software.aws.toolkits.jetbrains.core.coroutines.ioDispatcher -import software.aws.toolkits.jetbrains.services.amazonq.CHAT_EXPLICIT_PROJECT_CONTEXT_TIMEOUT -import software.aws.toolkits.jetbrains.services.amazonq.SUPPLEMENTAL_CONTEXT_TIMEOUT -import software.aws.toolkits.jetbrains.services.cwc.controller.chat.telemetry.getStartUrl -import software.aws.toolkits.jetbrains.settings.CodeWhispererSettings -import software.aws.toolkits.telemetry.AmazonqTelemetry -import java.io.OutputStreamWriter -import java.net.HttpURLConnection -import java.net.URI -import java.util.concurrent.atomic.AtomicBoolean -import java.util.concurrent.atomic.AtomicInteger -import kotlin.time.Duration.Companion.minutes - -class ProjectContextProvider(val project: Project, private val encoderServer: EncoderServer, private val cs: CoroutineScope) : Disposable { - private val retryCount = AtomicInteger(0) - val isIndexComplete = AtomicBoolean(false) - private val mapper = jacksonObjectMapper() - - // max number of requests that can be ongoing to an given server instance, excluding index() - private val ioDispatcher = ioDispatcher(20) - - init { - cs.launch { - if (ApplicationManager.getApplication().isUnitTestMode) { - return@launch - } - - // TODO: need better solution for this - @Suppress("LoopWithTooManyJumpStatements") - while (true) { - if (encoderServer.isNodeProcessRunning()) { - delay(10000) - initAndIndex() - break - } else { - delay(10000) - } - } - } - } - - data class FileCollectionResult( - val files: List, - val fileSize: Int, // in MB - ) - - // TODO: move to LspMessage.kt - @JsonIgnoreProperties(ignoreUnknown = true) - data class Usage( - @JsonProperty("memoryUsage") - val memoryUsage: Int? = null, - @JsonProperty("cpuUsage") - val cpuUsage: Int? = null, - ) - - // TODO: move to LspMessage.kt - @JsonIgnoreProperties(ignoreUnknown = true) - data class Chunk( - @JsonProperty("filePath") - val filePath: String? = null, - @JsonProperty("content") - val content: String? = null, - @JsonProperty("id") - val id: String? = null, - @JsonProperty("index") - val index: String? = null, - @JsonProperty("vec") - val vec: List? = null, - @JsonProperty("context") - val context: String? = null, - @JsonProperty("prev") - val prev: String? = null, - @JsonProperty("next") - val next: String? = null, - @JsonProperty("relativePath") - val relativePath: String? = null, - @JsonProperty("programmingLanguage") - val programmingLanguage: String? = null, - ) - - private suspend fun initAndIndex() { - while (retryCount.get() < 5) { - try { - logger.info { "project context: about to init key" } - val isInitSuccess = initEncryption() - if (isInitSuccess) { - logger.info { "project context index starting" } - delay(300) - val isIndexSuccess = index() - if (isIndexSuccess) { - isIndexComplete.set(true) - } - return - } - retryCount.incrementAndGet() - } catch (e: Exception) { - logger.warn(e) { "failed to init project context" } - if (e.stackTraceToString().contains("Connection refused")) { - retryCount.incrementAndGet() - delay(10000) - } else { - return - } - } - } - } - - private suspend fun initEncryption(): Boolean { - val request = encoderServer.getEncryptionRequest() - val response = sendMsgToLsp(LspMessage.Initialize, request) - logger.info { "received response to init encryption: $response" } - return response?.responseCode == 200 - } - - suspend fun index(): Boolean { - val projectRoot = project.basePath ?: return false - - val indexStartTime = System.currentTimeMillis() - val filesResult = collectFiles() - if (filesResult.files.isEmpty()) { - logger.warn { "No file found in workspace" } - return false - } - var duration = (System.currentTimeMillis() - indexStartTime).toDouble() - logger.debug { "time elapsed to collect project context files: ${duration}ms, collected ${filesResult.files.size} files" } - - val indexOption = if (CodeWhispererSettings.getInstance().isProjectContextEnabled()) IndexOption.ALL else IndexOption.DEFAULT - val encrypted = encryptRequest(IndexRequest(filesResult.files, projectRoot, indexOption.command, "")) - val response = sendMsgToLsp(LspMessage.Index, encrypted) - - duration = (System.currentTimeMillis() - indexStartTime).toDouble() - logger.debug { "project context index time: ${duration}ms" } - - val startUrl = getStartUrl(project) - if (response?.responseCode == 200) { - val usage = getUsage() - recordIndexWorkspace(duration, filesResult.files.size, filesResult.fileSize, true, usage?.memoryUsage, usage?.cpuUsage, startUrl) - logger.debug { "project context index finished for ${project.name}" } - return true - } else { - logger.debug { "project context index failed" } - recordIndexWorkspace(duration, filesResult.files.size, filesResult.fileSize, false, null, null, startUrl) - return false - } - } - - // TODO: rename queryChat - suspend fun query(prompt: String, timeout: Long?): List = withTimeout(timeout ?: CHAT_EXPLICIT_PROJECT_CONTEXT_TIMEOUT) { - val encrypted = encryptRequest(QueryChatRequest(prompt)) - val response = sendMsgToLsp(LspMessage.QueryChat, encrypted) ?: return@withTimeout emptyList() - val parsedResponse = mapper.readValue>(response.responseBody) - - return@withTimeout queryResultToRelevantDocuments(parsedResponse) - } - - suspend fun queryInline(query: String, filePath: String, target: InlineContextTarget): List = withTimeout(SUPPLEMENTAL_CONTEXT_TIMEOUT) { - val encrypted = encryptRequest(QueryInlineCompletionRequest(query, filePath, target.toString())) - val r = sendMsgToLsp(LspMessage.QueryInlineCompletion, encrypted) ?: return@withTimeout emptyList() - return@withTimeout mapper.readValue>(r.responseBody) - } - - suspend fun getUsage(): Usage? { - val response = sendMsgToLsp(LspMessage.GetUsageMetrics, request = null) ?: return null - return try { - val parsedResponse = mapper.readValue(response.responseBody) - parsedResponse - } catch (e: Exception) { - logger.warn { "error parsing query response ${e.message}" } - null - } - } - - @RequiresBackgroundThread - fun updateIndex(filePaths: List, mode: IndexUpdateMode) { - val encrypted = encryptRequest(UpdateIndexRequest(filePaths, mode.command)) - runBlocking { sendMsgToLsp(LspMessage.UpdateIndex, encrypted) } - } - - private fun recordIndexWorkspace( - duration: Double, - fileCount: Int = 0, - fileSize: Int = 0, - isSuccess: Boolean, - memoryUsage: Int? = 0, - cpuUsage: Int? = 0, - startUrl: String? = null, - ) { - AmazonqTelemetry.indexWorkspace( - project = null, - duration = duration, - amazonqIndexFileCount = fileCount.toLong(), - amazonqIndexFileSizeInMB = fileSize.toLong(), - success = isSuccess, - amazonqIndexMemoryUsageInMB = memoryUsage?.toLong(), - amazonqIndexCpuUsagePercentage = cpuUsage?.toLong(), - credentialStartUrl = startUrl - ) - } - - private fun setConnectionTimeout(connection: HttpURLConnection, timeout: Int) { - connection.connectTimeout = timeout - connection.readTimeout = timeout - } - - private fun setConnectionProperties(connection: HttpURLConnection) { - connection.requestMethod = "POST" - connection.setRequestProperty("Content-Type", "text/plain") - connection.setRequestProperty("Accept", "text/plain") - } - - private fun setConnectionRequest(connection: HttpURLConnection, payload: String) { - connection.doOutput = true - connection.outputStream.use { outputStream -> - OutputStreamWriter(outputStream).use { writer -> - writer.write(payload) - } - } - } - - fun collectFiles(): FileCollectionResult = collectFiles(project.getBaseDirectories(), ChangeListManager.getInstance(project)) - - private fun queryResultToRelevantDocuments(queryResult: List): List { - val documents: MutableList = mutableListOf() - queryResult.forEach { chunk -> - run { - val path = chunk.relativePath.orEmpty() - val text = chunk.context ?: chunk.content.orEmpty() - val document = RelevantDocument(path, text.take(10240)) - documents.add(document) - logger.info { "project context: query retrieved document $path with content: ${text.take(200)}" } - } - } - return documents - } - - private fun encryptRequest(r: LspRequest): String { - val payloadJson = mapper.writeValueAsString(r) - return encoderServer.encrypt(payloadJson) - } - - private suspend fun sendMsgToLsp(msgType: LspMessage, request: String?): LspResponse? { - if (!encoderServer.isNodeProcessRunning()) { - logger.warn { "language server for ${project.name} is not running" } - return null - } - logger.info { "sending message: ${msgType.endpoint} to lsp on port ${encoderServer.port}" } - val url = URI("http://127.0.0.1:${encoderServer.port}/${msgType.endpoint}").toURL() - // use 1h as timeout for index, 5 seconds for other APIs - val timeoutMs = if (msgType is LspMessage.Index) 60.minutes.inWholeMilliseconds.toInt() else 5000 - // dedicate single thread to index operation because it can be long running - val dispatcher = if (msgType is LspMessage.Index) ioDispatcher(1) else ioDispatcher - - return withContext(dispatcher) { - with(url.openConnection() as HttpURLConnection) { - setConnectionProperties(this) - setConnectionTimeout(this, timeoutMs) - request?.let { r -> - setConnectionRequest(this, r) - } - val responseCode = this.responseCode - logger.info { "receiving response for $msgType with responseCode $responseCode" } - - val responseBody = if (responseCode == 200) { - this.inputStream.bufferedReader().use { reader -> reader.readText() } - } else { - "" - } - - LspResponse(responseCode, responseBody) - } - } - } - - override fun dispose() { - retryCount.set(0) - } - - companion object { - private val logger = getLogger() - private val regex = Regex("""bin|build|node_modules|venv|\.venv|env|\.idea|\.conda""", RegexOption.IGNORE_CASE) - private val mega = (1024 * 1024).toULong() - private val tenMb = 10 * mega.toInt() - - private fun willExceedPayloadLimit(maxSize: ULong, currentTotalFileSize: ULong, currentFileSize: Long) = - currentTotalFileSize.let { totalSize -> totalSize > (maxSize - currentFileSize.toUInt()) } - - private fun isBuildOrBin(fileName: String): Boolean = - regex.find(fileName) != null - - fun collectFiles(projectBaseDirectories: Set, changeListManager: ChangeListManager): FileCollectionResult { - val maxSize = CodeWhispererSettings.getInstance() - .getProjectContextIndexMaxSize().toULong() * mega - val collectedFiles = mutableListOf() - var currentTotalFileSize = 0UL - val allFiles = mutableListOf() - - projectBaseDirectories.forEach { - VfsUtilCore.visitChildrenRecursively( - it, - object : VirtualFileVisitor(NO_FOLLOW_SYMLINKS) { - // TODO: refactor this along with /dev & codescan file traversing logic - override fun visitFile(file: VirtualFile): Boolean { - if ((file.isDirectory && isBuildOrBin(file.name)) || - !isWorkspaceSourceContent(file, projectBaseDirectories, changeListManager, additionalGlobalIgnoreRulesForStrictSources) || - (file.isFile && file.length > tenMb) - ) { - return false - } - if (file.isFile) { - allFiles.add(file) - return false - } - return true - } - } - ) - } - - for (file in allFiles) { - if (willExceedPayloadLimit(maxSize, currentTotalFileSize, file.length)) { - break - } - collectedFiles.add(file.path) - currentTotalFileSize += file.length.toUInt() - } - - return FileCollectionResult( - files = collectedFiles.toList(), - fileSize = (currentTotalFileSize / 1024u / 1024u).toInt() - ) - } - } -} diff --git a/plugins/amazonq/shared/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonq/project/manifest/ManifestManager.kt b/plugins/amazonq/shared/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonq/project/manifest/ManifestManager.kt deleted file mode 100644 index 484c319db52..00000000000 --- a/plugins/amazonq/shared/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonq/project/manifest/ManifestManager.kt +++ /dev/null @@ -1,116 +0,0 @@ -// Copyright 2024 Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: Apache-2.0 - -package software.aws.toolkits.jetbrains.services.amazonq.project.manifest - -import com.fasterxml.jackson.annotation.JsonProperty -import com.fasterxml.jackson.databind.DeserializationFeature -import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper -import com.fasterxml.jackson.module.kotlin.readValue -import com.intellij.openapi.util.SystemInfo -import com.intellij.util.system.CpuArch -import software.aws.toolkits.core.utils.getLogger -import software.aws.toolkits.core.utils.warn -import software.aws.toolkits.jetbrains.core.getTextFromUrl - -class ManifestManager { - private val cloudFrontUrl = "https://aws-toolkit-language-servers.amazonaws.com/q-context/manifest.json" - val currentVersion = "0.1.49" - val currentOs = getOs() - private val arch = CpuArch.CURRENT - private val mapper = jacksonObjectMapper().apply { configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false) } - - data class TargetContent( - @JsonProperty("filename") - val filename: String? = null, - @JsonProperty("url") - val url: String? = null, - @JsonProperty("hashes") - val hashes: List? = emptyList(), - @JsonProperty("bytes") - val bytes: Number? = null, - ) - - data class VersionTarget( - @JsonProperty("platform") - val platform: String? = null, - @JsonProperty("arch") - val arch: String? = null, - @JsonProperty("contents") - val contents: List? = emptyList(), - ) - - data class Version( - @JsonProperty("serverVersion") - val serverVersion: String? = null, - @JsonProperty("isDelisted") - val isDelisted: Boolean? = null, - @JsonProperty("targets") - val targets: List? = emptyList(), - ) - - data class Manifest( - @JsonProperty("manifestSchemaVersion") - val manifestSchemaVersion: String? = null, - @JsonProperty("artifactId") - val artifactId: String? = null, - @JsonProperty("artifactDescription") - val artifactDescription: String? = null, - @JsonProperty("isManifestDeprecated") - val isManifestDeprecated: Boolean? = null, - @JsonProperty("versions") - val versions: List? = emptyList(), - ) - - fun getManifest(): Manifest? = fetchFromRemoteAndSave() - - fun readManifestFile(content: String): Manifest? { - try { - return mapper.readValue(content) - } catch (e: Exception) { - logger.warn { "error parsing manifest file for project context ${e.message}" } - return null - } - } - - private fun getTargetFromManifest(manifest: Manifest): VersionTarget? { - val targets = manifest.versions?.find { version -> version.serverVersion != null && (version.serverVersion.contains(currentVersion)) }?.targets - if (targets.isNullOrEmpty()) { - return null - } - val targetArch = if (currentOs != "windows" && arch == CpuArch.ARM64) "arm64" else "x64" - return targets.find { target -> target.platform == currentOs && target.arch == targetArch } - } - - fun getNodeContentFromManifest(manifest: Manifest): TargetContent? { - val target = getTargetFromManifest(manifest) ?: return null - return target.contents?.find { content -> content.filename?.contains("node") == true } - } - - fun getZipContentFromManifest(manifest: Manifest): TargetContent? { - val target = getTargetFromManifest(manifest) ?: return null - return target.contents?.find { content -> content.filename?.contains("qserver") == true } - } - - private fun fetchFromRemoteAndSave(): Manifest? { - try { - val response = getTextFromUrl(cloudFrontUrl) - return readManifestFile(response) - } catch (e: Exception) { - logger.warn { "failed to save manifest from remote: ${e.message}" } - return null - } - } - - fun getOs(): String = - when { - SystemInfo.isWindows -> "windows" - SystemInfo.isMac -> "darwin" - SystemInfo.isLinux -> "linux" - else -> "linux" - } - - companion object { - private val logger = getLogger() - } -} diff --git a/plugins/amazonq/shared/jetbrains-community/tst/software/aws/toolkits/jetbrains/services/amazonq/lsp/artifacts/ArtifactHelperTest.kt b/plugins/amazonq/shared/jetbrains-community/tst/software/aws/toolkits/jetbrains/services/amazonq/lsp/artifacts/ArtifactHelperTest.kt index 5fede85b27e..4a161857de4 100644 --- a/plugins/amazonq/shared/jetbrains-community/tst/software/aws/toolkits/jetbrains/services/amazonq/lsp/artifacts/ArtifactHelperTest.kt +++ b/plugins/amazonq/shared/jetbrains-community/tst/software/aws/toolkits/jetbrains/services/amazonq/lsp/artifacts/ArtifactHelperTest.kt @@ -19,9 +19,7 @@ import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test import org.junit.jupiter.api.extension.ExtendWith import org.junit.jupiter.api.io.TempDir -import org.mockito.kotlin.mock import software.aws.toolkits.jetbrains.services.amazonq.lsp.artifacts.ArtifactManager.SupportedManifestVersionRange -import software.aws.toolkits.jetbrains.services.amazonq.project.manifest.ManifestManager import java.nio.file.Path @ExtendWith(ApplicationExtension::class) @@ -31,16 +29,14 @@ class ArtifactHelperTest { private lateinit var artifactHelper: ArtifactHelper private lateinit var manifestVersionRanges: SupportedManifestVersionRange - private lateinit var mockManifestManager: ManifestManager - private lateinit var contents: List + private lateinit var contents: List private lateinit var mockProject: Project @BeforeEach fun setUp() { artifactHelper = ArtifactHelper(tempDir, 3) - mockManifestManager = mock() contents = listOf( - ManifestManager.TargetContent( + TargetContent( filename = "server.zip", hashes = listOf("sha384:1234") ) @@ -57,7 +53,7 @@ class ArtifactHelperTest { val version2Dir = tempDir.resolve("2.0.0").apply { toFile().mkdirs() } val delistedVersions = listOf( - ManifestManager.Version(serverVersion = "1.0.0") + Version(serverVersion = "1.0.0") ) artifactHelper.removeDelistedVersions(delistedVersions) @@ -125,10 +121,10 @@ class ArtifactHelperTest { serverZipPath.toFile().createNewFile() val versions = listOf( - ManifestManager.Version(serverVersion = "1.0.0") + Version(serverVersion = "1.0.0") ) - val target = ManifestManager.VersionTarget(contents = contents) + val target = VersionTarget(contents = contents) mockkStatic("software.aws.toolkits.jetbrains.services.amazonq.lsp.artifacts.LspUtilsKt") every { generateSHA384Hash(any()) } returns "1234" @@ -149,10 +145,10 @@ class ArtifactHelperTest { serverZipPath.toFile().createNewFile() val versions = listOf( - ManifestManager.Version(serverVersion = "1.0.0") + Version(serverVersion = "1.0.0") ) - val target = ManifestManager.VersionTarget(contents = contents) + val target = VersionTarget(contents = contents) mockkStatic("software.aws.toolkits.jetbrains.services.amazonq.lsp.artifacts.LspUtilsKt") every { generateSHA384Hash(any()) } returns "1235" @@ -165,14 +161,14 @@ class ArtifactHelperTest { @Test fun `getExistingLspArtifacts should return false if versions are empty`() { - val versions = emptyList() + val versions = emptyList() assertThat(artifactHelper.getExistingLspArtifacts(versions, null)).isFalse() } @Test fun `getExistingLspArtifacts should return false if target does not have contents`() { val versions = listOf( - ManifestManager.Version(serverVersion = "1.0.0") + Version(serverVersion = "1.0.0") ) assertThat(artifactHelper.getExistingLspArtifacts(versions, null)).isFalse() } @@ -180,21 +176,21 @@ class ArtifactHelperTest { @Test fun `getExistingLspArtifacts should return false if Lsp path does not exist`() { val versions = listOf( - ManifestManager.Version(serverVersion = "1.0.0") + Version(serverVersion = "1.0.0") ) assertThat(artifactHelper.getExistingLspArtifacts(versions, null)).isFalse() } @Test fun `tryDownloadLspArtifacts should not download artifacts if target does not have contents`() { - val versions = listOf(ManifestManager.Version(serverVersion = "2.0.0")) + val versions = listOf(Version(serverVersion = "2.0.0")) assertThat(runBlocking { artifactHelper.tryDownloadLspArtifacts(mockProject, versions, null) }).isEqualTo(null) assertThat(tempDir.resolve("2.0.0").toFile().exists()).isFalse() } @Test fun `tryDownloadLspArtifacts should throw error if failed to download`() { - val versions = listOf(ManifestManager.Version(serverVersion = "1.0.0")) + val versions = listOf(Version(serverVersion = "1.0.0")) val spyArtifactHelper = spyk(artifactHelper) every { spyArtifactHelper.downloadLspArtifacts(any(), any()) } returns false @@ -204,8 +200,8 @@ class ArtifactHelperTest { @Test fun `tryDownloadLspArtifacts should throw error after attempts are exhausted`() { - val versions = listOf(ManifestManager.Version(serverVersion = "1.0.0")) - val target = ManifestManager.VersionTarget(contents = contents) + val versions = listOf(Version(serverVersion = "1.0.0")) + val target = VersionTarget(contents = contents) val spyArtifactHelper = spyk(artifactHelper) every { spyArtifactHelper.downloadLspArtifacts(any(), any()) } returns true diff --git a/plugins/amazonq/shared/jetbrains-community/tst/software/aws/toolkits/jetbrains/services/amazonq/lsp/artifacts/ArtifactManagerTest.kt b/plugins/amazonq/shared/jetbrains-community/tst/software/aws/toolkits/jetbrains/services/amazonq/lsp/artifacts/ArtifactManagerTest.kt index ae68c99d740..e16f102fbc6 100644 --- a/plugins/amazonq/shared/jetbrains-community/tst/software/aws/toolkits/jetbrains/services/amazonq/lsp/artifacts/ArtifactManagerTest.kt +++ b/plugins/amazonq/shared/jetbrains-community/tst/software/aws/toolkits/jetbrains/services/amazonq/lsp/artifacts/ArtifactManagerTest.kt @@ -21,7 +21,6 @@ import org.junit.jupiter.api.assertThrows import org.junit.jupiter.api.extension.RegisterExtension import org.junit.jupiter.api.io.TempDir import software.aws.toolkits.jetbrains.services.amazonq.lsp.artifacts.ArtifactManager.SupportedManifestVersionRange -import software.aws.toolkits.jetbrains.services.amazonq.project.manifest.ManifestManager import java.nio.file.Path class ArtifactManagerTest { @@ -64,7 +63,7 @@ class ArtifactManagerTest { @Test fun `fetch artifact does not have any valid lsp versions`() = runTest { - every { manifestFetcher.fetch() }.returns(ManifestManager.Manifest()) + every { manifestFetcher.fetch() }.returns(Manifest()) every { artifactManager.getLSPVersionsFromManifestWithSpecifiedRange(any()) }.returns( ArtifactManager.LSPVersions(deListedVersions = emptyList(), inRangeVersions = emptyList()) @@ -81,7 +80,7 @@ class ArtifactManagerTest { fun `fetch artifact if inRangeVersions are not available should fallback to local lsp`() = runTest { val expectedResult = listOf(Pair(tempDir, SemVer("1.0.0", 1, 0, 0))) - every { manifestFetcher.fetch() }.returns(ManifestManager.Manifest()) + every { manifestFetcher.fetch() }.returns(Manifest()) every { artifactHelper.getAllLocalLspArtifactsWithinManifestRange(any()) }.returns(expectedResult) artifactManager.fetchArtifact(projectExtension.project) @@ -92,13 +91,13 @@ class ArtifactManagerTest { @Test fun `fetch artifact have valid version in local system`() = runTest { - val target = ManifestManager.VersionTarget(platform = "temp", arch = "temp") - val versions = listOf(ManifestManager.Version("1.0.0", targets = listOf(target))) + val target = VersionTarget(platform = "temp", arch = "temp") + val versions = listOf(Version("1.0.0", targets = listOf(target))) every { artifactManager.getLSPVersionsFromManifestWithSpecifiedRange(any()) }.returns( ArtifactManager.LSPVersions(deListedVersions = emptyList(), inRangeVersions = versions) ) - every { manifestFetcher.fetch() }.returns(ManifestManager.Manifest()) + every { manifestFetcher.fetch() }.returns(Manifest()) mockkStatic("software.aws.toolkits.jetbrains.services.amazonq.lsp.artifacts.LspUtilsKt") every { getCurrentOS() }.returns("temp") @@ -116,14 +115,14 @@ class ArtifactManagerTest { @Test fun `fetch artifact does not have valid version in local system`() = runTest { - val target = ManifestManager.VersionTarget(platform = "temp", arch = "temp") - val versions = listOf(ManifestManager.Version("1.0.0", targets = listOf(target))) + val target = VersionTarget(platform = "temp", arch = "temp") + val versions = listOf(Version("1.0.0", targets = listOf(target))) val expectedResult = listOf(Pair(tempDir, SemVer("1.0.0", 1, 0, 0))) every { artifactManager.getLSPVersionsFromManifestWithSpecifiedRange(any()) }.returns( ArtifactManager.LSPVersions(deListedVersions = emptyList(), inRangeVersions = versions) ) - every { manifestFetcher.fetch() }.returns(ManifestManager.Manifest()) + every { manifestFetcher.fetch() }.returns(Manifest()) mockkStatic("software.aws.toolkits.jetbrains.services.amazonq.lsp.artifacts.LspUtilsKt") every { getCurrentOS() }.returns("temp") diff --git a/plugins/amazonq/shared/jetbrains-community/tst/software/aws/toolkits/jetbrains/services/amazonq/lsp/artifacts/ManifestFetcherTest.kt b/plugins/amazonq/shared/jetbrains-community/tst/software/aws/toolkits/jetbrains/services/amazonq/lsp/artifacts/ManifestFetcherTest.kt index b5a1bd32fac..d343f69037c 100644 --- a/plugins/amazonq/shared/jetbrains-community/tst/software/aws/toolkits/jetbrains/services/amazonq/lsp/artifacts/ManifestFetcherTest.kt +++ b/plugins/amazonq/shared/jetbrains-community/tst/software/aws/toolkits/jetbrains/services/amazonq/lsp/artifacts/ManifestFetcherTest.kt @@ -20,7 +20,6 @@ import org.mockito.kotlin.spy import org.mockito.kotlin.verify import org.mockito.kotlin.whenever import software.aws.toolkits.jetbrains.core.getTextFromUrl -import software.aws.toolkits.jetbrains.services.amazonq.project.manifest.ManifestManager import java.nio.file.Path import java.nio.file.Paths @@ -28,14 +27,12 @@ import java.nio.file.Paths class ManifestFetcherTest { private lateinit var manifestFetcher: ManifestFetcher - private lateinit var manifest: ManifestManager.Manifest - private lateinit var manifestManager: ManifestManager + private lateinit var manifest: Manifest @BeforeEach fun setup() { manifestFetcher = spy(ManifestFetcher()) - manifestManager = spy(ManifestManager()) - manifest = ManifestManager.Manifest() + manifest = Manifest() } @Test @@ -75,7 +72,7 @@ class ManifestFetcherTest { @Test fun `fetchManifestFromRemote should return manifest and update manifest`() { - val validManifest = ManifestManager.Manifest(manifestSchemaVersion = "1.0") + val validManifest = Manifest(manifestSchemaVersion = "1.0") mockkStatic("software.aws.toolkits.jetbrains.core.HttpUtilsKt") every { getTextFromUrl(any()) } returns "{ \"manifestSchemaVersion\": \"1.0\" }"