Skip to content

Commit 761ce71

Browse files
ashishrp-awsrli
ashishrp-aws
and
rli
authored
Adding ChangeListManager to parse ignored Files in Project and sortin… (#4453)
* Adding ChangeListManager to parse ignored Files in Project and sorting through project modules for accurate file parsing. * Correction for project code scan error. * Adding featureDevSessionContext to add more gitignore abilities. * Added comment for using both featureDevSessionContext and ChangeListManager. * Removed featureDevSessionContext and added test for module based file collection. * detekt correction. --------- Co-authored-by: Richard Li <[email protected]>
1 parent e04a605 commit 761ce71

File tree

3 files changed

+74
-42
lines changed

3 files changed

+74
-42
lines changed

‎plugins/core/jetbrains-community/tstFixtures/software/aws/toolkits/jetbrains/utils/rules/CodeInsightTestFixtureRule.kt‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,10 @@ package software.aws.toolkits.jetbrains.utils.rules
55

66
import com.intellij.openapi.application.ApplicationManager
77
import com.intellij.openapi.application.ModalityState
8+
import com.intellij.openapi.application.WriteAction
89
import com.intellij.openapi.application.runReadAction
910
import com.intellij.openapi.application.runWriteAction
11+
import com.intellij.openapi.module.JavaModuleType
1012
import com.intellij.openapi.module.Module
1113
import com.intellij.openapi.project.Project
1214
import com.intellij.openapi.roots.ModuleRootManager
@@ -17,6 +19,7 @@ import com.intellij.psi.PsiManager
1719
import com.intellij.testFramework.ApplicationRule
1820
import com.intellij.testFramework.LightProjectDescriptor
1921
import com.intellij.testFramework.PlatformTestUtil
22+
import com.intellij.testFramework.PsiTestUtil
2023
import com.intellij.testFramework.common.ThreadLeakTracker
2124
import com.intellij.testFramework.fixtures.CodeInsightTestFixture
2225
import com.intellij.testFramework.fixtures.IdeaTestFixtureFactory
@@ -148,6 +151,17 @@ fun CodeInsightTestFixture.openFile(relativePath: String, fileText: String): Vir
148151
return file
149152
}
150153

154+
fun CodeInsightTestFixture.addModule(moduleName: String): Module {
155+
val root = this.tempDirFixture.findOrCreateDir(moduleName)
156+
val module = PsiTestUtil.addModule(project, JavaModuleType.getModuleType(), moduleName, root)
157+
runInEdtAndWait {
158+
WriteAction.run<Exception> {
159+
PsiTestUtil.addContentRoot(module, root)
160+
}
161+
}
162+
return module
163+
}
164+
151165
fun CodeInsightTestFixture.addFileToModule(
152166
module: Module,
153167
relativePath: String,

‎plugins/toolkit/jetbrains-core/src/software/aws/toolkits/jetbrains/services/codewhisperer/codescan/sessionconfig/CodeScanSessionConfig.kt‎

Lines changed: 37 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -4,19 +4,20 @@
44
package software.aws.toolkits.jetbrains.services.codewhisperer.codescan.sessionconfig
55

66
import com.intellij.openapi.project.Project
7+
import com.intellij.openapi.project.guessModuleDir
78
import com.intellij.openapi.project.guessProjectDir
9+
import com.intellij.openapi.project.modules
10+
import com.intellij.openapi.vcs.changes.ChangeListManager
811
import com.intellij.openapi.vfs.LocalFileSystem
912
import com.intellij.openapi.vfs.VFileProperty
1013
import com.intellij.openapi.vfs.VfsUtil
1114
import com.intellij.openapi.vfs.VirtualFile
1215
import kotlinx.coroutines.TimeoutCancellationException
13-
import kotlinx.coroutines.runBlocking
1416
import kotlinx.coroutines.time.withTimeout
1517
import software.aws.toolkits.core.utils.createTemporaryZipFile
1618
import software.aws.toolkits.core.utils.debug
1719
import software.aws.toolkits.core.utils.getLogger
1820
import software.aws.toolkits.core.utils.putNextEntry
19-
import software.aws.toolkits.jetbrains.services.amazonq.FeatureDevSessionContext
2021
import software.aws.toolkits.jetbrains.services.codewhisperer.codescan.fileTooLarge
2122
import software.aws.toolkits.jetbrains.services.codewhisperer.codescan.noFileOpenError
2223
import software.aws.toolkits.jetbrains.services.codewhisperer.codescan.noSupportedFilesError
@@ -53,8 +54,6 @@ class CodeScanSessionConfig(
5354

5455
private var isProjectTruncated = false
5556

56-
private val featureDevSessionContext = FeatureDevSessionContext(project)
57-
5857
/**
5958
* Timeout for the overall job - "Run Security Scan".
6059
*/
@@ -149,9 +148,10 @@ class CodeScanSessionConfig(
149148
if (scope == CodeAnalysisScope.FILE) {
150149
totalSize = selectedFile?.length ?: 0L
151150
} else {
151+
val changeListManager = ChangeListManager.getInstance(project)
152152
VfsUtil.collectChildrenRecursively(projectRoot).filter {
153153
!it.isDirectory && !it.`is`((VFileProperty.SYMLINK)) && (
154-
!featureDevSessionContext.ignoreFile(it, this)
154+
!changeListManager.isIgnoredFile(it)
155155
)
156156
}.fold(0L) { acc, next ->
157157
totalSize = acc + next.length
@@ -175,39 +175,49 @@ class CodeScanSessionConfig(
175175

176176
fun getProjectPayloadMetadata(): PayloadMetadata {
177177
val files = mutableSetOf<String>()
178+
val traversedDirectories = mutableSetOf<VirtualFile>()
178179
val stack = Stack<VirtualFile>()
179180
var currentTotalFileSize = 0L
180181
var currentTotalLines = 0L
181182
val languageCounts = mutableMapOf<CodeWhispererProgrammingLanguage, Int>()
182183

183-
stack.push(projectRoot)
184-
while (stack.isNotEmpty()) {
185-
val current = stack.pop()
186-
187-
if (!current.isDirectory) {
188-
if (runBlocking { !featureDevSessionContext.ignoreFile(current, this) }) {
189-
if (willExceedPayloadLimit(currentTotalFileSize, current.length)) {
190-
break
184+
moduleLoop@ for (module in project.modules) {
185+
val changeListManager = ChangeListManager.getInstance(module.project)
186+
if (module.guessModuleDir() != null) {
187+
stack.push(module.guessModuleDir())
188+
while (stack.isNotEmpty()) {
189+
val current = stack.pop()
190+
191+
if (!current.isDirectory) {
192+
if (!changeListManager.isIgnoredFile(current) && !files.contains(current.path)
193+
) {
194+
if (willExceedPayloadLimit(currentTotalFileSize, current.length)) {
195+
break@moduleLoop
196+
} else {
197+
val language = current.programmingLanguage()
198+
if (language != CodeWhispererPlainText.INSTANCE && language != CodeWhispererUnknownLanguage.INSTANCE) {
199+
languageCounts[language] = (languageCounts[language] ?: 0) + 1
200+
}
201+
202+
files.add(current.path)
203+
currentTotalFileSize += current.length
204+
currentTotalLines += countLinesInVirtualFile(current)
205+
}
206+
}
191207
} else {
192-
val language = current.programmingLanguage()
193-
if (language != CodeWhispererPlainText.INSTANCE && language != CodeWhispererUnknownLanguage.INSTANCE) {
194-
languageCounts[language] = (languageCounts[language] ?: 0) + 1
208+
// Directory case: only traverse if not ignored
209+
if (!changeListManager.isIgnoredFile(current) && !traversedDirectories.contains(current)
210+
) {
211+
for (child in current.children) {
212+
stack.push(child)
213+
}
195214
}
196-
197-
files.add(current.path)
198-
currentTotalFileSize += current.length
199-
currentTotalLines += countLinesInVirtualFile(current)
200-
}
201-
}
202-
} else {
203-
// Directory case: only traverse if not ignored
204-
if (runBlocking { !featureDevSessionContext.ignoreFile(current, this) }) {
205-
for (child in current.children) {
206-
stack.push(child)
215+
traversedDirectories.add(current)
207216
}
208217
}
209218
}
210219
}
220+
211221
val maxCount = languageCounts.maxByOrNull { it.value }?.value ?: 0
212222
val maxCountLanguage = languageCounts.filter { it.value == maxCount }.keys.firstOrNull()
213223

‎plugins/toolkit/jetbrains-core/tst/software/aws/toolkits/jetbrains/services/codewhisperer/codescan/CodeWhispererProjectCodeScanTest.kt‎

Lines changed: 23 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,8 @@ import org.mockito.kotlin.stub
1313
import software.aws.toolkits.jetbrains.services.codewhisperer.codescan.sessionconfig.CodeScanSessionConfig
1414
import software.aws.toolkits.jetbrains.services.codewhisperer.util.CodeWhispererConstants
1515
import software.aws.toolkits.jetbrains.utils.rules.PythonCodeInsightTestFixtureRule
16+
import software.aws.toolkits.jetbrains.utils.rules.addFileToModule
17+
import software.aws.toolkits.jetbrains.utils.rules.addModule
1618
import software.aws.toolkits.telemetry.CodewhispererLanguage
1719
import java.io.BufferedInputStream
1820
import java.util.zip.ZipInputStream
@@ -130,7 +132,10 @@ class CodeWhispererProjectCodeScanTest : CodeWhispererCodeScanTestBase(PythonCod
130132
}
131133

132134
private fun setupCsharpProject() {
133-
testCs = projectRule.fixture.addFileToProject(
135+
val testModule = projectRule.fixture.addModule("testModule")
136+
val testModule2 = projectRule.fixture.addModule("testModule2")
137+
testCs = projectRule.fixture.addFileToModule(
138+
testModule,
134139
"/Test.cs",
135140
"""
136141
using Utils;
@@ -147,7 +152,8 @@ class CodeWhispererProjectCodeScanTest : CodeWhispererCodeScanTestBase(PythonCod
147152
totalSize += testCs.length
148153
totalLines += testCs.toNioPath().toFile().readLines().size
149154

150-
utilsCs = projectRule.fixture.addFileToProject(
155+
utilsCs = projectRule.fixture.addFileToModule(
156+
testModule,
151157
"/Utils.cs",
152158
"""
153159
public static class Utils
@@ -172,7 +178,8 @@ class CodeWhispererProjectCodeScanTest : CodeWhispererCodeScanTestBase(PythonCod
172178
totalSize += utilsCs.length
173179
totalLines += utilsCs.toNioPath().toFile().readLines().size
174180

175-
helperCs = projectRule.fixture.addFileToProject(
181+
helperCs = projectRule.fixture.addFileToModule(
182+
testModule,
176183
"/Helpers/Helper.cs",
177184
"""
178185
public static class Helper
@@ -215,7 +222,8 @@ class CodeWhispererProjectCodeScanTest : CodeWhispererCodeScanTestBase(PythonCod
215222
totalSize += helperCs.length
216223
totalLines += helperCs.toNioPath().toFile().readLines().size
217224

218-
helpGo = projectRule.fixture.addFileToProject(
225+
helpGo = projectRule.fixture.addFileToModule(
226+
testModule,
219227
"/help.go",
220228
"""
221229
package main
@@ -230,7 +238,8 @@ class CodeWhispererProjectCodeScanTest : CodeWhispererCodeScanTestBase(PythonCod
230238
totalSize += helpGo.length
231239
totalLines += helpGo.toNioPath().toFile().readLines().size
232240

233-
utilsJs = projectRule.fixture.addFileToProject(
241+
utilsJs = projectRule.fixture.addFileToModule(
242+
testModule,
234243
"/utils.js",
235244
"""
236245
function add(num1, num2) {
@@ -259,7 +268,8 @@ class CodeWhispererProjectCodeScanTest : CodeWhispererCodeScanTestBase(PythonCod
259268
totalSize += utilsJs.length
260269
totalLines += utilsJs.toNioPath().toFile().readLines().size
261270

262-
testJson = projectRule.fixture.addFileToProject(
271+
testJson = projectRule.fixture.addFileToModule(
272+
testModule,
263273
"/helpers/test3Json.json",
264274
"""
265275
{
@@ -314,7 +324,8 @@ class CodeWhispererProjectCodeScanTest : CodeWhispererCodeScanTestBase(PythonCod
314324
totalSize += testJson.length
315325
totalLines += testJson.toNioPath().toFile().readLines().size
316326

317-
helperPy = projectRule.fixture.addFileToProject(
327+
helperPy = projectRule.fixture.addFileToModule(
328+
testModule,
318329
"/helpers/helper.py",
319330
"""
320331
from helpers import helper as h
@@ -331,11 +342,12 @@ class CodeWhispererProjectCodeScanTest : CodeWhispererCodeScanTestBase(PythonCod
331342
totalSize += helperPy.length
332343
totalLines += helperPy.toNioPath().toFile().readLines().size
333344

334-
readMeMd = projectRule.fixture.addFileToProject("/ReadMe.md", "### Now included").virtualFile
345+
readMeMd = projectRule.fixture.addFileToModule(testModule, "/ReadMe.md", "### Now included").virtualFile
335346
totalSize += readMeMd.length
336347
totalLines += readMeMd.toNioPath().toFile().readLines().size
337348

338-
testTf = projectRule.fixture.addFileToProject(
349+
testTf = projectRule.fixture.addFileToModule(
350+
testModule2,
339351
"/testTf.tf",
340352
"""
341353
# Create example resource for three S3 buckets using for_each, where the bucket prefix are in variable with list containing [prod, staging, dev]
@@ -354,7 +366,8 @@ class CodeWhispererProjectCodeScanTest : CodeWhispererCodeScanTestBase(PythonCod
354366
totalSize += testTf.length
355367
totalLines += testTf.toNioPath().toFile().readLines().size
356368

357-
testYaml = projectRule.fixture.addFileToProject(
369+
testYaml = projectRule.fixture.addFileToModule(
370+
testModule2,
358371
"/testYaml.yaml",
359372
"""
360373
AWSTemplateFormatVersion: "2010-09-09"
@@ -378,10 +391,5 @@ class CodeWhispererProjectCodeScanTest : CodeWhispererCodeScanTestBase(PythonCod
378391
).virtualFile
379392
totalSize += testYaml.length
380393
totalLines += testYaml.toNioPath().toFile().readLines().size
381-
382-
// Adding gitignore file and gitignore file member for testing.
383-
// The tests include the markdown file but not these two files.
384-
projectRule.fixture.addFileToProject("/.gitignore", "node_modules\n.idea\n.vscode\n.DS_Store").virtualFile
385-
projectRule.fixture.addFileToProject("test.idea", "ref: refs/heads/main")
386394
}
387395
}

0 commit comments

Comments
 (0)