Skip to content

Commit b1d627f

Browse files
nnobelissschuberth
nnobelis
authored and
sschuberth
committed
fix(fossid-webapp): Remove the archive prefix from the scan result
When the scan is created in archive upload mode, all the file paths are prefixed with the name of the archive received by FossId e.g. fossid-source-archive832364312005325574.zip/test.txt. Consequently, this dynamic path names are visible in the snippet reports and no snippet choice can be made as the path always changes. This commit addresses the issue by removing this prefix when listing FossId scan results. When a request affecting a file is sent (mark as identified, add a comment), the prefix needs to be prepended to the file path again. Signed-off-by: Nicolas Nobelis <[email protected]>
1 parent 98b3895 commit b1d627f

File tree

4 files changed

+280
-12
lines changed

4 files changed

+280
-12
lines changed

‎plugins/scanners/fossid/src/main/kotlin/FossId.kt‎

Lines changed: 66 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@ import kotlinx.coroutines.withTimeoutOrNull
3939
import org.apache.logging.log4j.kotlin.logger
4040

4141
import org.ossreviewtoolkit.clients.fossid.FossIdRestService
42+
import org.ossreviewtoolkit.clients.fossid.PolymorphicList
4243
import org.ossreviewtoolkit.clients.fossid.addComponentIdentification
4344
import org.ossreviewtoolkit.clients.fossid.addFileComment
4445
import org.ossreviewtoolkit.clients.fossid.checkResponse
@@ -55,6 +56,9 @@ import org.ossreviewtoolkit.clients.fossid.listSnippets
5556
import org.ossreviewtoolkit.clients.fossid.markAsIdentified
5657
import org.ossreviewtoolkit.clients.fossid.model.Project
5758
import org.ossreviewtoolkit.clients.fossid.model.Scan
59+
import org.ossreviewtoolkit.clients.fossid.model.identification.identifiedFiles.IdentifiedFile
60+
import org.ossreviewtoolkit.clients.fossid.model.identification.ignored.IgnoredFile
61+
import org.ossreviewtoolkit.clients.fossid.model.identification.markedAsIdentified.MarkedAsIdentifiedFile
5862
import org.ossreviewtoolkit.clients.fossid.model.result.MatchType
5963
import org.ossreviewtoolkit.clients.fossid.model.result.MatchedLines
6064
import org.ossreviewtoolkit.clients.fossid.model.status.ScanStatus
@@ -820,10 +824,16 @@ class FossId internal constructor(
820824
}
821825
}
822826

827+
// Here the search for the archive prefix cannot be conditioned to config.isArchiveUploadMode because the
828+
// current run could be configured in clone repository mode but the scan is a previous scan created in archive
829+
// upload mode.
830+
val archivePrefix = getArchivePrefix(pendingFiles, identifiedFiles, markedAsIdentifiedFiles, listIgnoredFiles)
831+
823832
val matchedLines = mutableMapOf<Int, MatchedLines>()
824833
val pendingFilesIterator = pendingFiles.iterator()
825834
val snippets = flow {
826835
while (pendingFilesIterator.hasNext()) {
836+
// Here the pending files still have their prefix as it is required to list the snippets.
827837
val file = pendingFilesIterator.next()
828838
logger.info { "Listing snippet for $file..." }
829839

@@ -862,20 +872,50 @@ class FossId internal constructor(
862872
}
863873
}
864874

865-
emit(file to filteredSnippets.toSet())
875+
val fileWithoutPrefix = archivePrefix?.let { file.removePrefix(it) } ?: file
876+
877+
emit(fileWithoutPrefix to filteredSnippets.toSet())
866878
}
867879
}
868880

869-
return RawResults(
881+
return RawResults.createAndRemovePrefix(
870882
identifiedFiles,
871883
markedAsIdentifiedFiles,
872884
listIgnoredFiles,
873885
pendingFiles,
874886
snippets,
875-
matchedLines
887+
matchedLines,
888+
archivePrefix
876889
)
877890
}
878891

892+
/**
893+
* When the scan is created in Archive upload mode, all the file paths are prefixed with the archive name e.g.
894+
* fossid-source-archive832364312005325574.zip/test.txt. This function searches this prefix in the list of pending
895+
* files or in the list of identified files. If no such prefix is found, null is returned.
896+
*/
897+
internal fun getArchivePrefix(
898+
pendingFiles: List<String>,
899+
identifiedFiles: List<IdentifiedFile>,
900+
markedAsIdentifiedFiles: List<MarkedAsIdentifiedFile>,
901+
listIgnoredFiles: PolymorphicList<IgnoredFile>
902+
): String? {
903+
val prefix = pendingFiles.findPrefix { it }
904+
?: identifiedFiles.findPrefix { it.file.path }
905+
?: markedAsIdentifiedFiles.findPrefix { it.file.path }
906+
?: listIgnoredFiles.findPrefix { it.path }
907+
908+
return prefix?.let {
909+
logger.info { "Found archive prefix '$prefix' from snippets." }
910+
"$prefix/"
911+
}
912+
}
913+
914+
private fun <T> List<T>.findPrefix(selector: (T) -> String?): String? =
915+
firstOrNull()
916+
?.takeIf { selector(it)?.startsWith("fossid-source-archive") == true }
917+
?.let { selector(it)?.substringBefore("/") }
918+
879919
/**
880920
* Construct the [ScanSummary] for this FossID scan.
881921
*/
@@ -906,7 +946,8 @@ class FossId internal constructor(
906946
snippetChoices,
907947
snippetFindings,
908948
rawResults.listPendingFiles,
909-
snippetLicenseFindings
949+
snippetLicenseFindings,
950+
rawResults.archivePrefix
910951
)
911952

912953
val pendingFilesCount = (rawResults.listPendingFiles - newlyMarkedFiles.toSet()).size
@@ -948,7 +989,9 @@ class FossId internal constructor(
948989

949990
/**
950991
* Mark all the files in [snippetChoices] as identified, only after searching in [snippetFindings] that they have no
951-
* non-chosen source location remaining. Only files in [listPendingFiles] are marked.
992+
* non-chosen source location remaining. Only files in [listPendingFiles] are marked. Prefix [archivePrefix]
993+
* prepended to the files paths when the scan is done in archive upload mode needs to be prepended when sending
994+
* requests that affect a specific file.
952995
* Files marked as identified have a license identification and a source location (stored in a comment), using
953996
* [licenseFindings] as reference.
954997
* Returns the list of files that have been marked as identified.
@@ -958,7 +1001,8 @@ class FossId internal constructor(
9581001
snippetChoices: List<SnippetChoice> = emptyList(),
9591002
snippetFindings: Set<SnippetFinding>,
9601003
pendingFiles: List<String>,
961-
licenseFindings: Set<LicenseFinding>
1004+
licenseFindings: Set<LicenseFinding>,
1005+
archivePrefix: String?
9621006
): List<String> {
9631007
val licenseFindingsByPath = licenseFindings.groupBy { it.location.path }
9641008
val result = mutableListOf<String>()
@@ -982,8 +1026,15 @@ class FossId internal constructor(
9821026
"findings. The used reasons are: ${reasons.joinToString()}"
9831027
}
9841028

1029+
val pathForRequest = archivePrefix?.let { "$it$path" } ?: path
9851030
requests += async {
986-
service.markAsIdentified(config.user.value, config.apiKey.value, scanCode, path, false)
1031+
service.markAsIdentified(
1032+
config.user.value,
1033+
config.apiKey.value,
1034+
scanCode,
1035+
pathForRequest,
1036+
false
1037+
)
9871038
result += path
9881039
}
9891040

@@ -1012,7 +1063,7 @@ class FossId internal constructor(
10121063
config.user.value,
10131064
config.apiKey.value,
10141065
scanCode,
1015-
path,
1066+
pathForRequest,
10161067
artifact,
10171068
version,
10181069
false
@@ -1045,7 +1096,13 @@ class FossId internal constructor(
10451096
"relevant count $notRelevantChoicesCount."
10461097
}
10471098

1048-
service.addFileComment(config.user.value, config.apiKey.value, scanCode, path, jsonComment)
1099+
service.addFileComment(
1100+
config.user.value,
1101+
config.apiKey.value,
1102+
scanCode,
1103+
pathForRequest,
1104+
jsonComment
1105+
)
10491106
}
10501107
}
10511108
}

‎plugins/scanners/fossid/src/main/kotlin/FossIdScanResults.kt‎

Lines changed: 53 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,9 @@ package org.ossreviewtoolkit.plugins.scanners.fossid
2121

2222
import java.lang.invoke.MethodHandles
2323

24+
import kotlin.collections.map
25+
import kotlin.text.removePrefix
26+
2427
import kotlinx.coroutines.flow.Flow
2528
import kotlinx.coroutines.flow.transformWhile
2629

@@ -68,8 +71,56 @@ internal data class RawResults(
6871
val listIgnoredFiles: List<IgnoredFile>,
6972
val listPendingFiles: List<String>,
7073
val listSnippets: Flow<Pair<String, Set<Snippet>>>,
71-
val snippetMatchedLines: Map<Int, MatchedLines> = emptyMap()
72-
)
74+
val snippetMatchedLines: Map<Int, MatchedLines> = emptyMap(),
75+
76+
/**
77+
* If the scan was done in archive upload mode, this is the prefix FossId uses before each file path, e.g.
78+
* fossid-source-archive832364312005325574.zip/.
79+
*/
80+
val archivePrefix: String? = null
81+
) {
82+
companion object {
83+
@Suppress("LongParameterList")
84+
fun createAndRemovePrefix(
85+
identifiedFiles: List<IdentifiedFile>,
86+
markedAsIdentifiedFiles: List<MarkedAsIdentifiedFile>,
87+
listIgnoredFiles: List<IgnoredFile>,
88+
listPendingFiles: List<String>,
89+
listSnippets: Flow<Pair<String, Set<Snippet>>>,
90+
snippetMatchedLines: Map<Int, MatchedLines> = emptyMap(),
91+
archivePrefix: String?
92+
): RawResults =
93+
if (archivePrefix == null) {
94+
RawResults(
95+
identifiedFiles,
96+
markedAsIdentifiedFiles,
97+
listIgnoredFiles,
98+
listPendingFiles,
99+
listSnippets,
100+
snippetMatchedLines,
101+
null
102+
)
103+
} else {
104+
identifiedFiles.forEach {
105+
it.file = it.file.copy(path = it.file.path?.removePrefix(archivePrefix))
106+
}
107+
108+
markedAsIdentifiedFiles.forEach {
109+
it.file = it.file.copy(path = it.file.path?.removePrefix(archivePrefix))
110+
}
111+
112+
RawResults(
113+
identifiedFiles,
114+
markedAsIdentifiedFiles,
115+
listIgnoredFiles.map { it.copy(path = it.path.removePrefix(archivePrefix)) },
116+
listPendingFiles.map { it.removePrefix(archivePrefix) },
117+
listSnippets,
118+
snippetMatchedLines,
119+
archivePrefix
120+
)
121+
}
122+
}
123+
}
73124

74125
/**
75126
* A data class to hold FossID mapped results.

‎plugins/scanners/fossid/src/test/kotlin/FossIdSnippetChoiceTest.kt‎

Lines changed: 89 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,8 @@ import org.ossreviewtoolkit.clients.fossid.PolymorphicList
4141
import org.ossreviewtoolkit.clients.fossid.PolymorphicResponseBody
4242
import org.ossreviewtoolkit.clients.fossid.addComponentIdentification
4343
import org.ossreviewtoolkit.clients.fossid.addFileComment
44+
import org.ossreviewtoolkit.clients.fossid.createScan
45+
import org.ossreviewtoolkit.clients.fossid.extractArchives
4446
import org.ossreviewtoolkit.clients.fossid.listIdentifiedFiles
4547
import org.ossreviewtoolkit.clients.fossid.listIgnoredFiles
4648
import org.ossreviewtoolkit.clients.fossid.listMarkedAsIdentifiedFiles
@@ -53,7 +55,9 @@ import org.ossreviewtoolkit.clients.fossid.model.result.MatchType
5355
import org.ossreviewtoolkit.clients.fossid.model.result.MatchedLines
5456
import org.ossreviewtoolkit.clients.fossid.model.result.Snippet
5557
import org.ossreviewtoolkit.clients.fossid.model.status.ScanStatus
58+
import org.ossreviewtoolkit.clients.fossid.removeUploadedContent
5659
import org.ossreviewtoolkit.clients.fossid.unmarkAsIdentified
60+
import org.ossreviewtoolkit.clients.fossid.uploadFile
5761
import org.ossreviewtoolkit.downloader.VersionControlSystem
5862
import org.ossreviewtoolkit.model.Severity
5963
import org.ossreviewtoolkit.model.TextLocation
@@ -70,6 +74,7 @@ import org.ossreviewtoolkit.utils.spdx.toSpdx
7074
/** Sample files in the results. **/
7175
private const val FILE_1 = "a.java"
7276
private const val FILE_2 = "b.java"
77+
private const val FILE_1_ARCHIVE_MODE = "fossid-source-archive832364312005325574.zip/a.java"
7378

7479
/** A sample purl in the results. **/
7580
private const val PURL_1 = "pkg:github/fakeuser/[email protected]"
@@ -312,6 +317,89 @@ class FossIdSnippetChoiceTest : WordSpec({
312317
}
313318
}
314319

320+
"mark a file with all snippets chosen as identified when the scan has been created in archive mode" {
321+
val branchName = "aTestBranch"
322+
val projectCode = PROJECT
323+
val scanCode = scanCode(PROJECT, null)
324+
val config = createConfig(deltaScans = false, fetchSnippetMatchedLines = true, isArchiveMode = true)
325+
val vcsInfo = createVcsInfo()
326+
val scan = createScan(vcsInfo.url, "${vcsInfo.revision}_other", scanCode)
327+
val pkgId = createIdentifier(index = 42)
328+
329+
val service = FossIdRestService.create(config.serverUrl)
330+
.expectProjectRequest(projectCode)
331+
.expectListScans(projectCode, listOf(scan))
332+
.expectCheckScanStatus(scanCode, ScanStatus.FINISHED)
333+
.expectCreateScan(projectCode, scanCode, vcsInfo, branchName, isArchiveMode = true)
334+
.expectRemoveUploadedContent(scanCode)
335+
.expectUploadFile(scanCode)
336+
.expectExtractArchives(scanCode)
337+
.mockFiles(
338+
scanCode,
339+
pendingFiles = listOf(FILE_1_ARCHIVE_MODE),
340+
snippets = listOf(
341+
createSnippet(0, FILE_1, PURL_1)
342+
),
343+
matchedLines = mapOf(
344+
0 to MatchedLines.create((10..20).toList(), (10..20).toList())
345+
)
346+
)
347+
// The mark as identified call is made on the real snippet path.
348+
.expectMarkAsIdentified(scanCode, FILE_1_ARCHIVE_MODE)
349+
350+
// The snippet choice is make on the truncated snippet path.
351+
val choiceLocation = TextLocation(FILE_1, 10, 20)
352+
val snippetChoices = createSnippetChoices(
353+
vcsInfo.url,
354+
createSnippetChoice(choiceLocation, PURL_1, comment = "")
355+
)
356+
val fossId = createFossId(config)
357+
358+
val comment = createOrtScanComment(vcsInfo.url, vcsInfo.revision, branchName).asJsonString()
359+
val summary = fossId.scan(
360+
createPackage(pkgId, vcsInfo),
361+
snippetChoices = snippetChoices,
362+
labels = mapOf(FossId.PROJECT_REVISION_LABEL to branchName)
363+
).summary
364+
365+
summary.snippetFindings should beEmpty()
366+
coVerify {
367+
service.createScan(USER, API_KEY, projectCode, scanCode, null, null, comment)
368+
service.removeUploadedContent(USER, API_KEY, scanCode)
369+
service.uploadFile(USER, API_KEY, scanCode, any())
370+
service.extractArchives(USER, API_KEY, scanCode, any())
371+
service.markAsIdentified(USER, API_KEY, scanCode, FILE_1_ARCHIVE_MODE, any())
372+
service.addComponentIdentification(
373+
user = USER,
374+
apiKey = API_KEY,
375+
scanCode = scanCode,
376+
path = FILE_1_ARCHIVE_MODE,
377+
componentName = "fakepackage1",
378+
componentVersion = "1.0.0",
379+
isDirectory = false,
380+
preserveExistingIdentifications = true
381+
)
382+
383+
val payload = OrtCommentPayload(mapOf("MIT" to listOf(choiceLocation)), 1, 0)
384+
val jsonComment = jsonMapper.writeValueAsString(OrtComment(payload))
385+
386+
service.addFileComment(
387+
user = USER,
388+
apiKey = API_KEY,
389+
scanCode = scanCode,
390+
path = FILE_1_ARCHIVE_MODE,
391+
comment = jsonComment,
392+
isImportant = false,
393+
includeInReport = false
394+
)
395+
}
396+
397+
summary.issues.forAtLeastOne {
398+
it.message shouldBe "This scan has 0 file(s) pending identification in FossID. " +
399+
"Please review and resolve them at: https://www.example.org/fossid/index.html?action=scanview&sid=1"
400+
}
401+
}
402+
315403
"mark a file with only non relevant snippets for a given snippet location as identified" {
316404
val projectCode = PROJECT
317405
val scanCode = scanCode(PROJECT, null)
@@ -952,7 +1040,7 @@ private fun createSnippetChoice(location: TextLocation, purl: String? = null, co
9521040
)
9531041
)
9541042

955-
fun FossIdServiceWithVersion.mockFiles(
1043+
internal fun FossIdServiceWithVersion.mockFiles(
9561044
scanCode: String,
9571045
markedFiles: List<MarkedAsIdentifiedFile> = emptyList(),
9581046
pendingFiles: List<String> = emptyList(),

0 commit comments

Comments
 (0)