Skip to content

Commit b12d31f

Browse files
nnobelisnnobelis
committed
fix(fossid-webapp): Ignore the archive prefix for marked files
This is a fixup for b1d627f. Signed-off-by: Nicolas Nobelis <[email protected]>
1 parent 7f4f6c5 commit b12d31f

File tree

3 files changed

+127
-8
lines changed

3 files changed

+127
-8
lines changed

‎plugins/scanners/fossid/src/main/kotlin/FossId.kt‎

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -813,7 +813,16 @@ class FossId internal constructor(
813813
"${pendingFiles.size} pending files have been returned for scan '$scanCode'."
814814
}
815815

816-
pendingFiles += listUnmatchedSnippetChoices(markedAsIdentifiedFiles, snippetChoices).also { newPendingFiles ->
816+
// Here the search for the archive prefix cannot be conditioned to config.isArchiveUploadMode because the
817+
// current run could be configured in clone repository mode but the scan is a previous scan created in archive
818+
// upload mode.
819+
val archivePrefix = getArchivePrefix(pendingFiles, identifiedFiles, markedAsIdentifiedFiles, listIgnoredFiles)
820+
821+
pendingFiles += listUnmatchedSnippetChoices(
822+
markedAsIdentifiedFiles,
823+
snippetChoices,
824+
archivePrefix
825+
).also { newPendingFiles ->
817826
newPendingFiles.map {
818827
logger.info {
819828
"Marked as identified file '$it' is not in .ort.yml anymore or its configuration has been " +
@@ -824,11 +833,6 @@ class FossId internal constructor(
824833
}
825834
}
826835

827-
// Here the search for the archive prefix cannot be conditioned to config.isArchiveUploadMode because the
828-
// current run could be configured in clone repository mode but the scan is a previous scan created in archive
829-
// upload mode.
830-
val archivePrefix = getArchivePrefix(pendingFiles, identifiedFiles, markedAsIdentifiedFiles, listIgnoredFiles)
831-
832836
val matchedLines = mutableMapOf<Int, MatchedLines>()
833837
val pendingFilesIterator = pendingFiles.iterator()
834838
val snippets = flow {

‎plugins/scanners/fossid/src/main/kotlin/FossIdScanResults.kt‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -438,15 +438,19 @@ private fun getLicenseFindingFromSnippetChoice(
438438
* Check all [markedAsIdentifiedFiles] if their snippet choices locations count or non-relevant snippets locations count
439439
* matches the ones stored in the [OrtComment]: When not, it means some of this configuration has been removed and the
440440
* files should be considered as pending again. Such files are returned.
441+
* If present, the optional [archivePrefix] is removed from the marked files names when looking for their corresponding
442+
* snippet choices.
441443
*/
442444
internal fun listUnmatchedSnippetChoices(
443445
markedAsIdentifiedFiles: List<MarkedAsIdentifiedFile>,
444-
snippetChoices: List<SnippetChoice>
446+
snippetChoices: List<SnippetChoice>,
447+
archivePrefix: String?
445448
): List<String> =
446449
markedAsIdentifiedFiles.filterNot { markedAsIdentifiedFile ->
447450
val markedFileName = markedAsIdentifiedFile.getFileName()
451+
val markedFileNameWithoutPrefix = archivePrefix?.let { markedFileName.removePrefix(it) } ?: markedFileName
448452
val snippetChoicesByName = snippetChoices.filter {
449-
it.given.sourceLocation.path == markedFileName
453+
it.given.sourceLocation.path == markedFileNameWithoutPrefix
450454
}
451455

452456
val comment = markedAsIdentifiedFile.comments.values.firstOrNull {

‎plugins/scanners/fossid/src/test/kotlin/FossIdSnippetChoiceTest.kt‎

Lines changed: 111 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -678,6 +678,49 @@ class FossIdSnippetChoiceTest : WordSpec({
678678
summary.snippetFindings should beEmpty()
679679
}
680680

681+
"add the license of already marked file with a snippet choice to the license findings (scan in archive mode)" {
682+
val projectCode = PROJECT
683+
val scanCode = scanCode(PROJECT, null)
684+
val config = createConfig(deltaScans = false, fetchSnippetMatchedLines = true, isArchiveMode = true)
685+
val vcsInfo = createVcsInfo()
686+
val scan = createScan(vcsInfo.url, "${vcsInfo.revision}_other", scanCode)
687+
val pkgId = createIdentifier(index = 42)
688+
689+
val choiceLocation = TextLocation(FILE_1, 10, 20)
690+
val payload = OrtCommentPayload(mapOf("MIT" to listOf(choiceLocation)), 1, 0)
691+
val comment = jsonMapper.writeValueAsString(mapOf(ORT_NAME to payload))
692+
val markedAsIdentifiedFile = createMarkAsIdentifiedFile("MIT", FILE_1_ARCHIVE_MODE, comment)
693+
FossIdRestService.create(config.serverUrl)
694+
.expectProjectRequest(projectCode)
695+
.expectListScans(projectCode, listOf(scan))
696+
.expectCheckScanStatus(scanCode, ScanStatus.FINISHED)
697+
.expectCreateScan(projectCode, scanCode, vcsInfo, "", isArchiveMode = true)
698+
.expectRemoveUploadedContent(scanCode)
699+
.expectUploadFile(scanCode)
700+
.expectExtractArchives(scanCode)
701+
.mockFiles(
702+
scanCode,
703+
markedFiles = listOf(markedAsIdentifiedFile)
704+
)
705+
706+
val snippetChoices = createSnippetChoices(
707+
vcsInfo.url,
708+
createSnippetChoice(choiceLocation, PURL_1, "")
709+
)
710+
val fossId = createFossId(config)
711+
712+
val summary = fossId.scan(createPackage(pkgId, vcsInfo), snippetChoices = snippetChoices).summary
713+
714+
summary.licenseFindings shouldHaveSize 1
715+
summary.licenseFindings.first().apply {
716+
license shouldBe "MIT".toSpdx()
717+
location shouldBe choiceLocation
718+
}
719+
720+
summary.issues.filter { it.severity > Severity.HINT } should beEmpty()
721+
summary.snippetFindings should beEmpty()
722+
}
723+
681724
"add the license of marked as identified files that have been manually marked in the UI (legacy behavior)" {
682725
val projectCode = PROJECT
683726
val scanCode = scanCode(PROJECT, null)
@@ -775,6 +818,74 @@ class FossIdSnippetChoiceTest : WordSpec({
775818
}
776819
}
777820

821+
"put a marked as identified file back to pending if it has no snippet choice (scan in archive mode)" {
822+
val projectCode = PROJECT
823+
val scanCode = scanCode(PROJECT, null)
824+
val config = createConfig(deltaScans = false, fetchSnippetMatchedLines = true, isArchiveMode = true)
825+
val vcsInfo = createVcsInfo()
826+
val scan = createScan(vcsInfo.url, "${vcsInfo.revision}_other", scanCode)
827+
val pkgId = createIdentifier(index = 42)
828+
829+
val choiceLocation = TextLocation(FILE_1, 10, 20)
830+
val payload = OrtCommentPayload(mapOf("MIT" to listOf(choiceLocation)), 1, 0)
831+
val comment = jsonMapper.writeValueAsString(mapOf(ORT_NAME to payload))
832+
val markedAsIdentifiedFile = createMarkAsIdentifiedFile("MIT", FILE_1_ARCHIVE_MODE, comment)
833+
val service = FossIdRestService.create(config.serverUrl)
834+
.expectProjectRequest(projectCode)
835+
.expectListScans(projectCode, listOf(scan))
836+
.expectCheckScanStatus(scanCode, ScanStatus.FINISHED)
837+
.expectCreateScan(projectCode, scanCode, vcsInfo, "", isArchiveMode = true)
838+
.expectRemoveUploadedContent(scanCode)
839+
.expectUploadFile(scanCode)
840+
.expectExtractArchives(scanCode)
841+
.mockFiles(
842+
scanCode,
843+
markedFiles = listOf(markedAsIdentifiedFile),
844+
snippets = listOf(
845+
createSnippet(0, FILE_1, PURL_1),
846+
createSnippet(1, FILE_1, PURL_2),
847+
createSnippet(2, FILE_1, PURL_3)
848+
),
849+
matchedLines = mapOf(
850+
0 to MatchedLines.create((10..20).toList(), (10..20).toList()),
851+
1 to MatchedLines.create((10..20).toList(), (10..20).toList()),
852+
2 to MatchedLines.create((20..30).toList(), (20..30).toList())
853+
)
854+
)
855+
// The unmark as identified call is made on the real snippet path.
856+
.expectUnmarkAsIdentified(scanCode, FILE_1_ARCHIVE_MODE)
857+
858+
val fossId = createFossId(config)
859+
860+
val summary = fossId.scan(createPackage(pkgId, vcsInfo), snippetChoices = emptyList()).summary
861+
862+
summary.issues.forAtLeastOne {
863+
it.message shouldBe "This scan has 1 file(s) pending identification in FossID. " +
864+
"Please review and resolve them at: https://www.example.org/fossid/index.html?action=scanview&sid=1"
865+
}
866+
867+
summary.issues.filter { it.severity > Severity.HINT } should beEmpty()
868+
summary.snippetFindings shouldHaveSize 2
869+
summary.snippetFindings.first().apply {
870+
sourceLocation.path shouldBe FILE_1
871+
snippets shouldHaveSize 2
872+
snippets.map { it.purl } shouldBe listOf(PURL_1, PURL_2)
873+
}
874+
875+
summary.snippetFindings.last().apply {
876+
sourceLocation.path shouldBe FILE_1
877+
snippets shouldHaveSize 1
878+
snippets.map { it.purl } shouldBe listOf(PURL_3)
879+
}
880+
881+
coVerify {
882+
service.removeUploadedContent(USER, API_KEY, scanCode)
883+
service.uploadFile(USER, API_KEY, scanCode, any())
884+
service.extractArchives(USER, API_KEY, scanCode, any())
885+
service.unmarkAsIdentified(USER, API_KEY, scanCode, FILE_1_ARCHIVE_MODE, any())
886+
}
887+
}
888+
778889
"put a marked as identified file back to pending if some of its snippet choices have been deleted" {
779890
val projectCode = PROJECT
780891
val scanCode = scanCode(PROJECT, null)

0 commit comments

Comments
 (0)