Skip to content

Commit 845aa24

Browse files
nnobelisnnobelis
committed
fix(fossid-webapp): Ignore the archive prefix for marked files
This is a fixup for b1d627f. Signed-off-by: Nicolas Nobelis <[email protected]>
1 parent 7f4f6c5 commit 845aa24

File tree

3 files changed

+125
-8
lines changed

3 files changed

+125
-8
lines changed

‎plugins/scanners/fossid/src/main/kotlin/FossId.kt‎

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -813,7 +813,16 @@ class FossId internal constructor(
813813
"${pendingFiles.size} pending files have been returned for scan '$scanCode'."
814814
}
815815

816-
pendingFiles += listUnmatchedSnippetChoices(markedAsIdentifiedFiles, snippetChoices).also { newPendingFiles ->
816+
// Here the search for the archive prefix cannot be conditioned to config.isArchiveUploadMode because the
817+
// current run could be configured in clone repository mode but the scan is a previous scan created in archive
818+
// upload mode.
819+
val archivePrefix = getArchivePrefix(pendingFiles, identifiedFiles, markedAsIdentifiedFiles, listIgnoredFiles)
820+
821+
pendingFiles += listUnmatchedSnippetChoices(
822+
markedAsIdentifiedFiles,
823+
snippetChoices,
824+
archivePrefix
825+
).also { newPendingFiles ->
817826
newPendingFiles.map {
818827
logger.info {
819828
"Marked as identified file '$it' is not in .ort.yml anymore or its configuration has been " +
@@ -824,11 +833,6 @@ class FossId internal constructor(
824833
}
825834
}
826835

827-
// Here the search for the archive prefix cannot be conditioned to config.isArchiveUploadMode because the
828-
// current run could be configured in clone repository mode but the scan is a previous scan created in archive
829-
// upload mode.
830-
val archivePrefix = getArchivePrefix(pendingFiles, identifiedFiles, markedAsIdentifiedFiles, listIgnoredFiles)
831-
832836
val matchedLines = mutableMapOf<Int, MatchedLines>()
833837
val pendingFilesIterator = pendingFiles.iterator()
834838
val snippets = flow {

‎plugins/scanners/fossid/src/main/kotlin/FossIdScanResults.kt‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -438,15 +438,19 @@ private fun getLicenseFindingFromSnippetChoice(
438438
* Check all [markedAsIdentifiedFiles] if their snippet choices locations count or non-relevant snippets locations count
439439
* matches the ones stored in the [OrtComment]: When not, it means some of this configuration has been removed and the
440440
* files should be considered as pending again. Such files are returned.
441+
* If present, the optional [archivePrefix] is removed from the marked files names when looking for their corresponding
442+
* snippet choices.
441443
*/
442444
internal fun listUnmatchedSnippetChoices(
443445
markedAsIdentifiedFiles: List<MarkedAsIdentifiedFile>,
444-
snippetChoices: List<SnippetChoice>
446+
snippetChoices: List<SnippetChoice>,
447+
archivePrefix: String?
445448
): List<String> =
446449
markedAsIdentifiedFiles.filterNot { markedAsIdentifiedFile ->
447450
val markedFileName = markedAsIdentifiedFile.getFileName()
451+
val markedFileNameWithoutPrefix = archivePrefix?.let { markedFileName.removePrefix(it) } ?: markedFileName
448452
val snippetChoicesByName = snippetChoices.filter {
449-
it.given.sourceLocation.path == markedFileName
453+
it.given.sourceLocation.path == markedFileNameWithoutPrefix
450454
}
451455

452456
val comment = markedAsIdentifiedFile.comments.values.firstOrNull {

‎plugins/scanners/fossid/src/test/kotlin/FossIdSnippetChoiceTest.kt‎

Lines changed: 109 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ package org.ossreviewtoolkit.plugins.scanners.fossid
2222
import io.kotest.core.spec.style.WordSpec
2323
import io.kotest.inspectors.forAtLeastOne
2424
import io.kotest.matchers.collections.beEmpty
25+
import io.kotest.matchers.collections.shouldBeSingleton
2526
import io.kotest.matchers.collections.shouldHaveSize
2627
import io.kotest.matchers.should
2728
import io.kotest.matchers.shouldBe
@@ -678,6 +679,48 @@ class FossIdSnippetChoiceTest : WordSpec({
678679
summary.snippetFindings should beEmpty()
679680
}
680681

682+
"add the license of already marked file with a snippet choice to the license findings (scan in archive mode)" {
683+
val projectCode = PROJECT
684+
val scanCode = scanCode(PROJECT, null)
685+
val config = createConfig(deltaScans = false, fetchSnippetMatchedLines = true, isArchiveMode = true)
686+
val vcsInfo = createVcsInfo()
687+
val scan = createScan(vcsInfo.url, "${vcsInfo.revision}_other", scanCode)
688+
val pkgId = createIdentifier(index = 42)
689+
690+
val choiceLocation = TextLocation(FILE_1, 10, 20)
691+
val payload = OrtCommentPayload(mapOf("MIT" to listOf(choiceLocation)), 1, 0)
692+
val comment = jsonMapper.writeValueAsString(mapOf(ORT_NAME to payload))
693+
val markedAsIdentifiedFile = createMarkAsIdentifiedFile("MIT", FILE_1_ARCHIVE_MODE, comment)
694+
FossIdRestService.create(config.serverUrl)
695+
.expectProjectRequest(projectCode)
696+
.expectListScans(projectCode, listOf(scan))
697+
.expectCheckScanStatus(scanCode, ScanStatus.FINISHED)
698+
.expectCreateScan(projectCode, scanCode, vcsInfo, "", isArchiveMode = true)
699+
.expectRemoveUploadedContent(scanCode)
700+
.expectUploadFile(scanCode)
701+
.expectExtractArchives(scanCode)
702+
.mockFiles(
703+
scanCode,
704+
markedFiles = listOf(markedAsIdentifiedFile)
705+
)
706+
707+
val snippetChoices = createSnippetChoices(
708+
vcsInfo.url,
709+
createSnippetChoice(choiceLocation, PURL_1, "")
710+
)
711+
val fossId = createFossId(config)
712+
713+
val summary = fossId.scan(createPackage(pkgId, vcsInfo), snippetChoices = snippetChoices).summary
714+
715+
summary.licenseFindings.shouldBeSingleton {
716+
it.license shouldBe "MIT".toSpdx()
717+
it.location shouldBe choiceLocation
718+
}
719+
720+
summary.issues.filter { it.severity > Severity.HINT } should beEmpty()
721+
summary.snippetFindings should beEmpty()
722+
}
723+
681724
"add the license of marked as identified files that have been manually marked in the UI (legacy behavior)" {
682725
val projectCode = PROJECT
683726
val scanCode = scanCode(PROJECT, null)
@@ -775,6 +818,72 @@ class FossIdSnippetChoiceTest : WordSpec({
775818
}
776819
}
777820

821+
"put a marked as identified file back to pending if it has no snippet choice (scan in archive mode)" {
822+
val projectCode = PROJECT
823+
val scanCode = scanCode(PROJECT, null)
824+
val config = createConfig(deltaScans = false, fetchSnippetMatchedLines = true, isArchiveMode = true)
825+
val vcsInfo = createVcsInfo()
826+
val scan = createScan(vcsInfo.url, "${vcsInfo.revision}_other", scanCode)
827+
val pkgId = createIdentifier(index = 42)
828+
829+
val choiceLocation = TextLocation(FILE_1, 10, 20)
830+
val payload = OrtCommentPayload(mapOf("MIT" to listOf(choiceLocation)), 1, 0)
831+
val comment = jsonMapper.writeValueAsString(mapOf(ORT_NAME to payload))
832+
val markedAsIdentifiedFile = createMarkAsIdentifiedFile("MIT", FILE_1_ARCHIVE_MODE, comment)
833+
val service = FossIdRestService.create(config.serverUrl)
834+
.expectProjectRequest(projectCode)
835+
.expectListScans(projectCode, listOf(scan))
836+
.expectCheckScanStatus(scanCode, ScanStatus.FINISHED)
837+
.expectCreateScan(projectCode, scanCode, vcsInfo, "", isArchiveMode = true)
838+
.expectRemoveUploadedContent(scanCode)
839+
.expectUploadFile(scanCode)
840+
.expectExtractArchives(scanCode)
841+
.mockFiles(
842+
scanCode,
843+
markedFiles = listOf(markedAsIdentifiedFile),
844+
snippets = listOf(
845+
createSnippet(0, FILE_1, PURL_1),
846+
createSnippet(1, FILE_1, PURL_2),
847+
createSnippet(2, FILE_1, PURL_3)
848+
),
849+
matchedLines = mapOf(
850+
0 to MatchedLines.create((10..20).toList(), (10..20).toList()),
851+
1 to MatchedLines.create((10..20).toList(), (10..20).toList()),
852+
2 to MatchedLines.create((20..30).toList(), (20..30).toList())
853+
)
854+
)
855+
// The unmark as identified call is made on the real snippet path.
856+
.expectUnmarkAsIdentified(scanCode, FILE_1_ARCHIVE_MODE)
857+
858+
val fossId = createFossId(config)
859+
860+
val summary = fossId.scan(createPackage(pkgId, vcsInfo), snippetChoices = emptyList()).summary
861+
862+
summary.issues.forAtLeastOne {
863+
it.message shouldBe "This scan has 1 file(s) pending identification in FossID. " +
864+
"Please review and resolve them at: https://www.example.org/fossid/index.html?action=scanview&sid=1"
865+
}
866+
867+
summary.issues.filter { it.severity > Severity.HINT } should beEmpty()
868+
summary.snippetFindings shouldHaveSize 2
869+
summary.snippetFindings.first().apply {
870+
sourceLocation.path shouldBe FILE_1
871+
snippets.map { it.purl } shouldBe listOf(PURL_1, PURL_2)
872+
}
873+
874+
summary.snippetFindings.last().apply {
875+
sourceLocation.path shouldBe FILE_1
876+
snippets.map { it.purl } shouldBe listOf(PURL_3)
877+
}
878+
879+
coVerify {
880+
service.removeUploadedContent(USER, API_KEY, scanCode)
881+
service.uploadFile(USER, API_KEY, scanCode, any())
882+
service.extractArchives(USER, API_KEY, scanCode, any())
883+
service.unmarkAsIdentified(USER, API_KEY, scanCode, FILE_1_ARCHIVE_MODE, any())
884+
}
885+
}
886+
778887
"put a marked as identified file back to pending if some of its snippet choices have been deleted" {
779888
val projectCode = PROJECT
780889
val scanCode = scanCode(PROJECT, null)

0 commit comments

Comments
 (0)