diff --git a/plugins/scanners/fossid/src/main/kotlin/FossId.kt b/plugins/scanners/fossid/src/main/kotlin/FossId.kt index 0941d98e54768..1d4b0334cfc57 100644 --- a/plugins/scanners/fossid/src/main/kotlin/FossId.kt +++ b/plugins/scanners/fossid/src/main/kotlin/FossId.kt @@ -813,7 +813,16 @@ class FossId internal constructor( "${pendingFiles.size} pending files have been returned for scan '$scanCode'." } - pendingFiles += listUnmatchedSnippetChoices(markedAsIdentifiedFiles, snippetChoices).also { newPendingFiles -> + // Here the search for the archive prefix cannot be conditioned to config.isArchiveUploadMode because the + // current run could be configured in clone repository mode but the scan is a previous scan created in archive + // upload mode. + val archivePrefix = getArchivePrefix(pendingFiles, identifiedFiles, markedAsIdentifiedFiles, listIgnoredFiles) + + pendingFiles += listUnmatchedSnippetChoices( + markedAsIdentifiedFiles, + snippetChoices, + archivePrefix + ).also { newPendingFiles -> newPendingFiles.map { logger.info { "Marked as identified file '$it' is not in .ort.yml anymore or its configuration has been " + @@ -824,11 +833,6 @@ class FossId internal constructor( } } - // Here the search for the archive prefix cannot be conditioned to config.isArchiveUploadMode because the - // current run could be configured in clone repository mode but the scan is a previous scan created in archive - // upload mode. - val archivePrefix = getArchivePrefix(pendingFiles, identifiedFiles, markedAsIdentifiedFiles, listIgnoredFiles) - val matchedLines = mutableMapOf() val pendingFilesIterator = pendingFiles.iterator() val snippets = flow { diff --git a/plugins/scanners/fossid/src/main/kotlin/FossIdScanResults.kt b/plugins/scanners/fossid/src/main/kotlin/FossIdScanResults.kt index 739f4e0b64cc0..aff688017aaae 100644 --- a/plugins/scanners/fossid/src/main/kotlin/FossIdScanResults.kt +++ b/plugins/scanners/fossid/src/main/kotlin/FossIdScanResults.kt @@ -438,15 +438,19 @@ private fun getLicenseFindingFromSnippetChoice( * Check all [markedAsIdentifiedFiles] if their snippet choices locations count or non-relevant snippets locations count * matches the ones stored in the [OrtComment]: When not, it means some of this configuration has been removed and the * files should be considered as pending again. Such files are returned. + * If present, the optional [archivePrefix] is removed from the marked files names when looking for their corresponding + * snippet choices. */ internal fun listUnmatchedSnippetChoices( markedAsIdentifiedFiles: List, - snippetChoices: List + snippetChoices: List, + archivePrefix: String? ): List = markedAsIdentifiedFiles.filterNot { markedAsIdentifiedFile -> val markedFileName = markedAsIdentifiedFile.getFileName() + val markedFileNameWithoutPrefix = archivePrefix?.let { markedFileName.removePrefix(it) } ?: markedFileName val snippetChoicesByName = snippetChoices.filter { - it.given.sourceLocation.path == markedFileName + it.given.sourceLocation.path == markedFileNameWithoutPrefix } val comment = markedAsIdentifiedFile.comments.values.firstOrNull { diff --git a/plugins/scanners/fossid/src/test/kotlin/FossIdSnippetChoiceTest.kt b/plugins/scanners/fossid/src/test/kotlin/FossIdSnippetChoiceTest.kt index c7b155a1762b7..7eccde2af1bf4 100644 --- a/plugins/scanners/fossid/src/test/kotlin/FossIdSnippetChoiceTest.kt +++ b/plugins/scanners/fossid/src/test/kotlin/FossIdSnippetChoiceTest.kt @@ -22,6 +22,7 @@ package org.ossreviewtoolkit.plugins.scanners.fossid import io.kotest.core.spec.style.WordSpec import io.kotest.inspectors.forAtLeastOne import io.kotest.matchers.collections.beEmpty +import io.kotest.matchers.collections.shouldBeSingleton import io.kotest.matchers.collections.shouldHaveSize import io.kotest.matchers.should import io.kotest.matchers.shouldBe @@ -678,6 +679,48 @@ class FossIdSnippetChoiceTest : WordSpec({ summary.snippetFindings should beEmpty() } + "add the license of already marked file with a snippet choice to the license findings (scan in archive mode)" { + val projectCode = PROJECT + val scanCode = scanCode(PROJECT, null) + val config = createConfig(deltaScans = false, fetchSnippetMatchedLines = true, isArchiveMode = true) + val vcsInfo = createVcsInfo() + val scan = createScan(vcsInfo.url, "${vcsInfo.revision}_other", scanCode) + val pkgId = createIdentifier(index = 42) + + val choiceLocation = TextLocation(FILE_1, 10, 20) + val payload = OrtCommentPayload(mapOf("MIT" to listOf(choiceLocation)), 1, 0) + val comment = jsonMapper.writeValueAsString(mapOf(ORT_NAME to payload)) + val markedAsIdentifiedFile = createMarkAsIdentifiedFile("MIT", FILE_1_ARCHIVE_MODE, comment) + FossIdRestService.create(config.serverUrl) + .expectProjectRequest(projectCode) + .expectListScans(projectCode, listOf(scan)) + .expectCheckScanStatus(scanCode, ScanStatus.FINISHED) + .expectCreateScan(projectCode, scanCode, vcsInfo, "", isArchiveMode = true) + .expectRemoveUploadedContent(scanCode) + .expectUploadFile(scanCode) + .expectExtractArchives(scanCode) + .mockFiles( + scanCode, + markedFiles = listOf(markedAsIdentifiedFile) + ) + + val snippetChoices = createSnippetChoices( + vcsInfo.url, + createSnippetChoice(choiceLocation, PURL_1, "") + ) + val fossId = createFossId(config) + + val summary = fossId.scan(createPackage(pkgId, vcsInfo), snippetChoices = snippetChoices).summary + + summary.licenseFindings.shouldBeSingleton { + it.license shouldBe "MIT".toSpdx() + it.location shouldBe choiceLocation + } + + summary.issues.filter { it.severity > Severity.HINT } should beEmpty() + summary.snippetFindings should beEmpty() + } + "add the license of marked as identified files that have been manually marked in the UI (legacy behavior)" { val projectCode = PROJECT val scanCode = scanCode(PROJECT, null) @@ -775,6 +818,72 @@ class FossIdSnippetChoiceTest : WordSpec({ } } + "put a marked as identified file back to pending if it has no snippet choice (scan in archive mode)" { + val projectCode = PROJECT + val scanCode = scanCode(PROJECT, null) + val config = createConfig(deltaScans = false, fetchSnippetMatchedLines = true, isArchiveMode = true) + val vcsInfo = createVcsInfo() + val scan = createScan(vcsInfo.url, "${vcsInfo.revision}_other", scanCode) + val pkgId = createIdentifier(index = 42) + + val choiceLocation = TextLocation(FILE_1, 10, 20) + val payload = OrtCommentPayload(mapOf("MIT" to listOf(choiceLocation)), 1, 0) + val comment = jsonMapper.writeValueAsString(mapOf(ORT_NAME to payload)) + val markedAsIdentifiedFile = createMarkAsIdentifiedFile("MIT", FILE_1_ARCHIVE_MODE, comment) + val service = FossIdRestService.create(config.serverUrl) + .expectProjectRequest(projectCode) + .expectListScans(projectCode, listOf(scan)) + .expectCheckScanStatus(scanCode, ScanStatus.FINISHED) + .expectCreateScan(projectCode, scanCode, vcsInfo, "", isArchiveMode = true) + .expectRemoveUploadedContent(scanCode) + .expectUploadFile(scanCode) + .expectExtractArchives(scanCode) + .mockFiles( + scanCode, + markedFiles = listOf(markedAsIdentifiedFile), + snippets = listOf( + createSnippet(0, FILE_1, PURL_1), + createSnippet(1, FILE_1, PURL_2), + createSnippet(2, FILE_1, PURL_3) + ), + matchedLines = mapOf( + 0 to MatchedLines.create((10..20).toList(), (10..20).toList()), + 1 to MatchedLines.create((10..20).toList(), (10..20).toList()), + 2 to MatchedLines.create((20..30).toList(), (20..30).toList()) + ) + ) + // The unmark as identified call is made on the real snippet path. + .expectUnmarkAsIdentified(scanCode, FILE_1_ARCHIVE_MODE) + + val fossId = createFossId(config) + + val summary = fossId.scan(createPackage(pkgId, vcsInfo), snippetChoices = emptyList()).summary + + summary.issues.forAtLeastOne { + it.message shouldBe "This scan has 1 file(s) pending identification in FossID. " + + "Please review and resolve them at: https://www.example.org/fossid/index.html?action=scanview&sid=1" + } + + summary.issues.filter { it.severity > Severity.HINT } should beEmpty() + summary.snippetFindings shouldHaveSize 2 + summary.snippetFindings.first().apply { + sourceLocation.path shouldBe FILE_1 + snippets.map { it.purl } shouldBe listOf(PURL_1, PURL_2) + } + + summary.snippetFindings.last().apply { + sourceLocation.path shouldBe FILE_1 + snippets.map { it.purl } shouldBe listOf(PURL_3) + } + + coVerify { + service.removeUploadedContent(USER, API_KEY, scanCode) + service.uploadFile(USER, API_KEY, scanCode, any()) + service.extractArchives(USER, API_KEY, scanCode, any()) + service.unmarkAsIdentified(USER, API_KEY, scanCode, FILE_1_ARCHIVE_MODE, any()) + } + } + "put a marked as identified file back to pending if some of its snippet choices have been deleted" { val projectCode = PROJECT val scanCode = scanCode(PROJECT, null)