SCANOSS Scanner Improvements

plugins/scanners/scanoss/src/main/kotlin/ScanOssConfig.kt

@@ -25,11 +25,11 @@ import org.ossreviewtoolkit.plugins.api.OrtPluginOption
 import org.ossreviewtoolkit.plugins.api.Secret
 
 data class ScanOssConfig(
-    /** The URL of the ScanOSS server. */
+    /** The URL of the SCANOSS server. */
     @OrtPluginOption(defaultValue = ScanApi.DEFAULT_BASE_URL)
     val apiUrl: String,
 
-    /** The API key used to authenticate with the ScanOSS server. */
+    /** The API key used to authenticate with the SCANOSS server. */
     @OrtPluginOption(defaultValue = "")
     val apiKey: Secret,
 

plugins/scanners/scanoss/src/main/kotlin/ScanOssResultParser.kt

@@ -22,9 +22,13 @@ package org.ossreviewtoolkit.plugins.scanners.scanoss
 import com.scanoss.dto.ScanFileDetails
 import com.scanoss.dto.ScanFileResult
 import com.scanoss.dto.enums.MatchType
+import com.scanoss.dto.enums.StatusType
 
+import java.lang.invoke.MethodHandles
 import java.time.Instant
 
+import org.apache.logging.log4j.kotlin.loggerOf
+
 import org.ossreviewtoolkit.downloader.VcsHost
 import org.ossreviewtoolkit.model.CopyrightFinding
 import org.ossreviewtoolkit.model.LicenseFinding
@@ -38,6 +42,8 @@ import org.ossreviewtoolkit.utils.spdx.SpdxExpression
 import org.ossreviewtoolkit.utils.spdx.SpdxLicenseIdExpression
 import org.ossreviewtoolkit.utils.spdx.toExpression
 
+private val logger = loggerOf(MethodHandles.lookup().lookupClass())
+
 /**
  * Generate a summary from the given SCANOSS [result], using [startTime], [endTime] as metadata. This variant can be
  * used if the result is not read from a local file.
@@ -51,21 +57,19 @@ internal fun generateSummary(startTime: Instant, endTime: Instant, results: List
         result.fileDetails.forEach { details ->
             when (details.matchType) {
                 MatchType.file -> {
-                    licenseFindings += getLicenseFindings(details)
-                    copyrightFindings += getCopyrightFindings(details)
+                    val localFile = requireNotNull(result.filePath)
+                    licenseFindings += getLicenseFindings(details, localFile)
+                    copyrightFindings += getCopyrightFindings(details, localFile)
                 }
 
                 MatchType.snippet -> {
-                    val file = requireNotNull(details.file)
-                    val lines = requireNotNull(details.lines)
-                    val sourceLocations = convertLines(file, lines)
-                    val snippets = getSnippets(details)
-
-                    snippets.forEach { snippet ->
-                        sourceLocations.forEach { sourceLocation ->
-                            // TODO: Aggregate the snippet by source file location.
-                            snippetFindings += SnippetFinding(sourceLocation, setOf(snippet))
-                        }
+                    val localFile = requireNotNull(result.filePath)
+                    if (details.status == StatusType.pending) {
+                        snippetFindings += createSnippetFindings(details, localFile)
+                    } else {
+                        logger.info { "File '$localFile' is identified, not including in snippet findings." }
+                        licenseFindings += getLicenseFindings(details, result.filePath)
+                        copyrightFindings += getCopyrightFindings(details, result.filePath)
                     }
                 }
 
@@ -90,8 +94,7 @@ internal fun generateSummary(startTime: Instant, endTime: Instant, results: List
 /**
  * Get the license findings from the given [details].
  */
-private fun getLicenseFindings(details: ScanFileDetails): List<LicenseFinding> {
-    val path = details.file ?: return emptyList()
+private fun getLicenseFindings(details: ScanFileDetails, path: String): List<LicenseFinding> {
     val score = details.matched?.removeSuffix("%")?.toFloatOrNull()
 
     return details.licenseDetails.orEmpty().map { license ->
@@ -118,9 +121,7 @@ private fun getLicenseFindings(details: ScanFileDetails): List<LicenseFinding> {
 /**
  * Get the copyright findings from the given [details].
  */
-private fun getCopyrightFindings(details: ScanFileDetails): List<CopyrightFinding> {
-    val path = details.file ?: return emptyList()
-
+private fun getCopyrightFindings(details: ScanFileDetails, path: String): List<CopyrightFinding> {
     return details.copyrightDetails.orEmpty().map { copyright ->
         CopyrightFinding(
             statement = copyright.name,
@@ -134,39 +135,57 @@ private fun getCopyrightFindings(details: ScanFileDetails): List<CopyrightFindin
 }
 
 /**
- * Get the snippet findings from the given [details]. If a snippet returned by ScanOSS contains several Purls,
- * several snippets are created in ORT each containing a single Purl.
+ * Create snippet findings from given [details] and [localFilePath]. If a snippet returned by ScanOSS contains several
+ * PURLs the function extracts the first PURL as the primary identifier while storing the remaining PURLs in
+ * additionalData to preserve the complete information.
  */
-private fun getSnippets(details: ScanFileDetails): Set<Snippet> {
+private fun createSnippetFindings(details: ScanFileDetails, localFilePath: String): Set<SnippetFinding> {
     val matched = requireNotNull(details.matched)
-    val fileUrl = requireNotNull(details.fileUrl)
+    val ossFile = requireNotNull(details.file)
     val ossLines = requireNotNull(details.ossLines)
+    val localLines = requireNotNull(details.lines)
     val url = requireNotNull(details.url)
-    val purls = requireNotNull(details.purls)
-
-    val licenses = details.licenseDetails.orEmpty().mapTo(mutableSetOf()) { license ->
-        SpdxExpression.parse(license.name)
-    }
+    val purls = requireNotNull(details.purls).toMutableList()
 
     val score = matched.substringBeforeLast("%").toFloat()
-    val locations = convertLines(fileUrl, ossLines)
+    val primaryPurl = purls.removeFirstOrNull().orEmpty()
+
+    val license = details.licenseDetails.orEmpty()
+        .map { license -> SpdxExpression.parse(license.name) }
+        .toExpression()?.sorted() ?: SpdxLicenseIdExpression(SpdxConstants.NOASSERTION)
+
     // TODO: No resolved revision is available. Should a ArtifactProvenance be created instead ?
     val vcsInfo = VcsHost.parseUrl(url.takeUnless { it == "none" }.orEmpty())
     val provenance = RepositoryProvenance(vcsInfo, ".")
 
-    return buildSet {
-        purls.forEach { purl ->
-            locations.forEach { snippetLocation ->
-                val license = licenses.toExpression()?.sorted() ?: SpdxLicenseIdExpression(SpdxConstants.NOASSERTION)
+    val additionalData = purls.associateWith { "" }
 
-                add(Snippet(score, snippetLocation, provenance, purl, license))
-            }
+    // Convert both local and OSS line ranges to source locations.
+    val sourceLocations = convertLines(localFilePath, localLines)
+    val ossLocations = convertLines(ossFile, ossLines)
+
+    // The number of source locations should match the number of oss locations.
+    if (sourceLocations.size != ossLocations.size) {
+        logger.warn {
+            "Unexpected mismatch in '$localFilePath': " +
+                "${sourceLocations.size} source locations vs ${ossLocations.size} oss locations. " +
+                "This indicates a potential issue with line range conversion."
         }
     }
+
+    // Directly pair source locations with their corresponding OSS locations and create a SnippetFinding.
+    return sourceLocations.zip(ossLocations).mapTo(mutableSetOf()) { (sourceLocation, ossLocation) ->
+        SnippetFinding(
+            sourceLocation,
+            setOf(
+                Snippet(score, ossLocation, provenance, primaryPurl, license, additionalData)
+            )
+        )
+    }
 }
 
 /**
- * Split [lineRanges] returned by ScanOSS such as "32-105,117-199" into [TextLocation]s for the given [file].
+ * Split [lineRanges] returned by SCANOSS such as "32-105,117-199" into [TextLocation]s for the given [file].
  */
 private fun convertLines(file: String, lineRanges: String): List<TextLocation> =
     lineRanges.split(',').map { lineRange ->

plugins/scanners/scanoss/src/test/kotlin/ScanOssResultParserTest.kt

@@ -27,6 +27,7 @@ import io.kotest.matchers.collections.containExactlyInAnyOrder
 import io.kotest.matchers.collections.haveSize
 import io.kotest.matchers.collections.shouldContain
 import io.kotest.matchers.should
+import io.kotest.matchers.shouldBe
 
 import java.time.Instant
 
@@ -38,7 +39,9 @@ import org.ossreviewtoolkit.model.SnippetFinding
 import org.ossreviewtoolkit.model.TextLocation
 import org.ossreviewtoolkit.model.VcsInfo
 import org.ossreviewtoolkit.model.VcsType
+import org.ossreviewtoolkit.utils.spdx.SpdxConstants
 import org.ossreviewtoolkit.utils.spdx.SpdxExpression
+import org.ossreviewtoolkit.utils.spdx.toSpdx
 import org.ossreviewtoolkit.utils.test.readResource
 
 class ScanOssResultParserTest : WordSpec({
@@ -63,7 +66,8 @@ class ScanOssResultParserTest : WordSpec({
             summary.licenseFindings shouldContain LicenseFinding(
                 license = "Apache-2.0",
                 location = TextLocation(
-                    path = "hopscotch-rails-0.1.2.1/vendor/assets/javascripts/hopscotch.js",
+                    path = "/tmp/ort-ScanOss2759786101559527642/Maven/junit/junit/4.12/src/site/resources/" +
+                        "scripts/hopscotch-0.1.2.min.js",
                     startLine = TextLocation.UNKNOWN_LINE,
                     endLine = TextLocation.UNKNOWN_LINE
                 ),
@@ -74,7 +78,8 @@ class ScanOssResultParserTest : WordSpec({
             summary.copyrightFindings shouldContain CopyrightFinding(
                 statement = "Copyright 2013 LinkedIn Corp.",
                 location = TextLocation(
-                    path = "hopscotch-rails-0.1.2.1/vendor/assets/javascripts/hopscotch.js",
+                    path = "/tmp/ort-ScanOss2759786101559527642/Maven/junit/junit/4.12/src/site/resources/" +
+                        "scripts/hopscotch-0.1.2.min.js",
                     startLine = TextLocation.UNKNOWN_LINE,
                     endLine = TextLocation.UNKNOWN_LINE
                 )
@@ -101,7 +106,7 @@ class ScanOssResultParserTest : WordSpec({
             summary.licenseFindings shouldContain LicenseFinding(
                 license = "Apache-2.0",
                 location = TextLocation(
-                    path = "com/vdurmont/semver4j/Range.java",
+                    path = "src/main/java/com/vdurmont/semver4j/Range.java",
                     startLine = TextLocation.UNKNOWN_LINE,
                     endLine = TextLocation.UNKNOWN_LINE
                 ),
@@ -116,7 +121,7 @@ class ScanOssResultParserTest : WordSpec({
                         Snippet(
                             98.0f,
                             TextLocation(
-                                "https://osskb.org/api/file_contents/6ff2427335b985212c9b79dfa795799f",
+                                "src/main/java/com/vdurmont/semver4j/Requirement.java",
                                 1,
                                 710
                             ),
@@ -131,5 +136,107 @@ class ScanOssResultParserTest : WordSpec({
                 )
             )
         }
+
+        "handle multiple PURLs by extracting first as primary and storing remaining in additionalData" {
+            val results = readResource("/scanoss-multiple-purls.json").let {
+                JsonUtils.toScanFileResultsFromObject(JsonUtils.toJsonObject(it))
+            }
+
+            val time = Instant.now()
+            val summary = generateSummary(time, time, results)
+
+            // Verify we have one finding per source location, not per PURL.
+            summary.snippetFindings should haveSize(2)
+
+            with(summary.snippetFindings.first()) {
+                // Check source location (local file).
+                sourceLocation shouldBe TextLocation("hung_task.c", 12, 150)
+
+                // Verify first PURL is extracted as primary identifier.
+                snippets should haveSize(1)
+                snippets.first().purl shouldBe "pkg:github/kdrag0n/proton_bluecross"
+
+                // Verify remaining PURLs are stored in additionalData.
+                snippets.first().additionalData shouldBe
+                    mapOf(
+                        "pkg:github/fake/fake_repository" to ""
+                    )
+
+                // Check OSS location.
+                snippets.first().location shouldBe
+                    TextLocation("kernel/hung_task.c", 10, 148)
+            }
+
+            // Verify same behavior for second snippet.
+            with(summary.snippetFindings.last()) {
+                sourceLocation shouldBe TextLocation("hung_task.c", 540, 561)
+                snippets.first().purl shouldBe "pkg:github/kdrag0n/proton_bluecross"
+                snippets.first().location shouldBe
+                    TextLocation("kernel/hung_task.c", 86, 107)
+            }
+        }
+
+        "combine the same license from different sources into a single expression" {
+            // When the same license appears in multiple sources (like scancode and file_header),
+            // combine them into a single expression rather than duplicating.
+            val results = readResource("/scanoss-snippet-same-license-multiple-sources.json").let {
+                JsonUtils.toScanFileResultsFromObject(JsonUtils.toJsonObject(it))
+            }
+
+            val time = Instant.now()
+            val summary = generateSummary(time, time, results)
+
+            // Verify the snippet finding.
+            summary.snippetFindings should haveSize(1)
+            val snippet = summary.snippetFindings.first().snippets.first()
+
+            // Consolidate the license into a single expression
+            // even though it came from both "scancode" and "file_header" sources.
+            snippet.license shouldBe "LGPL-2.1-or-later".toSpdx()
+
+            // Preserve other snippet details correctly.
+            with(summary.snippetFindings.first()) {
+                sourceLocation.path shouldBe "src/check_error.c"
+                sourceLocation.startLine shouldBe 16
+                sourceLocation.endLine shouldBe 24
+            }
+        }
+
+        "handle empty license array with NOASSERTION" {
+            val results = readResource("/scanoss-snippet-no-license-data.json").let {
+                JsonUtils.toScanFileResultsFromObject(JsonUtils.toJsonObject(it))
+            }
+
+            val time = Instant.now()
+            val summary = generateSummary(time, time, results)
+
+            // Verify the snippet finding.
+            summary.snippetFindings should haveSize(1)
+            val snippet = summary.snippetFindings.first().snippets.first()
+
+            // Use NOASSERTION when no licenses are provided.
+            snippet.license shouldBe SpdxConstants.NOASSERTION.toSpdx()
+
+            // Preserve other snippet details correctly.
+            with(summary.snippetFindings.first()) {
+                sourceLocation.path shouldBe "fake_file.c"
+                sourceLocation.startLine shouldBe 16
+                sourceLocation.endLine shouldBe 24
+            }
+        }
+
+        "exclude identified snippets from snippet findings" {
+            // The scanoss-identified-snippet.json contains two snippets, but one is identified.
+            // Only unidentified snippets should be included in the SnippetFindings.
+            val results = readResource("/scanoss-identified-snippet.json").let {
+                JsonUtils.toScanFileResultsFromObject(JsonUtils.toJsonObject(it))
+            }
+
+            val time = Instant.now()
+            val summary = generateSummary(time, time, results)
+
+            // Should have only one finding because the identified snippet is excluded
+            summary.snippetFindings should haveSize(1)
+        }
     }
 })

plugins/scanners/scanoss/src/test/kotlin/ScanOssScannerDirectoryTest.kt

@@ -103,7 +103,7 @@ class ScanOssScannerDirectoryTest : StringSpec({
                         Snippet(
                             99.0f,
                             TextLocation(
-                                "https://osskb.org/api/file_contents/871fb0c5188c2f620d9b997e225b0095",
+                                "examples/example.rules.kts",
                                 128,
                                 367
                             ),
@@ -158,7 +158,7 @@ class ScanOssScannerDirectoryTest : StringSpec({
         // [fingerprint data for the file]
         // --boundary--
 
-        // Extract included filenames using a regex pattern from the ScanOSS HTTP POST.
+        // Extract included filenames using a regex pattern from the SCANOSS HTTP POST.
         // The pattern matches lines starting with "file=" followed by hash and size, then captures the filename.
         val filenamePattern = "file=.*?,.*?,(.+)".toRegex(RegexOption.MULTILINE)
         val includedFiles = requestBodies.flatMap { body ->