refactor(scanoss): Replace direct API calls with SCANOSS SDK

isasmendiagus · isasmendiagus · commit 83e5bed54bb2 · 2025-03-12T14:59:37.000+01:00
Replace custom direct API calls to SCANOSS with the official Java SDK.
This change improves maintainability by leveraging the SDK's functionality
instead of maintaining custom implementation for API interactions.

Signed-off-by: Agustin Isasmendi &lt;agustin.isasmendi@scanoss.com&gt;
diff --git a/plugins/scanners/scanoss/src/main/kotlin/ScanOss.kt b/plugins/scanners/scanoss/src/main/kotlin/ScanOss.kt
@@ -19,27 +19,20 @@
 
 package org.ossreviewtoolkit.plugins.scanners.scanoss
 
-import com.scanoss.Winnowing
-import com.scanoss.dto.ScanFileResult
-import com.scanoss.rest.ScanApi
+import com.scanoss.Scanner
 import com.scanoss.utils.JsonUtils
 import com.scanoss.utils.PackageDetails
 
 import java.io.File
 import java.time.Instant
-import java.util.UUID
-
-import org.apache.logging.log4j.kotlin.logger
 
 import org.ossreviewtoolkit.model.ScanSummary
 import org.ossreviewtoolkit.plugins.api.OrtPlugin
 import org.ossreviewtoolkit.plugins.api.PluginDescriptor
 import org.ossreviewtoolkit.scanner.PathScannerWrapper
 import org.ossreviewtoolkit.scanner.ScanContext
 import org.ossreviewtoolkit.scanner.ScannerMatcher
-import org.ossreviewtoolkit.scanner.ScannerMatcherConfig
 import org.ossreviewtoolkit.scanner.ScannerWrapperFactory
-import org.ossreviewtoolkit.utils.common.VCS_DIRECTORIES
 
 @OrtPlugin(
     id = "SCANOSS",
@@ -49,13 +42,8 @@ import org.ossreviewtoolkit.utils.common.VCS_DIRECTORIES
 )
 class ScanOss(
     override val descriptor: PluginDescriptor = ScanOssFactory.descriptor,
-    config: ScanOssConfig
+    private var config: ScanOssConfig
 ) : PathScannerWrapper {
-    private val service = ScanApi.builder()
-        // As there is only a single endpoint, the SCANOSS API client expects the path to be part of the API URL.
-        .url(config.apiUrl.removeSuffix("/") + "/scan/direct")
-        .apiKey(config.apiKey.value)
-        .build()
 
     override val version: String by lazy {
         // TODO: Find out the best / cheapest way to query the SCANOSS server for its version.
@@ -64,73 +52,27 @@ class ScanOss(
 
     override val configuration = ""
 
-    override val matcher by lazy {
-        ScannerMatcher.create(
-            details,
-            ScannerMatcherConfig(
-                config.regScannerName,
-                config.minVersion,
-                config.maxVersion,
-                configuration
-            )
-        )
-    }
+    override val matcher: ScannerMatcher? = null
 
     override val readFromStorage = config.readFromStorage
 
     override val writeToStorage = config.writeToStorage
 
-    /**
-     * The name of the file corresponding to the fingerprints can be sent to SCANOSS for more precise matches.
-     * However, for anonymity, a unique identifier should be generated and used instead. This property holds the
-     * mapping between the file paths and the unique identifiers. When receiving the response, the UUID will be
-     * replaced by the actual file path.
-     *
-     * TODO: This behavior should be driven by a configuration parameter enabled by default.
-     */
-    private val fileNamesAnonymizationMapping = mutableMapOf<UUID, String>()
-
     override fun scanPath(path: File, context: ScanContext): ScanSummary {
         val startTime = Instant.now()
 
-        val wfpString = buildString {
-            path.walk()
-                .onEnter { it.name !in VCS_DIRECTORIES }
-                .filterNot { it.isDirectory }
-                .forEach {
-                    logger.info { "Computing fingerprint for file ${it.absolutePath}..." }
-                    append(createWfpForFile(it))
-                }
-        }
-
-        val result = service.scan(
-            wfpString,
-            context.labels["scanOssContext"],
-            context.labels["scanOssId"]?.toIntOrNull() ?: Thread.currentThread().threadId().toInt()
-        )
-
-        // Replace the anonymized UUIDs by their file paths.
-        val results = JsonUtils.toScanFileResultsFromObject(JsonUtils.toJsonObject(result)).map {
-            val uuid = UUID.fromString(it.filePath)
+        val scanoss = Scanner.builder()
+            .url(config.apiUrl.removeSuffix("/") + "/scan/direct")
+            .apiKey(config.apiKey.value)
+            .build()
 
-            val fileName = fileNamesAnonymizationMapping[uuid] ?: throw IllegalArgumentException(
-                "The ${descriptor.id} server returned UUID '$uuid' which is not present in the mapping."
-            )
-
-            ScanFileResult(fileName, it.fileDetails)
+        val rawResults: List<String> = when {
+            path.isFile -> listOf(scanoss.scanFile(path.absolutePath))
+            else -> scanoss.scanFolder(path.absolutePath)
         }
 
+        val results = JsonUtils.toScanFileResults(rawResults)
         val endTime = Instant.now()
         return generateSummary(startTime, endTime, results)
     }
-
-    internal fun generateRandomUUID() = UUID.randomUUID()
-
-    internal fun createWfpForFile(file: File): String {
-        generateRandomUUID().let { uuid ->
-            // TODO: Let's keep the original file extension to give SCANOSS some hint about the mime type.
-            fileNamesAnonymizationMapping[uuid] = file.path
-            return Winnowing.builder().build().wfpForFile(file.path, uuid.toString())
-        }
-    }
 }
diff --git a/plugins/scanners/scanoss/src/test/assets/scanMulti/mappings/scanoss-multi-response.json b/plugins/scanners/scanoss/src/test/assets/scanMulti/mappings/scanoss-multi-response.json
@@ -10,7 +10,7 @@
   },
   "response" : {
     "status" : 200,
-    "body" : "{  \"c198b884-f6cf-496f-95eb-0e7968dd2ec6\": [    {      \"id\": \"snippet\",      \"status\": \"pending\",      \"lines\": \"1-240\",      \"oss_lines\": \"128-367\",      \"matched\": \"99%\",      \"purl\": [        \"pkg:github/scanoss/ort\"      ],      \"vendor\": \"scanoss\",      \"component\": \"ort\",      \"version\": \"e654028\",      \"latest\": \"b12f8ee\",      \"url\": \"https://github.com/scanoss/ort\",      \"release_date\": \"2021-03-18\",      \"file\": \"utils/src/main/kotlin/ArchiveUtils.kt\",      \"url_hash\": \"37faa38a820322fa93bf7a8fa8290bb8\",      \"file_hash\": \"871fb0c5188c2f620d9b997e225b0095\",      \"source_hash\": \"2e91edbe430c4eb195a977d326d6d6c0\",      \"file_url\": \"https://osskb.org/api/file_contents/871fb0c5188c2f620d9b997e225b0095\",      \"licenses\": [        {          \"name\": \"Apache-2.0\",          \"patent_hints\": \"yes\",          \"copyleft\": \"no\",          \"checklist_url\": \"https://www.osadl.org/fileadmin/checklists/unreflicenses/Apache-2.0.txt\",          \"osadl_updated\": \"2022-03-17 13:38\",          \"source\": \"file_spdx_tag\"        },        {          \"name\": \"Apache-2.0\",          \"patent_hints\": \"yes\",          \"copyleft\": \"no\",          \"checklist_url\": \"https://www.osadl.org/fileadmin/checklists/unreflicenses/Apache-2.0.txt\",          \"osadl_updated\": \"2022-03-17 13:38\",          \"source\": \"scancode\"        }      ],      \"server\": {        \"version\": \"4.4.2\",        \"kb_version\": {          \"monthly\": \"22.02\",          \"daily\": \"22.03.25\"        }      }    }  ],  \"5530105e-0752-4750-9c07-4e4604b879a5\": [    {      \"id\": \"file\",      \"status\": \"pending\",      \"lines\": \"all\",      \"oss_lines\": \"all\",      \"matched\": \"100%\",      \"purl\": [        \"pkg:github/scanoss/ort\"      ],      \"vendor\": \"scanoss\",      \"component\": \"ort\",      \"version\": \"e654028\",      \"latest\": \"b12f8ee\",      \"url\": \"https://github.com/scanoss/ort\",      \"release_date\": \"2021-03-18\",      \"file\": \"scanner/src/main/kotlin/ScannerFactory.kt\",      \"url_hash\": \"37faa38a820322fa93bf7a8fa8290bb8\",      \"file_hash\": \"5c8ab9be40df937e46c53509481107cd\",      \"source_hash\": \"5c8ab9be40df937e46c53509481107cd\",      \"file_url\": \"https://osskb.org/api/file_contents/5c8ab9be40df937e46c53509481107cd\",      \"licenses\": [        {          \"name\": \"Apache-2.0\",          \"patent_hints\": \"yes\",          \"copyleft\": \"no\",          \"checklist_url\": \"https://www.osadl.org/fileadmin/checklists/unreflicenses/Apache-2.0.txt\",          \"osadl_updated\": \"2022-03-17 13:38\",          \"source\": \"file_spdx_tag\"        },        {          \"name\": \"Apache-2.0\",          \"patent_hints\": \"yes\",          \"copyleft\": \"no\",          \"checklist_url\": \"https://www.osadl.org/fileadmin/checklists/unreflicenses/Apache-2.0.txt\",          \"osadl_updated\": \"2022-03-17 13:38\",          \"source\": \"scancode\"        }      ],      \"server\": {        \"version\": \"4.4.2\",        \"kb_version\": {          \"monthly\": \"22.02\",          \"daily\": \"22.03.25\"        }      }    }  ]}",
+    "body" : "{  \"utils/src/main/kotlin/ArchiveUtils.kt\": [    {      \"id\": \"snippet\",      \"status\": \"pending\",      \"lines\": \"1-240\",      \"oss_lines\": \"128-367\",      \"matched\": \"99%\",      \"purl\": [        \"pkg:github/scanoss/ort\"      ],      \"vendor\": \"scanoss\",      \"component\": \"ort\",      \"version\": \"e654028\",      \"latest\": \"b12f8ee\",      \"url\": \"https://github.com/scanoss/ort\",      \"release_date\": \"2021-03-18\",      \"file\": \"utils/src/main/kotlin/ArchiveUtils.kt\",      \"url_hash\": \"37faa38a820322fa93bf7a8fa8290bb8\",      \"file_hash\": \"871fb0c5188c2f620d9b997e225b0095\",      \"source_hash\": \"2e91edbe430c4eb195a977d326d6d6c0\",      \"file_url\": \"https://osskb.org/api/file_contents/871fb0c5188c2f620d9b997e225b0095\",      \"licenses\": [        {          \"name\": \"Apache-2.0\",          \"patent_hints\": \"yes\",          \"copyleft\": \"no\",          \"checklist_url\": \"https://www.osadl.org/fileadmin/checklists/unreflicenses/Apache-2.0.txt\",          \"osadl_updated\": \"2022-03-17 13:38\",          \"source\": \"file_spdx_tag\"        },        {          \"name\": \"Apache-2.0\",          \"patent_hints\": \"yes\",          \"copyleft\": \"no\",          \"checklist_url\": \"https://www.osadl.org/fileadmin/checklists/unreflicenses/Apache-2.0.txt\",          \"osadl_updated\": \"2022-03-17 13:38\",          \"source\": \"scancode\"        }      ],      \"server\": {        \"version\": \"4.4.2\",        \"kb_version\": {          \"monthly\": \"22.02\",          \"daily\": \"22.03.25\"        }      }    }  ],  \"5530105e-0752-4750-9c07-4e4604b879a5\": [    {      \"id\": \"file\",      \"status\": \"pending\",      \"lines\": \"all\",      \"oss_lines\": \"all\",      \"matched\": \"100%\",      \"purl\": [        \"pkg:github/scanoss/ort\"      ],      \"vendor\": \"scanoss\",      \"component\": \"ort\",      \"version\": \"e654028\",      \"latest\": \"b12f8ee\",      \"url\": \"https://github.com/scanoss/ort\",      \"release_date\": \"2021-03-18\",      \"file\": \"scanner/src/main/kotlin/ScannerFactory.kt\",      \"url_hash\": \"37faa38a820322fa93bf7a8fa8290bb8\",      \"file_hash\": \"5c8ab9be40df937e46c53509481107cd\",      \"source_hash\": \"5c8ab9be40df937e46c53509481107cd\",      \"file_url\": \"https://osskb.org/api/file_contents/5c8ab9be40df937e46c53509481107cd\",      \"licenses\": [        {          \"name\": \"Apache-2.0\",          \"patent_hints\": \"yes\",          \"copyleft\": \"no\",          \"checklist_url\": \"https://www.osadl.org/fileadmin/checklists/unreflicenses/Apache-2.0.txt\",          \"osadl_updated\": \"2022-03-17 13:38\",          \"source\": \"file_spdx_tag\"        },        {          \"name\": \"Apache-2.0\",          \"patent_hints\": \"yes\",          \"copyleft\": \"no\",          \"checklist_url\": \"https://www.osadl.org/fileadmin/checklists/unreflicenses/Apache-2.0.txt\",          \"osadl_updated\": \"2022-03-17 13:38\",          \"source\": \"scancode\"        }      ],      \"server\": {        \"version\": \"4.4.2\",        \"kb_version\": {          \"monthly\": \"22.02\",          \"daily\": \"22.03.25\"        }      }    }  ]}",
     "headers" : {
       "Server" : "nginx/1.14.2",
       "Date" : "Wed, 16 Mar 2022 13:07:04 GMT",
diff --git a/plugins/scanners/scanoss/src/test/kotlin/ScanOssScannerDirectoryTest.kt b/plugins/scanners/scanoss/src/test/kotlin/ScanOssScannerDirectoryTest.kt
@@ -27,12 +27,9 @@ import io.kotest.matchers.collections.containExactlyInAnyOrder
 import io.kotest.matchers.collections.shouldContainExactly
 import io.kotest.matchers.should
 
-import io.mockk.every
 import io.mockk.spyk
-import io.mockk.verify
 
 import java.io.File
-import java.util.UUID
 
 import org.ossreviewtoolkit.model.LicenseFinding
 import org.ossreviewtoolkit.model.PackageType
@@ -74,25 +71,11 @@ class ScanOssScannerDirectoryTest : StringSpec({
     }
 
     "The scanner should scan a directory" {
-        // Manipulate the UUID generation to have the same IDs as in the response.
-        every {
-            scanner.generateRandomUUID()
-        } answers {
-            UUID.fromString("5530105e-0752-4750-9c07-4e4604b879a5")
-        } andThenAnswer {
-            UUID.fromString("c198b884-f6cf-496f-95eb-0e7968dd2ec6")
-        }
-
         val summary = scanner.scanPath(
             TEST_DIRECTORY_TO_SCAN,
             ScanContext(labels = emptyMap(), packageType = PackageType.PACKAGE)
         )
 
-        verify(exactly = 1) {
-            scanner.createWfpForFile(TEST_DIRECTORY_TO_SCAN.resolve("ArchiveUtils.kt"))
-            scanner.createWfpForFile(TEST_DIRECTORY_TO_SCAN.resolve("ScannerFactory.kt"))
-        }
-
         with(summary) {
             licenseFindings should containExactlyInAnyOrder(
                 LicenseFinding(
diff --git a/plugins/scanners/scanoss/src/test/kotlin/ScanOssScannerFileTest.kt b/plugins/scanners/scanoss/src/test/kotlin/ScanOssScannerFileTest.kt
@@ -26,12 +26,9 @@ import io.kotest.core.spec.style.StringSpec
 import io.kotest.matchers.collections.containExactly
 import io.kotest.matchers.should
 
-import io.mockk.every
 import io.mockk.spyk
-import io.mockk.verify
 
 import java.io.File
-import java.util.UUID
 
 import org.ossreviewtoolkit.model.LicenseFinding
 import org.ossreviewtoolkit.model.PackageType
@@ -67,22 +64,11 @@ class ScanOssScannerFileTest : StringSpec({
     }
 
     "The scanner should scan a single file" {
-        // Manipulate the UUID generation to have the same IDs as in the response.
-        every {
-            scanner.generateRandomUUID()
-        } answers {
-            UUID.fromString("bf5401e9-03b3-4c91-906c-cadb90487b8c")
-        }
-
         val summary = scanner.scanPath(
             TEST_FILE_TO_SCAN,
             ScanContext(labels = emptyMap(), packageType = PackageType.PACKAGE)
         )
 
-        verify(exactly = 1) {
-            scanner.createWfpForFile(TEST_FILE_TO_SCAN)
-        }
-
         with(summary) {
             licenseFindings should containExactly(
                 LicenseFinding(
