feat(scanoss): Add exclusion pattern support to SCANOSS

isasmendiagus · isasmendiagus · commit f5a96acb5a01 · 2025-05-06T16:57:51.000+02:00
Implement exclusion filtering to respect path patterns specified in the
configuration. The scanner now properly excludes files matching the
patterns during the scan process.

Signed-off-by: Agustin Isasmendi &lt;agustin.isasmendi@scanoss.com&gt;
diff --git a/plugins/scanners/scanoss/src/main/kotlin/ScanOss.kt b/plugins/scanners/scanoss/src/main/kotlin/ScanOss.kt
@@ -20,12 +20,15 @@
 package org.ossreviewtoolkit.plugins.scanners.scanoss
 
 import com.scanoss.Scanner
+import com.scanoss.filters.FilterConfig
 import com.scanoss.utils.JsonUtils
 import com.scanoss.utils.PackageDetails
 
 import java.io.File
 import java.time.Instant
 
+import org.apache.logging.log4j.kotlin.logger
+
 import org.ossreviewtoolkit.model.ScanSummary
 import org.ossreviewtoolkit.plugins.api.OrtPlugin
 import org.ossreviewtoolkit.plugins.api.PluginDescriptor
@@ -65,8 +68,27 @@ class ScanOss(
     override fun scanPath(path: File, context: ScanContext): ScanSummary {
         val startTime = Instant.now()
 
+        val filterConfig = FilterConfig.builder()
+            .customFilter { currentPath ->
+                // The "currentPath" variable contains a path object representing the file or directory being evaluated
+                // by the filter.
+                // This is provided by the Scanner and represents individual files/directories during traversal.
+                try {
+                    val relativePath = currentPath.toFile().toRelativeString(path)
+                    val isExcluded = context.excludes?.isPathExcluded(relativePath) ?: false
+                    logger.debug { "Path: $currentPath, relative: $relativePath, isExcluded: $isExcluded" }
+                    isExcluded
+                } catch (e: IllegalArgumentException) {
+                    logger.warn { "Error processing path $currentPath: ${e.message}" }
+                    false
+                }
+            }
+            .build()
+
         // Build the scanner at function level in case any path-specific settings or filters are needed later.
-        val scanoss = scanossBuilder.build()
+        val scanoss = scanossBuilder
+            .filterConfig(filterConfig)
+            .build()
 
         val rawResults = when {
             path.isFile -> listOf(scanoss.scanFile(path.toString()))
diff --git a/plugins/scanners/scanoss/src/test/assets/exclusionTest/ArchiveUtils.kt b/plugins/scanners/scanoss/src/test/assets/exclusionTest/ArchiveUtils.kt
@@ -0,0 +1,31 @@
+/*
+ * This file contains random data generated using the following command:
+ * head -c 1k < /dev/urandom | base64
+ *
+ * The command takes 1 kilobyte of random bytes from the /dev/urandom device
+ * and then encodes it as base64 text.
+ *
+ * Generated on: Fri Mar 14 05:04:43 PM CET 2025
+ * Purpose: To create test data with completely random content that cannot
+ *          match any existing code in repositories, thereby avoiding false
+ *          positives when scanning ORT source code.
+ */
+
+7m8Y06QhHzmQ4ePs0UUUasqsc8SP1ayNTFdQb6wffQwMu605hXOGHbOoy5pUv7ksgf6sw5ET2qXp
+T23LF2yA1cdNeDt8DBDd3IDmLX/wGgXcQjcaCtfSsMWB7oqHBMGkzwC5fMcDKPLK6ec2MwX6WPkw
+E18ImifWtAmGPEFGxWuqIinhE1yGSN+ImqJPVmpYfMOaDIAaS3JpiHZDmJW5uyQ5DB6W7lpm0q+f
+ZbtGPBeimy1jWF0H6kEW/TIve8RzUjdHU/t//O9r0b2AP08shSrSDWGlbQzxTniLOp2VZxNUEcVM
+c9/Lx4OXEaM/3NDCdr4qQS/1kZpGKFrv06zzC8tlncGaxBfdZSCsh1i+LbZtvUmTSv/wz7g+mld5
+WB2lSzF1Ervzqnm2+3iY+9TvVxDWzZ27LWsd1kvFrJCM03jI1q0c7uJrnnovAOoZkH2QiMPNBQmB
+wShT36h3su/aiOXEquXi+DoTSYDNgXeHVGI2joLVWYLfeTcTTfdvZiwp0K+XQp6fKtWX8tpUibNq
+ngp2dOlzl5yiT+WAD2ETGuyEML/wM3oz+wB93me2YYLJqz/1gtlnnRvGnAukbFLpxxXGK7Vnz+FF
+KfcPWF+O/FNV3nJD+m2nlMVj1n4lRM/mUEdVDhDDtxhywvi6DdNQMcUoeXZRT3dLk27+efNLvMDk
+7TsW/asvMoPrioAkDiTHqWvy+OUImWqqzNpzxIMuTWZrApSklw2UeyXknvHBORUN95AM6Oe9iKb0
+7B2g8U9dIFo7v/AhaDqoQMw+Dz1KfH6+fPaqZEy2H1U7/9RSorKz0fycz7n7BtqWxjenqw11LLxy
+lO36udPuvtr2b/WB/4ch0LuoI2eA11iTeIG4DuTxvizU3lExBXP+e8EAjkWx6F2ymDrI21PYPp++
+uidSk3g/RmaRZGk8akcXbs3pDO/twfjaH3YWYZzBf8aP1TRYDp4NF5v2OhWDa5d2dqdQGDRGg/wy
+Gf0W8txn/fQ3QN7SS9qPftgD6OYIpxKjIWBq/zb5+SAzhBZVjFYw+KVi+zu/P7he7xLRko6APCum
+Ugk7wqohWVdbl2IG2RIuPUOH2zQdzVJvLisKhfq3q6ydGmjD/WRNOxbebpmSKcmZWVg0Ko7/e0ys
+ymV2Ud0tIZwfIH/7476SZAh0ym1U7mgyzm/jlxKm5gUIF1+NWQiqa80GmsAJfquf1Yj4i0ftF+eO
+6OPJqbkZERpu24u2HIfL6CvlUkx08mS+eqLzyRiRuidDcQGFOK+0xPUk01jOZnGiY1ptG4W+Fo5K
+OhcT2H14wQqiHsthzMhpSLXwMG2ddM7P69rHEAXB3iXyWopgdWopVekxHEuar0mv3D6uO/4HKQ==
diff --git a/plugins/scanners/scanoss/src/test/assets/exclusionTest/ScannerFactory.kt b/plugins/scanners/scanoss/src/test/assets/exclusionTest/ScannerFactory.kt
@@ -0,0 +1,31 @@
+/*
+ * This file contains random data generated using the following command:
+ * head -c 1k < /dev/urandom | base64
+ *
+ * The command takes 1 kilobyte of random bytes from the /dev/urandom device
+ * and then encodes it as base64 text.
+ *
+ * Generated on: Fri Mar 14 05:05:29 PM CET 2025
+ * Purpose: To create test data with completely random content that cannot
+ *          match any existing code in repositories, thereby avoiding false
+ *          positives when scanning ORT source code.
+ */
+
+IyaayUoMK28Ib11Z55hC2OShY3p3HzWQyGy199hx20oqZrypl9AuDhKtBdl+qozcZBNajzvkU3H/
+jh3vV/P9I2VLQNVqMCpjelQoXyVq/nmbwxdQBXGbLgcC4J05ujQ2hoXuF4jdtEttDxca8P/EpUub
+nmSO3zmz86LqyyYgFj3imketFw0GvnCYU/8VDjmLxnigspEVI7ZDOacKOshObwH+Br/XgFHr5tyc
+ulGqACTjGY3EEdAjC2+tcTqoI+4mXVxx4CcBD4lRn90khfFOcAM8Iu2pGaERHnAtUrf9EX3rLsOW
+wV+wYllChP71rI/4ueEch9X8ph1dA0nQQN1tLUi58pQlkCHY6K0QNFiD6K+RxaBl3yBt1IZqjfZi
+UVjTb7xJDrYnLPIASlPd0AduDik8pKn+GTqIFWgkkRr5mY6c9jTqHxY7rASDNi7LGKUE9gPFd1LD
+xPJmsl+8L+lcVJjJNU7Tkps/ZZJuo/EqlwbUd/Wq45S++YBBfYlFaOXn/bVMhxXi1SH3xMHSAjH1
+aYj0YHEdBHnEF1ouahyS4607cundZcSR29kITrUnFSi/ZP3zKREa3MGm/qrJS7qFSxlHVsYHBIjy
+VRx+teV4nQWKJyA6x/T9Sx63lM7duwhVRdh9JxhxnrKAyUBH5HwhpFXHreMjudNdY9nMaWaKP9Ge
+oD4Rr4iA3kvaHjtqSfhB55PgQO7Od/KLNTRfMMPl7IjbouQNCai++hV+p7BRAjtGUwTOXp9FHbv7
+YFGBFl3a0e1+YEoQA+0Psf1x2lENCJwH87DuZxuKI3kbcY6XA5kebt43m/eztRa17z/vmwyiQ/up
++RpMU9Xp1bv39h84QbyvZYN40xzHc8togJmPtSKCyEPcmHdt9t0LF6TCsb4k+kIBRUXMfnYpEDqv
+E6dldgWHjVy/4llWqyj3SsToERP1VhaloWyq8QRNke6lKzxMXOhmupKX195V2cA+6EGY3sK/ykhl
+fYOofbKcHwevHKgOJyj7Tj6+9qUgda/EI01lcJicTO8Nqb0LW+FfwIiws7WlsZWuxQGUZ0SOMBU4
+MnR9NPbS6rUSx2rMfSPn18Jd82D5eoM32ogRQb7C2pgXQbQoAegl98vtOjkze4wsa6CmW0rmbQrJ
+bpgPoWiZ1t/BlAUvxjRuzQSNNhnyvaC5nib6NYZAcr9BCm3yJ0sR/uSOUG8cCoJptMYhH9XxHqKl
+ACwfHgq7/mHBTxhQCmw5hkDWvY7FqzDPME3igab1Mda4lxOyUjJ3PeVzZbWZY2s/oUaSbntsSqRM
+z+zutj83Nm76iOSS0MXxCfi5VKYThzGdfXkYB2tZP8yPhh+sw0CpqeV5KB810C76abZbVZ+EDw==
diff --git a/plugins/scanners/scanoss/src/test/assets/exclusionTest/server.go b/plugins/scanners/scanoss/src/test/assets/exclusionTest/server.go
@@ -0,0 +1,31 @@
+/*
+ * This file contains random data generated using the following command:
+ * head -c 1k < /dev/urandom | base64
+ *
+ * The command takes 1 kilobyte of random bytes from the /dev/urandom device
+ * and then encodes it as base64 text.
+ *
+ * Generated on: Fri Mar 14 05:05:29 PM CET 2025
+ * Purpose: To create test data with completely random content that cannot
+ *          match any existing code in repositories, thereby avoiding false
+ *          positives when scanning ORT source code.
+ */
+
+IyaayUoMK28Ib11Z55hC2OShY3p3HzWQyGy199hx20oqZrypl9AuDhKtBdl+qozcZBNajzvkU3H/
+jh3vV/P9I2VLQNVqMCpjelQoXyVq/nmbwxdQBXGbLgcC4J05ujQ2hoXuF4jdtEttDxca8P/EpUub
+nmSO3zmz86LqyyYgFj3imketFw0GvnCYU/8VDjmLxnigspEVI7ZDOacKOshObwH+Br/XgFHr5tyc
+ulGqACTjGY3EEdAjC2+tcTqoI+4mXVxx4CcBD4lRn90khfFOcAM8Iu2pGaERHnAtUrf9EX3rLsOW
+wV+wYllChP71rI/4ueEch9X8ph1dA0nQQN1tLUi58pQlkCHY6K0QNFiD6K+RxaBl3yBt1IZqjfZi
+UVjTb7xJDrYnLPIASlPd0AduDik8pKn+GTqIFWgkkRr5mY6c9jTqHxY7rASDNi7LGKUE9gPFd1LD
+xPJmsl+8L+lcVJjJNU7Tkps/ZZJuo/EqlwbUd/Wq45S++YBBfYlFaOXn/bVMhxXi1SH3xMHSAjH1
+aYj0YHEdBHnEF1ouahyS4607cundZcSR29kITrUnFSi/ZP3zKREa3MGm/qrJS7qFSxlHVsYHBIjy
+VRx+teV4nQWKJyA6x/T9Sx63lM7duwhVRdh9JxhxnrKAyUBH5HwhpFXHreMjudNdY9nMaWaKP9Ge
+oD4Rr4iA3kvaHjtqSfhB55PgQO7Od/KLNTRfMMPl7IjbouQNCai++hV+p7BRAjtGUwTOXp9FHbv7
+YFGBFl3a0e1+YEoQA+0Psf1x2lENCJwH87DuZxuKI3kbcY6XA5kebt43m/eztRa17z/vmwyiQ/up
++RpMU9Xp1bv39h84QbyvZYN40xzHc8togJmPtSKCyEPcmHdt9t0LF6TCsb4k+kIBRUXMfnYpEDqv
+E6dldgWHjVy/4llWqyj3SsToERP1VhaloWyq8QRNke6lKzxMXOhmupKX195V2cA+6EGY3sK/ykhl
+fYOofbKcHwevHKgOJyj7Tj6+9qUgda/EI01lcJicTO8Nqb0LW+FfwIiws7WlsZWuxQGUZ0SOMBU4
+MnR9NPbS6rUSx2rMfSPn18Jd82D5eoM32ogRQb7C2pgXQbQoAegl98vtOjkze4wsa6CmW0rmbQrJ
+bpgPoWiZ1t/BlAUvxjRuzQSNNhnyvaC5nib6NYZAcr9BCm3yJ0sR/uSOUG8cCoJptMYhH9XxHqKl
+ACwfHgq7/mHBTxhQCmw5hkDWvY7FqzDPME3igab1Mda4lxOyUjJ3PeVzZbWZY2s/oUaSbntsSqRM
+z+zutj83Nm76iOSS0MXxCfi5VKYThzGdfXkYB2tZP8yPhh+sw0CpqeV5KB810C76abZbVZ+EDw==
diff --git a/plugins/scanners/scanoss/src/test/kotlin/ScanOssScannerDirectoryTest.kt b/plugins/scanners/scanoss/src/test/kotlin/ScanOssScannerDirectoryTest.kt
@@ -20,12 +20,15 @@
 package org.ossreviewtoolkit.plugins.scanners.scanoss
 
 import com.github.tomakehurst.wiremock.WireMockServer
+import com.github.tomakehurst.wiremock.client.WireMock
 import com.github.tomakehurst.wiremock.core.WireMockConfiguration
 
+import io.kotest.assertions.fail
 import io.kotest.core.spec.style.StringSpec
 import io.kotest.matchers.collections.containExactly
 import io.kotest.matchers.collections.containExactlyInAnyOrder
 import io.kotest.matchers.should
+import io.kotest.matchers.shouldBe
 
 import io.mockk.spyk
 
@@ -39,11 +42,16 @@ import org.ossreviewtoolkit.model.SnippetFinding
 import org.ossreviewtoolkit.model.TextLocation
 import org.ossreviewtoolkit.model.VcsInfo
 import org.ossreviewtoolkit.model.VcsType
+import org.ossreviewtoolkit.model.config.Excludes
+import org.ossreviewtoolkit.model.config.PathExclude
+import org.ossreviewtoolkit.model.config.PathExcludeReason
 import org.ossreviewtoolkit.plugins.api.Secret
 import org.ossreviewtoolkit.scanner.ScanContext
 import org.ossreviewtoolkit.utils.spdx.SpdxExpression
 
+// Define separate directories for different test scenarios.
 private val TEST_DIRECTORY_TO_SCAN = File("src/test/assets/filesToScan")
+private val EXCLUSION_TEST_DIRECTORY = File("src/test/assets/exclusionTest")
 
 /**
  * A test for scanning a directory with the [ScanOss] scanner.
@@ -110,4 +118,61 @@ class ScanOssScannerDirectoryTest : StringSpec({
             )
         }
     }
+
+    "Scanner should exclude only files matching the specified path pattern (**/*.kt)" {
+        val pathExcludes = listOf(
+            PathExclude(
+                pattern = "**/*.kt", // Glob pattern to match all .kt files in any directory.
+                reason = PathExcludeReason.BUILD_TOOL_OF,
+                comment = "Excluding .kt source files from scanning"
+            )
+        )
+
+        // Verify our test file exists. This file should be included in the scan since it does not match the exclusion
+        // pattern (it is a .go file, not a .kt file).
+        val includedFile = File(EXCLUSION_TEST_DIRECTORY, "server.go")
+        if (!includedFile.isFile) {
+            fail("The file ${includedFile.absolutePath} does not exist - test environment may not be properly set up")
+        }
+
+        // Run the scanner with our exclusion pattern. This will traverse the directory and should skip .kt files.
+        scanner.scanPath(
+            EXCLUSION_TEST_DIRECTORY,
+            ScanContext(
+                labels = emptyMap(),
+                packageType = PackageType.PACKAGE,
+                excludes = Excludes(paths = pathExcludes)
+            )
+        )
+
+        // Retrieve all HTTP POST requests captured by WireMock during the scan.
+        val requests = server.findAll(WireMock.postRequestedFor(WireMock.anyUrl()))
+        val requestBodies = requests.map { it.bodyAsString }
+
+        // The scanner sends files to the API in a multipart/form-data POST request with this format:
+        // --boundary
+        // Content-Disposition: form-data; name="file"; filename="[UUID].wfp"
+        // Content-Type: text/plain; charset=utf-8
+        // Content-Length: [length]
+        //
+        // file=[hash],[size],[filename]
+        // [fingerprint data for the file]
+        // --boundary--
+
+        // Extract included filenames using a regex pattern from the ScanOSS HTTP POST.
+        // The pattern matches lines starting with "file=" followed by hash and size, then captures the filename.
+        val filenamePattern = "file=.*?,.*?,(.+)".toRegex(RegexOption.MULTILINE)
+        val includedFiles = requestBodies.flatMap { body ->
+            filenamePattern.findAll(body).map { it.groupValues[1] }.toList()
+        }
+
+        // Verify that .kt files were excluded from the scan.
+        // These assertions check that Kotlin files are not present in the API requests.
+        includedFiles.any { it.contains("ArchiveUtils.kt") } shouldBe false
+        includedFiles.any { it.contains("ScannerFactory.kt") } shouldBe false
+
+        // Verify that non-.kt files were included in the scan.
+        // This assertion checks that our Go file was sent to the API.
+        includedFiles.any { it.contains("server.go") } shouldBe true
+    }
 })
