diff --git a/plugins/scanners/scanoss/src/main/kotlin/ScanOss.kt b/plugins/scanners/scanoss/src/main/kotlin/ScanOss.kt index 32287c82f850a..460ee1d5fa614 100644 --- a/plugins/scanners/scanoss/src/main/kotlin/ScanOss.kt +++ b/plugins/scanners/scanoss/src/main/kotlin/ScanOss.kt @@ -20,13 +20,24 @@ package org.ossreviewtoolkit.plugins.scanners.scanoss import com.scanoss.Scanner +import com.scanoss.filters.FilterConfig +import com.scanoss.settings.Bom +import com.scanoss.settings.RemoveRule +import com.scanoss.settings.ReplaceRule +import com.scanoss.settings.Rule +import com.scanoss.settings.ScanossSettings import com.scanoss.utils.JsonUtils import com.scanoss.utils.PackageDetails import java.io.File import java.time.Instant +import org.apache.logging.log4j.kotlin.logger + import org.ossreviewtoolkit.model.ScanSummary +import org.ossreviewtoolkit.model.config.SnippetChoices +import org.ossreviewtoolkit.model.config.snippet.SnippetChoice +import org.ossreviewtoolkit.model.config.snippet.SnippetChoiceReason import org.ossreviewtoolkit.plugins.api.OrtPlugin import org.ossreviewtoolkit.plugins.api.PluginDescriptor import org.ossreviewtoolkit.scanner.PathScannerWrapper @@ -65,8 +76,28 @@ class ScanOss( override fun scanPath(path: File, context: ScanContext): ScanSummary { val startTime = Instant.now() + val filterConfig = FilterConfig.builder() + .customFilter { currentPath -> + // The "currentPath" variable contains a path object representing the file or directory being evaluated + // by the filter. + // This is provided by the Scanner and represents individual files/directories during traversal. + try { + val relativePath = currentPath.toFile().toRelativeString(path) + val isExcluded = context.excludes?.isPathExcluded(relativePath) ?: false + logger.debug { "Path: $currentPath, relative: $relativePath, isExcluded: $isExcluded" } + isExcluded + } catch (e: IllegalArgumentException) { + logger.warn { "Error processing path $currentPath: ${e.message}" } + false + } + } + .build() + // Build the scanner at function level in case any path-specific settings or filters are needed later. - val scanoss = scanossBuilder.build() + val scanoss = scanossBuilder + .settings(buildSettingsFromORTContext(context)) + .filterConfig(filterConfig) + .build() val rawResults = when { path.isFile -> listOf(scanoss.scanFile(path.toString())) @@ -77,4 +108,70 @@ class ScanOss( val endTime = Instant.now() return generateSummary(startTime, endTime, results) } + + data class ProcessedRules( + val includeRules: List, + val ignoreRules: List, + val replaceRules: List, + val removeRules: List + ) + + private fun buildSettingsFromORTContext(context: ScanContext): ScanossSettings { + val rules = processSnippetChoices(context.snippetChoices) + val bom = Bom.builder() + .ignore(rules.ignoreRules) + .include(rules.includeRules) + .replace(rules.replaceRules) + .remove(rules.removeRules) + .build() + return ScanossSettings.builder().bom(bom).build() + } + + fun processSnippetChoices(snippetChoices: List): ProcessedRules { + val includeRules = mutableListOf() + val ignoreRules = mutableListOf() + val replaceRules = mutableListOf() + val removeRules = mutableListOf() + + snippetChoices.forEach { snippetChoice -> + snippetChoice.choices.forEach { choice -> + when (choice.choice.reason) { + SnippetChoiceReason.ORIGINAL_FINDING -> { + includeRules.includeFinding(choice) + } + + SnippetChoiceReason.NO_RELEVANT_FINDING -> { + removeRules.removeFinding(choice) + } + + SnippetChoiceReason.OTHER -> { + logger.info { + "Encountered OTHER reason for snippet choice in file ${choice.given.sourceLocation.path}" + } + } + } + } + } + + return ProcessedRules(includeRules, ignoreRules, replaceRules, removeRules) + } + + private fun MutableList.includeFinding(choice: SnippetChoice) { + this += Rule.builder() + .purl(choice.choice.purl) + .path(choice.given.sourceLocation.path) + .build() + } + + private fun MutableList.removeFinding(choice: SnippetChoice) { + this += RemoveRule.builder().apply { + path(choice.given.sourceLocation.path) + + // Set line range only if both line positions (startLine and endLine) are known. + if (choice.given.sourceLocation.hasLineRange) { + startLine(choice.given.sourceLocation.startLine) + endLine(choice.given.sourceLocation.endLine) + } + }.build() + } } diff --git a/plugins/scanners/scanoss/src/test/assets/exclusionTest/ArchiveUtils.kt b/plugins/scanners/scanoss/src/test/assets/exclusionTest/ArchiveUtils.kt new file mode 100644 index 0000000000000..fb5fef1c4962d --- /dev/null +++ b/plugins/scanners/scanoss/src/test/assets/exclusionTest/ArchiveUtils.kt @@ -0,0 +1,31 @@ +/* + * This file contains random data generated using the following command: + * head -c 1k < /dev/urandom | base64 + * + * The command takes 1 kilobyte of random bytes from the /dev/urandom device + * and then encodes it as base64 text. + * + * Generated on: Fri Mar 14 05:04:43 PM CET 2025 + * Purpose: To create test data with completely random content that cannot + * match any existing code in repositories, thereby avoiding false + * positives when scanning ORT source code. + */ + +7m8Y06QhHzmQ4ePs0UUUasqsc8SP1ayNTFdQb6wffQwMu605hXOGHbOoy5pUv7ksgf6sw5ET2qXp +T23LF2yA1cdNeDt8DBDd3IDmLX/wGgXcQjcaCtfSsMWB7oqHBMGkzwC5fMcDKPLK6ec2MwX6WPkw +E18ImifWtAmGPEFGxWuqIinhE1yGSN+ImqJPVmpYfMOaDIAaS3JpiHZDmJW5uyQ5DB6W7lpm0q+f +ZbtGPBeimy1jWF0H6kEW/TIve8RzUjdHU/t//O9r0b2AP08shSrSDWGlbQzxTniLOp2VZxNUEcVM +c9/Lx4OXEaM/3NDCdr4qQS/1kZpGKFrv06zzC8tlncGaxBfdZSCsh1i+LbZtvUmTSv/wz7g+mld5 +WB2lSzF1Ervzqnm2+3iY+9TvVxDWzZ27LWsd1kvFrJCM03jI1q0c7uJrnnovAOoZkH2QiMPNBQmB +wShT36h3su/aiOXEquXi+DoTSYDNgXeHVGI2joLVWYLfeTcTTfdvZiwp0K+XQp6fKtWX8tpUibNq +ngp2dOlzl5yiT+WAD2ETGuyEML/wM3oz+wB93me2YYLJqz/1gtlnnRvGnAukbFLpxxXGK7Vnz+FF +KfcPWF+O/FNV3nJD+m2nlMVj1n4lRM/mUEdVDhDDtxhywvi6DdNQMcUoeXZRT3dLk27+efNLvMDk +7TsW/asvMoPrioAkDiTHqWvy+OUImWqqzNpzxIMuTWZrApSklw2UeyXknvHBORUN95AM6Oe9iKb0 +7B2g8U9dIFo7v/AhaDqoQMw+Dz1KfH6+fPaqZEy2H1U7/9RSorKz0fycz7n7BtqWxjenqw11LLxy +lO36udPuvtr2b/WB/4ch0LuoI2eA11iTeIG4DuTxvizU3lExBXP+e8EAjkWx6F2ymDrI21PYPp++ +uidSk3g/RmaRZGk8akcXbs3pDO/twfjaH3YWYZzBf8aP1TRYDp4NF5v2OhWDa5d2dqdQGDRGg/wy +Gf0W8txn/fQ3QN7SS9qPftgD6OYIpxKjIWBq/zb5+SAzhBZVjFYw+KVi+zu/P7he7xLRko6APCum +Ugk7wqohWVdbl2IG2RIuPUOH2zQdzVJvLisKhfq3q6ydGmjD/WRNOxbebpmSKcmZWVg0Ko7/e0ys +ymV2Ud0tIZwfIH/7476SZAh0ym1U7mgyzm/jlxKm5gUIF1+NWQiqa80GmsAJfquf1Yj4i0ftF+eO +6OPJqbkZERpu24u2HIfL6CvlUkx08mS+eqLzyRiRuidDcQGFOK+0xPUk01jOZnGiY1ptG4W+Fo5K +OhcT2H14wQqiHsthzMhpSLXwMG2ddM7P69rHEAXB3iXyWopgdWopVekxHEuar0mv3D6uO/4HKQ== diff --git a/plugins/scanners/scanoss/src/test/assets/exclusionTest/ScannerFactory.kt b/plugins/scanners/scanoss/src/test/assets/exclusionTest/ScannerFactory.kt new file mode 100644 index 0000000000000..271420857d262 --- /dev/null +++ b/plugins/scanners/scanoss/src/test/assets/exclusionTest/ScannerFactory.kt @@ -0,0 +1,31 @@ +/* + * This file contains random data generated using the following command: + * head -c 1k < /dev/urandom | base64 + * + * The command takes 1 kilobyte of random bytes from the /dev/urandom device + * and then encodes it as base64 text. + * + * Generated on: Fri Mar 14 05:05:29 PM CET 2025 + * Purpose: To create test data with completely random content that cannot + * match any existing code in repositories, thereby avoiding false + * positives when scanning ORT source code. + */ + +IyaayUoMK28Ib11Z55hC2OShY3p3HzWQyGy199hx20oqZrypl9AuDhKtBdl+qozcZBNajzvkU3H/ +jh3vV/P9I2VLQNVqMCpjelQoXyVq/nmbwxdQBXGbLgcC4J05ujQ2hoXuF4jdtEttDxca8P/EpUub +nmSO3zmz86LqyyYgFj3imketFw0GvnCYU/8VDjmLxnigspEVI7ZDOacKOshObwH+Br/XgFHr5tyc +ulGqACTjGY3EEdAjC2+tcTqoI+4mXVxx4CcBD4lRn90khfFOcAM8Iu2pGaERHnAtUrf9EX3rLsOW +wV+wYllChP71rI/4ueEch9X8ph1dA0nQQN1tLUi58pQlkCHY6K0QNFiD6K+RxaBl3yBt1IZqjfZi +UVjTb7xJDrYnLPIASlPd0AduDik8pKn+GTqIFWgkkRr5mY6c9jTqHxY7rASDNi7LGKUE9gPFd1LD +xPJmsl+8L+lcVJjJNU7Tkps/ZZJuo/EqlwbUd/Wq45S++YBBfYlFaOXn/bVMhxXi1SH3xMHSAjH1 +aYj0YHEdBHnEF1ouahyS4607cundZcSR29kITrUnFSi/ZP3zKREa3MGm/qrJS7qFSxlHVsYHBIjy +VRx+teV4nQWKJyA6x/T9Sx63lM7duwhVRdh9JxhxnrKAyUBH5HwhpFXHreMjudNdY9nMaWaKP9Ge +oD4Rr4iA3kvaHjtqSfhB55PgQO7Od/KLNTRfMMPl7IjbouQNCai++hV+p7BRAjtGUwTOXp9FHbv7 +YFGBFl3a0e1+YEoQA+0Psf1x2lENCJwH87DuZxuKI3kbcY6XA5kebt43m/eztRa17z/vmwyiQ/up ++RpMU9Xp1bv39h84QbyvZYN40xzHc8togJmPtSKCyEPcmHdt9t0LF6TCsb4k+kIBRUXMfnYpEDqv +E6dldgWHjVy/4llWqyj3SsToERP1VhaloWyq8QRNke6lKzxMXOhmupKX195V2cA+6EGY3sK/ykhl +fYOofbKcHwevHKgOJyj7Tj6+9qUgda/EI01lcJicTO8Nqb0LW+FfwIiws7WlsZWuxQGUZ0SOMBU4 +MnR9NPbS6rUSx2rMfSPn18Jd82D5eoM32ogRQb7C2pgXQbQoAegl98vtOjkze4wsa6CmW0rmbQrJ +bpgPoWiZ1t/BlAUvxjRuzQSNNhnyvaC5nib6NYZAcr9BCm3yJ0sR/uSOUG8cCoJptMYhH9XxHqKl +ACwfHgq7/mHBTxhQCmw5hkDWvY7FqzDPME3igab1Mda4lxOyUjJ3PeVzZbWZY2s/oUaSbntsSqRM +z+zutj83Nm76iOSS0MXxCfi5VKYThzGdfXkYB2tZP8yPhh+sw0CpqeV5KB810C76abZbVZ+EDw== diff --git a/plugins/scanners/scanoss/src/test/assets/exclusionTest/server.go b/plugins/scanners/scanoss/src/test/assets/exclusionTest/server.go new file mode 100644 index 0000000000000..271420857d262 --- /dev/null +++ b/plugins/scanners/scanoss/src/test/assets/exclusionTest/server.go @@ -0,0 +1,31 @@ +/* + * This file contains random data generated using the following command: + * head -c 1k < /dev/urandom | base64 + * + * The command takes 1 kilobyte of random bytes from the /dev/urandom device + * and then encodes it as base64 text. + * + * Generated on: Fri Mar 14 05:05:29 PM CET 2025 + * Purpose: To create test data with completely random content that cannot + * match any existing code in repositories, thereby avoiding false + * positives when scanning ORT source code. + */ + +IyaayUoMK28Ib11Z55hC2OShY3p3HzWQyGy199hx20oqZrypl9AuDhKtBdl+qozcZBNajzvkU3H/ +jh3vV/P9I2VLQNVqMCpjelQoXyVq/nmbwxdQBXGbLgcC4J05ujQ2hoXuF4jdtEttDxca8P/EpUub +nmSO3zmz86LqyyYgFj3imketFw0GvnCYU/8VDjmLxnigspEVI7ZDOacKOshObwH+Br/XgFHr5tyc +ulGqACTjGY3EEdAjC2+tcTqoI+4mXVxx4CcBD4lRn90khfFOcAM8Iu2pGaERHnAtUrf9EX3rLsOW +wV+wYllChP71rI/4ueEch9X8ph1dA0nQQN1tLUi58pQlkCHY6K0QNFiD6K+RxaBl3yBt1IZqjfZi +UVjTb7xJDrYnLPIASlPd0AduDik8pKn+GTqIFWgkkRr5mY6c9jTqHxY7rASDNi7LGKUE9gPFd1LD +xPJmsl+8L+lcVJjJNU7Tkps/ZZJuo/EqlwbUd/Wq45S++YBBfYlFaOXn/bVMhxXi1SH3xMHSAjH1 +aYj0YHEdBHnEF1ouahyS4607cundZcSR29kITrUnFSi/ZP3zKREa3MGm/qrJS7qFSxlHVsYHBIjy +VRx+teV4nQWKJyA6x/T9Sx63lM7duwhVRdh9JxhxnrKAyUBH5HwhpFXHreMjudNdY9nMaWaKP9Ge +oD4Rr4iA3kvaHjtqSfhB55PgQO7Od/KLNTRfMMPl7IjbouQNCai++hV+p7BRAjtGUwTOXp9FHbv7 +YFGBFl3a0e1+YEoQA+0Psf1x2lENCJwH87DuZxuKI3kbcY6XA5kebt43m/eztRa17z/vmwyiQ/up ++RpMU9Xp1bv39h84QbyvZYN40xzHc8togJmPtSKCyEPcmHdt9t0LF6TCsb4k+kIBRUXMfnYpEDqv +E6dldgWHjVy/4llWqyj3SsToERP1VhaloWyq8QRNke6lKzxMXOhmupKX195V2cA+6EGY3sK/ykhl +fYOofbKcHwevHKgOJyj7Tj6+9qUgda/EI01lcJicTO8Nqb0LW+FfwIiws7WlsZWuxQGUZ0SOMBU4 +MnR9NPbS6rUSx2rMfSPn18Jd82D5eoM32ogRQb7C2pgXQbQoAegl98vtOjkze4wsa6CmW0rmbQrJ +bpgPoWiZ1t/BlAUvxjRuzQSNNhnyvaC5nib6NYZAcr9BCm3yJ0sR/uSOUG8cCoJptMYhH9XxHqKl +ACwfHgq7/mHBTxhQCmw5hkDWvY7FqzDPME3igab1Mda4lxOyUjJ3PeVzZbWZY2s/oUaSbntsSqRM +z+zutj83Nm76iOSS0MXxCfi5VKYThzGdfXkYB2tZP8yPhh+sw0CpqeV5KB810C76abZbVZ+EDw== diff --git a/plugins/scanners/scanoss/src/test/assets/filesToScan/ArchiveUtils.kt b/plugins/scanners/scanoss/src/test/assets/filesToScan/ArchiveUtils.kt index 58da835ead789..987dee0828263 100644 --- a/plugins/scanners/scanoss/src/test/assets/filesToScan/ArchiveUtils.kt +++ b/plugins/scanners/scanoss/src/test/assets/filesToScan/ArchiveUtils.kt @@ -1,245 +1,31 @@ /* - * Copyright (C) 2017 The ORT Project Authors (see ) + * This file contains random data generated using the following command: + * head -c 1k < /dev/urandom | base64 * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * The command takes 1 kilobyte of random bytes from the /dev/urandom device + * and then encodes it as base64 text. * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - * License-Filename: LICENSE - */ - -@file:Suppress("MatchingDeclarationName") - -package org.ossreviewtoolkit.utils - -import java.io.File -import java.io.IOException -import java.io.InputStream -import java.nio.file.FileVisitResult -import java.nio.file.Files -import java.nio.file.Path -import java.nio.file.SimpleFileVisitor -import java.nio.file.attribute.BasicFileAttributes -import java.util.zip.Deflater - -import org.apache.commons.compress.archivers.ArchiveEntry -import org.apache.commons.compress.archivers.ArchiveInputStream -import org.apache.commons.compress.archivers.sevenz.SevenZFile -import org.apache.commons.compress.archivers.tar.TarArchiveEntry -import org.apache.commons.compress.archivers.tar.TarArchiveInputStream -import org.apache.commons.compress.archivers.zip.ZipArchiveEntry -import org.apache.commons.compress.archivers.zip.ZipArchiveInputStream -import org.apache.commons.compress.archivers.zip.ZipArchiveOutputStream -import org.apache.commons.compress.archivers.zip.ZipFile -import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream -import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream -import org.apache.commons.compress.compressors.xz.XZCompressorInputStream -import org.apache.commons.compress.utils.SeekableInMemoryByteChannel - -enum class ArchiveType(vararg val extensions: String) { - TAR(".gem", ".tar"), - TAR_BZIP2(".tar.bz2", ".tbz2"), - TAR_GZIP(".crate", ".tar.gz", ".tgz"), - TAR_XZ(".tar.xz", ".txz"), - ZIP(".aar", ".egg", ".jar", ".war", ".whl", ".zip"), - SEVENZIP(".7z"), - NONE(""); - - companion object { - fun getType(filename: String): ArchiveType { - val lowerName = filename.toLowerCase() - return (ArchiveType.entries - NONE).find { type -> - type.extensions.any { lowerName.endsWith(it) } - } ?: NONE - } - } -} - -/** - * Unpack the [File] to [targetDirectory]. - */ -fun File.unpack(targetDirectory: File) = - when (ArchiveType.getType(name)) { - ArchiveType.SEVENZIP -> unpack7Zip(targetDirectory) - ArchiveType.ZIP -> unpackZip(targetDirectory) - - ArchiveType.TAR -> inputStream().unpackTar(targetDirectory) - ArchiveType.TAR_BZIP2 -> BZip2CompressorInputStream(inputStream()).unpackTar(targetDirectory) - ArchiveType.TAR_GZIP -> GzipCompressorInputStream(inputStream()).unpackTar(targetDirectory) - ArchiveType.TAR_XZ -> XZCompressorInputStream(inputStream()).unpackTar(targetDirectory) - - ArchiveType.NONE -> { - throw IOException("Unable to guess compression scheme from file name '$name'.") - } - } - -/** - * Unpack the [File] assuming it is a 7-Zip archive. This implementation ignores empty directories and symbolic links. - */ -fun File.unpack7Zip(targetDirectory: File) { - SevenZFile(this).use { zipFile -> - while (true) { - val entry = zipFile.nextEntry ?: break - - if (entry.isDirectory || entry.isAntiItem) { - continue - } - - val target = targetDirectory.resolve(entry.name) - - // There is no guarantee that directory entries appear before file entries, so ensure that the parent - // directory for a file exists. - target.parentFile.safeMkdirs() - - target.outputStream().use { output -> - zipFile.getInputStream(entry).copyTo(output) - } - } - } -} - -/** - * Unpack the [File] assuming it is a Zip archive. - */ -fun File.unpackZip(targetDirectory: File) = ZipFile(this).unpack(targetDirectory) - -/** - * Unpack the [ByteArray] assuming it is a Zip archive. - */ -fun ByteArray.unpackZip(targetDirectory: File) = ZipFile(SeekableInMemoryByteChannel(this)).unpack(targetDirectory) - -/** - * Pack the file into a ZIP [targetFile] using [Deflater.BEST_COMPRESSION]. If the file is a directory its content is - * recursively added to the archive. Only regular files are added, e.g. symbolic links or directories are skipped. If - * a [prefix] is specified, it is added to the file names in the ZIP file. - * If not all files shall be added to the archive a [filter] can be provided. - */ -fun File.packZip( - targetFile: File, - prefix: String = "", - overwrite: Boolean = false, - filter: (Path) -> Boolean = { true } -) { - require(overwrite || !targetFile.exists()) { - "The target ZIP file '${targetFile.absolutePath}' must not exist." - } - - ZipArchiveOutputStream(targetFile).use { output -> - output.setLevel(Deflater.BEST_COMPRESSION) - Files.walkFileTree(toPath(), object : SimpleFileVisitor() { - override fun visitFile(file: Path, attrs: BasicFileAttributes): FileVisitResult { - if (attrs.isRegularFile && filter(file)) { - val entry = ZipArchiveEntry(file.toFile(), "$prefix${this@packZip.toPath().relativize(file)}") - output.putArchiveEntry(entry) - file.toFile().inputStream().use { input -> input.copyTo(output) } - output.closeArchiveEntry() - } - - return FileVisitResult.CONTINUE - } - }) - } -} - -/** - * Unpack the [InputStream] to [targetDirectory] assuming that it is a tape archive (TAR). This implementation ignores - * empty directories and symbolic links. - */ -fun InputStream.unpackTar(targetDirectory: File) = - TarArchiveInputStream(this).unpack( - targetDirectory, - { entry -> !(entry as TarArchiveEntry).isFile }, - { entry -> (entry as TarArchiveEntry).mode } - ) - -/** - * Unpack the [InputStream] to [targetDirectory] assuming that it is a ZIP archive. This implementation ignores empty - * directories and symbolic links. - */ -fun InputStream.unpackZip(targetDirectory: File) = - ZipArchiveInputStream(this).unpack( - targetDirectory, - { entry -> (entry as ZipArchiveEntry).let { it.isDirectory || it.isUnixSymlink } }, - { entry -> (entry as ZipArchiveEntry).unixMode } - ) - -/** - * Copy the executable bit contained in [mode] to the [target] file's mode bits. - */ -private fun copyExecutableModeBit(target: File, mode: Int) { - if (Os.isWindows) return - - // Note: In contrast to Java, Kotlin does not support octal literals, see - // https://kotlinlang.org/docs/reference/basic-types.html#literal-constants. - // The bit-triplets from left to right stand for user, groups, other, respectively. - if (mode and 0b001_000_001 != 0) { - target.setExecutable(true, (mode and 0b000_000_001) == 0) - } -} - -/** - * Unpack this [ArchiveInputStream] to the [targetDirectory], skipping all entries for which [shouldSkip] returns true, - * and using what [mode] returns as the file mode bits. - */ -private fun ArchiveInputStream.unpack( - targetDirectory: File, - shouldSkip: (ArchiveEntry) -> Boolean, - mode: (ArchiveEntry) -> Int -) = - use { input -> - while (true) { - val entry = input.nextEntry ?: break - - if (shouldSkip(entry)) continue - - val target = targetDirectory.resolve(entry.name) - - // There is no guarantee that directory entries appear before file entries, so ensure that the parent - // directory for a file exists. - target.parentFile.safeMkdirs() - - target.outputStream().use { output -> - input.copyTo(output) - } - - copyExecutableModeBit(target, mode(entry)) - } - } - -/** - * Unpack the [ZipFile]. In contrast to [InputStream.unpackZip] this properly parses the ZIP's central directory, see - * https://commons.apache.org/proper/commons-compress/zip.html#ZipArchiveInputStream_vs_ZipFile. - */ -private fun ZipFile.unpack(targetDirectory: File) = - use { zipFile -> - val entries = zipFile.entries - - while (entries.hasMoreElements()) { - val entry = entries.nextElement() - - if (entry.isDirectory || entry.isUnixSymlink) { - continue - } - - val target = targetDirectory.resolve(entry.name) - - // There is no guarantee that directory entries appear before file entries, so ensure that the parent - // directory for a file exists. - target.parentFile.safeMkdirs() - - target.outputStream().use { output -> - zipFile.getInputStream(entry).copyTo(output) - } - - copyExecutableModeBit(target, entry.unixMode) - } - } + * Generated on: Fri Mar 14 05:06:11 PM CET 2025 + * Purpose: To create test data with completely random content that cannot + * match any existing code in repositories, thereby avoiding false + * positives when scanning ORT source code. +*/ + +DPygtEpjZMRFKqxbJXCG83PLBBIlyTp4VXOKLCVrCOtkvRQ40stMrMTjCQu8Og1dMsB6fUyn3ZFO +B6PxaWqBYPFxOw+mm3blj5ph/iFh8Ihfw0Hwunv2O/z0OiErz2XejOOWhyQqplTMwPtO5awYM7Vl +1jqOXJv/0dA+9JuCpi4sWw8Fb+FCkNai43viz0A7acL2aQYXkeBvxTgR8bsmu1G+VI8ANfeAb/Rc +IY1j1kFsY6PtxNzM3tZg/RiiLndP6cr1SNhodMWWY/11Fx5qYNxFPWTiPIlRX1F4KR3yYeaEgkpN +zHaKR3FJjO0am/YBliMCu23zIY8G3cbgTEb/88tqsEt3Midjv5blIFXturT7zc4loaBgDYJvbGqG +1NJSFsDQWiglMKJd78oR8XdSP8oWJs0BU4GQUSLA41bdkI1dP04JEyM8grsXog6SEdhSfJcr5ZXj +3WdwFQzZHvAwQ9PinXJm0UFPuEVv8QGGbM1q/YnMe9okd23owDWIWZksUmT896SOprNhxj3hQUbE +Nmm+ts80cHu9puyhwbrG5V/QoiC7l/oWwDQ1P8MXHyMphYv+jlTLVaD54ppWPK4vOfhCKpEIrMBh +RqCBYXIcx6LXg95aPGOP8CSCIBuv/cqI65WYUUUhVFer/DyIEsPlIT+AZyuhy7Z0R4hkP7jn6irl +PHkyUBCzyIAmcwNWKpQCUfwfvkjhM02KVPwUstelPrv0ArK5MWEN5d+rOZenjRenkkT1fBO+Q6qu +Wec0XWIBbz+82Ebb+hgSBtVIOnCr7hzYPoknhlvnClu/7NinjDmW7SeCbou8KUbmg3pHgDBG6XHG +azOZIDeaowL4Pfkzy3L+OpL+mTb3Iwi8TVNvCJDkMIUtCaW2fhdzaWdcZM/cJghOJk6JxxB8d7T1 +VWp4HhAF6n6jAhu1uqJ/1KS92DnSYH8UuSPBguOjnsOWwMwrwGrFIzH/O4ICnJ7kydywszHYvZtk +HeB1BVs/VMLn9R323VH5vh3uvjMptggTG5fgnsNlD+ZZFPe59xqZ2g8w8Evnc+SQGxr76krxPi8n +wLd30VcI5yrgaEkyU/90mU0zzGVR5H5WtyCCn3S+MeGSyNKHAx+V7cpvkorwqXOcty7qQRrqhIMv +PhGPGOxNvhaYuz9uWtwfr1trjjspkBRtpazDfO8vqZAACDqm8xoYRYwdaRUf4rJkK80EDqL2daAb +NQyIuCtqy9FjTPcQHV9QHNhysPiROuPpGA2Ew40DkTEgeazLOPKRjWLqieBuKe7FP7v/LBcIV1id +rn89bhabHPemm4BlKaWL3o12PrHYQPb/51E2DZoKw1Rk4IQN0tWAUMVt5hxaS5jlEJ+Vh388WQ== diff --git a/plugins/scanners/scanoss/src/test/assets/filesToScan/ScannerFactory.kt b/plugins/scanners/scanoss/src/test/assets/filesToScan/ScannerFactory.kt index a92c2e8453b08..3f988c795670e 100644 --- a/plugins/scanners/scanoss/src/test/assets/filesToScan/ScannerFactory.kt +++ b/plugins/scanners/scanoss/src/test/assets/filesToScan/ScannerFactory.kt @@ -1,54 +1,31 @@ /* - * Copyright (C) 2017 The ORT Project Authors (see ) + * This file contains random data generated using the following command: + * head -c 1k < /dev/urandom | base64 * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * The command takes 1 kilobyte of random bytes from the /dev/urandom device + * and then encodes it as base64 text. * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - * License-Filename: LICENSE - */ - -package org.ossreviewtoolkit.scanner - -import java.util.ServiceLoader - -import org.ossreviewtoolkit.model.config.ScannerConfiguration - -/** - * A common interface for use with [ServiceLoader] that all [AbstractScannerFactory] classes need to implement. - */ -interface ScannerFactory { - /** - * The name to use to refer to the scanner. - */ - val scannerName: String - - /** - * Create a [Scanner] using the specified [config]. - */ - fun create(config: ScannerConfiguration): Scanner -} - -/** - * A generic factory class for a [Scanner]. + * Generated on: Fri Mar 14 05:07:04 PM CET 2025 + * Purpose: To create test data with completely random content that cannot + * match any existing code in repositories, thereby avoiding false + * positives when scanning ORT source code. */ -abstract class AbstractScannerFactory( - override val scannerName: String -) : ScannerFactory { - abstract override fun create(config: ScannerConfiguration): T - /** - * Return the scanner's name here to allow Clikt to display something meaningful when listing the scanners - * which are enabled by default via their factories. - */ - override fun toString() = scannerName -} +mneGKU/gTsBek3i9p6pHaccif0G7AhDAF/w3j6QSXhUJACJY9h+yN+gBf0Duf1DONiZrOlt96w77 +yQH9J0inoH0rJr0EKXWVkjyHTBwhmMWDcnwrAEmQRn/kCtFNfMRFAyJsxBE2suK754qfhUbS4hjt +qB9JHimTQCqQ2bgLJvgMgZ9Bfs21lIhkpTlUsdEyEBjy7VB5DVS8Usal+lIJENJ/RaJmxULcPj1C +LEUSaEf+GRr8wDoysFcIGav2uJpVaZfAvbfWxwnMScjpaKeY+lzScmnnYNXhRBYYh+mI1qJY/ycW +b5jREjm3iF1RQ5x9/PeLpx5fRPL11G9aaHWCrrHT9xrV8Lb5WIf5aGaZG+p7c3CDdHGhT0E7ltY9 +tFKs9vstrgqhL/l0hFxvoZtjBE2tdt6+lhGt2puZ6XoWAZf4Baas2cTUr5i6WvdDKbPnytcCFMuN +UgBJS60JpNaWPV45QQS970G2BhCoeI5fYi2//0KFz2axIuVfRhEaojFS+CNVwpd9hXrNQSXFaS38 +oYZF/p09SiWJHHoaaKSTEGkSZZRHDhEnfRhOnTmSdmqftMywk5178mcJWhOfemK/IcILhNeB+5am +E+AKJy+rsYffep5KXPobFn3BntAdJilwe6R0CHf5d3wIwUIDxMiqSgnDJZthiVFYBBhYTrLZza+t +eXTfmFvdjcsUqNSUuglSjFBVUFRM4rKk7Y8zOQJ6PWge7LOqVxsrLY9KsRt1iI6JZ9Pq3G7yL9aD +qsivx4mnGmUvVPSbSGK4soWKYOKpMkYkyALyHJhmQMY6BxYmbHbqIaxWS67IEj00XjcIi2tsV8qN +juG7Rgyc33peN/Xu6EDEN+Uv4+EMB+VbwEsSgVhIdIAjiMMQT/r9u7hjXcDyZZNpHK3wBd3bUfn0 +MMPbWI4hIIj6/c5YM5SiNi8zPGWAbpc76FAg7cskvYwMOFfoaUxtkMD8IW5Y7X/m3XSE/O9cy7Wa +2+cq+YkIs+YmyixoiQV/eJimhmH6P2VYgDYUoRzfcKsE1QpjiWdLq2PEjD5iTsOgy/HdWKLAvdnW +j3jgzvZNBmfX/MYyFJn/pBALbAVFvQtQTLBmAJlyVjPfLCEhA6RhMmATvOhm3OLsacdFjrrz/Mhb +hkOCDPSyqoinHTcHKvpKORhCnlYPBGshboJ28h1oQIcRhGYGCxdZfIVClbUG3os0AzjHRFtI8f0Z +WQmL0LrqrdxwmdAJauLjglYARUkAB/UHRBvI3S8DMLo/E8dpfhvyzb/Jgu0lfXXPmdSDCRXDL6/0 +vpbI40itpGnQ52u8wOHM3tz11v2UqcYi/HSiLQEt1CKOLpNnJuJ7KcwybUqJoR3Sbn7cBmvVxg== diff --git a/plugins/scanners/scanoss/src/test/kotlin/ScanOssScannerDirectoryTest.kt b/plugins/scanners/scanoss/src/test/kotlin/ScanOssScannerDirectoryTest.kt index d2cef5f3f3dbd..4c910465a8129 100644 --- a/plugins/scanners/scanoss/src/test/kotlin/ScanOssScannerDirectoryTest.kt +++ b/plugins/scanners/scanoss/src/test/kotlin/ScanOssScannerDirectoryTest.kt @@ -20,12 +20,15 @@ package org.ossreviewtoolkit.plugins.scanners.scanoss import com.github.tomakehurst.wiremock.WireMockServer +import com.github.tomakehurst.wiremock.client.WireMock import com.github.tomakehurst.wiremock.core.WireMockConfiguration +import io.kotest.assertions.fail import io.kotest.core.spec.style.StringSpec import io.kotest.matchers.collections.containExactly import io.kotest.matchers.collections.containExactlyInAnyOrder import io.kotest.matchers.should +import io.kotest.matchers.shouldBe import io.mockk.spyk @@ -39,11 +42,16 @@ import org.ossreviewtoolkit.model.SnippetFinding import org.ossreviewtoolkit.model.TextLocation import org.ossreviewtoolkit.model.VcsInfo import org.ossreviewtoolkit.model.VcsType +import org.ossreviewtoolkit.model.config.Excludes +import org.ossreviewtoolkit.model.config.PathExclude +import org.ossreviewtoolkit.model.config.PathExcludeReason import org.ossreviewtoolkit.plugins.api.Secret import org.ossreviewtoolkit.scanner.ScanContext import org.ossreviewtoolkit.utils.spdx.SpdxExpression +// Define separate directories for different test scenarios. private val TEST_DIRECTORY_TO_SCAN = File("src/test/assets/filesToScan") +private val EXCLUSION_TEST_DIRECTORY = File("src/test/assets/exclusionTest") /** * A test for scanning a directory with the [ScanOss] scanner. @@ -110,4 +118,61 @@ class ScanOssScannerDirectoryTest : StringSpec({ ) } } + + "Scanner should exclude only files matching the specified path pattern (**/*.kt)" { + val pathExcludes = listOf( + PathExclude( + pattern = "**/*.kt", // Glob pattern to match all .kt files in any directory. + reason = PathExcludeReason.BUILD_TOOL_OF, + comment = "Excluding .kt source files from scanning" + ) + ) + + // Verify our test file exists. This file should be included in the scan since it does not match the exclusion + // pattern (it is a .go file, not a .kt file). + val includedFile = File(EXCLUSION_TEST_DIRECTORY, "server.go") + if (!includedFile.isFile) { + fail("The file ${includedFile.absolutePath} does not exist - test environment may not be properly set up") + } + + // Run the scanner with our exclusion pattern. This will traverse the directory and should skip .kt files. + scanner.scanPath( + EXCLUSION_TEST_DIRECTORY, + ScanContext( + labels = emptyMap(), + packageType = PackageType.PACKAGE, + excludes = Excludes(paths = pathExcludes) + ) + ) + + // Retrieve all HTTP POST requests captured by WireMock during the scan. + val requests = server.findAll(WireMock.postRequestedFor(WireMock.anyUrl())) + val requestBodies = requests.map { it.bodyAsString } + + // The scanner sends files to the API in a multipart/form-data POST request with this format: + // --boundary + // Content-Disposition: form-data; name="file"; filename="[UUID].wfp" + // Content-Type: text/plain; charset=utf-8 + // Content-Length: [length] + // + // file=[hash],[size],[filename] + // [fingerprint data for the file] + // --boundary-- + + // Extract included filenames using a regex pattern from the ScanOSS HTTP POST. + // The pattern matches lines starting with "file=" followed by hash and size, then captures the filename. + val filenamePattern = "file=.*?,.*?,(.+)".toRegex(RegexOption.MULTILINE) + val includedFiles = requestBodies.flatMap { body -> + filenamePattern.findAll(body).map { it.groupValues[1] }.toList() + } + + // Verify that .kt files were excluded from the scan. + // These assertions check that Kotlin files are not present in the API requests. + includedFiles.any { it.contains("ArchiveUtils.kt") } shouldBe false + includedFiles.any { it.contains("ScannerFactory.kt") } shouldBe false + + // Verify that non-.kt files were included in the scan. + // This assertion checks that our Go file was sent to the API. + includedFiles.any { it.contains("server.go") } shouldBe true + } }) diff --git a/plugins/scanners/scanoss/src/test/kotlin/ScanOssTest.kt b/plugins/scanners/scanoss/src/test/kotlin/ScanOssTest.kt new file mode 100644 index 0000000000000..e8a9a98ac1829 --- /dev/null +++ b/plugins/scanners/scanoss/src/test/kotlin/ScanOssTest.kt @@ -0,0 +1,193 @@ +/* + * Copyright (C) 2025 The ORT Project Authors (see ) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * License-Filename: LICENSE + */ + +package org.ossreviewtoolkit.plugins.scanners.scanoss + +import io.kotest.core.spec.style.WordSpec +import io.kotest.matchers.collections.beEmpty +import io.kotest.matchers.collections.shouldBeSingleton +import io.kotest.matchers.should +import io.kotest.matchers.shouldBe + +import org.ossreviewtoolkit.model.TextLocation +import org.ossreviewtoolkit.model.config.SnippetChoices +import org.ossreviewtoolkit.model.config.snippet.Choice +import org.ossreviewtoolkit.model.config.snippet.Given +import org.ossreviewtoolkit.model.config.snippet.Provenance +import org.ossreviewtoolkit.model.config.snippet.SnippetChoice +import org.ossreviewtoolkit.model.config.snippet.SnippetChoiceReason + +// Sample files in the results. +private const val FILE_1 = "a.java" +private const val FILE_2 = "b.java" + +// A sample purl in the results. +private const val PURL_1 = "pkg:github/fakeuser/fakepackage1@1.0.0" + +class ScanOssTest : WordSpec({ + "processSnippetChoices()" should { + "create empty rules when no snippet choices exist" { + val scanoss = createScanOss(createScanOssConfig()) + + val emptySnippetChoices: List = listOf() + + val rules = scanoss.processSnippetChoices(emptySnippetChoices) + + rules.ignoreRules should beEmpty() + rules.removeRules should beEmpty() + rules.includeRules should beEmpty() + rules.replaceRules should beEmpty() + } + + "create an include rule for snippet choices with ORIGINAL finding" { + val vcsInfo = createVcsInfo() + val scanoss = createScanOss(createScanOssConfig()) + + val location = TextLocation(FILE_1, 10, 20) + val snippetChoices = createSnippetChoices( + vcsInfo.url, + createSnippetChoice( + location, + PURL_1, + "This is an original finding" + ) + ) + + val rules = scanoss.processSnippetChoices(snippetChoices) + + rules.includeRules.shouldBeSingleton { rule -> + rule.purl shouldBe PURL_1 + rule.path shouldBe FILE_1 + } + + rules.removeRules should beEmpty() + rules.ignoreRules should beEmpty() + rules.replaceRules should beEmpty() + } + + "create a remove rule for snippet choices with NOT_FINDING reason" { + val vcsInfo = createVcsInfo() + + val scanoss = createScanOss(createScanOssConfig()) + + val location = TextLocation(FILE_2, 15, 30) + val snippetChoices = createSnippetChoices( + vcsInfo.url, + createSnippetChoice( + location, + null, // null PURL for NOT_FINDING. + "This is not a relevant finding" + ) + ) + + val rules = scanoss.processSnippetChoices(snippetChoices) + + rules.removeRules.shouldBeSingleton { rule -> + rule.path shouldBe FILE_2 + rule.startLine shouldBe 15 + rule.endLine shouldBe 30 + } + + rules.includeRules should beEmpty() + rules.ignoreRules should beEmpty() + rules.replaceRules should beEmpty() + } + + "handle multiple snippet choices with different reasons correctly" { + val vcsInfo = createVcsInfo() + val scanoss = createScanOss(createScanOssConfig()) + + val location1 = TextLocation(FILE_1, 10, 20) + val location2 = TextLocation(FILE_2, 15, 30) + + val snippetChoices = createSnippetChoices( + vcsInfo.url, + createSnippetChoice( + location1, + PURL_1, + "This is an original finding" + ), + createSnippetChoice( + location2, + null, + "This is not a relevant finding" + ) + ) + + val rules = scanoss.processSnippetChoices(snippetChoices) + + rules.includeRules.shouldBeSingleton { rule -> + rule.purl shouldBe PURL_1 + rule.path shouldBe FILE_1 + } + + rules.removeRules.shouldBeSingleton { rule -> + rule.path shouldBe FILE_2 + rule.startLine shouldBe 15 + rule.endLine shouldBe 30 + } + + rules.ignoreRules should beEmpty() + rules.replaceRules should beEmpty() + } + + "create a remove rule without line ranges when snippet choice has UNKNOWN_LINE (-1) values" { + val vcsInfo = createVcsInfo() + val scanoss = createScanOss(createScanOssConfig()) + + // Create a TextLocation with -1 for start and end lines. + val location = TextLocation(FILE_2, TextLocation.UNKNOWN_LINE, TextLocation.UNKNOWN_LINE) + val snippetChoices = createSnippetChoices( + vcsInfo.url, + createSnippetChoice( + location, + null, // null PURL for NOT_FINDING. + "This is a not relevant finding with no line ranges" + ) + ) + + val rules = scanoss.processSnippetChoices(snippetChoices) + + rules.removeRules.shouldBeSingleton { rule -> + rule.path shouldBe FILE_2 + rule.startLine shouldBe null + rule.endLine shouldBe null + } + + rules.includeRules should beEmpty() + rules.ignoreRules should beEmpty() + rules.replaceRules should beEmpty() + } + } +}) + +private fun createSnippetChoices(provenanceUrl: String, vararg snippetChoices: SnippetChoice) = + listOf(SnippetChoices(Provenance(provenanceUrl), snippetChoices.asList())) + +private fun createSnippetChoice(location: TextLocation, purl: String? = null, comment: String) = + SnippetChoice( + Given( + location + ), + Choice( + purl, + if (purl == null) SnippetChoiceReason.NO_RELEVANT_FINDING else SnippetChoiceReason.ORIGINAL_FINDING, + comment + ) + ) diff --git a/plugins/scanners/scanoss/src/test/kotlin/TestUtils.kt b/plugins/scanners/scanoss/src/test/kotlin/TestUtils.kt new file mode 100644 index 0000000000000..a199e4887ac12 --- /dev/null +++ b/plugins/scanners/scanoss/src/test/kotlin/TestUtils.kt @@ -0,0 +1,70 @@ +/* + * Copyright (C) 2025 The ORT Project Authors (see ) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * License-Filename: LICENSE + */ + +package org.ossreviewtoolkit.plugins.scanners.scanoss + +import com.scanoss.rest.ScanApi + +import org.ossreviewtoolkit.model.VcsInfo +import org.ossreviewtoolkit.model.VcsType +import org.ossreviewtoolkit.plugins.api.Secret + +// A test project name. +internal const val PROJECT = "scanoss-test-project" + +// A (resolved) test revision. +private const val REVISION = "0123456789012345678901234567890123456789" + +/** + * Create a new [ScanOss] instance with the specified [config]. + */ +internal fun createScanOss(config: ScanOssConfig): ScanOss = ScanOss(config = config) + +/** + * Create a standard [ScanOssConfig] whose properties can be partly specified. + */ +internal fun createScanOssConfig( + apiUrl: String = ScanApi.DEFAULT_BASE_URL, + apiKey: Secret = Secret(""), + regScannerName: String? = null, + minVersion: String? = null, + maxVersion: String? = null, + readFromStorage: Boolean = true, + writeToStorage: Boolean = true +): ScanOssConfig = + ScanOssConfig( + apiUrl = apiUrl, + apiKey = apiKey, + regScannerName = regScannerName, + minVersion = minVersion, + maxVersion = maxVersion, + readFromStorage = readFromStorage, + writeToStorage = writeToStorage + ) + +/** + * Create a [VcsInfo] object for a project with the given [name][projectName] and the optional parameters for [type], + * [path], and [revision]. + */ +internal fun createVcsInfo( + projectName: String = PROJECT, + type: VcsType = VcsType.GIT, + path: String = "", + revision: String = REVISION +): VcsInfo = VcsInfo(type = type, path = path, revision = revision, url = "https://github.com/test/$projectName.git")