diff --git a/buildSrc/src/main/kotlin/LicenseUtils.kt b/buildSrc/src/main/kotlin/LicenseUtils.kt index 26b99631b95f6..b9561321ad687 100644 --- a/buildSrc/src/main/kotlin/LicenseUtils.kt +++ b/buildSrc/src/main/kotlin/LicenseUtils.kt @@ -34,6 +34,7 @@ object CopyrightableFiles { "plugins/reporters/asciidoc/src/main/resources/pdf-theme/pdf-theme.yml", "plugins/reporters/asciidoc/src/main/resources/templates/freemarker_implicit.ftl", "plugins/reporters/fossid/src/main/resources/templates/freemarker_implicit.ftl", + "plugins/reporters/scanoss/src/main/resources/templates/freemarker_implicit.ftl", "plugins/reporters/freemarker/src/main/resources/templates/freemarker_implicit.ftl", "plugins/reporters/static-html/src/main/resources/prismjs/", "plugins/reporters/web-app-template/yarn.lock", diff --git a/integrations/completions/ort-completion.fish b/integrations/completions/ort-completion.fish index 67aba1cc1859c..5e565980cf0f0 100644 --- a/integrations/completions/ort-completion.fish +++ b/integrations/completions/ort-completion.fish @@ -149,7 +149,7 @@ complete -c ort -f -n __fish_use_subcommand -a report -d 'Present Analyzer, Scan ## Options for report complete -c ort -n "__fish_seen_subcommand_from report" -l ort-file -s i -r -F -d 'The ORT result file to use.' complete -c ort -n "__fish_seen_subcommand_from report" -l output-dir -s o -r -F -d 'The output directory to store the generated reports in.' -complete -c ort -n "__fish_seen_subcommand_from report" -l report-formats -s f -r -d 'A comma-separated list of report formats to generate, any of [AOSD2.0, AOSD2.1, CtrlXAutomation, CycloneDX, DocBookTemplate, EvaluatedModel, FossID, FossIdSnippet, HtmlTemplate, ManPageTemplate, Opossum, PdfTemplate, PlainTextTemplate, SpdxDocument, StaticHTML, TrustSource, WebApp].' +complete -c ort -n "__fish_seen_subcommand_from report" -l report-formats -s f -r -d 'A comma-separated list of report formats to generate, any of [AOSD2.0, AOSD2.1, CtrlXAutomation, CycloneDX, DocBookTemplate, EvaluatedModel, FossID, FossIdSnippet, HtmlTemplate, ManPageTemplate, Opossum, PdfTemplate, PlainTextTemplate, ScanossSnippet, SpdxDocument, StaticHTML, TrustSource, WebApp].' complete -c ort -n "__fish_seen_subcommand_from report" -l copyright-garbage-file -r -F -d 'A file containing copyright statements which are marked as garbage. This can make the output inconsistent with the evaluator output but is useful when testing copyright garbage.' complete -c ort -n "__fish_seen_subcommand_from report" -l custom-license-texts-dir -r -F -d 'A directory which maps custom license IDs to license texts. It should contain one text file per license with the license ID as the filename. A custom license text is used only if its ID has a \'LicenseRef-\' prefix and if the respective license text is not known by ORT.' complete -c ort -n "__fish_seen_subcommand_from report" -l how-to-fix-text-provider-script -r -F -d 'The path to a Kotlin script which returns an instance of a \'HowToFixTextProvider\'. That provider injects how-to-fix texts in Markdown format for ORT issues.' diff --git a/plugins/reporters/scanoss/build.gradle.kts b/plugins/reporters/scanoss/build.gradle.kts new file mode 100644 index 0000000000000..7feaf0d15d6da --- /dev/null +++ b/plugins/reporters/scanoss/build.gradle.kts @@ -0,0 +1,39 @@ +/* + * Copyright (C) 2025 The ORT Project Authors (see ) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * License-Filename: LICENSE + */ + +plugins { + // Apply precompiled plugins. + id("ort-plugin-conventions") +} + +dependencies { + api(projects.reporter) + + ksp(projects.reporter) + + implementation(projects.model) + implementation(projects.plugins.reporters.asciidocReporter) + implementation(projects.plugins.reporters.freemarkerReporter) + implementation(projects.utils.commonUtils) + implementation(projects.utils.ortUtils) + + implementation(libs.kotlinx.coroutines) + + testImplementation(libs.mockk) +} diff --git a/plugins/reporters/scanoss/src/main/kotlin/ScanossSnippetReporter.kt b/plugins/reporters/scanoss/src/main/kotlin/ScanossSnippetReporter.kt new file mode 100644 index 0000000000000..0f482fe3f9acc --- /dev/null +++ b/plugins/reporters/scanoss/src/main/kotlin/ScanossSnippetReporter.kt @@ -0,0 +1,52 @@ +/* + * Copyright (C) 2025 The ORT Project Authors (see ) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * License-Filename: LICENSE + */ + +package org.ossreviewtoolkit.plugins.reporters.scanoss + +import java.io.File + +import org.ossreviewtoolkit.plugins.api.OrtPlugin +import org.ossreviewtoolkit.plugins.api.PluginDescriptor +import org.ossreviewtoolkit.plugins.reporters.asciidoc.AsciiDocTemplateReporterConfig +import org.ossreviewtoolkit.plugins.reporters.asciidoc.HtmlTemplateReporter +import org.ossreviewtoolkit.reporter.Reporter +import org.ossreviewtoolkit.reporter.ReporterFactory +import org.ossreviewtoolkit.reporter.ReporterInput + +@OrtPlugin( + displayName = "SCANOSS Snippet Reporter", + description = "Generates a detailed report of the SCANOSS snippet findings.", + factory = ReporterFactory::class +) +class ScanossSnippetReporter(override val descriptor: PluginDescriptor = ScanossSnippetReporterFactory.descriptor) : + Reporter by delegateReporter { + companion object { + private val delegateReporter = HtmlTemplateReporter( + ScanossSnippetReporterFactory.descriptor, + AsciiDocTemplateReporterConfig(templateIds = listOf("scanoss_snippet"), templatePaths = null) + ) + } + + override fun generateReport(input: ReporterInput, outputDir: File): List> { + val hasScanossResults = input.ortResult.scanner?.scanResults?.any { it.scanner.name == "SCANOSS" } == true + require(hasScanossResults) { "No SCANOSS scan results have been found." } + + return delegateReporter.generateReport(input, outputDir) + } +} diff --git a/plugins/reporters/scanoss/src/main/resources/templates/.gitattributes b/plugins/reporters/scanoss/src/main/resources/templates/.gitattributes new file mode 100644 index 0000000000000..abca6329e9e97 --- /dev/null +++ b/plugins/reporters/scanoss/src/main/resources/templates/.gitattributes @@ -0,0 +1,2 @@ +# Use Unix line endings for Freemarker templates for consistency across platforms. +**/*.ftl text eol=lf diff --git a/plugins/reporters/scanoss/src/main/resources/templates/asciidoc/scanoss_snippet.ftl b/plugins/reporters/scanoss/src/main/resources/templates/asciidoc/scanoss_snippet.ftl new file mode 100644 index 0000000000000..cc580d9d8bb0c --- /dev/null +++ b/plugins/reporters/scanoss/src/main/resources/templates/asciidoc/scanoss_snippet.ftl @@ -0,0 +1,142 @@ +[#-- + Copyright (C) 2025 The ORT Project Authors (see ) + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + SPDX-License-Identifier: Apache-2.0 + License-Filename: LICENSE +--] + +:publisher: OSS Review Toolkit +[#assign now = .now] +:revdate: ${now?date?iso_local} + +:title-page: +:sectnums: +:toc: + += SCANOSS Snippets +List of all the provenances with their files and snippets. +[#list ortResult.scanner.scanResults as scanResult] + +[#assign snippetsLimitIssue = helper.getSnippetsLimitIssue()] + +[#if snippetsLimitIssue?has_content] +[WARNING] +==== +${snippetsLimitIssue} +==== +[/#if] + +[#if scanResult.provenance.vcsInfo??] + [#assign url = scanResult.provenance.vcsInfo.url] +[#else] + [#assign url = scanResult.provenance.sourceArtifact.url] +[/#if] +== Provenance '${url}' + +[#assign summary = scanResult.summary] + +Scan start time : ${summary.startTime} + +End time : ${summary.startTime} + +[#if scanResult.provenance.vcsInfo??] + [#assign gitRepoUrl = url] + [#assign gitRevision = scanResult.provenance.vcsInfo.revision] + Git repo URL: ${gitRepoUrl} + + Git revision: ${gitRevision} + + [#if gitRepoUrl?contains("github.com")] + [#assign githubBaseURL = '${gitRepoUrl?remove_ending(".git")}/blob/${gitRevision}'] + [/#if] +[/#if] + +[#list helper.groupSnippetsByFile(summary.snippetFindings) as filePath, snippetFindings ] + +[#if gitRepoUrl?? && gitRepoUrl?contains("github.com")] + [#assign localFileURL = '${githubBaseURL}/${filePath}[${filePath}]'] +[#else] + [#assign localFileURL = "${filePath}"] +[/#if] +[#assign licenses = helper.collectLicenses(snippetFindings)] + +*${localFileURL}* + +License(s): +[#list licenses as license] + ${license}[#sep], +[/#list] + +[#list helper.groupSnippetsBySourceLines(snippetFindings) as sourceLocation, snippetFinding] +[#assign snippetCount = snippetFinding.snippets?size] + +[width=100%] +[cols="1,3,4,1,2"] +|=== +| Source Location | pURL | License | Score | Release Date + +.${snippetCount*2}+| +Partial match + +${sourceLocation.startLine?c}-${sourceLocation.endLine?c} + + +[#list snippetFinding.snippets as snippet ] + +| ${snippet.purl!""} +| ${snippet.license!""} +| ${snippet.score!""} +| ${snippet.additionalData['release_date']} + +4+a| +.Create a snippet choice for this snippet or mark it as false positive +[%collapsible] +==== +Add the following lines to the *.ort.yml* file. + +To **choose** this snippet: +[source,yaml] +-- +snippet_choices: +- provenance: + url: "${scanResult.provenance.vcsInfo.url}" + choices: + - given: + source_location: + path: "${filePath}" + start_line: ${snippetFinding.sourceLocation.startLine?c} + end_line: ${snippetFinding.sourceLocation.endLine?c} + choice: + purl: "${snippet.purl!""}" + reason: "ORIGINAL_FINDING" + comment: "Explain why this snippet choice was made" +-- +Or to mark this location has having ONLY **false positives snippets**: +[source,yaml] +-- +snippet_choices: +- provenance: + url: "${scanResult.provenance.vcsInfo.url}" + choices: + - given: + source_location: + path: "${filePath}" + start_line: ${snippetFinding.sourceLocation.startLine?c} + end_line: ${snippetFinding.sourceLocation.endLine?c} + choice: + reason: "NO_RELEVANT_FINDING" + comment: "Explain why this location has only false positives snippets" +-- +==== +[/#list] +|=== +[/#list] +[/#list] +[/#list] diff --git a/plugins/reporters/scanoss/src/main/resources/templates/freemarker_implicit.ftl b/plugins/reporters/scanoss/src/main/resources/templates/freemarker_implicit.ftl new file mode 100644 index 0000000000000..a409081912143 --- /dev/null +++ b/plugins/reporters/scanoss/src/main/resources/templates/freemarker_implicit.ftl @@ -0,0 +1,10 @@ +[#ftl] +[#-- @implicitly included --] + +[#-- @ftlvariable name="projects" type="kotlin.collections.Set" --] +[#-- @ftlvariable name="packages" type="kotlin.collections.Set" --] +[#-- @ftlvariable name="ortResult" type="org.ossreviewtoolkit.model.OrtResult" --] +[#-- @ftlvariable name="licenseTextProvider" type="org.ossreviewtoolkit.reporter.LicenseTextProvider" --] +[#-- @ftlvariable name="LicenseView" type="org.ossreviewtoolkit.model.licenses.LicenseView" --] +[#-- @ftlvariable name="helper" type="org.ossreviewtoolkit.plugins.reporters.freemarker.FreemarkerTemplateProcessor.TemplateHelper" --] +[#-- @ftlvariable name="projectsAsPackages" type="kotlin.collections.Set" --] diff --git a/plugins/scanners/scanoss/src/main/kotlin/ScanOss.kt b/plugins/scanners/scanoss/src/main/kotlin/ScanOss.kt index 5d5e3282bdd24..3974a23ccc6ee 100644 --- a/plugins/scanners/scanoss/src/main/kotlin/ScanOss.kt +++ b/plugins/scanners/scanoss/src/main/kotlin/ScanOss.kt @@ -19,27 +19,31 @@ package org.ossreviewtoolkit.plugins.scanners.scanoss -import com.scanoss.Winnowing -import com.scanoss.dto.ScanFileResult -import com.scanoss.rest.ScanApi +import com.scanoss.Scanner +import com.scanoss.filters.FilterConfig +import com.scanoss.settings.Bom +import com.scanoss.settings.RemoveRule +import com.scanoss.settings.ReplaceRule +import com.scanoss.settings.Rule +import com.scanoss.settings.ScanossSettings import com.scanoss.utils.JsonUtils import com.scanoss.utils.PackageDetails import java.io.File import java.time.Instant -import java.util.UUID import org.apache.logging.log4j.kotlin.logger import org.ossreviewtoolkit.model.ScanSummary +import org.ossreviewtoolkit.model.config.SnippetChoices +import org.ossreviewtoolkit.model.config.snippet.SnippetChoice +import org.ossreviewtoolkit.model.config.snippet.SnippetChoiceReason import org.ossreviewtoolkit.plugins.api.OrtPlugin import org.ossreviewtoolkit.plugins.api.PluginDescriptor import org.ossreviewtoolkit.scanner.PathScannerWrapper import org.ossreviewtoolkit.scanner.ScanContext import org.ossreviewtoolkit.scanner.ScannerMatcher -import org.ossreviewtoolkit.scanner.ScannerMatcherConfig import org.ossreviewtoolkit.scanner.ScannerWrapperFactory -import org.ossreviewtoolkit.utils.common.VCS_DIRECTORIES @OrtPlugin( id = "SCANOSS", @@ -51,11 +55,10 @@ class ScanOss( override val descriptor: PluginDescriptor = ScanOssFactory.descriptor, config: ScanOssConfig ) : PathScannerWrapper { - private val service = ScanApi.builder() + private val scanossBuilder = Scanner.builder() // As there is only a single endpoint, the SCANOSS API client expects the path to be part of the API URL. .url(config.apiUrl.removeSuffix("/") + "/scan/direct") .apiKey(config.apiKey.value) - .build() override val version: String by lazy { // TODO: Find out the best / cheapest way to query the SCANOSS server for its version. @@ -64,73 +67,123 @@ class ScanOss( override val configuration = "" - override val matcher by lazy { - ScannerMatcher.create( - details, - ScannerMatcherConfig( - config.regScannerName, - config.minVersion, - config.maxVersion, - configuration - ) - ) - } + override val matcher: ScannerMatcher? = null override val readFromStorage = config.readFromStorage override val writeToStorage = config.writeToStorage - /** - * The name of the file corresponding to the fingerprints can be sent to SCANOSS for more precise matches. - * However, for anonymity, a unique identifier should be generated and used instead. This property holds the - * mapping between the file paths and the unique identifiers. When receiving the response, the UUID will be - * replaced by the actual file path. - * - * TODO: This behavior should be driven by a configuration parameter enabled by default. - */ - private val fileNamesAnonymizationMapping = mutableMapOf() - override fun scanPath(path: File, context: ScanContext): ScanSummary { val startTime = Instant.now() - val wfpString = buildString { - path.walk() - .onEnter { it.name !in VCS_DIRECTORIES } - .filterNot { it.isDirectory } - .forEach { - logger.info { "Computing fingerprint for file ${it.absolutePath}..." } - append(createWfpForFile(it)) + val filterConfig = FilterConfig.builder() + .customFilter { currentPath -> + // The "currentPath" variable contains a path object representing the file or directory being evaluated + // by the filter. + // This is provided by the Scanner and represents individual files/directories during traversal. + try { + val relativePath = currentPath.toFile().toRelativeString(path) + val isExcluded = context.excludes?.isPathExcluded(relativePath) ?: false + logger.debug { "Path: $currentPath, relative: $relativePath, isExcluded: $isExcluded" } + isExcluded + } catch (e: IllegalArgumentException) { + logger.warn { "Error processing path $currentPath: ${e.message}" } + false } + } + .build() + + // Build the scanner at function level in case any path-specific settings or filters are needed later. + val scanoss = scanossBuilder + .settings(buildSettingsFromORTContext(context)) + .filterConfig(filterConfig) + .build() + + val rawResults = when { + path.isFile -> listOf(scanoss.scanFile(path.toString())) + else -> scanoss.scanFolder(path.toString()) } - val result = service.scan( - wfpString, - context.labels["scanOssContext"], - context.labels["scanOssId"]?.toIntOrNull() ?: Thread.currentThread().threadId().toInt() - ) - - // Replace the anonymized UUIDs by their file paths. - val results = JsonUtils.toScanFileResultsFromObject(JsonUtils.toJsonObject(result)).map { - val uuid = UUID.fromString(it.filePath) + val results = JsonUtils.toScanFileResults(rawResults) + val endTime = Instant.now() + return generateSummary(startTime, endTime, results) + } - val fileName = fileNamesAnonymizationMapping[uuid] ?: throw IllegalArgumentException( - "The ${descriptor.id} server returned UUID '$uuid' which is not present in the mapping." - ) + data class ProcessedRules( + val includeRules: List, + val ignoreRules: List, + val replaceRules: List, + val removeRules: List + ) + + private fun buildSettingsFromORTContext(context: ScanContext): ScanossSettings { + val rules = processSnippetChoices(context.snippetChoices) + val bom = Bom.builder() + .ignore(rules.ignoreRules) + .include(rules.includeRules) + .replace(rules.replaceRules) + .remove(rules.removeRules) + .build() + return ScanossSettings.builder().bom(bom).build() + } - ScanFileResult(fileName, it.fileDetails) + fun processSnippetChoices(snippetChoices: List): ProcessedRules { + val includeRules = mutableListOf() + val ignoreRules = mutableListOf() + val replaceRules = mutableListOf() + val removeRules = mutableListOf() + + snippetChoices.forEach { snippetChoice -> + snippetChoice.choices.forEach { choice -> + when (choice.choice.reason) { + SnippetChoiceReason.ORIGINAL_FINDING -> { + processOriginalFinding( + choice = choice, + includeRules = includeRules + ) + } + + SnippetChoiceReason.NO_RELEVANT_FINDING -> { + processNoRelevantFinding( + choice = choice, + removeRules = removeRules + ) + } + + SnippetChoiceReason.OTHER -> { + logger.info { + "Encountered OTHER reason for snippet choice in file ${choice.given.sourceLocation.path}" + } + } + } + } } - val endTime = Instant.now() - return generateSummary(startTime, endTime, results) + return ProcessedRules(includeRules, ignoreRules, replaceRules, removeRules) + } + + private fun processOriginalFinding(choice: SnippetChoice, includeRules: MutableList) { + includeRules.add( + Rule.builder() + .purl(choice.choice.purl) + .path(choice.given.sourceLocation.path) + .build() + ) } - internal fun generateRandomUUID() = UUID.randomUUID() + private fun processNoRelevantFinding(choice: SnippetChoice, removeRules: MutableList) { + val builder = RemoveRule.builder() - internal fun createWfpForFile(file: File): String { - generateRandomUUID().let { uuid -> - // TODO: Let's keep the original file extension to give SCANOSS some hint about the mime type. - fileNamesAnonymizationMapping[uuid] = file.path - return Winnowing.builder().build().wfpForFile(file.path, uuid.toString()) + builder.path(choice.given.sourceLocation.path) + + // Set line range only if both `sourceLocation.startLine` and `sourceLocation.endLine` are positive numbers. + // If either line is zero or negative, the rule will apply to the entire file. + if (choice.given.sourceLocation.startLine > 0 && choice.given.sourceLocation.endLine > 0) { + builder.startLine(choice.given.sourceLocation.startLine) + builder.endLine(choice.given.sourceLocation.endLine) } + + val rule = builder.build() + removeRules.add(rule) } } diff --git a/plugins/scanners/scanoss/src/main/kotlin/ScanOssResultParser.kt b/plugins/scanners/scanoss/src/main/kotlin/ScanOssResultParser.kt index 9522a0b3e0016..d81250cc0df1b 100644 --- a/plugins/scanners/scanoss/src/main/kotlin/ScanOssResultParser.kt +++ b/plugins/scanners/scanoss/src/main/kotlin/ScanOssResultParser.kt @@ -18,13 +18,17 @@ */ package org.ossreviewtoolkit.plugins.scanners.scanoss - +import com.scanoss.dto.LicenseDetails import com.scanoss.dto.ScanFileDetails import com.scanoss.dto.ScanFileResult import com.scanoss.dto.enums.MatchType +import com.scanoss.dto.enums.StatusType +import java.lang.invoke.MethodHandles import java.time.Instant +import org.apache.logging.log4j.kotlin.loggerOf + import org.ossreviewtoolkit.downloader.VcsHost import org.ossreviewtoolkit.model.CopyrightFinding import org.ossreviewtoolkit.model.LicenseFinding @@ -36,7 +40,8 @@ import org.ossreviewtoolkit.model.TextLocation import org.ossreviewtoolkit.utils.spdx.SpdxConstants import org.ossreviewtoolkit.utils.spdx.SpdxExpression import org.ossreviewtoolkit.utils.spdx.SpdxLicenseIdExpression -import org.ossreviewtoolkit.utils.spdx.toExpression + +private val logger = loggerOf(MethodHandles.lookup().lookupClass()) /** * Generate a summary from the given SCANOSS [result], using [startTime], [endTime] as metadata. This variant can be @@ -56,16 +61,29 @@ internal fun generateSummary(startTime: Instant, endTime: Instant, results: List } MatchType.snippet -> { - val file = requireNotNull(details.file) - val lines = requireNotNull(details.lines) - val sourceLocations = convertLines(file, lines) - val snippets = getSnippets(details) - - snippets.forEach { snippet -> - sourceLocations.forEach { sourceLocation -> - // TODO: Aggregate the snippet by source file location. - snippetFindings += SnippetFinding(sourceLocation, setOf(snippet)) + val file = requireNotNull(result.filePath) + if (details.status == StatusType.pending) { + val lines = requireNotNull(details.lines) + val sourceLocations = convertLines(file, lines) + val snippets = getSnippets(details) + + // The number of snippets should match the number of source locations. + if (sourceLocations.size != snippets.size) { + logger.warn { + "Unexpected mismatch in '$file': " + + "${sourceLocations.size} source locations vs ${snippets.size} snippets. " + + "This indicates a potential issue with line range conversion." + } } + + // Associate each source location with its corresponding snippet. + sourceLocations.zip(snippets).forEach { (location, snippet) -> + snippetFindings += SnippetFinding(location, setOf(snippet)) + } + } else { + logger.warn { "File '$file' is identified, not including on snippet findings" } + licenseFindings += getLicenseFindings(details) + copyrightFindings += getCopyrightFindings(details) } } @@ -134,19 +152,18 @@ private fun getCopyrightFindings(details: ScanFileDetails): List { +private fun getSnippets(details: ScanFileDetails): List { val matched = requireNotNull(details.matched) val fileUrl = requireNotNull(details.fileUrl) val ossLines = requireNotNull(details.ossLines) val url = requireNotNull(details.url) val purls = requireNotNull(details.purls) - val licenses = details.licenseDetails.orEmpty().mapTo(mutableSetOf()) { license -> - SpdxExpression.parse(license.name) - } + val license = getUniqueLicenseExpression(details.licenseDetails.toList()) val score = matched.substringBeforeLast("%").toFloat() val locations = convertLines(fileUrl, ossLines) @@ -154,14 +171,15 @@ private fun getSnippets(details: ScanFileDetails): Set { val vcsInfo = VcsHost.parseUrl(url.takeUnless { it == "none" }.orEmpty()) val provenance = RepositoryProvenance(vcsInfo, ".") - return buildSet { - purls.forEach { purl -> - locations.forEach { snippetLocation -> - val license = licenses.toExpression()?.sorted() ?: SpdxLicenseIdExpression(SpdxConstants.NOASSERTION) + // Store all PURLs in additionalData to preserve the complete information. + val additionalData = mapOf( + "release_date" to details.releaseDate, + "all_purls" to purls.joinToString(" ") + ) - add(Snippet(score, snippetLocation, provenance, purl, license)) - } - } + // Create one snippet per location, using the first PURL as the primary identifier. + return locations.map { snippetLocation -> + Snippet(score, snippetLocation, provenance, purls.firstOrNull().orEmpty(), license, additionalData) } } @@ -178,3 +196,14 @@ private fun convertLines(file: String, lineRanges: String): List = else -> throw IllegalArgumentException("Unsupported line range '$lineRange'.") } } + +fun getUniqueLicenseExpression(licensesDetails: List): SpdxExpression { + if (licensesDetails.isEmpty()) { + return SpdxLicenseIdExpression(SpdxConstants.NOASSERTION) + } + + return licensesDetails + .map { license -> SpdxExpression.parse(license.name) } + .reduce { acc, expr -> acc and expr } + .simplify() +} diff --git a/plugins/scanners/scanoss/src/test/assets/exclusionTest/ArchiveUtils.kt b/plugins/scanners/scanoss/src/test/assets/exclusionTest/ArchiveUtils.kt new file mode 100644 index 0000000000000..fb5fef1c4962d --- /dev/null +++ b/plugins/scanners/scanoss/src/test/assets/exclusionTest/ArchiveUtils.kt @@ -0,0 +1,31 @@ +/* + * This file contains random data generated using the following command: + * head -c 1k < /dev/urandom | base64 + * + * The command takes 1 kilobyte of random bytes from the /dev/urandom device + * and then encodes it as base64 text. + * + * Generated on: Fri Mar 14 05:04:43 PM CET 2025 + * Purpose: To create test data with completely random content that cannot + * match any existing code in repositories, thereby avoiding false + * positives when scanning ORT source code. + */ + +7m8Y06QhHzmQ4ePs0UUUasqsc8SP1ayNTFdQb6wffQwMu605hXOGHbOoy5pUv7ksgf6sw5ET2qXp +T23LF2yA1cdNeDt8DBDd3IDmLX/wGgXcQjcaCtfSsMWB7oqHBMGkzwC5fMcDKPLK6ec2MwX6WPkw +E18ImifWtAmGPEFGxWuqIinhE1yGSN+ImqJPVmpYfMOaDIAaS3JpiHZDmJW5uyQ5DB6W7lpm0q+f +ZbtGPBeimy1jWF0H6kEW/TIve8RzUjdHU/t//O9r0b2AP08shSrSDWGlbQzxTniLOp2VZxNUEcVM +c9/Lx4OXEaM/3NDCdr4qQS/1kZpGKFrv06zzC8tlncGaxBfdZSCsh1i+LbZtvUmTSv/wz7g+mld5 +WB2lSzF1Ervzqnm2+3iY+9TvVxDWzZ27LWsd1kvFrJCM03jI1q0c7uJrnnovAOoZkH2QiMPNBQmB +wShT36h3su/aiOXEquXi+DoTSYDNgXeHVGI2joLVWYLfeTcTTfdvZiwp0K+XQp6fKtWX8tpUibNq +ngp2dOlzl5yiT+WAD2ETGuyEML/wM3oz+wB93me2YYLJqz/1gtlnnRvGnAukbFLpxxXGK7Vnz+FF +KfcPWF+O/FNV3nJD+m2nlMVj1n4lRM/mUEdVDhDDtxhywvi6DdNQMcUoeXZRT3dLk27+efNLvMDk +7TsW/asvMoPrioAkDiTHqWvy+OUImWqqzNpzxIMuTWZrApSklw2UeyXknvHBORUN95AM6Oe9iKb0 +7B2g8U9dIFo7v/AhaDqoQMw+Dz1KfH6+fPaqZEy2H1U7/9RSorKz0fycz7n7BtqWxjenqw11LLxy +lO36udPuvtr2b/WB/4ch0LuoI2eA11iTeIG4DuTxvizU3lExBXP+e8EAjkWx6F2ymDrI21PYPp++ +uidSk3g/RmaRZGk8akcXbs3pDO/twfjaH3YWYZzBf8aP1TRYDp4NF5v2OhWDa5d2dqdQGDRGg/wy +Gf0W8txn/fQ3QN7SS9qPftgD6OYIpxKjIWBq/zb5+SAzhBZVjFYw+KVi+zu/P7he7xLRko6APCum +Ugk7wqohWVdbl2IG2RIuPUOH2zQdzVJvLisKhfq3q6ydGmjD/WRNOxbebpmSKcmZWVg0Ko7/e0ys +ymV2Ud0tIZwfIH/7476SZAh0ym1U7mgyzm/jlxKm5gUIF1+NWQiqa80GmsAJfquf1Yj4i0ftF+eO +6OPJqbkZERpu24u2HIfL6CvlUkx08mS+eqLzyRiRuidDcQGFOK+0xPUk01jOZnGiY1ptG4W+Fo5K +OhcT2H14wQqiHsthzMhpSLXwMG2ddM7P69rHEAXB3iXyWopgdWopVekxHEuar0mv3D6uO/4HKQ== diff --git a/plugins/scanners/scanoss/src/test/assets/exclusionTest/ScannerFactory.kt b/plugins/scanners/scanoss/src/test/assets/exclusionTest/ScannerFactory.kt new file mode 100644 index 0000000000000..271420857d262 --- /dev/null +++ b/plugins/scanners/scanoss/src/test/assets/exclusionTest/ScannerFactory.kt @@ -0,0 +1,31 @@ +/* + * This file contains random data generated using the following command: + * head -c 1k < /dev/urandom | base64 + * + * The command takes 1 kilobyte of random bytes from the /dev/urandom device + * and then encodes it as base64 text. + * + * Generated on: Fri Mar 14 05:05:29 PM CET 2025 + * Purpose: To create test data with completely random content that cannot + * match any existing code in repositories, thereby avoiding false + * positives when scanning ORT source code. + */ + +IyaayUoMK28Ib11Z55hC2OShY3p3HzWQyGy199hx20oqZrypl9AuDhKtBdl+qozcZBNajzvkU3H/ +jh3vV/P9I2VLQNVqMCpjelQoXyVq/nmbwxdQBXGbLgcC4J05ujQ2hoXuF4jdtEttDxca8P/EpUub +nmSO3zmz86LqyyYgFj3imketFw0GvnCYU/8VDjmLxnigspEVI7ZDOacKOshObwH+Br/XgFHr5tyc +ulGqACTjGY3EEdAjC2+tcTqoI+4mXVxx4CcBD4lRn90khfFOcAM8Iu2pGaERHnAtUrf9EX3rLsOW +wV+wYllChP71rI/4ueEch9X8ph1dA0nQQN1tLUi58pQlkCHY6K0QNFiD6K+RxaBl3yBt1IZqjfZi +UVjTb7xJDrYnLPIASlPd0AduDik8pKn+GTqIFWgkkRr5mY6c9jTqHxY7rASDNi7LGKUE9gPFd1LD +xPJmsl+8L+lcVJjJNU7Tkps/ZZJuo/EqlwbUd/Wq45S++YBBfYlFaOXn/bVMhxXi1SH3xMHSAjH1 +aYj0YHEdBHnEF1ouahyS4607cundZcSR29kITrUnFSi/ZP3zKREa3MGm/qrJS7qFSxlHVsYHBIjy +VRx+teV4nQWKJyA6x/T9Sx63lM7duwhVRdh9JxhxnrKAyUBH5HwhpFXHreMjudNdY9nMaWaKP9Ge +oD4Rr4iA3kvaHjtqSfhB55PgQO7Od/KLNTRfMMPl7IjbouQNCai++hV+p7BRAjtGUwTOXp9FHbv7 +YFGBFl3a0e1+YEoQA+0Psf1x2lENCJwH87DuZxuKI3kbcY6XA5kebt43m/eztRa17z/vmwyiQ/up ++RpMU9Xp1bv39h84QbyvZYN40xzHc8togJmPtSKCyEPcmHdt9t0LF6TCsb4k+kIBRUXMfnYpEDqv +E6dldgWHjVy/4llWqyj3SsToERP1VhaloWyq8QRNke6lKzxMXOhmupKX195V2cA+6EGY3sK/ykhl +fYOofbKcHwevHKgOJyj7Tj6+9qUgda/EI01lcJicTO8Nqb0LW+FfwIiws7WlsZWuxQGUZ0SOMBU4 +MnR9NPbS6rUSx2rMfSPn18Jd82D5eoM32ogRQb7C2pgXQbQoAegl98vtOjkze4wsa6CmW0rmbQrJ +bpgPoWiZ1t/BlAUvxjRuzQSNNhnyvaC5nib6NYZAcr9BCm3yJ0sR/uSOUG8cCoJptMYhH9XxHqKl +ACwfHgq7/mHBTxhQCmw5hkDWvY7FqzDPME3igab1Mda4lxOyUjJ3PeVzZbWZY2s/oUaSbntsSqRM +z+zutj83Nm76iOSS0MXxCfi5VKYThzGdfXkYB2tZP8yPhh+sw0CpqeV5KB810C76abZbVZ+EDw== diff --git a/plugins/scanners/scanoss/src/test/assets/exclusionTest/server.go b/plugins/scanners/scanoss/src/test/assets/exclusionTest/server.go new file mode 100644 index 0000000000000..d5d121c5e19c9 --- /dev/null +++ b/plugins/scanners/scanoss/src/test/assets/exclusionTest/server.go @@ -0,0 +1,31 @@ +/* + * This file contains random data generated using the following command: + * head -c 1k < /dev/urandom | base64 + * + * The command takes 1 kilobyte of random bytes from the /dev/urandom device + * and then encodes it as base64 text. + * + * Generated on: Fri Mar 14 05:05:40 PM CET 2025 + * Purpose: To create test data with completely random content that cannot + * match any existing code in repositories, thereby avoiding false + * positives when scanning ORT source code. + */ + +O8uBEiQRmSvrKRn33rErjHcbg7fTCq2ZFFNDs9WiTlEArfr7g2anf0cRFmQkq3/LqViXDBLaVw8y +NQU3wNIbCuqFNzF2Yo0/i2v54MZWIvsnqaXj1hZBHUlXXJvfT+HLQzJAJRS8QJ066Ruui0njNsE6 +Xk2UM05WlS75ZwhO41G6tZtp1qfCvUYDzc5gCkWhce0IIftGjz2a8c6cFv6t0Cudknwqj5p59UPJ +bV445vLh9aGE5SZqwEZKReSyjtzXctz0t33JrbEBsYEBaVLBH2FGbWzMj5iOBTZ6/YEdY4sVKmbk +xPlsS2pW9kURX3BhPmfdfLRXgO2tFC2s+5tmwhtOQzisplaHv+feZVytxv2jRMy+qLWYrDDTA7z7 +PoxrfXckupavs4NMXlbar/wlODwbu3VbFIatY5vJ0bWj63QbzP23J1iE+26+4IH8uhW8HST02LRP +z72f9f8a0JdEyDPxM3mz+WAkVjc8gw3sdmOw/6WCOmFrU08U2jdAISE47ft+SI7CM4YkQxcCRdND +WpxGl/ARXd9JJLdemmLrXjRa9RltSAvcJAkGCYb/YHOKW9MVubYPLC+cFDX8pLrNr3XGRdoYuH2q +Uoz3A/LBYK2nI6UkDCfcigFvi3gaeTmf5muMEa4PBmruLM5aIpd8NyqoS+sbsSsJIRaJQHR+COf6 +ZQuOvBn5Z6siSelZUn/PDruCIlutHNnV7eF2SjVj7mU6WsawpE73oLIpeQ37NHGZoXbxg0qP3Cf7 +Q0ukwki+tDKb82JNJ02eu2Ht3//iGevGJWr0Qz02Jt9iknIjIBHBLkypOOEqMZQWKPXKDwtcB950 +BNr5SFh0Ml4nPDm+vmfkOylHYOWCvH9gcQDOoYWZWPXhUQUBHhpXvPVB3bLqUunByKzRufB0JuGV +K9whoy78dNLBrDC/Zk+CU3B0cB8FSdBIcWZqtKSjISHrXB3eflWjsErRZxvZWutRINd79oGYkbBB +6rWRu6a7XJzbNogYiuF5eAh3cIeGWVISwmhF77AH+c6ojatciC+6JpwJTBI8d6hcahDGxs9k9sWQ +opAWyMWvBMJHKZ15V9UbNBl/WSNGjL37UIlQLAfTy64WCFzlnIbKcBFaV8E969DTjbxjKfCPto3u +/u/ABCrkbJAVKxBKgZ7XTX0Ww3TJh8q7Qp8G+ZTmlAVDvvXa+Jx94dyEiRoihecDcA+5OjCiod+X +qaoGgTuQ3TKAO2sKWNy5cUuYzPKOfCtUim4cj1M7FbR3p8QHs8fqyYp4Wm+n1ub1TerFiS2v34Cx +bdQCmdoUSDOmatm50Gw/CiCFotHtJeGG0EyEZ50hP1C2v0zYoTSuwi/lY6RfhMZjkBN3WUCHDg== diff --git a/plugins/scanners/scanoss/src/test/assets/filesToScan/ArchiveUtils.kt b/plugins/scanners/scanoss/src/test/assets/filesToScan/ArchiveUtils.kt index 58da835ead789..987dee0828263 100644 --- a/plugins/scanners/scanoss/src/test/assets/filesToScan/ArchiveUtils.kt +++ b/plugins/scanners/scanoss/src/test/assets/filesToScan/ArchiveUtils.kt @@ -1,245 +1,31 @@ /* - * Copyright (C) 2017 The ORT Project Authors (see ) + * This file contains random data generated using the following command: + * head -c 1k < /dev/urandom | base64 * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * The command takes 1 kilobyte of random bytes from the /dev/urandom device + * and then encodes it as base64 text. * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - * License-Filename: LICENSE - */ - -@file:Suppress("MatchingDeclarationName") - -package org.ossreviewtoolkit.utils - -import java.io.File -import java.io.IOException -import java.io.InputStream -import java.nio.file.FileVisitResult -import java.nio.file.Files -import java.nio.file.Path -import java.nio.file.SimpleFileVisitor -import java.nio.file.attribute.BasicFileAttributes -import java.util.zip.Deflater - -import org.apache.commons.compress.archivers.ArchiveEntry -import org.apache.commons.compress.archivers.ArchiveInputStream -import org.apache.commons.compress.archivers.sevenz.SevenZFile -import org.apache.commons.compress.archivers.tar.TarArchiveEntry -import org.apache.commons.compress.archivers.tar.TarArchiveInputStream -import org.apache.commons.compress.archivers.zip.ZipArchiveEntry -import org.apache.commons.compress.archivers.zip.ZipArchiveInputStream -import org.apache.commons.compress.archivers.zip.ZipArchiveOutputStream -import org.apache.commons.compress.archivers.zip.ZipFile -import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream -import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream -import org.apache.commons.compress.compressors.xz.XZCompressorInputStream -import org.apache.commons.compress.utils.SeekableInMemoryByteChannel - -enum class ArchiveType(vararg val extensions: String) { - TAR(".gem", ".tar"), - TAR_BZIP2(".tar.bz2", ".tbz2"), - TAR_GZIP(".crate", ".tar.gz", ".tgz"), - TAR_XZ(".tar.xz", ".txz"), - ZIP(".aar", ".egg", ".jar", ".war", ".whl", ".zip"), - SEVENZIP(".7z"), - NONE(""); - - companion object { - fun getType(filename: String): ArchiveType { - val lowerName = filename.toLowerCase() - return (ArchiveType.entries - NONE).find { type -> - type.extensions.any { lowerName.endsWith(it) } - } ?: NONE - } - } -} - -/** - * Unpack the [File] to [targetDirectory]. - */ -fun File.unpack(targetDirectory: File) = - when (ArchiveType.getType(name)) { - ArchiveType.SEVENZIP -> unpack7Zip(targetDirectory) - ArchiveType.ZIP -> unpackZip(targetDirectory) - - ArchiveType.TAR -> inputStream().unpackTar(targetDirectory) - ArchiveType.TAR_BZIP2 -> BZip2CompressorInputStream(inputStream()).unpackTar(targetDirectory) - ArchiveType.TAR_GZIP -> GzipCompressorInputStream(inputStream()).unpackTar(targetDirectory) - ArchiveType.TAR_XZ -> XZCompressorInputStream(inputStream()).unpackTar(targetDirectory) - - ArchiveType.NONE -> { - throw IOException("Unable to guess compression scheme from file name '$name'.") - } - } - -/** - * Unpack the [File] assuming it is a 7-Zip archive. This implementation ignores empty directories and symbolic links. - */ -fun File.unpack7Zip(targetDirectory: File) { - SevenZFile(this).use { zipFile -> - while (true) { - val entry = zipFile.nextEntry ?: break - - if (entry.isDirectory || entry.isAntiItem) { - continue - } - - val target = targetDirectory.resolve(entry.name) - - // There is no guarantee that directory entries appear before file entries, so ensure that the parent - // directory for a file exists. - target.parentFile.safeMkdirs() - - target.outputStream().use { output -> - zipFile.getInputStream(entry).copyTo(output) - } - } - } -} - -/** - * Unpack the [File] assuming it is a Zip archive. - */ -fun File.unpackZip(targetDirectory: File) = ZipFile(this).unpack(targetDirectory) - -/** - * Unpack the [ByteArray] assuming it is a Zip archive. - */ -fun ByteArray.unpackZip(targetDirectory: File) = ZipFile(SeekableInMemoryByteChannel(this)).unpack(targetDirectory) - -/** - * Pack the file into a ZIP [targetFile] using [Deflater.BEST_COMPRESSION]. If the file is a directory its content is - * recursively added to the archive. Only regular files are added, e.g. symbolic links or directories are skipped. If - * a [prefix] is specified, it is added to the file names in the ZIP file. - * If not all files shall be added to the archive a [filter] can be provided. - */ -fun File.packZip( - targetFile: File, - prefix: String = "", - overwrite: Boolean = false, - filter: (Path) -> Boolean = { true } -) { - require(overwrite || !targetFile.exists()) { - "The target ZIP file '${targetFile.absolutePath}' must not exist." - } - - ZipArchiveOutputStream(targetFile).use { output -> - output.setLevel(Deflater.BEST_COMPRESSION) - Files.walkFileTree(toPath(), object : SimpleFileVisitor() { - override fun visitFile(file: Path, attrs: BasicFileAttributes): FileVisitResult { - if (attrs.isRegularFile && filter(file)) { - val entry = ZipArchiveEntry(file.toFile(), "$prefix${this@packZip.toPath().relativize(file)}") - output.putArchiveEntry(entry) - file.toFile().inputStream().use { input -> input.copyTo(output) } - output.closeArchiveEntry() - } - - return FileVisitResult.CONTINUE - } - }) - } -} - -/** - * Unpack the [InputStream] to [targetDirectory] assuming that it is a tape archive (TAR). This implementation ignores - * empty directories and symbolic links. - */ -fun InputStream.unpackTar(targetDirectory: File) = - TarArchiveInputStream(this).unpack( - targetDirectory, - { entry -> !(entry as TarArchiveEntry).isFile }, - { entry -> (entry as TarArchiveEntry).mode } - ) - -/** - * Unpack the [InputStream] to [targetDirectory] assuming that it is a ZIP archive. This implementation ignores empty - * directories and symbolic links. - */ -fun InputStream.unpackZip(targetDirectory: File) = - ZipArchiveInputStream(this).unpack( - targetDirectory, - { entry -> (entry as ZipArchiveEntry).let { it.isDirectory || it.isUnixSymlink } }, - { entry -> (entry as ZipArchiveEntry).unixMode } - ) - -/** - * Copy the executable bit contained in [mode] to the [target] file's mode bits. - */ -private fun copyExecutableModeBit(target: File, mode: Int) { - if (Os.isWindows) return - - // Note: In contrast to Java, Kotlin does not support octal literals, see - // https://kotlinlang.org/docs/reference/basic-types.html#literal-constants. - // The bit-triplets from left to right stand for user, groups, other, respectively. - if (mode and 0b001_000_001 != 0) { - target.setExecutable(true, (mode and 0b000_000_001) == 0) - } -} - -/** - * Unpack this [ArchiveInputStream] to the [targetDirectory], skipping all entries for which [shouldSkip] returns true, - * and using what [mode] returns as the file mode bits. - */ -private fun ArchiveInputStream.unpack( - targetDirectory: File, - shouldSkip: (ArchiveEntry) -> Boolean, - mode: (ArchiveEntry) -> Int -) = - use { input -> - while (true) { - val entry = input.nextEntry ?: break - - if (shouldSkip(entry)) continue - - val target = targetDirectory.resolve(entry.name) - - // There is no guarantee that directory entries appear before file entries, so ensure that the parent - // directory for a file exists. - target.parentFile.safeMkdirs() - - target.outputStream().use { output -> - input.copyTo(output) - } - - copyExecutableModeBit(target, mode(entry)) - } - } - -/** - * Unpack the [ZipFile]. In contrast to [InputStream.unpackZip] this properly parses the ZIP's central directory, see - * https://commons.apache.org/proper/commons-compress/zip.html#ZipArchiveInputStream_vs_ZipFile. - */ -private fun ZipFile.unpack(targetDirectory: File) = - use { zipFile -> - val entries = zipFile.entries - - while (entries.hasMoreElements()) { - val entry = entries.nextElement() - - if (entry.isDirectory || entry.isUnixSymlink) { - continue - } - - val target = targetDirectory.resolve(entry.name) - - // There is no guarantee that directory entries appear before file entries, so ensure that the parent - // directory for a file exists. - target.parentFile.safeMkdirs() - - target.outputStream().use { output -> - zipFile.getInputStream(entry).copyTo(output) - } - - copyExecutableModeBit(target, entry.unixMode) - } - } + * Generated on: Fri Mar 14 05:06:11 PM CET 2025 + * Purpose: To create test data with completely random content that cannot + * match any existing code in repositories, thereby avoiding false + * positives when scanning ORT source code. +*/ + +DPygtEpjZMRFKqxbJXCG83PLBBIlyTp4VXOKLCVrCOtkvRQ40stMrMTjCQu8Og1dMsB6fUyn3ZFO +B6PxaWqBYPFxOw+mm3blj5ph/iFh8Ihfw0Hwunv2O/z0OiErz2XejOOWhyQqplTMwPtO5awYM7Vl +1jqOXJv/0dA+9JuCpi4sWw8Fb+FCkNai43viz0A7acL2aQYXkeBvxTgR8bsmu1G+VI8ANfeAb/Rc +IY1j1kFsY6PtxNzM3tZg/RiiLndP6cr1SNhodMWWY/11Fx5qYNxFPWTiPIlRX1F4KR3yYeaEgkpN +zHaKR3FJjO0am/YBliMCu23zIY8G3cbgTEb/88tqsEt3Midjv5blIFXturT7zc4loaBgDYJvbGqG +1NJSFsDQWiglMKJd78oR8XdSP8oWJs0BU4GQUSLA41bdkI1dP04JEyM8grsXog6SEdhSfJcr5ZXj +3WdwFQzZHvAwQ9PinXJm0UFPuEVv8QGGbM1q/YnMe9okd23owDWIWZksUmT896SOprNhxj3hQUbE +Nmm+ts80cHu9puyhwbrG5V/QoiC7l/oWwDQ1P8MXHyMphYv+jlTLVaD54ppWPK4vOfhCKpEIrMBh +RqCBYXIcx6LXg95aPGOP8CSCIBuv/cqI65WYUUUhVFer/DyIEsPlIT+AZyuhy7Z0R4hkP7jn6irl +PHkyUBCzyIAmcwNWKpQCUfwfvkjhM02KVPwUstelPrv0ArK5MWEN5d+rOZenjRenkkT1fBO+Q6qu +Wec0XWIBbz+82Ebb+hgSBtVIOnCr7hzYPoknhlvnClu/7NinjDmW7SeCbou8KUbmg3pHgDBG6XHG +azOZIDeaowL4Pfkzy3L+OpL+mTb3Iwi8TVNvCJDkMIUtCaW2fhdzaWdcZM/cJghOJk6JxxB8d7T1 +VWp4HhAF6n6jAhu1uqJ/1KS92DnSYH8UuSPBguOjnsOWwMwrwGrFIzH/O4ICnJ7kydywszHYvZtk +HeB1BVs/VMLn9R323VH5vh3uvjMptggTG5fgnsNlD+ZZFPe59xqZ2g8w8Evnc+SQGxr76krxPi8n +wLd30VcI5yrgaEkyU/90mU0zzGVR5H5WtyCCn3S+MeGSyNKHAx+V7cpvkorwqXOcty7qQRrqhIMv +PhGPGOxNvhaYuz9uWtwfr1trjjspkBRtpazDfO8vqZAACDqm8xoYRYwdaRUf4rJkK80EDqL2daAb +NQyIuCtqy9FjTPcQHV9QHNhysPiROuPpGA2Ew40DkTEgeazLOPKRjWLqieBuKe7FP7v/LBcIV1id +rn89bhabHPemm4BlKaWL3o12PrHYQPb/51E2DZoKw1Rk4IQN0tWAUMVt5hxaS5jlEJ+Vh388WQ== diff --git a/plugins/scanners/scanoss/src/test/assets/filesToScan/ScannerFactory.kt b/plugins/scanners/scanoss/src/test/assets/filesToScan/ScannerFactory.kt index a92c2e8453b08..3f988c795670e 100644 --- a/plugins/scanners/scanoss/src/test/assets/filesToScan/ScannerFactory.kt +++ b/plugins/scanners/scanoss/src/test/assets/filesToScan/ScannerFactory.kt @@ -1,54 +1,31 @@ /* - * Copyright (C) 2017 The ORT Project Authors (see ) + * This file contains random data generated using the following command: + * head -c 1k < /dev/urandom | base64 * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * The command takes 1 kilobyte of random bytes from the /dev/urandom device + * and then encodes it as base64 text. * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - * License-Filename: LICENSE - */ - -package org.ossreviewtoolkit.scanner - -import java.util.ServiceLoader - -import org.ossreviewtoolkit.model.config.ScannerConfiguration - -/** - * A common interface for use with [ServiceLoader] that all [AbstractScannerFactory] classes need to implement. - */ -interface ScannerFactory { - /** - * The name to use to refer to the scanner. - */ - val scannerName: String - - /** - * Create a [Scanner] using the specified [config]. - */ - fun create(config: ScannerConfiguration): Scanner -} - -/** - * A generic factory class for a [Scanner]. + * Generated on: Fri Mar 14 05:07:04 PM CET 2025 + * Purpose: To create test data with completely random content that cannot + * match any existing code in repositories, thereby avoiding false + * positives when scanning ORT source code. */ -abstract class AbstractScannerFactory( - override val scannerName: String -) : ScannerFactory { - abstract override fun create(config: ScannerConfiguration): T - /** - * Return the scanner's name here to allow Clikt to display something meaningful when listing the scanners - * which are enabled by default via their factories. - */ - override fun toString() = scannerName -} +mneGKU/gTsBek3i9p6pHaccif0G7AhDAF/w3j6QSXhUJACJY9h+yN+gBf0Duf1DONiZrOlt96w77 +yQH9J0inoH0rJr0EKXWVkjyHTBwhmMWDcnwrAEmQRn/kCtFNfMRFAyJsxBE2suK754qfhUbS4hjt +qB9JHimTQCqQ2bgLJvgMgZ9Bfs21lIhkpTlUsdEyEBjy7VB5DVS8Usal+lIJENJ/RaJmxULcPj1C +LEUSaEf+GRr8wDoysFcIGav2uJpVaZfAvbfWxwnMScjpaKeY+lzScmnnYNXhRBYYh+mI1qJY/ycW +b5jREjm3iF1RQ5x9/PeLpx5fRPL11G9aaHWCrrHT9xrV8Lb5WIf5aGaZG+p7c3CDdHGhT0E7ltY9 +tFKs9vstrgqhL/l0hFxvoZtjBE2tdt6+lhGt2puZ6XoWAZf4Baas2cTUr5i6WvdDKbPnytcCFMuN +UgBJS60JpNaWPV45QQS970G2BhCoeI5fYi2//0KFz2axIuVfRhEaojFS+CNVwpd9hXrNQSXFaS38 +oYZF/p09SiWJHHoaaKSTEGkSZZRHDhEnfRhOnTmSdmqftMywk5178mcJWhOfemK/IcILhNeB+5am +E+AKJy+rsYffep5KXPobFn3BntAdJilwe6R0CHf5d3wIwUIDxMiqSgnDJZthiVFYBBhYTrLZza+t +eXTfmFvdjcsUqNSUuglSjFBVUFRM4rKk7Y8zOQJ6PWge7LOqVxsrLY9KsRt1iI6JZ9Pq3G7yL9aD +qsivx4mnGmUvVPSbSGK4soWKYOKpMkYkyALyHJhmQMY6BxYmbHbqIaxWS67IEj00XjcIi2tsV8qN +juG7Rgyc33peN/Xu6EDEN+Uv4+EMB+VbwEsSgVhIdIAjiMMQT/r9u7hjXcDyZZNpHK3wBd3bUfn0 +MMPbWI4hIIj6/c5YM5SiNi8zPGWAbpc76FAg7cskvYwMOFfoaUxtkMD8IW5Y7X/m3XSE/O9cy7Wa +2+cq+YkIs+YmyixoiQV/eJimhmH6P2VYgDYUoRzfcKsE1QpjiWdLq2PEjD5iTsOgy/HdWKLAvdnW +j3jgzvZNBmfX/MYyFJn/pBALbAVFvQtQTLBmAJlyVjPfLCEhA6RhMmATvOhm3OLsacdFjrrz/Mhb +hkOCDPSyqoinHTcHKvpKORhCnlYPBGshboJ28h1oQIcRhGYGCxdZfIVClbUG3os0AzjHRFtI8f0Z +WQmL0LrqrdxwmdAJauLjglYARUkAB/UHRBvI3S8DMLo/E8dpfhvyzb/Jgu0lfXXPmdSDCRXDL6/0 +vpbI40itpGnQ52u8wOHM3tz11v2UqcYi/HSiLQEt1CKOLpNnJuJ7KcwybUqJoR3Sbn7cBmvVxg== diff --git a/plugins/scanners/scanoss/src/test/assets/scanMulti/mappings/scanoss-multi-response.json b/plugins/scanners/scanoss/src/test/assets/scanMulti/mappings/scanoss-multi-response.json index 2db4f0820d6d0..ec812d4c7887c 100644 --- a/plugins/scanners/scanoss/src/test/assets/scanMulti/mappings/scanoss-multi-response.json +++ b/plugins/scanners/scanoss/src/test/assets/scanMulti/mappings/scanoss-multi-response.json @@ -10,7 +10,7 @@ }, "response" : { "status" : 200, - "body" : "{ \"c198b884-f6cf-496f-95eb-0e7968dd2ec6\": [ { \"id\": \"snippet\", \"status\": \"pending\", \"lines\": \"1-240\", \"oss_lines\": \"128-367\", \"matched\": \"99%\", \"purl\": [ \"pkg:github/scanoss/ort\" ], \"vendor\": \"scanoss\", \"component\": \"ort\", \"version\": \"e654028\", \"latest\": \"b12f8ee\", \"url\": \"https://github.com/scanoss/ort\", \"release_date\": \"2021-03-18\", \"file\": \"utils/src/main/kotlin/ArchiveUtils.kt\", \"url_hash\": \"37faa38a820322fa93bf7a8fa8290bb8\", \"file_hash\": \"871fb0c5188c2f620d9b997e225b0095\", \"source_hash\": \"2e91edbe430c4eb195a977d326d6d6c0\", \"file_url\": \"https://osskb.org/api/file_contents/871fb0c5188c2f620d9b997e225b0095\", \"licenses\": [ { \"name\": \"Apache-2.0\", \"patent_hints\": \"yes\", \"copyleft\": \"no\", \"checklist_url\": \"https://www.osadl.org/fileadmin/checklists/unreflicenses/Apache-2.0.txt\", \"osadl_updated\": \"2022-03-17 13:38\", \"source\": \"file_spdx_tag\" }, { \"name\": \"Apache-2.0\", \"patent_hints\": \"yes\", \"copyleft\": \"no\", \"checklist_url\": \"https://www.osadl.org/fileadmin/checklists/unreflicenses/Apache-2.0.txt\", \"osadl_updated\": \"2022-03-17 13:38\", \"source\": \"scancode\" } ], \"server\": { \"version\": \"4.4.2\", \"kb_version\": { \"monthly\": \"22.02\", \"daily\": \"22.03.25\" } } } ], \"5530105e-0752-4750-9c07-4e4604b879a5\": [ { \"id\": \"file\", \"status\": \"pending\", \"lines\": \"all\", \"oss_lines\": \"all\", \"matched\": \"100%\", \"purl\": [ \"pkg:github/scanoss/ort\" ], \"vendor\": \"scanoss\", \"component\": \"ort\", \"version\": \"e654028\", \"latest\": \"b12f8ee\", \"url\": \"https://github.com/scanoss/ort\", \"release_date\": \"2021-03-18\", \"file\": \"scanner/src/main/kotlin/ScannerFactory.kt\", \"url_hash\": \"37faa38a820322fa93bf7a8fa8290bb8\", \"file_hash\": \"5c8ab9be40df937e46c53509481107cd\", \"source_hash\": \"5c8ab9be40df937e46c53509481107cd\", \"file_url\": \"https://osskb.org/api/file_contents/5c8ab9be40df937e46c53509481107cd\", \"licenses\": [ { \"name\": \"Apache-2.0\", \"patent_hints\": \"yes\", \"copyleft\": \"no\", \"checklist_url\": \"https://www.osadl.org/fileadmin/checklists/unreflicenses/Apache-2.0.txt\", \"osadl_updated\": \"2022-03-17 13:38\", \"source\": \"file_spdx_tag\" }, { \"name\": \"Apache-2.0\", \"patent_hints\": \"yes\", \"copyleft\": \"no\", \"checklist_url\": \"https://www.osadl.org/fileadmin/checklists/unreflicenses/Apache-2.0.txt\", \"osadl_updated\": \"2022-03-17 13:38\", \"source\": \"scancode\" } ], \"server\": { \"version\": \"4.4.2\", \"kb_version\": { \"monthly\": \"22.02\", \"daily\": \"22.03.25\" } } } ]}", + "body" : "{ \"utils/src/main/kotlin/ArchiveUtils.kt\": [ { \"id\": \"snippet\", \"status\": \"pending\", \"lines\": \"1-240\", \"oss_lines\": \"128-367\", \"matched\": \"99%\", \"purl\": [ \"pkg:github/scanoss/ort\" ], \"vendor\": \"scanoss\", \"component\": \"ort\", \"version\": \"e654028\", \"latest\": \"b12f8ee\", \"url\": \"https://github.com/scanoss/ort\", \"release_date\": \"2021-03-18\", \"file\": \"utils/src/main/kotlin/ArchiveUtils.kt\", \"url_hash\": \"37faa38a820322fa93bf7a8fa8290bb8\", \"file_hash\": \"871fb0c5188c2f620d9b997e225b0095\", \"source_hash\": \"2e91edbe430c4eb195a977d326d6d6c0\", \"file_url\": \"https://osskb.org/api/file_contents/871fb0c5188c2f620d9b997e225b0095\", \"licenses\": [ { \"name\": \"Apache-2.0\", \"patent_hints\": \"yes\", \"copyleft\": \"no\", \"checklist_url\": \"https://www.osadl.org/fileadmin/checklists/unreflicenses/Apache-2.0.txt\", \"osadl_updated\": \"2022-03-17 13:38\", \"source\": \"file_spdx_tag\" }, { \"name\": \"Apache-2.0\", \"patent_hints\": \"yes\", \"copyleft\": \"no\", \"checklist_url\": \"https://www.osadl.org/fileadmin/checklists/unreflicenses/Apache-2.0.txt\", \"osadl_updated\": \"2022-03-17 13:38\", \"source\": \"scancode\" } ], \"server\": { \"version\": \"4.4.2\", \"kb_version\": { \"monthly\": \"22.02\", \"daily\": \"22.03.25\" } } } ], \"5530105e-0752-4750-9c07-4e4604b879a5\": [ { \"id\": \"file\", \"status\": \"pending\", \"lines\": \"all\", \"oss_lines\": \"all\", \"matched\": \"100%\", \"purl\": [ \"pkg:github/scanoss/ort\" ], \"vendor\": \"scanoss\", \"component\": \"ort\", \"version\": \"e654028\", \"latest\": \"b12f8ee\", \"url\": \"https://github.com/scanoss/ort\", \"release_date\": \"2021-03-18\", \"file\": \"scanner/src/main/kotlin/ScannerFactory.kt\", \"url_hash\": \"37faa38a820322fa93bf7a8fa8290bb8\", \"file_hash\": \"5c8ab9be40df937e46c53509481107cd\", \"source_hash\": \"5c8ab9be40df937e46c53509481107cd\", \"file_url\": \"https://osskb.org/api/file_contents/5c8ab9be40df937e46c53509481107cd\", \"licenses\": [ { \"name\": \"Apache-2.0\", \"patent_hints\": \"yes\", \"copyleft\": \"no\", \"checklist_url\": \"https://www.osadl.org/fileadmin/checklists/unreflicenses/Apache-2.0.txt\", \"osadl_updated\": \"2022-03-17 13:38\", \"source\": \"file_spdx_tag\" }, { \"name\": \"Apache-2.0\", \"patent_hints\": \"yes\", \"copyleft\": \"no\", \"checklist_url\": \"https://www.osadl.org/fileadmin/checklists/unreflicenses/Apache-2.0.txt\", \"osadl_updated\": \"2022-03-17 13:38\", \"source\": \"scancode\" } ], \"server\": { \"version\": \"4.4.2\", \"kb_version\": { \"monthly\": \"22.02\", \"daily\": \"22.03.25\" } } } ]}", "headers" : { "Server" : "nginx/1.14.2", "Date" : "Wed, 16 Mar 2022 13:07:04 GMT", diff --git a/plugins/scanners/scanoss/src/test/assets/scanoss-multiple-purls.json b/plugins/scanners/scanoss/src/test/assets/scanoss-multiple-purls.json new file mode 100644 index 0000000000000..7e02115508b8a --- /dev/null +++ b/plugins/scanners/scanoss/src/test/assets/scanoss-multiple-purls.json @@ -0,0 +1,60 @@ +{ + "hung_task.c": [ + { + "component": "proton_bluecross", + "file": "kernel/hung_task.c", + "file_hash": "581734935cfbe570d280a1265aaa2a6b", + "file_url": "https://api.scanoss.com/file_contents/581734935cfbe570d280a1265aaa2a6b", + "id": "snippet", + "latest": "17", + "licenses": [ + { + "checklist_url": "https://www.osadl.org/fileadmin/checklists/unreflicenses/GPL-2.0-only.txt", + "copyleft": "yes", + "incompatible_with": "Apache-1.0, Apache-1.1, Apache-2.0, BSD-4-Clause, BSD-4-Clause-UC, BSD-4.3TAHOE, ECL-2.0, FTL, IJG, LicenseRef-scancode-bsla-no-advert, Minpack, OpenSSL, PHP-3.01, Python-2.0, zlib-acknowledgement, XFree86-1.1", + "name": "GPL-2.0-only", + "osadl_updated": "2025-02-10T14:26:00+0000", + "patent_hints": "yes", + "source": "scancode", + "url": "https://spdx.org/licenses/GPL-2.0-only.html" + }, + { + "name": "GPL-2.0-only WITH Linux-syscall-note", + "source": "scancode", + "url": "https://spdx.org/licenses/GPL-2.0-only WITH Linux-syscall-note.html" + }, + { + "checklist_url": "https://www.osadl.org/fileadmin/checklists/unreflicenses/GPL-2.0-only.txt", + "copyleft": "yes", + "incompatible_with": "Apache-1.0, Apache-1.1, Apache-2.0, BSD-4-Clause, BSD-4-Clause-UC, BSD-4.3TAHOE, ECL-2.0, FTL, IJG, LicenseRef-scancode-bsla-no-advert, Minpack, OpenSSL, PHP-3.01, Python-2.0, zlib-acknowledgement, XFree86-1.1", + "name": "GPL-2.0-only", + "osadl_updated": "2025-02-10T14:26:00+0000", + "patent_hints": "yes", + "source": "scancode", + "url": "https://spdx.org/licenses/GPL-2.0-only.html" + } + ], + "lines": "12-150,540-561", + "matched": "35%", + "oss_lines": "10-148,86-107", + "purl": [ + "pkg:github/kdrag0n/proton_bluecross", + "pkg:github/fake/fake_repository" + ], + "release_date": "2019-02-21", + "server": { + "kb_version": { + "daily": "25.03.27", + "monthly": "25.03" + }, + "version": "5.4.10" + }, + "source_hash": "45dd1e50621a8a32f88fbe0251a470ab", + "status": "pending", + "url": "https://github.com/kdrag0n/proton_bluecross", + "url_hash": "a9c1c67f0930dc42dbd40c29e565bcdd", + "vendor": "kdrag0n", + "version": "15" + } + ] +} diff --git a/plugins/scanners/scanoss/src/test/kotlin/ScanOssResultParserTest.kt b/plugins/scanners/scanoss/src/test/kotlin/ScanOssResultParserTest.kt index 3fbef55245ddf..4d3a06d001946 100644 --- a/plugins/scanners/scanoss/src/test/kotlin/ScanOssResultParserTest.kt +++ b/plugins/scanners/scanoss/src/test/kotlin/ScanOssResultParserTest.kt @@ -19,6 +19,7 @@ package org.ossreviewtoolkit.plugins.scanners.scanoss +import com.scanoss.dto.LicenseDetails import com.scanoss.utils.JsonUtils import io.kotest.core.spec.style.WordSpec @@ -27,6 +28,7 @@ import io.kotest.matchers.collections.containExactlyInAnyOrder import io.kotest.matchers.collections.haveSize import io.kotest.matchers.collections.shouldContain import io.kotest.matchers.should +import io.kotest.matchers.shouldBe import java.io.File import java.time.Instant @@ -39,9 +41,44 @@ import org.ossreviewtoolkit.model.SnippetFinding import org.ossreviewtoolkit.model.TextLocation import org.ossreviewtoolkit.model.VcsInfo import org.ossreviewtoolkit.model.VcsType +import org.ossreviewtoolkit.utils.spdx.SpdxConstants import org.ossreviewtoolkit.utils.spdx.SpdxExpression +import org.ossreviewtoolkit.utils.spdx.SpdxLicenseIdExpression class ScanOssResultParserTest : WordSpec({ + "getUniqueLicenseDetails()" should { + "deduplicate complex license expressions" { + val uniqueLicenses = getUniqueLicenseExpression( + listOf( + LicenseDetails.builder().name("MIT").build(), + LicenseDetails.builder().name("MIT").build(), + LicenseDetails.builder().name("GPL-2.0-only").build(), + LicenseDetails.builder().name("GPL-2.0-only WITH Linux-syscall-note").build(), + LicenseDetails.builder().name("GPL-2.0-only AND MIT").build() + ) + ) + + val decomposed = uniqueLicenses.decompose().toList() + + val expressionStrings = decomposed.map { it.toString() } + + // Check that each license appears exactly once + expressionStrings.count { it == "MIT" } shouldBe 1 + expressionStrings.count { it == "GPL-2.0-only" } shouldBe 1 + expressionStrings.count { it == "GPL-2.0-only WITH Linux-syscall-note" } shouldBe 1 + + // Ensure no unexpected elements + expressionStrings.size shouldBe 3 + } + + "handle empty license list" { + val emptyLicenses = getUniqueLicenseExpression(listOf()) + + // Verify empty license list returns NOASSERTION + emptyLicenses shouldBe SpdxLicenseIdExpression(SpdxConstants.NOASSERTION) + } + } + "generateSummary()" should { "properly summarize JUnit 4.12 findings" { val results = File("src/test/assets/scanoss-junit-4.12.json").readText().let { @@ -125,11 +162,52 @@ class ScanOssResultParserTest : WordSpec({ "." ), "pkg:github/vdurmont/semver4j", - SpdxExpression.parse("CC-BY-SA-2.0") + SpdxExpression.parse("CC-BY-SA-2.0"), + additionalData = mapOf( + "release_date" to "2019-09-13", + "all_purls" to "pkg:github/vdurmont/semver4j" + ) ) ) ) ) } + + "should handle multiple PURLs by selecting first as primary and preserving all in metadata" { + val results = File("src/test/assets/scanoss-multiple-purls.json").readText().let { + JsonUtils.toScanFileResultsFromObject(JsonUtils.toJsonObject(it)) + } + + val time = Instant.now() + val summary = generateSummary(time, time, results) + + // Should have one finding per source location, not per PURL. + summary.snippetFindings should haveSize(2) + + with(summary.snippetFindings.first()) { + // Check source location (local file). + sourceLocation shouldBe TextLocation("hung_task.c", 12, 150) + + // Should use first PURL as primary identifier. + snippets should haveSize(1) + snippets.first().purl shouldBe "pkg:github/kdrag0n/proton_bluecross" + + // Should preserve all PURLs in additionalData. + snippets.first().additionalData["all_purls"] shouldBe + "pkg:github/kdrag0n/proton_bluecross pkg:github/fake/fake_repository" + + // Check OSS location. + snippets.first().location shouldBe + TextLocation("https://api.scanoss.com/file_contents/581734935cfbe570d280a1265aaa2a6b", 10, 148) + } + + // Verify same behavior for second snippet. + with(summary.snippetFindings.last()) { + sourceLocation shouldBe TextLocation("hung_task.c", 540, 561) + snippets.first().purl shouldBe "pkg:github/kdrag0n/proton_bluecross" + snippets.first().location shouldBe + TextLocation("https://api.scanoss.com/file_contents/581734935cfbe570d280a1265aaa2a6b", 86, 107) + } + } } }) diff --git a/plugins/scanners/scanoss/src/test/kotlin/ScanOssScannerDirectoryTest.kt b/plugins/scanners/scanoss/src/test/kotlin/ScanOssScannerDirectoryTest.kt index b47bb6b05b9c3..00a90039a0116 100644 --- a/plugins/scanners/scanoss/src/test/kotlin/ScanOssScannerDirectoryTest.kt +++ b/plugins/scanners/scanoss/src/test/kotlin/ScanOssScannerDirectoryTest.kt @@ -20,19 +20,19 @@ package org.ossreviewtoolkit.plugins.scanners.scanoss import com.github.tomakehurst.wiremock.WireMockServer +import com.github.tomakehurst.wiremock.client.WireMock import com.github.tomakehurst.wiremock.core.WireMockConfiguration +import io.kotest.assertions.fail import io.kotest.core.spec.style.StringSpec import io.kotest.matchers.collections.containExactly import io.kotest.matchers.collections.containExactlyInAnyOrder import io.kotest.matchers.should +import io.kotest.matchers.shouldBe -import io.mockk.every import io.mockk.spyk -import io.mockk.verify import java.io.File -import java.util.UUID import org.ossreviewtoolkit.model.LicenseFinding import org.ossreviewtoolkit.model.PackageType @@ -42,11 +42,16 @@ import org.ossreviewtoolkit.model.SnippetFinding import org.ossreviewtoolkit.model.TextLocation import org.ossreviewtoolkit.model.VcsInfo import org.ossreviewtoolkit.model.VcsType +import org.ossreviewtoolkit.model.config.Excludes +import org.ossreviewtoolkit.model.config.PathExclude +import org.ossreviewtoolkit.model.config.PathExcludeReason import org.ossreviewtoolkit.plugins.api.Secret import org.ossreviewtoolkit.scanner.ScanContext import org.ossreviewtoolkit.utils.spdx.SpdxExpression +// Define separate directories for different test scenarios. private val TEST_DIRECTORY_TO_SCAN = File("src/test/assets/filesToScan") +private val EXCLUSION_TEST_DIRECTORY = File("src/test/assets/exclusionTest") /** * A test for scanning a directory with the [ScanOss] scanner. @@ -74,25 +79,11 @@ class ScanOssScannerDirectoryTest : StringSpec({ } "The scanner should scan a directory" { - // Manipulate the UUID generation to have the same IDs as in the response. - every { - scanner.generateRandomUUID() - } answers { - UUID.fromString("5530105e-0752-4750-9c07-4e4604b879a5") - } andThenAnswer { - UUID.fromString("c198b884-f6cf-496f-95eb-0e7968dd2ec6") - } - val summary = scanner.scanPath( TEST_DIRECTORY_TO_SCAN, ScanContext(labels = emptyMap(), packageType = PackageType.PACKAGE) ) - verify(exactly = 1) { - scanner.createWfpForFile(TEST_DIRECTORY_TO_SCAN.resolve("ArchiveUtils.kt")) - scanner.createWfpForFile(TEST_DIRECTORY_TO_SCAN.resolve("ScannerFactory.kt")) - } - with(summary) { licenseFindings should containExactlyInAnyOrder( LicenseFinding( @@ -120,11 +111,73 @@ class ScanOssScannerDirectoryTest : StringSpec({ VcsInfo(VcsType.GIT, "https://github.com/scanoss/ort.git", ""), "." ), "pkg:github/scanoss/ort", - SpdxExpression.parse("Apache-2.0") + SpdxExpression.parse("Apache-2.0"), + additionalData = mapOf( + "release_date" to "2021-03-18", + "all_purls" to "pkg:github/scanoss/ort" + ) + ) ) ) ) } } + + "Scanner should exclude only files matching the specified path pattern (**/*.kt)" { + val pathExcludes = listOf( + PathExclude( + pattern = "**/*.kt", // Glob pattern to match all .kt files in any directory. + reason = PathExcludeReason.BUILD_TOOL_OF, + comment = "Excluding .kt source files from scanning" + ) + ) + + // Verify our test file exists. This file should be included in the scan since it does not match the exclusion + // pattern (it is a .go file, not a .kt file). + val includedFile = File(EXCLUSION_TEST_DIRECTORY, "server.go") + if (!includedFile.isFile) { + fail("The test file ${includedFile.absolutePath} does not exist - test environment may not be properly set up") + } + + // Run the scanner with our exclusion pattern. This will traverse the directory and should skip .kt files. + scanner.scanPath( + EXCLUSION_TEST_DIRECTORY, + ScanContext( + labels = emptyMap(), + packageType = PackageType.PACKAGE, + excludes = Excludes(paths = pathExcludes) + ) + ) + + // Retrieve all HTTP POST requests captured by WireMock during the scan. + val requests = server.findAll(WireMock.postRequestedFor(WireMock.anyUrl())) + val requestBodies = requests.map { it.bodyAsString } + + // The scanner sends files to the API in a multipart/form-data POST request with this format: + // --boundary + // Content-Disposition: form-data; name="file"; filename="[UUID].wfp" + // Content-Type: text/plain; charset=utf-8 + // Content-Length: [length] + // + // file=[hash],[size],[filename] + // [fingerprint data for the file] + // --boundary-- + + // Extract included filenames using a regex pattern from the ScanOSS HTTP POST. + // The pattern matches lines starting with "file=" followed by hash and size, then captures the filename. + val filenamePattern = "file=.*?,.*?,(.+)".toRegex(RegexOption.MULTILINE) + val includedFiles = requestBodies.flatMap { body -> + filenamePattern.findAll(body).map { it.groupValues[1] }.toList() + } + + // Verify that .kt files were excluded from the scan. + // These assertions check that Kotlin files are not present in the API requests. + includedFiles.any { it.contains("ArchiveUtils.kt") } shouldBe false + includedFiles.any { it.contains("ScannerFactory.kt") } shouldBe false + + // Verify that non-.kt files were included in the scan. + // This assertion checks that our Go file was sent to the API. + includedFiles.any { it.contains("server.go") } shouldBe true + } }) diff --git a/plugins/scanners/scanoss/src/test/kotlin/ScanOssScannerFileTest.kt b/plugins/scanners/scanoss/src/test/kotlin/ScanOssScannerFileTest.kt index 9a32a156831db..95f6ba781922b 100644 --- a/plugins/scanners/scanoss/src/test/kotlin/ScanOssScannerFileTest.kt +++ b/plugins/scanners/scanoss/src/test/kotlin/ScanOssScannerFileTest.kt @@ -26,12 +26,9 @@ import io.kotest.core.spec.style.StringSpec import io.kotest.matchers.collections.containExactly import io.kotest.matchers.should -import io.mockk.every import io.mockk.spyk -import io.mockk.verify import java.io.File -import java.util.UUID import org.ossreviewtoolkit.model.LicenseFinding import org.ossreviewtoolkit.model.PackageType @@ -67,22 +64,11 @@ class ScanOssScannerFileTest : StringSpec({ } "The scanner should scan a single file" { - // Manipulate the UUID generation to have the same IDs as in the response. - every { - scanner.generateRandomUUID() - } answers { - UUID.fromString("bf5401e9-03b3-4c91-906c-cadb90487b8c") - } - val summary = scanner.scanPath( TEST_FILE_TO_SCAN, ScanContext(labels = emptyMap(), packageType = PackageType.PACKAGE) ) - verify(exactly = 1) { - scanner.createWfpForFile(TEST_FILE_TO_SCAN) - } - with(summary) { licenseFindings should containExactly( LicenseFinding( diff --git a/plugins/scanners/scanoss/src/test/kotlin/ScanOssTest.kt b/plugins/scanners/scanoss/src/test/kotlin/ScanOssTest.kt new file mode 100644 index 0000000000000..5308ffec45a95 --- /dev/null +++ b/plugins/scanners/scanoss/src/test/kotlin/ScanOssTest.kt @@ -0,0 +1,193 @@ +/* + * Copyright (C) 2025 The ORT Project Authors (see ) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * License-Filename: LICENSE + */ + +package org.ossreviewtoolkit.plugins.scanners.scanoss + +import io.kotest.core.spec.style.WordSpec +import io.kotest.matchers.collections.beEmpty +import io.kotest.matchers.collections.shouldBeSingleton +import io.kotest.matchers.should +import io.kotest.matchers.shouldBe + +import org.ossreviewtoolkit.model.TextLocation +import org.ossreviewtoolkit.model.config.SnippetChoices +import org.ossreviewtoolkit.model.config.snippet.Choice +import org.ossreviewtoolkit.model.config.snippet.Given +import org.ossreviewtoolkit.model.config.snippet.Provenance +import org.ossreviewtoolkit.model.config.snippet.SnippetChoice +import org.ossreviewtoolkit.model.config.snippet.SnippetChoiceReason + +// Sample files in the results. +private const val FILE_1 = "a.java" +private const val FILE_2 = "b.java" + +// A sample purl in the results. +private const val PURL_1 = "pkg:github/fakeuser/fakepackage1@1.0.0" + +class ScanOssTest : WordSpec({ + "processSnippetChoices" should { + "create empty rules when no snippet choices exist" { + val scanoss = createScanOss(createScanOssConfig()) + + val emptySnippetChoices: List = listOf() + + val rules = scanoss.processSnippetChoices(emptySnippetChoices) + + rules.ignoreRules should beEmpty() + rules.removeRules should beEmpty() + rules.includeRules should beEmpty() + rules.replaceRules should beEmpty() + } + + "create an include rule for snippet choices with ORIGINAL finding" { + val vcsInfo = createVcsInfo() + val scanoss = createScanOss(createScanOssConfig()) + + val location = TextLocation(FILE_1, 10, 20) + val snippetChoices = createSnippetChoices( + vcsInfo.url, + createSnippetChoice( + location, + PURL_1, + "This is an original finding" + ) + ) + + val rules = scanoss.processSnippetChoices(snippetChoices) + + rules.includeRules.shouldBeSingleton { rule -> + rule.purl shouldBe PURL_1 + rule.path shouldBe FILE_1 + } + + rules.removeRules should beEmpty() + rules.ignoreRules should beEmpty() + rules.replaceRules should beEmpty() + } + + "create a remove rule for snippet choices with NOT_FINDING reason" { + val vcsInfo = createVcsInfo() + + val scanoss = createScanOss(createScanOssConfig()) + + val location = TextLocation(FILE_2, 15, 30) + val snippetChoices = createSnippetChoices( + vcsInfo.url, + createSnippetChoice( + location, + null, // null PURL for NOT_FINDING + "This is not a relevant finding" + ) + ) + + val rules = scanoss.processSnippetChoices(snippetChoices) + + rules.removeRules.shouldBeSingleton { rule -> + rule.path shouldBe FILE_2 + rule.startLine shouldBe 15 + rule.endLine shouldBe 30 + } + + rules.includeRules should beEmpty() + rules.ignoreRules should beEmpty() + rules.replaceRules should beEmpty() + } + + "handle multiple snippet choices with different reasons correctly" { + val vcsInfo = createVcsInfo() + val scanoss = createScanOss(createScanOssConfig()) + + val location1 = TextLocation(FILE_1, 10, 20) + val location2 = TextLocation(FILE_2, 15, 30) + + val snippetChoices = createSnippetChoices( + vcsInfo.url, + createSnippetChoice( + location1, + PURL_1, + "This is an original finding" + ), + createSnippetChoice( + location2, + null, + "This is not a relevant finding" + ) + ) + + val rules = scanoss.processSnippetChoices(snippetChoices) + + rules.includeRules.shouldBeSingleton { rule -> + rule.purl shouldBe PURL_1 + rule.path shouldBe FILE_1 + } + + rules.removeRules.shouldBeSingleton { rule -> + rule.path shouldBe FILE_2 + rule.startLine shouldBe 15 + rule.endLine shouldBe 30 + } + + rules.ignoreRules should beEmpty() + rules.replaceRules should beEmpty() + } + + "should create a remove rule without line ranges when snippet choice has UNKNOWN_LINE (-1) values" { + val vcsInfo = createVcsInfo() + val scanoss = createScanOss(createScanOssConfig()) + + // Create a TextLocation with -1 for start and end lines + val location = TextLocation(FILE_2, TextLocation.UNKNOWN_LINE, TextLocation.UNKNOWN_LINE) + val snippetChoices = createSnippetChoices( + vcsInfo.url, + createSnippetChoice( + location, + null, // null PURL for NOT_FINDING + "This is a not relevant finding with no line ranges" + ) + ) + + val rules = scanoss.processSnippetChoices(snippetChoices) + + rules.removeRules.shouldBeSingleton { rule -> + rule.path shouldBe FILE_2 + rule.startLine shouldBe null + rule.endLine shouldBe null + } + + rules.includeRules should beEmpty() + rules.ignoreRules should beEmpty() + rules.replaceRules should beEmpty() + } + } +}) + +private fun createSnippetChoices(provenanceUrl: String, vararg snippetChoices: SnippetChoice) = + listOf(SnippetChoices(Provenance(provenanceUrl), snippetChoices.asList())) + +private fun createSnippetChoice(location: TextLocation, purl: String? = null, comment: String) = + SnippetChoice( + Given( + location + ), + Choice( + purl, + if (purl == null) SnippetChoiceReason.NO_RELEVANT_FINDING else SnippetChoiceReason.ORIGINAL_FINDING, + comment + ) + ) diff --git a/plugins/scanners/scanoss/src/test/kotlin/TestUtils.kt b/plugins/scanners/scanoss/src/test/kotlin/TestUtils.kt new file mode 100644 index 0000000000000..94fc0487c976a --- /dev/null +++ b/plugins/scanners/scanoss/src/test/kotlin/TestUtils.kt @@ -0,0 +1,69 @@ +/* + * Copyright (C) 2025 The ORT Project Authors (see ) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * License-Filename: LICENSE + */ + +package org.ossreviewtoolkit.plugins.scanners.scanoss + +import com.scanoss.rest.ScanApi + +import org.ossreviewtoolkit.model.VcsInfo +import org.ossreviewtoolkit.model.VcsType +import org.ossreviewtoolkit.plugins.api.Secret + +// A test project name. +internal const val PROJECT = "scanoss-test-project" + +// A (resolved) test revision. +private const val REVISION = "0123456789012345678901234567890123456789" + +/** + * Create a new [ScanOss] instance with the specified [config]. + */ +internal fun createScanOss(config: ScanOssConfig): ScanOss = ScanOss(config = config) + +/** + * Create a standard [ScanOssConfig] whose properties can be partly specified. + */ +internal fun createScanOssConfig( + apiUrl: String = ScanApi.DEFAULT_BASE_URL, + apiKey: Secret = Secret(""), + regScannerName: String? = null, + minVersion: String? = null, + maxVersion: String? = null, + readFromStorage: Boolean = true, + writeToStorage: Boolean = true +): ScanOssConfig = ScanOssConfig( + apiUrl = apiUrl, + apiKey = apiKey, + regScannerName = regScannerName, + minVersion = minVersion, + maxVersion = maxVersion, + readFromStorage = readFromStorage, + writeToStorage = writeToStorage +) + +/** + * Create a [VcsInfo] object for a project with the given [name][projectName] and the optional parameters for [type], + * [path], and [revision]. + */ +internal fun createVcsInfo( + projectName: String = PROJECT, + type: VcsType = VcsType.GIT, + path: String = "", + revision: String = REVISION +): VcsInfo = VcsInfo(type = type, path = path, revision = revision, url = "https://github.com/test/$projectName.git")