feat(Provenance): Add DirectoryProvenance as a LocalProvenance

In contrast to the previously added `RemoteProvenance` stands the
`LocalProvenance`, which has no remote source, but instead references
a local input of some kind.
The `DirectoryProvenance` references a local project directory,
which is lacking supported (remote) version control.
It is defined by its canonical path only.

Since ORT needs further refactoring until `DirectoryProvenance` can
be fully utilized, the new class can not be used right now.
However the presence of a new `KnownProvenance` class results in
`when` conditional cases not being exhaustive anymore.

To circumvent this issue, the following changes were made:
1. Wherever possible `RemoteProvenance` is used as parameter type
   instead of `KnownProvenance`.
2. When necessary `KnownProvenance`s are cast to `RemoteProvenance`.
3. If `Provenance` is expected, `DirectoryProvenance` is handled
   like `UnknownProvenance`.

For instances of `Package` the new default data structure should be
`RemoteProvenance`, as a `Package` by definition requires a remote.

The exception being `hash` and `storageKey`, which both required a
default value, which was set to the `canonicalPath`.
However, since the rest of the code does not handle
`DirectoryProvenance`, this should remain unused for now.

See [1] for more context on the new provenance hierarchy.

[1]: #8803 (comment)

Signed-off-by: Jens Keim <[email protected]>

Author: pepper-jk

model/src/main/kotlin/Provenance.kt

@@ -26,6 +26,8 @@ import com.fasterxml.jackson.databind.annotation.JsonDeserialize
 import com.fasterxml.jackson.databind.deser.std.StdDeserializer
 import com.fasterxml.jackson.module.kotlin.treeToValue
 
+import java.io.File
+
 /**
  * Provenance information about the origin of source code.
  */
@@ -45,6 +47,8 @@ sealed interface KnownProvenance : Provenance
 
 sealed interface RemoteProvenance : KnownProvenance
 
+sealed interface LocalProvenance : KnownProvenance
+
 /**
  * Provenance information for a source artifact.
  */
@@ -83,6 +87,19 @@ data class RepositoryProvenance(
     override fun matches(pkg: Package): Boolean = vcsInfo == pkg.vcsProcessed
 }
 
+/**
+ * Provenance information for a local directory path.
+ */
+data class DirectoryProvenance(
+    val canonicalPath: File
+) : LocalProvenance {
+    /**
+     * Return false for any [pkg] as a [Package] by definition is coming from a remote, and is not stored in a local
+     * directory path (which would be a [Project]).
+     */
+    override fun matches(pkg: Package): Boolean = false
+}
+
 /**
  * A custom deserializer for polymorphic deserialization of [Provenance] without requiring type information.
  */

model/src/main/kotlin/ProvenanceResolutionResult.kt

@@ -37,7 +37,7 @@ data class ProvenanceResolutionResult(
      * The resolved provenance of the package. Can only be null if a [packageProvenanceResolutionIssue] occurred.
      */
     @JsonInclude(JsonInclude.Include.NON_NULL)
-    val packageProvenance: KnownProvenance? = null,
+    val packageProvenance: RemoteProvenance? = null,
 
     /**
      * The (recursive) sub-repositories of [packageProvenance]. The map can only be empty if a

model/src/main/kotlin/utils/FileProvenanceFileStorage.kt

@@ -24,6 +24,7 @@ import java.io.InputStream
 import org.apache.logging.log4j.kotlin.logger
 
 import org.ossreviewtoolkit.model.ArtifactProvenance
+import org.ossreviewtoolkit.model.DirectoryProvenance
 import org.ossreviewtoolkit.model.HashAlgorithm
 import org.ossreviewtoolkit.model.KnownProvenance
 import org.ossreviewtoolkit.model.RepositoryProvenance
@@ -81,6 +82,7 @@ private fun KnownProvenance.hash(): String {
     val key = when (this) {
         is ArtifactProvenance -> "${sourceArtifact.url}${sourceArtifact.hash.value}"
         is RepositoryProvenance -> "${vcsInfo.type}${vcsInfo.url}$resolvedRevision"
+        is DirectoryProvenance -> "$canonicalPath"
     }
 
     return HashAlgorithm.SHA1.calculate(key.toByteArray())

model/src/main/kotlin/utils/PostgresProvenanceFileStorage.kt

@@ -36,6 +36,7 @@ import org.jetbrains.exposed.sql.insert
 import org.jetbrains.exposed.sql.selectAll
 
 import org.ossreviewtoolkit.model.ArtifactProvenance
+import org.ossreviewtoolkit.model.DirectoryProvenance
 import org.ossreviewtoolkit.model.KnownProvenance
 import org.ossreviewtoolkit.model.RepositoryProvenance
 import org.ossreviewtoolkit.model.utils.DatabaseUtils.checkDatabaseEncoding
@@ -121,4 +122,5 @@ private fun KnownProvenance.storageKey(): String =
         is ArtifactProvenance -> "source-artifact|${sourceArtifact.url}|${sourceArtifact.hash}"
         // The trailing "|" is kept for backward compatibility because there used to be an additional parameter.
         is RepositoryProvenance -> "vcs|${vcsInfo.type}|${vcsInfo.url}|$resolvedRevision|"
+        is DirectoryProvenance -> "directory|$canonicalPath"
     }

model/src/main/kotlin/utils/PurlExtensions.kt

@@ -27,6 +27,7 @@ import org.ossreviewtoolkit.model.Identifier
 import org.ossreviewtoolkit.model.Package
 import org.ossreviewtoolkit.model.Provenance
 import org.ossreviewtoolkit.model.RemoteArtifact
+import org.ossreviewtoolkit.model.RemoteProvenance
 import org.ossreviewtoolkit.model.RepositoryProvenance
 import org.ossreviewtoolkit.model.UnknownProvenance
 import org.ossreviewtoolkit.model.VcsInfo
@@ -101,7 +102,11 @@ fun Provenance.toPurlExtras(): PurlExtras =
             )
         }
 
-        is UnknownProvenance -> PurlExtras()
+        /**
+         * Purls refer to packages that have been published and thus always have a remove provenance. So just return
+         * empty extras in all other cases.
+         */
+        !is RemoteProvenance -> PurlExtras()
     }
 
 /**

scanner/src/main/kotlin/ScanController.kt

@@ -24,6 +24,7 @@ import org.ossreviewtoolkit.model.Issue
 import org.ossreviewtoolkit.model.KnownProvenance
 import org.ossreviewtoolkit.model.Package
 import org.ossreviewtoolkit.model.Provenance
+import org.ossreviewtoolkit.model.RemoteProvenance
 import org.ossreviewtoolkit.model.RepositoryProvenance
 import org.ossreviewtoolkit.model.ScanResult
 import org.ossreviewtoolkit.model.ScanSummary
@@ -79,7 +80,7 @@ internal class ScanController(
      * A map of package [Identifier]s to their resolved [KnownProvenance]s. These provenances are used to filter the
      * scan results for a package based on the VCS path.
      */
-    private val packageProvenances = mutableMapOf<Identifier, KnownProvenance>()
+    private val packageProvenances = mutableMapOf<Identifier, RemoteProvenance>()
 
     /**
      * A map of package [Identifier]s to their resolved [KnownProvenance]s with the VCS path removed. These provenances
@@ -130,7 +131,7 @@ internal class ScanController(
     /**
      * Set the [provenance] for the package denoted by [id], overwriting any existing values.
      */
-    fun putPackageProvenance(id: Identifier, provenance: KnownProvenance) {
+    fun putPackageProvenance(id: Identifier, provenance: RemoteProvenance) {
         packageProvenances[id] = provenance
         packageProvenancesWithoutVcsPath[id] = when (provenance) {
             is RepositoryProvenance -> provenance.copy(vcsInfo = provenance.vcsInfo.copy(path = ""))
@@ -267,7 +268,7 @@ internal class ScanController(
     fun getPackagesForProvenanceWithoutVcsPath(provenance: KnownProvenance): Set<Identifier> =
         packageProvenancesWithoutVcsPath.filter { (_, packageProvenance) -> packageProvenance == provenance }.keys
 
-    fun getPackageProvenance(id: Identifier): KnownProvenance? = packageProvenances[id]
+    fun getPackageProvenance(id: Identifier): RemoteProvenance? = packageProvenances[id]
 
     /**
      * Return the package provenanceResolutionIssue associated with the given [id].

scanner/src/main/kotlin/Scanner.kt

@@ -44,6 +44,7 @@ import org.ossreviewtoolkit.model.OrtResult
 import org.ossreviewtoolkit.model.Package
 import org.ossreviewtoolkit.model.PackageType
 import org.ossreviewtoolkit.model.ProvenanceResolutionResult
+import org.ossreviewtoolkit.model.RemoteProvenance
 import org.ossreviewtoolkit.model.ScanResult
 import org.ossreviewtoolkit.model.ScannerRun
 import org.ossreviewtoolkit.model.TextLocation
@@ -235,7 +236,7 @@ class Scanner(
                 }.awaitAll()
             }.forEach { (pkg, result) ->
                 result.onSuccess { provenance ->
-                    controller.putPackageProvenance(pkg.id, provenance)
+                    controller.putPackageProvenance(pkg.id, provenance as RemoteProvenance)
                 }.onFailure {
                     controller.putPackageProvenanceResolutionIssue(
                         pkg.id,
@@ -256,7 +257,7 @@ class Scanner(
                 controller.getPackageProvenancesWithoutVcsPath().map { provenance ->
                     async {
                         provenance to runCatching {
-                            nestedProvenanceResolver.resolveNestedProvenance(provenance)
+                            nestedProvenanceResolver.resolveNestedProvenance(provenance as RemoteProvenance)
                         }.onFailure {
                             if (it is CancellationException) currentCoroutineContext().ensureActive()
                         }
@@ -591,7 +592,7 @@ class Scanner(
         controller: ScanController
     ): Map<PathScannerWrapper, ScanResult> {
         val downloadDir = try {
-            provenanceDownloader.download(provenance)
+            provenanceDownloader.download(provenance as RemoteProvenance)
         } catch (e: DownloadException) {
             val issue = createAndLogIssue(
                 "Downloader", "Could not download provenance $provenance: ${e.collectMessages()}"

scanner/src/main/kotlin/provenance/NestedProvenanceResolver.kt

@@ -23,8 +23,8 @@ import org.apache.logging.log4j.kotlin.logger
 
 import org.ossreviewtoolkit.downloader.WorkingTreeCache
 import org.ossreviewtoolkit.model.ArtifactProvenance
-import org.ossreviewtoolkit.model.KnownProvenance
 import org.ossreviewtoolkit.model.Provenance
+import org.ossreviewtoolkit.model.RemoteProvenance
 import org.ossreviewtoolkit.model.RepositoryProvenance
 
 /**
@@ -36,7 +36,7 @@ interface NestedProvenanceResolver {
      * [NestedProvenance] always contains only the provided [ArtifactProvenance]. For a [RepositoryProvenance] the
      * resolver looks for nested repositories, for example Git submodules or Mercurial subrepositories.
      */
-    suspend fun resolveNestedProvenance(provenance: KnownProvenance): NestedProvenance
+    suspend fun resolveNestedProvenance(provenance: RemoteProvenance): NestedProvenance
 }
 
 /**
@@ -46,7 +46,7 @@ class DefaultNestedProvenanceResolver(
     private val storage: NestedProvenanceStorage,
     private val workingTreeCache: WorkingTreeCache
 ) : NestedProvenanceResolver {
-    override suspend fun resolveNestedProvenance(provenance: KnownProvenance): NestedProvenance {
+    override suspend fun resolveNestedProvenance(provenance: RemoteProvenance): NestedProvenance {
         return when (provenance) {
             is ArtifactProvenance -> NestedProvenance(root = provenance, subRepositories = emptyMap())
             is RepositoryProvenance -> resolveNestedRepository(provenance)

scanner/src/main/kotlin/provenance/ProvenanceDownloader.kt

@@ -32,8 +32,8 @@ import org.ossreviewtoolkit.downloader.DownloadException
 import org.ossreviewtoolkit.downloader.Downloader
 import org.ossreviewtoolkit.downloader.WorkingTreeCache
 import org.ossreviewtoolkit.model.ArtifactProvenance
-import org.ossreviewtoolkit.model.KnownProvenance
 import org.ossreviewtoolkit.model.Package
+import org.ossreviewtoolkit.model.RemoteProvenance
 import org.ossreviewtoolkit.model.RepositoryProvenance
 import org.ossreviewtoolkit.model.config.DownloaderConfiguration
 import org.ossreviewtoolkit.utils.common.safeDeleteRecursively
@@ -51,7 +51,7 @@ fun interface ProvenanceDownloader {
      *
      * Throws a [DownloadException] if the download fails.
      */
-    fun download(provenance: KnownProvenance): File
+    fun download(provenance: RemoteProvenance): File
 
     /**
      * Download the source code specified by the provided [nestedProvenance] incl. sub-repositories and return the path
@@ -64,7 +64,7 @@ fun interface ProvenanceDownloader {
         // Use the provenanceDownloader to download each provenance from nestedProvenance separately, because they are
         // likely already cached if a path scanner wrapper is used.
 
-        val root = download(nestedProvenance.root)
+        val root = download(nestedProvenance.root as RemoteProvenance)
 
         nestedProvenance.subRepositories.forEach { (path, provenance) ->
             val tempDir = download(provenance)
@@ -86,7 +86,7 @@ class DefaultProvenanceDownloader(
 ) : ProvenanceDownloader {
     private val downloader = Downloader(config)
 
-    override fun download(provenance: KnownProvenance): File {
+    override fun download(provenance: RemoteProvenance): File {
         val downloadDir = createOrtTempDir()
 
         when (provenance) {

scanner/src/main/kotlin/storages/ProvenanceBasedFileStorage.kt

@@ -29,6 +29,7 @@ import org.apache.logging.log4j.kotlin.logger
 
 import org.ossreviewtoolkit.model.ArtifactProvenance
 import org.ossreviewtoolkit.model.KnownProvenance
+import org.ossreviewtoolkit.model.RemoteProvenance
 import org.ossreviewtoolkit.model.RepositoryProvenance
 import org.ossreviewtoolkit.model.ScanResult
 import org.ossreviewtoolkit.model.yamlMapper
@@ -45,7 +46,7 @@ class ProvenanceBasedFileStorage(private val backend: FileStorage) : ProvenanceB
     override fun read(provenance: KnownProvenance, scannerMatcher: ScannerMatcher?): List<ScanResult> {
         requireEmptyVcsPath(provenance)
 
-        val path = storagePath(provenance)
+        val path = storagePath(provenance as RemoteProvenance)
 
         return runCatching {
             backend.read(path).use { input ->
@@ -97,7 +98,7 @@ class ProvenanceBasedFileStorage(private val backend: FileStorage) : ProvenanceB
 
         val scanResults = existingScanResults + scanResult
 
-        val path = storagePath(provenance)
+        val path = storagePath(provenance as RemoteProvenance)
         val yamlBytes = yamlMapper.writeValueAsBytes(scanResults)
         val input = ByteArrayInputStream(yamlBytes)
 
@@ -123,7 +124,7 @@ class ProvenanceBasedFileStorage(private val backend: FileStorage) : ProvenanceB
     }
 }
 
-private fun storagePath(provenance: KnownProvenance) =
+private fun storagePath(provenance: RemoteProvenance) =
     when (provenance) {
         is ArtifactProvenance -> "artifact/${provenance.sourceArtifact.url.fileSystemEncode()}/$SCAN_RESULTS_FILE_NAME"
         is RepositoryProvenance -> {