feat(model): Detect and exclude binary license files

Detect and exclude binary license files using Apache Tika. When a
non-text file is found during the license info creation process, a
warning is printed, and it is excluded from the final report.

This prevents the inclusion of binary files that previously caused the
reporter to enter an endless loop during report generation.

Signed-off-by: Julian Olderdissen <[email protected]>

Author: Juli0q

gradle/libs.versions.toml

@@ -67,6 +67,7 @@ slf4j = "2.0.17"
 springCore = "6.2.7"
 svnkit = "1.10.12"
 sw360Client = "17.0.1-m2"
+tika = "3.1.0"
 wagonHttp = "3.5.3"
 wiremock = "3.13.0"
 xmlutil = "0.91.1"
@@ -186,6 +187,7 @@ slf4j = { module = "org.slf4j:slf4j-api ", version.ref = "slf4j" }
 springCore = { module = "org.springframework:spring-core", version.ref = "springCore" }
 svnkit = { module = "com.tmatesoft.svnkit:svnkit", version.ref = "svnkit" }
 sw360Client = { module = "org.eclipse.sw360:client", version.ref = "sw360Client" }
+tika = { module = "org.apache.tika:tika-core", version.ref = "tika" }
 wagon-http = { module = "org.apache.maven.wagon:wagon-http", version.ref = "wagonHttp" }
 wiremock = { module = "org.wiremock:wiremock", version.ref = "wiremock" }
 xz = { module = "org.tukaani:xz", version.ref = "xz" }

model/build.gradle.kts

@@ -42,6 +42,7 @@ dependencies {
     implementation(libs.jackson.module.kotlin)
     implementation(libs.postgres)
     implementation(libs.semver4j)
+    implementation(libs.tika)
 
     testFixturesImplementation(projects.utils.testUtils)
 

model/src/main/kotlin/utils/FileArchiver.kt

@@ -25,6 +25,8 @@ import kotlin.time.measureTime
 import kotlin.time.measureTimedValue
 
 import org.apache.logging.log4j.kotlin.logger
+import org.apache.tika.Tika
+import org.apache.tika.mime.MimeTypes
 
 import org.ossreviewtoolkit.model.KnownProvenance
 import org.ossreviewtoolkit.utils.common.FileMatcher
@@ -72,11 +74,17 @@ class FileArchiver(
         logger.info { "Archiving files matching ${matcher.patterns} from '$directory'..." }
 
         val zipFile = createOrtTempFile(suffix = ".zip")
+        val tika = Tika()
 
         val zipDuration = measureTime {
             directory.packZip(zipFile, overwrite = true) { file ->
                 val relativePath = file.relativeTo(directory).invariantSeparatorsPath
 
+                if (tika.detect(file) != MimeTypes.PLAIN_TEXT) {
+                    logger.warn { "Not adding file '$relativePath' to archive because it is not a text file." }
+                    return@packZip false
+                }
+
                 matcher.matches(relativePath).also { result ->
                     logger.debug {
                         if (result) {

model/src/test/kotlin/utils/FileArchiverTest.kt

@@ -23,8 +23,10 @@ import io.kotest.core.spec.style.StringSpec
 import io.kotest.core.test.TestCase
 import io.kotest.engine.spec.tempdir
 import io.kotest.matchers.file.aFile
+import io.kotest.matchers.file.containFile
 import io.kotest.matchers.file.exist
 import io.kotest.matchers.file.shouldContainNFiles
+import io.kotest.matchers.should
 import io.kotest.matchers.shouldBe
 import io.kotest.matchers.shouldNot
 
@@ -60,10 +62,10 @@ class FileArchiverTest : StringSpec() {
         storage = FileProvenanceFileStorage(LocalFileStorage(storageDir), FileArchiverConfiguration.ARCHIVE_FILENAME)
     }
 
-    private fun createFile(path: String) {
+    private fun createFile(path: String, write: File.() -> Unit = { writeText(path) }) {
         val file = workingDir.resolve(path)
         file.parentFile.safeMkdirs()
-        file.writeText(path)
+        file.write()
     }
 
     /**
@@ -162,5 +164,27 @@ class FileArchiverTest : StringSpec() {
             archiver.unarchive(targetDir, PROVENANCE) shouldBe true
             targetDir shouldContainNFiles 0
         }
+
+        "exclude basic binary license file" {
+            createFile("License") { writeBytes(byteArrayOf(0xFF.toByte(), 0xD8.toByte())) }
+
+            val archiver = FileArchiver.createDefault()
+            archiver.archive(workingDir, PROVENANCE)
+            val result = archiver.unarchive(targetDir, PROVENANCE)
+
+            result shouldBe true
+            targetDir shouldNot containFile("License")
+        }
+
+        "include utf8 file with japanese chars" {
+            createFile("License") { writeText("ぁあぃいぅうぇえぉおかが") }
+
+            val archiver = FileArchiver.createDefault()
+            archiver.archive(workingDir, PROVENANCE)
+            val result = archiver.unarchive(targetDir, PROVENANCE)
+
+            result shouldBe true
+            targetDir should containFile("License")
+        }
     }
 }