Refactor file visiting logic in StreamAction (#1233)

* Remove ShadowJar.rootPatternSet

Seems it's unused now.

* Test `duplicatesStrategyFailShouldFailBuildForDuplicatedEntries`

* Convert the resolution result to ZipTree

* Cleanups

* Reduce RelativeArchivePath and ArchiveFileTreeElement

* Defer includedZipTrees

* Fix `failBuildIfProcessingBadJar`

* Cleanups

* Remove recordVisit

* Test more strategies in ServiceFileTransformerTest

* Test `shadowJarIsCachedCorrectlyAfterDuplicatesStrategyChanged`

* Update baseline

* Refactor and simplify visitFile

* Show how to add extra files into shadowed jars

* Unify from and into orders

* Update changelog

* Remove TODO for `injectMultiReleaseAttrIfPresent`

* Fix merge

* Merge `RealStreamAction` back into `ShadowCopyAction`

* Cleanups

* Adjust ShadowCopyAction ctor

* Seems we have no need to close context.inputStream in transformers

* Fix `canAddExtraFilesIntoShadowJar`

* Mark `withDebug` for testing

* Revert "Mark `withDebug` for testing"

This reverts commit ec3df52.

* Cleanups

* Drop `ZipCompressor`

* Drop `rootPatternSet` property

* Shorten obtaining relative paths

> Returns the path of this file, relative to the root of the containing file tree. Always uses '/' as the hierarchy
  separator, regardless of platform file separator. Same as calling getRelativePath().getPathString().

* Fix merge

* Fix merge

* Cleanups

* Add dir entries in the last

Author: Goooler

api/shadow.api

@@ -64,10 +64,6 @@ public abstract class com/github/jengelman/gradle/plugins/shadow/ShadowPlugin :
 	public fun apply (Lorg/gradle/api/Project;)V
 }
 
-public abstract interface class com/github/jengelman/gradle/plugins/shadow/internal/ZipCompressor {
-	public abstract fun createArchiveOutputStream (Ljava/io/File;)Lorg/apache/tools/zip/ZipOutputStream;
-}
-
 public abstract class com/github/jengelman/gradle/plugins/shadow/legacy/LegacyShadowPlugin : org/gradle/api/Plugin {
 	public fun <init> ()V
 	public synthetic fun apply (Ljava/lang/Object;)V
@@ -175,45 +171,14 @@ public abstract interface class com/github/jengelman/gradle/plugins/shadow/tasks
 
 public class com/github/jengelman/gradle/plugins/shadow/tasks/ShadowCopyAction : org/gradle/api/internal/file/copy/CopyAction {
 	public static final field Companion Lcom/github/jengelman/gradle/plugins/shadow/tasks/ShadowCopyAction$Companion;
-	public fun <init> (Ljava/io/File;Lcom/github/jengelman/gradle/plugins/shadow/internal/ZipCompressor;Lorg/gradle/api/internal/DocumentationRegistry;Ljava/lang/String;Ljava/util/Set;Ljava/util/Set;Lorg/gradle/api/tasks/util/PatternSet;ZLjava/util/Set;)V
+	public fun <init> (Ljava/io/File;Lkotlin/jvm/functions/Function1;Lorg/gradle/api/internal/DocumentationRegistry;Ljava/util/Set;Ljava/util/Set;Ljava/util/Set;ZLjava/lang/String;)V
 	public fun execute (Lorg/gradle/api/internal/file/copy/CopyActionProcessingStream;)Lorg/gradle/api/tasks/WorkResult;
 }
 
-public class com/github/jengelman/gradle/plugins/shadow/tasks/ShadowCopyAction$ArchiveFileTreeElement : org/gradle/api/file/FileTreeElement {
-	public fun <init> (Lcom/github/jengelman/gradle/plugins/shadow/tasks/ShadowCopyAction$RelativeArchivePath;)V
-	public fun asFileTreeElement ()Lorg/gradle/api/file/FileTreeElement;
-	public fun copyTo (Ljava/io/File;)Z
-	public fun copyTo (Ljava/io/OutputStream;)V
-	public fun getFile ()Ljava/io/File;
-	public fun getLastModified ()J
-	public fun getMode ()I
-	public fun getName ()Ljava/lang/String;
-	public fun getPath ()Ljava/lang/String;
-	public fun getPermissions ()Lorg/gradle/api/file/FilePermissions;
-	public fun getRelativePath ()Lcom/github/jengelman/gradle/plugins/shadow/tasks/ShadowCopyAction$RelativeArchivePath;
-	public synthetic fun getRelativePath ()Lorg/gradle/api/file/RelativePath;
-	public fun getSize ()J
-	public fun isClass ()Z
-	public fun isDirectory ()Z
-	public fun open ()Ljava/io/InputStream;
-}
-
 public final class com/github/jengelman/gradle/plugins/shadow/tasks/ShadowCopyAction$Companion {
 	public final fun getCONSTANT_TIME_FOR_ZIP_ENTRIES ()J
 }
 
-public class com/github/jengelman/gradle/plugins/shadow/tasks/ShadowCopyAction$RelativeArchivePath : org/gradle/api/file/RelativePath {
-	public fun <init> (Lorg/apache/tools/zip/ZipEntry;Z)V
-	public final fun charAt (I)C
-	public fun get (I)C
-	public fun getEntry ()Lorg/apache/tools/zip/ZipEntry;
-	public fun getLength ()I
-	public fun getParent ()Lcom/github/jengelman/gradle/plugins/shadow/tasks/ShadowCopyAction$RelativeArchivePath;
-	public synthetic fun getParent ()Lorg/gradle/api/file/RelativePath;
-	public fun isClass ()Z
-	public final fun length ()I
-}
-
 public abstract class com/github/jengelman/gradle/plugins/shadow/tasks/ShadowJar : org/gradle/api/tasks/bundling/Jar, com/github/jengelman/gradle/plugins/shadow/tasks/ShadowSpec {
 	public fun <init> ()V
 	public fun append (Ljava/lang/String;)Lcom/github/jengelman/gradle/plugins/shadow/tasks/ShadowJar;
@@ -231,13 +196,11 @@ public abstract class com/github/jengelman/gradle/plugins/shadow/tasks/ShadowJar
 	public fun getExcludes ()Ljava/util/Set;
 	public fun getIncludedDependencies ()Lorg/gradle/api/file/ConfigurableFileCollection;
 	public fun getIncludes ()Ljava/util/Set;
-	protected fun getInternalCompressor ()Lcom/github/jengelman/gradle/plugins/shadow/internal/ZipCompressor;
 	public fun getManifest ()Lcom/github/jengelman/gradle/plugins/shadow/tasks/InheritManifest;
 	public synthetic fun getManifest ()Lorg/gradle/api/java/archives/Manifest;
 	public fun getMinimizeJar ()Lorg/gradle/api/provider/Property;
 	public fun getRelocationPrefix ()Lorg/gradle/api/provider/Property;
 	public fun getRelocators ()Lorg/gradle/api/provider/SetProperty;
-	protected fun getRootPatternSet ()Lorg/gradle/api/tasks/util/PatternSet;
 	public fun getSourceSetsClassesDirs ()Lorg/gradle/api/file/ConfigurableFileCollection;
 	public fun getToMinimize ()Lorg/gradle/api/file/ConfigurableFileCollection;
 	public fun getTransformers ()Lorg/gradle/api/provider/SetProperty;

lint-baseline.xml

@@ -41,7 +41,7 @@
         errorLine2="~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~">
         <location
             file="src/main/kotlin/com/github/jengelman/gradle/plugins/shadow/tasks/ShadowCopyAction.kt"
-            line="26"
+            line="17"
             column="1"/>
     </issue>
 
@@ -52,18 +52,7 @@
         errorLine2="~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~">
         <location
             file="src/main/kotlin/com/github/jengelman/gradle/plugins/shadow/tasks/ShadowCopyAction.kt"
-            line="27"
-            column="1"/>
-    </issue>
-
-    <issue
-        id="InternalGradleApiUsage"
-        message="Avoid using internal Gradle APIs"
-        errorLine1="import org.gradle.api.internal.file.DefaultFilePermissions"
-        errorLine2="~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~">
-        <location
-            file="src/main/kotlin/com/github/jengelman/gradle/plugins/shadow/tasks/ShadowCopyAction.kt"
-            line="28"
+            line="18"
             column="1"/>
     </issue>
 
@@ -74,7 +63,7 @@
         errorLine2="~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~">
         <location
             file="src/main/kotlin/com/github/jengelman/gradle/plugins/shadow/tasks/ShadowCopyAction.kt"
-            line="29"
+            line="19"
             column="1"/>
     </issue>
 
@@ -85,7 +74,7 @@
         errorLine2="~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~">
         <location
             file="src/main/kotlin/com/github/jengelman/gradle/plugins/shadow/tasks/ShadowCopyAction.kt"
-            line="30"
+            line="20"
             column="1"/>
     </issue>
 
@@ -96,7 +85,7 @@
         errorLine2="~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~">
         <location
             file="src/main/kotlin/com/github/jengelman/gradle/plugins/shadow/tasks/ShadowCopyAction.kt"
-            line="31"
+            line="21"
             column="1"/>
     </issue>
 
@@ -107,7 +96,7 @@
         errorLine2="~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~">
         <location
             file="src/main/kotlin/com/github/jengelman/gradle/plugins/shadow/tasks/ShadowJar.kt"
-            line="34"
+            line="33"
             column="1"/>
     </issue>
 
@@ -118,7 +107,7 @@
         errorLine2="~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~">
         <location
             file="src/main/kotlin/com/github/jengelman/gradle/plugins/shadow/tasks/ShadowJar.kt"
-            line="35"
+            line="34"
             column="1"/>
     </issue>
 
@@ -129,62 +118,7 @@
         errorLine2="~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~">
         <location
             file="src/main/kotlin/com/github/jengelman/gradle/plugins/shadow/tasks/ShadowJar.kt"
-            line="36"
-            column="1"/>
-    </issue>
-
-    <issue
-        id="InternalGradleApiUsage"
-        message="Avoid using internal Gradle APIs"
-        errorLine1="import org.gradle.api.internal.file.copy.DefaultCopySpec"
-        errorLine2="~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~">
-        <location
-            file="src/main/kotlin/com/github/jengelman/gradle/plugins/shadow/tasks/ShadowJar.kt"
-            line="37"
-            column="1"/>
-    </issue>
-
-    <issue
-        id="InternalGradleApiUsage"
-        message="Avoid using internal Gradle APIs"
-        errorLine1="import org.gradle.api.internal.file.DefaultFileTreeElement"
-        errorLine2="~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~">
-        <location
-            file="src/main/kotlin/com/github/jengelman/gradle/plugins/shadow/internal/Utils.kt"
-            line="16"
-            column="1"/>
-    </issue>
-
-    <issue
-        id="InternalGradleApiUsage"
-        message="Avoid using internal Gradle APIs"
-        errorLine1="import org.gradle.internal.file.Chmod"
-        errorLine2="~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~">
-        <location
-            file="src/main/kotlin/com/github/jengelman/gradle/plugins/shadow/internal/Utils.kt"
-            line="17"
-            column="1"/>
-    </issue>
-
-    <issue
-        id="InternalGradleApiUsage"
-        message="Avoid using internal Gradle APIs"
-        errorLine1="import org.gradle.internal.file.FileMetadata"
-        errorLine2="~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~">
-        <location
-            file="src/main/kotlin/com/github/jengelman/gradle/plugins/shadow/internal/Utils.kt"
-            line="18"
-            column="1"/>
-    </issue>
-
-    <issue
-        id="InternalGradleApiUsage"
-        message="Avoid using internal Gradle APIs"
-        errorLine1="import org.gradle.internal.file.Stat"
-        errorLine2="~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~">
-        <location
-            file="src/main/kotlin/com/github/jengelman/gradle/plugins/shadow/internal/Utils.kt"
-            line="19"
+            line="35"
             column="1"/>
     </issue>
 

src/docs/changes/README.md

@@ -12,11 +12,18 @@
 - **BREAKING CHANGE:** Move tracking unused classes logic out of `ShadowCopyAction`. ([#1257](https://github.com/GradleUp/shadow/pull/1257))
 - Reduce duplicated `SimpleRelocator` to improve performance. ([#1271](https://github.com/GradleUp/shadow/pull/1271))
 - **BREAKING CHANGE:** Move `DependencyFilter` from `com.github.jengelman.gradle.plugins.shadow.internal` into `com.github.jengelman.gradle.plugins.shadow.tasks`. ([#1272](https://github.com/GradleUp/shadow/pull/1272))
+- Handle file unzipping via `Project.zipTree`. ([#1233](https://github.com/GradleUp/shadow/pull/1233))
+
+**Fixed**
+
+- Honor `DuplicatesStrategy`. ([#1233](https://github.com/GradleUp/shadow/pull/1233))
+- Honor unzipped jars via `from`. ([#1233](https://github.com/GradleUp/shadow/pull/1233))
 
 **Removed**
 
 - **BREAKING CHANGE:** Remove `BaseStreamAction`. ([#1258](https://github.com/GradleUp/shadow/pull/1258))
 - **BREAKING CHANGE:** Remove `ShadowStats`. ([#1264](https://github.com/GradleUp/shadow/pull/1264))
+- **BREAKING CHANGE:** Remove `ShadowCopyAction.ArchiveFileTreeElement` and `RelativeArchivePath`. ([#1233](https://github.com/GradleUp/shadow/pull/1233))
 
 
 ## [v9.0.0-beta8] (2025-02-08)

src/docs/configuration/README.md

@@ -99,3 +99,18 @@ tasks.named('shadowJar', com.github.jengelman.gradle.plugins.shadow.tasks.Shadow
   manifest.inheritFrom(testJar.get().manifest)
 }
 ```
+
+## Adding Extra Files
+
+The `shadowJar` task is a subclass of the `Jar` task, which means that the 
+[from](https://docs.gradle.org/current/dsl/org.gradle.jvm.tasks.Jar.html#org.gradle.jvm.tasks.Jar:from(java.lang.Object,%20groovy.lang.Closure)) 
+method can be used to add extra files.
+
+```groovy
+tasks.named('shadowJar', com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar) {
+  from('extra.jar') {
+    // Copy the contents of the extra.jar file into META-INF/ in the shadowed JAR.
+    into('META-INF')
+  }
+}
+```

src/functionalTest/kotlin/com/github/jengelman/gradle/plugins/shadow/JavaPluginTest.kt

@@ -3,6 +3,7 @@ package com.github.jengelman.gradle.plugins.shadow
 import assertk.all
 import assertk.assertThat
 import assertk.assertions.contains
+import assertk.assertions.containsMatch
 import assertk.assertions.isEqualTo
 import assertk.assertions.isGreaterThan
 import assertk.assertions.isNotEmpty
@@ -20,8 +21,11 @@ import com.github.jengelman.gradle.plugins.shadow.util.Issue
 import com.github.jengelman.gradle.plugins.shadow.util.containsEntries
 import com.github.jengelman.gradle.plugins.shadow.util.doesNotContainEntries
 import com.github.jengelman.gradle.plugins.shadow.util.getMainAttr
+import com.github.jengelman.gradle.plugins.shadow.util.getStream
 import com.github.jengelman.gradle.plugins.shadow.util.runProcess
 import kotlin.io.path.appendText
+import kotlin.io.path.name
+import kotlin.io.path.outputStream
 import kotlin.io.path.writeText
 import org.gradle.api.plugins.JavaPlugin
 import org.gradle.testfixtures.ProjectBuilder
@@ -595,9 +599,7 @@ class JavaPluginTest : BasePluginTest() {
 
     assertThat(result).all {
       taskOutcomeEquals(shadowJarTask, FAILED)
-      transform { it.output }.contains(
-        "java.util.zip.ZipException: archive is not a ZIP archive",
-      )
+      transform { it.output }.containsMatch("Cannot expand ZIP '.*bad\\.jar'".toRegex())
     }
   }
 
@@ -653,4 +655,38 @@ class JavaPluginTest : BasePluginTest() {
       getMainAttr("Bar-Attr").isEqualTo("Bar-Value")
     }
   }
+
+  @Test
+  fun canAddExtraFilesIntoShadowJar() {
+    writeMainClass()
+    projectScriptPath.appendText(
+      """
+        $shadowJar {
+          from(files('${artifactAJar.toUri().toURL().path}')) {
+            into('META-INF')
+          }
+        }
+      """.trimIndent(),
+    )
+
+    run(shadowJarTask)
+
+    assertThat(outputShadowJar).useAll {
+      containsEntries(
+        "my/Main.class",
+        "META-INF/a-1.0.jar",
+      )
+      doesNotContainEntries(*entriesInA)
+    }
+    val unzipped = path("unzipped")
+    outputShadowJar.use {
+      it.getStream("META-INF/a-1.0.jar").use { inputStream ->
+        inputStream.copyTo(unzipped.outputStream())
+      }
+    }
+    assertThat(jarPath(unzipped.name)).useAll {
+      containsEntries(*entriesInA)
+      doesNotContainEntries("my/Main.class")
+    }
+  }
 }

src/functionalTest/kotlin/com/github/jengelman/gradle/plugins/shadow/caching/ShadowJarCachingTest.kt

@@ -8,6 +8,7 @@ import com.github.jengelman.gradle.plugins.shadow.util.doesNotContainEntries
 import kotlin.io.path.appendText
 import kotlin.io.path.readText
 import kotlin.io.path.writeText
+import org.gradle.api.file.DuplicatesStrategy
 import org.junit.jupiter.api.Test
 
 class ShadowJarCachingTest : BaseCachingTest() {
@@ -210,4 +211,23 @@ class ShadowJarCachingTest : BaseCachingTest() {
 
     assertions()
   }
+
+  @Test
+  fun shadowJarIsCachedCorrectlyAfterDuplicatesStrategyChanged() {
+    listOf(
+      DuplicatesStrategy.EXCLUDE,
+      DuplicatesStrategy.INCLUDE,
+      DuplicatesStrategy.WARN,
+    ).forEach { strategy ->
+      projectScriptPath.appendText(
+        """
+          $shadowJar {
+            duplicatesStrategy = DuplicatesStrategy.$strategy
+          }
+        """.trimIndent() + System.lineSeparator(),
+      )
+
+      assertCompositeExecutions()
+    }
+  }
 }