Upstream update and shadow plugin configuration (#794)

* Bumping versions and fixing symbols

* Merging services

* Using resolvable configuration

* Renaming upstream distro name & version provider

Author: LikeTheSalad

agent/build.gradle.kts

@@ -1,9 +1,3 @@
-import com.github.jk1.license.filter.LicenseBundleNormalizer
-import com.github.jk1.license.render.InventoryMarkdownReportRenderer
-import java.nio.file.Files
-import java.util.*
-import java.util.stream.Collectors
-
 plugins {
   id("elastic-otel.agent-packaging-conventions")
   id("elastic-otel.sign-and-publish-conventions")
@@ -28,8 +22,8 @@ dependencies {
 
 licenseReport {
   configurations = arrayOf(
-    project.configurations.bootstrapLibs.name,
-    project.configurations.javaagentLibs.name
+    project.configurations.bootstrapLibsClasspath.name,
+    project.configurations.javaagentLibsClasspath.name
     // No need to include the upstreamAgent, because it already has all dependent licenses packaged
     // Those will be preserved in our JAR
   )

buildSrc/src/main/kotlin/elastic-otel.agent-packaging-conventions.gradle.kts

@@ -1,3 +1,4 @@
+import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowCopyAction
 import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar
 import com.github.jengelman.gradle.plugins.shadow.transformers.TransformerContext
 import org.apache.tools.zip.ZipEntry
@@ -20,12 +21,30 @@ plugins {
 // this configuration collects libs that will be placed in the bootstrap classloader
 val bootstrapLibs: Configuration by configurations.creating {
   isCanBeConsumed = false
+  isCanBeResolved = false
 }
 val javaagentLibs: Configuration by configurations.creating {
   isCanBeConsumed = false
+  isCanBeResolved = false
 }
 val upstreamAgent: Configuration by configurations.creating {
   isCanBeConsumed = false
+  isCanBeResolved = false
+}
+val bootstrapLibsClasspath: Configuration by configurations.creating {
+  extendsFrom(bootstrapLibs)
+  isCanBeConsumed = false
+  isCanBeResolved = true
+}
+val javaagentLibsClasspath: Configuration by configurations.creating {
+  extendsFrom(javaagentLibs)
+  isCanBeConsumed = false
+  isCanBeResolved = true
+}
+val upstreamAgentClasspath: Configuration by configurations.creating {
+  extendsFrom(upstreamAgent)
+  isCanBeConsumed = false
+  isCanBeResolved = true
 }
 
 dependencies {
@@ -70,14 +89,15 @@ tasks {
 
   // 1. all distro specific javaagent libs are relocated
   val relocateJavaagentLibs = register<ShadowJar>("relocateJavaagentLibs") {
-    configurations = listOf(javaagentLibs)
+    configurations = listOf(javaagentLibsClasspath)
 
-    duplicatesStrategy = DuplicatesStrategy.FAIL
+    duplicatesStrategy = DuplicatesStrategy.INCLUDE
+    failOnDuplicateEntries = true
 
     archiveFileName.set("javaagentLibs-relocated.jar")
 
     mergeServiceFiles()
-    exclude("**/module-info.class")
+    exclude("**/module-info.class", "META-INF/LICENSE*", "META-INF/NOTICE*")
     relocatePackages(this)
 
     // exclude known bootstrap dependencies - they can't appear in the inst/ directory
@@ -116,7 +136,7 @@ tasks {
   // This transformer injects a new Field into the Opentelemetry SdkSpan class to be used
   // as efficient storage for co.elastic.otel.common.SpanValues
   // Check the FieldBackedSpanValueStorageProvider for details
-  val injectSpanValueFieldTransformer = object: com.github.jengelman.gradle.plugins.shadow.transformers.Transformer {
+  val injectSpanValueFieldTransformer = object: com.github.jengelman.gradle.plugins.shadow.transformers.ResourceTransformer {
 
     @Internal
     val SDK_SPAN_CLASS_FILE = "inst/io/opentelemetry/sdk/trace/SdkSpan.classdata"
@@ -131,15 +151,15 @@ tasks {
     }
 
     override fun canTransformResource(element: FileTreeElement): Boolean {
-      return element.name.equals(SDK_SPAN_CLASS_FILE)
+      return element.path == SDK_SPAN_CLASS_FILE
     }
 
     override fun transform(context: TransformerContext) {
       if(bytecode != null) {
         throw IllegalStateException("Multiple SdkSpan classes detected")
       }
 
-      val inputStream = context.getIs()
+      val inputStream = context.inputStream
       val reader = ClassReader(inputStream)
       val writer = ClassWriter(reader, 0)
       val visitor = object : ClassVisitor(Opcodes.ASM9, writer) {
@@ -168,33 +188,38 @@ tasks {
       }
 
       val entry = ZipEntry(SDK_SPAN_CLASS_FILE)
-      entry.time = TransformerContext.getEntryTimestamp(preserveFileTimestamps, entry.time)
+      entry.time = getEntryTimestamp(preserveFileTimestamps, entry.time)
       os.putNextEntry(entry)
       os.write(bytecode)
     }
 
+    private fun getEntryTimestamp(preserveFileTimestamps: Boolean, entryTime: Long): Long {
+      return if (preserveFileTimestamps) entryTime else ShadowCopyAction.CONSTANT_TIME_FOR_ZIP_ENTRIES
+    }
   }
 
   // 3. the relocated and isolated javaagent libs are merged together with the bootstrap libs (which undergo relocation
   // in this task) and the upstream javaagent jar; duplicates are removed
   shadowJar {
-
-    dependsOn(isolateJavaagentLibs)
-    configurations = listOf(bootstrapLibs, upstreamAgent)
+    configurations = listOf(bootstrapLibsClasspath, upstreamAgentClasspath)
 
     // exclude slf4j-simple from the shadow jar as we use log4j2-slf4j with internal-logging instead
     exclude("inst/io/opentelemetry/javaagent/slf4j/simple/**")
 
-    from(isolateJavaagentLibs.get().outputs)
+    from(isolateJavaagentLibs)
 
     archiveClassifier.set("")
 
-    duplicatesStrategy = DuplicatesStrategy.EXCLUDE
+    duplicatesStrategy = DuplicatesStrategy.INCLUDE
+    failOnDuplicateEntries = true
 
-    mergeServiceFiles {
-      include("inst/META-INF/services/*")
+    mergeServiceFiles{
+      include("inst/META-INF/services/**")
+      path = "inst/META-INF/services"
+    }
+    filesNotMatching("inst/META-INF/services/**") {
+      duplicatesStrategy = DuplicatesStrategy.EXCLUDE
     }
-    exclude("**/module-info.class")
     relocatePackages(this)
     transform(injectSpanValueFieldTransformer)
 

custom/src/main/java/co/elastic/otel/ElasticAutoConfigurationCustomizerProvider.java

@@ -142,7 +142,7 @@ private static void resourceProviders(
 
     // disable upstream distro name & version provider
     disabledResourceProviders.add(
-        "io.opentelemetry.javaagent.tooling.DistroVersionResourceProvider");
+        "io.opentelemetry.javaagent.tooling.resources.DistroResourceProvider");
     config.put(DISABLED_RESOURCE_PROVIDERS, String.join(",", disabledResourceProviders));
   }
 

custom/src/test/java/co/elastic/otel/ElasticAutoConfigurationCustomizerProviderTest.java

@@ -46,7 +46,7 @@ void defaultConfiguration() {
         .describedAs("upstream distro version resource provider must be disabled")
         .containsEntry(
             "otel.java.disabled.resource.providers",
-            "io.opentelemetry.javaagent.tooling.DistroVersionResourceProvider");
+            "io.opentelemetry.javaagent.tooling.resources.DistroResourceProvider");
 
     assertThat(config)
         .describedAs("runtime experimental metrics must be enabled")
@@ -66,7 +66,7 @@ void disableCustomResourceProvider() {
             v ->
                 assertThat(v.split(","))
                     .containsExactly(
-                        "io.opentelemetry.javaagent.tooling.DistroVersionResourceProvider",
+                        "io.opentelemetry.javaagent.tooling.resources.DistroResourceProvider",
                         "my.disabled.provider.Provider"));
   }
 

gradle/libs.versions.toml

@@ -1,5 +1,5 @@
 [versions]
-shadow = "8.3.9"
+shadow = "9.1.0"
 jib = "3.4.5"
 spotless = "7.2.1"
 junit = "5.13.4"
@@ -11,18 +11,18 @@ opentelemetryProto = "1.3.2-alpha"
 
 # otel agent, we rely on the '*-alpha' and get the non-alpha dependencies transitively
 # updated from upstream agent with gradle/update-upstream.sh
-opentelemetryJavaagentAlpha = "2.19.0-alpha"
+opentelemetryJavaagentAlpha = "2.20.0-alpha"
 
 # otel contrib
 # updated from upstream agent with gradle/update-upstream.sh
-opentelemetryContribAlpha = "1.48.0-alpha"
+opentelemetryContribAlpha = "1.49.0-alpha"
 
 # otel semconv
 # updated from upstream agent with gradle/update-upstream.sh
 # While the semconv stable/incubating artifacts are provided as transitive dependencies, keeping
 # an explicit version here allows to easily override to a not-yet-released version.
-opentelemetrySemconv = "1.34.0"
-opentelemetrySemconvAlpha = "1.34.0-alpha"
+opentelemetrySemconv = "1.37.0"
+opentelemetrySemconvAlpha = "1.37.0-alpha"
 
 [libraries]
 
@@ -78,8 +78,8 @@ ant = "org.apache.ant:ant:1.10.15"
 asm = "org.ow2.asm:asm:9.8"
 
 slf4j-api = "org.slf4j:slf4j-api:2.0.17"
-log4j2-slf4j = { group= "org.apache.logging.log4j", name="log4j-slf4j2-impl", version.ref="log4j2"}
-log4j2-core = { group= "org.apache.logging.log4j", name="log4j-core", version.ref="log4j2"}
+log4j2-slf4j = { group = "org.apache.logging.log4j", name = "log4j-slf4j2-impl", version.ref = "log4j2" }
+log4j2-core = { group = "org.apache.logging.log4j", name = "log4j-core", version.ref = "log4j2" }
 
 # Instrumented libraries
 openaiClient = "com.openai:openai-java:2.20.1"