Instrument Mulesoft 4.5.0+ (#7981)

* Instrument mulesoft 4.5.0+

* add muzzle checks

* fix helper classes

* Enable grizzly when mule is enabled

* Test interactions with otel exporter

* Limit 4.5.0 tests to <=11

* add lockfile

* test with latest 4

* codenarc

* enforce logback-core

* Separate domain dir per mule version

* Clean up build

* exclude old pull parser lib

* clean latestDep

* Get doc name to enrich resource name

* change parent-child propagatoin

* fix span ordering in tests

* Refactor

* fix thread propagation on late scheduled stuff

* fix comparator

* cleanup

* review

Author: amarziali

dd-java-agent/instrumentation/grizzly-client-1.9/src/main/java/datadog/trace/instrumentation/grizzly/client/AsyncHttpClientInstrumentation.java

@@ -15,8 +15,10 @@
 import com.ning.http.client.Request;
 import datadog.trace.agent.tooling.Instrumenter;
 import datadog.trace.agent.tooling.InstrumenterModule;
+import datadog.trace.api.InstrumenterConfig;
 import datadog.trace.bootstrap.instrumentation.api.AgentSpan;
 import datadog.trace.bootstrap.instrumentation.decorator.HttpClientDecorator;
+import java.util.Collections;
 import net.bytebuddy.asm.Advice;
 
 @AutoService(InstrumenterModule.class)
@@ -29,7 +31,7 @@ public AsyncHttpClientInstrumentation() {
 
   @Override
   protected boolean defaultEnabled() {
-    return false;
+    return InstrumenterConfig.get().isIntegrationEnabled(Collections.singleton("mule"), false);
   }
 
   @Override

dd-java-agent/instrumentation/grizzly-http-2.3.20/src/main/java/datadog/trace/instrumentation/grizzlyhttp232/DefaultFilterChainInstrumentation.java

@@ -8,6 +8,8 @@
 import com.google.auto.service.AutoService;
 import datadog.trace.agent.tooling.Instrumenter;
 import datadog.trace.agent.tooling.InstrumenterModule;
+import datadog.trace.api.InstrumenterConfig;
+import java.util.Collections;
 
 @AutoService(InstrumenterModule.class)
 public class DefaultFilterChainInstrumentation extends InstrumenterModule.Tracing
@@ -37,7 +39,7 @@ public String[] helperClassNames() {
 
   @Override
   protected boolean defaultEnabled() {
-    return false;
+    return InstrumenterConfig.get().isIntegrationEnabled(Collections.singleton("mule"), false);
   }
 
   @Override

dd-java-agent/instrumentation/grizzly-http-2.3.20/src/main/java/datadog/trace/instrumentation/grizzlyhttp232/HttpCodecFilterInstrumentation.java

@@ -7,6 +7,8 @@
 import com.google.auto.service.AutoService;
 import datadog.trace.agent.tooling.Instrumenter;
 import datadog.trace.agent.tooling.InstrumenterModule;
+import datadog.trace.api.InstrumenterConfig;
+import java.util.Collections;
 
 @AutoService(InstrumenterModule.class)
 public final class HttpCodecFilterInstrumentation extends InstrumenterModule.Tracing
@@ -23,7 +25,7 @@ public String instrumentedType() {
 
   @Override
   protected boolean defaultEnabled() {
-    return false;
+    return InstrumenterConfig.get().isIntegrationEnabled(Collections.singleton("mule"), false);
   }
 
   @Override

dd-java-agent/instrumentation/grizzly-http-2.3.20/src/main/java/datadog/trace/instrumentation/grizzlyhttp232/HttpServerFilterInstrumentation.java

@@ -7,6 +7,8 @@
 import com.google.auto.service.AutoService;
 import datadog.trace.agent.tooling.Instrumenter;
 import datadog.trace.agent.tooling.InstrumenterModule;
+import datadog.trace.api.InstrumenterConfig;
+import java.util.Collections;
 
 @AutoService(InstrumenterModule.class)
 public class HttpServerFilterInstrumentation extends InstrumenterModule.Tracing
@@ -23,7 +25,7 @@ public String instrumentedType() {
 
   @Override
   protected boolean defaultEnabled() {
-    return false;
+    return InstrumenterConfig.get().isIntegrationEnabled(Collections.singleton("mule"), false);
   }
 
   @Override

dd-java-agent/instrumentation/mule-4/application/pom.xml

@@ -6,8 +6,8 @@
   <properties>
     <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
     <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
-    <app.runtime>4.2.2-20210129</app.runtime>
-    <mule.maven.plugin.version>3.4.2</mule.maven.plugin.version>
+    <app.runtime>4.5.0</app.runtime>
+    <mule.maven.plugin.version>3.8.6</mule.maven.plugin.version>
     <datadog.builddir>${project.basedir}</datadog.builddir>
     <datadog.name>mule-test-application</datadog.name>
     <datadog.version>1.0.0-SNAPSHOT</datadog.version>
@@ -57,19 +57,19 @@
     <dependency>
       <groupId>org.mule.connectors</groupId>
       <artifactId>mule-http-connector</artifactId>
-      <version>1.5.23</version>
+      <version>1.9.2</version>
       <classifier>mule-plugin</classifier>
     </dependency>
     <dependency>
       <groupId>org.mule.connectors</groupId>
       <artifactId>mule-sockets-connector</artifactId>
-      <version>1.2.0</version>
+      <version>1.2.4</version>
       <classifier>mule-plugin</classifier>
     </dependency>
     <dependency>
       <groupId>org.mule.module</groupId>
       <artifactId>mule-java-module</artifactId>
-      <version>1.2.7</version>
+      <version>1.2.13</version>
       <classifier>mule-plugin</classifier>
     </dependency>
   </dependencies>

dd-java-agent/instrumentation/mule-4/build.gradle

@@ -1,111 +1,199 @@
 ext {
-  // TODO Java 17: Mule 4 doesn't support Java 17
-  maxJavaVersionForTests = JavaVersion.VERSION_15
+  forkedTestMaxJavaVersionForTests = JavaVersion.VERSION_11
+  mule46ForkedTestMaxJavaVersionForTests = JavaVersion.VERSION_11
+  latestDepForkedTestMinJavaVersionForTests = JavaVersion.VERSION_17
+  latestDepForkedTestMaxJavaVersionForTests = JavaVersion.VERSION_17
 }
+def muleVersion = '4.5.0'
+def appDir = "$projectDir/application"
 
 muzzle {
-  extraRepository('anypoint-releases', 'https://maven.anypoint.mulesoft.com/api/v2/maven')
   extraRepository('mulesoft-releases', 'https://repository.mulesoft.org/releases')
   extraRepository('mulesoft-public', 'https://repository.mulesoft.org/nexus/content/repositories/public')
 
   pass {
     group = 'org.mule.runtime'
     module = 'mule-core'
-    versions = '[,]'
-    assertInverse = true
-
-    excludeDependency 'com.google.code.findbugs:jsr305' // avoid dependency issue with mule-4.5.0
+    versions = "[$muleVersion,)"
+    javaVersion = "17"
+    excludeDependency 'om.google.guava:guava'
+    excludeDependency 'com.google.code.findbugs:jsr305'
+    additionalDependencies +="org.mule.runtime:mule-tracer-customization-impl:$muleVersion"
   }
-
-  fail {
-    group = 'org.mule'
+  pass {
+    group = 'org.mule.runtime'
+    module = 'mule-tracer-customization-impl'
+    versions = "[$muleVersion,)"
+    javaVersion = "17"
+    excludeDependency 'om.google.guava:guava'
+    excludeDependency 'com.google.code.findbugs:jsr305'
+    additionalDependencies +="org.mule.runtime:mule-core:$muleVersion"
+  }
+  pass {
+    name = 'before-4.5.0'
+    group = 'org.mule.runtime'
     module = 'mule-core'
-    versions = '[3.2.0,)' // First version where muzzle can resolve all dependencies
-    skipVersions += "3.9.5" // bad release
+    versions = "[$muleVersion,)"
+    javaVersion = "17"
+    assertInverse true
+    excludeDependency 'om.google.guava:guava'
+    excludeDependency 'com.google.code.findbugs:jsr305'
+    additionalDependencies +="org.mule.runtime:mule-tracer-customization-impl:$muleVersion"
   }
 }
 
 apply from: "$rootDir/gradle/java.gradle"
 
-def muleVersion = '4.2.2'
-def muleBaseDir = "$buildDir/mule"
-def appDir = "$projectDir/application"
-def generatedResourcesDir = "$buildDir/generated-resources/test"
-
-clean.doFirst {
-  delete muleBaseDir
-}
+addTestSuiteForDir('mule46ForkedTest', 'test')
+addTestSuiteForDir('latestDepForkedTest', 'test')
 
 repositories {
-  maven {
-    name "anypoint-releases"
-    url "https://maven.anypoint.mulesoft.com/api/v2/maven"
-  }
   maven {
     name "mulesoft-releases"
     url "https://repository.mulesoft.org/releases"
+    mavenContent {
+      releasesOnly()
+    }
   }
   maven {
     name "mulesoft-public"
     url "https://repository.mulesoft.org/nexus/content/repositories/public"
+    mavenContent {
+      releasesOnly()
+    }
   }
 }
 
 configurations {
   muleServices
+  mule46Services
+  latestMuleServices
 }
 
 configurations.all {
-  // don't allow mule to do slf4j over log4j
-  exclude group: 'org.apache.logging.log4j', module: 'log4j-slf4j-impl'
+  exclude group: 'pull-parser', module: 'pull-parser'
+
+  resolutionStrategy {
+    // avoid bringing in logback based on java 11
+    force libs.logback.classic
+    force libs.logback.core
+  }
 }
 
 sourceSets {
   test {
-    output.dir(generatedResourcesDir, builtBy: 'generateAppResources')
+    output.dir("$buildDir/generated-resources/test", builtBy: 'generateAppResources')
+  }
+  mule46ForkedTest {
+    output.dir("$buildDir/generated-resources/mule46ForkedTest", builtBy: 'generateAppResources46')
+  }
+  latestDepForkedTest {
+    output.dir("$buildDir/generated-resources/latestDepForkedTest", builtBy: 'generateAppResourcesLatest')
   }
 }
 
 tasks.named("compileTestGroovy").configure {
   dependsOn 'mvnPackage', 'extractMuleServices'
 }
 
+tasks.named("compileMule46ForkedTestGroovy").configure {
+  dependsOn 'mvnPackage', 'extractMule46Services'
+}
+
+tasks.named("compileLatestDepForkedTestGroovy").configure {
+  dependsOn 'mvnPackage', 'extractLatestMuleServices'
+  setJavaVersion(it, 17)
+}
+
+tasks.named("compileLatestDepForkedTestJava").configure {
+  setJavaVersion(it, 17)
+}
+
 dependencies {
   compileOnly group: 'org.mule.runtime', name: 'mule-core', version: muleVersion
-  testImplementation project(':dd-java-agent:instrumentation:aws-common')
+  compileOnly group: 'org.mule.runtime', name: 'mule-tracer-customization-impl', version: muleVersion
 
+  testImplementation project(':dd-java-agent:instrumentation:aws-common')
   testImplementation project(':dd-java-agent:instrumentation:reactor-core-3.1')
   testImplementation project(':dd-java-agent:instrumentation:reactive-streams')
+  testImplementation project(':dd-java-agent:instrumentation:grizzly-2')
   testImplementation project(':dd-java-agent:instrumentation:grizzly-http-2.3.20')
   testImplementation project(':dd-java-agent:instrumentation:grizzly-client-1.9')
   testImplementation project(':dd-java-agent:instrumentation:caffeine')
+  testImplementation project(':dd-java-agent:instrumentation:quartz-2')
+
 
   testImplementation group: 'org.mule.runtime', name: 'mule-module-launcher', version: muleVersion
   testImplementation group: 'org.mule.runtime', name: 'mule-core', version: muleVersion
-  testImplementation group: 'org.mule.runtime', name: 'mule-module-extensions-spring-support', version: muleVersion
+  //testImplementation group: 'org.mule.runtime', name: 'mule-module-extensions-spring-support', version: muleVersion
   testImplementation group: 'org.mule.runtime', name: 'mule-module-service', version: muleVersion
   // this is needed for mule to find removed javax.* APIs on Java 11+
   testImplementation group: 'org.mule.runtime', name: 'mule-module-javaee', version: muleVersion
-
-  muleServices group: 'org.mule.services', name: 'mule-service-http', version: '1.4.7', classifier: 'mule-service'
-  muleServices group: 'org.mule.services', name: 'mule-service-scheduler', version: '1.2.4', classifier: 'mule-service'
-  muleServices group: 'org.mule.services', name: 'mule-service-weave', version: '2.2.2', classifier: 'mule-service'
+  testImplementation group: 'org.mule.runtime', name: 'mule-dwb-api', version: '2.5.0'
+  // testImplementation group: 'xml-apis', name: 'xml-apis', version: '1.4.01'
+  muleServices group: 'org.mule.services', name: 'mule-service-http', version: '1.5.21', classifier: 'mule-service'
+  muleServices group: 'org.mule.services', name: 'mule-service-scheduler', version: '1.5.0', classifier: 'mule-service'
+  muleServices group: 'org.mule.services', name: 'mule-service-weave', version: '2.5.0', classifier: 'mule-service'
+
+  mule46ForkedTestImplementation group: 'org.mule.runtime', name: 'mule-module-launcher', version: '4.6.0'
+  mule46ForkedTestImplementation group: 'org.mule.runtime', name: 'mule-core', version: '4.6.0'
+  mule46ForkedTestImplementation group: 'org.mule.runtime', name: 'mule-module-extensions-spring-support', version: '4.6.0'
+  mule46ForkedTestImplementation group: 'org.mule.runtime', name: 'mule-module-service', version: '4.6.0'
+  // this is needed for mule to find removed javax.* APIs on Java 11+
+  mule46ForkedTestImplementation group: 'org.mule.runtime', name: 'mule-module-javaee', version: '4.6.0'
+  mule46Services group: 'org.mule.services', name: 'mule-service-http', version: '1.5.21', classifier: 'mule-service'
+  mule46Services group: 'org.mule.services', name: 'mule-service-scheduler', version: '1.5.0', classifier: 'mule-service'
+  mule46Services group: 'org.mule.services', name: 'mule-service-weave', version: '2.5.0', classifier: 'mule-service'
+
+  latestDepForkedTestImplementation group: 'org.mule.runtime', name: 'mule-module-launcher', version: '4.8.+'
+  latestDepForkedTestImplementation group: 'org.mule.runtime', name: 'mule-core', version: '4.8.+'
+  latestDepForkedTestImplementation group: 'org.mule.runtime', name: 'mule-module-extensions-spring-support', version: '4.8.+'
+  latestDepForkedTestImplementation group: 'org.mule.runtime', name: 'mule-module-service', version: '4.8.+'
+  latestDepForkedTestImplementation group: 'org.mule.runtime', name: 'mule-module-javaee', version: '4.8.+'
+
+
+  //TODO: 4.9.0 is not yet out but it looks like using 4.+ instead of above 4.8.+ brings in 4.9.0-SNAPSHOT artifacts.
+  /*
+   When testing with them, the mule container does not bootstrap because of:
+   None of [protected void java.lang.Object.finalize() throws java.lang.Throwable, public final void java.lang.Object.wait(long,int) throws java.lang.InterruptedException, public final void java.lang.Object.wait() throws java.lang.InterruptedException, public final native void java.lang.Object.wait(long) throws java.lang.InterruptedException, public boolean java.lang.Object.equals(java.lang.Object), public java.lang.String java.lang.Object.toString(), public native int java.lang.Object.hashCode(), public final native java.lang.Class java.lang.Object.getClass(), protected native java.lang.Object java.lang.Object.clone() throws java.lang.CloneNotSupportedException, public final native void java.lang.Object.notify(), public final native void java.lang.Object.notifyAll(), public abstract java.lang.Object java.lang.reflect.InvocationHandler.invoke(java.lang.Object,java.lang.reflect.Method,java.lang.Object[]) throws java.lang.Throwable] allows for delegation from public boolean org.springframework.beans.factory.SmartFactoryBean.isEagerInit()
+   */
+  latestDepForkedTestImplementation group: 'org.mule.runtime', name: 'mule-extensions-soap-api', version: '1.8.+'
+
+  latestDepForkedTestImplementation group: 'jakarta.xml.ws', name: 'jakarta.xml.ws-api', version: '4.0.2'
+  // back to the future
+  latestDepForkedTestImplementation group: 'javax.xml.bind', name: 'jaxb-api', version: '2.3.0'
+  latestDepForkedTestImplementation group: 'com.sun.xml.bind', name: 'jaxb-impl', version: '2.3.0'
+  latestDepForkedTestImplementation group: 'com.sun.xml.bind', name: 'jaxb-core', version: '2.3.0'
+
+  latestMuleServices group: 'org.mule.services', name: 'mule-service-http', version: '1.+', classifier: 'mule-service'
+  latestMuleServices group: 'org.mule.services', name: 'mule-service-scheduler', version: '1.+', classifier: 'mule-service'
+  latestMuleServices group: 'org.mule.services', name: 'mule-service-weave', version: '2.8.1', classifier: 'mule-service'
 }
 
-// extract the enabled services into the mule base directory
-tasks.register('extractMuleServices', Sync) {
-  dependsOn configurations.muleServices
-
-  configurations.muleServices.resolvedConfiguration.resolvedArtifacts.findAll {
+def copyMuleArtifacts(String configName, String muleBaseDir, Sync sync) {
+  configurations[configName].resolvedConfiguration.resolvedArtifacts.findAll {
     it.classifier == "mule-service"
   } collect { artifact ->
     def id = artifact.moduleVersion.id
-    from(zipTree(artifact.file)) {
+    sync.from(zipTree(artifact.file)) {
       into("${id.name}-${id.version}")
     }
   }
+  sync.into "$muleBaseDir/services"
+}
 
-  into "$muleBaseDir/services"
+// extract the enabled services into the mule base directory
+tasks.register('extractMuleServices', Sync) {
+  dependsOn configurations.muleServices
+  copyMuleArtifacts("muleServices", "$buildDir/mule/test", it)
+}
+tasks.register('extractMule46Services', Sync) {
+  dependsOn configurations.mule46Services
+  copyMuleArtifacts("mule46Services", "$buildDir/mule/mule46ForkedTest", it)
+}
+tasks.register('extractLatestMuleServices', Sync) {
+  dependsOn configurations.latestMuleServices
+  copyMuleArtifacts("latestMuleServices", "$buildDir/mule/latestDepForkedTest", it)
 }
 
 // build the mule application via maven
@@ -118,23 +206,29 @@ tasks.register('mvnPackage', Exec) {
   inputs.file("$appDir/mule-artifact.json")
 }
 
+
 // generate a properties file so the test knows where to run mule, and what jar to deploy
 tasks.register('generateAppResources') {
-  outputs.dir generatedResourcesDir
-  doLast {
+  createAppResourceTask(it, "test")
+}
+tasks.register('generateAppResources46') {
+  createAppResourceTask(it, "mule46ForkedTest")
+}
+tasks.register('generateAppResourcesLatest') {
+  createAppResourceTask(it, "latestDepForkedTest")
+}
+
+def createAppResourceTask(Task task, String name) {
+  def generatedResourcesDir = "$buildDir/generated-resources/$name"
+  task.outputs.dir generatedResourcesDir
+  task.doLast {
     def generated = new File(generatedResourcesDir, "test-build.properties")
-    generated.text = """|mule.base=$muleBaseDir
+    generated.text = """|mule.base=$buildDir/mule/$name
                         |MuleTestApplicationConstants.jar=$buildDir/target/mule-test-application-$version-mule-application.jar
                         |MuleTestApplicationConstants.name=mule-test-application-$version-mule-application""".stripMargin()
   }
 }
 
-tasks.named("test").configure {
-  outputs.upToDateWhen {
-    !mvnPackage.didWork && !extractMuleServices.didWork
-  }
-}
-
 spotless {
   java {
     target "**/*.java"