Fix testng order and test templates

Author: amarziali

dd-java-agent/instrumentation/testng/testng-6.4/src/main/java/datadog/trace/instrumentation/testng6/TestNGClassListenerInstrumentation.java

@@ -9,6 +9,7 @@
 import datadog.trace.agent.tooling.Instrumenter;
 import datadog.trace.agent.tooling.InstrumenterModule;
 import datadog.trace.instrumentation.testng.TestNGClassListener;
+import datadog.trace.instrumentation.testng.TestNGInstrumentation;
 import datadog.trace.instrumentation.testng.TestNGUtils;
 import datadog.trace.util.Strings;
 import net.bytebuddy.asm.Advice;
@@ -64,6 +65,13 @@ public String[] helperClassNames() {
     };
   }
 
+  @Override
+  public int order() {
+    // Depends on datadog.trace.instrumentation.testng.TestNGInstrumentation,
+    // as it needs datadog.trace.instrumentation.testng.TestEventsHandlerHolder.start to be called;
+    return TestNGInstrumentation.ORDER - 1;
+  }
+
   public static class InvokeBeforeClassAdvice {
     @SuppressWarnings("bytebuddy-exception-suppression")
     @Advice.OnMethodEnter

dd-java-agent/instrumentation/testng/testng-6.4/src/test/resources/test-class-skipped/events.ftl

@@ -12,7 +12,7 @@
       "test.codeowners" : "[\"owner1\",\"owner2\"]",
       "test.framework" : "testng",
       "test.framework_version" : ${content_meta_test_framework_version},
-      "test.module" : "testng-6",
+      "test.module" : "testng-6.4",
       "test.source.file" : "dummy_source_path",
       "test.status" : "skip",
       "test.suite" : "org.example.TestSkippedClass",
@@ -51,7 +51,7 @@
       "test.codeowners" : "[\"owner1\",\"owner2\"]",
       "test.framework" : "testng",
       "test.framework_version" : ${content_meta_test_framework_version},
-      "test.module" : "testng-6",
+      "test.module" : "testng-6.4",
       "test.name" : "test_class_another_skipped",
       "test.source.file" : "dummy_source_path",
       "test.source.method" : "test_class_another_skipped()V",
@@ -98,7 +98,7 @@
       "test.codeowners" : "[\"owner1\",\"owner2\"]",
       "test.framework" : "testng",
       "test.framework_version" : ${content_meta_test_framework_version},
-      "test.module" : "testng-6",
+      "test.module" : "testng-6.4",
       "test.name" : "test_class_skipped",
       "test.source.file" : "dummy_source_path",
       "test.source.method" : "test_class_skipped()V",
@@ -143,7 +143,7 @@
       "library_version" : ${content_meta_library_version},
       "runtime-id" : ${content_meta_runtime_id},
       "span.kind" : "test_session_end",
-      "test.command" : "testng-6",
+      "test.command" : "testng-6.4",
       "test.framework" : "testng",
       "test.framework_version" : ${content_meta_test_framework_version},
       "test.status" : "skip",
@@ -157,7 +157,7 @@
       "process_id" : ${content_metrics_process_id}
     },
     "name" : "testng.test_session",
-    "resource" : "testng-6",
+    "resource" : "testng-6.4",
     "service" : "worker.org.gradle.process.internal.worker.gradleworkermain",
     "start" : ${content_start_4},
     "test_session_id" : ${content_test_session_id}
@@ -177,7 +177,7 @@
       "span.kind" : "test_module_end",
       "test.framework" : "testng",
       "test.framework_version" : ${content_meta_test_framework_version},
-      "test.module" : "testng-6",
+      "test.module" : "testng-6.4",
       "test.status" : "skip",
       "test.type" : "test",
       "test_session.name" : "session-name"
@@ -186,12 +186,12 @@
       "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_5}
     },
     "name" : "testng.test_module",
-    "resource" : "testng-6",
+    "resource" : "testng-6.4",
     "service" : "worker.org.gradle.process.internal.worker.gradleworkermain",
     "start" : ${content_start_5},
     "test_module_id" : ${content_test_module_id},
     "test_session_id" : ${content_test_session_id}
   },
   "type" : "test_module_end",
   "version" : 1
-} ]
+} ]

dd-java-agent/instrumentation/testng/testng-6.4/src/test/resources/test-disabled-failed-parameterized/events.ftl

@@ -12,7 +12,7 @@
       "test.codeowners" : "[\"owner1\",\"owner2\"]",
       "test.framework" : "testng",
       "test.framework_version" : ${content_meta_test_framework_version},
-      "test.module" : "testng-6",
+      "test.module" : "testng-6.4",
       "test.source.file" : "dummy_source_path",
       "test.status" : "skip",
       "test.suite" : "org.example.TestFailedParameterized",
@@ -51,7 +51,7 @@
       "test.codeowners" : "[\"owner1\",\"owner2\"]",
       "test.framework" : "testng",
       "test.framework_version" : ${content_meta_test_framework_version},
-      "test.module" : "testng-6",
+      "test.module" : "testng-6.4",
       "test.name" : "parameterized_test_succeed",
       "test.parameters" : "{\"arguments\":{\"0\":\"hello\",\"1\":\"true\"}}",
       "test.skip_reason" : "Flaky test is disabled by Datadog",
@@ -101,7 +101,7 @@
       "test.codeowners" : "[\"owner1\",\"owner2\"]",
       "test.framework" : "testng",
       "test.framework_version" : ${content_meta_test_framework_version},
-      "test.module" : "testng-6",
+      "test.module" : "testng-6.4",
       "test.name" : "parameterized_test_succeed",
       "test.parameters" : "{\"arguments\":{\"0\":\"\\\"goodbye\\\"\",\"1\":\"false\"}}",
       "test.skip_reason" : "Flaky test is disabled by Datadog",
@@ -149,7 +149,7 @@
       "library_version" : ${content_meta_library_version},
       "runtime-id" : ${content_meta_runtime_id},
       "span.kind" : "test_session_end",
-      "test.command" : "testng-6",
+      "test.command" : "testng-6.4",
       "test.framework" : "testng",
       "test.framework_version" : ${content_meta_test_framework_version},
       "test.status" : "skip",
@@ -164,7 +164,7 @@
       "process_id" : ${content_metrics_process_id}
     },
     "name" : "testng.test_session",
-    "resource" : "testng-6",
+    "resource" : "testng-6.4",
     "service" : "worker.org.gradle.process.internal.worker.gradleworkermain",
     "start" : ${content_start_4},
     "test_session_id" : ${content_test_session_id}
@@ -184,7 +184,7 @@
       "span.kind" : "test_module_end",
       "test.framework" : "testng",
       "test.framework_version" : ${content_meta_test_framework_version},
-      "test.module" : "testng-6",
+      "test.module" : "testng-6.4",
       "test.status" : "skip",
       "test.test_management.enabled" : "true",
       "test.type" : "test",
@@ -194,12 +194,12 @@
       "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_5}
     },
     "name" : "testng.test_module",
-    "resource" : "testng-6",
+    "resource" : "testng-6.4",
     "service" : "worker.org.gradle.process.internal.worker.gradleworkermain",
     "start" : ${content_start_5},
     "test_module_id" : ${content_test_module_id},
     "test_session_id" : ${content_test_session_id}
   },
   "type" : "test_module_end",
   "version" : 1
-} ]
+} ]

dd-java-agent/instrumentation/testng/testng-6.4/src/test/resources/test-disabled-failed/events.ftl

@@ -12,7 +12,7 @@
       "test.codeowners" : "[\"owner1\",\"owner2\"]",
       "test.framework" : "testng",
       "test.framework_version" : ${content_meta_test_framework_version},
-      "test.module" : "testng-6",
+      "test.module" : "testng-6.4",
       "test.source.file" : "dummy_source_path",
       "test.status" : "skip",
       "test.suite" : "org.example.TestFailed",
@@ -51,7 +51,7 @@
       "test.codeowners" : "[\"owner1\",\"owner2\"]",
       "test.framework" : "testng",
       "test.framework_version" : ${content_meta_test_framework_version},
-      "test.module" : "testng-6",
+      "test.module" : "testng-6.4",
       "test.name" : "test_failed",
       "test.skip_reason" : "Flaky test is disabled by Datadog",
       "test.source.file" : "dummy_source_path",
@@ -98,7 +98,7 @@
       "library_version" : ${content_meta_library_version},
       "runtime-id" : ${content_meta_runtime_id},
       "span.kind" : "test_session_end",
-      "test.command" : "testng-6",
+      "test.command" : "testng-6.4",
       "test.framework" : "testng",
       "test.framework_version" : ${content_meta_test_framework_version},
       "test.status" : "skip",
@@ -113,7 +113,7 @@
       "process_id" : ${content_metrics_process_id}
     },
     "name" : "testng.test_session",
-    "resource" : "testng-6",
+    "resource" : "testng-6.4",
     "service" : "worker.org.gradle.process.internal.worker.gradleworkermain",
     "start" : ${content_start_3},
     "test_session_id" : ${content_test_session_id}
@@ -133,7 +133,7 @@
       "span.kind" : "test_module_end",
       "test.framework" : "testng",
       "test.framework_version" : ${content_meta_test_framework_version},
-      "test.module" : "testng-6",
+      "test.module" : "testng-6.4",
       "test.status" : "skip",
       "test.test_management.enabled" : "true",
       "test.type" : "test",
@@ -143,12 +143,12 @@
       "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_4}
     },
     "name" : "testng.test_module",
-    "resource" : "testng-6",
+    "resource" : "testng-6.4",
     "service" : "worker.org.gradle.process.internal.worker.gradleworkermain",
     "start" : ${content_start_4},
     "test_module_id" : ${content_test_module_id},
     "test_session_id" : ${content_test_session_id}
   },
   "type" : "test_module_end",
   "version" : 1
-} ]
+} ]

dd-java-agent/instrumentation/testng/testng-6.4/src/test/resources/test-error/events.ftl

@@ -12,7 +12,7 @@
       "test.codeowners" : "[\"owner1\",\"owner2\"]",
       "test.framework" : "testng",
       "test.framework_version" : ${content_meta_test_framework_version},
-      "test.module" : "testng-6",
+      "test.module" : "testng-6.4",
       "test.source.file" : "dummy_source_path",
       "test.status" : "fail",
       "test.suite" : "org.example.TestError",
@@ -54,7 +54,7 @@
       "test.codeowners" : "[\"owner1\",\"owner2\"]",
       "test.framework" : "testng",
       "test.framework_version" : ${content_meta_test_framework_version},
-      "test.module" : "testng-6",
+      "test.module" : "testng-6.4",
       "test.name" : "test_error",
       "test.source.file" : "dummy_source_path",
       "test.source.method" : "test_error()V",
@@ -99,7 +99,7 @@
       "library_version" : ${content_meta_library_version},
       "runtime-id" : ${content_meta_runtime_id},
       "span.kind" : "test_session_end",
-      "test.command" : "testng-6",
+      "test.command" : "testng-6.4",
       "test.framework" : "testng",
       "test.framework_version" : ${content_meta_test_framework_version},
       "test.status" : "fail",
@@ -113,7 +113,7 @@
       "process_id" : ${content_metrics_process_id}
     },
     "name" : "testng.test_session",
-    "resource" : "testng-6",
+    "resource" : "testng-6.4",
     "service" : "worker.org.gradle.process.internal.worker.gradleworkermain",
     "start" : ${content_start_3},
     "test_session_id" : ${content_test_session_id}
@@ -133,7 +133,7 @@
       "span.kind" : "test_module_end",
       "test.framework" : "testng",
       "test.framework_version" : ${content_meta_test_framework_version},
-      "test.module" : "testng-6",
+      "test.module" : "testng-6.4",
       "test.status" : "fail",
       "test.type" : "test",
       "test_session.name" : "session-name"
@@ -142,12 +142,12 @@
       "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_4}
     },
     "name" : "testng.test_module",
-    "resource" : "testng-6",
+    "resource" : "testng-6.4",
     "service" : "worker.org.gradle.process.internal.worker.gradleworkermain",
     "start" : ${content_start_4},
     "test_module_id" : ${content_test_module_id},
     "test_session_id" : ${content_test_session_id}
   },
   "type" : "test_module_end",
   "version" : 1
-} ]
+} ]

dd-java-agent/instrumentation/testng/testng-6.4/src/test/resources/test-factory-data-provider/events.ftl

@@ -12,7 +12,7 @@
       "test.codeowners" : "[\"owner1\",\"owner2\"]",
       "test.framework" : "testng",
       "test.framework_version" : ${content_meta_test_framework_version},
-      "test.module" : "testng-6",
+      "test.module" : "testng-6.4",
       "test.source.file" : "dummy_source_path",
       "test.status" : "pass",
       "test.suite" : "org.example.TestSucceedDataProvider",
@@ -51,7 +51,7 @@
       "test.codeowners" : "[\"owner1\",\"owner2\"]",
       "test.framework" : "testng",
       "test.framework_version" : ${content_meta_test_framework_version},
-      "test.module" : "testng-6",
+      "test.module" : "testng-6.4",
       "test.name" : "testMethod",
       "test.source.file" : "dummy_source_path",
       "test.source.method" : "testMethod()V",
@@ -96,7 +96,7 @@
       "library_version" : ${content_meta_library_version},
       "runtime-id" : ${content_meta_runtime_id},
       "span.kind" : "test_session_end",
-      "test.command" : "testng-6",
+      "test.command" : "testng-6.4",
       "test.framework" : "testng",
       "test.framework_version" : ${content_meta_test_framework_version},
       "test.status" : "pass",
@@ -110,7 +110,7 @@
       "process_id" : ${content_metrics_process_id}
     },
     "name" : "testng.test_session",
-    "resource" : "testng-6",
+    "resource" : "testng-6.4",
     "service" : "worker.org.gradle.process.internal.worker.gradleworkermain",
     "start" : ${content_start_3},
     "test_session_id" : ${content_test_session_id}
@@ -130,7 +130,7 @@
       "span.kind" : "test_module_end",
       "test.framework" : "testng",
       "test.framework_version" : ${content_meta_test_framework_version},
-      "test.module" : "testng-6",
+      "test.module" : "testng-6.4",
       "test.status" : "pass",
       "test.type" : "test",
       "test_session.name" : "session-name"
@@ -139,12 +139,12 @@
       "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_4}
     },
     "name" : "testng.test_module",
-    "resource" : "testng-6",
+    "resource" : "testng-6.4",
     "service" : "worker.org.gradle.process.internal.worker.gradleworkermain",
     "start" : ${content_start_4},
     "test_module_id" : ${content_test_module_id},
     "test_session_id" : ${content_test_session_id}
   },
   "type" : "test_module_end",
   "version" : 1
-} ]
+} ]