diff --git a/docs/instrumentation-list.yaml b/docs/instrumentation-list.yaml index f0689db59089..2531a97646a9 100644 --- a/docs/instrumentation-list.yaml +++ b/docs/instrumentation-list.yaml @@ -14,6 +14,49 @@ libraries: target_versions: javaagent: - io.activej:activej-http:[6.0,) + telemetry: + - when: default + metrics: + - name: http.server.request.duration + description: Duration of HTTP server requests. + type: HISTOGRAM + unit: s + attributes: + - name: http.request.method + type: STRING + - name: http.response.status_code + type: LONG + - name: network.protocol.version + type: STRING + - name: url.scheme + type: STRING + spans: + - span_kind: SERVER + attributes: + - name: client.address + type: STRING + - name: error.type + type: STRING + - name: http.request.method + type: STRING + - name: http.response.status_code + type: LONG + - name: network.peer.address + type: STRING + - name: network.protocol.version + type: STRING + - name: server.address + type: STRING + - name: server.port + type: LONG + - name: url.path + type: STRING + - name: url.query + type: STRING + - name: url.scheme + type: STRING + - name: user_agent.original + type: STRING akka: - name: akka-actor-2.3 source_path: instrumentation/akka/akka-actor-2.3 @@ -214,6 +257,9 @@ libraries: type: STRING apache: - name: apache-dbcp-2.0 + description: | + This instrumentation provides database connection pools metrics for Apache DBCP. + The instrumentation uses `MBeanRegistration` methods for lifecycle detection, therefore it only activates if the `BasicDataSource` is registered to an `MBeanServer`. If using Spring Boot, this happens automatically as all Spring beans that support JMX registration are automatically registered by default. source_path: instrumentation/apache-dbcp-2.0 scope: name: io.opentelemetry.apache-dbcp-2.0 @@ -334,6 +380,8 @@ libraries: - name: rpc.system type: STRING - name: apache-httpasyncclient-4.1 + description: This instrumentation provides CLIENT spans and metrics for the Apache + HttpAsyncClient. source_path: instrumentation/apache-httpasyncclient-4.1 scope: name: io.opentelemetry.apache-httpasyncclient-4.1 @@ -358,14 +406,72 @@ libraries: type: STRING - name: server.port type: LONG + spans: + - span_kind: CLIENT + attributes: + - name: error.type + type: STRING + - name: http.request.method + type: STRING + - name: http.request.method_original + type: STRING + - name: http.response.status_code + type: LONG + - name: network.protocol.version + type: STRING + - name: server.address + type: STRING + - name: server.port + type: LONG + - name: url.full + type: STRING - name: apache-httpclient-2.0 + description: This instrumentation provides CLIENT spans and metrics for versions + 2 and 3 of the Apache HttpClient. source_path: instrumentation/apache-httpclient/apache-httpclient-2.0 scope: name: io.opentelemetry.apache-httpclient-2.0 target_versions: javaagent: - commons-httpclient:commons-httpclient:[2.0,4.0) + telemetry: + - when: default + metrics: + - name: http.client.request.duration + description: Duration of HTTP client requests. + type: HISTOGRAM + unit: s + attributes: + - name: http.request.method + type: STRING + - name: http.response.status_code + type: LONG + - name: network.protocol.version + type: STRING + - name: server.address + type: STRING + - name: server.port + type: LONG + spans: + - span_kind: CLIENT + attributes: + - name: error.type + type: STRING + - name: http.request.method + type: STRING + - name: http.response.status_code + type: LONG + - name: network.protocol.version + type: STRING + - name: server.address + type: STRING + - name: server.port + type: LONG + - name: url.full + type: STRING - name: apache-httpclient-4.0 + description: This instrumentation provides CLIENT spans and metrics for version + 4 of the Apache HttpClient. source_path: instrumentation/apache-httpclient/apache-httpclient-4.0 scope: name: io.opentelemetry.apache-httpclient-4.0 @@ -373,13 +479,91 @@ libraries: javaagent: - io.dropwizard:dropwizard-client:(,3.0.0) - org.apache.httpcomponents:httpclient:[4.0,) + telemetry: + - when: default + metrics: + - name: http.client.request.duration + description: Duration of HTTP client requests. + type: HISTOGRAM + unit: s + attributes: + - name: http.request.method + type: STRING + - name: http.response.status_code + type: LONG + - name: network.protocol.version + type: STRING + - name: server.address + type: STRING + - name: server.port + type: LONG + spans: + - span_kind: CLIENT + attributes: + - name: error.type + type: STRING + - name: http.request.method + type: STRING + - name: http.request.method_original + type: STRING + - name: http.response.status_code + type: LONG + - name: network.protocol.version + type: STRING + - name: server.address + type: STRING + - name: server.port + type: LONG + - name: url.full + type: STRING - name: apache-httpclient-4.3 + description: This instrumentation provides a library integration that enables + CLIENT spans and metrics for the Apache HttpClient. source_path: instrumentation/apache-httpclient/apache-httpclient-4.3 scope: name: io.opentelemetry.apache-httpclient-4.3 target_versions: library: - org.apache.httpcomponents:httpclient:[4.3,4.+) + telemetry: + - when: default + metrics: + - name: http.client.request.duration + description: Duration of HTTP client requests. + type: HISTOGRAM + unit: s + attributes: + - name: http.request.method + type: STRING + - name: http.response.status_code + type: LONG + - name: network.protocol.version + type: STRING + - name: server.address + type: STRING + - name: server.port + type: LONG + spans: + - span_kind: CLIENT + attributes: + - name: error.type + type: STRING + - name: http.request.method + type: STRING + - name: http.request.method_original + type: STRING + - name: http.request.resend_count + type: LONG + - name: http.response.status_code + type: LONG + - name: network.protocol.version + type: STRING + - name: server.address + type: STRING + - name: server.port + type: LONG + - name: url.full + type: STRING - name: apache-httpclient-5.0 source_path: instrumentation/apache-httpclient/apache-httpclient-5.0 scope: @@ -2857,6 +3041,8 @@ libraries: - jakarta.servlet:jakarta.servlet-api:[5.0.0,) spark: - name: spark-2.3 + description: | + This instrumentation does not emit telemetry on its own. Instead, it extracts the HTTP route and attaches it to SERVER spans and HTTP server metrics. source_path: instrumentation/spark-2.3 scope: name: io.opentelemetry.spark-2.3 diff --git a/instrumentation-docs/collect.sh b/instrumentation-docs/collect.sh index dea252b28f84..832b3626a880 100755 --- a/instrumentation-docs/collect.sh +++ b/instrumentation-docs/collect.sh @@ -14,12 +14,16 @@ fi readonly INSTRUMENTATIONS=( # : : [ gradle-task-suffix ] + "activej-http-6.0:javaagent:test" "akka:akka-http-10.0:javaagent:test" "apache-httpasyncclient-4.1:javaagent:test" "alibaba-druid-1.0:javaagent:test" "alibaba-druid-1.0:javaagent:testStableSemconv" "apache-dbcp-2.0:javaagent:test" "apache-dbcp-2.0:javaagent:testStableSemconv" + "apache-httpclient:apache-httpclient-2.0:javaagent:test" + "apache-httpclient:apache-httpclient-4.0:javaagent:test" + "apache-httpclient:apache-httpclient-4.3:library:test" "apache-httpclient:apache-httpclient-5.0:javaagent:test" "apache-dubbo-2.7:javaagent:testDubbo" "c3p0-0.9:javaagent:test" @@ -170,7 +174,7 @@ echo ./gradlew "${gradle_tasks[@]}" \ -PcollectMetadata=true \ - --rerun-tasks + --rerun-tasks --continue # uncomment the next line to remove all .telemetry directories #find_and_remove_all_telemetry diff --git a/instrumentation-docs/readme.md b/instrumentation-docs/readme.md index b7810c19492a..4772b83bd886 100644 --- a/instrumentation-docs/readme.md +++ b/instrumentation-docs/readme.md @@ -8,10 +8,11 @@ Run the analysis to update the instrumentation-list.yaml: `./gradlew :instrumentation-docs:runAnalysis` -### Metric collection +### Telemetry collection Until this process is ready for all instrumentations, each module will be modified to include a -system property feature flag configured for when the tests run: +system property feature flag configured for when the tests run. By enabling the following flag you +will enable metric collection: ```kotlin tasks { @@ -22,6 +23,17 @@ tasks { } ``` +In order to collect spans, add the `collectSpans` property (along with `collectMetadata`): + +```kotlin +tasks { + test { + systemProperty("collectMetadata", collectMetadata) + systemProperty("collectSpans", true) + } +} +``` + Sometimes instrumentation will behave differently based on configuration options, and we can differentiate between these configurations by using the `metaDataConfig` system property. When the telemetry is written to a file, the value of this property will be included, or it will default to @@ -130,6 +142,9 @@ public class SpringWebInstrumentationModule extends InstrumentationModule * metrics * List of metrics that the instrumentation module collects, including the metric name, description, type, and attributes. * Separate lists for the metrics emitted by default vs via configuration options. +* spans + * List of spans kinds the instrumentation module generates, including the attributes and their types. + * Separate lists for the spans emitted by default vs via configuration options. ## Methodology @@ -168,16 +183,16 @@ name is determined by the instrumentation module name: `io.opentelemetry.{instr We will implement gatherers for the schemaUrl and scope attributes when instrumentations start implementing them. -### Metrics +### Spans and Metrics -In order to identify what metrics are emitted from instrumentations, we can hook into the -`InstrumentationTestRunner` class and collect the metrics generated during runs. We can then +In order to identify what telemetry is emitted from instrumentations, we can hook into the +`InstrumentationTestRunner` class and collect the metrics and spans generated during runs. We can then leverage the `afterTestClass()` in the Agent and library test runners to then write this information into temporary files. When we analyze the instrumentation modules, we can read these files and -generate the metrics section of the instrumentation-list.yaml file. +generate the telemetry section of the instrumentation-list.yaml file. The data is written into a `.telemetry` directory in the root of each instrumentation module. This data will be excluded from git and just generated on demand. -Each file has a `when` value along with the list of metrics that indicates whether the telemetry is emitted by default or via a -configuration option. +Each file has a `when` value along with the list of metrics that indicates whether the telemetry is +emitted by default or via a configuration option. diff --git a/instrumentation-docs/src/main/java/io/opentelemetry/instrumentation/docs/DocGeneratorApplication.java b/instrumentation-docs/src/main/java/io/opentelemetry/instrumentation/docs/DocGeneratorApplication.java index adf17c29cfcd..db51b6ce0e99 100644 --- a/instrumentation-docs/src/main/java/io/opentelemetry/instrumentation/docs/DocGeneratorApplication.java +++ b/instrumentation-docs/src/main/java/io/opentelemetry/instrumentation/docs/DocGeneratorApplication.java @@ -14,6 +14,7 @@ import java.io.IOException; import java.nio.file.Files; import java.nio.file.Paths; +import java.util.Comparator; import java.util.List; import java.util.Locale; import java.util.TreeMap; @@ -107,5 +108,48 @@ private static String getPercentage(String label, long numerator, long denominat + "%)"; } + @SuppressWarnings("unused") // temporary helper method used for project tracking + private static String listAllModules(List modules) { + return modules.stream() + .map(InstrumentationModule::getInstrumentationName) + .sorted() + .map(name -> "- [ ] " + name) + .collect(Collectors.joining("\n")); + } + + @SuppressWarnings("unused") // temporary helper method used for project tracking + private static String modulesWithDescriptions(List modules) { + // checklist of all modules sorted by name, with a check if description is set + return modules.stream() + .sorted(Comparator.comparing(InstrumentationModule::getInstrumentationName)) + .map( + module -> { + boolean hasDescription = + module.getMetadata() != null + && module.getMetadata().getDescription() != null + && !module.getMetadata().getDescription().isEmpty(); + String checkbox = hasDescription ? "- [x] " : "- [ ] "; + return checkbox + module.getInstrumentationName(); + }) + .collect(Collectors.joining("\n")); + } + + @SuppressWarnings("unused") // temporary helper method used for project tracking + private static String modulesWithConfigs(List modules) { + // checklist of all modules sorted by name, with a check if config is set + return modules.stream() + .sorted(Comparator.comparing(InstrumentationModule::getInstrumentationName)) + .map( + module -> { + boolean hasDescription = + module.getMetadata() != null + && module.getMetadata().getConfigurations() != null + && !module.getMetadata().getConfigurations().isEmpty(); + String checkbox = hasDescription ? "- [x] " : "- [ ] "; + return checkbox + module.getInstrumentationName(); + }) + .collect(Collectors.joining("\n")); + } + private DocGeneratorApplication() {} } diff --git a/instrumentation-docs/src/main/java/io/opentelemetry/instrumentation/docs/utils/YamlHelper.java b/instrumentation-docs/src/main/java/io/opentelemetry/instrumentation/docs/utils/YamlHelper.java index 73a8b40c2c62..1a1b24dc0017 100644 --- a/instrumentation-docs/src/main/java/io/opentelemetry/instrumentation/docs/utils/YamlHelper.java +++ b/instrumentation-docs/src/main/java/io/opentelemetry/instrumentation/docs/utils/YamlHelper.java @@ -179,9 +179,14 @@ private static Map baseProperties(InstrumentationModule module) telemetryEntry.put("spans", spanList); } - telemetryList.add(telemetryEntry); + if (!spanList.isEmpty() || !metricsList.isEmpty()) { + telemetryList.add(telemetryEntry); + } + } + + if (!telemetryList.isEmpty()) { + moduleMap.put("telemetry", telemetryList); } - moduleMap.put("telemetry", telemetryList); } return moduleMap; } diff --git a/instrumentation/activej-http-6.0/javaagent/build.gradle.kts b/instrumentation/activej-http-6.0/javaagent/build.gradle.kts index afe8d68968e3..e824d7d3cc0d 100644 --- a/instrumentation/activej-http-6.0/javaagent/build.gradle.kts +++ b/instrumentation/activej-http-6.0/javaagent/build.gradle.kts @@ -19,3 +19,10 @@ dependencies { otelJava { minJavaVersionSupported.set(JavaVersion.VERSION_17) } + +tasks { + test { + systemProperty("collectMetadata", findProperty("collectMetadata")?.toString() ?: "false") + systemProperty("collectSpans", true) + } +} diff --git a/instrumentation/akka/akka-actor-2.3/javaagent/src/test/scala/io/opentelemetry/instrumentation/akkaactor/AkkaActors.scala b/instrumentation/akka/akka-actor-2.3/javaagent/src/test/scala/io/opentelemetry/instrumentation/akkaactor/AkkaActors.scala index 3350838b178b..6d523f3007c5 100644 --- a/instrumentation/akka/akka-actor-2.3/javaagent/src/test/scala/io/opentelemetry/instrumentation/akkaactor/AkkaActors.scala +++ b/instrumentation/akka/akka-actor-2.3/javaagent/src/test/scala/io/opentelemetry/instrumentation/akkaactor/AkkaActors.scala @@ -5,11 +5,6 @@ package io.opentelemetry.instrumentation.akkaactor -/* - * Copyright The OpenTelemetry Authors - * SPDX-License-Identifier: Apache-2.0 - */ - import akka.actor.{Actor, ActorLogging, ActorRef, ActorSystem, Props} import akka.pattern.ask import akka.util.Timeout diff --git a/instrumentation/apache-dbcp-2.0/javaagent/build.gradle.kts b/instrumentation/apache-dbcp-2.0/javaagent/build.gradle.kts index e7c30b98ced1..7dc3f2098c64 100644 --- a/instrumentation/apache-dbcp-2.0/javaagent/build.gradle.kts +++ b/instrumentation/apache-dbcp-2.0/javaagent/build.gradle.kts @@ -27,10 +27,12 @@ tasks { systemProperty("collectMetadata", collectMetadata) systemProperty("metaDataConfig", "otel.semconv-stability.opt-in=database") + systemProperty("collectSpans", true) } test { systemProperty("collectMetadata", collectMetadata) + systemProperty("collectSpans", true) } check { diff --git a/instrumentation/apache-dbcp-2.0/metadata.yaml b/instrumentation/apache-dbcp-2.0/metadata.yaml new file mode 100644 index 000000000000..a7411e026fe3 --- /dev/null +++ b/instrumentation/apache-dbcp-2.0/metadata.yaml @@ -0,0 +1,7 @@ +description: > + This instrumentation provides database connection pools metrics for Apache DBCP. + + The instrumentation uses `MBeanRegistration` methods for lifecycle detection, therefore it + only activates if the `BasicDataSource` is registered to an `MBeanServer`. If using Spring Boot, + this happens automatically as all Spring beans that support JMX registration are automatically + registered by default. diff --git a/instrumentation/apache-httpasyncclient-4.1/javaagent/build.gradle.kts b/instrumentation/apache-httpasyncclient-4.1/javaagent/build.gradle.kts index dfecfc737f31..553480583ad4 100644 --- a/instrumentation/apache-httpasyncclient-4.1/javaagent/build.gradle.kts +++ b/instrumentation/apache-httpasyncclient-4.1/javaagent/build.gradle.kts @@ -20,5 +20,6 @@ dependencies { tasks { test { systemProperty("collectMetadata", findProperty("collectMetadata")?.toString() ?: "false") + systemProperty("collectSpans", true) } } diff --git a/instrumentation/apache-httpasyncclient-4.1/metadata.yaml b/instrumentation/apache-httpasyncclient-4.1/metadata.yaml new file mode 100644 index 000000000000..8a1d021ba818 --- /dev/null +++ b/instrumentation/apache-httpasyncclient-4.1/metadata.yaml @@ -0,0 +1 @@ +description: This instrumentation provides CLIENT spans and metrics for the Apache HttpAsyncClient. diff --git a/instrumentation/apache-httpclient/apache-httpclient-2.0/javaagent/build.gradle.kts b/instrumentation/apache-httpclient/apache-httpclient-2.0/javaagent/build.gradle.kts index 544a7a6c08fb..c964677ce35f 100644 --- a/instrumentation/apache-httpclient/apache-httpclient-2.0/javaagent/build.gradle.kts +++ b/instrumentation/apache-httpclient/apache-httpclient-2.0/javaagent/build.gradle.kts @@ -19,5 +19,7 @@ dependencies { tasks { withType().configureEach { systemProperty("testLatestDeps", findProperty("testLatestDeps") as Boolean) + systemProperty("collectMetadata", findProperty("collectMetadata")?.toString() ?: "false") + systemProperty("collectSpans", true) } } diff --git a/instrumentation/apache-httpclient/apache-httpclient-2.0/metadata.yaml b/instrumentation/apache-httpclient/apache-httpclient-2.0/metadata.yaml new file mode 100644 index 000000000000..c9b610946760 --- /dev/null +++ b/instrumentation/apache-httpclient/apache-httpclient-2.0/metadata.yaml @@ -0,0 +1 @@ +description: This instrumentation provides CLIENT spans and metrics for versions 2 and 3 of the Apache HttpClient. diff --git a/instrumentation/apache-httpclient/apache-httpclient-4.0/javaagent/build.gradle.kts b/instrumentation/apache-httpclient/apache-httpclient-4.0/javaagent/build.gradle.kts index efa347eeefe5..004d3953ab6e 100644 --- a/instrumentation/apache-httpclient/apache-httpclient-4.0/javaagent/build.gradle.kts +++ b/instrumentation/apache-httpclient/apache-httpclient-4.0/javaagent/build.gradle.kts @@ -29,3 +29,10 @@ dependencies { library("org.apache.httpcomponents:httpclient:4.0") testCompileOnly("net.jcip:jcip-annotations:1.0") } + +tasks { + test { + systemProperty("collectMetadata", findProperty("collectMetadata")?.toString() ?: "false") + systemProperty("collectSpans", true) + } +} diff --git a/instrumentation/apache-httpclient/apache-httpclient-4.0/metadata.yaml b/instrumentation/apache-httpclient/apache-httpclient-4.0/metadata.yaml new file mode 100644 index 000000000000..517afd52ffaf --- /dev/null +++ b/instrumentation/apache-httpclient/apache-httpclient-4.0/metadata.yaml @@ -0,0 +1 @@ +description: This instrumentation provides CLIENT spans and metrics for version 4 of the Apache HttpClient. diff --git a/instrumentation/apache-httpclient/apache-httpclient-4.3/library/build.gradle.kts b/instrumentation/apache-httpclient/apache-httpclient-4.3/library/build.gradle.kts index bd9927795b0e..98140363a7ed 100644 --- a/instrumentation/apache-httpclient/apache-httpclient-4.3/library/build.gradle.kts +++ b/instrumentation/apache-httpclient/apache-httpclient-4.3/library/build.gradle.kts @@ -11,3 +11,10 @@ dependencies { latestDepTestLibrary("org.apache.httpcomponents:httpclient:4.+") // see apache-httpclient-5.0 module } + +tasks { + test { + systemProperty("collectMetadata", findProperty("collectMetadata")?.toString() ?: "false") + systemProperty("collectSpans", true) + } +} diff --git a/instrumentation/apache-httpclient/apache-httpclient-4.3/metadata.yaml b/instrumentation/apache-httpclient/apache-httpclient-4.3/metadata.yaml new file mode 100644 index 000000000000..7ca71c47fe24 --- /dev/null +++ b/instrumentation/apache-httpclient/apache-httpclient-4.3/metadata.yaml @@ -0,0 +1 @@ +description: This instrumentation provides a library integration that enables CLIENT spans and metrics for the Apache HttpClient. diff --git a/instrumentation/spark-2.3/metadata.yaml b/instrumentation/spark-2.3/metadata.yaml new file mode 100644 index 000000000000..e4523dffa326 --- /dev/null +++ b/instrumentation/spark-2.3/metadata.yaml @@ -0,0 +1,3 @@ +description: > + This instrumentation does not emit telemetry on its own. Instead, it extracts the HTTP route and + attaches it to SERVER spans and HTTP server metrics. diff --git a/testing-common/src/main/java/io/opentelemetry/instrumentation/testing/LibraryTestRunner.java b/testing-common/src/main/java/io/opentelemetry/instrumentation/testing/LibraryTestRunner.java index f7cb08c5484b..690632bceba4 100644 --- a/testing-common/src/main/java/io/opentelemetry/instrumentation/testing/LibraryTestRunner.java +++ b/testing-common/src/main/java/io/opentelemetry/instrumentation/testing/LibraryTestRunner.java @@ -20,6 +20,7 @@ import io.opentelemetry.context.propagation.TextMapPropagator; import io.opentelemetry.contrib.baggage.processor.BaggageSpanProcessor; import io.opentelemetry.exporter.logging.LoggingSpanExporter; +import io.opentelemetry.instrumentation.testing.internal.MetaDataCollector; import io.opentelemetry.sdk.OpenTelemetrySdk; import io.opentelemetry.sdk.common.CompletableResultCode; import io.opentelemetry.sdk.logs.SdkLoggerProvider; @@ -39,6 +40,9 @@ import io.opentelemetry.sdk.trace.SpanProcessor; import io.opentelemetry.sdk.trace.data.SpanData; import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor; +import java.io.IOException; +import java.net.URL; +import java.nio.file.Paths; import java.time.Duration; import java.util.List; import java.util.concurrent.TimeUnit; @@ -116,7 +120,19 @@ public void beforeTestClass() { } @Override - public void afterTestClass() {} + public void afterTestClass() throws IOException { + // Generates files in a `.telemetry` directory within the instrumentation module with all + // captured emitted metadata to be used by the instrumentation-docs Doc generator. + if (Boolean.getBoolean("collectMetadata")) { + URL resource = this.getClass().getClassLoader().getResource(""); + if (resource == null) { + return; + } + String path = Paths.get(resource.getPath()).toString(); + + MetaDataCollector.writeTelemetryToFiles(path, metrics, tracesByScope); + } + } @Override public void clearAllExportedData() {