🐛 gRPC header capture with shaded netty module and body capture without com.google.protobuf.util.JsonFormat (#339)

* ✅ add test tasks for gRPC 1.30.0

* ♻️ move GrpcSemanticAttributes to common library for access across grpc instrumentation modules

* ✨ add instrumentation for grpc-shaded-netty-1.9

* 🎨 format grpc-1.6 build

* 🐛 fix classpath for versioned test source set

* ✅ add muzzle check for module

* 🎨 format shaded netty module

* ♻️ move usages of grpc-common to util class

* 🐛 add names to instrumentation module

* 🎨 format utils

* 🚚 rename NettyUtils

* 🚚 move around classes

* 🚚 move instrumentation classes to separate file

* 🐛 copy-paste build script bug

* 🔥 remove unneeded helper class method

* ✅ update muzzle policy to be more restrictive, we need to work without JsonFormat available on the application class loader

* ✨ add shaded project for protobuf-java-util

* ♻️ use shaded protobuf java util in grpc instrumentation

* 🔥 remove unneeded relocations

Author: ryandens

instrumentation/build.gradle.kts

@@ -34,6 +34,7 @@ dependencies{
     implementation(project(":instrumentation:servlet:servlet-3.0"))
     implementation(project(":instrumentation:spark-2.3"))
     implementation(project(":instrumentation:grpc-1.6"))
+    implementation(project(":instrumentation:grpc-shaded-netty-1.9"))
     implementation(project(":instrumentation:okhttp:okhttp-3.0"))
     implementation(project(":instrumentation:apache-httpclient-4.0"))
     implementation(project(":instrumentation:jaxrs-client-2.0"))

instrumentation/grpc-1.6/build.gradle.kts

@@ -14,8 +14,8 @@ muzzle {
         group = "io.grpc"
         module = "grpc-core"
         versions = "[1.6.0,)"
-        // for body capture via com.google.protobuf.util.JsonFormat
-        extraDependency("io.grpc:grpc-protobuf:1.6.0")
+        // for body capture via com.google.protobuf.Message to exist always
+        extraDependency("com.google.protobuf:protobuf-java:3.3.1")
         extraDependency("io.grpc:grpc-netty:1.6.0")
     }
 }
@@ -34,6 +34,8 @@ idea {
     }
 }
 
+val testGrpcVersion = "1.30.0"
+
 protobuf {
     protoc {
         // The artifact spec for the Protobuf Compiler
@@ -60,15 +62,21 @@ val grpcVersion = "1.6.0"
 dependencies {
     api("io.opentelemetry.javaagent.instrumentation:opentelemetry-javaagent-grpc-1.6:${versions["opentelemetry_java_agent"]}")
     api("io.opentelemetry.instrumentation:opentelemetry-grpc-1.6:${versions["opentelemetry_java_agent"]}")
+    implementation(project(":instrumentation:grpc-common"))
+    implementation(project(":shaded-protobuf-java-util", "shadow"))
 
     compileOnly("io.grpc:grpc-core:${grpcVersion}")
-    compileOnly("io.grpc:grpc-protobuf:${grpcVersion}")
+    compileOnly("io.grpc:grpc-protobuf:${grpcVersion}") {
+        exclude("com.google.protobuf", "protobuf-java-util")
+    }
     compileOnly("io.grpc:grpc-stub:${grpcVersion}")
     compileOnly("io.grpc:grpc-netty:${grpcVersion}")
 
     implementation("javax.annotation:javax.annotation-api:1.3.2")
 
-     testImplementation(testFixtures(project(":testing-common")))
+    testImplementation(testFixtures(project(":testing-common")))
+    testImplementation(project(":instrumentation:grpc-shaded-netty-1.9"))
+
     testImplementation("io.grpc:grpc-core:${grpcVersion}") {
         version {
             strictly(grpcVersion)
@@ -90,3 +98,46 @@ dependencies {
         }
     }
 }
+
+val grpcVersions = listOf(grpcVersion, "1.30.0")
+
+sourceSets {
+    for (version in grpcVersions) {
+        create("test_$version") {
+            dependencies {
+                implementationConfigurationName("io.grpc:grpc-core:$version")
+            }
+        }
+    }
+}
+
+tasks.compileTestJava {
+    this.classpath += sourceSets.named("test_$grpcVersion").get().output
+}
+tasks.test {
+    classpath += sourceSets.named("test_$grpcVersion").get().output
+}
+
+for (version in listOf("1.30.0")) {
+    val versionedConfiguration = configurations.create("test_${version}") {
+        extendsFrom(configurations.runtimeClasspath.get())
+    }
+    dependencies {
+        versionedConfiguration(testFixtures(project(":testing-common")))
+        versionedConfiguration("io.opentelemetry.javaagent.instrumentation:opentelemetry-javaagent-grpc-1.6:${versions["opentelemetry_java_agent"]}")
+        versionedConfiguration("io.opentelemetry.instrumentation:opentelemetry-grpc-1.6:${versions["opentelemetry_java_agent"]}")
+        versionedConfiguration(project(":instrumentation:grpc-shaded-netty-1.9"))
+        versionedConfiguration(platform("io.grpc:grpc-bom:$version"))
+        versionedConfiguration("io.grpc:grpc-core")
+        versionedConfiguration("io.grpc:grpc-protobuf")
+        versionedConfiguration("io.grpc:grpc-stub")
+        versionedConfiguration("io.grpc:grpc-netty")
+
+    }
+    val versionedTest = task<Test>("test_${version}") {
+        group = "verification"
+        classpath = versionedConfiguration + sourceSets.main.get().output + sourceSets.test.get().output + sourceSets.named("test_$version").get().output
+        useJUnitPlatform()
+    }
+    tasks.check { dependsOn(versionedTest) }
+}

instrumentation/grpc-1.6/src/main/java/io/opentelemetry/javaagent/instrumentation/hypertrace/grpc/v1_6/GrpcSpanDecorator.java

@@ -17,11 +17,12 @@
 package io.opentelemetry.javaagent.instrumentation.hypertrace.grpc.v1_6;
 
 import com.google.protobuf.Message;
-import com.google.protobuf.util.JsonFormat;
 import io.grpc.Metadata;
 import io.grpc.Metadata.Key;
 import io.opentelemetry.api.common.AttributeKey;
 import io.opentelemetry.api.trace.Span;
+import io.opentelemetry.javaagent.instrumentation.hypertrace.com.google.protobuf.util.JsonFormat;
+import io.opentelemetry.javaagent.instrumentation.hypertrace.grpc.GrpcSemanticAttributes;
 import java.io.IOException;
 import java.util.LinkedHashMap;
 import java.util.Map;

instrumentation/grpc-1.6/src/main/java/io/opentelemetry/javaagent/instrumentation/hypertrace/grpc/v1_6/NettyHttp2HeadersInstrumentationModule.java

@@ -28,6 +28,7 @@
 import io.opentelemetry.javaagent.extension.instrumentation.InstrumentationModule;
 import io.opentelemetry.javaagent.extension.instrumentation.TypeInstrumentation;
 import io.opentelemetry.javaagent.instrumentation.api.Java8BytecodeBridge;
+import io.opentelemetry.javaagent.instrumentation.hypertrace.grpc.GrpcSemanticAttributes;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.HashMap;

instrumentation/grpc-1.6/src/test/java/io/opentelemetry/javaagent/instrumentation/hypertrace/grpc/v1_6/GrpcInstrumentationTest.java

@@ -19,7 +19,6 @@
 import com.google.protobuf.util.JsonFormat;
 import io.grpc.ForwardingServerCall;
 import io.grpc.ManagedChannel;
-import io.grpc.ManagedChannelBuilder;
 import io.grpc.Metadata;
 import io.grpc.Server;
 import io.grpc.ServerBuilder;
@@ -30,6 +29,7 @@
 import io.grpc.Status;
 import io.grpc.StatusRuntimeException;
 import io.grpc.stub.MetadataUtils;
+import io.opentelemetry.javaagent.instrumentation.hypertrace.grpc.GrpcSemanticAttributes;
 import io.opentelemetry.sdk.trace.data.SpanData;
 import java.io.IOException;
 import java.net.URL;
@@ -104,10 +104,7 @@ public void sendHeaders(Metadata headers) {
             .build();
     SERVER.start();
 
-    CHANNEL =
-        ManagedChannelBuilder.forTarget(String.format("localhost:%d", SERVER.getPort()))
-            .usePlaintext(true)
-            .build();
+    CHANNEL = new GrpcChannelProvider(SERVER.getPort()).get();
   }
 
   @AfterAll

instrumentation/grpc-1.6/src/test_1.30.0/java/io/opentelemetry/javaagent/instrumentation/hypertrace/grpc/v1_6/GrpcChannelProvider.java

@@ -0,0 +1,37 @@
+/*
+ * Copyright The Hypertrace Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.opentelemetry.javaagent.instrumentation.hypertrace.grpc.v1_6;
+
+import io.grpc.ManagedChannel;
+import io.grpc.ManagedChannelBuilder;
+import java.util.function.Supplier;
+
+final class GrpcChannelProvider implements Supplier<ManagedChannel> {
+
+  private final int port;
+
+  GrpcChannelProvider(final int port) {
+    this.port = port;
+  }
+
+  @Override
+  public ManagedChannel get() {
+    return ManagedChannelBuilder.forTarget(String.format("localhost:%d", port))
+        .usePlaintext()
+        .build();
+  }
+}

instrumentation/grpc-1.6/src/test_1.6.0/java/io/opentelemetry/javaagent/instrumentation/hypertrace/grpc/v1_6/GrpcChannelProvider.java

@@ -0,0 +1,37 @@
+/*
+ * Copyright The Hypertrace Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.opentelemetry.javaagent.instrumentation.hypertrace.grpc.v1_6;
+
+import io.grpc.ManagedChannel;
+import io.grpc.ManagedChannelBuilder;
+import java.util.function.Supplier;
+
+final class GrpcChannelProvider implements Supplier<ManagedChannel> {
+
+  private final int port;
+
+  GrpcChannelProvider(final int port) {
+    this.port = port;
+  }
+
+  @Override
+  public ManagedChannel get() {
+    return ManagedChannelBuilder.forTarget(String.format("localhost:%d", port))
+        .usePlaintext(true)
+        .build();
+  }
+}

instrumentation/grpc-common/build.gradle.kts

@@ -0,0 +1,7 @@
+plugins {
+    `java-library`
+}
+
+dependencies {
+    compileOnly("io.grpc:grpc-core:1.6.0")
+}

instrumentation/grpc-1.6/src/main/java/io/opentelemetry/javaagent/instrumentation/hypertrace/grpc/v1_6/GrpcSemanticAttributes.java renamed to instrumentation/grpc-common/src/main/java/io/opentelemetry/javaagent/instrumentation/hypertrace/grpc/GrpcSemanticAttributes.java

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package io.opentelemetry.javaagent.instrumentation.hypertrace.grpc.v1_6;
+package io.opentelemetry.javaagent.instrumentation.hypertrace.grpc;
 
 import io.grpc.Metadata;
 

instrumentation/grpc-shaded-netty-1.9/build.gradle.kts

@@ -0,0 +1,29 @@
+plugins {
+    `java-library`
+    id("net.bytebuddy.byte-buddy")
+    id("io.opentelemetry.instrumentation.auto-instrumentation")
+    muzzle
+}
+
+muzzle {
+    pass {
+        group = "io.grpc"
+        module = "grpc-netty-shaded"
+        versions = "[1.9.0,)"
+        assertInverse = true
+    }
+}
+
+afterEvaluate{
+    io.opentelemetry.instrumentation.gradle.bytebuddy.ByteBuddyPluginConfigurator(project,
+            sourceSets.main.get(),
+            "io.opentelemetry.javaagent.tooling.muzzle.collector.MuzzleCodeGenerationPlugin",
+            project(":javaagent-tooling").configurations["instrumentationMuzzle"] + configurations.runtimeClasspath
+    ).configure()
+}
+
+dependencies {
+    compileOnly("io.grpc:grpc-core:1.9.0")
+    compileOnly("io.grpc:grpc-netty-shaded:1.9.0")
+    implementation(project(":instrumentation:grpc-common"))
+}