Skip to content

Commit 0b73bff

Browse files
link04link04
authored
[OTEL] Vendoring OtlpGrpcExportClient And Enabling OTLP Metrics gRPC Tests (#7666)
## Summary of changes Added gRPC protocol support to the OTLP metrics exporter by vendoring OpenTelemetry's gRPC transport client. The implementation now supports `grpc` which also was updated to be the default protocol value. ## Reason for change Customers need gRPC support for OTLP metrics export to maintain compatibility with OpenTelemetry ecosystem standards and their existing infrastructure that may require or prefer gRPC transport. ## Implementation details **Vendored dependencies:** - Vendored `OpenTelemetry.Exporter.OpenTelemetryProtocol` v1.13.1(lastest ATM with included fixes) gRPC transport client (`OtlpGrpcExportClient` and related helpers) - Created `OpenTelemetryStubs.cs` with minimal stub implementations to avoid vendoring the entire OpenTelemetry SDK: - `Guard` and `UriExtensions` from `OpenTelemetry.Internal` (Would need to discard deletion after usng vendoring command) - `OtlpExporterOptions` (minimal configuration class) - `OtlpExportProtocol` enum (kept in stub due to namespace issues with the original being in parent namespace `OpenTelemetry.Exporter`) - `OpenTelemetryProtocolExporterEventSource` (stub EventSource for logging) **Core changes:** - Updated `OtlpExporter.cs` to instantiate and use the vendored `OtlpGrpcExportClient` when protocol is `Grpc` - Added 5-byte gRPC message frame prefix (compression flag + message length) to protobuf payloads for gRPC transport - Modified `OtlpMetricsSerializer.cs` to optionally reserve bytes at the start of the buffer for the gRPC frame header **Vendoring infrastructure:** - Added `AddOpenTelemetryUsings` transform to inject common `using` directives needed by vendored OTel files - Configured exclusions to vendor only the gRPC transport client, not the full OTLP exporter ## Test coverage - Updated `OpenTelemetrySdkTests.SubmitsOtlpMetrics` to test both `http/protobuf` and `grpc` protocols - gRPC tests use the `dd-apm-test-agent` container - All tests validate metrics are correctly exported and received by the respective agents ## DLL File Size Difference After building both `master` branch and this branch locally then comparing file size I got: |Target Framework | Master (bytes) | Feature (bytes) | Difference (bytes) | Difference (%) | Impact | |------------------|----------------|-----------------|-------------------|----------------|--------| | net6.0 | 8,252,416 | 8,270,336 | **+17,920** | +0.22% | +17.5 KB | | netcoreapp3.1 | 8,173,056 | 8,190,976 | **+17,920** | +0.22% | +17.5 KB | ## Other Details [APMAPI-1679](https://datadoghq.atlassian.net/browse/APMAPI-1679) [APMAPI-1679]: https://datadoghq.atlassian.net/browse/APMAPI-1679?atlOrigin=eyJpIjoiNWRkNTljNzYxNjVmNDY3MDlhMDU5Y2ZhYzA5YTRkZjUiLCJwIjoiZ2l0aHViLWNvbS1KU1cifQ
1 parent abe7838 commit 0b73bff

29 files changed

+1795
-44
lines changed

‎docker-compose.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -445,6 +445,7 @@ services:
445445
- COUCHBASE_HOST=couchbase
446446
- COUCHBASE_PORT=8091
447447
- ASB_CONNECTION_STRING=Endpoint=sb://azureservicebus-emulator:5672;SharedAccessKeyName=RootManageSharedAccessKey;SharedAccessKey=SAS_KEY_VALUE;UseDevelopmentEmulator=true;
448+
- TEST_AGENT_HOST=test-agent
448449
- CONTAINER_HOSTNAME=http://integrationtests
449450
- DD_LOGGER_DD_API_KEY
450451
- DD_LOGGER_DD_TRACE_LOG_DIRECTORY=/project/artifacts/build_data/infra_logs

‎tracer/build/_build/UpdateVendors/VendoredDependency.cs‎

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -153,6 +153,52 @@ static VendoredDependency()
153153
downloadUrl: "https://github.com/icsharpcode/SharpZipLib/archive/refs/tags/v1.3.3.zip",
154154
pathToSrc: new[] { "SharpZipLib-1.3.3", "src", "ICSharpCode.SharpZipLib" },
155155
transform: filePath => RewriteCsFileWithStandardTransform(filePath, originalNamespace: "ICSharpCode.SharpZipLib", AddIfNetFramework));
156+
157+
Add(
158+
libraryName: "OpenTelemetry.Exporter.OpenTelemetryProtocol",
159+
version: "core-1.13.1",
160+
downloadUrl: "https://github.com/open-telemetry/opentelemetry-dotnet/archive/refs/tags/core-1.13.1.zip",
161+
pathToSrc: new[] { "opentelemetry-dotnet-core-1.13.1", "src", "OpenTelemetry.Exporter.OpenTelemetryProtocol" },
162+
transform: filePath => RewriteCsFileWithStandardTransform(
163+
filePath,
164+
originalNamespace: "OpenTelemetry.Exporter.OpenTelemetryProtocol",
165+
AddIfNetcoreapp31OrGreater,
166+
AddNullableDirectiveTransform,
167+
AddOpenTelemetryUsings),
168+
relativePathsToExclude: new[]
169+
{
170+
// Vendor only the gRPC transport client - exclude everything else
171+
".publicApi/",
172+
"Builder/",
173+
"PersistentStorage/",
174+
"Implementation/Serializer/",
175+
"Implementation/Transmission/",
176+
"Implementation/ActivityExtensions.cs",
177+
"Implementation/ExperimentalOptions.cs",
178+
"Implementation/SdkLimitOptions.cs",
179+
"Implementation/TelemetryType.cs",
180+
"Implementation/OpenTelemetryProtocolExporterEventSource.cs",
181+
"Implementation/OtlpServiceCollectionExtensions.cs",
182+
"Implementation/OtlpExporterOptionsConfigurationType.cs",
183+
"Implementation/OtlpSpecConfigDefinitions.cs",
184+
"Implementation/TimestampHelpers.cs",
185+
"Implementation/ExportClient/OtlpHttpExportClient.cs", // We only need gRPC for this PR
186+
"Implementation/ExportClient/OtlpRetry.cs",
187+
"CHANGELOG.md",
188+
"README.md",
189+
"IOtlpExporterOptions.cs",
190+
"OtlpExporterOptions.cs",
191+
"OtlpExporterOptionsExtensions.cs",
192+
"OtlpExportProtocol.cs", // In parent namespace OpenTelemetry.Exporter; keep in stub instead
193+
"OtlpExportProtocolParser.cs",
194+
"OtlpLogExporter.cs",
195+
"OtlpLogExporterHelperExtensions.cs",
196+
"OtlpMetricExporter.cs",
197+
"OtlpMetricExporterExtensions.cs",
198+
"OtlpSignalType.cs",
199+
"OtlpTraceExporter.cs",
200+
"OtlpTraceExporterHelperExtensions.cs"
201+
});
156202
}
157203

158204
public static List<VendoredDependency> All { get; set; } = new List<VendoredDependency>();
@@ -398,5 +444,31 @@ private static void RewriteFileWithTransform(string filePath, Func<string, strin
398444
fileContent,
399445
new UTF8Encoding(encoderShouldEmitUTF8Identifier: false));
400446
}
447+
448+
private static string AddOpenTelemetryUsings(string filePath, string contents)
449+
{
450+
// Find the namespace declaration
451+
var namespaceIndex = contents.IndexOf("\nnamespace ");
452+
if (namespaceIndex < 0)
453+
{
454+
return contents; // No namespace found, skip
455+
}
456+
457+
// Move to the start of the line
458+
namespaceIndex = contents.LastIndexOf('\n', namespaceIndex) + 1;
459+
460+
// Add all common using directives needed by OTel files
461+
// Compiler ignores duplicates (CS0105 is suppressed in auto-generated header)
462+
var usings =
463+
"using System;\n" +
464+
"using System.Collections.Generic;\n" +
465+
"using System.IO;\n" +
466+
"using System.Linq;\n" +
467+
"using System.Net.Http;\n" +
468+
"using System.Threading;\n" +
469+
"using System.Threading.Tasks;\n\n";
470+
471+
return contents.Insert(namespaceIndex, usings);
472+
}
401473
}
402474
}

‎tracer/dependabot/Datadog.Dependabot.Vendors.csproj‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
</PropertyGroup>
77

88
<ItemGroup>
9+
910
<!-- https://www.nuget.org/packages/Microsoft.OpenApi/1.6.23 -->
1011
<PackageReference Include="Microsoft.OpenApi" Version="1.6.23" />
1112

@@ -27,11 +28,8 @@
2728
<!-- https://www.nuget.org/packages/dnlib/3.4.0 -->
2829
<PackageReference Include="dnlib" Version="3.4.0" />
2930

30-
<!-- https://www.nuget.org/packages/Datadog.Sketches/1.0.0 -->
31-
<PackageReference Include="Datadog.Sketches" Version="1.0.0" />
32-
33-
<!-- https://www.nuget.org/packages/IndieSystem.Text.RegularExpressions/0.6 -->
34-
<PackageReference Include="IndieSystem.Text.RegularExpressions" Version="0.6" />
31+
<!-- https://www.nuget.org/packages/Datadog.Sketches/main -->
32+
<PackageReference Include="Datadog.Sketches" Version="main" />
3533

3634
<!-- https://www.nuget.org/packages/System.Collections.Immutable/7.0.0 -->
3735
<PackageReference Include="System.Collections.Immutable" Version="7.0.0" />
@@ -45,12 +43,14 @@
4543
<!-- https://www.nuget.org/packages/System.Reflection.Metadata/7.0.2 -->
4644
<PackageReference Include="System.Reflection.Metadata" Version="7.0.2" />
4745

48-
<!-- https://www.nuget.org/packages/ICSharpCode.SharpZipLib/1.3.3 -->
49-
<PackageReference Include="ICSharpCode.SharpZipLib" Version="1.3.3" />
50-
5146
<!-- https://www.nuget.org/packages/System.Runtime.CompilerServices.Unsafe/1.0.0 -->
5247
<PackageReference Include="System.Runtime.CompilerServices.Unsafe" Version="1.0.0" />
5348

49+
<!-- https://www.nuget.org/packages/ICSharpCode.SharpZipLib/1.3.3 -->
50+
<PackageReference Include="ICSharpCode.SharpZipLib" Version="1.3.3" />
51+
52+
<!-- https://www.nuget.org/packages/OpenTelemetry.Exporter.OpenTelemetryProtocol/core-1.13.1 -->
53+
<PackageReference Include="OpenTelemetry.Exporter.OpenTelemetryProtocol" Version="core-1.13.1" />
5454
</ItemGroup>
5555

5656
</Project>

‎tracer/src/Datadog.Trace.Trimming/build/Datadog.Trace.Trimming.xml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -366,8 +366,11 @@
366366
<assembly fullname="System.Messaging" />
367367
<assembly fullname="System.Net.Http">
368368
<type fullname="System.Net.Http.ByteArrayContent" />
369+
<type fullname="System.Net.Http.Headers.HeaderStringValues" />
370+
<type fullname="System.Net.Http.Headers.HeaderStringValues/Enumerator" />
369371
<type fullname="System.Net.Http.Headers.HttpContentHeaders" />
370372
<type fullname="System.Net.Http.Headers.HttpHeaders" />
373+
<type fullname="System.Net.Http.Headers.HttpHeadersNonValidated" />
371374
<type fullname="System.Net.Http.Headers.HttpRequestHeaders" />
372375
<type fullname="System.Net.Http.Headers.HttpResponseHeaders" />
373376
<type fullname="System.Net.Http.Headers.MediaTypeHeaderValue" />
@@ -379,6 +382,7 @@
379382
<type fullname="System.Net.Http.HttpMessageHandler" />
380383
<type fullname="System.Net.Http.HttpMessageInvoker" />
381384
<type fullname="System.Net.Http.HttpMethod" />
385+
<type fullname="System.Net.Http.HttpRequestException" />
382386
<type fullname="System.Net.Http.HttpRequestMessage" />
383387
<type fullname="System.Net.Http.HttpResponseMessage" />
384388
<type fullname="System.Net.Http.HttpVersionPolicy" />
@@ -645,6 +649,7 @@
645649
<type fullname="System.IO.FileStream" />
646650
<type fullname="System.IO.FileSystemInfo" />
647651
<type fullname="System.IO.HandleInheritability" />
652+
<type fullname="System.IO.InvalidDataException" />
648653
<type fullname="System.IO.IOException" />
649654
<type fullname="System.IO.MemoryStream" />
650655
<type fullname="System.IO.Path" />
@@ -911,6 +916,7 @@
911916
<type fullname="System.IO.BinaryReader" />
912917
<type fullname="System.IO.BinaryWriter" />
913918
<type fullname="System.IO.EndOfStreamException" />
919+
<type fullname="System.IO.InvalidDataException" />
914920
<type fullname="System.IO.MemoryStream" />
915921
<type fullname="System.IO.Path" />
916922
<type fullname="System.IO.StreamReader" />

‎tracer/src/Datadog.Trace/Configuration/ConfigurationKeys.OpenTelemetry.cs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -88,7 +88,7 @@ internal class OpenTelemetry
8888
/// <summary>
8989
/// Configuration key to set the OTLP protocol for metrics export.
9090
/// Takes precedence over <see cref="ExporterOtlpProtocol"/>.
91-
/// Valid values: grpc, http/protobuf, http/json, defaults to http/protobuf.
91+
/// Valid values: grpc, http/protobuf, http/json, defaults to grpc.
9292
/// </summary>
9393
public const string ExporterOtlpMetricsProtocol = "OTEL_EXPORTER_OTLP_METRICS_PROTOCOL";
9494

@@ -126,7 +126,7 @@ internal class OpenTelemetry
126126
/// <summary>
127127
/// Configuration key to set the OTLP protocol (fallback for metrics-specific protocol).
128128
/// Used when <see cref="ExporterOtlpMetricsProtocol"/> is not set.
129-
/// Valid values: grpc, http/protobuf, http/json, defaults to http/protobuf.
129+
/// Valid values: grpc, http/protobuf, http/json, defaults to grpc.
130130
/// </summary>
131131
public const string ExporterOtlpProtocol = "OTEL_EXPORTER_OTLP_PROTOCOL";
132132

‎tracer/src/Datadog.Trace/Configuration/TracerSettings.cs‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -205,11 +205,11 @@ not null when string.Equals(value, "otlp", StringComparison.OrdinalIgnoreCase) =
205205
OtlpMetricsProtocol = config
206206
.WithKeys(ConfigurationKeys.OpenTelemetry.ExporterOtlpMetricsProtocol, ConfigurationKeys.OpenTelemetry.ExporterOtlpProtocol)
207207
.GetAs(
208-
defaultValue: new(OtlpProtocol.HttpProtobuf, "http/protobuf"),
208+
defaultValue: new(OtlpProtocol.Grpc, "grpc"),
209209
converter: x => x switch
210210
{
211-
not null when string.Equals(x, "http/protobuf", StringComparison.OrdinalIgnoreCase) => OtlpProtocol.HttpProtobuf,
212211
not null when string.Equals(x, "grpc", StringComparison.OrdinalIgnoreCase) => OtlpProtocol.Grpc,
212+
not null when string.Equals(x, "http/protobuf", StringComparison.OrdinalIgnoreCase) => OtlpProtocol.HttpProtobuf,
213213
not null when string.Equals(x, "http/json", StringComparison.OrdinalIgnoreCase) => OtlpProtocol.HttpJson,
214214
_ => UnsupportedOtlpProtocol(inputValue: x ?? "null"),
215215
},
@@ -1279,7 +1279,7 @@ internal string GetDefaultHttpClientExclusions()
12791279

12801280
private static ParsingResult<OtlpProtocol> UnsupportedOtlpProtocol(string inputValue)
12811281
{
1282-
Log.Warning("Unsupported OTLP protocol '{Protocol}'. Supported values are 'http/protobuf', 'grpc', 'http/json'. Using default: http/protobuf", inputValue);
1282+
Log.Warning("Unsupported OTLP protocol '{Protocol}'. Supported values are 'grpc', 'http/protobuf' and 'http/json'. Using default: http/protobuf", inputValue);
12831283
return ParsingResult<OtlpProtocol>.Failure();
12841284
}
12851285

0 commit comments

Comments
 (0)