feat: HTTPX semantic convention stability opt in

instrumentation/httpx/Appraisals

@@ -4,6 +4,15 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 
-appraise 'httpx-1' do
-  gem 'httpx', '~> 1.0'
+# To faclitate HTTP semantic convention stability migration, we are using
+# appraisal to test the different semantic convention modes along with different
+# gem versions. For more information on the semantic convention modes, see:
+# https://opentelemetry.io/docs/specs/semconv/non-normative/http-migration/
+
+semconv_stability = %w[dup stable old]
+
+semconv_stability.each do |mode|
+  appraise "httpx-1-#{mode}" do
+    gem 'httpx', '~> 1.0'
+  end
 end

instrumentation/httpx/README.md

@@ -48,3 +48,19 @@ The `opentelemetry-instrumentation-httpx` gem is distributed under the Apache 2.
 [slack-channel]: https://cloud-native.slack.com/archives/C01NWKKMKMY
 [discussions-url]: https://github.com/open-telemetry/opentelemetry-ruby/discussions
 [httpx-home]: https://github.com/HoneyryderChuck/httpx
+
+## HTTP semantic convention stability
+
+In the OpenTelemetry ecosystem, HTTP semantic conventions have now reached a stable state. However, the initial HTTPX instrumentation was introduced before this stability was achieved, which resulted in HTTP attributes being based on an older version of the semantic conventions.
+
+To facilitate the migration to stable semantic conventions, you can use the `OTEL_SEMCONV_STABILITY_OPT_IN` environment variable. This variable allows you to opt-in to the new stable conventions, ensuring compatibility and future-proofing your instrumentation.
+
+When setting the value for `OTEL_SEMCONV_STABILITY_OPT_IN`, you can specify which conventions you wish to adopt:
+
+- `http` - Emits the stable HTTP and networking conventions and ceases emitting the old conventions previously emitted by the instrumentation.
+- `http/dup` - Emits both the old and stable HTTP and networking conventions, enabling a phased rollout of the stable semantic conventions.
+- Default behavior (in the absence of either value) is to continue emitting the old HTTP and networking conventions the instrumentation previously emitted.
+
+During the transition from old to stable conventions, HTTPX instrumentation code comes in three patch versions: `dup`, `old`, and `stable`. These versions are identical except for the attributes they send. Any changes to HTTPX instrumentation should consider all three patches.
+
+For additional information on migration, please refer to our [documentation](https://opentelemetry.io/docs/specs/semconv/non-normative/http-migration/).

instrumentation/httpx/lib/opentelemetry/instrumentation/httpx/dup/plugin.rb

@@ -0,0 +1,136 @@
+# frozen_string_literal: true
+
+# Copyright The OpenTelemetry Authors
+#
+# SPDX-License-Identifier: Apache-2.0
+
+module OpenTelemetry
+  module Instrumentation
+    module HTTPX
+      module Dup
+        module Plugin
+          # Instruments around HTTPX's request/response lifecycle in order to generate
+          # an OTEL trace.
+          module RequestTracer
+            module_function
+
+            # initializes tracing on the +request+.
+            def call(request)
+              span = nil
+
+              # request objects are reused, when already buffered requests get rerouted to a different
+              # connection due to connection issues, or when they already got a response, but need to
+              # be retried. In such situations, the original span needs to be extended for the former,
+              # while a new is required for the latter.
+              request.on(:idle) do
+                span = nil
+              end
+              # the span is initialized when the request is buffered in the parser, which is the closest
+              # one gets to actually sending the request.
+              request.on(:headers) do
+                next if span
+
+                span = initialize_span(request)
+              end
+
+              request.on(:response) do |response|
+                unless span
+                  next unless response.is_a?(::HTTPX::ErrorResponse) && response.error.respond_to?(:connection)
+
+                  # handles the case when the +error+ happened during name resolution, which means
+                  # that the tracing start point hasn't been triggered yet; in such cases, the approximate
+                  # initial resolving time is collected from the connection, and used as span start time,
+                  # and the tracing object in inserted before the on response callback is called.
+                  span = initialize_span(request, response.error.connection.init_time)
+
+                end
+
+                finish(response, span)
+              end
+            end
+
+            def finish(response, span)
+              if response.is_a?(::HTTPX::ErrorResponse)
+                span.record_exception(response.error)
+                span.status = Trace::Status.error(response.error.to_s)
+              else
+                span.set_attribute(OpenTelemetry::SemanticConventions::Trace::HTTP_STATUS_CODE, response.status)
+                span.set_attribute('http.response.status_code', response.status)
+
+                if response.status.between?(400, 599)
+                  err = ::HTTPX::HTTPError.new(response)
+                  span.record_exception(err)
+                  span.status = Trace::Status.error(err.to_s)
+                end
+              end
+
+              span.finish
+            end
+
+            # return a span initialized with the +@request+ state.
+            def initialize_span(request, start_time = ::Time.now)
+              verb = request.verb
+              uri = request.uri
+
+              config = HTTPX::Instrumentation.instance.config
+
+              attributes = {
+                OpenTelemetry::SemanticConventions::Trace::HTTP_HOST => uri.host,
+                OpenTelemetry::SemanticConventions::Trace::HTTP_METHOD => verb,
+                OpenTelemetry::SemanticConventions::Trace::HTTP_SCHEME => uri.scheme,
+                OpenTelemetry::SemanticConventions::Trace::HTTP_TARGET => uri.path,
+                OpenTelemetry::SemanticConventions::Trace::HTTP_URL => "#{uri.scheme}://#{uri.host}",
+                OpenTelemetry::SemanticConventions::Trace::NET_PEER_NAME => uri.host,
+                OpenTelemetry::SemanticConventions::Trace::NET_PEER_PORT => uri.port,
+                'http.request.method' => verb,
+                'url.scheme' => uri.scheme,
+                'url.path' => uri.path,
+                'url.full' => "#{uri.scheme}://#{uri.host}",
+                'server.address' => uri.host,
+                'server.port' => uri.port
+              }
+
+              attributes['url.query'] = uri.query unless uri.query.nil?
+              attributes[OpenTelemetry::SemanticConventions::Trace::PEER_SERVICE] = config[:peer_service] if config[:peer_service]
+              attributes.merge!(OpenTelemetry::Common::HTTP::ClientContext.attributes)
+
+              span = tracer.start_span(verb, attributes: attributes, kind: :client, start_timestamp: start_time)
+
+              OpenTelemetry::Trace.with_span(span) do
+                OpenTelemetry.propagation.inject(request.headers)
+              end
+
+              span
+            rescue StandardError => e
+              OpenTelemetry.handle_error(exception: e)
+            end
+
+            def tracer
+              HTTPX::Instrumentation.instance.tracer
+            end
+          end
+
+          # Request patch to initiate the trace on initialization.
+          module RequestMethods
+            def initialize(*)
+              super
+
+              RequestTracer.call(self)
+            end
+          end
+
+          # Connection patch to start monitoring on initialization.
+          module ConnectionMethods
+            attr_reader :init_time
+
+            def initialize(*)
+              super
+
+              @init_time = ::Time.now
+            end
+          end
+        end
+      end
+    end
+  end
+end

instrumentation/httpx/lib/opentelemetry/instrumentation/httpx/instrumentation.rb

@@ -10,8 +10,9 @@ module HTTPX
       # The Instrumentation class contains logic to detect and install the Http instrumentation
       class Instrumentation < OpenTelemetry::Instrumentation::Base
         install do |_config|
-          require_dependencies
-          patch
+          patch_type = determine_semconv
+          send(:"require_dependencies_#{patch_type}")
+          send(:"patch_#{patch_type}")
         end
 
         compatible do
@@ -24,15 +25,50 @@ class Instrumentation < OpenTelemetry::Instrumentation::Base
 
         option :peer_service, default: nil, validate: :string
 
-        def patch
-          otel_session = ::HTTPX.plugin(Plugin)
+        def determine_semconv
+          stability_opt_in = ENV.fetch('OTEL_SEMCONV_STABILITY_OPT_IN', '')
+          values = stability_opt_in.split(',').map(&:strip)
+
+          if values.include?('http/dup')
+            'dup'
+          elsif values.include?('http')
+            'stable'
+          else
+            'old'
+          end
+        end
+
+        def patch_old
+          otel_session = ::HTTPX.plugin(Old::Plugin)
 
           ::HTTPX.send(:remove_const, :Session)
           ::HTTPX.send(:const_set, :Session, otel_session.class)
         end
 
-        def require_dependencies
-          require_relative 'plugin'
+        def patch_stable
+          otel_session = ::HTTPX.plugin(Stable::Plugin)
+
+          ::HTTPX.send(:remove_const, :Session)
+          ::HTTPX.send(:const_set, :Session, otel_session.class)
+        end
+
+        def patch_dup
+          otel_session = ::HTTPX.plugin(Dup::Plugin)
+
+          ::HTTPX.send(:remove_const, :Session)
+          ::HTTPX.send(:const_set, :Session, otel_session.class)
+        end
+
+        def require_dependencies_old
+          require_relative 'old/plugin'
+        end
+
+        def require_dependencies_stable
+          require_relative 'stable/plugin'
+        end
+
+        def require_dependencies_dup
+          require_relative 'dup/plugin'
         end
       end
     end

instrumentation/httpx/lib/opentelemetry/instrumentation/httpx/old/plugin.rb

@@ -0,0 +1,128 @@
+# frozen_string_literal: true
+
+# Copyright The OpenTelemetry Authors
+#
+# SPDX-License-Identifier: Apache-2.0
+
+module OpenTelemetry
+  module Instrumentation
+    module HTTPX
+      module Old
+        module Plugin
+          # Instruments around HTTPX's request/response lifecycle in order to generate
+          # an OTEL trace.
+          module RequestTracer
+            module_function
+
+            # initializes tracing on the +request+.
+            def call(request)
+              span = nil
+
+              # request objects are reused, when already buffered requests get rerouted to a different
+              # connection due to connection issues, or when they already got a response, but need to
+              # be retried. In such situations, the original span needs to be extended for the former,
+              # while a new is required for the latter.
+              request.on(:idle) do
+                span = nil
+              end
+              # the span is initialized when the request is buffered in the parser, which is the closest
+              # one gets to actually sending the request.
+              request.on(:headers) do
+                next if span
+
+                span = initialize_span(request)
+              end
+
+              request.on(:response) do |response|
+                unless span
+                  next unless response.is_a?(::HTTPX::ErrorResponse) && response.error.respond_to?(:connection)
+
+                  # handles the case when the +error+ happened during name resolution, which means
+                  # that the tracing start point hasn't been triggered yet; in such cases, the approximate
+                  # initial resolving time is collected from the connection, and used as span start time,
+                  # and the tracing object in inserted before the on response callback is called.
+                  span = initialize_span(request, response.error.connection.init_time)
+
+                end
+
+                finish(response, span)
+              end
+            end
+
+            def finish(response, span)
+              if response.is_a?(::HTTPX::ErrorResponse)
+                span.record_exception(response.error)
+                span.status = Trace::Status.error(response.error.to_s)
+              else
+                span.set_attribute(OpenTelemetry::SemanticConventions::Trace::HTTP_STATUS_CODE, response.status)
+
+                if response.status.between?(400, 599)
+                  err = ::HTTPX::HTTPError.new(response)
+                  span.record_exception(err)
+                  span.status = Trace::Status.error(err.to_s)
+                end
+              end
+
+              span.finish
+            end
+
+            # return a span initialized with the +@request+ state.
+            def initialize_span(request, start_time = ::Time.now)
+              verb = request.verb
+              uri = request.uri
+
+              config = HTTPX::Instrumentation.instance.config
+
+              attributes = {
+                OpenTelemetry::SemanticConventions::Trace::HTTP_HOST => uri.host,
+                OpenTelemetry::SemanticConventions::Trace::HTTP_METHOD => verb,
+                OpenTelemetry::SemanticConventions::Trace::HTTP_SCHEME => uri.scheme,
+                OpenTelemetry::SemanticConventions::Trace::HTTP_TARGET => uri.path,
+                OpenTelemetry::SemanticConventions::Trace::HTTP_URL => "#{uri.scheme}://#{uri.host}",
+                OpenTelemetry::SemanticConventions::Trace::NET_PEER_NAME => uri.host,
+                OpenTelemetry::SemanticConventions::Trace::NET_PEER_PORT => uri.port
+              }
+
+              attributes[OpenTelemetry::SemanticConventions::Trace::PEER_SERVICE] = config[:peer_service] if config[:peer_service]
+              attributes.merge!(OpenTelemetry::Common::HTTP::ClientContext.attributes)
+
+              span = tracer.start_span("HTTP #{verb}", attributes: attributes, kind: :client, start_timestamp: start_time)
+
+              OpenTelemetry::Trace.with_span(span) do
+                OpenTelemetry.propagation.inject(request.headers)
+              end
+
+              span
+            rescue StandardError => e
+              OpenTelemetry.handle_error(exception: e)
+            end
+
+            def tracer
+              HTTPX::Instrumentation.instance.tracer
+            end
+          end
+
+          # Request patch to initiate the trace on initialization.
+          module RequestMethods
+            def initialize(*)
+              super
+
+              RequestTracer.call(self)
+            end
+          end
+
+          # Connection patch to start monitoring on initialization.
+          module ConnectionMethods
+            attr_reader :init_time
+
+            def initialize(*)
+              super
+
+              @init_time = ::Time.now
+            end
+          end
+        end
+      end
+    end
+  end
+end