Make encoding values in UTF-8 more robust

Do not crash on invalid unicode values

Author: adikus

lib/logtail-rack/http_context.rb

@@ -1,6 +1,7 @@
 require "logtail/contexts/http"
 require "logtail/current_context"
 require "logtail-rack/middleware"
+require "logtail-rack/util/encoding"
 require "logtail-rack/util/request"
 
 module Logtail
@@ -11,10 +12,10 @@ class HTTPContext < Middleware
         def call(env)
           request = Util::Request.new(env)
           context = Contexts::HTTP.new(
-            host: request.host.force_encoding('UTF-8'),
-            method: request.request_method.force_encoding('UTF-8'),
+            host: Util::Encoding.force_utf8_encoding(request),
+            method: Util::Encoding.force_utf8_encoding(request.request_method),
             path: request.path,
-            remote_addr: request.ip.force_encoding('UTF-8'),
+            remote_addr: Util::Encoding.force_utf8_encoding(request.ip),
             request_id: request.request_id
           )
 

lib/logtail-rack/http_events.rb

@@ -6,6 +6,7 @@
 require "logtail-rack/http_request"
 require "logtail-rack/http_response"
 require "logtail-rack/middleware"
+require "logtail-rack/util/encoding"
 
 module Logtail
   module Integrations
@@ -179,13 +180,13 @@ def call(env)
                 body: event_body,
                 content_length: safe_to_i(request.content_length),
                 headers: filter_http_headers(request.headers),
-                host: force_encoding(request.host),
+                host: Util::Encoding.force_utf8_encoding(request.host),
                 method: request.request_method,
                 path: request.path,
                 port: request.port,
-                query_string: force_encoding(request.query_string),
+                query_string: Util::Encoding.force_utf8_encoding(request.query_string),
                 request_id: request.request_id,
-                scheme: force_encoding(request.scheme),
+                scheme: Util::Encoding.force_utf8_encoding(request.scheme),
               )
 
               {
@@ -277,14 +278,6 @@ def filter_http_headers(headers)
           def safe_to_i(val)
             val.nil? ? nil : val.to_i
           end
-
-          def force_encoding(value)
-            if value.respond_to?(:force_encoding)
-              value.dup.force_encoding('UTF-8')
-            else
-              value
-            end
-          end
       end
     end
   end

lib/logtail-rack/util/encoding.rb

@@ -3,7 +3,10 @@ module Util
     class Encoding
       def self.force_utf8_encoding(data)
         if data.respond_to?(:force_encoding)
-          data.dup.force_encoding('UTF-8')
+          encoded_data = data.dup.force_encoding('UTF-8')
+          encoded_data = data.dup.force_encoding("ISO-8859-1").encode("UTF-8") unless encoded_data.valid_encoding?
+          encoded_data = data.dup.encode('UTF-8', invalid: :replace, undef: :replace) unless encoded_data.valid_encoding?
+          encoded_data
         elsif data.respond_to?(:transform_values)
           data.transform_values { |val| Logtail::Util::Encoding.force_utf8_encoding(val) }
         else