add better estimation for non-ascii characters

Author: liustve

aws-opentelemetry-distro/src/amazon/opentelemetry/distro/exporter/otlp/aws/logs/aws_batch_log_record_processor.py

@@ -175,12 +175,12 @@ def _estimate_log_size(self, log: LogData, depth: int = 3) -> int:  # pylint: di
                 if next_val is None:
                     continue
 
-                if isinstance(next_val, (str, bytes)):
+                if isinstance(next_val, bytes):
                     size += len(next_val)
                     continue
 
-                if isinstance(next_val, (float, int, bool)):
-                    size += len(str(next_val))
+                if isinstance(next_val, (str, float, int, bool)):
+                    size += AwsCloudWatchOtlpBatchLogRecordProcessor._estimate_utf8_size(str(next_val))
                     continue
 
                 # next_val must be Sequence["AnyValue"] or Mapping[str, "AnyValue"]
@@ -210,3 +210,17 @@ def _estimate_log_size(self, log: LogData, depth: int = 3) -> int:  # pylint: di
             queue = new_queue
 
         return size
+
+    @staticmethod
+    def _estimate_utf8_size(s: str):
+        ascii_count = 0
+        non_ascii_count = 0
+
+        for char in s:
+            if ord(char) < 128:
+                ascii_count += 1
+            else:
+                non_ascii_count += 1
+
+        # Estimate: ASCII chars (1 byte) + upper bound of non-ASCII chars 4 bytes
+        return ascii_count + (non_ascii_count * 4)

aws-opentelemetry-distro/tests/amazon/opentelemetry/distro/exporter/otlp/aws/logs/test_aws_batch_log_record_processor.py

@@ -116,8 +116,8 @@ def test_process_log_data_nested_structure_size_exceeds_max_log_size(self):
 
     def test_process_log_data_primitive(self):
 
-        primitives: List[AnyValue] = ["test", b"test", 1, 1.2, True, False, None]
-        expected_sizes = [4, 4, 1, 3, 4, 5, 0]
+        primitives: List[AnyValue] = ["test", b"test", 1, 1.2, True, False, None, "深入 Python", "café"]
+        expected_sizes = [4, 4, 1, 3, 4, 5, 0, 2 * 4 + len(" Python"), 1 * 4 + len("caf")]
 
         for index, primitive in enumerate(primitives):
             log = self.generate_test_log_data(log_body=primitive, count=1)