fix: decode bytes strings when using JSON encoder (backport #3117) (#3177)

This is an automatic backport of pull request #3117 done by [Mergify](https://mergify.com).


---


<details>
<summary>Mergify commands and options</summary>

<br />

More conditions and actions can be found in the [documentation](https://docs.mergify.com/).

You can also trigger Mergify actions by commenting on this pull request:

- `@Mergifyio refresh` will re-evaluate the rules
- `@Mergifyio rebase` will rebase this PR on its base branch
- `@Mergifyio update` will merge the base branch into this PR
- `@Mergifyio backport <destination>` will backport this PR on `<destination>` branch

Additionally, on Mergify [dashboard](https://dashboard.mergify.com/) you can:

- look at your merge queues
- generate the Mergify configuration with the config editor.

Finally, you can contact us on https://mergify.com
</details>

Author: mergify[bot]

ddtrace/internal/encoding.py

@@ -8,6 +8,9 @@
 from ._encoding import ListStringTable
 from ._encoding import MsgpackEncoderV03
 from ._encoding import MsgpackEncoderV05
+from .compat import PY3
+from .compat import binary_type
+from .compat import ensure_text
 from .logger import get_logger
 
 
@@ -48,17 +51,33 @@ def encode(self, obj):
         raise NotImplementedError()
 
 
-class JSONEncoder(_EncoderBase):
+class JSONEncoder(json.JSONEncoder, _EncoderBase):
     content_type = "application/json"
 
     def encode_traces(self, traces):
-        normalized_traces = [[span.to_dict() for span in trace] for trace in traces]
+        normalized_traces = [[JSONEncoder._normalize_span(span.to_dict()) for span in trace] for trace in traces]
         return self.encode(normalized_traces)
 
     @staticmethod
-    def encode(obj):
-        # type: (Any) -> str
-        return json.dumps(obj)
+    def _normalize_span(span):
+        # Ensure all string attributes are actually strings and not bytes
+        # DEV: We are deferring meta/metrics to reduce any performance issues.
+        #      Meta/metrics may still contain `bytes` and have encoding issues.
+        span["resource"] = JSONEncoder._normalize_str(span["resource"])
+        span["name"] = JSONEncoder._normalize_str(span["name"])
+        span["service"] = JSONEncoder._normalize_str(span["service"])
+        return span
+
+    @staticmethod
+    def _normalize_str(obj):
+        if obj is None:
+            return obj
+
+        if PY3:
+            return ensure_text(obj, errors="backslashreplace")
+        elif isinstance(obj, binary_type):
+            return obj.decode("utf-8", errors="replace")
+        return obj
 
 
 class JSONEncoderV2(JSONEncoder):
@@ -77,6 +96,7 @@ def encode_traces(self, traces):
     def _convert_span(span):
         # type: (Span) -> Dict[str, Any]
         sp = span.to_dict()
+        sp = JSONEncoderV2._normalize_span(sp)
         sp["trace_id"] = JSONEncoderV2._encode_id_to_hex(sp.get("trace_id"))
         sp["parent_id"] = JSONEncoderV2._encode_id_to_hex(sp.get("parent_id"))
         sp["span_id"] = JSONEncoderV2._encode_id_to_hex(sp.get("span_id"))

releasenotes/notes/fix-json-encoder-bytes-d35121d00f2b8caf.yaml

@@ -0,0 +1,4 @@
+---
+fixes:
+  - |
+    Fix JSON encoding error when a ``bytes`` string is used for span metadata.

tests/tracer/test_encoders.py

@@ -161,19 +161,20 @@ class TestEncoders(TestCase):
 
     def test_encode_traces_json(self):
         # test encoding for JSON format
-        traces = []
-        traces.append(
+        traces = [
             [
                 Span(name="client.testing", tracer=None),
                 Span(name="client.testing", tracer=None),
-            ]
-        )
-        traces.append(
+            ],
             [
                 Span(name="client.testing", tracer=None),
                 Span(name="client.testing", tracer=None),
-            ]
-        )
+            ],
+            [
+                Span(name=b"client.testing", tracer=None),
+                Span(name=b"client.testing", tracer=None),
+            ],
+        ]
 
         encoder = JSONEncoder()
         spans = encoder.encode_traces(traces)
@@ -182,39 +183,42 @@ def test_encode_traces_json(self):
         # test the encoded output that should be a string
         # and the output must be flatten
         assert isinstance(spans, string_type)
-        assert len(items) == 2
+        assert len(items) == 3
         assert len(items[0]) == 2
         assert len(items[1]) == 2
-        for i in range(2):
+        assert len(items[2]) == 2
+        for i in range(3):
             for j in range(2):
                 assert "client.testing" == items[i][j]["name"]
 
     def test_encode_traces_json_v2(self):
         # test encoding for JSON format
-        traces = []
-        traces.append(
+        traces = [
             [
                 Span(name="client.testing", tracer=None, span_id=0xAAAAAA),
                 Span(name="client.testing", tracer=None, span_id=0xAAAAAA),
-            ]
-        )
-        traces.append(
+            ],
             [
                 Span(name="client.testing", tracer=None, span_id=0xAAAAAA),
                 Span(name="client.testing", tracer=None, span_id=0xAAAAAA),
-            ]
-        )
+            ],
+            [
+                Span(name=b"client.testing", tracer=None, span_id=0xAAAAAA),
+                Span(name=b"client.testing", tracer=None, span_id=0xAAAAAA),
+            ],
+        ]
 
         encoder = JSONEncoderV2()
         spans = encoder.encode_traces(traces)
         items = json.loads(spans)["traces"]
         # test the encoded output that should be a string
         # and the output must be flatten
         assert isinstance(spans, string_type)
-        assert len(items) == 2
+        assert len(items) == 3
         assert len(items[0]) == 2
         assert len(items[1]) == 2
-        for i in range(2):
+        assert len(items[2]) == 2
+        for i in range(3):
             for j in range(2):
                 assert "client.testing" == items[i][j]["name"]
                 assert isinstance(items[i][j]["span_id"], string_type)
@@ -235,17 +239,24 @@ def test_encode_traces_msgpack_v03(self):
                 Span(name="client.testing", tracer=None),
             ]
         )
+        encoder.put(
+            [
+                Span(name=b"client.testing", tracer=None),
+                Span(name=b"client.testing", tracer=None),
+            ]
+        )
 
         spans = encoder.encode()
         items = encoder._decode(spans)
 
         # test the encoded output that should be a string
         # and the output must be flatten
         assert isinstance(spans, msgpack_type)
-        assert len(items) == 2
+        assert len(items) == 3
         assert len(items[0]) == 2
         assert len(items[1]) == 2
-        for i in range(2):
+        assert len(items[2]) == 2
+        for i in range(3):
             for j in range(2):
                 assert b"client.testing" == items[i][j][b"name"]
 
@@ -628,3 +639,53 @@ def run(self):
 
     unpacked = decode(encoder.encode(), reconstruct=True)
     assert unpacked is not None
+
+
+@pytest.mark.parametrize("encoder_cls", ["JSONEncoder", "JSONEncoderV2"])
+def test_json_encoder_traces_bytes(encoder_cls, run_python_code_in_subprocess):
+    """
+    Regression test for: https://github.com/DataDog/dd-trace-py/issues/3115
+
+    Ensure we properly decode `bytes` objects when encoding with the JSONEncoder
+    """
+    # Run test in a subprocess to test without setting file encoding to utf-8
+    code = """
+import json
+
+from ddtrace.internal.compat import PY3
+from ddtrace.internal.encoding import {0}
+from ddtrace.span import Span
+
+encoder = {0}()
+data = encoder.encode_traces(
+         [
+             [
+                 Span(name=b"\\x80span.a", tracer=None),
+                 Span(name=u"\\x80span.b", tracer=None),
+                 Span(name="\\x80span.b", tracer=None),
+             ]
+         ]
+    )
+traces = json.loads(data)
+if "{0}" == "JSONEncoderV2":
+    traces = traces["traces"]
+
+assert len(traces) == 1
+span_a, span_b, span_c = traces[0]
+
+if PY3:
+    assert "\\\\x80span.a" == span_a["name"]
+    assert u"\\x80span.b" == span_b["name"]
+    assert u"\\x80span.b" == span_c["name"]
+else:
+    assert u"\\ufffdspan.a" == span_a["name"]
+    assert u"\\x80span.b" == span_b["name"]
+    assert u"\\ufffdspan.b" == span_c["name"]
+""".format(
+        encoder_cls
+    )
+
+    out, err, status, pid = run_python_code_in_subprocess(code)
+    assert status == 0, err
+    assert out == b""
+    assert err == b""