Validate every event created by the elasticapm_client test client (#1091)

* Validate every event created by the elasticapm_client test client

So far, only when using sending_elasticapm_client, events were validated.
This made us miss things like #1088 and #1089. With this change,
every single event is validated against the apm-server JSONSchema.

* fix encoding error in tornado revealed by validation

Author: beniwohli

elasticapm/contrib/tornado/utils.py

@@ -60,7 +60,7 @@ def get_data_from_request(request_handler, request, config, event_type):
             try:
                 body = tornado.escape.json_decode(request.body)
             except Exception:
-                body = request.body
+                body = str(request.body, errors="ignore")
 
             if body is not None:
                 result["body"] = body if config.capture_body in ("all", event_type) else "[REDACTED]"

tests/contrib/asyncio/tornado/tornado_tests.py

@@ -220,10 +220,10 @@ async def test_capture_headers_body_is_dynamic(app, base_url, http_client):
     errors = elasticapm_client.events[constants.ERROR]
 
     assert "headers" in transactions[0]["context"]["request"]
-    assert transactions[0]["context"]["request"]["body"] == b"xyz"
+    assert transactions[0]["context"]["request"]["body"] == "xyz"
     assert "headers" in transactions[0]["context"]["response"]
     assert "headers" in errors[0]["context"]["request"]
-    assert errors[0]["context"]["request"]["body"] == b"xyz"
+    assert errors[0]["context"]["request"]["body"] == "xyz"
 
     assert "headers" not in transactions[2]["context"]["request"]
     assert "headers" not in transactions[2]["context"]["response"]

tests/fixtures.py

@@ -67,12 +67,15 @@
 ERRORS_SCHEMA = os.path.join(cur_dir, "upstream", "json-specs", "error.json")
 TRANSACTIONS_SCHEMA = os.path.join(cur_dir, "upstream", "json-specs", "transaction.json")
 SPAN_SCHEMA = os.path.join(cur_dir, "upstream", "json-specs", "span.json")
+METRICSET_SCHEMA = os.path.join(cur_dir, "upstream", "json-specs", "metricset.json")
 METADATA_SCHEMA = os.path.join(cur_dir, "upstream", "json-specs", "metadata.json")
 
 
 with codecs.open(ERRORS_SCHEMA, encoding="utf8") as errors_json, codecs.open(
     TRANSACTIONS_SCHEMA, encoding="utf8"
 ) as transactions_json, codecs.open(SPAN_SCHEMA, encoding="utf8") as span_json, codecs.open(
+    METRICSET_SCHEMA, encoding="utf8"
+) as metricset_json, codecs.open(
     METADATA_SCHEMA, encoding="utf8"
 ) as metadata_json:
     VALIDATORS = {
@@ -95,6 +98,12 @@
                 base_uri="file:" + pathname2url(SPAN_SCHEMA), referrer="file:" + pathname2url(SPAN_SCHEMA)
             ),
         ),
+        "metricset": jsonschema.Draft4Validator(
+            json.load(metricset_json),
+            resolver=jsonschema.RefResolver(
+                base_uri="file:" + pathname2url(METRICSET_SCHEMA), referrer="file:" + pathname2url(METRICSET_SCHEMA)
+            ),
+        ),
         "metadata": jsonschema.Draft4Validator(
             json.load(metadata_json),
             resolver=jsonschema.RefResolver(
@@ -187,6 +196,7 @@ def elasticapm_client(request):
     sys.excepthook = original_exceptionhook
     execution_context.set_transaction(None)
     execution_context.set_span(None)
+    assert not client._transport.validation_errors
 
 
 @pytest.fixture()
@@ -295,12 +305,19 @@ class DummyTransport(HTTPTransportBase):
     def __init__(self, url, *args, **kwargs):
         super(DummyTransport, self).__init__(url, *args, **kwargs)
         self.events = defaultdict(list)
+        self.validation_errors = defaultdict(list)
 
     def queue(self, event_type, data, flush=False):
         self._flushed.clear()
         data = self._process_event(event_type, data)
         self.events[event_type].append(data)
         self._flushed.set()
+        if data is not None:
+            validator = VALIDATORS[event_type]
+            try:
+                validator.validate(data)
+            except jsonschema.ValidationError as e:
+                self.validation_errors[event_type].append(e.message)
 
     def start_thread(self, pid=None):
         # don't call the parent method, but the one from ThreadManager