Enable log forwarding by default. (#587)

* Enable log forwarding by default. (#583)

* Enable log forwarding by default.

* Fix catmap fixture for log forwarding compatibility.

* Use isinstance check in fixtures.

* Update data usage supportability metrics. (#582)

* Update data usage supportability metric name.

* Fix compression bug and add testing.

Co-authored-by: Lalleh Rafeei <[email protected]>
Co-authored-by: Nyenty Ayuk-Enow <[email protected]>

* Update harvest_loop tests.

* Update compression test to check method1 payload length for consistency.

Co-authored-by: Lalleh Rafeei <[email protected]>
Co-authored-by: Nyenty Ayuk-Enow <[email protected]>

Co-authored-by: Lalleh Rafeei <[email protected]>
Co-authored-by: Nyenty Ayuk-Enow <[email protected]>

Author: 3 people

newrelic/common/agent_http.py

@@ -524,24 +524,32 @@ def _supportability_request(params, payload, body, compression_time):
         # *********
         # Used only for supportability metrics. Do not use to drive business
         # logic!
+        # payload: uncompressed
+        # body: compressed
         agent_method = params and params.get("method")
         # *********
 
-        if agent_method and body:
+        if agent_method and payload:
             # Compression was applied
             if compression_time is not None:
                 internal_metric(
-                    "Supportability/Python/Collector/ZLIB/Bytes/%s" % agent_method,
-                    len(payload),
+                    "Supportability/Python/Collector/%s/ZLIB/Bytes" % agent_method,
+                    len(body),
                 )
                 internal_metric(
-                    "Supportability/Python/Collector/ZLIB/Compress/%s" % agent_method,
+                    "Supportability/Python/Collector/ZLIB/Bytes", len(body)
+                )
+                internal_metric(
+                    "Supportability/Python/Collector/%s/ZLIB/Compress" % agent_method,
                     compression_time,
                 )
-
             internal_metric(
-                "Supportability/Python/Collector/Output/Bytes/%s" % agent_method,
-                len(body),
+                "Supportability/Python/Collector/%s/Output/Bytes" % agent_method,
+                len(payload),
+            )
+            # Top level metric to aggregate overall bytes being sent
+            internal_metric(
+                "Supportability/Python/Collector/Output/Bytes", len(payload)
             )
 
     @staticmethod

newrelic/core/config.py

@@ -812,7 +812,7 @@ def default_host(license_key):
 
 _settings.application_logging.enabled = _environ_as_bool("NEW_RELIC_APPLICATION_LOGGING_ENABLED", default=True)
 _settings.application_logging.forwarding.enabled = _environ_as_bool(
-    "NEW_RELIC_APPLICATION_LOGGING_FORWARDING_ENABLED", default=False
+    "NEW_RELIC_APPLICATION_LOGGING_FORWARDING_ENABLED", default=True
 )
 _settings.application_logging.metrics.enabled = _environ_as_bool(
     "NEW_RELIC_APPLICATION_LOGGING_METRICS_ENABLED", default=True

tests/agent_unittests/test_harvest_loop.py

@@ -896,7 +896,7 @@ def test_default_events_harvested(allowlist_event):
     num_seen = 0 if (allowlist_event != "span_event_data") else 1
     assert app._stats_engine.span_events.num_seen == num_seen
 
-    assert app._stats_engine.metrics_count() == 1
+    assert app._stats_engine.metrics_count() == 4
 
 
 @failing_endpoint("analytic_event_data")

tests/agent_unittests/test_http_client.py

@@ -289,32 +289,47 @@ def test_http_payload_compression(server, client_cls, method, threshold):
         compression_threshold=threshold,
     ) as client:
         with InternalTraceContext(internal_metrics):
-            status, data = client.send_request(payload=payload, params={"method": "test"})
+            status, data = client.send_request(payload=payload, params={"method": "method1"})
+
+    # Sending one additional request to valid metric aggregation for top level data usage supportability metrics
+    with client_cls(
+        "localhost",
+        server.port,
+        disable_certificate_validation=True,
+        compression_method=method,
+        compression_threshold=threshold,
+    ) as client:
+        with InternalTraceContext(internal_metrics):
+            status, data = client.send_request(payload=payload, params={"method": "method2"})
 
     assert status == 200
     data = data.split(b"\n")
     sent_payload = data[-1]
     payload_byte_len = len(sent_payload)
-
     internal_metrics = dict(internal_metrics.metrics())
     if client_cls is ApplicationModeClient:
-        assert internal_metrics["Supportability/Python/Collector/Output/Bytes/test"][:2] == [
+        assert internal_metrics["Supportability/Python/Collector/method1/Output/Bytes"][:2] == [
             1,
-            payload_byte_len,
+            len(payload),
+        ]
+        assert internal_metrics["Supportability/Python/Collector/Output/Bytes"][:2] == [
+            2,
+            len(payload)*2,
         ]
 
         if threshold < 20:
             # Verify compression time is recorded
-            assert internal_metrics["Supportability/Python/Collector/ZLIB/Compress/test"][0] == 1
-            assert internal_metrics["Supportability/Python/Collector/ZLIB/Compress/test"][1] > 0
-
-            # Verify the original payload length is recorded
-            assert internal_metrics["Supportability/Python/Collector/ZLIB/Bytes/test"][:2] == [1, len(payload)]
-
-            assert len(internal_metrics) == 3
+            assert internal_metrics["Supportability/Python/Collector/method1/ZLIB/Compress"][0] == 1
+            assert internal_metrics["Supportability/Python/Collector/method1/ZLIB/Compress"][1] > 0
+
+            # Verify the compressed payload length is recorded
+            assert internal_metrics["Supportability/Python/Collector/method1/ZLIB/Bytes"][:2] == [1, payload_byte_len]
+            assert internal_metrics["Supportability/Python/Collector/ZLIB/Bytes"][:2] == [2, payload_byte_len*2]
+            
+            assert len(internal_metrics) == 8
         else:
             # Verify no ZLIB compression metrics were sent
-            assert len(internal_metrics) == 1
+            assert len(internal_metrics) == 3
     else:
         assert not internal_metrics
 

tests/testing_support/fixtures.py

@@ -2625,8 +2625,8 @@ def _validate_analytics_sample_data(wrapped, instance, args, kwargs):
         _new_wrapped = _capture_samples(wrapped)
 
         result = _new_wrapped(*args, **kwargs)
-
-        _samples = [s for s in samples if s[0]["type"] == "Transaction"]
+        # Check type of s[0] because it returns an integer if s is a LogEventNode
+        _samples = [s for s in samples if not isinstance(s[0], int) and s[0]["type"] == "Transaction"]
         assert _samples, "No Transaction events captured."
         for sample in _samples:
             assert isinstance(sample, list)