From 6b6bf08ae01492f16cfeb81858af6f8acfc60845 Mon Sep 17 00:00:00 2001 From: Riccardo Magliocchetti Date: Mon, 17 Feb 2025 14:52:49 +0100 Subject: [PATCH 1/2] elasticapm: properly cleanup buffer and data views With Python 3.13 our pattern of buffering revealed some issues because the underlying BytesIO fileobj may get released before gzip.GzipFile. This requires a fix in CPython but also some improvements on our side by properly closing the GzipFile in case of error and also releasing the memoryview we can from the BytesIO buffer. These problems manifests as following warnings from unraisable exceptions running tests: The closing of the gzip buffer helps with: Traceback (most recent call last): File "/usr/lib/python3.13/gzip.py", line 362, in close fileobj.write(self.compress.flush()) ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^ ValueError: I/O operation on closed file. Exception ignored in: <_io.BytesIO object at 0x7fbc4335fbf0> Traceback (most recent call last): File "/venv313/lib/python3.13/site-packages/ecs_logging/_stdlib.py", line 272, in _record_attribute def _record_attribute( BufferError: Existing exports of data: object cannot be re-sized Python 3.12 shows the same warnings with the `-X dev` flag. --- elasticapm/transport/base.py | 3 +++ tests/transports/test_base.py | 17 ++++++++++++----- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/elasticapm/transport/base.py b/elasticapm/transport/base.py index 24911c395..b81960907 100644 --- a/elasticapm/transport/base.py +++ b/elasticapm/transport/base.py @@ -250,6 +250,7 @@ def _flush(self, buffer, forced_flush=False) -> None: """ if not self.state.should_try(): logger.error("dropping flushed data due to transport failure back-off") + buffer.close() else: fileobj = buffer.fileobj # get a reference to the fileobj before closing the gzip file buffer.close() @@ -261,6 +262,8 @@ def _flush(self, buffer, forced_flush=False) -> None: except Exception as e: self.handle_transport_fail(e) + data.release() + def start_thread(self, pid=None) -> None: super(Transport, self).start_thread(pid=pid) if (not self._thread or self.pid != self._thread.pid) and not self._closed: diff --git a/tests/transports/test_base.py b/tests/transports/test_base.py index 457f68613..fca033fac 100644 --- a/tests/transports/test_base.py +++ b/tests/transports/test_base.py @@ -107,18 +107,25 @@ def test_empty_queue_flush(mock_send, elasticapm_client): transport.close() -@mock.patch("elasticapm.transport.base.Transport.send") +@mock.patch("elasticapm.transport.base.Transport._flush") @pytest.mark.parametrize("elasticapm_client", [{"api_request_time": "5s"}], indirect=True) -def test_metadata_prepended(mock_send, elasticapm_client): +def test_metadata_prepended(mock_flush, elasticapm_client): transport = Transport(client=elasticapm_client, compress_level=0) transport.start_thread() transport.queue("error", {}, flush=True) transport.close() - assert mock_send.call_count == 1 - args, kwargs = mock_send.call_args - data = gzip.decompress(args[0]) + assert mock_flush.call_count == 1 + args, kwargs = mock_flush.call_args + buffer = args[0] + # this test used to mock send but after we fixed a leak for not releasing the memoryview containing + # the gzipped data we cannot read it anymore. So reimplement _flush and read the data ourselves + fileobj = buffer.fileobj + buffer.close() + compressed_data = fileobj.getbuffer() + data = gzip.decompress(compressed_data) data = data.decode("utf-8").split("\n") assert "metadata" in data[0] + compressed_data.release() @mock.patch("elasticapm.transport.base.Transport.send") From 0545d164fe84e5239e5470c8df7dc1b2a4ec61e2 Mon Sep 17 00:00:00 2001 From: Riccardo Magliocchetti Date: Fri, 7 Feb 2025 11:21:07 +0100 Subject: [PATCH 2/2] Fix get_name_from_func for partialmethod in Python 3.13 Handle _partialmethod moving to __partialmethod__ in Python 3.13 --- elasticapm/utils/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/elasticapm/utils/__init__.py b/elasticapm/utils/__init__.py index 0f7b52c0d..4403f5abd 100644 --- a/elasticapm/utils/__init__.py +++ b/elasticapm/utils/__init__.py @@ -78,6 +78,8 @@ def get_name_from_func(func: FunctionType) -> str: return "partial({})".format(get_name_from_func(func.func)) elif hasattr(func, "_partialmethod") and hasattr(func._partialmethod, "func"): return "partial({})".format(get_name_from_func(func._partialmethod.func)) + elif hasattr(func, "__partialmethod__") and hasattr(func.__partialmethod__, "func"): + return "partial({})".format(get_name_from_func(func.__partialmethod__.func)) module = func.__module__