From a2778a8128911c2eaf41ad7d9e42135b1ed3bb16 Mon Sep 17 00:00:00 2001 From: Piotr Gabryjeluk Date: Thu, 11 Sep 2025 22:05:28 +0200 Subject: [PATCH 1/7] Make a warning class for the type inference warnings --- src/neptune_query/internal/composition/type_inference.py | 4 ++-- src/neptune_query/warnings.py | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/neptune_query/internal/composition/type_inference.py b/src/neptune_query/internal/composition/type_inference.py index b580f1b5..270ea607 100644 --- a/src/neptune_query/internal/composition/type_inference.py +++ b/src/neptune_query/internal/composition/type_inference.py @@ -28,6 +28,7 @@ from neptune_api.client import AuthenticatedClient from ...exceptions import AttributeTypeInferenceError +from ...warnings import AttributeWarning from .. import ( filters, identifiers, @@ -156,8 +157,7 @@ def emit_warnings(self) -> None: for attr_state in self.attributes: if attr_state.warning_text: msg = f"Attribute '{attr_state.original_attribute.name}': {attr_state.warning_text}" - # TODO: Add category to warnings.py - warnings.warn(msg, stacklevel=3) + warnings.warn(msg, category=AttributeWarning, stacklevel=3) def infer_attribute_types_in_filter( diff --git a/src/neptune_query/warnings.py b/src/neptune_query/warnings.py index da0e6588..1b45768a 100644 --- a/src/neptune_query/warnings.py +++ b/src/neptune_query/warnings.py @@ -16,3 +16,7 @@ class ExperimentalWarning(UserWarning): """Warning for use of experimental API elements.""" + + +class AttributeWarning(UserWarning): + """Warning for attribute issues.""" From 5ea4b578081df9fa47dcdb7a287c9849b27ef01e Mon Sep 17 00:00:00 2001 From: Piotr Gabryjeluk Date: Thu, 11 Sep 2025 22:20:52 +0200 Subject: [PATCH 2/7] Add mechanism for throttled warnings and use that for the ExperimentalWarning --- src/neptune_query/internal/experimental.py | 18 +++++------- src/neptune_query/internal/warnings.py | 34 ++++++++++++++++++++++ 2 files changed, 41 insertions(+), 11 deletions(-) create mode 100644 src/neptune_query/internal/warnings.py diff --git a/src/neptune_query/internal/experimental.py b/src/neptune_query/internal/experimental.py index 4868791d..56ad2cf6 100644 --- a/src/neptune_query/internal/experimental.py +++ b/src/neptune_query/internal/experimental.py @@ -14,18 +14,15 @@ # limitations under the License. import functools -import warnings from typing import ( Callable, ParamSpec, TypeVar, ) +from neptune_query.internal.warnings import throttled_warn from neptune_query.warnings import ExperimentalWarning -# registry of functions already warned -_warned_experimentals = set() - T = ParamSpec("T") R = TypeVar("R") @@ -38,14 +35,13 @@ def experimental(func: Callable[T, R]) -> Callable[T, R]: @functools.wraps(func) def wrapper(*args: T.args, **kwargs: T.kwargs) -> R: - if func not in _warned_experimentals: - warnings.warn( + throttled_warn( + ExperimentalWarning( f"{func.__qualname__} is experimental and might change or be removed " - "in a future minor release. Use with caution in production code.", - category=ExperimentalWarning, - stacklevel=2, - ) - _warned_experimentals.add(func) + "in a future minor release. Use with caution in production code." + ), + stacklevel=3, + ) return func(*args, **kwargs) return wrapper diff --git a/src/neptune_query/internal/warnings.py b/src/neptune_query/internal/warnings.py new file mode 100644 index 00000000..34ca60ec --- /dev/null +++ b/src/neptune_query/internal/warnings.py @@ -0,0 +1,34 @@ +# +# Copyright (c) 2025, Neptune Labs Sp. z o.o. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import warnings +from typing import Type + +from neptune_query.warnings import ExperimentalWarning + +# registry of warnings that were already emitted with the (type, message) tuple +WARNING_CLASSES_EMITTED_ONCE_PER_MESSAGE = (ExperimentalWarning,) + +_silence_warnings_msg: set[tuple[Type[Warning], str]] = set() + + +def throttled_warn(warning: Warning, stacklevel: int = 2) -> None: + if isinstance(warning, WARNING_CLASSES_EMITTED_ONCE_PER_MESSAGE): + key = (type(warning), str(warning)) + if key in _silence_warnings_msg: + return + _silence_warnings_msg.add(key) + + warnings.warn(warning, stacklevel=stacklevel) From 99f2653538127de068236466ab2dea0a3e7e2ce5 Mon Sep 17 00:00:00 2001 From: Piotr Gabryjeluk Date: Thu, 11 Sep 2025 22:20:52 +0200 Subject: [PATCH 3/7] Add throttled warnings for retries --- .../internal/composition/type_inference.py | 8 +- src/neptune_query/internal/retrieval/retry.py | 32 +++- src/neptune_query/internal/warnings.py | 22 ++- src/neptune_query/warnings.py | 12 ++ tests/unit/internal/test_warnings.py | 161 ++++++++++++++++++ 5 files changed, 225 insertions(+), 10 deletions(-) create mode 100644 tests/unit/internal/test_warnings.py diff --git a/src/neptune_query/internal/composition/type_inference.py b/src/neptune_query/internal/composition/type_inference.py index 270ea607..61accf1b 100644 --- a/src/neptune_query/internal/composition/type_inference.py +++ b/src/neptune_query/internal/composition/type_inference.py @@ -13,7 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. import copy -import warnings from collections import defaultdict from concurrent.futures import Executor from dataclasses import dataclass @@ -40,6 +39,7 @@ HISTOGRAM_SERIES_AGGREGATIONS, STRING_SERIES_AGGREGATIONS, ) +from ..warnings import throttled_warn from .attributes import fetch_attribute_definitions T = TypeVar("T", bound=Union[filters._Filter, filters._Attribute, None]) @@ -156,8 +156,10 @@ def get_result_or_raise(self) -> T: def emit_warnings(self) -> None: for attr_state in self.attributes: if attr_state.warning_text: - msg = f"Attribute '{attr_state.original_attribute.name}': {attr_state.warning_text}" - warnings.warn(msg, category=AttributeWarning, stacklevel=3) + throttled_warn( + AttributeWarning(f"Attribute '{attr_state.original_attribute.name}': {attr_state.warning_text}"), + stacklevel=3, + ) def infer_attribute_types_in_filter( diff --git a/src/neptune_query/internal/retrieval/retry.py b/src/neptune_query/internal/retrieval/retry.py index f8d7ba61..4a53e73c 100644 --- a/src/neptune_query/internal/retrieval/retry.py +++ b/src/neptune_query/internal/retrieval/retry.py @@ -32,8 +32,14 @@ from neptune_api.types import Response from ... import exceptions +from ...warnings import ( + Http5xxWarning, + Http429Warning, + HttpOtherWarning, +) from .. import env from ..logger import get_logger +from ..warnings import throttled_warn logger = get_logger() @@ -107,15 +113,29 @@ def wrapper(*args: T.args, **kwargs: T.kwargs) -> Response[R]: sleep_time = float(response.headers["retry-after"]) rate_limit_time_extension += sleep_time backoff_tries = 0 # reset backoff tries counter when using a different strategy - logger.debug( - f"Neptune API request was rate limited. Retry-after header value: {sleep_time} seconds. " - f"Total time spent on rate limiting so far: {rate_limit_time_extension} seconds." + throttled_warn( + Http429Warning( + f"Neptune API request was rate limited. Retry-after header value: {sleep_time} seconds. " + f"Total time spent on rate limiting so far: {rate_limit_time_extension:.2f} seconds." + ) + ) + elif response is not None and 500 <= response.status_code.value < 600: + sleep_time = backoff_strategy(backoff_tries) + throttled_warn( + Http5xxWarning( + f"Neptune API request failed with {response.status_code.value}. " + f"Backoff strategy recommends backing off for {sleep_time:.2f} seconds. " + f"Response: {response}. Last exception: {last_exc}." + ) ) else: sleep_time = backoff_strategy(backoff_tries) - logger.debug( - f"Neptune API request failed. Backoff strategy recommends backing off for {sleep_time:.2f} " - f"seconds. Response: {response}. Last exception: {last_exc}." + throttled_warn( + HttpOtherWarning( + f"Neptune API request failed. " + f"Backoff strategy recommends backing off for {sleep_time:.2f} seconds. " + f"Response: {response}. Last exception: {last_exc}." + ) ) elapsed_time = time.monotonic() - start_time diff --git a/src/neptune_query/internal/warnings.py b/src/neptune_query/internal/warnings.py index 34ca60ec..6cd39f95 100644 --- a/src/neptune_query/internal/warnings.py +++ b/src/neptune_query/internal/warnings.py @@ -14,14 +14,27 @@ # limitations under the License. import warnings +from datetime import ( + datetime, + timedelta, +) from typing import Type -from neptune_query.warnings import ExperimentalWarning +from neptune_query.warnings import ( + ExperimentalWarning, + Http5xxWarning, + Http429Warning, +) # registry of warnings that were already emitted with the (type, message) tuple WARNING_CLASSES_EMITTED_ONCE_PER_MESSAGE = (ExperimentalWarning,) +# registry of warning types that were already emitted with the time they should be silenced until +WARNING_CLASSES_EMITTED_ONCE_PER_MINUTE = (Http429Warning, Http5xxWarning) +ONE_MINUTE = timedelta(seconds=60) + _silence_warnings_msg: set[tuple[Type[Warning], str]] = set() +_silence_warnings_until: dict[Type[Warning], datetime] = {} def throttled_warn(warning: Warning, stacklevel: int = 2) -> None: @@ -31,4 +44,11 @@ def throttled_warn(warning: Warning, stacklevel: int = 2) -> None: return _silence_warnings_msg.add(key) + if isinstance(warning, WARNING_CLASSES_EMITTED_ONCE_PER_MINUTE): + warning_type = type(warning) + now = datetime.now() + if warning_type in _silence_warnings_until and _silence_warnings_until[warning_type] > now: + return + _silence_warnings_until[warning_type] = now + ONE_MINUTE + warnings.warn(warning, stacklevel=stacklevel) diff --git a/src/neptune_query/warnings.py b/src/neptune_query/warnings.py index 1b45768a..4f95b873 100644 --- a/src/neptune_query/warnings.py +++ b/src/neptune_query/warnings.py @@ -20,3 +20,15 @@ class ExperimentalWarning(UserWarning): class AttributeWarning(UserWarning): """Warning for attribute issues.""" + + +class Http429Warning(UserWarning): + """Warning for HTTP 429 responses.""" + + +class Http5xxWarning(UserWarning): + """Warning for HTTP 5xx responses.""" + + +class HttpOtherWarning(UserWarning): + """Warning for other HTTP issues.""" diff --git a/tests/unit/internal/test_warnings.py b/tests/unit/internal/test_warnings.py new file mode 100644 index 00000000..3c555be8 --- /dev/null +++ b/tests/unit/internal/test_warnings.py @@ -0,0 +1,161 @@ +# +# Copyright (c) 2025, Neptune Labs Sp. z o.o. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from datetime import datetime +from unittest.mock import patch + +import pytest + +from neptune_query.internal.warnings import ( + _silence_warnings_msg, + _silence_warnings_until, + throttled_warn, +) +from neptune_query.warnings import ( + ExperimentalWarning, + Http5xxWarning, + Http429Warning, +) + +TIME_001 = datetime(2025, 9, 11, 15, 0, 0) +TIME_010 = datetime(2025, 9, 11, 15, 0, 10) +TIME_020 = datetime(2025, 9, 11, 15, 0, 20) +TIME_030 = datetime(2025, 9, 11, 15, 0, 30) +TIME_055 = datetime(2025, 9, 11, 15, 0, 55) +TIME_065 = datetime(2025, 9, 11, 15, 1, 5) +TIME_075 = datetime(2025, 9, 11, 15, 1, 15) + + +@pytest.fixture(autouse=True) +def clear_warning_registries(): + # Clear the warning registries before each test + _silence_warnings_msg.clear() + _silence_warnings_until.clear() + + +def emit_warning_at_time(time, warning): + """Helper function to emit a warning at a specific time by mocking INITIAL_TIME""" + with patch("neptune_query.internal.warnings.datetime") as mock_datetime: + mock_datetime.now.return_value = time + throttled_warn(warning) + + +@patch("neptune_query.internal.warnings.warnings.warn") +def test_regular_warning(mock_warn): + """Regular warnings should be emitted every time""" + warning = Warning("Test warning") + emit_warning_at_time(TIME_001, warning) + mock_warn.assert_called_once_with(warning, stacklevel=2) + + # Second emission should also go through + emit_warning_at_time(TIME_001, warning) + assert mock_warn.call_count == 2 + + +@patch("neptune_query.internal.warnings.warnings.warn") +def test_experimental_warning_once_per_message(mock_warn): + """ExperimentalWarning with the same message should be emitted only once""" + warning = ExperimentalWarning("Test experimental feature") + different_warning = ExperimentalWarning("Different message") + + # First emission should go through + emit_warning_at_time(TIME_001, warning) + mock_warn.assert_called_once_with(warning, stacklevel=2) + + # Second emission with same message should be suppressed + emit_warning_at_time(TIME_010, warning) + assert mock_warn.call_count == 1 + + # Different message should go through + emit_warning_at_time(TIME_010, different_warning) + assert mock_warn.call_count == 2 + + # Emitting the same warnings again should still be suppressed + emit_warning_at_time(TIME_030, warning) + emit_warning_at_time(TIME_030, different_warning) + assert mock_warn.call_count == 2 + + +@patch("neptune_query.internal.warnings.warnings.warn") +def test_http429_warning_once_per_minute(mock_warn): + """Http429Warning should be emitted once per minute""" + warning = Http429Warning("Rate limit exceeded") + different_warning = Http429Warning("Another rate limit message") + non_throttled_warning = Warning("Non-throttled warning") + + # First emission should go through + emit_warning_at_time(TIME_001, warning) + mock_warn.assert_called_once_with(warning, stacklevel=2) + + # Further emissions within a minute should be suppressed + emit_warning_at_time(TIME_010, warning) + emit_warning_at_time(TIME_020, warning) + emit_warning_at_time(TIME_055, warning) + assert mock_warn.call_count == 1 + + # Non-throttled warning should go through + emit_warning_at_time(TIME_020, non_throttled_warning) + emit_warning_at_time(TIME_030, non_throttled_warning) + assert mock_warn.call_count == 3 + + # Different message but same type should also be suppressed + emit_warning_at_time(TIME_055, different_warning) + assert mock_warn.call_count == 3 + + # After a minute it should go through again + emit_warning_at_time(TIME_065, warning) + assert mock_warn.call_count == 4 + + +@patch("neptune_query.internal.warnings.warnings.warn") +def test_http429_and_http5xx_warnings(mock_warn): + """Http429Warning and Http5xxWarning should be throttled independently""" + warning_429 = Http429Warning("Rate limit exceeded") + warning_5xx = Http5xxWarning("Server error") + non_throttled_warning = Warning("Non-throttled warning") + + # Emit Http429Warning + emit_warning_at_time(TIME_001, warning_429) + assert mock_warn.call_count == 1 + mock_warn.assert_called_once_with(warning_429, stacklevel=2) + + # Emit Http5xxWarning 9 seconds later + emit_warning_at_time(TIME_010, warning_5xx) + assert mock_warn.call_count == 2 + mock_warn.assert_called_with(warning_5xx, stacklevel=2) + + # Further emissions within a minute should be suppressed + emit_warning_at_time(TIME_020, warning_429) + emit_warning_at_time(TIME_030, warning_5xx) + assert mock_warn.call_count == 2 + + # Non-throttled warning should go through + emit_warning_at_time(TIME_030, non_throttled_warning) + emit_warning_at_time(TIME_055, non_throttled_warning) + assert mock_warn.call_count == 4 + + # After a minute both should go through again + emit_warning_at_time(TIME_065, warning_429) + assert mock_warn.call_count == 5 + mock_warn.assert_called_with(warning_429, stacklevel=2) + + # Time for 5xx warnings is not yet up + emit_warning_at_time(TIME_065, warning_5xx) + assert mock_warn.call_count == 5 + + # After another 10 seconds (total 1m15s), 5xx warning should go through + emit_warning_at_time(TIME_075, warning_5xx) + assert mock_warn.call_count == 6 + mock_warn.assert_called_with(warning_5xx, stacklevel=2) From ffd3bc4f9008ed91a7f8bf736039d43e222ad9ab Mon Sep 17 00:00:00 2001 From: Piotr Gabryjeluk Date: Mon, 15 Sep 2025 09:18:12 +0200 Subject: [PATCH 4/7] Updates to the warning throttling --- src/neptune_query/internal/experimental.py | 2 +- src/neptune_query/internal/retrieval/retry.py | 44 +++++++---- src/neptune_query/internal/warnings.py | 36 ++++++--- src/neptune_query/warnings.py | 16 ++-- tests/unit/internal/test_warnings.py | 76 +++++++++---------- 5 files changed, 107 insertions(+), 67 deletions(-) diff --git a/src/neptune_query/internal/experimental.py b/src/neptune_query/internal/experimental.py index 56ad2cf6..1890cc38 100644 --- a/src/neptune_query/internal/experimental.py +++ b/src/neptune_query/internal/experimental.py @@ -37,7 +37,7 @@ def experimental(func: Callable[T, R]) -> Callable[T, R]: def wrapper(*args: T.args, **kwargs: T.kwargs) -> R: throttled_warn( ExperimentalWarning( - f"{func.__qualname__} is experimental and might change or be removed " + f"`{func.__module__}.{func.__qualname__}` is experimental and might change or be removed " "in a future minor release. Use with caution in production code." ), stacklevel=3, diff --git a/src/neptune_query/internal/retrieval/retry.py b/src/neptune_query/internal/retrieval/retry.py index 4a53e73c..294216be 100644 --- a/src/neptune_query/internal/retrieval/retry.py +++ b/src/neptune_query/internal/retrieval/retry.py @@ -35,6 +35,7 @@ from ...warnings import ( Http5xxWarning, Http429Warning, + Http503Warning, HttpOtherWarning, ) from .. import env @@ -93,7 +94,6 @@ def wrapper(*args: T.args, **kwargs: T.kwargs) -> Response[R]: response = None try: response = func(*args, **kwargs) - if 200 <= response.status_code.value < 300: return response except exceptions.NeptuneError: @@ -113,29 +113,47 @@ def wrapper(*args: T.args, **kwargs: T.kwargs) -> Response[R]: sleep_time = float(response.headers["retry-after"]) rate_limit_time_extension += sleep_time backoff_tries = 0 # reset backoff tries counter when using a different strategy + logger.debug( + f"Neptune API request was rate limited. Retry-after header value: {sleep_time} seconds. " + f"Total time spent on rate limiting so far: {rate_limit_time_extension} seconds." + ) throttled_warn( Http429Warning( - f"Neptune API request was rate limited. Retry-after header value: {sleep_time} seconds. " - f"Total time spent on rate limiting so far: {rate_limit_time_extension:.2f} seconds." - ) + "Neptune API request was rate limited. We will slow down and retry automatically." + ), + stacklevel=4, + ) + elif response is not None and response.status_code.value == 503: + sleep_time = backoff_strategy(backoff_tries) + logger.debug( + f"Neptune API request failed. Backoff strategy recommends backing off for {sleep_time:.2f} " + f"seconds. Response: {response}. Last exception: {last_exc}." + ) + throttled_warn( + Http503Warning("Neptune API is temporarily unavailable. We will retry automatically."), + stacklevel=4, ) elif response is not None and 500 <= response.status_code.value < 600: sleep_time = backoff_strategy(backoff_tries) + logger.debug( + f"Neptune API request failed. Backoff strategy recommends backing off for {sleep_time:.2f} " + f"seconds. Response: {response}. Last exception: {last_exc}." + ) throttled_warn( Http5xxWarning( - f"Neptune API request failed with {response.status_code.value}. " - f"Backoff strategy recommends backing off for {sleep_time:.2f} seconds. " - f"Response: {response}. Last exception: {last_exc}." - ) + f"Neptune API request failed with HTTP code {response.status_code.value}. " + "We will retry automatically." + ), + stacklevel=4, ) else: sleep_time = backoff_strategy(backoff_tries) + logger.debug( + f"Neptune API request failed. Backoff strategy recommends backing off for {sleep_time:.2f} " + f"seconds. Response: {response}. Last exception: {last_exc}." + ) throttled_warn( - HttpOtherWarning( - f"Neptune API request failed. " - f"Backoff strategy recommends backing off for {sleep_time:.2f} seconds. " - f"Response: {response}. Last exception: {last_exc}." - ) + HttpOtherWarning("Neptune API request failed. We will retry automatically"), stacklevel=4 ) elapsed_time = time.monotonic() - start_time diff --git a/src/neptune_query/internal/warnings.py b/src/neptune_query/internal/warnings.py index 6cd39f95..1772dc59 100644 --- a/src/neptune_query/internal/warnings.py +++ b/src/neptune_query/internal/warnings.py @@ -12,7 +12,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - +import sys import warnings from datetime import ( datetime, @@ -24,31 +24,49 @@ ExperimentalWarning, Http5xxWarning, Http429Warning, + HttpOtherWarning, ) # registry of warnings that were already emitted with the (type, message) tuple -WARNING_CLASSES_EMITTED_ONCE_PER_MESSAGE = (ExperimentalWarning,) +WARNING_TYPES_EMITTED_ONCE_PER_MESSAGE = (ExperimentalWarning,) # registry of warning types that were already emitted with the time they should be silenced until -WARNING_CLASSES_EMITTED_ONCE_PER_MINUTE = (Http429Warning, Http5xxWarning) -ONE_MINUTE = timedelta(seconds=60) +WARNING_TYPES_EMITTED_ONCE_PER_WHILE = (Http429Warning, Http5xxWarning, HttpOtherWarning) +WARNING_SUPPRESSION_DURATION = timedelta(seconds=20) _silence_warnings_msg: set[tuple[Type[Warning], str]] = set() _silence_warnings_until: dict[Type[Warning], datetime] = {} -def throttled_warn(warning: Warning, stacklevel: int = 2) -> None: - if isinstance(warning, WARNING_CLASSES_EMITTED_ONCE_PER_MESSAGE): +def get_thread_id() -> int: + import threading + + return threading.get_ident() % 123 + + +def format_warning(warning: Warning) -> Warning: + # check if stderr is a terminal: + if sys.stderr.isatty(): + orange_bold = "\033[1;38;2;255;165;0m" + end = "\033[0m" + msg = f"{orange_bold}{str(warning)}{end}" + return type(warning)(msg) + else: + return warning + + +def throttled_warn(warning: Warning, stacklevel: int = 3) -> None: + if isinstance(warning, WARNING_TYPES_EMITTED_ONCE_PER_MESSAGE): key = (type(warning), str(warning)) if key in _silence_warnings_msg: return _silence_warnings_msg.add(key) - if isinstance(warning, WARNING_CLASSES_EMITTED_ONCE_PER_MINUTE): + if isinstance(warning, WARNING_TYPES_EMITTED_ONCE_PER_WHILE): warning_type = type(warning) now = datetime.now() if warning_type in _silence_warnings_until and _silence_warnings_until[warning_type] > now: return - _silence_warnings_until[warning_type] = now + ONE_MINUTE + _silence_warnings_until[warning_type] = now + WARNING_SUPPRESSION_DURATION - warnings.warn(warning, stacklevel=stacklevel) + warnings.warn(format_warning(warning), stacklevel=stacklevel) diff --git a/src/neptune_query/warnings.py b/src/neptune_query/warnings.py index 4f95b873..026bf592 100644 --- a/src/neptune_query/warnings.py +++ b/src/neptune_query/warnings.py @@ -14,21 +14,25 @@ # limitations under the License. +class AttributeWarning(UserWarning): + """Warning for attribute issues.""" + + class ExperimentalWarning(UserWarning): """Warning for use of experimental API elements.""" -class AttributeWarning(UserWarning): - """Warning for attribute issues.""" +class Http429Warning(UserWarning): + """Warning for retryable HTTP 429 responses (rate limiting).""" -class Http429Warning(UserWarning): - """Warning for HTTP 429 responses.""" +class Http503Warning(UserWarning): + """Warning for retryable HTTP 503 responses (service unavailable).""" class Http5xxWarning(UserWarning): - """Warning for HTTP 5xx responses.""" + """Warning for retryable HTTP 5xx responses (server errors).""" class HttpOtherWarning(UserWarning): - """Warning for other HTTP issues.""" + """Warning for other retryable HTTP issues.""" diff --git a/tests/unit/internal/test_warnings.py b/tests/unit/internal/test_warnings.py index 3c555be8..48cb0ab3 100644 --- a/tests/unit/internal/test_warnings.py +++ b/tests/unit/internal/test_warnings.py @@ -29,13 +29,13 @@ Http429Warning, ) -TIME_001 = datetime(2025, 9, 11, 15, 0, 0) -TIME_010 = datetime(2025, 9, 11, 15, 0, 10) -TIME_020 = datetime(2025, 9, 11, 15, 0, 20) -TIME_030 = datetime(2025, 9, 11, 15, 0, 30) -TIME_055 = datetime(2025, 9, 11, 15, 0, 55) -TIME_065 = datetime(2025, 9, 11, 15, 1, 5) -TIME_075 = datetime(2025, 9, 11, 15, 1, 15) +TIME_00_01 = datetime(2025, 9, 11, 15, 0, 1) +TIME_00_03 = datetime(2025, 9, 11, 15, 0, 3) +TIME_00_07 = datetime(2025, 9, 11, 15, 0, 7) +TIME_00_10 = datetime(2025, 9, 11, 15, 0, 10) +TIME_00_18 = datetime(2025, 9, 11, 15, 0, 18) +TIME_00_22 = datetime(2025, 9, 11, 15, 0, 22) +TIME_00_25 = datetime(2025, 9, 11, 15, 0, 25) @pytest.fixture(autouse=True) @@ -56,11 +56,11 @@ def emit_warning_at_time(time, warning): def test_regular_warning(mock_warn): """Regular warnings should be emitted every time""" warning = Warning("Test warning") - emit_warning_at_time(TIME_001, warning) - mock_warn.assert_called_once_with(warning, stacklevel=2) + emit_warning_at_time(TIME_00_01, warning) + mock_warn.assert_called_once_with(warning, stacklevel=3) # Second emission should also go through - emit_warning_at_time(TIME_001, warning) + emit_warning_at_time(TIME_00_01, warning) assert mock_warn.call_count == 2 @@ -71,20 +71,20 @@ def test_experimental_warning_once_per_message(mock_warn): different_warning = ExperimentalWarning("Different message") # First emission should go through - emit_warning_at_time(TIME_001, warning) - mock_warn.assert_called_once_with(warning, stacklevel=2) + emit_warning_at_time(TIME_00_01, warning) + mock_warn.assert_called_once_with(warning, stacklevel=3) # Second emission with same message should be suppressed - emit_warning_at_time(TIME_010, warning) + emit_warning_at_time(TIME_00_03, warning) assert mock_warn.call_count == 1 # Different message should go through - emit_warning_at_time(TIME_010, different_warning) + emit_warning_at_time(TIME_00_03, different_warning) assert mock_warn.call_count == 2 # Emitting the same warnings again should still be suppressed - emit_warning_at_time(TIME_030, warning) - emit_warning_at_time(TIME_030, different_warning) + emit_warning_at_time(TIME_00_10, warning) + emit_warning_at_time(TIME_00_10, different_warning) assert mock_warn.call_count == 2 @@ -96,26 +96,26 @@ def test_http429_warning_once_per_minute(mock_warn): non_throttled_warning = Warning("Non-throttled warning") # First emission should go through - emit_warning_at_time(TIME_001, warning) - mock_warn.assert_called_once_with(warning, stacklevel=2) + emit_warning_at_time(TIME_00_01, warning) + mock_warn.assert_called_once_with(warning, stacklevel=3) # Further emissions within a minute should be suppressed - emit_warning_at_time(TIME_010, warning) - emit_warning_at_time(TIME_020, warning) - emit_warning_at_time(TIME_055, warning) + emit_warning_at_time(TIME_00_03, warning) + emit_warning_at_time(TIME_00_07, warning) + emit_warning_at_time(TIME_00_18, warning) assert mock_warn.call_count == 1 # Non-throttled warning should go through - emit_warning_at_time(TIME_020, non_throttled_warning) - emit_warning_at_time(TIME_030, non_throttled_warning) + emit_warning_at_time(TIME_00_07, non_throttled_warning) + emit_warning_at_time(TIME_00_10, non_throttled_warning) assert mock_warn.call_count == 3 # Different message but same type should also be suppressed - emit_warning_at_time(TIME_055, different_warning) + emit_warning_at_time(TIME_00_18, different_warning) assert mock_warn.call_count == 3 # After a minute it should go through again - emit_warning_at_time(TIME_065, warning) + emit_warning_at_time(TIME_00_22, warning) assert mock_warn.call_count == 4 @@ -127,35 +127,35 @@ def test_http429_and_http5xx_warnings(mock_warn): non_throttled_warning = Warning("Non-throttled warning") # Emit Http429Warning - emit_warning_at_time(TIME_001, warning_429) + emit_warning_at_time(TIME_00_01, warning_429) assert mock_warn.call_count == 1 - mock_warn.assert_called_once_with(warning_429, stacklevel=2) + mock_warn.assert_called_once_with(warning_429, stacklevel=3) # Emit Http5xxWarning 9 seconds later - emit_warning_at_time(TIME_010, warning_5xx) + emit_warning_at_time(TIME_00_03, warning_5xx) assert mock_warn.call_count == 2 - mock_warn.assert_called_with(warning_5xx, stacklevel=2) + mock_warn.assert_called_with(warning_5xx, stacklevel=3) # Further emissions within a minute should be suppressed - emit_warning_at_time(TIME_020, warning_429) - emit_warning_at_time(TIME_030, warning_5xx) + emit_warning_at_time(TIME_00_07, warning_429) + emit_warning_at_time(TIME_00_10, warning_5xx) assert mock_warn.call_count == 2 # Non-throttled warning should go through - emit_warning_at_time(TIME_030, non_throttled_warning) - emit_warning_at_time(TIME_055, non_throttled_warning) + emit_warning_at_time(TIME_00_10, non_throttled_warning) + emit_warning_at_time(TIME_00_18, non_throttled_warning) assert mock_warn.call_count == 4 # After a minute both should go through again - emit_warning_at_time(TIME_065, warning_429) + emit_warning_at_time(TIME_00_22, warning_429) assert mock_warn.call_count == 5 - mock_warn.assert_called_with(warning_429, stacklevel=2) + mock_warn.assert_called_with(warning_429, stacklevel=3) # Time for 5xx warnings is not yet up - emit_warning_at_time(TIME_065, warning_5xx) + emit_warning_at_time(TIME_00_22, warning_5xx) assert mock_warn.call_count == 5 # After another 10 seconds (total 1m15s), 5xx warning should go through - emit_warning_at_time(TIME_075, warning_5xx) + emit_warning_at_time(TIME_00_25, warning_5xx) assert mock_warn.call_count == 6 - mock_warn.assert_called_with(warning_5xx, stacklevel=2) + mock_warn.assert_called_with(warning_5xx, stacklevel=3) From fd3fe38f4d4a2c0902add868950157ebccb9831b Mon Sep 17 00:00:00 2001 From: Piotr Gabryjeluk Date: Mon, 15 Sep 2025 15:28:56 +0200 Subject: [PATCH 5/7] Remove unused debug-only function --- src/neptune_query/internal/warnings.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/neptune_query/internal/warnings.py b/src/neptune_query/internal/warnings.py index 1772dc59..79b9ab25 100644 --- a/src/neptune_query/internal/warnings.py +++ b/src/neptune_query/internal/warnings.py @@ -38,12 +38,6 @@ _silence_warnings_until: dict[Type[Warning], datetime] = {} -def get_thread_id() -> int: - import threading - - return threading.get_ident() % 123 - - def format_warning(warning: Warning) -> Warning: # check if stderr is a terminal: if sys.stderr.isatty(): From 20d2b1873781bc3cbc8c6dff8063c0cf4c50cb17 Mon Sep 17 00:00:00 2001 From: Piotr Gabryjeluk Date: Mon, 15 Sep 2025 15:41:36 +0200 Subject: [PATCH 6/7] Update comments in `test_warnings.py` --- tests/unit/internal/test_warnings.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/tests/unit/internal/test_warnings.py b/tests/unit/internal/test_warnings.py index 48cb0ab3..14b63e9d 100644 --- a/tests/unit/internal/test_warnings.py +++ b/tests/unit/internal/test_warnings.py @@ -89,7 +89,7 @@ def test_experimental_warning_once_per_message(mock_warn): @patch("neptune_query.internal.warnings.warnings.warn") -def test_http429_warning_once_per_minute(mock_warn): +def test_http429_warning_once_per_20s(mock_warn): """Http429Warning should be emitted once per minute""" warning = Http429Warning("Rate limit exceeded") different_warning = Http429Warning("Another rate limit message") @@ -99,10 +99,10 @@ def test_http429_warning_once_per_minute(mock_warn): emit_warning_at_time(TIME_00_01, warning) mock_warn.assert_called_once_with(warning, stacklevel=3) - # Further emissions within a minute should be suppressed - emit_warning_at_time(TIME_00_03, warning) - emit_warning_at_time(TIME_00_07, warning) - emit_warning_at_time(TIME_00_18, warning) + # Further emissions within seconds should be suppressed + emit_warning_at_time(TIME_00_03, warning) # 2s after first emission + emit_warning_at_time(TIME_00_07, warning) # 6s after first emission + emit_warning_at_time(TIME_00_18, warning) # 17s after first emission assert mock_warn.call_count == 1 # Non-throttled warning should go through @@ -114,7 +114,7 @@ def test_http429_warning_once_per_minute(mock_warn): emit_warning_at_time(TIME_00_18, different_warning) assert mock_warn.call_count == 3 - # After a minute it should go through again + # After 21 seconds since first emission, it should go through again emit_warning_at_time(TIME_00_22, warning) assert mock_warn.call_count == 4 @@ -131,14 +131,14 @@ def test_http429_and_http5xx_warnings(mock_warn): assert mock_warn.call_count == 1 mock_warn.assert_called_once_with(warning_429, stacklevel=3) - # Emit Http5xxWarning 9 seconds later + # Emit Http5xxWarning 2 seconds later emit_warning_at_time(TIME_00_03, warning_5xx) assert mock_warn.call_count == 2 mock_warn.assert_called_with(warning_5xx, stacklevel=3) - # Further emissions within a minute should be suppressed - emit_warning_at_time(TIME_00_07, warning_429) - emit_warning_at_time(TIME_00_10, warning_5xx) + # Further emissions within 20 seconds should be suppressed + emit_warning_at_time(TIME_00_07, warning_429) # 6s after first warning_429 + emit_warning_at_time(TIME_00_10, warning_5xx) # 7s after first warning_5xx assert mock_warn.call_count == 2 # Non-throttled warning should go through @@ -146,7 +146,7 @@ def test_http429_and_http5xx_warnings(mock_warn): emit_warning_at_time(TIME_00_18, non_throttled_warning) assert mock_warn.call_count == 4 - # After a minute both should go through again + # After 21 seconds since first emission, warning_429 should go through again emit_warning_at_time(TIME_00_22, warning_429) assert mock_warn.call_count == 5 mock_warn.assert_called_with(warning_429, stacklevel=3) @@ -155,7 +155,7 @@ def test_http429_and_http5xx_warnings(mock_warn): emit_warning_at_time(TIME_00_22, warning_5xx) assert mock_warn.call_count == 5 - # After another 10 seconds (total 1m15s), 5xx warning should go through + # After another 3 seconds (total 22s from first 5xx warning), 5xx warning should go through emit_warning_at_time(TIME_00_25, warning_5xx) assert mock_warn.call_count == 6 mock_warn.assert_called_with(warning_5xx, stacklevel=3) From e24f035a78bad44c1cd4bca495ccac3aa6fa7926 Mon Sep 17 00:00:00 2001 From: Piotr Gabryjeluk Date: Mon, 15 Sep 2025 17:10:32 +0200 Subject: [PATCH 7/7] Format code --- tests/unit/internal/test_warnings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/internal/test_warnings.py b/tests/unit/internal/test_warnings.py index 14b63e9d..55abd650 100644 --- a/tests/unit/internal/test_warnings.py +++ b/tests/unit/internal/test_warnings.py @@ -138,7 +138,7 @@ def test_http429_and_http5xx_warnings(mock_warn): # Further emissions within 20 seconds should be suppressed emit_warning_at_time(TIME_00_07, warning_429) # 6s after first warning_429 - emit_warning_at_time(TIME_00_10, warning_5xx) # 7s after first warning_5xx + emit_warning_at_time(TIME_00_10, warning_5xx) # 7s after first warning_5xx assert mock_warn.call_count == 2 # Non-throttled warning should go through