Skip to content

Commit e77a53e

Browse files
AdityaAudiyaythomas
authored andcommitted
fix(waits): repair strategy, fail on exhaustion
1 parent 1c80cd5 commit e77a53e

4 files changed

Lines changed: 177 additions & 95 deletions

File tree

packages/aws-durable-execution-sdk-python/src/aws_durable_execution_sdk_python/operation/wait_for_condition.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -242,7 +242,7 @@ def execute(self, checkpointed_result: CheckpointedResult) -> T:
242242
if delay_seconds is not None and delay_seconds < 1:
243243
logger.warning(
244244
(
245-
"WaitDecision delay_seconds step for id: %s, name: %s,"
245+
"wait_for_condition delay_seconds for id: %s, name: %s,"
246246
"is %d < 1. Setting to minimum of 1 seconds."
247247
),
248248
self.operation_identifier.operation_id,

packages/aws-durable-execution-sdk-python/src/aws_durable_execution_sdk_python/waits.py

Lines changed: 36 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
from typing import TYPE_CHECKING, Generic
88

99
from aws_durable_execution_sdk_python.config import Duration, JitterStrategy, T
10+
from aws_durable_execution_sdk_python.exceptions import WaitForConditionError
1011

1112
if TYPE_CHECKING:
1213
from collections.abc import Callable
@@ -16,29 +17,6 @@
1617
Numeric = int | float
1718

1819

19-
@dataclass
20-
class WaitDecision:
21-
"""Decision about whether to wait a step and with what delay."""
22-
23-
should_wait: bool
24-
delay: Duration
25-
26-
@property
27-
def delay_seconds(self) -> int:
28-
"""Get delay in seconds."""
29-
return self.delay.to_seconds()
30-
31-
@classmethod
32-
def wait(cls, delay: Duration) -> WaitDecision:
33-
"""Create a wait decision."""
34-
return cls(should_wait=True, delay=delay)
35-
36-
@classmethod
37-
def no_wait(cls) -> WaitDecision:
38-
"""Create a no-wait decision."""
39-
return cls(should_wait=False, delay=Duration())
40-
41-
4220
@dataclass
4321
class WaitStrategyConfig(Generic[T]):
4422
should_continue_polling: Callable[[T], bool]
@@ -69,35 +47,6 @@ def timeout_seconds(self) -> int | None:
6947
return self.timeout.to_seconds()
7048

7149

72-
def create_wait_strategy(
73-
config: WaitStrategyConfig[T],
74-
) -> Callable[[T, int], WaitDecision]:
75-
def wait_strategy(result: T, attempts_made: int) -> WaitDecision:
76-
# Check if condition is met
77-
if not config.should_continue_polling(result):
78-
return WaitDecision.no_wait()
79-
80-
# Check if we've exceeded max attempts
81-
if attempts_made >= config.max_attempts:
82-
return WaitDecision.no_wait()
83-
84-
# Calculate delay with exponential backoff
85-
base_delay: float = min(
86-
config.initial_delay_seconds * (config.backoff_rate ** (attempts_made - 1)),
87-
config.max_delay_seconds,
88-
)
89-
90-
# Apply jitter to get final delay
91-
delay_with_jitter: float = config.jitter_strategy.apply_jitter(base_delay)
92-
93-
# Round up and ensure minimum of 1 second
94-
final_delay: int = max(1, math.ceil(delay_with_jitter))
95-
96-
return WaitDecision.wait(Duration(seconds=final_delay))
97-
98-
return wait_strategy
99-
100-
10150
@dataclass(frozen=True)
10251
class WaitForConditionDecision:
10352
"""Decision about whether to continue waiting."""
@@ -121,6 +70,41 @@ def stop_polling(cls) -> WaitForConditionDecision:
12170
return cls(should_continue=False, delay=Duration())
12271

12372

73+
def create_wait_strategy(
74+
config: WaitStrategyConfig[T],
75+
) -> Callable[[T, int], WaitForConditionDecision]:
76+
def wait_strategy(result: T, attempts_made: int) -> WaitForConditionDecision:
77+
# Condition satisfied wins over exhaustion, so a condition met on the final
78+
# attempt still succeeds (matches the JS and Java SDKs).
79+
if not config.should_continue_polling(result):
80+
return WaitForConditionDecision.stop_polling()
81+
82+
# Out of attempts: fail rather than stop, otherwise the executor would treat
83+
# this as success and return partial state.
84+
if attempts_made >= config.max_attempts:
85+
msg = (
86+
f"wait_for_condition exhausted {config.max_attempts} attempts "
87+
"before the condition was met"
88+
)
89+
raise WaitForConditionError(msg)
90+
91+
# Calculate delay with exponential backoff
92+
base_delay: float = min(
93+
config.initial_delay_seconds * (config.backoff_rate ** (attempts_made - 1)),
94+
config.max_delay_seconds,
95+
)
96+
97+
# Apply jitter to get final delay
98+
delay_with_jitter: float = config.jitter_strategy.apply_jitter(base_delay)
99+
100+
# Round up and ensure minimum of 1 second
101+
final_delay: int = max(1, math.ceil(delay_with_jitter))
102+
103+
return WaitForConditionDecision.continue_waiting(Duration(seconds=final_delay))
104+
105+
return wait_strategy
106+
107+
124108
@dataclass(frozen=True)
125109
class WaitForConditionConfig(Generic[T]):
126110
"""Configuration for wait_for_condition."""

packages/aws-durable-execution-sdk-python/tests/operation/wait_for_condition_test.py

Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,8 @@
3535
from aws_durable_execution_sdk_python.waits import (
3636
WaitForConditionConfig,
3737
WaitForConditionDecision,
38+
WaitStrategyConfig,
39+
create_wait_strategy,
3840
)
3941
from tests.serdes_test import CustomDictSerDes
4042

@@ -1571,6 +1573,95 @@ def mock_wait_strategy(state, attempt):
15711573
assert mock_state.create_checkpoint.call_count == 2 # START + SUCCESS checkpoints
15721574

15731575

1576+
def test_wait_for_condition_exhaustion_raises_and_checkpoints_fail():
1577+
"""Live path: the built-in strategy runs out of attempts, so it raises
1578+
WaitForConditionError, which is checkpointed as a FAIL and propagated."""
1579+
mock_state = Mock(spec=ExecutionState)
1580+
mock_state.durable_execution_arn = "arn:aws:test"
1581+
mock_state.get_checkpoint_result.return_value = (
1582+
CheckpointedResult.create_not_found()
1583+
)
1584+
1585+
mock_logger = Mock(spec=Logger)
1586+
mock_logger.with_log_info.return_value = mock_logger
1587+
1588+
op_id = OperationIdentifier(
1589+
"op1", OperationSubType.WAIT_FOR_CONDITION, None, "test_wait"
1590+
)
1591+
1592+
def check_func(state, context):
1593+
return state + 1
1594+
1595+
mock_state.wrap_user_function.return_value = check_func
1596+
1597+
# max_attempts=1 means attempt 1 is already the last one.
1598+
config = WaitForConditionConfig(
1599+
initial_state=5,
1600+
wait_strategy=create_wait_strategy(
1601+
WaitStrategyConfig(should_continue_polling=lambda x: True, max_attempts=1)
1602+
),
1603+
)
1604+
1605+
with pytest.raises(WaitForConditionError):
1606+
wait_for_condition_handler(
1607+
state=mock_state,
1608+
operation_identifier=op_id,
1609+
check=check_func,
1610+
config=config,
1611+
context_logger=mock_logger,
1612+
)
1613+
1614+
assert mock_state.create_checkpoint.call_count == 2 # START and FAIL
1615+
fail_operation = mock_state.create_checkpoint.call_args_list[1][1][
1616+
"operation_update"
1617+
]
1618+
assert fail_operation.error.type == "WaitForConditionError"
1619+
1620+
1621+
def test_wait_for_condition_exhaustion_surfaces_on_replay():
1622+
"""Replay path: the FAILED checkpoint short-circuits on the next invocation
1623+
and is reconstructed as the typed WaitForConditionError carrying the original
1624+
error_type, without re-running the check."""
1625+
mock_state = Mock(spec=ExecutionState)
1626+
mock_state.durable_execution_arn = "test_arn"
1627+
operation = Operation(
1628+
operation_id="op1",
1629+
operation_type=OperationType.STEP,
1630+
status=OperationStatus.FAILED,
1631+
step_details=StepDetails(
1632+
error=ErrorObject("exhausted attempts", "WaitForConditionError", None, None)
1633+
),
1634+
)
1635+
mock_result = CheckpointedResult.create_from_operation(operation)
1636+
mock_state.get_checkpoint_result.return_value = mock_result
1637+
1638+
mock_logger = Mock(spec=Logger)
1639+
op_id = OperationIdentifier(
1640+
"op1", OperationSubType.WAIT_FOR_CONDITION, None, "test_wait"
1641+
)
1642+
1643+
def check_func(state, context):
1644+
msg = "Check function should not be called on replay of a failure"
1645+
raise AssertionError(msg)
1646+
1647+
config = WaitForConditionConfig(
1648+
initial_state=5,
1649+
wait_strategy=lambda s, a: WaitForConditionDecision.stop_polling(),
1650+
)
1651+
1652+
with pytest.raises(WaitForConditionError) as exc_info:
1653+
wait_for_condition_handler(
1654+
state=mock_state,
1655+
operation_identifier=op_id,
1656+
check=check_func,
1657+
config=config,
1658+
context_logger=mock_logger,
1659+
)
1660+
1661+
assert exc_info.value.error_type == "WaitForConditionError"
1662+
assert mock_state.create_checkpoint.call_count == 0 # Nothing new on replay
1663+
1664+
15741665
def test_wait_for_condition_executes_check_when_checkpoint_not_terminal_duplicate():
15751666
"""Test backward compatibility: when checkpoint is not terminal (STARTED),
15761667
the wait_for_condition operation executes the check function normally.

0 commit comments

Comments
 (0)