From 6312a02c827882620473d6d233bdd346705984c0 Mon Sep 17 00:00:00 2001 From: Dan Davison Date: Mon, 15 Sep 2025 21:57:30 -0400 Subject: [PATCH 01/81] Drive-by: get rid of always-true condition --- temporalio/worker/_workflow.py | 53 +++++++++++++++++----------------- 1 file changed, 26 insertions(+), 27 deletions(-) diff --git a/temporalio/worker/_workflow.py b/temporalio/worker/_workflow.py index 1e178f015..1918496b8 100644 --- a/temporalio/worker/_workflow.py +++ b/temporalio/worker/_workflow.py @@ -285,34 +285,33 @@ async def _handle_activation( # Run activation in separate thread so we can check if it's # deadlocked - if workflow: - activate_task = asyncio.get_running_loop().run_in_executor( - self._workflow_task_executor, - workflow.activate, - act, - ) + activate_task = asyncio.get_running_loop().run_in_executor( + self._workflow_task_executor, + workflow.activate, + act, + ) - # Run activation task with deadlock timeout - try: - completion = await asyncio.wait_for( - activate_task, self._deadlock_timeout_seconds - ) - except asyncio.TimeoutError: - # Need to create the deadlock exception up here so it - # captures the trace now instead of later after we may have - # interrupted it - deadlock_exc = _DeadlockError.from_deadlocked_workflow( - workflow.instance, self._deadlock_timeout_seconds - ) - # When we deadlock, we will raise an exception to fail - # the task. But before we do that, we want to try to - # interrupt the thread and put this activation task on - # the workflow so that the successive eviction can wait - # on it before trying to evict. - workflow.attempt_deadlock_interruption() - # Set the task and raise - workflow.deadlocked_activation_task = activate_task - raise deadlock_exc from None + # Run activation task with deadlock timeout + try: + completion = await asyncio.wait_for( + activate_task, self._deadlock_timeout_seconds + ) + except asyncio.TimeoutError: + # Need to create the deadlock exception up here so it + # captures the trace now instead of later after we may have + # interrupted it + deadlock_exc = _DeadlockError.from_deadlocked_workflow( + workflow.instance, self._deadlock_timeout_seconds + ) + # When we deadlock, we will raise an exception to fail + # the task. But before we do that, we want to try to + # interrupt the thread and put this activation task on + # the workflow so that the successive eviction can wait + # on it before trying to evict. + workflow.attempt_deadlock_interruption() + # Set the task and raise + workflow.deadlocked_activation_task = activate_task + raise deadlock_exc from None except Exception as err: if isinstance(err, _DeadlockError): From 5b08112a36c76f13bb8ed373a3849f83adebf57c Mon Sep 17 00:00:00 2001 From: Dan Davison Date: Tue, 9 Sep 2025 19:33:56 -0400 Subject: [PATCH 02/81] Support serialization context for all ser/de operations --- temporalio/client.py | 198 +++- temporalio/converter.py | 114 +- temporalio/worker/_activity.py | 55 +- temporalio/worker/_workflow.py | 44 +- temporalio/worker/_workflow_instance.py | 228 ++-- tests/test_serialization_context.py | 1328 +++++++++++++++++++++++ 6 files changed, 1810 insertions(+), 157 deletions(-) create mode 100644 tests/test_serialization_context.py diff --git a/temporalio/client.py b/temporalio/client.py index f9735cfb2..39d05e2bf 100644 --- a/temporalio/client.py +++ b/temporalio/client.py @@ -62,6 +62,11 @@ import temporalio.service import temporalio.workflow from temporalio.activity import ActivityCancellationDetails +from temporalio.converter import ( + ActivitySerializationContext, + DataConverter, + WorkflowSerializationContext, +) from temporalio.service import ( HttpConnectProxyConfig, KeepAliveConfig, @@ -1592,6 +1597,11 @@ def __init__( ) -> None: """Create workflow handle.""" self._client = client + self._data_converter = client.data_converter._with_context( + temporalio.converter.WorkflowSerializationContext( + namespace=client.namespace, workflow_id=id + ) + ) self._id = id self._run_id = run_id self._result_run_id = result_run_id @@ -1701,7 +1711,7 @@ async def result( break # Ignoring anything after the first response like TypeScript type_hints = [self._result_type] if self._result_type else None - results = await self._client.data_converter.decode_wrapper( + results = await self._data_converter.decode_wrapper( complete_attr.result, type_hints, ) @@ -1717,7 +1727,7 @@ async def result( hist_run_id = fail_attr.new_execution_run_id break raise WorkflowFailureError( - cause=await self._client.data_converter.decode_failure( + cause=await self._data_converter.decode_failure( fail_attr.failure ), ) @@ -1727,7 +1737,7 @@ async def result( cause=temporalio.exceptions.CancelledError( "Workflow cancelled", *( - await self._client.data_converter.decode_wrapper( + await self._data_converter.decode_wrapper( cancel_attr.details ) ), @@ -1739,7 +1749,7 @@ async def result( cause=temporalio.exceptions.TerminatedError( term_attr.reason or "Workflow terminated", *( - await self._client.data_converter.decode_wrapper( + await self._data_converter.decode_wrapper( term_attr.details ) ), @@ -3190,7 +3200,15 @@ async def fetch_next_page(self, *, page_size: Optional[int] = None) -> None: timeout=self._input.rpc_timeout, ) self._current_page = [ - WorkflowExecution._from_raw_info(v, self._client.data_converter) + WorkflowExecution._from_raw_info( + v, + self._client.data_converter._with_context( + WorkflowSerializationContext( + namespace=self._client.namespace, + workflow_id=v.execution.workflow_id, + ) + ), + ) for v in resp.executions ] self._current_page_index = 0 @@ -4148,37 +4166,47 @@ async def _to_proto( priority: Optional[temporalio.api.common.v1.Priority] = None if self.priority: priority = self.priority._to_proto() + data_converter = client.data_converter._with_context( + WorkflowSerializationContext( + namespace=client.namespace, + workflow_id=self.id, + ) + ) action = temporalio.api.schedule.v1.ScheduleAction( start_workflow=temporalio.api.workflow.v1.NewWorkflowExecutionInfo( workflow_id=self.id, workflow_type=temporalio.api.common.v1.WorkflowType(name=self.workflow), task_queue=temporalio.api.taskqueue.v1.TaskQueue(name=self.task_queue), - input=None - if not self.args - else temporalio.api.common.v1.Payloads( - payloads=[ - a - if isinstance(a, temporalio.api.common.v1.Payload) - else (await client.data_converter.encode([a]))[0] - for a in self.args - ] + input=( + temporalio.api.common.v1.Payloads( + payloads=[ + a + if isinstance(a, temporalio.api.common.v1.Payload) + else (await data_converter.encode([a]))[0] + for a in self.args + ] + ) + if self.args + else None ), workflow_execution_timeout=execution_timeout, workflow_run_timeout=run_timeout, workflow_task_timeout=task_timeout, retry_policy=retry_policy, - memo=None - if not self.memo - else temporalio.api.common.v1.Memo( - fields={ - k: v - if isinstance(v, temporalio.api.common.v1.Payload) - else (await client.data_converter.encode([v]))[0] - for k, v in self.memo.items() - }, + memo=( + temporalio.api.common.v1.Memo( + fields={ + k: v + if isinstance(v, temporalio.api.common.v1.Payload) + else (await data_converter.encode([v]))[0] + for k, v in self.memo.items() + }, + ) + if self.memo + else None ), user_metadata=await _encode_user_metadata( - client.data_converter, self.static_summary, self.static_details + data_converter, self.static_summary, self.static_details ), priority=priority, ), @@ -4994,6 +5022,12 @@ def __init__( self._workflow_run_id = workflow_run_id self._result_type = result_type self._known_outcome = known_outcome + self._data_converter = self._client.data_converter._with_context( + WorkflowSerializationContext( + namespace=self._client.namespace, + workflow_id=self.workflow_id, + ) + ) @property def id(self) -> str: @@ -5041,14 +5075,12 @@ async def result( assert self._known_outcome if self._known_outcome.HasField("failure"): raise WorkflowUpdateFailedError( - await self._client.data_converter.decode_failure( - self._known_outcome.failure - ), + await self._data_converter.decode_failure(self._known_outcome.failure), ) if not self._known_outcome.success.payloads: return None # type: ignore type_hints = [self._result_type] if self._result_type else None - results = await self._client.data_converter.decode( + results = await self._data_converter.decode( self._known_outcome.success.payloads, type_hints ) if not results: @@ -5900,12 +5932,18 @@ async def _build_signal_with_start_workflow_execution_request( self, input: StartWorkflowInput ) -> temporalio.api.workflowservice.v1.SignalWithStartWorkflowExecutionRequest: assert input.start_signal + data_converter = self._client.data_converter._with_context( + WorkflowSerializationContext( + namespace=self._client.namespace, + workflow_id=input.id, + ) + ) req = temporalio.api.workflowservice.v1.SignalWithStartWorkflowExecutionRequest( signal_name=input.start_signal ) if input.start_signal_args: req.signal_input.payloads.extend( - await self._client.data_converter.encode(input.start_signal_args) + await data_converter.encode(input.start_signal_args) ) await self._populate_start_workflow_execution_request(req, input) return req @@ -5925,14 +5963,18 @@ async def _populate_start_workflow_execution_request( ], input: Union[StartWorkflowInput, UpdateWithStartStartWorkflowInput], ) -> None: + data_converter = self._client.data_converter._with_context( + WorkflowSerializationContext( + namespace=self._client.namespace, + workflow_id=input.id, + ) + ) req.namespace = self._client.namespace req.workflow_id = input.id req.workflow_type.name = input.workflow req.task_queue.name = input.task_queue if input.args: - req.input.payloads.extend( - await self._client.data_converter.encode(input.args) - ) + req.input.payloads.extend(await data_converter.encode(input.args)) if input.execution_timeout is not None: req.workflow_execution_timeout.FromTimedelta(input.execution_timeout) if input.run_timeout is not None: @@ -5955,15 +5997,13 @@ async def _populate_start_workflow_execution_request( req.cron_schedule = input.cron_schedule if input.memo is not None: for k, v in input.memo.items(): - req.memo.fields[k].CopyFrom( - (await self._client.data_converter.encode([v]))[0] - ) + req.memo.fields[k].CopyFrom((await data_converter.encode([v]))[0]) if input.search_attributes is not None: temporalio.converter.encode_search_attributes( input.search_attributes, req.search_attributes ) metadata = await _encode_user_metadata( - self._client.data_converter, input.static_summary, input.static_details + data_converter, input.static_summary, input.static_details ) if metadata is not None: req.user_metadata.CopyFrom(metadata) @@ -6009,7 +6049,12 @@ async def describe_workflow( metadata=input.rpc_metadata, timeout=input.rpc_timeout, ), - self._client.data_converter, + self._client.data_converter._with_context( + WorkflowSerializationContext( + namespace=self._client.namespace, + workflow_id=input.id, + ) + ), ) def fetch_workflow_history_events( @@ -6038,6 +6083,12 @@ async def count_workflows( ) async def query_workflow(self, input: QueryWorkflowInput) -> Any: + data_converter = self._client.data_converter._with_context( + WorkflowSerializationContext( + namespace=self._client.namespace, + workflow_id=input.id, + ) + ) req = temporalio.api.workflowservice.v1.QueryWorkflowRequest( namespace=self._client.namespace, execution=temporalio.api.common.v1.WorkflowExecution( @@ -6053,7 +6104,7 @@ async def query_workflow(self, input: QueryWorkflowInput) -> Any: req.query.query_type = input.query if input.args: req.query.query_args.payloads.extend( - await self._client.data_converter.encode(input.args) + await data_converter.encode(input.args) ) if input.headers is not None: await self._apply_headers(input.headers, req.query.header.fields) @@ -6077,9 +6128,7 @@ async def query_workflow(self, input: QueryWorkflowInput) -> Any: if not resp.query_result.payloads: return None type_hints = [input.ret_type] if input.ret_type else None - results = await self._client.data_converter.decode( - resp.query_result.payloads, type_hints - ) + results = await data_converter.decode(resp.query_result.payloads, type_hints) if not results: return None elif len(results) > 1: @@ -6087,6 +6136,12 @@ async def query_workflow(self, input: QueryWorkflowInput) -> Any: return results[0] async def signal_workflow(self, input: SignalWorkflowInput) -> None: + data_converter = self._client.data_converter._with_context( + WorkflowSerializationContext( + namespace=self._client.namespace, + workflow_id=input.id, + ) + ) req = temporalio.api.workflowservice.v1.SignalWorkflowExecutionRequest( namespace=self._client.namespace, workflow_execution=temporalio.api.common.v1.WorkflowExecution( @@ -6098,9 +6153,7 @@ async def signal_workflow(self, input: SignalWorkflowInput) -> None: request_id=str(uuid.uuid4()), ) if input.args: - req.input.payloads.extend( - await self._client.data_converter.encode(input.args) - ) + req.input.payloads.extend(await data_converter.encode(input.args)) if input.headers is not None: await self._apply_headers(input.headers, req.header.fields) await self._client.workflow_service.signal_workflow_execution( @@ -6108,6 +6161,12 @@ async def signal_workflow(self, input: SignalWorkflowInput) -> None: ) async def terminate_workflow(self, input: TerminateWorkflowInput) -> None: + data_converter = self._client.data_converter._with_context( + WorkflowSerializationContext( + namespace=self._client.namespace, + workflow_id=input.id, + ) + ) req = temporalio.api.workflowservice.v1.TerminateWorkflowExecutionRequest( namespace=self._client.namespace, workflow_execution=temporalio.api.common.v1.WorkflowExecution( @@ -6119,9 +6178,7 @@ async def terminate_workflow(self, input: TerminateWorkflowInput) -> None: first_execution_run_id=input.first_execution_run_id or "", ) if input.args: - req.details.payloads.extend( - await self._client.data_converter.encode(input.args) - ) + req.details.payloads.extend(await data_converter.encode(input.args)) await self._client.workflow_service.terminate_workflow_execution( req, retry=True, metadata=input.rpc_metadata, timeout=input.rpc_timeout ) @@ -6178,6 +6235,12 @@ async def _build_update_workflow_execution_request( input: Union[StartWorkflowUpdateInput, UpdateWithStartUpdateWorkflowInput], workflow_id: str, ) -> temporalio.api.workflowservice.v1.UpdateWorkflowExecutionRequest: + data_converter = self._client.data_converter._with_context( + WorkflowSerializationContext( + namespace=self._client.namespace, + workflow_id=workflow_id, + ) + ) run_id, first_execution_run_id = ( ( input.run_id, @@ -6210,7 +6273,7 @@ async def _build_update_workflow_execution_request( ) if input.args: req.request.input.args.payloads.extend( - await self._client.data_converter.encode(input.args) + await data_converter.encode(input.args) ) if input.headers is not None: await self._apply_headers(input.headers, req.request.input.header.fields) @@ -6354,10 +6417,11 @@ async def _start_workflow_update_with_start( async def heartbeat_async_activity( self, input: HeartbeatAsyncActivityInput ) -> None: + data_converter = self._async_activity_data_converter(input.id_or_token) details = ( None if not input.details - else await self._client.data_converter.encode_wrapper(input.details) + else await data_converter.encode_wrapper(input.details) ) if isinstance(input.id_or_token, AsyncActivityIDReference): resp_by_id = await self._client.workflow_service.record_activity_task_heartbeat_by_id( @@ -6408,10 +6472,11 @@ async def heartbeat_async_activity( ) async def complete_async_activity(self, input: CompleteAsyncActivityInput) -> None: + data_converter = self._async_activity_data_converter(input.id_or_token) result = ( None if input.result is temporalio.common._arg_unset - else await self._client.data_converter.encode_wrapper([input.result]) + else await data_converter.encode_wrapper([input.result]) ) if isinstance(input.id_or_token, AsyncActivityIDReference): await self._client.workflow_service.respond_activity_task_completed_by_id( @@ -6441,14 +6506,14 @@ async def complete_async_activity(self, input: CompleteAsyncActivityInput) -> No ) async def fail_async_activity(self, input: FailAsyncActivityInput) -> None: + data_converter = self._async_activity_data_converter(input.id_or_token) + failure = temporalio.api.failure.v1.Failure() - await self._client.data_converter.encode_failure(input.error, failure) + await data_converter.encode_failure(input.error, failure) last_heartbeat_details = ( - None - if not input.last_heartbeat_details - else await self._client.data_converter.encode_wrapper( - input.last_heartbeat_details - ) + await data_converter.encode_wrapper(input.last_heartbeat_details) + if input.last_heartbeat_details + else None ) if isinstance(input.id_or_token, AsyncActivityIDReference): await self._client.workflow_service.respond_activity_task_failed_by_id( @@ -6482,10 +6547,11 @@ async def fail_async_activity(self, input: FailAsyncActivityInput) -> None: async def report_cancellation_async_activity( self, input: ReportCancellationAsyncActivityInput ) -> None: + data_converter = self._async_activity_data_converter(input.id_or_token) details = ( None if not input.details - else await self._client.data_converter.encode_wrapper(input.details) + else await data_converter.encode_wrapper(input.details) ) if isinstance(input.id_or_token, AsyncActivityIDReference): await self._client.workflow_service.respond_activity_task_canceled_by_id( @@ -6514,6 +6580,24 @@ async def report_cancellation_async_activity( timeout=input.rpc_timeout, ) + def _async_activity_data_converter( + self, id_or_token: Union[AsyncActivityIDReference, bytes] + ) -> DataConverter: + return self._client.data_converter._with_context( + ActivitySerializationContext( + namespace=self._client.namespace, + workflow_id=( + id_or_token.workflow_id + if isinstance(id_or_token, AsyncActivityIDReference) + else "" + ), + workflow_type="", + activity_type="", + activity_task_queue="", + is_local=False, + ) + ) + ### Schedule calls async def create_schedule(self, input: CreateScheduleInput) -> ScheduleHandle: diff --git a/temporalio/converter.py b/temporalio/converter.py index a9f8c0c98..03a81660d 100644 --- a/temporalio/converter.py +++ b/temporalio/converter.py @@ -43,6 +43,7 @@ import google.protobuf.symbol_database import nexusrpc import typing_extensions +from typing_extensions import Self import temporalio.api.common.v1 import temporalio.api.enums.v1 @@ -65,6 +66,91 @@ logger = getLogger(__name__) +class SerializationContext(ABC): + """Base serialization context. + + Provides contextual information during serialization and deserialization operations. + + Examples: + - In client code, when starting a workflow, or sending a signal/update/query to a workflow, or + receiving the result of an update/query, or handling an exception from a workflow, the context + type is :py:class:`WorkflowSerializationContext` and the workflow ID set of the target + workflow will be set in the context. + - In workflow code, when operating on a payload being sent/received to/from a child workflow, or + handling an exception from a child workflow, the context type is + :py:class:`WorkflowSerializationContext` and the workflow ID is that of the child workflow, + not of the currently executing (i.e. parent) workflow. + - In workflow code, when operating on a payload to be sent/received to/from an activity, the + context type is :py:class:`ActivitySerializationContext` and the workflow ID is that of the + currently-executing workflow. ActivitySerializationContext is also set on operations + """ + + pass + + +@dataclass(frozen=True) +class WorkflowSerializationContext(SerializationContext): + """Serialization context for workflows. + + See :py:class:`SerializationContext` for more details. + + Attributes: + namespace: The namespace the workflow is running in. + workflow_id: The ID of the workflow. Note that this is the ID of the workflow of which the + payload being operated on is an input or output. Note also that when creating/describing + schedules, this may be the workflow ID prefix as configured, not the final workflow ID + when the workflow is created by the schedule. + """ + + namespace: str + workflow_id: str + + +@dataclass(frozen=True) +class ActivitySerializationContext(SerializationContext): + """Serialization context for activities. + + See :py:class:`SerializationContext` for more details. + + Attributes: + namespace: Workflow/activity namespace. + workflow_id: Workflow ID. Note, when creating/describing schedules, + this may be the workflow ID prefix as configured, not the final + workflow ID when the workflow is created by the schedule. + workflow_type: Workflow Type. + activity_type: Activity Type. + activity_task_queue: Activity task queue. + is_local: Whether the activity is a local activity. + """ + + namespace: str + workflow_id: str + workflow_type: str + activity_type: str + activity_task_queue: str + is_local: bool + + +class WithSerializationContext(ABC): + """Interface for objects that can use serialization context. + + This is similar to the .NET IWithSerializationContext interface. + Objects implementing this interface can receive contextual information + during serialization and deserialization. + """ + + def with_context(self, context: Optional[SerializationContext]) -> Self: + """Return a copy of this object configured to use the given context. + + Args: + context: The serialization context to use, or None for no context. + + Returns: + A new instance configured with the context. + """ + raise NotImplementedError() + + class PayloadConverter(ABC): """Base payload converter to/from multiple payloads/values.""" @@ -232,7 +318,7 @@ def from_payload( raise NotImplementedError -class CompositePayloadConverter(PayloadConverter): +class CompositePayloadConverter(PayloadConverter, WithSerializationContext): """Composite payload converter that delegates to a list of encoding payload converters. Encoding/decoding are attempted on each payload converter successively until @@ -315,6 +401,16 @@ def from_payloads( ) from err return values + def with_context(self, context: Optional[SerializationContext]) -> Self: + """Return a new instance with the given context.""" + converters = [ + c.with_context(context) if isinstance(c, WithSerializationContext) else c + for c in self.converters.values() + ] + instance = type(self).__new__(type(self)) + CompositePayloadConverter.__init__(instance, *converters) + return instance + class DefaultPayloadConverter(CompositePayloadConverter): """Default payload converter compatible with other Temporal SDKs. @@ -1212,6 +1308,22 @@ async def decode_failure( await self.payload_codec.decode_failure(failure) return self.failure_converter.from_failure(failure, self.payload_converter) + def _with_context(self, context: Optional[SerializationContext]) -> Self: + cloned = dataclasses.replace(self) + payload_converter = self.payload_converter + payload_codec = self.payload_codec + failure_converter = self.failure_converter + if isinstance(payload_converter, WithSerializationContext): + payload_converter = payload_converter.with_context(context) + if isinstance(payload_codec, WithSerializationContext): + payload_codec = payload_codec.with_context(context) + if isinstance(failure_converter, WithSerializationContext): + failure_converter = failure_converter.with_context(context) + object.__setattr__(cloned, "payload_converter", payload_converter) + object.__setattr__(cloned, "payload_codec", payload_codec) + object.__setattr__(cloned, "failure_converter", failure_converter) + return cloned + DefaultPayloadConverter.default_encoding_payload_converters = ( BinaryNullPayloadConverter(), diff --git a/temporalio/worker/_activity.py b/temporalio/worker/_activity.py index 4ccb56ca6..39507759a 100644 --- a/temporalio/worker/_activity.py +++ b/temporalio/worker/_activity.py @@ -252,6 +252,18 @@ async def _heartbeat_async( if details is None: return + data_converter = self._data_converter + if activity.info: + context = temporalio.converter.ActivitySerializationContext( + namespace=activity.info.workflow_namespace, + workflow_id=activity.info.workflow_id, + workflow_type=activity.info.workflow_type, + activity_type=activity.info.activity_type, + activity_task_queue=self._task_queue, + is_local=activity.info.is_local, + ) + data_converter = data_converter._with_context(context) + # Perform the heartbeat try: heartbeat = temporalio.bridge.proto.ActivityHeartbeat( # type: ignore[reportAttributeAccessIssue] @@ -259,7 +271,7 @@ async def _heartbeat_async( ) if details: # Convert to core payloads - heartbeat.details.extend(await self._data_converter.encode(details)) + heartbeat.details.extend(await data_converter.encode(details)) logger.debug("Recording heartbeat with details %s", details) self._bridge_worker().record_activity_heartbeat(heartbeat) except Exception as err: @@ -293,9 +305,21 @@ async def _handle_start_activity_task( completion = temporalio.bridge.proto.ActivityTaskCompletion( # type: ignore[reportAttributeAccessIssue] task_token=task_token ) + # Create serialization context for the activity + context = temporalio.converter.ActivitySerializationContext( + namespace=start.workflow_namespace, + workflow_id=start.workflow_execution.workflow_id, + workflow_type=start.workflow_type, + activity_type=start.activity_type, + activity_task_queue=self._task_queue, + is_local=start.is_local, + ) + data_converter = self._data_converter._with_context(context) try: - result = await self._execute_activity(start, running_activity, task_token) - [payload] = await self._data_converter.encode([result]) + result = await self._execute_activity( + start, running_activity, task_token, data_converter + ) + [payload] = await data_converter.encode([result]) completion.result.completed.result.CopyFrom(payload) except BaseException as err: try: @@ -313,7 +337,7 @@ async def _handle_start_activity_task( temporalio.activity.logger.warning( f"Completing as failure during heartbeat with error of type {type(err)}: {err}", ) - await self._data_converter.encode_failure( + await data_converter.encode_failure( err, completion.result.failed.failure ) elif ( @@ -327,7 +351,7 @@ async def _handle_start_activity_task( temporalio.activity.logger.warning( "Completing as failure due to unhandled cancel error produced by activity pause", ) - await self._data_converter.encode_failure( + await data_converter.encode_failure( temporalio.exceptions.ApplicationError( type="ActivityPause", message="Unhandled activity cancel error produced by activity pause", @@ -345,7 +369,7 @@ async def _handle_start_activity_task( temporalio.activity.logger.warning( "Completing as failure due to unhandled cancel error produced by activity reset", ) - await self._data_converter.encode_failure( + await data_converter.encode_failure( temporalio.exceptions.ApplicationError( type="ActivityReset", message="Unhandled activity cancel error produced by activity reset", @@ -360,7 +384,7 @@ async def _handle_start_activity_task( and running_activity.cancelled_by_request ): temporalio.activity.logger.debug("Completing as cancelled") - await self._data_converter.encode_failure( + await data_converter.encode_failure( # TODO(cretz): Should use some other message? temporalio.exceptions.CancelledError("Cancelled"), completion.result.cancelled.failure, @@ -386,7 +410,7 @@ async def _handle_start_activity_task( exc_info=True, extra={"__temporal_error_identifier": "ActivityFailure"}, ) - await self._data_converter.encode_failure( + await data_converter.encode_failure( err, completion.result.failed.failure ) # For broken executors, we have to fail the entire worker @@ -428,6 +452,7 @@ async def _execute_activity( start: temporalio.bridge.proto.activity_task.Start, # type: ignore[reportAttributeAccessIssue] running_activity: _RunningActivity, task_token: bytes, + data_converter: temporalio.converter.DataConverter, ) -> Any: """Invoke the user's activity function. @@ -501,9 +526,7 @@ async def _execute_activity( args = ( [] if not start.input - else await self._data_converter.decode( - start.input, type_hints=arg_types - ) + else await data_converter.decode(start.input, type_hints=arg_types) ) except Exception as err: raise temporalio.exceptions.ApplicationError( @@ -519,7 +542,7 @@ async def _execute_activity( heartbeat_details = ( [] if not start.heartbeat_details - else await self._data_converter.decode(start.heartbeat_details) + else await data_converter.decode(start.heartbeat_details) ) except Exception as err: raise temporalio.exceptions.ApplicationError( @@ -563,11 +586,9 @@ async def _execute_activity( else None, ) - if self._encode_headers and self._data_converter.payload_codec is not None: + if self._encode_headers and data_converter.payload_codec is not None: for payload in start.header_fields.values(): - new_payload = ( - await self._data_converter.payload_codec.decode([payload]) - )[0] + new_payload = (await data_converter.payload_codec.decode([payload]))[0] payload.CopyFrom(new_payload) running_activity.info = info @@ -591,7 +612,7 @@ async def _execute_activity( if not running_activity.cancel_thread_raiser else running_activity.cancel_thread_raiser.shielded ), - payload_converter_class_or_instance=self._data_converter.payload_converter, + payload_converter_class_or_instance=data_converter.payload_converter, runtime_metric_meter=None if sync_non_threaded else self._metric_meter, client=self._client if not running_activity.sync else None, cancellation_details=running_activity.cancellation_details, diff --git a/temporalio/worker/_workflow.py b/temporalio/worker/_workflow.py index 1918496b8..fdf1e86d8 100644 --- a/temporalio/worker/_workflow.py +++ b/temporalio/worker/_workflow.py @@ -248,20 +248,13 @@ async def _handle_activation( await self._handle_cache_eviction(act, cache_remove_job) return + data_converter = self._data_converter # Build default success completion (e.g. remove-job-only activations) completion = ( temporalio.bridge.proto.workflow_completion.WorkflowActivationCompletion() ) completion.successful.SetInParent() try: - # Decode the activation if there's a codec and not cache remove job - if self._data_converter.payload_codec: - await temporalio.bridge.worker.decode_activation( - act, - self._data_converter.payload_codec, - decode_headers=self._encode_headers, - ) - if LOG_PROTOS: logger.debug("Received workflow activation:\n%s", act) @@ -273,9 +266,8 @@ async def _handle_activation( raise RuntimeError( "Missing initialize workflow, workflow could have unexpectedly been removed from cache" ) - workflow = _RunningWorkflow( - self._create_workflow_instance(act, init_job) - ) + workflow_instance, det = self._create_workflow_instance(act, init_job) + workflow = _RunningWorkflow(workflow_instance, det.info.workflow_id) self._running_workflows[act.run_id] = workflow elif init_job: # This should never happen @@ -283,6 +275,19 @@ async def _handle_activation( "Cache already exists for activation with initialize job" ) + data_converter = self._data_converter._with_context( + temporalio.converter.WorkflowSerializationContext( + namespace=self._namespace, + workflow_id=workflow.workflow_id, + ) + ) + if data_converter.payload_codec: + await temporalio.bridge.worker.decode_activation( + act, + data_converter.payload_codec, + decode_headers=self._encode_headers, + ) + # Run activation in separate thread so we can check if it's # deadlocked activate_task = asyncio.get_running_loop().run_in_executor( @@ -324,9 +329,9 @@ async def _handle_activation( # Set completion failure completion.failed.failure.SetInParent() try: - self._data_converter.failure_converter.to_failure( + data_converter.failure_converter.to_failure( err, - self._data_converter.payload_converter, + data_converter.payload_converter, completion.failed.failure, ) except Exception as inner_err: @@ -342,11 +347,11 @@ async def _handle_activation( completion.run_id = act.run_id # Encode the completion if there's a codec and not cache remove job - if self._data_converter.payload_codec: + if data_converter.payload_codec: try: await temporalio.bridge.worker.encode_completion( completion, - self._data_converter.payload_codec, + data_converter.payload_codec, encode_headers=self._encode_headers, ) except Exception as err: @@ -490,7 +495,7 @@ def _create_workflow_instance( self, act: temporalio.bridge.proto.workflow_activation.WorkflowActivation, init: temporalio.bridge.proto.workflow_activation.InitializeWorkflow, - ) -> WorkflowInstance: + ) -> tuple[WorkflowInstance, WorkflowInstanceDetails]: # Get the definition defn = self._workflows.get(init.workflow_type, self._dynamic_workflow) if not defn: @@ -570,9 +575,9 @@ def _create_workflow_instance( last_failure=last_failure, ) if defn.sandboxed: - return self._workflow_runner.create_instance(det) + return self._workflow_runner.create_instance(det), det else: - return self._unsandboxed_workflow_runner.create_instance(det) + return self._unsandboxed_workflow_runner.create_instance(det), det def nondeterminism_as_workflow_fail(self) -> bool: return any( @@ -666,8 +671,9 @@ def _gen_tb_helper( class _RunningWorkflow: - def __init__(self, instance: WorkflowInstance): + def __init__(self, instance: WorkflowInstance, workflow_id: str): self.instance = instance + self.workflow_id = workflow_id self.deadlocked_activation_task: Optional[Awaitable] = None self._deadlock_can_be_interrupted_lock = threading.Lock() self._deadlock_can_be_interrupted = False diff --git a/temporalio/worker/_workflow_instance.py b/temporalio/worker/_workflow_instance.py index 118966b34..56914199f 100644 --- a/temporalio/worker/_workflow_instance.py +++ b/temporalio/worker/_workflow_instance.py @@ -207,11 +207,18 @@ def __init__(self, det: WorkflowInstanceDetails) -> None: # No init for AbstractEventLoop WorkflowInstance.__init__(self) temporalio.workflow._Runtime.__init__(self) - self._payload_converter = det.payload_converter_class() - self._failure_converter = det.failure_converter_class() self._defn = det.defn self._workflow_input: Optional[ExecuteWorkflowInput] = None self._info = det.info + self._payload_converter_class = det.payload_converter_class + self._failure_converter_class = det.failure_converter_class + self._payload_converter, self._failure_converter = self._converters( + temporalio.converter.WorkflowSerializationContext( + namespace=det.info.namespace, + workflow_id=det.info.workflow_id, + ) + ) + self._extern_functions = det.extern_functions self._disable_eager_activity_execution = det.disable_eager_activity_execution self._worker_level_failure_exception_types = ( @@ -236,8 +243,8 @@ def __init__(self, det: WorkflowInstanceDetails) -> None: self._pending_activities: Dict[int, _ActivityHandle] = {} self._pending_child_workflows: Dict[int, _ChildWorkflowHandle] = {} self._pending_nexus_operations: Dict[int, _NexusOperationHandle] = {} - self._pending_external_signals: Dict[int, asyncio.Future] = {} - self._pending_external_cancels: Dict[int, asyncio.Future] = {} + self._pending_external_signals: Dict[int, Tuple[asyncio.Future, str]] = {} + self._pending_external_cancels: Dict[int, Tuple[asyncio.Future, str]] = {} # Keyed by type self._curr_seqs: Dict[str, int] = {} # TODO(cretz): Any concerns about not sharing this? Maybe the types I @@ -692,6 +699,7 @@ async def run_query() -> None: ) # Create input + # TODO: why do we deserialize query input in workflow but not signal? args = self._process_handler_args( job.query_type, job.arguments, @@ -754,6 +762,20 @@ def _apply_resolve_activity( handle = self._pending_activities.pop(job.seq, None) if not handle: raise RuntimeError(f"Failed finding activity handle for sequence {job.seq}") + payload_converter, failure_converter = self._converters( + temporalio.converter.ActivitySerializationContext( + namespace=self._info.namespace, + workflow_id=self._info.workflow_id, + workflow_type=self._info.workflow_type, + activity_type=handle._input.activity, + activity_task_queue=( + handle._input.task_queue or self._info.task_queue + if isinstance(handle._input, StartActivityInput) + else self._info.task_queue + ), + is_local=isinstance(handle._input, StartLocalActivityInput), + ) + ) if job.result.HasField("completed"): ret: Optional[Any] = None if job.result.completed.HasField("result"): @@ -761,19 +783,20 @@ def _apply_resolve_activity( ret_vals = self._convert_payloads( [job.result.completed.result], ret_types, + payload_converter, ) ret = ret_vals[0] handle._resolve_success(ret) elif job.result.HasField("failed"): handle._resolve_failure( - self._failure_converter.from_failure( - job.result.failed.failure, self._payload_converter + failure_converter.from_failure( + job.result.failed.failure, payload_converter ) ) elif job.result.HasField("cancelled"): handle._resolve_failure( - self._failure_converter.from_failure( - job.result.cancelled.failure, self._payload_converter + failure_converter.from_failure( + job.result.cancelled.failure, payload_converter ) ) elif job.result.HasField("backoff"): @@ -790,6 +813,12 @@ def _apply_resolve_child_workflow_execution( raise RuntimeError( f"Failed finding child workflow handle for sequence {job.seq}" ) + payload_converter, failure_converter = self._converters( + temporalio.converter.WorkflowSerializationContext( + namespace=self._info.namespace, + workflow_id=handle._input.id, + ) + ) if job.result.HasField("completed"): ret: Optional[Any] = None if job.result.completed.HasField("result"): @@ -797,19 +826,20 @@ def _apply_resolve_child_workflow_execution( ret_vals = self._convert_payloads( [job.result.completed.result], ret_types, + payload_converter, ) ret = ret_vals[0] handle._resolve_success(ret) elif job.result.HasField("failed"): handle._resolve_failure( - self._failure_converter.from_failure( - job.result.failed.failure, self._payload_converter + failure_converter.from_failure( + job.result.failed.failure, payload_converter ) ) elif job.result.HasField("cancelled"): handle._resolve_failure( - self._failure_converter.from_failure( - job.result.cancelled.failure, self._payload_converter + failure_converter.from_failure( + job.result.cancelled.failure, payload_converter ) ) else: @@ -845,11 +875,15 @@ def _apply_resolve_child_workflow_execution_start( ) elif job.HasField("cancelled"): self._pending_child_workflows.pop(job.seq) - handle._resolve_failure( - self._failure_converter.from_failure( - job.cancelled.failure, self._payload_converter + payload_converter, failure_converter = self._converters( + temporalio.converter.WorkflowSerializationContext( + namespace=self._info.namespace, + workflow_id=handle._input.id, ) ) + handle._resolve_failure( + failure_converter.from_failure(job.cancelled.failure, payload_converter) + ) else: raise RuntimeError("Child workflow start did not have a known status") @@ -862,6 +896,10 @@ def _apply_resolve_nexus_operation_start( raise RuntimeError( f"Failed to find nexus operation handle for job sequence number {job.seq}" ) + # We not set a serialization context for nexus operations on the caller side because it is + # not possible to do so on the handler side. + payload_converter, failure_converter = self._converters(None) + if job.HasField("operation_token"): # The nexus operation started asynchronously. A `ResolveNexusOperation` job # will follow in a future activation. @@ -874,9 +912,7 @@ def _apply_resolve_nexus_operation_start( # The nexus operation start failed; no ResolveNexusOperation will follow. self._pending_nexus_operations.pop(job.seq, None) handle._resolve_failure( - self._failure_converter.from_failure( - job.failed, self._payload_converter - ) + failure_converter.from_failure(job.failed, payload_converter) ) else: raise ValueError(f"Unknown Nexus operation start status: {job}") @@ -899,31 +935,29 @@ def _apply_resolve_nexus_operation( # completed / failed, but it has already been resolved. return + # We not set a serialization context for nexus operations on the caller side because it is + # not possible to do so on the handler side. + payload_converter, failure_converter = self._converters(None) # Handle the four oneof variants of NexusOperationResult result = job.result if result.HasField("completed"): [output] = self._convert_payloads( [result.completed], [handle._input.output_type] if handle._input.output_type else None, + payload_converter, ) handle._resolve_success(output) elif result.HasField("failed"): handle._resolve_failure( - self._failure_converter.from_failure( - result.failed, self._payload_converter - ) + failure_converter.from_failure(result.failed, payload_converter) ) elif result.HasField("cancelled"): handle._resolve_failure( - self._failure_converter.from_failure( - result.cancelled, self._payload_converter - ) + failure_converter.from_failure(result.cancelled, payload_converter) ) elif result.HasField("timed_out"): handle._resolve_failure( - self._failure_converter.from_failure( - result.timed_out, self._payload_converter - ) + failure_converter.from_failure(result.timed_out, payload_converter) ) else: raise RuntimeError("Nexus operation did not have a result") @@ -932,18 +966,23 @@ def _apply_resolve_request_cancel_external_workflow( self, job: temporalio.bridge.proto.workflow_activation.ResolveRequestCancelExternalWorkflow, ) -> None: - fut = self._pending_external_cancels.pop(job.seq, None) - if not fut: + pending = self._pending_external_cancels.pop(job.seq, None) + if not pending: raise RuntimeError( f"Failed finding pending external cancel for sequence {job.seq}" ) + fut, external_workflow_id = pending # We intentionally let this error if future is already done if job.HasField("failure"): - fut.set_exception( - self._failure_converter.from_failure( - job.failure, self._payload_converter + payload_converter, failure_converter = self._converters( + temporalio.converter.WorkflowSerializationContext( + namespace=self._info.namespace, + workflow_id=external_workflow_id, ) ) + fut.set_exception( + failure_converter.from_failure(job.failure, payload_converter) + ) else: fut.set_result(None) @@ -951,18 +990,23 @@ def _apply_resolve_signal_external_workflow( self, job: temporalio.bridge.proto.workflow_activation.ResolveSignalExternalWorkflow, ) -> None: - fut = self._pending_external_signals.pop(job.seq, None) - if not fut: + pending = self._pending_external_signals.pop(job.seq, None) + if not pending: raise RuntimeError( f"Failed finding pending external signal for sequence {job.seq}" ) + fut, external_workflow_id = pending # We intentionally let this error if future is already done if job.HasField("failure"): - fut.set_exception( - self._failure_converter.from_failure( - job.failure, self._payload_converter + payload_converter, failure_converter = self._converters( + temporalio.converter.WorkflowSerializationContext( + namespace=self._info.namespace, + workflow_id=external_workflow_id, ) ) + fut.set_exception( + failure_converter.from_failure(job.failure, payload_converter) + ) else: fut.set_result(None) @@ -1022,7 +1066,10 @@ def _make_workflow_input( if not self._defn.name: # Dynamic is just the raw value for each input value arg_types = [temporalio.common.RawValue] * len(init_job.arguments) - args = self._convert_payloads(init_job.arguments, arg_types) + + args = self._convert_payloads( + init_job.arguments, arg_types, self._payload_converter + ) # Put args in a list if dynamic if not self._defn.name: args = [args] @@ -1806,9 +1853,13 @@ async def run_activity() -> Any: async def _outbound_signal_child_workflow( self, input: SignalChildWorkflowInput ) -> None: - payloads = ( - self._payload_converter.to_payloads(input.args) if input.args else None + payload_converter, _ = self._converters( + temporalio.converter.WorkflowSerializationContext( + namespace=self._info.namespace, + workflow_id=input.child_workflow_id, + ) ) + payloads = payload_converter.to_payloads(input.args) if input.args else None command = self._add_command() v = command.signal_external_workflow_execution v.child_workflow_id = input.child_workflow_id @@ -1822,9 +1873,13 @@ async def _outbound_signal_child_workflow( async def _outbound_signal_external_workflow( self, input: SignalExternalWorkflowInput ) -> None: - payloads = ( - self._payload_converter.to_payloads(input.args) if input.args else None + payload_converter, _ = self._converters( + temporalio.converter.WorkflowSerializationContext( + namespace=input.namespace, + workflow_id=input.workflow_id, + ) ) + payloads = payload_converter.to_payloads(input.args) if input.args else None command = self._add_command() v = command.signal_external_workflow_execution v.workflow_execution.namespace = input.namespace @@ -1858,7 +1913,10 @@ def apply_child_cancel_error() -> None: # TODO(cretz): Nothing waits on this future, so how # if at all should we report child-workflow cancel # request failure? - self._pending_external_cancels[cancel_seq] = self.create_future() + self._pending_external_cancels[cancel_seq] = ( + self.create_future(), + input.id, + ) # Function that runs in the handle async def run_child() -> Any: @@ -1964,8 +2022,9 @@ async def _cancel_external_workflow( done_fut = self.create_future() command.request_cancel_external_workflow_execution.seq = seq - # Set as pending - self._pending_external_cancels[seq] = done_fut + # Set as pending with the target workflow ID for later context use + target_workflow_id = command.request_cancel_external_workflow_execution.workflow_execution.workflow_id + self._pending_external_cancels[seq] = (done_fut, target_workflow_id) # Wait until done (there is no cancelling a cancel request) await done_fut @@ -1980,6 +2039,7 @@ def _convert_payloads( self, payloads: Sequence[temporalio.api.common.v1.Payload], types: Optional[List[Type]], + payload_converter: temporalio.converter.PayloadConverter, ) -> List[Any]: if not payloads: return [] @@ -1987,10 +2047,7 @@ def _convert_payloads( if types and len(types) != len(payloads): types = None try: - return self._payload_converter.from_payloads( - payloads, - type_hints=types, - ) + return payload_converter.from_payloads(payloads, type_hints=types) except temporalio.exceptions.FailureError: # Don't wrap payload conversion errors that would fail the workflow raise @@ -1999,6 +2056,21 @@ def _convert_payloads( raise raise RuntimeError("Failed decoding arguments") from err + def _converters( + self, context: Optional[temporalio.converter.SerializationContext] + ) -> Tuple[ + temporalio.converter.PayloadConverter, + temporalio.converter.FailureConverter, + ]: + """Construct workflow payload and failure converters with the given context.""" + payload_converter = self._payload_converter_class() + failure_converter = self._failure_converter_class() + if isinstance(payload_converter, temporalio.converter.WithSerializationContext): + payload_converter = payload_converter.with_context(context) + if isinstance(failure_converter, temporalio.converter.WithSerializationContext): + failure_converter = failure_converter.with_context(context) + return payload_converter, failure_converter + def _instantiate_workflow_object(self) -> Any: if not self._workflow_input: raise RuntimeError("Expected workflow input. This is a Python SDK bug.") @@ -2082,15 +2154,21 @@ def _process_handler_args( if not defn_name and defn_dynamic_vararg: # Take off the string type hint for conversion arg_types = defn_arg_types[1:] if defn_arg_types else None - return [job_name] + self._convert_payloads(job_input, arg_types) + return [job_name] + self._convert_payloads( + job_input, arg_types, self._payload_converter + ) if not defn_name: return [ job_name, self._convert_payloads( - job_input, [temporalio.common.RawValue] * len(job_input) + job_input, + [temporalio.common.RawValue] * len(job_input), + self._payload_converter, ), ] - return self._convert_payloads(job_input, defn_arg_types) + return self._convert_payloads( + job_input, defn_arg_types, self._payload_converter + ) def _process_signal_job( self, @@ -2256,8 +2334,13 @@ async def _signal_external_workflow( done_fut = self.create_future() command.signal_external_workflow_execution.seq = seq - # Set as pending - self._pending_external_signals[seq] = done_fut + # Set as pending with the target workflow ID for later context use + # Extract the workflow ID from the command + target_workflow_id = ( + command.signal_external_workflow_execution.child_workflow_id + or command.signal_external_workflow_execution.workflow_execution.workflow_id + ) + self._pending_external_signals[seq] = (done_fut, target_workflow_id) # Wait until completed or cancelled while True: @@ -2724,6 +2807,20 @@ def __init__( self._result_fut = instance.create_future() self._started = False instance._register_task(self, name=f"activity: {input.activity}") + self._payload_converter, _ = self._instance._converters( + temporalio.converter.ActivitySerializationContext( + namespace=self._instance._info.namespace, + workflow_id=self._instance._info.workflow_id, + workflow_type=self._instance._info.workflow_type, + activity_type=self._input.activity, + activity_task_queue=( + self._input.task_queue or self._instance._info.task_queue + if isinstance(self._input, StartActivityInput) + else self._instance._info.task_queue + ), + is_local=isinstance(self._input, StartLocalActivityInput), + ) + ) def cancel(self, msg: Optional[Any] = None) -> bool: # Allow the cancel to go through for the task even if we're deleting, @@ -2771,7 +2868,7 @@ def _apply_schedule_command( ) -> None: # Convert arguments before creating command in case it raises error payloads = ( - self._instance._payload_converter.to_payloads(self._input.args) + self._payload_converter.to_payloads(self._input.args) if self._input.args else None ) @@ -2807,7 +2904,7 @@ def _apply_schedule_command( self._input.retry_policy.apply_to_proto(v.retry_policy) if self._input.summary: command.user_metadata.summary.CopyFrom( - self._instance._payload_converter.to_payload(self._input.summary) + self._payload_converter.to_payload(self._input.summary) ) v.cancellation_type = cast( temporalio.bridge.proto.workflow_commands.ActivityCancellationType.ValueType, @@ -2871,6 +2968,12 @@ def __init__( self._result_fut: asyncio.Future[Any] = instance.create_future() self._first_execution_run_id = "" instance._register_task(self, name=f"child: {input.workflow}") + self._payload_converter, _ = self._instance._converters( + temporalio.converter.WorkflowSerializationContext( + namespace=self._instance._info.namespace, + workflow_id=self._input.id, + ) + ) @property def id(self) -> str: @@ -2921,7 +3024,7 @@ def _resolve_failure(self, err: BaseException) -> None: def _apply_start_command(self) -> None: # Convert arguments before creating command in case it raises error payloads = ( - self._instance._payload_converter.to_payloads(self._input.args) + self._payload_converter.to_payloads(self._input.args) if self._input.args else None ) @@ -2956,9 +3059,7 @@ def _apply_start_command(self) -> None: temporalio.common._apply_headers(self._input.headers, v.headers) if self._input.memo: for k, val in self._input.memo.items(): - v.memo[k].CopyFrom( - self._instance._payload_converter.to_payloads([val])[0] - ) + v.memo[k].CopyFrom(self._payload_converter.to_payloads([val])[0]) if self._input.search_attributes: _encode_search_attributes( self._input.search_attributes, v.search_attributes @@ -2971,11 +3072,11 @@ def _apply_start_command(self) -> None: v.versioning_intent = self._input.versioning_intent._to_proto() if self._input.static_summary: command.user_metadata.summary.CopyFrom( - self._instance._payload_converter.to_payload(self._input.static_summary) + self._payload_converter.to_payload(self._input.static_summary) ) if self._input.static_details: command.user_metadata.details.CopyFrom( - self._instance._payload_converter.to_payload(self._input.static_details) + self._payload_converter.to_payload(self._input.static_details) ) if self._input.priority: v.priority.CopyFrom(self._input.priority._to_proto()) @@ -3057,6 +3158,7 @@ def __init__( self._task = asyncio.Task(fn) self._start_fut: asyncio.Future[Optional[str]] = instance.create_future() self._result_fut: asyncio.Future[Optional[OutputT]] = instance.create_future() + self._payload_converter, _ = self._instance._converters(None) @property def operation_token(self) -> Optional[str]: @@ -3089,7 +3191,7 @@ def _resolve_failure(self, err: BaseException) -> None: self._result_fut.set_result(None) def _apply_schedule_command(self) -> None: - payload = self._instance._payload_converter.to_payload(self._input.input) + payload = self._payload_converter.to_payload(self._input.input) command = self._instance._add_command() v = command.schedule_nexus_operation v.seq = self._seq diff --git a/tests/test_serialization_context.py b/tests/test_serialization_context.py new file mode 100644 index 000000000..e70f17818 --- /dev/null +++ b/tests/test_serialization_context.py @@ -0,0 +1,1328 @@ +""" +Test context-aware serde/codec operations. + +Serialization context should be available on all serde/codec operations, but testing all of them is +infeasible; this test suite only covers a selection. +""" + +from __future__ import annotations + +import asyncio +import dataclasses +import uuid +from collections import defaultdict +from dataclasses import dataclass, field +from datetime import timedelta +from typing import Any, List, Literal, Optional, Sequence, Type + +import pytest +from pydantic import BaseModel +from typing_extensions import Never + +from temporalio import activity, workflow +from temporalio.api.common.v1 import Payload +from temporalio.api.failure.v1 import Failure +from temporalio.client import Client, WorkflowFailureError, WorkflowUpdateFailedError +from temporalio.common import RetryPolicy +from temporalio.contrib.pydantic import PydanticJSONPlainPayloadConverter +from temporalio.converter import ( + ActivitySerializationContext, + CompositePayloadConverter, + DataConverter, + DefaultFailureConverter, + DefaultPayloadConverter, + EncodingPayloadConverter, + JSONPlainPayloadConverter, + PayloadCodec, + PayloadConverter, + SerializationContext, + WithSerializationContext, + WorkflowSerializationContext, +) +from temporalio.exceptions import ApplicationError +from temporalio.worker import Worker +from temporalio.worker._workflow_instance import UnsandboxedWorkflowRunner + + +@dataclass +class TraceItem: + method: Literal[ + "to_payload", + "from_payload", + "to_failure", + "from_failure", + "encode", + "decode", + ] + context: dict[str, Any] + + +@dataclass +class TraceData: + items: list[TraceItem] = field(default_factory=list) + + +class SerializationContextPayloadConverter( + EncodingPayloadConverter, WithSerializationContext +): + def __init__(self): + self.context: Optional[SerializationContext] = None + + @property + def encoding(self) -> str: + return "test-serialization-context" + + def with_context( + self, context: Optional[SerializationContext] + ) -> SerializationContextPayloadConverter: + converter = SerializationContextPayloadConverter() + converter.context = context + return converter + + def to_payload(self, value: Any) -> Optional[Payload]: + if not isinstance(value, TraceData): + return None + if not self.context: + raise Exception("Context is None") + if isinstance(self.context, WorkflowSerializationContext): + value.items.append( + TraceItem( + method="to_payload", + context=dataclasses.asdict(self.context), + ) + ) + elif isinstance(self.context, ActivitySerializationContext): + value.items.append( + TraceItem( + method="to_payload", + context=dataclasses.asdict(self.context), + ) + ) + else: + raise Exception(f"Unexpected context type: {type(self.context)}") + payload = JSONPlainPayloadConverter().to_payload(value) + assert payload + payload.metadata["encoding"] = self.encoding.encode() + return payload + + def from_payload(self, payload: Payload, type_hint: Optional[Type] = None) -> Any: + # Always deserialize as TraceData since that's what this converter handles + value = JSONPlainPayloadConverter().from_payload(payload, TraceData) + assert isinstance(value, TraceData) + if not self.context: + raise Exception("Context is None") + if isinstance(self.context, WorkflowSerializationContext): + value.items.append( + TraceItem( + method="from_payload", + context=dataclasses.asdict(self.context), + ) + ) + elif isinstance(self.context, ActivitySerializationContext): + value.items.append( + TraceItem( + method="from_payload", + context=dataclasses.asdict(self.context), + ) + ) + else: + raise Exception(f"Unexpected context type: {type(self.context)}") + return value + + +class SerializationContextCompositePayloadConverter( + CompositePayloadConverter, WithSerializationContext +): + def __init__(self): + super().__init__( + SerializationContextPayloadConverter(), + *DefaultPayloadConverter.default_encoding_payload_converters, + ) + + +# Test payload conversion + + +@activity.defn +async def passthrough_activity(input: TraceData) -> TraceData: + activity.heartbeat(input) + # Wait for the heartbeat to be processed so that it modifies the data before the activity returns + await asyncio.sleep(0.2) + return input + + +@workflow.defn +class EchoWorkflow: + @workflow.run + async def run(self, data: TraceData) -> TraceData: + return data + + +@workflow.defn +class PayloadConversionWorkflow: + @workflow.run + async def run(self, data: TraceData) -> TraceData: + data = await workflow.execute_activity( + passthrough_activity, + data, + start_to_close_timeout=timedelta(seconds=10), + heartbeat_timeout=timedelta(seconds=2), + ) + data = await workflow.execute_child_workflow( + EchoWorkflow.run, data, id=f"{workflow.info().workflow_id}_child" + ) + return data + + +async def test_workflow_payload_conversion( + client: Client, +): + workflow_id = str(uuid.uuid4()) + task_queue = str(uuid.uuid4()) + + config = client.config() + config["data_converter"] = dataclasses.replace( + DataConverter.default, + payload_converter_class=SerializationContextCompositePayloadConverter, + ) + client = Client(**config) + + async with Worker( + client, + task_queue=task_queue, + workflows=[PayloadConversionWorkflow, EchoWorkflow], + activities=[passthrough_activity], + workflow_runner=UnsandboxedWorkflowRunner(), # so that we can use isinstance + ): + result = await client.execute_workflow( + PayloadConversionWorkflow.run, + TraceData(), + id=workflow_id, + task_queue=task_queue, + ) + + workflow_context = dataclasses.asdict( + WorkflowSerializationContext( + namespace="default", + workflow_id=workflow_id, + ) + ) + child_workflow_context = dataclasses.asdict( + WorkflowSerializationContext( + namespace="default", + workflow_id=f"{workflow_id}_child", + ) + ) + activity_context = dataclasses.asdict( + ActivitySerializationContext( + namespace="default", + workflow_id=workflow_id, + workflow_type=PayloadConversionWorkflow.__name__, + activity_type=passthrough_activity.__name__, + activity_task_queue=task_queue, + is_local=False, + ) + ) + assert result.items == [ + TraceItem( + method="to_payload", + context=workflow_context, # Outbound workflow input + ), + TraceItem( + method="from_payload", + context=workflow_context, # Inbound workflow input + ), + TraceItem( + method="to_payload", + context=activity_context, # Outbound activity input + ), + TraceItem( + method="from_payload", + context=activity_context, # Inbound activity input + ), + TraceItem( + method="to_payload", + context=activity_context, # Outbound heartbeat + ), + TraceItem( + method="to_payload", + context=activity_context, # Outbound activity result + ), + TraceItem( + method="from_payload", + context=activity_context, # Inbound activity result + ), + TraceItem( + method="to_payload", + context=child_workflow_context, # Outbound child workflow input + ), + TraceItem( + method="from_payload", + context=child_workflow_context, # Inbound child workflow input + ), + TraceItem( + method="to_payload", + context=child_workflow_context, # Outbound child workflow result + ), + TraceItem( + method="from_payload", + context=child_workflow_context, # Inbound child workflow result + ), + TraceItem( + method="to_payload", + context=workflow_context, # Outbound workflow result + ), + TraceItem( + method="from_payload", + context=workflow_context, # Inbound workflow result + ), + ] + + +# Activity with heartbeat details test + + +@activity.defn +async def activity_with_heartbeat_details() -> TraceData: + """Activity that checks heartbeat details are decoded with proper context.""" + info = activity.info() + + if info.heartbeat_details: + assert len(info.heartbeat_details) == 1 + heartbeat_data = info.heartbeat_details[0] + assert isinstance(heartbeat_data, TraceData) + return heartbeat_data + + data = TraceData() + activity.heartbeat(data) + await asyncio.sleep(0.1) + raise Exception("Intentional failure to test heartbeat details") + + +@workflow.defn +class HeartbeatDetailsSerializationContextTestWorkflow: + @workflow.run + async def run(self) -> TraceData: + return await workflow.execute_activity( + activity_with_heartbeat_details, + start_to_close_timeout=timedelta(seconds=10), + retry_policy=RetryPolicy( + initial_interval=timedelta(milliseconds=100), + maximum_attempts=2, + ), + ) + + +async def test_heartbeat_details_payload_conversion(client: Client): + """Test that heartbeat details are decoded with activity context.""" + workflow_id = str(uuid.uuid4()) + task_queue = str(uuid.uuid4()) + + config = client.config() + config["data_converter"] = dataclasses.replace( + DataConverter.default, + payload_converter_class=SerializationContextCompositePayloadConverter, + ) + + client = Client(**config) + + async with Worker( + client, + task_queue=task_queue, + workflows=[HeartbeatDetailsSerializationContextTestWorkflow], + activities=[activity_with_heartbeat_details], + workflow_runner=UnsandboxedWorkflowRunner(), # so that we can use isinstance + ): + result = await client.execute_workflow( + HeartbeatDetailsSerializationContextTestWorkflow.run, + id=workflow_id, + task_queue=task_queue, + ) + + activity_context = dataclasses.asdict( + ActivitySerializationContext( + namespace="default", + workflow_id=workflow_id, + workflow_type=HeartbeatDetailsSerializationContextTestWorkflow.__name__, + activity_type=activity_with_heartbeat_details.__name__, + activity_task_queue=task_queue, + is_local=False, + ) + ) + + found_heartbeat_decode = False + for item in result.items: + if item.method == "from_payload" and item.context == activity_context: + found_heartbeat_decode = True + break + + assert ( + found_heartbeat_decode + ), "Heartbeat details should be decoded with activity context" + + +# Local activity test + + +@activity.defn +async def local_activity(input: TraceData) -> TraceData: + return input + + +@workflow.defn +class LocalActivityWorkflow: + @workflow.run + async def run(self, data: TraceData) -> TraceData: + return await workflow.execute_local_activity( + local_activity, + data, + start_to_close_timeout=timedelta(seconds=10), + ) + + +async def test_local_activity_payload_conversion(client: Client): + workflow_id = str(uuid.uuid4()) + task_queue = str(uuid.uuid4()) + + config = client.config() + config["data_converter"] = dataclasses.replace( + DataConverter.default, + payload_converter_class=SerializationContextCompositePayloadConverter, + ) + client = Client(**config) + + async with Worker( + client, + task_queue=task_queue, + workflows=[LocalActivityWorkflow], + activities=[local_activity], + workflow_runner=UnsandboxedWorkflowRunner(), # so that we can use isinstance + ): + result = await client.execute_workflow( + LocalActivityWorkflow.run, + TraceData(), + id=workflow_id, + task_queue=task_queue, + ) + + workflow_context = dataclasses.asdict( + WorkflowSerializationContext( + namespace="default", + workflow_id=workflow_id, + ) + ) + local_activity_context = dataclasses.asdict( + ActivitySerializationContext( + namespace="default", + workflow_id=workflow_id, + workflow_type=LocalActivityWorkflow.__name__, + activity_type=local_activity.__name__, + activity_task_queue=task_queue, + is_local=True, + ) + ) + + assert ( + result.items + == [ + TraceItem( + method="to_payload", + context=workflow_context, # Outbound workflow input + ), + TraceItem( + method="from_payload", + context=workflow_context, # Inbound workflow input + ), + TraceItem( + method="to_payload", + context=local_activity_context, # Outbound local activity input (is_local=True) + ), + TraceItem( + method="from_payload", + context=local_activity_context, # Inbound local activity input (is_local=True) + ), + TraceItem( + method="to_payload", + context=local_activity_context, # Outbound local activity result (is_local=True) + ), + TraceItem( + method="from_payload", + context=local_activity_context, # Inbound local activity result (is_local=True) + ), + TraceItem( + method="to_payload", + context=workflow_context, # Outbound workflow result + ), + TraceItem( + method="from_payload", + context=workflow_context, # Inbound workflow result + ), + ] + ) + + +# Async activity completion test + + +@activity.defn +async def async_activity() -> TraceData: + # Signal that activity has started via heartbeat + activity.heartbeat("started") + activity.raise_complete_async() + + +@workflow.defn +class AsyncActivityCompletionSerializationContextTestWorkflow: + @workflow.run + async def run(self) -> TraceData: + return await workflow.execute_activity( + async_activity, + start_to_close_timeout=timedelta(seconds=10), + activity_id="async-activity-id", + ) + + +async def test_async_activity_completion_payload_conversion( + client: Client, +): + workflow_id = str(uuid.uuid4()) + task_queue = str(uuid.uuid4()) + + config = client.config() + config["data_converter"] = dataclasses.replace( + DataConverter.default, + payload_converter_class=SerializationContextCompositePayloadConverter, + ) + + client = Client(**config) + + async with Worker( + client, + task_queue=task_queue, + workflows=[AsyncActivityCompletionSerializationContextTestWorkflow], + activities=[async_activity], + workflow_runner=UnsandboxedWorkflowRunner(), # so that we can use isinstance + ): + wf_handle = await client.start_workflow( + AsyncActivityCompletionSerializationContextTestWorkflow.run, + id=workflow_id, + task_queue=task_queue, + ) + activity_handle = client.get_async_activity_handle( + workflow_id=workflow_id, + run_id=wf_handle.first_execution_run_id, + activity_id="async-activity-id", + ) + # Wait a bit for the activity to start + await asyncio.sleep(0.5) + data = TraceData() + await activity_handle.heartbeat(data) + await activity_handle.complete(data) + result = await wf_handle.result() + + # project down since activity completion by a client does not have access to most activity + # context fields + def project(trace_item: TraceItem) -> str: + return trace_item.method + + assert [project(item) for item in result.items] == [ + "to_payload", # Outbound activity input + "to_payload", # Outbound activity heartbeat data + "from_payload", # Inbound activity result + "to_payload", # Outbound workflow result + "from_payload", # Inbound workflow result + ] + + +# Signal test + + +@workflow.defn(sandboxed=False) # so that we can use isinstance +class SignalSerializationContextTestWorkflow: + def __init__(self) -> None: + self.signal_received: Optional[TraceData] = None + + @workflow.run + async def run(self) -> TraceData: + await workflow.wait_condition(lambda: self.signal_received is not None) + assert self.signal_received is not None + return self.signal_received + + @workflow.signal + async def my_signal(self, data: TraceData) -> None: + self.signal_received = data + + +async def test_signal_payload_conversion( + client: Client, +): + workflow_id = str(uuid.uuid4()) + task_queue = str(uuid.uuid4()) + + config = client.config() + config["data_converter"] = dataclasses.replace( + DataConverter.default, + payload_converter_class=SerializationContextCompositePayloadConverter, + ) + + custom_client = Client(**config) + + async with Worker( + custom_client, + task_queue=task_queue, + workflows=[SignalSerializationContextTestWorkflow], + activities=[], + workflow_runner=UnsandboxedWorkflowRunner(), # so that we can use isinstance + ): + handle = await custom_client.start_workflow( + SignalSerializationContextTestWorkflow.run, + id=workflow_id, + task_queue=task_queue, + ) + await handle.signal( + SignalSerializationContextTestWorkflow.my_signal, + TraceData(), + ) + result = await handle.result() + + workflow_context = dataclasses.asdict( + WorkflowSerializationContext( + namespace="default", + workflow_id=workflow_id, + ) + ) + assert result.items == [ + TraceItem( + method="to_payload", + context=workflow_context, # Outbound signal input + ), + TraceItem( + method="from_payload", + context=workflow_context, # Inbound signal input + ), + TraceItem( + method="to_payload", + context=workflow_context, # Outbound workflow result + ), + TraceItem( + method="from_payload", + context=workflow_context, # Inbound workflow result + ), + ] + + +# Query test + + +@workflow.defn +class QuerySerializationContextTestWorkflow: + @workflow.run + async def run(self) -> None: + await asyncio.Event().wait() + + @workflow.query + def my_query(self, input: TraceData) -> TraceData: + return input + + +async def test_query_payload_conversion( + client: Client, +): + workflow_id = str(uuid.uuid4()) + task_queue = str(uuid.uuid4()) + + config = client.config() + config["data_converter"] = dataclasses.replace( + DataConverter.default, + payload_converter_class=SerializationContextCompositePayloadConverter, + ) + custom_client = Client(**config) + + async with Worker( + custom_client, + task_queue=task_queue, + workflows=[QuerySerializationContextTestWorkflow], + activities=[], + workflow_runner=UnsandboxedWorkflowRunner(), # so that we can use isinstance + ): + handle = await custom_client.start_workflow( + QuerySerializationContextTestWorkflow.run, + id=workflow_id, + task_queue=task_queue, + ) + result = await handle.query( + QuerySerializationContextTestWorkflow.my_query, TraceData() + ) + + workflow_context = dataclasses.asdict( + WorkflowSerializationContext( + namespace="default", + workflow_id=workflow_id, + ) + ) + assert result.items == [ + TraceItem( + method="to_payload", + context=workflow_context, # Outbound query input + ), + TraceItem( + method="from_payload", + context=workflow_context, # Inbound query input + ), + TraceItem( + method="to_payload", + context=workflow_context, # Outbound query result + ), + TraceItem( + method="from_payload", + context=workflow_context, # Inbound query result + ), + ] + + +# Update test + + +@workflow.defn +class UpdateSerializationContextTestWorkflow: + @workflow.init + def __init__(self, pass_validation: bool) -> None: + self.pass_validation = pass_validation + self.input: Optional[TraceData] = None + + @workflow.run + async def run(self, pass_validation: bool) -> TraceData: + await workflow.wait_condition(lambda: self.input is not None) + assert self.input + return self.input + + @workflow.update + def my_update(self, input: TraceData) -> TraceData: + return input + + @my_update.validator + def my_update_validator(self, input: TraceData) -> None: + self.input = input # for test purposes; update validators should not mutate workflow state + if not self.pass_validation: + raise ValueError("Rejected") + + +@pytest.mark.parametrize("pass_validation", [True, False]) +async def test_update_payload_conversion( + client: Client, + pass_validation: bool, +): + workflow_id = str(uuid.uuid4()) + task_queue = str(uuid.uuid4()) + + config = client.config() + config["data_converter"] = dataclasses.replace( + DataConverter.default, + payload_converter_class=SerializationContextCompositePayloadConverter, + ) + custom_client = Client(**config) + + async with Worker( + custom_client, + task_queue=task_queue, + workflows=[UpdateSerializationContextTestWorkflow], + activities=[], + workflow_runner=UnsandboxedWorkflowRunner(), # so that we can use isinstance + ): + wf_handle = await custom_client.start_workflow( + UpdateSerializationContextTestWorkflow.run, + pass_validation, + id=workflow_id, + task_queue=task_queue, + ) + if pass_validation: + result = await wf_handle.execute_update( + UpdateSerializationContextTestWorkflow.my_update, TraceData() + ) + else: + try: + await wf_handle.execute_update( + UpdateSerializationContextTestWorkflow.my_update, TraceData() + ) + raise AssertionError("Expected WorkflowUpdateFailedError") + except WorkflowUpdateFailedError: + pass + + result = await wf_handle.result() + + workflow_context = dataclasses.asdict( + WorkflowSerializationContext( + namespace="default", + workflow_id=workflow_id, + ) + ) + assert result.items == [ + TraceItem( + method="to_payload", + context=workflow_context, # Outbound update input + ), + TraceItem( + method="from_payload", + context=workflow_context, # Inbound update input + ), + TraceItem( + method="to_payload", + context=workflow_context, # Outbound update/workflow result + ), + TraceItem( + method="from_payload", + context=workflow_context, # Inbound update/workflow result + ), + ] + + +# External workflow test + + +@workflow.defn +class ExternalWorkflowTarget: + def __init__(self) -> None: + self.signal_received: Optional[TraceData] = None + + @workflow.run + async def run(self) -> TraceData: + try: + await workflow.wait_condition(lambda: self.signal_received is not None) + return self.signal_received or TraceData() + except asyncio.CancelledError: + return TraceData() + + @workflow.signal + async def external_signal(self, data: TraceData) -> None: + self.signal_received = data + + +@workflow.defn +class ExternalWorkflowSignaler: + @workflow.run + async def run(self, target_id: str, data: TraceData) -> TraceData: + handle = workflow.get_external_workflow_handle(target_id) + await handle.signal(ExternalWorkflowTarget.external_signal, data) + return data + + +@workflow.defn +class ExternalWorkflowCanceller: + @workflow.run + async def run(self, target_id: str) -> TraceData: + handle = workflow.get_external_workflow_handle(target_id) + await handle.cancel() + return TraceData() + + +@pytest.mark.timeout(10) +async def test_external_workflow_signal_and_cancel_payload_conversion( + client: Client, +): + target_workflow_id = str(uuid.uuid4()) + signaler_workflow_id = str(uuid.uuid4()) + task_queue = str(uuid.uuid4()) + + config = client.config() + config["data_converter"] = dataclasses.replace( + DataConverter.default, + payload_converter_class=SerializationContextCompositePayloadConverter, + ) + client = Client(**config) + + async with Worker( + client, + task_queue=task_queue, + workflows=[ + ExternalWorkflowTarget, + ExternalWorkflowSignaler, + ExternalWorkflowCanceller, + ], + activities=[], + workflow_runner=UnsandboxedWorkflowRunner(), # so that we can use isinstance + ): + target_handle = await client.start_workflow( + ExternalWorkflowTarget.run, + id=target_workflow_id, + task_queue=task_queue, + ) + + signaler_handle = await client.start_workflow( + ExternalWorkflowSignaler.run, + args=[target_workflow_id, TraceData()], + id=signaler_workflow_id, + task_queue=task_queue, + ) + + signaler_result = await signaler_handle.result() + await target_handle.result() + + signaler_context = dataclasses.asdict( + WorkflowSerializationContext( + namespace="default", + workflow_id=signaler_workflow_id, + ) + ) + target_context = dataclasses.asdict( + WorkflowSerializationContext( + namespace="default", + workflow_id=target_workflow_id, + ) + ) + + assert ( + signaler_result.items + == [ + TraceItem( + method="to_payload", + context=signaler_context, # Outbound signaler workflow input + ), + TraceItem( + method="from_payload", + context=signaler_context, # Inbound signaler workflow input + ), + TraceItem( + method="to_payload", + context=target_context, # Should use target workflow's context for external signal + ), + TraceItem( + method="to_payload", + context=signaler_context, # Outbound signaler workflow result + ), + TraceItem( + method="from_payload", + context=signaler_context, # Inbound signaler workflow result + ), + ] + ) + + +# Failure conversion + + +@activity.defn +async def failing_activity() -> Never: + raise ApplicationError("test error", dataclasses.asdict(TraceData())) + + +@workflow.defn +class FailureConverterTestWorkflow: + @workflow.run + async def run(self) -> Never: + await workflow.execute_activity( + failing_activity, + start_to_close_timeout=timedelta(seconds=10), + retry_policy=RetryPolicy(maximum_attempts=1), + ) + raise Exception("Unreachable") + + +test_traces: dict[str, list[TraceItem]] = defaultdict(list) + + +class FailureConverterWithContext(DefaultFailureConverter, WithSerializationContext): + def __init__(self): + super().__init__(encode_common_attributes=False) + self.context: Optional[SerializationContext] = None + + def with_context( + self, context: Optional[SerializationContext] + ) -> "FailureConverterWithContext": + converter = FailureConverterWithContext() + converter.context = context + return converter + + def to_failure( + self, + exception: BaseException, + payload_converter: PayloadConverter, + failure: Failure, + ) -> None: + assert isinstance( + self.context, (WorkflowSerializationContext, ActivitySerializationContext) + ) + test_traces[self.context.workflow_id].append( + TraceItem( + method="to_failure", + context=dataclasses.asdict(self.context), + ) + ) + super().to_failure(exception, payload_converter, failure) + + def from_failure( + self, failure: Failure, payload_converter: PayloadConverter + ) -> BaseException: + assert isinstance( + self.context, (WorkflowSerializationContext, ActivitySerializationContext) + ) + test_traces[self.context.workflow_id].append( + TraceItem( + method="from_failure", + context=dataclasses.asdict(self.context), + ) + ) + return super().from_failure(failure, payload_converter) + + +async def test_failure_converter_with_context(client: Client): + workflow_id = str(uuid.uuid4()) + task_queue = str(uuid.uuid4()) + + data_converter = dataclasses.replace( + DataConverter.default, + failure_converter_class=FailureConverterWithContext, + ) + config = client.config() + config["data_converter"] = data_converter + client = Client(**config) + + async with Worker( + client, + task_queue=task_queue, + workflows=[FailureConverterTestWorkflow], + activities=[failing_activity], + workflow_runner=UnsandboxedWorkflowRunner(), + ): + try: + await client.execute_workflow( + FailureConverterTestWorkflow.run, + id=workflow_id, + task_queue=task_queue, + ) + raise AssertionError("unreachable") + except WorkflowFailureError: + pass + + assert isinstance(data_converter.failure_converter, FailureConverterWithContext) + + workflow_context = dataclasses.asdict( + WorkflowSerializationContext( + namespace="default", + workflow_id=workflow_id, + ) + ) + activity_context = dataclasses.asdict( + ActivitySerializationContext( + namespace="default", + workflow_id=workflow_id, + workflow_type=FailureConverterTestWorkflow.__name__, + activity_type=failing_activity.__name__, + activity_task_queue=task_queue, + is_local=False, + ) + ) + assert test_traces[workflow_id] == ( + [ + TraceItem( + context=activity_context, + method="to_failure", # outbound activity result + ) + ] + + ( + [ + TraceItem( + context=activity_context, + method="from_failure", # inbound activity result + ) + ] + * 2 # from_failure deserializes the error and error cause + ) + + [ + TraceItem( + context=workflow_context, + method="to_failure", # outbound workflow result + ) + ] + + ( + [ + TraceItem( + context=workflow_context, + method="from_failure", # inbound workflow result + ) + ] + * 2 # from_failure deserializes the error and error cause + ) + ) + del test_traces[workflow_id] + + +class PayloadCodecWithContext(PayloadCodec, WithSerializationContext): + def __init__(self): + self.context: Optional[SerializationContext] = None + self.encode_called_with_context = False + self.decode_called_with_context = False + + def with_context( + self, context: Optional[SerializationContext] + ) -> "PayloadCodecWithContext": + codec = PayloadCodecWithContext() + codec.context = context + return codec + + async def encode(self, payloads: Sequence[Payload]) -> List[Payload]: + assert self.context + if isinstance(self.context, ActivitySerializationContext): + test_traces[self.context.workflow_id].append( + TraceItem( + context=dataclasses.asdict(self.context), + method="encode", + ) + ) + else: + assert isinstance(self.context, WorkflowSerializationContext) + test_traces[self.context.workflow_id].append( + TraceItem( + context=dataclasses.asdict(self.context), + method="encode", + ) + ) + return list(payloads) + + async def decode(self, payloads: Sequence[Payload]) -> List[Payload]: + assert self.context + if isinstance(self.context, ActivitySerializationContext): + test_traces[self.context.workflow_id].append( + TraceItem( + context=dataclasses.asdict(self.context), + method="decode", + ) + ) + else: + assert isinstance(self.context, WorkflowSerializationContext) + test_traces[self.context.workflow_id].append( + TraceItem( + context=dataclasses.asdict(self.context), + method="decode", + ) + ) + return list(payloads) + + +@workflow.defn +class CodecTestWorkflow: + @workflow.run + async def run(self, data: str) -> str: + return data + + +async def test_codec_with_context(client: Client): + workflow_id = str(uuid.uuid4()) + task_queue = str(uuid.uuid4()) + + client_config = client.config() + client_config["data_converter"] = dataclasses.replace( + DataConverter.default, payload_codec=PayloadCodecWithContext() + ) + client = Client(**client_config) + async with Worker( + client, + task_queue=task_queue, + workflows=[CodecTestWorkflow], + ): + await client.execute_workflow( + CodecTestWorkflow.run, + "data", + id=workflow_id, + task_queue=task_queue, + ) + workflow_context = dataclasses.asdict( + WorkflowSerializationContext( + namespace=client.namespace, + workflow_id=workflow_id, + ) + ) + assert test_traces[workflow_id] == [ + TraceItem( + context=workflow_context, + method="encode", + ), + TraceItem( + context=workflow_context, + method="decode", + ), + TraceItem( + context=workflow_context, + method="encode", + ), + TraceItem( + context=workflow_context, + method="decode", + ), + ] + del test_traces[workflow_id] + + +@activity.defn +async def codec_test_local_activity(data: str) -> str: + return data + + +@workflow.defn +class LocalActivityCodecTestWorkflow: + @workflow.run + async def run(self, data: str) -> str: + return await workflow.execute_local_activity( + codec_test_local_activity, + data, + start_to_close_timeout=timedelta(seconds=10), + ) + + +async def test_local_activity_codec_with_context(client: Client): + """Test that codec gets correct context with is_local=True for local activities.""" + workflow_id = str(uuid.uuid4()) + task_queue = str(uuid.uuid4()) + + client_config = client.config() + client_config["data_converter"] = dataclasses.replace( + DataConverter.default, payload_codec=PayloadCodecWithContext() + ) + client = Client(**client_config) + async with Worker( + client, + task_queue=task_queue, + workflows=[LocalActivityCodecTestWorkflow], + activities=[codec_test_local_activity], + ): + await client.execute_workflow( + LocalActivityCodecTestWorkflow.run, + "data", + id=workflow_id, + task_queue=task_queue, + ) + + workflow_context = dataclasses.asdict( + WorkflowSerializationContext( + namespace=client.namespace, + workflow_id=workflow_id, + ) + ) + local_activity_context = dataclasses.asdict( + ActivitySerializationContext( + namespace=client.namespace, + workflow_id=workflow_id, + workflow_type=LocalActivityCodecTestWorkflow.__name__, + activity_type=codec_test_local_activity.__name__, + activity_task_queue=task_queue, + is_local=True, # Should be True for local activities + ) + ) + + # Note: Local activities have partial activity context support through codec + # The input encode uses workflow context, but the decode uses activity context + # The result encode uses activity context, but the decode uses workflow context + assert test_traces[workflow_id] == [ + # Workflow input + TraceItem( + context=workflow_context, + method="encode", + ), + TraceItem( + context=workflow_context, + method="decode", + ), + # Local activity input - encode uses workflow context + TraceItem( + context=workflow_context, + method="encode", + ), + # Local activity input - decode uses activity context with is_local=True + TraceItem( + context=local_activity_context, + method="decode", + ), + # Local activity result - encode uses activity context with is_local=True + TraceItem( + context=local_activity_context, + method="encode", + ), + # Local activity result - decode uses workflow context + TraceItem( + context=workflow_context, + method="decode", + ), + # Workflow result + TraceItem( + context=workflow_context, + method="encode", + ), + TraceItem( + context=workflow_context, + method="decode", + ), + ] + del test_traces[workflow_id] + + +# Pydantic + + +class PydanticData(BaseModel): + value: str + trace: List[str] = [] + + +class PydanticJSONConverterWithContext( + PydanticJSONPlainPayloadConverter, WithSerializationContext +): + def __init__(self): + super().__init__() + self.context: Optional[SerializationContext] = None + + def with_context( + self, context: Optional[SerializationContext] + ) -> "PydanticJSONConverterWithContext": + converter = PydanticJSONConverterWithContext() + converter.context = context + return converter + + def to_payload(self, value: Any) -> Optional[Payload]: + if isinstance(value, PydanticData) and self.context: + if isinstance(self.context, WorkflowSerializationContext): + value.trace.append(f"wf_{self.context.workflow_id}") + return super().to_payload(value) + + +class PydanticConverterWithContext(CompositePayloadConverter, WithSerializationContext): + def __init__(self): + super().__init__( + *( + c + if not isinstance(c, JSONPlainPayloadConverter) + else PydanticJSONConverterWithContext() + for c in DefaultPayloadConverter.default_encoding_payload_converters + ) + ) + self.context: Optional[SerializationContext] = None + + +@workflow.defn +class PydanticContextWorkflow: + @workflow.run + async def run(self, data: PydanticData) -> PydanticData: + return data + + +async def test_pydantic_converter_with_context(client: Client): + wf_id = str(uuid.uuid4()) + task_queue = str(uuid.uuid4()) + + client_config = client.config() + client_config["data_converter"] = dataclasses.replace( + DataConverter.default, + payload_converter_class=PydanticConverterWithContext, + ) + client = Client(**client_config) + + async with Worker( + client, + task_queue=task_queue, + workflows=[PydanticContextWorkflow], + ): + result = await client.execute_workflow( + PydanticContextWorkflow.run, + PydanticData(value="test"), + id=wf_id, + task_queue=task_queue, + ) + assert f"wf_{wf_id}" in result.trace From 80119de8b3c63ea496df9674bfa25ad74f1ddee2 Mon Sep 17 00:00:00 2001 From: Dan Davison Date: Mon, 15 Sep 2025 11:54:55 -0400 Subject: [PATCH 03/81] Partial revert --- temporalio/worker/_workflow.py | 33 ++++++++++++++++++++++----------- 1 file changed, 22 insertions(+), 11 deletions(-) diff --git a/temporalio/worker/_workflow.py b/temporalio/worker/_workflow.py index fdf1e86d8..b37eb342d 100644 --- a/temporalio/worker/_workflow.py +++ b/temporalio/worker/_workflow.py @@ -255,19 +255,36 @@ async def _handle_activation( ) completion.successful.SetInParent() try: + # Decode the activation if there's a codec and not cache remove job + if LOG_PROTOS: logger.debug("Received workflow activation:\n%s", act) # If the workflow is not running yet, create it workflow = self._running_workflows.get(act.run_id) + + if data_converter.payload_codec: + await temporalio.bridge.worker.decode_activation( + act, + data_converter.payload_codec, + decode_headers=self._encode_headers, + ) if not workflow: # Must have a initialize job to create instance if not init_job: raise RuntimeError( "Missing initialize workflow, workflow could have unexpectedly been removed from cache" ) - workflow_instance, det = self._create_workflow_instance(act, init_job) - workflow = _RunningWorkflow(workflow_instance, det.info.workflow_id) + data_converter = self._data_converter._with_context( + temporalio.converter.WorkflowSerializationContext( + namespace=self._namespace, + workflow_id=init_job.workflow_id, + ) + ) + workflow = _RunningWorkflow( + self._create_workflow_instance(act, init_job), + init_job.workflow_id, + ) self._running_workflows[act.run_id] = workflow elif init_job: # This should never happen @@ -281,12 +298,6 @@ async def _handle_activation( workflow_id=workflow.workflow_id, ) ) - if data_converter.payload_codec: - await temporalio.bridge.worker.decode_activation( - act, - data_converter.payload_codec, - decode_headers=self._encode_headers, - ) # Run activation in separate thread so we can check if it's # deadlocked @@ -495,7 +506,7 @@ def _create_workflow_instance( self, act: temporalio.bridge.proto.workflow_activation.WorkflowActivation, init: temporalio.bridge.proto.workflow_activation.InitializeWorkflow, - ) -> tuple[WorkflowInstance, WorkflowInstanceDetails]: + ) -> WorkflowInstance: # Get the definition defn = self._workflows.get(init.workflow_type, self._dynamic_workflow) if not defn: @@ -575,9 +586,9 @@ def _create_workflow_instance( last_failure=last_failure, ) if defn.sandboxed: - return self._workflow_runner.create_instance(det), det + return self._workflow_runner.create_instance(det) else: - return self._unsandboxed_workflow_runner.create_instance(det), det + return self._unsandboxed_workflow_runner.create_instance(det) def nondeterminism_as_workflow_fail(self) -> bool: return any( From cebff4f0e133102560c27404d732f2796bb1caf9 Mon Sep 17 00:00:00 2001 From: Dan Davison Date: Mon, 15 Sep 2025 23:11:14 -0400 Subject: [PATCH 04/81] Refactor activation processing --- temporalio/worker/_workflow.py | 52 ++++++++++++++-------------------- 1 file changed, 22 insertions(+), 30 deletions(-) diff --git a/temporalio/worker/_workflow.py b/temporalio/worker/_workflow.py index b37eb342d..24099a998 100644 --- a/temporalio/worker/_workflow.py +++ b/temporalio/worker/_workflow.py @@ -255,49 +255,43 @@ async def _handle_activation( ) completion.successful.SetInParent() try: - # Decode the activation if there's a codec and not cache remove job - if LOG_PROTOS: logger.debug("Received workflow activation:\n%s", act) - # If the workflow is not running yet, create it workflow = self._running_workflows.get(act.run_id) - - if data_converter.payload_codec: - await temporalio.bridge.worker.decode_activation( - act, - data_converter.payload_codec, - decode_headers=self._encode_headers, - ) if not workflow: - # Must have a initialize job to create instance if not init_job: raise RuntimeError( "Missing initialize workflow, workflow could have unexpectedly been removed from cache" ) - data_converter = self._data_converter._with_context( - temporalio.converter.WorkflowSerializationContext( - namespace=self._namespace, - workflow_id=init_job.workflow_id, + workflow_id = init_job.workflow_id + else: + workflow_id = workflow.workflow_id + if init_job: + # Should never happen + logger.warning( + "Cache already exists for activation with initialize job" ) - ) - workflow = _RunningWorkflow( - self._create_workflow_instance(act, init_job), - init_job.workflow_id, - ) - self._running_workflows[act.run_id] = workflow - elif init_job: - # This should never happen - logger.warning( - "Cache already exists for activation with initialize job" - ) data_converter = self._data_converter._with_context( temporalio.converter.WorkflowSerializationContext( namespace=self._namespace, - workflow_id=workflow.workflow_id, + workflow_id=workflow_id, ) ) + if data_converter.payload_codec: + await temporalio.bridge.worker.decode_activation( + act, + data_converter.payload_codec, + decode_headers=self._encode_headers, + ) + if not workflow: + assert init_job + workflow = _RunningWorkflow( + self._create_workflow_instance(act, init_job), + workflow_id, + ) + self._running_workflows[act.run_id] = workflow # Run activation in separate thread so we can check if it's # deadlocked @@ -337,7 +331,6 @@ async def _handle_activation( "Failed handling activation on workflow with run ID %s", act.run_id ) - # Set completion failure completion.failed.failure.SetInParent() try: data_converter.failure_converter.to_failure( @@ -354,10 +347,9 @@ async def _handle_activation( f"Failed converting activation exception: {inner_err}" ) - # Always set the run ID on the completion completion.run_id = act.run_id - # Encode the completion if there's a codec and not cache remove job + # Encode completion if data_converter.payload_codec: try: await temporalio.bridge.worker.encode_completion( From 03cae3795635650074f978d3d3dced7a2ac8e0da Mon Sep 17 00:00:00 2001 From: Dan Davison Date: Fri, 19 Sep 2025 08:05:57 -0400 Subject: [PATCH 05/81] Do not support None context --- temporalio/converter.py | 17 +++++++++-------- temporalio/worker/_workflow_instance.py | 13 +++++++++---- 2 files changed, 18 insertions(+), 12 deletions(-) diff --git a/temporalio/converter.py b/temporalio/converter.py index 03a81660d..93d31b2d4 100644 --- a/temporalio/converter.py +++ b/temporalio/converter.py @@ -139,7 +139,7 @@ class WithSerializationContext(ABC): during serialization and deserialization. """ - def with_context(self, context: Optional[SerializationContext]) -> Self: + def with_context(self, context: SerializationContext) -> Self: """Return a copy of this object configured to use the given context. Args: @@ -401,7 +401,7 @@ def from_payloads( ) from err return values - def with_context(self, context: Optional[SerializationContext]) -> Self: + def with_context(self, context: SerializationContext) -> Self: """Return a new instance with the given context.""" converters = [ c.with_context(context) if isinstance(c, WithSerializationContext) else c @@ -1313,12 +1313,13 @@ def _with_context(self, context: Optional[SerializationContext]) -> Self: payload_converter = self.payload_converter payload_codec = self.payload_codec failure_converter = self.failure_converter - if isinstance(payload_converter, WithSerializationContext): - payload_converter = payload_converter.with_context(context) - if isinstance(payload_codec, WithSerializationContext): - payload_codec = payload_codec.with_context(context) - if isinstance(failure_converter, WithSerializationContext): - failure_converter = failure_converter.with_context(context) + if context: + if isinstance(payload_converter, WithSerializationContext): + payload_converter = payload_converter.with_context(context) + if isinstance(payload_codec, WithSerializationContext): + payload_codec = payload_codec.with_context(context) + if isinstance(failure_converter, WithSerializationContext): + failure_converter = failure_converter.with_context(context) object.__setattr__(cloned, "payload_converter", payload_converter) object.__setattr__(cloned, "payload_codec", payload_codec) object.__setattr__(cloned, "failure_converter", failure_converter) diff --git a/temporalio/worker/_workflow_instance.py b/temporalio/worker/_workflow_instance.py index 56914199f..36fb67acd 100644 --- a/temporalio/worker/_workflow_instance.py +++ b/temporalio/worker/_workflow_instance.py @@ -2065,10 +2065,15 @@ def _converters( """Construct workflow payload and failure converters with the given context.""" payload_converter = self._payload_converter_class() failure_converter = self._failure_converter_class() - if isinstance(payload_converter, temporalio.converter.WithSerializationContext): - payload_converter = payload_converter.with_context(context) - if isinstance(failure_converter, temporalio.converter.WithSerializationContext): - failure_converter = failure_converter.with_context(context) + if context: + if isinstance( + payload_converter, temporalio.converter.WithSerializationContext + ): + payload_converter = payload_converter.with_context(context) + if isinstance( + failure_converter, temporalio.converter.WithSerializationContext + ): + failure_converter = failure_converter.with_context(context) return payload_converter, failure_converter def _instantiate_workflow_object(self) -> Any: From 3bd3bab1e395616ca74d7b37623553a8a580e005 Mon Sep 17 00:00:00 2001 From: Dan Davison Date: Fri, 19 Sep 2025 08:40:27 -0400 Subject: [PATCH 06/81] Never reinstantiate converters --- temporalio/worker/_workflow_instance.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/temporalio/worker/_workflow_instance.py b/temporalio/worker/_workflow_instance.py index 36fb67acd..10b9a78f8 100644 --- a/temporalio/worker/_workflow_instance.py +++ b/temporalio/worker/_workflow_instance.py @@ -210,8 +210,8 @@ def __init__(self, det: WorkflowInstanceDetails) -> None: self._defn = det.defn self._workflow_input: Optional[ExecuteWorkflowInput] = None self._info = det.info - self._payload_converter_class = det.payload_converter_class - self._failure_converter_class = det.failure_converter_class + self._context_free_payload_converter = det.payload_converter_class() + self._context_free_failure_converter = det.failure_converter_class() self._payload_converter, self._failure_converter = self._converters( temporalio.converter.WorkflowSerializationContext( namespace=det.info.namespace, @@ -2063,8 +2063,8 @@ def _converters( temporalio.converter.FailureConverter, ]: """Construct workflow payload and failure converters with the given context.""" - payload_converter = self._payload_converter_class() - failure_converter = self._failure_converter_class() + payload_converter = self._context_free_payload_converter + failure_converter = self._context_free_failure_converter if context: if isinstance( payload_converter, temporalio.converter.WithSerializationContext From aa60bbe084ab17e2ed95b177e365b48bd60ea2a3 Mon Sep 17 00:00:00 2001 From: Dan Davison Date: Fri, 19 Sep 2025 09:21:37 -0400 Subject: [PATCH 07/81] Stack contexts I --- temporalio/worker/_workflow_instance.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/temporalio/worker/_workflow_instance.py b/temporalio/worker/_workflow_instance.py index 10b9a78f8..263c98326 100644 --- a/temporalio/worker/_workflow_instance.py +++ b/temporalio/worker/_workflow_instance.py @@ -210,8 +210,8 @@ def __init__(self, det: WorkflowInstanceDetails) -> None: self._defn = det.defn self._workflow_input: Optional[ExecuteWorkflowInput] = None self._info = det.info - self._context_free_payload_converter = det.payload_converter_class() - self._context_free_failure_converter = det.failure_converter_class() + self._payload_converter = det.payload_converter_class() + self._failure_converter = det.failure_converter_class() self._payload_converter, self._failure_converter = self._converters( temporalio.converter.WorkflowSerializationContext( namespace=det.info.namespace, @@ -2063,8 +2063,8 @@ def _converters( temporalio.converter.FailureConverter, ]: """Construct workflow payload and failure converters with the given context.""" - payload_converter = self._context_free_payload_converter - failure_converter = self._context_free_failure_converter + payload_converter = self._payload_converter + failure_converter = self._failure_converter if context: if isinstance( payload_converter, temporalio.converter.WithSerializationContext From 814f0da783858a2248f7d62b34d8c6c56946494f Mon Sep 17 00:00:00 2001 From: Dan Davison Date: Fri, 19 Sep 2025 09:25:54 -0400 Subject: [PATCH 08/81] Reuse and rename helper --- temporalio/worker/_workflow_instance.py | 59 ++++++++++++++----------- 1 file changed, 32 insertions(+), 27 deletions(-) diff --git a/temporalio/worker/_workflow_instance.py b/temporalio/worker/_workflow_instance.py index 263c98326..314d07ffe 100644 --- a/temporalio/worker/_workflow_instance.py +++ b/temporalio/worker/_workflow_instance.py @@ -210,12 +210,14 @@ def __init__(self, det: WorkflowInstanceDetails) -> None: self._defn = det.defn self._workflow_input: Optional[ExecuteWorkflowInput] = None self._info = det.info - self._payload_converter = det.payload_converter_class() - self._failure_converter = det.failure_converter_class() - self._payload_converter, self._failure_converter = self._converters( - temporalio.converter.WorkflowSerializationContext( - namespace=det.info.namespace, - workflow_id=det.info.workflow_id, + self._payload_converter, self._failure_converter = ( + self._converters_with_context( + temporalio.converter.WorkflowSerializationContext( + namespace=det.info.namespace, + workflow_id=det.info.workflow_id, + ), + det.payload_converter_class(), + det.failure_converter_class(), ) ) @@ -762,7 +764,7 @@ def _apply_resolve_activity( handle = self._pending_activities.pop(job.seq, None) if not handle: raise RuntimeError(f"Failed finding activity handle for sequence {job.seq}") - payload_converter, failure_converter = self._converters( + payload_converter, failure_converter = self._converters_with_context( temporalio.converter.ActivitySerializationContext( namespace=self._info.namespace, workflow_id=self._info.workflow_id, @@ -813,7 +815,7 @@ def _apply_resolve_child_workflow_execution( raise RuntimeError( f"Failed finding child workflow handle for sequence {job.seq}" ) - payload_converter, failure_converter = self._converters( + payload_converter, failure_converter = self._converters_with_context( temporalio.converter.WorkflowSerializationContext( namespace=self._info.namespace, workflow_id=handle._input.id, @@ -875,7 +877,7 @@ def _apply_resolve_child_workflow_execution_start( ) elif job.HasField("cancelled"): self._pending_child_workflows.pop(job.seq) - payload_converter, failure_converter = self._converters( + payload_converter, failure_converter = self._converters_with_context( temporalio.converter.WorkflowSerializationContext( namespace=self._info.namespace, workflow_id=handle._input.id, @@ -898,7 +900,7 @@ def _apply_resolve_nexus_operation_start( ) # We not set a serialization context for nexus operations on the caller side because it is # not possible to do so on the handler side. - payload_converter, failure_converter = self._converters(None) + payload_converter, failure_converter = self._converters_with_context(None) if job.HasField("operation_token"): # The nexus operation started asynchronously. A `ResolveNexusOperation` job @@ -937,7 +939,7 @@ def _apply_resolve_nexus_operation( # We not set a serialization context for nexus operations on the caller side because it is # not possible to do so on the handler side. - payload_converter, failure_converter = self._converters(None) + payload_converter, failure_converter = self._converters_with_context(None) # Handle the four oneof variants of NexusOperationResult result = job.result if result.HasField("completed"): @@ -974,7 +976,7 @@ def _apply_resolve_request_cancel_external_workflow( fut, external_workflow_id = pending # We intentionally let this error if future is already done if job.HasField("failure"): - payload_converter, failure_converter = self._converters( + payload_converter, failure_converter = self._converters_with_context( temporalio.converter.WorkflowSerializationContext( namespace=self._info.namespace, workflow_id=external_workflow_id, @@ -998,7 +1000,7 @@ def _apply_resolve_signal_external_workflow( fut, external_workflow_id = pending # We intentionally let this error if future is already done if job.HasField("failure"): - payload_converter, failure_converter = self._converters( + payload_converter, failure_converter = self._converters_with_context( temporalio.converter.WorkflowSerializationContext( namespace=self._info.namespace, workflow_id=external_workflow_id, @@ -1853,7 +1855,7 @@ async def run_activity() -> Any: async def _outbound_signal_child_workflow( self, input: SignalChildWorkflowInput ) -> None: - payload_converter, _ = self._converters( + payload_converter, _ = self._converters_with_context( temporalio.converter.WorkflowSerializationContext( namespace=self._info.namespace, workflow_id=input.child_workflow_id, @@ -1873,7 +1875,7 @@ async def _outbound_signal_child_workflow( async def _outbound_signal_external_workflow( self, input: SignalExternalWorkflowInput ) -> None: - payload_converter, _ = self._converters( + payload_converter, _ = self._converters_with_context( temporalio.converter.WorkflowSerializationContext( namespace=input.namespace, workflow_id=input.workflow_id, @@ -2056,25 +2058,28 @@ def _convert_payloads( raise raise RuntimeError("Failed decoding arguments") from err - def _converters( - self, context: Optional[temporalio.converter.SerializationContext] + def _converters_with_context( + self, + context: Optional[temporalio.converter.SerializationContext], + base_payload_converter: Optional[temporalio.converter.PayloadConverter] = None, + base_failure_converter: Optional[temporalio.converter.FailureConverter] = None, ) -> Tuple[ temporalio.converter.PayloadConverter, temporalio.converter.FailureConverter, ]: """Construct workflow payload and failure converters with the given context.""" - payload_converter = self._payload_converter - failure_converter = self._failure_converter + base_payload_converter = base_payload_converter or self._payload_converter + base_failure_converter = base_failure_converter or self._failure_converter if context: if isinstance( - payload_converter, temporalio.converter.WithSerializationContext + base_payload_converter, temporalio.converter.WithSerializationContext ): - payload_converter = payload_converter.with_context(context) + base_payload_converter = base_payload_converter.with_context(context) if isinstance( - failure_converter, temporalio.converter.WithSerializationContext + base_failure_converter, temporalio.converter.WithSerializationContext ): - failure_converter = failure_converter.with_context(context) - return payload_converter, failure_converter + base_failure_converter = base_failure_converter.with_context(context) + return base_payload_converter, base_failure_converter def _instantiate_workflow_object(self) -> Any: if not self._workflow_input: @@ -2812,7 +2817,7 @@ def __init__( self._result_fut = instance.create_future() self._started = False instance._register_task(self, name=f"activity: {input.activity}") - self._payload_converter, _ = self._instance._converters( + self._payload_converter, _ = self._instance._converters_with_context( temporalio.converter.ActivitySerializationContext( namespace=self._instance._info.namespace, workflow_id=self._instance._info.workflow_id, @@ -2973,7 +2978,7 @@ def __init__( self._result_fut: asyncio.Future[Any] = instance.create_future() self._first_execution_run_id = "" instance._register_task(self, name=f"child: {input.workflow}") - self._payload_converter, _ = self._instance._converters( + self._payload_converter, _ = self._instance._converters_with_context( temporalio.converter.WorkflowSerializationContext( namespace=self._instance._info.namespace, workflow_id=self._input.id, @@ -3163,7 +3168,7 @@ def __init__( self._task = asyncio.Task(fn) self._start_fut: asyncio.Future[Optional[str]] = instance.create_future() self._result_fut: asyncio.Future[Optional[OutputT]] = instance.create_future() - self._payload_converter, _ = self._instance._converters(None) + self._payload_converter, _ = self._instance._converters_with_context(None) @property def operation_token(self) -> Optional[str]: From fa36908bff5f7bba8eb4e6edd742535aaf03c50b Mon Sep 17 00:00:00 2001 From: Dan Davison Date: Fri, 19 Sep 2025 09:33:26 -0400 Subject: [PATCH 09/81] Don't assume user overrides with same constructor signature --- temporalio/converter.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/temporalio/converter.py b/temporalio/converter.py index 93d31b2d4..8964bae95 100644 --- a/temporalio/converter.py +++ b/temporalio/converter.py @@ -401,15 +401,16 @@ def from_payloads( ) from err return values - def with_context(self, context: SerializationContext) -> Self: + def with_context(self, context: SerializationContext) -> CompositePayloadConverter: """Return a new instance with the given context.""" - converters = [ - c.with_context(context) if isinstance(c, WithSerializationContext) else c - for c in self.converters.values() - ] - instance = type(self).__new__(type(self)) - CompositePayloadConverter.__init__(instance, *converters) - return instance + return CompositePayloadConverter( + *( + c.with_context(context) + if isinstance(c, WithSerializationContext) + else c + for c in self.converters.values() + ) + ) class DefaultPayloadConverter(CompositePayloadConverter): From 647d9ab500c1b64fdfd6d05bdb37b26b939355b9 Mon Sep 17 00:00:00 2001 From: Dan Davison Date: Fri, 19 Sep 2025 11:03:38 -0400 Subject: [PATCH 10/81] Simplify --- temporalio/converter.py | 15 +++++++-------- temporalio/worker/_workflow_instance.py | 2 +- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/temporalio/converter.py b/temporalio/converter.py index 8964bae95..3ad972049 100644 --- a/temporalio/converter.py +++ b/temporalio/converter.py @@ -1309,18 +1309,17 @@ async def decode_failure( await self.payload_codec.decode_failure(failure) return self.failure_converter.from_failure(failure, self.payload_converter) - def _with_context(self, context: Optional[SerializationContext]) -> Self: + def _with_context(self, context: SerializationContext) -> Self: cloned = dataclasses.replace(self) payload_converter = self.payload_converter payload_codec = self.payload_codec failure_converter = self.failure_converter - if context: - if isinstance(payload_converter, WithSerializationContext): - payload_converter = payload_converter.with_context(context) - if isinstance(payload_codec, WithSerializationContext): - payload_codec = payload_codec.with_context(context) - if isinstance(failure_converter, WithSerializationContext): - failure_converter = failure_converter.with_context(context) + if isinstance(payload_converter, WithSerializationContext): + payload_converter = payload_converter.with_context(context) + if isinstance(payload_codec, WithSerializationContext): + payload_codec = payload_codec.with_context(context) + if isinstance(failure_converter, WithSerializationContext): + failure_converter = failure_converter.with_context(context) object.__setattr__(cloned, "payload_converter", payload_converter) object.__setattr__(cloned, "payload_codec", payload_codec) object.__setattr__(cloned, "failure_converter", failure_converter) diff --git a/temporalio/worker/_workflow_instance.py b/temporalio/worker/_workflow_instance.py index 314d07ffe..f4534f845 100644 --- a/temporalio/worker/_workflow_instance.py +++ b/temporalio/worker/_workflow_instance.py @@ -3168,7 +3168,7 @@ def __init__( self._task = asyncio.Task(fn) self._start_fut: asyncio.Future[Optional[str]] = instance.create_future() self._result_fut: asyncio.Future[Optional[OutputT]] = instance.create_future() - self._payload_converter, _ = self._instance._converters_with_context(None) + self._payload_converter = self._instance._payload_converter @property def operation_token(self) -> Optional[str]: From 4adbb364809939d48526824b28a303bd8793b8c4 Mon Sep 17 00:00:00 2001 From: Dan Davison Date: Mon, 22 Sep 2025 03:57:58 -0400 Subject: [PATCH 11/81] Command-aware serialization context during payload visitor traversal - Change visitor API - Add command seq IDs to async context during payload visitor traversal - Store data_converter on WorkflowInstanceDetails instead of converter classes - Command-aware codec for payload visitor --- scripts/gen_payload_visitor.py | 69 ++- temporalio/bridge/_visitor.py | 130 +++-- temporalio/bridge/worker.py | 21 +- temporalio/client.py | 1 + temporalio/worker/_workflow.py | 47 +- temporalio/worker/_workflow_instance.py | 116 ++++- .../worker/workflow_sandbox/_in_sandbox.py | 9 +- temporalio/worker/workflow_sandbox/_runner.py | 19 +- tests/test_serialization_context.py | 446 +++++++++++++++--- tests/worker/test_visitor.py | 2 +- tests/worker/test_workflow.py | 5 + 11 files changed, 737 insertions(+), 128 deletions(-) diff --git a/scripts/gen_payload_visitor.py b/scripts/gen_payload_visitor.py index 790169afd..8900afb88 100644 --- a/scripts/gen_payload_visitor.py +++ b/scripts/gen_payload_visitor.py @@ -1,7 +1,7 @@ import subprocess import sys from pathlib import Path -from typing import Optional, Tuple +from typing import Optional from google.protobuf.descriptor import Descriptor, FieldDescriptor @@ -67,6 +67,20 @@ def emit_singular( await self._visit_{child_method}(fs, {access_expr})""" +def emit_singular_with_seq( + field_name: str, access_expr: str, child_method: str, presence_word: str +) -> str: + # Helper to emit a singular field visit that sets the seq contextvar, with presence check but + # without headers guard since this is used for commands only. + return f"""\ + {presence_word} o.HasField("{field_name}"): + token = current_command_seq.set({access_expr}.seq) + try: + await self._visit_{child_method}(fs, {access_expr}) + finally: + current_command_seq.reset(token)""" + + class VisitorGenerator: def generate(self, roots: list[Descriptor]) -> str: """ @@ -85,10 +99,16 @@ def generate(self, roots: list[Descriptor]) -> str: header = """ # This file is generated by gen_payload_visitor.py. Changes should be made there. import abc -from typing import Any, MutableSequence +import contextvars +from typing import Any, MutableSequence, Optional from temporalio.api.common.v1.message_pb2 import Payload +# Current workflow command sequence number +current_command_seq: contextvars.ContextVar[Optional[int]] = contextvars.ContextVar( + "current_command_seq", default=None +) + class VisitorFunctions(abc.ABC): \"\"\"Set of functions which can be called by the visitor. Allows handling payloads as a sequence. @@ -258,6 +278,29 @@ def walk(self, desc: Descriptor) -> bool: ) ) + commands_with_seq = { + "start_timer", + "cancel_timer", + "schedule_activity", + "schedule_local_activity", + "request_cancel_activity", + "request_cancel_local_activity", + "start_child_workflow_execution", + "request_cancel_external_workflow_execution", + "signal_external_workflow_execution", + "cancel_signal_workflow", + "schedule_nexus_operation", + "request_cancel_nexus_operation", + } + activation_jobs_with_seq = { + "resolve_activity", + "resolve_child_workflow_execution", + "resolve_child_workflow_execution_start", + "resolve_request_cancel_external_workflow", + "resolve_signal_external_workflow", + "resolve_nexus_operation", + "resolve_nexus_operation_start", + } # Process oneof fields as if/elif chains for oneof_idx, fields in oneof_fields.items(): oneof_lines = [] @@ -269,9 +312,25 @@ def walk(self, desc: Descriptor) -> bool: if child_has_payload: if_word = "if" if first else "elif" first = False - line = emit_singular( - field.name, f"o.{field.name}", name_for(child_desc), if_word - ) + if ( + desc.full_name == "coresdk.workflow_commands.WorkflowCommand" + and field.name in commands_with_seq + ): + line = emit_singular_with_seq( + field.name, f"o.{field.name}", name_for(child_desc), if_word + ) + elif ( + desc.full_name + == "coresdk.workflow_activation.WorkflowActivationJob" + and field.name in activation_jobs_with_seq + ): + line = emit_singular_with_seq( + field.name, f"o.{field.name}", name_for(child_desc), if_word + ) + else: + line = emit_singular( + field.name, f"o.{field.name}", name_for(child_desc), if_word + ) oneof_lines.append(line) if oneof_lines: lines.extend(oneof_lines) diff --git a/temporalio/bridge/_visitor.py b/temporalio/bridge/_visitor.py index 0491b5e88..ba3f8fe1d 100644 --- a/temporalio/bridge/_visitor.py +++ b/temporalio/bridge/_visitor.py @@ -1,9 +1,15 @@ # This file is generated by gen_payload_visitor.py. Changes should be made there. import abc -from typing import Any, MutableSequence +import contextvars +from typing import Any, MutableSequence, Optional from temporalio.api.common.v1.message_pb2 import Payload +# Current workflow command sequence number +current_command_seq: contextvars.ContextVar[Optional[int]] = contextvars.ContextVar( + "current_command_seq", default=None +) + class VisitorFunctions(abc.ABC): """Set of functions which can be called by the visitor. @@ -247,35 +253,69 @@ async def _visit_coresdk_workflow_activation_WorkflowActivationJob(self, fs, o): fs, o.signal_workflow ) elif o.HasField("resolve_activity"): - await self._visit_coresdk_workflow_activation_ResolveActivity( - fs, o.resolve_activity - ) + token = current_command_seq.set(o.resolve_activity.seq) + try: + await self._visit_coresdk_workflow_activation_ResolveActivity( + fs, o.resolve_activity + ) + finally: + current_command_seq.reset(token) elif o.HasField("resolve_child_workflow_execution_start"): - await self._visit_coresdk_workflow_activation_ResolveChildWorkflowExecutionStart( - fs, o.resolve_child_workflow_execution_start + token = current_command_seq.set( + o.resolve_child_workflow_execution_start.seq ) + try: + await self._visit_coresdk_workflow_activation_ResolveChildWorkflowExecutionStart( + fs, o.resolve_child_workflow_execution_start + ) + finally: + current_command_seq.reset(token) elif o.HasField("resolve_child_workflow_execution"): - await self._visit_coresdk_workflow_activation_ResolveChildWorkflowExecution( - fs, o.resolve_child_workflow_execution - ) + token = current_command_seq.set(o.resolve_child_workflow_execution.seq) + try: + await self._visit_coresdk_workflow_activation_ResolveChildWorkflowExecution( + fs, o.resolve_child_workflow_execution + ) + finally: + current_command_seq.reset(token) elif o.HasField("resolve_signal_external_workflow"): - await self._visit_coresdk_workflow_activation_ResolveSignalExternalWorkflow( - fs, o.resolve_signal_external_workflow - ) + token = current_command_seq.set(o.resolve_signal_external_workflow.seq) + try: + await self._visit_coresdk_workflow_activation_ResolveSignalExternalWorkflow( + fs, o.resolve_signal_external_workflow + ) + finally: + current_command_seq.reset(token) elif o.HasField("resolve_request_cancel_external_workflow"): - await self._visit_coresdk_workflow_activation_ResolveRequestCancelExternalWorkflow( - fs, o.resolve_request_cancel_external_workflow + token = current_command_seq.set( + o.resolve_request_cancel_external_workflow.seq ) + try: + await self._visit_coresdk_workflow_activation_ResolveRequestCancelExternalWorkflow( + fs, o.resolve_request_cancel_external_workflow + ) + finally: + current_command_seq.reset(token) elif o.HasField("do_update"): await self._visit_coresdk_workflow_activation_DoUpdate(fs, o.do_update) elif o.HasField("resolve_nexus_operation_start"): - await self._visit_coresdk_workflow_activation_ResolveNexusOperationStart( - fs, o.resolve_nexus_operation_start - ) + token = current_command_seq.set(o.resolve_nexus_operation_start.seq) + try: + await ( + self._visit_coresdk_workflow_activation_ResolveNexusOperationStart( + fs, o.resolve_nexus_operation_start + ) + ) + finally: + current_command_seq.reset(token) elif o.HasField("resolve_nexus_operation"): - await self._visit_coresdk_workflow_activation_ResolveNexusOperation( - fs, o.resolve_nexus_operation - ) + token = current_command_seq.set(o.resolve_nexus_operation.seq) + try: + await self._visit_coresdk_workflow_activation_ResolveNexusOperation( + fs, o.resolve_nexus_operation + ) + finally: + current_command_seq.reset(token) async def _visit_coresdk_workflow_activation_WorkflowActivation(self, fs, o): for v in o.jobs: @@ -374,9 +414,13 @@ async def _visit_coresdk_workflow_commands_WorkflowCommand(self, fs, o): if o.HasField("user_metadata"): await self._visit_temporal_api_sdk_v1_UserMetadata(fs, o.user_metadata) if o.HasField("schedule_activity"): - await self._visit_coresdk_workflow_commands_ScheduleActivity( - fs, o.schedule_activity - ) + token = current_command_seq.set(o.schedule_activity.seq) + try: + await self._visit_coresdk_workflow_commands_ScheduleActivity( + fs, o.schedule_activity + ) + finally: + current_command_seq.reset(token) elif o.HasField("respond_to_query"): await self._visit_coresdk_workflow_commands_QueryResult( fs, o.respond_to_query @@ -394,17 +438,29 @@ async def _visit_coresdk_workflow_commands_WorkflowCommand(self, fs, o): fs, o.continue_as_new_workflow_execution ) elif o.HasField("start_child_workflow_execution"): - await self._visit_coresdk_workflow_commands_StartChildWorkflowExecution( - fs, o.start_child_workflow_execution - ) + token = current_command_seq.set(o.start_child_workflow_execution.seq) + try: + await self._visit_coresdk_workflow_commands_StartChildWorkflowExecution( + fs, o.start_child_workflow_execution + ) + finally: + current_command_seq.reset(token) elif o.HasField("signal_external_workflow_execution"): - await self._visit_coresdk_workflow_commands_SignalExternalWorkflowExecution( - fs, o.signal_external_workflow_execution - ) + token = current_command_seq.set(o.signal_external_workflow_execution.seq) + try: + await self._visit_coresdk_workflow_commands_SignalExternalWorkflowExecution( + fs, o.signal_external_workflow_execution + ) + finally: + current_command_seq.reset(token) elif o.HasField("schedule_local_activity"): - await self._visit_coresdk_workflow_commands_ScheduleLocalActivity( - fs, o.schedule_local_activity - ) + token = current_command_seq.set(o.schedule_local_activity.seq) + try: + await self._visit_coresdk_workflow_commands_ScheduleLocalActivity( + fs, o.schedule_local_activity + ) + finally: + current_command_seq.reset(token) elif o.HasField("upsert_workflow_search_attributes"): await self._visit_coresdk_workflow_commands_UpsertWorkflowSearchAttributes( fs, o.upsert_workflow_search_attributes @@ -418,9 +474,13 @@ async def _visit_coresdk_workflow_commands_WorkflowCommand(self, fs, o): fs, o.update_response ) elif o.HasField("schedule_nexus_operation"): - await self._visit_coresdk_workflow_commands_ScheduleNexusOperation( - fs, o.schedule_nexus_operation - ) + token = current_command_seq.set(o.schedule_nexus_operation.seq) + try: + await self._visit_coresdk_workflow_commands_ScheduleNexusOperation( + fs, o.schedule_nexus_operation + ) + finally: + current_command_seq.reset(token) async def _visit_coresdk_workflow_completion_Success(self, fs, o): for v in o.commands: diff --git a/temporalio/bridge/worker.py b/temporalio/bridge/worker.py index 6fff9878c..ea35191c8 100644 --- a/temporalio/bridge/worker.py +++ b/temporalio/bridge/worker.py @@ -7,11 +7,9 @@ from dataclasses import dataclass from typing import ( - TYPE_CHECKING, Awaitable, Callable, List, - Mapping, MutableSequence, Optional, Sequence, @@ -20,7 +18,6 @@ Union, ) -import google.protobuf.internal.containers from typing_extensions import TypeAlias import temporalio.api.common.v1 @@ -35,7 +32,7 @@ import temporalio.bridge.temporal_sdk_bridge import temporalio.converter import temporalio.exceptions -from temporalio.api.common.v1.message_pb2 import Payload, Payloads +from temporalio.api.common.v1.message_pb2 import Payload from temporalio.bridge._visitor import PayloadVisitor, VisitorFunctions from temporalio.bridge.temporal_sdk_bridge import ( CustomSlotSupplier as BridgeCustomSlotSupplier, @@ -299,22 +296,22 @@ async def visit_payloads(self, payloads: MutableSequence[Payload]) -> None: async def decode_activation( - act: temporalio.bridge.proto.workflow_activation.WorkflowActivation, - codec: temporalio.converter.PayloadCodec, + activation: temporalio.bridge.proto.workflow_activation.WorkflowActivation, + decode: Callable[[Sequence[Payload]], Awaitable[List[Payload]]], decode_headers: bool, ) -> None: - """Decode the given activation with the codec.""" + """Decode all payloads in the activation.""" await PayloadVisitor( skip_search_attributes=True, skip_headers=not decode_headers - ).visit(_Visitor(codec.decode), act) + ).visit(_Visitor(decode), activation) async def encode_completion( - comp: temporalio.bridge.proto.workflow_completion.WorkflowActivationCompletion, - codec: temporalio.converter.PayloadCodec, + completion: temporalio.bridge.proto.workflow_completion.WorkflowActivationCompletion, + encode: Callable[[Sequence[Payload]], Awaitable[List[Payload]]], encode_headers: bool, ) -> None: - """Recursively encode the given completion with the codec.""" + """Encode all payloads in the completion.""" await PayloadVisitor( skip_search_attributes=True, skip_headers=not encode_headers - ).visit(_Visitor(codec.encode), comp) + ).visit(_Visitor(encode), completion) diff --git a/temporalio/client.py b/temporalio/client.py index 39d05e2bf..f17fd35e4 100644 --- a/temporalio/client.py +++ b/temporalio/client.py @@ -5974,6 +5974,7 @@ async def _populate_start_workflow_execution_request( req.workflow_type.name = input.workflow req.task_queue.name = input.task_queue if input.args: + # client encode wf input req.input.payloads.extend(await data_converter.encode(input.args)) if input.execution_timeout is not None: req.workflow_execution_timeout.FromTimedelta(input.execution_timeout) diff --git a/temporalio/worker/_workflow.py b/temporalio/worker/_workflow.py index 24099a998..3b04ad511 100644 --- a/temporalio/worker/_workflow.py +++ b/temporalio/worker/_workflow.py @@ -24,6 +24,7 @@ import temporalio.activity import temporalio.api.common.v1 +import temporalio.bridge._visitor import temporalio.bridge.client import temporalio.bridge.proto.workflow_activation import temporalio.bridge.proto.workflow_completion @@ -254,6 +255,7 @@ async def _handle_activation( temporalio.bridge.proto.workflow_completion.WorkflowActivationCompletion() ) completion.successful.SetInParent() + workflow = None try: if LOG_PROTOS: logger.debug("Received workflow activation:\n%s", act) @@ -280,9 +282,13 @@ async def _handle_activation( ) ) if data_converter.payload_codec: + if not workflow: + payload_codec = data_converter.payload_codec + else: + payload_codec = _CommandAwarePayloadCodec(workflow.instance) await temporalio.bridge.worker.decode_activation( act, - data_converter.payload_codec, + payload_codec.decode, decode_headers=self._encode_headers, ) if not workflow: @@ -348,13 +354,15 @@ async def _handle_activation( ) completion.run_id = act.run_id + assert workflow # Encode completion if data_converter.payload_codec: + payload_codec = _CommandAwarePayloadCodec(workflow.instance) try: await temporalio.bridge.worker.encode_completion( completion, - data_converter.payload_codec, + payload_codec.encode, encode_headers=self._encode_headers, ) except Exception as err: @@ -565,8 +573,7 @@ def _create_workflow_instance( # Create instance from details det = WorkflowInstanceDetails( - payload_converter_class=self._data_converter.payload_converter_class, - failure_converter_class=self._data_converter.failure_converter_class, + data_converter=self._data_converter, interceptor_classes=self._interceptor_classes, defn=defn, info=info, @@ -706,5 +713,37 @@ def attempt_deadlock_interruption(self) -> None: ) +class _CommandAwarePayloadCodec(temporalio.converter.PayloadCodec): + """A payload codec that sets serialization context for the associated command. + + This codec responds to the :py:data:`temporalio.bridge._visitor.current_command_seq` context + variable set by the payload visitor. + """ + + def __init__( + self, + instance: WorkflowInstance, + ): + self.instance = instance + + async def encode( + self, + payloads: Sequence[temporalio.api.common.v1.Payload], + ) -> List[temporalio.api.common.v1.Payload]: + return await self._get_current_command_codec().encode(payloads) + + async def decode( + self, + payloads: Sequence[temporalio.api.common.v1.Payload], + ) -> List[temporalio.api.common.v1.Payload]: + return await self._get_current_command_codec().decode(payloads) + + def _get_current_command_codec(self) -> temporalio.converter.PayloadCodec: + seq = temporalio.bridge._visitor.current_command_seq.get() + codec = self.instance.get_payload_codec(seq) + assert codec, "Payload codec must be set on the data converter" + return codec + + class _InterruptDeadlockError(BaseException): pass diff --git a/temporalio/worker/_workflow_instance.py b/temporalio/worker/_workflow_instance.py index f4534f845..ef7dd2acd 100644 --- a/temporalio/worker/_workflow_instance.py +++ b/temporalio/worker/_workflow_instance.py @@ -135,8 +135,7 @@ def set_worker_level_failure_exception_types( class WorkflowInstanceDetails: """Immutable details for creating a workflow instance.""" - payload_converter_class: Type[temporalio.converter.PayloadConverter] - failure_converter_class: Type[temporalio.converter.FailureConverter] + data_converter: temporalio.converter.DataConverter interceptor_classes: Sequence[Type[WorkflowInboundInterceptor]] defn: temporalio.workflow._Definition info: temporalio.workflow.Info @@ -168,6 +167,21 @@ def activate( """ raise NotImplementedError + @abstractmethod + def get_payload_codec( + self, command_seq: Optional[int] + ) -> Optional[temporalio.converter.PayloadCodec]: + """Return a payload codec with appropriate serialization context. + + Args: + command_seq: Optional sequence number of the associated command. If set, the payload + codec will have serialization context set appropriately for that command. + + Returns: + The payload codec. + """ + raise NotImplementedError + def get_thread_id(self) -> Optional[int]: """Return the thread identifier that this workflow is running on. @@ -210,14 +224,21 @@ def __init__(self, det: WorkflowInstanceDetails) -> None: self._defn = det.defn self._workflow_input: Optional[ExecuteWorkflowInput] = None self._info = det.info + self._context_free_payload_codec = det.data_converter.payload_codec + self._context_free_payload_converter = ( + det.data_converter.payload_converter_class() + ) + self._context_free_failure_converter = ( + det.data_converter.failure_converter_class() + ) self._payload_converter, self._failure_converter = ( self._converters_with_context( temporalio.converter.WorkflowSerializationContext( namespace=det.info.namespace, workflow_id=det.info.workflow_id, ), - det.payload_converter_class(), - det.failure_converter_class(), + self._context_free_payload_converter, + self._context_free_failure_converter, ) ) @@ -701,7 +722,6 @@ async def run_query() -> None: ) # Create input - # TODO: why do we deserialize query input in workflow but not signal? args = self._process_handler_args( job.query_type, job.arguments, @@ -939,7 +959,10 @@ def _apply_resolve_nexus_operation( # We not set a serialization context for nexus operations on the caller side because it is # not possible to do so on the handler side. - payload_converter, failure_converter = self._converters_with_context(None) + payload_converter, failure_converter = ( + self._context_free_payload_converter, + self._context_free_failure_converter, + ) # Handle the four oneof variants of NexusOperationResult result = job.result if result.HasField("completed"): @@ -2068,18 +2091,83 @@ def _converters_with_context( temporalio.converter.FailureConverter, ]: """Construct workflow payload and failure converters with the given context.""" - base_payload_converter = base_payload_converter or self._payload_converter - base_failure_converter = base_failure_converter or self._failure_converter + payload_converter = base_payload_converter or self._payload_converter + failure_converter = base_failure_converter or self._failure_converter if context: if isinstance( - base_payload_converter, temporalio.converter.WithSerializationContext + payload_converter, temporalio.converter.WithSerializationContext ): - base_payload_converter = base_payload_converter.with_context(context) + payload_converter = payload_converter.with_context(context) if isinstance( - base_failure_converter, temporalio.converter.WithSerializationContext + failure_converter, temporalio.converter.WithSerializationContext ): - base_failure_converter = base_failure_converter.with_context(context) - return base_payload_converter, base_failure_converter + failure_converter = failure_converter.with_context(context) + return payload_converter, failure_converter + + # _WorkflowInstanceImpl.get_pending_command_serialization_context + def get_payload_codec( + self, command_seq: Optional[int] + ) -> Optional[temporalio.converter.PayloadCodec]: + # This function is only called when the user's payload codec supports serialization context. + if not isinstance( + self._context_free_payload_codec, + temporalio.converter.WithSerializationContext, + ): + return self._context_free_payload_codec + + workflow_context = temporalio.converter.WorkflowSerializationContext( + namespace=self._info.namespace, + workflow_id=self._info.workflow_id, + ) + + if command_seq is None: + # Use payload codec with workflow context by default (i.e. for payloads not associated + # with a pending command) + return self._context_free_payload_codec.with_context(workflow_context) + + if command_seq in self._pending_activities: + handle = self._pending_activities[command_seq] + context = temporalio.converter.ActivitySerializationContext( + namespace=self._info.namespace, + workflow_id=self._info.workflow_id, + workflow_type=self._info.workflow_type, + activity_type=handle._input.activity, + activity_task_queue=( + handle._input.task_queue + if isinstance(handle._input, StartActivityInput) + and handle._input.task_queue + else self._info.task_queue + ), + is_local=isinstance(handle._input, StartLocalActivityInput), + ) + return self._context_free_payload_codec.with_context(context) + + elif command_seq in self._pending_child_workflows: + handle = self._pending_child_workflows[command_seq] + context = temporalio.converter.WorkflowSerializationContext( + namespace=self._info.namespace, + workflow_id=handle._input.id, + ) + return self._context_free_payload_codec.with_context(context) + + elif command_seq in self._pending_external_signals: + # Use the target workflow's context for external signals + _, workflow_id = self._pending_external_signals[command_seq] + context = temporalio.converter.WorkflowSerializationContext( + namespace=self._info.namespace, + workflow_id=workflow_id, + ) + return self._context_free_payload_codec.with_context(context) + + elif command_seq in self._pending_nexus_operations: + # Use empty context for nexus operations: users will never want to encrypt using a + # key derived from caller workflow context because the caller workflow context is + # not available on the handler side for decryption. + return self._context_free_payload_codec + + else: + # Use payload codec with workflow context for all other payloads + return self._context_free_payload_codec.with_context(workflow_context) def _instantiate_workflow_object(self) -> Any: if not self._workflow_input: @@ -3168,7 +3256,7 @@ def __init__( self._task = asyncio.Task(fn) self._start_fut: asyncio.Future[Optional[str]] = instance.create_future() self._result_fut: asyncio.Future[Optional[OutputT]] = instance.create_future() - self._payload_converter = self._instance._payload_converter + self._payload_converter = self._instance._context_free_payload_converter @property def operation_token(self) -> Optional[str]: diff --git a/temporalio/worker/workflow_sandbox/_in_sandbox.py b/temporalio/worker/workflow_sandbox/_in_sandbox.py index 3091cef1d..0188e309b 100644 --- a/temporalio/worker/workflow_sandbox/_in_sandbox.py +++ b/temporalio/worker/workflow_sandbox/_in_sandbox.py @@ -6,10 +6,11 @@ import dataclasses import logging -from typing import Any, Type +from typing import Any, Optional, Type import temporalio.bridge.proto.workflow_activation import temporalio.bridge.proto.workflow_completion +import temporalio.converter import temporalio.worker._workflow_instance import temporalio.workflow @@ -79,3 +80,9 @@ def activate( ) -> temporalio.bridge.proto.workflow_completion.WorkflowActivationCompletion: """Send activation to this instance.""" return self.instance.activate(act) + + def get_payload_codec( + self, command_seq: Optional[int] + ) -> Optional[temporalio.converter.PayloadCodec]: + """Get payload codec.""" + return self.instance.get_payload_codec(command_seq) diff --git a/temporalio/worker/workflow_sandbox/_runner.py b/temporalio/worker/workflow_sandbox/_runner.py index c656e3041..4e48002d0 100644 --- a/temporalio/worker/workflow_sandbox/_runner.py +++ b/temporalio/worker/workflow_sandbox/_runner.py @@ -77,8 +77,7 @@ def prepare_workflow(self, defn: temporalio.workflow._Definition) -> None: # Just create with fake info which validates self.create_instance( WorkflowInstanceDetails( - payload_converter_class=temporalio.converter.DataConverter.default.payload_converter_class, - failure_converter_class=temporalio.converter.DataConverter.default.failure_converter_class, + data_converter=temporalio.converter.DataConverter.default, interceptor_classes=[], defn=defn, # Just use fake info during validation @@ -185,3 +184,19 @@ def _run_code(self, code: str, **extra_globals: Any) -> None: def get_thread_id(self) -> Optional[int]: return self._current_thread_id + + def get_payload_codec( + self, command_seq: Optional[int] + ) -> Optional[temporalio.converter.PayloadCodec]: + # Forward call to the sandboxed instance + self.importer.restriction_context.is_runtime = True + try: + self._run_code( + "with __temporal_importer.applied():\n" + " __temporal_codec = __temporal_in_sandbox.get_payload_codec(__temporal_command_seq)\n", + __temporal_importer=self.importer, + __temporal_command_seq=command_seq, + ) + return self.globals_and_locals.pop("__temporal_codec", None) # type: ignore + finally: + self.importer.restriction_context.is_runtime = False diff --git a/tests/test_serialization_context.py b/tests/test_serialization_context.py index e70f17818..4ffe0dd20 100644 --- a/tests/test_serialization_context.py +++ b/tests/test_serialization_context.py @@ -9,12 +9,14 @@ import asyncio import dataclasses +import json import uuid from collections import defaultdict from dataclasses import dataclass, field from datetime import timedelta from typing import Any, List, Literal, Optional, Sequence, Type +import nexusrpc import pytest from pydantic import BaseModel from typing_extensions import Never @@ -42,6 +44,7 @@ from temporalio.exceptions import ApplicationError from temporalio.worker import Worker from temporalio.worker._workflow_instance import UnsandboxedWorkflowRunner +from tests.helpers.nexus import create_nexus_endpoint, make_nexus_endpoint_name @dataclass @@ -422,43 +425,40 @@ async def test_local_activity_payload_conversion(client: Client): ) ) - assert ( - result.items - == [ - TraceItem( - method="to_payload", - context=workflow_context, # Outbound workflow input - ), - TraceItem( - method="from_payload", - context=workflow_context, # Inbound workflow input - ), - TraceItem( - method="to_payload", - context=local_activity_context, # Outbound local activity input (is_local=True) - ), - TraceItem( - method="from_payload", - context=local_activity_context, # Inbound local activity input (is_local=True) - ), - TraceItem( - method="to_payload", - context=local_activity_context, # Outbound local activity result (is_local=True) - ), - TraceItem( - method="from_payload", - context=local_activity_context, # Inbound local activity result (is_local=True) - ), - TraceItem( - method="to_payload", - context=workflow_context, # Outbound workflow result - ), - TraceItem( - method="from_payload", - context=workflow_context, # Inbound workflow result - ), - ] - ) + assert result.items == [ + TraceItem( + method="to_payload", + context=workflow_context, # Outbound workflow input + ), + TraceItem( + method="from_payload", + context=workflow_context, # Inbound workflow input + ), + TraceItem( + method="to_payload", + context=local_activity_context, # Outbound local activity input + ), + TraceItem( + method="from_payload", + context=local_activity_context, # Inbound local activity input + ), + TraceItem( + method="to_payload", + context=local_activity_context, # Outbound local activity result + ), + TraceItem( + method="from_payload", + context=local_activity_context, # Inbound local activity result + ), + TraceItem( + method="to_payload", + context=workflow_context, # Outbound workflow result + ), + TraceItem( + method="from_payload", + context=workflow_context, # Inbound workflow result + ), + ] # Async activity completion test @@ -1046,6 +1046,9 @@ async def test_failure_converter_with_context(client: Client): del test_traces[workflow_id] +## Test payload codec + + class PayloadCodecWithContext(PayloadCodec, WithSerializationContext): def __init__(self): self.context: Optional[SerializationContext] = None @@ -1204,56 +1207,391 @@ async def test_local_activity_codec_with_context(client: Client): workflow_type=LocalActivityCodecTestWorkflow.__name__, activity_type=codec_test_local_activity.__name__, activity_task_queue=task_queue, - is_local=True, # Should be True for local activities + is_local=True, ) ) - # Note: Local activities have partial activity context support through codec - # The input encode uses workflow context, but the decode uses activity context - # The result encode uses activity context, but the decode uses workflow context assert test_traces[workflow_id] == [ - # Workflow input TraceItem( context=workflow_context, - method="encode", + method="encode", # outbound workflow input ), TraceItem( context=workflow_context, - method="decode", + method="decode", # inbound workflow input + ), + TraceItem( + context=local_activity_context, + method="encode", # outbound local activity input + ), + TraceItem( + context=local_activity_context, + method="decode", # inbound local activity input + ), + TraceItem( + context=local_activity_context, + method="encode", # outbound local activity result + ), + TraceItem( + context=local_activity_context, + method="decode", # inbound local activity result + ), + TraceItem( + context=workflow_context, + method="encode", # outbound workflow result ), - # Local activity input - encode uses workflow context TraceItem( context=workflow_context, + method="decode", # inbound workflow result + ), + ] + del test_traces[workflow_id] + + +# Child workflow codec test + + +@workflow.defn +class ChildWorkflowCodecTestWorkflow: + @workflow.run + async def run(self, data: TraceData) -> TraceData: + return await workflow.execute_child_workflow( + EchoWorkflow.run, + data, + id=f"{workflow.info().workflow_id}-child", + ) + + +async def test_child_workflow_codec_with_context(client: Client): + workflow_id = str(uuid.uuid4()) + task_queue = str(uuid.uuid4()) + + config = client.config() + config["data_converter"] = dataclasses.replace( + DataConverter.default, + payload_codec=PayloadCodecWithContext(), + ) + client = Client(**config) + + async with Worker( + client, + task_queue=task_queue, + workflows=[ChildWorkflowCodecTestWorkflow, EchoWorkflow], + workflow_runner=UnsandboxedWorkflowRunner(), + ): + await client.execute_workflow( + ChildWorkflowCodecTestWorkflow.run, + TraceData(), + id=workflow_id, + task_queue=task_queue, + ) + + parent_workflow_context = dataclasses.asdict( + WorkflowSerializationContext( + namespace=client.namespace, + workflow_id=workflow_id, + ) + ) + + child_workflow_context = dataclasses.asdict( + WorkflowSerializationContext( + namespace=client.namespace, + workflow_id=f"{workflow_id}-child", + ) + ) + + # The expectation is that child workflows should use their own context for encoding/decoding, + # similar to how .NET and Java handle it + # Traces are stored under both parent and child workflow IDs + child_workflow_id = f"{workflow_id}-child" + + # Combine traces from parent and child workflows + all_traces = ( + test_traces[workflow_id][:2] # Parent workflow input + + test_traces[child_workflow_id] # All child workflow operations + + test_traces[workflow_id][2:] # Parent workflow result + ) + + assert all_traces == [ + # Parent workflow input + TraceItem( + context=parent_workflow_context, method="encode", ), - # Local activity input - decode uses activity context with is_local=True TraceItem( - context=local_activity_context, + context=parent_workflow_context, method="decode", ), - # Local activity result - encode uses activity context with is_local=True + # Child workflow input - should use child's context TraceItem( - context=local_activity_context, + context=child_workflow_context, method="encode", ), - # Local activity result - decode uses workflow context TraceItem( - context=workflow_context, + context=child_workflow_context, method="decode", ), - # Workflow result + # Child workflow result - should use child's context TraceItem( - context=workflow_context, + context=child_workflow_context, method="encode", ), TraceItem( - context=workflow_context, + context=child_workflow_context, + method="decode", + ), + # Parent workflow result + TraceItem( + context=parent_workflow_context, + method="encode", + ), + TraceItem( + context=parent_workflow_context, method="decode", ), ] del test_traces[workflow_id] +# Payload encryption test + + +class PayloadEncryptionCodec(PayloadCodec, WithSerializationContext): + """ + The outbound data for encoding must always be the string "outbound". "Encrypt" it by replacing + it with a key that is derived from the context available during encoding. On decryption, assert + that the same key can be derived from the context available during decoding, and return the + string "inbound". + """ + + def __init__(self): + self.context: Optional[SerializationContext] = None + + def with_context( + self, context: Optional[SerializationContext] + ) -> PayloadEncryptionCodec: + codec = PayloadEncryptionCodec() + codec.context = context + return codec + + async def encode(self, payloads: Sequence[Payload]) -> List[Payload]: + [payload] = payloads + return [ + Payload( + metadata=payload.metadata, + data=json.dumps(self._get_encryption_key()).encode(), + ) + ] + + async def decode(self, payloads: Sequence[Payload]) -> List[Payload]: + [payload] = payloads + assert json.loads(payload.data.decode()) == self._get_encryption_key() + metadata = dict(payload.metadata) + return [Payload(metadata=metadata, data=b'"inbound"')] + + def _get_encryption_key(self) -> str: + context = ( + dataclasses.asdict(self.context) + if isinstance( + self.context, + (WorkflowSerializationContext, ActivitySerializationContext), + ) + else {} + ) + return json.dumps({k: v for k, v in sorted(context.items())}) + + +@activity.defn +async def payload_encryption_activity(data: str) -> str: + assert data == "inbound" + return "outbound" + + +@workflow.defn +class PayloadEncryptionChildWorkflow: + @workflow.run + async def run(self, data: str) -> str: + assert data == "inbound" + return "outbound" + + +@nexusrpc.service +class PayloadEncryptionService: + payload_encryption_operation: nexusrpc.Operation[str, str] + + +@nexusrpc.handler.service_handler +class PayloadEncryptionServiceHandler: + @nexusrpc.handler.sync_operation + async def payload_encryption_operation( + self, _: nexusrpc.handler.StartOperationContext, data: str + ) -> str: + assert data == "inbound" + return "outbound" + + +@workflow.defn +class PayloadEncryptionWorkflow: + def __init__(self): + self.received_signal = False + self.received_update = False + + @workflow.run + async def run(self, data: str) -> str: + await workflow.wait_condition( + lambda: (self.received_signal and self.received_update) + ) + assert "inbound" == await workflow.execute_activity( + payload_encryption_activity, + "outbound", + start_to_close_timeout=timedelta(seconds=10), + ) + assert "inbound" == await workflow.execute_child_workflow( + PayloadEncryptionChildWorkflow.run, + "outbound", + id=f"{workflow.info().workflow_id}_child", + ) + return "outbound" + + @workflow.query + def query(self, data: str) -> str: + assert data == "inbound" + return "outbound" + + @workflow.signal + def signal(self, data: str) -> None: + assert data == "inbound" + self.received_signal = True + + @workflow.update + def update(self, data: str) -> str: + assert data == "inbound" + self.received_update = True + return "outbound" + + @update.validator + def update_validator(self, data: str) -> None: + assert data == "inbound" + + +async def test_payload_encryption_with_context( + client: Client, +): + """ + "Encrypt" outbound payloads with a key using all available context fields, in order to demonstrate + that the same context is available to decrypt inbound payloads. + """ + workflow_id = str(uuid.uuid4()) + task_queue = str(uuid.uuid4()) + + config = client.config() + config["data_converter"] = dataclasses.replace( + DataConverter.default, + payload_codec=PayloadEncryptionCodec(), + ) + client = Client(**config) + + async with Worker( + client, + task_queue=task_queue, + workflows=[PayloadEncryptionWorkflow, PayloadEncryptionChildWorkflow], + activities=[payload_encryption_activity], + nexus_service_handlers=[PayloadEncryptionServiceHandler()], + ): + wf_handle = await client.start_workflow( + PayloadEncryptionWorkflow.run, + "outbound", + id=workflow_id, + task_queue=task_queue, + ) + assert "inbound" == await wf_handle.query( + PayloadEncryptionWorkflow.query, "outbound" + ) + await wf_handle.signal(PayloadEncryptionWorkflow.signal, "outbound") + assert "inbound" == await wf_handle.execute_update( + PayloadEncryptionWorkflow.update, "outbound" + ) + assert "inbound" == await wf_handle.result() + + +# Test outbound Nexus operations do not have any context set + + +class AssertNexusLacksContextPayloadCodec(PayloadCodec, WithSerializationContext): + def __init__(self): + self.context = None + + def with_context( + self, context: SerializationContext + ) -> AssertNexusLacksContextPayloadCodec: + codec = AssertNexusLacksContextPayloadCodec() + codec.context = context + return codec + + async def encode(self, payloads: Sequence[Payload]) -> List[Payload]: + [payload] = payloads + assert bool(self.context) == (payload.data.decode() != '"nexus-data"') + return list(payloads) + + async def decode(self, payloads: Sequence[Payload]) -> List[Payload]: + [payload] = payloads + assert bool(self.context) == (payload.data.decode() != '"nexus-data"') + return list(payloads) + + +@nexusrpc.handler.service_handler +class NexusOperationTestServiceHandler: + @nexusrpc.handler.sync_operation + async def operation( + self, _: nexusrpc.handler.StartOperationContext, data: str + ) -> str: + return data + + +@workflow.defn +class NexusOperationTestWorkflow: + @workflow.run + async def run(self, data: str) -> None: + nexus_client = workflow.create_nexus_client( + service=NexusOperationTestServiceHandler, + endpoint=make_nexus_endpoint_name(workflow.info().task_queue), + ) + await nexus_client.start_operation( + NexusOperationTestServiceHandler.operation, input="nexus-data" + ) + + +async def test_nexus_payload_codec_operations_lack_context( + client: Client, +): + """ + encode() and decode() on nexus payloads should not have any context set. + """ + workflow_id = str(uuid.uuid4()) + task_queue = str(uuid.uuid4()) + + config = client.config() + config["data_converter"] = dataclasses.replace( + DataConverter.default, + payload_codec=AssertNexusLacksContextPayloadCodec(), + ) + client = Client(**config) + + async with Worker( + client, + task_queue=task_queue, + workflows=[NexusOperationTestWorkflow], + nexus_service_handlers=[NexusOperationTestServiceHandler()], + ): + await create_nexus_endpoint(task_queue, client) + await client.execute_workflow( + NexusOperationTestWorkflow.run, + "workflow-data", + id=workflow_id, + task_queue=task_queue, + ) + + # Pydantic diff --git a/tests/worker/test_visitor.py b/tests/worker/test_visitor.py index c59a0248b..be2b991cd 100644 --- a/tests/worker/test_visitor.py +++ b/tests/worker/test_visitor.py @@ -236,7 +236,7 @@ async def test_bridge_encoding(): ), ) - await temporalio.bridge.worker.encode_completion(comp, SimpleCodec(), True) + await temporalio.bridge.worker.encode_completion(comp, SimpleCodec().encode, True) cmd = comp.successful.commands[0] sa = cmd.schedule_activity diff --git a/tests/worker/test_workflow.py b/tests/worker/test_workflow.py index 9661ad7cc..2b8f468c6 100644 --- a/tests/worker/test_workflow.py +++ b/tests/worker/test_workflow.py @@ -1610,6 +1610,11 @@ def activate(self, act: WorkflowActivation) -> WorkflowActivationCompletion: self._runner._pairs.append((act, comp)) return comp + def get_payload_codec( + self, command_seq: Optional[int] + ) -> Optional[temporalio.converter.PayloadCodec]: + return self._unsandboxed.get_payload_codec(command_seq) + async def test_workflow_with_custom_runner(client: Client): runner = CustomWorkflowRunner() From 00f8d0d5bf28bcef26a740668fb64be997eb4716 Mon Sep 17 00:00:00 2001 From: Dan Davison Date: Tue, 23 Sep 2025 18:30:33 -0400 Subject: [PATCH 12/81] Cleanup --- tests/test_serialization_context.py | 106 +++++++++++----------------- 1 file changed, 42 insertions(+), 64 deletions(-) diff --git a/tests/test_serialization_context.py b/tests/test_serialization_context.py index 4ffe0dd20..3629ae187 100644 --- a/tests/test_serialization_context.py +++ b/tests/test_serialization_context.py @@ -85,8 +85,6 @@ def with_context( def to_payload(self, value: Any) -> Optional[Payload]: if not isinstance(value, TraceData): return None - if not self.context: - raise Exception("Context is None") if isinstance(self.context, WorkflowSerializationContext): value.items.append( TraceItem( @@ -109,11 +107,8 @@ def to_payload(self, value: Any) -> Optional[Payload]: return payload def from_payload(self, payload: Payload, type_hint: Optional[Type] = None) -> Any: - # Always deserialize as TraceData since that's what this converter handles value = JSONPlainPayloadConverter().from_payload(payload, TraceData) assert isinstance(value, TraceData) - if not self.context: - raise Exception("Context is None") if isinstance(self.context, WorkflowSerializationContext): value.items.append( TraceItem( @@ -143,7 +138,9 @@ def __init__(self): ) -# Test payload conversion +# Payload conversion tests + +# Test misc payload conversion calls @activity.defn @@ -177,7 +174,7 @@ async def run(self, data: TraceData) -> TraceData: return data -async def test_workflow_payload_conversion( +async def test_payload_conversion_calls_follow_expected_sequence_and_contexts( client: Client, ): workflow_id = str(uuid.uuid4()) @@ -520,12 +517,7 @@ async def test_async_activity_completion_payload_conversion( await activity_handle.complete(data) result = await wf_handle.result() - # project down since activity completion by a client does not have access to most activity - # context fields - def project(trace_item: TraceItem) -> str: - return trace_item.method - - assert [project(item) for item in result.items] == [ + assert [item.method for item in result.items] == [ "to_payload", # Outbound activity input "to_payload", # Outbound activity heartbeat data "from_payload", # Inbound activity result @@ -1046,7 +1038,7 @@ async def test_failure_converter_with_context(client: Client): del test_traces[workflow_id] -## Test payload codec +# Test payload codec class PayloadCodecWithContext(PayloadCodec, WithSerializationContext): @@ -1155,6 +1147,9 @@ async def test_codec_with_context(client: Client): del test_traces[workflow_id] +# Local activity codec test + + @activity.defn async def codec_test_local_activity(data: str) -> str: return data @@ -1265,6 +1260,7 @@ async def run(self, data: TraceData) -> TraceData: async def test_child_workflow_codec_with_context(client: Client): workflow_id = str(uuid.uuid4()) task_queue = str(uuid.uuid4()) + child_workflow_id = f"{workflow_id}-child" config = client.config() config["data_converter"] = dataclasses.replace( @@ -1292,68 +1288,54 @@ async def test_child_workflow_codec_with_context(client: Client): workflow_id=workflow_id, ) ) - child_workflow_context = dataclasses.asdict( WorkflowSerializationContext( namespace=client.namespace, - workflow_id=f"{workflow_id}-child", + workflow_id=child_workflow_id, ) ) - # The expectation is that child workflows should use their own context for encoding/decoding, - # similar to how .NET and Java handle it - # Traces are stored under both parent and child workflow IDs - child_workflow_id = f"{workflow_id}-child" - - # Combine traces from parent and child workflows - all_traces = ( - test_traces[workflow_id][:2] # Parent workflow input - + test_traces[child_workflow_id] # All child workflow operations - + test_traces[workflow_id][2:] # Parent workflow result - ) - - assert all_traces == [ - # Parent workflow input + assert test_traces[workflow_id] == [ TraceItem( context=parent_workflow_context, - method="encode", + method="encode", # outbound workflow input ), TraceItem( context=parent_workflow_context, - method="decode", + method="decode", # inbound workflow input ), - # Child workflow input - should use child's context TraceItem( - context=child_workflow_context, - method="encode", + context=parent_workflow_context, + method="encode", # outbound workflow result ), TraceItem( - context=child_workflow_context, - method="decode", + context=parent_workflow_context, + method="decode", # inbound workflow result ), - # Child workflow result - should use child's context + ] + assert test_traces[child_workflow_id] == [ TraceItem( context=child_workflow_context, - method="encode", + method="encode", # outbound child workflow input ), TraceItem( context=child_workflow_context, - method="decode", + method="decode", # inbound child workflow input ), - # Parent workflow result TraceItem( - context=parent_workflow_context, - method="encode", + context=child_workflow_context, + method="encode", # outbound child workflow result ), TraceItem( - context=parent_workflow_context, - method="decode", + context=child_workflow_context, + method="decode", # inbound child workflow result ), ] del test_traces[workflow_id] + del test_traces[child_workflow_id] -# Payload encryption test +# Payload codec: test decode context matches encode context class PayloadEncryptionCodec(PayloadCodec, WithSerializationContext): @@ -1474,12 +1456,12 @@ def update_validator(self, data: str) -> None: assert data == "inbound" -async def test_payload_encryption_with_context( +async def test_decode_context_matches_encode_context( client: Client, ): """ - "Encrypt" outbound payloads with a key using all available context fields, in order to demonstrate - that the same context is available to decrypt inbound payloads. + Encode outbound payloads with a key using all available context fields, in order to demonstrate + that the same context is available to decode inbound payloads. """ workflow_id = str(uuid.uuid4()) task_queue = str(uuid.uuid4()) @@ -1514,7 +1496,7 @@ async def test_payload_encryption_with_context( assert "inbound" == await wf_handle.result() -# Test outbound Nexus operations do not have any context set +# Test nexus payload codec class AssertNexusLacksContextPayloadCodec(PayloadCodec, WithSerializationContext): @@ -1528,15 +1510,14 @@ def with_context( codec.context = context return codec - async def encode(self, payloads: Sequence[Payload]) -> List[Payload]: + async def _assert_context_iff_not_nexus( + self, payloads: Sequence[Payload] + ) -> List[Payload]: [payload] = payloads assert bool(self.context) == (payload.data.decode() != '"nexus-data"') return list(payloads) - async def decode(self, payloads: Sequence[Payload]) -> List[Payload]: - [payload] = payloads - assert bool(self.context) == (payload.data.decode() != '"nexus-data"') - return list(payloads) + encode = decode = _assert_context_iff_not_nexus @nexusrpc.handler.service_handler @@ -1567,9 +1548,6 @@ async def test_nexus_payload_codec_operations_lack_context( """ encode() and decode() on nexus payloads should not have any context set. """ - workflow_id = str(uuid.uuid4()) - task_queue = str(uuid.uuid4()) - config = client.config() config["data_converter"] = dataclasses.replace( DataConverter.default, @@ -1579,20 +1557,20 @@ async def test_nexus_payload_codec_operations_lack_context( async with Worker( client, - task_queue=task_queue, + task_queue=str(uuid.uuid4()), workflows=[NexusOperationTestWorkflow], nexus_service_handlers=[NexusOperationTestServiceHandler()], - ): - await create_nexus_endpoint(task_queue, client) + ) as worker: + await create_nexus_endpoint(worker.task_queue, client) await client.execute_workflow( NexusOperationTestWorkflow.run, "workflow-data", - id=workflow_id, - task_queue=task_queue, + id=str(uuid.uuid4()), + task_queue=worker.task_queue, ) -# Pydantic +# Test pydantic converter with context class PydanticData(BaseModel): From cd549297bc89479da6ba7a9f5a9879e3ac61887b Mon Sep 17 00:00:00 2001 From: Dan Davison Date: Wed, 24 Sep 2025 04:39:20 -0400 Subject: [PATCH 13/81] heartbeat test --- tests/test_serialization_context.py | 60 ++++++++++++++++++++--------- 1 file changed, 41 insertions(+), 19 deletions(-) diff --git a/tests/test_serialization_context.py b/tests/test_serialization_context.py index 3629ae187..00fd39895 100644 --- a/tests/test_serialization_context.py +++ b/tests/test_serialization_context.py @@ -284,19 +284,17 @@ async def test_payload_conversion_calls_follow_expected_sequence_and_contexts( @activity.defn async def activity_with_heartbeat_details() -> TraceData: - """Activity that checks heartbeat details are decoded with proper context.""" info = activity.info() - - if info.heartbeat_details: - assert len(info.heartbeat_details) == 1 - heartbeat_data = info.heartbeat_details[0] + if info.attempt == 1: + data = TraceData() + activity.heartbeat(data) + raise Exception("Intentional error to force retry") + elif info.attempt == 2: + [heartbeat_data] = info.heartbeat_details assert isinstance(heartbeat_data, TraceData) return heartbeat_data - - data = TraceData() - activity.heartbeat(data) - await asyncio.sleep(0.1) - raise Exception("Intentional failure to test heartbeat details") + else: + raise AssertionError(f"Unexpected attempt number: {info.attempt}") @workflow.defn @@ -339,6 +337,13 @@ async def test_heartbeat_details_payload_conversion(client: Client): task_queue=task_queue, ) + workflow_context = dataclasses.asdict( + WorkflowSerializationContext( + namespace="default", + workflow_id=workflow_id, + ) + ) + activity_context = dataclasses.asdict( ActivitySerializationContext( namespace="default", @@ -350,15 +355,32 @@ async def test_heartbeat_details_payload_conversion(client: Client): ) ) - found_heartbeat_decode = False - for item in result.items: - if item.method == "from_payload" and item.context == activity_context: - found_heartbeat_decode = True - break - - assert ( - found_heartbeat_decode - ), "Heartbeat details should be decoded with activity context" + assert result.items == [ + TraceItem( + method="to_payload", + context=activity_context, # Outbound heartbeat + ), + TraceItem( + method="from_payload", + context=activity_context, # Inbound heartbeart detail + ), + TraceItem( + method="to_payload", + context=activity_context, # Outbound activity result + ), + TraceItem( + method="from_payload", + context=activity_context, # Inbound activity result + ), + TraceItem( + method="to_payload", + context=workflow_context, # Outbound workflow result + ), + TraceItem( + method="from_payload", + context=workflow_context, # Inbound workflow result + ), + ] # Local activity test From ebe168183c88defa0db057a8c899097bb039e120 Mon Sep 17 00:00:00 2001 From: Dan Davison Date: Wed, 24 Sep 2025 05:53:05 -0400 Subject: [PATCH 14/81] Cleanup --- tests/test_serialization_context.py | 45 ++++++++++++++++++++++------- 1 file changed, 35 insertions(+), 10 deletions(-) diff --git a/tests/test_serialization_context.py b/tests/test_serialization_context.py index 00fd39895..30af6d444 100644 --- a/tests/test_serialization_context.py +++ b/tests/test_serialization_context.py @@ -140,7 +140,7 @@ def __init__(self): # Payload conversion tests -# Test misc payload conversion calls +## Misc payload conversion @activity.defn @@ -279,7 +279,7 @@ async def test_payload_conversion_calls_follow_expected_sequence_and_contexts( ] -# Activity with heartbeat details test +## Activity heartbeat payload conversion @activity.defn @@ -383,7 +383,7 @@ async def test_heartbeat_details_payload_conversion(client: Client): ] -# Local activity test +## Local activity payload conversion @activity.defn @@ -480,13 +480,32 @@ async def test_local_activity_payload_conversion(client: Client): ] -# Async activity completion test +## Async activity completion payload conversion + + +@workflow.defn +class EventWorkflow: + # Like a global asyncio.Event() + + def __init__(self) -> None: + self.signal_received = asyncio.Event() + + @workflow.run + async def run(self) -> None: + await self.signal_received.wait() + + @workflow.signal + def signal(self) -> None: + self.signal_received.set() @activity.defn async def async_activity() -> TraceData: - # Signal that activity has started via heartbeat - activity.heartbeat("started") + await ( + activity.client() + .get_workflow_handle("activity-started-wf-id") + .signal(EventWorkflow.signal) + ) activity.raise_complete_async() @@ -512,16 +531,23 @@ async def test_async_activity_completion_payload_conversion( DataConverter.default, payload_converter_class=SerializationContextCompositePayloadConverter, ) - client = Client(**config) async with Worker( client, task_queue=task_queue, - workflows=[AsyncActivityCompletionSerializationContextTestWorkflow], + workflows=[ + AsyncActivityCompletionSerializationContextTestWorkflow, + EventWorkflow, + ], activities=[async_activity], workflow_runner=UnsandboxedWorkflowRunner(), # so that we can use isinstance ): + act_started_wf_handle = await client.start_workflow( + EventWorkflow.run, + id="activity-started-wf-id", + task_queue=task_queue, + ) wf_handle = await client.start_workflow( AsyncActivityCompletionSerializationContextTestWorkflow.run, id=workflow_id, @@ -532,8 +558,7 @@ async def test_async_activity_completion_payload_conversion( run_id=wf_handle.first_execution_run_id, activity_id="async-activity-id", ) - # Wait a bit for the activity to start - await asyncio.sleep(0.5) + await act_started_wf_handle.result() data = TraceData() await activity_handle.heartbeat(data) await activity_handle.complete(data) From 51e3d4980a111104ee28cf85cdad1cf0a563c06d Mon Sep 17 00:00:00 2001 From: Dan Davison Date: Wed, 24 Sep 2025 07:44:27 -0400 Subject: [PATCH 15/81] Requirements --- requirements.md | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 requirements.md diff --git a/requirements.md b/requirements.md new file mode 100644 index 000000000..36840a438 --- /dev/null +++ b/requirements.md @@ -0,0 +1,22 @@ +# Serialization Context Requirements + +## Simple Workflow Context + +**Sequence:** `client --start--> workflow --result--> client` + +**Rule:** All operations use `WorkflowSerializationContext` with the target workflow's ID and namespace. + +| Operation | Stage | Context Type | .NET | Java | +|-----------|-------|--------------|------|------| +| **1. Client Starts Workflow** | | | | | +| Serialize | DataConverter | WorkflowSerializationContext | [TemporalClient.Workflow.cs:693-696](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Client/TemporalClient.Workflow.cs#L693-L696) | [RootWorkflowClientInvoker.java:62-66](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/client/RootWorkflowClientInvoker.java#L62-L66) | +| Encode | PayloadCodec | WorkflowSerializationContext | [TemporalClient.Workflow.cs:699-700](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Client/TemporalClient.Workflow.cs#L699-L700) | [RootWorkflowClientInvoker.java:69](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/client/RootWorkflowClientInvoker.java#L69) | +| **2. Workflow Receives Input** | | | | | +| Decode | PayloadCodec | WorkflowSerializationContext | [WorkflowCodecHelper.cs:125-126](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowCodecHelper.cs#L125-L126) | [ReplayWorkflowTaskHandler.java:160-162](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/replay/ReplayWorkflowTaskHandler.java#L160-L162) | +| Deserialize | DataConverter | WorkflowSerializationContext | [WorkflowInstance.cs:395-397](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowInstance.cs#L395-L397) | [SyncWorkflow.java:75-77](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/sync/SyncWorkflow.java#L75-L77) | +| **3. Workflow Returns Result** | | | | | +| Serialize | DataConverter | WorkflowSerializationContext | [WorkflowInstance.cs:1182-1184](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowInstance.cs#L1182-L1184) | [POJOWorkflowImplementationFactory.java:290-292](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/sync/POJOWorkflowImplementationFactory.java#L290-L292) | +| Encode | PayloadCodec | WorkflowSerializationContext | [WorkflowCodecHelper.cs:217-220](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowCodecHelper.cs#L217-L220) | [WorkflowWorker.java:702-704](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/worker/WorkflowWorker.java#L702-L704) | +| **4. Client Receives Result** | | | | | +| Decode | PayloadCodec | WorkflowSerializationContext | [WorkflowHandle.cs:84-87](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Client/WorkflowHandle.cs#L84-L87) | [RootWorkflowClientInvoker.java:324-327](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/client/RootWorkflowClientInvoker.java#L324-L327) | +| Deserialize | DataConverter | WorkflowSerializationContext | [WorkflowHandle.cs:129-130](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Client/WorkflowHandle.cs#L129-L130) | [RootWorkflowClientInvoker.java:318-322](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/client/RootWorkflowClientInvoker.java#L318-L322) | From 30520e43b66cf7f4c6afe347ecc8620194ea9b08 Mon Sep 17 00:00:00 2001 From: Dan Davison Date: Wed, 24 Sep 2025 08:01:10 -0400 Subject: [PATCH 16/81] Document serialization context flow for workflows, activities, child workflows, and nexus operations The document provides: - 16 code locations for simple workflow execution flow - 16 code locations for activity execution flow - 16 code locations for child workflow execution flow - 16 code locations for nexus operation execution flow Each flow shows exactly where .NET and Java apply serialization context for both DataConverter and PayloadCodec operations. Key insights documented: - Workflow operations always use WorkflowSerializationContext - Activity operations use ActivitySerializationContext with is_local flag - Child workflows use WorkflowSerializationContext with child's ID - Nexus operations have no serialization context (null) --- requirements.md | 70 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/requirements.md b/requirements.md index 36840a438..520020bae 100644 --- a/requirements.md +++ b/requirements.md @@ -20,3 +20,73 @@ | **4. Client Receives Result** | | | | | | Decode | PayloadCodec | WorkflowSerializationContext | [WorkflowHandle.cs:84-87](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Client/WorkflowHandle.cs#L84-L87) | [RootWorkflowClientInvoker.java:324-327](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/client/RootWorkflowClientInvoker.java#L324-L327) | | Deserialize | DataConverter | WorkflowSerializationContext | [WorkflowHandle.cs:129-130](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Client/WorkflowHandle.cs#L129-L130) | [RootWorkflowClientInvoker.java:318-322](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/client/RootWorkflowClientInvoker.java#L318-L322) | + +## Activity Context + +**Sequence:** `workflow --scheduleActivity--> activity --result--> workflow` + +**Rule:** All operations use `ActivitySerializationContext` with activity type, task queue, and is_local flag. + +| Operation | Stage | Context Type | .NET | Java | +|-----------|-------|--------------|------|------| +| **1. Workflow Schedules Activity** | | | | | +| Serialize | DataConverter | ActivitySerializationContext | [WorkflowInstance.cs:2084-2093](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowInstance.cs#L2084-L2093) | [SyncWorkflowContext.java:257-267](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/sync/SyncWorkflowContext.java#L257-L267) | +| Encode | PayloadCodec | ActivitySerializationContext | [WorkflowCodecHelper.cs:243-253](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowCodecHelper.cs#L243-L253) | [SyncWorkflowContext.java:268-270](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/sync/SyncWorkflowContext.java#L268-L270) | +| **2. Activity Receives Input** | | | | | +| Decode | PayloadCodec | ActivitySerializationContext | Worker-side (not in workflow) | Worker-side (not in workflow) | +| Deserialize | DataConverter | ActivitySerializationContext | Worker-side (not in workflow) | Worker-side (not in workflow) | +| **3. Activity Returns Result** | | | | | +| Serialize | DataConverter | ActivitySerializationContext | Worker-side (not in workflow) | Worker-side (not in workflow) | +| Encode | PayloadCodec | ActivitySerializationContext | Worker-side (not in workflow) | Worker-side (not in workflow) | +| **4. Workflow Receives Result** | | | | | +| Decode | PayloadCodec | ActivitySerializationContext | [WorkflowCodecHelper.cs:68-76](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowCodecHelper.cs#L68-L76) | [SyncWorkflowContext.java:273-274](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/sync/SyncWorkflowContext.java#L273-L274) | +| Deserialize | DataConverter | ActivitySerializationContext | [WorkflowInstance.cs:2720](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowInstance.cs#L2720) | [SyncWorkflowContext.java:286-288](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/sync/SyncWorkflowContext.java#L286-L288) | + +## Child Workflow Context + +**Sequence:** `workflow --startChildWorkflow--> childWorkflow --result--> workflow` + +**Rule:** Child operations use `WorkflowSerializationContext` with the child workflow's ID. + +| Operation | Stage | Context Type | .NET | Java | +|-----------|-------|--------------|------|------| +| **1. Workflow Starts Child** | | | | | +| Serialize | DataConverter | WorkflowSerializationContext (child ID) | [WorkflowInstance.cs:2319-2326](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowInstance.cs#L2319-L2326) | [SyncWorkflowContext.java:687-690](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/sync/SyncWorkflowContext.java#L687-L690) | +| Encode | PayloadCodec | WorkflowSerializationContext (child ID) | [WorkflowCodecHelper.cs:301-310](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowCodecHelper.cs#L301-L310) | [SyncWorkflowContext.java:690](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/sync/SyncWorkflowContext.java#L690) | +| **2. Child Receives Input** | | | | | +| Decode | PayloadCodec | WorkflowSerializationContext (child ID) | [WorkflowCodecHelper.cs:125-126](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowCodecHelper.cs#L125-L126) | [ReplayWorkflowTaskHandler.java:160-162](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/replay/ReplayWorkflowTaskHandler.java#L160-L162) | +| Deserialize | DataConverter | WorkflowSerializationContext (child ID) | [WorkflowInstance.cs:395-397](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowInstance.cs#L395-L397) | [SyncWorkflow.java:75-77](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/sync/SyncWorkflow.java#L75-L77) | +| **3. Child Returns Result** | | | | | +| Serialize | DataConverter | WorkflowSerializationContext (child ID) | [WorkflowInstance.cs:1182-1184](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowInstance.cs#L1182-L1184) | [POJOWorkflowImplementationFactory.java:290-292](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/sync/POJOWorkflowImplementationFactory.java#L290-L292) | +| Encode | PayloadCodec | WorkflowSerializationContext (child ID) | [WorkflowCodecHelper.cs:217-220](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowCodecHelper.cs#L217-L220) | [WorkflowWorker.java:702-704](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/worker/WorkflowWorker.java#L702-L704) | +| **4. Workflow Receives Child Result** | | | | | +| Decode | PayloadCodec | WorkflowSerializationContext (child ID) | [WorkflowCodecHelper.cs:78-85](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowCodecHelper.cs#L78-L85) | [SyncWorkflowContext.java:1245-1248](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/sync/SyncWorkflowContext.java#L1245-L1248) | +| Deserialize | DataConverter | WorkflowSerializationContext (child ID) | [WorkflowInstance.cs:2808](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowInstance.cs#L2808) | [SyncWorkflowContext.java:1245-1248](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/sync/SyncWorkflowContext.java#L1245-L1248) | + +## Nexus Operation Context + +**Sequence:** `workflow --scheduleNexusOperation--> nexusOperation --result--> workflow` + +**Rule:** Nexus operations have **no serialization context** (null/none). + +| Operation | Stage | Context Type | .NET | Java | +|-----------|-------|--------------|------|------| +| **1. Workflow Schedules Nexus Op** | | | | | +| Serialize | DataConverter | None | [WorkflowInstance.cs:2525](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowInstance.cs#L2525) | [SyncWorkflowContext.java:790](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/sync/SyncWorkflowContext.java#L790) | +| Encode | PayloadCodec | None | [WorkflowCodecHelper.cs:355-356](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowCodecHelper.cs#L355-L356) | [SyncWorkflowContext.java:791-794](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/sync/SyncWorkflowContext.java#L791-L794) | +| **2. Nexus Operation Receives Input** | | | | | +| Decode | PayloadCodec | None | N/A (handler-side) | N/A (handler-side) | +| Deserialize | DataConverter | None | N/A (handler-side) | N/A (handler-side) | +| **3. Nexus Operation Returns Result** | | | | | +| Serialize | DataConverter | None | N/A (handler-side) | N/A (handler-side) | +| Encode | PayloadCodec | None | N/A (handler-side) | N/A (handler-side) | +| **4. Workflow Receives Result** | | | | | +| Decode | PayloadCodec | None | [WorkflowCodecHelper.cs:112-113](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowCodecHelper.cs#L112-L113) | N/A (codec not applied) | +| Deserialize | DataConverter | None | [WorkflowInstance.cs:2954](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowInstance.cs#L2954) | [SyncWorkflowContext.java:855-856](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/sync/SyncWorkflowContext.java#L855-L856) | + +## Summary of Context Rules + +1. **Workflow operations**: Always use `WorkflowSerializationContext` with the target workflow's ID +2. **Activity operations**: Use `ActivitySerializationContext` with activity details and `is_local` flag +3. **Child workflow operations**: Use `WorkflowSerializationContext` with the child's workflow ID +4. **Nexus operations**: No serialization context (null/none) From 71d6dcc5b8b6b83da3b629444cd27673b477bb4f Mon Sep 17 00:00:00 2001 From: Dan Davison Date: Wed, 24 Sep 2025 08:08:28 -0400 Subject: [PATCH 17/81] Add memo, search attributes, and user-accessible data converter documentation Added three new sections to requirements.md: 1. Memo and Search Attribute Context: - Documents that memos always use WorkflowSerializationContext - Shows search attributes don't use any serialization context (direct proto) - Includes references for client, workflow upsert, child workflow, and schedule operations 2. User-Accessible Data Converter - Workflow Context: - Shows how .NET exposes Workflow.PayloadConverter with workflow context - Documents that Java doesn't expose data converter to workflow code - Shows Python's workflow.payload_converter() API with workflow context 3. User-Accessible Data Converter - Activity Context: - Shows how .NET exposes ActivityExecutionContext.PayloadConverter - Documents that Java doesn't expose data converter to activity code - Shows Python's activity.payload_converter() API with activity context All references include verified GitHub links to the exact source locations. --- requirements.md | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/requirements.md b/requirements.md index 520020bae..cd6de6d0b 100644 --- a/requirements.md +++ b/requirements.md @@ -84,9 +84,55 @@ | Decode | PayloadCodec | None | [WorkflowCodecHelper.cs:112-113](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowCodecHelper.cs#L112-L113) | N/A (codec not applied) | | Deserialize | DataConverter | None | [WorkflowInstance.cs:2954](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowInstance.cs#L2954) | [SyncWorkflowContext.java:855-856](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/sync/SyncWorkflowContext.java#L855-L856) | +## Memo and Search Attribute Context + +### Memos + +**Rule:** Memos always use `WorkflowSerializationContext` with the workflow's ID when set or accessed. + +| Operation | Context Type | .NET | Java | +|-----------|--------------|------|------| +| **Client sets memo on start** | WorkflowSerializationContext | [TemporalClient.Workflow.cs:827](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Client/TemporalClient.Workflow.cs#L827) | [RootWorkflowClientInvoker.java:292-294](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/client/RootWorkflowClientInvoker.java#L292-L294) | +| **Workflow upserts memo** | WorkflowSerializationContext | [WorkflowInstance.cs:472](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowInstance.cs#L472) | [SyncWorkflowContext.java:1416](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/sync/SyncWorkflowContext.java#L1416) | +| **Child workflow memo** | WorkflowSerializationContext (child ID) | [WorkflowInstance.cs:2359-2360](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowInstance.cs#L2359-L2360) | [SyncWorkflowContext.java:693-700](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/sync/SyncWorkflowContext.java#L693-L700) | +| **Schedule sets memo** | WorkflowSerializationContext | [ScheduleActionStartWorkflow.cs:199](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Client/Schedules/ScheduleActionStartWorkflow.cs#L199) | [ScheduleProtoUtil.java:134](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/client/ScheduleProtoUtil.java#L134) | + +### Search Attributes + +**Rule:** Search attributes do NOT use serialization context - they use specialized converters for indexing. + +| Operation | Context Type | .NET | Java | +|-----------|--------------|------|------| +| **All operations** | None (direct proto conversion) | Uses `ToProto()` | Uses `toSearchAttributes()` | + +## User-Accessible Data Converter + +### Workflow Context + +**Rule:** Data converters exposed to workflow code have `WorkflowSerializationContext` applied. + +| SDK | API | Context | Reference | +|-----|-----|---------|-----------| +| **.NET** | `Workflow.PayloadConverter` | WorkflowSerializationContext | [Workflow.cs:185](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Workflows/Workflow.cs#L185) | +| **Java** | Not directly exposed | N/A | Workflow code cannot access data converter | +| **Python** | `workflow.payload_converter()` | WorkflowSerializationContext | [workflow.py:1148](https://github.com/temporalio/sdk-python/blob/main/temporalio/workflow.py#L1148) | + +### Activity Context + +**Rule:** Data converters exposed to activity code have `ActivitySerializationContext` applied. + +| SDK | API | Context | Reference | +|-----|-----|---------|-----------| +| **.NET** | `ActivityExecutionContext.PayloadConverter` | ActivitySerializationContext | [ActivityExecutionContext.cs:49](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Activities/ActivityExecutionContext.cs#L49) | +| **Java** | Not directly exposed | N/A | Activity code cannot access data converter | +| **Python** | `activity.payload_converter()` | ActivitySerializationContext | [activity.py:470](https://github.com/temporalio/sdk-python/blob/main/temporalio/activity.py#L470) | + ## Summary of Context Rules 1. **Workflow operations**: Always use `WorkflowSerializationContext` with the target workflow's ID 2. **Activity operations**: Use `ActivitySerializationContext` with activity details and `is_local` flag 3. **Child workflow operations**: Use `WorkflowSerializationContext` with the child's workflow ID 4. **Nexus operations**: No serialization context (null/none) +5. **Memos**: Always use workflow context +6. **Search attributes**: Never use context (indexing-specific conversion) +7. **User-exposed converters**: Have appropriate context pre-applied From 59f7ff1bc0b81f12d1b88733e2bb156f78ba0058 Mon Sep 17 00:00:00 2001 From: Dan Davison Date: Wed, 24 Sep 2025 08:12:23 -0400 Subject: [PATCH 18/81] Add async activity completion and heartbeating documentation Added two comprehensive sections documenting serialization context for: 1. **Async Activity Completion**: - Documents how .NET, Java, and Python apply ActivitySerializationContext - Shows complete, fail, and report cancellation operations - Notes that .NET/Java support explicit WithContext() methods - Explains Python applies context internally with partial info 2. **Activity Heartbeating**: - Separates heartbeating during execution vs async heartbeating - Shows full context available during activity execution - Documents partial context for async heartbeating - Includes verified code references for all three SDKs Key insights: - Context may be incomplete for async operations (e.g., missing activity type) - Python applies context automatically, while .NET/Java allow explicit control - All SDKs ensure proper data conversion for heartbeat details --- requirements.md | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/requirements.md b/requirements.md index cd6de6d0b..fa9db2053 100644 --- a/requirements.md +++ b/requirements.md @@ -127,6 +127,36 @@ | **Java** | Not directly exposed | N/A | Activity code cannot access data converter | | **Python** | `activity.payload_converter()` | ActivitySerializationContext | [activity.py:470](https://github.com/temporalio/sdk-python/blob/main/temporalio/activity.py#L470) | +## Async Activity Completion + +**Rule:** Async activity completion uses `ActivitySerializationContext` with available activity information. + +| Operation | Context Type | .NET | Java | Python | +|-----------|--------------|------|------|--------| +| **Complete** | ActivitySerializationContext | [AsyncActivityHandle.cs:42](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Client/AsyncActivityHandle.cs#L42) | [ActivityCompletionClientImpl.java:51-56](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/client/ActivityCompletionClientImpl.java#L51-L56) | [client.py:6477-6481](https://github.com/temporalio/sdk-python/blob/main/temporalio/client.py#L6477-L6481) | +| **Fail** | ActivitySerializationContext | [AsyncActivityHandle.cs:51](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Client/AsyncActivityHandle.cs#L51) | [ActivityCompletionClientImpl.java:63-65](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/client/ActivityCompletionClientImpl.java#L63-L65) | [client.py:6511-6514](https://github.com/temporalio/sdk-python/blob/main/temporalio/client.py#L6511-L6514) | +| **Report Cancellation** | ActivitySerializationContext | [AsyncActivityHandle.cs:62](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Client/AsyncActivityHandle.cs#L62) | [ActivityCompletionClientImpl.java:74](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/client/ActivityCompletionClientImpl.java#L74) | [client.py:6588-6600](https://github.com/temporalio/sdk-python/blob/main/temporalio/client.py#L6588-L6600) | +| **WithContext Method** | Creates context-aware handle | [AsyncActivityHandle.cs:71-73](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Client/AsyncActivityHandle.cs#L71-L73) | [ActivityCompletionClient.java:107-108](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/client/ActivityCompletionClientImpl.java#L107-L108) | N/A (context applied internally) | + +### Notes +- Python applies partial context internally in `_async_activity_data_converter()` with workflow ID when available +- .NET and Java allow explicit context application via `WithSerializationContext()`/`withContext()` methods +- Context may be incomplete for async completion (e.g., missing activity type) when using task token + +## Activity Heartbeating + +**Rule:** Heartbeating uses `ActivitySerializationContext` with full activity information when available. + +| Operation | Context Source | .NET | Java | Python | +|-----------|---------------|------|------|--------| +| **During Execution** | From running activity info | Same as async completion | [HeartbeatContextImpl.java:67-75](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/activity/HeartbeatContextImpl.java#L67-L75) | [_activity.py:257-265](https://github.com/temporalio/sdk-python/blob/main/temporalio/worker/_activity.py#L257-L265) | +| **Async Heartbeat** | From task token or activity ID | [AsyncActivityHandle.cs:30-32](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Client/AsyncActivityHandle.cs#L30-L32) | [ActivityCompletionClientImpl.java:94-102](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/client/ActivityCompletionClientImpl.java#L94-L102) | [client.py:6422-6426](https://github.com/temporalio/sdk-python/blob/main/temporalio/client.py#L6422-L6426) | + +### Notes +- Heartbeating during activity execution has full context information +- Async heartbeating (outside activity execution) may have partial context +- All SDKs apply context to ensure proper data conversion for heartbeat details + ## Summary of Context Rules 1. **Workflow operations**: Always use `WorkflowSerializationContext` with the target workflow's ID @@ -136,3 +166,5 @@ 5. **Memos**: Always use workflow context 6. **Search attributes**: Never use context (indexing-specific conversion) 7. **User-exposed converters**: Have appropriate context pre-applied +8. **Async activity completion**: Use activity context with available information +9. **Heartbeating**: Use activity context with full info during execution, partial for async From 43be40ff64927788f302c72f3c2323bf76f203ed Mon Sep 17 00:00:00 2001 From: Dan Davison Date: Wed, 24 Sep 2025 07:44:38 -0400 Subject: [PATCH 19/81] Async activity completion --- tests/test_serialization_context.py | 50 +++++++++++++++++++++++++---- 1 file changed, 43 insertions(+), 7 deletions(-) diff --git a/tests/test_serialization_context.py b/tests/test_serialization_context.py index 30af6d444..89424e5d3 100644 --- a/tests/test_serialization_context.py +++ b/tests/test_serialization_context.py @@ -543,6 +543,19 @@ async def test_async_activity_completion_payload_conversion( activities=[async_activity], workflow_runner=UnsandboxedWorkflowRunner(), # so that we can use isinstance ): + workflow_context = WorkflowSerializationContext( + namespace="default", + workflow_id=workflow_id, + ) + activity_context = ActivitySerializationContext( + namespace="default", + workflow_id=workflow_id, + workflow_type=AsyncActivityCompletionSerializationContextTestWorkflow.__name__, + activity_type=async_activity.__name__, + activity_task_queue=task_queue, + is_local=False, + ) + act_started_wf_handle = await client.start_workflow( EventWorkflow.run, id="activity-started-wf-id", @@ -557,19 +570,42 @@ async def test_async_activity_completion_payload_conversion( workflow_id=workflow_id, run_id=wf_handle.first_execution_run_id, activity_id="async-activity-id", - ) + ).with_context(activity_context) + await act_started_wf_handle.result() data = TraceData() await activity_handle.heartbeat(data) await activity_handle.complete(data) result = await wf_handle.result() - assert [item.method for item in result.items] == [ - "to_payload", # Outbound activity input - "to_payload", # Outbound activity heartbeat data - "from_payload", # Inbound activity result - "to_payload", # Outbound workflow result - "from_payload", # Inbound workflow result + print() + for item in result.items: + print(item) + + activity_context_dict = dataclasses.asdict(activity_context) + workflow_context_dict = dataclasses.asdict(workflow_context) + + assert result.items == [ + TraceItem( + method="to_payload", + context=activity_context_dict, # Outbound activity heartbeat + ), + TraceItem( + method="to_payload", + context=activity_context_dict, # Outbound activity completion + ), + TraceItem( + method="from_payload", + context=activity_context_dict, # Inbound activity result + ), + TraceItem( + method="to_payload", + context=workflow_context_dict, # Outbound workflow result + ), + TraceItem( + method="from_payload", + context=workflow_context_dict, # Inbound workflow result + ), ] From d41cee6bf2ca6940132867605ba91f15ca1257d5 Mon Sep 17 00:00:00 2001 From: Dan Davison Date: Wed, 24 Sep 2025 08:36:37 -0400 Subject: [PATCH 20/81] Implement WithSerializationContext on AsyncActivityHandle --- temporalio/client.py | 59 +++++++++++++++++++++++++------------------- 1 file changed, 34 insertions(+), 25 deletions(-) diff --git a/temporalio/client.py b/temporalio/client.py index f17fd35e4..7005f40c3 100644 --- a/temporalio/client.py +++ b/temporalio/client.py @@ -63,8 +63,9 @@ import temporalio.workflow from temporalio.activity import ActivityCancellationDetails from temporalio.converter import ( - ActivitySerializationContext, DataConverter, + SerializationContext, + WithSerializationContext, WorkflowSerializationContext, ) from temporalio.service import ( @@ -2732,15 +2733,19 @@ class AsyncActivityIDReference: activity_id: str -class AsyncActivityHandle: +class AsyncActivityHandle(WithSerializationContext): """Handle representing an external activity for completion and heartbeat.""" def __init__( - self, client: Client, id_or_token: Union[AsyncActivityIDReference, bytes] + self, + client: Client, + id_or_token: Union[AsyncActivityIDReference, bytes], + data_converter_override: Optional[DataConverter] = None, ) -> None: """Create an async activity handle.""" self._client = client self._id_or_token = id_or_token + self._data_converter_override = data_converter_override async def heartbeat( self, @@ -2762,6 +2767,7 @@ async def heartbeat( details=details, rpc_metadata=rpc_metadata, rpc_timeout=rpc_timeout, + data_converter_override=self._data_converter_override, ), ) @@ -2786,6 +2792,7 @@ async def complete( result=result, rpc_metadata=rpc_metadata, rpc_timeout=rpc_timeout, + data_converter_override=self._data_converter_override, ), ) @@ -2813,6 +2820,7 @@ async def fail( last_heartbeat_details=last_heartbeat_details, rpc_metadata=rpc_metadata, rpc_timeout=rpc_timeout, + data_converter_override=self._data_converter_override, ), ) @@ -2836,9 +2844,24 @@ async def report_cancellation( details=details, rpc_metadata=rpc_metadata, rpc_timeout=rpc_timeout, + data_converter_override=self._data_converter_override, ), ) + def with_context(self, context: SerializationContext) -> AsyncActivityHandle: + """Create a new AsyncActivityHandle with a different serialization context. + + Payloads received by the activity will be decoded and deserialized using a data converter + with :py:class:`ActivitySerializationContext` set as context. If you are using a custom data + converter that makes use of this context then you can use this method to supply matching + context data to the data converter used to serialize and encode the outbound payloads. + """ + return AsyncActivityHandle( + self._client, + self._id_or_token, + self._client.data_converter._with_context(context), + ) + @dataclass class WorkflowExecution: @@ -5486,6 +5509,7 @@ class HeartbeatAsyncActivityInput: details: Sequence[Any] rpc_metadata: Mapping[str, Union[str, bytes]] rpc_timeout: Optional[timedelta] + data_converter_override: Optional[DataConverter] = None @dataclass @@ -5496,6 +5520,7 @@ class CompleteAsyncActivityInput: result: Optional[Any] rpc_metadata: Mapping[str, Union[str, bytes]] rpc_timeout: Optional[timedelta] + data_converter_override: Optional[DataConverter] = None @dataclass @@ -5507,6 +5532,7 @@ class FailAsyncActivityInput: last_heartbeat_details: Sequence[Any] rpc_metadata: Mapping[str, Union[str, bytes]] rpc_timeout: Optional[timedelta] + data_converter_override: Optional[DataConverter] = None @dataclass @@ -5517,6 +5543,7 @@ class ReportCancellationAsyncActivityInput: details: Sequence[Any] rpc_metadata: Mapping[str, Union[str, bytes]] rpc_timeout: Optional[timedelta] + data_converter_override: Optional[DataConverter] = None @dataclass @@ -6418,7 +6445,7 @@ async def _start_workflow_update_with_start( async def heartbeat_async_activity( self, input: HeartbeatAsyncActivityInput ) -> None: - data_converter = self._async_activity_data_converter(input.id_or_token) + data_converter = input.data_converter_override or self._client.data_converter details = ( None if not input.details @@ -6473,7 +6500,7 @@ async def heartbeat_async_activity( ) async def complete_async_activity(self, input: CompleteAsyncActivityInput) -> None: - data_converter = self._async_activity_data_converter(input.id_or_token) + data_converter = input.data_converter_override or self._client.data_converter result = ( None if input.result is temporalio.common._arg_unset @@ -6507,7 +6534,7 @@ async def complete_async_activity(self, input: CompleteAsyncActivityInput) -> No ) async def fail_async_activity(self, input: FailAsyncActivityInput) -> None: - data_converter = self._async_activity_data_converter(input.id_or_token) + data_converter = input.data_converter_override or self._client.data_converter failure = temporalio.api.failure.v1.Failure() await data_converter.encode_failure(input.error, failure) @@ -6548,7 +6575,7 @@ async def fail_async_activity(self, input: FailAsyncActivityInput) -> None: async def report_cancellation_async_activity( self, input: ReportCancellationAsyncActivityInput ) -> None: - data_converter = self._async_activity_data_converter(input.id_or_token) + data_converter = input.data_converter_override or self._client.data_converter details = ( None if not input.details @@ -6581,24 +6608,6 @@ async def report_cancellation_async_activity( timeout=input.rpc_timeout, ) - def _async_activity_data_converter( - self, id_or_token: Union[AsyncActivityIDReference, bytes] - ) -> DataConverter: - return self._client.data_converter._with_context( - ActivitySerializationContext( - namespace=self._client.namespace, - workflow_id=( - id_or_token.workflow_id - if isinstance(id_or_token, AsyncActivityIDReference) - else "" - ), - workflow_type="", - activity_type="", - activity_task_queue="", - is_local=False, - ) - ) - ### Schedule calls async def create_schedule(self, input: CreateScheduleInput) -> ScheduleHandle: From dab89995ce94a0d98dfffd47ae00947f753eceb7 Mon Sep 17 00:00:00 2001 From: Dan Davison Date: Wed, 24 Sep 2025 09:02:34 -0400 Subject: [PATCH 21/81] Introduce shared base context class --- temporalio/converter.py | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/temporalio/converter.py b/temporalio/converter.py index 3ad972049..6d1c07f63 100644 --- a/temporalio/converter.py +++ b/temporalio/converter.py @@ -89,7 +89,15 @@ class SerializationContext(ABC): @dataclass(frozen=True) -class WorkflowSerializationContext(SerializationContext): +class BaseWorkflowSerializationContext(SerializationContext): + """Base serialization context shared by workflow and activity serialization contexts.""" + + namespace: str + workflow_id: str + + +@dataclass(frozen=True) +class WorkflowSerializationContext(BaseWorkflowSerializationContext): """Serialization context for workflows. See :py:class:`SerializationContext` for more details. @@ -102,12 +110,11 @@ class WorkflowSerializationContext(SerializationContext): when the workflow is created by the schedule. """ - namespace: str - workflow_id: str + pass @dataclass(frozen=True) -class ActivitySerializationContext(SerializationContext): +class ActivitySerializationContext(BaseWorkflowSerializationContext): """Serialization context for activities. See :py:class:`SerializationContext` for more details. @@ -115,16 +122,14 @@ class ActivitySerializationContext(SerializationContext): Attributes: namespace: Workflow/activity namespace. workflow_id: Workflow ID. Note, when creating/describing schedules, - this may be the workflow ID prefix as configured, not the final - workflow ID when the workflow is created by the schedule. + this may be the workflow ID prefix as configured, not the final workflow ID when the + workflow is created by the schedule. workflow_type: Workflow Type. activity_type: Activity Type. activity_task_queue: Activity task queue. is_local: Whether the activity is a local activity. """ - namespace: str - workflow_id: str workflow_type: str activity_type: str activity_task_queue: str From 87ca957d527453e9ce3d59bced2b7899ecb8b3b3 Mon Sep 17 00:00:00 2001 From: Dan Davison Date: Wed, 24 Sep 2025 09:30:11 -0400 Subject: [PATCH 22/81] Add python column to requirements.md --- requirements.md | 145 ++++++++++++++++++++++++++---------------------- 1 file changed, 80 insertions(+), 65 deletions(-) diff --git a/requirements.md b/requirements.md index fa9db2053..ccce0c4ae 100644 --- a/requirements.md +++ b/requirements.md @@ -6,20 +6,20 @@ **Rule:** All operations use `WorkflowSerializationContext` with the target workflow's ID and namespace. -| Operation | Stage | Context Type | .NET | Java | -|-----------|-------|--------------|------|------| -| **1. Client Starts Workflow** | | | | | -| Serialize | DataConverter | WorkflowSerializationContext | [TemporalClient.Workflow.cs:693-696](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Client/TemporalClient.Workflow.cs#L693-L696) | [RootWorkflowClientInvoker.java:62-66](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/client/RootWorkflowClientInvoker.java#L62-L66) | -| Encode | PayloadCodec | WorkflowSerializationContext | [TemporalClient.Workflow.cs:699-700](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Client/TemporalClient.Workflow.cs#L699-L700) | [RootWorkflowClientInvoker.java:69](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/client/RootWorkflowClientInvoker.java#L69) | -| **2. Workflow Receives Input** | | | | | -| Decode | PayloadCodec | WorkflowSerializationContext | [WorkflowCodecHelper.cs:125-126](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowCodecHelper.cs#L125-L126) | [ReplayWorkflowTaskHandler.java:160-162](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/replay/ReplayWorkflowTaskHandler.java#L160-L162) | -| Deserialize | DataConverter | WorkflowSerializationContext | [WorkflowInstance.cs:395-397](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowInstance.cs#L395-L397) | [SyncWorkflow.java:75-77](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/sync/SyncWorkflow.java#L75-L77) | -| **3. Workflow Returns Result** | | | | | -| Serialize | DataConverter | WorkflowSerializationContext | [WorkflowInstance.cs:1182-1184](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowInstance.cs#L1182-L1184) | [POJOWorkflowImplementationFactory.java:290-292](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/sync/POJOWorkflowImplementationFactory.java#L290-L292) | -| Encode | PayloadCodec | WorkflowSerializationContext | [WorkflowCodecHelper.cs:217-220](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowCodecHelper.cs#L217-L220) | [WorkflowWorker.java:702-704](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/worker/WorkflowWorker.java#L702-L704) | -| **4. Client Receives Result** | | | | | -| Decode | PayloadCodec | WorkflowSerializationContext | [WorkflowHandle.cs:84-87](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Client/WorkflowHandle.cs#L84-L87) | [RootWorkflowClientInvoker.java:324-327](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/client/RootWorkflowClientInvoker.java#L324-L327) | -| Deserialize | DataConverter | WorkflowSerializationContext | [WorkflowHandle.cs:129-130](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Client/WorkflowHandle.cs#L129-L130) | [RootWorkflowClientInvoker.java:318-322](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/client/RootWorkflowClientInvoker.java#L318-L322) | +| Operation | Stage | Context Type | .NET | Java | Python | +|-----------|-------|--------------|------|------|--------| +| **1. Client Starts Workflow** | | | | | | +| Serialize | DataConverter | WorkflowSerializationContext | [TemporalClient.Workflow.cs:693-696](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Client/TemporalClient.Workflow.cs#L693-L696) | [RootWorkflowClientInvoker.java:62-66](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/client/RootWorkflowClientInvoker.java#L62-L66) | [client.py:5993-5998](https://github.com/temporalio/sdk-python/blob/main/temporalio/client.py#L5993-L5998) | +| Encode | PayloadCodec | WorkflowSerializationContext | [TemporalClient.Workflow.cs:699-700](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Client/TemporalClient.Workflow.cs#L699-L700) | [RootWorkflowClientInvoker.java:69](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/client/RootWorkflowClientInvoker.java#L69) | [client.py:6005](https://github.com/temporalio/sdk-python/blob/main/temporalio/client.py#L6005) | +| **2. Workflow Receives Input** | | | | | | +| Decode | PayloadCodec | WorkflowSerializationContext | [WorkflowCodecHelper.cs:125-126](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowCodecHelper.cs#L125-L126) | [ReplayWorkflowTaskHandler.java:160-162](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/replay/ReplayWorkflowTaskHandler.java#L160-L162) | [_workflow.py:289-293](https://github.com/temporalio/sdk-python/blob/main/temporalio/worker/_workflow.py#L289-L293) | +| Deserialize | DataConverter | WorkflowSerializationContext | [WorkflowInstance.cs:395-397](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowInstance.cs#L395-L397) | [SyncWorkflow.java:75-77](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/sync/SyncWorkflow.java#L75-L77) | Via context-aware converter | +| **3. Workflow Returns Result** | | | | | | +| Serialize | DataConverter | WorkflowSerializationContext | [WorkflowInstance.cs:1182-1184](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowInstance.cs#L1182-L1184) | [POJOWorkflowImplementationFactory.java:290-292](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/sync/POJOWorkflowImplementationFactory.java#L290-L292) | Via context-aware converter | +| Encode | PayloadCodec | WorkflowSerializationContext | [WorkflowCodecHelper.cs:217-220](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowCodecHelper.cs#L217-L220) | [WorkflowWorker.java:702-704](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/worker/WorkflowWorker.java#L702-L704) | [_workflow.py:363-367](https://github.com/temporalio/sdk-python/blob/main/temporalio/worker/_workflow.py#L363-L367) | +| **4. Client Receives Result** | | | | | | +| Decode | PayloadCodec | WorkflowSerializationContext | [WorkflowHandle.cs:84-87](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Client/WorkflowHandle.cs#L84-L87) | [RootWorkflowClientInvoker.java:324-327](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/client/RootWorkflowClientInvoker.java#L324-L327) | Via context-aware handle | +| Deserialize | DataConverter | WorkflowSerializationContext | [WorkflowHandle.cs:129-130](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Client/WorkflowHandle.cs#L129-L130) | [RootWorkflowClientInvoker.java:318-322](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/client/RootWorkflowClientInvoker.java#L318-L322) | [client.py:1715-1718](https://github.com/temporalio/sdk-python/blob/main/temporalio/client.py#L1715-L1718) | ## Activity Context @@ -27,20 +27,20 @@ **Rule:** All operations use `ActivitySerializationContext` with activity type, task queue, and is_local flag. -| Operation | Stage | Context Type | .NET | Java | -|-----------|-------|--------------|------|------| -| **1. Workflow Schedules Activity** | | | | | -| Serialize | DataConverter | ActivitySerializationContext | [WorkflowInstance.cs:2084-2093](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowInstance.cs#L2084-L2093) | [SyncWorkflowContext.java:257-267](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/sync/SyncWorkflowContext.java#L257-L267) | -| Encode | PayloadCodec | ActivitySerializationContext | [WorkflowCodecHelper.cs:243-253](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowCodecHelper.cs#L243-L253) | [SyncWorkflowContext.java:268-270](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/sync/SyncWorkflowContext.java#L268-L270) | -| **2. Activity Receives Input** | | | | | -| Decode | PayloadCodec | ActivitySerializationContext | Worker-side (not in workflow) | Worker-side (not in workflow) | -| Deserialize | DataConverter | ActivitySerializationContext | Worker-side (not in workflow) | Worker-side (not in workflow) | -| **3. Activity Returns Result** | | | | | -| Serialize | DataConverter | ActivitySerializationContext | Worker-side (not in workflow) | Worker-side (not in workflow) | -| Encode | PayloadCodec | ActivitySerializationContext | Worker-side (not in workflow) | Worker-side (not in workflow) | -| **4. Workflow Receives Result** | | | | | -| Decode | PayloadCodec | ActivitySerializationContext | [WorkflowCodecHelper.cs:68-76](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowCodecHelper.cs#L68-L76) | [SyncWorkflowContext.java:273-274](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/sync/SyncWorkflowContext.java#L273-L274) | -| Deserialize | DataConverter | ActivitySerializationContext | [WorkflowInstance.cs:2720](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowInstance.cs#L2720) | [SyncWorkflowContext.java:286-288](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/sync/SyncWorkflowContext.java#L286-L288) | +| Operation | Stage | Context Type | .NET | Java | Python | +|-----------|-------|--------------|------|------|--------| +| **1. Workflow Schedules Activity** | | | | | | +| Serialize | DataConverter | ActivitySerializationContext | [WorkflowInstance.cs:2084-2093](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowInstance.cs#L2084-L2093) | [SyncWorkflowContext.java:257-267](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/sync/SyncWorkflowContext.java#L257-L267) | [_workflow_instance.py:2968-2972](https://github.com/temporalio/sdk-python/blob/main/temporalio/worker/_workflow_instance.py#L2968-L2972) | +| Encode | PayloadCodec | ActivitySerializationContext | [WorkflowCodecHelper.cs:243-253](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowCodecHelper.cs#L243-L253) | [SyncWorkflowContext.java:268-270](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/sync/SyncWorkflowContext.java#L268-L270) | Via command-aware codec | +| **2. Activity Receives Input** | | | | | | +| Decode | PayloadCodec | ActivitySerializationContext | Worker-side (not in workflow) | Worker-side (not in workflow) | Worker-side (not in workflow) | +| Deserialize | DataConverter | ActivitySerializationContext | Worker-side (not in workflow) | Worker-side (not in workflow) | Worker-side (not in workflow) | +| **3. Activity Returns Result** | | | | | | +| Serialize | DataConverter | ActivitySerializationContext | Worker-side (not in workflow) | Worker-side (not in workflow) | Worker-side (not in workflow) | +| Encode | PayloadCodec | ActivitySerializationContext | Worker-side (not in workflow) | Worker-side (not in workflow) | Worker-side (not in workflow) | +| **4. Workflow Receives Result** | | | | | | +| Decode | PayloadCodec | ActivitySerializationContext | [WorkflowCodecHelper.cs:68-76](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowCodecHelper.cs#L68-L76) | [SyncWorkflowContext.java:273-274](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/sync/SyncWorkflowContext.java#L273-L274) | Via command-aware codec | +| Deserialize | DataConverter | ActivitySerializationContext | [WorkflowInstance.cs:2720](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowInstance.cs#L2720) | [SyncWorkflowContext.java:286-288](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/sync/SyncWorkflowContext.java#L286-L288) | Via context-aware converter | ## Child Workflow Context @@ -48,20 +48,20 @@ **Rule:** Child operations use `WorkflowSerializationContext` with the child workflow's ID. -| Operation | Stage | Context Type | .NET | Java | -|-----------|-------|--------------|------|------| -| **1. Workflow Starts Child** | | | | | -| Serialize | DataConverter | WorkflowSerializationContext (child ID) | [WorkflowInstance.cs:2319-2326](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowInstance.cs#L2319-L2326) | [SyncWorkflowContext.java:687-690](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/sync/SyncWorkflowContext.java#L687-L690) | -| Encode | PayloadCodec | WorkflowSerializationContext (child ID) | [WorkflowCodecHelper.cs:301-310](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowCodecHelper.cs#L301-L310) | [SyncWorkflowContext.java:690](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/sync/SyncWorkflowContext.java#L690) | -| **2. Child Receives Input** | | | | | -| Decode | PayloadCodec | WorkflowSerializationContext (child ID) | [WorkflowCodecHelper.cs:125-126](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowCodecHelper.cs#L125-L126) | [ReplayWorkflowTaskHandler.java:160-162](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/replay/ReplayWorkflowTaskHandler.java#L160-L162) | -| Deserialize | DataConverter | WorkflowSerializationContext (child ID) | [WorkflowInstance.cs:395-397](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowInstance.cs#L395-L397) | [SyncWorkflow.java:75-77](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/sync/SyncWorkflow.java#L75-L77) | -| **3. Child Returns Result** | | | | | -| Serialize | DataConverter | WorkflowSerializationContext (child ID) | [WorkflowInstance.cs:1182-1184](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowInstance.cs#L1182-L1184) | [POJOWorkflowImplementationFactory.java:290-292](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/sync/POJOWorkflowImplementationFactory.java#L290-L292) | -| Encode | PayloadCodec | WorkflowSerializationContext (child ID) | [WorkflowCodecHelper.cs:217-220](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowCodecHelper.cs#L217-L220) | [WorkflowWorker.java:702-704](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/worker/WorkflowWorker.java#L702-L704) | -| **4. Workflow Receives Child Result** | | | | | -| Decode | PayloadCodec | WorkflowSerializationContext (child ID) | [WorkflowCodecHelper.cs:78-85](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowCodecHelper.cs#L78-L85) | [SyncWorkflowContext.java:1245-1248](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/sync/SyncWorkflowContext.java#L1245-L1248) | -| Deserialize | DataConverter | WorkflowSerializationContext (child ID) | [WorkflowInstance.cs:2808](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowInstance.cs#L2808) | [SyncWorkflowContext.java:1245-1248](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/sync/SyncWorkflowContext.java#L1245-L1248) | +| Operation | Stage | Context Type | .NET | Java | Python | +|-----------|-------|--------------|------|------|--------| +| **1. Workflow Starts Child** | | | | | | +| Serialize | DataConverter | WorkflowSerializationContext (child ID) | [WorkflowInstance.cs:2319-2326](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowInstance.cs#L2319-L2326) | [SyncWorkflowContext.java:687-690](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/sync/SyncWorkflowContext.java#L687-L690) | [_workflow_instance.py:3124-3128](https://github.com/temporalio/sdk-python/blob/main/temporalio/worker/_workflow_instance.py#L3124-L3128) | +| Encode | PayloadCodec | WorkflowSerializationContext (child ID) | [WorkflowCodecHelper.cs:301-310](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowCodecHelper.cs#L301-L310) | [SyncWorkflowContext.java:690](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/sync/SyncWorkflowContext.java#L690) | Via command-aware codec | +| **2. Child Receives Input** | | | | | | +| Decode | PayloadCodec | WorkflowSerializationContext (child ID) | [WorkflowCodecHelper.cs:125-126](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowCodecHelper.cs#L125-L126) | [ReplayWorkflowTaskHandler.java:160-162](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/replay/ReplayWorkflowTaskHandler.java#L160-L162) | [_workflow.py:289-293](https://github.com/temporalio/sdk-python/blob/main/temporalio/worker/_workflow.py#L289-L293) | +| Deserialize | DataConverter | WorkflowSerializationContext (child ID) | [WorkflowInstance.cs:395-397](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowInstance.cs#L395-L397) | [SyncWorkflow.java:75-77](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/sync/SyncWorkflow.java#L75-L77) | Via context-aware converter | +| **3. Child Returns Result** | | | | | | +| Serialize | DataConverter | WorkflowSerializationContext (child ID) | [WorkflowInstance.cs:1182-1184](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowInstance.cs#L1182-L1184) | [POJOWorkflowImplementationFactory.java:290-292](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/sync/POJOWorkflowImplementationFactory.java#L290-L292) | Via context-aware converter | +| Encode | PayloadCodec | WorkflowSerializationContext (child ID) | [WorkflowCodecHelper.cs:217-220](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowCodecHelper.cs#L217-L220) | [WorkflowWorker.java:702-704](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/worker/WorkflowWorker.java#L702-L704) | [_workflow.py:363-367](https://github.com/temporalio/sdk-python/blob/main/temporalio/worker/_workflow.py#L363-L367) | +| **4. Workflow Receives Child Result** | | | | | | +| Decode | PayloadCodec | WorkflowSerializationContext (child ID) | [WorkflowCodecHelper.cs:78-85](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowCodecHelper.cs#L78-L85) | [SyncWorkflowContext.java:1245-1248](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/sync/SyncWorkflowContext.java#L1245-L1248) | Via command-aware codec | +| Deserialize | DataConverter | WorkflowSerializationContext (child ID) | [WorkflowInstance.cs:2808](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowInstance.cs#L2808) | [SyncWorkflowContext.java:1245-1248](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/sync/SyncWorkflowContext.java#L1245-L1248) | Via context-aware converter | ## Nexus Operation Context @@ -69,20 +69,20 @@ **Rule:** Nexus operations have **no serialization context** (null/none). -| Operation | Stage | Context Type | .NET | Java | -|-----------|-------|--------------|------|------| -| **1. Workflow Schedules Nexus Op** | | | | | -| Serialize | DataConverter | None | [WorkflowInstance.cs:2525](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowInstance.cs#L2525) | [SyncWorkflowContext.java:790](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/sync/SyncWorkflowContext.java#L790) | -| Encode | PayloadCodec | None | [WorkflowCodecHelper.cs:355-356](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowCodecHelper.cs#L355-L356) | [SyncWorkflowContext.java:791-794](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/sync/SyncWorkflowContext.java#L791-L794) | -| **2. Nexus Operation Receives Input** | | | | | -| Decode | PayloadCodec | None | N/A (handler-side) | N/A (handler-side) | -| Deserialize | DataConverter | None | N/A (handler-side) | N/A (handler-side) | -| **3. Nexus Operation Returns Result** | | | | | -| Serialize | DataConverter | None | N/A (handler-side) | N/A (handler-side) | -| Encode | PayloadCodec | None | N/A (handler-side) | N/A (handler-side) | -| **4. Workflow Receives Result** | | | | | -| Decode | PayloadCodec | None | [WorkflowCodecHelper.cs:112-113](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowCodecHelper.cs#L112-L113) | N/A (codec not applied) | -| Deserialize | DataConverter | None | [WorkflowInstance.cs:2954](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowInstance.cs#L2954) | [SyncWorkflowContext.java:855-856](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/sync/SyncWorkflowContext.java#L855-L856) | +| Operation | Stage | Context Type | .NET | Java | Python | +|-----------|-------|--------------|------|------|--------| +| **1. Workflow Schedules Nexus Op** | | | | | | +| Serialize | DataConverter | None | [WorkflowInstance.cs:2525](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowInstance.cs#L2525) | [SyncWorkflowContext.java:790](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/sync/SyncWorkflowContext.java#L790) | Via converter (no context) | +| Encode | PayloadCodec | None | [WorkflowCodecHelper.cs:355-356](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowCodecHelper.cs#L355-L356) | [SyncWorkflowContext.java:791-794](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/sync/SyncWorkflowContext.java#L791-L794) | Via codec (no context) | +| **2. Nexus Operation Receives Input** | | | | | | +| Decode | PayloadCodec | None | N/A (handler-side) | N/A (handler-side) | N/A (handler-side) | +| Deserialize | DataConverter | None | N/A (handler-side) | N/A (handler-side) | N/A (handler-side) | +| **3. Nexus Operation Returns Result** | | | | | | +| Serialize | DataConverter | None | N/A (handler-side) | N/A (handler-side) | N/A (handler-side) | +| Encode | PayloadCodec | None | N/A (handler-side) | N/A (handler-side) | N/A (handler-side) | +| **4. Workflow Receives Result** | | | | | | +| Decode | PayloadCodec | None | [WorkflowCodecHelper.cs:112-113](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowCodecHelper.cs#L112-L113) | N/A (codec not applied) | Via codec (no context) | +| Deserialize | DataConverter | None | [WorkflowInstance.cs:2954](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowInstance.cs#L2954) | [SyncWorkflowContext.java:855-856](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/sync/SyncWorkflowContext.java#L855-L856) | Via converter (no context) | ## Memo and Search Attribute Context @@ -90,20 +90,20 @@ **Rule:** Memos always use `WorkflowSerializationContext` with the workflow's ID when set or accessed. -| Operation | Context Type | .NET | Java | -|-----------|--------------|------|------| -| **Client sets memo on start** | WorkflowSerializationContext | [TemporalClient.Workflow.cs:827](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Client/TemporalClient.Workflow.cs#L827) | [RootWorkflowClientInvoker.java:292-294](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/client/RootWorkflowClientInvoker.java#L292-L294) | -| **Workflow upserts memo** | WorkflowSerializationContext | [WorkflowInstance.cs:472](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowInstance.cs#L472) | [SyncWorkflowContext.java:1416](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/sync/SyncWorkflowContext.java#L1416) | -| **Child workflow memo** | WorkflowSerializationContext (child ID) | [WorkflowInstance.cs:2359-2360](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowInstance.cs#L2359-L2360) | [SyncWorkflowContext.java:693-700](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/sync/SyncWorkflowContext.java#L693-L700) | -| **Schedule sets memo** | WorkflowSerializationContext | [ScheduleActionStartWorkflow.cs:199](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Client/Schedules/ScheduleActionStartWorkflow.cs#L199) | [ScheduleProtoUtil.java:134](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/client/ScheduleProtoUtil.java#L134) | +| Operation | Context Type | .NET | Java | Python | +|-----------|--------------|------|------|--------| +| **Client sets memo on start** | WorkflowSerializationContext | [TemporalClient.Workflow.cs:827](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Client/TemporalClient.Workflow.cs#L827) | [RootWorkflowClientInvoker.java:292-294](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/client/RootWorkflowClientInvoker.java#L292-L294) | [client.py:6027-6028](https://github.com/temporalio/sdk-python/blob/main/temporalio/client.py#L6027-L6028) | +| **Workflow upserts memo** | WorkflowSerializationContext | [WorkflowInstance.cs:472](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowInstance.cs#L472) | [SyncWorkflowContext.java:1416](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/sync/SyncWorkflowContext.java#L1416) | Via context-aware converter | +| **Child workflow memo** | WorkflowSerializationContext (child ID) | [WorkflowInstance.cs:2359-2360](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowInstance.cs#L2359-L2360) | [SyncWorkflowContext.java:693-700](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/sync/SyncWorkflowContext.java#L693-L700) | Via child context converter | +| **Schedule sets memo** | WorkflowSerializationContext | [ScheduleActionStartWorkflow.cs:199](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Client/Schedules/ScheduleActionStartWorkflow.cs#L199) | [ScheduleProtoUtil.java:134](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/client/ScheduleProtoUtil.java#L134) | [client.py:4220-4229](https://github.com/temporalio/sdk-python/blob/main/temporalio/client.py#L4220-L4229) | ### Search Attributes **Rule:** Search attributes do NOT use serialization context - they use specialized converters for indexing. -| Operation | Context Type | .NET | Java | -|-----------|--------------|------|------| -| **All operations** | None (direct proto conversion) | Uses `ToProto()` | Uses `toSearchAttributes()` | +| Operation | Context Type | .NET | Java | Python | +|-----------|--------------|------|------|--------| +| **All operations** | None (direct proto conversion) | Uses `ToProto()` | Uses `toSearchAttributes()` | Uses `encode_search_attributes()` | ## User-Accessible Data Converter @@ -168,3 +168,18 @@ 7. **User-exposed converters**: Have appropriate context pre-applied 8. **Async activity completion**: Use activity context with available information 9. **Heartbeating**: Use activity context with full info during execution, partial for async + +## Important Note About .NET + +⚠️ **Current .NET Bug** ([temporalio/sdk-dotnet#523](https://github.com/temporalio/sdk-dotnet/issues/523) - **OPEN**) + +The .NET SDK currently has a bug where it incorrectly applies `WorkflowSerializationContext` to **all** data conversion operations within a workflow context, including: +- Activities (should use `ActivitySerializationContext`) +- Nexus operations (should use no context) + +**This document shows the intended/correct behavior**, which is: +- How Java currently works ✅ +- How Python should work after alignment ✅ +- How .NET **should** work (but doesn't yet) ⚠️ + +The tables above reflect the desired state where each SDK applies context selectively based on the operation type. From e63afbe8a901e23f33394bd3e5ff31516e1c661a Mon Sep 17 00:00:00 2001 From: Dan Davison Date: Wed, 24 Sep 2025 09:36:14 -0400 Subject: [PATCH 23/81] Add Python column links to all requirements.md tables Added comprehensive Python SDK references for all serialization context operations: 1. **Simple Workflow Context**: Added links to Python code for client start/result and workflow input/output operations 2. **Activity Context**: Added links to activity worker code where payloads are encoded/decoded 3. **Child Workflow Context**: Added links showing how child workflow contexts are applied 4. **Nexus Operations**: Marked as 'N/A (not yet implemented)' since nexus operations are not yet in Python SDK 5. **Memos and Search Attributes**: Added links to relevant converter and instance code 6. **Command-Aware Codec**: Many operations use the _CommandAwarePayloadCodec which dynamically applies context based on the current command sequence 7. **Clarified .NET Bug Status**: Added section explaining that .NET issue #523 is still OPEN and the document shows intended (not current) .NET behavior All Python links point to specific line ranges in the main branch for: - client.py (client operations) - _workflow.py (workflow worker operations) - _workflow_instance.py (workflow instance operations) - _activity.py (activity worker operations) - converter.py (search attribute encoding) --- requirements.md | 44 ++++++++++++++++++++++---------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/requirements.md b/requirements.md index ccce0c4ae..4addc60e2 100644 --- a/requirements.md +++ b/requirements.md @@ -13,12 +13,12 @@ | Encode | PayloadCodec | WorkflowSerializationContext | [TemporalClient.Workflow.cs:699-700](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Client/TemporalClient.Workflow.cs#L699-L700) | [RootWorkflowClientInvoker.java:69](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/client/RootWorkflowClientInvoker.java#L69) | [client.py:6005](https://github.com/temporalio/sdk-python/blob/main/temporalio/client.py#L6005) | | **2. Workflow Receives Input** | | | | | | | Decode | PayloadCodec | WorkflowSerializationContext | [WorkflowCodecHelper.cs:125-126](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowCodecHelper.cs#L125-L126) | [ReplayWorkflowTaskHandler.java:160-162](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/replay/ReplayWorkflowTaskHandler.java#L160-L162) | [_workflow.py:289-293](https://github.com/temporalio/sdk-python/blob/main/temporalio/worker/_workflow.py#L289-L293) | -| Deserialize | DataConverter | WorkflowSerializationContext | [WorkflowInstance.cs:395-397](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowInstance.cs#L395-L397) | [SyncWorkflow.java:75-77](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/sync/SyncWorkflow.java#L75-L77) | Via context-aware converter | +| Deserialize | DataConverter | WorkflowSerializationContext | [WorkflowInstance.cs:395-397](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowInstance.cs#L395-L397) | [SyncWorkflow.java:75-77](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/sync/SyncWorkflow.java#L75-L77) | [_workflow.py:278-283](https://github.com/temporalio/sdk-python/blob/main/temporalio/worker/_workflow.py#L278-L283) | | **3. Workflow Returns Result** | | | | | | -| Serialize | DataConverter | WorkflowSerializationContext | [WorkflowInstance.cs:1182-1184](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowInstance.cs#L1182-L1184) | [POJOWorkflowImplementationFactory.java:290-292](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/sync/POJOWorkflowImplementationFactory.java#L290-L292) | Via context-aware converter | +| Serialize | DataConverter | WorkflowSerializationContext | [WorkflowInstance.cs:1182-1184](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowInstance.cs#L1182-L1184) | [POJOWorkflowImplementationFactory.java:290-292](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/sync/POJOWorkflowImplementationFactory.java#L290-L292) | [_workflow.py:340-346](https://github.com/temporalio/sdk-python/blob/main/temporalio/worker/_workflow.py#L340-L346) | | Encode | PayloadCodec | WorkflowSerializationContext | [WorkflowCodecHelper.cs:217-220](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowCodecHelper.cs#L217-L220) | [WorkflowWorker.java:702-704](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/worker/WorkflowWorker.java#L702-L704) | [_workflow.py:363-367](https://github.com/temporalio/sdk-python/blob/main/temporalio/worker/_workflow.py#L363-L367) | | **4. Client Receives Result** | | | | | | -| Decode | PayloadCodec | WorkflowSerializationContext | [WorkflowHandle.cs:84-87](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Client/WorkflowHandle.cs#L84-L87) | [RootWorkflowClientInvoker.java:324-327](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/client/RootWorkflowClientInvoker.java#L324-L327) | Via context-aware handle | +| Decode | PayloadCodec | WorkflowSerializationContext | [WorkflowHandle.cs:84-87](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Client/WorkflowHandle.cs#L84-L87) | [RootWorkflowClientInvoker.java:324-327](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/client/RootWorkflowClientInvoker.java#L324-L327) | [client.py:1601-1605](https://github.com/temporalio/sdk-python/blob/main/temporalio/client.py#L1601-L1605) | | Deserialize | DataConverter | WorkflowSerializationContext | [WorkflowHandle.cs:129-130](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Client/WorkflowHandle.cs#L129-L130) | [RootWorkflowClientInvoker.java:318-322](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/client/RootWorkflowClientInvoker.java#L318-L322) | [client.py:1715-1718](https://github.com/temporalio/sdk-python/blob/main/temporalio/client.py#L1715-L1718) | ## Activity Context @@ -31,16 +31,16 @@ |-----------|-------|--------------|------|------|--------| | **1. Workflow Schedules Activity** | | | | | | | Serialize | DataConverter | ActivitySerializationContext | [WorkflowInstance.cs:2084-2093](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowInstance.cs#L2084-L2093) | [SyncWorkflowContext.java:257-267](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/sync/SyncWorkflowContext.java#L257-L267) | [_workflow_instance.py:2968-2972](https://github.com/temporalio/sdk-python/blob/main/temporalio/worker/_workflow_instance.py#L2968-L2972) | -| Encode | PayloadCodec | ActivitySerializationContext | [WorkflowCodecHelper.cs:243-253](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowCodecHelper.cs#L243-L253) | [SyncWorkflowContext.java:268-270](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/sync/SyncWorkflowContext.java#L268-L270) | Via command-aware codec | +| Encode | PayloadCodec | ActivitySerializationContext | [WorkflowCodecHelper.cs:243-253](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowCodecHelper.cs#L243-L253) | [SyncWorkflowContext.java:268-270](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/sync/SyncWorkflowContext.java#L268-L270) | [_workflow.py:716-733](https://github.com/temporalio/sdk-python/blob/main/temporalio/worker/_workflow.py#L716-L733) | | **2. Activity Receives Input** | | | | | | -| Decode | PayloadCodec | ActivitySerializationContext | Worker-side (not in workflow) | Worker-side (not in workflow) | Worker-side (not in workflow) | -| Deserialize | DataConverter | ActivitySerializationContext | Worker-side (not in workflow) | Worker-side (not in workflow) | Worker-side (not in workflow) | +| Decode | PayloadCodec | ActivitySerializationContext | Worker-side (not in workflow) | Worker-side (not in workflow) | [_activity.py:317-318](https://github.com/temporalio/sdk-python/blob/main/temporalio/worker/_activity.py#L317-L318) | +| Deserialize | DataConverter | ActivitySerializationContext | Worker-side (not in workflow) | Worker-side (not in workflow) | [_activity.py:526-530](https://github.com/temporalio/sdk-python/blob/main/temporalio/worker/_activity.py#L526-L530) | | **3. Activity Returns Result** | | | | | | -| Serialize | DataConverter | ActivitySerializationContext | Worker-side (not in workflow) | Worker-side (not in workflow) | Worker-side (not in workflow) | -| Encode | PayloadCodec | ActivitySerializationContext | Worker-side (not in workflow) | Worker-side (not in workflow) | Worker-side (not in workflow) | +| Serialize | DataConverter | ActivitySerializationContext | Worker-side (not in workflow) | Worker-side (not in workflow) | [_activity.py:322-323](https://github.com/temporalio/sdk-python/blob/main/temporalio/worker/_activity.py#L322-L323) | +| Encode | PayloadCodec | ActivitySerializationContext | Worker-side (not in workflow) | Worker-side (not in workflow) | [_activity.py:317-318](https://github.com/temporalio/sdk-python/blob/main/temporalio/worker/_activity.py#L317-L318) | | **4. Workflow Receives Result** | | | | | | -| Decode | PayloadCodec | ActivitySerializationContext | [WorkflowCodecHelper.cs:68-76](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowCodecHelper.cs#L68-L76) | [SyncWorkflowContext.java:273-274](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/sync/SyncWorkflowContext.java#L273-L274) | Via command-aware codec | -| Deserialize | DataConverter | ActivitySerializationContext | [WorkflowInstance.cs:2720](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowInstance.cs#L2720) | [SyncWorkflowContext.java:286-288](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/sync/SyncWorkflowContext.java#L286-L288) | Via context-aware converter | +| Decode | PayloadCodec | ActivitySerializationContext | [WorkflowCodecHelper.cs:68-76](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowCodecHelper.cs#L68-L76) | [SyncWorkflowContext.java:273-274](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/sync/SyncWorkflowContext.java#L273-L274) | [_workflow.py:716-745](https://github.com/temporalio/sdk-python/blob/main/temporalio/worker/_workflow.py#L716-L745) | +| Deserialize | DataConverter | ActivitySerializationContext | [WorkflowInstance.cs:2720](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowInstance.cs#L2720) | [SyncWorkflowContext.java:286-288](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/sync/SyncWorkflowContext.java#L286-L288) | [_workflow_instance.py:804-810](https://github.com/temporalio/sdk-python/blob/main/temporalio/worker/_workflow_instance.py#L804-L810) | ## Child Workflow Context @@ -52,16 +52,16 @@ |-----------|-------|--------------|------|------|--------| | **1. Workflow Starts Child** | | | | | | | Serialize | DataConverter | WorkflowSerializationContext (child ID) | [WorkflowInstance.cs:2319-2326](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowInstance.cs#L2319-L2326) | [SyncWorkflowContext.java:687-690](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/sync/SyncWorkflowContext.java#L687-L690) | [_workflow_instance.py:3124-3128](https://github.com/temporalio/sdk-python/blob/main/temporalio/worker/_workflow_instance.py#L3124-L3128) | -| Encode | PayloadCodec | WorkflowSerializationContext (child ID) | [WorkflowCodecHelper.cs:301-310](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowCodecHelper.cs#L301-L310) | [SyncWorkflowContext.java:690](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/sync/SyncWorkflowContext.java#L690) | Via command-aware codec | +| Encode | PayloadCodec | WorkflowSerializationContext (child ID) | [WorkflowCodecHelper.cs:301-310](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowCodecHelper.cs#L301-L310) | [SyncWorkflowContext.java:690](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/sync/SyncWorkflowContext.java#L690) | [_workflow.py:716-745](https://github.com/temporalio/sdk-python/blob/main/temporalio/worker/_workflow.py#L716-L745) | | **2. Child Receives Input** | | | | | | | Decode | PayloadCodec | WorkflowSerializationContext (child ID) | [WorkflowCodecHelper.cs:125-126](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowCodecHelper.cs#L125-L126) | [ReplayWorkflowTaskHandler.java:160-162](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/replay/ReplayWorkflowTaskHandler.java#L160-L162) | [_workflow.py:289-293](https://github.com/temporalio/sdk-python/blob/main/temporalio/worker/_workflow.py#L289-L293) | -| Deserialize | DataConverter | WorkflowSerializationContext (child ID) | [WorkflowInstance.cs:395-397](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowInstance.cs#L395-L397) | [SyncWorkflow.java:75-77](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/sync/SyncWorkflow.java#L75-L77) | Via context-aware converter | +| Deserialize | DataConverter | WorkflowSerializationContext (child ID) | [WorkflowInstance.cs:395-397](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowInstance.cs#L395-L397) | [SyncWorkflow.java:75-77](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/sync/SyncWorkflow.java#L75-L77) | [_workflow.py:278-283](https://github.com/temporalio/sdk-python/blob/main/temporalio/worker/_workflow.py#L278-L283) | | **3. Child Returns Result** | | | | | | -| Serialize | DataConverter | WorkflowSerializationContext (child ID) | [WorkflowInstance.cs:1182-1184](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowInstance.cs#L1182-L1184) | [POJOWorkflowImplementationFactory.java:290-292](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/sync/POJOWorkflowImplementationFactory.java#L290-L292) | Via context-aware converter | +| Serialize | DataConverter | WorkflowSerializationContext (child ID) | [WorkflowInstance.cs:1182-1184](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowInstance.cs#L1182-L1184) | [POJOWorkflowImplementationFactory.java:290-292](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/sync/POJOWorkflowImplementationFactory.java#L290-L292) | [_workflow.py:340-346](https://github.com/temporalio/sdk-python/blob/main/temporalio/worker/_workflow.py#L340-L346) | | Encode | PayloadCodec | WorkflowSerializationContext (child ID) | [WorkflowCodecHelper.cs:217-220](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowCodecHelper.cs#L217-L220) | [WorkflowWorker.java:702-704](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/worker/WorkflowWorker.java#L702-L704) | [_workflow.py:363-367](https://github.com/temporalio/sdk-python/blob/main/temporalio/worker/_workflow.py#L363-L367) | | **4. Workflow Receives Child Result** | | | | | | -| Decode | PayloadCodec | WorkflowSerializationContext (child ID) | [WorkflowCodecHelper.cs:78-85](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowCodecHelper.cs#L78-L85) | [SyncWorkflowContext.java:1245-1248](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/sync/SyncWorkflowContext.java#L1245-L1248) | Via command-aware codec | -| Deserialize | DataConverter | WorkflowSerializationContext (child ID) | [WorkflowInstance.cs:2808](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowInstance.cs#L2808) | [SyncWorkflowContext.java:1245-1248](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/sync/SyncWorkflowContext.java#L1245-L1248) | Via context-aware converter | +| Decode | PayloadCodec | WorkflowSerializationContext (child ID) | [WorkflowCodecHelper.cs:78-85](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowCodecHelper.cs#L78-L85) | [SyncWorkflowContext.java:1245-1248](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/sync/SyncWorkflowContext.java#L1245-L1248) | [_workflow.py:716-745](https://github.com/temporalio/sdk-python/blob/main/temporalio/worker/_workflow.py#L716-L745) | +| Deserialize | DataConverter | WorkflowSerializationContext (child ID) | [WorkflowInstance.cs:2808](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowInstance.cs#L2808) | [SyncWorkflowContext.java:1245-1248](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/sync/SyncWorkflowContext.java#L1245-L1248) | [_workflow_instance.py:842-858](https://github.com/temporalio/sdk-python/blob/main/temporalio/worker/_workflow_instance.py#L842-L858) | ## Nexus Operation Context @@ -72,8 +72,8 @@ | Operation | Stage | Context Type | .NET | Java | Python | |-----------|-------|--------------|------|------|--------| | **1. Workflow Schedules Nexus Op** | | | | | | -| Serialize | DataConverter | None | [WorkflowInstance.cs:2525](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowInstance.cs#L2525) | [SyncWorkflowContext.java:790](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/sync/SyncWorkflowContext.java#L790) | Via converter (no context) | -| Encode | PayloadCodec | None | [WorkflowCodecHelper.cs:355-356](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowCodecHelper.cs#L355-L356) | [SyncWorkflowContext.java:791-794](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/sync/SyncWorkflowContext.java#L791-L794) | Via codec (no context) | +| Serialize | DataConverter | None | [WorkflowInstance.cs:2525](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowInstance.cs#L2525) | [SyncWorkflowContext.java:790](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/sync/SyncWorkflowContext.java#L790) | N/A (not yet implemented) | +| Encode | PayloadCodec | None | [WorkflowCodecHelper.cs:355-356](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowCodecHelper.cs#L355-L356) | [SyncWorkflowContext.java:791-794](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/sync/SyncWorkflowContext.java#L791-L794) | N/A (not yet implemented) | | **2. Nexus Operation Receives Input** | | | | | | | Decode | PayloadCodec | None | N/A (handler-side) | N/A (handler-side) | N/A (handler-side) | | Deserialize | DataConverter | None | N/A (handler-side) | N/A (handler-side) | N/A (handler-side) | @@ -81,8 +81,8 @@ | Serialize | DataConverter | None | N/A (handler-side) | N/A (handler-side) | N/A (handler-side) | | Encode | PayloadCodec | None | N/A (handler-side) | N/A (handler-side) | N/A (handler-side) | | **4. Workflow Receives Result** | | | | | | -| Decode | PayloadCodec | None | [WorkflowCodecHelper.cs:112-113](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowCodecHelper.cs#L112-L113) | N/A (codec not applied) | Via codec (no context) | -| Deserialize | DataConverter | None | [WorkflowInstance.cs:2954](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowInstance.cs#L2954) | [SyncWorkflowContext.java:855-856](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/sync/SyncWorkflowContext.java#L855-L856) | Via converter (no context) | +| Decode | PayloadCodec | None | [WorkflowCodecHelper.cs:112-113](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowCodecHelper.cs#L112-L113) | N/A (codec not applied) | N/A (not yet implemented) | +| Deserialize | DataConverter | None | [WorkflowInstance.cs:2954](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowInstance.cs#L2954) | [SyncWorkflowContext.java:855-856](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/sync/SyncWorkflowContext.java#L855-L856) | N/A (not yet implemented) | ## Memo and Search Attribute Context @@ -93,8 +93,8 @@ | Operation | Context Type | .NET | Java | Python | |-----------|--------------|------|------|--------| | **Client sets memo on start** | WorkflowSerializationContext | [TemporalClient.Workflow.cs:827](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Client/TemporalClient.Workflow.cs#L827) | [RootWorkflowClientInvoker.java:292-294](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/client/RootWorkflowClientInvoker.java#L292-L294) | [client.py:6027-6028](https://github.com/temporalio/sdk-python/blob/main/temporalio/client.py#L6027-L6028) | -| **Workflow upserts memo** | WorkflowSerializationContext | [WorkflowInstance.cs:472](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowInstance.cs#L472) | [SyncWorkflowContext.java:1416](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/sync/SyncWorkflowContext.java#L1416) | Via context-aware converter | -| **Child workflow memo** | WorkflowSerializationContext (child ID) | [WorkflowInstance.cs:2359-2360](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowInstance.cs#L2359-L2360) | [SyncWorkflowContext.java:693-700](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/sync/SyncWorkflowContext.java#L693-L700) | Via child context converter | +| **Workflow upserts memo** | WorkflowSerializationContext | [WorkflowInstance.cs:472](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowInstance.cs#L472) | [SyncWorkflowContext.java:1416](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/sync/SyncWorkflowContext.java#L1416) | [_workflow_instance.py:908-912](https://github.com/temporalio/sdk-python/blob/main/temporalio/worker/_workflow_instance.py#L908-L912) | +| **Child workflow memo** | WorkflowSerializationContext (child ID) | [WorkflowInstance.cs:2359-2360](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowInstance.cs#L2359-L2360) | [SyncWorkflowContext.java:693-700](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/sync/SyncWorkflowContext.java#L693-L700) | [_workflow_instance.py:3156-3159](https://github.com/temporalio/sdk-python/blob/main/temporalio/worker/_workflow_instance.py#L3156-L3159) | | **Schedule sets memo** | WorkflowSerializationContext | [ScheduleActionStartWorkflow.cs:199](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Client/Schedules/ScheduleActionStartWorkflow.cs#L199) | [ScheduleProtoUtil.java:134](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/client/ScheduleProtoUtil.java#L134) | [client.py:4220-4229](https://github.com/temporalio/sdk-python/blob/main/temporalio/client.py#L4220-L4229) | ### Search Attributes @@ -103,7 +103,7 @@ | Operation | Context Type | .NET | Java | Python | |-----------|--------------|------|------|--------| -| **All operations** | None (direct proto conversion) | Uses `ToProto()` | Uses `toSearchAttributes()` | Uses `encode_search_attributes()` | +| **All operations** | None (direct proto conversion) | Uses `ToProto()` | Uses `toSearchAttributes()` | [converter.py:1358-1363](https://github.com/temporalio/sdk-python/blob/main/temporalio/converter.py#L1358-L1363) | ## User-Accessible Data Converter From 2de5bfd9356cb06c7c37341b22ad60bc97e447a8 Mon Sep 17 00:00:00 2001 From: Dan Davison Date: Wed, 24 Sep 2025 09:46:33 -0400 Subject: [PATCH 24/81] Do not "stack" payload/failure conversion contexts --- temporalio/worker/_workflow_instance.py | 36 +++++++++++-------------- 1 file changed, 15 insertions(+), 21 deletions(-) diff --git a/temporalio/worker/_workflow_instance.py b/temporalio/worker/_workflow_instance.py index ef7dd2acd..9c092fd2c 100644 --- a/temporalio/worker/_workflow_instance.py +++ b/temporalio/worker/_workflow_instance.py @@ -236,9 +236,7 @@ def __init__(self, det: WorkflowInstanceDetails) -> None: temporalio.converter.WorkflowSerializationContext( namespace=det.info.namespace, workflow_id=det.info.workflow_id, - ), - self._context_free_payload_converter, - self._context_free_failure_converter, + ) ) ) @@ -918,9 +916,12 @@ def _apply_resolve_nexus_operation_start( raise RuntimeError( f"Failed to find nexus operation handle for job sequence number {job.seq}" ) - # We not set a serialization context for nexus operations on the caller side because it is - # not possible to do so on the handler side. - payload_converter, failure_converter = self._converters_with_context(None) + # We don't set a serialization context for nexus operations on the caller side because it's + # not possible to set a matching context on the handler side. + payload_converter, failure_converter = ( + self._context_free_payload_converter, + self._context_free_failure_converter, + ) if job.HasField("operation_token"): # The nexus operation started asynchronously. A `ResolveNexusOperation` job @@ -957,7 +958,7 @@ def _apply_resolve_nexus_operation( # completed / failed, but it has already been resolved. return - # We not set a serialization context for nexus operations on the caller side because it is + # We don't set a serialization context for nexus operations on the caller side because it is # not possible to do so on the handler side. payload_converter, failure_converter = ( self._context_free_payload_converter, @@ -2083,25 +2084,18 @@ def _convert_payloads( def _converters_with_context( self, - context: Optional[temporalio.converter.SerializationContext], - base_payload_converter: Optional[temporalio.converter.PayloadConverter] = None, - base_failure_converter: Optional[temporalio.converter.FailureConverter] = None, + context: temporalio.converter.SerializationContext, ) -> Tuple[ temporalio.converter.PayloadConverter, temporalio.converter.FailureConverter, ]: """Construct workflow payload and failure converters with the given context.""" - payload_converter = base_payload_converter or self._payload_converter - failure_converter = base_failure_converter or self._failure_converter - if context: - if isinstance( - payload_converter, temporalio.converter.WithSerializationContext - ): - payload_converter = payload_converter.with_context(context) - if isinstance( - failure_converter, temporalio.converter.WithSerializationContext - ): - failure_converter = failure_converter.with_context(context) + payload_converter = self._context_free_payload_converter + failure_converter = self._context_free_failure_converter + if isinstance(payload_converter, temporalio.converter.WithSerializationContext): + payload_converter = payload_converter.with_context(context) + if isinstance(failure_converter, temporalio.converter.WithSerializationContext): + failure_converter = failure_converter.with_context(context) return payload_converter, failure_converter # _WorkflowInstanceImpl.get_pending_command_serialization_context From 792c26e0f92f7207a454d3f3cdd7420d686282da Mon Sep 17 00:00:00 2001 From: Dan Davison Date: Wed, 24 Sep 2025 09:50:48 -0400 Subject: [PATCH 25/81] Cleanup --- temporalio/worker/_workflow_instance.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/temporalio/worker/_workflow_instance.py b/temporalio/worker/_workflow_instance.py index 9c092fd2c..dc849e8f1 100644 --- a/temporalio/worker/_workflow_instance.py +++ b/temporalio/worker/_workflow_instance.py @@ -2102,12 +2102,12 @@ def _converters_with_context( def get_payload_codec( self, command_seq: Optional[int] ) -> Optional[temporalio.converter.PayloadCodec]: - # This function is only called when the user's payload codec supports serialization context. + payload_codec = self._context_free_payload_codec if not isinstance( - self._context_free_payload_codec, + payload_codec, temporalio.converter.WithSerializationContext, ): - return self._context_free_payload_codec + return payload_codec workflow_context = temporalio.converter.WorkflowSerializationContext( namespace=self._info.namespace, @@ -2117,7 +2117,7 @@ def get_payload_codec( if command_seq is None: # Use payload codec with workflow context by default (i.e. for payloads not associated # with a pending command) - return self._context_free_payload_codec.with_context(workflow_context) + return payload_codec.with_context(workflow_context) if command_seq in self._pending_activities: handle = self._pending_activities[command_seq] @@ -2134,7 +2134,7 @@ def get_payload_codec( ), is_local=isinstance(handle._input, StartLocalActivityInput), ) - return self._context_free_payload_codec.with_context(context) + return payload_codec.with_context(context) elif command_seq in self._pending_child_workflows: handle = self._pending_child_workflows[command_seq] @@ -2142,7 +2142,7 @@ def get_payload_codec( namespace=self._info.namespace, workflow_id=handle._input.id, ) - return self._context_free_payload_codec.with_context(context) + return payload_codec.with_context(context) elif command_seq in self._pending_external_signals: # Use the target workflow's context for external signals @@ -2151,17 +2151,17 @@ def get_payload_codec( namespace=self._info.namespace, workflow_id=workflow_id, ) - return self._context_free_payload_codec.with_context(context) + return payload_codec.with_context(context) elif command_seq in self._pending_nexus_operations: # Use empty context for nexus operations: users will never want to encrypt using a # key derived from caller workflow context because the caller workflow context is # not available on the handler side for decryption. - return self._context_free_payload_codec + return payload_codec else: # Use payload codec with workflow context for all other payloads - return self._context_free_payload_codec.with_context(workflow_context) + return payload_codec.with_context(workflow_context) def _instantiate_workflow_object(self) -> Any: if not self._workflow_input: From 75f804c3c406f1e270c70a6ab3c52a32d272614f Mon Sep 17 00:00:00 2001 From: Dan Davison Date: Wed, 24 Sep 2025 09:53:56 -0400 Subject: [PATCH 26/81] Remove redundant Stage column and add missing .NET/Java links 1. **Removed Stage column**: The Stage column was redundant since the Operation column already specifies whether it's DataConverter or PayloadCodec. This simplifies the tables and improves readability. 2. **Added missing .NET and Java links** for activity worker operations: - .NET ActivityWorker.cs:389-395 for codec operations - .NET ActivityWorker.cs:376-380 for input deserialization - .NET ActivityWorker.cs:417-418 for result serialization - Java ActivityTaskExecutors.java:70-71 for context creation - Java ActivityTaskExecutors.java:83 for input deserialization - Java ActivityTaskExecutors.java:159-161 for result serialization 3. **Fixed table formatting**: Corrected section headers to have the right number of columns (5 total: Operation, Context Type, .NET, Java, Python) The document now provides complete code references for all three SDKs, making it easier to compare serialization context handling across implementations. --- requirements.md | 112 ++++++++++++++++++++++++------------------------ 1 file changed, 56 insertions(+), 56 deletions(-) diff --git a/requirements.md b/requirements.md index 4addc60e2..b8be9517a 100644 --- a/requirements.md +++ b/requirements.md @@ -6,20 +6,20 @@ **Rule:** All operations use `WorkflowSerializationContext` with the target workflow's ID and namespace. -| Operation | Stage | Context Type | .NET | Java | Python | -|-----------|-------|--------------|------|------|--------| -| **1. Client Starts Workflow** | | | | | | -| Serialize | DataConverter | WorkflowSerializationContext | [TemporalClient.Workflow.cs:693-696](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Client/TemporalClient.Workflow.cs#L693-L696) | [RootWorkflowClientInvoker.java:62-66](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/client/RootWorkflowClientInvoker.java#L62-L66) | [client.py:5993-5998](https://github.com/temporalio/sdk-python/blob/main/temporalio/client.py#L5993-L5998) | -| Encode | PayloadCodec | WorkflowSerializationContext | [TemporalClient.Workflow.cs:699-700](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Client/TemporalClient.Workflow.cs#L699-L700) | [RootWorkflowClientInvoker.java:69](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/client/RootWorkflowClientInvoker.java#L69) | [client.py:6005](https://github.com/temporalio/sdk-python/blob/main/temporalio/client.py#L6005) | -| **2. Workflow Receives Input** | | | | | | -| Decode | PayloadCodec | WorkflowSerializationContext | [WorkflowCodecHelper.cs:125-126](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowCodecHelper.cs#L125-L126) | [ReplayWorkflowTaskHandler.java:160-162](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/replay/ReplayWorkflowTaskHandler.java#L160-L162) | [_workflow.py:289-293](https://github.com/temporalio/sdk-python/blob/main/temporalio/worker/_workflow.py#L289-L293) | -| Deserialize | DataConverter | WorkflowSerializationContext | [WorkflowInstance.cs:395-397](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowInstance.cs#L395-L397) | [SyncWorkflow.java:75-77](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/sync/SyncWorkflow.java#L75-L77) | [_workflow.py:278-283](https://github.com/temporalio/sdk-python/blob/main/temporalio/worker/_workflow.py#L278-L283) | -| **3. Workflow Returns Result** | | | | | | -| Serialize | DataConverter | WorkflowSerializationContext | [WorkflowInstance.cs:1182-1184](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowInstance.cs#L1182-L1184) | [POJOWorkflowImplementationFactory.java:290-292](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/sync/POJOWorkflowImplementationFactory.java#L290-L292) | [_workflow.py:340-346](https://github.com/temporalio/sdk-python/blob/main/temporalio/worker/_workflow.py#L340-L346) | -| Encode | PayloadCodec | WorkflowSerializationContext | [WorkflowCodecHelper.cs:217-220](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowCodecHelper.cs#L217-L220) | [WorkflowWorker.java:702-704](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/worker/WorkflowWorker.java#L702-L704) | [_workflow.py:363-367](https://github.com/temporalio/sdk-python/blob/main/temporalio/worker/_workflow.py#L363-L367) | -| **4. Client Receives Result** | | | | | | -| Decode | PayloadCodec | WorkflowSerializationContext | [WorkflowHandle.cs:84-87](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Client/WorkflowHandle.cs#L84-L87) | [RootWorkflowClientInvoker.java:324-327](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/client/RootWorkflowClientInvoker.java#L324-L327) | [client.py:1601-1605](https://github.com/temporalio/sdk-python/blob/main/temporalio/client.py#L1601-L1605) | -| Deserialize | DataConverter | WorkflowSerializationContext | [WorkflowHandle.cs:129-130](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Client/WorkflowHandle.cs#L129-L130) | [RootWorkflowClientInvoker.java:318-322](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/client/RootWorkflowClientInvoker.java#L318-L322) | [client.py:1715-1718](https://github.com/temporalio/sdk-python/blob/main/temporalio/client.py#L1715-L1718) | +| Operation | Context Type | .NET | Java | Python | +|-----------|--------------|------|------|--------| +| **1. Client Starts Workflow** | | | | | +| Serialize DataConverter | WorkflowSerializationContext | [TemporalClient.Workflow.cs:693-696](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Client/TemporalClient.Workflow.cs#L693-L696) | [RootWorkflowClientInvoker.java:62-66](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/client/RootWorkflowClientInvoker.java#L62-L66) | [client.py:5993-5998](https://github.com/temporalio/sdk-python/blob/main/temporalio/client.py#L5993-L5998) | +| Encode PayloadCodec | WorkflowSerializationContext | [TemporalClient.Workflow.cs:699-700](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Client/TemporalClient.Workflow.cs#L699-L700) | [RootWorkflowClientInvoker.java:69](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/client/RootWorkflowClientInvoker.java#L69) | [client.py:6005](https://github.com/temporalio/sdk-python/blob/main/temporalio/client.py#L6005) | +| **2. Workflow Receives Input** | | | | | +| Decode PayloadCodec | WorkflowSerializationContext | [WorkflowCodecHelper.cs:125-126](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowCodecHelper.cs#L125-L126) | [ReplayWorkflowTaskHandler.java:160-162](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/replay/ReplayWorkflowTaskHandler.java#L160-L162) | [_workflow.py:289-293](https://github.com/temporalio/sdk-python/blob/main/temporalio/worker/_workflow.py#L289-L293) | +| Deserialize DataConverter | WorkflowSerializationContext | [WorkflowInstance.cs:395-397](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowInstance.cs#L395-L397) | [SyncWorkflow.java:75-77](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/sync/SyncWorkflow.java#L75-L77) | [_workflow.py:278-283](https://github.com/temporalio/sdk-python/blob/main/temporalio/worker/_workflow.py#L278-L283) | +| **3. Workflow Returns Result** | | | | | +| Serialize DataConverter | WorkflowSerializationContext | [WorkflowInstance.cs:1182-1184](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowInstance.cs#L1182-L1184) | [POJOWorkflowImplementationFactory.java:290-292](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/sync/POJOWorkflowImplementationFactory.java#L290-L292) | [_workflow.py:340-346](https://github.com/temporalio/sdk-python/blob/main/temporalio/worker/_workflow.py#L340-L346) | +| Encode PayloadCodec | WorkflowSerializationContext | [WorkflowCodecHelper.cs:217-220](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowCodecHelper.cs#L217-L220) | [WorkflowWorker.java:702-704](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/worker/WorkflowWorker.java#L702-L704) | [_workflow.py:363-367](https://github.com/temporalio/sdk-python/blob/main/temporalio/worker/_workflow.py#L363-L367) | +| **4. Client Receives Result** | | | | | +| Decode PayloadCodec | WorkflowSerializationContext | [WorkflowHandle.cs:84-87](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Client/WorkflowHandle.cs#L84-L87) | [RootWorkflowClientInvoker.java:324-327](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/client/RootWorkflowClientInvoker.java#L324-L327) | [client.py:1601-1605](https://github.com/temporalio/sdk-python/blob/main/temporalio/client.py#L1601-L1605) | +| Deserialize DataConverter | WorkflowSerializationContext | [WorkflowHandle.cs:129-130](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Client/WorkflowHandle.cs#L129-L130) | [RootWorkflowClientInvoker.java:318-322](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/client/RootWorkflowClientInvoker.java#L318-L322) | [client.py:1715-1718](https://github.com/temporalio/sdk-python/blob/main/temporalio/client.py#L1715-L1718) | ## Activity Context @@ -27,20 +27,20 @@ **Rule:** All operations use `ActivitySerializationContext` with activity type, task queue, and is_local flag. -| Operation | Stage | Context Type | .NET | Java | Python | -|-----------|-------|--------------|------|------|--------| -| **1. Workflow Schedules Activity** | | | | | | -| Serialize | DataConverter | ActivitySerializationContext | [WorkflowInstance.cs:2084-2093](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowInstance.cs#L2084-L2093) | [SyncWorkflowContext.java:257-267](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/sync/SyncWorkflowContext.java#L257-L267) | [_workflow_instance.py:2968-2972](https://github.com/temporalio/sdk-python/blob/main/temporalio/worker/_workflow_instance.py#L2968-L2972) | -| Encode | PayloadCodec | ActivitySerializationContext | [WorkflowCodecHelper.cs:243-253](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowCodecHelper.cs#L243-L253) | [SyncWorkflowContext.java:268-270](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/sync/SyncWorkflowContext.java#L268-L270) | [_workflow.py:716-733](https://github.com/temporalio/sdk-python/blob/main/temporalio/worker/_workflow.py#L716-L733) | -| **2. Activity Receives Input** | | | | | | -| Decode | PayloadCodec | ActivitySerializationContext | Worker-side (not in workflow) | Worker-side (not in workflow) | [_activity.py:317-318](https://github.com/temporalio/sdk-python/blob/main/temporalio/worker/_activity.py#L317-L318) | -| Deserialize | DataConverter | ActivitySerializationContext | Worker-side (not in workflow) | Worker-side (not in workflow) | [_activity.py:526-530](https://github.com/temporalio/sdk-python/blob/main/temporalio/worker/_activity.py#L526-L530) | -| **3. Activity Returns Result** | | | | | | -| Serialize | DataConverter | ActivitySerializationContext | Worker-side (not in workflow) | Worker-side (not in workflow) | [_activity.py:322-323](https://github.com/temporalio/sdk-python/blob/main/temporalio/worker/_activity.py#L322-L323) | -| Encode | PayloadCodec | ActivitySerializationContext | Worker-side (not in workflow) | Worker-side (not in workflow) | [_activity.py:317-318](https://github.com/temporalio/sdk-python/blob/main/temporalio/worker/_activity.py#L317-L318) | -| **4. Workflow Receives Result** | | | | | | -| Decode | PayloadCodec | ActivitySerializationContext | [WorkflowCodecHelper.cs:68-76](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowCodecHelper.cs#L68-L76) | [SyncWorkflowContext.java:273-274](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/sync/SyncWorkflowContext.java#L273-L274) | [_workflow.py:716-745](https://github.com/temporalio/sdk-python/blob/main/temporalio/worker/_workflow.py#L716-L745) | -| Deserialize | DataConverter | ActivitySerializationContext | [WorkflowInstance.cs:2720](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowInstance.cs#L2720) | [SyncWorkflowContext.java:286-288](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/sync/SyncWorkflowContext.java#L286-L288) | [_workflow_instance.py:804-810](https://github.com/temporalio/sdk-python/blob/main/temporalio/worker/_workflow_instance.py#L804-L810) | +| Operation | Context Type | .NET | Java | Python | +|-----------|--------------|------|------|--------| +| **1. Workflow Schedules Activity** | | | | | +| Serialize DataConverter | ActivitySerializationContext | [WorkflowInstance.cs:2084-2093](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowInstance.cs#L2084-L2093) | [SyncWorkflowContext.java:257-267](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/sync/SyncWorkflowContext.java#L257-L267) | [_workflow_instance.py:2968-2972](https://github.com/temporalio/sdk-python/blob/main/temporalio/worker/_workflow_instance.py#L2968-L2972) | +| Encode PayloadCodec | ActivitySerializationContext | [WorkflowCodecHelper.cs:243-253](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowCodecHelper.cs#L243-L253) | [SyncWorkflowContext.java:268-270](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/sync/SyncWorkflowContext.java#L268-L270) | [_workflow.py:716-733](https://github.com/temporalio/sdk-python/blob/main/temporalio/worker/_workflow.py#L716-L733) | +| **2. Activity Receives Input** | | | | | +| Decode PayloadCodec | ActivitySerializationContext | [ActivityWorker.cs:389-395](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/ActivityWorker.cs#L389-L395) | [ActivityTaskExecutors.java:70-71](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/activity/ActivityTaskExecutors.java#L70-L71) | [_activity.py:317-318](https://github.com/temporalio/sdk-python/blob/main/temporalio/worker/_activity.py#L317-L318) | +| Deserialize DataConverter | ActivitySerializationContext | [ActivityWorker.cs:376-380](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/ActivityWorker.cs#L376-L380) | [ActivityTaskExecutors.java:83](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/activity/ActivityTaskExecutors.java#L83) | [_activity.py:526-530](https://github.com/temporalio/sdk-python/blob/main/temporalio/worker/_activity.py#L526-L530) | +| **3. Activity Returns Result** | | | | | +| Serialize DataConverter | ActivitySerializationContext | [ActivityWorker.cs:417-418](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/ActivityWorker.cs#L417-L418) | [ActivityTaskExecutors.java:159-161](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/activity/ActivityTaskExecutors.java#L159-L161) | [_activity.py:322-323](https://github.com/temporalio/sdk-python/blob/main/temporalio/worker/_activity.py#L322-L323) | +| Encode PayloadCodec | ActivitySerializationContext | [ActivityWorker.cs:389-395](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/ActivityWorker.cs#L389-L395) | [ActivityTaskExecutors.java:70-71](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/activity/ActivityTaskExecutors.java#L70-L71) | [_activity.py:317-318](https://github.com/temporalio/sdk-python/blob/main/temporalio/worker/_activity.py#L317-L318) | +| **4. Workflow Receives Result** | | | | | +| Decode PayloadCodec | ActivitySerializationContext | [WorkflowCodecHelper.cs:68-76](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowCodecHelper.cs#L68-L76) | [SyncWorkflowContext.java:273-274](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/sync/SyncWorkflowContext.java#L273-L274) | [_workflow.py:716-745](https://github.com/temporalio/sdk-python/blob/main/temporalio/worker/_workflow.py#L716-L745) | +| Deserialize DataConverter | ActivitySerializationContext | [WorkflowInstance.cs:2720](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowInstance.cs#L2720) | [SyncWorkflowContext.java:286-288](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/sync/SyncWorkflowContext.java#L286-L288) | [_workflow_instance.py:804-810](https://github.com/temporalio/sdk-python/blob/main/temporalio/worker/_workflow_instance.py#L804-L810) | ## Child Workflow Context @@ -48,20 +48,20 @@ **Rule:** Child operations use `WorkflowSerializationContext` with the child workflow's ID. -| Operation | Stage | Context Type | .NET | Java | Python | -|-----------|-------|--------------|------|------|--------| -| **1. Workflow Starts Child** | | | | | | -| Serialize | DataConverter | WorkflowSerializationContext (child ID) | [WorkflowInstance.cs:2319-2326](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowInstance.cs#L2319-L2326) | [SyncWorkflowContext.java:687-690](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/sync/SyncWorkflowContext.java#L687-L690) | [_workflow_instance.py:3124-3128](https://github.com/temporalio/sdk-python/blob/main/temporalio/worker/_workflow_instance.py#L3124-L3128) | -| Encode | PayloadCodec | WorkflowSerializationContext (child ID) | [WorkflowCodecHelper.cs:301-310](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowCodecHelper.cs#L301-L310) | [SyncWorkflowContext.java:690](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/sync/SyncWorkflowContext.java#L690) | [_workflow.py:716-745](https://github.com/temporalio/sdk-python/blob/main/temporalio/worker/_workflow.py#L716-L745) | -| **2. Child Receives Input** | | | | | | -| Decode | PayloadCodec | WorkflowSerializationContext (child ID) | [WorkflowCodecHelper.cs:125-126](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowCodecHelper.cs#L125-L126) | [ReplayWorkflowTaskHandler.java:160-162](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/replay/ReplayWorkflowTaskHandler.java#L160-L162) | [_workflow.py:289-293](https://github.com/temporalio/sdk-python/blob/main/temporalio/worker/_workflow.py#L289-L293) | -| Deserialize | DataConverter | WorkflowSerializationContext (child ID) | [WorkflowInstance.cs:395-397](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowInstance.cs#L395-L397) | [SyncWorkflow.java:75-77](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/sync/SyncWorkflow.java#L75-L77) | [_workflow.py:278-283](https://github.com/temporalio/sdk-python/blob/main/temporalio/worker/_workflow.py#L278-L283) | -| **3. Child Returns Result** | | | | | | -| Serialize | DataConverter | WorkflowSerializationContext (child ID) | [WorkflowInstance.cs:1182-1184](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowInstance.cs#L1182-L1184) | [POJOWorkflowImplementationFactory.java:290-292](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/sync/POJOWorkflowImplementationFactory.java#L290-L292) | [_workflow.py:340-346](https://github.com/temporalio/sdk-python/blob/main/temporalio/worker/_workflow.py#L340-L346) | -| Encode | PayloadCodec | WorkflowSerializationContext (child ID) | [WorkflowCodecHelper.cs:217-220](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowCodecHelper.cs#L217-L220) | [WorkflowWorker.java:702-704](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/worker/WorkflowWorker.java#L702-L704) | [_workflow.py:363-367](https://github.com/temporalio/sdk-python/blob/main/temporalio/worker/_workflow.py#L363-L367) | -| **4. Workflow Receives Child Result** | | | | | | -| Decode | PayloadCodec | WorkflowSerializationContext (child ID) | [WorkflowCodecHelper.cs:78-85](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowCodecHelper.cs#L78-L85) | [SyncWorkflowContext.java:1245-1248](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/sync/SyncWorkflowContext.java#L1245-L1248) | [_workflow.py:716-745](https://github.com/temporalio/sdk-python/blob/main/temporalio/worker/_workflow.py#L716-L745) | -| Deserialize | DataConverter | WorkflowSerializationContext (child ID) | [WorkflowInstance.cs:2808](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowInstance.cs#L2808) | [SyncWorkflowContext.java:1245-1248](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/sync/SyncWorkflowContext.java#L1245-L1248) | [_workflow_instance.py:842-858](https://github.com/temporalio/sdk-python/blob/main/temporalio/worker/_workflow_instance.py#L842-L858) | +| Operation | Context Type | .NET | Java | Python | +|-----------|--------------|------|------|--------| +| **1. Workflow Starts Child** | | | | | +| Serialize DataConverter | WorkflowSerializationContext (child ID) | [WorkflowInstance.cs:2319-2326](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowInstance.cs#L2319-L2326) | [SyncWorkflowContext.java:687-690](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/sync/SyncWorkflowContext.java#L687-L690) | [_workflow_instance.py:3124-3128](https://github.com/temporalio/sdk-python/blob/main/temporalio/worker/_workflow_instance.py#L3124-L3128) | +| Encode PayloadCodec | WorkflowSerializationContext (child ID) | [WorkflowCodecHelper.cs:301-310](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowCodecHelper.cs#L301-L310) | [SyncWorkflowContext.java:690](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/sync/SyncWorkflowContext.java#L690) | [_workflow.py:716-745](https://github.com/temporalio/sdk-python/blob/main/temporalio/worker/_workflow.py#L716-L745) | +| **2. Child Receives Input** | | | | | +| Decode PayloadCodec | WorkflowSerializationContext (child ID) | [WorkflowCodecHelper.cs:125-126](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowCodecHelper.cs#L125-L126) | [ReplayWorkflowTaskHandler.java:160-162](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/replay/ReplayWorkflowTaskHandler.java#L160-L162) | [_workflow.py:289-293](https://github.com/temporalio/sdk-python/blob/main/temporalio/worker/_workflow.py#L289-L293) | +| Deserialize DataConverter | WorkflowSerializationContext (child ID) | [WorkflowInstance.cs:395-397](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowInstance.cs#L395-L397) | [SyncWorkflow.java:75-77](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/sync/SyncWorkflow.java#L75-L77) | [_workflow.py:278-283](https://github.com/temporalio/sdk-python/blob/main/temporalio/worker/_workflow.py#L278-L283) | +| **3. Child Returns Result** | | | | | +| Serialize DataConverter | WorkflowSerializationContext (child ID) | [WorkflowInstance.cs:1182-1184](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowInstance.cs#L1182-L1184) | [POJOWorkflowImplementationFactory.java:290-292](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/sync/POJOWorkflowImplementationFactory.java#L290-L292) | [_workflow.py:340-346](https://github.com/temporalio/sdk-python/blob/main/temporalio/worker/_workflow.py#L340-L346) | +| Encode PayloadCodec | WorkflowSerializationContext (child ID) | [WorkflowCodecHelper.cs:217-220](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowCodecHelper.cs#L217-L220) | [WorkflowWorker.java:702-704](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/worker/WorkflowWorker.java#L702-L704) | [_workflow.py:363-367](https://github.com/temporalio/sdk-python/blob/main/temporalio/worker/_workflow.py#L363-L367) | +| **4. Workflow Receives Child Result** | | | | | +| Decode PayloadCodec | WorkflowSerializationContext (child ID) | [WorkflowCodecHelper.cs:78-85](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowCodecHelper.cs#L78-L85) | [SyncWorkflowContext.java:1245-1248](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/sync/SyncWorkflowContext.java#L1245-L1248) | [_workflow.py:716-745](https://github.com/temporalio/sdk-python/blob/main/temporalio/worker/_workflow.py#L716-L745) | +| Deserialize DataConverter | WorkflowSerializationContext (child ID) | [WorkflowInstance.cs:2808](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowInstance.cs#L2808) | [SyncWorkflowContext.java:1245-1248](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/sync/SyncWorkflowContext.java#L1245-L1248) | [_workflow_instance.py:842-858](https://github.com/temporalio/sdk-python/blob/main/temporalio/worker/_workflow_instance.py#L842-L858) | ## Nexus Operation Context @@ -69,20 +69,20 @@ **Rule:** Nexus operations have **no serialization context** (null/none). -| Operation | Stage | Context Type | .NET | Java | Python | -|-----------|-------|--------------|------|------|--------| -| **1. Workflow Schedules Nexus Op** | | | | | | -| Serialize | DataConverter | None | [WorkflowInstance.cs:2525](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowInstance.cs#L2525) | [SyncWorkflowContext.java:790](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/sync/SyncWorkflowContext.java#L790) | N/A (not yet implemented) | -| Encode | PayloadCodec | None | [WorkflowCodecHelper.cs:355-356](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowCodecHelper.cs#L355-L356) | [SyncWorkflowContext.java:791-794](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/sync/SyncWorkflowContext.java#L791-L794) | N/A (not yet implemented) | -| **2. Nexus Operation Receives Input** | | | | | | -| Decode | PayloadCodec | None | N/A (handler-side) | N/A (handler-side) | N/A (handler-side) | -| Deserialize | DataConverter | None | N/A (handler-side) | N/A (handler-side) | N/A (handler-side) | -| **3. Nexus Operation Returns Result** | | | | | | -| Serialize | DataConverter | None | N/A (handler-side) | N/A (handler-side) | N/A (handler-side) | -| Encode | PayloadCodec | None | N/A (handler-side) | N/A (handler-side) | N/A (handler-side) | -| **4. Workflow Receives Result** | | | | | | -| Decode | PayloadCodec | None | [WorkflowCodecHelper.cs:112-113](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowCodecHelper.cs#L112-L113) | N/A (codec not applied) | N/A (not yet implemented) | -| Deserialize | DataConverter | None | [WorkflowInstance.cs:2954](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowInstance.cs#L2954) | [SyncWorkflowContext.java:855-856](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/sync/SyncWorkflowContext.java#L855-L856) | N/A (not yet implemented) | +| Operation | Context Type | .NET | Java | Python | +|-----------|--------------|------|------|--------| +| **1. Workflow Schedules Nexus Op** | | | | | +| Serialize DataConverter | None | [WorkflowInstance.cs:2525](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowInstance.cs#L2525) | [SyncWorkflowContext.java:790](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/sync/SyncWorkflowContext.java#L790) | N/A (not yet implemented) | +| Encode PayloadCodec | None | [WorkflowCodecHelper.cs:355-356](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowCodecHelper.cs#L355-L356) | [SyncWorkflowContext.java:791-794](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/sync/SyncWorkflowContext.java#L791-L794) | N/A (not yet implemented) | +| **2. Nexus Operation Receives Input** | | | | | +| Decode PayloadCodec | None | N/A (handler-side) | N/A (handler-side) | N/A (handler-side) | +| Deserialize DataConverter | None | N/A (handler-side) | N/A (handler-side) | N/A (handler-side) | +| **3. Nexus Operation Returns Result** | | | | | +| Serialize DataConverter | None | N/A (handler-side) | N/A (handler-side) | N/A (handler-side) | +| Encode PayloadCodec | None | N/A (handler-side) | N/A (handler-side) | N/A (handler-side) | +| **4. Workflow Receives Result** | | | | | +| Decode PayloadCodec | None | [WorkflowCodecHelper.cs:112-113](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowCodecHelper.cs#L112-L113) | N/A (codec not applied) | N/A (not yet implemented) | +| Deserialize DataConverter | None | [WorkflowInstance.cs:2954](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowInstance.cs#L2954) | [SyncWorkflowContext.java:855-856](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/sync/SyncWorkflowContext.java#L855-L856) | N/A (not yet implemented) | ## Memo and Search Attribute Context From 297b4d0780db2986563ca69022d7fd9a2f036052 Mon Sep 17 00:00:00 2001 From: Dan Davison Date: Wed, 24 Sep 2025 11:18:55 -0400 Subject: [PATCH 27/81] Fix assertion error when workflow initialization fails The serialization context changes introduced an incorrect 'assert workflow' statement that would fail when workflow initialization fails (e.g., when the workflow class isn't registered). This caused tests to hang with an unhandled AssertionError. The workflow variable can legitimately be None when initialization fails, and we only need it when applying a payload codec. Changed to check for workflow existence as part of the condition for applying the codec. --- temporalio/worker/_workflow.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/temporalio/worker/_workflow.py b/temporalio/worker/_workflow.py index 3b04ad511..a6be3688e 100644 --- a/temporalio/worker/_workflow.py +++ b/temporalio/worker/_workflow.py @@ -354,10 +354,9 @@ async def _handle_activation( ) completion.run_id = act.run_id - assert workflow # Encode completion - if data_converter.payload_codec: + if data_converter.payload_codec and workflow: payload_codec = _CommandAwarePayloadCodec(workflow.instance) try: await temporalio.bridge.worker.encode_completion( From fe8c0d481e334f61da8e76c15952456fb90308e2 Mon Sep 17 00:00:00 2001 From: Dan Davison Date: Wed, 24 Sep 2025 11:46:16 -0400 Subject: [PATCH 28/81] Cleanup: sort lines --- scripts/gen_payload_visitor.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/scripts/gen_payload_visitor.py b/scripts/gen_payload_visitor.py index 8900afb88..9e4d2d71e 100644 --- a/scripts/gen_payload_visitor.py +++ b/scripts/gen_payload_visitor.py @@ -279,27 +279,27 @@ def walk(self, desc: Descriptor) -> bool: ) commands_with_seq = { - "start_timer", + "cancel_signal_workflow", "cancel_timer", - "schedule_activity", - "schedule_local_activity", "request_cancel_activity", - "request_cancel_local_activity", - "start_child_workflow_execution", "request_cancel_external_workflow_execution", - "signal_external_workflow_execution", - "cancel_signal_workflow", - "schedule_nexus_operation", + "request_cancel_local_activity", "request_cancel_nexus_operation", + "schedule_activity", + "schedule_local_activity", + "schedule_nexus_operation", + "signal_external_workflow_execution", + "start_child_workflow_execution", + "start_timer", } activation_jobs_with_seq = { "resolve_activity", - "resolve_child_workflow_execution", "resolve_child_workflow_execution_start", + "resolve_child_workflow_execution", + "resolve_nexus_operation_start", + "resolve_nexus_operation", "resolve_request_cancel_external_workflow", "resolve_signal_external_workflow", - "resolve_nexus_operation", - "resolve_nexus_operation_start", } # Process oneof fields as if/elif chains for oneof_idx, fields in oneof_fields.items(): From fc25a9eaeda1cf9050bd294784a27d27ece4e49e Mon Sep 17 00:00:00 2001 From: Dan Davison Date: Wed, 24 Sep 2025 11:51:17 -0400 Subject: [PATCH 29/81] Lazy construction of data converter with context on WorkflowHandle --- temporalio/client.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/temporalio/client.py b/temporalio/client.py index 7005f40c3..015fff95b 100644 --- a/temporalio/client.py +++ b/temporalio/client.py @@ -6,6 +6,7 @@ import asyncio import copy import dataclasses +import functools import inspect import json import re @@ -1598,11 +1599,6 @@ def __init__( ) -> None: """Create workflow handle.""" self._client = client - self._data_converter = client.data_converter._with_context( - temporalio.converter.WorkflowSerializationContext( - namespace=client.namespace, workflow_id=id - ) - ) self._id = id self._run_id = run_id self._result_run_id = result_run_id @@ -1611,6 +1607,14 @@ def __init__( self._start_workflow_response = start_workflow_response self.__temporal_eagerly_started = False + @functools.cached_property + def _data_converter(self) -> temporalio.converter.DataConverter: + return self._client.data_converter._with_context( + temporalio.converter.WorkflowSerializationContext( + namespace=self._client.namespace, workflow_id=self._id + ) + ) + @property def id(self) -> str: """ID for the workflow.""" From 95b7b24e6c74b5dcf3ba225c31852cb1b244146d Mon Sep 17 00:00:00 2001 From: Dan Davison Date: Wed, 24 Sep 2025 13:35:29 -0400 Subject: [PATCH 30/81] memoize data converter construction in paging iterator --- temporalio/client.py | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/temporalio/client.py b/temporalio/client.py index 015fff95b..05413828f 100644 --- a/temporalio/client.py +++ b/temporalio/client.py @@ -3226,15 +3226,24 @@ async def fetch_next_page(self, *, page_size: Optional[int] = None) -> None: metadata=self._input.rpc_metadata, timeout=self._input.rpc_timeout, ) + + data_converter_cache = {} + + def get_data_converter(workflow_id: str) -> temporalio.converter.DataConverter: + if workflow_id not in data_converter_cache: + data_converter_cache[workflow_id] = ( + self._client.data_converter._with_context( + WorkflowSerializationContext( + namespace=self._client.namespace, + workflow_id=workflow_id, + ) + ) + ) + return data_converter_cache[workflow_id] + self._current_page = [ WorkflowExecution._from_raw_info( - v, - self._client.data_converter._with_context( - WorkflowSerializationContext( - namespace=self._client.namespace, - workflow_id=v.execution.workflow_id, - ) - ), + v, get_data_converter(v.execution.workflow_id) ) for v in resp.executions ] From e0484b87feef3756db5ef0c9815e8a82ccc5f8f6 Mon Sep 17 00:00:00 2001 From: Dan Davison Date: Wed, 24 Sep 2025 12:34:59 -0400 Subject: [PATCH 31/81] AI fix for uv run pytest 'tests/worker/test_workflow.py::test_workflow_activity_outbound_conversion_failure' uv run pytest 'tests/worker/test_workflow.py::test_exception_raising_converter_param' --- temporalio/converter.py | 15 +++++++++++++++ temporalio/worker/_workflow_instance.py | 8 ++------ 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/temporalio/converter.py b/temporalio/converter.py index 6d1c07f63..652962c6d 100644 --- a/temporalio/converter.py +++ b/temporalio/converter.py @@ -435,6 +435,21 @@ def __init__(self) -> None: """Create a default payload converter.""" super().__init__(*DefaultPayloadConverter.default_encoding_payload_converters) + def with_context(self, context: SerializationContext) -> Self: + """Return a new instance with the given context, preserving the class type.""" + # Create a new instance of the same class (works for subclasses too) + instance = self.__class__() + # Update the converters with context + instance.converters = { + encoding: ( + converter.with_context(context) + if isinstance(converter, WithSerializationContext) + else converter + ) + for encoding, converter in self.converters.items() + } + return instance + class BinaryNullPayloadConverter(EncodingPayloadConverter): """Converter for 'binary/null' payloads supporting None values.""" diff --git a/temporalio/worker/_workflow_instance.py b/temporalio/worker/_workflow_instance.py index dc849e8f1..37d2daa58 100644 --- a/temporalio/worker/_workflow_instance.py +++ b/temporalio/worker/_workflow_instance.py @@ -225,12 +225,8 @@ def __init__(self, det: WorkflowInstanceDetails) -> None: self._workflow_input: Optional[ExecuteWorkflowInput] = None self._info = det.info self._context_free_payload_codec = det.data_converter.payload_codec - self._context_free_payload_converter = ( - det.data_converter.payload_converter_class() - ) - self._context_free_failure_converter = ( - det.data_converter.failure_converter_class() - ) + self._context_free_payload_converter = det.data_converter.payload_converter + self._context_free_failure_converter = det.data_converter.failure_converter self._payload_converter, self._failure_converter = ( self._converters_with_context( temporalio.converter.WorkflowSerializationContext( From 2597c8b9ed3402217bce7cfbc5f7372f81644b71 Mon Sep 17 00:00:00 2001 From: Dan Davison Date: Wed, 24 Sep 2025 14:20:41 -0400 Subject: [PATCH 32/81] Revert "AI fix for" This reverts commit 09584f840d6bc633fc0f3ba9acc958cafe1dae03. --- temporalio/converter.py | 15 --------------- temporalio/worker/_workflow_instance.py | 8 ++++++-- 2 files changed, 6 insertions(+), 17 deletions(-) diff --git a/temporalio/converter.py b/temporalio/converter.py index 652962c6d..6d1c07f63 100644 --- a/temporalio/converter.py +++ b/temporalio/converter.py @@ -435,21 +435,6 @@ def __init__(self) -> None: """Create a default payload converter.""" super().__init__(*DefaultPayloadConverter.default_encoding_payload_converters) - def with_context(self, context: SerializationContext) -> Self: - """Return a new instance with the given context, preserving the class type.""" - # Create a new instance of the same class (works for subclasses too) - instance = self.__class__() - # Update the converters with context - instance.converters = { - encoding: ( - converter.with_context(context) - if isinstance(converter, WithSerializationContext) - else converter - ) - for encoding, converter in self.converters.items() - } - return instance - class BinaryNullPayloadConverter(EncodingPayloadConverter): """Converter for 'binary/null' payloads supporting None values.""" diff --git a/temporalio/worker/_workflow_instance.py b/temporalio/worker/_workflow_instance.py index 37d2daa58..dc849e8f1 100644 --- a/temporalio/worker/_workflow_instance.py +++ b/temporalio/worker/_workflow_instance.py @@ -225,8 +225,12 @@ def __init__(self, det: WorkflowInstanceDetails) -> None: self._workflow_input: Optional[ExecuteWorkflowInput] = None self._info = det.info self._context_free_payload_codec = det.data_converter.payload_codec - self._context_free_payload_converter = det.data_converter.payload_converter - self._context_free_failure_converter = det.data_converter.failure_converter + self._context_free_payload_converter = ( + det.data_converter.payload_converter_class() + ) + self._context_free_failure_converter = ( + det.data_converter.failure_converter_class() + ) self._payload_converter, self._failure_converter = ( self._converters_with_context( temporalio.converter.WorkflowSerializationContext( From 6b4081ffc7b608dfedfa0fab3f05f742b0c64e7a Mon Sep 17 00:00:00 2001 From: Dan Davison Date: Wed, 24 Sep 2025 15:38:35 -0400 Subject: [PATCH 33/81] Fix payload codec usage - Revert "Store data_converter on WorkflowInstanceDetails instead of converter classes" - Don't require payload codec on workflow instance - Don't stack contexts Revert "Store data_converter on WorkflowInstanceDetails instead of converter classes" This reverts commit f869af87ba5bffb891f8abb3189e655921247ca3. --- temporalio/worker/_workflow.py | 46 +++++++++++-------- temporalio/worker/_workflow_instance.py | 29 ++++++------ .../worker/workflow_sandbox/_in_sandbox.py | 12 +++-- temporalio/worker/workflow_sandbox/_runner.py | 14 ++++-- tests/worker/test_workflow.py | 12 +++-- 5 files changed, 66 insertions(+), 47 deletions(-) diff --git a/temporalio/worker/_workflow.py b/temporalio/worker/_workflow.py index a6be3688e..7a0a3c56d 100644 --- a/temporalio/worker/_workflow.py +++ b/temporalio/worker/_workflow.py @@ -249,13 +249,12 @@ async def _handle_activation( await self._handle_cache_eviction(act, cache_remove_job) return - data_converter = self._data_converter # Build default success completion (e.g. remove-job-only activations) completion = ( temporalio.bridge.proto.workflow_completion.WorkflowActivationCompletion() ) completion.successful.SetInParent() - workflow = None + workflow = workflow_id = None try: if LOG_PROTOS: logger.debug("Received workflow activation:\n%s", act) @@ -275,17 +274,14 @@ async def _handle_activation( "Cache already exists for activation with initialize job" ) - data_converter = self._data_converter._with_context( - temporalio.converter.WorkflowSerializationContext( - namespace=self._namespace, - workflow_id=workflow_id, - ) - ) - if data_converter.payload_codec: + if self._data_converter.payload_codec: if not workflow: - payload_codec = data_converter.payload_codec + payload_codec = self._data_converter.payload_codec else: - payload_codec = _CommandAwarePayloadCodec(workflow.instance) + payload_codec = _CommandAwarePayloadCodec( + workflow.instance, + self._data_converter.payload_codec, + ) await temporalio.bridge.worker.decode_activation( act, payload_codec.decode, @@ -339,6 +335,14 @@ async def _handle_activation( completion.failed.failure.SetInParent() try: + data_converter = self._data_converter + if workflow_id: + data_converter = data_converter._with_context( + temporalio.converter.WorkflowSerializationContext( + namespace=self._namespace, + workflow_id=workflow_id, + ) + ) data_converter.failure_converter.to_failure( err, data_converter.payload_converter, @@ -356,8 +360,11 @@ async def _handle_activation( completion.run_id = act.run_id # Encode completion - if data_converter.payload_codec and workflow: - payload_codec = _CommandAwarePayloadCodec(workflow.instance) + if self._data_converter.payload_codec and workflow: + payload_codec = _CommandAwarePayloadCodec( + workflow.instance, + self._data_converter.payload_codec, + ) try: await temporalio.bridge.worker.encode_completion( completion, @@ -572,7 +579,8 @@ def _create_workflow_instance( # Create instance from details det = WorkflowInstanceDetails( - data_converter=self._data_converter, + payload_converter_class=self._data_converter.payload_converter_class, + failure_converter_class=self._data_converter.failure_converter_class, interceptor_classes=self._interceptor_classes, defn=defn, info=info, @@ -722,8 +730,10 @@ class _CommandAwarePayloadCodec(temporalio.converter.PayloadCodec): def __init__( self, instance: WorkflowInstance, + context_free_payload_codec: temporalio.converter.PayloadCodec, ): self.instance = instance + self.context_free_payload_codec = context_free_payload_codec async def encode( self, @@ -738,10 +748,10 @@ async def decode( return await self._get_current_command_codec().decode(payloads) def _get_current_command_codec(self) -> temporalio.converter.PayloadCodec: - seq = temporalio.bridge._visitor.current_command_seq.get() - codec = self.instance.get_payload_codec(seq) - assert codec, "Payload codec must be set on the data converter" - return codec + return self.instance.get_payload_codec_with_context( + self.context_free_payload_codec, + temporalio.bridge._visitor.current_command_seq.get(), + ) class _InterruptDeadlockError(BaseException): diff --git a/temporalio/worker/_workflow_instance.py b/temporalio/worker/_workflow_instance.py index dc849e8f1..3cba6bee1 100644 --- a/temporalio/worker/_workflow_instance.py +++ b/temporalio/worker/_workflow_instance.py @@ -135,7 +135,8 @@ def set_worker_level_failure_exception_types( class WorkflowInstanceDetails: """Immutable details for creating a workflow instance.""" - data_converter: temporalio.converter.DataConverter + payload_converter_class: Type[temporalio.converter.PayloadConverter] + failure_converter_class: Type[temporalio.converter.FailureConverter] interceptor_classes: Sequence[Type[WorkflowInboundInterceptor]] defn: temporalio.workflow._Definition info: temporalio.workflow.Info @@ -168,9 +169,11 @@ def activate( raise NotImplementedError @abstractmethod - def get_payload_codec( - self, command_seq: Optional[int] - ) -> Optional[temporalio.converter.PayloadCodec]: + def get_payload_codec_with_context( + self, + payload_codec: temporalio.converter.PayloadCodec, + command_seq: Optional[int], + ) -> temporalio.converter.PayloadCodec: """Return a payload codec with appropriate serialization context. Args: @@ -224,13 +227,8 @@ def __init__(self, det: WorkflowInstanceDetails) -> None: self._defn = det.defn self._workflow_input: Optional[ExecuteWorkflowInput] = None self._info = det.info - self._context_free_payload_codec = det.data_converter.payload_codec - self._context_free_payload_converter = ( - det.data_converter.payload_converter_class() - ) - self._context_free_failure_converter = ( - det.data_converter.failure_converter_class() - ) + self._context_free_payload_converter = det.payload_converter_class() + self._context_free_failure_converter = det.failure_converter_class() self._payload_converter, self._failure_converter = ( self._converters_with_context( temporalio.converter.WorkflowSerializationContext( @@ -2099,10 +2097,11 @@ def _converters_with_context( return payload_converter, failure_converter # _WorkflowInstanceImpl.get_pending_command_serialization_context - def get_payload_codec( - self, command_seq: Optional[int] - ) -> Optional[temporalio.converter.PayloadCodec]: - payload_codec = self._context_free_payload_codec + def get_payload_codec_with_context( + self, + payload_codec: temporalio.converter.PayloadCodec, + command_seq: Optional[int], + ) -> temporalio.converter.PayloadCodec: if not isinstance( payload_codec, temporalio.converter.WithSerializationContext, diff --git a/temporalio/worker/workflow_sandbox/_in_sandbox.py b/temporalio/worker/workflow_sandbox/_in_sandbox.py index 0188e309b..f34b54424 100644 --- a/temporalio/worker/workflow_sandbox/_in_sandbox.py +++ b/temporalio/worker/workflow_sandbox/_in_sandbox.py @@ -81,8 +81,10 @@ def activate( """Send activation to this instance.""" return self.instance.activate(act) - def get_payload_codec( - self, command_seq: Optional[int] - ) -> Optional[temporalio.converter.PayloadCodec]: - """Get payload codec.""" - return self.instance.get_payload_codec(command_seq) + def get_payload_codec_with_context( + self, + payload_codec: temporalio.converter.PayloadCodec, + command_seq: Optional[int], + ) -> temporalio.converter.PayloadCodec: + """Get payload codec with context.""" + return self.instance.get_payload_codec_with_context(payload_codec, command_seq) diff --git a/temporalio/worker/workflow_sandbox/_runner.py b/temporalio/worker/workflow_sandbox/_runner.py index 4e48002d0..1fce6d04b 100644 --- a/temporalio/worker/workflow_sandbox/_runner.py +++ b/temporalio/worker/workflow_sandbox/_runner.py @@ -77,7 +77,8 @@ def prepare_workflow(self, defn: temporalio.workflow._Definition) -> None: # Just create with fake info which validates self.create_instance( WorkflowInstanceDetails( - data_converter=temporalio.converter.DataConverter.default, + payload_converter_class=temporalio.converter.DataConverter.default.payload_converter_class, + failure_converter_class=temporalio.converter.DataConverter.default.failure_converter_class, interceptor_classes=[], defn=defn, # Just use fake info during validation @@ -185,16 +186,19 @@ def _run_code(self, code: str, **extra_globals: Any) -> None: def get_thread_id(self) -> Optional[int]: return self._current_thread_id - def get_payload_codec( - self, command_seq: Optional[int] - ) -> Optional[temporalio.converter.PayloadCodec]: + def get_payload_codec_with_context( + self, + payload_codec: temporalio.converter.PayloadCodec, + command_seq: Optional[int], + ) -> temporalio.converter.PayloadCodec: # Forward call to the sandboxed instance self.importer.restriction_context.is_runtime = True try: self._run_code( "with __temporal_importer.applied():\n" - " __temporal_codec = __temporal_in_sandbox.get_payload_codec(__temporal_command_seq)\n", + " __temporal_codec = __temporal_in_sandbox.get_payload_codec_with_context(__temporal_payload_codec, __temporal_command_seq)\n", __temporal_importer=self.importer, + __temporal_payload_codec=payload_codec, __temporal_command_seq=command_seq, ) return self.globals_and_locals.pop("__temporal_codec", None) # type: ignore diff --git a/tests/worker/test_workflow.py b/tests/worker/test_workflow.py index 2b8f468c6..b406a6229 100644 --- a/tests/worker/test_workflow.py +++ b/tests/worker/test_workflow.py @@ -1610,10 +1610,14 @@ def activate(self, act: WorkflowActivation) -> WorkflowActivationCompletion: self._runner._pairs.append((act, comp)) return comp - def get_payload_codec( - self, command_seq: Optional[int] - ) -> Optional[temporalio.converter.PayloadCodec]: - return self._unsandboxed.get_payload_codec(command_seq) + def get_payload_codec_with_context( + self, + payload_codec: temporalio.converter.PayloadCodec, + command_seq: Optional[int], + ) -> temporalio.converter.PayloadCodec: + return self._unsandboxed.get_payload_codec_with_context( + payload_codec, command_seq + ) async def test_workflow_with_custom_runner(client: Client): From 6bf9472e035bb5c1ebb891c86442c1db0cfe0ef7 Mon Sep 17 00:00:00 2001 From: Dan Davison Date: Wed, 24 Sep 2025 17:54:58 -0400 Subject: [PATCH 34/81] Bug fix --- temporalio/worker/_workflow.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/temporalio/worker/_workflow.py b/temporalio/worker/_workflow.py index 7a0a3c56d..c386556eb 100644 --- a/temporalio/worker/_workflow.py +++ b/temporalio/worker/_workflow.py @@ -254,7 +254,8 @@ async def _handle_activation( temporalio.bridge.proto.workflow_completion.WorkflowActivationCompletion() ) completion.successful.SetInParent() - workflow = workflow_id = None + workflow = None + data_converter = self._data_converter try: if LOG_PROTOS: logger.debug("Received workflow activation:\n%s", act) @@ -274,9 +275,16 @@ async def _handle_activation( "Cache already exists for activation with initialize job" ) + data_converter = self._data_converter._with_context( + temporalio.converter.WorkflowSerializationContext( + namespace=self._namespace, + workflow_id=workflow_id, + ) + ) if self._data_converter.payload_codec: + assert data_converter.payload_codec if not workflow: - payload_codec = self._data_converter.payload_codec + payload_codec = data_converter.payload_codec else: payload_codec = _CommandAwarePayloadCodec( workflow.instance, @@ -335,14 +343,6 @@ async def _handle_activation( completion.failed.failure.SetInParent() try: - data_converter = self._data_converter - if workflow_id: - data_converter = data_converter._with_context( - temporalio.converter.WorkflowSerializationContext( - namespace=self._namespace, - workflow_id=workflow_id, - ) - ) data_converter.failure_converter.to_failure( err, data_converter.payload_converter, From afbc0fbaa58b5f7e1d996f0862c429f4e0791413 Mon Sep 17 00:00:00 2001 From: Dan Davison Date: Thu, 25 Sep 2025 08:42:27 -0400 Subject: [PATCH 35/81] Cleanup --- temporalio/converter.py | 4 +- temporalio/worker/_workflow_instance.py | 7 +++- tests/test_serialization_context.py | 50 ++++++++++++++++--------- 3 files changed, 40 insertions(+), 21 deletions(-) diff --git a/temporalio/converter.py b/temporalio/converter.py index 6d1c07f63..f0fb8c4e0 100644 --- a/temporalio/converter.py +++ b/temporalio/converter.py @@ -136,6 +136,7 @@ class ActivitySerializationContext(BaseWorkflowSerializationContext): is_local: bool +# TODO: duck typing or nominal typing? class WithSerializationContext(ABC): """Interface for objects that can use serialization context. @@ -341,7 +342,6 @@ def __init__(self, *converters: EncodingPayloadConverter) -> None: Args: converters: Payload converters to delegate to, in order. """ - # Insertion order preserved here since Python 3.7 self.converters = {c.encoding.encode(): c for c in converters} def to_payloads( @@ -407,7 +407,7 @@ def from_payloads( return values def with_context(self, context: SerializationContext) -> CompositePayloadConverter: - """Return a new instance with the given context.""" + """Return a new instance with context set on the component converters""" return CompositePayloadConverter( *( c.with_context(context) diff --git a/temporalio/worker/_workflow_instance.py b/temporalio/worker/_workflow_instance.py index 3cba6bee1..f25d0e290 100644 --- a/temporalio/worker/_workflow_instance.py +++ b/temporalio/worker/_workflow_instance.py @@ -2087,7 +2087,12 @@ def _converters_with_context( temporalio.converter.PayloadConverter, temporalio.converter.FailureConverter, ]: - """Construct workflow payload and failure converters with the given context.""" + """Construct workflow payload and failure converters with the given context. + + This plays a similar role to DataConverter._with_context, but operates on PayloadConverter + and FailureConverter only (since payload encoding/decoding is done by the worker, outside + the workflowsandbox). + """ payload_converter = self._context_free_payload_converter failure_converter = self._context_free_failure_converter if isinstance(payload_converter, temporalio.converter.WithSerializationContext): diff --git a/tests/test_serialization_context.py b/tests/test_serialization_context.py index 89424e5d3..6a7429f94 100644 --- a/tests/test_serialization_context.py +++ b/tests/test_serialization_context.py @@ -21,9 +21,9 @@ from pydantic import BaseModel from typing_extensions import Never +import temporalio.api.common.v1 +import temporalio.api.failure.v1 from temporalio import activity, workflow -from temporalio.api.common.v1 import Payload -from temporalio.api.failure.v1 import Failure from temporalio.client import Client, WorkflowFailureError, WorkflowUpdateFailedError from temporalio.common import RetryPolicy from temporalio.contrib.pydantic import PydanticJSONPlainPayloadConverter @@ -82,7 +82,7 @@ def with_context( converter.context = context return converter - def to_payload(self, value: Any) -> Optional[Payload]: + def to_payload(self, value: Any) -> Optional[temporalio.api.common.v1.Payload]: if not isinstance(value, TraceData): return None if isinstance(self.context, WorkflowSerializationContext): @@ -106,7 +106,11 @@ def to_payload(self, value: Any) -> Optional[Payload]: payload.metadata["encoding"] = self.encoding.encode() return payload - def from_payload(self, payload: Payload, type_hint: Optional[Type] = None) -> Any: + def from_payload( + self, + payload: temporalio.api.common.v1.Payload, + type_hint: Optional[Type] = None, + ) -> Any: value = JSONPlainPayloadConverter().from_payload(payload, TraceData) assert isinstance(value, TraceData) if isinstance(self.context, WorkflowSerializationContext): @@ -1002,7 +1006,7 @@ def __init__(self): def with_context( self, context: Optional[SerializationContext] - ) -> "FailureConverterWithContext": + ) -> FailureConverterWithContext: converter = FailureConverterWithContext() converter.context = context return converter @@ -1011,7 +1015,7 @@ def to_failure( self, exception: BaseException, payload_converter: PayloadConverter, - failure: Failure, + failure: temporalio.api.failure.v1.Failure, ) -> None: assert isinstance( self.context, (WorkflowSerializationContext, ActivitySerializationContext) @@ -1025,7 +1029,9 @@ def to_failure( super().to_failure(exception, payload_converter, failure) def from_failure( - self, failure: Failure, payload_converter: PayloadConverter + self, + failure: temporalio.api.failure.v1.Failure, + payload_converter: PayloadConverter, ) -> BaseException: assert isinstance( self.context, (WorkflowSerializationContext, ActivitySerializationContext) @@ -1132,12 +1138,14 @@ def __init__(self): def with_context( self, context: Optional[SerializationContext] - ) -> "PayloadCodecWithContext": + ) -> PayloadCodecWithContext: codec = PayloadCodecWithContext() codec.context = context return codec - async def encode(self, payloads: Sequence[Payload]) -> List[Payload]: + async def encode( + self, payloads: Sequence[temporalio.api.common.v1.Payload] + ) -> List[temporalio.api.common.v1.Payload]: assert self.context if isinstance(self.context, ActivitySerializationContext): test_traces[self.context.workflow_id].append( @@ -1156,7 +1164,9 @@ async def encode(self, payloads: Sequence[Payload]) -> List[Payload]: ) return list(payloads) - async def decode(self, payloads: Sequence[Payload]) -> List[Payload]: + async def decode( + self, payloads: Sequence[temporalio.api.common.v1.Payload] + ) -> List[temporalio.api.common.v1.Payload]: assert self.context if isinstance(self.context, ActivitySerializationContext): test_traces[self.context.workflow_id].append( @@ -1439,20 +1449,24 @@ def with_context( codec.context = context return codec - async def encode(self, payloads: Sequence[Payload]) -> List[Payload]: + async def encode( + self, payloads: Sequence[temporalio.api.common.v1.Payload] + ) -> List[temporalio.api.common.v1.Payload]: [payload] = payloads return [ - Payload( + temporalio.api.common.v1.Payload( metadata=payload.metadata, data=json.dumps(self._get_encryption_key()).encode(), ) ] - async def decode(self, payloads: Sequence[Payload]) -> List[Payload]: + async def decode( + self, payloads: Sequence[temporalio.api.common.v1.Payload] + ) -> List[temporalio.api.common.v1.Payload]: [payload] = payloads assert json.loads(payload.data.decode()) == self._get_encryption_key() metadata = dict(payload.metadata) - return [Payload(metadata=metadata, data=b'"inbound"')] + return [temporalio.api.common.v1.Payload(metadata=metadata, data=b'"inbound"')] def _get_encryption_key(self) -> str: context = ( @@ -1594,8 +1608,8 @@ def with_context( return codec async def _assert_context_iff_not_nexus( - self, payloads: Sequence[Payload] - ) -> List[Payload]: + self, payloads: Sequence[temporalio.api.common.v1.Payload] + ) -> List[temporalio.api.common.v1.Payload]: [payload] = payloads assert bool(self.context) == (payload.data.decode() != '"nexus-data"') return list(payloads) @@ -1670,12 +1684,12 @@ def __init__(self): def with_context( self, context: Optional[SerializationContext] - ) -> "PydanticJSONConverterWithContext": + ) -> PydanticJSONConverterWithContext: converter = PydanticJSONConverterWithContext() converter.context = context return converter - def to_payload(self, value: Any) -> Optional[Payload]: + def to_payload(self, value: Any) -> Optional[temporalio.api.common.v1.Payload]: if isinstance(value, PydanticData) and self.context: if isinstance(self.context, WorkflowSerializationContext): value.trace.append(f"wf_{self.context.workflow_id}") From 0be8d6e78bf644649f2b64733db0d4fdf864cedc Mon Sep 17 00:00:00 2001 From: Dan Davison Date: Thu, 25 Sep 2025 15:30:17 -0400 Subject: [PATCH 36/81] Appease mypy --- temporalio/worker/_workflow_instance.py | 28 ++++++++++++------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/temporalio/worker/_workflow_instance.py b/temporalio/worker/_workflow_instance.py index f25d0e290..4e69551e4 100644 --- a/temporalio/worker/_workflow_instance.py +++ b/temporalio/worker/_workflow_instance.py @@ -2124,38 +2124,38 @@ def get_payload_codec_with_context( return payload_codec.with_context(workflow_context) if command_seq in self._pending_activities: - handle = self._pending_activities[command_seq] - context = temporalio.converter.ActivitySerializationContext( + act_handle = self._pending_activities[command_seq] + act_context = temporalio.converter.ActivitySerializationContext( namespace=self._info.namespace, workflow_id=self._info.workflow_id, workflow_type=self._info.workflow_type, - activity_type=handle._input.activity, + activity_type=act_handle._input.activity, activity_task_queue=( - handle._input.task_queue - if isinstance(handle._input, StartActivityInput) - and handle._input.task_queue + act_handle._input.task_queue + if isinstance(act_handle._input, StartActivityInput) + and act_handle._input.task_queue else self._info.task_queue ), - is_local=isinstance(handle._input, StartLocalActivityInput), + is_local=isinstance(act_handle._input, StartLocalActivityInput), ) - return payload_codec.with_context(context) + return payload_codec.with_context(act_context) elif command_seq in self._pending_child_workflows: - handle = self._pending_child_workflows[command_seq] - context = temporalio.converter.WorkflowSerializationContext( + cwf_handle = self._pending_child_workflows[command_seq] + wf_context = temporalio.converter.WorkflowSerializationContext( namespace=self._info.namespace, - workflow_id=handle._input.id, + workflow_id=cwf_handle._input.id, ) - return payload_codec.with_context(context) + return payload_codec.with_context(wf_context) elif command_seq in self._pending_external_signals: # Use the target workflow's context for external signals _, workflow_id = self._pending_external_signals[command_seq] - context = temporalio.converter.WorkflowSerializationContext( + wf_context = temporalio.converter.WorkflowSerializationContext( namespace=self._info.namespace, workflow_id=workflow_id, ) - return payload_codec.with_context(context) + return payload_codec.with_context(wf_context) elif command_seq in self._pending_nexus_operations: # Use empty context for nexus operations: users will never want to encrypt using a From 6bb4ae2b4d656a9a40afaac4429c0098c3816c89 Mon Sep 17 00:00:00 2001 From: Dan Davison Date: Thu, 25 Sep 2025 15:35:45 -0400 Subject: [PATCH 37/81] cleanup --- requirements.md | 185 ---------------------------------------- temporalio/client.py | 2 + temporalio/converter.py | 19 +++-- 3 files changed, 15 insertions(+), 191 deletions(-) delete mode 100644 requirements.md diff --git a/requirements.md b/requirements.md deleted file mode 100644 index b8be9517a..000000000 --- a/requirements.md +++ /dev/null @@ -1,185 +0,0 @@ -# Serialization Context Requirements - -## Simple Workflow Context - -**Sequence:** `client --start--> workflow --result--> client` - -**Rule:** All operations use `WorkflowSerializationContext` with the target workflow's ID and namespace. - -| Operation | Context Type | .NET | Java | Python | -|-----------|--------------|------|------|--------| -| **1. Client Starts Workflow** | | | | | -| Serialize DataConverter | WorkflowSerializationContext | [TemporalClient.Workflow.cs:693-696](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Client/TemporalClient.Workflow.cs#L693-L696) | [RootWorkflowClientInvoker.java:62-66](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/client/RootWorkflowClientInvoker.java#L62-L66) | [client.py:5993-5998](https://github.com/temporalio/sdk-python/blob/main/temporalio/client.py#L5993-L5998) | -| Encode PayloadCodec | WorkflowSerializationContext | [TemporalClient.Workflow.cs:699-700](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Client/TemporalClient.Workflow.cs#L699-L700) | [RootWorkflowClientInvoker.java:69](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/client/RootWorkflowClientInvoker.java#L69) | [client.py:6005](https://github.com/temporalio/sdk-python/blob/main/temporalio/client.py#L6005) | -| **2. Workflow Receives Input** | | | | | -| Decode PayloadCodec | WorkflowSerializationContext | [WorkflowCodecHelper.cs:125-126](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowCodecHelper.cs#L125-L126) | [ReplayWorkflowTaskHandler.java:160-162](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/replay/ReplayWorkflowTaskHandler.java#L160-L162) | [_workflow.py:289-293](https://github.com/temporalio/sdk-python/blob/main/temporalio/worker/_workflow.py#L289-L293) | -| Deserialize DataConverter | WorkflowSerializationContext | [WorkflowInstance.cs:395-397](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowInstance.cs#L395-L397) | [SyncWorkflow.java:75-77](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/sync/SyncWorkflow.java#L75-L77) | [_workflow.py:278-283](https://github.com/temporalio/sdk-python/blob/main/temporalio/worker/_workflow.py#L278-L283) | -| **3. Workflow Returns Result** | | | | | -| Serialize DataConverter | WorkflowSerializationContext | [WorkflowInstance.cs:1182-1184](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowInstance.cs#L1182-L1184) | [POJOWorkflowImplementationFactory.java:290-292](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/sync/POJOWorkflowImplementationFactory.java#L290-L292) | [_workflow.py:340-346](https://github.com/temporalio/sdk-python/blob/main/temporalio/worker/_workflow.py#L340-L346) | -| Encode PayloadCodec | WorkflowSerializationContext | [WorkflowCodecHelper.cs:217-220](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowCodecHelper.cs#L217-L220) | [WorkflowWorker.java:702-704](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/worker/WorkflowWorker.java#L702-L704) | [_workflow.py:363-367](https://github.com/temporalio/sdk-python/blob/main/temporalio/worker/_workflow.py#L363-L367) | -| **4. Client Receives Result** | | | | | -| Decode PayloadCodec | WorkflowSerializationContext | [WorkflowHandle.cs:84-87](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Client/WorkflowHandle.cs#L84-L87) | [RootWorkflowClientInvoker.java:324-327](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/client/RootWorkflowClientInvoker.java#L324-L327) | [client.py:1601-1605](https://github.com/temporalio/sdk-python/blob/main/temporalio/client.py#L1601-L1605) | -| Deserialize DataConverter | WorkflowSerializationContext | [WorkflowHandle.cs:129-130](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Client/WorkflowHandle.cs#L129-L130) | [RootWorkflowClientInvoker.java:318-322](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/client/RootWorkflowClientInvoker.java#L318-L322) | [client.py:1715-1718](https://github.com/temporalio/sdk-python/blob/main/temporalio/client.py#L1715-L1718) | - -## Activity Context - -**Sequence:** `workflow --scheduleActivity--> activity --result--> workflow` - -**Rule:** All operations use `ActivitySerializationContext` with activity type, task queue, and is_local flag. - -| Operation | Context Type | .NET | Java | Python | -|-----------|--------------|------|------|--------| -| **1. Workflow Schedules Activity** | | | | | -| Serialize DataConverter | ActivitySerializationContext | [WorkflowInstance.cs:2084-2093](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowInstance.cs#L2084-L2093) | [SyncWorkflowContext.java:257-267](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/sync/SyncWorkflowContext.java#L257-L267) | [_workflow_instance.py:2968-2972](https://github.com/temporalio/sdk-python/blob/main/temporalio/worker/_workflow_instance.py#L2968-L2972) | -| Encode PayloadCodec | ActivitySerializationContext | [WorkflowCodecHelper.cs:243-253](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowCodecHelper.cs#L243-L253) | [SyncWorkflowContext.java:268-270](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/sync/SyncWorkflowContext.java#L268-L270) | [_workflow.py:716-733](https://github.com/temporalio/sdk-python/blob/main/temporalio/worker/_workflow.py#L716-L733) | -| **2. Activity Receives Input** | | | | | -| Decode PayloadCodec | ActivitySerializationContext | [ActivityWorker.cs:389-395](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/ActivityWorker.cs#L389-L395) | [ActivityTaskExecutors.java:70-71](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/activity/ActivityTaskExecutors.java#L70-L71) | [_activity.py:317-318](https://github.com/temporalio/sdk-python/blob/main/temporalio/worker/_activity.py#L317-L318) | -| Deserialize DataConverter | ActivitySerializationContext | [ActivityWorker.cs:376-380](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/ActivityWorker.cs#L376-L380) | [ActivityTaskExecutors.java:83](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/activity/ActivityTaskExecutors.java#L83) | [_activity.py:526-530](https://github.com/temporalio/sdk-python/blob/main/temporalio/worker/_activity.py#L526-L530) | -| **3. Activity Returns Result** | | | | | -| Serialize DataConverter | ActivitySerializationContext | [ActivityWorker.cs:417-418](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/ActivityWorker.cs#L417-L418) | [ActivityTaskExecutors.java:159-161](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/activity/ActivityTaskExecutors.java#L159-L161) | [_activity.py:322-323](https://github.com/temporalio/sdk-python/blob/main/temporalio/worker/_activity.py#L322-L323) | -| Encode PayloadCodec | ActivitySerializationContext | [ActivityWorker.cs:389-395](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/ActivityWorker.cs#L389-L395) | [ActivityTaskExecutors.java:70-71](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/activity/ActivityTaskExecutors.java#L70-L71) | [_activity.py:317-318](https://github.com/temporalio/sdk-python/blob/main/temporalio/worker/_activity.py#L317-L318) | -| **4. Workflow Receives Result** | | | | | -| Decode PayloadCodec | ActivitySerializationContext | [WorkflowCodecHelper.cs:68-76](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowCodecHelper.cs#L68-L76) | [SyncWorkflowContext.java:273-274](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/sync/SyncWorkflowContext.java#L273-L274) | [_workflow.py:716-745](https://github.com/temporalio/sdk-python/blob/main/temporalio/worker/_workflow.py#L716-L745) | -| Deserialize DataConverter | ActivitySerializationContext | [WorkflowInstance.cs:2720](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowInstance.cs#L2720) | [SyncWorkflowContext.java:286-288](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/sync/SyncWorkflowContext.java#L286-L288) | [_workflow_instance.py:804-810](https://github.com/temporalio/sdk-python/blob/main/temporalio/worker/_workflow_instance.py#L804-L810) | - -## Child Workflow Context - -**Sequence:** `workflow --startChildWorkflow--> childWorkflow --result--> workflow` - -**Rule:** Child operations use `WorkflowSerializationContext` with the child workflow's ID. - -| Operation | Context Type | .NET | Java | Python | -|-----------|--------------|------|------|--------| -| **1. Workflow Starts Child** | | | | | -| Serialize DataConverter | WorkflowSerializationContext (child ID) | [WorkflowInstance.cs:2319-2326](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowInstance.cs#L2319-L2326) | [SyncWorkflowContext.java:687-690](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/sync/SyncWorkflowContext.java#L687-L690) | [_workflow_instance.py:3124-3128](https://github.com/temporalio/sdk-python/blob/main/temporalio/worker/_workflow_instance.py#L3124-L3128) | -| Encode PayloadCodec | WorkflowSerializationContext (child ID) | [WorkflowCodecHelper.cs:301-310](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowCodecHelper.cs#L301-L310) | [SyncWorkflowContext.java:690](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/sync/SyncWorkflowContext.java#L690) | [_workflow.py:716-745](https://github.com/temporalio/sdk-python/blob/main/temporalio/worker/_workflow.py#L716-L745) | -| **2. Child Receives Input** | | | | | -| Decode PayloadCodec | WorkflowSerializationContext (child ID) | [WorkflowCodecHelper.cs:125-126](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowCodecHelper.cs#L125-L126) | [ReplayWorkflowTaskHandler.java:160-162](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/replay/ReplayWorkflowTaskHandler.java#L160-L162) | [_workflow.py:289-293](https://github.com/temporalio/sdk-python/blob/main/temporalio/worker/_workflow.py#L289-L293) | -| Deserialize DataConverter | WorkflowSerializationContext (child ID) | [WorkflowInstance.cs:395-397](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowInstance.cs#L395-L397) | [SyncWorkflow.java:75-77](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/sync/SyncWorkflow.java#L75-L77) | [_workflow.py:278-283](https://github.com/temporalio/sdk-python/blob/main/temporalio/worker/_workflow.py#L278-L283) | -| **3. Child Returns Result** | | | | | -| Serialize DataConverter | WorkflowSerializationContext (child ID) | [WorkflowInstance.cs:1182-1184](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowInstance.cs#L1182-L1184) | [POJOWorkflowImplementationFactory.java:290-292](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/sync/POJOWorkflowImplementationFactory.java#L290-L292) | [_workflow.py:340-346](https://github.com/temporalio/sdk-python/blob/main/temporalio/worker/_workflow.py#L340-L346) | -| Encode PayloadCodec | WorkflowSerializationContext (child ID) | [WorkflowCodecHelper.cs:217-220](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowCodecHelper.cs#L217-L220) | [WorkflowWorker.java:702-704](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/worker/WorkflowWorker.java#L702-L704) | [_workflow.py:363-367](https://github.com/temporalio/sdk-python/blob/main/temporalio/worker/_workflow.py#L363-L367) | -| **4. Workflow Receives Child Result** | | | | | -| Decode PayloadCodec | WorkflowSerializationContext (child ID) | [WorkflowCodecHelper.cs:78-85](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowCodecHelper.cs#L78-L85) | [SyncWorkflowContext.java:1245-1248](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/sync/SyncWorkflowContext.java#L1245-L1248) | [_workflow.py:716-745](https://github.com/temporalio/sdk-python/blob/main/temporalio/worker/_workflow.py#L716-L745) | -| Deserialize DataConverter | WorkflowSerializationContext (child ID) | [WorkflowInstance.cs:2808](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowInstance.cs#L2808) | [SyncWorkflowContext.java:1245-1248](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/sync/SyncWorkflowContext.java#L1245-L1248) | [_workflow_instance.py:842-858](https://github.com/temporalio/sdk-python/blob/main/temporalio/worker/_workflow_instance.py#L842-L858) | - -## Nexus Operation Context - -**Sequence:** `workflow --scheduleNexusOperation--> nexusOperation --result--> workflow` - -**Rule:** Nexus operations have **no serialization context** (null/none). - -| Operation | Context Type | .NET | Java | Python | -|-----------|--------------|------|------|--------| -| **1. Workflow Schedules Nexus Op** | | | | | -| Serialize DataConverter | None | [WorkflowInstance.cs:2525](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowInstance.cs#L2525) | [SyncWorkflowContext.java:790](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/sync/SyncWorkflowContext.java#L790) | N/A (not yet implemented) | -| Encode PayloadCodec | None | [WorkflowCodecHelper.cs:355-356](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowCodecHelper.cs#L355-L356) | [SyncWorkflowContext.java:791-794](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/sync/SyncWorkflowContext.java#L791-L794) | N/A (not yet implemented) | -| **2. Nexus Operation Receives Input** | | | | | -| Decode PayloadCodec | None | N/A (handler-side) | N/A (handler-side) | N/A (handler-side) | -| Deserialize DataConverter | None | N/A (handler-side) | N/A (handler-side) | N/A (handler-side) | -| **3. Nexus Operation Returns Result** | | | | | -| Serialize DataConverter | None | N/A (handler-side) | N/A (handler-side) | N/A (handler-side) | -| Encode PayloadCodec | None | N/A (handler-side) | N/A (handler-side) | N/A (handler-side) | -| **4. Workflow Receives Result** | | | | | -| Decode PayloadCodec | None | [WorkflowCodecHelper.cs:112-113](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowCodecHelper.cs#L112-L113) | N/A (codec not applied) | N/A (not yet implemented) | -| Deserialize DataConverter | None | [WorkflowInstance.cs:2954](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowInstance.cs#L2954) | [SyncWorkflowContext.java:855-856](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/sync/SyncWorkflowContext.java#L855-L856) | N/A (not yet implemented) | - -## Memo and Search Attribute Context - -### Memos - -**Rule:** Memos always use `WorkflowSerializationContext` with the workflow's ID when set or accessed. - -| Operation | Context Type | .NET | Java | Python | -|-----------|--------------|------|------|--------| -| **Client sets memo on start** | WorkflowSerializationContext | [TemporalClient.Workflow.cs:827](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Client/TemporalClient.Workflow.cs#L827) | [RootWorkflowClientInvoker.java:292-294](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/client/RootWorkflowClientInvoker.java#L292-L294) | [client.py:6027-6028](https://github.com/temporalio/sdk-python/blob/main/temporalio/client.py#L6027-L6028) | -| **Workflow upserts memo** | WorkflowSerializationContext | [WorkflowInstance.cs:472](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowInstance.cs#L472) | [SyncWorkflowContext.java:1416](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/sync/SyncWorkflowContext.java#L1416) | [_workflow_instance.py:908-912](https://github.com/temporalio/sdk-python/blob/main/temporalio/worker/_workflow_instance.py#L908-L912) | -| **Child workflow memo** | WorkflowSerializationContext (child ID) | [WorkflowInstance.cs:2359-2360](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Worker/WorkflowInstance.cs#L2359-L2360) | [SyncWorkflowContext.java:693-700](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/sync/SyncWorkflowContext.java#L693-L700) | [_workflow_instance.py:3156-3159](https://github.com/temporalio/sdk-python/blob/main/temporalio/worker/_workflow_instance.py#L3156-L3159) | -| **Schedule sets memo** | WorkflowSerializationContext | [ScheduleActionStartWorkflow.cs:199](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Client/Schedules/ScheduleActionStartWorkflow.cs#L199) | [ScheduleProtoUtil.java:134](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/client/ScheduleProtoUtil.java#L134) | [client.py:4220-4229](https://github.com/temporalio/sdk-python/blob/main/temporalio/client.py#L4220-L4229) | - -### Search Attributes - -**Rule:** Search attributes do NOT use serialization context - they use specialized converters for indexing. - -| Operation | Context Type | .NET | Java | Python | -|-----------|--------------|------|------|--------| -| **All operations** | None (direct proto conversion) | Uses `ToProto()` | Uses `toSearchAttributes()` | [converter.py:1358-1363](https://github.com/temporalio/sdk-python/blob/main/temporalio/converter.py#L1358-L1363) | - -## User-Accessible Data Converter - -### Workflow Context - -**Rule:** Data converters exposed to workflow code have `WorkflowSerializationContext` applied. - -| SDK | API | Context | Reference | -|-----|-----|---------|-----------| -| **.NET** | `Workflow.PayloadConverter` | WorkflowSerializationContext | [Workflow.cs:185](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Workflows/Workflow.cs#L185) | -| **Java** | Not directly exposed | N/A | Workflow code cannot access data converter | -| **Python** | `workflow.payload_converter()` | WorkflowSerializationContext | [workflow.py:1148](https://github.com/temporalio/sdk-python/blob/main/temporalio/workflow.py#L1148) | - -### Activity Context - -**Rule:** Data converters exposed to activity code have `ActivitySerializationContext` applied. - -| SDK | API | Context | Reference | -|-----|-----|---------|-----------| -| **.NET** | `ActivityExecutionContext.PayloadConverter` | ActivitySerializationContext | [ActivityExecutionContext.cs:49](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Activities/ActivityExecutionContext.cs#L49) | -| **Java** | Not directly exposed | N/A | Activity code cannot access data converter | -| **Python** | `activity.payload_converter()` | ActivitySerializationContext | [activity.py:470](https://github.com/temporalio/sdk-python/blob/main/temporalio/activity.py#L470) | - -## Async Activity Completion - -**Rule:** Async activity completion uses `ActivitySerializationContext` with available activity information. - -| Operation | Context Type | .NET | Java | Python | -|-----------|--------------|------|------|--------| -| **Complete** | ActivitySerializationContext | [AsyncActivityHandle.cs:42](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Client/AsyncActivityHandle.cs#L42) | [ActivityCompletionClientImpl.java:51-56](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/client/ActivityCompletionClientImpl.java#L51-L56) | [client.py:6477-6481](https://github.com/temporalio/sdk-python/blob/main/temporalio/client.py#L6477-L6481) | -| **Fail** | ActivitySerializationContext | [AsyncActivityHandle.cs:51](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Client/AsyncActivityHandle.cs#L51) | [ActivityCompletionClientImpl.java:63-65](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/client/ActivityCompletionClientImpl.java#L63-L65) | [client.py:6511-6514](https://github.com/temporalio/sdk-python/blob/main/temporalio/client.py#L6511-L6514) | -| **Report Cancellation** | ActivitySerializationContext | [AsyncActivityHandle.cs:62](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Client/AsyncActivityHandle.cs#L62) | [ActivityCompletionClientImpl.java:74](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/client/ActivityCompletionClientImpl.java#L74) | [client.py:6588-6600](https://github.com/temporalio/sdk-python/blob/main/temporalio/client.py#L6588-L6600) | -| **WithContext Method** | Creates context-aware handle | [AsyncActivityHandle.cs:71-73](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Client/AsyncActivityHandle.cs#L71-L73) | [ActivityCompletionClient.java:107-108](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/client/ActivityCompletionClientImpl.java#L107-L108) | N/A (context applied internally) | - -### Notes -- Python applies partial context internally in `_async_activity_data_converter()` with workflow ID when available -- .NET and Java allow explicit context application via `WithSerializationContext()`/`withContext()` methods -- Context may be incomplete for async completion (e.g., missing activity type) when using task token - -## Activity Heartbeating - -**Rule:** Heartbeating uses `ActivitySerializationContext` with full activity information when available. - -| Operation | Context Source | .NET | Java | Python | -|-----------|---------------|------|------|--------| -| **During Execution** | From running activity info | Same as async completion | [HeartbeatContextImpl.java:67-75](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/internal/activity/HeartbeatContextImpl.java#L67-L75) | [_activity.py:257-265](https://github.com/temporalio/sdk-python/blob/main/temporalio/worker/_activity.py#L257-L265) | -| **Async Heartbeat** | From task token or activity ID | [AsyncActivityHandle.cs:30-32](https://github.com/temporalio/sdk-dotnet/blob/main/src/Temporalio/Client/AsyncActivityHandle.cs#L30-L32) | [ActivityCompletionClientImpl.java:94-102](https://github.com/temporalio/sdk-java/blob/master/temporal-sdk/src/main/java/io/temporal/client/ActivityCompletionClientImpl.java#L94-L102) | [client.py:6422-6426](https://github.com/temporalio/sdk-python/blob/main/temporalio/client.py#L6422-L6426) | - -### Notes -- Heartbeating during activity execution has full context information -- Async heartbeating (outside activity execution) may have partial context -- All SDKs apply context to ensure proper data conversion for heartbeat details - -## Summary of Context Rules - -1. **Workflow operations**: Always use `WorkflowSerializationContext` with the target workflow's ID -2. **Activity operations**: Use `ActivitySerializationContext` with activity details and `is_local` flag -3. **Child workflow operations**: Use `WorkflowSerializationContext` with the child's workflow ID -4. **Nexus operations**: No serialization context (null/none) -5. **Memos**: Always use workflow context -6. **Search attributes**: Never use context (indexing-specific conversion) -7. **User-exposed converters**: Have appropriate context pre-applied -8. **Async activity completion**: Use activity context with available information -9. **Heartbeating**: Use activity context with full info during execution, partial for async - -## Important Note About .NET - -⚠️ **Current .NET Bug** ([temporalio/sdk-dotnet#523](https://github.com/temporalio/sdk-dotnet/issues/523) - **OPEN**) - -The .NET SDK currently has a bug where it incorrectly applies `WorkflowSerializationContext` to **all** data conversion operations within a workflow context, including: -- Activities (should use `ActivitySerializationContext`) -- Nexus operations (should use no context) - -**This document shows the intended/correct behavior**, which is: -- How Java currently works ✅ -- How Python should work after alignment ✅ -- How .NET **should** work (but doesn't yet) ⚠️ - -The tables above reflect the desired state where each SDK applies context selectively based on the operation type. diff --git a/temporalio/client.py b/temporalio/client.py index 05413828f..d574b590d 100644 --- a/temporalio/client.py +++ b/temporalio/client.py @@ -2852,6 +2852,8 @@ async def report_cancellation( ), ) + # TODO(dan): should this return Self (requiring that the user's subclass has the same + # constructor signature)? def with_context(self, context: SerializationContext) -> AsyncActivityHandle: """Create a new AsyncActivityHandle with a different serialization context. diff --git a/temporalio/converter.py b/temporalio/converter.py index f0fb8c4e0..c82a15b9c 100644 --- a/temporalio/converter.py +++ b/temporalio/converter.py @@ -138,18 +138,25 @@ class ActivitySerializationContext(BaseWorkflowSerializationContext): # TODO: duck typing or nominal typing? class WithSerializationContext(ABC): - """Interface for objects that can use serialization context. - - This is similar to the .NET IWithSerializationContext interface. - Objects implementing this interface can receive contextual information - during serialization and deserialization. + """Interface for classes that can use serialization context. + + The following classes may implement this interface: + - :py:class:`PayloadConverter` + - :py:class:`PayloadCodec` + - :py:class:`FailureConverter` + - :py:class:`EncodingPayloadConverter` + + During data converter operations (encoding/decoding, serialization/deserialization, and failure + conversion), instances of classes implementing this interface will be replaced by the result of + calling with_context(context). This allows overridden methods (encode/decode, + to_payload/from_payload, etc) to use the context. """ def with_context(self, context: SerializationContext) -> Self: """Return a copy of this object configured to use the given context. Args: - context: The serialization context to use, or None for no context. + context: The serialization context to use. Returns: A new instance configured with the context. From 1e691abec56e78427abcdca53bd7cdc81b64d5a2 Mon Sep 17 00:00:00 2001 From: Dan Davison Date: Fri, 26 Sep 2025 05:35:11 -0400 Subject: [PATCH 38/81] Skip nexus tests under Java test server --- tests/test_serialization_context.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/test_serialization_context.py b/tests/test_serialization_context.py index 6a7429f94..e9d3d69b8 100644 --- a/tests/test_serialization_context.py +++ b/tests/test_serialization_context.py @@ -42,6 +42,7 @@ WorkflowSerializationContext, ) from temporalio.exceptions import ApplicationError +from temporalio.testing import WorkflowEnvironment from temporalio.worker import Worker from temporalio.worker._workflow_instance import UnsandboxedWorkflowRunner from tests.helpers.nexus import create_nexus_endpoint, make_nexus_endpoint_name @@ -1640,12 +1641,15 @@ async def run(self, data: str) -> None: async def test_nexus_payload_codec_operations_lack_context( - client: Client, + env: WorkflowEnvironment, ): """ encode() and decode() on nexus payloads should not have any context set. """ - config = client.config() + if env.supports_time_skipping: + pytest.skip("Nexus tests don't work with the Java test server") + + config = env.client.config() config["data_converter"] = dataclasses.replace( DataConverter.default, payload_codec=AssertNexusLacksContextPayloadCodec(), From b6f49d54bed9562bd88614292fb124a86e85b43f Mon Sep 17 00:00:00 2001 From: Dan Davison Date: Thu, 25 Sep 2025 15:42:54 -0400 Subject: [PATCH 39/81] Install clippy --- .github/workflows/ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6289dbcd0..44e3741d0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -148,6 +148,8 @@ jobs: with: submodules: recursive - uses: dtolnay/rust-toolchain@stable + with: + components: "clippy" - uses: Swatinem/rust-cache@v2 with: workspaces: temporalio/bridge -> target From febca8b598215dbea6afe6f3f8e3e3a200b93614 Mon Sep 17 00:00:00 2001 From: Dan Davison Date: Thu, 25 Sep 2025 09:25:29 -0400 Subject: [PATCH 40/81] Failing test: customized default payload converter --- tests/test_serialization_context.py | 81 +++++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) diff --git a/tests/test_serialization_context.py b/tests/test_serialization_context.py index e9d3d69b8..3c88d17da 100644 --- a/tests/test_serialization_context.py +++ b/tests/test_serialization_context.py @@ -1743,3 +1743,84 @@ async def test_pydantic_converter_with_context(client: Client): task_queue=task_queue, ) assert f"wf_{wf_id}" in result.trace + + +# Test customized DefaultPayloadConverter + +# The SDK's CompositePayloadConverter comes with a with_context implementation that ensures that its +# component EncodingPayloadConverters will be replaced with the results of calling with_context() on +# them, if they support with_context (this happens when we call data_converter._with_context). In +# this test, the user has subclassed CompositePayloadConverter. The test confirms that the +# CompositePayloadConverter's with_context yields an instance of the user's subclass. + + +class UserMethodCalledError(Exception): + pass + + +class CustomEncodingPayloadConverter( + JSONPlainPayloadConverter, WithSerializationContext +): + @property + def encoding(self) -> str: + return "custom-encoding-that-does-not-clash-with-default-converters" + + def __init__(self): + super().__init__() + self.context: Optional[SerializationContext] = None + + def with_context( + self, context: Optional[SerializationContext] + ) -> CustomEncodingPayloadConverter: + converter = CustomEncodingPayloadConverter() + converter.context = context + return converter + + +class CustomPayloadConverter(CompositePayloadConverter): + def __init__(self): + # Add a context-aware EncodingPayloadConverter so that + # CompositePayloadConverter.with_context is forced to construct and return a new instance. + super().__init__( + CustomEncodingPayloadConverter(), + *DefaultPayloadConverter.default_encoding_payload_converters, + ) + + def to_payloads( + self, values: Sequence[Any] + ) -> List[temporalio.api.common.v1.Payload]: + raise UserMethodCalledError + + def from_payloads( + self, + payloads: Sequence[temporalio.api.common.v1.Payload], + type_hints: Optional[List[Type]] = None, + ) -> List[Any]: + raise NotImplementedError + + +async def test_user_customization_of_default_payload_converter( + client: Client, +): + wf_id = str(uuid.uuid4()) + task_queue = str(uuid.uuid4()) + + client_config = client.config() + client_config["data_converter"] = dataclasses.replace( + DataConverter.default, + payload_converter_class=CustomPayloadConverter, + ) + client = Client(**client_config) + + async with Worker( + client, + task_queue=task_queue, + workflows=[EchoWorkflow], + ): + with pytest.raises(UserMethodCalledError): + await client.execute_workflow( + EchoWorkflow.run, + TraceData(), + id=wf_id, + task_queue=task_queue, + ) From ff732287b40e9894bac099afcb7e14e8918c471b Mon Sep 17 00:00:00 2001 From: Dan Davison Date: Thu, 25 Sep 2025 15:17:18 -0400 Subject: [PATCH 41/81] Fix test: return Self --- temporalio/converter.py | 44 ++++++++++++++++++++++++++++++----------- 1 file changed, 33 insertions(+), 11 deletions(-) diff --git a/temporalio/converter.py b/temporalio/converter.py index c82a15b9c..ad063c16d 100644 --- a/temporalio/converter.py +++ b/temporalio/converter.py @@ -349,6 +349,9 @@ def __init__(self, *converters: EncodingPayloadConverter) -> None: Args: converters: Payload converters to delegate to, in order. """ + self._set_converters(*converters) + + def _set_converters(self, *converters: EncodingPayloadConverter) -> None: self.converters = {c.encoding.encode(): c for c in converters} def to_payloads( @@ -413,16 +416,26 @@ def from_payloads( ) from err return values - def with_context(self, context: SerializationContext) -> CompositePayloadConverter: - """Return a new instance with context set on the component converters""" - return CompositePayloadConverter( - *( - c.with_context(context) - if isinstance(c, WithSerializationContext) - else c - for c in self.converters.values() - ) - ) + def with_context(self, context: SerializationContext) -> Self: + """Return a new instance with context set on the component converters. + + If none of the component converters support with_context, return self. + """ + converters: list[EncodingPayloadConverter] = [] + any_with_context = False + for c in self.converters.values(): + if isinstance(c, WithSerializationContext): + converters.append(c.with_context(context)) + any_with_context = True + else: + converters.append(c) + + if not any_with_context: + return self + + new_instance = type(self)() + new_instance._set_converters(*converters) + return new_instance class DefaultPayloadConverter(CompositePayloadConverter): @@ -1322,7 +1335,6 @@ async def decode_failure( return self.failure_converter.from_failure(failure, self.payload_converter) def _with_context(self, context: SerializationContext) -> Self: - cloned = dataclasses.replace(self) payload_converter = self.payload_converter payload_codec = self.payload_codec failure_converter = self.failure_converter @@ -1332,6 +1344,16 @@ def _with_context(self, context: SerializationContext) -> Self: payload_codec = payload_codec.with_context(context) if isinstance(failure_converter, WithSerializationContext): failure_converter = failure_converter.with_context(context) + if all( + new == orig + for new, orig in [ + (payload_converter, self.payload_converter), + (payload_codec, self.payload_codec), + (failure_converter, self.failure_converter), + ] + ): + return self + cloned = dataclasses.replace(self) object.__setattr__(cloned, "payload_converter", payload_converter) object.__setattr__(cloned, "payload_codec", payload_codec) object.__setattr__(cloned, "failure_converter", failure_converter) From 192281d79b1cfbd3c87de81bac2daaf087257955 Mon Sep 17 00:00:00 2001 From: Dan Davison Date: Thu, 25 Sep 2025 17:43:25 -0400 Subject: [PATCH 42/81] Break feature --- temporalio/converter.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/temporalio/converter.py b/temporalio/converter.py index ad063c16d..c6c36923c 100644 --- a/temporalio/converter.py +++ b/temporalio/converter.py @@ -433,9 +433,11 @@ def with_context(self, context: SerializationContext) -> Self: if not any_with_context: return self - new_instance = type(self)() + new_instance = ( + CompositePayloadConverter() + ) # FIXME: deliberate temporary wrong class new_instance._set_converters(*converters) - return new_instance + return new_instance # type: ignore class DefaultPayloadConverter(CompositePayloadConverter): From 5e155495136ed78fb3af1063281b1cea49e77e24 Mon Sep 17 00:00:00 2001 From: Dan Davison Date: Fri, 26 Sep 2025 06:11:19 -0400 Subject: [PATCH 43/81] Revert "Break feature" This reverts commit f677ef0d739884f9b760fcce21bf0e3c10f6c4a1. --- temporalio/converter.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/temporalio/converter.py b/temporalio/converter.py index c6c36923c..ad063c16d 100644 --- a/temporalio/converter.py +++ b/temporalio/converter.py @@ -433,11 +433,9 @@ def with_context(self, context: SerializationContext) -> Self: if not any_with_context: return self - new_instance = ( - CompositePayloadConverter() - ) # FIXME: deliberate temporary wrong class + new_instance = type(self)() new_instance._set_converters(*converters) - return new_instance # type: ignore + return new_instance class DefaultPayloadConverter(CompositePayloadConverter): From a0d6127bb65924970559145925d3a57aa8721eb9 Mon Sep 17 00:00:00 2001 From: Dan Davison Date: Fri, 26 Sep 2025 12:27:32 -0400 Subject: [PATCH 44/81] Use structural typing for the interface --- temporalio/client.py | 3 +-- temporalio/converter.py | 11 ++++++++--- tests/test_serialization_context.py | 27 +++++++++------------------ 3 files changed, 18 insertions(+), 23 deletions(-) diff --git a/temporalio/client.py b/temporalio/client.py index d574b590d..5b5256a4d 100644 --- a/temporalio/client.py +++ b/temporalio/client.py @@ -66,7 +66,6 @@ from temporalio.converter import ( DataConverter, SerializationContext, - WithSerializationContext, WorkflowSerializationContext, ) from temporalio.service import ( @@ -2737,7 +2736,7 @@ class AsyncActivityIDReference: activity_id: str -class AsyncActivityHandle(WithSerializationContext): +class AsyncActivityHandle: """Handle representing an external activity for completion and heartbeat.""" def __init__( diff --git a/temporalio/converter.py b/temporalio/converter.py index ad063c16d..3f8eaf08e 100644 --- a/temporalio/converter.py +++ b/temporalio/converter.py @@ -28,6 +28,7 @@ Mapping, NewType, Optional, + Protocol, Sequence, Tuple, Type, @@ -35,6 +36,7 @@ Union, get_type_hints, overload, + runtime_checkable, ) import google.protobuf.duration_pb2 @@ -136,10 +138,13 @@ class ActivitySerializationContext(BaseWorkflowSerializationContext): is_local: bool -# TODO: duck typing or nominal typing? -class WithSerializationContext(ABC): +@runtime_checkable +class WithSerializationContext(Protocol): """Interface for classes that can use serialization context. + To use serialization context in your class, implementing this interface is sufficient; you do + not need to inherit from this class. + The following classes may implement this interface: - :py:class:`PayloadConverter` - :py:class:`PayloadCodec` @@ -331,7 +336,7 @@ def from_payload( raise NotImplementedError -class CompositePayloadConverter(PayloadConverter, WithSerializationContext): +class CompositePayloadConverter(PayloadConverter): """Composite payload converter that delegates to a list of encoding payload converters. Encoding/decoding are attempted on each payload converter successively until diff --git a/tests/test_serialization_context.py b/tests/test_serialization_context.py index 3c88d17da..62dc0c6c2 100644 --- a/tests/test_serialization_context.py +++ b/tests/test_serialization_context.py @@ -38,7 +38,6 @@ PayloadCodec, PayloadConverter, SerializationContext, - WithSerializationContext, WorkflowSerializationContext, ) from temporalio.exceptions import ApplicationError @@ -66,9 +65,7 @@ class TraceData: items: list[TraceItem] = field(default_factory=list) -class SerializationContextPayloadConverter( - EncodingPayloadConverter, WithSerializationContext -): +class SerializationContextPayloadConverter(EncodingPayloadConverter): def __init__(self): self.context: Optional[SerializationContext] = None @@ -133,9 +130,7 @@ def from_payload( return value -class SerializationContextCompositePayloadConverter( - CompositePayloadConverter, WithSerializationContext -): +class SerializationContextCompositePayloadConverter(CompositePayloadConverter): def __init__(self): super().__init__( SerializationContextPayloadConverter(), @@ -1000,7 +995,7 @@ async def run(self) -> Never: test_traces: dict[str, list[TraceItem]] = defaultdict(list) -class FailureConverterWithContext(DefaultFailureConverter, WithSerializationContext): +class FailureConverterWithContext(DefaultFailureConverter): def __init__(self): super().__init__(encode_common_attributes=False) self.context: Optional[SerializationContext] = None @@ -1131,7 +1126,7 @@ async def test_failure_converter_with_context(client: Client): # Test payload codec -class PayloadCodecWithContext(PayloadCodec, WithSerializationContext): +class PayloadCodecWithContext(PayloadCodec): def __init__(self): self.context: Optional[SerializationContext] = None self.encode_called_with_context = False @@ -1432,7 +1427,7 @@ async def test_child_workflow_codec_with_context(client: Client): # Payload codec: test decode context matches encode context -class PayloadEncryptionCodec(PayloadCodec, WithSerializationContext): +class PayloadEncryptionCodec(PayloadCodec): """ The outbound data for encoding must always be the string "outbound". "Encrypt" it by replacing it with a key that is derived from the context available during encoding. On decryption, assert @@ -1597,7 +1592,7 @@ async def test_decode_context_matches_encode_context( # Test nexus payload codec -class AssertNexusLacksContextPayloadCodec(PayloadCodec, WithSerializationContext): +class AssertNexusLacksContextPayloadCodec(PayloadCodec): def __init__(self): self.context = None @@ -1679,9 +1674,7 @@ class PydanticData(BaseModel): trace: List[str] = [] -class PydanticJSONConverterWithContext( - PydanticJSONPlainPayloadConverter, WithSerializationContext -): +class PydanticJSONConverterWithContext(PydanticJSONPlainPayloadConverter): def __init__(self): super().__init__() self.context: Optional[SerializationContext] = None @@ -1700,7 +1693,7 @@ def to_payload(self, value: Any) -> Optional[temporalio.api.common.v1.Payload]: return super().to_payload(value) -class PydanticConverterWithContext(CompositePayloadConverter, WithSerializationContext): +class PydanticConverterWithContext(CompositePayloadConverter): def __init__(self): super().__init__( *( @@ -1758,9 +1751,7 @@ class UserMethodCalledError(Exception): pass -class CustomEncodingPayloadConverter( - JSONPlainPayloadConverter, WithSerializationContext -): +class CustomEncodingPayloadConverter(JSONPlainPayloadConverter): @property def encoding(self) -> str: return "custom-encoding-that-does-not-clash-with-default-converters" From 1ea65b09b5c83994e751ff47a4ed767635a09918 Mon Sep 17 00:00:00 2001 From: Dan Davison Date: Fri, 26 Sep 2025 14:15:18 -0400 Subject: [PATCH 45/81] Cleanup --- temporalio/client.py | 2 +- temporalio/converter.py | 5 ++++- temporalio/worker/_workflow_instance.py | 7 ++----- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/temporalio/client.py b/temporalio/client.py index 5b5256a4d..98e4d8ad8 100644 --- a/temporalio/client.py +++ b/temporalio/client.py @@ -2852,7 +2852,7 @@ async def report_cancellation( ) # TODO(dan): should this return Self (requiring that the user's subclass has the same - # constructor signature)? + # constructor signature)? CompositePayloadConverter.with_context does. def with_context(self, context: SerializationContext) -> AsyncActivityHandle: """Create a new AsyncActivityHandle with a different serialization context. diff --git a/temporalio/converter.py b/temporalio/converter.py index 3f8eaf08e..e8464ac15 100644 --- a/temporalio/converter.py +++ b/temporalio/converter.py @@ -84,7 +84,8 @@ class SerializationContext(ABC): not of the currently executing (i.e. parent) workflow. - In workflow code, when operating on a payload to be sent/received to/from an activity, the context type is :py:class:`ActivitySerializationContext` and the workflow ID is that of the - currently-executing workflow. ActivitySerializationContext is also set on operations + currently-executing workflow. ActivitySerializationContext is also set on data converter + operations in the activity context. """ pass @@ -438,6 +439,8 @@ def with_context(self, context: SerializationContext) -> Self: if not any_with_context: return self + # A user who has created a subclass with a non-nullary constructor must override this + # method. new_instance = type(self)() new_instance._set_converters(*converters) return new_instance diff --git a/temporalio/worker/_workflow_instance.py b/temporalio/worker/_workflow_instance.py index 4e69551e4..c33c78fbe 100644 --- a/temporalio/worker/_workflow_instance.py +++ b/temporalio/worker/_workflow_instance.py @@ -957,7 +957,7 @@ def _apply_resolve_nexus_operation( return # We don't set a serialization context for nexus operations on the caller side because it is - # not possible to do so on the handler side. + # not possible to set the same context on the handler side. payload_converter, failure_converter = ( self._context_free_payload_converter, self._context_free_failure_converter, @@ -2091,7 +2091,7 @@ def _converters_with_context( This plays a similar role to DataConverter._with_context, but operates on PayloadConverter and FailureConverter only (since payload encoding/decoding is done by the worker, outside - the workflowsandbox). + the workflow sandbox). """ payload_converter = self._context_free_payload_converter failure_converter = self._context_free_failure_converter @@ -2101,7 +2101,6 @@ def _converters_with_context( failure_converter = failure_converter.with_context(context) return payload_converter, failure_converter - # _WorkflowInstanceImpl.get_pending_command_serialization_context def get_payload_codec_with_context( self, payload_codec: temporalio.converter.PayloadCodec, @@ -2430,8 +2429,6 @@ async def _signal_external_workflow( done_fut = self.create_future() command.signal_external_workflow_execution.seq = seq - # Set as pending with the target workflow ID for later context use - # Extract the workflow ID from the command target_workflow_id = ( command.signal_external_workflow_execution.child_workflow_id or command.signal_external_workflow_execution.workflow_execution.workflow_id From 49a96b942a90d681c612dba51d97ea7de430c881 Mon Sep 17 00:00:00 2001 From: Dan Davison Date: Fri, 26 Sep 2025 14:26:42 -0400 Subject: [PATCH 46/81] Make data converter lazy on update handle --- temporalio/client.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/temporalio/client.py b/temporalio/client.py index 98e4d8ad8..25e91a1ba 100644 --- a/temporalio/client.py +++ b/temporalio/client.py @@ -5059,7 +5059,10 @@ def __init__( self._workflow_run_id = workflow_run_id self._result_type = result_type self._known_outcome = known_outcome - self._data_converter = self._client.data_converter._with_context( + + @functools.cached_property + def _data_converter(self) -> temporalio.converter.DataConverter: + return self._client.data_converter._with_context( WorkflowSerializationContext( namespace=self._client.namespace, workflow_id=self.workflow_id, From f4f8cbd528003a24ccc5b00fc4453b7b69536f90 Mon Sep 17 00:00:00 2001 From: Dan Davison Date: Fri, 26 Sep 2025 14:30:34 -0400 Subject: [PATCH 47/81] Use object identity comparison semantics --- temporalio/converter.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/temporalio/converter.py b/temporalio/converter.py index e8464ac15..f0a6ce5ee 100644 --- a/temporalio/converter.py +++ b/temporalio/converter.py @@ -425,14 +425,14 @@ def from_payloads( def with_context(self, context: SerializationContext) -> Self: """Return a new instance with context set on the component converters. - If none of the component converters support with_context, return self. + If none of the component converters returned new instances, return self. """ converters: list[EncodingPayloadConverter] = [] any_with_context = False for c in self.converters.values(): if isinstance(c, WithSerializationContext): converters.append(c.with_context(context)) - any_with_context = True + any_with_context |= converters[-1] != c else: converters.append(c) From 3f1f9c1146f3d2c5586590ad816f3f38a265e88c Mon Sep 17 00:00:00 2001 From: Dan Davison Date: Fri, 26 Sep 2025 15:21:33 -0400 Subject: [PATCH 48/81] Revert "Use structural typing for the interface" This reverts commit b8ab990603a3ad47437f553a7fe8113fa9ee86ab. --- temporalio/client.py | 3 ++- temporalio/converter.py | 11 +++-------- tests/test_serialization_context.py | 27 ++++++++++++++++++--------- 3 files changed, 23 insertions(+), 18 deletions(-) diff --git a/temporalio/client.py b/temporalio/client.py index 25e91a1ba..d4c054bc2 100644 --- a/temporalio/client.py +++ b/temporalio/client.py @@ -66,6 +66,7 @@ from temporalio.converter import ( DataConverter, SerializationContext, + WithSerializationContext, WorkflowSerializationContext, ) from temporalio.service import ( @@ -2736,7 +2737,7 @@ class AsyncActivityIDReference: activity_id: str -class AsyncActivityHandle: +class AsyncActivityHandle(WithSerializationContext): """Handle representing an external activity for completion and heartbeat.""" def __init__( diff --git a/temporalio/converter.py b/temporalio/converter.py index f0a6ce5ee..41a1c64cb 100644 --- a/temporalio/converter.py +++ b/temporalio/converter.py @@ -28,7 +28,6 @@ Mapping, NewType, Optional, - Protocol, Sequence, Tuple, Type, @@ -36,7 +35,6 @@ Union, get_type_hints, overload, - runtime_checkable, ) import google.protobuf.duration_pb2 @@ -139,13 +137,10 @@ class ActivitySerializationContext(BaseWorkflowSerializationContext): is_local: bool -@runtime_checkable -class WithSerializationContext(Protocol): +# TODO: duck typing or nominal typing? +class WithSerializationContext(ABC): """Interface for classes that can use serialization context. - To use serialization context in your class, implementing this interface is sufficient; you do - not need to inherit from this class. - The following classes may implement this interface: - :py:class:`PayloadConverter` - :py:class:`PayloadCodec` @@ -337,7 +332,7 @@ def from_payload( raise NotImplementedError -class CompositePayloadConverter(PayloadConverter): +class CompositePayloadConverter(PayloadConverter, WithSerializationContext): """Composite payload converter that delegates to a list of encoding payload converters. Encoding/decoding are attempted on each payload converter successively until diff --git a/tests/test_serialization_context.py b/tests/test_serialization_context.py index 62dc0c6c2..3c88d17da 100644 --- a/tests/test_serialization_context.py +++ b/tests/test_serialization_context.py @@ -38,6 +38,7 @@ PayloadCodec, PayloadConverter, SerializationContext, + WithSerializationContext, WorkflowSerializationContext, ) from temporalio.exceptions import ApplicationError @@ -65,7 +66,9 @@ class TraceData: items: list[TraceItem] = field(default_factory=list) -class SerializationContextPayloadConverter(EncodingPayloadConverter): +class SerializationContextPayloadConverter( + EncodingPayloadConverter, WithSerializationContext +): def __init__(self): self.context: Optional[SerializationContext] = None @@ -130,7 +133,9 @@ def from_payload( return value -class SerializationContextCompositePayloadConverter(CompositePayloadConverter): +class SerializationContextCompositePayloadConverter( + CompositePayloadConverter, WithSerializationContext +): def __init__(self): super().__init__( SerializationContextPayloadConverter(), @@ -995,7 +1000,7 @@ async def run(self) -> Never: test_traces: dict[str, list[TraceItem]] = defaultdict(list) -class FailureConverterWithContext(DefaultFailureConverter): +class FailureConverterWithContext(DefaultFailureConverter, WithSerializationContext): def __init__(self): super().__init__(encode_common_attributes=False) self.context: Optional[SerializationContext] = None @@ -1126,7 +1131,7 @@ async def test_failure_converter_with_context(client: Client): # Test payload codec -class PayloadCodecWithContext(PayloadCodec): +class PayloadCodecWithContext(PayloadCodec, WithSerializationContext): def __init__(self): self.context: Optional[SerializationContext] = None self.encode_called_with_context = False @@ -1427,7 +1432,7 @@ async def test_child_workflow_codec_with_context(client: Client): # Payload codec: test decode context matches encode context -class PayloadEncryptionCodec(PayloadCodec): +class PayloadEncryptionCodec(PayloadCodec, WithSerializationContext): """ The outbound data for encoding must always be the string "outbound". "Encrypt" it by replacing it with a key that is derived from the context available during encoding. On decryption, assert @@ -1592,7 +1597,7 @@ async def test_decode_context_matches_encode_context( # Test nexus payload codec -class AssertNexusLacksContextPayloadCodec(PayloadCodec): +class AssertNexusLacksContextPayloadCodec(PayloadCodec, WithSerializationContext): def __init__(self): self.context = None @@ -1674,7 +1679,9 @@ class PydanticData(BaseModel): trace: List[str] = [] -class PydanticJSONConverterWithContext(PydanticJSONPlainPayloadConverter): +class PydanticJSONConverterWithContext( + PydanticJSONPlainPayloadConverter, WithSerializationContext +): def __init__(self): super().__init__() self.context: Optional[SerializationContext] = None @@ -1693,7 +1700,7 @@ def to_payload(self, value: Any) -> Optional[temporalio.api.common.v1.Payload]: return super().to_payload(value) -class PydanticConverterWithContext(CompositePayloadConverter): +class PydanticConverterWithContext(CompositePayloadConverter, WithSerializationContext): def __init__(self): super().__init__( *( @@ -1751,7 +1758,9 @@ class UserMethodCalledError(Exception): pass -class CustomEncodingPayloadConverter(JSONPlainPayloadConverter): +class CustomEncodingPayloadConverter( + JSONPlainPayloadConverter, WithSerializationContext +): @property def encoding(self) -> str: return "custom-encoding-that-does-not-clash-with-default-converters" From 622b7788b37bb047863cc8a8bbf6c6c39f711363 Mon Sep 17 00:00:00 2001 From: Dan Davison Date: Fri, 26 Sep 2025 15:48:53 -0400 Subject: [PATCH 49/81] Make DataConverter implement the interface --- temporalio/client.py | 24 ++++++++++++------------ temporalio/converter.py | 5 +++-- temporalio/worker/_activity.py | 4 ++-- temporalio/worker/_workflow.py | 2 +- 4 files changed, 18 insertions(+), 17 deletions(-) diff --git a/temporalio/client.py b/temporalio/client.py index d4c054bc2..18dc77974 100644 --- a/temporalio/client.py +++ b/temporalio/client.py @@ -1609,7 +1609,7 @@ def __init__( @functools.cached_property def _data_converter(self) -> temporalio.converter.DataConverter: - return self._client.data_converter._with_context( + return self._client.data_converter.with_context( temporalio.converter.WorkflowSerializationContext( namespace=self._client.namespace, workflow_id=self._id ) @@ -2865,7 +2865,7 @@ def with_context(self, context: SerializationContext) -> AsyncActivityHandle: return AsyncActivityHandle( self._client, self._id_or_token, - self._client.data_converter._with_context(context), + self._client.data_converter.with_context(context), ) @@ -3234,7 +3234,7 @@ async def fetch_next_page(self, *, page_size: Optional[int] = None) -> None: def get_data_converter(workflow_id: str) -> temporalio.converter.DataConverter: if workflow_id not in data_converter_cache: data_converter_cache[workflow_id] = ( - self._client.data_converter._with_context( + self._client.data_converter.with_context( WorkflowSerializationContext( namespace=self._client.namespace, workflow_id=workflow_id, @@ -4204,7 +4204,7 @@ async def _to_proto( priority: Optional[temporalio.api.common.v1.Priority] = None if self.priority: priority = self.priority._to_proto() - data_converter = client.data_converter._with_context( + data_converter = client.data_converter.with_context( WorkflowSerializationContext( namespace=client.namespace, workflow_id=self.id, @@ -5063,7 +5063,7 @@ def __init__( @functools.cached_property def _data_converter(self) -> temporalio.converter.DataConverter: - return self._client.data_converter._with_context( + return self._client.data_converter.with_context( WorkflowSerializationContext( namespace=self._client.namespace, workflow_id=self.workflow_id, @@ -5977,7 +5977,7 @@ async def _build_signal_with_start_workflow_execution_request( self, input: StartWorkflowInput ) -> temporalio.api.workflowservice.v1.SignalWithStartWorkflowExecutionRequest: assert input.start_signal - data_converter = self._client.data_converter._with_context( + data_converter = self._client.data_converter.with_context( WorkflowSerializationContext( namespace=self._client.namespace, workflow_id=input.id, @@ -6008,7 +6008,7 @@ async def _populate_start_workflow_execution_request( ], input: Union[StartWorkflowInput, UpdateWithStartStartWorkflowInput], ) -> None: - data_converter = self._client.data_converter._with_context( + data_converter = self._client.data_converter.with_context( WorkflowSerializationContext( namespace=self._client.namespace, workflow_id=input.id, @@ -6095,7 +6095,7 @@ async def describe_workflow( metadata=input.rpc_metadata, timeout=input.rpc_timeout, ), - self._client.data_converter._with_context( + self._client.data_converter.with_context( WorkflowSerializationContext( namespace=self._client.namespace, workflow_id=input.id, @@ -6129,7 +6129,7 @@ async def count_workflows( ) async def query_workflow(self, input: QueryWorkflowInput) -> Any: - data_converter = self._client.data_converter._with_context( + data_converter = self._client.data_converter.with_context( WorkflowSerializationContext( namespace=self._client.namespace, workflow_id=input.id, @@ -6182,7 +6182,7 @@ async def query_workflow(self, input: QueryWorkflowInput) -> Any: return results[0] async def signal_workflow(self, input: SignalWorkflowInput) -> None: - data_converter = self._client.data_converter._with_context( + data_converter = self._client.data_converter.with_context( WorkflowSerializationContext( namespace=self._client.namespace, workflow_id=input.id, @@ -6207,7 +6207,7 @@ async def signal_workflow(self, input: SignalWorkflowInput) -> None: ) async def terminate_workflow(self, input: TerminateWorkflowInput) -> None: - data_converter = self._client.data_converter._with_context( + data_converter = self._client.data_converter.with_context( WorkflowSerializationContext( namespace=self._client.namespace, workflow_id=input.id, @@ -6281,7 +6281,7 @@ async def _build_update_workflow_execution_request( input: Union[StartWorkflowUpdateInput, UpdateWithStartUpdateWorkflowInput], workflow_id: str, ) -> temporalio.api.workflowservice.v1.UpdateWorkflowExecutionRequest: - data_converter = self._client.data_converter._with_context( + data_converter = self._client.data_converter.with_context( WorkflowSerializationContext( namespace=self._client.namespace, workflow_id=workflow_id, diff --git a/temporalio/converter.py b/temporalio/converter.py index 41a1c64cb..608ef8bba 100644 --- a/temporalio/converter.py +++ b/temporalio/converter.py @@ -1233,7 +1233,7 @@ def __init__(self) -> None: @dataclass(frozen=True) -class DataConverter: +class DataConverter(WithSerializationContext): """Data converter for converting and encoding payloads to/from Python values. This combines :py:class:`PayloadConverter` which converts values with @@ -1337,7 +1337,8 @@ async def decode_failure( await self.payload_codec.decode_failure(failure) return self.failure_converter.from_failure(failure, self.payload_converter) - def _with_context(self, context: SerializationContext) -> Self: + def with_context(self, context: SerializationContext) -> Self: + """Return an instance with context set on the component converters.""" payload_converter = self.payload_converter payload_codec = self.payload_codec failure_converter = self.failure_converter diff --git a/temporalio/worker/_activity.py b/temporalio/worker/_activity.py index 39507759a..44bfb6910 100644 --- a/temporalio/worker/_activity.py +++ b/temporalio/worker/_activity.py @@ -262,7 +262,7 @@ async def _heartbeat_async( activity_task_queue=self._task_queue, is_local=activity.info.is_local, ) - data_converter = data_converter._with_context(context) + data_converter = data_converter.with_context(context) # Perform the heartbeat try: @@ -314,7 +314,7 @@ async def _handle_start_activity_task( activity_task_queue=self._task_queue, is_local=start.is_local, ) - data_converter = self._data_converter._with_context(context) + data_converter = self._data_converter.with_context(context) try: result = await self._execute_activity( start, running_activity, task_token, data_converter diff --git a/temporalio/worker/_workflow.py b/temporalio/worker/_workflow.py index c386556eb..89c1286b0 100644 --- a/temporalio/worker/_workflow.py +++ b/temporalio/worker/_workflow.py @@ -275,7 +275,7 @@ async def _handle_activation( "Cache already exists for activation with initialize job" ) - data_converter = self._data_converter._with_context( + data_converter = self._data_converter.with_context( temporalio.converter.WorkflowSerializationContext( namespace=self._namespace, workflow_id=workflow_id, From b72f2b736304b98d91b341388f0518826bf61296 Mon Sep 17 00:00:00 2001 From: Dan Davison Date: Sat, 27 Sep 2025 06:44:18 -0400 Subject: [PATCH 50/81] Cleanup --- temporalio/converter.py | 3 +-- tests/test_serialization_context.py | 4 ---- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/temporalio/converter.py b/temporalio/converter.py index 608ef8bba..381239189 100644 --- a/temporalio/converter.py +++ b/temporalio/converter.py @@ -434,8 +434,7 @@ def with_context(self, context: SerializationContext) -> Self: if not any_with_context: return self - # A user who has created a subclass with a non-nullary constructor must override this - # method. + # Must have a nullary constructor new_instance = type(self)() new_instance._set_converters(*converters) return new_instance diff --git a/tests/test_serialization_context.py b/tests/test_serialization_context.py index 3c88d17da..cf07a5b27 100644 --- a/tests/test_serialization_context.py +++ b/tests/test_serialization_context.py @@ -583,10 +583,6 @@ async def test_async_activity_completion_payload_conversion( await activity_handle.complete(data) result = await wf_handle.result() - print() - for item in result.items: - print(item) - activity_context_dict = dataclasses.asdict(activity_context) workflow_context_dict = dataclasses.asdict(workflow_context) From 0201fba221352c031b87e26d04ad7c5ad16e4a95 Mon Sep 17 00:00:00 2001 From: Dan Davison Date: Sat, 27 Sep 2025 06:15:21 -0400 Subject: [PATCH 51/81] Return self in AsyncActivityHandle.with_handle --- temporalio/client.py | 17 +++++--- tests/test_serialization_context.py | 61 ++++++++++++++++++++++++++++- 2 files changed, 72 insertions(+), 6 deletions(-) diff --git a/temporalio/client.py b/temporalio/client.py index 18dc77974..752e3e99e 100644 --- a/temporalio/client.py +++ b/temporalio/client.py @@ -41,7 +41,7 @@ import google.protobuf.json_format import google.protobuf.timestamp_pb2 from google.protobuf.internal.containers import MessageMap -from typing_extensions import Concatenate, Required, TypedDict +from typing_extensions import Concatenate, Required, Self, TypedDict import temporalio.api.common.v1 import temporalio.api.enums.v1 @@ -2852,9 +2852,7 @@ async def report_cancellation( ), ) - # TODO(dan): should this return Self (requiring that the user's subclass has the same - # constructor signature)? CompositePayloadConverter.with_context does. - def with_context(self, context: SerializationContext) -> AsyncActivityHandle: + def with_context(self, context: SerializationContext) -> Self: """Create a new AsyncActivityHandle with a different serialization context. Payloads received by the activity will be decoded and deserialized using a data converter @@ -2862,7 +2860,16 @@ def with_context(self, context: SerializationContext) -> AsyncActivityHandle: converter that makes use of this context then you can use this method to supply matching context data to the data converter used to serialize and encode the outbound payloads. """ - return AsyncActivityHandle( + data_converter = self._client.data_converter.with_context(context) + if data_converter == self._client.data_converter: + return self + cls = type(self) + if cls.__init__ is not AsyncActivityHandle.__init__: + raise TypeError( + "If you have subclassed AsyncActivityHandle and overridden the __init__ method " + "then you must override with_context to return an instance of your class." + ) + return cls( self._client, self._id_or_token, self._client.data_converter.with_context(context), diff --git a/tests/test_serialization_context.py b/tests/test_serialization_context.py index cf07a5b27..cc0866bb2 100644 --- a/tests/test_serialization_context.py +++ b/tests/test_serialization_context.py @@ -24,7 +24,12 @@ import temporalio.api.common.v1 import temporalio.api.failure.v1 from temporalio import activity, workflow -from temporalio.client import Client, WorkflowFailureError, WorkflowUpdateFailedError +from temporalio.client import ( + AsyncActivityHandle, + Client, + WorkflowFailureError, + WorkflowUpdateFailedError, +) from temporalio.common import RetryPolicy from temporalio.contrib.pydantic import PydanticJSONPlainPayloadConverter from temporalio.converter import ( @@ -610,6 +615,60 @@ async def test_async_activity_completion_payload_conversion( ] +class MyAsyncActivityHandle(AsyncActivityHandle): + def my_method(self) -> None: + pass + + +class MyAsyncActivityHandleWithOverriddenConstructor(AsyncActivityHandle): + def __init__(self, *args: Any, **kwargs: Any) -> None: + super().__init__(*args, **kwargs) + + def my_method(self) -> None: + pass + + +def test_subclassed_async_activity_handle(client: Client): + activity_context = ActivitySerializationContext( + namespace="default", + workflow_id="workflow-id", + workflow_type="workflow-type", + activity_type="activity-type", + activity_task_queue="activity-task-queue", + is_local=False, + ) + handle = MyAsyncActivityHandle(client=client, id_or_token=b"task-token") + # This works because the data converter does not use context so AsyncActivityHandle.with_context + # returns self + assert isinstance(handle.with_context(activity_context), MyAsyncActivityHandle) + + # This time the data converter uses context so AsyncActivityHandle.with_context attempts to + # return a new instance of the user's subclass. It works, because they have not overridden the + # constructor. + client_config = client.config() + client_config["data_converter"] = dataclasses.replace( + DataConverter.default, + payload_converter_class=SerializationContextCompositePayloadConverter, + ) + client = Client(**client_config) + handle = MyAsyncActivityHandle(client=client, id_or_token=b"task-token") + assert isinstance(handle.with_context(activity_context), MyAsyncActivityHandle) + + # Finally, a user attempts the same but having overridden the constructor. This fails: + # AsyncActivityHandle.with_context refuses to attempt to create an instance of their subclass. + handle2 = MyAsyncActivityHandleWithOverriddenConstructor( + client=client, id_or_token=b"task-token" + ) + with pytest.raises( + TypeError, + match="you must override with_context to return an instance of your class", + ): + assert isinstance( + handle2.with_context(activity_context), + MyAsyncActivityHandleWithOverriddenConstructor, + ) + + # Signal test From e854537100b23155055db3057d079653af1953d4 Mon Sep 17 00:00:00 2001 From: Dan Davison Date: Sat, 27 Sep 2025 09:16:27 -0400 Subject: [PATCH 52/81] Use object equality as was intended --- temporalio/client.py | 2 +- temporalio/converter.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/temporalio/client.py b/temporalio/client.py index 752e3e99e..207849ef2 100644 --- a/temporalio/client.py +++ b/temporalio/client.py @@ -2861,7 +2861,7 @@ def with_context(self, context: SerializationContext) -> Self: context data to the data converter used to serialize and encode the outbound payloads. """ data_converter = self._client.data_converter.with_context(context) - if data_converter == self._client.data_converter: + if data_converter is self._client.data_converter: return self cls = type(self) if cls.__init__ is not AsyncActivityHandle.__init__: diff --git a/temporalio/converter.py b/temporalio/converter.py index 381239189..88693e04b 100644 --- a/temporalio/converter.py +++ b/temporalio/converter.py @@ -427,7 +427,7 @@ def with_context(self, context: SerializationContext) -> Self: for c in self.converters.values(): if isinstance(c, WithSerializationContext): converters.append(c.with_context(context)) - any_with_context |= converters[-1] != c + any_with_context |= converters[-1] is not c else: converters.append(c) @@ -1348,7 +1348,7 @@ def with_context(self, context: SerializationContext) -> Self: if isinstance(failure_converter, WithSerializationContext): failure_converter = failure_converter.with_context(context) if all( - new == orig + new is orig for new, orig in [ (payload_converter, self.payload_converter), (payload_codec, self.payload_codec), From 46328d28e77568e14db90cbae0bf9605942d05c6 Mon Sep 17 00:00:00 2001 From: Dan Davison Date: Sat, 27 Sep 2025 09:38:40 -0400 Subject: [PATCH 53/81] Factor out get_converters_with_context --- temporalio/converter.py | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/temporalio/converter.py b/temporalio/converter.py index 88693e04b..6067436c6 100644 --- a/temporalio/converter.py +++ b/temporalio/converter.py @@ -422,6 +422,20 @@ def with_context(self, context: SerializationContext) -> Self: If none of the component converters returned new instances, return self. """ + converters = self.get_converters_with_context(context) + if converters is None: + return self + new_instance = type(self)() # Must have a nullary constructor + new_instance._set_converters(*converters) + return new_instance + + def get_converters_with_context( + self, context: SerializationContext + ) -> Optional[List[EncodingPayloadConverter]]: + """Return converter instances with context set. + + If no converter uses context, return None. + """ converters: list[EncodingPayloadConverter] = [] any_with_context = False for c in self.converters.values(): @@ -431,13 +445,7 @@ def with_context(self, context: SerializationContext) -> Self: else: converters.append(c) - if not any_with_context: - return self - - # Must have a nullary constructor - new_instance = type(self)() - new_instance._set_converters(*converters) - return new_instance + return converters if any_with_context else None class DefaultPayloadConverter(CompositePayloadConverter): From 38ac5c9aa345c49d86617923a2e49bc42624af54 Mon Sep 17 00:00:00 2001 From: Dan Davison Date: Sat, 27 Sep 2025 09:49:37 -0400 Subject: [PATCH 54/81] Fix bug spotted by AI (Cursor Bot) --- temporalio/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/temporalio/client.py b/temporalio/client.py index 207849ef2..db163632e 100644 --- a/temporalio/client.py +++ b/temporalio/client.py @@ -2872,7 +2872,7 @@ def with_context(self, context: SerializationContext) -> Self: return cls( self._client, self._id_or_token, - self._client.data_converter.with_context(context), + data_converter, ) From 9011d9c15acb8d3e1fae38d444ff3defd31368e9 Mon Sep 17 00:00:00 2001 From: Dan Davison Date: Sat, 27 Sep 2025 10:51:29 -0400 Subject: [PATCH 55/81] appease pydoctor --- temporalio/converter.py | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/temporalio/converter.py b/temporalio/converter.py index 6067436c6..8e55dd4a6 100644 --- a/temporalio/converter.py +++ b/temporalio/converter.py @@ -72,18 +72,20 @@ class SerializationContext(ABC): Provides contextual information during serialization and deserialization operations. Examples: - - In client code, when starting a workflow, or sending a signal/update/query to a workflow, or - receiving the result of an update/query, or handling an exception from a workflow, the context - type is :py:class:`WorkflowSerializationContext` and the workflow ID set of the target - workflow will be set in the context. - - In workflow code, when operating on a payload being sent/received to/from a child workflow, or - handling an exception from a child workflow, the context type is - :py:class:`WorkflowSerializationContext` and the workflow ID is that of the child workflow, - not of the currently executing (i.e. parent) workflow. - - In workflow code, when operating on a payload to be sent/received to/from an activity, the - context type is :py:class:`ActivitySerializationContext` and the workflow ID is that of the - currently-executing workflow. ActivitySerializationContext is also set on data converter - operations in the activity context. + In client code, when starting a workflow, or sending a signal/update/query to a workflow, + or receiving the result of an update/query, or handling an exception from a workflow, the + context type is :py:class:`WorkflowSerializationContext` and the workflow ID set of the + target workflow will be set in the context. + + In workflow code, when operating on a payload being sent/received to/from a child workflow, + or handling an exception from a child workflow, the context type is + :py:class:`WorkflowSerializationContext` and the workflow ID is that of the child workflow, + not of the currently executing (i.e. parent) workflow. + + In workflow code, when operating on a payload to be sent/received to/from an activity, the + context type is :py:class:`ActivitySerializationContext` and the workflow ID is that of the + currently-executing workflow. ActivitySerializationContext is also set on data converter + operations in the activity context. """ pass From 0ecc96fda2b6b2bc1dcb177a647d961285021e44 Mon Sep 17 00:00:00 2001 From: Dan Davison Date: Mon, 29 Sep 2025 12:29:26 -0400 Subject: [PATCH 56/81] Formatting --- temporalio/client.py | 46 +++++++++++++++++++++++++++----------------- 1 file changed, 28 insertions(+), 18 deletions(-) diff --git a/temporalio/client.py b/temporalio/client.py index db163632e..17e20e382 100644 --- a/temporalio/client.py +++ b/temporalio/client.py @@ -2943,27 +2943,37 @@ def _from_raw_info( **additional_fields: Any, ) -> WorkflowExecution: return cls( - close_time=info.close_time.ToDatetime().replace(tzinfo=timezone.utc) - if info.HasField("close_time") - else None, + close_time=( + info.close_time.ToDatetime().replace(tzinfo=timezone.utc) + if info.HasField("close_time") + else None + ), data_converter=converter, - execution_time=info.execution_time.ToDatetime().replace(tzinfo=timezone.utc) - if info.HasField("execution_time") - else None, + execution_time=( + info.execution_time.ToDatetime().replace(tzinfo=timezone.utc) + if info.HasField("execution_time") + else None + ), history_length=info.history_length, id=info.execution.workflow_id, - parent_id=info.parent_execution.workflow_id - if info.HasField("parent_execution") - else None, - parent_run_id=info.parent_execution.run_id - if info.HasField("parent_execution") - else None, - root_id=info.root_execution.workflow_id - if info.HasField("root_execution") - else None, - root_run_id=info.root_execution.run_id - if info.HasField("root_execution") - else None, + parent_id=( + info.parent_execution.workflow_id + if info.HasField("parent_execution") + else None + ), + parent_run_id=( + info.parent_execution.run_id + if info.HasField("parent_execution") + else None + ), + root_id=( + info.root_execution.workflow_id + if info.HasField("root_execution") + else None + ), + root_run_id=( + info.root_execution.run_id if info.HasField("root_execution") else None + ), raw_info=info, run_id=info.execution.run_id, search_attributes=temporalio.converter.decode_search_attributes( From deab876b9d9322138b7234a1e70de8b78cae595f Mon Sep 17 00:00:00 2001 From: Dan Davison Date: Mon, 29 Sep 2025 12:43:16 -0400 Subject: [PATCH 57/81] Improvements from code review --- temporalio/converter.py | 7 ++++++- temporalio/worker/_workflow.py | 4 ++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/temporalio/converter.py b/temporalio/converter.py index 8e55dd4a6..b4bded364 100644 --- a/temporalio/converter.py +++ b/temporalio/converter.py @@ -353,6 +353,9 @@ def __init__(self, *converters: EncodingPayloadConverter) -> None: converters: Payload converters to delegate to, in order. """ self._set_converters(*converters) + self._any_converter_takes_context = any( + isinstance(c, WithSerializationContext) for c in converters + ) def _set_converters(self, *converters: EncodingPayloadConverter) -> None: self.converters = {c.encoding.encode(): c for c in converters} @@ -433,11 +436,13 @@ def with_context(self, context: SerializationContext) -> Self: def get_converters_with_context( self, context: SerializationContext - ) -> Optional[List[EncodingPayloadConverter]]: + ) -> Optional[list[EncodingPayloadConverter]]: """Return converter instances with context set. If no converter uses context, return None. """ + if not self._any_converter_takes_context: + return None converters: list[EncodingPayloadConverter] = [] any_with_context = False for c in self.converters.values(): diff --git a/temporalio/worker/_workflow.py b/temporalio/worker/_workflow.py index 89c1286b0..7c0d7f0ff 100644 --- a/temporalio/worker/_workflow.py +++ b/temporalio/worker/_workflow.py @@ -288,7 +288,7 @@ async def _handle_activation( else: payload_codec = _CommandAwarePayloadCodec( workflow.instance, - self._data_converter.payload_codec, + context_free_payload_codec=self._data_converter.payload_codec, ) await temporalio.bridge.worker.decode_activation( act, @@ -363,7 +363,7 @@ async def _handle_activation( if self._data_converter.payload_codec and workflow: payload_codec = _CommandAwarePayloadCodec( workflow.instance, - self._data_converter.payload_codec, + context_free_payload_codec=self._data_converter.payload_codec, ) try: await temporalio.bridge.worker.encode_completion( From 6fe8dbe91bdab8c4d7f057875d4dd4e57c92537b Mon Sep 17 00:00:00 2001 From: Dan Davison Date: Mon, 29 Sep 2025 12:54:22 -0400 Subject: [PATCH 58/81] Make it a dataclass --- temporalio/worker/_workflow.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/temporalio/worker/_workflow.py b/temporalio/worker/_workflow.py index 7c0d7f0ff..cb54466f9 100644 --- a/temporalio/worker/_workflow.py +++ b/temporalio/worker/_workflow.py @@ -8,6 +8,7 @@ import os import sys import threading +from dataclasses import dataclass from datetime import timezone from types import TracebackType from typing import ( @@ -720,6 +721,7 @@ def attempt_deadlock_interruption(self) -> None: ) +@dataclass(frozen=True) class _CommandAwarePayloadCodec(temporalio.converter.PayloadCodec): """A payload codec that sets serialization context for the associated command. @@ -727,13 +729,8 @@ class _CommandAwarePayloadCodec(temporalio.converter.PayloadCodec): variable set by the payload visitor. """ - def __init__( - self, - instance: WorkflowInstance, - context_free_payload_codec: temporalio.converter.PayloadCodec, - ): - self.instance = instance - self.context_free_payload_codec = context_free_payload_codec + instance: WorkflowInstance + context_free_payload_codec: temporalio.converter.PayloadCodec async def encode( self, From 3103cd066cafdaedd64f18aac62243d4bab7c9f8 Mon Sep 17 00:00:00 2001 From: Dan Davison Date: Mon, 29 Sep 2025 13:05:30 -0400 Subject: [PATCH 59/81] Revert to mapping codec over structure --- temporalio/bridge/worker.py | 8 ++++---- temporalio/worker/_workflow.py | 4 ++-- tests/worker/test_visitor.py | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/temporalio/bridge/worker.py b/temporalio/bridge/worker.py index ea35191c8..bdf128cca 100644 --- a/temporalio/bridge/worker.py +++ b/temporalio/bridge/worker.py @@ -297,21 +297,21 @@ async def visit_payloads(self, payloads: MutableSequence[Payload]) -> None: async def decode_activation( activation: temporalio.bridge.proto.workflow_activation.WorkflowActivation, - decode: Callable[[Sequence[Payload]], Awaitable[List[Payload]]], + codec: temporalio.converter.PayloadCodec, decode_headers: bool, ) -> None: """Decode all payloads in the activation.""" await PayloadVisitor( skip_search_attributes=True, skip_headers=not decode_headers - ).visit(_Visitor(decode), activation) + ).visit(_Visitor(codec.decode), activation) async def encode_completion( completion: temporalio.bridge.proto.workflow_completion.WorkflowActivationCompletion, - encode: Callable[[Sequence[Payload]], Awaitable[List[Payload]]], + codec: temporalio.converter.PayloadCodec, encode_headers: bool, ) -> None: """Encode all payloads in the completion.""" await PayloadVisitor( skip_search_attributes=True, skip_headers=not encode_headers - ).visit(_Visitor(encode), completion) + ).visit(_Visitor(codec.encode), completion) diff --git a/temporalio/worker/_workflow.py b/temporalio/worker/_workflow.py index cb54466f9..5a2549725 100644 --- a/temporalio/worker/_workflow.py +++ b/temporalio/worker/_workflow.py @@ -293,7 +293,7 @@ async def _handle_activation( ) await temporalio.bridge.worker.decode_activation( act, - payload_codec.decode, + payload_codec, decode_headers=self._encode_headers, ) if not workflow: @@ -369,7 +369,7 @@ async def _handle_activation( try: await temporalio.bridge.worker.encode_completion( completion, - payload_codec.encode, + payload_codec, encode_headers=self._encode_headers, ) except Exception as err: diff --git a/tests/worker/test_visitor.py b/tests/worker/test_visitor.py index be2b991cd..c59a0248b 100644 --- a/tests/worker/test_visitor.py +++ b/tests/worker/test_visitor.py @@ -236,7 +236,7 @@ async def test_bridge_encoding(): ), ) - await temporalio.bridge.worker.encode_completion(comp, SimpleCodec().encode, True) + await temporalio.bridge.worker.encode_completion(comp, SimpleCodec(), True) cmd = comp.successful.commands[0] sa = cmd.schedule_activity From 3bb9950ee6c4349d90d506b099c6b87de318e3c7 Mon Sep 17 00:00:00 2001 From: Dan Davison Date: Mon, 29 Sep 2025 13:20:38 -0400 Subject: [PATCH 60/81] Use existing workflow context payload codec --- temporalio/worker/_workflow.py | 5 ++ temporalio/worker/_workflow_instance.py | 82 ++++++++++--------- .../worker/workflow_sandbox/_in_sandbox.py | 9 +- temporalio/worker/workflow_sandbox/_runner.py | 8 +- tests/worker/test_workflow.py | 7 +- 5 files changed, 66 insertions(+), 45 deletions(-) diff --git a/temporalio/worker/_workflow.py b/temporalio/worker/_workflow.py index 5a2549725..9836c6e5b 100644 --- a/temporalio/worker/_workflow.py +++ b/temporalio/worker/_workflow.py @@ -290,6 +290,7 @@ async def _handle_activation( payload_codec = _CommandAwarePayloadCodec( workflow.instance, context_free_payload_codec=self._data_converter.payload_codec, + workflow_context_payload_codec=data_converter.payload_codec, ) await temporalio.bridge.worker.decode_activation( act, @@ -362,9 +363,11 @@ async def _handle_activation( # Encode completion if self._data_converter.payload_codec and workflow: + assert data_converter.payload_codec payload_codec = _CommandAwarePayloadCodec( workflow.instance, context_free_payload_codec=self._data_converter.payload_codec, + workflow_context_payload_codec=data_converter.payload_codec, ) try: await temporalio.bridge.worker.encode_completion( @@ -731,6 +734,7 @@ class _CommandAwarePayloadCodec(temporalio.converter.PayloadCodec): instance: WorkflowInstance context_free_payload_codec: temporalio.converter.PayloadCodec + workflow_context_payload_codec: temporalio.converter.PayloadCodec async def encode( self, @@ -747,6 +751,7 @@ async def decode( def _get_current_command_codec(self) -> temporalio.converter.PayloadCodec: return self.instance.get_payload_codec_with_context( self.context_free_payload_codec, + self.workflow_context_payload_codec, temporalio.bridge._visitor.current_command_seq.get(), ) diff --git a/temporalio/worker/_workflow_instance.py b/temporalio/worker/_workflow_instance.py index c33c78fbe..cc095b075 100644 --- a/temporalio/worker/_workflow_instance.py +++ b/temporalio/worker/_workflow_instance.py @@ -171,14 +171,17 @@ def activate( @abstractmethod def get_payload_codec_with_context( self, - payload_codec: temporalio.converter.PayloadCodec, + base_payload_codec: temporalio.converter.PayloadCodec, + workflow_context_payload_codec: temporalio.converter.PayloadCodec, command_seq: Optional[int], ) -> temporalio.converter.PayloadCodec: """Return a payload codec with appropriate serialization context. Args: + base_payload_codec: The base payload codec to apply context to. + workflow_context_payload_codec: A payload codec that already has workflow context set. command_seq: Optional sequence number of the associated command. If set, the payload - codec will have serialization context set appropriately for that command. + codec will have serialization context set appropriately for that command. Returns: The payload codec. @@ -2103,68 +2106,71 @@ def _converters_with_context( def get_payload_codec_with_context( self, - payload_codec: temporalio.converter.PayloadCodec, + base_payload_codec: temporalio.converter.PayloadCodec, + workflow_context_payload_codec: temporalio.converter.PayloadCodec, command_seq: Optional[int], ) -> temporalio.converter.PayloadCodec: if not isinstance( - payload_codec, + base_payload_codec, temporalio.converter.WithSerializationContext, ): - return payload_codec - - workflow_context = temporalio.converter.WorkflowSerializationContext( - namespace=self._info.namespace, - workflow_id=self._info.workflow_id, - ) + return base_payload_codec if command_seq is None: # Use payload codec with workflow context by default (i.e. for payloads not associated # with a pending command) - return payload_codec.with_context(workflow_context) + return workflow_context_payload_codec if command_seq in self._pending_activities: - act_handle = self._pending_activities[command_seq] - act_context = temporalio.converter.ActivitySerializationContext( - namespace=self._info.namespace, - workflow_id=self._info.workflow_id, - workflow_type=self._info.workflow_type, - activity_type=act_handle._input.activity, - activity_task_queue=( - act_handle._input.task_queue - if isinstance(act_handle._input, StartActivityInput) - and act_handle._input.task_queue - else self._info.task_queue - ), - is_local=isinstance(act_handle._input, StartLocalActivityInput), + # Use the activity's context + activity_handle = self._pending_activities[command_seq] + return base_payload_codec.with_context( + temporalio.converter.ActivitySerializationContext( + namespace=self._info.namespace, + workflow_id=self._info.workflow_id, + workflow_type=self._info.workflow_type, + activity_type=activity_handle._input.activity, + activity_task_queue=( + activity_handle._input.task_queue + if isinstance(activity_handle._input, StartActivityInput) + and activity_handle._input.task_queue + else self._info.task_queue + ), + is_local=isinstance( + activity_handle._input, StartLocalActivityInput + ), + ) ) - return payload_codec.with_context(act_context) elif command_seq in self._pending_child_workflows: - cwf_handle = self._pending_child_workflows[command_seq] - wf_context = temporalio.converter.WorkflowSerializationContext( - namespace=self._info.namespace, - workflow_id=cwf_handle._input.id, + # Use the child workflow's context + child_wf_handle = self._pending_child_workflows[command_seq] + return base_payload_codec.with_context( + temporalio.converter.WorkflowSerializationContext( + namespace=self._info.namespace, + workflow_id=child_wf_handle._input.id, + ) ) - return payload_codec.with_context(wf_context) elif command_seq in self._pending_external_signals: - # Use the target workflow's context for external signals - _, workflow_id = self._pending_external_signals[command_seq] - wf_context = temporalio.converter.WorkflowSerializationContext( - namespace=self._info.namespace, - workflow_id=workflow_id, + # Use the target workflow's context + _, target_workflow_id = self._pending_external_signals[command_seq] + return base_payload_codec.with_context( + temporalio.converter.WorkflowSerializationContext( + namespace=self._info.namespace, + workflow_id=target_workflow_id, + ) ) - return payload_codec.with_context(wf_context) elif command_seq in self._pending_nexus_operations: # Use empty context for nexus operations: users will never want to encrypt using a # key derived from caller workflow context because the caller workflow context is # not available on the handler side for decryption. - return payload_codec + return base_payload_codec else: # Use payload codec with workflow context for all other payloads - return payload_codec.with_context(workflow_context) + return workflow_context_payload_codec def _instantiate_workflow_object(self) -> Any: if not self._workflow_input: diff --git a/temporalio/worker/workflow_sandbox/_in_sandbox.py b/temporalio/worker/workflow_sandbox/_in_sandbox.py index f34b54424..3905694db 100644 --- a/temporalio/worker/workflow_sandbox/_in_sandbox.py +++ b/temporalio/worker/workflow_sandbox/_in_sandbox.py @@ -83,8 +83,13 @@ def activate( def get_payload_codec_with_context( self, - payload_codec: temporalio.converter.PayloadCodec, + base_payload_codec: temporalio.converter.PayloadCodec, + workflow_context_payload_codec: temporalio.converter.PayloadCodec, command_seq: Optional[int], ) -> temporalio.converter.PayloadCodec: """Get payload codec with context.""" - return self.instance.get_payload_codec_with_context(payload_codec, command_seq) + return self.instance.get_payload_codec_with_context( + base_payload_codec, + workflow_context_payload_codec, + command_seq, + ) diff --git a/temporalio/worker/workflow_sandbox/_runner.py b/temporalio/worker/workflow_sandbox/_runner.py index 1fce6d04b..724f0489f 100644 --- a/temporalio/worker/workflow_sandbox/_runner.py +++ b/temporalio/worker/workflow_sandbox/_runner.py @@ -188,7 +188,8 @@ def get_thread_id(self) -> Optional[int]: def get_payload_codec_with_context( self, - payload_codec: temporalio.converter.PayloadCodec, + base_payload_codec: temporalio.converter.PayloadCodec, + workflow_context_payload_codec: temporalio.converter.PayloadCodec, command_seq: Optional[int], ) -> temporalio.converter.PayloadCodec: # Forward call to the sandboxed instance @@ -196,9 +197,10 @@ def get_payload_codec_with_context( try: self._run_code( "with __temporal_importer.applied():\n" - " __temporal_codec = __temporal_in_sandbox.get_payload_codec_with_context(__temporal_payload_codec, __temporal_command_seq)\n", + " __temporal_codec = __temporal_in_sandbox.get_payload_codec_with_context(__temporal_base_payload_codec, __temporal_workflow_context_payload_codec, __temporal_command_seq)\n", __temporal_importer=self.importer, - __temporal_payload_codec=payload_codec, + __temporal_base_payload_codec=base_payload_codec, + __temporal_workflow_context_payload_codec=workflow_context_payload_codec, __temporal_command_seq=command_seq, ) return self.globals_and_locals.pop("__temporal_codec", None) # type: ignore diff --git a/tests/worker/test_workflow.py b/tests/worker/test_workflow.py index b406a6229..7df4a2d66 100644 --- a/tests/worker/test_workflow.py +++ b/tests/worker/test_workflow.py @@ -1612,11 +1612,14 @@ def activate(self, act: WorkflowActivation) -> WorkflowActivationCompletion: def get_payload_codec_with_context( self, - payload_codec: temporalio.converter.PayloadCodec, + base_payload_codec: temporalio.converter.PayloadCodec, + workflow_context_payload_codec: temporalio.converter.PayloadCodec, command_seq: Optional[int], ) -> temporalio.converter.PayloadCodec: return self._unsandboxed.get_payload_codec_with_context( - payload_codec, command_seq + base_payload_codec, + workflow_context_payload_codec, + command_seq, ) From 0b9025866193fedad17eaf312b7f3750caf34d35 Mon Sep 17 00:00:00 2001 From: Dan Davison Date: Mon, 29 Sep 2025 14:20:49 -0400 Subject: [PATCH 61/81] Rename converters for explicitness --- temporalio/worker/_workflow_instance.py | 87 +++++++++++++++---------- 1 file changed, 52 insertions(+), 35 deletions(-) diff --git a/temporalio/worker/_workflow_instance.py b/temporalio/worker/_workflow_instance.py index cc095b075..9c2014eb1 100644 --- a/temporalio/worker/_workflow_instance.py +++ b/temporalio/worker/_workflow_instance.py @@ -232,12 +232,13 @@ def __init__(self, det: WorkflowInstanceDetails) -> None: self._info = det.info self._context_free_payload_converter = det.payload_converter_class() self._context_free_failure_converter = det.failure_converter_class() - self._payload_converter, self._failure_converter = ( - self._converters_with_context( - temporalio.converter.WorkflowSerializationContext( - namespace=det.info.namespace, - workflow_id=det.info.workflow_id, - ) + ( + self._workflow_context_payload_converter, + self._workflow_context_failure_converter, + ) = self._converters_with_context( + temporalio.converter.WorkflowSerializationContext( + namespace=det.info.namespace, + workflow_id=det.info.workflow_id, ) ) @@ -495,9 +496,9 @@ def activate( # Set completion failure self._current_completion.failed.failure.SetInParent() try: - self._failure_converter.to_failure( + self._workflow_context_failure_converter.to_failure( activation_err, - self._payload_converter, + self._workflow_context_payload_converter, self._current_completion.failed.failure, ) except Exception as inner_err: @@ -639,7 +640,9 @@ async def run_update() -> None: # Run the handler success = await self._inbound.handle_update_handler(handler_input) - result_payloads = self._payload_converter.to_payloads([success]) + result_payloads = self._workflow_context_payload_converter.to_payloads( + [success] + ) if len(result_payloads) != 1: raise ValueError( f"Expected 1 result payload, got {len(result_payloads)}" @@ -671,9 +674,9 @@ async def run_update() -> None: job.protocol_instance_id ) command.update_response.rejected.SetInParent() - self._failure_converter.to_failure( + self._workflow_context_failure_converter.to_failure( err, - self._payload_converter, + self._workflow_context_payload_converter, command.update_response.rejected, ) else: @@ -735,7 +738,9 @@ async def run_query() -> None: headers=job.headers, ) success = await self._inbound.handle_query(input) - result_payloads = self._payload_converter.to_payloads([success]) + result_payloads = ( + self._workflow_context_payload_converter.to_payloads([success]) + ) if len(result_payloads) != 1: raise ValueError( f"Expected 1 result payload, got {len(result_payloads)}" @@ -747,9 +752,9 @@ async def run_query() -> None: try: command = self._add_command() command.respond_to_query.query_id = job.query_id - self._failure_converter.to_failure( + self._workflow_context_failure_converter.to_failure( err, - self._payload_converter, + self._workflow_context_payload_converter, command.respond_to_query.failed, ) except Exception as inner_err: @@ -1055,7 +1060,9 @@ def _apply_initialize_workflow( async def run_workflow(input: ExecuteWorkflowInput) -> None: try: result = await self._inbound.execute_workflow(input) - result_payloads = self._payload_converter.to_payloads([result]) + result_payloads = self._workflow_context_payload_converter.to_payloads( + [result] + ) if len(result_payloads) != 1: raise ValueError( f"Expected 1 result payload, got {len(result_payloads)}" @@ -1095,7 +1102,7 @@ def _make_workflow_input( arg_types = [temporalio.common.RawValue] * len(init_job.arguments) args = self._convert_payloads( - init_job.arguments, arg_types, self._payload_converter + init_job.arguments, arg_types, self._workflow_context_payload_converter ) # Put args in a list if dynamic if not self._defn.name: @@ -1235,7 +1242,7 @@ def workflow_is_replaying(self) -> bool: def workflow_memo(self) -> Mapping[str, Any]: if self._untyped_converted_memo is None: self._untyped_converted_memo = { - k: self._payload_converter.from_payload(v) + k: self._workflow_context_payload_converter.from_payload(v) for k, v in self._info.raw_memo.items() } return self._untyped_converted_memo @@ -1248,7 +1255,7 @@ def workflow_memo_value( if default is temporalio.common._arg_unset: raise KeyError(f"Memo does not have a value for key {key}") return default - return self._payload_converter.from_payload( + return self._workflow_context_payload_converter.from_payload( payload, type_hint, # type: ignore[arg-type] ) @@ -1262,7 +1269,9 @@ def workflow_upsert_memo(self, updates: Mapping[str, Any]) -> None: # Intentionally not checking if memo exists, so that no-op removals show up in history too. removals.append(k) else: - update_payloads[k] = self._payload_converter.to_payload(v) + update_payloads[k] = ( + self._workflow_context_payload_converter.to_payload(v) + ) if not update_payloads and not removals: return @@ -1281,7 +1290,7 @@ def workflow_upsert_memo(self, updates: Mapping[str, Any]) -> None: mut_raw_memo[k] = v if removals: - null_payload = self._payload_converter.to_payload(None) + null_payload = self._workflow_context_payload_converter.to_payload(None) for k in removals: fields[k].CopyFrom(null_payload) mut_raw_memo.pop(k, None) @@ -1289,8 +1298,8 @@ def workflow_upsert_memo(self, updates: Mapping[str, Any]) -> None: # Keeping deserialized memo dict in sync, if exists if self._untyped_converted_memo is not None: for k, v in update_payloads.items(): - self._untyped_converted_memo[k] = self._payload_converter.from_payload( - v + self._untyped_converted_memo[k] = ( + self._workflow_context_payload_converter.from_payload(v) ) for k in removals: self._untyped_converted_memo.pop(k, None) @@ -1328,7 +1337,7 @@ def workflow_patch(self, id: str, *, deprecated: bool) -> bool: return use_patch def workflow_payload_converter(self) -> temporalio.converter.PayloadConverter: - return self._payload_converter + return self._workflow_context_payload_converter def workflow_random(self) -> random.Random: self._assert_not_read_only("random") @@ -1723,7 +1732,7 @@ async def workflow_sleep( ) -> None: user_metadata = ( temporalio.api.sdk.v1.UserMetadata( - summary=self._payload_converter.to_payload(summary) + summary=self._workflow_context_payload_converter.to_payload(summary) ) if summary else None @@ -1748,7 +1757,9 @@ async def workflow_wait_condition( self._conditions.append((fn, fut)) user_metadata = ( temporalio.api.sdk.v1.UserMetadata( - summary=self._payload_converter.to_payload(timeout_summary) + summary=self._workflow_context_payload_converter.to_payload( + timeout_summary + ) ) if timeout_summary else None @@ -1800,18 +1811,18 @@ def workflow_last_completion_result( return None if type_hint is None: - return self._payload_converter.from_payload( + return self._workflow_context_payload_converter.from_payload( self._last_completion_result.payloads[0] ) else: - return self._payload_converter.from_payload( + return self._workflow_context_payload_converter.from_payload( self._last_completion_result.payloads[0], type_hint ) def workflow_last_failure(self) -> Optional[BaseException]: if self._last_failure: - return self._failure_converter.from_failure( - self._last_failure, self._payload_converter + return self._workflow_context_failure_converter.from_failure( + self._last_failure, self._workflow_context_payload_converter ) return None @@ -2256,7 +2267,7 @@ def _process_handler_args( # Take off the string type hint for conversion arg_types = defn_arg_types[1:] if defn_arg_types else None return [job_name] + self._convert_payloads( - job_input, arg_types, self._payload_converter + job_input, arg_types, self._workflow_context_payload_converter ) if not defn_name: return [ @@ -2264,11 +2275,11 @@ def _process_handler_args( self._convert_payloads( job_input, [temporalio.common.RawValue] * len(job_input), - self._payload_converter, + self._workflow_context_payload_converter, ), ] return self._convert_payloads( - job_input, defn_arg_types, self._payload_converter + job_input, defn_arg_types, self._workflow_context_payload_converter ) def _process_signal_job( @@ -2422,7 +2433,9 @@ def _set_workflow_failure(self, err: BaseException) -> None: failure = self._add_command().fail_workflow_execution.failure failure.SetInParent() try: - self._failure_converter.to_failure(err, self._payload_converter, failure) + self._workflow_context_failure_converter.to_failure( + err, self._workflow_context_payload_converter, failure + ) except Exception as inner_err: raise ValueError("Failed converting workflow exception") from inner_err @@ -3329,13 +3342,17 @@ def __init__( def _apply_command(self) -> None: # Convert arguments before creating command in case it raises error payloads = ( - self._instance._payload_converter.to_payloads(self._input.args) + self._instance._workflow_context_payload_converter.to_payloads( + self._input.args + ) if self._input.args else None ) memo_payloads = ( { - k: self._instance._payload_converter.to_payloads([val])[0] + k: self._instance._workflow_context_payload_converter.to_payloads( + [val] + )[0] for k, val in self._input.memo.items() } if self._input.memo From c80452b6479393b87a3200277e372701769fbd1d Mon Sep 17 00:00:00 2001 From: Dan Davison Date: Mon, 29 Sep 2025 16:25:51 -0400 Subject: [PATCH 62/81] Failing test: execute activity and child workflow concurrently to catch bug --- tests/test_serialization_context.py | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/tests/test_serialization_context.py b/tests/test_serialization_context.py index cc0866bb2..45597a81f 100644 --- a/tests/test_serialization_context.py +++ b/tests/test_serialization_context.py @@ -1576,16 +1576,25 @@ async def run(self, data: str) -> str: await workflow.wait_condition( lambda: (self.received_signal and self.received_update) ) - assert "inbound" == await workflow.execute_activity( - payload_encryption_activity, - "outbound", - start_to_close_timeout=timedelta(seconds=10), - ) - assert "inbound" == await workflow.execute_child_workflow( - PayloadEncryptionChildWorkflow.run, - "outbound", - id=f"{workflow.info().workflow_id}_child", + # Run them in parallel to check that data converter operations do not mix up contexts when + # there are multiple concurrent payload types. + coros = [ + workflow.execute_activity( + payload_encryption_activity, + "outbound", + start_to_close_timeout=timedelta(seconds=10), + ), + workflow.execute_child_workflow( + PayloadEncryptionChildWorkflow.run, + "outbound", + id=f"{workflow.info().workflow_id}_child", + ), + ] + [act_result, cw_result], _ = await workflow.wait( + [asyncio.create_task(c) for c in coros] ) + assert await act_result == "inbound" + assert await cw_result == "inbound" return "outbound" @workflow.query From 88f797a4aa502026983b1bfb26b1a47bbcfb2f86 Mon Sep 17 00:00:00 2001 From: Dan Davison Date: Mon, 29 Sep 2025 16:46:43 -0400 Subject: [PATCH 63/81] Use (command_type, command_seq) dataclass --- scripts/gen_payload_visitor.py | 55 +++++++- temporalio/bridge/_visitor.py | 124 ++++++++++++++---- temporalio/worker/_workflow.py | 2 +- temporalio/worker/_workflow_instance.py | 41 ++++-- .../worker/workflow_sandbox/_in_sandbox.py | 5 +- temporalio/worker/workflow_sandbox/_runner.py | 7 +- tests/worker/test_workflow.py | 5 +- 7 files changed, 186 insertions(+), 53 deletions(-) diff --git a/scripts/gen_payload_visitor.py b/scripts/gen_payload_visitor.py index 9e4d2d71e..ec641165c 100644 --- a/scripts/gen_payload_visitor.py +++ b/scripts/gen_payload_visitor.py @@ -72,13 +72,48 @@ def emit_singular_with_seq( ) -> str: # Helper to emit a singular field visit that sets the seq contextvar, with presence check but # without headers guard since this is used for commands only. + + # Map field names to command types + field_to_command_type = { + # Commands + "schedule_activity": "COMMAND_TYPE_SCHEDULE_ACTIVITY_TASK", + "schedule_local_activity": "COMMAND_TYPE_SCHEDULE_ACTIVITY_TASK", + "start_child_workflow_execution": "COMMAND_TYPE_START_CHILD_WORKFLOW_EXECUTION", + "signal_external_workflow_execution": "COMMAND_TYPE_SIGNAL_EXTERNAL_WORKFLOW_EXECUTION", + "schedule_nexus_operation": "COMMAND_TYPE_SCHEDULE_NEXUS_OPERATION", + "request_cancel_activity": "COMMAND_TYPE_REQUEST_CANCEL_ACTIVITY_TASK", + "request_cancel_local_activity": "COMMAND_TYPE_REQUEST_CANCEL_ACTIVITY_TASK", + "request_cancel_external_workflow_execution": "COMMAND_TYPE_REQUEST_CANCEL_EXTERNAL_WORKFLOW_EXECUTION", + "request_cancel_nexus_operation": "COMMAND_TYPE_REQUEST_CANCEL_NEXUS_OPERATION", + "cancel_timer": "COMMAND_TYPE_CANCEL_TIMER", + "cancel_signal_workflow": "COMMAND_TYPE_CANCEL_SIGNAL_WORKFLOW", + "start_timer": "COMMAND_TYPE_START_TIMER", + # Resolutions (use the corresponding command type) + "resolve_activity": "COMMAND_TYPE_SCHEDULE_ACTIVITY_TASK", + "resolve_child_workflow_execution_start": "COMMAND_TYPE_START_CHILD_WORKFLOW_EXECUTION", + "resolve_child_workflow_execution": "COMMAND_TYPE_START_CHILD_WORKFLOW_EXECUTION", + "resolve_signal_external_workflow": "COMMAND_TYPE_SIGNAL_EXTERNAL_WORKFLOW_EXECUTION", + "resolve_request_cancel_external_workflow": "COMMAND_TYPE_REQUEST_CANCEL_EXTERNAL_WORKFLOW_EXECUTION", + "resolve_nexus_operation_start": "COMMAND_TYPE_SCHEDULE_NEXUS_OPERATION", + "resolve_nexus_operation": "COMMAND_TYPE_SCHEDULE_NEXUS_OPERATION", + } + + command_type = field_to_command_type.get(field_name) + if not command_type: + raise ValueError(f"Unknown field with seq: {field_name}") + return f"""\ {presence_word} o.HasField("{field_name}"): - token = current_command_seq.set({access_expr}.seq) + token = current_command_info.set( + CommandInfo( + command_type=temporalio.api.enums.v1.command_type_pb2.CommandType.{command_type}, + command_seq={access_expr}.seq, + ) + ) try: await self._visit_{child_method}(fs, {access_expr}) finally: - current_command_seq.reset(token)""" + current_command_info.reset(token)""" class VisitorGenerator: @@ -100,13 +135,23 @@ def generate(self, roots: list[Descriptor]) -> str: # This file is generated by gen_payload_visitor.py. Changes should be made there. import abc import contextvars +from dataclasses import dataclass from typing import Any, MutableSequence, Optional +import temporalio.api.enums.v1.command_type_pb2 from temporalio.api.common.v1.message_pb2 import Payload -# Current workflow command sequence number -current_command_seq: contextvars.ContextVar[Optional[int]] = contextvars.ContextVar( - "current_command_seq", default=None + +@dataclass(frozen=True) +class CommandInfo: + \"\"\"Information identifying a specific command instance.\"\"\" + + command_type: temporalio.api.enums.v1.command_type_pb2.CommandType + command_seq: int + + +current_command_info: contextvars.ContextVar[Optional[CommandInfo]] = ( + contextvars.ContextVar("current_command_info", default=None) ) class VisitorFunctions(abc.ABC): diff --git a/temporalio/bridge/_visitor.py b/temporalio/bridge/_visitor.py index ba3f8fe1d..d1d0ed2a3 100644 --- a/temporalio/bridge/_visitor.py +++ b/temporalio/bridge/_visitor.py @@ -1,13 +1,23 @@ # This file is generated by gen_payload_visitor.py. Changes should be made there. import abc import contextvars +from dataclasses import dataclass from typing import Any, MutableSequence, Optional +import temporalio.api.enums.v1.command_type_pb2 from temporalio.api.common.v1.message_pb2 import Payload -# Current workflow command sequence number -current_command_seq: contextvars.ContextVar[Optional[int]] = contextvars.ContextVar( - "current_command_seq", default=None + +@dataclass(frozen=True) +class CommandInfo: + """Information identifying a specific command instance.""" + + command_type: temporalio.api.enums.v1.command_type_pb2.CommandType.ValueType + command_seq: int + + +current_command_info: contextvars.ContextVar[Optional[CommandInfo]] = ( + contextvars.ContextVar("current_command_info", default=None) ) @@ -253,53 +263,79 @@ async def _visit_coresdk_workflow_activation_WorkflowActivationJob(self, fs, o): fs, o.signal_workflow ) elif o.HasField("resolve_activity"): - token = current_command_seq.set(o.resolve_activity.seq) + token = current_command_info.set( + CommandInfo( + command_type=temporalio.api.enums.v1.command_type_pb2.CommandType.COMMAND_TYPE_SCHEDULE_ACTIVITY_TASK, + command_seq=o.resolve_activity.seq, + ) + ) try: await self._visit_coresdk_workflow_activation_ResolveActivity( fs, o.resolve_activity ) finally: - current_command_seq.reset(token) + current_command_info.reset(token) elif o.HasField("resolve_child_workflow_execution_start"): - token = current_command_seq.set( - o.resolve_child_workflow_execution_start.seq + token = current_command_info.set( + CommandInfo( + command_type=temporalio.api.enums.v1.command_type_pb2.CommandType.COMMAND_TYPE_START_CHILD_WORKFLOW_EXECUTION, + command_seq=o.resolve_child_workflow_execution_start.seq, + ) ) try: await self._visit_coresdk_workflow_activation_ResolveChildWorkflowExecutionStart( fs, o.resolve_child_workflow_execution_start ) finally: - current_command_seq.reset(token) + current_command_info.reset(token) elif o.HasField("resolve_child_workflow_execution"): - token = current_command_seq.set(o.resolve_child_workflow_execution.seq) + token = current_command_info.set( + CommandInfo( + command_type=temporalio.api.enums.v1.command_type_pb2.CommandType.COMMAND_TYPE_START_CHILD_WORKFLOW_EXECUTION, + command_seq=o.resolve_child_workflow_execution.seq, + ) + ) try: await self._visit_coresdk_workflow_activation_ResolveChildWorkflowExecution( fs, o.resolve_child_workflow_execution ) finally: - current_command_seq.reset(token) + current_command_info.reset(token) elif o.HasField("resolve_signal_external_workflow"): - token = current_command_seq.set(o.resolve_signal_external_workflow.seq) + token = current_command_info.set( + CommandInfo( + command_type=temporalio.api.enums.v1.command_type_pb2.CommandType.COMMAND_TYPE_SIGNAL_EXTERNAL_WORKFLOW_EXECUTION, + command_seq=o.resolve_signal_external_workflow.seq, + ) + ) try: await self._visit_coresdk_workflow_activation_ResolveSignalExternalWorkflow( fs, o.resolve_signal_external_workflow ) finally: - current_command_seq.reset(token) + current_command_info.reset(token) elif o.HasField("resolve_request_cancel_external_workflow"): - token = current_command_seq.set( - o.resolve_request_cancel_external_workflow.seq + token = current_command_info.set( + CommandInfo( + command_type=temporalio.api.enums.v1.command_type_pb2.CommandType.COMMAND_TYPE_REQUEST_CANCEL_EXTERNAL_WORKFLOW_EXECUTION, + command_seq=o.resolve_request_cancel_external_workflow.seq, + ) ) try: await self._visit_coresdk_workflow_activation_ResolveRequestCancelExternalWorkflow( fs, o.resolve_request_cancel_external_workflow ) finally: - current_command_seq.reset(token) + current_command_info.reset(token) elif o.HasField("do_update"): await self._visit_coresdk_workflow_activation_DoUpdate(fs, o.do_update) elif o.HasField("resolve_nexus_operation_start"): - token = current_command_seq.set(o.resolve_nexus_operation_start.seq) + token = current_command_info.set( + CommandInfo( + command_type=temporalio.api.enums.v1.command_type_pb2.CommandType.COMMAND_TYPE_SCHEDULE_NEXUS_OPERATION, + command_seq=o.resolve_nexus_operation_start.seq, + ) + ) try: await ( self._visit_coresdk_workflow_activation_ResolveNexusOperationStart( @@ -307,15 +343,20 @@ async def _visit_coresdk_workflow_activation_WorkflowActivationJob(self, fs, o): ) ) finally: - current_command_seq.reset(token) + current_command_info.reset(token) elif o.HasField("resolve_nexus_operation"): - token = current_command_seq.set(o.resolve_nexus_operation.seq) + token = current_command_info.set( + CommandInfo( + command_type=temporalio.api.enums.v1.command_type_pb2.CommandType.COMMAND_TYPE_SCHEDULE_NEXUS_OPERATION, + command_seq=o.resolve_nexus_operation.seq, + ) + ) try: await self._visit_coresdk_workflow_activation_ResolveNexusOperation( fs, o.resolve_nexus_operation ) finally: - current_command_seq.reset(token) + current_command_info.reset(token) async def _visit_coresdk_workflow_activation_WorkflowActivation(self, fs, o): for v in o.jobs: @@ -414,13 +455,18 @@ async def _visit_coresdk_workflow_commands_WorkflowCommand(self, fs, o): if o.HasField("user_metadata"): await self._visit_temporal_api_sdk_v1_UserMetadata(fs, o.user_metadata) if o.HasField("schedule_activity"): - token = current_command_seq.set(o.schedule_activity.seq) + token = current_command_info.set( + CommandInfo( + command_type=temporalio.api.enums.v1.command_type_pb2.CommandType.COMMAND_TYPE_SCHEDULE_ACTIVITY_TASK, + command_seq=o.schedule_activity.seq, + ) + ) try: await self._visit_coresdk_workflow_commands_ScheduleActivity( fs, o.schedule_activity ) finally: - current_command_seq.reset(token) + current_command_info.reset(token) elif o.HasField("respond_to_query"): await self._visit_coresdk_workflow_commands_QueryResult( fs, o.respond_to_query @@ -438,29 +484,44 @@ async def _visit_coresdk_workflow_commands_WorkflowCommand(self, fs, o): fs, o.continue_as_new_workflow_execution ) elif o.HasField("start_child_workflow_execution"): - token = current_command_seq.set(o.start_child_workflow_execution.seq) + token = current_command_info.set( + CommandInfo( + command_type=temporalio.api.enums.v1.command_type_pb2.CommandType.COMMAND_TYPE_START_CHILD_WORKFLOW_EXECUTION, + command_seq=o.start_child_workflow_execution.seq, + ) + ) try: await self._visit_coresdk_workflow_commands_StartChildWorkflowExecution( fs, o.start_child_workflow_execution ) finally: - current_command_seq.reset(token) + current_command_info.reset(token) elif o.HasField("signal_external_workflow_execution"): - token = current_command_seq.set(o.signal_external_workflow_execution.seq) + token = current_command_info.set( + CommandInfo( + command_type=temporalio.api.enums.v1.command_type_pb2.CommandType.COMMAND_TYPE_SIGNAL_EXTERNAL_WORKFLOW_EXECUTION, + command_seq=o.signal_external_workflow_execution.seq, + ) + ) try: await self._visit_coresdk_workflow_commands_SignalExternalWorkflowExecution( fs, o.signal_external_workflow_execution ) finally: - current_command_seq.reset(token) + current_command_info.reset(token) elif o.HasField("schedule_local_activity"): - token = current_command_seq.set(o.schedule_local_activity.seq) + token = current_command_info.set( + CommandInfo( + command_type=temporalio.api.enums.v1.command_type_pb2.CommandType.COMMAND_TYPE_SCHEDULE_ACTIVITY_TASK, + command_seq=o.schedule_local_activity.seq, + ) + ) try: await self._visit_coresdk_workflow_commands_ScheduleLocalActivity( fs, o.schedule_local_activity ) finally: - current_command_seq.reset(token) + current_command_info.reset(token) elif o.HasField("upsert_workflow_search_attributes"): await self._visit_coresdk_workflow_commands_UpsertWorkflowSearchAttributes( fs, o.upsert_workflow_search_attributes @@ -474,13 +535,18 @@ async def _visit_coresdk_workflow_commands_WorkflowCommand(self, fs, o): fs, o.update_response ) elif o.HasField("schedule_nexus_operation"): - token = current_command_seq.set(o.schedule_nexus_operation.seq) + token = current_command_info.set( + CommandInfo( + command_type=temporalio.api.enums.v1.command_type_pb2.CommandType.COMMAND_TYPE_SCHEDULE_NEXUS_OPERATION, + command_seq=o.schedule_nexus_operation.seq, + ) + ) try: await self._visit_coresdk_workflow_commands_ScheduleNexusOperation( fs, o.schedule_nexus_operation ) finally: - current_command_seq.reset(token) + current_command_info.reset(token) async def _visit_coresdk_workflow_completion_Success(self, fs, o): for v in o.commands: diff --git a/temporalio/worker/_workflow.py b/temporalio/worker/_workflow.py index 9836c6e5b..be230243e 100644 --- a/temporalio/worker/_workflow.py +++ b/temporalio/worker/_workflow.py @@ -752,7 +752,7 @@ def _get_current_command_codec(self) -> temporalio.converter.PayloadCodec: return self.instance.get_payload_codec_with_context( self.context_free_payload_codec, self.workflow_context_payload_codec, - temporalio.bridge._visitor.current_command_seq.get(), + temporalio.bridge._visitor.current_command_info.get(), ) diff --git a/temporalio/worker/_workflow_instance.py b/temporalio/worker/_workflow_instance.py index 9c2014eb1..94949ab98 100644 --- a/temporalio/worker/_workflow_instance.py +++ b/temporalio/worker/_workflow_instance.py @@ -51,6 +51,7 @@ import temporalio.api.common.v1 import temporalio.api.enums.v1 import temporalio.api.sdk.v1 +import temporalio.bridge._visitor import temporalio.bridge.proto.activity_result import temporalio.bridge.proto.child_workflow import temporalio.bridge.proto.common @@ -173,14 +174,14 @@ def get_payload_codec_with_context( self, base_payload_codec: temporalio.converter.PayloadCodec, workflow_context_payload_codec: temporalio.converter.PayloadCodec, - command_seq: Optional[int], + command_info: Optional[temporalio.bridge._visitor.CommandInfo], ) -> temporalio.converter.PayloadCodec: """Return a payload codec with appropriate serialization context. Args: base_payload_codec: The base payload codec to apply context to. workflow_context_payload_codec: A payload codec that already has workflow context set. - command_seq: Optional sequence number of the associated command. If set, the payload + command_info: Optional information identifying the associated command. If set, the payload codec will have serialization context set appropriately for that command. Returns: @@ -2119,7 +2120,7 @@ def get_payload_codec_with_context( self, base_payload_codec: temporalio.converter.PayloadCodec, workflow_context_payload_codec: temporalio.converter.PayloadCodec, - command_seq: Optional[int], + command_info: Optional[temporalio.bridge._visitor.CommandInfo], ) -> temporalio.converter.PayloadCodec: if not isinstance( base_payload_codec, @@ -2127,14 +2128,18 @@ def get_payload_codec_with_context( ): return base_payload_codec - if command_seq is None: + if command_info is None: # Use payload codec with workflow context by default (i.e. for payloads not associated # with a pending command) return workflow_context_payload_codec - if command_seq in self._pending_activities: + if ( + command_info.command_type + == temporalio.api.enums.v1.command_type_pb2.CommandType.COMMAND_TYPE_SCHEDULE_ACTIVITY_TASK + and command_info.command_seq in self._pending_activities + ): # Use the activity's context - activity_handle = self._pending_activities[command_seq] + activity_handle = self._pending_activities[command_info.command_seq] return base_payload_codec.with_context( temporalio.converter.ActivitySerializationContext( namespace=self._info.namespace, @@ -2153,9 +2158,13 @@ def get_payload_codec_with_context( ) ) - elif command_seq in self._pending_child_workflows: + elif ( + command_info.command_type + == temporalio.api.enums.v1.command_type_pb2.CommandType.COMMAND_TYPE_START_CHILD_WORKFLOW_EXECUTION + and command_info.command_seq in self._pending_child_workflows + ): # Use the child workflow's context - child_wf_handle = self._pending_child_workflows[command_seq] + child_wf_handle = self._pending_child_workflows[command_info.command_seq] return base_payload_codec.with_context( temporalio.converter.WorkflowSerializationContext( namespace=self._info.namespace, @@ -2163,9 +2172,15 @@ def get_payload_codec_with_context( ) ) - elif command_seq in self._pending_external_signals: + elif ( + command_info.command_type + == temporalio.api.enums.v1.command_type_pb2.CommandType.COMMAND_TYPE_SIGNAL_EXTERNAL_WORKFLOW_EXECUTION + and command_info.command_seq in self._pending_external_signals + ): # Use the target workflow's context - _, target_workflow_id = self._pending_external_signals[command_seq] + _, target_workflow_id = self._pending_external_signals[ + command_info.command_seq + ] return base_payload_codec.with_context( temporalio.converter.WorkflowSerializationContext( namespace=self._info.namespace, @@ -2173,7 +2188,11 @@ def get_payload_codec_with_context( ) ) - elif command_seq in self._pending_nexus_operations: + elif ( + command_info.command_type + == temporalio.api.enums.v1.command_type_pb2.CommandType.COMMAND_TYPE_SCHEDULE_NEXUS_OPERATION + and command_info.command_seq in self._pending_nexus_operations + ): # Use empty context for nexus operations: users will never want to encrypt using a # key derived from caller workflow context because the caller workflow context is # not available on the handler side for decryption. diff --git a/temporalio/worker/workflow_sandbox/_in_sandbox.py b/temporalio/worker/workflow_sandbox/_in_sandbox.py index 3905694db..0843c7e92 100644 --- a/temporalio/worker/workflow_sandbox/_in_sandbox.py +++ b/temporalio/worker/workflow_sandbox/_in_sandbox.py @@ -8,6 +8,7 @@ import logging from typing import Any, Optional, Type +import temporalio.bridge._visitor import temporalio.bridge.proto.workflow_activation import temporalio.bridge.proto.workflow_completion import temporalio.converter @@ -85,11 +86,11 @@ def get_payload_codec_with_context( self, base_payload_codec: temporalio.converter.PayloadCodec, workflow_context_payload_codec: temporalio.converter.PayloadCodec, - command_seq: Optional[int], + command_info: Optional[temporalio.bridge._visitor.CommandInfo], ) -> temporalio.converter.PayloadCodec: """Get payload codec with context.""" return self.instance.get_payload_codec_with_context( base_payload_codec, workflow_context_payload_codec, - command_seq, + command_info, ) diff --git a/temporalio/worker/workflow_sandbox/_runner.py b/temporalio/worker/workflow_sandbox/_runner.py index 724f0489f..e1fe0adc8 100644 --- a/temporalio/worker/workflow_sandbox/_runner.py +++ b/temporalio/worker/workflow_sandbox/_runner.py @@ -11,6 +11,7 @@ from datetime import datetime, timedelta, timezone from typing import Any, Optional, Sequence, Type +import temporalio.bridge._visitor import temporalio.bridge.proto.workflow_activation import temporalio.bridge.proto.workflow_completion import temporalio.common @@ -190,18 +191,18 @@ def get_payload_codec_with_context( self, base_payload_codec: temporalio.converter.PayloadCodec, workflow_context_payload_codec: temporalio.converter.PayloadCodec, - command_seq: Optional[int], + command_info: Optional[temporalio.bridge._visitor.CommandInfo], ) -> temporalio.converter.PayloadCodec: # Forward call to the sandboxed instance self.importer.restriction_context.is_runtime = True try: self._run_code( "with __temporal_importer.applied():\n" - " __temporal_codec = __temporal_in_sandbox.get_payload_codec_with_context(__temporal_base_payload_codec, __temporal_workflow_context_payload_codec, __temporal_command_seq)\n", + " __temporal_codec = __temporal_in_sandbox.get_payload_codec_with_context(__temporal_base_payload_codec, __temporal_workflow_context_payload_codec, __temporal_command_info)\n", __temporal_importer=self.importer, __temporal_base_payload_codec=base_payload_codec, __temporal_workflow_context_payload_codec=workflow_context_payload_codec, - __temporal_command_seq=command_seq, + __temporal_command_info=command_info, ) return self.globals_and_locals.pop("__temporal_codec", None) # type: ignore finally: diff --git a/tests/worker/test_workflow.py b/tests/worker/test_workflow.py index 7df4a2d66..7454de124 100644 --- a/tests/worker/test_workflow.py +++ b/tests/worker/test_workflow.py @@ -43,6 +43,7 @@ import temporalio.activity import temporalio.api.sdk.v1 +import temporalio.bridge._visitor import temporalio.client import temporalio.converter import temporalio.worker @@ -1614,12 +1615,12 @@ def get_payload_codec_with_context( self, base_payload_codec: temporalio.converter.PayloadCodec, workflow_context_payload_codec: temporalio.converter.PayloadCodec, - command_seq: Optional[int], + command_info: Optional[temporalio.bridge._visitor.CommandInfo], ) -> temporalio.converter.PayloadCodec: return self._unsandboxed.get_payload_codec_with_context( base_payload_codec, workflow_context_payload_codec, - command_seq, + command_info, ) From e802435926ad2bacecc586c93870a34bfab67edb Mon Sep 17 00:00:00 2001 From: Dan Davison Date: Mon, 29 Sep 2025 18:08:34 -0400 Subject: [PATCH 64/81] Compute context, not payload codec --- temporalio/worker/_workflow.py | 16 ++-- temporalio/worker/_workflow_instance.py | 78 ++++++++----------- .../worker/workflow_sandbox/_in_sandbox.py | 14 +--- temporalio/worker/workflow_sandbox/_runner.py | 12 +-- tests/worker/test_workflow.py | 12 +-- 5 files changed, 52 insertions(+), 80 deletions(-) diff --git a/temporalio/worker/_workflow.py b/temporalio/worker/_workflow.py index be230243e..2afd2f7e5 100644 --- a/temporalio/worker/_workflow.py +++ b/temporalio/worker/_workflow.py @@ -290,7 +290,6 @@ async def _handle_activation( payload_codec = _CommandAwarePayloadCodec( workflow.instance, context_free_payload_codec=self._data_converter.payload_codec, - workflow_context_payload_codec=data_converter.payload_codec, ) await temporalio.bridge.worker.decode_activation( act, @@ -367,7 +366,6 @@ async def _handle_activation( payload_codec = _CommandAwarePayloadCodec( workflow.instance, context_free_payload_codec=self._data_converter.payload_codec, - workflow_context_payload_codec=data_converter.payload_codec, ) try: await temporalio.bridge.worker.encode_completion( @@ -734,7 +732,6 @@ class _CommandAwarePayloadCodec(temporalio.converter.PayloadCodec): instance: WorkflowInstance context_free_payload_codec: temporalio.converter.PayloadCodec - workflow_context_payload_codec: temporalio.converter.PayloadCodec async def encode( self, @@ -749,11 +746,18 @@ async def decode( return await self._get_current_command_codec().decode(payloads) def _get_current_command_codec(self) -> temporalio.converter.PayloadCodec: - return self.instance.get_payload_codec_with_context( + if not isinstance( self.context_free_payload_codec, - self.workflow_context_payload_codec, + temporalio.converter.WithSerializationContext, + ): + return self.context_free_payload_codec + + if context := self.instance.get_serialization_context( temporalio.bridge._visitor.current_command_info.get(), - ) + ): + return self.context_free_payload_codec.with_context(context) + + return self.context_free_payload_codec class _InterruptDeadlockError(BaseException): diff --git a/temporalio/worker/_workflow_instance.py b/temporalio/worker/_workflow_instance.py index 94949ab98..8e98fe757 100644 --- a/temporalio/worker/_workflow_instance.py +++ b/temporalio/worker/_workflow_instance.py @@ -170,22 +170,18 @@ def activate( raise NotImplementedError @abstractmethod - def get_payload_codec_with_context( + def get_serialization_context( self, - base_payload_codec: temporalio.converter.PayloadCodec, - workflow_context_payload_codec: temporalio.converter.PayloadCodec, command_info: Optional[temporalio.bridge._visitor.CommandInfo], - ) -> temporalio.converter.PayloadCodec: - """Return a payload codec with appropriate serialization context. + ) -> Optional[temporalio.converter.SerializationContext]: + """Return appropriate serialization context. Args: - base_payload_codec: The base payload codec to apply context to. - workflow_context_payload_codec: A payload codec that already has workflow context set. command_info: Optional information identifying the associated command. If set, the payload codec will have serialization context set appropriately for that command. Returns: - The payload codec. + The serialization context, or None if no context should be set. """ raise NotImplementedError @@ -2116,22 +2112,18 @@ def _converters_with_context( failure_converter = failure_converter.with_context(context) return payload_converter, failure_converter - def get_payload_codec_with_context( + def get_serialization_context( self, - base_payload_codec: temporalio.converter.PayloadCodec, - workflow_context_payload_codec: temporalio.converter.PayloadCodec, command_info: Optional[temporalio.bridge._visitor.CommandInfo], - ) -> temporalio.converter.PayloadCodec: - if not isinstance( - base_payload_codec, - temporalio.converter.WithSerializationContext, - ): - return base_payload_codec - + ) -> Optional[temporalio.converter.SerializationContext]: + workflow_context = temporalio.converter.WorkflowSerializationContext( + namespace=self._info.namespace, + workflow_id=self._info.workflow_id, + ) if command_info is None: # Use payload codec with workflow context by default (i.e. for payloads not associated # with a pending command) - return workflow_context_payload_codec + return workflow_context if ( command_info.command_type @@ -2140,22 +2132,18 @@ def get_payload_codec_with_context( ): # Use the activity's context activity_handle = self._pending_activities[command_info.command_seq] - return base_payload_codec.with_context( - temporalio.converter.ActivitySerializationContext( - namespace=self._info.namespace, - workflow_id=self._info.workflow_id, - workflow_type=self._info.workflow_type, - activity_type=activity_handle._input.activity, - activity_task_queue=( - activity_handle._input.task_queue - if isinstance(activity_handle._input, StartActivityInput) - and activity_handle._input.task_queue - else self._info.task_queue - ), - is_local=isinstance( - activity_handle._input, StartLocalActivityInput - ), - ) + return temporalio.converter.ActivitySerializationContext( + namespace=self._info.namespace, + workflow_id=self._info.workflow_id, + workflow_type=self._info.workflow_type, + activity_type=activity_handle._input.activity, + activity_task_queue=( + activity_handle._input.task_queue + if isinstance(activity_handle._input, StartActivityInput) + and activity_handle._input.task_queue + else self._info.task_queue + ), + is_local=isinstance(activity_handle._input, StartLocalActivityInput), ) elif ( @@ -2165,11 +2153,9 @@ def get_payload_codec_with_context( ): # Use the child workflow's context child_wf_handle = self._pending_child_workflows[command_info.command_seq] - return base_payload_codec.with_context( - temporalio.converter.WorkflowSerializationContext( - namespace=self._info.namespace, - workflow_id=child_wf_handle._input.id, - ) + return temporalio.converter.WorkflowSerializationContext( + namespace=self._info.namespace, + workflow_id=child_wf_handle._input.id, ) elif ( @@ -2181,11 +2167,9 @@ def get_payload_codec_with_context( _, target_workflow_id = self._pending_external_signals[ command_info.command_seq ] - return base_payload_codec.with_context( - temporalio.converter.WorkflowSerializationContext( - namespace=self._info.namespace, - workflow_id=target_workflow_id, - ) + return temporalio.converter.WorkflowSerializationContext( + namespace=self._info.namespace, + workflow_id=target_workflow_id, ) elif ( @@ -2196,11 +2180,11 @@ def get_payload_codec_with_context( # Use empty context for nexus operations: users will never want to encrypt using a # key derived from caller workflow context because the caller workflow context is # not available on the handler side for decryption. - return base_payload_codec + return None else: # Use payload codec with workflow context for all other payloads - return workflow_context_payload_codec + return workflow_context def _instantiate_workflow_object(self) -> Any: if not self._workflow_input: diff --git a/temporalio/worker/workflow_sandbox/_in_sandbox.py b/temporalio/worker/workflow_sandbox/_in_sandbox.py index 0843c7e92..689d77716 100644 --- a/temporalio/worker/workflow_sandbox/_in_sandbox.py +++ b/temporalio/worker/workflow_sandbox/_in_sandbox.py @@ -82,15 +82,9 @@ def activate( """Send activation to this instance.""" return self.instance.activate(act) - def get_payload_codec_with_context( + def get_serialization_context( self, - base_payload_codec: temporalio.converter.PayloadCodec, - workflow_context_payload_codec: temporalio.converter.PayloadCodec, command_info: Optional[temporalio.bridge._visitor.CommandInfo], - ) -> temporalio.converter.PayloadCodec: - """Get payload codec with context.""" - return self.instance.get_payload_codec_with_context( - base_payload_codec, - workflow_context_payload_codec, - command_info, - ) + ) -> Optional[temporalio.converter.SerializationContext]: + """Get serialization context.""" + return self.instance.get_serialization_context(command_info) diff --git a/temporalio/worker/workflow_sandbox/_runner.py b/temporalio/worker/workflow_sandbox/_runner.py index e1fe0adc8..d9d0601f6 100644 --- a/temporalio/worker/workflow_sandbox/_runner.py +++ b/temporalio/worker/workflow_sandbox/_runner.py @@ -187,23 +187,19 @@ def _run_code(self, code: str, **extra_globals: Any) -> None: def get_thread_id(self) -> Optional[int]: return self._current_thread_id - def get_payload_codec_with_context( + def get_serialization_context( self, - base_payload_codec: temporalio.converter.PayloadCodec, - workflow_context_payload_codec: temporalio.converter.PayloadCodec, command_info: Optional[temporalio.bridge._visitor.CommandInfo], - ) -> temporalio.converter.PayloadCodec: + ) -> Optional[temporalio.converter.SerializationContext]: # Forward call to the sandboxed instance self.importer.restriction_context.is_runtime = True try: self._run_code( "with __temporal_importer.applied():\n" - " __temporal_codec = __temporal_in_sandbox.get_payload_codec_with_context(__temporal_base_payload_codec, __temporal_workflow_context_payload_codec, __temporal_command_info)\n", + " __temporal_context = __temporal_in_sandbox.get_serialization_context(__temporal_command_info)\n", __temporal_importer=self.importer, - __temporal_base_payload_codec=base_payload_codec, - __temporal_workflow_context_payload_codec=workflow_context_payload_codec, __temporal_command_info=command_info, ) - return self.globals_and_locals.pop("__temporal_codec", None) # type: ignore + return self.globals_and_locals.pop("__temporal_context", None) # type: ignore finally: self.importer.restriction_context.is_runtime = False diff --git a/tests/worker/test_workflow.py b/tests/worker/test_workflow.py index 7454de124..15b340380 100644 --- a/tests/worker/test_workflow.py +++ b/tests/worker/test_workflow.py @@ -1611,17 +1611,11 @@ def activate(self, act: WorkflowActivation) -> WorkflowActivationCompletion: self._runner._pairs.append((act, comp)) return comp - def get_payload_codec_with_context( + def get_serialization_context( self, - base_payload_codec: temporalio.converter.PayloadCodec, - workflow_context_payload_codec: temporalio.converter.PayloadCodec, command_info: Optional[temporalio.bridge._visitor.CommandInfo], - ) -> temporalio.converter.PayloadCodec: - return self._unsandboxed.get_payload_codec_with_context( - base_payload_codec, - workflow_context_payload_codec, - command_info, - ) + ) -> Optional[temporalio.converter.SerializationContext]: + return self._unsandboxed.get_serialization_context(command_info) async def test_workflow_with_custom_runner(client: Client): From d877aa8842fc5fa93f51da8825f9dedd50e23858 Mon Sep 17 00:00:00 2001 From: Dan Davison Date: Mon, 29 Sep 2025 18:25:09 -0400 Subject: [PATCH 65/81] Cleanup: use converters from nexus / child workflow handles --- temporalio/worker/_workflow_instance.py | 69 +++++++++++-------------- 1 file changed, 29 insertions(+), 40 deletions(-) diff --git a/temporalio/worker/_workflow_instance.py b/temporalio/worker/_workflow_instance.py index 8e98fe757..27fc2546d 100644 --- a/temporalio/worker/_workflow_instance.py +++ b/temporalio/worker/_workflow_instance.py @@ -836,12 +836,7 @@ def _apply_resolve_child_workflow_execution( raise RuntimeError( f"Failed finding child workflow handle for sequence {job.seq}" ) - payload_converter, failure_converter = self._converters_with_context( - temporalio.converter.WorkflowSerializationContext( - namespace=self._info.namespace, - workflow_id=handle._input.id, - ) - ) + if job.result.HasField("completed"): ret: Optional[Any] = None if job.result.completed.HasField("result"): @@ -849,20 +844,20 @@ def _apply_resolve_child_workflow_execution( ret_vals = self._convert_payloads( [job.result.completed.result], ret_types, - payload_converter, + handle._payload_converter, ) ret = ret_vals[0] handle._resolve_success(ret) elif job.result.HasField("failed"): handle._resolve_failure( - failure_converter.from_failure( - job.result.failed.failure, payload_converter + handle._failure_converter.from_failure( + job.result.failed.failure, handle._payload_converter ) ) elif job.result.HasField("cancelled"): handle._resolve_failure( - failure_converter.from_failure( - job.result.cancelled.failure, payload_converter + handle._failure_converter.from_failure( + job.result.cancelled.failure, handle._payload_converter ) ) else: @@ -898,14 +893,10 @@ def _apply_resolve_child_workflow_execution_start( ) elif job.HasField("cancelled"): self._pending_child_workflows.pop(job.seq) - payload_converter, failure_converter = self._converters_with_context( - temporalio.converter.WorkflowSerializationContext( - namespace=self._info.namespace, - workflow_id=handle._input.id, - ) - ) handle._resolve_failure( - failure_converter.from_failure(job.cancelled.failure, payload_converter) + handle._failure_converter.from_failure( + job.cancelled.failure, handle._payload_converter + ) ) else: raise RuntimeError("Child workflow start did not have a known status") @@ -919,13 +910,6 @@ def _apply_resolve_nexus_operation_start( raise RuntimeError( f"Failed to find nexus operation handle for job sequence number {job.seq}" ) - # We don't set a serialization context for nexus operations on the caller side because it's - # not possible to set a matching context on the handler side. - payload_converter, failure_converter = ( - self._context_free_payload_converter, - self._context_free_failure_converter, - ) - if job.HasField("operation_token"): # The nexus operation started asynchronously. A `ResolveNexusOperation` job # will follow in a future activation. @@ -938,7 +922,9 @@ def _apply_resolve_nexus_operation_start( # The nexus operation start failed; no ResolveNexusOperation will follow. self._pending_nexus_operations.pop(job.seq, None) handle._resolve_failure( - failure_converter.from_failure(job.failed, payload_converter) + handle._failure_converter.from_failure( + job.failed, handle._payload_converter + ) ) else: raise ValueError(f"Unknown Nexus operation start status: {job}") @@ -961,32 +947,32 @@ def _apply_resolve_nexus_operation( # completed / failed, but it has already been resolved. return - # We don't set a serialization context for nexus operations on the caller side because it is - # not possible to set the same context on the handler side. - payload_converter, failure_converter = ( - self._context_free_payload_converter, - self._context_free_failure_converter, - ) # Handle the four oneof variants of NexusOperationResult result = job.result if result.HasField("completed"): [output] = self._convert_payloads( [result.completed], [handle._input.output_type] if handle._input.output_type else None, - payload_converter, + handle._payload_converter, ) handle._resolve_success(output) elif result.HasField("failed"): handle._resolve_failure( - failure_converter.from_failure(result.failed, payload_converter) + handle._failure_converter.from_failure( + result.failed, handle._payload_converter + ) ) elif result.HasField("cancelled"): handle._resolve_failure( - failure_converter.from_failure(result.cancelled, payload_converter) + handle._failure_converter.from_failure( + result.cancelled, handle._payload_converter + ) ) elif result.HasField("timed_out"): handle._resolve_failure( - failure_converter.from_failure(result.timed_out, payload_converter) + handle._failure_converter.from_failure( + result.timed_out, handle._payload_converter + ) ) else: raise RuntimeError("Nexus operation did not have a result") @@ -3083,10 +3069,12 @@ def __init__( self._result_fut: asyncio.Future[Any] = instance.create_future() self._first_execution_run_id = "" instance._register_task(self, name=f"child: {input.workflow}") - self._payload_converter, _ = self._instance._converters_with_context( - temporalio.converter.WorkflowSerializationContext( - namespace=self._instance._info.namespace, - workflow_id=self._input.id, + self._payload_converter, self._failure_converter = ( + self._instance._converters_with_context( + temporalio.converter.WorkflowSerializationContext( + namespace=self._instance._info.namespace, + workflow_id=self._input.id, + ) ) ) @@ -3274,6 +3262,7 @@ def __init__( self._start_fut: asyncio.Future[Optional[str]] = instance.create_future() self._result_fut: asyncio.Future[Optional[OutputT]] = instance.create_future() self._payload_converter = self._instance._context_free_payload_converter + self._failure_converter = self._instance._context_free_failure_converter @property def operation_token(self) -> Optional[str]: From b9ce8c879a4ca7231b40e4d7e78d7af322ee5a32 Mon Sep 17 00:00:00 2001 From: Dan Davison Date: Mon, 29 Sep 2025 18:52:29 -0400 Subject: [PATCH 66/81] Refactor: don't use default PayloadVisitor to set command context --- scripts/gen_payload_visitor.py | 111 +---------- temporalio/bridge/_visitor.py | 176 +++--------------- temporalio/bridge/worker.py | 7 +- temporalio/client.py | 1 - temporalio/worker/_command_aware_visitor.py | 145 +++++++++++++++ temporalio/worker/_workflow.py | 9 +- temporalio/worker/_workflow_instance.py | 6 +- .../worker/workflow_sandbox/_in_sandbox.py | 4 +- temporalio/worker/workflow_sandbox/_runner.py | 5 +- tests/worker/test_workflow.py | 4 +- 10 files changed, 192 insertions(+), 276 deletions(-) create mode 100644 temporalio/worker/_command_aware_visitor.py diff --git a/scripts/gen_payload_visitor.py b/scripts/gen_payload_visitor.py index ec641165c..e8ddf38bd 100644 --- a/scripts/gen_payload_visitor.py +++ b/scripts/gen_payload_visitor.py @@ -67,55 +67,6 @@ def emit_singular( await self._visit_{child_method}(fs, {access_expr})""" -def emit_singular_with_seq( - field_name: str, access_expr: str, child_method: str, presence_word: str -) -> str: - # Helper to emit a singular field visit that sets the seq contextvar, with presence check but - # without headers guard since this is used for commands only. - - # Map field names to command types - field_to_command_type = { - # Commands - "schedule_activity": "COMMAND_TYPE_SCHEDULE_ACTIVITY_TASK", - "schedule_local_activity": "COMMAND_TYPE_SCHEDULE_ACTIVITY_TASK", - "start_child_workflow_execution": "COMMAND_TYPE_START_CHILD_WORKFLOW_EXECUTION", - "signal_external_workflow_execution": "COMMAND_TYPE_SIGNAL_EXTERNAL_WORKFLOW_EXECUTION", - "schedule_nexus_operation": "COMMAND_TYPE_SCHEDULE_NEXUS_OPERATION", - "request_cancel_activity": "COMMAND_TYPE_REQUEST_CANCEL_ACTIVITY_TASK", - "request_cancel_local_activity": "COMMAND_TYPE_REQUEST_CANCEL_ACTIVITY_TASK", - "request_cancel_external_workflow_execution": "COMMAND_TYPE_REQUEST_CANCEL_EXTERNAL_WORKFLOW_EXECUTION", - "request_cancel_nexus_operation": "COMMAND_TYPE_REQUEST_CANCEL_NEXUS_OPERATION", - "cancel_timer": "COMMAND_TYPE_CANCEL_TIMER", - "cancel_signal_workflow": "COMMAND_TYPE_CANCEL_SIGNAL_WORKFLOW", - "start_timer": "COMMAND_TYPE_START_TIMER", - # Resolutions (use the corresponding command type) - "resolve_activity": "COMMAND_TYPE_SCHEDULE_ACTIVITY_TASK", - "resolve_child_workflow_execution_start": "COMMAND_TYPE_START_CHILD_WORKFLOW_EXECUTION", - "resolve_child_workflow_execution": "COMMAND_TYPE_START_CHILD_WORKFLOW_EXECUTION", - "resolve_signal_external_workflow": "COMMAND_TYPE_SIGNAL_EXTERNAL_WORKFLOW_EXECUTION", - "resolve_request_cancel_external_workflow": "COMMAND_TYPE_REQUEST_CANCEL_EXTERNAL_WORKFLOW_EXECUTION", - "resolve_nexus_operation_start": "COMMAND_TYPE_SCHEDULE_NEXUS_OPERATION", - "resolve_nexus_operation": "COMMAND_TYPE_SCHEDULE_NEXUS_OPERATION", - } - - command_type = field_to_command_type.get(field_name) - if not command_type: - raise ValueError(f"Unknown field with seq: {field_name}") - - return f"""\ - {presence_word} o.HasField("{field_name}"): - token = current_command_info.set( - CommandInfo( - command_type=temporalio.api.enums.v1.command_type_pb2.CommandType.{command_type}, - command_seq={access_expr}.seq, - ) - ) - try: - await self._visit_{child_method}(fs, {access_expr}) - finally: - current_command_info.reset(token)""" - - class VisitorGenerator: def generate(self, roots: list[Descriptor]) -> str: """ @@ -134,26 +85,11 @@ def generate(self, roots: list[Descriptor]) -> str: header = """ # This file is generated by gen_payload_visitor.py. Changes should be made there. import abc -import contextvars -from dataclasses import dataclass -from typing import Any, MutableSequence, Optional +from typing import Any, MutableSequence -import temporalio.api.enums.v1.command_type_pb2 from temporalio.api.common.v1.message_pb2 import Payload -@dataclass(frozen=True) -class CommandInfo: - \"\"\"Information identifying a specific command instance.\"\"\" - - command_type: temporalio.api.enums.v1.command_type_pb2.CommandType - command_seq: int - - -current_command_info: contextvars.ContextVar[Optional[CommandInfo]] = ( - contextvars.ContextVar("current_command_info", default=None) -) - class VisitorFunctions(abc.ABC): \"\"\"Set of functions which can be called by the visitor. Allows handling payloads as a sequence. @@ -323,29 +259,6 @@ def walk(self, desc: Descriptor) -> bool: ) ) - commands_with_seq = { - "cancel_signal_workflow", - "cancel_timer", - "request_cancel_activity", - "request_cancel_external_workflow_execution", - "request_cancel_local_activity", - "request_cancel_nexus_operation", - "schedule_activity", - "schedule_local_activity", - "schedule_nexus_operation", - "signal_external_workflow_execution", - "start_child_workflow_execution", - "start_timer", - } - activation_jobs_with_seq = { - "resolve_activity", - "resolve_child_workflow_execution_start", - "resolve_child_workflow_execution", - "resolve_nexus_operation_start", - "resolve_nexus_operation", - "resolve_request_cancel_external_workflow", - "resolve_signal_external_workflow", - } # Process oneof fields as if/elif chains for oneof_idx, fields in oneof_fields.items(): oneof_lines = [] @@ -357,25 +270,9 @@ def walk(self, desc: Descriptor) -> bool: if child_has_payload: if_word = "if" if first else "elif" first = False - if ( - desc.full_name == "coresdk.workflow_commands.WorkflowCommand" - and field.name in commands_with_seq - ): - line = emit_singular_with_seq( - field.name, f"o.{field.name}", name_for(child_desc), if_word - ) - elif ( - desc.full_name - == "coresdk.workflow_activation.WorkflowActivationJob" - and field.name in activation_jobs_with_seq - ): - line = emit_singular_with_seq( - field.name, f"o.{field.name}", name_for(child_desc), if_word - ) - else: - line = emit_singular( - field.name, f"o.{field.name}", name_for(child_desc), if_word - ) + line = emit_singular( + field.name, f"o.{field.name}", name_for(child_desc), if_word + ) oneof_lines.append(line) if oneof_lines: lines.extend(oneof_lines) diff --git a/temporalio/bridge/_visitor.py b/temporalio/bridge/_visitor.py index d1d0ed2a3..0491b5e88 100644 --- a/temporalio/bridge/_visitor.py +++ b/temporalio/bridge/_visitor.py @@ -1,26 +1,10 @@ # This file is generated by gen_payload_visitor.py. Changes should be made there. import abc -import contextvars -from dataclasses import dataclass -from typing import Any, MutableSequence, Optional +from typing import Any, MutableSequence -import temporalio.api.enums.v1.command_type_pb2 from temporalio.api.common.v1.message_pb2 import Payload -@dataclass(frozen=True) -class CommandInfo: - """Information identifying a specific command instance.""" - - command_type: temporalio.api.enums.v1.command_type_pb2.CommandType.ValueType - command_seq: int - - -current_command_info: contextvars.ContextVar[Optional[CommandInfo]] = ( - contextvars.ContextVar("current_command_info", default=None) -) - - class VisitorFunctions(abc.ABC): """Set of functions which can be called by the visitor. Allows handling payloads as a sequence. @@ -263,100 +247,35 @@ async def _visit_coresdk_workflow_activation_WorkflowActivationJob(self, fs, o): fs, o.signal_workflow ) elif o.HasField("resolve_activity"): - token = current_command_info.set( - CommandInfo( - command_type=temporalio.api.enums.v1.command_type_pb2.CommandType.COMMAND_TYPE_SCHEDULE_ACTIVITY_TASK, - command_seq=o.resolve_activity.seq, - ) + await self._visit_coresdk_workflow_activation_ResolveActivity( + fs, o.resolve_activity ) - try: - await self._visit_coresdk_workflow_activation_ResolveActivity( - fs, o.resolve_activity - ) - finally: - current_command_info.reset(token) elif o.HasField("resolve_child_workflow_execution_start"): - token = current_command_info.set( - CommandInfo( - command_type=temporalio.api.enums.v1.command_type_pb2.CommandType.COMMAND_TYPE_START_CHILD_WORKFLOW_EXECUTION, - command_seq=o.resolve_child_workflow_execution_start.seq, - ) + await self._visit_coresdk_workflow_activation_ResolveChildWorkflowExecutionStart( + fs, o.resolve_child_workflow_execution_start ) - try: - await self._visit_coresdk_workflow_activation_ResolveChildWorkflowExecutionStart( - fs, o.resolve_child_workflow_execution_start - ) - finally: - current_command_info.reset(token) elif o.HasField("resolve_child_workflow_execution"): - token = current_command_info.set( - CommandInfo( - command_type=temporalio.api.enums.v1.command_type_pb2.CommandType.COMMAND_TYPE_START_CHILD_WORKFLOW_EXECUTION, - command_seq=o.resolve_child_workflow_execution.seq, - ) + await self._visit_coresdk_workflow_activation_ResolveChildWorkflowExecution( + fs, o.resolve_child_workflow_execution ) - try: - await self._visit_coresdk_workflow_activation_ResolveChildWorkflowExecution( - fs, o.resolve_child_workflow_execution - ) - finally: - current_command_info.reset(token) elif o.HasField("resolve_signal_external_workflow"): - token = current_command_info.set( - CommandInfo( - command_type=temporalio.api.enums.v1.command_type_pb2.CommandType.COMMAND_TYPE_SIGNAL_EXTERNAL_WORKFLOW_EXECUTION, - command_seq=o.resolve_signal_external_workflow.seq, - ) + await self._visit_coresdk_workflow_activation_ResolveSignalExternalWorkflow( + fs, o.resolve_signal_external_workflow ) - try: - await self._visit_coresdk_workflow_activation_ResolveSignalExternalWorkflow( - fs, o.resolve_signal_external_workflow - ) - finally: - current_command_info.reset(token) elif o.HasField("resolve_request_cancel_external_workflow"): - token = current_command_info.set( - CommandInfo( - command_type=temporalio.api.enums.v1.command_type_pb2.CommandType.COMMAND_TYPE_REQUEST_CANCEL_EXTERNAL_WORKFLOW_EXECUTION, - command_seq=o.resolve_request_cancel_external_workflow.seq, - ) + await self._visit_coresdk_workflow_activation_ResolveRequestCancelExternalWorkflow( + fs, o.resolve_request_cancel_external_workflow ) - try: - await self._visit_coresdk_workflow_activation_ResolveRequestCancelExternalWorkflow( - fs, o.resolve_request_cancel_external_workflow - ) - finally: - current_command_info.reset(token) elif o.HasField("do_update"): await self._visit_coresdk_workflow_activation_DoUpdate(fs, o.do_update) elif o.HasField("resolve_nexus_operation_start"): - token = current_command_info.set( - CommandInfo( - command_type=temporalio.api.enums.v1.command_type_pb2.CommandType.COMMAND_TYPE_SCHEDULE_NEXUS_OPERATION, - command_seq=o.resolve_nexus_operation_start.seq, - ) + await self._visit_coresdk_workflow_activation_ResolveNexusOperationStart( + fs, o.resolve_nexus_operation_start ) - try: - await ( - self._visit_coresdk_workflow_activation_ResolveNexusOperationStart( - fs, o.resolve_nexus_operation_start - ) - ) - finally: - current_command_info.reset(token) elif o.HasField("resolve_nexus_operation"): - token = current_command_info.set( - CommandInfo( - command_type=temporalio.api.enums.v1.command_type_pb2.CommandType.COMMAND_TYPE_SCHEDULE_NEXUS_OPERATION, - command_seq=o.resolve_nexus_operation.seq, - ) + await self._visit_coresdk_workflow_activation_ResolveNexusOperation( + fs, o.resolve_nexus_operation ) - try: - await self._visit_coresdk_workflow_activation_ResolveNexusOperation( - fs, o.resolve_nexus_operation - ) - finally: - current_command_info.reset(token) async def _visit_coresdk_workflow_activation_WorkflowActivation(self, fs, o): for v in o.jobs: @@ -455,18 +374,9 @@ async def _visit_coresdk_workflow_commands_WorkflowCommand(self, fs, o): if o.HasField("user_metadata"): await self._visit_temporal_api_sdk_v1_UserMetadata(fs, o.user_metadata) if o.HasField("schedule_activity"): - token = current_command_info.set( - CommandInfo( - command_type=temporalio.api.enums.v1.command_type_pb2.CommandType.COMMAND_TYPE_SCHEDULE_ACTIVITY_TASK, - command_seq=o.schedule_activity.seq, - ) + await self._visit_coresdk_workflow_commands_ScheduleActivity( + fs, o.schedule_activity ) - try: - await self._visit_coresdk_workflow_commands_ScheduleActivity( - fs, o.schedule_activity - ) - finally: - current_command_info.reset(token) elif o.HasField("respond_to_query"): await self._visit_coresdk_workflow_commands_QueryResult( fs, o.respond_to_query @@ -484,44 +394,17 @@ async def _visit_coresdk_workflow_commands_WorkflowCommand(self, fs, o): fs, o.continue_as_new_workflow_execution ) elif o.HasField("start_child_workflow_execution"): - token = current_command_info.set( - CommandInfo( - command_type=temporalio.api.enums.v1.command_type_pb2.CommandType.COMMAND_TYPE_START_CHILD_WORKFLOW_EXECUTION, - command_seq=o.start_child_workflow_execution.seq, - ) + await self._visit_coresdk_workflow_commands_StartChildWorkflowExecution( + fs, o.start_child_workflow_execution ) - try: - await self._visit_coresdk_workflow_commands_StartChildWorkflowExecution( - fs, o.start_child_workflow_execution - ) - finally: - current_command_info.reset(token) elif o.HasField("signal_external_workflow_execution"): - token = current_command_info.set( - CommandInfo( - command_type=temporalio.api.enums.v1.command_type_pb2.CommandType.COMMAND_TYPE_SIGNAL_EXTERNAL_WORKFLOW_EXECUTION, - command_seq=o.signal_external_workflow_execution.seq, - ) + await self._visit_coresdk_workflow_commands_SignalExternalWorkflowExecution( + fs, o.signal_external_workflow_execution ) - try: - await self._visit_coresdk_workflow_commands_SignalExternalWorkflowExecution( - fs, o.signal_external_workflow_execution - ) - finally: - current_command_info.reset(token) elif o.HasField("schedule_local_activity"): - token = current_command_info.set( - CommandInfo( - command_type=temporalio.api.enums.v1.command_type_pb2.CommandType.COMMAND_TYPE_SCHEDULE_ACTIVITY_TASK, - command_seq=o.schedule_local_activity.seq, - ) + await self._visit_coresdk_workflow_commands_ScheduleLocalActivity( + fs, o.schedule_local_activity ) - try: - await self._visit_coresdk_workflow_commands_ScheduleLocalActivity( - fs, o.schedule_local_activity - ) - finally: - current_command_info.reset(token) elif o.HasField("upsert_workflow_search_attributes"): await self._visit_coresdk_workflow_commands_UpsertWorkflowSearchAttributes( fs, o.upsert_workflow_search_attributes @@ -535,18 +418,9 @@ async def _visit_coresdk_workflow_commands_WorkflowCommand(self, fs, o): fs, o.update_response ) elif o.HasField("schedule_nexus_operation"): - token = current_command_info.set( - CommandInfo( - command_type=temporalio.api.enums.v1.command_type_pb2.CommandType.COMMAND_TYPE_SCHEDULE_NEXUS_OPERATION, - command_seq=o.schedule_nexus_operation.seq, - ) + await self._visit_coresdk_workflow_commands_ScheduleNexusOperation( + fs, o.schedule_nexus_operation ) - try: - await self._visit_coresdk_workflow_commands_ScheduleNexusOperation( - fs, o.schedule_nexus_operation - ) - finally: - current_command_info.reset(token) async def _visit_coresdk_workflow_completion_Success(self, fs, o): for v in o.commands: diff --git a/temporalio/bridge/worker.py b/temporalio/bridge/worker.py index bdf128cca..8e20b670a 100644 --- a/temporalio/bridge/worker.py +++ b/temporalio/bridge/worker.py @@ -33,11 +33,12 @@ import temporalio.converter import temporalio.exceptions from temporalio.api.common.v1.message_pb2 import Payload -from temporalio.bridge._visitor import PayloadVisitor, VisitorFunctions +from temporalio.bridge._visitor import VisitorFunctions from temporalio.bridge.temporal_sdk_bridge import ( CustomSlotSupplier as BridgeCustomSlotSupplier, ) from temporalio.bridge.temporal_sdk_bridge import PollShutdownError # type: ignore +from temporalio.worker._command_aware_visitor import CommandAwarePayloadVisitor @dataclass @@ -301,7 +302,7 @@ async def decode_activation( decode_headers: bool, ) -> None: """Decode all payloads in the activation.""" - await PayloadVisitor( + await CommandAwarePayloadVisitor( skip_search_attributes=True, skip_headers=not decode_headers ).visit(_Visitor(codec.decode), activation) @@ -312,6 +313,6 @@ async def encode_completion( encode_headers: bool, ) -> None: """Encode all payloads in the completion.""" - await PayloadVisitor( + await CommandAwarePayloadVisitor( skip_search_attributes=True, skip_headers=not encode_headers ).visit(_Visitor(codec.encode), completion) diff --git a/temporalio/client.py b/temporalio/client.py index 17e20e382..37d8c6641 100644 --- a/temporalio/client.py +++ b/temporalio/client.py @@ -6036,7 +6036,6 @@ async def _populate_start_workflow_execution_request( req.workflow_type.name = input.workflow req.task_queue.name = input.task_queue if input.args: - # client encode wf input req.input.payloads.extend(await data_converter.encode(input.args)) if input.execution_timeout is not None: req.workflow_execution_timeout.FromTimedelta(input.execution_timeout) diff --git a/temporalio/worker/_command_aware_visitor.py b/temporalio/worker/_command_aware_visitor.py new file mode 100644 index 000000000..fe0255a3f --- /dev/null +++ b/temporalio/worker/_command_aware_visitor.py @@ -0,0 +1,145 @@ +"""Visitor that sets command context during payload traversal.""" + +import contextvars +from contextlib import contextmanager +from dataclasses import dataclass +from typing import Any, Iterator, Optional, Type + +from temporalio.api.enums.v1.command_type_pb2 import CommandType +from temporalio.bridge._visitor import PayloadVisitor, VisitorFunctions +from temporalio.bridge.proto.workflow_activation import workflow_activation_pb2 +from temporalio.bridge.proto.workflow_activation.workflow_activation_pb2 import ( + FireTimer, + ResolveActivity, + ResolveChildWorkflowExecution, + ResolveChildWorkflowExecutionStart, + ResolveNexusOperation, + ResolveNexusOperationStart, + ResolveRequestCancelExternalWorkflow, + ResolveSignalExternalWorkflow, +) +from temporalio.bridge.proto.workflow_commands import workflow_commands_pb2 +from temporalio.bridge.proto.workflow_commands.workflow_commands_pb2 import ( + CancelSignalWorkflow, + CancelTimer, + RequestCancelActivity, + RequestCancelExternalWorkflowExecution, + RequestCancelLocalActivity, + RequestCancelNexusOperation, + ScheduleActivity, + ScheduleLocalActivity, + ScheduleNexusOperation, + SignalExternalWorkflowExecution, + StartChildWorkflowExecution, + StartTimer, +) + + +@dataclass(frozen=True) +class CommandInfo: + """Information identifying a specific command instance.""" + + command_type: CommandType.ValueType + command_seq: int + + +current_command_info: contextvars.ContextVar[Optional[CommandInfo]] = ( + contextvars.ContextVar("current_command_info", default=None) +) + + +class CommandAwarePayloadVisitor(PayloadVisitor): + """Payload visitor that sets command context during traversal. + + Overridden methods are created for all workflow commands and activation jobs that have a 'seq' + field. + """ + + _COMMAND_TYPE_MAP: dict[type[Any], Optional[CommandType.ValueType]] = { + # Commands + ScheduleActivity: CommandType.COMMAND_TYPE_SCHEDULE_ACTIVITY_TASK, + ScheduleLocalActivity: CommandType.COMMAND_TYPE_SCHEDULE_ACTIVITY_TASK, + StartChildWorkflowExecution: CommandType.COMMAND_TYPE_START_CHILD_WORKFLOW_EXECUTION, + SignalExternalWorkflowExecution: CommandType.COMMAND_TYPE_SIGNAL_EXTERNAL_WORKFLOW_EXECUTION, + RequestCancelExternalWorkflowExecution: CommandType.COMMAND_TYPE_REQUEST_CANCEL_EXTERNAL_WORKFLOW_EXECUTION, + ScheduleNexusOperation: CommandType.COMMAND_TYPE_SCHEDULE_NEXUS_OPERATION, + RequestCancelNexusOperation: CommandType.COMMAND_TYPE_REQUEST_CANCEL_NEXUS_OPERATION, + StartTimer: CommandType.COMMAND_TYPE_START_TIMER, + CancelTimer: CommandType.COMMAND_TYPE_CANCEL_TIMER, + RequestCancelActivity: CommandType.COMMAND_TYPE_REQUEST_CANCEL_ACTIVITY_TASK, + RequestCancelLocalActivity: CommandType.COMMAND_TYPE_REQUEST_CANCEL_ACTIVITY_TASK, + CancelSignalWorkflow: None, + # Workflow activation jobs + ResolveActivity: CommandType.COMMAND_TYPE_SCHEDULE_ACTIVITY_TASK, + ResolveChildWorkflowExecutionStart: CommandType.COMMAND_TYPE_START_CHILD_WORKFLOW_EXECUTION, + ResolveChildWorkflowExecution: CommandType.COMMAND_TYPE_START_CHILD_WORKFLOW_EXECUTION, + ResolveSignalExternalWorkflow: CommandType.COMMAND_TYPE_SIGNAL_EXTERNAL_WORKFLOW_EXECUTION, + ResolveRequestCancelExternalWorkflow: CommandType.COMMAND_TYPE_REQUEST_CANCEL_EXTERNAL_WORKFLOW_EXECUTION, + ResolveNexusOperationStart: CommandType.COMMAND_TYPE_SCHEDULE_NEXUS_OPERATION, + ResolveNexusOperation: CommandType.COMMAND_TYPE_SCHEDULE_NEXUS_OPERATION, + FireTimer: CommandType.COMMAND_TYPE_START_TIMER, + } + + def __init__(self, **kwargs: Any) -> None: + super().__init__(**kwargs) + self._create_override_methods() + + def _create_override_methods(self) -> None: + """Dynamically create override methods for all protos with seq fields.""" + for proto_class in _get_workflow_command_protos_with_seq(): + if command_type := self._COMMAND_TYPE_MAP[proto_class]: + self._add_override( + proto_class, "coresdk_workflow_commands", command_type + ) + for proto_class in _get_workflow_activation_job_protos_with_seq(): + if command_type := self._COMMAND_TYPE_MAP[proto_class]: + self._add_override( + proto_class, "coresdk_workflow_activation", command_type + ) + + def _add_override( + self, proto_class: Type[Any], module: str, command_type: CommandType.ValueType + ) -> None: + """Add an override method that sets command context.""" + method_name = f"_visit_{module}_{proto_class.__name__}" + parent_method = getattr(PayloadVisitor, method_name, None) + + if not parent_method: + # No visitor method means no payload fields to visit + return + + async def override_method(fs: VisitorFunctions, o: Any) -> None: + with current_command(command_type, o.seq): + assert parent_method + await parent_method(self, fs, o) + + setattr(self, method_name, override_method) + + +def _get_workflow_command_protos_with_seq() -> Iterator[Type[Any]]: + """Get concrete classes of all workflow command protos with a seq field.""" + for descriptor in workflow_commands_pb2.DESCRIPTOR.message_types_by_name.values(): + if "seq" in descriptor.fields_by_name: + yield descriptor._concrete_class + + +def _get_workflow_activation_job_protos_with_seq() -> Iterator[Type[Any]]: + """Get concrete classes of all workflow activation job protos with a seq field.""" + for descriptor in workflow_activation_pb2.DESCRIPTOR.message_types_by_name.values(): + if "seq" in descriptor.fields_by_name: + yield descriptor._concrete_class + + +@contextmanager +def current_command( + command_type: CommandType.ValueType, command_seq: int +) -> Iterator[None]: + """Context manager for setting command info.""" + token = current_command_info.set( + CommandInfo(command_type=command_type, command_seq=command_seq) + ) + try: + yield + finally: + if token: + current_command_info.reset(token) diff --git a/temporalio/worker/_workflow.py b/temporalio/worker/_workflow.py index 2afd2f7e5..8c7c13505 100644 --- a/temporalio/worker/_workflow.py +++ b/temporalio/worker/_workflow.py @@ -37,6 +37,7 @@ import temporalio.exceptions import temporalio.workflow +from . import _command_aware_visitor from ._interceptor import ( Interceptor, WorkflowInboundInterceptor, @@ -724,10 +725,10 @@ def attempt_deadlock_interruption(self) -> None: @dataclass(frozen=True) class _CommandAwarePayloadCodec(temporalio.converter.PayloadCodec): - """A payload codec that sets serialization context for the associated command. + """A payload codec that sets serialization context for the command associated with each payload. - This codec responds to the :py:data:`temporalio.bridge._visitor.current_command_seq` context - variable set by the payload visitor. + This codec responds to the context variable set by + :py:class:`_command_aware_visitor.CommandAwarePayloadVisitor`. """ instance: WorkflowInstance @@ -753,7 +754,7 @@ def _get_current_command_codec(self) -> temporalio.converter.PayloadCodec: return self.context_free_payload_codec if context := self.instance.get_serialization_context( - temporalio.bridge._visitor.current_command_info.get(), + _command_aware_visitor.current_command_info.get(), ): return self.context_free_payload_codec.with_context(context) diff --git a/temporalio/worker/_workflow_instance.py b/temporalio/worker/_workflow_instance.py index 27fc2546d..01d7ef032 100644 --- a/temporalio/worker/_workflow_instance.py +++ b/temporalio/worker/_workflow_instance.py @@ -51,7 +51,6 @@ import temporalio.api.common.v1 import temporalio.api.enums.v1 import temporalio.api.sdk.v1 -import temporalio.bridge._visitor import temporalio.bridge.proto.activity_result import temporalio.bridge.proto.child_workflow import temporalio.bridge.proto.common @@ -66,6 +65,7 @@ from temporalio.service import __version__ from ..api.failure.v1.message_pb2 import Failure +from . import _command_aware_visitor from ._interceptor import ( ContinueAsNewInput, ExecuteWorkflowInput, @@ -172,7 +172,7 @@ def activate( @abstractmethod def get_serialization_context( self, - command_info: Optional[temporalio.bridge._visitor.CommandInfo], + command_info: Optional[_command_aware_visitor.CommandInfo], ) -> Optional[temporalio.converter.SerializationContext]: """Return appropriate serialization context. @@ -2100,7 +2100,7 @@ def _converters_with_context( def get_serialization_context( self, - command_info: Optional[temporalio.bridge._visitor.CommandInfo], + command_info: Optional[_command_aware_visitor.CommandInfo], ) -> Optional[temporalio.converter.SerializationContext]: workflow_context = temporalio.converter.WorkflowSerializationContext( namespace=self._info.namespace, diff --git a/temporalio/worker/workflow_sandbox/_in_sandbox.py b/temporalio/worker/workflow_sandbox/_in_sandbox.py index 689d77716..17a5c5742 100644 --- a/temporalio/worker/workflow_sandbox/_in_sandbox.py +++ b/temporalio/worker/workflow_sandbox/_in_sandbox.py @@ -8,12 +8,12 @@ import logging from typing import Any, Optional, Type -import temporalio.bridge._visitor import temporalio.bridge.proto.workflow_activation import temporalio.bridge.proto.workflow_completion import temporalio.converter import temporalio.worker._workflow_instance import temporalio.workflow +from temporalio.worker import _command_aware_visitor logger = logging.getLogger(__name__) @@ -84,7 +84,7 @@ def activate( def get_serialization_context( self, - command_info: Optional[temporalio.bridge._visitor.CommandInfo], + command_info: Optional[_command_aware_visitor.CommandInfo], ) -> Optional[temporalio.converter.SerializationContext]: """Get serialization context.""" return self.instance.get_serialization_context(command_info) diff --git a/temporalio/worker/workflow_sandbox/_runner.py b/temporalio/worker/workflow_sandbox/_runner.py index d9d0601f6..e1a48871d 100644 --- a/temporalio/worker/workflow_sandbox/_runner.py +++ b/temporalio/worker/workflow_sandbox/_runner.py @@ -11,13 +11,12 @@ from datetime import datetime, timedelta, timezone from typing import Any, Optional, Sequence, Type -import temporalio.bridge._visitor import temporalio.bridge.proto.workflow_activation import temporalio.bridge.proto.workflow_completion import temporalio.common import temporalio.converter -import temporalio.worker._workflow_instance import temporalio.workflow +from temporalio.worker import _command_aware_visitor from ...api.common.v1.message_pb2 import Payloads from ...api.failure.v1.message_pb2 import Failure @@ -189,7 +188,7 @@ def get_thread_id(self) -> Optional[int]: def get_serialization_context( self, - command_info: Optional[temporalio.bridge._visitor.CommandInfo], + command_info: Optional[_command_aware_visitor.CommandInfo], ) -> Optional[temporalio.converter.SerializationContext]: # Forward call to the sandboxed instance self.importer.restriction_context.is_runtime = True diff --git a/tests/worker/test_workflow.py b/tests/worker/test_workflow.py index 15b340380..3ecd6c63b 100644 --- a/tests/worker/test_workflow.py +++ b/tests/worker/test_workflow.py @@ -43,10 +43,10 @@ import temporalio.activity import temporalio.api.sdk.v1 -import temporalio.bridge._visitor import temporalio.client import temporalio.converter import temporalio.worker +import temporalio.worker._command_aware_visitor import temporalio.workflow from temporalio import activity, workflow from temporalio.api.common.v1 import Payload, Payloads, WorkflowExecution @@ -1613,7 +1613,7 @@ def activate(self, act: WorkflowActivation) -> WorkflowActivationCompletion: def get_serialization_context( self, - command_info: Optional[temporalio.bridge._visitor.CommandInfo], + command_info: Optional[temporalio.worker._command_aware_visitor.CommandInfo], ) -> Optional[temporalio.converter.SerializationContext]: return self._unsandboxed.get_serialization_context(command_info) From 9cd5351587e7152f682de0d4e0c5ec8212a7a551 Mon Sep 17 00:00:00 2001 From: Dan Davison Date: Tue, 30 Sep 2025 08:46:27 -0400 Subject: [PATCH 67/81] Construct context-aware data converter lazily on WorkflowExecution --- temporalio/client.py | 48 +++++++++++++++++++++++--------------------- 1 file changed, 25 insertions(+), 23 deletions(-) diff --git a/temporalio/client.py b/temporalio/client.py index 37d8c6641..d70ccfd5f 100644 --- a/temporalio/client.py +++ b/temporalio/client.py @@ -2883,9 +2883,6 @@ class WorkflowExecution: close_time: Optional[datetime] """When the workflow was closed if closed.""" - data_converter: temporalio.converter.DataConverter - """Data converter from when this description was created.""" - execution_time: Optional[datetime] """When this workflow run started or should start.""" @@ -2895,6 +2892,9 @@ class WorkflowExecution: id: str """ID for the workflow.""" + namespace: str + """Namespace for the workflow.""" + parent_id: Optional[str] """ID for the parent workflow if this was started as a child.""" @@ -2935,20 +2935,31 @@ class WorkflowExecution: workflow_type: str """Type name for the workflow.""" + _context_free_data_converter: temporalio.converter.DataConverter + + @property + def data_converter(self) -> temporalio.converter.DataConverter: + return self._context_free_data_converter.with_context( + WorkflowSerializationContext( + namespace=self.namespace, + workflow_id=self.id, + ) + ) + @classmethod def _from_raw_info( cls, info: temporalio.api.workflow.v1.WorkflowExecutionInfo, + namespace: str, converter: temporalio.converter.DataConverter, **additional_fields: Any, - ) -> WorkflowExecution: + ) -> Self: return cls( close_time=( info.close_time.ToDatetime().replace(tzinfo=timezone.utc) if info.HasField("close_time") else None ), - data_converter=converter, execution_time=( info.execution_time.ToDatetime().replace(tzinfo=timezone.utc) if info.HasField("execution_time") @@ -2956,6 +2967,7 @@ def _from_raw_info( ), history_length=info.history_length, id=info.execution.workflow_id, + namespace=namespace, parent_id=( info.parent_execution.workflow_id if info.HasField("parent_execution") @@ -2986,6 +2998,7 @@ def _from_raw_info( info.search_attributes ), workflow_type=info.type.name, + _context_free_data_converter=converter, **additional_fields, ) @@ -3091,11 +3104,13 @@ async def _decode_metadata(self) -> None: @staticmethod async def _from_raw_description( description: temporalio.api.workflowservice.v1.DescribeWorkflowExecutionResponse, + namespace: str, converter: temporalio.converter.DataConverter, ) -> WorkflowExecutionDescription: - return WorkflowExecutionDescription._from_raw_info( # type: ignore + return WorkflowExecutionDescription._from_raw_info( description.workflow_execution_info, - converter, + namespace=namespace, + converter=converter, raw_description=description, ) @@ -3246,23 +3261,9 @@ async def fetch_next_page(self, *, page_size: Optional[int] = None) -> None: timeout=self._input.rpc_timeout, ) - data_converter_cache = {} - - def get_data_converter(workflow_id: str) -> temporalio.converter.DataConverter: - if workflow_id not in data_converter_cache: - data_converter_cache[workflow_id] = ( - self._client.data_converter.with_context( - WorkflowSerializationContext( - namespace=self._client.namespace, - workflow_id=workflow_id, - ) - ) - ) - return data_converter_cache[workflow_id] - self._current_page = [ WorkflowExecution._from_raw_info( - v, get_data_converter(v.execution.workflow_id) + v, self._client.namespace, self._client.data_converter ) for v in resp.executions ] @@ -6111,7 +6112,8 @@ async def describe_workflow( metadata=input.rpc_metadata, timeout=input.rpc_timeout, ), - self._client.data_converter.with_context( + namespace=self._client.namespace, + converter=self._client.data_converter.with_context( WorkflowSerializationContext( namespace=self._client.namespace, workflow_id=input.id, From b90491915d5cf0db081744f410af389ab0dbe650 Mon Sep 17 00:00:00 2001 From: Dan Davison Date: Tue, 30 Sep 2025 08:52:45 -0400 Subject: [PATCH 68/81] Check we have test coverage --- temporalio/client.py | 1 + 1 file changed, 1 insertion(+) diff --git a/temporalio/client.py b/temporalio/client.py index d70ccfd5f..ecd8d73a7 100644 --- a/temporalio/client.py +++ b/temporalio/client.py @@ -2939,6 +2939,7 @@ class WorkflowExecution: @property def data_converter(self) -> temporalio.converter.DataConverter: + 0 / 0 # type: ignore return self._context_free_data_converter.with_context( WorkflowSerializationContext( namespace=self.namespace, From 896a6fcc46ebca7c5f734c9f0441fc935b2ca013 Mon Sep 17 00:00:00 2001 From: Dan Davison Date: Tue, 30 Sep 2025 09:25:48 -0400 Subject: [PATCH 69/81] Avoid constructing workflow-context payload codec twice --- temporalio/worker/_workflow.py | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/temporalio/worker/_workflow.py b/temporalio/worker/_workflow.py index 8c7c13505..6e7c254aa 100644 --- a/temporalio/worker/_workflow.py +++ b/temporalio/worker/_workflow.py @@ -277,12 +277,11 @@ async def _handle_activation( "Cache already exists for activation with initialize job" ) - data_converter = self._data_converter.with_context( - temporalio.converter.WorkflowSerializationContext( - namespace=self._namespace, - workflow_id=workflow_id, - ) + workflow_context = temporalio.converter.WorkflowSerializationContext( + namespace=self._namespace, + workflow_id=workflow_id, ) + data_converter = self._data_converter.with_context(workflow_context) if self._data_converter.payload_codec: assert data_converter.payload_codec if not workflow: @@ -291,6 +290,8 @@ async def _handle_activation( payload_codec = _CommandAwarePayloadCodec( workflow.instance, context_free_payload_codec=self._data_converter.payload_codec, + workflow_context_payload_codec=data_converter.payload_codec, + workflow_context=workflow_context, ) await temporalio.bridge.worker.decode_activation( act, @@ -367,6 +368,11 @@ async def _handle_activation( payload_codec = _CommandAwarePayloadCodec( workflow.instance, context_free_payload_codec=self._data_converter.payload_codec, + workflow_context_payload_codec=data_converter.payload_codec, + workflow_context=temporalio.converter.WorkflowSerializationContext( + namespace=self._namespace, + workflow_id=workflow.workflow_id, + ), ) try: await temporalio.bridge.worker.encode_completion( @@ -733,6 +739,8 @@ class _CommandAwarePayloadCodec(temporalio.converter.PayloadCodec): instance: WorkflowInstance context_free_payload_codec: temporalio.converter.PayloadCodec + workflow_context_payload_codec: temporalio.converter.PayloadCodec + workflow_context: temporalio.converter.WorkflowSerializationContext async def encode( self, @@ -756,6 +764,8 @@ def _get_current_command_codec(self) -> temporalio.converter.PayloadCodec: if context := self.instance.get_serialization_context( _command_aware_visitor.current_command_info.get(), ): + if context == self.workflow_context: + return self.workflow_context_payload_codec return self.context_free_payload_codec.with_context(context) return self.context_free_payload_codec From 7fecf51b73017e2300c051cec848eeb2e46353b4 Mon Sep 17 00:00:00 2001 From: Dan Davison Date: Tue, 30 Sep 2025 09:58:40 -0400 Subject: [PATCH 70/81] Appease docstring linter --- temporalio/client.py | 1 + temporalio/worker/_command_aware_visitor.py | 1 + 2 files changed, 2 insertions(+) diff --git a/temporalio/client.py b/temporalio/client.py index ecd8d73a7..8d521b6f4 100644 --- a/temporalio/client.py +++ b/temporalio/client.py @@ -2939,6 +2939,7 @@ class WorkflowExecution: @property def data_converter(self) -> temporalio.converter.DataConverter: + """Data converter for the workflow.""" 0 / 0 # type: ignore return self._context_free_data_converter.with_context( WorkflowSerializationContext( diff --git a/temporalio/worker/_command_aware_visitor.py b/temporalio/worker/_command_aware_visitor.py index fe0255a3f..bf5a16aaf 100644 --- a/temporalio/worker/_command_aware_visitor.py +++ b/temporalio/worker/_command_aware_visitor.py @@ -81,6 +81,7 @@ class CommandAwarePayloadVisitor(PayloadVisitor): } def __init__(self, **kwargs: Any) -> None: + """Initialize the command-aware payload visitor.""" super().__init__(**kwargs) self._create_override_methods() From 2257c57ef4f0901d640d1f0d81e63eddb2e0b044 Mon Sep 17 00:00:00 2001 From: Dan Davison Date: Tue, 30 Sep 2025 10:13:06 -0400 Subject: [PATCH 71/81] Test workflow and activity payload converters have context --- tests/test_serialization_context.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/test_serialization_context.py b/tests/test_serialization_context.py index 45597a81f..ee7be8684 100644 --- a/tests/test_serialization_context.py +++ b/tests/test_serialization_context.py @@ -155,6 +155,7 @@ def __init__(self): @activity.defn async def passthrough_activity(input: TraceData) -> TraceData: + activity.payload_converter().to_payload(input) activity.heartbeat(input) # Wait for the heartbeat to be processed so that it modifies the data before the activity returns await asyncio.sleep(0.2) @@ -172,6 +173,7 @@ async def run(self, data: TraceData) -> TraceData: class PayloadConversionWorkflow: @workflow.run async def run(self, data: TraceData) -> TraceData: + workflow.payload_converter().to_payload(data) data = await workflow.execute_activity( passthrough_activity, data, @@ -242,6 +244,10 @@ async def test_payload_conversion_calls_follow_expected_sequence_and_contexts( method="from_payload", context=workflow_context, # Inbound workflow input ), + TraceItem( + method="to_payload", + context=workflow_context, # workflow payload converter + ), TraceItem( method="to_payload", context=activity_context, # Outbound activity input @@ -250,6 +256,10 @@ async def test_payload_conversion_calls_follow_expected_sequence_and_contexts( method="from_payload", context=activity_context, # Inbound activity input ), + TraceItem( + method="to_payload", + context=activity_context, # activity payload converter + ), TraceItem( method="to_payload", context=activity_context, # Outbound heartbeat From c1143dfb013acd2b923d6e5fdca49680939cee36 Mon Sep 17 00:00:00 2001 From: Dan Davison Date: Tue, 30 Sep 2025 11:41:22 -0400 Subject: [PATCH 72/81] Static method generation --- temporalio/worker/_command_aware_visitor.py | 82 ++++++++++++--------- 1 file changed, 47 insertions(+), 35 deletions(-) diff --git a/temporalio/worker/_command_aware_visitor.py b/temporalio/worker/_command_aware_visitor.py index bf5a16aaf..5c15242bd 100644 --- a/temporalio/worker/_command_aware_visitor.py +++ b/temporalio/worker/_command_aware_visitor.py @@ -48,11 +48,23 @@ class CommandInfo: ) +def _create_override_method( + parent_method: Any, command_type: CommandType.ValueType +) -> Any: + """Create an override method that sets command context.""" + + async def override_method(self: Any, fs: VisitorFunctions, o: Any) -> None: + with current_command(command_type, o.seq): + await parent_method(self, fs, o) + + return override_method + + class CommandAwarePayloadVisitor(PayloadVisitor): """Payload visitor that sets command context during traversal. - Overridden methods are created for all workflow commands and activation jobs that have a 'seq' - field. + Override methods are created at class definition time for all workflow + commands and activation jobs that have a 'seq' field. """ _COMMAND_TYPE_MAP: dict[type[Any], Optional[CommandType.ValueType]] = { @@ -80,41 +92,37 @@ class CommandAwarePayloadVisitor(PayloadVisitor): FireTimer: CommandType.COMMAND_TYPE_START_TIMER, } - def __init__(self, **kwargs: Any) -> None: - """Initialize the command-aware payload visitor.""" - super().__init__(**kwargs) - self._create_override_methods() - - def _create_override_methods(self) -> None: - """Dynamically create override methods for all protos with seq fields.""" - for proto_class in _get_workflow_command_protos_with_seq(): - if command_type := self._COMMAND_TYPE_MAP[proto_class]: - self._add_override( - proto_class, "coresdk_workflow_commands", command_type - ) - for proto_class in _get_workflow_activation_job_protos_with_seq(): - if command_type := self._COMMAND_TYPE_MAP[proto_class]: - self._add_override( - proto_class, "coresdk_workflow_activation", command_type - ) - - def _add_override( - self, proto_class: Type[Any], module: str, command_type: CommandType.ValueType - ) -> None: - """Add an override method that sets command context.""" - method_name = f"_visit_{module}_{proto_class.__name__}" - parent_method = getattr(PayloadVisitor, method_name, None) - - if not parent_method: - # No visitor method means no payload fields to visit - return - async def override_method(fs: VisitorFunctions, o: Any) -> None: - with current_command(command_type, o.seq): - assert parent_method - await parent_method(self, fs, o) +# Add override methods to CommandAwarePayloadVisitor at class definition time +def _add_class_overrides() -> None: + """Add override methods to CommandAwarePayloadVisitor class.""" + # Process workflow commands + for proto_class in _get_workflow_command_protos_with_seq(): + if command_type := CommandAwarePayloadVisitor._COMMAND_TYPE_MAP.get( + proto_class + ): + method_name = f"_visit_coresdk_workflow_commands_{proto_class.__name__}" + parent_method = getattr(PayloadVisitor, method_name, None) + if parent_method: + setattr( + CommandAwarePayloadVisitor, + method_name, + _create_override_method(parent_method, command_type), + ) - setattr(self, method_name, override_method) + # Process activation jobs + for proto_class in _get_workflow_activation_job_protos_with_seq(): + if command_type := CommandAwarePayloadVisitor._COMMAND_TYPE_MAP.get( + proto_class + ): + method_name = f"_visit_coresdk_workflow_activation_{proto_class.__name__}" + parent_method = getattr(PayloadVisitor, method_name, None) + if parent_method: + setattr( + CommandAwarePayloadVisitor, + method_name, + _create_override_method(parent_method, command_type), + ) def _get_workflow_command_protos_with_seq() -> Iterator[Type[Any]]: @@ -144,3 +152,7 @@ def current_command( finally: if token: current_command_info.reset(token) + + +# Create all override methods on the class when the module is imported +_add_class_overrides() From b055d23bbdf332e63497ee8f7d9623a55b58ff55 Mon Sep 17 00:00:00 2001 From: Dan Davison Date: Tue, 30 Sep 2025 11:51:22 -0400 Subject: [PATCH 73/81] Revert back to explicitly statically defined methods --- temporalio/worker/_command_aware_visitor.py | 243 ++++++++++++++------ tests/worker/test_command_aware_visitor.py | 46 ++++ 2 files changed, 215 insertions(+), 74 deletions(-) create mode 100644 tests/worker/test_command_aware_visitor.py diff --git a/temporalio/worker/_command_aware_visitor.py b/temporalio/worker/_command_aware_visitor.py index 5c15242bd..b93daa724 100644 --- a/temporalio/worker/_command_aware_visitor.py +++ b/temporalio/worker/_command_aware_visitor.py @@ -48,81 +48,180 @@ class CommandInfo: ) -def _create_override_method( - parent_method: Any, command_type: CommandType.ValueType -) -> Any: - """Create an override method that sets command context.""" - - async def override_method(self: Any, fs: VisitorFunctions, o: Any) -> None: - with current_command(command_type, o.seq): - await parent_method(self, fs, o) - - return override_method - - class CommandAwarePayloadVisitor(PayloadVisitor): """Payload visitor that sets command context during traversal. - Override methods are created at class definition time for all workflow - commands and activation jobs that have a 'seq' field. + Override methods are explicitly defined for all workflow commands and + activation jobs that have a 'seq' field. """ - _COMMAND_TYPE_MAP: dict[type[Any], Optional[CommandType.ValueType]] = { - # Commands - ScheduleActivity: CommandType.COMMAND_TYPE_SCHEDULE_ACTIVITY_TASK, - ScheduleLocalActivity: CommandType.COMMAND_TYPE_SCHEDULE_ACTIVITY_TASK, - StartChildWorkflowExecution: CommandType.COMMAND_TYPE_START_CHILD_WORKFLOW_EXECUTION, - SignalExternalWorkflowExecution: CommandType.COMMAND_TYPE_SIGNAL_EXTERNAL_WORKFLOW_EXECUTION, - RequestCancelExternalWorkflowExecution: CommandType.COMMAND_TYPE_REQUEST_CANCEL_EXTERNAL_WORKFLOW_EXECUTION, - ScheduleNexusOperation: CommandType.COMMAND_TYPE_SCHEDULE_NEXUS_OPERATION, - RequestCancelNexusOperation: CommandType.COMMAND_TYPE_REQUEST_CANCEL_NEXUS_OPERATION, - StartTimer: CommandType.COMMAND_TYPE_START_TIMER, - CancelTimer: CommandType.COMMAND_TYPE_CANCEL_TIMER, - RequestCancelActivity: CommandType.COMMAND_TYPE_REQUEST_CANCEL_ACTIVITY_TASK, - RequestCancelLocalActivity: CommandType.COMMAND_TYPE_REQUEST_CANCEL_ACTIVITY_TASK, - CancelSignalWorkflow: None, - # Workflow activation jobs - ResolveActivity: CommandType.COMMAND_TYPE_SCHEDULE_ACTIVITY_TASK, - ResolveChildWorkflowExecutionStart: CommandType.COMMAND_TYPE_START_CHILD_WORKFLOW_EXECUTION, - ResolveChildWorkflowExecution: CommandType.COMMAND_TYPE_START_CHILD_WORKFLOW_EXECUTION, - ResolveSignalExternalWorkflow: CommandType.COMMAND_TYPE_SIGNAL_EXTERNAL_WORKFLOW_EXECUTION, - ResolveRequestCancelExternalWorkflow: CommandType.COMMAND_TYPE_REQUEST_CANCEL_EXTERNAL_WORKFLOW_EXECUTION, - ResolveNexusOperationStart: CommandType.COMMAND_TYPE_SCHEDULE_NEXUS_OPERATION, - ResolveNexusOperation: CommandType.COMMAND_TYPE_SCHEDULE_NEXUS_OPERATION, - FireTimer: CommandType.COMMAND_TYPE_START_TIMER, - } - - -# Add override methods to CommandAwarePayloadVisitor at class definition time -def _add_class_overrides() -> None: - """Add override methods to CommandAwarePayloadVisitor class.""" - # Process workflow commands - for proto_class in _get_workflow_command_protos_with_seq(): - if command_type := CommandAwarePayloadVisitor._COMMAND_TYPE_MAP.get( - proto_class + # Workflow commands + async def _visit_coresdk_workflow_commands_ScheduleActivity( + self, fs: VisitorFunctions, o: ScheduleActivity + ) -> None: + with current_command(CommandType.COMMAND_TYPE_SCHEDULE_ACTIVITY_TASK, o.seq): + await super()._visit_coresdk_workflow_commands_ScheduleActivity(fs, o) + + async def _visit_coresdk_workflow_commands_ScheduleLocalActivity( + self, fs: VisitorFunctions, o: ScheduleLocalActivity + ) -> None: + with current_command(CommandType.COMMAND_TYPE_SCHEDULE_ACTIVITY_TASK, o.seq): + await super()._visit_coresdk_workflow_commands_ScheduleLocalActivity(fs, o) + + async def _visit_coresdk_workflow_commands_StartChildWorkflowExecution( + self, fs: VisitorFunctions, o: StartChildWorkflowExecution + ) -> None: + with current_command( + CommandType.COMMAND_TYPE_START_CHILD_WORKFLOW_EXECUTION, o.seq + ): + await super()._visit_coresdk_workflow_commands_StartChildWorkflowExecution( + fs, o + ) + + async def _visit_coresdk_workflow_commands_SignalExternalWorkflowExecution( + self, fs: VisitorFunctions, o: SignalExternalWorkflowExecution + ) -> None: + with current_command( + CommandType.COMMAND_TYPE_SIGNAL_EXTERNAL_WORKFLOW_EXECUTION, o.seq + ): + await super()._visit_coresdk_workflow_commands_SignalExternalWorkflowExecution( + fs, o + ) + + async def _visit_coresdk_workflow_commands_RequestCancelExternalWorkflowExecution( + self, fs: VisitorFunctions, o: RequestCancelExternalWorkflowExecution + ) -> None: + with current_command( + CommandType.COMMAND_TYPE_REQUEST_CANCEL_EXTERNAL_WORKFLOW_EXECUTION, o.seq + ): + # Note: Base class doesn't have this visitor (no payloads to visit) + pass + + async def _visit_coresdk_workflow_commands_ScheduleNexusOperation( + self, fs: VisitorFunctions, o: ScheduleNexusOperation + ) -> None: + with current_command(CommandType.COMMAND_TYPE_SCHEDULE_NEXUS_OPERATION, o.seq): + await super()._visit_coresdk_workflow_commands_ScheduleNexusOperation(fs, o) + + async def _visit_coresdk_workflow_commands_RequestCancelNexusOperation( + self, fs: VisitorFunctions, o: RequestCancelNexusOperation + ) -> None: + with current_command( + CommandType.COMMAND_TYPE_REQUEST_CANCEL_NEXUS_OPERATION, o.seq ): - method_name = f"_visit_coresdk_workflow_commands_{proto_class.__name__}" - parent_method = getattr(PayloadVisitor, method_name, None) - if parent_method: - setattr( - CommandAwarePayloadVisitor, - method_name, - _create_override_method(parent_method, command_type), - ) - - # Process activation jobs - for proto_class in _get_workflow_activation_job_protos_with_seq(): - if command_type := CommandAwarePayloadVisitor._COMMAND_TYPE_MAP.get( - proto_class + # Note: Base class doesn't have this visitor (no payloads to visit) + pass + + async def _visit_coresdk_workflow_commands_StartTimer( + self, fs: VisitorFunctions, o: StartTimer + ) -> None: + with current_command(CommandType.COMMAND_TYPE_START_TIMER, o.seq): + # Note: Base class doesn't have this visitor (no payloads to visit) + pass + + async def _visit_coresdk_workflow_commands_CancelTimer( + self, fs: VisitorFunctions, o: CancelTimer + ) -> None: + with current_command(CommandType.COMMAND_TYPE_CANCEL_TIMER, o.seq): + # Note: Base class doesn't have this visitor (no payloads to visit) + pass + + async def _visit_coresdk_workflow_commands_RequestCancelActivity( + self, fs: VisitorFunctions, o: RequestCancelActivity + ) -> None: + with current_command( + CommandType.COMMAND_TYPE_REQUEST_CANCEL_ACTIVITY_TASK, o.seq ): - method_name = f"_visit_coresdk_workflow_activation_{proto_class.__name__}" - parent_method = getattr(PayloadVisitor, method_name, None) - if parent_method: - setattr( - CommandAwarePayloadVisitor, - method_name, - _create_override_method(parent_method, command_type), - ) + # Note: Base class doesn't have this visitor (no payloads to visit) + pass + + async def _visit_coresdk_workflow_commands_RequestCancelLocalActivity( + self, fs: VisitorFunctions, o: RequestCancelLocalActivity + ) -> None: + with current_command( + CommandType.COMMAND_TYPE_REQUEST_CANCEL_ACTIVITY_TASK, o.seq + ): + # Note: Base class doesn't have this visitor (no payloads to visit) + pass + + async def _visit_coresdk_workflow_commands_CancelSignalWorkflow( + self, fs: VisitorFunctions, o: CancelSignalWorkflow + ) -> None: + # CancelSignalWorkflow has seq but no server command type + # (it's an internal SDK command). Set context to None. + with current_command(None, o.seq): # type: ignore + # Note: Base class doesn't have this visitor (no payloads to visit) + pass + + # Workflow activation jobs + async def _visit_coresdk_workflow_activation_ResolveActivity( + self, fs: VisitorFunctions, o: ResolveActivity + ) -> None: + with current_command(CommandType.COMMAND_TYPE_SCHEDULE_ACTIVITY_TASK, o.seq): + await super()._visit_coresdk_workflow_activation_ResolveActivity(fs, o) + + async def _visit_coresdk_workflow_activation_ResolveChildWorkflowExecutionStart( + self, fs: VisitorFunctions, o: ResolveChildWorkflowExecutionStart + ) -> None: + with current_command( + CommandType.COMMAND_TYPE_START_CHILD_WORKFLOW_EXECUTION, o.seq + ): + await super()._visit_coresdk_workflow_activation_ResolveChildWorkflowExecutionStart( + fs, o + ) + + async def _visit_coresdk_workflow_activation_ResolveChildWorkflowExecution( + self, fs: VisitorFunctions, o: ResolveChildWorkflowExecution + ) -> None: + with current_command( + CommandType.COMMAND_TYPE_START_CHILD_WORKFLOW_EXECUTION, o.seq + ): + await super()._visit_coresdk_workflow_activation_ResolveChildWorkflowExecution( + fs, o + ) + + async def _visit_coresdk_workflow_activation_ResolveSignalExternalWorkflow( + self, fs: VisitorFunctions, o: ResolveSignalExternalWorkflow + ) -> None: + with current_command( + CommandType.COMMAND_TYPE_SIGNAL_EXTERNAL_WORKFLOW_EXECUTION, o.seq + ): + await super()._visit_coresdk_workflow_activation_ResolveSignalExternalWorkflow( + fs, o + ) + + async def _visit_coresdk_workflow_activation_ResolveRequestCancelExternalWorkflow( + self, fs: VisitorFunctions, o: ResolveRequestCancelExternalWorkflow + ) -> None: + with current_command( + CommandType.COMMAND_TYPE_REQUEST_CANCEL_EXTERNAL_WORKFLOW_EXECUTION, o.seq + ): + await super()._visit_coresdk_workflow_activation_ResolveRequestCancelExternalWorkflow( + fs, o + ) + + async def _visit_coresdk_workflow_activation_ResolveNexusOperationStart( + self, fs: VisitorFunctions, o: ResolveNexusOperationStart + ) -> None: + with current_command(CommandType.COMMAND_TYPE_SCHEDULE_NEXUS_OPERATION, o.seq): + await super()._visit_coresdk_workflow_activation_ResolveNexusOperationStart( + fs, o + ) + + async def _visit_coresdk_workflow_activation_ResolveNexusOperation( + self, fs: VisitorFunctions, o: ResolveNexusOperation + ) -> None: + with current_command(CommandType.COMMAND_TYPE_SCHEDULE_NEXUS_OPERATION, o.seq): + await super()._visit_coresdk_workflow_activation_ResolveNexusOperation( + fs, o + ) + + async def _visit_coresdk_workflow_activation_FireTimer( + self, fs: VisitorFunctions, o: FireTimer + ) -> None: + with current_command(CommandType.COMMAND_TYPE_START_TIMER, o.seq): + # Note: Base class doesn't have this visitor (no payloads to visit) + pass def _get_workflow_command_protos_with_seq() -> Iterator[Type[Any]]: @@ -141,18 +240,14 @@ def _get_workflow_activation_job_protos_with_seq() -> Iterator[Type[Any]]: @contextmanager def current_command( - command_type: CommandType.ValueType, command_seq: int + command_type: Optional[CommandType.ValueType], command_seq: int ) -> Iterator[None]: """Context manager for setting command info.""" token = current_command_info.set( - CommandInfo(command_type=command_type, command_seq=command_seq) + CommandInfo(command_type=command_type, command_seq=command_seq) # type: ignore ) try: yield finally: if token: current_command_info.reset(token) - - -# Create all override methods on the class when the module is imported -_add_class_overrides() diff --git a/tests/worker/test_command_aware_visitor.py b/tests/worker/test_command_aware_visitor.py new file mode 100644 index 000000000..8a1628641 --- /dev/null +++ b/tests/worker/test_command_aware_visitor.py @@ -0,0 +1,46 @@ +"""Test that CommandAwarePayloadVisitor handles all commands with seq fields.""" + +from temporalio.worker._command_aware_visitor import ( + CommandAwarePayloadVisitor, + _get_workflow_activation_job_protos_with_seq, + _get_workflow_command_protos_with_seq, +) + + +def test_command_aware_visitor_has_methods_for_all_seq_protos(): + """Verify CommandAwarePayloadVisitor has methods for all protos with seq fields.""" + visitor = CommandAwarePayloadVisitor() + + # Check all workflow commands with seq have corresponding methods + + command_protos = list(_get_workflow_command_protos_with_seq()) + job_protos = list(_get_workflow_activation_job_protos_with_seq()) + assert command_protos, "Should find workflow commands with seq" + assert job_protos, "Should find workflow activation jobs with seq" + + commands_missing = [] + for proto_class in command_protos: + method_name = f"_visit_coresdk_workflow_commands_{proto_class.__name__}" + if not hasattr(visitor, method_name): + commands_missing.append(proto_class.__name__) + + # Check all workflow activation jobs with seq have corresponding methods + jobs_missing = [] + for proto_class in job_protos: + method_name = f"_visit_coresdk_workflow_activation_{proto_class.__name__}" + if not hasattr(visitor, method_name): + jobs_missing.append(proto_class.__name__) + + errors = [] + if commands_missing: + errors.append( + f"Missing visitor methods for commands with seq: {commands_missing}\n" + f"Add methods to CommandAwarePayloadVisitor for these commands." + ) + if jobs_missing: + errors.append( + f"Missing visitor methods for activation jobs with seq: {jobs_missing}\n" + f"Add methods to CommandAwarePayloadVisitor for these jobs." + ) + + assert not errors, "\n".join(errors) From 880b487507ce5c00248ce9c341e564d9674999bc Mon Sep 17 00:00:00 2001 From: Dan Davison Date: Tue, 30 Sep 2025 13:10:16 -0400 Subject: [PATCH 74/81] Remove no-op methods --- temporalio/worker/_command_aware_visitor.py | 82 +-------------------- tests/worker/test_command_aware_visitor.py | 53 ++++++++++--- 2 files changed, 45 insertions(+), 90 deletions(-) diff --git a/temporalio/worker/_command_aware_visitor.py b/temporalio/worker/_command_aware_visitor.py index b93daa724..e6653ef9b 100644 --- a/temporalio/worker/_command_aware_visitor.py +++ b/temporalio/worker/_command_aware_visitor.py @@ -9,7 +9,6 @@ from temporalio.bridge._visitor import PayloadVisitor, VisitorFunctions from temporalio.bridge.proto.workflow_activation import workflow_activation_pb2 from temporalio.bridge.proto.workflow_activation.workflow_activation_pb2 import ( - FireTimer, ResolveActivity, ResolveChildWorkflowExecution, ResolveChildWorkflowExecutionStart, @@ -20,18 +19,11 @@ ) from temporalio.bridge.proto.workflow_commands import workflow_commands_pb2 from temporalio.bridge.proto.workflow_commands.workflow_commands_pb2 import ( - CancelSignalWorkflow, - CancelTimer, - RequestCancelActivity, - RequestCancelExternalWorkflowExecution, - RequestCancelLocalActivity, - RequestCancelNexusOperation, ScheduleActivity, ScheduleLocalActivity, ScheduleNexusOperation, SignalExternalWorkflowExecution, StartChildWorkflowExecution, - StartTimer, ) @@ -51,11 +43,11 @@ class CommandInfo: class CommandAwarePayloadVisitor(PayloadVisitor): """Payload visitor that sets command context during traversal. - Override methods are explicitly defined for all workflow commands and - activation jobs that have a 'seq' field. + Override methods are explicitly defined for workflow commands and + activation jobs that have both a 'seq' field and payloads to visit. """ - # Workflow commands + # Workflow commands with payloads async def _visit_coresdk_workflow_commands_ScheduleActivity( self, fs: VisitorFunctions, o: ScheduleActivity ) -> None: @@ -88,72 +80,13 @@ async def _visit_coresdk_workflow_commands_SignalExternalWorkflowExecution( fs, o ) - async def _visit_coresdk_workflow_commands_RequestCancelExternalWorkflowExecution( - self, fs: VisitorFunctions, o: RequestCancelExternalWorkflowExecution - ) -> None: - with current_command( - CommandType.COMMAND_TYPE_REQUEST_CANCEL_EXTERNAL_WORKFLOW_EXECUTION, o.seq - ): - # Note: Base class doesn't have this visitor (no payloads to visit) - pass - async def _visit_coresdk_workflow_commands_ScheduleNexusOperation( self, fs: VisitorFunctions, o: ScheduleNexusOperation ) -> None: with current_command(CommandType.COMMAND_TYPE_SCHEDULE_NEXUS_OPERATION, o.seq): await super()._visit_coresdk_workflow_commands_ScheduleNexusOperation(fs, o) - async def _visit_coresdk_workflow_commands_RequestCancelNexusOperation( - self, fs: VisitorFunctions, o: RequestCancelNexusOperation - ) -> None: - with current_command( - CommandType.COMMAND_TYPE_REQUEST_CANCEL_NEXUS_OPERATION, o.seq - ): - # Note: Base class doesn't have this visitor (no payloads to visit) - pass - - async def _visit_coresdk_workflow_commands_StartTimer( - self, fs: VisitorFunctions, o: StartTimer - ) -> None: - with current_command(CommandType.COMMAND_TYPE_START_TIMER, o.seq): - # Note: Base class doesn't have this visitor (no payloads to visit) - pass - - async def _visit_coresdk_workflow_commands_CancelTimer( - self, fs: VisitorFunctions, o: CancelTimer - ) -> None: - with current_command(CommandType.COMMAND_TYPE_CANCEL_TIMER, o.seq): - # Note: Base class doesn't have this visitor (no payloads to visit) - pass - - async def _visit_coresdk_workflow_commands_RequestCancelActivity( - self, fs: VisitorFunctions, o: RequestCancelActivity - ) -> None: - with current_command( - CommandType.COMMAND_TYPE_REQUEST_CANCEL_ACTIVITY_TASK, o.seq - ): - # Note: Base class doesn't have this visitor (no payloads to visit) - pass - - async def _visit_coresdk_workflow_commands_RequestCancelLocalActivity( - self, fs: VisitorFunctions, o: RequestCancelLocalActivity - ) -> None: - with current_command( - CommandType.COMMAND_TYPE_REQUEST_CANCEL_ACTIVITY_TASK, o.seq - ): - # Note: Base class doesn't have this visitor (no payloads to visit) - pass - - async def _visit_coresdk_workflow_commands_CancelSignalWorkflow( - self, fs: VisitorFunctions, o: CancelSignalWorkflow - ) -> None: - # CancelSignalWorkflow has seq but no server command type - # (it's an internal SDK command). Set context to None. - with current_command(None, o.seq): # type: ignore - # Note: Base class doesn't have this visitor (no payloads to visit) - pass - - # Workflow activation jobs + # Workflow activation jobs with payloads async def _visit_coresdk_workflow_activation_ResolveActivity( self, fs: VisitorFunctions, o: ResolveActivity ) -> None: @@ -216,13 +149,6 @@ async def _visit_coresdk_workflow_activation_ResolveNexusOperation( fs, o ) - async def _visit_coresdk_workflow_activation_FireTimer( - self, fs: VisitorFunctions, o: FireTimer - ) -> None: - with current_command(CommandType.COMMAND_TYPE_START_TIMER, o.seq): - # Note: Base class doesn't have this visitor (no payloads to visit) - pass - def _get_workflow_command_protos_with_seq() -> Iterator[Type[Any]]: """Get concrete classes of all workflow command protos with a seq field.""" diff --git a/tests/worker/test_command_aware_visitor.py b/tests/worker/test_command_aware_visitor.py index 8a1628641..f15a7733b 100644 --- a/tests/worker/test_command_aware_visitor.py +++ b/tests/worker/test_command_aware_visitor.py @@ -1,5 +1,6 @@ -"""Test that CommandAwarePayloadVisitor handles all commands with seq fields.""" +"""Test that CommandAwarePayloadVisitor handles all commands with seq fields that have payloads.""" +from temporalio.bridge._visitor import PayloadVisitor from temporalio.worker._command_aware_visitor import ( CommandAwarePayloadVisitor, _get_workflow_activation_job_protos_with_seq, @@ -7,40 +8,68 @@ ) -def test_command_aware_visitor_has_methods_for_all_seq_protos(): - """Verify CommandAwarePayloadVisitor has methods for all protos with seq fields.""" - visitor = CommandAwarePayloadVisitor() +def test_command_aware_visitor_has_methods_for_all_seq_protos_with_payloads(): + """Verify CommandAwarePayloadVisitor has methods for all protos with seq fields that have payloads. - # Check all workflow commands with seq have corresponding methods + We only override methods when the base class has a visitor method (i.e., there are payloads to visit). + Commands without payloads don't need overrides since there's nothing to visit. + """ + visitor = CommandAwarePayloadVisitor() + # Find all protos with seq command_protos = list(_get_workflow_command_protos_with_seq()) job_protos = list(_get_workflow_activation_job_protos_with_seq()) assert command_protos, "Should find workflow commands with seq" assert job_protos, "Should find workflow activation jobs with seq" + # Check workflow commands - only ones with payloads need overrides commands_missing = [] + commands_with_payloads = [] for proto_class in command_protos: method_name = f"_visit_coresdk_workflow_commands_{proto_class.__name__}" - if not hasattr(visitor, method_name): - commands_missing.append(proto_class.__name__) + # Only check if base class has this visitor (meaning there are payloads) + if hasattr(PayloadVisitor, method_name): + commands_with_payloads.append(proto_class.__name__) + # Check if CommandAwarePayloadVisitor has its own override (not just inherited) + if method_name not in CommandAwarePayloadVisitor.__dict__: + commands_missing.append(proto_class.__name__) - # Check all workflow activation jobs with seq have corresponding methods + # Check workflow activation jobs - only ones with payloads need overrides jobs_missing = [] + jobs_with_payloads = [] for proto_class in job_protos: method_name = f"_visit_coresdk_workflow_activation_{proto_class.__name__}" - if not hasattr(visitor, method_name): - jobs_missing.append(proto_class.__name__) + # Only check if base class has this visitor (meaning there are payloads) + if hasattr(PayloadVisitor, method_name): + jobs_with_payloads.append(proto_class.__name__) + # Check if CommandAwarePayloadVisitor has its own override (not just inherited) + if method_name not in CommandAwarePayloadVisitor.__dict__: + jobs_missing.append(proto_class.__name__) errors = [] if commands_missing: errors.append( - f"Missing visitor methods for commands with seq: {commands_missing}\n" + f"Missing visitor methods for commands with seq and payloads: {commands_missing}\n" f"Add methods to CommandAwarePayloadVisitor for these commands." ) if jobs_missing: errors.append( - f"Missing visitor methods for activation jobs with seq: {jobs_missing}\n" + f"Missing visitor methods for activation jobs with seq and payloads: {jobs_missing}\n" f"Add methods to CommandAwarePayloadVisitor for these jobs." ) assert not errors, "\n".join(errors) + + # Verify we found the expected commands/jobs with payloads + assert len(commands_with_payloads) > 0, "Should find commands with payloads" + assert len(jobs_with_payloads) > 0, "Should find activation jobs with payloads" + + # Sanity check: we should have fewer overrides than total protos with seq + # (because some don't have payloads) + assert len(commands_with_payloads) < len( + command_protos + ), "Should have some commands without payloads" + # All activation jobs except FireTimer have payloads + assert ( + len(jobs_with_payloads) == len(job_protos) - 1 + ), "Should have exactly one activation job without payloads (FireTimer)" From eb5daff0863727575c7e127f41298ab306d9e7e9 Mon Sep 17 00:00:00 2001 From: Dan Davison Date: Tue, 30 Sep 2025 13:17:26 -0400 Subject: [PATCH 75/81] Code golf --- temporalio/worker/_command_aware_visitor.py | 105 ++++++++++---------- 1 file changed, 53 insertions(+), 52 deletions(-) diff --git a/temporalio/worker/_command_aware_visitor.py b/temporalio/worker/_command_aware_visitor.py index e6653ef9b..e09722349 100644 --- a/temporalio/worker/_command_aware_visitor.py +++ b/temporalio/worker/_command_aware_visitor.py @@ -1,6 +1,7 @@ """Visitor that sets command context during payload traversal.""" import contextvars +import sys from contextlib import contextmanager from dataclasses import dataclass from typing import Any, Iterator, Optional, Type @@ -51,103 +52,103 @@ class CommandAwarePayloadVisitor(PayloadVisitor): async def _visit_coresdk_workflow_commands_ScheduleActivity( self, fs: VisitorFunctions, o: ScheduleActivity ) -> None: - with current_command(CommandType.COMMAND_TYPE_SCHEDULE_ACTIVITY_TASK, o.seq): - await super()._visit_coresdk_workflow_commands_ScheduleActivity(fs, o) + await self._visit_with_context( + CommandType.COMMAND_TYPE_SCHEDULE_ACTIVITY_TASK, o.seq, fs, o + ) async def _visit_coresdk_workflow_commands_ScheduleLocalActivity( self, fs: VisitorFunctions, o: ScheduleLocalActivity ) -> None: - with current_command(CommandType.COMMAND_TYPE_SCHEDULE_ACTIVITY_TASK, o.seq): - await super()._visit_coresdk_workflow_commands_ScheduleLocalActivity(fs, o) + await self._visit_with_context( + CommandType.COMMAND_TYPE_SCHEDULE_ACTIVITY_TASK, o.seq, fs, o + ) async def _visit_coresdk_workflow_commands_StartChildWorkflowExecution( self, fs: VisitorFunctions, o: StartChildWorkflowExecution ) -> None: - with current_command( - CommandType.COMMAND_TYPE_START_CHILD_WORKFLOW_EXECUTION, o.seq - ): - await super()._visit_coresdk_workflow_commands_StartChildWorkflowExecution( - fs, o - ) + await self._visit_with_context( + CommandType.COMMAND_TYPE_START_CHILD_WORKFLOW_EXECUTION, o.seq, fs, o + ) async def _visit_coresdk_workflow_commands_SignalExternalWorkflowExecution( self, fs: VisitorFunctions, o: SignalExternalWorkflowExecution ) -> None: - with current_command( - CommandType.COMMAND_TYPE_SIGNAL_EXTERNAL_WORKFLOW_EXECUTION, o.seq - ): - await super()._visit_coresdk_workflow_commands_SignalExternalWorkflowExecution( - fs, o - ) + await self._visit_with_context( + CommandType.COMMAND_TYPE_SIGNAL_EXTERNAL_WORKFLOW_EXECUTION, o.seq, fs, o + ) async def _visit_coresdk_workflow_commands_ScheduleNexusOperation( self, fs: VisitorFunctions, o: ScheduleNexusOperation ) -> None: - with current_command(CommandType.COMMAND_TYPE_SCHEDULE_NEXUS_OPERATION, o.seq): - await super()._visit_coresdk_workflow_commands_ScheduleNexusOperation(fs, o) + await self._visit_with_context( + CommandType.COMMAND_TYPE_SCHEDULE_NEXUS_OPERATION, o.seq, fs, o + ) # Workflow activation jobs with payloads async def _visit_coresdk_workflow_activation_ResolveActivity( self, fs: VisitorFunctions, o: ResolveActivity ) -> None: - with current_command(CommandType.COMMAND_TYPE_SCHEDULE_ACTIVITY_TASK, o.seq): - await super()._visit_coresdk_workflow_activation_ResolveActivity(fs, o) + await self._visit_with_context( + CommandType.COMMAND_TYPE_SCHEDULE_ACTIVITY_TASK, o.seq, fs, o + ) async def _visit_coresdk_workflow_activation_ResolveChildWorkflowExecutionStart( self, fs: VisitorFunctions, o: ResolveChildWorkflowExecutionStart ) -> None: - with current_command( - CommandType.COMMAND_TYPE_START_CHILD_WORKFLOW_EXECUTION, o.seq - ): - await super()._visit_coresdk_workflow_activation_ResolveChildWorkflowExecutionStart( - fs, o - ) + await self._visit_with_context( + CommandType.COMMAND_TYPE_START_CHILD_WORKFLOW_EXECUTION, o.seq, fs, o + ) async def _visit_coresdk_workflow_activation_ResolveChildWorkflowExecution( self, fs: VisitorFunctions, o: ResolveChildWorkflowExecution ) -> None: - with current_command( - CommandType.COMMAND_TYPE_START_CHILD_WORKFLOW_EXECUTION, o.seq - ): - await super()._visit_coresdk_workflow_activation_ResolveChildWorkflowExecution( - fs, o - ) + await self._visit_with_context( + CommandType.COMMAND_TYPE_START_CHILD_WORKFLOW_EXECUTION, o.seq, fs, o + ) async def _visit_coresdk_workflow_activation_ResolveSignalExternalWorkflow( self, fs: VisitorFunctions, o: ResolveSignalExternalWorkflow ) -> None: - with current_command( - CommandType.COMMAND_TYPE_SIGNAL_EXTERNAL_WORKFLOW_EXECUTION, o.seq - ): - await super()._visit_coresdk_workflow_activation_ResolveSignalExternalWorkflow( - fs, o - ) + await self._visit_with_context( + CommandType.COMMAND_TYPE_SIGNAL_EXTERNAL_WORKFLOW_EXECUTION, o.seq, fs, o + ) async def _visit_coresdk_workflow_activation_ResolveRequestCancelExternalWorkflow( self, fs: VisitorFunctions, o: ResolveRequestCancelExternalWorkflow ) -> None: - with current_command( - CommandType.COMMAND_TYPE_REQUEST_CANCEL_EXTERNAL_WORKFLOW_EXECUTION, o.seq - ): - await super()._visit_coresdk_workflow_activation_ResolveRequestCancelExternalWorkflow( - fs, o - ) + await self._visit_with_context( + CommandType.COMMAND_TYPE_REQUEST_CANCEL_EXTERNAL_WORKFLOW_EXECUTION, + o.seq, + fs, + o, + ) async def _visit_coresdk_workflow_activation_ResolveNexusOperationStart( self, fs: VisitorFunctions, o: ResolveNexusOperationStart ) -> None: - with current_command(CommandType.COMMAND_TYPE_SCHEDULE_NEXUS_OPERATION, o.seq): - await super()._visit_coresdk_workflow_activation_ResolveNexusOperationStart( - fs, o - ) + await self._visit_with_context( + CommandType.COMMAND_TYPE_SCHEDULE_NEXUS_OPERATION, o.seq, fs, o + ) async def _visit_coresdk_workflow_activation_ResolveNexusOperation( self, fs: VisitorFunctions, o: ResolveNexusOperation ) -> None: - with current_command(CommandType.COMMAND_TYPE_SCHEDULE_NEXUS_OPERATION, o.seq): - await super()._visit_coresdk_workflow_activation_ResolveNexusOperation( - fs, o - ) + await self._visit_with_context( + CommandType.COMMAND_TYPE_SCHEDULE_NEXUS_OPERATION, o.seq, fs, o + ) + + async def _visit_with_context( + self, + command_type: CommandType.ValueType, + seq: int, + fs: VisitorFunctions, + o: Any, + ) -> None: + """Helper to call parent method with command context.""" + method_name = sys._getframe(1).f_code.co_name + parent_method = getattr(super(), method_name) + with current_command(command_type, seq): + await parent_method(fs, o) def _get_workflow_command_protos_with_seq() -> Iterator[Type[Any]]: From b826097f70af85ab6cd7eb835254968478e7e589 Mon Sep 17 00:00:00 2001 From: Dan Davison Date: Tue, 30 Sep 2025 13:17:50 -0400 Subject: [PATCH 76/81] Revert "Code golf" This reverts commit e8e62bf6db3d5d054f31fa03141bb18c39a4bdb1. --- temporalio/worker/_command_aware_visitor.py | 105 ++++++++++---------- 1 file changed, 52 insertions(+), 53 deletions(-) diff --git a/temporalio/worker/_command_aware_visitor.py b/temporalio/worker/_command_aware_visitor.py index e09722349..e6653ef9b 100644 --- a/temporalio/worker/_command_aware_visitor.py +++ b/temporalio/worker/_command_aware_visitor.py @@ -1,7 +1,6 @@ """Visitor that sets command context during payload traversal.""" import contextvars -import sys from contextlib import contextmanager from dataclasses import dataclass from typing import Any, Iterator, Optional, Type @@ -52,103 +51,103 @@ class CommandAwarePayloadVisitor(PayloadVisitor): async def _visit_coresdk_workflow_commands_ScheduleActivity( self, fs: VisitorFunctions, o: ScheduleActivity ) -> None: - await self._visit_with_context( - CommandType.COMMAND_TYPE_SCHEDULE_ACTIVITY_TASK, o.seq, fs, o - ) + with current_command(CommandType.COMMAND_TYPE_SCHEDULE_ACTIVITY_TASK, o.seq): + await super()._visit_coresdk_workflow_commands_ScheduleActivity(fs, o) async def _visit_coresdk_workflow_commands_ScheduleLocalActivity( self, fs: VisitorFunctions, o: ScheduleLocalActivity ) -> None: - await self._visit_with_context( - CommandType.COMMAND_TYPE_SCHEDULE_ACTIVITY_TASK, o.seq, fs, o - ) + with current_command(CommandType.COMMAND_TYPE_SCHEDULE_ACTIVITY_TASK, o.seq): + await super()._visit_coresdk_workflow_commands_ScheduleLocalActivity(fs, o) async def _visit_coresdk_workflow_commands_StartChildWorkflowExecution( self, fs: VisitorFunctions, o: StartChildWorkflowExecution ) -> None: - await self._visit_with_context( - CommandType.COMMAND_TYPE_START_CHILD_WORKFLOW_EXECUTION, o.seq, fs, o - ) + with current_command( + CommandType.COMMAND_TYPE_START_CHILD_WORKFLOW_EXECUTION, o.seq + ): + await super()._visit_coresdk_workflow_commands_StartChildWorkflowExecution( + fs, o + ) async def _visit_coresdk_workflow_commands_SignalExternalWorkflowExecution( self, fs: VisitorFunctions, o: SignalExternalWorkflowExecution ) -> None: - await self._visit_with_context( - CommandType.COMMAND_TYPE_SIGNAL_EXTERNAL_WORKFLOW_EXECUTION, o.seq, fs, o - ) + with current_command( + CommandType.COMMAND_TYPE_SIGNAL_EXTERNAL_WORKFLOW_EXECUTION, o.seq + ): + await super()._visit_coresdk_workflow_commands_SignalExternalWorkflowExecution( + fs, o + ) async def _visit_coresdk_workflow_commands_ScheduleNexusOperation( self, fs: VisitorFunctions, o: ScheduleNexusOperation ) -> None: - await self._visit_with_context( - CommandType.COMMAND_TYPE_SCHEDULE_NEXUS_OPERATION, o.seq, fs, o - ) + with current_command(CommandType.COMMAND_TYPE_SCHEDULE_NEXUS_OPERATION, o.seq): + await super()._visit_coresdk_workflow_commands_ScheduleNexusOperation(fs, o) # Workflow activation jobs with payloads async def _visit_coresdk_workflow_activation_ResolveActivity( self, fs: VisitorFunctions, o: ResolveActivity ) -> None: - await self._visit_with_context( - CommandType.COMMAND_TYPE_SCHEDULE_ACTIVITY_TASK, o.seq, fs, o - ) + with current_command(CommandType.COMMAND_TYPE_SCHEDULE_ACTIVITY_TASK, o.seq): + await super()._visit_coresdk_workflow_activation_ResolveActivity(fs, o) async def _visit_coresdk_workflow_activation_ResolveChildWorkflowExecutionStart( self, fs: VisitorFunctions, o: ResolveChildWorkflowExecutionStart ) -> None: - await self._visit_with_context( - CommandType.COMMAND_TYPE_START_CHILD_WORKFLOW_EXECUTION, o.seq, fs, o - ) + with current_command( + CommandType.COMMAND_TYPE_START_CHILD_WORKFLOW_EXECUTION, o.seq + ): + await super()._visit_coresdk_workflow_activation_ResolveChildWorkflowExecutionStart( + fs, o + ) async def _visit_coresdk_workflow_activation_ResolveChildWorkflowExecution( self, fs: VisitorFunctions, o: ResolveChildWorkflowExecution ) -> None: - await self._visit_with_context( - CommandType.COMMAND_TYPE_START_CHILD_WORKFLOW_EXECUTION, o.seq, fs, o - ) + with current_command( + CommandType.COMMAND_TYPE_START_CHILD_WORKFLOW_EXECUTION, o.seq + ): + await super()._visit_coresdk_workflow_activation_ResolveChildWorkflowExecution( + fs, o + ) async def _visit_coresdk_workflow_activation_ResolveSignalExternalWorkflow( self, fs: VisitorFunctions, o: ResolveSignalExternalWorkflow ) -> None: - await self._visit_with_context( - CommandType.COMMAND_TYPE_SIGNAL_EXTERNAL_WORKFLOW_EXECUTION, o.seq, fs, o - ) + with current_command( + CommandType.COMMAND_TYPE_SIGNAL_EXTERNAL_WORKFLOW_EXECUTION, o.seq + ): + await super()._visit_coresdk_workflow_activation_ResolveSignalExternalWorkflow( + fs, o + ) async def _visit_coresdk_workflow_activation_ResolveRequestCancelExternalWorkflow( self, fs: VisitorFunctions, o: ResolveRequestCancelExternalWorkflow ) -> None: - await self._visit_with_context( - CommandType.COMMAND_TYPE_REQUEST_CANCEL_EXTERNAL_WORKFLOW_EXECUTION, - o.seq, - fs, - o, - ) + with current_command( + CommandType.COMMAND_TYPE_REQUEST_CANCEL_EXTERNAL_WORKFLOW_EXECUTION, o.seq + ): + await super()._visit_coresdk_workflow_activation_ResolveRequestCancelExternalWorkflow( + fs, o + ) async def _visit_coresdk_workflow_activation_ResolveNexusOperationStart( self, fs: VisitorFunctions, o: ResolveNexusOperationStart ) -> None: - await self._visit_with_context( - CommandType.COMMAND_TYPE_SCHEDULE_NEXUS_OPERATION, o.seq, fs, o - ) + with current_command(CommandType.COMMAND_TYPE_SCHEDULE_NEXUS_OPERATION, o.seq): + await super()._visit_coresdk_workflow_activation_ResolveNexusOperationStart( + fs, o + ) async def _visit_coresdk_workflow_activation_ResolveNexusOperation( self, fs: VisitorFunctions, o: ResolveNexusOperation ) -> None: - await self._visit_with_context( - CommandType.COMMAND_TYPE_SCHEDULE_NEXUS_OPERATION, o.seq, fs, o - ) - - async def _visit_with_context( - self, - command_type: CommandType.ValueType, - seq: int, - fs: VisitorFunctions, - o: Any, - ) -> None: - """Helper to call parent method with command context.""" - method_name = sys._getframe(1).f_code.co_name - parent_method = getattr(super(), method_name) - with current_command(command_type, seq): - await parent_method(fs, o) + with current_command(CommandType.COMMAND_TYPE_SCHEDULE_NEXUS_OPERATION, o.seq): + await super()._visit_coresdk_workflow_activation_ResolveNexusOperation( + fs, o + ) def _get_workflow_command_protos_with_seq() -> Iterator[Type[Any]]: From 3c37838e34b2d4ee36ea0c4c49bd79dfac27ff55 Mon Sep 17 00:00:00 2001 From: Dan Davison Date: Tue, 30 Sep 2025 13:20:39 -0400 Subject: [PATCH 77/81] Cleanup --- temporalio/worker/_command_aware_visitor.py | 22 +++---------------- tests/worker/test_command_aware_visitor.py | 24 ++++++++++++++++----- 2 files changed, 22 insertions(+), 24 deletions(-) diff --git a/temporalio/worker/_command_aware_visitor.py b/temporalio/worker/_command_aware_visitor.py index e6653ef9b..450445999 100644 --- a/temporalio/worker/_command_aware_visitor.py +++ b/temporalio/worker/_command_aware_visitor.py @@ -3,11 +3,10 @@ import contextvars from contextlib import contextmanager from dataclasses import dataclass -from typing import Any, Iterator, Optional, Type +from typing import Iterator, Optional from temporalio.api.enums.v1.command_type_pb2 import CommandType from temporalio.bridge._visitor import PayloadVisitor, VisitorFunctions -from temporalio.bridge.proto.workflow_activation import workflow_activation_pb2 from temporalio.bridge.proto.workflow_activation.workflow_activation_pb2 import ( ResolveActivity, ResolveChildWorkflowExecution, @@ -17,7 +16,6 @@ ResolveRequestCancelExternalWorkflow, ResolveSignalExternalWorkflow, ) -from temporalio.bridge.proto.workflow_commands import workflow_commands_pb2 from temporalio.bridge.proto.workflow_commands.workflow_commands_pb2 import ( ScheduleActivity, ScheduleLocalActivity, @@ -150,27 +148,13 @@ async def _visit_coresdk_workflow_activation_ResolveNexusOperation( ) -def _get_workflow_command_protos_with_seq() -> Iterator[Type[Any]]: - """Get concrete classes of all workflow command protos with a seq field.""" - for descriptor in workflow_commands_pb2.DESCRIPTOR.message_types_by_name.values(): - if "seq" in descriptor.fields_by_name: - yield descriptor._concrete_class - - -def _get_workflow_activation_job_protos_with_seq() -> Iterator[Type[Any]]: - """Get concrete classes of all workflow activation job protos with a seq field.""" - for descriptor in workflow_activation_pb2.DESCRIPTOR.message_types_by_name.values(): - if "seq" in descriptor.fields_by_name: - yield descriptor._concrete_class - - @contextmanager def current_command( - command_type: Optional[CommandType.ValueType], command_seq: int + command_type: CommandType.ValueType, command_seq: int ) -> Iterator[None]: """Context manager for setting command info.""" token = current_command_info.set( - CommandInfo(command_type=command_type, command_seq=command_seq) # type: ignore + CommandInfo(command_type=command_type, command_seq=command_seq) ) try: yield diff --git a/tests/worker/test_command_aware_visitor.py b/tests/worker/test_command_aware_visitor.py index f15a7733b..92c67b218 100644 --- a/tests/worker/test_command_aware_visitor.py +++ b/tests/worker/test_command_aware_visitor.py @@ -1,11 +1,11 @@ """Test that CommandAwarePayloadVisitor handles all commands with seq fields that have payloads.""" +from typing import Any, Iterator, Type + from temporalio.bridge._visitor import PayloadVisitor -from temporalio.worker._command_aware_visitor import ( - CommandAwarePayloadVisitor, - _get_workflow_activation_job_protos_with_seq, - _get_workflow_command_protos_with_seq, -) +from temporalio.bridge.proto.workflow_activation import workflow_activation_pb2 +from temporalio.bridge.proto.workflow_commands import workflow_commands_pb2 +from temporalio.worker._command_aware_visitor import CommandAwarePayloadVisitor def test_command_aware_visitor_has_methods_for_all_seq_protos_with_payloads(): @@ -73,3 +73,17 @@ def test_command_aware_visitor_has_methods_for_all_seq_protos_with_payloads(): assert ( len(jobs_with_payloads) == len(job_protos) - 1 ), "Should have exactly one activation job without payloads (FireTimer)" + + +def _get_workflow_command_protos_with_seq() -> Iterator[Type[Any]]: + """Get concrete classes of all workflow command protos with a seq field.""" + for descriptor in workflow_commands_pb2.DESCRIPTOR.message_types_by_name.values(): + if "seq" in descriptor.fields_by_name: + yield descriptor._concrete_class + + +def _get_workflow_activation_job_protos_with_seq() -> Iterator[Type[Any]]: + """Get concrete classes of all workflow activation job protos with a seq field.""" + for descriptor in workflow_activation_pb2.DESCRIPTOR.message_types_by_name.values(): + if "seq" in descriptor.fields_by_name: + yield descriptor._concrete_class From d8c4f175e318e3f50dc9bb199801a1faa7949dc5 Mon Sep 17 00:00:00 2001 From: Dan Davison Date: Tue, 30 Sep 2025 13:28:47 -0400 Subject: [PATCH 78/81] Remove temp coverage check --- temporalio/client.py | 1 - 1 file changed, 1 deletion(-) diff --git a/temporalio/client.py b/temporalio/client.py index 8d521b6f4..20a9b3c6d 100644 --- a/temporalio/client.py +++ b/temporalio/client.py @@ -2940,7 +2940,6 @@ class WorkflowExecution: @property def data_converter(self) -> temporalio.converter.DataConverter: """Data converter for the workflow.""" - 0 / 0 # type: ignore return self._context_free_data_converter.with_context( WorkflowSerializationContext( namespace=self.namespace, From 09be82ec88a6ab44a0575e52b415f5d734685949 Mon Sep 17 00:00:00 2001 From: Dan Davison Date: Wed, 1 Oct 2025 12:18:29 -0400 Subject: [PATCH 79/81] Add docstrings, cleanup --- temporalio/activity.py | 1 + temporalio/converter.py | 1 - temporalio/workflow.py | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) diff --git a/temporalio/activity.py b/temporalio/activity.py index 72ee81ac0..d726b9ef2 100644 --- a/temporalio/activity.py +++ b/temporalio/activity.py @@ -470,6 +470,7 @@ class _CompleteAsyncError(BaseException): def payload_converter() -> temporalio.converter.PayloadConverter: """Get the payload converter for the current activity. + The returned converter has :py:class:`temporalio.converter.ActivitySerializationContext` set. This is often used for dynamic activities to convert payloads. """ return _Context.current().payload_converter diff --git a/temporalio/converter.py b/temporalio/converter.py index b4bded364..dda9ffc9d 100644 --- a/temporalio/converter.py +++ b/temporalio/converter.py @@ -139,7 +139,6 @@ class ActivitySerializationContext(BaseWorkflowSerializationContext): is_local: bool -# TODO: duck typing or nominal typing? class WithSerializationContext(ABC): """Interface for classes that can use serialization context. diff --git a/temporalio/workflow.py b/temporalio/workflow.py index 98b45e367..e5bb25ba6 100644 --- a/temporalio/workflow.py +++ b/temporalio/workflow.py @@ -61,7 +61,6 @@ import temporalio.workflow from temporalio.nexus._util import ServiceHandlerT -from .api.failure.v1.message_pb2 import Failure from .types import ( AnyType, CallableAsyncNoParam, @@ -1148,6 +1147,7 @@ def patched(id: str) -> bool: def payload_converter() -> temporalio.converter.PayloadConverter: """Get the payload converter for the current workflow. + The returned converter has :py:class:`temporalio.converter.WorkflowSerializationContext` set. This is often used for dynamic workflows/signals/queries to convert payloads. """ From a8e3c6777efa100bb3e6ef984473f10cd2a45303 Mon Sep 17 00:00:00 2001 From: Dan Davison Date: Wed, 1 Oct 2025 12:47:26 -0400 Subject: [PATCH 80/81] Reduce unnecessary instantiations --- temporalio/worker/_workflow_instance.py | 128 +++++++++++++----------- 1 file changed, 70 insertions(+), 58 deletions(-) diff --git a/temporalio/worker/_workflow_instance.py b/temporalio/worker/_workflow_instance.py index 01d7ef032..44eb443ff 100644 --- a/temporalio/worker/_workflow_instance.py +++ b/temporalio/worker/_workflow_instance.py @@ -229,14 +229,15 @@ def __init__(self, det: WorkflowInstanceDetails) -> None: self._info = det.info self._context_free_payload_converter = det.payload_converter_class() self._context_free_failure_converter = det.failure_converter_class() - ( - self._workflow_context_payload_converter, - self._workflow_context_failure_converter, - ) = self._converters_with_context( - temporalio.converter.WorkflowSerializationContext( - namespace=det.info.namespace, - workflow_id=det.info.workflow_id, - ) + workflow_context = temporalio.converter.WorkflowSerializationContext( + namespace=det.info.namespace, + workflow_id=det.info.workflow_id, + ) + self._workflow_context_payload_converter = self._payload_converter_with_context( + workflow_context + ) + self._workflow_context_failure_converter = self._failure_converter_with_context( + workflow_context ) self._extern_functions = det.extern_functions @@ -785,20 +786,20 @@ def _apply_resolve_activity( handle = self._pending_activities.pop(job.seq, None) if not handle: raise RuntimeError(f"Failed finding activity handle for sequence {job.seq}") - payload_converter, failure_converter = self._converters_with_context( - temporalio.converter.ActivitySerializationContext( - namespace=self._info.namespace, - workflow_id=self._info.workflow_id, - workflow_type=self._info.workflow_type, - activity_type=handle._input.activity, - activity_task_queue=( - handle._input.task_queue or self._info.task_queue - if isinstance(handle._input, StartActivityInput) - else self._info.task_queue - ), - is_local=isinstance(handle._input, StartLocalActivityInput), - ) - ) + activity_context = temporalio.converter.ActivitySerializationContext( + namespace=self._info.namespace, + workflow_id=self._info.workflow_id, + workflow_type=self._info.workflow_type, + activity_type=handle._input.activity, + activity_task_queue=( + handle._input.task_queue or self._info.task_queue + if isinstance(handle._input, StartActivityInput) + else self._info.task_queue + ), + is_local=isinstance(handle._input, StartLocalActivityInput), + ) + payload_converter = self._payload_converter_with_context(activity_context) + failure_converter = self._failure_converter_with_context(activity_context) if job.result.HasField("completed"): ret: Optional[Any] = None if job.result.completed.HasField("result"): @@ -989,12 +990,12 @@ def _apply_resolve_request_cancel_external_workflow( fut, external_workflow_id = pending # We intentionally let this error if future is already done if job.HasField("failure"): - payload_converter, failure_converter = self._converters_with_context( - temporalio.converter.WorkflowSerializationContext( - namespace=self._info.namespace, - workflow_id=external_workflow_id, - ) + workflow_context = temporalio.converter.WorkflowSerializationContext( + namespace=self._info.namespace, + workflow_id=external_workflow_id, ) + payload_converter = self._payload_converter_with_context(workflow_context) + failure_converter = self._failure_converter_with_context(workflow_context) fut.set_exception( failure_converter.from_failure(job.failure, payload_converter) ) @@ -1013,12 +1014,12 @@ def _apply_resolve_signal_external_workflow( fut, external_workflow_id = pending # We intentionally let this error if future is already done if job.HasField("failure"): - payload_converter, failure_converter = self._converters_with_context( - temporalio.converter.WorkflowSerializationContext( - namespace=self._info.namespace, - workflow_id=external_workflow_id, - ) + workflow_context = temporalio.converter.WorkflowSerializationContext( + namespace=self._info.namespace, + workflow_id=external_workflow_id, ) + payload_converter = self._payload_converter_with_context(workflow_context) + failure_converter = self._failure_converter_with_context(workflow_context) fut.set_exception( failure_converter.from_failure(job.failure, payload_converter) ) @@ -1874,7 +1875,7 @@ async def run_activity() -> Any: async def _outbound_signal_child_workflow( self, input: SignalChildWorkflowInput ) -> None: - payload_converter, _ = self._converters_with_context( + payload_converter = self._payload_converter_with_context( temporalio.converter.WorkflowSerializationContext( namespace=self._info.namespace, workflow_id=input.child_workflow_id, @@ -1894,7 +1895,7 @@ async def _outbound_signal_child_workflow( async def _outbound_signal_external_workflow( self, input: SignalExternalWorkflowInput ) -> None: - payload_converter, _ = self._converters_with_context( + payload_converter = self._payload_converter_with_context( temporalio.converter.WorkflowSerializationContext( namespace=input.namespace, workflow_id=input.workflow_id, @@ -2077,39 +2078,45 @@ def _convert_payloads( raise raise RuntimeError("Failed decoding arguments") from err - def _converters_with_context( + def _payload_converter_with_context( self, context: temporalio.converter.SerializationContext, - ) -> Tuple[ - temporalio.converter.PayloadConverter, - temporalio.converter.FailureConverter, - ]: - """Construct workflow payload and failure converters with the given context. + ) -> temporalio.converter.PayloadConverter: + """Construct workflow payload converter with the given context. This plays a similar role to DataConverter._with_context, but operates on PayloadConverter - and FailureConverter only (since payload encoding/decoding is done by the worker, outside - the workflow sandbox). + only (payload encoding/decoding is done by the worker, outside the workflow sandbox). """ payload_converter = self._context_free_payload_converter - failure_converter = self._context_free_failure_converter if isinstance(payload_converter, temporalio.converter.WithSerializationContext): payload_converter = payload_converter.with_context(context) + return payload_converter + + def _failure_converter_with_context( + self, + context: temporalio.converter.SerializationContext, + ) -> temporalio.converter.FailureConverter: + """Construct workflow failure converter with the given context. + + This plays a similar role to DataConverter._with_context, but operates on FailureConverter + only (payload encoding/decoding is done by the worker, outside the workflow sandbox). + """ + failure_converter = self._context_free_failure_converter if isinstance(failure_converter, temporalio.converter.WithSerializationContext): failure_converter = failure_converter.with_context(context) - return payload_converter, failure_converter + return failure_converter def get_serialization_context( self, command_info: Optional[_command_aware_visitor.CommandInfo], ) -> Optional[temporalio.converter.SerializationContext]: - workflow_context = temporalio.converter.WorkflowSerializationContext( - namespace=self._info.namespace, - workflow_id=self._info.workflow_id, - ) if command_info is None: # Use payload codec with workflow context by default (i.e. for payloads not associated # with a pending command) - return workflow_context + return temporalio.converter.WorkflowSerializationContext( + namespace=self._info.namespace, + workflow_id=self._info.workflow_id, + ) if ( command_info.command_type @@ -2170,7 +2177,10 @@ def get_serialization_context( else: # Use payload codec with workflow context for all other payloads - return workflow_context + return temporalio.converter.WorkflowSerializationContext( + namespace=self._info.namespace, + workflow_id=self._info.workflow_id, + ) def _instantiate_workflow_object(self) -> Any: if not self._workflow_input: @@ -2908,7 +2918,7 @@ def __init__( self._result_fut = instance.create_future() self._started = False instance._register_task(self, name=f"activity: {input.activity}") - self._payload_converter, _ = self._instance._converters_with_context( + self._payload_converter = self._instance._payload_converter_with_context( temporalio.converter.ActivitySerializationContext( namespace=self._instance._info.namespace, workflow_id=self._instance._info.workflow_id, @@ -3069,13 +3079,15 @@ def __init__( self._result_fut: asyncio.Future[Any] = instance.create_future() self._first_execution_run_id = "" instance._register_task(self, name=f"child: {input.workflow}") - self._payload_converter, self._failure_converter = ( - self._instance._converters_with_context( - temporalio.converter.WorkflowSerializationContext( - namespace=self._instance._info.namespace, - workflow_id=self._input.id, - ) - ) + workflow_context = temporalio.converter.WorkflowSerializationContext( + namespace=self._instance._info.namespace, + workflow_id=self._input.id, + ) + self._payload_converter = self._instance._payload_converter_with_context( + workflow_context + ) + self._failure_converter = self._instance._failure_converter_with_context( + workflow_context ) @property From c78016eba2d9e0ed4e30eca3eda38779989405ed Mon Sep 17 00:00:00 2001 From: Dan Davison Date: Wed, 1 Oct 2025 13:04:41 -0400 Subject: [PATCH 81/81] Ensure _any_converter_takes_context is set and not stale --- temporalio/converter.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/temporalio/converter.py b/temporalio/converter.py index dda9ffc9d..29eb35566 100644 --- a/temporalio/converter.py +++ b/temporalio/converter.py @@ -5,6 +5,7 @@ import collections import collections.abc import dataclasses +import functools import inspect import json import sys @@ -352,9 +353,6 @@ def __init__(self, *converters: EncodingPayloadConverter) -> None: converters: Payload converters to delegate to, in order. """ self._set_converters(*converters) - self._any_converter_takes_context = any( - isinstance(c, WithSerializationContext) for c in converters - ) def _set_converters(self, *converters: EncodingPayloadConverter) -> None: self.converters = {c.encoding.encode(): c for c in converters} @@ -453,6 +451,12 @@ def get_converters_with_context( return converters if any_with_context else None + @functools.cached_property + def _any_converter_takes_context(self) -> bool: + return any( + isinstance(c, WithSerializationContext) for c in self.converters.values() + ) + class DefaultPayloadConverter(CompositePayloadConverter): """Default payload converter compatible with other Temporal SDKs.